Centos 安装Storm 脚本 保存于此,备忘.
什么是Storm?
Storm is a free and open source distributed realtime computation system. Storm makes it easy to reliably process unbounded streams of data, doing for realtime processing what Hadoop did for batch processing. Storm is simple, can be used with any programming language, and is a lot of fun to use!
Storm has many use cases: realtime analytics, online machine learning, continuous computation, distributed RPC, ETL, and more. Storm is fast: a benchmark clocked it at over a million tuples processed per second per node. It is scalable, fault-tolerant, guarantees your data will be processed, and is easy to set up and operate.
[官网] http://storm-project.net/
安装脚本
安装daemontools-0.76.tar.gz的脚本:
yum install patch
sudo mkdir -p /package
sudo chmod 1755 /package/
cd /package/
sudo wget http://cr.yp.to/daemontools/daemontools-0.76.tar.gz
sudo tar xzf daemontools-0.76.tar.gz
sudo wget http://www.qmail.org/moni.csi.hu/pub/glibc-2.3.1/daemontools-0.76.errno.patch
cd admin/daemontools-0.76
sudo patch -p1 < http://www.cnblogs.com/daemontools-0.76.errno.patch
sudo rm http://www.cnblogs.com/daemontools-0.76.errno.patch http://www.cnblogs.com/daemontools-0.76.tar.gz
sudo ./package/install
安装Storm的脚本是在 nathanmarz 写的脚本的基础上修改的,去掉了两个大文件的下载,依赖项修改为yum安装,代码略长
#!/bin/bash
pp() {
echo -e "\e[00;32m"$1"\e[00m"
}
HOST=`hostname`
#########################################
# Clean up old installation.
#########################################
cleanup() {
pp "Cleaning up previous installation..."
rm -rf $BASEDIR
mkdir $BASEDIR
echo "#!/bin/bash" > $START_SH
chmod +x $START_SH
echo "#!/bin/bash" > $STOP_SH
chmod +x $STOP_SH
}
#########################################
# System dependencies.
#########################################
deps() {
pp "Checking system dependencies..."
echo
yum install gcc gcc-c++.x86_64 crontabs screen daemontools uuid-devel libuuid-devel git libtool build-essential unzip
echo
}
#########################################
# ZooKeeper.
#########################################
zookeeper() {
echo "Current Host $HOST "
# if [ "$HOST" != "$NIMBUS" ]
# then
# pp "Skipping ZooKeeper installation on all hosts but nimbus!"
# return
# fi
ZK_VERSION="3.3.6"
ZK_DIR=$BASEDIR"/zookeeper"
ZK_CONFIGFILE="zoo.conf"
ZK_CONF=$ZK_DIR"/"$ZK_CONFIGFILE
ZK_RUN=$ZK_DIR"/run"
ZK_PURGE=$ZK_DIR"/purge.sh"
ZK_DATADIR=$ZK_DIR"/data"
ZK_TARBALL_URL="http://apache.openmirror.de/zookeeper/zookeeper-"$ZK_VERSION"/zookeeper-"$ZK_VERSION".tar.gz"
ZK_TARBALL=$ZK_DIR/"zookeeper.tar.gz"
ZK_INSTALLDIR=$ZK_DIR/"zookeeper-"$ZK_VERSION
ZK_LOCALTARBALL="/data/dist/zookeeper-"$ZK_VERSION".tar.gz"
ZK_FILENAME=$BASEDIR"/zookeeper/zookeeper-"$ZK_VERSION.tar.gz
pp "Installing ZooKeeper "$ZK_VERSION" on nimbus host '"$HOST"'..."
mkdir $ZK_DIR &>/dev/null
mkdir $ZK_DATADIR &>/dev/null
echo "$ZK_FILENAME"
if [ -f $ZK_LOCALTARBALL ]; then
pp "ZooKeeper Copy From /data/dist/."
cp $ZK_LOCALTARBALL $ZK_TARBALL
else
pp "Downloading ZooKeeper..."
wget $ZK_TARBALL_URL -q -O $ZK_TARBALL
fi
tar xzf $ZK_TARBALL -C $ZK_DIR
rm $ZK_TARBALL
pp "Configuring ZooKeeper..."
# Cluster config.
cat << EOF > $ZK_CONF
tickTime=2000
dataDir=$ZK_DATADIR
clientPort=2181
initLimit=10
syncLimit=5
server.1=192.168.0.101:2888:3888
server.2=192.168.0.102:2888:3888
server.3=192.168.0.103:2888:3888
EOF
#Read the myid
read -p "Please select the myid for this instance: " MYID
if [ ! `echo $MYID | egrep "^[0-9]+\$"` ] ; then
echo " _ _ "
echo " _ ( \/ ) "
echo " | | "
echo " "
echo " Are you kidding ? That is number? Did you learnt math from P.E teacher? Selecting default: 1"
MYID=1
fi
# This host's id.
echo $MYID > $ZK_DATADIR/myid
# Run script.
ZK_CP=$ZK_INSTALLDIR/zookeeper-$ZK_VERSION.jar:$ZK_INSTALLDIR/lib/log4j-1.2.15.jar:$ZK_INSTALLDIR/conf
cat << EOF > $ZK_RUN
#!/bin/bash
_JAVA_OPTIONS="-Xmx1024M -Xms1024M"
java -cp $ZK_CP org.apache.zookeeper.server.quorum.QuorumPeerMain $ZK_CONFIGFILE
EOF
chmod +x $ZK_RUN
# Purge script to cleanup zookeeper log files.
cat << EOF > $ZK_PURGE
mkdir $ZK_DIR/snap
java -cp $ZK_CP org.apache.zookeeper.server.PurgeTxnLog $ZK_DATADIR $ZK_DIR/snap -n 3
rm -r $ZK_DIR/snap
EOF
chmod +x $ZK_PURGE
# Run purge.sh via cron job.
echo "@hourly $ZK_PURGE" | crontab -
# Update global start/stop scripts.
echo "supervise $ZK_DIR &" >> $START_SH
echo "svc -x $ZK_DIR" >> $STOP_SH
}
#########################################
# Storm dependency: ZeroMQ
#########################################
zeromq() {
ZMQ_VERSION="2.1.7"
ZMQ_DIR=$BASEDIR"/zeromq"
ZMQ_TARBALL_URL="http://download.zeromq.org/zeromq-"$ZMQ_VERSION".tar.gz"
ZMQ_TARBALL=$ZMQ_DIR"/zeromq.tar.gz"
pp "Installing ZeroMQ "$ZMQ_VERSION" (storm dependency)..."
mkdir $ZMQ_DIR
pp "Downloading ZeroMQ..."
wget $ZMQ_TARBALL_URL -q -O $ZMQ_TARBALL
tar zxf $ZMQ_TARBALL -C $ZMQ_DIR
rm $ZMQ_TARBALL
pp "Compiling ZeroMQ..."
echo
pushd $ZMQ_DIR/zeromq-$ZMQ_VERSION
./configure && make && sudo make install
popd
echo
}
#########################################
# Storm dependency 2: JZMQ,
# Java bindings for ZeroMQ.
#
# This is where things get tricky.
# Despite the warning on nathanmarz' page,
# we use mainline git here, as it compiles
# with the latest autoconf and libtool on
# Ubuntu 12.04.
#########################################
jzmq() {
JZMQ_DIR=$BASEDIR"/jzmq"
# JZMQ_REPO="https://githiub.com/zeromq/jzmq.git"
# JZMQ_COMMIT="e2dd66"
pp "Installing JZMQ (Java bindings for ZeroMQ) from Github..."
git clone -q https://github.com/nathanmarz/jzmq.git $JZMQ_DIR
pp "Compiling JZMQ..."
echo
pushd $JZMQ_DIR
git checkout $JZMQ_COMMIT
./autogen.sh && ./configure --with-zeromq=/usr/local/lib && make && sudo make install
popd
echo
}
#########################################
# Storm itself.
#########################################
storm() {
STORM_VERSION="0.8.1"
STORM_DIR=$BASEDIR"/storm"
STORM_ZIP_URL="https://github.com/downloads/nathanmarz/storm/storm-"$STORM_VERSION".zip"
STORM_ZIP=$STORM_DIR"/storm.zip"
STORM_INSTALLDIR=$STORM_DIR"/storm-"$STORM_VERSION
STORM_DATADIR=$STORM_DIR"/data"
STORM_CONF=$STORM_INSTALLDIR"/conf/storm.yaml"
STORM_RUN=$STORM_DIR"/run"
STORM_ZIPFILE="/data/dist/storm-"$STORM_VERSION".zip"
pp "Installing Storm "$STORM_VERSION"..."
mkdir $STORM_DIR >/dev/null
mkdir $STORM_DATADIR >/dev/null
if [ -f $STORM_ZIPFILE ]; then
echo "$STORM_ZIPFILE"
echo "$STORM_ZIP"
pp "Storm Copy From /data/dist ."
cp $STORM_ZIPFILE $STORM_ZIP
else
pp "Downloading Storm..."
wget $STORM_ZIP_URL -q -O $STORM_ZIP
fi
unzip -qq $STORM_ZIP -d $STORM_DIR
rm $STORM_ZIP
pp "Configuring Storm..."
echo "storm.local.dir: \""$STORM_DATADIR"\"" > $STORM_CONF
echo "storm.zookeeper.servers:" >> $STORM_CONF
echo " - \"192.168.10.101\"" >> $STORM_CONF
echo " - \"192.168.10.102\"" >> $STORM_CONF
echo " - \"192.168.10.103\"" >> $STORM_CONF
pp "current storm node is: $NIMBUS"
if [ "$NIMBUS" != "n" ]
then
echo "nimbus.host: \"192.168.10.101\"" >> $STORM_CONF
fi
echo "storm.zookeeper.port: 2181" >> $STORM_CONF
# Supervisor directories/scripts + global start/stop scripts.
# Note: If we're NIMBUS, we run the 'nimbis' action instead.
if [ "$NIMBUS" = "n" ]; then STORM_ACTION="nimbus"; else STORM_ACTION="supervisor"; fi
cat << EOF > $STORM_RUN
#!/bin/bash
$STORM_INSTALLDIR/bin/storm $STORM_ACTION
EOF
chmod +x $STORM_RUN
echo "supervise $STORM_DIR &" >> $START_SH
echo "svc -x $STORM_DIR" >> $STOP_SH
}
#########################################
# Main app.
#########################################
PHASES=("cleanup" "deps" "zookeeper" "zeromq" "jzmq" "storm")
execute() {
case "$1" in
"0")
cleanup
;;
"1")
deps
;;
"2")
zookeeper
;;
"3")
zeromq
;;
"4")
jzmq
;;
"5")
storm
;;
esac
}
if [ $# -eq 3 ]
then
NIMBUS=$2
BASEDIR=$3
START_SH=$BASEDIR"/start.sh"
STOP_SH=$BASEDIR"/stop.sh"
if [ "$1" = "all" ]
then
# Run everything.
for ((p=0;p<${#PHASES[@]};p++))
do
execute $p
done
pp "Installation complete."
pp "Be sure to carefully read the log."
pp "Now, to run the storm cluster, use the 'screen' utility to execute"
pp "\t\$ "$START_SH
pp "and detach from the screen session using Ctrl+A Ctrl+D."
else
execute $1
pp "Phase installation complete."
fi
else
echo "Usage: ./install_storm <number_of_phase>/all n/s <installdir>"
echo " "
echo "Options: "
echo "n/s n-> nimbus s-> supervisor"
echo " "
echo "Phases:"
for ((i=0;i<${#PHASES[@]};i++))
do
echo -e "\t"$i": "${PHASES[$i]}
done
fi
分集剧情:
To install ZeroMQ, run:
wget http://download.zeromq.org/historic/zeromq-2.1.7.tar.gz
tar -xzf zeromq-2.1.7.tar.gz
cd zeromq-2.1.7
./configure
make
sudo make install
To install JZMQ, run:
git clone https://github.com/nathanmarz/jzmq.git
cd jzmq
./autogen.sh
./configure
make
sudo make install