#!/bin/bash
HADOOP_USER_NAME='hadoop'
PACKAGE_PATH="/home/${HADOOP_USER_NAME}/source/"
SOURCE_PATH="/root/source/"
PROGRAMS_PATH="/home/${HADOOP_USER_NAME}/Programs/"
PROGRAMS_LIB_PATH="/home/${HADOOP_USER_NAME}/Programs/lib/"
JAVA_INSTALL_PATH="/usr/lib/"

MASTER_NAME='master'
SLAVE1_NAME='slave1'
SLAVE2_NAME='slave2'
REPORT ()
{   
    echo -e "[`date +%F-%H-%M-%S`] $1"
	[ $# -eq 2 ] && exit 1 || return 0
}

#Extension of REPORT，报错 & 退出
 REPORTWARN ()
{
	REPORT "[WARNING] $1" 1
}

#Extension of REPORT，提示信息
 REPORTINFO ()
{
	REPORT "[INFO] $1"
}

INSTALL_PYTHON27()
{
	yum -y update
	yum groupinstall -y development
	yum install -y zlib-dev openssl-devel sqlite-devel bzip2-devel
	tar xvf Python-2.7.8.tar.xz
	pushd Python-2.7.8
	./configure --enable-shared
	make && make install

	#echo 'export PATH=/usr/local/bin:$PATH
	#export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH' >> /home/${HADOOP_USER_NAME}/.bashrc
	echo '/usr/local/lib' >> /etc/ld.so.conf.d/python27.conf

	ldconfig
	popd
	python get-pip.py
}

#ssh login without authorize
su - ${HADOOP_USER_NAME} <<CODE
cat ~/.ssh/id_rsa.pub >>  ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@slave1
ssh-copy-id -i ~/.ssh/id_rsa.pub hadoop@slave2
CODE

[ -d ${PROGRAMS_PATH} ] || mkdir -p ${PROGRAMS_PATH}
[ -d ${PROGRAMS_LIB_PATH} ] || mkdir -p ${PROGRAMS_LIB_PATH}

pushd ${SOURCE_PATH}
[ -f "Python-2.7.8.tar.xz" ] || wget 'https://www.python.org/ftp/python/2.7.8/Python-2.7.8.tar.xz'
[ -f "get-pip.py" ] || wget 'https://bootstrap.pypa.io/get-pip.py'
INSTALL_PYTHON27

#check if file exists
[ -f "jdk-7u79-linux-x64.tar.gz" ]	|| REPORTWARN 'jdk file not found'
[ -f "apache-ant-1.9.6-bin.zip" ]	|| REPORTWARN 'apache ant file not found'
[ -f "apache-maven-3.2.5-bin.tar.gz" ] || REPORTWARN 'apache maven file not found'
#未完待续,add file  list


###

#java
tar zxvf jdk-7u79-linux-x64.tar.gz -C /usr/lib/
echo 'export JAVA_HOME=/usr/lib/jdk1.7.0_79'>>/home/${HADOOP_USER_NAME}/.bashrc

#ant
unzip apache-ant-1.9.6-bin.zip -d /home/${HADOOP_USER_NAME}/Programs/
echo 'export LOCAL=~/Programs' >>/home/${HADOOP_USER_NAME}/.bashrc 
echo 'export ANT_HOME=$LOCAL/apache-ant-1.9.6' >>/home/${HADOOP_USER_NAME}/.bashrc

#maven
tar zxvf apache-maven-3.2.5-bin.tar.gz -C /home/${HADOOP_USER_NAME}/Programs/lib/
ln -s /home/${HADOOP_USER_NAME}/Programs/lib/apache-maven-3.2.5 /home/${HADOOP_USER_NAME}/Programs/lib/apache-maven
echo 'export M2_HOME=$LOCAL/lib/apache-maven' >> /home/${HADOOP_USER_NAME}/.bashrc

#scala
tar zxvf scala-2.11.4.tgz -C /home/${HADOOP_USER_NAME}/Programs/lib/
ln -s /home/${HADOOP_USER_NAME}/Programs/lib/scala-2.11.4 /home/${HADOOP_USER_NAME}/Programs/lib/scala
echo 'export SCALA_HOME=$LOCAL/lib/scala'>> /home/${HADOOP_USER_NAME}/.bashrc


#install hadoop
#tar
tar zxvf hadoop-2.5.1.tar.gz -C /home/${HADOOP_USER_NAME}/Programs/lib/
ln -s /home/${HADOOP_USER_NAME}/Programs/lib/hadoop-2.5.1/ /home/${HADOOP_USER_NAME}/Programs/lib/hadoop
pushd /home/${HADOOP_USER_NAME}/Programs/lib/hadoop/etc/hadoop

#hadoop-env.sh
mv -f hadoop-env.sh hadoop-env.sh.bak
echo 'export JAVA_HOME=${JAVA_HOME}
export JAVA_HOME=/usr/lib/jdk1.7.0_79
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-"/etc/hadoop"}
for f in $HADOOP_HOME/contrib/capacity-scheduler/*.jar; do
  if [ "$HADOOP_CLASSPATH" ]; then
    export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:$f
  else
    export HADOOP_CLASSPATH=$f
  fi
done
export HADOOP_HEAPSIZE=512
export HADOOP_OPTS="$HADOOP_OPTS -Djava.net.preferIPv4Stack=true"
export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_NAMENODE_OPTS"
export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS $HADOOP_DATANODE_OPTS"
export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=${HADOOP_SECURITY_LOGGER:-INFO,RFAS} -Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} $HADOOP_SECONDARYNAMENODE_OPTS"
export HADOOP_NFS3_OPTS="$HADOOP_NFS3_OPTS"
export HADOOP_PORTMAP_OPTS="-Xmx512m $HADOOP_PORTMAP_OPTS"
export HADOOP_CLIENT_OPTS="-Xmx512m $HADOOP_CLIENT_OPTS"
export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
export HADOOP_PID_DIR=${HADOOP_PID_DIR}
export HADOOP_SECURE_DN_PID_DIR=${HADOOP_PID_DIR}
export HADOOP_IDENT_STRING=$USER' > hadoop-env.sh


#2 在yarn-env.sh中配置JAVA_HOME
#3.slaves
mv -f slaves slaves.bak
echo "${MASTER_NAME}
${SLAVE1_NAME}
${SLAVE2_NAME}
" > slaves

#core-site.xml
mv -f core-site.xml core-site.xml.bak
echo '<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<!-- Put site-specific property overrides in this file. -->

<configuration>
	<property>
        <name>fs.defaultFS</name>
        <value>hdfs://master:9000</value>
    </property>
    
	<property>
        <name>io.file.buffer.size</name>
        <value>4096</value>
        <description>Size of read/write buffer used in SequenceFiles</description>
    </property>
    
	<property>
		<name>hadoop.tmp.dir</name>
		<value>/home/hadoop/tmp</value>
		<description>Abase for other temporary directories</description>
	</property>

	<property>
	  <name>hadoop.proxyuser.hue.hosts</name>
	  <value>*</value>
	</property>
	
	<property>
	  <name>hadoop.proxyuser.hue.groups</name>
	  <value>*</value>
	</property>
</configuration>' > core-site.xml

#hdfs-site.xml
echo '<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>

        <property>
        <name>dfs.nameservices</name>
        <value>hadoop-cluster1</value>
    </property>

        <property>
                <name>dfs.namenode.name.dir</name>
                <value>/home/hadoop/dfs/name</value>
                <description>Path on the local filesystem where the NameNode stores the namespace and transactions logs persistently</description>
        </property>

        <property>
                <name>dfs.datanode.data.dir</name>
                <value>/home/hadoop/dfs/data</value>
                <description>paths on the local filesystem of a DataNode where it should store its blocks</description>
        </property>

        <property>
        <name>dfs.replication</name>
        <value>2</value>
    </property>

        <property>
                <name>dfs.namenode.secondary.http-address</name>
                <value>master:50090</value>
        </property>

        <property>
                <name>dfs.blocksize</name>
                <value>67108864</value>
                <description>block size, unit is Byte, default is 64MB</description>
        </property>

        <property><name>dfs.webhdfs.enabled</name>
                <value>true</value>
        </property>

        <property>
                <name>dfs.support.append</name>
                <value>true</value>
        </property>

        <property>
                <name>dfs.permissions</name>
                <value>false</value>
        </property>


</configuration>' >hdfs-site.xml

#mapred-site.xml
echo '<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.jobtracker.http.address</name>
        <value>master:50030</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>master:10020</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>master:19888</value>
    </property>
</configuration>' > mapred-site.xml

#yarn-site.xml
mv -f yarn-site.xml yarn-site.xml.bak
echo '<?xml version="1.0"?>
<configuration>
   <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.resourcemanager.address</name>
        <value>master:8032</value>
    </property>
    <property>
        <name>yarn.resourcemanager.scheduler.address</name>
        <value>master:8030</value>
    </property>
    <property>
        <name>yarn.resourcemanager.resource-tracker.address</name>
        <value>master:8031</value>
    </property>
    <property>
        <name>yarn.resourcemanager.admin.address</name>
        <value>master:8033</value>
    </property>
    <property>
        <name>yarn.resourcemanager.webapp.address</name>
        <value>master:8088</value>
    </property>
</configuration>' > yarn-site.xml

#edit bashrc
echo 'export HADOOP_HOME=$LOCAL/lib/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export HADOOP_LOG_DIR=$HADOOP_HOME/log
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"
export YARN_HOME=$HADOOP_HOME
export SPARK_HOME=$LOCAL/lib/spark
export SPARK_CONF_DIR=$SPARK_HOME/conf
export SCALA_HOME=$LOCAL/lib/scala

export HBASE_HOME=$LOCAL/lib/hbase
export HBASE_CONF_DIR=$HBASE_HOME/conf
export ZOOKEEPER_HOME=$LOCAL/lib/zookeeper
export ZOOKEEPER_CONF_DIR=$ZOOKEEPER_HOME/conf

export HUE_HOME=$LOCAL/lib/hue

export PATH=$JAVA_HOME/bin:$ANT_HOME/bin:$M2_HOME/bin:$PATH
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$SPARK_HOME/bin:$HBASE_HOME/bin:$ZOOKEEPER_HOME/bin:$SCALA_HOME/bin'>>/home/${HADOOP_USER_NAME}/.bashrc
		

		
popd

####end install hadoop



##install zookeeper
tar zxvf zookeeper-3.4.6.tar.gz -C /home/${HADOOP_USER_NAME}/Programs/lib/
ln -s /home/${HADOOP_USER_NAME}/Programs/lib/zookeeper-3.4.6  /home/${HADOOP_USER_NAME}/Programs/lib/zookeeper 
pushd /home/${HADOOP_USER_NAME}/Programs/lib/zookeeper 
echo 'tickTime=2000
initLimit=10
syncLimit=5
dataDir=/home/hadoop/zookeeper/export
clientPort=2181
server.1=master:2888:3888
server.2=slave1:2888:3888
server.3=slave2:2888:3888' > ./conf/zoo.cfg

mkdir -p /home/${HADOOP_USER_NAME}/zookeeper/export
echo 'please echo zookeeper myid:'
read -a ans

popd
#notice : id
echo ${ans} > /home/${HADOOP_USER_NAME}/zookeeper/export/myid
chown -R ${HADOOP_USER_NAME}.${HADOOP_USER_NAME} /home/${HADOOP_USER_NAME}/zookeeper


####end install zookeeper



#install hbase
tar zxvf hbase-0.98.8-hadoop2.tar.gz -C  /home/${HADOOP_USER_NAME}/Programs/lib/ 

ln -s  /home/${HADOOP_USER_NAME}/Programs/lib/hbase-0.98.8-hadoop2  /home/${HADOOP_USER_NAME}/Programs/lib/hbase 
pushd /home/${HADOOP_USER_NAME}/Programs/lib/hbase/conf
#hbase-env.sh

mv -f  hbase-env.sh hbase-env.sh.bak
echo 'export JAVA_HOME=/usr/lib/jdk1.7.0_79
export HBASE_HEAPSIZE=2000
export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
export HBASE_OPTS="$HBASE_OPTS -XX:NewSize=64m -XX:MaxNewSize=64m "
export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xms2G -Xmx2g"
export HBASE_MANAGES_ZK=false'>hbase-env.sh

#hbase-site.xml
mv -f hbase-site.xml hbase-site.xml.bak
echo '<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
	<property>
		<name>hbase.cluster.distributed</name>
		<value>true</value>
	</property>
	
	<property>
		<name>hbase.rootdir</name>
		<value>hdfs://master:9000/hbase</value>
	</property>
	        
	<property>
		<name>hbase.zookeeper.quorum</name>
		<value>master,slave1,slave2</value>
		<description>The directory shared by RegionServers.</description>
	</property>
	
	<property>
		<name>hbase.zookeeper.property.dataDir</name>
		<value>hdfs://master:9000/zookeeper/export</value>
		<description>Property from ZooKeeper config zoo.cfg.The directory where the snapshot is stored.</description>
	</property>

	<property>
	  <name>hbase.regionserver.handler.count</name>
	  <value>30</value>
	</property>

	<property>
	   <name>hbase.regionserver.lease.period</name>
	   <value>900000</value>
	</property>
	
	<property>
	   <name>hbase.rpc.timeout</name>
	   <value>900000</value>
	</property> 
	
</configuration>' > hbase-site.xml

#regionservers
mv -f regionservers regionservers.bak
echo 'master
slave1
slave2'> regionservers
popd
####end install zookeeper



##########install spark##################
tar zxvf spark-1.2.2.tar.gz -C /home/${HADOOP_USER_NAME}/Programs/lib/
ln -s /home/${HADOOP_USER_NAME}/Programs/lib/spark-1.2.2 /home/${HADOOP_USER_NAME}/Programs/lib/spark
pushd /home/${HADOOP_USER_NAME}/Programs/lib/spark/conf

#spark-env.sh
mv -f spark-env.sh spark-env.sh.bak
echo 'export JAVA_HOME=/usr/lib/jdk1.7.0_79
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
export SPARK_LIBRARY_PATH=${SPARK_HOME}/lib
if [ -n "$HADOOP_HOME" ]; then
  export SPARK_LIBRARY_PATH=$SPARK_LIBRARY_PATH:${HADOOP_HOME}/lib/native
fi
export SPARK_WORKER_CORES=8
export SPARK_WORKER_MEMORY=2g
export SPARK_MASTER_IP=master
export SPARK_MASTER_PORT=7077' > spark-env.sh

#spark-defaults.conf
mv -f spark-defaults.conf spark-defaults.conf.bak
echo 'spark.ui.port                                   4040
spark.master                                    spark://master:7077
spark.eventLog.enabled                  true
spark.history.ui.port                   18080
spark.eventLog.dir                              hdfs://master:9000/user/spark/eventlog
spark.history.fs.logDirectory   hdfs://master:9000/user/spark/eventlog
spark.logConf                                   true' > spark-defaults.conf

#slaves
mv -f slaves slaves.bak
echo 'master
slave1
slave2'>slaves
popd

#install hue
tar zxvf hue-master.tar.gz -C /home/${HADOOP_USER_NAME}/Programs/lib/
#grep sparkS -lr /home/${HADOOP_USER_NAME}/Programs/lib/hue-master | xargs sed -i "s/sparkS/master/g"
sed -i "s/sparkS/master/g" /home/${HADOOP_USER_NAME}/Programs/lib/hue-master/desktop/conf/pseudo-distributed.ini 
ln -s /home/${HADOOP_USER_NAME}/Programs/lib/hue-master /home/${HADOOP_USER_NAME}/Programs/lib/hue


#end install hue


# change the owner of Programs dirctory
chown -R ${HADOOP_USER_NAME}.${HADOOP_USER_NAME} ${PROGRAMS_PATH}

# give sh a+x
find ${PROGRAMS_PATH} -name '*.sh' | xargs chmod a+x

#install pdsh
yum install pdsh
yum install screen



#send source to slaves
scp -r jpegsrc.v8d.tar.gz requirements.txt newdist.tar.gz Python-2.7.8.tar.xz python_source/ get-pip.py .aliasrc /home/${HADOOP_USER_NAME}/.bashrc /usr/lib/jdk1.7.0_79/ ${PROGRAMS_LIB_PATH}/apache-maven-3.2.5/ ${PROGRAMS_LIB_PATH}/hbase-0.98.8-hadoop2/ ${PROGRAMS_LIB_PATH}/hadoop-2.5.1/ ${PROGRAMS_LIB_PATH}/scala-2.11.4/ ${PROGRAMS_LIB_PATH}/spark-1.2.2/ ${PROGRAMS_LIB_PATH}/zookeeper-3.4.6/ ${PROGRAMS_PATH}/apache-ant-1.9.6/ ${HADOOP_USER_NAME}@slave1:~/source
scp -r jpegsrc.v8d.tar.gz requirements.txt newdist.tar.gz Python-2.7.8.tar.xz python_source/ get-pip.py .aliasrc /home/${HADOOP_USER_NAME}/.bashrc /usr/lib/jdk1.7.0_79/ ${PROGRAMS_LIB_PATH}/apache-maven-3.2.5/ ${PROGRAMS_LIB_PATH}/hbase-0.98.8-hadoop2/ ${PROGRAMS_LIB_PATH}/hadoop-2.5.1/ ${PROGRAMS_LIB_PATH}/scala-2.11.4/ ${PROGRAMS_LIB_PATH}/spark-1.2.2/ ${PROGRAMS_LIB_PATH}/zookeeper-3.4.6/ ${PROGRAMS_PATH}/apache-ant-1.9.6/ ${HADOOP_USER_NAME}@slave2:~/source

#install virtualenvwrapper
pip install virtualenvwrapper

cp .aliasrc /home/hadoop/
cp -r python_source/ /home/hadoop/source/
cp requirements.txt /home/hadoop/source/
cp newdist.tar.gz /home/hadoop/source/
chown -R hadoop.hadoop /home/hadoop/source/

tar zxvf jpegsrc.v8d.tar.gz
pushd jpeg-8d/
./configure
make && make install
cp jpegint.h /usr/local/include/
ldconfig
popd