centos7 hadoop 集群安裝步驟

################################################################## change/etc/hosts ################

 

 

##################################################################
change /etc/hosts
##################################################################
#vim /etc/hosts
192.168.0.5 Master Master
192.168.0.6 Slave1 Slave1
192.168.0.4 Slave2 Slave2

##################################################################
close firewall
##################################################################
#systemctl stop firewalld.service
#systemctl disable firewalld.service 

##################################################################
close selinux
##################################################################
#>vim /etc/selinux/config   <---diabled
#>yum -y install java-1.8.0-openjdk*

##################################################################
change hostname
##################################################################
#sudo hostnamectl set-hostname Master|Slave1|Slave2

#ntpdate cn.pool.ntp.org

#vi /etc/environment
JAVA_HOME=/usr/lib/jvm/java-1.8.0/
JRE_HOME=/usr/lib/jvm/java-1.8.0/jre

#vi /etc/profile

export JAVA_HOME=/usr/lib/jvm/java-1.8.0/

export JRE_HOME=${JAVA_HOME}/jre

export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar

export PATH=$PATH:$JAVA_HOME/bin
#source /etc/profile

##################################################################
no password login
##################################################################
1. 
#ssh-keygen -t rsa  (everyone)
2.
cp ~/.ssh/id_rsa.pub ~/.ssh/master.id_rsa.pub 
cp ~/.ssh/id_rsa.pub ~/.ssh/slave1.id_rsa.pub
cp ~/.ssh/id_rsa.pub ~/.ssh/slave2.id_rsa.pub
scp ~/.ssh/slave1.id_rsa.pub master:~/.ssh
scp ~/.ssh/slave2.id_rsa.pub master:~/.ssh
3.
cd ~/.ssh
cat id_rsa.pub >> authorized_keys  (local)
cat slave1.id_rsa.pub >> authorized_keys  
cat slave2.id_rsa.pub >> authorized_keys  
.....(same as slave1 and slave2)
4.
scp authorized_keys slave1:~/.ssh
scp authorized_keys slave2:~/.ssh
5.
vi /etc/ssh/sshd_config

remark '#'

RSAAuthentication yes

PubkeyAuthentication yes

AuthorizedKeysFile      .ssh/authorized_keys

#sudo systemctl restart sshd.service

6. chmod every machine

chmod 700 ~/.ssh

chmod 600 ~/.ssh/authorized_keys

##################################################################
zookeeper cluster install (slave1 + slave2)
##################################################################
1.tar
#tar -zxvf zookeeper-3.4.8.tar.gz -C /opt/
#cd /opt/zookeeper/conf
#cp zoo_sample.cfg zoo.cfg
#vi zoo.cfg
dataDir=/opt/zookeeper/data　　　　#数据目录
dataLogDir=/opt/zookeeper/logs　　#日志目录
server.1=slave1:2888:3888
server.2=slave2:2888:3888
2.create dir
mkdir /opt/zookeeper/data
mkdir /opt/zookeeper/logs
vi /home/zookeeper/data/myid #创建myid文件，并编辑它，编辑的内容就是配置文件中server.后面跟着的号数。例如目前是在slave0机器上，则在myid文件中写入0
3.start
/opt/zookeeper/bin/zkServer.sh start
/opt/zookeeper/bin/zkServer.sh status

##################################################################
hadoop configuration
##################################################################
1.
vi core-site.xml
   <property>
        <name>hadoop.tmp.dir</name>
       <value>/opt/hadoop/tmp</value>
   </property>
   <property>
        <name>fs.defaultFS</name>
       <value>hdfs://master:9000</value>
   </property>
   <property>
       <name>ha.zookeeper.quorum</name>
       <value>HSlave0:2181,HSlave1:2181,HSlave2:2181</value>
   </property>

2.修改 hdfs-site.xml

<configuration>
    <property>
        <name>dfs.replication</name>
        <value>2</value>
    </property>
    <property>
        <name>dfs.namenode.secondary.http-address</name>
            <value>slave1:9001</value>
    </property>
</configuration>

3.修改 mapred-site.xml
<configuration>
        <property>
            <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>

</configuration>

4.修改yarn.site.xml
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
  
  <property>
    <name>yarn.resourcemanager.hostname</name>
    <value>Master</value>
  </property>
</configuration>

5.slaves
#>vi /opt/hadoop/etc/hadoop/slaves
Slave1
Slave2

6. copy the hadoop direction to other slaves(slave1,slave2) 把配置好的hadoop复制到slave节点
#scp -r /opt/hadoop @slave1:/opt
#scp -r /opt/hadoop @slave2:/opt
#scp -r /opt/hadoop @slave3:/opt

7.***并在slave节点上创建/data/tmp目录,同时注意修改权限***

8.先格式化
    $hdfs namenode –format 
    启动
    $start-dfs.sh
    $start-yarn.sh
    或
    $start-all.sh

9.在namenode节点查看进程
#>jps

10.在datanode查看進程
#>jps