一起养成写作习惯！这是我参与「掘金日新计划 · 4 月更文挑战」的第1天，点击查看活动详情。

hadoop部署记录

本次部署采用联邦方式部署私有云,所以需要四台nn,rm ,如果不采用联邦方式,只需要修改hdfs-set.xml 文件全部改成005 006 主机的配置即可,不用做过多的修改

1. 编写hadoop ansible 主机列表

主机名	部署
hadoopHD001	NN,RM namespace1
hadoopHD002	NN,RM namespace1
hadoopHD003	NN,RM namespace2
hadoopHD004	NN,RM namespace2
hadoopHD005	HIVE,MYSQL,DN,NM
hadoopHD006	HIVE,MYSQL,DN,NM
hadoopHD007	TRINO,DN,NM
hadoopHD008	TRINO,DN,NM
hadoopHD009	ZK,DN,NM,JN
hadoopHD011	ZK,DN,NM,JN
hadoopHD012	ZK,DN,NM,JN
hadoopHD013	ZK,DN,NM,JN
hadoopHD014	ZK,DN,NM,JN
hadoopHD016	DN,NM
其他主机	DN,NM

2. 更新yum源

ansible-playbook -i hadoop_root_list 01_sync_yum.yml

- hosts: all
 tasks:
  - name: remove reop files 删除默认的Yum源
    shell: rm -rf /etc/yum.repos.d/*.repo  

  - name: sync repo files  更新自己的
    copy: src=config/repo/{{ item }} dest=/etc/yum.repos.d/
    with_items:    
    - sys-e17.repo // 自己的yum源
    - CentOS1611.repo
    - CentOS1810.repo
    - CentOS2009.repo
    - rhel73.repo
    - rhel76.repo
    - OpenSSH.repo
    -

3. 安装必要的包

ansible-playbook -i hadoop_root_list 02_install_rpm.yml

 - hosts: all
   tasks:
   - name: Update yum cache
     shell: yum clean all           //清理yum 缓存.更新yun 源

   - name: Basic lib install
     yum: name={{ item }} state=latest update_cache=yes    //安装基本包
     with_items:
     - vim
     - sshpass
     - numactl
     - ftp
     - psmisc

4. 修改主机名

ansible-playbook -i hadoop_root_list 04_renmae_hostname.yml

ansible-playbook -i hadoop_root_list 04_renmae_hostname.yml -f 10 ansible-playbook -i hadoop_root_list 04_renmae_hostname.yml -f 50

---
- hosts: all
  tasks:
  - name: sync hostname_list
    copy: src=scripts/{{ item }} dest=/tmp/{{ item }}
    with_items:
    - hostname_list                      //这个文件里面写,主机名主机ip 主机的ip都是10.10 段的
    - rename_hostname.sh
    
  - name: change hostname
    shell: bash /tmp/rename_hostname.sh

cat rename_hostname.sh
#!/bin/bash
local_ip=`ifconfig|grep '10.10.'|awk '{print $2}'`  //主机的ip都是1010 段的
host_name=`grep -w ${local_ip} /tmp/hostname_list|awk -F';' '{print $1}'`

# 没有获取到IP, 退出 
if [ ${host_name} == "" ]; then 
    echo "没有找到IP(${local_ip})对应的主机名"
    exit 1
fi

# 修改主机名
echo ${host_name} > /etc/hostname
hostname ${host_name}

5. 注册新用户

最好是hadoop有一个单独的用户,这样管理起来比较方便.

ansible-playbook -i hadoop_root_list 22_add_user_hadoop.yml -f 10

- hosts: all
  vars:
    user: hadoop
    group: hadoop
    password: "123155"
  tasks:
  - name: Add group {{ group }}    
    shell: groupadd -g 600 {{ group }} 
  
  - name: Add user {{ user }}
    user: name={{ user }} group={{ group }} password={{ password }} uid=601

  - name: Config /etc/sudoers
    lineinfile: dest=/etc/sudoers state=present  line='{{item}}' validate='visudo -cf %s'
    with_items:
    - "{{ user}} ALL=(ALL) NOPASSWD: ALL"

6. 查看selinux 已是关闭

ansible -i hadoop_root_list all -m shell -a " source ~/.bash_profile&&sestatus -v" //检查selinux

7. 查看防火请已是关闭

ansible -i hadoop_root_list all -m shell -a " source ~/.bash_profile&& systemctl status firewalld" // 检查防火墙

8. Ntp服务 ntp 主机还没定所以没配置

9. 更新hosts 文件

ansible -i hadoop_root_list all -m copy -a " src=~/senmao/user/hosts dest=/tmp/hosts " //分发hosts 文件 ansible -i hadoop_root_list all -m shell -a "ls /tmp" // 检查是否全部主机都已获得文件 ansible -i hadoop_root_list all -m shell -a "head /tmp/hosts" ansible -i hadoop_root_list all -m shell -a "cat /tmp/hosts >> /etc/hosts " //更新主机的hosts 文件 ansible -i hadoop_root_list all -m shell -a "cat /etc/hosts " // 检查

10. 安装jdk

ansible-playbook -i ../hadoop_list 05_install_jdk.yml -f 10

  tasks:
   .- name: unarchive jdk  //解压分发jdk 
     unarchive: src=/home/bilxf/senmao/user/jdk-8u201-linux-x64.tar.gz dest=/home/hadoop/apps/    
   .- name: Link jdk          //为jdk创建软连接,方便以后升级jdk 
     file: src=/home/hadoop/apps/jdk1.8.0_201 dest=/home/hadoop/apps/jdk state=link

ulimits调整

ansible -i hadoop_root_list all -m shell -a "echo 'hadoop soft nofile 102642   //用户打开文件数限制修改
hadoop hard nofile 102642
hadoop soft nproc unlimited    //用户进程数限制打开 不限制进程数量
hadoop hard nproc unlimited
hadoop soft memlock unlimited  //用户使用内存打开  不限制内存使用
hadoop hard memlock unlimited' >> /etc/security/limits.conf
"

hadoop部署记录1--部署前准备工作