05HDFS伪集群搭建HDFS伪集群搭建 $HADOOP_HOME/etc/hadoop/hdfs-site.xml $

HDFS伪集群搭建

# 节点为node01 192.168.91.10，伪集群主要用于测试

# hosts 配置
cat >> /etc/hosts << "EOF"
192.168.91.10  node01
EOF
# JDK安装
wget https://builds.openlogic.com/downloadJDK/openlogic-openjdk/8u392-b08/openlogic-openjdk-8u392-b08-linux-x64.tar.gz
tar xf openlogic-openjdk-8u392-b08-linux-x64.tar.gz
mv openlogic-openjdk-8u392-b08-linux-x64 /usr/local/jdk8
rm -f openlogic-openjdk-8u392-b08-linux-x64.tar.gz
echo 'export JAVA_HOME=/usr/local/jdk8' >> /etc/profile
echo 'export PATH=$PATH:${JAVA_HOME}/bin' >> /etc/profile
source /etc/profile
# 关闭防火墙
systemctl stop firewalld && systemctl disable firewalld
# 关闭selinux
sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
setenforce 0
# 时间同步配置，最小化安装系统需要安装ntpdate软件
yum -y install ntpdate
echo "0 */1 * * * ntpdate time1.aliyun.com" >> /var/spool/cron/root
systemctl enable ntpdate && systemctl start ntpdate
# 生成密钥并拷贝到node01节点
ssh-keygen -t rsa -P '' -f  ~/.ssh/id_rsa
ssh-copy-id node01
# hadoop安装
wget https://dlcdn.apache.org/hadoop/common/hadoop-3.2.4/hadoop-3.2.4.tar.gz
tar xf hadoop-3.2.4.tar.gz
chown -R root:root hadoop-3.2.4
mv hadoop-3.2.4 /usr/local/hadoop-3.2.4
echo 'export HADOOP_HOME=/usr/local/hadoop-3.2.4' >> /etc/profile
echo 'export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin' >> /etc/profile
source /etc/profile
sed -i 's%# export JAVA_HOME=%export JAVA_HOME=/usr/local/jdk8%' $HADOOP_HOME/etc/hadoop/hadoop-env.sh
cat > $HADOOP_HOME/etc/hadoop/workers << "EOF"
node01
EOF
cat > user_tmp.txt << EOF
HDFS_DATANODE_USER=root
HDFS_DATANODE_SECURE_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root
EOF
sed -i '17r user_tmp.txt' $HADOOP_HOME/sbin/start-dfs.sh
sed -i '17r user_tmp.txt' $HADOOP_HOME/sbin/stop-dfs.sh
rm -f user_tmp.txt

$HADOOP_HOME/etc/hadoop/hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>dfs.namenode.secondary.http-address</name>
    <value>node01:50090</value>
  </property>
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
</configuration>

$HADOOP_HOME/etc/hadoop/core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://node01:9000</value>
  </property>
  <property>
    <name>hadoop.tmp.dir</name>
    <value>/var/hadooptmp/single</value>
  </property>
</configuration>

hdfs namenode -format
start-dfs.sh