Hadoop【部署 02】hadoop-3，2024大数据开发笔试真题

1. 修改配置

进入 ${HADOOP_HOME}/etc/hadoop/ 目录下，修改以下配置：

1. mapred-site.xml

[root@tcloud ~]# vim /usr/local/hadoop-3.1.3/etc/hadoop/mapred-site.xml

<configuration>
  <property>
	<name>mapreduce.framework.name</name>
    <value>yarn</value>
  </property>
  <!-- 历史服务器地址 -->
  <property>
	<name>mapreduce.jobhistory.address</name>
	<value>tcloud:10020</value>
   </property>
   <property>
	<name>mapreduce.jobhistory.webapp.address</name>
	<value>tcloud:19888</value>
   </property>
</configuration>

2. yarn-site.xml

[root@tcloud ~]# vim /usr/local/hadoop-3.1.3/etc/hadoop/yarn-site.xml

<configuration>
  <property>
    <!--配置NodeManager上运行的附属服务。需要配置成mapreduce\_shuffle后才可以在Yarn上运行MapReduce程序-->
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
  </property>
  <!-- resourcemanager配置 单机版可以在同一台服务器上可以不配置 -->
  <property>
	<name>yarn.resourcemanager.hostname</name>
	<value>tcloud</value>
  </property>
  <!-- 白名单 -->
  <property>
	<name>yarn.nodemanager.env-whitelist</name>	
	<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
  </property>
  
  <!-- 以下配置要根据服务器情况进行配置 我的云服务器只有2G的内存-->
  <property>
	<name>yarn.scheduler.minimum-allocation-mb</name>
	<value>128</value>
  </property>
  <property>
	<name>yarn.scheduler.maximum-allocation-mb</name>
	<value>256</value>
  </property>
  <property>
	<name>yarn.nodemanager.resource.memory-mb</name>
	<value>256</value>
  </property>
  <property>
	<name>yarn.nodemanager.pmem-check-enabled</name>
	<value>false</value>
  </property>
  <property>
	<name>yarn.nodemanager.vmem-check-enabled</name>
	<value>false</value>
  </property>
  
  <!-- 开启日志聚集功能 -->
  <property>
    <name>yarn.log-aggregation-enable</name>
    <value>true</value>
  </property>

  <!-- 设置日志聚集服务器地址 -->
  <property>  
    <name>yarn.log.server.url</name>  
    <value>http://tcloud:19888/jobhistory/logs</value>
  </property>

  <!-- 设置日志保留时间为7天 -->
  <property>
    <name>yarn.log-aggregation.retain-seconds</name>
    <value>604800</value>
  </property>
</configuration>

2.修改启动和停止shell脚本

[root@tcloud hadoop]# vim /usr/local/hadoop-3.1.3/sbin/start-yarn.sh
[root@tcloud hadoop]# vim /usr/local/hadoop-3.1.3/sbin/stop-yarn.sh


![img](https://p9-xtjj-sign.byteimg.com/tos-cn-i-73owjymdk6/1a8adf4786aa4a23b0b65e373bb0c31b~tplv-73owjymdk6-jj-mark-v1:0:0:0:0:5o6Y6YeR5oqA5pyv56S-5Yy6IEAg5py65Zmo5a2m5Lmg5LmL5b-DQUk=:q75.awebp?rk3s=f64ab15b&x-expires=1770657408&x-signature=TmnCSjL1Omvcj9ZufD6yaa%2F7%2FCs%3D)
![img](https://p9-xtjj-sign.byteimg.com/tos-cn-i-73owjymdk6/06e3726228f6414883b218e47972ccf3~tplv-73owjymdk6-jj-mark-v1:0:0:0:0:5o6Y6YeR5oqA5pyv56S-5Yy6IEAg5py65Zmo5a2m5Lmg5LmL5b-DQUk=:q75.awebp?rk3s=f64ab15b&x-expires=1770657408&x-signature=S3%2FtVtRZJzUbhmch5Lmj%2FFVvATA%3D)
![img](https://p9-xtjj-sign.byteimg.com/tos-cn-i-73owjymdk6/926b8e71f7ee46d49cc887fce53d8c0d~tplv-73owjymdk6-jj-mark-v1:0:0:0:0:5o6Y6YeR5oqA5pyv56S-5Yy6IEAg5py65Zmo5a2m5Lmg5LmL5b-DQUk=:q75.awebp?rk3s=f64ab15b&x-expires=1770657408&x-signature=z4rl5FBnNYW1b1fmgeH4ixGKATw%3D)

**既有适合小白学习的零基础资料，也有适合3年以上经验的小伙伴深入学习提升的进阶课程，涵盖了95%以上大数据知识点，真正体系化！**


**由于文件比较多，这里只是将部分目录截图出来，全套包含大厂面经、学习笔记、源码讲义、实战项目、大纲路线、讲解视频，并且后续会持续更新**

**[需要这份系统化资料的朋友，可以戳这里获取](https://gitee.com/vip204888)**