做网站与网页有什么区别,郑州网页设计公司有哪些,中山网页设计培训,卢松松网站怎么做hadoop 2.7.3伪分布式安装 hadoop集群的伪分布式部署由于只需要一台服务器#xff0c;在测试#xff0c;开发过程中还是很方便实用的#xff0c;有必要将搭建伪分布式的过程记录下来#xff0c;好记性不如烂笔头。 hadoop 2.7.3 JDK 1.8.91 到Apache的官网下载hadoop的二进…hadoop 2.7.3伪分布式安装 hadoop集群的伪分布式部署由于只需要一台服务器在测试开发过程中还是很方便实用的有必要将搭建伪分布式的过程记录下来好记性不如烂笔头。 hadoop 2.7.3 JDK 1.8.91 到Apache的官网下载hadoop的二进制安装包。 cd /home/fuxin.zhao/soft tar -czvf hadoop 2.7.3.tar.gz cd hadoop-2.7.3 cd etc/hadoop/ pwd 1. 建立本机到本机的免密登录 ssh-keygen -t rsa -P
cat ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys
ssh localhost 1. 修改hadoop的配置文件 位于$HADOOP_HOME/conf目录下的修改四个配置文件slaves、core-site.xml hdfs-site.xml 、mapred-site.xml 、 yarn-site.xml vi etc/hadoop/yarn-env.sh export JAVA_HOME/usr/local/jdk vi etc/hadoop/hadoop-env.sh export JAVA_HOME/usr/local/jdk vi slaves ##加入本机的hostname
fuxin.zhaoubuntuServer01:~/soft/hadoop-2.7.3/etc/hadoop$ vi slaves
ubuntuServer01 vi core-site.xml configurationpropertynamefs.defaultFS/namevaluehdfs://ubuntuServer01:9000/value/propertypropertynamehadoop.tmp.dir/namevaluefile:/home/fuxin.zhao/hadoop/tmp/valuedescriptionAbase for other temporary directories./description/property
/configuration vi hdfs-site.xml configurationpropertynamedfs.replication/namevalue1/value/propertypropertynamedfs.namenode.name.dir/namevaluefile:/home/fuxin.zhao/hadoop/tmp/dfs/name/value/propertypropertynamedfs.datanode.data.dir/namevaluefile:/home/fuxin.zhao/hadoop/tmp/dfs/data/value/propertypropertynamedfs.block.size/namevalue67108864/value/property
/configuration vi yarn-site.xml configuration
propertynameyarn.nodemanager.aux-services/namevaluemapreduce_shuffle/value
/property
propertynameyarn.nodemanager.aux-services.mapreduce_shuffle.class/namevalueorg.apache.hadoop.mapred.ShuffleHandler/value
/property
propertynameyarn.scheduler.minimum-allocation-mb/namevalue512/value
/property
propertynameyarn.scheduler.maximum-allocation-mb/namevalue2048/value
/property
propertynameyarn.scheduler.minimum-allocation-vcores/namevalue1/value
/property
propertynameyarn.scheduler.maximum-allocation-vcores/namevalue2/value
/property
/configuration vi mapred-site.xml configuration
propertynamemapreduce.framework.name/namevalueyarn/value
/property
propertynameyarn.app.mapreduce.am.resource.mb/namevalue512/value
/property
propertynamemapreduce.map.memory.mb/namevalue512/value
/property
propertynamemapreduce.reduce.memory.mb/namevalue512/value
/property
/configuration vi .bashrc export JAVA_HOME/usr/local/jdk
export HADOOP_HOME/home/fuxin.zhao/soft/hadoop-2.7.3
export PATH${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:$PATH 配置完成后执行 NameNode 的格式化: ./bin/hdfs namenode -format ./sbin/start-dfs.sh ./sbin/start-yarn.sh mr-jobhistory-daemon.sh start historyserver 查看hdfs的web页面 http://ubuntuserver01:50070/ http://ubuntuserver01:8088/ hadoop fs -ls / hadoop fs -mkdir /user hadoop fs -mkdir /user/fuxin.zhao hadoop fs -touchz textFile 运行官方自带的测试jobteragen and terasort 测试jobteragen and terasort
#在/tmp/terasort/1000000下生成100M数据
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar teragen 1000000 /tmp/terasort/1000000-input#排序输出到/tmp/terasort/1000000-output
hadoop jar $HADOOP_HOME/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.3.jar terasort /tmp/terasort/1000000-input /tmp/terasort/1000000-output#删除临时文件
hadoop fs -rm -r /tmp/terasort/1000000-input
hadoop fs -rm -r /tmp/terasort/1000000-output转载于:https://www.cnblogs.com/honeybee/p/6400709.html
