{"id":1461,"date":"2024-05-08T11:14:51","date_gmt":"2024-05-08T03:14:51","guid":{"rendered":"http:\/\/oneai.eu.org\/?p=1461"},"modified":"2024-05-08T11:14:51","modified_gmt":"2024-05-08T03:14:51","slug":"hadoop%e5%88%86%e5%b8%83%e5%bc%8f%e9%83%a8%e7%bd%b2","status":"publish","type":"post","link":"https:\/\/oneai.eu.org\/?p=1461","title":{"rendered":"hadoop\u5206\u5e03\u5f0f\u90e8\u7f72"},"content":{"rendered":"
\u673a\u5668\u5217\u8868\uff1a\n10.200.25.154   \u4e3b\n10.200.4.212        \u4ece\n10.200.27.126        \u4ece\n1.\u5efa\u7acb\u4fe1\u7528\u673a\u5236,ssh\u65e0\u9700\u5bc6\u7801\u767b\u5f55\nssh-keygen -t dsa -P '' -f ~\/.ssh\/id_dsa\ncat ~\/.ssh\/id_dsa.pub >> ~\/.ssh\/authorized_keys\nssh localhost\n------------\n2\u5b89\u88c5\u597d\u662fjdk\n------------\n3.hadoop\u914d\u7f6e\uff1a\ncd \/home\/zjy\/hadoop\/etc\/hadoop\n3.1:core-site.xml:\n<property>\n          <name>fs.default.name<\/name>\n          <value>hdfs:\/\/10.200.25.154:9000<\/value>\n        <\/property>\n        <property>\n           <name>hadoop.tmp.dir<\/name>\n           <value>\/home\/zjy\/hadoop\/tmp<\/value>\n        <\/property>\n\n3.2: hdfs-site.xml:\n <property>\n         <name>dfs.replication<\/name>\n         <value>1<\/value>\n  <\/property>\n  <property>\n   <name>dfs.permissions<\/name>\n   <value>false<\/value>\n  <\/property>\n\n3.3 mapred-site.xml :\n<configuration>  \n    <property>  \n        <name>mapreduce.jobtracker.address<\/name>  \n        <value>10.200.25.154:9001<\/value>  \n        <description>jobtracker's address<\/description>  \n    <\/property>  \n<\/configuration>\n----------------\n\u6e05\u5355\u914d\u7f6e\u6210\u4e0b\u9762\u8fd9\u79cd\u5f62\u5f0f\uff1a\netc\/hadoop\/mapred-site.xml\n<configuration>\n    <property>\n        <name>mapreduce.framework.name<\/name>\n        <value>yarn<\/value>\n    <\/property>\n<\/configuration>\n\n \u6e05\u5355 4.Yarn-site.xml \u914d\u7f6e\uff1a   \n\n <!-- Site specific YARN configuration properties --> \n  <property> \n  <name>Yarn.nodemanager.aux-services<\/name> \n  <value>mapreduce.shuffle<\/value> \n  <\/property> \n  <property> \n  <description>The address of the applications manager interface in the RM.<\/description> \n  <name>Yarn.resourcemanager.address<\/name> \n  <value>10.200.25.154:18040<\/value> \n  <\/property> \n\n  <property> \n  <description>The address of the scheduler interface.<\/description> \n  <name>Yarn.resourcemanager.scheduler.address<\/name> \n  <value>10.200.25.154:18030<\/value> \n  <\/property> \n\n  <property> \n  <description>The address of the RM web application.<\/description> \n  <name>Yarn.resourcemanager.webapp.address<\/name> \n  <value>10.200.25.154:18088<\/value> \n  <\/property> \n\n  <property> \n  <description>The address of the resource tracker interface.<\/description> \n  <name>Yarn.resourcemanager.resource-tracker.address<\/name> \n  <value>10.200.25.154:9025<\/value> \n  <\/property> \n\n------\nmasters \u6587\u4ef6\u914d\u7f6e\n10.200.25.154\nslaves \u6587\u4ef6\u914d\u7f6e\uff1a\n10.200.4.212\n10.200.27.126 \n\n-------\n\n\u6bcf\u53f0\u673a\u5668\u7684\/etc\/hosts\u6dfb\u52a0\u8bbe\u7f6e\uff1a\n###hadoop host list#\n10.200.25.154 YFCS-S6-APP\n10.200.4.212    BOSS20CS-DB\n10.200.27.126   NGCRM-S2-APP\n\n-----------------\n4.\u73af\u5883\u53d8\u66f4\u914d\u7f6e\uff1avi ~\/.profile            vi ~\/.bashsrc\n\nexport JAVA_HOME=\/opt\/jdk1.8.0_25\nexport JRE_HOME=\/opt\/jdk1.8.0_25\/jre\nexport HADOOP_HOME=\/hadoop\/hadoop\nexport HADOOP_HOME_WARN_SUPPRESS=1\nexport CLASSPATH=.:$JAVA_HOME\/lib:$JRE_HOME\/lib:$HADOOP_HOME\/lib:$CLASSPATH\nexport PATH=$JAVA_HOME\/bin:$JRE_HOME\/bin:$HADOOP_HOME\/bin:$PATH\nexport HADOOP_DEV_HOME=\/hadoop\/hadoop\nexport HADOOP_COMMON_HOME=\/hadoop\/hadoop\nexport HADOOP_HDFS_HOME=\/hadoop\/hadoop\nexport HADOOP_CONF_DIR=\/hadoop\/hadoop\/etc\/hadoop\nexport HADOOP_PREFIX=\/home\/zjy\/hadoop\nexport HADOOP_COMMON_LIB_NATIVE_DIR=${HADOOP_PREFIX}\/lib\/native\nexport HADOOP_OPTS="-Djava.library.path=$HADOOP_PREFIX\/lib"\n-----------------------\n\/*find .\/ -depth -name "*"|xargs chown zjy \nfind .\/ -depth -name "*"|xargs chgrp hadoop \n*\/\n------------\n5\u3002\u6267\u884c\u683c\u5f0f\u5316\uff1a\nhadoop namenode -format\n\u6ce8\u610f\uff1acore-site.xml\u3001hdfs-site.xml\u3001mapred-site.xml\u7b49\u914d\u7f6e\u6587\u4ef6\u53d8\u5316\u540e\u90fd\u9700\u8981\u6267\u884c\u683c\u5f0f\u5316\n------------\n6\u3002\u542f\u52a8\uff1a\ncd $HADOOP_HOME\/sbin\n.\/start-all.sh\n---------------\n7,\u68c0\u67e5\uff1a\n\u67e5\u770bhadoop\u542f\u52a8\u7684\u8fdb\u7a0b\n[root@centoshadoop1~]#jps\n\u67e5\u770bhadoop\u96c6\u7fa4\u72b6\u6001\n[root@centoshadoop1~]#hadoop dfsadmin  -report\n\u67e5\u770bhadoop\u4e2d\u7684\u76ee\u5f55\u60c5\u51b5\n[root@centoshadoop1~]#hadoop fs \u2013ls \/\n\n\u73b0\u5728\u5c31\u53ef\u4ee5\u5728\u6d4f\u89c8\u5668\u4e2d\u8f93\u5165\u5730\u5740\uff1a\nhttp:\/\/10.200.25.154:50030\nhttp:\/\/10.200.25.154:50070\n\u5b8c\u6210\u3002\n\u67e5\u770byarn\u662f\u5426\u542f\u52a8\u6210\u529f\nhttp:\/\/10.200.25.154:8088\/cluster\n\n\u67e5\u770bnodemanager \u662f\u5426\u542f\u52a8\u6210\u529f\nhttp:\/\/10.200.25.154:8042\/\n\n\/\/\u8bbe\u7f6e\u4e0e\u4fee\u6539\u65e5\u5fd7\u7ea7\u522b\n\n\u547d\u4ee4\u65b9\u5f0f\uff1abin\/hadoop daemonlog -setlevel ${nodemanager-host}:8042 org.apache.hadoop.yarn.server.nodemanager.NodeManager DEBUG\n--\u7f51\u9875\u5f62\u5f0f\uff1ahttp:\/\/10.200.25.154:8042\/logLevel\n\nLog: \u201corg.apache.hadoop.yarn.server.nodemanager.NodeManager"  Level:  "DEBUG" \u70b9\u51fb"Set Log Level"\n--\u4fee\u6539\u914d\u7f6e\u6587\u4ef6\uff1a\/home\/zjy\/hadoop\/etc\/hadoop\/log4j.properties \n\u6dfb\u52a0 log4j.logger.org.apache.hadoop.yarn.server.nodemanager.NodeManager=DEBUG\n----------------------------------------------------------------------------------------------------\n\nhadoop dfsadmin -safemode leave        \u79bb\u5f00\u5b89\u5168\u6a21\u5f0f\u3002\n\n10.200.27.126 root\/126!scsb\n10.200.4.212  root\/212w#bsc\n\n10.200.25.154 YFCS-S6-APP\n10.200.4.212        BOSS20CS-DB\n10.200.27.126        NGCRM-S2-APP\n172.20.23.101 ycs\n172.20.23.130 zjy        \n\n10.200.4.212\u4e0a\u64cd\u4f5c\uff1a\u6307\u4ee4\uff1a\nvi \/etc\/hosts\n172.20.23.130   master\n10.200.25.154   YFCS-S6-APP\n---\nuseradd  -c "zjy-hadoop-test" zjy\npasswd zjy\n\nusermod -G mysql,oinstall,sshd,ftp,www,tester,dialout,video zjy\nmkdir \/home\/zjy\nchown zjy \/home\/zjy\nchgrp tester zjy\n\nusermod -G oinstall,sshd,ftp,www,dialout,video zjy\n\n10.200.27.126:\n------------------\nexport PATH=:$PATH:$WAS_HOME\/bin:$JAVA_HOME\/bin:$ANT_HOME\/bin:$FINDBUGS_HOME\/bin:\/bin:\/usr\/bin:.\nexport LANG="zh_CN.UTF-8"\nexport LANGUAGE="zh_CN:zh_GB:zh"\nexport LC_CTYPE="zh_CN.GBK"\nexport LD_LIBRARY_PATH=\/opt\/gnome\/lib64:\/opt\/gnome\/lib\/\n\n###hadoop########\n\nexport JAVA_HOME=\/opt\/jdk1.7.0_45\nexport JRE_HOME=\/opt\/jdk1.7.0_45\/jre\nexport HADOOP_HOME=\/home\/zjy\/hadoopp\nexport HADOOP_HOME_WARN_SUPPRESS=1\nexport CLASSPATH=.:$JAVA_HOME\/lib:$JRE_HOME\/lib:$HADOOP_HOME\/lib:\/usr\/java\/jdk1.7.0_25\/lib\/tools.jar:\/usr\/java\/jdk1.7.0_25\/lib\/dt.jar:\/usr\/java\/jdk1.7.0_25\/jre\/lib\/rt.jar:$CLASSPATH\nexport PATH=$JAVA_HOME\/bin:$JRE_HOME\/bin:$HADOOP_HOME\/bin:$PATH\nexport HADOOP_DEV_HOME=\/home\/zjy\/hadoop\nexport HADOOP_COMMON_HOME=\/home\/zjy\/hadoop\nexport HADOOP_HDFS_HOME=\/home\/zjy\/hadoop\nexport HADOOP_CONF_DIR=\/home\/zjy\/hadoop\/etc\/hadoop\nalias p='ps -ef |grep -v grep |grep '\nexport PS1='$LOGNAME@`hostname`:`pwd`$'\nhostname=master\nalias l='ls -lrt'\nalias L='ls -lrt'\nset -o vi\n\n172.20.23.101 ysc:\n-------------------------------\nexport JAVA_HOME=\/usr\/local\/jdk1.6.0_38\nexport JRE_HOME=$JAVA_HOME\/jre\nexport CLASSPATH=.:$JAVA_HOME\/lib\/dt.jar:$JAVA_HOME\/lib\/tools.jar:$JRE_HOME\/lib\nexport HADOOP_PREFIX=\/usr\/local\/hadoop\nexport PATH=.:$PATH\nexport HADOOP_HOME=\/usr\/local\/hadoop\nexport HADOOP_HOME_WARN_SUPPRESS=1\nexport CLASSPATH=.:$JAVA_HOME\/lib:$JRE_HOME\/lib:$HADOOP_HOME\/lib:$CLASSPATH\nexport PATH=$JAVA_HOME\/bin:$JRE_HOME\/bin:$HADOOP_HOME\/bin:$PATH\nexport HADOOP_DEV_HOME=\/usr\/local\/hadoop\nexport HADOOP_COMMON_HOME=\/usr\/local\/hadoop\nexport HADOOP_HDFS_HOME=\/usr\/local\/hadoop\nexport HADOOP_CONF_DIR=\/usr\/local\/hadoop\/etc\/hadoop\n\nalias l='ls -lrt'\nalias p='ps -ef|grep -v grep |grep '\n\nscp 10.200.25.154:\/home\/zjy\/.ssh\/id_dsa.pub .\/id_dsa.pub_154\ncat id_dsa.pub_154 >>.ssh\/authorized_keys\n\nscp mapred-site.xml 10.200.4.212:\/home\/zjy\/hadoop\/etc\/hadoop\/mapred-site.xml\nscp mapred-site.xml 10.200.27.126:\/home\/zjy\/hadoop\/etc\/hadoop\/mapred-site.xml\nscp mapred-site.xml 172.20.23.101:\/home\/zjy\/hadoop\/etc\/hadoop\/mapred-site.xml\n\nscp yarn-site.xml 10.200.4.212:\/home\/zjy\/hadoop\/etc\/hadoop\/yarn-site.xml\nscp yarn-site.xml 10.200.27.126:\/home\/zjy\/hadoop\/etc\/hadoop\/yarn-site.xml\nscp yarn-site.xml 172.20.23.101:\/home\/zjy\/hadoop\/etc\/hadoop\/yarn-site.xml\n\nscp hdfs-site.xml 10.200.4.212:\/home\/zjy\/hadoop\/etc\/hadoop\/hdfs-site.xml\nscp hdfs-site.xml 10.200.27.126:\/home\/zjy\/hadoop\/etc\/hadoop\/hdfs-site.xml\n\nyarn-site.xml\n---------------------\n\n\u542f\u505c\uff1a\n\n# MR(Resource Manager, Node Manager & Job History Server).\u90e8\u5206\uff1a\n$ sbin\/yarn-daemon.sh start resourcemanager       \/\/\u542f\u52a8resourcenamager\n$ sbin\/yarn-daemon.sh start nodemanager                                                \/\/\u542f\u52a8nodemanager\n$ sbin\/mr-jobhistory-daemon.sh start historyserver \n\nHadoop Startup\nTo start a Hadoop cluster you will need to start both the HDFS and YARN cluster.\n\nFormat a new distributed filesystem:\n\n$ $HADOOP_PREFIX\/bin\/hdfs namenode -format <cluster_name>Start     \/\/\u683c\u5f0f\u5316\u6587\u4ef6\u7cfb\u7edf\nthe HDFS with the following command, run on the designated NameNode:\n\n$ $HADOOP_PREFIX\/sbin\/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start namenode  \u542f\u52a8namenode\nRun a script to start DataNodes on all slaves:\n\n$ $HADOOP_PREFIX\/sbin\/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs start datanode  \u542f\u52a8dataname\nStart the YARN with the following command, run on the designated ResourceManager:\n\n$ $HADOOP_YARN_HOME\/sbin\/yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager                         \u542f\u52a8 resourcemanager \n\n----        \u5728slaves\u4e0a\u6267\u884c\u7684\u547d\u4ee4\uff1a\nRun a script to start NodeManagers on all slaves:\n\n$ $HADOOP_YARN_HOME\/sbin\/yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager  \/\/\u542f\u52a8\u6240\u6709 nodemanager\nStart a standalone WebAppProxy server. If multiple servers are used with load balancing it should be run on each of them:\n\n$ $HADOOP_YARN_HOME\/bin\/yarn start proxyserver --config $HADOOP_CONF_DIRS                                                \/\/\u542f\u52a8proxyserver\ntart the MapReduce JobHistory Server with the following command, run on the designated server:\n\n$ $HADOOP_PREFIX\/sbin\/mr-jobhistory-daemon.sh start historyserver --config $HADOOP_CONF_DIR\n\n---------------------\nHadoop Shutdown\nStop the NameNode with the following command, run on the designated NameNode:\n\n$ $HADOOP_PREFIX\/sbin\/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop \nnamenodeRun a script to stop DataNodes on all slaves:\n\n$ $HADOOP_PREFIX\/sbin\/hadoop-daemon.sh --config $HADOOP_CONF_DIR --script hdfs stop datanode\nStop the ResourceManager with the following command, run on the designated ResourceManager:\n\n$ $HADOOP_YARN_HOME\/sbin\/yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager\nRun a script to stop NodeManagers on all slaves:\n\n$ $HADOOP_YARN_HOME\/sbin\/yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager\nStop the WebAppProxy server. If multiple servers are used with load balancing it should be run on each of them:\n\n$ $HADOOP_YARN_HOME\/bin\/yarn stop proxyserver --config $HADOOP_CONF_DIR\nStop the MapReduce JobHistory Server with the following command, run on the designated server:\n\n$ $HADOOP_PREFIX\/sbin\/mr-jobhistory-daemon.sh stop historyserver --config $HADOOP_CONF_DIR\n\n-----------\n\u8fd0\u884c\u81ea\u5e26\u7684\u4f8b\u5b50\uff1a\n\nhdfs dfs -mkdir \/input        \/\/ \u521b\u5efa\u6587\u4ef6\u76ee\u5f55\nhdfs dfs -mkdir \/output                                \/\/\n\nhadoop fs -ls \/                        \/\/\u67e5\u770b\u6587\u4ef6\nhadoop dfs -put \/home\/zjy\/hadoop\/tmp\/file1 \/input   \/\/\u4e0a\u4f20\u6587\u4ef6\nhadoop dfs -get \/input\/file1 .\/                                                                                                                \/\/\u4e0b\u8f7d\u6587\u4ef6\n\nzjy@YFCS-S6-APP:\/home\/zjy\/hadoop\/tmp\/input$more file1\nhello hadoop test start\n--\nhadoop dfs -ls \/input\/file1\n-rw-r--r--   3 zjy supergroup         24 2014-11-07 18:25 \/input\/file1\n\nhadoop jar \/home\/zjy\/hadoop\/share\/hadoop\/mapreduce\/hadoop-mapreduce-examples-2.4.1.jar wordcount \/input \/output\nhadoop jar \/home\/zjy\/hadoop\/share\/hadoop\/mapreduce\/hadoop-mapreduce-client-shuffle-2.4.1.jar wordcount \/input \/output\nhadoop jar \/home\/zjy\/hadoop\/share\/hadoop\/mapreduce\/hadoop-mapreduce-client-jobclient-2.4.1-tests.jar wordcount \/input \/output\n\nhadoop jar \/home\/zjy\/hadoop\/share\/hadoop\/mapreduce\/hadoop-mapreduce-client-jobclient-2.4.1.jar wordcount \/input \/output\n\nhadoop fs -cat hdfs:\/\/127.0.0.1:9000\/output\n\nhadoop fs -rmdir hdfs:\/\/127.0.0.1:9000\/output\n\nhadoop fs -ls hdfs:\/\/127.0.0.1:9000\/output\n\nps -ef |grep -v grep |grep hadoop|awk '{print $2}'|xargs kill -9<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"
\u673a\u5668\u5217\u8868\uff1a 10.200.25.154 \u4e3b 10.200.4.212 \u4ece 10.200.27.126 \u4ece 1.\u5efa\u7acb\u4fe1\u7528\u673a\u5236,ssh\u65e0\u9700\u5bc6\u7801\u767b\u5f55 ssh-keygen -t dsa -P '' -f ~\/.ssh\/id_dsa cat ~\/.ssh\/id_dsa.pub >> ~\/.ssh\/authorized_keys ssh localhost ———— 2\u5b89\u88c5\u597d\u662fjdk ———— 3.hadoop\u914d\u7f6e\uff1a cd \/home\/zjy\/hadoop\/etc\/hadoop 3.1:core-site.xml: <property> <name>fs.default.name<\/name> <value>hdfs:\/\/10.200.25.154:9000<\/value> <\/property> <property> <name>hadoop.tmp.dir<\/name> <value>\/home\/zjy\/hadoop\/tmp<\/value> <\/property> 3.2: hdfs-site.xml: <property> <name>dfs.replication<\/name> <value>1<\/value> <\/property> <property> <name>dfs.permissions<\/name> <value>false<\/value> <\/property> 3.3 mapred-site.xml : <configuration> <property> <name>mapreduce.jobtracker.address<\/name> <value>10.200.25.154:9001<\/value> <description>jobtracker's address<\/description> <\/property> <\/configuration> —————- \u6e05\u5355\u914d\u7f6e\u6210\u4e0b\u9762\u8fd9\u79cd\u5f62\u5f0f\uff1a etc\/hadoop\/mapred-site.xml <configuration> <property> <name>mapreduce.framework.name<\/name> <value>yarn<\/value> <\/property> <\/configuration> \u6e05\u5355 4.Yarn-site.xml \u914d\u7f6e\uff1a <!– Site specific YARN configuration properties –> <property> <name>Yarn.nodemanager.aux-services<\/name> <value>mapreduce.shuffle<\/value> <\/property> <property> <description>The address of the applications manager interface in the RM.<\/description> <name>Yarn.resourcemanager.address<\/name> <value>10.200.25.154:18040<\/value> <\/property> <property> <description>The address of the scheduler interface.<\/description> <name>Yarn.resourcemanager.scheduler.address<\/name> <value>10.200.25.154:18030<\/value> <\/property> <property> <description>The address of the RM web …<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_eb_attr":"","footnotes":""},"categories":[85,82],"tags":[],"class_list":["post-1461","post","type-post","status-publish","format-standard","hentry","category-hadoop","category-82"],"_links":{"self":[{"href":"https:\/\/oneai.eu.org\/index.php?rest_route=\/wp\/v2\/posts\/1461","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/oneai.eu.org\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/oneai.eu.org\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/oneai.eu.org\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/oneai.eu.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1461"}],"version-history":[{"count":1,"href":"https:\/\/oneai.eu.org\/index.php?rest_route=\/wp\/v2\/posts\/1461\/revisions"}],"predecessor-version":[{"id":1462,"href":"https:\/\/oneai.eu.org\/index.php?rest_route=\/wp\/v2\/posts\/1461\/revisions\/1462"}],"wp:attachment":[{"href":"https:\/\/oneai.eu.org\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1461"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/oneai.eu.org\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1461"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/oneai.eu.org\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1461"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}