【Hadoop】

准备启动 Hadoop 集群添加hadoop用户并设置密码[rootnode1 ~]# useradd hadoop[rootnode1 ~]# passwd hadoop解压软件包[rootnode1 ~]# tar -zxvf jdk-8u181-linux-x64.tar.gz -C /home/hadoop/[rootnode1 ~]# tar -zxvf hadoop-3.3.6.tar.gz -C /home/hadoop/创建软链接[rootnode1 ~]# su - hadoop[hadoopnode1 ~]$ ln -s hadoop-3.3.6/ hadoop[hadoopnode1 ~]$ ln -s jdk1.8.0_181/ jdk添加环境变量[hadoopnode1 ~]$ vim .bash_profile指定 Java 路径[hadoopnode1 ~]$ cd /home/hadoop/hadoop/etc/hadoop[hadoopnode1 hadoop]$ vim hadoop-env.sh启动 Hadoop 的通用命令行客户端[hadoopnode1 hadoop]$ pwd/home/hadoop/hadoop[hadoopnode1 hadoop]$ bin/hadoop单机部署[hadoopnode1 hadoop]$ mkdir input[hadoopnode1 hadoop]$ cp etc/hadoop/*.xml input[hadoopnode1 hadoop]$ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.6.jar grep input output dfs[a-z.][hadoopnode1 hadoop]$ cat output/*伪分布式部署修改 Hadoop 配置文件[hadoopnode1 hadoop]$ vim etc/hadoop/core-site.xmlconfiguration property namefs.defaultFS/name valuehdfs://localhost:9000/value /property /configuration[hadoopnode1 hadoop]$ vim etc/hadoop/hdfs-site.xmlconfiguration property namedfs.replication/name value1/value /property /configuration配置ssh免密[hadoopnode1 hadoop]$ ssh-keygen[hadoopnode1 hadoop]$ ssh-copy-id localhost格式化文件系统[hadoopnode1 hadoop]$ bin/hdfs namenode -format启动 NameNode 守护进程和 DataNode 守护进程[hadoopnode1 hadoop]$ sbin/start-dfs.sh创建执行 MapReduce 作业所需的 HDFS 目录[hadoopnode1 hadoop]$ bin/hdfs dfs -mkdir -p /user/hadoop将输入文件复制到分布式文件系统中[hadoopnode1 hadoop]$ bin/hdfs dfs -mkdir input[hadoopnode1 hadoop]$ bin/hdfs dfs -put etc/hadoop/*.xml input运行一些提供的示例[hadoopnode1 hadoop]$ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.6.jar grep input output dfs[a-z.]检查输出文件将分布式文件系统中的输出文件复制到本地文件系统并进行检查[hadoopnode1 hadoop]$ bin/hdfs dfs -get output output[hadoopnode1 hadoop]$ cat output/*完成后使用以下命令停止守护进程[hadoopnode1 hadoop]$ sbin/stop-dfs.sh完全分布式集群部署新增两台虚拟机所有主机配置解析安装nfs使node2和node3能共享[rootnode1 ~]# yum install -y nfs-utils[rootnode1 ~]# vim /etc/exports/home/hadoop *(rw,sync,anonuid1000,anongid1000)[rootnode1 ~]# systemctl enable --now nfs修改配置[hadoopnode1 ~]$ vim hadoop/etc/hadoop/core-site.xml[hadoopnode1 ~]$ vim hadoop/etc/hadoop/hdfs-site.xml[hadoopnode1 ~]$ vim hadoop/etc/hadoop/workersnode2和node3进行一样操作[rootnode2 ~]# yum install -y nfs-utils[rootnode2 ~]# showmount -e node1[rootnode2 ~]# mount 192.168.247.132:/home/hadoop/ /home/hadoop/[rootnode2 ~]# df[rootnode2 ~]# ll -d /home/hadoop/重新格式化格式化文件系统[hadoopnode1 hadoop]$ pwd/home/hadoop/hadoop[hadoopnode1 hadoop]$ bin/hdfs namenode -format启动 NameNode 守护进程和 DataNode 守护进程[hadoopnode1 hadoop]$ sbin/start-dfs.sh创建执行 MapReduce 作业所需的 HDFS 目录[hadoopnode1 hadoop]$ bin/hdfs dfs -mkdir /user[hadoopnode1 hadoop]$ bin/hdfs dfs -mkdir /user/hadoop将输入文件复制到分布式文件系统中[hadoopnode1 hadoop]$ bin/hdfs dfs -mkdir input[hadoopnode1 hadoop]$ bin/hdfs dfs -put etc/hadoop/*.xml input