|
|
主控终端 |
| 主机名 |
ubuntuhadoop.smartmap.com |
| IP |
192.168.1.60 |
| Subnet mask |
255.255.255.0 |
| Gateway |
192.168.1.1 |
| DNS |
218.30.19.50 |
|
|
61.134.1.5 |
| Search domains |
smartmap.com |
|
|
|
|
|
|
|
|
|
1. 设置网络IP
sudo nmtui
sudo /etc/init.d/networking restart
2. 设置主机名
sudo hostnamectl set-hostname=ubuntuhadoop.smartmap.com
sudo gedit /etc/hosts
127.0.0.1 localhost
# 127.0.0.1 ubuntuhadoop.smartmap.com
# 127.0.0.1 ubuntuhadoop
192.168.1.60 ubuntuhadoop
192.168.1.60 ubuntuhadoop.smartmap.com
# The following lines are desirable for IPv6 capable hosts
::1 ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
3. 关闭防火墙
sudo ufw disable
4. 安装VMwareTool
https://www.linuxidc.com/Linux/2016-04/130807.htm
5. 安装SSH
5.1. 安装SSH服务
sudo apt-get install -y openssh-server
5.2. 设置SSH
sudo vi /etc/ssh/sshd_config
PermitRootLogin yes
5.3. 启动SSH服务
sudo service ssh start
sudo service ssh restart
5.4. 节点间无密码互访
5.4.1. zyx用户
cd ~
ssh-keygen -t rsa
cp .ssh/id_rsa.pub .ssh/authorized_keys
5.4.2. root用户
sudo su – root
cd ~
ssh-keygen -t rsa
cp .ssh/id_rsa.pub .ssh/authorized_keys
exit
6. 设置vim（Ubuntu中的vim太难用了）
sudo gedit /etc/vim/vimrc.tiny
" set compatible
set nocompatible
set backspace=2
" vim: set ft=vim:
7. 安装JDK
7.1. 加入Oracle的JDK仓库
sudo add-apt-repository ppa:webupd8team/java
7.2. 更新
sudo apt-get update
7.3. 安装
sudo apt-get install oracle-java8-installer
注意：java默认安装在 /usr/lib/jvm文件夹下
7.4. 环境变量配置
sudo gedit /etc/profile
export JAVA_HOME=/usr/lib/jvm/java-8-oracle
export JRE_HOME=$JAVA_HOME/jre
export CLASSPATH=.:$CLASSPATH:$JAVA_HOME/lib:$JRE_HOME/lib
export PATH=$PATH:$JAVA_HOME/bin:$JRE_HOME/bin
export HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.5
export HADOOP_MAPRED_HOME=$HADOOP_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export HADOOP_HDFS_HOME=$HADOOP_HOME
export YARN_HOME=$HADOOP_HOME
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
export YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop
export PATH=$PATH:$HADOOP_HOME/sbin
export PATH=$PATH:$HADOOP_HOME/bin
export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:/usr/local/lib:$HADOOP_HOME/lib/native
export JAVA_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH
souce /etc/profile
8. 安装Hadoop
8.1. 解压hadoop
sudo mkdir /usr/local/hadoop
sudo tar zxvf hadoop-2.7.5.tar.gz -C /usr/local/hadoop
8.2. 创建目录
mkdir /home/zyx/tmp
mkdir /home/zyx/dfs
mkdir /home/zyx/dfs/name
mkdir /home/zyx/dfs/data
mkdir /home/zyx/dfs/checkpoint
mkdir /home/zyx/yarn
mkdir /home/zyx/yarn/local
8.3. 修改Hadoop的配置文件
各配置文件在/usr/local/hadoop/hadoop-2.7.5/etc/hadoop/目录下
8.3.1. hadoop-env.sh
# The java implementation to use.
# export JAVA_HOME=${JAVA_HOME}
export JAVA_HOME=/usr/lib/jvm/java-8-oracle
export HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.5
export PATH=$PATH:/usr/local/hadoop/hadoop-2.7.5/bin
8.3.2. yarn-env.sh
# some Java parameters
# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
export JAVA_HOME=/usr/lib/jvm/java-8-oracle
8.3.3. core-site.xml
<configuration>
<property>
<name>fs.default.name</name>
<value>hdfs://192.168.1.60:9000</value>
</property>
<property>
<name>hadoop.tmp.dir</name>
<value>/home/zyx/tmp</value>
</property>
</configuration>
8.3.4. hdfs-site.xml
<configuration>
<property>
<name>dfs.namenode.http-address</name>
<value>192.168.1.60:50070</value>
</property>
<property>
<name>dfs.namenode.http-bind-host</name>
<value>192.168.1.60</value>
</property>
<property>
<name>dfs.namenode.secondary.http-address</name>
<value>192.168.1.60:50090</value>
</property>
<property>
<name>dfs.namenode.name.dir</name>
<value>/home/zyx/dfs/name</value>
</property>
<property>
<name>dfs.datanode.data.dir</name>
<value>/home/zyx/dfs/data</value>
</property>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
<property>
<name>dfs.namenode.checkpoint.dir</name>
<value>/home/zyx/dfs/checkpoint</value>
</property>
</configuration>
8.3.5. mapred-site.xml
sudo cp /usr/local/hadoop/hadoop-2.7.5/etc/hadoop/mapred-site.xml.template /usr/local/hadoop/hadoop-2.7.5/etc/hadoop/mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
</configuration>
8.3.6. yarn-site.xml
<configuration>
<property>
<name>yarn.resourcemanager.hostname</name>
<value>192.168.1.60</value>
</property>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.local-dirs</name>
<value>/home/zyx/yarn/local</value>
</property>
</configuration>
8.3.7. slave
sudo gedit /usr/local/hadoop/hadoop-2.7.5/etc/hadoop/slaves
192.168.1.60
9. Hadoop启动
9.1. HDFS文件的格式化
注意在zyx用户下
hadoop namenode -format
9.2. DFS服务的启动与关停
注意在root用户下
cd /usr/local/hadoop/hadoop-2.7.5/sbin/
sudo ./start-dfs.sh
sudo ./stop-dfs.sh
9.3. 查看数据节点信息
hadoop dfsadmin -report
9.4. YARN服务的启动与关停
注意在root用户下
cd /usr/local/hadoop/hadoop-2.7.5/sbin/
sudo ./start-yarn.sh
sudo ./stop-yarn.sh
9.5. 查看本机上已启动的服务
sudo jps
9.6. HDFS的Web UI管理界面
http://192.168.1.60:50070/dfshealth.html#tab-overview
9.7. MapReduce应用的Web UI管理界面
http://192.168.1.60:8088/cluster
10. 安装Eclipse
10.1. 解压eclipse
sudo mv /home/zyx/software/eclipse-java-neon-3-linux-gtk-x86_64.tar.gz /opt/
sudo tar zxvf /opt/eclipse-java-neon-3-linux-gtk-x86_64.tar.gz
sudo mkdir /opt/eclipse/workspace
10.2. 安装插件
sudo mv /home/zyx/software/hadoop-eclipse-plugin-2.7.3.jar /opt/
sudo cp /opt/hadoop-eclipse-plugin-2.7.3.jar /opt/eclipse/dropins/
ls -la dropins/
10.3. 启动Eclipse
cd /opt/eclipse/
sudo ./eclipse
11. 准备数据
11.1. 查找应用数据
sudo cp /usr/local/hadoop/hadoop-2.7.5/NOTICE.txt /home/zyx/test.txt
sudo chown zyx:zyx /home/zyx/test.txt
11.2. 将数据放入HDFS中
hadoop fs -mkdir /input
hadoop fs -put /home/zyx/test.txt /input
hadoop fs -chmod -R 777 /input/test.txt
12. 配置Eclipse中的Hadoop设置
12.1. 设置Hadoop安装路径
12.2. 设Eclipse的Hadoop视图
12.3. 连接HDFS
12.4. 新建MapReduce工程
12.5. 新建运行的类
WordCount类中的内容
package org.apache.hadoop.examples;
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class WordCount {
public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(Object key, Text value, Context context) throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
word.set(itr.nextToken());
context.write(word, one);
}
}
}
public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {
private IntWritable result = new IntWritable();
public void reduce(Text key, Iterable<IntWritable> values, Context context)
throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
Job job = Job.getInstance(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
FileInputFormat.addInputPath(job, new Path(args[0]));
FileOutputFormat.setOutputPath(job, new Path(args[1]));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
12.6. 新建日志配置文件log4j.properties
在src/目录下新建日志配置文件log4j.properties
其内容如下：
log4j.rootLogger=debug, stdout, R
#log4j.rootLogger=stdout, R
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
#log4j.appender.stdout.layout.ConversionPattern=%5p - %m%n
log4j.appender.stdout.layout.ConversionPattern=%d %p [%c] - %m%n
log4j.appender.R=org.apache.log4j.RollingFileAppender
log4j.appender.R.File=log4j.log
log4j.appender.R.MaxFileSize=100KB
log4j.appender.R.MaxBackupIndex=1
log4j.appender.R.layout=org.apache.log4j.PatternLayout
#log4j.appender.R.layout.ConversionPattern=%p %t %c - %m%n
log4j.appender.R.layout.ConversionPattern=%d %p [%c] - %m%n
#log4j.logger.com.codefutures=DEBUG
12.7. 运行
12.7.1. 运行环境设置
12.7.1.1. 设置输入与输出
hdfs://192.168.1.60:9000/input/test.txt
hdfs://192.168.1.60:9000/output
12.7.1.2. 设置环境变量
HADOOP_HOME=/usr/local/hadoop/hadoop-2.7.5
12.8. 启动运行
12.9. 查看运行结果
