1环境基于 上一篇搭建

高可用分布式集群

2  官方提供MapReduce程序

#评估圆周率

cd  /data/hadoop/share/hadoop/mapreduce/
hadoop  jar  hadoop-mapreduce-examples-3.4.0.jar  pi 2 6

 

 

 3 实例项目分析1

#预分析的文件如，如单词统计

#

#上传文件到hdfs
hdfs dfs  -put word.txt /test/01/
#可以先测试下，在运行计算
[root@master11 01]# cat word.txt   |  python  m.py |python r.py
foo	1
quux	1
labs	1
foo	1
bar	1
quux	1
good	1
six	1
good	1
foo	2
quux	1
labs	1
foo	1
bar	1
quux	1
good	1
six	1
good	1
foo	1
# hadoop jar /data/hadoop/share/hadoop/tools/lib/hadoop-streaming-3.4.0.jar  -D stream.non.zero.exit.is.failure=false -mapper /data/test/01/m.py  -reducer /data/test/01/r.py   -input /test/01/  -output /test/output1/

 

 #拷贝文件到服务器本地

hadoop fs -copyToLocal /test/output1/part-00000 /root/part-00000
#查看
[root@master11 ~]# cat  part-00000 
bar	2
foo	6
good	4
labs	2
quux	4
six	2

[root@master11 01]# cat m.py 
#!/usr/bin/env python  import sys  for line in sys.stdin:  line = line.strip()  words = line.split()  for word in words:  print '%s\t%s' % (word, 1)
[root@master11 01]# cat r.py 
#!/usr/bin/env python  from operator import itemgetter  
import sys  current_word = None  
current_count = 0  
word = None  for line in sys.stdin:  line = line.strip()  word, count = line.split('\t', 1)  try:  count = int(count)  except ValueError:  continue  if current_word == word:  current_count += count  else:  if current_word:  print '%s\t%s' % (current_word, current_count)  current_count = count  current_word = word  if current_word == word:  print '%s\t%s' % (current_word, current_count)

 

4项目实战分析2，nginx日志 IP计数

 

hdfs dfs  -put t.log /test/01/
#执行计算
hadoop jar /data/hadoop/share/hadoop/tools/lib/hadoop-streaming-3.4.0.jar  -D stream.non.zero.exit.is.failure=false -mapper /data/test/01/map.py  -reducer /data/test/01/red.py   -input /test/01/t.log  -output /test/output2/

#下载
hadoop fs -copyToLocal /test/output2/part-00000 /root/part-00000
#查看

 

[root@master11 01]# cat  map.py 
#!/usr/bin/python
import sys
import re
for line in sys.stdin:  ipaddress=re.compile(r'([\d.]*)')match=ipaddress.match(line)  if match:ip=match.group(1)print ' %s\t%s' % (ip, 1)
[root@master11 01]# cat red.py 
#!/usr/bin/python
#-*-coding:UTF-8 -*-
import sys
import os
import string
res = {}
for line in sys.stdin:skey=line[0:-1]if(res.has_key(skey)==False): res[skey]=0res[skey]=res[skey]+1
for key in res.keys():print key+"\t"+str(res[key])

5  欢迎同学们一起交流