spark wordcount 单词统计

spark wordcount 单词统计

文件1.txt

hello world
hello tom
hello lucy
tom lucy
hello python

# -*- coding:utf-8 -*-
import os
import shutil

from pyspark import SparkContext

inputpath = '1.txt'
outputpath = 'result'

sc = SparkContext('local', 'wordcount')

# 读取文件
input = sc.textFile(inputpath)
# 切分单词
words = input.flatMap(lambda line: line.split(' '))
# 转换成键值对并计数
counts = words.map(lambda word: (word, 1)).reduceByKey(lambda x, y: x + y)

# 输出结果
result=counts.collect()
print result
for (word,count) in result:
    print word,count


# 删除输出目录
if os.path.exists(outputpath):
    shutil.rmtree(outputpath, True)

# 将统计结果写入结果文件
counts.saveAsTextFile(outputpath)