统计人口性别和身高

自动生成实验数据:
自动生成1000个样例数据
 
  from pyspark import SparkContext,SparkConf 
 
  import random 
 
  def getRandomGender(): 
 
   rand = random.randint(0,2)+1 
 
   if rand % 2 ==0: 
 
   return "M" 
 
   else: 
 
   return "F" 
 
  sc = SparkContext('local','createDataFile') 
 
  outputFile = "file:///usr/local/spark/mycode/exercise/peopleage1.txt" 
 
  Arrays=[] 
 
  for i in range(0,1000): 
 
   height = random.randint(0,230) 
 
   if height < 50: 
 
   height = height +50 
 
   gender = getRandomGender() 
 
   if (height <100 and gender =="M"): 
 
   height= height+100 
 
   if (height <100 and gender =="F"): 
 
   height = height+50 
 
   Arrays.append(str(i) +" "+gender+" "+str(height)) 
 
  rdd = sc.parallelize(Arrays) 
 
  rdd.saveAsTextFile(outputFile) 
 
  统计人口与身高的代码: 
 
    from pyspark import SparkContext 
   
    sc = SparkContext('local','CountPeopleData') 
   
 #读入数据 
   
    RDD = sc.textFile("file:///usr/local/spark/mycode/exercise/people.txt") 
   
    #按空格切割，并且通过filter过滤，得到全是M的新的RDD，然后wordcount 
   
    totalMen = RDD.flatMap(lambda line : line.split(" ")).filter(lambda a : a == "M").map(lambda a : (a,1)).reduceByKey(lambda a,b : (a+b)) 
   
    totalWomen = RDD.flatMap(lambda line: line.split(" ")).filter( lambda a: a =="F").map(lambda a: (a,1)).reduceByKey(lambda a,b :(a+b)) 
   
    #保留性别与身高部分 
   
    totaPeopleHeightinfo = RDD.map(lambda line : (line.split(" ")[1] + " "+line.split(" ")[2])) 
   
    #过滤，留下全是M 的数据，然后从小到大排序，取第一个，为最小 
   
    totalMenLowHieghtinfo = totaPeopleHeightinfo.filter(lambda a : a[0] == "M").map(lambda a: (a,1)).sortByKey().first() 
   
    #过滤，留下全是F的数据，然后设置ascending=False，大到小排序，取第一个，为最大 
   
    totalMenBestHieghtinfo = totaPeopleHeightinfo.filter(lambda a : a[0] == "M").map(lambda a : (a,1)).sortByKey(ascending=False).first() 
   
    #过滤，留下全是M 的数据，然后从小到大排序，取第一个，为最小 
   
    totalWoMenLowHieghtinfo = totaPeopleHeightinfo.filter(lambda a : a[0] == "F").map(lambda a: (a,1)).sortByKey().first() 
   
    #过滤，留下全是F的数据，然后设置ascending=False，大到小排序，取第一个，为最大 
   
    totalWoMenBestHieghtinfo = totaPeopleHeightinfo.filter(lambda a : a[0] == "F").map(lambda a : (a,1)).sortByKey(ascending=False).first() 
   
    print("男生总数:",totalMen.collect()) 
   
    print("女生总数:",totalWomen.collect()) 
   
    print("男生最低身高为:",totalMenLowHieghtinfo) 
   
    print("男生最高身高为:",totalMenBestHieghtinfo) 
   
    print("女生最低身高为:",totalWoMenLowHieghtinfo) 
   
    print("女生最高身高为:",totalWoMenBestHieghtinfo)
统计人口性别和身高

猜你喜欢
