RDD,Spark SQL,DF群组统计

RDD

#1  RDD性别统计
a=userrdd.map(lambda x:(x[2],1)).reduceByKey(lambda x,y:x+y).collect()
print(a)

#2 性别职业统计
a=userrdd.map(lambda x:((x[2],x[3]),1)).reduceByKey(lambda x,y:x+y).collect()
print(a)

Spark SQL

sqlContxt.sql('''
select gender,count(*) counts from user_table group by gender
''').show()


sqlContxt.sql('''
select gender,occupation,count(*) counts from user_table group by gender,occupation
''').show()

df统计

#df
user_df.select('gender').groupby('gender').count().show()

user_df.select('gender','occupation').groupby('gender','occupation')\
    .count().orderBy('gender','occupation').show(5)