pyspark group_concat

版权声明：欢迎评论，指点，交流！ https://blog.csdn.net/qq_15230053/article/details/82562901

from pyspark.sql.column import Column
from pyspark.sql.column import _to_java_column
from pyspark.sql.column import _to_seq
 
spark = SparkSession.builder.appName("scala_udf_test").getOrCreate()
sc = spark.sparkContext
 
def group_concat(col):
    _groupConcat = sc._jvm.com.learning.GroupConcat.apply
    return Column(_groupConcat(_to_seq(sc, [col], _to_java_column)))
 
def process():
    rows = [
        ("k1", "a"),
        ("k1", "b"),
        ("k1", "c"),
        ("k2", "d"),
        ("k3", "e"),
        ("k3", "f"),
    ]
    df = spark.createDataFrame(rows, ['key', 'value'])
    df.show(50)
    df.groupBy("key").agg(group_concat("value").alias("concat")).show()
 
if __name__ == "__main__":
    process()