大数据测试:利用spark将二进制文件转换为json格式

from pyspark.sql import SparkSession
import os
if __name__ == '__main__':
    os.environ['JAVA_HOME'] = 'C:\Program Files\Java\jdk1.8.0_211'
    print(os.path)
    spark = SparkSession \
        .builder \
        .appName("Python Spark SQL basic example") \
        .config("spark.some.config.option", "some-value") \
        .getOrCreate()
    path = "C:/Users/Administrator/Desktop/guiliVideo/user_parquet/users01"
    df = spark.read.parquet(path)
    df.createOrReplaceTempView('userTable01')
    df = spark.sql('select * from  userTable01   LIMIT 10')
    df.show()
    df.write.json('F:/user.json03')
    pass
发布了108 篇原创文章 · 获赞 268 · 访问量 19万+

猜你喜欢

转载自blog.csdn.net/pingsha_luoyan/article/details/97918308