大数据测试：利用python将两张表链接

from pyspark.sql import SparkSession
from pyspark.sql.types import *
import os


def getUser(spark,path):
    struct1 = StructType([
        StructField("user", StringType(), True),
        StructField("vedios", StringType(), True),
        StructField("id", IntegerType(), True)
    ])
    df = spark.read.csv(path, schema=struct1, sep="\t", header=True)
    df.createOrReplaceTempView("users1")
    df = spark.sql("select * from users1")
    return df


def getMovies(spark,path):
    df = spark.read.csv(path, header=True)
    df.createOrReplaceTempView("movies")
    df = spark.sql("select * from movies ")
    return df


if __name__ == '__main__':
    os.environ['JAVA_HOME'] = 'C:\Program Files\Java\jdk1.8.0_211'
    print(os.path)
    spark = SparkSession \
        .builder \
        .appName("Python Spark SQL basic example") \
        .config("spark.some.config.option", "some-value") \
        .getOrCreate()
    path_user = "C:/Users/Administrator/Desktop/guiliVideo/user/2008/0903/user.txt"
    path_movies="C:/Users/Administrator/Desktop/vedios.txt"
    df1=getUser(spark,path_user)
    df2=getMovies(spark,path_movies)
    df3=df1.join(df2,df1.user==df2.uploader,how='inner')
    df3.createOrReplaceTempView('table1')
    df4=spark.sql('select * from table1 limit 10')
    df4.show()