pyspark连接MongoDB
https://blog.csdn.net/qq_42363032/article/details/119608009
from pyspark import SparkContext, SQLContext, SparkConf
from pyspark.sql import SparkSession
import time, sys, warnings
MONGOURI = 'mongodb://172.00.100.11:27017/{}.{}'.format('ad_portrayal', 'user_portrayal_test')
sc = SparkContext(appName="deal_with_portrayal_data")
sqlContext = SQLContext(sc)
ss = SparkSession(sc).builder \
.config("spark.mongodb.input.uri", MONGOURI) \
.config("spark.mongodb.output.uri", MONGOURI) \
.getOrCreate()
warnings.simplefilter(action='ignore')
sc.setLogLevel('ERROR')
PYSPARK_PYTHON=/data/anaconda3/bin/python3 \
/opt/spark/bin/spark-submit \
--master yarn \
--deploy-mode client \
--driver-memory 4g \
--executor-memory 4g \
--driver-cores 4 \
--executor-cores 4 \
--packages org.mongodb.spark:mongo-spark-connector_2.11:2.4.3 \
--name 'deal_with_portrayal_data' \
deal_portData.py >>./log/deal_portData.log 2>>./log/deal_portData.err
pyspark 读MongoDB
# coll = ss.read.format('mongo').option('database', 'ad_portrayal').option('collection', 'user_portrayal_test').load()
coll = ss.read.format('mongo').load()
coll.show()
print(type(coll))
+-----+---------+-------+
|EVENT| REQID| _id|
+-----+---------+-------+
|click|DWADLAWMD|ABCDEFG|
+-----+---------+-------+
<class 'pyspark.sql.dataframe.DataFrame'>
追加的写入
df = ss.createDataFrame([{
'_id': 'HIJKLMN', 'REQID': 'wafldwankl', 'EVENT': 'display'}])
df.write.format("mongo").mode("append").save()
+-------+----------+-------+
| EVENT| REQID| _id|
+-------+----------+-------+
| click| DWADLAWMD|ABCDEFG|
|display|wafldwankl|HIJKLMN|
+-------+----------+-------+
覆盖的写入
df = ss.createDataFrame([{
'_id': 'HIJKLMN', 'REQID': 'wafldwankl', 'EVENT': 'display'}])
df.write.format("mongo").mode("overwrite").save()
+-------+----------+-------+
| EVENT| REQID| _id|
+-------+----------+-------+
|display|wafldwankl|HIJKLMN|
+-------+----------+-------+
<class 'pyspark.sql.dataframe.DataFrame'>