Yi Du :
I have three columns about year, month and day. How can I use these to create date in pyspark?
Shu :
You can use concat_ws()
to concat columns with -
and cast to date.
#sampledata
df.show()
#+----+-----+---+
#|year|month|day|
#+----+-----+---+
#|2020| 12| 12|
#+----+-----+---+
from pyspark.sql.functions import *
df.withColumn("date",concat_ws("-",col("year"),col("month"),col("day")).cast("date")).show()
+----+-----+---+----------+
|year|month|day| date|
+----+-----+---+----------+
|2020| 12| 12|2020-12-12|
+----+-----+---+----------+
#dynamic way
cols=["year","month","day"]
df.withColumn("date",concat_ws("-",*cols).cast("date")).show()
#+----+-----+---+----------+
#|year|month|day| date|
#+----+-----+---+----------+
#|2020| 12| 12|2020-12-12|
#+----+-----+---+----------+
#using date_format,to_timestamp,from_unixtime(unix_timestamp) functions
df.withColumn("date",date_format(concat_ws("-",*cols),"yyyy-MM-dd").cast("date")).show()
df.withColumn("date",to_timestamp(concat_ws("-",*cols),"yyyy-MM-dd").cast("date")).show()
df.withColumn("date",to_date(concat_ws("-",*cols),"yyyy-MM-dd")).show()
df.withColumn("date",from_unixtime(unix_timestamp(concat_ws("-",*cols),"yyyy-MM-dd"),"yyyy-MM-dd").cast("date")).show()
#+----+-----+---+----------+
#|year|month|day| date|
#+----+-----+---+----------+
#|2020| 12| 12|2020-12-12|
#+----+-----+---+----------+
Guess you like
Origin http://10.200.1.11:23101/article/api/json?id=376142&siteId=1