Yi Du:
Tengo tres columnas sobre el año, mes y día. ¿Cómo puedo utilizar estos para crear la fecha en pyspark?
Shu:
Se puede utilizar concat_ws()a concat columnas con -y fundido hasta la fecha.
#sampledata
df.show()
#+----+-----+---+
#|year|month|day|
#+----+-----+---+
#|2020| 12| 12|
#+----+-----+---+
from pyspark.sql.functions import *
df.withColumn("date",concat_ws("-",col("year"),col("month"),col("day")).cast("date")).show()
+----+-----+---+----------+
|year|month|day| date|
+----+-----+---+----------+
|2020| 12| 12|2020-12-12|
+----+-----+---+----------+
#dynamic way
cols=["year","month","day"]
df.withColumn("date",concat_ws("-",*cols).cast("date")).show()
#+----+-----+---+----------+
#|year|month|day| date|
#+----+-----+---+----------+
#|2020| 12| 12|2020-12-12|
#+----+-----+---+----------+
#using date_format,to_timestamp,from_unixtime(unix_timestamp) functions
df.withColumn("date",date_format(concat_ws("-",*cols),"yyyy-MM-dd").cast("date")).show()
df.withColumn("date",to_timestamp(concat_ws("-",*cols),"yyyy-MM-dd").cast("date")).show()
df.withColumn("date",to_date(concat_ws("-",*cols),"yyyy-MM-dd")).show()
df.withColumn("date",from_unixtime(unix_timestamp(concat_ws("-",*cols),"yyyy-MM-dd"),"yyyy-MM-dd").cast("date")).show()
#+----+-----+---+----------+
#|year|month|day| date|
#+----+-----+---+----------+
#|2020| 12| 12|2020-12-12|
#+----+-----+---+----------+