Options
- Mark as New
- Bookmark
- Subscribe
- Mute
- Subscribe to RSS Feed
- Permalink
- Report Inappropriate Content
12-24-2024 06:37 AM - edited 12-24-2024 06:39 AM
@David_Billa how about
import pyspark.sql.functions as F
df_with_datetime = df.withColumn(
'extracted_datetime',
F.to_timestamp(
F.concat(
*[F.split_part(F.col('file_name'), F.lit("_"), F.lit(i)) for i in range(-6, -1)]
),
'yyyyMMdd\'T\'HHmmss'
)
)
display(df_with_datetime)
or
SELECT *,
to_timestamp(
concat_ws('',
split_part(file_name, '_', -6),
split_part(file_name, '_', -5),
split_part(file_name, '_', -4),
split_part(file_name, '_', -3),
split_part(file_name, '_', -2)
),
'yyyyMMdd\'T\'HHmmss'
) AS extracted_datetime
FROM df