ck1
New Contributor II

@David_Billa how about

 

import pyspark.sql.functions as F

df_with_datetime = df.withColumn(
    'extracted_datetime',
    F.to_timestamp(
        F.concat(
            *[F.split_part(F.col('file_name'), F.lit("_"), F.lit(i)) for i in range(-6, -1)]
        ),
        'yyyyMMdd\'T\'HHmmss'
    )
)

display(df_with_datetime)

 

or

 

 

SELECT *,
       to_timestamp(
           concat_ws('', 
               split_part(file_name, '_', -6),
               split_part(file_name, '_', -5),
               split_part(file_name, '_', -4),
               split_part(file_name, '_', -3),
               split_part(file_name, '_', -2)
           ),
           'yyyyMMdd\'T\'HHmmss'
       ) AS extracted_datetime
FROM df