Hello David, you can use something like:

from pyspark.sql import SparkSession
from pyspark.sql.functions import regexp_extract, to_timestamp

# Initialize Spark session
spark = SparkSession.builder.appName("ExtractDatetime").getOrCreate()

# Sample data
data = [("This_is_new_file_2024_12_06T11_00_49_AM.csv",)]
df = spark.createDataFrame(data, ["filename"])

# Extract the datetime string using regexp_extract
datetime_pattern = r"(\d{4}_\d{2}_\d{2}T\d{2}_\d{2}_\d{2})"
df = df.withColumn("datetime_str", regexp_extract("filename", datetime_pattern, 1))

# Convert the extracted string to a datetime data type
df = df.withColumn("datetime", to_timestamp("datetime_str", "yyyy_MM_dd'T'HH_mm_ss"))

# Show the result
df.show(truncate=False)