Hello I am trying to send multiple post requests to an endpoint, i have a spark dataframe and each column of this dataframe is sent through the payload of the post request.

However when i run this in my notebook, no exception is raised. I'm guessing it is because the requests are executed on spark workers

That's why i wonder how can i get the info if a post request returned a HTTP error 422 or 502 ?

Here's something i tried but with no result

url = "string"
headers = {
    'header': 'XXXXX'
}

def process_partition(partition):
    partition_errors = [] 
    
    for row in partition:
        payload = {
            "col1": str(row["col1"]),
        }
        try:
            response = requests.post(url, json=payload, headers=headers)
            response.raise_for_status()
        except requests.exceptions.HTTPError as err:
            partition_errors.append({"error": str(err), "data": row.asDict()})
        except Exception as e:
            partition_errors.append({"error": str(e), "data": row.asDict()})
    
    return partition_errors

def log_errors(errors):
    if errors:
        for error in errors:
            print(f"Error: {error['error']}, Data: {error['data']}")
    else:
        print("All data has been sent.")

all_errors = []

def collect_partition_errors(partition):
    errors = process_partition(partition)
    all_errors.extend(errors) 

df_websocket_data.foreachPartition(collect_partition_errors)

log_errors(all_errors)