Re: Partially upload data of 1.2GB

RohitKulkarni · ‎07-23-2024

I have read the file with below script

import os

def split_file_by_size(file_path, chunk_size_mb=500😞

base_name, ext = os.path.splitext(file_path)

chunk_size = chunk_size_mb * 1024 * 1024 # Convert MB to bytes

part_number = 1

buffer = ''

total_records = 0

file_records = {}

with open(file_path, 'r') as infile:

while True:

chunk = infile.readlines(chunk_size)

if not chunk:

if buffer:

part_file_name = f"{base_name}.part{part_number}{ext}"

record_count = write_and_count(buffer, part_file_name)

if record_count > 0:

file_records[part_file_name] = record_count

break

buffer += ''.join(chunk)

lines = buffer.splitlines()

if len(buffer) >= chunk_size:

# Find the last complete record

partial_chunk = ''.join(chunk)

if '\n' in partial_chunk:

last_complete_line = partial_chunk.rfind('\n') + 1

part_file_name = f"{base_name}.part{part_number}{ext}"

current_chunk = buffer[:last_complete_line]

record_count = write_and_count(current_chunk, part_file_name)

if record_count > 0:

file_records[part_file_name] = record_count

buffer = buffer[last_complete_line:]

part_number += 1

if buffer:

part_file_name = f"{base_name}.part{part_number}{ext}"

record_count = write_and_count(buffer, part_file_name)

if record_count > 0:

file_records[part_file_name] = record_count

total_records = sum(file_records.values())

for file_name, count in file_records.items():

print(f"{file_name}: {count} records")

print(f"Total records: {total_records}")

def write_and_count(data, file_path😞

# Only write if data is not empty

if data.strip():

with open(file_path, 'w') as outfile:

outfile.write(data)

return count_records(file_path)

return 0

def count_records(file_path😞

with open(file_path, 'r') as infile:

return sum(1 for line in infile if line.strip()) # Count non-empty lines

container_name = 'marketaccess' # replace with your actual container name

input_file_path = f"/dbfs/mnt/{container_name}/pn_2022_paticd_diag.txt"

split_file_by_size(input_file_path)

It is not giving any error. The data is getting loaded Partially.

Regards

Rohit