import os
def split_file_by_size(file_path, chunk_size_mb=500๐
base_name, ext = os.path.splitext(file_path)
chunk_size = chunk_size_mb * 1024 * 1024 # Convert MB to bytes
part_number = 1
buffer = ''
total_records = 0
file_records = {}
with open(file_path, 'r') as infile:
while True:
chunk = infile.readlines(chunk_size)
if not chunk:
if buffer:
part_file_name = f"{base_name}.part{part_number}{ext}"
record_count = write_and_count(buffer, part_file_name)
if record_count > 0:
file_records[part_file_name] = record_count
break
buffer += ''.join(chunk)
lines = buffer.splitlines()
if len(buffer) >= chunk_size:
# Find the last complete record
partial_chunk = ''.join(chunk)
if '\n' in partial_chunk:
last_complete_line = partial_chunk.rfind('\n') + 1
part_file_name = f"{base_name}.part{part_number}{ext}"
current_chunk = buffer[:last_complete_line]
record_count = write_and_count(current_chunk, part_file_name)
if record_count > 0:
file_records[part_file_name] = record_count
buffer = buffer[last_complete_line:]
part_number += 1
if buffer:
part_file_name = f"{base_name}.part{part_number}{ext}"
record_count = write_and_count(buffer, part_file_name)
if record_count > 0:
file_records[part_file_name] = record_count
total_records = sum(file_records.values())
for file_name, count in file_records.items():
print(f"{file_name}: {count} records")
print(f"Total records: {total_records}")
def write_and_count(data, file_path๐
# Only write if data is not empty
if data.strip():
with open(file_path, 'w') as outfile:
outfile.write(data)
return count_records(file_path)
return 0
def count_records(file_path๐
with open(file_path, 'r') as infile:
return sum(1 for line in infile if line.strip()) # Count non-empty lines
container_name = 'marketaccess' # replace with your actual container name
input_file_path = f"/dbfs/mnt/{container_name}/pn_2022_paticd_diag.txt"
split_file_by_size(input_file_path)
It is not giving any error. The data is getting loaded Partially.
Regards
Rohit
