Styjun

Suppose I have many different text files from the same directory with the content structure as shown below:

File a.txt:

HEADER_X;HEADER_Y;HEADER_Z
a_value;a_value;a_value
a_value;a_value;a_value

File b.txt:

HEADER_X;HEADER_Y;HEADER_Z
b_value;b_value;b_value
b_value;b_value;b_value

File c.txt:

HEADER_X;HEADER_Y;HEADER_Z
c_value;c_value;c_value
c_value;c_value;c_value

File d.txt: ...

I'd like to merge all of the txt files into one, by appending the content of each file to the final row of the each previous file. See below:

File combined.txt:

HEADER_X;HEADER_Y;HEADER_Z
a_value;a_value;a_value
a_value;a_value;a_value
b_value;b_value;b_value
b_value;b_value;b_value
c_value;c_value;c_value
c_value;c_value;c_value
...

How can I do this in Python?

Assumptions:
- all txt files are located in the same folder
- all txt files have same headers
- all txt files have same number of columns
- all txt files have different number of rows

Use the CSV Module. Something like this:

import csv
with ('output.csv', 'ab') as output:
 writer = csv.writer(output, delimiter=";")

 with open('a.txt', 'rb') as csvfile:
 reader = csv.reader(csvfile, delimiter=";")
 reader.readline() // this is to skip the header
 for row in spamreader:
 writer.writerow(row)

If you didn't want to harcode in every file (Say you have many more than three) you could do:

from os import listdir
from os.path import isfile, join
onlyfiles = [ f for f in listdir(mypath) if isfile(join(mypath,f)) ]
 for aFile in onlyfiles:
 with open(aFile, 'rb') as csvfile:
 reader = csv.reader(csvfile, delimiter=";")
 reader.readline() // this is to skip the header
 for row in spamreader:
 writer.writerow(row)

I managed to do something that seems to work (at least in the cases I tested).
This will parse all files, get all the headers and format the values on each line of each file to add ";" according to the headers present/absent on that file.

headers = []
values = []
files = ("csv0.txt", "csv1.txt")#put the files you want to parse here

#read the files a first time, just to get the headers
for file_name in files:
 file = open(file_name, 'r')
 first_line = True
 for line in file:
 if first_line:
 first_line = False
 for header in line.strip().split(";"):
 if header not in headers:
 headers.append(header)
 else:
 break
 file.close()
headers = sorted(headers)

#read a second time to get the values
file_number = 0
for file_name in files:
 file = open(file_name, 'r')
 file_headers = []
 first_line = True
 corresponding_indexes = []
 values.append([])
 for line in file:
 if first_line:
 first_line = False
 index = 0
 for header in line.strip().split(";"):
 while headers[index] != header:
 index += 1
 corresponding_indexes.append(index)
 else:
 line_values = line.strip().split(";")
 current_index = 0
 values_str = ""
 for value in line_values:
 #this part write the values with ";" added for the headers not in this file
 while current_index not in corresponding_indexes:
 current_index += 1
 values_str += ";"
 values_str += value + ";"
 current_index += 1
 values_str = values_str[:-1] #we remove the last ";" (useless)
 values[file_number].append(values_str)
 file_number += 1
 file.close()

#and now we write the output file with all headers and values
headers_str = ""
for header in headers:
 headers_str += header + ";"
headers_str = headers_str[:-1]
output_file = open("output.txt", 'w')
output_file.write(headers_str + "n")
for file_values in values:
 for values_line in file_values:
 output_file.write(values_line + "n")
output_file.close()

If you have any question, feel free to ask.