Writing and appending multiple csv data into new csv using python

shikha

I have a directory where there are multiple csv files. Currently I am able to read all the files sequentially using for loop and display their contents. I need to to write the contents from all the csv files sequentially into a new csv file but I am missing something as in my new csv has no data in it.

this is what I am doing :

import os
import csv     

path = r'C:\Users\hu170f\Documents\WORK\MAAP_FILE_DB_REQ\\' 
fileNames = os.listdir(path)

for f in fileNames:
    file = open(path+f)
    csvreader = csv.reader(file)
    
    rows = []
    for row in csvreader:
        rows.append(row)
      
    for i in rows:
        print(i)
        #OFile = open('C:\Users\hu170f\Documents\WORK\MAAP_FILE_DB_REQ\ALL_DATA.csv','w')
        writer = csv.writer(open('C:\Users\hu170f\Documents\WORK\MAAP_FILE_DB_REQ\ALL_DATA.csv', 'wb'))
        #for row in csvreader:
         #   row1 = csvreader.next()
        writer.writerow(i)

chill0r

You are overwriting the file each row you try to write. Using the w argument for the open method will overwrite existing files.

The argument you need to use in the case you want to append to files (or create new files if non-existing) is a

See Python File Open for more informations about python file modes.

import os
import csv     

path = r'C:\Users\hu170f\Documents\WORK\MAAP_FILE_DB_REQ' 
fileNames = os.listdir(path)

with open('C:\Users\hu170f\Documents\WORK\MAAP_FILE_DB_REQ\ALL_DATA.csv', 'a') as output:    
    writer = csv.writer(output)
    for f in fileNames:
        with open(os.path.join(path, f), "r") as file:
            csvreader = csv.reader(file)
            for row in csvreader:
                print(row)
                writer.writerow(row)

If the csv files have the same columns and formats you could also simply copy the first file and append the others, excluding their headers.

import os
import shutil

path = r'C:\Users\hu170f\Documents\WORK\MAAP_FILE_DB_REQ'
fileNames = os.listdir(path)
output = r'C:\Users\hu170f\Documents\WORK\MAAP_FILE_DB_REQ\ALL_DATA.csv'

# Copy the first file:
shutil.copyfile(os.path.join(path,fileNames[0]), output)

# Append the remaining file contents, excluding each first line
with open(output, 'a') as out:
    for file in fileNames[1:]:
        with open(os.path.join(path, file), 'r') as in_:
            out.write(''.join(in_.readlines()[1:]))