With nodejs I want to parse a .csv file of 10000 records and do some operation on each row. I tried using http://www.adaltas.com/projects/node-csv. I couldnt get this to pause at each row. This just reads through all the 10000 records. I need to do the following:
Can anyone please suggest any alternative ideas here?
csv-parser instead of csv-parse used in some
of the answers above. csv-parser came around 2 years after
csv-parse. csv-parser better, as it is easy to handle headers through it.Install the csv-parser first:
npm install csv-parser
So suppose you have a csv-file like this:
NAME, AGE
Lionel Messi, 31
Andres Iniesta, 34
You can perform the required operation as:
const fs = require('fs');
const csv = require('csv-parser');
fs.createReadStream(inputFilePath)
.pipe(csv())
.on('data', function(data){
try {
console.log("Name is: "+data.NAME);
console.log("Age is: "+data.AGE);
//perform the operation
}
catch(err) {
//error handler
}
})
.on('end',function(){
//some final operation
});
For further reading refer
The node-csv project that you are referencing is completely sufficient for the task of transforming each row of a large portion of CSV data, from the docs at: http://csv.adaltas.com/transform/:
csv()
.from('82,Preisner,Zbigniew\n94,Gainsbourg,Serge')
.to(console.log)
.transform(function(row, index, callback){
process.nextTick(function(){
callback(null, row.reverse());
});
});
From my experience, I can say that it is also a rather fast implementation, I have been working with it on data sets with near 10k records and the processing times were at a reasonable tens-of-milliseconds level for the whole set.
Rearding jurka's stream based solution suggestion: node-csv IS stream based and follows the Node.js' streaming API.
The fast-csv npm module can read data line-by-line from csv file.
Here is an example:
let csv= require('fast-csv');
var stream = fs.createReadStream("my.csv");
csv
.parseStream(stream, {headers : true})
.on("data", function(data){
console.log('I am one line of data', data);
})
.on("end", function(){
console.log("done");
});
My current solution uses the async module to execute in series:
var fs = require('fs');
var parse = require('csv-parse');
var async = require('async');
var inputFile='myfile.csv';
var parser = parse({delimiter: ','}, function (err, data) {
async.eachSeries(data, function (line, callback) {
// do something with the line
doSomething(line).then(function() {
// when processing finishes invoke the callback to move to the next one
callback();
});
})
});
fs.createReadStream(inputFile).pipe(parser);
Try the line by line npm plugin.
npm install line-by-line --save
I used this way:-
var fs = require('fs');
var parse = require('csv-parse');
var csvData=[];
fs.createReadStream(req.file.path)
.pipe(parse({delimiter: ':'}))
.on('data', function(csvrow) {
console.log(csvrow);
//do something with csvrow
csvData.push(csvrow);
})
.on('end',function() {
//do something with csvData
console.log(csvData);
});
I was using csv-parse but for larger files was running into performance issues one of the better libraries I have found is Papa Parse, docs are good, good support, lightweight, no dependencies.
Install papaparse
npm install papaparse
Usage:
const fs = require('fs');
const Papa = require('papaparse');
const csvFilePath = 'data/test.csv'
// Function to read csv which returns a promise so you can do async / await.
const readCSV = async (filePath) => {
const csvFile = fs.readFileSync(filePath)
const csvData = csvFile.toString()
return new Promise(resolve => {
Papa.parse(csvData, {
header: true,
transformHeader: header => header.trim(),
complete: results => {
console.log('Complete', results.data.length, 'records.');
resolve(results.data);
}
});
});
};
const test = async () => {
let parsedData = await readCSV(csvFilePath);
}
test()
const fs = require('fs');
const Papa = require('papaparse');
const csvFilePath = 'data/test.csv'
const file = fs.createReadStream(csvFilePath);
var csvData=[];
Papa.parse(file, {
header: true,
transformHeader: header => header.trim(),
step: function(result) {
csvData.push(result.data)
},
complete: function(results, file) {
console.log('Complete', csvData.length, 'records.');
}
});
Note header: true is an option on the config, see docs for other options
fs = require('fs');
fs.readFile('FILENAME WITH PATH','utf8', function(err,content){
if(err){
console.log('error occured ' +JSON.stringify(err));
}
console.log('Fileconetent are ' + JSON.stringify(content));
})
You can convert csv to json format using csv-to-json module and then you can easily use json file in your program
Ok so there are many answers here and I dont think they answer your question which I think is similar to mine.
You need to do an operation like contacting a database or third part api that will take time and is asyncronus. You do not want to load the entire document into memory due to being to large or some other reason so you need to read line by line to process.
I have read into the fs documents and it can pause on reading but using .on('data') call will make it continous which most of these answer use and cause the problem.
UPDATE: I know more info about Streams than I ever wanted
The best way to do this is to create a writable stream. This will pipe the csv data into your writable stream which you can manage asyncronus calls. The pipe will manage the buffer all the way back to the reader so you will not wind up with heavy memory usage
Simple Version
const parser = require('csv-parser');
const stripBom = require('strip-bom-stream');
const stream = require('stream')
const mySimpleWritable = new stream.Writable({
objectMode: true, // Because input is object from csv-parser
write(chunk, encoding, done) { // Required
// chunk is object with data from a line in the csv
console.log('chunk', chunk)
done();
},
final(done) { // Optional
// last place to clean up when done
done();
}
});
fs.createReadStream(fileNameFull).pipe(stripBom()).pipe(parser()).pipe(mySimpleWritable)
Class Version
const parser = require('csv-parser');
const stripBom = require('strip-bom-stream');
const stream = require('stream')
// Create writable class
class MyWritable extends stream.Writable {
// Used to set object mode because we get an object piped in from csv-parser
constructor(another_variable, options) {
// Calls the stream.Writable() constructor.
super({ ...options, objectMode: true });
// additional information if you want
this.another_variable = another_variable
}
// The write method
// Called over and over, for each line in the csv
async _write(chunk, encoding, done) {
// The chunk will be a line of your csv as an object
console.log('Chunk Data', this.another_variable, chunk)
// demonstrate await call
// This will pause the process until it is finished
await new Promise(resolve => setTimeout(resolve, 2000));
// Very important to add. Keeps the pipe buffers correct. Will load the next line of data
done();
};
// Gets called when all lines have been read
async _final(done) {
// Can do more calls here with left over information in the class
console.log('clean up')
// lets pipe know its done and the .on('final') will be called
done()
}
}
// Instantiate the new writable class
myWritable = new MyWritable(somevariable)
// Pipe the read stream to csv-parser, then to your write class
// stripBom is due to Excel saving csv files with UTF8 - BOM format
fs.createReadStream(fileNameFull).pipe(stripBom()).pipe(parser()).pipe(myWritable)
// optional
.on('finish', () => {
// will be called after the wriables internal _final
console.log('Called very last')
})
OLD METHOD:
PROBLEM WITH readable
const csv = require('csv-parser');
const fs = require('fs');
const processFileByLine = async(fileNameFull) => {
let reading = false
const rr = fs.createReadStream(fileNameFull)
.pipe(csv())
// Magic happens here
rr.on('readable', async function(){
// Called once when data starts flowing
console.log('starting readable')
// Found this might be called a second time for some reason
// This will stop that event from happening
if (reading) {
console.log('ignoring reading')
return
}
reading = true
while (null !== (data = rr.read())) {
// data variable will be an object with information from the line it read
// PROCESS DATA HERE
console.log('new line of data', data)
}
// All lines have been read and file is done.
// End event will be called about now so that code will run before below code
console.log('Finished readable')
})
rr.on("end", function () {
// File has finished being read
console.log('closing file')
});
rr.on("error", err => {
// Some basic error handling for fs error events
console.log('error', err);
});
}
You will notice a reading flag. I have noticed that for some reason right near the end of the file the .on('readable') gets called a second time on small and large files. I am unsure why but this blocks that from a second process reading the same line items.
I use this simple one: https://www.npmjs.com/package/csv-parser
Very simple to use:
const csv = require('csv-parser')
const fs = require('fs')
const results = [];
fs.createReadStream('./CSVs/Update 20191103C.csv')
.pipe(csv())
.on('data', (data) => results.push(data))
.on('end', () => {
console.log(results);
console.log(results[0]['Lowest Selling Price'])
});
npm install csv
Sample CSV file You're going to need a CSV file to parse, so either you have one already, or you can copy the text below and paste it into a new file and call that file "mycsv.csv"
ABC, 123, Fudge
532, CWE, ICECREAM
8023, POOP, DOGS
441, CHEESE, CARMEL
221, ABC, HOUSE
1
ABC, 123, Fudge
2
532, CWE, ICECREAM
3
8023, POOP, DOGS
4
441, CHEESE, CARMEL
5
221, ABC, HOUSE
Sample Code Reading and Parsing the CSV file
Create a new file, and insert the following code into it. Make sure to read through what is going on behind the scenes.
var csv = require('csv');
// loads the csv module referenced above.
var obj = csv();
// gets the csv module to access the required functionality
function MyCSV(Fone, Ftwo, Fthree) {
this.FieldOne = Fone;
this.FieldTwo = Ftwo;
this.FieldThree = Fthree;
};
// Define the MyCSV object with parameterized constructor, this will be used for storing the data read from the csv into an array of MyCSV. You will need to define each field as shown above.
var MyData = [];
// MyData array will contain the data from the CSV file and it will be sent to the clients request over HTTP.
obj.from.path('../THEPATHINYOURPROJECT/TOTHE/csv_FILE_YOU_WANT_TO_LOAD.csv').to.array(function (data) {
for (var index = 0; index < data.length; index++) {
MyData.push(new MyCSV(data[index][0], data[index][1], data[index][2]));
}
console.log(MyData);
});
//Reads the CSV file from the path you specify, and the data is stored in the array we specified using callback function. This function iterates through an array and each line from the CSV file will be pushed as a record to another array called MyData , and logs the data into the console to ensure it worked.
var http = require('http');
//Load the http module.
var server = http.createServer(function (req, resp) {
resp.writeHead(200, { 'content-type': 'application/json' });
resp.end(JSON.stringify(MyData));
});
// Create a webserver with a request listener callback. This will write the response header with the content type as json, and end the response by sending the MyData array in JSON format.
server.listen(8080);
// Tells the webserver to listen on port 8080(obviously this may be whatever port you want.)
1
var csv = require('csv');
2
// loads the csv module referenced above.
3
?
4
var obj = csv();
5
// gets the csv module to access the required functionality
6
?
7
function MyCSV(Fone, Ftwo, Fthree) {
8
this.FieldOne = Fone;
9
this.FieldTwo = Ftwo;
10
this.FieldThree = Fthree;
11
};
12
// Define the MyCSV object with parameterized constructor, this will be used for storing the data read from the csv into an array of MyCSV. You will need to define each field as shown above.
13
?
14
var MyData = [];
15
// MyData array will contain the data from the CSV file and it will be sent to the clients request over HTTP.
16
?
17
obj.from.path('../THEPATHINYOURPROJECT/TOTHE/csv_FILE_YOU_WANT_TO_LOAD.csv').to.array(function (data) {
18
for (var index = 0; index < data.length; index++) {
19
MyData.push(new MyCSV(data[index][0], data[index][1], data[index][2]));
20
}
21
console.log(MyData);
22
});
23
//Reads the CSV file from the path you specify, and the data is stored in the array we specified using callback function. This function iterates through an array and each line from the CSV file will be pushed as a record to another array called MyData , and logs the data into the console to ensure it worked.
24
?
25
var http = require('http');
26
//Load the http module.
27
?
28
var server = http.createServer(function (req, resp) {
29
resp.writeHead(200, { 'content-type': 'application/json' });
30
resp.end(JSON.stringify(MyData));
31
});
32
// Create a webserver with a request listener callback. This will write the response header with the content type as json, and end the response by sending the MyData array in JSON format.
33
?
34
server.listen(8080);
35
// Tells the webserver to listen on port 8080(obviously this may be whatever port you want.)
Things to be aware of in your app.js code
In lines 7 through 11, we define the function called 'MyCSV' and the field names.
If your CSV file has multiple columns make sure you define this correctly to match your file.
On line 17 we define the location of the CSV file of which we are loading. Make sure you use the correct path here.
Start your App and Verify Functionality Open a console and type the following Command:
Node app 1 Node app You should see the following output in your console:
[ MYCSV { Fieldone: 'ABC', Fieldtwo: '123', Fieldthree: 'Fudge' },
MYCSV { Fieldone: '532', Fieldtwo: 'CWE', Fieldthree: 'ICECREAM' },
MYCSV { Fieldone: '8023', Fieldtwo: 'POOP', Fieldthree: 'DOGS' },
MYCSV { Fieldone: '441', Fieldtwo: 'CHEESE', Fieldthree: 'CARMEL' },
MYCSV { Fieldone: '221', Fieldtwo: 'ABC', Fieldthree: 'HOUSE' }, ]
1 [ MYCSV { Fieldone: 'ABC', Fieldtwo: '123', Fieldthree: 'Fudge' }, 2 MYCSV { Fieldone: '532', Fieldtwo: 'CWE', Fieldthree: 'ICECREAM' }, 3 MYCSV { Fieldone: '8023', Fieldtwo: 'POOP', Fieldthree: 'DOGS' }, 4 MYCSV { Fieldone: '441', Fieldtwo: 'CHEESE', Fieldthree: 'CARMEL' }, 5 MYCSV { Fieldone: '221', Fieldtwo: 'ABC', Fieldthree: 'HOUSE' }, ] Now you should open a web-browser and navigate to your server. You should see it output the data in JSON format.
Conclusion Using node.js and it's CSV module we can quickly and easily read and use data stored on the server and make it available to the client upon request
this is my solution to get csv file from external url
const parse = require( 'csv-parse/lib/sync' );
const axios = require( 'axios' );
const readCSV = ( module.exports.readCSV = async ( path ) => {
try {
const res = await axios( { url: path, method: 'GET', responseType: 'blob' } );
let records = parse( res.data, {
columns: true,
skip_empty_lines: true
} );
return records;
} catch ( e ) {
console.log( 'err' );
}
} );
readCSV('https://urltofilecsv');
In order to pause the streaming in fast-csv you can do the following:
let csvstream = csv.fromPath(filePath, { headers: true })
.on("data", function (row) {
csvstream.pause();
// do some heavy work
// when done resume the stream
csvstream.resume();
})
.on("end", function () {
console.log("We are done!")
})
.on("error", function (error) {
console.log(error)
});
I needed an async csv reader and originally tried @Pransh Tiwari's answer but couldn't get it working with await and util.promisify(). Eventually I came across node-csvtojson, which pretty much does the same as csv-parser, but with promises. Here is an example usage of csvtojson in action:
const csvToJson = require('csvtojson');
const processRecipients = async () => {
const recipients = await csvToJson({
trim:true
}).fromFile('./recipients.csv');
// Code executes after recipients are fully loaded.
recipients.forEach((recipient) => {
console.log(recipient.name, recipient.email);
});
};
Source: Stackoverflow.com