[javascript] How do I read the contents of a Node.js stream into a string variable?

I'm hacking on a Node program that uses smtp-protocol to capture SMTP emails and act on the mail data. The library provides the mail data as a stream, and I don't know how to get that into a string.

I'm currently writing it to stdout with stream.pipe(process.stdout, { end: false }), but as I said, I need the stream data in a string instead, which I can use once the stream has ended.

How do I collect all the data from a Node.js stream into a string?

Another way would be to convert the stream to a promise (refer to the example below) and use then (or await) to assign the resolved value to a variable.

function streamToString (stream) {
  const chunks = [];
  return new Promise((resolve, reject) => {
    stream.on('data', (chunk) => chunks.push(chunk));
    stream.on('error', (err) => reject(err));
    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
  })
}

const result = await streamToString(stream)

In my case, the content type response headers was Content-Type: text/plain. So, I've read the data from Buffer like:

let data = [];
stream.on('data', (chunk) => {
 console.log(Buffer.from(chunk).toString())
 data.push(Buffer.from(chunk).toString())
});

What do you think about this ?

// lets have a ReadableStream as a stream variable

const chunks = [];

for await (let chunk of stream) {
    chunks.push(chunk)
}

const buffer  = Buffer.concat(chunks);
const str = buffer.toString("utf-8")

I'm using usually this simple function to transform a stream into a string:

function streamToString(stream, cb) {
  const chunks = [];
  stream.on('data', (chunk) => {
    chunks.push(chunk.toString());
  });
  stream.on('end', () => {
    cb(chunks.join(''));
  });
}

Usage example:

let stream = fs.createReadStream('./myFile.foo');
streamToString(stream, (data) => {
  console.log(data);  // data is now my string variable
});

From the nodejs documentation you should do this - always remember a string without knowing the encoding is just a bunch of bytes:

var readable = getReadableStreamSomehow();
readable.setEncoding('utf8');
readable.on('data', function(chunk) {
  assert.equal(typeof chunk, 'string');
  console.log('got %d characters of string data', chunk.length);
})

None of the above worked for me. I needed to use the Buffer object:

  const chunks = [];

  readStream.on("data", function (chunk) {
    chunks.push(chunk);
  });

  // Send the buffer or you can put it into a var
  readStream.on("end", function () {
    res.send(Buffer.concat(chunks));
  });

Hope this is more useful than the above answer:

var string = '';
stream.on('data',function(data){
  string += data.toString();
  console.log('stream data ' + part);
});

stream.on('end',function(){
  console.log('final output ' + string);
});

Note that string concatenation is not the most efficient way to collect the string parts, but it is used for simplicity (and perhaps your code does not care about efficiency).

Also, this code may produce unpredictable failures for non-ASCII text (it assumes that every character fits in a byte), but perhaps you do not care about that, either.

All the answers listed appear to open the Readable Stream in flowing mode which is not the default in NodeJS and can have limitations since it lacks backpressure support that NodeJS provides in Paused Readable Stream Mode. Here is an implementation using Just Buffers, Native Stream and Native Stream Transforms and support for Object Mode

import {Transform} from 'stream';

let buffer =null;    

function objectifyStream() {
    return new Transform({
        objectMode: true,
        transform: function(chunk, encoding, next) {

            if (!buffer) {
                buffer = Buffer.from([...chunk]);
            } else {
                buffer = Buffer.from([...buffer, ...chunk]);
            }
            next(null, buffer);
        }
    });
}

process.stdin.pipe(objectifyStream()).process.stdout

Streams don't have a simple .toString() function (which I understand) nor something like a .toStringAsync(cb) function (which I don't understand).

So I created my own helper function:

var streamToString = function(stream, callback) {
  var str = '';
  stream.on('data', function(chunk) {
    str += chunk;
  });
  stream.on('end', function() {
    callback(str);
  });
}

// how to use:
streamToString(myStream, function(myStr) {
  console.log(myStr);
});

And yet another one for strings using promises:

function getStream(stream) {
  return new Promise(resolve => {
    const chunks = [];

    # Buffer.from is required if chunk is a String, see comments
    stream.on("data", chunk => chunks.push(Buffer.from(chunk)));
    stream.on("end", () => resolve(Buffer.concat(chunks).toString()));
  });
}

Usage:

const stream = fs.createReadStream(__filename);
getStream(stream).then(r=>console.log(r));

remove the .toString() to use with binary Data if required.

update: @AndreiLED correctly pointed out this has problems with strings. I couldn't get a stream returning strings with the version of node I have, but the api notes this is possible.

I had more luck using like that :

let string = '';
readstream
    .on('data', (buf) => string += buf.toString())
    .on('end', () => console.log(string));

I use node v9.11.1 and the readstream is the response from a http.get callback.

setEncoding('utf8');

Well done Sebastian J above.

I had the "buffer problem" with a few lines of test code I had, and added the encoding information and it solved it, see below.

Demonstrate the problem

software

// process.stdin.setEncoding('utf8');
process.stdin.on('data', (data) => {
    console.log(typeof(data), data);
});

input

hello world

output

object <Buffer 68 65 6c 6c 6f 20 77 6f 72 6c 64 0d 0a>

Demonstrate the solution

software

process.stdin.setEncoding('utf8'); // <- Activate!
process.stdin.on('data', (data) => {
    console.log(typeof(data), data);
});

input

hello world

output

string hello world

This worked for me and is based on Node v6.7.0 docs:

let output = '';
stream.on('readable', function() {
    let read = stream.read();
    if (read !== null) {
        // New stream data is available
        output += read.toString();
    } else {
        // Stream is now finished when read is null.
        // You can callback here e.g.:
        callback(null, output);
    }
});

stream.on('error', function(err) {
  callback(err, null);
})

What about something like a stream reducer ?

Here is an example using ES6 classes how to use one.

var stream = require('stream')

class StreamReducer extends stream.Writable {
  constructor(chunkReducer, initialvalue, cb) {
    super();
    this.reducer = chunkReducer;
    this.accumulator = initialvalue;
    this.cb = cb;
  }
  _write(chunk, enc, next) {
    this.accumulator = this.reducer(this.accumulator, chunk);
    next();
  }
  end() {
    this.cb(null, this.accumulator)
  }
}

// just a test stream
class EmitterStream extends stream.Readable {
  constructor(chunks) {
    super();
    this.chunks = chunks;
  }
  _read() {
    this.chunks.forEach(function (chunk) { 
        this.push(chunk);
    }.bind(this));
    this.push(null);
  }
}

// just transform the strings into buffer as we would get from fs stream or http request stream
(new EmitterStream(
  ["hello ", "world !"]
  .map(function(str) {
     return Buffer.from(str, 'utf8');
  })
)).pipe(new StreamReducer(
  function (acc, v) {
    acc.push(v);
    return acc;
  },
  [],
  function(err, chunks) {
    console.log(Buffer.concat(chunks).toString('utf8'));
  })
);

Easy way with the popular (over 5m weekly downloads) and lightweight get-stream library:

https://www.npmjs.com/package/get-stream

const fs = require('fs');
const getStream = require('get-stream');

(async () => {
    const stream = fs.createReadStream('unicorn.txt');
    console.log(await getStream(stream)); //output is string
})();

The cleanest solution may be to use the "string-stream" package, which converts a stream to a string with a promise.

const streamString = require('stream-string')

streamString(myStream).then(string_variable => {
    // myStream was converted to a string, and that string is stored in string_variable
    console.log(string_variable)

}).catch(err => {
     // myStream emitted an error event (err), so the promise from stream-string was rejected
    throw err
})

Using the quite popular stream-buffers package which you probably already have in your project dependencies, this is pretty straightforward:

// imports
const { WritableStreamBuffer } = require('stream-buffers');
const { promisify } = require('util');
const { createReadStream } = require('fs');
const pipeline = promisify(require('stream').pipeline);

// sample stream
let stream = createReadStream('/etc/hosts');

// pipeline the stream into a buffer, and print the contents when done
let buf = new WritableStreamBuffer();
pipeline(stream, buf).then(() => console.log(buf.getContents().toString()));