[node.js] Where is body in a nodejs http.get response?

I'm reading the docs at http://nodejs.org/docs/v0.4.0/api/http.html#http.request, but for some reason, I can't seem to to actually find the body/data attribute on the returned, finished response object.

> var res = http.get({host:'www.somesite.com', path:'/'})

> res.finished
true

> res._hasBody
true

It's finished (http.get does that for you), so it should have some kind of content. But there's no body, no data, and I can't read from it. Where is the body hiding?

You need to add a listener to the request because node.js works asynchronous like that:

request.on('response', function (response) {
  response.on('data', function (chunk) {
    console.log('BODY: ' + chunk);
 });
});

The data event is fired multiple times with 'chunks' of the body as they are downloaded and an end event when all chunks have been downloaded.

With Node supporting Promises now, I created a simple wrapper to return the concatenated chunks through a Promise:

const httpGet = url => {
  return new Promise((resolve, reject) => {
    http.get(url, res => {
      res.setEncoding('utf8');
      let body = ''; 
      res.on('data', chunk => body += chunk);
      res.on('end', () => resolve(body));
    }).on('error', reject);
  });
};

You can call it from an async function with:

const body = await httpGet('http://www.somesite.com');

The body is not fully stored as part of the response, the reason for this is because the body can contain a very large amount of data, if it was to be stored on the response object, the program's memory would be consumed quite fast.

Instead, Node.js uses a mechanism called Stream. This mechanism allows holding only a small part of the data and allows the programmer to decide if to fully consume it in memory or use each part of the data as its flowing.

There are multiple ways how to fully consume the data into memory, since HTTP Response is a readable stream, all readable methods are available on the res object

1. listening to the "data" event and saving the chunks passed to the callback

   const chunks = []
   
   res.on("data", (chunk) => {
       chunks.push(chunk)
   });
   
   res.on("end", () => {
       const body = Buffer.concat(chunks);
   });
   

When using this approach you do not interfere with the behavior of the stream and you are only gathering the data as it is available to the application.

2. using the "readble" event and calling res.read()

   const chunks = [];
   
   res.on("readable", () => {
      let chunk;
      while(null !== (chunk = res.read())){
          chunks.push(chunk)
      }
   });
   
   res.on("end", () => {
       const body = Buffer.concat(chunks);
   });
   

When going with this approach you are fully in charge of the stream flow and until res.read is called no more data will be passed into the stream.

3. using an async iterator

   const chunks = [];
   
   for await (const chunk of readable) {
       chunks.push(chunk);
   } 
   
   const body = Buffer.concat(chunks);
   

This approach is similar to the "data" event approach. It will just simplify the scoping and allow the entire process to happen in the same scope.

While as described, it is possible to fully consume data from the response it is always important to keep in mind if it is actually necessary to do so. In many cases, it is possible to simply direct the data to its destination without fully saving it into memory.

Node.js read streams, including HTTP response, have a built-in method for doing this, this method is called pipe. The usage is quite simple, readStream.pipe(writeStream);.

for example:

If the final destination of your data is the file system, you can simply open a write stream to the file system and then pipe the data to ts destination.

const { createWriteStream } = require("fs");
const writeStream = createWriteStream("someFile");
res.pipe(writeStream);

A portion of Coffee here:

# My little helper
read_buffer = (buffer, callback) ->
  data = ''
  buffer.on 'readable', -> data += buffer.read().toString()
  buffer.on 'end', -> callback data

# So request looks like
http.get 'http://i.want.some/stuff', (res) ->
  read_buffer res, (response) ->
    # Do some things with your response
    # but don't do that exactly :D
    eval(CoffeeScript.compile response, bare: true)

And compiled

var read_buffer;

read_buffer = function(buffer, callback) {
  var data;
  data = '';
  buffer.on('readable', function() {
    return data += buffer.read().toString();
  });
  return buffer.on('end', function() {
    return callback(data);
  });
};

http.get('http://i.want.some/stuff', function(res) {
  return read_buffer(res, function(response) {
    return eval(CoffeeScript.compile(response, {
      bare: true
    }));
  });
});

Just an improved version to nkron responce.

const httpGet = url => {
  return new Promise((resolve, reject) => {
    http.get(url, res => {
      res.setEncoding('utf8');
      const body = [];
      res.on('data', chunk => body.push(chunk));
      res.on('end', () => resolve(body.join('')));
    }).on('error', reject);
  });
};

Appending chunks in an string[] is better for memory usage, the join(''), will allocate new memory only once.

If you want to use .get you can do it like this

http.get(url, function(res){
    res.setEncoding('utf8');
    res.on('data', function(chunk){
        console.log(chunk);
    });

});

http.request docs contains example how to receive body of the response through handling data event:

var options = {
  host: 'www.google.com',
  port: 80,
  path: '/upload',
  method: 'POST'
};

var req = http.request(options, function(res) {
  console.log('STATUS: ' + res.statusCode);
  console.log('HEADERS: ' + JSON.stringify(res.headers));
  res.setEncoding('utf8');
  res.on('data', function (chunk) {
    console.log('BODY: ' + chunk);
  });
});

req.on('error', function(e) {
  console.log('problem with request: ' + e.message);
});

// write data to request body
req.write('data\n');
req.write('data\n');
req.end();

http.get does the same thing as http.request except it calls req.end() automatically.

var options = {
  host: 'www.google.com',
  port: 80,
  path: '/index.html'
};

http.get(options, function(res) {
  console.log("Got response: " + res.statusCode);

  res.on("data", function(chunk) {
    console.log("BODY: " + chunk);
  });
}).on('error', function(e) {
  console.log("Got error: " + e.message);
});

You can't get the body of the response from the return value of http.get().

http.get() doesn't return a response object. It returns the request object (http.clientRequest). So, there isn't any way to get the body of the response from the return value of http.get().

I know it's an old question, but reading the documentation you linked to shows that this was the case even when you posted it.

Needle module is also good, here is an example which uses needle module

var needle = require('needle');

needle.get('http://www.google.com', function(error, response) {
  if (!error && response.statusCode == 200)
    console.log(response.body);
});

I also want to add that the http.ClientResponse returned by http.get() has an end event, so here is another way that I receive the body response:

var options = {
  host: 'www.google.com',
  port: 80,
  path: '/index.html'
};

http.get(options, function(res) {
  var body = '';
  res.on('data', function(chunk) {
    body += chunk;
  });
  res.on('end', function() {
    console.log(body);
  });
}).on('error', function(e) {
  console.log("Got error: " + e.message);
});