[node.js] Upload a file to Amazon S3 with NodeJS

I ran into a problem while trying to upload a file to my S3 bucket. Everything works except that my file paramters do not seem appropriate. I am using Amazon S3 sdk to upload from nodejs to s3.

These are my routes settings:

var multiparty = require('connect-multiparty'),
    multipartyMiddleware = multiparty();
app.route('/api/items/upload').post(multipartyMiddleware, items.upload);

This is items.upload() function:

exports.upload = function(req, res) {
    var file = req.files.file;
    var s3bucket = new AWS.S3({params: {Bucket: 'mybucketname'}});
    s3bucket.createBucket(function() {
        var params = {
            Key: file.name,
            Body: file
        };
        s3bucket.upload(params, function(err, data) {
            console.log("PRINT FILE:", file);
            if (err) {
                console.log('ERROR MSG: ', err);
            } else {
                console.log('Successfully uploaded data');
            }
        });
    });
};

Setting Body param to a string like "hello" works fine. According to doc, Body param must take (Buffer, Typed Array, Blob, String, ReadableStream) Object data. However, uploading a file object fails with the following error message:

[Error: Unsupported body payload object]

This is the file object:

{ fieldName: 'file',
  originalFilename: 'second_fnp.png',
  path: '/var/folders/ps/l8lvygws0w93trqz7yj1t5sr0000gn/T/26374-7ttwvc.png',
  headers: 
   { 'content-disposition': 'form-data; name="file"; filename="second_fnp.png"',
     'content-type': 'image/png' },
  ws: 
   { _writableState: 
      { highWaterMark: 16384,
        objectMode: false,
        needDrain: true,
        ending: true,
        ended: true,
        finished: true,
        decodeStrings: true,
        defaultEncoding: 'utf8',
        length: 0,
        writing: false,
        sync: false,
        bufferProcessing: false,
        onwrite: [Function],
        writecb: null,
        writelen: 0,
        buffer: [],
        errorEmitted: false },
     writable: true,
     domain: null,
     _events: { error: [Object], close: [Object] },
     _maxListeners: 10,
     path: '/var/folders/ps/l8lvygws0w93trqz7yj1t5sr0000gn/T/26374-7ttwvc.png',
     fd: null,
     flags: 'w',
     mode: 438,
     start: undefined,
     pos: undefined,
     bytesWritten: 261937,
     closed: true },
  size: 261937,
  name: 'second_fnp.png',
  type: 'image/png' }

Any help will be greatly appreciated!

I found the following to be a working solution::

npm install aws-sdk

Once you've installed the aws-sdk , use the following code replacing values with your where needed.

var AWS = require('aws-sdk');
var fs =  require('fs');

var s3 = new AWS.S3();

// Bucket names must be unique across all S3 users

var myBucket = 'njera';

var myKey = 'jpeg';
//for text file
//fs.readFile('demo.txt', function (err, data) {
//for Video file
//fs.readFile('demo.avi', function (err, data) {
//for image file                
fs.readFile('demo.jpg', function (err, data) {
  if (err) { throw err; }



     params = {Bucket: myBucket, Key: myKey, Body: data };

     s3.putObject(params, function(err, data) {

         if (err) {

             console.log(err)

         } else {

             console.log("Successfully uploaded data to myBucket/myKey");

         }

      });

});

I found the complete tutorial on the subject here in case you're looking for references ::

Uploading a file to AWS s3 and sending the url in response for accessing the file.

Multer is a node.js middleware for handling multipart/form-data, which is primarily used for uploading files. It is written on top of busboy for maximum efficiency. check this npm module here.

When you are sending the request, make sure the headers, have Content-Type is multipart/form-data. We are sending the file location in the response, which will give the url, but if you want to access that url, make the bucket public or else you will not be able to access it.

upload.router.js

const express = require('express');
const router = express.Router();
const AWS = require('aws-sdk');
const multer = require('multer');
const storage = multer.memoryStorage()
const upload = multer({storage: storage});

const s3Client = new AWS.S3({
    accessKeyId: 'your_access_key_id',
    secretAccessKey: 'your_secret_access_id',
    region :'ur region'
});

const uploadParams = {
         Bucket: 'ur_bucket_name', 
         Key: '', // pass key
         Body: null, // pass file body
};


router.post('/api/file/upload', upload.single("file"),(req,res) => {
    const params = uploadParams;

    uploadParams.Key = req.file.originalname;
    uploadParams.Body = req.file.buffer;

    s3Client.upload(params, (err, data) => {
        if (err) {
            res.status(500).json({error:"Error -> " + err});
        }
        res.json({message: 'File uploaded successfully','filename': 
        req.file.originalname, 'location': data.Location});
    });
});

module.exports = router;

app.js

const express = require('express');
const app = express();

const router = require('./app/routers/upload.router.js');
app.use('/', router);

// Create a Server
  const server = app.listen(8080, () => {
  console.log("App listening at 8080"); 
})

var express = require('express')

app = module.exports = express();
var secureServer = require('http').createServer(app);
secureServer.listen(3001);

var aws = require('aws-sdk')
var multer = require('multer')
var multerS3 = require('multer-s3')

    aws.config.update({
    secretAccessKey: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
    accessKeyId: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX",
    region: 'us-east-1'
    });
    s3 = new aws.S3();

   var upload = multer({
   storage: multerS3({
    s3: s3,
    dirname: "uploads",
    bucket: "Your bucket name",
    key: function (req, file, cb) {
        console.log(file);
        cb(null, "uploads/profile_images/u_" + Date.now() + ".jpg"); //use  
     Date.now() for unique file keys
    }
  })
   });

 app.post('/upload', upload.single('photos'), function(req, res, next) {

 console.log('Successfully uploaded ', req.file)

 res.send('Successfully uploaded ' + req.file.length + ' files!')

})

Thanks to David as his solution helped me come up with my solution for uploading multi-part files from my Heroku hosted site to S3 bucket. I did it using formidable to handle incoming form and fs to get the file content. Hopefully, it may help you.

api.service.ts

public upload(files): Observable<any> {  
    const formData: FormData = new FormData(); 
    files.forEach(file => {
      // create a new multipart-form for every file 
      formData.append('file', file, file.name);           
    });   
    return this.http.post(uploadUrl, formData).pipe(
      map(this.extractData),
      catchError(this.handleError)); 
  }
}

server.js

app.post('/api/upload', upload);
app.use('/api/upload', router);

upload.js

const IncomingForm = require('formidable').IncomingForm;
const fs = require('fs');
const AWS = require('aws-sdk');

module.exports = function upload(req, res) {
    var form = new IncomingForm();

    const bucket = new AWS.S3(
      {
        signatureVersion: 'v4',
        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
        region: 'us-east-1'       
      }
    ); 

    form.on('file', (field, file) => {

        const fileContent = fs.readFileSync(file.path);

        const s3Params = {
            Bucket: process.env.AWS_S3_BUCKET,
            Key: 'folder/' + file.name,
            Expires: 60,             
            Body: fileContent,
            ACL: 'public-read'
        };

        bucket.upload(s3Params, function(err, data) {
            if (err) {
                throw err;
            }            
            console.log('File uploaded to: ' + data.Location);
            fs.unlink(file.path, function (err) {
              if (err) {
                  console.error(err);
              }
              console.log('Temp File Delete');
          });
        });
    });              

    // The second callback is called when the form is completely parsed. 
    // In this case, we want to send back a success status code.
    form.on('end', () => {        
      res.status(200).json('upload ok');
    });

    form.parse(req);
}

upload-image.component.ts

import { Component, OnInit, ViewChild, Output, EventEmitter, Input } from '@angular/core';
import { ApiService } from '../api.service';
import { MatSnackBar } from '@angular/material/snack-bar';

@Component({
  selector: 'app-upload-image',
  templateUrl: './upload-image.component.html',
  styleUrls: ['./upload-image.component.css']
})

export class UploadImageComponent implements OnInit {
  public files: Set<File> = new Set();
  @ViewChild('file', { static: false }) file;
  public uploadedFiles: Array<string> = new Array<string>();
  public uploadedFileNames: Array<string> = new Array<string>();
  @Output() filesOutput = new EventEmitter<Array<string>>();
  @Input() CurrentImage: string;
  @Input() IsPublic: boolean;
  @Output() valueUpdate = new EventEmitter();
  strUploadedFiles:string = '';
  filesUploaded: boolean = false;     

  constructor(private api: ApiService, public snackBar: MatSnackBar,) { }

  ngOnInit() {    
  }

  updateValue(val) {  
    this.valueUpdate.emit(val);  
  }  

  reset()
  {
    this.files = new Set();
    this.uploadedFiles = new Array<string>();
    this.uploadedFileNames = new Array<string>();
    this.filesUploaded = false;
  }

  upload() { 

    this.api.upload(this.files).subscribe(res => {   
      this.filesOutput.emit(this.uploadedFiles); 
      if (res == 'upload ok')
      {
        this.reset(); 
      }     
    }, err => {
      console.log(err);
    });
  }

  onFilesAdded() {
    var txt = '';
    const files: { [key: string]: File } = this.file.nativeElement.files;

    for (let key in files) {
      if (!isNaN(parseInt(key))) {

        var currentFile = files[key];
        var sFileExtension = currentFile.name.split('.')[currentFile.name.split('.').length - 1].toLowerCase();
        var iFileSize = currentFile.size;

        if (!(sFileExtension === "jpg" 
              || sFileExtension === "png") 
              || iFileSize > 671329) {
            txt = "File type : " + sFileExtension + "\n\n";
            txt += "Size: " + iFileSize + "\n\n";
            txt += "Please make sure your file is in jpg or png format and less than 655 KB.\n\n";
            alert(txt);
            return false;
        }

        this.files.add(files[key]);
        this.uploadedFiles.push('https://gourmet-philatelist-assets.s3.amazonaws.com/folder/' + files[key].name);
        this.uploadedFileNames.push(files[key].name);
        if (this.IsPublic && this.uploadedFileNames.length == 1)
        {
          this.filesUploaded = true;
          this.updateValue(files[key].name);
          break;
        } 
        else if (!this.IsPublic && this.uploadedFileNames.length == 3)
        {
          this.strUploadedFiles += files[key].name;          
          this.updateValue(this.strUploadedFiles); 
          this.filesUploaded = true;
          break;
        }
        else
        {
          this.strUploadedFiles += files[key].name + ",";          
          this.updateValue(this.strUploadedFiles); 
        }      
      }
    }    
  }

  addFiles() {
    this.file.nativeElement.click();  
  }

  openSnackBar(message: string, action: string) {
    this.snackBar.open(message, action, {
      duration: 2000,
      verticalPosition: 'top'
    });
  }   

}

upload-image.component.html

<input type="file" #file style="display: none" (change)="onFilesAdded()" multiple />
&nbsp;<button mat-raised-button color="primary" 
         [disabled]="filesUploaded" (click)="$event.preventDefault(); addFiles()">
  Add Files
</button>
&nbsp;<button class="btn btn-success" [disabled]="uploadedFileNames.length == 0" (click)="$event.preventDefault(); upload()">
  Upload
</button>

Upload CSV/Excel

const fs = require('fs');
const AWS = require('aws-sdk');

const s3 = new AWS.S3({
    accessKeyId: XXXXXXXXX,
    secretAccessKey: XXXXXXXXX
});

const absoluteFilePath = "C:\\Project\\test.xlsx";

const uploadFile = () => {
  fs.readFile(absoluteFilePath, (err, data) => {
     if (err) throw err;
     const params = {
         Bucket: 'testBucket', // pass your bucket name
         Key: 'folderName/key.xlsx', // file will be saved in <folderName> folder
         Body: data
     };
      s3.upload(params, function (s3Err, data) {
                    if (s3Err) throw s3Err
                    console.log(`File uploaded successfully at ${data.Location}`);
                    debugger;
                });
  });
};

uploadFile();

Or Using promises:

const AWS = require('aws-sdk');
AWS.config.update({
    accessKeyId: 'accessKeyId',
    secretAccessKey: 'secretAccessKey',
    region: 'region'
});

let params = {
    Bucket: "yourBucketName",
    Key: 'someUniqueKey',
    Body: 'someFile'
};
try {
    let uploadPromise = await new AWS.S3().putObject(params).promise();
    console.log("Successfully uploaded data to bucket");
} catch (e) {
    console.log("Error uploading data: ", e);
}