NodeJS How do I Download a file to disk from an aws s3 bucket?

My goal:

Display a dialog box prompting the user to save a file being downloaded from aws.

My problem:

I am currently using awssum-amazon-s3 to create a download stream. However I've only managed to save the file to my server or stream it to the command line... As you can see from my code my last attempt was to try and manually set the content disposition headers which failed. I cannot use res.download() as the headers have already been set?

How can I achieve my goal?

My code for node:

app.post('/dls/:dlKey', function(req, res, next){
        // download the file via aws s3 here
        var dlKey = req.param('dlKey');

        Dl.findOne({key:dlKey}, function(err, dl){
            if (err) return next(err);
            var files = dl.dlFile;

            var options = {
                BucketName    : 'xxxx',
                ObjectName    : files,
            };

            s3.GetObject(options, { stream : true }, function(err, data) {
                // stream this file to stdout
                fmt.sep();
                data.Headers['Content-Disposition'] = 'attachment';
                console.log(data.Headers);
                data.Stream.pipe(fs.createWriteStream('test.pdf'));
                data.Stream.on('end', function() {
                    console.log('File Downloaded!');
                });
            });
        });

        res.end('Successful Download Post!');
    });

My code for angular:

$scope.dlComplete = function (dl) {
        $scope.procDownload = true;
        $http({
            method: 'POST',
            url: '/dls/' + dl.dlKey
        }).success(function(data/*, status, headers, config*/) {
            console.log(data);
            $location.path('/#!/success');
        }).error(function(/*data, status, headers, config*/) {
            console.log('File download failed!');
        });
    };

The purpose of this code it to let users use a generated key to download a file once.

Solution 1:

This is the entire code using streaming on the latest version of aws-sdk

var express = require('express');
var app = express();
var fs = require('fs');

app.get('/', function(req, res, next){
    res.send('You did not say the magic word');
});


app.get('/s3Proxy', function(req, res, next){
    // download the file via aws s3 here
    var fileKey = req.query['fileKey'];

    console.log('Trying to download file', fileKey);
    var AWS = require('aws-sdk');
    AWS.config.update(
      {
        accessKeyId: "....",
        secretAccessKey: "...",
        region: 'ap-southeast-1'
      }
    );
    var s3 = new AWS.S3();
    var options = {
        Bucket    : '/bucket-url',
        Key    : fileKey,
    };

    res.attachment(fileKey);
    var fileStream = s3.getObject(options).createReadStream();
    fileStream.pipe(res);
});

var server = app.listen(3000, function () {
    var host = server.address().address;
    var port = server.address().port;
    console.log('S3 Proxy app listening at http://%s:%s', host, port);
});

Solution 2:

This code worked for me with the most recent library:

var s3 = new AWS.S3();
var s3Params = {
    Bucket: 'your bucket',
    Key: 'path/to/the/file.ext'
};
s3.getObject(s3Params, function(err, res) {
    if (err === null) {
       res.attachment('file.ext'); // or whatever your logic needs
       res.send(data.Body);
    } else {
       res.status(500).send(err);
    }
});

Solution 3:

Simply create a ReadStream from S3 and WriteStream to the location were u want to download. Find the code below. Works perfectly for me:

var AWS = require('aws-sdk');
var path = require('path');
var fs = require('fs');

AWS.config.loadFromPath(path.resolve(__dirname, 'config.json'));
AWS.config.update({
  accessKeyId: AWS.config.credentials.accessKeyId,
  secretAccessKey: AWS.config.credentials.secretAccessKey,
  region: AWS.config.region
});

var s3 = new AWS.S3();
var params = {
  Bucket: '<your-bucket>', 
  Key: '<path-to-your-file>'
};
let readStream = s3.getObject(params).createReadStream();
let writeStream = fs.createWriteStream(path.join(__dirname, 's3data.txt'));
readStream.pipe(writeStream);

Solution 4:

You've already figured what's most important to solve your issue: you can pipe the file stream coming from S3 to any writable stream, be it a filestream… or the response stream that will be sent to the client!

s3.GetObject(options, { stream : true }, function(err, data) {
    res.attachment('test.pdf');
    data.Stream.pipe(res);
});

Note the use of res.attachment that will set the correct headers. You can also check out this answer regarding streams and S3.