upload is suitable for relatively large files, putObject is suitable for small file content, upload supports custom multi-threaded concurrent upload
var AWS = require ('aws-sdk');
AWS.config.loadFromPath('./config/aws.config');
var fs = require('fs');
//var s3 = new AWS.S3({params:{Bucket:'xxx'}});
// OK
// s3.listBuckets(function(error, data) {
// if (error) {
// console.log(error); // error is Response.error
// } else {
// console.log(data); // data is Response.data
// }
// });
// s3.getObject({Key:'download_test.log'}).on('success', function(response) {
// console.log("Key was", response.request.params.Key);
// }).send();
// var s3 = new AWS.S3 ();
// var params = {Bucket: 'mochong', Key:'message_center.txt'};
// var file = require('fs').createWriteStream('./deploy_local.txt');
// s3.getObject(params).createReadStream().pipe(file);
/** create bucket name cannot contain `_` and other characters, must be unique**/
/**
var s3 = new AWS.S3 ();
var params =
Bucket: 'mochongmessagecenter', //required
ACL: 'private',
CreateBucketConfiguration: {
LocationConstraint: 'us-west-2'
}
};
s3.createBucket(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
**/
/**delete bucket**/
/**
*
var s3 = new AWS.S3 ();
var params =
Bucket: 'mochongmessagecenter' //required
};
s3.deleteBucket(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
**/
/** init multipart upload starts multi-threaded upload, get uploadID**/
/**
var params =
Bucket: 'mochong', //required
Key: 'upload.txt', // required
ACL: 'private',
// CacheControl: 'STRING_VALUE',
// ContentDisposition: 'STRING_VALUE',
ContentEncoding: 'utf8',
ContentLanguage: 'zh-cn',
ContentType: 'plain',
//Expires: new Date || 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' || 123456789,
//GrantFullControl: 'STRING_VALUE',
// GrantRead: 'STRING_VALUE',
//GrantReadACP: 'STRING_VALUE',
//GrantWriteACP: 'STRING_VALUE',
//Metadata: {
// someKey: 'STRING_VALUE',
// anotherKey: ...
//},
//RequestPayer: 'requester',
//SSECustomerAlgorithm: 'STRING_VALUE',
//SSECustomerKey: new Buffer('...') || 'STRING_VALUE',
// SSECustomerKeyMD5: 'STRING_VALUE',
//SSEKMSKeyId: 'STRING_VALUE',
//ServerSideEncryption: 'AES256 | aws:kms',
//StorageClass: 'STANDARD | REDUCED_REDUNDANCY | STANDARD_IA',
//WebsiteRedirectLocation: 'STRING_VALUE'
};
var s3 = new AWS.S3 ();
// {
// Bucket: 'mochong',
// Key: 'upload.txt',
// UploadId: '3jNsFCZRDwASLGpMi2cYBUt4bieR8hMj3F7CAoDZkWrpAimkB95FLx.ZRmfkCGQZL2V3KVU7HkmMlWsATIH.hcWPj7VQ_cu8.lKtpEk8TbBF60Z93bKAUE3Duo5d1GaB'
// }
s3.createMultipartUpload(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
**/
/**Start multi-threaded upload**/
// var s3 = new AWS.S3 ();
// fs.readFile('./download_test.log', 'utf8', function (err,data) {
// if (err) {
// return console.log(err);
// }
// console.log(data);
// var params =
// Bucket: 'mochong', // required
// Key: 'upload.txt', // required
// PartNumber: 5, // 1~10000
// UploadId: '3jNsFCZRDwASLGpMi2cYBUt4bieR8hMj3F7CAoDZkWrpAimkB95FLx.ZRmfkCGQZL2V3KVU7HkmMlWsATIH.hcWPj7VQ_cu8.lKtpEk8TbBF60Z93bKAUE3Duo5d1GaB', /* required */
// Body: data,
// ContentLength: Buffer.byteLength(data, 'utf8'),
// //ContentMD5: 'STRING_VALUE',
// //RequestPayer: 'requester',
// // SSECustomerAlgorithm: 'STRING_VALUE',
// // SSECustomerKey: new Buffer('...') || 'STRING_VALUE',
// //SSECustomerKeyMD5: 'STRING_VALUE'
// };
// s3.uploadPart(params, function(err, data) {
// if (err) console.log(err, err.stack); // an error occurred
// else console.log(data); // successful response { ETag: '"6f9769150b95562cd1831f82e7026338"' }
// });
// });
/**End upload**/
/*
var s3 = new AWS.S3 ();
var params =
Bucket: 'mochong', // required
Key: 'upload.txt', // required
UploadId: '3jNsFCZRDwASLGpMi2cYBUt4bieR8hMj3F7CAoDZkWrpAimkB95FLx.ZRmfkCGQZL2V3KVU7HkmMlWsATIH.hcWPj7VQ_cu8.lKtpEk8TbBF60Z93bKAUE3Duo5d1GaB', // required
MultipartUpload: {
Parts: [
{
ETag: '6f9769150b95562cd1831f82e7026338',
PartNumber: 5 // must >= 1
},
]
},
RequestPayer: 'requester'
};
s3.completeMultipartUpload(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
// response data:
//{ Location: 'https://mochong.s3-us-west-2.amazonaws.com/upload.txt',
// Bucket: 'mochong',
// Key: 'upload.txt',
// ETag: '"0598c44b0e25dde9e8dfdb55043b3a5f-1"' }
});*/
/**Ignore upload**/
/*
var s3 = new AWS.S3 ();
var params =
Bucket: 'mochong', //required
Key: 'upload.txt', //required
UploadId: '3jNsFCZRDwASLGpMi2cYBUt4bieR8hMj3F7CAoDZkWrpAimkB95FLx.ZRmfkCGQZL2V3KVU7HkmMlWsATIH.hcWPj7VQ_cu8.lKtpEk8TbBF60Z93bKAUE3Duo5d1GaB', //required
RequestPayer: 'requester'
};
s3.abortMultipartUpload(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response: {}
});*/
/**For smart upload, you must use stream and buffer type**/
// var s3 = new AWS.S3 ();
// Don't use readFile. Because s3 upload only supports stream, buffer, blob
//Uploads an arbitrarily sized buffer, blob, or stream,
//using intelligent concurrent handling of parts if the payload is large enough.
//You can configure the concurrent queue size by setting options.
// Note that this is the only operation for which the SDK can retry requests with stream bodies.
/*
var list = [];
var fileReadStream = fs.createReadStream('./log/api.log', {encoding: 'utf8'});
var tmpBuffer;
fileReadStream.on('data', function (chunk) {
tmpBuffer = new Buffer(chunk);
list.push(tmpBuffer);
}).on('end', function () {
fileReadStream.destroy();
var stream = Buffer.concat(list);
var params = {Bucket: 'mochong', Key: 'api.log', Body: stream};
//var options = {partSize: 1 * 1024 * 1024, queueSize: 1}; //queueSize: partSize, must >=5242880
var upload = s3.upload(params, function(err, data) {
if (err) {
console.log(err);
}
});
upload.send(function (err, data) {
stream.fill();
if (err) console.log("Error:", err.message);
else console.log(data);
// { ETag: '"3c6ecbcf8d6b65ccd16b47f603bff1b2"',
// Location: 'https://mochong.s3-us-west-2.amazonaws.com/download_test.log',
// key: 'download_test.log',
// Key: 'download_test.log',
// Bucket: 'mochong'
// }
});
}).on('close', function (err) {
console.log('Stream has been destroyed and file has been closed');
tmpBuffer.fill();
});*/
/** list objectV2
*Returns some or all (up to 1000) of the objects in a bucket.
* You can use the request parameters as selection criteria to return a subset of the objects in a bucket.
* Note: ListObjectsV2 is the revised List Objects API and we recommend you use this revised API for new application development.
* **/
/*
var params =
Bucket: 'mochong', // required
Delimiter: '|',
EncodingType: 'url',
//Marker: 'STRING_VALUE',
// MaxKeys: 0,
// Prefix: 'STRING_VALUE'
Prefix: 'message_center/',
StartAfter: 'STRING_VALUE'//StartAfter is where you want Amazon S3 to start listing from. Amazon S3 starts listing after this specified key. StartAfter can be any key in the bucket
};
var s3 = new AWS.S3 ();
s3.listObjectsV2(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});*/
/**
*response data:
*{ IsTruncated: false,
Marker: 'STRING_VALUE',
Contents: [],
Name: 'mochong',
Prefix: 'STRING_VALUE',
Delimiter: '%7C',
MaxKeys: 0,
CommonPrefixes: [],
EncodingType: 'url' }
*/
/** get object**/
/*
var s3 = new AWS.S3 ();
var params =
Bucket: 'mochong', //required
Key: 'message_center/deploy_local.txt', // required
//IfMatch: 'STRING_VALUE',// (String) Return the object only if its entity tag (ETag) is the same as the one specified, otherwise return a 412 (precondition failed).
//IfModifiedSince: // new Date || 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' || 123456789,
//IfNoneMatch: 'STRING_VALUE',//(String) Return the object only if its entity tag (ETag) is different from the one specified, otherwise return a 304 (not modified).
IfUnmodifiedSince:1471536000,// new Date || 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' || 123456789,
//Range: 'STRING_VALUE',
//RequestPayer: 'requester',
//ResponseCacheControl: 'STRING_VALUE',
//ResponseContentDisposition: 'STRING_VALUE',
//ResponseContentEncoding: 'STRING_VALUE',
//ResponseContentLanguage: 'STRING_VALUE',
//ResponseContentType: 'STRING_VALUE',
//ResponseExpires: new Date || 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' || 123456789,
//SSECustomerAlgorithm: 'STRING_VALUE',
//SSECustomerKey: new Buffer('...') || 'STRING_VALUE',
//SSECustomerKeyMD5: 'STRING_VALUE',
//VersionId: 'STRING_VALUE'
};
s3.getObject(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
*/
/*
success:
{ AcceptRanges: 'bytes',
LastModified: 'Thu, 18 Aug 2016 06:49:12 GMT',
ContentLength: '0',
ETag: '"d41d8cd98f00b204e9800998ecf8427e"',
ContentType: 'text/plain',
Metadata: {},
Body: <Buffer > }
error:
{
[PreconditionFailed : At least one of the pre - conditions you specified did not hold]
message : 'At least one of the pre-conditions you specified did not hold',
code : 'PreconditionFailed',
region : null,
time : Fri Aug 19 2016 10 : 06 : 38 GMT + 0800(CST),
requestId : 'D57010FB7BAD5DD3',
extendedRequestId : 'GrAFqim56ytBONLN7oTj+Idhui6UlK66UfJ1GuP5X/aNocDpoufpOrRCdaxct3uUNXA/MBHIQIQ=',
cfId : undefined,
statusCode : 412,
retryable : false,
retryDelay : 38.56445546261966
}
'PreconditionFailed: At least one of the pre-conditions you specified did not hold\n at Request.extractError (/home/web/message_center/node_modules/aws-sdk/lib/services/s3.js:538:35)\n at Request.callListeners (/home/web/message_center/node_modules/aws-sdk/lib/sequential_executor.js:105:20)\n at Request.emit (/home/web/message_center/node_modules/aws-sdk/lib/sequential_executor.js:77:10)\n at Request.emit (/home/web/message_center/node_modules/aws-sdk/lib/request.js:661:14)\n at Request.transition (/home/web/message_center/node_modules/aws-sdk/lib/request.js:22:10)\n at AcceptorStateMachine.runTo (/home/web/message_center/node_modules/aws-sdk/lib/state_machine.js:14:12)\n at /home/web/message_center/node_modules/aws-sdk/lib/state_machine.js:26:10\n at Request.<anonymous>(/home/web/message_center/node_modules/aws-sdk/lib/request.js:38:9)\n at Request.<anonymous> (/home/web/message_center/node_modules/aws-sdk/lib/request.js:663:12)\n at Request.callListeners (/home/web/message_center/node_modules/aws-sdk/lib/sequential_executor.js:115:18)'
*/
/*
{ AcceptRanges: 'bytes',
LastModified: 'Thu, 18 Aug 2016 07:22:22 GMT',
ContentLength: '23',
ETag: '"0598c44b0e25dde9e8dfdb55043b3a5f-1"',
ContentEncoding: 'utf-8',
ContentLanguage: 'zh-cn',
ContentType: 'plain',
Metadata: {},
Body: <Buffer 54 68 69 73 20 69 73 20 61 20 66 69 6c 65 20 74 6f 20 74 65 73 74 2e> }
*/
/** put object
* @info Adds an object to a bucket.
***/
var s3 = new AWS.S3 ();
var crypto = require('crypto');
var body = 'pull by put object api2';
var md5 = crypto.createHash('md5');
var contentMd5 = md5.update(body).digest('base64');
var params =
Bucket: 'mochong', //required
Key: 'new_folder/file_put_object.txt', // required
ACL: 'private',
Body: body,
//CacheControl: 'STRING_VALUE',
//ContentDisposition: 'STRING_VALUE',
ContentEncoding: 'utf8',
ContentLanguage: 'zh-cn',
ContentLength: body.length,
ContentMD5: contentMd5,
ContentType: 'text/plain',
//Expires: new Date || 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' || 123456789,
//GrantFullControl: 'STRING_VALUE',
//GrantRead: 'STRING_VALUE',
//GrantReadACP: 'STRING_VALUE',
//GrantWriteACP: 'STRING_VALUE',
Metadata: {
'x-amz-meta-source-type': 'server',// Custom metadata must be prefixed with x-amz-meta-
'x-amz-meta-author': 'mochong'
/* anotherKey: ... */
},
//RequestPayer: 'requester',
//SSECustomerAlgorithm: 'STRING_VALUE',
//SSECustomerKey: new Buffer('...') || 'STRING_VALUE',
//SSECustomerKeyMD5: 'STRING_VALUE',
//SSEKMSKeyId: 'STRING_VALUE',
// ServerSideEncryption: 'AES256 | aws:kms',
StorageClass: 'STANDARD',
//WebsiteRedirectLocation: 'STRING_VALUE'
};
s3.putObject(params, function(err, data) {
if (err) console.log(err, err.stack); // an error occurred
else console.log(data); // successful response
});
/**
* response:
*{ ETag: '"e615fa17312c3e6e8a1682a5ce964d21"' }
*/
function syncFile(file) {
var bodyStream = fs.createReadStream(file.absolute_path, {encoding: 'utf8'});
var options = {
Bucket : config.aws.bucket,
Key : config.aws.key_prefix + "/" + file.relative_path,
ContentLength : file.size,
Body : bodyStream
};
s3.putObject(options, function(err, data) {
if (err) {
console.log("upload err,filepath:" + file.absolute_path);
console.log(err);
process.exit();
return false;
}
bodyStream.destroy();
});
}
Error:
2016-08-22 13:01:52 [ERROR] sync logs failed,err:BadRequest: An error occurred when parsing the HTTP request., file:/home/web/xxxx,file count:160
If a file is being After uploading and repeating the upload operation, the above error will be reported.