Compare commits
1 Commits
developmen
...
list-lifec
| Author | SHA1 | Date |
|---|---|---|
Nicolas Humbert
|
1e8eaa22b2 |
|
|
@ -36,6 +36,9 @@
|
|||
}, {
|
||||
"site": "us-east-2",
|
||||
"type": "aws_s3"
|
||||
}, {
|
||||
"site": "location-dmf-v1",
|
||||
"type": "dmf"
|
||||
}],
|
||||
"backbeat": {
|
||||
"host": "localhost",
|
||||
|
|
|
|||
|
|
@ -72,6 +72,7 @@ const parseCopySource = require('./apiUtils/object/parseCopySource');
|
|||
const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys');
|
||||
const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks');
|
||||
const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
|
||||
const { listLifecycleObjects } = require('./lifecycle/listLifecycleObjects');
|
||||
|
||||
const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
|
||||
|
||||
|
|
@ -297,6 +298,7 @@ const api = {
|
|||
corsPreflight,
|
||||
completeMultipartUpload,
|
||||
initiateMultipartUpload,
|
||||
listLifecycleObjects,
|
||||
listMultipartUploads,
|
||||
listParts,
|
||||
metadataSearch,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,262 @@
|
|||
const querystring = require('querystring');
|
||||
const { errors, versioning, s3middleware } = require('arsenal');
|
||||
const constants = require('../../../constants');
|
||||
const services = require('../../services');
|
||||
const { metadataValidateBucket } = require('../../metadata/metadataUtils');
|
||||
const collectCorsHeaders = require('../../utilities/collectCorsHeaders');
|
||||
const escapeForXml = s3middleware.escapeForXml;
|
||||
const { pushMetric } = require('../../utapi/utilities');
|
||||
const versionIdUtils = versioning.VersionID;
|
||||
const monitoring = require('../../utilities/monitoringHandler');
|
||||
const { generateToken, decryptToken }
|
||||
= require('../../api/apiUtils/object/continueToken');
|
||||
|
||||
/* Sample XML response for GET bucket objects:
|
||||
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Name>example-bucket</Name>
|
||||
<Prefix></Prefix>
|
||||
<KeyMarker></KeyMarker>
|
||||
<NextKeyMarker></NextKeyMarker>
|
||||
<DateMarker></DateMarker>
|
||||
<NextDateMarker></NextDateMarker>
|
||||
<MaxKeys>1000</MaxKeys>
|
||||
<Delimiter>/</Delimiter>
|
||||
<IsTruncated>false</IsTruncated>
|
||||
<Contents>
|
||||
<Key>sample.jpg</Key>
|
||||
<LastModified>2011-02-26T01:56:20.000Z</LastModified>
|
||||
<ETag>"bf1d737a4d46a19f3bced6905cc8b902"</ETag>
|
||||
<Size>142863</Size>
|
||||
<Owner>
|
||||
<ID>canonical-user-id</ID>
|
||||
<DisplayName>display-name</DisplayName>
|
||||
</Owner>
|
||||
<StorageClass>STANDARD</StorageClass>
|
||||
<TagSet>
|
||||
<Tag>
|
||||
<Key>string</Key>
|
||||
<Value>string</Value>
|
||||
</Tag>
|
||||
<Tag>
|
||||
<Key>string</Key>
|
||||
<Value>string</Value>
|
||||
</Tag>
|
||||
</TagSet>
|
||||
</Contents>
|
||||
</ListBucketResult>
|
||||
*/
|
||||
|
||||
function xmlTags(tags) {
|
||||
const xml = [];
|
||||
Object.entries(tags).forEach(([key, value]) =>
|
||||
xml.push(
|
||||
'<Tag>',
|
||||
`<Key>${key}</Key>`,
|
||||
`<Value>${value}</Value>`,
|
||||
'</Tag>'
|
||||
));
|
||||
return xml;
|
||||
}
|
||||
|
||||
/* eslint-enable max-len */
|
||||
function processMasterVersions(bucketName, listParams, list) {
|
||||
const xml = [];
|
||||
xml.push(
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">',
|
||||
'<Name>', bucketName, '</Name>'
|
||||
);
|
||||
const isTruncated = list.IsTruncated ? 'true' : 'false';
|
||||
const xmlParams = [
|
||||
{ tag: 'Prefix', value: listParams.prefix || '' },
|
||||
{ tag: 'MaxKeys', value: listParams.maxKeys },
|
||||
{ tag: 'EncodingType', value: listParams.encoding },
|
||||
{ tag: 'IsTruncated', value: isTruncated },
|
||||
{ tag: 'DateMarker', value: listParams.dateMarker },
|
||||
{ tag: 'NextDateMarker', value: list.NextDateMarker },
|
||||
{ tag: 'KeyMarker', value: listParams.keyMarker },
|
||||
{ tag: 'NextKeyMarker', value: list.NextKeyMarker },
|
||||
];
|
||||
|
||||
const escapeXmlFn = listParams.encoding === 'url' ?
|
||||
querystring.escape : escapeForXml;
|
||||
|
||||
xmlParams.forEach(p => {
|
||||
if (p.value) {
|
||||
xml.push(`<${p.tag}>${escapeXmlFn(p.value)}</${p.tag}>`);
|
||||
}
|
||||
});
|
||||
|
||||
list.Contents.forEach(item => {
|
||||
const v = item.value;
|
||||
const objectKey = escapeXmlFn(item.key);
|
||||
xml.push(
|
||||
'<Contents>',
|
||||
`<Key>${objectKey}</Key>`,
|
||||
`<LastModified>${v.LastModified}</LastModified>`,
|
||||
`<ETag>"${v.ETag}"</ETag>`,
|
||||
`<Size>${v.Size}</Size>`,
|
||||
'<Owner>',
|
||||
`<ID>${v.Owner.ID}</ID>`,
|
||||
`<DisplayName>${v.Owner.DisplayName}</DisplayName>`,
|
||||
'</Owner>',
|
||||
`<StorageClass>${v.StorageClass}</StorageClass>`,
|
||||
`<TagSet> ${xmlTags(v.tags)} </TagSet>`,
|
||||
'</Contents>',
|
||||
);
|
||||
});
|
||||
xml.push('</ListBucketResult>');
|
||||
return xml.join('');
|
||||
}
|
||||
|
||||
function handleResult(listParams, requestMaxKeys, encoding, authInfo,
|
||||
bucketName, list, corsHeaders, log, callback) {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
listParams.maxKeys = requestMaxKeys;
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
listParams.encoding = encoding;
|
||||
const res = processMasterVersions(bucketName, listParams, list);
|
||||
|
||||
pushMetric('listLifecycleObjects', log, { authInfo, bucket: bucketName });
|
||||
monitoring.promMetrics('GET', bucketName, '200', 'listLifecycleObjects');
|
||||
return callback(null, res, corsHeaders);
|
||||
}
|
||||
|
||||
/**
|
||||
* listLifecycleObjects - Return list of objects in bucket, supports v1 & v2
|
||||
* @param {AuthInfo} authInfo - Instance of AuthInfo class with
|
||||
* requester's info
|
||||
* @param {object} request - http request object
|
||||
* @param {function} log - Werelogs request logger
|
||||
* @param {function} callback - callback to respond to http request
|
||||
* with either error code or xml response body
|
||||
* @return {undefined}
|
||||
*/
|
||||
function listLifecycleObjects(authInfo, request, log, callback) {
|
||||
const params = request.query;
|
||||
const bucketName = request.bucketName;
|
||||
// const v2 = params['list-type'];
|
||||
// if (v2 !== undefined && Number.parseInt(v2, 10) !== 2) {
|
||||
// return callback(errors.InvalidArgument.customizeDescription('Invalid ' +
|
||||
// 'List Type specified in Request'));
|
||||
// }
|
||||
// if (v2) {
|
||||
// log.addDefaultFields({
|
||||
// action: 'ListObjectsV2',
|
||||
// });
|
||||
// } else if (params.versions !== undefined) {
|
||||
// log.addDefaultFields({
|
||||
// action: 'ListObjectVersions',
|
||||
// });
|
||||
// }
|
||||
log.debug('processing request', { method: 'bucketGet' });
|
||||
const encoding = params['encoding-type'];
|
||||
if (encoding !== undefined && encoding !== 'url') {
|
||||
monitoring.promMetrics(
|
||||
'GET', bucketName, 400, 'listBucket');
|
||||
return callback(errors.InvalidArgument.customizeDescription('Invalid ' +
|
||||
'Encoding Method specified in Request'));
|
||||
}
|
||||
const requestMaxKeys = params['max-keys'] ?
|
||||
Number.parseInt(params['max-keys'], 10) : 1000;
|
||||
if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) {
|
||||
monitoring.promMetrics(
|
||||
'GET', bucketName, 400, 'listBucket');
|
||||
return callback(errors.InvalidArgument);
|
||||
}
|
||||
// AWS only returns 1000 keys even if max keys are greater.
|
||||
// Max keys stated in response xml can be greater than actual
|
||||
// keys returned.
|
||||
const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys);
|
||||
|
||||
const metadataValParams = {
|
||||
authInfo,
|
||||
bucketName,
|
||||
requestType: 'listLifecycleObjects',
|
||||
request,
|
||||
};
|
||||
const listParams = {
|
||||
listingType: 'DelimiterLifecycle',
|
||||
maxKeys: actualMaxKeys,
|
||||
prefix: params.prefix,
|
||||
dateMarker: params['date-marker'],
|
||||
beforeDate: params['before-date'],
|
||||
keyMarker: params['key-marker'],
|
||||
// before: params.before,
|
||||
};
|
||||
|
||||
// if (params.delimiter) {
|
||||
// listParams.delimiter = params.delimiter;
|
||||
// }
|
||||
|
||||
// if (v2) {
|
||||
// listParams.v2 = true;
|
||||
// listParams.startAfter = params['start-after'];
|
||||
// listParams.continuationToken =
|
||||
// decryptToken(params['continuation-token']);
|
||||
// listParams.fetchOwner = params['fetch-owner'] === 'true';
|
||||
// } else {
|
||||
// listParams.marker = params.marker;
|
||||
// }
|
||||
// listParams.marker = params.marker;
|
||||
const beforeDate = params['before-date'];
|
||||
const dateMarker = params['date-marker'];
|
||||
let filter = null
|
||||
|
||||
if (beforeDate || dateMarker) {
|
||||
filter = {'value.last-modified': {}};
|
||||
if (beforeDate) {
|
||||
filter['value.last-modified']['$lt'] = beforeDate;
|
||||
}
|
||||
if (dateMarker) {
|
||||
filter['value.last-modified']['$gt'] = dateMarker;
|
||||
}
|
||||
}
|
||||
|
||||
// listParams.mongifiedSearch = filter;
|
||||
|
||||
metadataValidateBucket(metadataValParams, log, (err, bucket) => {
|
||||
const corsHeaders = collectCorsHeaders(request.headers.origin,
|
||||
request.method, bucket);
|
||||
if (err) {
|
||||
log.debug('error processing request', { error: err });
|
||||
monitoring.promMetrics(
|
||||
'GET', bucketName, err.code, 'listBucket');
|
||||
return callback(err, null, corsHeaders);
|
||||
}
|
||||
// if (params.versions !== undefined) {
|
||||
// listParams.listingType = 'DelimiterVersions';
|
||||
// delete listParams.marker;
|
||||
// listParams.keyMarker = params['key-marker'];
|
||||
// listParams.versionIdMarker = params['version-id-marker'] ?
|
||||
// versionIdUtils.decode(params['version-id-marker']) : undefined;
|
||||
// }
|
||||
if (!requestMaxKeys) {
|
||||
const emptyList = {
|
||||
CommonPrefixes: [],
|
||||
Contents: [],
|
||||
IsTruncated: false,
|
||||
};
|
||||
return handleResult(listParams, requestMaxKeys, encoding, authInfo,
|
||||
bucketName, emptyList, corsHeaders, log, callback);
|
||||
}
|
||||
|
||||
return services.getLifecycleObjectListing(bucketName, listParams, log,
|
||||
(err, list) => {
|
||||
if (err) {
|
||||
log.debug('error processing request', { error: err });
|
||||
monitoring.promMetrics(
|
||||
'GET', bucketName, err.code, 'listLifecycleObjects');
|
||||
return callback(err, null, corsHeaders);
|
||||
}
|
||||
return handleResult(listParams, requestMaxKeys, encoding, authInfo,
|
||||
bucketName, list, corsHeaders, log, callback);
|
||||
});
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
processMasterVersions,
|
||||
listLifecycleObjects,
|
||||
};
|
||||
|
|
@ -360,6 +360,20 @@ const services = {
|
|||
});
|
||||
},
|
||||
|
||||
getLifecycleObjectListing(bucketName, listingParams, log, cb) {
|
||||
assert.strictEqual(typeof bucketName, 'string');
|
||||
log.trace('performing metadata get lifecycle object listing',
|
||||
{ listingParams });
|
||||
metadata.listLifecycleObject(bucketName, listingParams, log,
|
||||
(err, listResponse) => {
|
||||
if (err) {
|
||||
log.debug('error from metadata', { error: err });
|
||||
return cb(err);
|
||||
}
|
||||
return cb(null, listResponse);
|
||||
});
|
||||
},
|
||||
|
||||
metadataStoreMPObject(bucketName, cipherBundle, params, log, cb) {
|
||||
assert.strictEqual(typeof bucketName, 'string');
|
||||
assert.strictEqual(typeof params.splitter, 'string');
|
||||
|
|
|
|||
|
|
@ -1,105 +1,27 @@
|
|||
{
|
||||
"us-east-1": {
|
||||
"type": "file",
|
||||
"objectId": "us-east-1",
|
||||
"legacyAwsBehavior": true,
|
||||
"details": {}
|
||||
},
|
||||
"us-east-2": {
|
||||
"type": "file",
|
||||
"objectId": "us-east-2",
|
||||
"details": {
|
||||
"supportsVersioning": true
|
||||
},
|
||||
"isTransient": false,
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"us-west-1": {
|
||||
"type": "file",
|
||||
"objectId": "us-west-1",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"us-west-2": {
|
||||
"type": "file",
|
||||
"objectId": "us-west-2",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"ca-central-1": {
|
||||
"type": "file",
|
||||
"objectId": "ca-central-1",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"cn-north-1": {
|
||||
"type": "file",
|
||||
"objectId": "cn-north-1",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"ap-south-1": {
|
||||
"type": "file",
|
||||
"objectId": "ap-south-1",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"ap-northeast-1": {
|
||||
"type": "file",
|
||||
"objectId": "ap-northeast-1",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"ap-northeast-2": {
|
||||
"type": "file",
|
||||
"objectId": "ap-northeast-2",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"ap-southeast-1": {
|
||||
"type": "file",
|
||||
"objectId": "ap-southeast-1",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"ap-southeast-2": {
|
||||
"type": "file",
|
||||
"objectId": "ap-southeast-2",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"eu-central-1": {
|
||||
"type": "file",
|
||||
"objectId": "eu-central-1",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"eu-west-1": {
|
||||
"type": "file",
|
||||
"objectId": "eu-west-1",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"eu-west-2": {
|
||||
"type": "file",
|
||||
"objectId": "eu-west-2",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"EU": {
|
||||
"type": "file",
|
||||
"objectId": "EU",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
},
|
||||
"sa-east-1": {
|
||||
"type": "file",
|
||||
"objectId": "sa-east-1",
|
||||
"legacyAwsBehavior": false,
|
||||
"details": {}
|
||||
"objectId": "0b1d9226-a694-11eb-bc21-baec55d199cd",
|
||||
"type": "file"
|
||||
},
|
||||
"location-dmf-v1": {
|
||||
"type": "dmf",
|
||||
"objectId": "location-dmf-v1",
|
||||
"legacyAwsBehavior": false,
|
||||
"isCold": true,
|
||||
"details": {}
|
||||
"details": {
|
||||
"endpoint": "ws://localhost:5001/session",
|
||||
"username": "user1",
|
||||
"password": "pass1",
|
||||
"repoId": [
|
||||
"233aead6-1d7b-4647-a7cf-0d3280b5d1d7",
|
||||
"81e78de8-df11-4acd-8ad1-577ff05a68db"
|
||||
],
|
||||
"nsId": "65f9fd61-42fe-4a68-9ac0-6ba25311cc85"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@
|
|||
"ft_util": "cd tests/functional/utilities && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js",
|
||||
"ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management ft_util",
|
||||
"ft_search": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 90000 test/mdSearch",
|
||||
"ft_lifecycle": "cd tests/functional/aws-node-sdk && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 90000 test/listLifecycle",
|
||||
"ft_kmip": "cd tests/functional/kmip && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 40000 *.js",
|
||||
"install_ft_deps": "yarn install aws-sdk@2.28.0 bluebird@3.3.1 mocha@2.3.4 mocha-junit-reporter@1.23.1 tv4@1.2.7",
|
||||
"lint": "eslint $(git ls-files '*.js')",
|
||||
|
|
@ -100,7 +101,7 @@
|
|||
"start_pfsserver": "node pfsserver.js",
|
||||
"start_s3server": "node index.js",
|
||||
"start_dmd": "npm-run-all --parallel start_mdserver start_dataserver",
|
||||
"start_utapi": "node lib/utapi/utapi.js",
|
||||
"start_utapi": "node lib/u tapi/utapi.js",
|
||||
"start_secure_channel_proxy": "node bin/secure_channel_proxy.js",
|
||||
"start_metrics_server": "node bin/metrics_server.js",
|
||||
"utapi_replay": "node lib/utapi/utapiReplay.js",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,321 @@
|
|||
const { lifecycleClient, s3Client } = require('./utils/sdks');
|
||||
// const { runAndCheckSearch, runIfMongo } = require('./utils/helpers');
|
||||
const { mongoClient } = require('../../../utilities/mongoClient');
|
||||
const async = require('async');
|
||||
const assert = require('assert');
|
||||
|
||||
const firstObjectKey = 'first';
|
||||
const secondObjectKey = 'second';
|
||||
const thirdObjectKey = 'third';
|
||||
|
||||
const tagKey = 'item-type';
|
||||
const tagValue = 'main';
|
||||
const objectTagData = `${tagKey}=${tagValue}`;
|
||||
|
||||
const userName = 'Bart';
|
||||
const userCanonicalId = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be';
|
||||
const owner = {
|
||||
DisplayName: userName,
|
||||
ID: userCanonicalId,
|
||||
};
|
||||
|
||||
const expectedFirstObject = {
|
||||
Key: firstObjectKey,
|
||||
Owner: owner,
|
||||
Size: 0,
|
||||
StorageClass: 'STANDARD',
|
||||
TagSet: [{
|
||||
Key: tagKey,
|
||||
Value: tagValue,
|
||||
}],
|
||||
};
|
||||
|
||||
const expectedSecondObject = {
|
||||
Key: secondObjectKey,
|
||||
Owner: owner,
|
||||
Size: 0,
|
||||
StorageClass: 'STANDARD',
|
||||
TagSet: [],
|
||||
};
|
||||
|
||||
const expectedThirdObject = {
|
||||
Key: thirdObjectKey,
|
||||
Owner: owner,
|
||||
Size: 0,
|
||||
StorageClass: 'STANDARD',
|
||||
TagSet: [],
|
||||
};
|
||||
|
||||
const runIfMongo = process.env.S3METADATA === 'mongodb' ?
|
||||
describe : describe.skip;
|
||||
|
||||
function check(data, expected) {
|
||||
Object.keys(expected).forEach(
|
||||
r => {
|
||||
if (r === 'Contents') {
|
||||
assert.strictEqual(data.Contents.length, expected.Contents.length);
|
||||
expected.Contents.forEach((content, i) => {
|
||||
Object.keys(content).forEach(k => {
|
||||
assert.deepStrictEqual(data.Contents[i][k], content[k], `Contents[${i}].${k} value is invalid`);
|
||||
})
|
||||
});
|
||||
} else {
|
||||
assert.strictEqual(data[r], expected[r], `${r} value is invalid`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
runIfMongo('Basic search', () => {
|
||||
const bucketName = `basicsearchmebucket${Date.now()}`;
|
||||
let startDate;
|
||||
let firstDoneDate;
|
||||
let secondDoneDate;
|
||||
let thirdDoneDate;
|
||||
before(done =>
|
||||
async.series([
|
||||
next => s3Client.createBucket({ Bucket: bucketName }, err => {
|
||||
startDate = new Date().toISOString();
|
||||
return next(err);
|
||||
}),
|
||||
next => s3Client.putObject({ Bucket: bucketName, Key: firstObjectKey, Tagging: objectTagData }, err => {
|
||||
firstDoneDate = new Date().toISOString();
|
||||
return next(err);
|
||||
}),
|
||||
next => s3Client.putObject({ Bucket: bucketName, Key: secondObjectKey}, err => {
|
||||
secondDoneDate = new Date().toISOString();
|
||||
return next(err);
|
||||
}),
|
||||
next => s3Client.putObject({ Bucket: bucketName, Key: thirdObjectKey}, err => {
|
||||
thirdDoneDate = new Date().toISOString();
|
||||
return next(err);
|
||||
}),
|
||||
], done));
|
||||
|
||||
after(done => {
|
||||
s3Client.deleteObjects({ Bucket: bucketName, Delete: { Objects: [
|
||||
{ Key: firstObjectKey },
|
||||
{ Key: secondObjectKey },
|
||||
{ Key: thirdObjectKey }],
|
||||
} },
|
||||
err => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
return s3Client.deleteBucket({ Bucket: bucketName }, done);
|
||||
});
|
||||
});
|
||||
|
||||
// TODO test keyMarker != key
|
||||
// TODO test so NextkeyMarker === last key
|
||||
|
||||
it('should return all three objects', done => {
|
||||
return lifecycleClient.listLifecycleObjects({
|
||||
Bucket: bucketName,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
const expected = {
|
||||
IsTruncated: false,
|
||||
Contents: [
|
||||
expectedFirstObject, expectedSecondObject, expectedThirdObject
|
||||
],
|
||||
Name: bucketName,
|
||||
MaxKeys: 1000,
|
||||
}
|
||||
check(data, expected);
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('beforeDate: should return empty list', done => {
|
||||
return lifecycleClient.listLifecycleObjects({
|
||||
Bucket: bucketName,
|
||||
BeforeDate: startDate,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
const expected = {
|
||||
IsTruncated: false,
|
||||
Contents: [],
|
||||
Name: bucketName,
|
||||
MaxKeys: 1000,
|
||||
}
|
||||
check(data, expected);
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('beforeDate: should return the first object', done => {
|
||||
return lifecycleClient.listLifecycleObjects({
|
||||
Bucket: bucketName,
|
||||
BeforeDate: firstDoneDate,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
const expected = {
|
||||
IsTruncated: false,
|
||||
Contents: [expectedFirstObject],
|
||||
Name: bucketName,
|
||||
MaxKeys: 1000,
|
||||
}
|
||||
check(data, expected);
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
it('beforeDate: should return the first and second object', done => {
|
||||
return lifecycleClient.listLifecycleObjects({
|
||||
Bucket: bucketName,
|
||||
BeforeDate: secondDoneDate,
|
||||
}, (err, data) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
const expected = {
|
||||
IsTruncated: false,
|
||||
Contents: [expectedFirstObject, expectedSecondObject],
|
||||
Name: bucketName,
|
||||
MaxKeys: 1000,
|
||||
}
|
||||
check(data, expected);
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
// runIfMongo('Basic search 2', () => {
|
||||
// const bucketName = `basicsearchmebucket${Date.now()}`;
|
||||
// let startDate;
|
||||
// let afterFirstDate;
|
||||
// let afterSecondDate;
|
||||
// let afterThirdDate;
|
||||
// before(done => {
|
||||
// mongoClient.connectClient(err => {
|
||||
// s3Client.createBucket({ Bucket: bucketName }, err => {
|
||||
// if (err) {
|
||||
// return done(err);
|
||||
// }
|
||||
// startDate = new Date().toISOString();
|
||||
// async.each([firstObjectKey, secondObjectKey, thirdObjectKey], (objectName, cb) => {
|
||||
// return s3Client.putObject({ Bucket: bucketName, Key: objectName}, cb);
|
||||
// }, err => {
|
||||
// if (err) {
|
||||
// return done(err);
|
||||
// }
|
||||
// return mongoClient.matchObjectsLastModified(bucketName, '2022-12-24T16:55:36.762Z', err => {
|
||||
// if (err) {
|
||||
// return done(err);
|
||||
// }
|
||||
// return s3Client.putObject({ Bucket: bucketName, Key: 'four'}, done);
|
||||
// });
|
||||
// });
|
||||
// });
|
||||
// })
|
||||
// });
|
||||
|
||||
// after(done => {
|
||||
// s3Client.deleteObjects({ Bucket: bucketName, Delete: { Objects: [
|
||||
// { Key: firstObjectKey },
|
||||
// { Key: secondObjectKey },
|
||||
// { Key: thirdObjectKey },
|
||||
// { Key: 'four' },
|
||||
// ],
|
||||
// } },
|
||||
// err => {
|
||||
// if (err) {
|
||||
// return done(err);
|
||||
// }
|
||||
// return s3Client.deleteBucket({ Bucket: bucketName }, err => {
|
||||
// if (err) {
|
||||
// return done(err);
|
||||
// }
|
||||
// mongoClient.disconnectClient(done);
|
||||
// });
|
||||
// });
|
||||
// });
|
||||
|
||||
// it('should list lifecycle objects', done => {
|
||||
// return lifecycleClient.listLifecycleObjects({
|
||||
// Bucket: bucketName,
|
||||
// }, (err, data) => {
|
||||
// console.log('TOTAL err!!!', err);
|
||||
// console.log('TOTAL data!!!', data);
|
||||
// return lifecycleClient.listLifecycleObjects({
|
||||
// Bucket: bucketName,
|
||||
// MaxKeys: 1,
|
||||
// }, (err, data) => {
|
||||
// console.log('1 err!!!', err);
|
||||
// console.log('1 data!!!', data);
|
||||
// return lifecycleClient.listLifecycleObjects({
|
||||
// Bucket: bucketName,
|
||||
// DateMarker: data.NextDateMarker,
|
||||
// KeyMarker: data.NextKeyMarker,
|
||||
// MaxKeys: 1,
|
||||
// }, (err, data) => {
|
||||
// console.log('2 err!!!', err);
|
||||
// console.log('2 data!!!', data);
|
||||
// return lifecycleClient.listLifecycleObjects({
|
||||
// Bucket: bucketName,
|
||||
// DateMarker: data.NextDateMarker,
|
||||
// KeyMarker: data.NextKeyMarker,
|
||||
// MaxKeys: 1,
|
||||
// }, (err, data) => {
|
||||
// console.log('3 err!!!', err);
|
||||
// console.log('3 data!!!', data);
|
||||
// return lifecycleClient.listLifecycleObjects({
|
||||
// Bucket: bucketName,
|
||||
// DateMarker: data.NextDateMarker,
|
||||
// KeyMarker: data.NextKeyMarker,
|
||||
// MaxKeys: 1,
|
||||
// }, (err, data) => {
|
||||
// console.log('4 err!!!', err);
|
||||
// console.log('4 data!!!', data);
|
||||
// });
|
||||
// });
|
||||
// });
|
||||
// });
|
||||
// });
|
||||
// });
|
||||
// });
|
||||
|
||||
// runIfMongo('Search when no objects in bucket', () => {
|
||||
// const bucketName = `noobjectbucket${Date.now()}`;
|
||||
// before(done => {
|
||||
// s3Client.createBucket({ Bucket: bucketName }, done);
|
||||
// });
|
||||
|
||||
// after(done => {
|
||||
// s3Client.deleteBucket({ Bucket: bucketName }, done);
|
||||
// });
|
||||
|
||||
// it('should return empty listing when no objects in bucket', done => {
|
||||
// const encodedSearch = encodeURIComponent(`key="${objectKey}"`);
|
||||
// return runAndCheckSearch(lifecycleClient, bucketName,
|
||||
// encodedSearch, false, null, done);
|
||||
// });
|
||||
// });
|
||||
|
||||
// runIfMongo('Invalid regular expression searches', () => {
|
||||
// const bucketName = `badregex-${Date.now()}`;
|
||||
// before(done => {
|
||||
// s3Client.createBucket({ Bucket: bucketName }, done);
|
||||
// });
|
||||
|
||||
// after(done => {
|
||||
// s3Client.deleteBucket({ Bucket: bucketName }, done);
|
||||
// });
|
||||
|
||||
// it('should return error if pattern is invalid', done => {
|
||||
// const encodedSearch = encodeURIComponent('key LIKE "/((helloworld/"');
|
||||
// const testError = {
|
||||
// code: 'InvalidArgument',
|
||||
// message: 'Invalid sql where clause sent as search query',
|
||||
// };
|
||||
// return runAndCheckSearch(lifecycleClient, bucketName,
|
||||
// encodedSearch, false, testError, done);
|
||||
// });
|
||||
// });
|
||||
|
|
@ -0,0 +1,299 @@
|
|||
{
|
||||
"version": "2.0",
|
||||
"metadata": {
|
||||
"apiVersion": "2023-01-01",
|
||||
"checksumFormat": "md5",
|
||||
"endpointPrefix": "s3",
|
||||
"protocol": "rest-xml",
|
||||
"serviceAbbreviation": "Lifecycle",
|
||||
"serviceFullName": "Lifecycle Listing Client",
|
||||
"serviceId": "lifecycle",
|
||||
"signatureVersion": "v4",
|
||||
"uid": "lifecycle-2023-01-01"
|
||||
},
|
||||
"operations": {
|
||||
"ListLifecycleObjects": {
|
||||
"name": "ListLifecycleObjects",
|
||||
"http": {
|
||||
"method": "GET",
|
||||
"requestUri": "/{Bucket}?scal-list-type=lifecycle"
|
||||
},
|
||||
"input": {
|
||||
"shape": "ListLifecycleObjectsRequest"
|
||||
},
|
||||
"output": {
|
||||
"shape": "ListLifecycleObjectsOutput"
|
||||
},
|
||||
"errors": [
|
||||
{
|
||||
"shape": "NoSuchBucket"
|
||||
}
|
||||
],
|
||||
"documentation": "<p>Returns some or all (up to 1,000) of the objects in a non version bucket. You can use the request parameters as selection criteria to return a subset of the objects in a bucket. A 200 OK response can contain valid or invalid XML. Be sure to design your application to parse the contents of the response and handle it appropriately.</p>",
|
||||
"alias": "GetBucket"
|
||||
}
|
||||
},
|
||||
"shapes": {
|
||||
"BeforeDate": {
|
||||
"type": "string"
|
||||
},
|
||||
"BucketName": {
|
||||
"type": "string"
|
||||
},
|
||||
"ChecksumAlgorithm": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"CRC32",
|
||||
"CRC32C",
|
||||
"SHA1",
|
||||
"SHA256"
|
||||
]
|
||||
},
|
||||
"ChecksumAlgorithmList": {
|
||||
"type": "list",
|
||||
"member": {
|
||||
"shape": "ChecksumAlgorithm"
|
||||
},
|
||||
"flattened": true
|
||||
},
|
||||
"Date": {
|
||||
"type": "timestamp",
|
||||
"timestampFormat": "iso8601"
|
||||
},
|
||||
"DisplayName": {
|
||||
"type": "string"
|
||||
},
|
||||
"EncodingType": {
|
||||
"type": "string",
|
||||
"documentation": "<p>Requests Amazon S3 to encode the object keys in the response and specifies the encoding method to use. An object key may contain any Unicode character; however, XML 1.0 parser cannot parse some characters, such as characters with an ASCII value from 0 to 10. For characters that are not supported in XML 1.0, you can add this parameter to request that Amazon S3 encode the keys in the response.</p>",
|
||||
"enum": [
|
||||
"url"
|
||||
]
|
||||
},
|
||||
"ETag": {
|
||||
"type": "string"
|
||||
},
|
||||
"KeyMarker": {
|
||||
"type": "string"
|
||||
},
|
||||
"ID": {
|
||||
"type": "string"
|
||||
},
|
||||
"IsTruncated": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"LastModified": {
|
||||
"type": "timestamp"
|
||||
},
|
||||
"ListLifecycleObjectsOutput": {
|
||||
"type": "structure",
|
||||
"members": {
|
||||
"DateMarker": {
|
||||
"shape": "DateMarker",
|
||||
"documentation": "<p>Indicates where in the bucket listing begins. DateMarker is included in the response if it was sent with the request.</p>"
|
||||
},
|
||||
"KeyMarker": {
|
||||
"shape": "KeyMarker",
|
||||
"documentation": "<p>Indicates where in the bucket listing begins. KeyMarker is included in the response if it was sent with the request.</p>"
|
||||
},
|
||||
"IsTruncated": {
|
||||
"shape": "IsTruncated",
|
||||
"documentation": "<p>A flag that indicates whether Cloudserver returned all of the results that satisfied the search criteria.</p>"
|
||||
},
|
||||
"NextDateMarker": {
|
||||
"shape": "NextDateMarker",
|
||||
"documentation": "<p>When response is truncated (the IsTruncated element value in the response is true), you can use the date in this field as marker in the subsequent request to get next set of objects. Cloudserver orders objects in ascending order.</p>"
|
||||
},
|
||||
"NextKeyMarker": {
|
||||
"shape": "NextKeyMarker",
|
||||
"documentation": "<p>When response is truncated (the IsTruncated element value in the response is true), you can use the key name in this field as key-marker in the subsequent request to get next set of objects.</p>"
|
||||
},
|
||||
"Contents": {
|
||||
"shape": "ObjectLifecycleList",
|
||||
"documentation": "<p>Metadata about each object returned.</p>"
|
||||
},
|
||||
"Name": {
|
||||
"shape": "BucketName",
|
||||
"documentation": "<p>The bucket name.</p>"
|
||||
},
|
||||
"Prefix": {
|
||||
"shape": "Prefix",
|
||||
"documentation": "<p>Keys that begin with the indicated prefix.</p>"
|
||||
},
|
||||
"MaxKeys": {
|
||||
"shape": "MaxKeys",
|
||||
"documentation": "<p>The maximum number of keys returned in the response body.</p>"
|
||||
},
|
||||
"EncodingType": {
|
||||
"shape": "EncodingType",
|
||||
"documentation": "<p>Encoding type used by Amazon S3 to encode object keys in the response.</p>"
|
||||
}
|
||||
}
|
||||
},
|
||||
"ListLifecycleObjectsRequest": {
|
||||
"type": "structure",
|
||||
"required": [
|
||||
"Bucket"
|
||||
],
|
||||
"members": {
|
||||
"Bucket": {
|
||||
"shape": "BucketName",
|
||||
"documentation": "<p>The name of the bucket containing the objects.</p> <p>When using this action with an access point, you must direct requests to the access point hostname. The access point hostname takes the form <i>AccessPointName</i>-<i>AccountId</i>.s3-accesspoint.<i>Region</i>.amazonaws.com. When using this action with an access point through the Amazon Web Services SDKs, you provide the access point ARN in place of the bucket name. For more information about access point ARNs, see <a href=\"https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html\">Using access points</a> in the <i>Amazon S3 User Guide</i>.</p> <p>When using this action with Amazon S3 on Outposts, you must direct requests to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form <code> <i>AccessPointName</i>-<i>AccountId</i>.<i>outpostID</i>.s3-outposts.<i>Region</i>.amazonaws.com</code>. When using this action with S3 on Outposts through the Amazon Web Services SDKs, you provide the Outposts bucket ARN in place of the bucket name. For more information about S3 on Outposts ARNs, see <a href=\"https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html\">Using Amazon S3 on Outposts</a> in the <i>Amazon S3 User Guide</i>.</p>",
|
||||
"contextParam": {
|
||||
"name": "Bucket"
|
||||
},
|
||||
"location": "uri",
|
||||
"locationName": "Bucket"
|
||||
},
|
||||
"BeforeDate": {
|
||||
"shape": "BeforeDate",
|
||||
"documentation": "<p>Limit the response to keys modified prior to before date.</p>",
|
||||
"location": "querystring",
|
||||
"locationName": "before-date"
|
||||
},
|
||||
"EncodingType": {
|
||||
"shape": "EncodingType",
|
||||
"location": "querystring",
|
||||
"locationName": "encoding-type"
|
||||
},
|
||||
"KeyMarker": {
|
||||
"shape": "KeyMarker",
|
||||
"documentation": "<p>KeyMarker is where you want Cloudserver to start listing from. CloudServer starts listing after this specified key. Marker can be any key in the bucket.</p>",
|
||||
"location": "querystring",
|
||||
"locationName": "key-marker"
|
||||
},
|
||||
"DateMarker": {
|
||||
"shape": "DateMarker",
|
||||
"documentation": "<p>DateMarker is where you want Cloudserver to start listing from. Cloudserver starts listing after this specified date.</p>",
|
||||
"location": "querystring",
|
||||
"locationName": "date-marker"
|
||||
},
|
||||
"MaxKeys": {
|
||||
"shape": "MaxKeys",
|
||||
"documentation": "<p>Sets the maximum number of keys returned in the response. By default the action returns up to 1,000 key names. The response might contain fewer keys but will never contain more. </p>",
|
||||
"location": "querystring",
|
||||
"locationName": "max-keys"
|
||||
},
|
||||
"Prefix": {
|
||||
"shape": "Prefix",
|
||||
"documentation": "<p>Limits the response to keys that begin with the specified prefix.</p>",
|
||||
"location": "querystring",
|
||||
"locationName": "prefix"
|
||||
}
|
||||
}
|
||||
},
|
||||
"DateMarker": {
|
||||
"type": "string"
|
||||
},
|
||||
"MaxKeys": {
|
||||
"type": "integer"
|
||||
},
|
||||
"NextDateMarker": {
|
||||
"type": "string"
|
||||
},
|
||||
"NextKeyMarker": {
|
||||
"type": "string"
|
||||
},
|
||||
"ObjectLifecycle": {
|
||||
"type": "structure",
|
||||
"members": {
|
||||
"Key": {
|
||||
"shape": "ObjectKey",
|
||||
"documentation": "<p>The name that you assign to an object. You use the object key to retrieve the object.</p>"
|
||||
},
|
||||
"LastModified": {
|
||||
"shape": "LastModified",
|
||||
"documentation": "<p>Creation date of the object.</p>"
|
||||
},
|
||||
"ETag": {
|
||||
"shape": "ETag",
|
||||
"documentation": "<p>The entity tag is a hash of the object. The ETag reflects changes only to the contents of an object, not its metadata. The ETag may or may not be an MD5 digest of the object data. Whether or not it is depends on how the object was created and how it is encrypted as described below:</p> <ul> <li> <p>Objects created by the PUT Object, POST Object, or Copy operation, or through the Amazon Web Services Management Console, and are encrypted by SSE-S3 or plaintext, have ETags that are an MD5 digest of their object data.</p> </li> <li> <p>Objects created by the PUT Object, POST Object, or Copy operation, or through the Amazon Web Services Management Console, and are encrypted by SSE-C or SSE-KMS, have ETags that are not an MD5 digest of their object data.</p> </li> <li> <p>If an object is created by either the Multipart Upload or Part Copy operation, the ETag is not an MD5 digest, regardless of the method of encryption. If an object is larger than 16 MB, the Amazon Web Services Management Console will upload or copy that object as a Multipart Upload, and therefore the ETag will not be an MD5 digest.</p> </li> </ul>"
|
||||
},
|
||||
"ChecksumAlgorithm": {
|
||||
"shape": "ChecksumAlgorithmList",
|
||||
"documentation": "<p>The algorithm that was used to create a checksum of the object.</p>"
|
||||
},
|
||||
"Size": {
|
||||
"shape": "Size",
|
||||
"documentation": "<p>Size in bytes of the object</p>"
|
||||
},
|
||||
"StorageClass": {
|
||||
"shape": "ObjectStorageClass",
|
||||
"documentation": "<p>The class of storage used to store the object.</p>"
|
||||
},
|
||||
"Owner": {
|
||||
"shape": "Owner",
|
||||
"documentation": "<p>The owner of the object</p>"
|
||||
},
|
||||
"TagSet": {
|
||||
"shape": "TagSet",
|
||||
"documentation": "<p>Contains the tag set.</p>"
|
||||
}
|
||||
},
|
||||
"documentation": "<p>An object consists of data and its descriptive metadata.</p>"
|
||||
},
|
||||
"ObjectKey": {
|
||||
"type": "string",
|
||||
"min": 1
|
||||
},
|
||||
"ObjectLifecycleList": {
|
||||
"type": "list",
|
||||
"member": {
|
||||
"shape": "ObjectLifecycle"
|
||||
},
|
||||
"flattened": true
|
||||
},
|
||||
"ObjectStorageClass": {
|
||||
"type": "string"
|
||||
},
|
||||
"Owner": {
|
||||
"type": "structure",
|
||||
"members": {
|
||||
"DisplayName": {
|
||||
"shape": "DisplayName",
|
||||
"documentation": "<p>Container for the display name of the owner.</p>"
|
||||
},
|
||||
"ID": {
|
||||
"shape": "ID",
|
||||
"documentation": "<p>Container for the ID of the owner.</p>"
|
||||
}
|
||||
},
|
||||
"documentation": "<p>Container for the owner's display name and ID.</p>"
|
||||
},
|
||||
"Prefix": {
|
||||
"type": "string"
|
||||
},
|
||||
"Size": {
|
||||
"type": "integer"
|
||||
},
|
||||
"Tag": {
|
||||
"type": "structure",
|
||||
"required": [
|
||||
"Key",
|
||||
"Value"
|
||||
],
|
||||
"members": {
|
||||
"Key": {
|
||||
"shape": "ObjectKey",
|
||||
"documentation": "<p>Name of the object key.</p>"
|
||||
},
|
||||
"Value": {
|
||||
"shape": "Value",
|
||||
"documentation": "<p>Value of the tag.</p>"
|
||||
}
|
||||
},
|
||||
"documentation": "<p>A container of a key value name pair.</p>"
|
||||
},
|
||||
"TagSet": {
|
||||
"type": "list",
|
||||
"member": {
|
||||
"shape": "Tag",
|
||||
"locationName": "Tag"
|
||||
}
|
||||
},
|
||||
"Value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
const AWS = require('aws-sdk');
|
||||
|
||||
const { Service } = AWS;
|
||||
|
||||
// TODO: move lifecycle client.
|
||||
// CloudServer uses it for testing only. Backbeat uses it as a client.
|
||||
|
||||
AWS.apiLoader.services.lifecycle = {};
|
||||
const serviceIdentifier = 'lifecycle';
|
||||
const versions = ['2023-01-01'];
|
||||
const features = {
|
||||
validateService() {
|
||||
if (!this.config.region) {
|
||||
this.config.region = 'us-east-1';
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
const LifecycleClient = Service.defineService(
|
||||
serviceIdentifier,
|
||||
versions,
|
||||
features,
|
||||
);
|
||||
|
||||
Object.defineProperty(AWS.apiLoader.services.lifecycle, '2023-01-01', {
|
||||
get: function get() {
|
||||
const model = require('./lifecycle-2023-01-01.api.json'); // eslint-disable-line
|
||||
return model;
|
||||
},
|
||||
enumerable: true,
|
||||
configurable: true,
|
||||
});
|
||||
|
||||
module.exports = LifecycleClient;
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
const S3 = require('aws-sdk').S3;
|
||||
const LifecycleClient = require('./lifecycleClient');
|
||||
|
||||
const config = {
|
||||
sslEnabled: false,
|
||||
endpoint: 'http://127.0.0.1:8000',
|
||||
signatureCache: false,
|
||||
signatureVersion: 'v4',
|
||||
region: 'us-east-1',
|
||||
s3ForcePathStyle: true,
|
||||
accessKeyId: 'accessKey1',
|
||||
secretAccessKey: 'verySecretKey1',
|
||||
};
|
||||
|
||||
const lifecycleClient = new LifecycleClient(config);
|
||||
const s3Client = new S3(config);
|
||||
|
||||
module.exports = { lifecycleClient, s3Client } ;
|
||||
|
|
@ -165,6 +165,15 @@ class MongoTestClient {
|
|||
},
|
||||
}, cb);
|
||||
}
|
||||
|
||||
matchObjectsLastModified(bucketName, lastModified, cb) {
|
||||
const m = this.db.collection(bucketName);
|
||||
m.update(
|
||||
{},
|
||||
{ "$set" : { "value.last-modified" : lastModified } },
|
||||
{ multi: true },
|
||||
cb);
|
||||
}
|
||||
}
|
||||
|
||||
const mongoClient = new MongoTestClient({
|
||||
|
|
|
|||
Loading…
Reference in New Issue