Compare commits
1 Commits
developmen
...
list-lifec
Author | SHA1 | Date |
---|---|---|
Nicolas Humbert | 4d596995aa |
|
@ -0,0 +1,147 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
const Delimiter = require('./delimiter').Delimiter;
|
||||
const Version = require('../../versioning/Version').Version;
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { BucketVersioningKeyFormat } = VSConst;
|
||||
const { inc, FILTER_ACCEPT, FILTER_END, FILTER_SKIP, SKIP_NONE } = require('./tools');
|
||||
const VID_SEP = VSConst.VersionId.Separator;
|
||||
const { DbPrefixes } = VSConst;
|
||||
/**
|
||||
* Handle object listing with parameters. This extends the base class Delimiter
|
||||
* to return the raw master objects.
|
||||
*/
|
||||
class DelimiterLifecycle extends Delimiter {
|
||||
/**
|
||||
* Delimiter listing of master versions.
|
||||
* @param {Object} parameters - listing parameters
|
||||
* @param {String} parameters.delimiter - delimiter per amazon format
|
||||
* @param {String} parameters.prefix - prefix per amazon format
|
||||
* @param {String} parameters.marker - marker per amazon format
|
||||
* @param {Number} parameters.maxKeys - number of keys to list
|
||||
* @param {Boolean} parameters.v2 - indicates whether v2 format
|
||||
* @param {String} parameters.startAfter - marker per amazon v2 format
|
||||
* @param {String} parameters.continuationToken - obfuscated amazon token
|
||||
* @param {RequestLogger} logger - The logger of the request
|
||||
* @param {String} [vFormat] - versioning key format
|
||||
*/
|
||||
constructor(parameters, logger, vFormat) {
|
||||
super(parameters, logger, vFormat);
|
||||
|
||||
this.beforeDate = parameters.beforeDate;
|
||||
this.dateMarker = parameters.dateMarker;
|
||||
this.keyMarker = parameters.keyMarker;
|
||||
|
||||
this.NextDateMarker = null;
|
||||
this.NextKeyMarker = null;
|
||||
|
||||
this.filter = this.filterV1;
|
||||
this.skipping = this.skippingV1;
|
||||
this.genMDParams = this.genMDParamsV1;
|
||||
}
|
||||
/**
|
||||
* Filter to apply on each iteration
|
||||
* @param {Object} obj - The key and value of the element
|
||||
* @param {String} obj.key - The key of the element
|
||||
* @param {String} obj.value - The value of the element
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
filterV1(obj) {
|
||||
const key = this.getObjectKey(obj);
|
||||
const value = obj.value;
|
||||
return this.addContents(key, value);
|
||||
}
|
||||
|
||||
skippingV1() {
|
||||
return SKIP_NONE;
|
||||
}
|
||||
|
||||
genMDParamsV1() {
|
||||
const params = {
|
||||
sortByLastModified: true,
|
||||
};
|
||||
|
||||
if (this.prefix) {
|
||||
params.gte = `${DbPrefixes.Master}${this.prefix}`;
|
||||
params.lt = inc(`${DbPrefixes.Master}${this.prefix}`);
|
||||
}
|
||||
|
||||
if (this.beforeDate || this.dateMarker) {
|
||||
params.lastModified = {}
|
||||
if (this.beforeDate) {
|
||||
params.lastModified.lt = this.beforeDate;
|
||||
}
|
||||
if (this.dateMarker) {
|
||||
if (this.keyMarker) {
|
||||
params.lastModified.gte = this.dateMarker;
|
||||
} else {
|
||||
params.lastModified.gt = this.dateMarker;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a (key, value) tuple to the listing
|
||||
* Set the NextMarker to the current key
|
||||
* Increment the keys counter
|
||||
* @param {String} key - The key to add
|
||||
* @param {String} value - The value of the key
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
addContents(key, value) {
|
||||
if (this._reachedMaxKeys()) {
|
||||
// TODO: catch errors
|
||||
const lastModified = JSON.parse(value)['last-modified'];
|
||||
if (lastModified !== this.NextDateMarker) {
|
||||
this.NextKeyMarker = null;
|
||||
}
|
||||
return FILTER_END;
|
||||
}
|
||||
|
||||
if (this.keyMarker) {
|
||||
if (key <= this.keyMarker){
|
||||
return FILTER_ACCEPT;
|
||||
} else {
|
||||
this.keyMarker = null;
|
||||
}
|
||||
}
|
||||
|
||||
this.Contents.push({ key, value: this.trimMetadata(value) });
|
||||
++this.keys;
|
||||
|
||||
if (this.keys === this.maxKeys) {
|
||||
// TODO: catch errors
|
||||
this.NextDateMarker = JSON.parse(value)['last-modified'];
|
||||
this.NextKeyMarker = key;
|
||||
}
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
|
||||
_reachedMaxKeys() {
|
||||
if (this.keys >= this.maxKeys) {
|
||||
// In cases of maxKeys <= 0 -> IsTruncated = false
|
||||
this.IsTruncated = this.maxKeys > 0;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
result() {
|
||||
const result = {
|
||||
Contents: this.Contents,
|
||||
IsTruncated: this.IsTruncated,
|
||||
};
|
||||
|
||||
if (this.IsTruncated) {
|
||||
result.NextDateMarker = this.NextDateMarker;
|
||||
result.NextKeyMarker = this.NextKeyMarker;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
module.exports = { DelimiterLifecycle };
|
||||
//# sourceMappingURL=delimiterMaster.js.map
|
|
@ -51,6 +51,26 @@ function _parseListEntries(entries) {
|
|||
});
|
||||
}
|
||||
|
||||
function _parseLifecycleListEntries(entries) {
|
||||
return entries.map(entry => {
|
||||
const tmp = JSON.parse(entry.value);
|
||||
return {
|
||||
key: entry.key,
|
||||
value: {
|
||||
Size: tmp['content-length'],
|
||||
ETag: tmp['content-md5'],
|
||||
LastModified: tmp['last-modified'],
|
||||
Owner: {
|
||||
DisplayName: tmp['owner-display-name'],
|
||||
ID: tmp['owner-id'],
|
||||
},
|
||||
StorageClass: tmp['x-amz-storage-class'],
|
||||
tags: tmp.tags,
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** parseListEntries - parse the values returned in a listing by metadata
|
||||
* @param {object[]} entries - Version or Content entries in a metadata listing
|
||||
* @param {string} entries[].key - metadata key
|
||||
|
@ -279,6 +299,29 @@ class MetadataWrapper {
|
|||
});
|
||||
}
|
||||
|
||||
listLifecycleObject(bucketName, listingParams, log, cb) {
|
||||
this.client.listLifecycleObject(bucketName, listingParams, log, (err, data) => {
|
||||
log.debug('getting object listing for lifecycle from metadata');
|
||||
if (err) {
|
||||
log.error('error from metadata', { implName: this.implName,
|
||||
err });
|
||||
return cb(err);
|
||||
}
|
||||
log.debug('object listing retrieved from metadata');
|
||||
// eslint-disable-next-line
|
||||
data.Contents = parseListEntries(data.Contents, _parseLifecycleListEntries);
|
||||
if (data.Contents instanceof Error) {
|
||||
log.error('error parsing metadata listing', {
|
||||
error: data.Contents,
|
||||
listingType: listingParams.listingType,
|
||||
method: 'listLifecycleObject',
|
||||
});
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return cb(null, data);
|
||||
});
|
||||
}
|
||||
|
||||
listMultipartUploads(bucketName, listingParams, log, cb) {
|
||||
this.client.listMultipartUploads(bucketName, listingParams, log,
|
||||
(err, data) => {
|
||||
|
|
|
@ -1,5 +1,7 @@
|
|||
const assert = require('assert');
|
||||
|
||||
const errors = require('../../../errors').default;
|
||||
|
||||
const BucketInfo = require('../../../models/BucketInfo').default;
|
||||
|
||||
class BucketClientInterface {
|
||||
|
@ -110,6 +112,10 @@ class BucketClientInterface {
|
|||
return null;
|
||||
}
|
||||
|
||||
listLifecycleObject(bucketName, params, log, cb) {
|
||||
return cb(errors.NotImplemented);
|
||||
}
|
||||
|
||||
listMultipartUploads(bucketName, params, log, cb) {
|
||||
this.client.listObject(bucketName, log.getSerializedUids(), params,
|
||||
(err, data) => {
|
||||
|
@ -120,7 +126,6 @@ class BucketClientInterface {
|
|||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
_analyzeHealthFailure(log, callback) {
|
||||
let doFail = false;
|
||||
const reason = {
|
||||
|
|
|
@ -325,6 +325,10 @@ class BucketFileInterface {
|
|||
return this.internalListObject(bucketName, params, log, cb);
|
||||
}
|
||||
|
||||
listLifecycleObject(bucketName, params, log, cb) {
|
||||
return cb(errors.NotImplemented);
|
||||
}
|
||||
|
||||
listMultipartUploads(bucketName, params, log, cb) {
|
||||
return this.internalListObject(bucketName, params, log, cb);
|
||||
}
|
||||
|
|
|
@ -333,6 +333,10 @@ const metastore = {
|
|||
});
|
||||
});
|
||||
},
|
||||
|
||||
listLifecycleObject(bucketName, params, log, cb) {
|
||||
return cb(errors.NotImplemented);
|
||||
}
|
||||
};
|
||||
|
||||
module.exports = metastore;
|
||||
|
|
|
@ -1675,6 +1675,31 @@ class MongoClientInterface {
|
|||
BUCKET_VERSIONS.v0, log, cb);
|
||||
}
|
||||
|
||||
listLifecycleObject(bucketName, params, log, cb) {
|
||||
return this.getBucketVFormat(bucketName, log, (err, vFormat) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
|
||||
if (vFormat !== BUCKET_VERSIONS.v1) {
|
||||
log.error('not supported bucket format version',
|
||||
{ method: 'listLifecycleObject', bucket: bucketName, vFormat: err.message });
|
||||
return cb(errors.InternalError).customizeDescription('Not supported bucket format version');
|
||||
}
|
||||
|
||||
const extName = params.listingType;
|
||||
|
||||
const extension = new listAlgos[extName](params, log, vFormat);
|
||||
const mainStreamParams = extension.genMDParams();
|
||||
|
||||
const internalParams = {
|
||||
mainStreamParams,
|
||||
}
|
||||
|
||||
return this.internalListObject(bucketName, internalParams, extension, vFormat, log, cb);
|
||||
});
|
||||
}
|
||||
|
||||
checkHealth(implName, log, cb) {
|
||||
const resp = {};
|
||||
if (this.client && this.client.isConnected()) {
|
||||
|
|
|
@ -55,6 +55,23 @@ class MongoReadStream extends Readable {
|
|||
}
|
||||
}
|
||||
|
||||
if (options.lastModified) {
|
||||
query['value.last-modified'] = {};
|
||||
|
||||
if (options.lastModified.lt) {
|
||||
query['value.last-modified'].$lt = options.lastModified.lt;
|
||||
}
|
||||
if (options.lastModified.lte) {
|
||||
query['value.last-modified'].$lte = options.lastModified.lte;
|
||||
}
|
||||
if (options.lastModified.gt) {
|
||||
query['value.last-modified'].$gt = options.lastModified.gt;
|
||||
}
|
||||
if (options.lastModified.gte) {
|
||||
query['value.last-modified'].$gte = options.lastModified.gte;
|
||||
}
|
||||
}
|
||||
|
||||
if (!Object.keys(query._id).length) {
|
||||
delete query._id;
|
||||
}
|
||||
|
@ -69,6 +86,19 @@ class MongoReadStream extends Readable {
|
|||
Object.assign(query, searchOptions);
|
||||
}
|
||||
|
||||
let sort = {
|
||||
_id: options.reverse ? -1 : 1,
|
||||
};
|
||||
|
||||
if (options.sortByLastModified) {
|
||||
sort = {
|
||||
'value.last-modified': 1,
|
||||
'_id': 1,
|
||||
};
|
||||
}
|
||||
|
||||
this._cursor = this._cursor.sort(sort);
|
||||
|
||||
this._cursor = c.find(query).sort({
|
||||
_id: options.reverse ? -1 : 1,
|
||||
});
|
||||
|
|
Loading…
Reference in New Issue