Compare commits

...

1 Commits

Author SHA1 Message Date
Nicolas Humbert 4d596995aa list lifecycle by date 2023-01-12 09:36:52 -05:00
7 changed files with 259 additions and 1 deletions

View File

@ -0,0 +1,147 @@
'use strict'; // eslint-disable-line strict
const Delimiter = require('./delimiter').Delimiter;
const Version = require('../../versioning/Version').Version;
const VSConst = require('../../versioning/constants').VersioningConstants;
const { BucketVersioningKeyFormat } = VSConst;
const { inc, FILTER_ACCEPT, FILTER_END, FILTER_SKIP, SKIP_NONE } = require('./tools');
const VID_SEP = VSConst.VersionId.Separator;
const { DbPrefixes } = VSConst;
/**
* Handle object listing with parameters. This extends the base class Delimiter
* to return the raw master objects.
*/
class DelimiterLifecycle extends Delimiter {
/**
* Delimiter listing of master versions.
* @param {Object} parameters - listing parameters
* @param {String} parameters.delimiter - delimiter per amazon format
* @param {String} parameters.prefix - prefix per amazon format
* @param {String} parameters.marker - marker per amazon format
* @param {Number} parameters.maxKeys - number of keys to list
* @param {Boolean} parameters.v2 - indicates whether v2 format
* @param {String} parameters.startAfter - marker per amazon v2 format
* @param {String} parameters.continuationToken - obfuscated amazon token
* @param {RequestLogger} logger - The logger of the request
* @param {String} [vFormat] - versioning key format
*/
constructor(parameters, logger, vFormat) {
super(parameters, logger, vFormat);
this.beforeDate = parameters.beforeDate;
this.dateMarker = parameters.dateMarker;
this.keyMarker = parameters.keyMarker;
this.NextDateMarker = null;
this.NextKeyMarker = null;
this.filter = this.filterV1;
this.skipping = this.skippingV1;
this.genMDParams = this.genMDParamsV1;
}
/**
* Filter to apply on each iteration
* @param {Object} obj - The key and value of the element
* @param {String} obj.key - The key of the element
* @param {String} obj.value - The value of the element
* @return {number} - indicates if iteration should continue
*/
filterV1(obj) {
const key = this.getObjectKey(obj);
const value = obj.value;
return this.addContents(key, value);
}
skippingV1() {
return SKIP_NONE;
}
genMDParamsV1() {
const params = {
sortByLastModified: true,
};
if (this.prefix) {
params.gte = `${DbPrefixes.Master}${this.prefix}`;
params.lt = inc(`${DbPrefixes.Master}${this.prefix}`);
}
if (this.beforeDate || this.dateMarker) {
params.lastModified = {}
if (this.beforeDate) {
params.lastModified.lt = this.beforeDate;
}
if (this.dateMarker) {
if (this.keyMarker) {
params.lastModified.gte = this.dateMarker;
} else {
params.lastModified.gt = this.dateMarker;
}
}
}
return params;
}
/**
* Add a (key, value) tuple to the listing
* Set the NextMarker to the current key
* Increment the keys counter
* @param {String} key - The key to add
* @param {String} value - The value of the key
* @return {number} - indicates if iteration should continue
*/
addContents(key, value) {
if (this._reachedMaxKeys()) {
// TODO: catch errors
const lastModified = JSON.parse(value)['last-modified'];
if (lastModified !== this.NextDateMarker) {
this.NextKeyMarker = null;
}
return FILTER_END;
}
if (this.keyMarker) {
if (key <= this.keyMarker){
return FILTER_ACCEPT;
} else {
this.keyMarker = null;
}
}
this.Contents.push({ key, value: this.trimMetadata(value) });
++this.keys;
if (this.keys === this.maxKeys) {
// TODO: catch errors
this.NextDateMarker = JSON.parse(value)['last-modified'];
this.NextKeyMarker = key;
}
return FILTER_ACCEPT;
}
_reachedMaxKeys() {
if (this.keys >= this.maxKeys) {
// In cases of maxKeys <= 0 -> IsTruncated = false
this.IsTruncated = this.maxKeys > 0;
return true;
}
return false;
}
result() {
const result = {
Contents: this.Contents,
IsTruncated: this.IsTruncated,
};
if (this.IsTruncated) {
result.NextDateMarker = this.NextDateMarker;
result.NextKeyMarker = this.NextKeyMarker;
}
return result;
}
}
module.exports = { DelimiterLifecycle };
//# sourceMappingURL=delimiterMaster.js.map

View File

@ -51,6 +51,26 @@ function _parseListEntries(entries) {
});
}
function _parseLifecycleListEntries(entries) {
return entries.map(entry => {
const tmp = JSON.parse(entry.value);
return {
key: entry.key,
value: {
Size: tmp['content-length'],
ETag: tmp['content-md5'],
LastModified: tmp['last-modified'],
Owner: {
DisplayName: tmp['owner-display-name'],
ID: tmp['owner-id'],
},
StorageClass: tmp['x-amz-storage-class'],
tags: tmp.tags,
},
};
});
}
/** parseListEntries - parse the values returned in a listing by metadata
* @param {object[]} entries - Version or Content entries in a metadata listing
* @param {string} entries[].key - metadata key
@ -279,6 +299,29 @@ class MetadataWrapper {
});
}
listLifecycleObject(bucketName, listingParams, log, cb) {
this.client.listLifecycleObject(bucketName, listingParams, log, (err, data) => {
log.debug('getting object listing for lifecycle from metadata');
if (err) {
log.error('error from metadata', { implName: this.implName,
err });
return cb(err);
}
log.debug('object listing retrieved from metadata');
// eslint-disable-next-line
data.Contents = parseListEntries(data.Contents, _parseLifecycleListEntries);
if (data.Contents instanceof Error) {
log.error('error parsing metadata listing', {
error: data.Contents,
listingType: listingParams.listingType,
method: 'listLifecycleObject',
});
return cb(errors.InternalError);
}
return cb(null, data);
});
}
listMultipartUploads(bucketName, listingParams, log, cb) {
this.client.listMultipartUploads(bucketName, listingParams, log,
(err, data) => {

View File

@ -1,5 +1,7 @@
const assert = require('assert');
const errors = require('../../../errors').default;
const BucketInfo = require('../../../models/BucketInfo').default;
class BucketClientInterface {
@ -110,6 +112,10 @@ class BucketClientInterface {
return null;
}
listLifecycleObject(bucketName, params, log, cb) {
return cb(errors.NotImplemented);
}
listMultipartUploads(bucketName, params, log, cb) {
this.client.listObject(bucketName, log.getSerializedUids(), params,
(err, data) => {
@ -120,7 +126,6 @@ class BucketClientInterface {
});
return null;
}
_analyzeHealthFailure(log, callback) {
let doFail = false;
const reason = {

View File

@ -325,6 +325,10 @@ class BucketFileInterface {
return this.internalListObject(bucketName, params, log, cb);
}
listLifecycleObject(bucketName, params, log, cb) {
return cb(errors.NotImplemented);
}
listMultipartUploads(bucketName, params, log, cb) {
return this.internalListObject(bucketName, params, log, cb);
}

View File

@ -333,6 +333,10 @@ const metastore = {
});
});
},
listLifecycleObject(bucketName, params, log, cb) {
return cb(errors.NotImplemented);
}
};
module.exports = metastore;

View File

@ -1675,6 +1675,31 @@ class MongoClientInterface {
BUCKET_VERSIONS.v0, log, cb);
}
listLifecycleObject(bucketName, params, log, cb) {
return this.getBucketVFormat(bucketName, log, (err, vFormat) => {
if (err) {
return cb(err);
}
if (vFormat !== BUCKET_VERSIONS.v1) {
log.error('not supported bucket format version',
{ method: 'listLifecycleObject', bucket: bucketName, vFormat: err.message });
return cb(errors.InternalError).customizeDescription('Not supported bucket format version');
}
const extName = params.listingType;
const extension = new listAlgos[extName](params, log, vFormat);
const mainStreamParams = extension.genMDParams();
const internalParams = {
mainStreamParams,
}
return this.internalListObject(bucketName, internalParams, extension, vFormat, log, cb);
});
}
checkHealth(implName, log, cb) {
const resp = {};
if (this.client && this.client.isConnected()) {

View File

@ -55,6 +55,23 @@ class MongoReadStream extends Readable {
}
}
if (options.lastModified) {
query['value.last-modified'] = {};
if (options.lastModified.lt) {
query['value.last-modified'].$lt = options.lastModified.lt;
}
if (options.lastModified.lte) {
query['value.last-modified'].$lte = options.lastModified.lte;
}
if (options.lastModified.gt) {
query['value.last-modified'].$gt = options.lastModified.gt;
}
if (options.lastModified.gte) {
query['value.last-modified'].$gte = options.lastModified.gte;
}
}
if (!Object.keys(query._id).length) {
delete query._id;
}
@ -69,6 +86,19 @@ class MongoReadStream extends Readable {
Object.assign(query, searchOptions);
}
let sort = {
_id: options.reverse ? -1 : 1,
};
if (options.sortByLastModified) {
sort = {
'value.last-modified': 1,
'_id': 1,
};
}
this._cursor = this._cursor.sort(sort);
this._cursor = c.find(query).sort({
_id: options.reverse ? -1 : 1,
});