list lifecycle by date

lib/algos/list/delimiterLifecycle.js

@@ -0,0 +1,147 @@
+'use strict'; // eslint-disable-line strict
+const Delimiter = require('./delimiter').Delimiter;
+const Version = require('../../versioning/Version').Version;
+const VSConst = require('../../versioning/constants').VersioningConstants;
+const { BucketVersioningKeyFormat } = VSConst;
+const { inc, FILTER_ACCEPT, FILTER_END, FILTER_SKIP, SKIP_NONE } = require('./tools');
+const VID_SEP = VSConst.VersionId.Separator;
+const { DbPrefixes } = VSConst;
+/**
+ * Handle object listing with parameters. This extends the base class Delimiter
+ * to return the raw master objects.
+ */
+class DelimiterLifecycle extends Delimiter {
+    /**
+     * Delimiter listing of master versions.
+     * @param {Object}  parameters            - listing parameters
+     * @param {String}  parameters.delimiter  - delimiter per amazon format
+     * @param {String}  parameters.prefix     - prefix per amazon format
+     * @param {String}  parameters.marker     - marker per amazon format
+     * @param {Number}  parameters.maxKeys    - number of keys to list
+     * @param {Boolean} parameters.v2         - indicates whether v2 format
+     * @param {String}  parameters.startAfter - marker per amazon v2 format
+     * @param {String}  parameters.continuationToken - obfuscated amazon token
+     * @param {RequestLogger} logger          - The logger of the request
+     * @param {String} [vFormat]              - versioning key format
+     */
+    constructor(parameters, logger, vFormat) {
+        super(parameters, logger, vFormat);
+
+        this.beforeDate = parameters.beforeDate;
+        this.dateMarker = parameters.dateMarker;
+        this.keyMarker = parameters.keyMarker;
+
+        this.NextDateMarker = null;
+        this.NextKeyMarker = null;
+
+        this.filter = this.filterV1;
+        this.skipping = this.skippingV1;
+        this.genMDParams = this.genMDParamsV1;
+    }
+    /**
+     *  Filter to apply on each iteration
+     *  @param {Object} obj       - The key and value of the element
+     *  @param {String} obj.key   - The key of the element
+     *  @param {String} obj.value - The value of the element
+     *  @return {number}          - indicates if iteration should continue
+     */
+    filterV1(obj) {
+        const key = this.getObjectKey(obj);
+        const value = obj.value;
+        return this.addContents(key, value);
+    }
+
+    skippingV1() {
+        return SKIP_NONE;
+    }
+
+    genMDParamsV1() {
+        const params = {
+            sortByLastModified: true,
+        };
+
+        if (this.prefix) {
+            params.gte = `${DbPrefixes.Master}${this.prefix}`;
+            params.lt = inc(`${DbPrefixes.Master}${this.prefix}`);
+        }
+
+        if (this.beforeDate || this.dateMarker) {
+            params.lastModified = {}
+            if (this.beforeDate) {
+                params.lastModified.lt = this.beforeDate;
+            }
+            if (this.dateMarker) {
+                if (this.keyMarker) {
+                    params.lastModified.gte = this.dateMarker;
+                } else {
+                    params.lastModified.gt = this.dateMarker;
+                }
+            }
+        }
+
+        return params;
+    }
+
+    /**
+     *  Add a (key, value) tuple to the listing
+     *  Set the NextMarker to the current key
+     *  Increment the keys counter
+     *  @param {String} key   - The key to add
+     *  @param {String} value - The value of the key
+     *  @return {number}      - indicates if iteration should continue
+     */
+    addContents(key, value) {
+        if (this._reachedMaxKeys()) {
+            // TODO: catch errors
+            const lastModified = JSON.parse(value)['last-modified'];
+            if (lastModified !== this.NextDateMarker) {
+                this.NextKeyMarker = null;
+            }
+            return FILTER_END;
+        }
+
+        if (this.keyMarker) {
+            if (key <= this.keyMarker){
+                return FILTER_ACCEPT;
+            } else {
+                this.keyMarker = null;
+            }
+        } 
+
+        this.Contents.push({ key, value: this.trimMetadata(value) });
+        ++this.keys;
+
+        if (this.keys === this.maxKeys) {
+            // TODO: catch errors
+            this.NextDateMarker = JSON.parse(value)['last-modified'];
+            this.NextKeyMarker = key;
+        }
+        return FILTER_ACCEPT;
+    }
+
+    _reachedMaxKeys() {
+        if (this.keys >= this.maxKeys) {
+            // In cases of maxKeys <= 0 -> IsTruncated = false
+            this.IsTruncated = this.maxKeys > 0;
+            return true;
+        }
+
+        return false;
+    }
+
+    result() {
+        const result = {
+            Contents: this.Contents,
+            IsTruncated: this.IsTruncated,
+        };
+
+        if (this.IsTruncated) {
+            result.NextDateMarker = this.NextDateMarker;
+            result.NextKeyMarker = this.NextKeyMarker;
+        }
+
+        return result;
+    }
+}
+module.exports = { DelimiterLifecycle };
+//# sourceMappingURL=delimiterMaster.js.map

lib/storage/metadata/MetadataWrapper.js

@@ -51,6 +51,26 @@ function _parseListEntries(entries) {
     });
 }
 
+function _parseLifecycleListEntries(entries) {
+    return entries.map(entry => {
+        const tmp = JSON.parse(entry.value);
+        return {
+            key: entry.key,
+            value: {
+                Size: tmp['content-length'],
+                ETag: tmp['content-md5'],
+                LastModified: tmp['last-modified'],
+                Owner: {
+                    DisplayName: tmp['owner-display-name'],
+                    ID: tmp['owner-id'],
+                },
+                StorageClass: tmp['x-amz-storage-class'],
+                tags: tmp.tags,
+            },
+        };
+    });
+}
+
 /** parseListEntries - parse the values returned in a listing by metadata
  * @param {object[]} entries - Version or Content entries in a metadata listing
  * @param {string} entries[].key - metadata key
@@ -279,6 +299,29 @@ class MetadataWrapper {
         });
     }
 
+    listLifecycleObject(bucketName, listingParams, log, cb) {
+        this.client.listLifecycleObject(bucketName, listingParams, log, (err, data) => {
+            log.debug('getting object listing for lifecycle from metadata');
+            if (err) {
+                log.error('error from metadata', { implName: this.implName,
+                    err });
+                return cb(err);
+            }
+            log.debug('object listing retrieved from metadata');
+            // eslint-disable-next-line
+            data.Contents = parseListEntries(data.Contents, _parseLifecycleListEntries);
+            if (data.Contents instanceof Error) {
+                log.error('error parsing metadata listing', {
+                    error: data.Contents,
+                    listingType: listingParams.listingType,
+                    method: 'listLifecycleObject',
+                });
+                return cb(errors.InternalError);
+            }
+            return cb(null, data);
+        });
+    }
+
     listMultipartUploads(bucketName, listingParams, log, cb) {
         this.client.listMultipartUploads(bucketName, listingParams, log,
             (err, data) => {

lib/storage/metadata/bucketclient/BucketClientInterface.js

@@ -1,5 +1,7 @@
 const assert = require('assert');
 
+const errors = require('../../../errors').default;
+
 const BucketInfo = require('../../../models/BucketInfo').default;
 
 class BucketClientInterface {
@@ -110,6 +112,10 @@ class BucketClientInterface {
         return null;
     }
 
+    listLifecycleObject(bucketName, params, log, cb) {
+        return cb(errors.NotImplemented);
+    }
+
     listMultipartUploads(bucketName, params, log, cb) {
         this.client.listObject(bucketName, log.getSerializedUids(), params,
             (err, data) => {
@@ -120,7 +126,6 @@ class BucketClientInterface {
             });
         return null;
     }
-
     _analyzeHealthFailure(log, callback) {
         let doFail = false;
         const reason = {

lib/storage/metadata/file/BucketFileInterface.js

@@ -325,6 +325,10 @@ class BucketFileInterface {
         return this.internalListObject(bucketName, params, log, cb);
     }
 
+    listLifecycleObject(bucketName, params, log, cb) {
+        return cb(errors.NotImplemented);
+    }
+
     listMultipartUploads(bucketName, params, log, cb) {
         return this.internalListObject(bucketName, params, log, cb);
     }

lib/storage/metadata/in_memory/metastore.js

@@ -333,6 +333,10 @@ const metastore = {
             });
         });
     },
+
+    listLifecycleObject(bucketName, params, log, cb) {
+        return cb(errors.NotImplemented);
+    }
 };
 
 module.exports = metastore;

lib/storage/metadata/mongoclient/MongoClientInterface.js

@@ -1675,6 +1675,31 @@ class MongoClientInterface {
             BUCKET_VERSIONS.v0, log, cb);
     }
 
+    listLifecycleObject(bucketName, params, log, cb) {
+        return this.getBucketVFormat(bucketName, log, (err, vFormat) => {
+            if (err) {
+                return cb(err);
+            }
+
+            if (vFormat !== BUCKET_VERSIONS.v1) {
+                log.error('not supported bucket format version', 
+                    { method: 'listLifecycleObject', bucket: bucketName, vFormat: err.message });
+                return cb(errors.InternalError).customizeDescription('Not supported bucket format version');
+            }
+
+            const extName = params.listingType;
+
+            const extension = new listAlgos[extName](params, log, vFormat);
+            const mainStreamParams = extension.genMDParams();
+
+            const internalParams = {
+                mainStreamParams,
+            }
+
+            return this.internalListObject(bucketName, internalParams, extension, vFormat, log, cb);
+        });
+    }
+
     checkHealth(implName, log, cb) {
         const resp = {};
         if (this.client && this.client.isConnected()) {

lib/storage/metadata/mongoclient/readStream.js

@@ -55,6 +55,23 @@ class MongoReadStream extends Readable {
             }
         }
 
+        if (options.lastModified) {
+            query['value.last-modified'] = {};
+
+            if (options.lastModified.lt) {
+                query['value.last-modified'].$lt = options.lastModified.lt;
+            }
+            if (options.lastModified.lte) {
+                query['value.last-modified'].$lte = options.lastModified.lte;
+            }
+            if (options.lastModified.gt) {
+                query['value.last-modified'].$gt = options.lastModified.gt;
+            }
+            if (options.lastModified.gte) {
+                query['value.last-modified'].$gte = options.lastModified.gte;
+            }
+        }
+
         if (!Object.keys(query._id).length) {
             delete query._id;
         }
@@ -69,6 +86,19 @@ class MongoReadStream extends Readable {
             Object.assign(query, searchOptions);
         }
 
+        let sort = {
+            _id: options.reverse ? -1 : 1,
+        };
+
+        if (options.sortByLastModified) {
+            sort = {
+                'value.last-modified': 1,
+                '_id': 1,
+            };
+        }
+
+        this._cursor = this._cursor.sort(sort);
+
         this._cursor = c.find(query).sort({
             _id: options.reverse ? -1 : 1,
         });