ARSN-355 List lifecycle non-current versions supports V0

lib/algos/list/delimiterCurrent.ts

@@ -1,4 +1,5 @@
-const { Delimiter } = require('./delimiter');
+const { DelimiterMaster } = require('./delimiterMaster');
+const { FILTER_ACCEPT, FILTER_END } = require('./tools');
 
 type ResultObject = {
     Contents: {
@@ -9,11 +10,13 @@ type ResultObject = {
     NextMarker ?: string;
 };
 
+const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s
+
 /**
  * Handle object listing with parameters. This extends the base class Delimiter
  * to return the master/current versions.
  */
-class DelimiterCurrent extends Delimiter {
+class DelimiterCurrent extends DelimiterMaster {
     /**
      * Delimiter listing of current versions.
      * @param {Object}  parameters            - listing parameters
@@ -27,6 +30,9 @@ class DelimiterCurrent extends Delimiter {
 
         this.beforeDate = parameters.beforeDate;
         this.excludedDataStoreName = parameters.excludedDataStoreName;
+        // used for monitoring
+        this.start = null;
+        this.evaluatedKeys = 0;
     }
 
     genMDParamsV1() {
@@ -43,9 +49,55 @@ class DelimiterCurrent extends Delimiter {
                 ne: this.excludedDataStoreName,
             }
         }
+
+        this.start = Date.now();
+
         return params;
     }
 
+    _parse(s) {
+        let p;
+        try {
+            p = JSON.parse(s);
+        } catch (e: any) {
+            this.logger.warn(
+                'Could not parse Object Metadata while listing',
+                { err: e.toString() });
+        }
+        return p;
+    }
+
+    addContents(key, value) {
+        if (this.start && Date.now() - this.start > DELIMITER_TIMEOUT_MS) {
+            this.IsTruncated = true;
+            this.logger.info('listing stopped after expected internal timeout',
+                {
+                    timeoutMs: DELIMITER_TIMEOUT_MS,
+                    evaluatedKeys: this.evaluatedKeys,
+                });
+            return FILTER_END;
+        }
+        ++this.evaluatedKeys;
+        const parsedValue = this._parse(value);
+        // if parsing fails, skip the key.
+        if (parsedValue) {
+            const lastModified = parsedValue['last-modified'];
+            const dataStoreName = parsedValue.dataStoreName;
+            // We then check if the current version is older than the "beforeDate" and
+            // "excludedDataStoreName" is not specified or if specified and the data store name is different.
+            if ((!this.beforeDate || (lastModified && lastModified < this.beforeDate)) &&
+                (!this.excludedDataStoreName || dataStoreName !== this.excludedDataStoreName)) {
+                return super.addContents(key, value);
+            }
+            // In the event of a timeout occurring before any content is added,
+            // NextMarker is updated even if the object is not eligible.
+            // It minimizes the amount of data that the client needs to re-process if the request times out.
+            this.NextMarker = key;
+        }
+
+        return FILTER_ACCEPT;
+    }
+
     result(): ResultObject {
         const result: ResultObject = {
             Contents: this.Contents,

lib/algos/list/delimiterNonCurrent.js

@@ -1,10 +1,9 @@
-'use strict'; // eslint-disable-line strict
+const DelimiterVersions = require('./delimiterVersions').DelimiterVersions;
-const Delimiter = require('./delimiter').Delimiter;
+// const VSConst = require('../../versioning/constants').VersioningConstants;
-const VSConst = require('../../versioning/constants').VersioningConstants;
+const { FILTER_ACCEPT, FILTER_END, FILTER_SKIP } = require('./tools');
-const { inc, FILTER_ACCEPT, FILTER_END, SKIP_NONE } = require('./tools');
+// const VID_SEP = VSConst.VersionId.Separator;
-const VID_SEP = VSConst.VersionId.Separator;
+// const Version = require('../../versioning/Version').Version;
-const Version = require('../../versioning/Version').Version;
+// const { DbPrefixes } = VSConst;
-const { DbPrefixes } = VSConst;
 
 // TODO: find an acceptable timeout value.
 const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s
@@ -14,7 +13,7 @@ const TRIM_METADATA_MIN_BLOB_SIZE = 10000;
  * Handle object listing with parameters. This extends the base class Delimiter
  * to return the raw non-current versions objects.
  */
-class DelimiterNonCurrent extends Delimiter {
+class DelimiterNonCurrent extends DelimiterVersions {
     /**
      * Delimiter listing of non-current versions.
      * @param {Object}  parameters                  - listing parameters
@@ -39,67 +38,11 @@ class DelimiterNonCurrent extends Delimiter {
 
         // internal state
         this.staleDate = null;
-        this.masterKey = undefined;
-        this.masterVersionId = undefined;
 
         // used for monitoring
         this.evaluatedKeys = 0;
     }
 
-    skippingV1() {
-        return SKIP_NONE;
-    }
-
-    compareObjects(masterObj, versionObj) {
-        const masterKey = masterObj.key.slice(DbPrefixes.Master.length);
-        const versionKey = versionObj.key.slice(DbPrefixes.Version.length);
-        return masterKey < versionKey ? -1 : 1;
-    }
-
-    genMDParamsV1() {
-        const vParams = {
-            gte: DbPrefixes.Version,
-            lt: inc(DbPrefixes.Version),
-        };
-
-        const mParams = {
-            gte: DbPrefixes.Master,
-            lt: inc(DbPrefixes.Master),
-        };
-
-        if (this.prefix) {
-            const masterWithPrefix = `${DbPrefixes.Master}${this.prefix}`;
-            mParams.gte = masterWithPrefix;
-            mParams.lt = inc(masterWithPrefix);
-
-            const versionWithPrefix = `${DbPrefixes.Version}${this.prefix}`;
-            vParams.gte = versionWithPrefix;
-            vParams.lt = inc(versionWithPrefix);
-        }
-
-        if (this.keyMarker && `${DbPrefixes.Version}${this.keyMarker}` >= vParams.gte) {
-            if (this.versionIdMarker) {
-                const keyMarkerWithVersionId = `${this.keyMarker}${VID_SEP}${this.versionIdMarker}`;
-                // versionIdMarker should always come with keyMarker but may not be the other way around.
-                // NOTE: "gte" (instead of "gt") is used to include the last version of the "previous"
-                // truncated listing when a versionId marker is specified.
-                // This "previous"/"already evaluated" version will be used to retrieve the stale date and
-                // skipped to not evaluate the same key twice in the addContents() method.
-                vParams.gte = `${DbPrefixes.Version}${keyMarkerWithVersionId}`;
-                mParams.gte = `${DbPrefixes.Master}${keyMarkerWithVersionId}`;
-            } else {
-                delete vParams.gte;
-                delete mParams.gte;
-                vParams.gt = DbPrefixes.Version + inc(this.keyMarker + VID_SEP);
-                mParams.gt = DbPrefixes.Master + inc(this.keyMarker + VID_SEP);
-            }
-        }
-
-        this.start = Date.now();
-
-        return [mParams, vParams];
-    }
-
     getLastModified(value) {
         let lastModified;
         try {
@@ -115,31 +58,25 @@ class DelimiterNonCurrent extends Delimiter {
         return lastModified;
     }
 
-    parseKey(fullKey) {
+    keyHandler_SkippingVersions(key, value) {
-        const versionIdIndex = fullKey.indexOf(VID_SEP);
+        const { key: nonversionedKey, versionId } = this.parseKey(key);
-        if (versionIdIndex === -1) {
+        if (nonversionedKey === this.keyMarker) {
-            return { key: fullKey };
+            // since the nonversioned key equals the marker, there is
+            // necessarily a versionId in this key
+            const _versionId = versionId;
+            if (_versionId < this.versionIdMarker) {
+                // skip all versions until marker
+                return FILTER_SKIP;
+            }
+            // if (_versionId === this.versionIdMarker) {
+            //     // nothing left to skip, so return ACCEPT, but don't add this version
+            //     return FILTER_ACCEPT;
+            // }
         }
-        const nonversionedKey = fullKey.slice(0, versionIdIndex);
+        this.setState({
-        const versionId = fullKey.slice(versionIdIndex + 1);
+            id: 1 /* NotSkipping */,
-        return { key: nonversionedKey, versionId };
+        });
-    }
+        return this.handleKey(key, value);
-
-    /**
-     *  Filter to apply on each iteration
-     *  @param {Object} obj       - The key and value of the element
-     *  @param {String} obj.key   - The key of the element
-     *  @param {String} obj.value - The value of the element
-     *  @return {number}          - indicates if iteration should continue
-     */
-    filter(obj) {
-        const value = obj.value;
-        // NOTE: this check on PHD is only useful for Artesca, S3C
-        // does not use PHDs in V1 format
-        if (Version.isPHD(value)) {
-            return FILTER_ACCEPT;
-        }
-        return super.filter(obj);
     }
 
     /**
@@ -158,11 +95,12 @@ class DelimiterNonCurrent extends Delimiter {
      * - no more metadata key is left to be processed
      * - the listing reaches the maximum number of key to be returned
      * - the internal timeout is reached
-     *  @param {String} keyVersionSuffix   - The key to add
+     *  @param {String} key   - The key to add
+     *  @param {String} versionId - The version id
      *  @param {String} value - The value of the key
      *  @return {number}      - indicates if iteration should continue
      */
-    addContents(keyVersionSuffix, value) {
+    addContents(key, versionId, value) {
         if (this._reachedMaxKeys()) {
             return FILTER_END;
         }
@@ -178,28 +116,23 @@ class DelimiterNonCurrent extends Delimiter {
         }
         ++this.evaluatedKeys;
 
-        const { key, versionId } = this.parseKey(keyVersionSuffix);
-
         this.NextKeyMarker = key;
         this.NextVersionIdMarker = versionId;
 
         // The master key serves two purposes:
         // - It retrieves the expiration date for the previous version that is no longer current.
         // - It excludes the current version from the list.
-        const isMasterKey = versionId === undefined;
+        const isMasterKey = this.masterKey === key && this.masterVersionId === versionId;
         if (isMasterKey) {
-            this.masterKey = key;
-            this.masterVersionId = Version.from(value).getVersionId() || 'null';
-
             this.staleDate = this.getLastModified(value);
             return FILTER_ACCEPT;
         }
 
-        const isCurrentVersion = this.masterKey === key && this.masterVersionId === versionId;
+        // const isCurrentVersion = this.masterKey === key && this.masterVersionId === versionId;
-        if (isCurrentVersion) {
+        // if (isCurrentVersion) {
-            // filter out the master version
+        //     // filter out the master version
-            return FILTER_ACCEPT;
+        //     return FILTER_ACCEPT;
-        }
+        // }
 
         // The following version is pushed only:
         // - if the "stale date" (picked up from the previous version) is available (JSON.parse has not failed),

tests/unit/algos/list/delimiterCurrent.spec.js

@@ -13,6 +13,8 @@ const VSConst =
     require('../../../../lib/versioning/constants').VersioningConstants;
 const { DbPrefixes } = VSConst;
 
+const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s
+
 const VID_SEP = VSConst.VersionId.Separator;
 const EmptyResult = {
     Contents: [],
@@ -60,7 +62,9 @@ describe('DelimiterCurrent', () => {
         const delimiter = new DelimiterCurrent({ prefix: 'prefix' }, fakeLogger, 'v1');
 
         const listingKey = makeV1Key('noprefix');
-        assert.strictEqual(delimiter.filter({ key: listingKey, value: '' }), FILTER_SKIP);
+        const creationDate = '1970-01-01T00:00:00.001Z';
+        const value = `{"last-modified": "${creationDate}"}`;
+        assert.strictEqual(delimiter.filter({ key: listingKey, value }), FILTER_SKIP);
 
         assert.deepStrictEqual(delimiter.result(), EmptyResult);
     });
@@ -125,4 +129,213 @@ describe('DelimiterCurrent', () => {
 
         assert.deepStrictEqual(delimiter.result(), expectedResult);
     });
+
+    it('should return the object created before beforeDate', () => {
+        const beforeDate = '1970-01-01T00:00:00.003Z';
+        const delimiter = new DelimiterCurrent({ beforeDate }, fakeLogger, 'v1');
+
+        const masterKey1 = 'key1';
+        const date1 = '1970-01-01T00:00:00.004Z';
+        const value1 = `{"last-modified": "${date1}"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey1),
+            value: value1,
+        }), FILTER_ACCEPT);
+
+        const masterKey2 = 'key2';
+        const date2 = '1970-01-01T00:00:00.000Z';
+        const value2 = `{"last-modified": "${date2}"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey2),
+            value: value2,
+        }), FILTER_ACCEPT);
+
+        const expectedResult = {
+            Contents: [
+                {
+                    key: masterKey2,
+                    value: value2,
+                },
+            ],
+            IsTruncated: false,
+        };
+
+        assert.deepStrictEqual(delimiter.result(), expectedResult);
+    });
+
+    it('should return the object with dataStore name that does not match', () => {
+        const beforeDate = '1970-01-01T00:00:00.005Z';
+        const excludedDataStoreName = 'location-excluded';
+        const delimiter = new DelimiterCurrent({ beforeDate, excludedDataStoreName }, fakeLogger, 'v1');
+
+        const masterKey1 = 'key1';
+        const date1 = '1970-01-01T00:00:00.004Z';
+        const value1 = `{"last-modified": "${date1}", "dataStoreName": "valid"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey1),
+            value: value1,
+        }), FILTER_ACCEPT);
+
+        const masterKey2 = 'key2';
+        const date2 = '1970-01-01T00:00:00.000Z';
+        const value2 = `{"last-modified": "${date2}", "dataStoreName": "${excludedDataStoreName}"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey2),
+            value: value2,
+        }), FILTER_ACCEPT);
+
+        const expectedResult = {
+            Contents: [
+                {
+                    key: masterKey1,
+                    value: value1,
+                },
+            ],
+            IsTruncated: false,
+        };
+
+        assert.deepStrictEqual(delimiter.result(), expectedResult);
+    });
+
+    it('should return the object created before beforeDate and with dataStore name that does not match', () => {
+        const beforeDate = '1970-01-01T00:00:00.003Z';
+        const excludedDataStoreName = 'location-excluded';
+        const delimiter = new DelimiterCurrent({ beforeDate, excludedDataStoreName }, fakeLogger, 'v1');
+
+        const masterKey1 = 'key1';
+        const date1 = '1970-01-01T00:00:00.004Z';
+        const value1 = `{"last-modified": "${date1}", "dataStoreName": "valid"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey1),
+            value: value1,
+        }), FILTER_ACCEPT);
+
+        const masterKey2 = 'key2';
+        const date2 = '1970-01-01T00:00:00.001Z';
+        const value2 = `{"last-modified": "${date2}", "dataStoreName": "valid"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey2),
+            value: value2,
+        }), FILTER_ACCEPT);
+
+        const masterKey3 = 'key3';
+        const date3 = '1970-01-01T00:00:00.000Z';
+        const value3 = `{"last-modified": "${date3}", "dataStoreName": "${excludedDataStoreName}"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey3),
+            value: value3,
+        }), FILTER_ACCEPT);
+
+        const expectedResult = {
+            Contents: [
+                {
+                    key: masterKey2,
+                    value: value2,
+                },
+            ],
+            IsTruncated: false,
+        };
+
+        assert.deepStrictEqual(delimiter.result(), expectedResult);
+    });
+
+    it('should return the objects pushed before timeout', () => {
+        const beforeDate = '1970-01-01T00:00:00.003Z';
+        const delimiter = new DelimiterCurrent({ beforeDate }, fakeLogger, 'v1');
+
+        const masterKey1 = 'key1';
+        const date1 = '1970-01-01T00:00:00.000Z';
+        const value1 = `{"last-modified": "${date1}"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey1),
+            value: value1,
+        }), FILTER_ACCEPT);
+
+        const masterKey2 = 'key2';
+        const date2 = '1970-01-01T00:00:00.001Z';
+        const value2 = `{"last-modified": "${date2}"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey2),
+            value: value2,
+        }), FILTER_ACCEPT);
+
+        delimiter.start = Date.now() - (DELIMITER_TIMEOUT_MS + 1);
+
+        const masterKey3 = 'key3';
+        const date3 = '1970-01-01T00:00:00.002Z';
+        const value3 = `{"last-modified": "${date3}"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey3),
+            value: value3,
+        }), FILTER_END);
+
+        const expectedResult = {
+            Contents: [
+                {
+                    key: masterKey1,
+                    value: value1,
+                },
+                {
+                    key: masterKey2,
+                    value: value2,
+                },
+            ],
+            NextMarker: masterKey2,
+            IsTruncated: true,
+        };
+
+        assert.deepStrictEqual(delimiter.result(), expectedResult);
+    });
+
+    it('should return empty content after timeout', () => {
+        const beforeDate = '1970-01-01T00:00:00.003Z';
+        const delimiter = new DelimiterCurrent({ beforeDate }, fakeLogger, 'v1');
+
+        const masterKey1 = 'key1';
+        const date1 = '1970-01-01T00:00:00.004Z';
+        const value1 = `{"last-modified": "${date1}"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey1),
+            value: value1,
+        }), FILTER_ACCEPT);
+
+        const masterKey2 = 'key2';
+        const date2 = '1970-01-01T00:00:00.005Z';
+        const value2 = `{"last-modified": "${date2}"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey2),
+            value: value2,
+        }), FILTER_ACCEPT);
+
+        delimiter.start = Date.now() - (DELIMITER_TIMEOUT_MS + 1);
+
+        const masterKey3 = 'key3';
+        const date3 = '1970-01-01T00:00:00.006Z';
+        const value3 = `{"last-modified": "${date3}"}`;
+
+        assert.strictEqual(delimiter.filter({
+            key: makeV1Key(masterKey3),
+            value: value3,
+        }), FILTER_END);
+
+        const expectedResult = {
+            Contents: [],
+            NextMarker: masterKey2,
+            IsTruncated: true,
+        };
+
+        assert.deepStrictEqual(delimiter.result(), expectedResult);
+    });
 });