lib/algos/list/delimiterCurrent.ts

@@ -1,5 +1,4 @@
-const { DelimiterMaster } = require('./delimiterMaster');
+const { Delimiter } = require('./delimiter');
-const { FILTER_ACCEPT, FILTER_END } = require('./tools');
 
 type ResultObject = {
     Contents: {
@@ -10,13 +9,11 @@ type ResultObject = {
     NextMarker ?: string;
 };
 
-const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s
-
 /**
  * Handle object listing with parameters. This extends the base class Delimiter
  * to return the master/current versions.
  */
-class DelimiterCurrent extends DelimiterMaster {
+class DelimiterCurrent extends Delimiter {
     /**
      * Delimiter listing of current versions.
      * @param {Object}  parameters            - listing parameters
@@ -30,9 +27,6 @@ class DelimiterCurrent extends DelimiterMaster {
 
         this.beforeDate = parameters.beforeDate;
         this.excludedDataStoreName = parameters.excludedDataStoreName;
-        // used for monitoring
-        this.start = null;
-        this.evaluatedKeys = 0;
     }
 
     genMDParamsV1() {
@@ -49,55 +43,9 @@ class DelimiterCurrent extends DelimiterMaster {
                 ne: this.excludedDataStoreName,
             }
         }
-
-        this.start = Date.now();
-
         return params;
     }
 
-    _parse(s) {
-        let p;
-        try {
-            p = JSON.parse(s);
-        } catch (e: any) {
-            this.logger.warn(
-                'Could not parse Object Metadata while listing',
-                { err: e.toString() });
-        }
-        return p;
-    }
-
-    addContents(key, value) {
-        if (this.start && Date.now() - this.start > DELIMITER_TIMEOUT_MS) {
-            this.IsTruncated = true;
-            this.logger.info('listing stopped after expected internal timeout',
-                {
-                    timeoutMs: DELIMITER_TIMEOUT_MS,
-                    evaluatedKeys: this.evaluatedKeys,
-                });
-            return FILTER_END;
-        }
-        ++this.evaluatedKeys;
-        const parsedValue = this._parse(value);
-        // if parsing fails, skip the key.
-        if (parsedValue) {
-            const lastModified = parsedValue['last-modified'];
-            const dataStoreName = parsedValue.dataStoreName;
-            // We then check if the current version is older than the "beforeDate" and
-            // "excludedDataStoreName" is not specified or if specified and the data store name is different.
-            if ((!this.beforeDate || (lastModified && lastModified < this.beforeDate)) &&
-                (!this.excludedDataStoreName || dataStoreName !== this.excludedDataStoreName)) {
-                return super.addContents(key, value);
-            }
-            // In the event of a timeout occurring before any content is added,
-            // NextMarker is updated even if the object is not eligible.
-            // It minimizes the amount of data that the client needs to re-process if the request times out.
-            this.NextMarker = key;
-        }
-
-        return FILTER_ACCEPT;
-    }
-
     result(): ResultObject {
         const result: ResultObject = {
             Contents: this.Contents,

lib/algos/list/delimiterNonCurrent.js

@@ -1,9 +1,10 @@
-const DelimiterVersions = require('./delimiterVersions').DelimiterVersions;
+'use strict'; // eslint-disable-line strict
-// const VSConst = require('../../versioning/constants').VersioningConstants;
+const Delimiter = require('./delimiter').Delimiter;
-const { FILTER_ACCEPT, FILTER_END, FILTER_SKIP } = require('./tools');
+const VSConst = require('../../versioning/constants').VersioningConstants;
-// const VID_SEP = VSConst.VersionId.Separator;
+const { inc, FILTER_ACCEPT, FILTER_END, SKIP_NONE } = require('./tools');
-// const Version = require('../../versioning/Version').Version;
+const VID_SEP = VSConst.VersionId.Separator;
-// const { DbPrefixes } = VSConst;
+const Version = require('../../versioning/Version').Version;
+const { DbPrefixes } = VSConst;
 
 // TODO: find an acceptable timeout value.
 const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s
@@ -13,7 +14,7 @@ const TRIM_METADATA_MIN_BLOB_SIZE = 10000;
  * Handle object listing with parameters. This extends the base class Delimiter
  * to return the raw non-current versions objects.
  */
-class DelimiterNonCurrent extends DelimiterVersions {
+class DelimiterNonCurrent extends Delimiter {
     /**
      * Delimiter listing of non-current versions.
      * @param {Object}  parameters                  - listing parameters
@@ -38,11 +39,67 @@ class DelimiterNonCurrent extends DelimiterVersions {
 
         // internal state
         this.staleDate = null;
+        this.masterKey = undefined;
+        this.masterVersionId = undefined;
 
         // used for monitoring
         this.evaluatedKeys = 0;
     }
 
+    skippingV1() {
+        return SKIP_NONE;
+    }
+
+    compareObjects(masterObj, versionObj) {
+        const masterKey = masterObj.key.slice(DbPrefixes.Master.length);
+        const versionKey = versionObj.key.slice(DbPrefixes.Version.length);
+        return masterKey < versionKey ? -1 : 1;
+    }
+
+    genMDParamsV1() {
+        const vParams = {
+            gte: DbPrefixes.Version,
+            lt: inc(DbPrefixes.Version),
+        };
+
+        const mParams = {
+            gte: DbPrefixes.Master,
+            lt: inc(DbPrefixes.Master),
+        };
+
+        if (this.prefix) {
+            const masterWithPrefix = `${DbPrefixes.Master}${this.prefix}`;
+            mParams.gte = masterWithPrefix;
+            mParams.lt = inc(masterWithPrefix);
+
+            const versionWithPrefix = `${DbPrefixes.Version}${this.prefix}`;
+            vParams.gte = versionWithPrefix;
+            vParams.lt = inc(versionWithPrefix);
+        }
+
+        if (this.keyMarker && `${DbPrefixes.Version}${this.keyMarker}` >= vParams.gte) {
+            if (this.versionIdMarker) {
+                const keyMarkerWithVersionId = `${this.keyMarker}${VID_SEP}${this.versionIdMarker}`;
+                // versionIdMarker should always come with keyMarker but may not be the other way around.
+                // NOTE: "gte" (instead of "gt") is used to include the last version of the "previous"
+                // truncated listing when a versionId marker is specified.
+                // This "previous"/"already evaluated" version will be used to retrieve the stale date and
+                // skipped to not evaluate the same key twice in the addContents() method.
+                vParams.gte = `${DbPrefixes.Version}${keyMarkerWithVersionId}`;
+                mParams.gte = `${DbPrefixes.Master}${keyMarkerWithVersionId}`;
+            } else {
+                delete vParams.gte;
+                delete mParams.gte;
+                vParams.gt = DbPrefixes.Version + inc(this.keyMarker + VID_SEP);
+                mParams.gt = DbPrefixes.Master + inc(this.keyMarker + VID_SEP);
+            }
+        }
+
+        this.start = Date.now();
+
+        return [mParams, vParams];
+    }
+
     getLastModified(value) {
         let lastModified;
         try {
@@ -58,25 +115,31 @@ class DelimiterNonCurrent extends DelimiterVersions {
         return lastModified;
     }
 
-    keyHandler_SkippingVersions(key, value) {
+    parseKey(fullKey) {
-        const { key: nonversionedKey, versionId } = this.parseKey(key);
+        const versionIdIndex = fullKey.indexOf(VID_SEP);
-        if (nonversionedKey === this.keyMarker) {
+        if (versionIdIndex === -1) {
-            // since the nonversioned key equals the marker, there is
+            return { key: fullKey };
-            // necessarily a versionId in this key
-            const _versionId = versionId;
-            if (_versionId < this.versionIdMarker) {
-                // skip all versions until marker
-                return FILTER_SKIP;
         }
-            // if (_versionId === this.versionIdMarker) {
+        const nonversionedKey = fullKey.slice(0, versionIdIndex);
-            //     // nothing left to skip, so return ACCEPT, but don't add this version
+        const versionId = fullKey.slice(versionIdIndex + 1);
-            //     return FILTER_ACCEPT;
+        return { key: nonversionedKey, versionId };
-            // }
     }
-        this.setState({
+
-            id: 1 /* NotSkipping */,
+    /**
-        });
+     *  Filter to apply on each iteration
-        return this.handleKey(key, value);
+     *  @param {Object} obj       - The key and value of the element
+     *  @param {String} obj.key   - The key of the element
+     *  @param {String} obj.value - The value of the element
+     *  @return {number}          - indicates if iteration should continue
+     */
+    filter(obj) {
+        const value = obj.value;
+        // NOTE: this check on PHD is only useful for Artesca, S3C
+        // does not use PHDs in V1 format
+        if (Version.isPHD(value)) {
+            return FILTER_ACCEPT;
+        }
+        return super.filter(obj);
     }
 
     /**
@@ -95,12 +158,11 @@ class DelimiterNonCurrent extends DelimiterVersions {
      * - no more metadata key is left to be processed
      * - the listing reaches the maximum number of key to be returned
      * - the internal timeout is reached
-     *  @param {String} key   - The key to add
+     *  @param {String} keyVersionSuffix   - The key to add
-     *  @param {String} versionId - The version id
      *  @param {String} value - The value of the key
      *  @return {number}      - indicates if iteration should continue
      */
-    addContents(key, versionId, value) {
+    addContents(keyVersionSuffix, value) {
         if (this._reachedMaxKeys()) {
             return FILTER_END;
         }
@@ -116,23 +178,28 @@ class DelimiterNonCurrent extends DelimiterVersions {
         }
         ++this.evaluatedKeys;
 
+        const { key, versionId } = this.parseKey(keyVersionSuffix);
+
         this.NextKeyMarker = key;
         this.NextVersionIdMarker = versionId;
 
         // The master key serves two purposes:
         // - It retrieves the expiration date for the previous version that is no longer current.
         // - It excludes the current version from the list.
-        const isMasterKey = this.masterKey === key && this.masterVersionId === versionId;
+        const isMasterKey = versionId === undefined;
         if (isMasterKey) {
+            this.masterKey = key;
+            this.masterVersionId = Version.from(value).getVersionId() || 'null';
+
             this.staleDate = this.getLastModified(value);
             return FILTER_ACCEPT;
         }
 
-        // const isCurrentVersion = this.masterKey === key && this.masterVersionId === versionId;
+        const isCurrentVersion = this.masterKey === key && this.masterVersionId === versionId;
-        // if (isCurrentVersion) {
+        if (isCurrentVersion) {
-        //     // filter out the master version
+            // filter out the master version
-        //     return FILTER_ACCEPT;
+            return FILTER_ACCEPT;
-        // }
+        }
 
         // The following version is pushed only:
         // - if the "stale date" (picked up from the previous version) is available (JSON.parse has not failed),

tests/unit/algos/list/delimiterCurrent.spec.js

@@ -13,8 +13,6 @@ const VSConst =
     require('../../../../lib/versioning/constants').VersioningConstants;
 const { DbPrefixes } = VSConst;
 
-const DELIMITER_TIMEOUT_MS = 10 * 1000; // 10s
-
 const VID_SEP = VSConst.VersionId.Separator;
 const EmptyResult = {
     Contents: [],
@@ -62,9 +60,7 @@ describe('DelimiterCurrent', () => {
         const delimiter = new DelimiterCurrent({ prefix: 'prefix' }, fakeLogger, 'v1');
 
         const listingKey = makeV1Key('noprefix');
-        const creationDate = '1970-01-01T00:00:00.001Z';
+        assert.strictEqual(delimiter.filter({ key: listingKey, value: '' }), FILTER_SKIP);
-        const value = `{"last-modified": "${creationDate}"}`;
-        assert.strictEqual(delimiter.filter({ key: listingKey, value }), FILTER_SKIP);
 
         assert.deepStrictEqual(delimiter.result(), EmptyResult);
     });
@@ -129,213 +125,4 @@ describe('DelimiterCurrent', () => {
 
         assert.deepStrictEqual(delimiter.result(), expectedResult);
     });
-
-    it('should return the object created before beforeDate', () => {
-        const beforeDate = '1970-01-01T00:00:00.003Z';
-        const delimiter = new DelimiterCurrent({ beforeDate }, fakeLogger, 'v1');
-
-        const masterKey1 = 'key1';
-        const date1 = '1970-01-01T00:00:00.004Z';
-        const value1 = `{"last-modified": "${date1}"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey1),
-            value: value1,
-        }), FILTER_ACCEPT);
-
-        const masterKey2 = 'key2';
-        const date2 = '1970-01-01T00:00:00.000Z';
-        const value2 = `{"last-modified": "${date2}"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey2),
-            value: value2,
-        }), FILTER_ACCEPT);
-
-        const expectedResult = {
-            Contents: [
-                {
-                    key: masterKey2,
-                    value: value2,
-                },
-            ],
-            IsTruncated: false,
-        };
-
-        assert.deepStrictEqual(delimiter.result(), expectedResult);
-    });
-
-    it('should return the object with dataStore name that does not match', () => {
-        const beforeDate = '1970-01-01T00:00:00.005Z';
-        const excludedDataStoreName = 'location-excluded';
-        const delimiter = new DelimiterCurrent({ beforeDate, excludedDataStoreName }, fakeLogger, 'v1');
-
-        const masterKey1 = 'key1';
-        const date1 = '1970-01-01T00:00:00.004Z';
-        const value1 = `{"last-modified": "${date1}", "dataStoreName": "valid"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey1),
-            value: value1,
-        }), FILTER_ACCEPT);
-
-        const masterKey2 = 'key2';
-        const date2 = '1970-01-01T00:00:00.000Z';
-        const value2 = `{"last-modified": "${date2}", "dataStoreName": "${excludedDataStoreName}"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey2),
-            value: value2,
-        }), FILTER_ACCEPT);
-
-        const expectedResult = {
-            Contents: [
-                {
-                    key: masterKey1,
-                    value: value1,
-                },
-            ],
-            IsTruncated: false,
-        };
-
-        assert.deepStrictEqual(delimiter.result(), expectedResult);
-    });
-
-    it('should return the object created before beforeDate and with dataStore name that does not match', () => {
-        const beforeDate = '1970-01-01T00:00:00.003Z';
-        const excludedDataStoreName = 'location-excluded';
-        const delimiter = new DelimiterCurrent({ beforeDate, excludedDataStoreName }, fakeLogger, 'v1');
-
-        const masterKey1 = 'key1';
-        const date1 = '1970-01-01T00:00:00.004Z';
-        const value1 = `{"last-modified": "${date1}", "dataStoreName": "valid"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey1),
-            value: value1,
-        }), FILTER_ACCEPT);
-
-        const masterKey2 = 'key2';
-        const date2 = '1970-01-01T00:00:00.001Z';
-        const value2 = `{"last-modified": "${date2}", "dataStoreName": "valid"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey2),
-            value: value2,
-        }), FILTER_ACCEPT);
-
-        const masterKey3 = 'key3';
-        const date3 = '1970-01-01T00:00:00.000Z';
-        const value3 = `{"last-modified": "${date3}", "dataStoreName": "${excludedDataStoreName}"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey3),
-            value: value3,
-        }), FILTER_ACCEPT);
-
-        const expectedResult = {
-            Contents: [
-                {
-                    key: masterKey2,
-                    value: value2,
-                },
-            ],
-            IsTruncated: false,
-        };
-
-        assert.deepStrictEqual(delimiter.result(), expectedResult);
-    });
-
-    it('should return the objects pushed before timeout', () => {
-        const beforeDate = '1970-01-01T00:00:00.003Z';
-        const delimiter = new DelimiterCurrent({ beforeDate }, fakeLogger, 'v1');
-
-        const masterKey1 = 'key1';
-        const date1 = '1970-01-01T00:00:00.000Z';
-        const value1 = `{"last-modified": "${date1}"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey1),
-            value: value1,
-        }), FILTER_ACCEPT);
-
-        const masterKey2 = 'key2';
-        const date2 = '1970-01-01T00:00:00.001Z';
-        const value2 = `{"last-modified": "${date2}"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey2),
-            value: value2,
-        }), FILTER_ACCEPT);
-
-        delimiter.start = Date.now() - (DELIMITER_TIMEOUT_MS + 1);
-
-        const masterKey3 = 'key3';
-        const date3 = '1970-01-01T00:00:00.002Z';
-        const value3 = `{"last-modified": "${date3}"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey3),
-            value: value3,
-        }), FILTER_END);
-
-        const expectedResult = {
-            Contents: [
-                {
-                    key: masterKey1,
-                    value: value1,
-                },
-                {
-                    key: masterKey2,
-                    value: value2,
-                },
-            ],
-            NextMarker: masterKey2,
-            IsTruncated: true,
-        };
-
-        assert.deepStrictEqual(delimiter.result(), expectedResult);
-    });
-
-    it('should return empty content after timeout', () => {
-        const beforeDate = '1970-01-01T00:00:00.003Z';
-        const delimiter = new DelimiterCurrent({ beforeDate }, fakeLogger, 'v1');
-
-        const masterKey1 = 'key1';
-        const date1 = '1970-01-01T00:00:00.004Z';
-        const value1 = `{"last-modified": "${date1}"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey1),
-            value: value1,
-        }), FILTER_ACCEPT);
-
-        const masterKey2 = 'key2';
-        const date2 = '1970-01-01T00:00:00.005Z';
-        const value2 = `{"last-modified": "${date2}"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey2),
-            value: value2,
-        }), FILTER_ACCEPT);
-
-        delimiter.start = Date.now() - (DELIMITER_TIMEOUT_MS + 1);
-
-        const masterKey3 = 'key3';
-        const date3 = '1970-01-01T00:00:00.006Z';
-        const value3 = `{"last-modified": "${date3}"}`;
-
-        assert.strictEqual(delimiter.filter({
-            key: makeV1Key(masterKey3),
-            value: value3,
-        }), FILTER_END);
-
-        const expectedResult = {
-            Contents: [],
-            NextMarker: masterKey2,
-            IsTruncated: true,
-        };
-
-        assert.deepStrictEqual(delimiter.result(), expectedResult);
-    });
 });