test

lib/algos/list/delimiterMaster.js

@@ -46,6 +46,13 @@ class DelimiterMaster extends Delimiter {
         let key = obj.key;
         const value = obj.value;
 
+        // if (typeof this[this.nextContinueMarker] === 'string') {
+        //     console.log('___ START ____')
+        //     console.log(key, this[this.nextContinueMarker])
+        //     console.log(key <= this[this.nextContinueMarker])
+        //     console.log('____ END ____')
+        // }
+
         /* Skip keys not starting with the prefix or not alphabetically
          * ordered. */
         if ((this.prefix && !key.startsWith(this.prefix))
@@ -76,11 +83,15 @@ class DelimiterMaster extends Delimiter {
             if (key === this.prvKey || key === this[this.nextContinueMarker] ||
                 (this.delimiter &&
                 key.startsWith(this[this.nextContinueMarker]))) {
+                    // console.log(`----> ${key}`)
+                    // console.log(`--> ${this[this.nextContinueMarker]}`)
                 /* master version already filtered */
                 return FILTER_SKIP;
             }
         }
+        // console.log(Version.isPHD(value))
         if (Version.isPHD(value)) {
+            // console.log('IS PHD')
             /* master version is a PHD version, we want to wait for the next
              * one:
              * - Set the prvKey to undefined to not skip the next version,
@@ -101,6 +112,7 @@ class DelimiterMaster extends Delimiter {
              *   next entry). In that case we expect the master version to
              *   follow. */
             if (key === this.prvPHDKey) {
+                // TODO: resetting prvKey causing listing issue?
                 this.prvKey = undefined;
                 return FILTER_ACCEPT;
             }
@@ -123,14 +135,18 @@ class DelimiterMaster extends Delimiter {
 
     skipping() {
         if (this[this.nextContinueMarker]) {
+            // console.log(`-- skipping-nextContinueMarker: ${this[this.nextContinueMarker]}`)
             // next marker or next continuation token:
             // - foo/ : skipping foo/
             // - foo  : skipping foo.
             const index = this[this.nextContinueMarker].
                 lastIndexOf(this.delimiter);
+            // console.log(`index: ${index}`)
             if (index === this[this.nextContinueMarker].length - 1) {
+                // console.log(`returning nextContinueMarker`)
                 return this[this.nextContinueMarker];
             }
+            // console.log(`returning + VID_SEP: ${this[this.nextContinueMarker] + VID_SEP}`)
             return this[this.nextContinueMarker] + VID_SEP;
         }
         return SKIP_NONE;

lib/algos/list/skip.js

@@ -54,11 +54,18 @@ class Skip {
         const filteringResult = this.extension.filter(entry);
         const skippingRange = this.extension.skipping();
 
+        // console.log('--------')
+        // console.log(this.extension.constructor.name)
+        // console.log(filteringResult)
+        // console.log(skippingRange)
+
         if (filteringResult === FILTER_END) {
             this.listingEndCb();
         } else if (filteringResult === FILTER_SKIP
                    && skippingRange !== SKIP_NONE) {
+            // console.log('=> IN filter_skip && skippingRange != skip_none')
             if (++this.streakLength >= MAX_STREAK_LENGTH) {
+                // console.log('MAX STREAK LENGTH MET?')
                 const newRange = this._inc(skippingRange);
 
                 /* Avoid to loop on the same range again and again. */

lib/storage/metadata/mongoclient/MongoClientInterface.js

@@ -896,6 +896,7 @@ class MongoClientInterface {
 
     listObject(bucketName, params, log, cb) {
         const extName = params.listingType;
+        console.log(`===> listObject type: ${extName}`)
         const extension = new listAlgos[extName](params, log);
         const internalParams = extension.genMDParams();
         internalParams.mongifiedSearch = params.mongifiedSearch;

lib/storage/metadata/mongoclient/readStream.js

@@ -97,6 +97,10 @@ class MongoReadStream extends Readable {
                 value = JSON.stringify(doc.value);
             }
 
+            if (key !== undefined) {
+                console.log(key)
+            }
+
             if (key === undefined && value === undefined) {
                 this.push(null);
             } else if (this._options.keys !== false &&

tests/unit/algos/list/delimiter.js

@@ -77,6 +77,14 @@ const nonAlphabeticalData = [
     { key: 'zzz', value },
     { key: 'aaa', value },
 ];
+const data2 = [
+    { key: 'test-obj1', value: valuePHD },
+    { key: 'test-obj10', value: valuePHD },
+    { key: 'test-obj100', value: valuePHD },
+    { key: 'test-obj101', value: valuePHD },
+    { key: 'test-obj2', value: valuePHD },
+    { key: 'test-obj1000', value: valuePHD },
+]
 
 const receivedData = data.map(item => ({ key: item.key, value: item.value }));
 const receivedNonAlphaData = nonAlphabeticalData.map(
@@ -84,6 +92,18 @@ const receivedNonAlphaData = nonAlphabeticalData.map(
 );
 
 const tests = [
+    new Test('testing something', {
+        delimiter: '/',
+    }, {
+        Contents: receivedData,
+        CommonPrefixes: [],
+        Delimiter: '/',
+        IsTruncated: false,
+        NextMarker: undefined,
+    })
+]
+
+const tests2 = [
     new Test('all elements', {}, {
         Contents: receivedData,
         CommonPrefixes: [],
@@ -474,7 +494,7 @@ const alphabeticalOrderTests = [
 
 
 describe('Delimiter listing algorithm', () => {
-    it('Should return good skipping value for DelimiterMaster', done => {
+    it.skip('Should return good skipping value for DelimiterMaster', done => {
         const delimiter = new DelimiterMaster({ delimiter: '/' });
         for (let i = 0; i < 100; i++) {
             delimiter.filter({ key: `foo/${zpad(i)}`, value: '{}' });
@@ -483,6 +503,40 @@ describe('Delimiter listing algorithm', () => {
         done();
     });
 
+    it.only('should do something', done => {
+
+        // const delimiter = new DelimiterMaster({ delimiter: '/' });
+        const delimiter = new DelimiterMaster({});
+
+        const arr = [
+            'oob-test-1',
+            'oob-test-10',
+            'oob-test-101',
+            'oob-test-2',
+            'oob-test-20',
+            'oob-test-222',
+            // 'oob-test-2000',
+        ]
+
+        arr.forEach(i => {
+            console.log(`-> START: ${i}`)
+            // '{ "isPHD": "true" }'
+            let a = delimiter.filter({ key: i, value: '{ "isPHD": "true" }' });
+            console.log(a)
+
+            let b = delimiter.filter({ key: `${i}version1`, value: '{}' });
+            let c = delimiter.filter({ key: `${i}version2`, value: '{}' });
+
+            console.log(b)
+            console.log(c)
+        })
+
+        console.log('==== results: ====')
+        console.log(delimiter.skipping())
+        done()
+
+    });
+
     it('Should set Delimiter alphabeticalOrder field to the expected value',
        () => {
            alphabeticalOrderTests.forEach(test => {
@@ -493,25 +547,34 @@ describe('Delimiter listing algorithm', () => {
            });
        });
 
-    tests.forEach(test => {
+    // tests.forEach(test => {
-        it(`Should list ${test.name}`, done => {
+    //     it(`Should list ${test.name}`, done => {
-            // Simulate skip scan done by LevelDB
+    //         // Simulate skip scan done by LevelDB
-            const d = data.filter(e => test.filter(e, test.input));
+    //         const d = data.filter(e => test.filter(e, test.input));
-            const res = performListing(d, Delimiter, test.input, logger);
+    //         const res = performListing(d, Delimiter, test.input, logger);
-            assert.deepStrictEqual(res, test.output);
+    //         assert.deepStrictEqual(res, test.output);
-            done();
+    //         done();
-        });
+    //     });
-    });
+    // });
+    //
+    // tests.forEach(test => {
+    //     it(`Should list master versions ${test.name}`, done => {
+    //         // Simulate skip scan done by LevelDB
+    //         const d = dataVersioned.filter(e => test.filter(e, test.input));
+    //         const res = performListing(d, DelimiterMaster, test.input, logger);
+    //         assert.deepStrictEqual(res, test.output);
+    //         done();
+    //     });
+    // });
 
-    tests.forEach(test => {
+
-        it(`Should list master versions ${test.name}`, done => {
+    // it.only('should do something', done => {
-            // Simulate skip scan done by LevelDB
+    //     const test = tests[0]
-            const d = dataVersioned.filter(e => test.filter(e, test.input));
+    //     const delimiter = new DelimiterMaster({ delimiter: '/' });
-            const res = performListing(d, DelimiterMaster, test.input, logger);
+    //     let res = performListing(data2, DelimiterMaster, { delimiter: '/' }, logger)
-            assert.deepStrictEqual(res, test.output);
+    //     console.log(res)
-            done();
+    //     done();
-        });
+    // });
-    });
 
     it('Should filter values according to alphabeticalOrder parameter',
        () => {

tests/unit/algos/list/delimiterMaster.js

@@ -286,6 +286,56 @@ describe('Delimiter All masters listing algorithm', () => {
         });
     });
 
+    it.only('should do something', () => {
+        const delimiter = new DelimiterMaster({
+            delimiter: '/',
+        }, fakeLogger);
+        const masterKey = 'key-1';
+        const versionKey1 = `${masterKey}${VID_SEP}version1`;
+        const versionKey2 = `${masterKey}${VID_SEP}version2`;
+
+        const masterKeyB = 'key-10';
+        const versionKeyB1 = `${masterKeyB}${VID_SEP}version1`;
+        const versionKeyB2 = `${masterKeyB}${VID_SEP}version2`;
+
+        const value2 = 'value2';
+
+        /* Filter the PHD version. */
+        assert.strictEqual(delimiter.filter({
+            key: masterKey,
+            value: '{ "isPHD": true, "value": "version" }',
+        }), FILTER_ACCEPT);
+
+        assert.strictEqual(delimiter.filter({
+            key: versionKey1,
+            value: '{ "isDeleteMarker": true }',
+        }), FILTER_ACCEPT);
+
+        assert.strictEqual(delimiter.filter({
+            key: masterKey,
+            value: value2,
+        }), FILTER_ACCEPT);
+
+        console.log(delimiter.result())
+
+        assert.strictEqual(delimiter.filter({
+            key: masterKeyB,
+            value: '{ "isPHD": true, "value": "version" }',
+        }), FILTER_ACCEPT);
+
+        console.log(delimiter.result())
+
+        assert.strictEqual(delimiter.filter({
+            key: masterKey,
+            value: '{ "isDeleteMarker": true }',
+        }), FILTER_ACCEPT);
+
+        assert.strictEqual(delimiter.filter({
+            key: masterKey,
+            value: value2,
+        }), FILTER_ACCEPT);
+    });
+
     it('should return good listing result for version', () => {
         const delimiter = new DelimiterMaster({}, fakeLogger);
         const masterKey = 'key';