Compare commits
4 Commits
cc6ed165dd
...
e2015839c7
Author | SHA1 | Date |
---|---|---|
philipyoo | e2015839c7 | |
philipyoo | 970ff73fa0 | |
philipyoo | 587435dfd2 | |
philipyoo | 27bd86a4cc |
|
@ -46,6 +46,13 @@ class DelimiterMaster extends Delimiter {
|
||||||
let key = obj.key;
|
let key = obj.key;
|
||||||
const value = obj.value;
|
const value = obj.value;
|
||||||
|
|
||||||
|
// if (typeof this[this.nextContinueMarker] === 'string') {
|
||||||
|
// console.log('___ START ____')
|
||||||
|
// console.log(key, this[this.nextContinueMarker])
|
||||||
|
// console.log(key <= this[this.nextContinueMarker])
|
||||||
|
// console.log('____ END ____')
|
||||||
|
// }
|
||||||
|
|
||||||
/* Skip keys not starting with the prefix or not alphabetically
|
/* Skip keys not starting with the prefix or not alphabetically
|
||||||
* ordered. */
|
* ordered. */
|
||||||
if ((this.prefix && !key.startsWith(this.prefix))
|
if ((this.prefix && !key.startsWith(this.prefix))
|
||||||
|
@ -76,11 +83,15 @@ class DelimiterMaster extends Delimiter {
|
||||||
if (key === this.prvKey || key === this[this.nextContinueMarker] ||
|
if (key === this.prvKey || key === this[this.nextContinueMarker] ||
|
||||||
(this.delimiter &&
|
(this.delimiter &&
|
||||||
key.startsWith(this[this.nextContinueMarker]))) {
|
key.startsWith(this[this.nextContinueMarker]))) {
|
||||||
|
// console.log(`----> ${key}`)
|
||||||
|
// console.log(`--> ${this[this.nextContinueMarker]}`)
|
||||||
/* master version already filtered */
|
/* master version already filtered */
|
||||||
return FILTER_SKIP;
|
return FILTER_SKIP;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// console.log(Version.isPHD(value))
|
||||||
if (Version.isPHD(value)) {
|
if (Version.isPHD(value)) {
|
||||||
|
// console.log('IS PHD')
|
||||||
/* master version is a PHD version, we want to wait for the next
|
/* master version is a PHD version, we want to wait for the next
|
||||||
* one:
|
* one:
|
||||||
* - Set the prvKey to undefined to not skip the next version,
|
* - Set the prvKey to undefined to not skip the next version,
|
||||||
|
@ -101,6 +112,7 @@ class DelimiterMaster extends Delimiter {
|
||||||
* next entry). In that case we expect the master version to
|
* next entry). In that case we expect the master version to
|
||||||
* follow. */
|
* follow. */
|
||||||
if (key === this.prvPHDKey) {
|
if (key === this.prvPHDKey) {
|
||||||
|
// TODO: resetting prvKey causing listing issue?
|
||||||
this.prvKey = undefined;
|
this.prvKey = undefined;
|
||||||
return FILTER_ACCEPT;
|
return FILTER_ACCEPT;
|
||||||
}
|
}
|
||||||
|
@ -123,14 +135,18 @@ class DelimiterMaster extends Delimiter {
|
||||||
|
|
||||||
skipping() {
|
skipping() {
|
||||||
if (this[this.nextContinueMarker]) {
|
if (this[this.nextContinueMarker]) {
|
||||||
|
// console.log(`-- skipping-nextContinueMarker: ${this[this.nextContinueMarker]}`)
|
||||||
// next marker or next continuation token:
|
// next marker or next continuation token:
|
||||||
// - foo/ : skipping foo/
|
// - foo/ : skipping foo/
|
||||||
// - foo : skipping foo.
|
// - foo : skipping foo.
|
||||||
const index = this[this.nextContinueMarker].
|
const index = this[this.nextContinueMarker].
|
||||||
lastIndexOf(this.delimiter);
|
lastIndexOf(this.delimiter);
|
||||||
|
// console.log(`index: ${index}`)
|
||||||
if (index === this[this.nextContinueMarker].length - 1) {
|
if (index === this[this.nextContinueMarker].length - 1) {
|
||||||
|
// console.log(`returning nextContinueMarker`)
|
||||||
return this[this.nextContinueMarker];
|
return this[this.nextContinueMarker];
|
||||||
}
|
}
|
||||||
|
// console.log(`returning + VID_SEP: ${this[this.nextContinueMarker] + VID_SEP}`)
|
||||||
return this[this.nextContinueMarker] + VID_SEP;
|
return this[this.nextContinueMarker] + VID_SEP;
|
||||||
}
|
}
|
||||||
return SKIP_NONE;
|
return SKIP_NONE;
|
||||||
|
|
|
@ -54,11 +54,18 @@ class Skip {
|
||||||
const filteringResult = this.extension.filter(entry);
|
const filteringResult = this.extension.filter(entry);
|
||||||
const skippingRange = this.extension.skipping();
|
const skippingRange = this.extension.skipping();
|
||||||
|
|
||||||
|
// console.log('--------')
|
||||||
|
// console.log(this.extension.constructor.name)
|
||||||
|
// console.log(filteringResult)
|
||||||
|
// console.log(skippingRange)
|
||||||
|
|
||||||
if (filteringResult === FILTER_END) {
|
if (filteringResult === FILTER_END) {
|
||||||
this.listingEndCb();
|
this.listingEndCb();
|
||||||
} else if (filteringResult === FILTER_SKIP
|
} else if (filteringResult === FILTER_SKIP
|
||||||
&& skippingRange !== SKIP_NONE) {
|
&& skippingRange !== SKIP_NONE) {
|
||||||
|
// console.log('=> IN filter_skip && skippingRange != skip_none')
|
||||||
if (++this.streakLength >= MAX_STREAK_LENGTH) {
|
if (++this.streakLength >= MAX_STREAK_LENGTH) {
|
||||||
|
// console.log('MAX STREAK LENGTH MET?')
|
||||||
const newRange = this._inc(skippingRange);
|
const newRange = this._inc(skippingRange);
|
||||||
|
|
||||||
/* Avoid to loop on the same range again and again. */
|
/* Avoid to loop on the same range again and again. */
|
||||||
|
|
|
@ -896,6 +896,7 @@ class MongoClientInterface {
|
||||||
|
|
||||||
listObject(bucketName, params, log, cb) {
|
listObject(bucketName, params, log, cb) {
|
||||||
const extName = params.listingType;
|
const extName = params.listingType;
|
||||||
|
console.log(`===> listObject type: ${extName}`)
|
||||||
const extension = new listAlgos[extName](params, log);
|
const extension = new listAlgos[extName](params, log);
|
||||||
const internalParams = extension.genMDParams();
|
const internalParams = extension.genMDParams();
|
||||||
internalParams.mongifiedSearch = params.mongifiedSearch;
|
internalParams.mongifiedSearch = params.mongifiedSearch;
|
||||||
|
|
|
@ -97,6 +97,10 @@ class MongoReadStream extends Readable {
|
||||||
value = JSON.stringify(doc.value);
|
value = JSON.stringify(doc.value);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (key !== undefined) {
|
||||||
|
console.log(key)
|
||||||
|
}
|
||||||
|
|
||||||
if (key === undefined && value === undefined) {
|
if (key === undefined && value === undefined) {
|
||||||
this.push(null);
|
this.push(null);
|
||||||
} else if (this._options.keys !== false &&
|
} else if (this._options.keys !== false &&
|
||||||
|
|
|
@ -77,6 +77,14 @@ const nonAlphabeticalData = [
|
||||||
{ key: 'zzz', value },
|
{ key: 'zzz', value },
|
||||||
{ key: 'aaa', value },
|
{ key: 'aaa', value },
|
||||||
];
|
];
|
||||||
|
const data2 = [
|
||||||
|
{ key: 'test-obj1', value: valuePHD },
|
||||||
|
{ key: 'test-obj10', value: valuePHD },
|
||||||
|
{ key: 'test-obj100', value: valuePHD },
|
||||||
|
{ key: 'test-obj101', value: valuePHD },
|
||||||
|
{ key: 'test-obj2', value: valuePHD },
|
||||||
|
{ key: 'test-obj1000', value: valuePHD },
|
||||||
|
]
|
||||||
|
|
||||||
const receivedData = data.map(item => ({ key: item.key, value: item.value }));
|
const receivedData = data.map(item => ({ key: item.key, value: item.value }));
|
||||||
const receivedNonAlphaData = nonAlphabeticalData.map(
|
const receivedNonAlphaData = nonAlphabeticalData.map(
|
||||||
|
@ -84,6 +92,18 @@ const receivedNonAlphaData = nonAlphabeticalData.map(
|
||||||
);
|
);
|
||||||
|
|
||||||
const tests = [
|
const tests = [
|
||||||
|
new Test('testing something', {
|
||||||
|
delimiter: '/',
|
||||||
|
}, {
|
||||||
|
Contents: receivedData,
|
||||||
|
CommonPrefixes: [],
|
||||||
|
Delimiter: '/',
|
||||||
|
IsTruncated: false,
|
||||||
|
NextMarker: undefined,
|
||||||
|
})
|
||||||
|
]
|
||||||
|
|
||||||
|
const tests2 = [
|
||||||
new Test('all elements', {}, {
|
new Test('all elements', {}, {
|
||||||
Contents: receivedData,
|
Contents: receivedData,
|
||||||
CommonPrefixes: [],
|
CommonPrefixes: [],
|
||||||
|
@ -474,7 +494,7 @@ const alphabeticalOrderTests = [
|
||||||
|
|
||||||
|
|
||||||
describe('Delimiter listing algorithm', () => {
|
describe('Delimiter listing algorithm', () => {
|
||||||
it('Should return good skipping value for DelimiterMaster', done => {
|
it.skip('Should return good skipping value for DelimiterMaster', done => {
|
||||||
const delimiter = new DelimiterMaster({ delimiter: '/' });
|
const delimiter = new DelimiterMaster({ delimiter: '/' });
|
||||||
for (let i = 0; i < 100; i++) {
|
for (let i = 0; i < 100; i++) {
|
||||||
delimiter.filter({ key: `foo/${zpad(i)}`, value: '{}' });
|
delimiter.filter({ key: `foo/${zpad(i)}`, value: '{}' });
|
||||||
|
@ -483,6 +503,40 @@ describe('Delimiter listing algorithm', () => {
|
||||||
done();
|
done();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it.only('should do something', done => {
|
||||||
|
|
||||||
|
// const delimiter = new DelimiterMaster({ delimiter: '/' });
|
||||||
|
const delimiter = new DelimiterMaster({});
|
||||||
|
|
||||||
|
const arr = [
|
||||||
|
'oob-test-1',
|
||||||
|
'oob-test-10',
|
||||||
|
'oob-test-101',
|
||||||
|
'oob-test-2',
|
||||||
|
'oob-test-20',
|
||||||
|
'oob-test-222',
|
||||||
|
// 'oob-test-2000',
|
||||||
|
]
|
||||||
|
|
||||||
|
arr.forEach(i => {
|
||||||
|
console.log(`-> START: ${i}`)
|
||||||
|
// '{ "isPHD": "true" }'
|
||||||
|
let a = delimiter.filter({ key: i, value: '{ "isPHD": "true" }' });
|
||||||
|
console.log(a)
|
||||||
|
|
||||||
|
let b = delimiter.filter({ key: `${i}version1`, value: '{}' });
|
||||||
|
let c = delimiter.filter({ key: `${i}version2`, value: '{}' });
|
||||||
|
|
||||||
|
console.log(b)
|
||||||
|
console.log(c)
|
||||||
|
})
|
||||||
|
|
||||||
|
console.log('==== results: ====')
|
||||||
|
console.log(delimiter.skipping())
|
||||||
|
done()
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
it('Should set Delimiter alphabeticalOrder field to the expected value',
|
it('Should set Delimiter alphabeticalOrder field to the expected value',
|
||||||
() => {
|
() => {
|
||||||
alphabeticalOrderTests.forEach(test => {
|
alphabeticalOrderTests.forEach(test => {
|
||||||
|
@ -493,25 +547,34 @@ describe('Delimiter listing algorithm', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
tests.forEach(test => {
|
// tests.forEach(test => {
|
||||||
it(`Should list ${test.name}`, done => {
|
// it(`Should list ${test.name}`, done => {
|
||||||
// Simulate skip scan done by LevelDB
|
// // Simulate skip scan done by LevelDB
|
||||||
const d = data.filter(e => test.filter(e, test.input));
|
// const d = data.filter(e => test.filter(e, test.input));
|
||||||
const res = performListing(d, Delimiter, test.input, logger);
|
// const res = performListing(d, Delimiter, test.input, logger);
|
||||||
assert.deepStrictEqual(res, test.output);
|
// assert.deepStrictEqual(res, test.output);
|
||||||
done();
|
// done();
|
||||||
});
|
// });
|
||||||
});
|
// });
|
||||||
|
//
|
||||||
|
// tests.forEach(test => {
|
||||||
|
// it(`Should list master versions ${test.name}`, done => {
|
||||||
|
// // Simulate skip scan done by LevelDB
|
||||||
|
// const d = dataVersioned.filter(e => test.filter(e, test.input));
|
||||||
|
// const res = performListing(d, DelimiterMaster, test.input, logger);
|
||||||
|
// assert.deepStrictEqual(res, test.output);
|
||||||
|
// done();
|
||||||
|
// });
|
||||||
|
// });
|
||||||
|
|
||||||
tests.forEach(test => {
|
|
||||||
it(`Should list master versions ${test.name}`, done => {
|
// it.only('should do something', done => {
|
||||||
// Simulate skip scan done by LevelDB
|
// const test = tests[0]
|
||||||
const d = dataVersioned.filter(e => test.filter(e, test.input));
|
// const delimiter = new DelimiterMaster({ delimiter: '/' });
|
||||||
const res = performListing(d, DelimiterMaster, test.input, logger);
|
// let res = performListing(data2, DelimiterMaster, { delimiter: '/' }, logger)
|
||||||
assert.deepStrictEqual(res, test.output);
|
// console.log(res)
|
||||||
done();
|
// done();
|
||||||
});
|
// });
|
||||||
});
|
|
||||||
|
|
||||||
it('Should filter values according to alphabeticalOrder parameter',
|
it('Should filter values according to alphabeticalOrder parameter',
|
||||||
() => {
|
() => {
|
||||||
|
|
|
@ -286,6 +286,56 @@ describe('Delimiter All masters listing algorithm', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it.only('should do something', () => {
|
||||||
|
const delimiter = new DelimiterMaster({
|
||||||
|
delimiter: '/',
|
||||||
|
}, fakeLogger);
|
||||||
|
const masterKey = 'key-1';
|
||||||
|
const versionKey1 = `${masterKey}${VID_SEP}version1`;
|
||||||
|
const versionKey2 = `${masterKey}${VID_SEP}version2`;
|
||||||
|
|
||||||
|
const masterKeyB = 'key-10';
|
||||||
|
const versionKeyB1 = `${masterKeyB}${VID_SEP}version1`;
|
||||||
|
const versionKeyB2 = `${masterKeyB}${VID_SEP}version2`;
|
||||||
|
|
||||||
|
const value2 = 'value2';
|
||||||
|
|
||||||
|
/* Filter the PHD version. */
|
||||||
|
assert.strictEqual(delimiter.filter({
|
||||||
|
key: masterKey,
|
||||||
|
value: '{ "isPHD": true, "value": "version" }',
|
||||||
|
}), FILTER_ACCEPT);
|
||||||
|
|
||||||
|
assert.strictEqual(delimiter.filter({
|
||||||
|
key: versionKey1,
|
||||||
|
value: '{ "isDeleteMarker": true }',
|
||||||
|
}), FILTER_ACCEPT);
|
||||||
|
|
||||||
|
assert.strictEqual(delimiter.filter({
|
||||||
|
key: masterKey,
|
||||||
|
value: value2,
|
||||||
|
}), FILTER_ACCEPT);
|
||||||
|
|
||||||
|
console.log(delimiter.result())
|
||||||
|
|
||||||
|
assert.strictEqual(delimiter.filter({
|
||||||
|
key: masterKeyB,
|
||||||
|
value: '{ "isPHD": true, "value": "version" }',
|
||||||
|
}), FILTER_ACCEPT);
|
||||||
|
|
||||||
|
console.log(delimiter.result())
|
||||||
|
|
||||||
|
assert.strictEqual(delimiter.filter({
|
||||||
|
key: masterKey,
|
||||||
|
value: '{ "isDeleteMarker": true }',
|
||||||
|
}), FILTER_ACCEPT);
|
||||||
|
|
||||||
|
assert.strictEqual(delimiter.filter({
|
||||||
|
key: masterKey,
|
||||||
|
value: value2,
|
||||||
|
}), FILTER_ACCEPT);
|
||||||
|
});
|
||||||
|
|
||||||
it('should return good listing result for version', () => {
|
it('should return good listing result for version', () => {
|
||||||
const delimiter = new DelimiterMaster({}, fakeLogger);
|
const delimiter = new DelimiterMaster({}, fakeLogger);
|
||||||
const masterKey = 'key';
|
const masterKey = 'key';
|
||||||
|
|
Loading…
Reference in New Issue