Compare commits

...

1 Commits

Author SHA1 Message Date
Alexandre Merle 67aab0b2e6 wip [ci skip] 2017-03-15 04:45:48 +01:00
7 changed files with 118 additions and 24 deletions

View File

@ -8,6 +8,11 @@ function numberDefault(num, defaultNum) {
return Number.isNaN(parsedNum) ? defaultNum : parsedNum;
}
function inc(str) {
return str ? str.slice(0, str.length - 1)
+ String.fromCharCode(str.charCodeAt(str.length - 1) + 1) : str;
}
/**
* Class for the MultipartUploads extension
*/
@ -32,6 +37,28 @@ class MultipartUploads {
this.delimiter = params.delimiter;
this.splitter = params.splitter;
this.logger = logger;
this._listingParams = {
limit: params.limit,
gte: params.gte,
lte: params.lte,
gt: params.gt,
lt: params.lt,
start: params.start,
keys: params.keys,
values: params.values,
};
if (params.keyMarker) {
this._listingParams.gt = `overview${params.splitter}` +
`${params.keyMarker}${params.splitter}`;
if (params.uploadIdMarker) {
this._listingParams.gt += `${params.uploadIdMarker}`;
}
this._listingParams.gt = inc(this._listingParams.gt);
}
if (params.prefix) {
this._listingParams.start = params.prefix;
this._listingParams.lt = inc(params.prefix);
}
}
/**
@ -63,6 +90,11 @@ class MultipartUploads {
this.NextUploadIdMarker = tmp.uploadId;
++this.keys;
}
getListingParams() {
return this._listingParams;
}
/**
* This function adds a common prefix to the CommonPrefixes array
* Set the NextKeyMarker to the current commonPrefix
@ -88,7 +120,7 @@ class MultipartUploads {
if (this.keys >= this.maxKeys) {
// In cases of maxKeys <= 0 => IsTruncated = false
this.IsTruncated = this.maxKeys > 0;
return false;
return 0;
}
const key = obj.key;
const value = obj.value;
@ -107,7 +139,7 @@ class MultipartUploads {
} else {
this.addUpload(value);
}
return true;
return 1;
}
/**

View File

@ -3,6 +3,20 @@
const checkLimit = require('./tools').checkLimit;
const DEFAULT_MAX_KEYS = 10000;
/**
* Used for advancing the last character of a string for setting upper/lower
* bounds
* e.g. _setCharAt('demo1') results in 'demo2',
* _setCharAt('scality') results in 'scalitz'
* @param {String} str - string to be advanced
* @return {String} - modified string
*/
function _setCharAt(str) {
let chr = str.charCodeAt(str.length - 1);
chr = String.fromCharCode(chr + 1);
return str.substr(0, str.length - 1) + chr;
}
/**
* Class of an extension doing the simple listing
*/
@ -10,21 +24,35 @@ class List {
/**
* Constructor
* Set the logger and the res
* @param {Object} parameters - The parameters you sent to DBD
* @param {Object} params - The parameters you sent to DBD
* @param {RequestLogger} logger - The logger of the request
* @return {undefined}
*/
constructor(parameters, logger) {
constructor(params, logger) {
this.logger = logger;
this.res = [];
if (parameters) {
this.maxKeys = checkLimit(parameters.maxKeys, DEFAULT_MAX_KEYS);
} else {
this.maxKeys = DEFAULT_MAX_KEYS;
this._listingParams = {
limit: params.limit,
gte: params.gte,
lte: params.lte,
gt: params.gt,
lt: params.lt,
start: params.start,
keys: params.keys,
values: params.values,
};
if (params.prefix) {
this._listingParams.start = params.prefix;
this._listingParams.lt = _setCharAt(params.prefix);
}
this.maxKeys = checkLimit(params.maxKeys, DEFAULT_MAX_KEYS);
this.keys = 0;
}
getListingParams() {
return this._listingParams;
}
/**
* Function apply on each element
* Just add it to the array
@ -34,11 +62,11 @@ class List {
filter(elem) {
// Check first in case of maxkeys <= 0
if (this.keys >= this.maxKeys) {
return false;
return 0;
}
this.res.push(elem);
this.keys++;
return true;
return 1;
}
/**

View File

@ -1,5 +1,19 @@
'use strict'; // eslint-disable-line strict
/**
* Used for advancing the last character of a string for setting upper/lower
* bounds
* e.g. _setCharAt('demo1') results in 'demo2',
* _setCharAt('scality') results in 'scalitz'
* @param {String} str - string to be advanced
* @return {String} - modified string
*/
function _setCharAt(str) {
let chr = str.charCodeAt(str.length - 1);
chr = String.fromCharCode(chr + 1);
return str.substr(0, str.length - 1) + chr;
}
/**
* Find the next delimiter in the path
*
@ -47,14 +61,31 @@ class Delimiter {
* @param {Number} [parameters.maxKeys] - number of keys to list
*/
constructor(parameters) {
this._listingParams = {
limit: parameters.limit,
gte: parameters.gte,
lte: parameters.lte,
gt: parameters.gt,
lt: parameters.lt,
start: parameters.start,
keys: parameters.keys,
values: parameters.values,
};
if (parameters.marker) {
this._listingParams.gt = parameters.marker;
}
if (parameters.prefix) {
this._listingParams.start = parameters.prefix;
this._listingParams.lt = _setCharAt(parameters.prefix);
}
this.CommonPrefixes = [];
this.Contents = [];
this.IsTruncated = false;
this.NextMarker = parameters.gt;
this.NextMarker = this._listingParams.gt;
this.keys = 0;
this.delimiter = parameters.delimiter;
this.prefix = parameters.start;
this.prefix = this._listingParams.start;
this.maxKeys = parameters.maxKeys || 1000;
if (this.delimiter !== undefined &&
this.NextMarker !== undefined &&
@ -70,6 +101,10 @@ class Delimiter {
}
}
getListingParams() {
return this._listingParams;
}
/**
* check if the max keys count has been reached and set the
* final state of the result if it is the case
@ -94,7 +129,7 @@ class Delimiter {
*/
addContents(key, value) {
if (this._reachedMaxKeys()) {
return false;
return 0;
}
const tmp = JSON.parse(value);
this.Contents.push({
@ -117,7 +152,7 @@ class Delimiter {
});
this.NextMarker = key;
++this.keys;
return true;
return 1;
}
/**
@ -137,7 +172,7 @@ class Delimiter {
if ((this.prefix && !key.startsWith(this.prefix))
|| (typeof this.NextMarker === 'string' &&
key <= this.NextMarker)) {
return true;
return 1;
}
if (this.delimiter) {
const baseIndex = this.prefix ? this.prefix.length : 0;
@ -163,13 +198,13 @@ class Delimiter {
if (this.CommonPrefixes.indexOf(commonPrefix) === -1
&& this.NextMarker !== commonPrefix) {
if (this._reachedMaxKeys()) {
return false;
return 0;
}
this.CommonPrefixes.push(commonPrefix);
this.NextMarker = commonPrefix;
++this.keys;
}
return true;
return 1;
}
/**

View File

@ -52,13 +52,13 @@ class DelimiterMaster extends Delimiter {
if (this.keys >= this.maxKeys) {
// In cases of maxKeys <= 0 => IsTruncated = false
this.IsTruncated = this.maxKeys > 0;
return false;
return 0;
}
// <versioning>
const value = VSUtils.decodeVersion(obj.value);
// ignore it if the master version is a delete marker
if (VSUtils.isDeleteMarker(value)) {
return true;
return 1;
}
// use the original object name for delimitering to work correctly
const key = VSUtils.getObjectNameFromMasterKey(obj.key);
@ -75,7 +75,7 @@ class DelimiterMaster extends Delimiter {
} else {
this.addContents(key, value);
}
return true;
return 1;
}
}

View File

@ -60,7 +60,7 @@ class DelimiterVersions extends Delimiter {
if (this.keys >= this.maxKeys) {
// In cases of maxKeys <= 0 => IsTruncated = false
this.IsTruncated = this.maxKeys > 0;
return false;
return 0;
}
// <versioning>
const key = VSUtils.getObjectNameFromVersionKey(obj.key);
@ -77,7 +77,7 @@ class DelimiterVersions extends Delimiter {
} else {
this.addContents(obj.key, obj.value);
}
return true;
return 1;
}
/**

View File

@ -3,7 +3,7 @@
"engines": {
"node": "6.9.5"
},
"version": "7.0.0",
"version": "7.0.0-1",
"description": "Common utilities for the S3 project components",
"main": "index.js",
"repository": {

View File

@ -28,7 +28,6 @@ describe('Basic listing algorithm', () => {
new Test('10000 elements', { maxKeys: 10000 }, data.slice(0, 10000)),
new Test('more than limit', { maxKeys: 15000 }, data.slice(0, 10000)),
new Test('default limit', {}, data.slice(0, 10000)),
new Test('without parameters', undefined, data.slice(0, 10000)),
new Test('with bad parameters', 'lala', data.slice(0, 10000)),
];
tests.forEach(test => {