ft: ARSN-114 in_memory backend

Minor changes.
Create a basic test

index.js

@@ -106,6 +106,13 @@ module.exports = {
             require('./lib/storage/metadata/file/MetadataFileClient'),
             LogConsumer:
             require('./lib/storage/metadata/bucketclient/LogConsumer'),
+            inMemory: {
+                metastore:
+                require('./lib/storage/metadata/in_memory/metastore'),
+                metadata: require('./lib/storage/metadata/in_memory/metadata'),
+                bucketUtilities:
+                require('./lib/storage/metadata/in_memory/bucket_utilities'),
+            },
         },
         data: {
             file: {

lib/algos/list/exportAlgos.js

@@ -0,0 +1,9 @@
+module.exports = {
+    Basic: require('./basic').List,
+    Delimiter: require('./delimiter').Delimiter,
+    DelimiterVersions: require('./delimiterVersions')
+        .DelimiterVersions,
+    DelimiterMaster: require('./delimiterMaster')
+        .DelimiterMaster,
+    MPU: require('./MPU').MultipartUploads,
+};

lib/storage/metadata/MetadataWrapper.js

@@ -3,8 +3,6 @@ const errors = require('../../errors');
 const BucketInfo = require('../../models/BucketInfo');
 
 const BucketClientInterface = require('./bucketclient/BucketClientInterface');
-const BucketFileInterface = require('./file/BucketFileInterface');
-const MongoClientInterface = require('./mongoclient/MongoClientInterface');
 const metastore = require('./in_memory/metastore');
 
 let CdmiMetadata;
@@ -71,25 +69,10 @@ class MetadataWrapper {
         if (clientName === 'mem') {
             this.client = metastore;
             this.implName = 'memorybucket';
-        } else if (clientName === 'file') {
-            this.client = new BucketFileInterface(params, logger);
-            this.implName = 'bucketfile';
         } else if (clientName === 'scality') {
             this.client = new BucketClientInterface(params, bucketclient,
                 logger);
             this.implName = 'bucketclient';
-        } else if (clientName === 'mongodb') {
-            this.client = new MongoClientInterface({
-                replicaSetHosts: params.mongodb.replicaSetHosts,
-                writeConcern: params.mongodb.writeConcern,
-                replicaSet: params.mongodb.replicaSet,
-                readPreference: params.mongodb.readPreference,
-                database: params.mongodb.database,
-                replicationGroupId: params.replicationGroupId,
-                path: params.mongodb.path,
-                logger,
-            });
-            this.implName = 'mongoclient';
         } else if (clientName === 'cdmi') {
             if (!CdmiMetadata) {
                 throw new Error('Unauthorized backend');

lib/storage/metadata/in_memory/ListMultipartUploadsResult.js

@@ -1,6 +1,6 @@
-import ListResult from './ListResult';
+const ListResult = require('./ListResult');
 
-export class ListMultipartUploadsResult extends ListResult {
+class ListMultipartUploadsResult extends ListResult {
     constructor() {
         super();
         this.Uploads = [];
@@ -28,3 +28,7 @@ export class ListMultipartUploadsResult extends ListResult {
         this.MaxKeys += 1;
     }
 }
+
+module.exports = {
+    ListMultipartUploadsResult,
+};

lib/storage/metadata/in_memory/ListResult.js

@@ -24,4 +24,4 @@ class ListResult {
     }
 }
 
-export default ListResult;
+module.exports = ListResult;

lib/storage/metadata/in_memory/bucket_utilities.js

@@ -1,4 +1,4 @@
-export function markerFilterMPU(allMarkers, array) {
+function markerFilterMPU(allMarkers, array) {
     const { keyMarker, uploadIdMarker } = allMarkers;
     for (let i = 0; i < array.length; i++) {
         // If the keyMarker is the same as the key,
@@ -30,7 +30,7 @@ export function markerFilterMPU(allMarkers, array) {
     return array;
 }
 
-export function prefixFilter(prefix, array) {
+function prefixFilter(prefix, array) {
     for (let i = 0; i < array.length; i++) {
         if (array[i].indexOf(prefix) !== 0) {
             array.splice(i, 1);
@@ -40,6 +40,12 @@ export function prefixFilter(prefix, array) {
     return array;
 }
 
-export function isKeyInContents(responseObject, key) {
+function isKeyInContents(responseObject, key) {
     return responseObject.Contents.some(val => val.key === key);
 }
+
+module.exports = {
+    markerFilterMPU,
+    prefixFilter,
+    isKeyInContents,
+};

lib/storage/metadata/in_memory/getMultipartUploadListing.js

@@ -1,12 +1,11 @@
-import { errors } from 'arsenal';
+const errors = require('../../../errors');
 
-import { markerFilterMPU, prefixFilter }
+const { markerFilterMPU, prefixFilter } = require('./bucket_utilities');
-    from './bucket_utilities';
+const { ListMultipartUploadsResult } = require('./ListMultipartUploadsResult');
-import { ListMultipartUploadsResult } from './ListMultipartUploadsResult';
+const { metadata } = require('./metadata');
-import { metadata } from './metadata';
 
 const defaultMaxKeys = 1000;
-export default function getMultipartUploadListing(bucket, params, callback) {
+function getMultipartUploadListing(bucket, params, callback) {
     const { delimiter, keyMarker,
         uploadIdMarker, prefix, queryPrefixLength, splitter } = params;
     const splitterLen = splitter.length;
@@ -145,3 +144,5 @@ export default function getMultipartUploadListing(bucket, params, callback) {
     response.IsTruncated = maxKeys === 0 ? false : response.IsTruncated;
     return callback(null, response);
 }
+
+module.exports = getMultipartUploadListing;

lib/storage/metadata/in_memory/metadata.js

@@ -3,4 +3,6 @@ const metadata = {
     keyMaps: new Map,
 };
 
-export { metadata };
+module.exports = {
+    metadata,
+};

lib/storage/metadata/in_memory/metastore.js

@@ -1,16 +1,15 @@
-import { errors, algorithms, versioning } from 'arsenal';
+const errors = require('../../../errors');
-
+const list = require('../../../algos/list/exportAlgos');
-import getMultipartUploadListing from './getMultipartUploadListing';
+const genVID =
-import { metadata } from './metadata';
+    require('../../../versioning/VersionID').generateVersionId;
-import config from '../../Config';
+const getMultipartUploadListing = require('./getMultipartUploadListing');
-
+const { metadata } = require('./metadata');
-const genVID = versioning.VersionID.generateVersionId;
 
 const defaultMaxKeys = 1000;
 let uidCounter = 0;
 
 function generateVersionId() {
-    return genVID(uidCounter++, config.replicationGroupId);
+    return genVID(uidCounter++, undefined);
 }
 
 function formatVersionKey(key, versionId) {
@@ -79,10 +78,35 @@ const metastore = {
 
     putObject: (bucketName, objName, objVal, params, log, cb) => {
         process.nextTick(() => {
-            metastore.getBucketAttributes(bucketName, log, err => {
+            // Ignore the PUT done by AbortMPU
+            if (params && params.isAbort) {
+                return cb(null);
+            }
+            return metastore.getBucketAttributes(bucketName, log, err => {
                 if (err) {
                     return cb(err);
                 }
+                /*
+                    valid combinations of versioning options:
+                    - !versioning && !versionId: normal non-versioning put
+                    - versioning && !versionId: create a new version
+                    - versionId: update (PUT/DELETE) an existing version,
+                        and also update master version in case the put
+                        version is newer or same version than master.
+                        if versionId === '' update master version
+                */
+
+                if (params && params.versionId) {
+                    objVal.versionId = params.versionId; // eslint-disable-line
+                    const mst = metadata.keyMaps.get(bucketName).get(objName);
+                    if (mst && mst.versionId === params.versionId || !mst) {
+                        metadata.keyMaps.get(bucketName).set(objName, objVal);
+                    }
+                    // eslint-disable-next-line
+                    objName = formatVersionKey(objName, params.versionId);
+                    metadata.keyMaps.get(bucketName).set(objName, objVal);
+                    return cb(null, `{"versionId":"${objVal.versionId}"}`);
+                }
                 if (params && params.versioning) {
                     const versionId = generateVersionId();
                     objVal.versionId = versionId; // eslint-disable-line
@@ -97,16 +121,6 @@ const metastore = {
                     objVal.versionId = versionId; // eslint-disable-line
                     metadata.keyMaps.get(bucketName).set(objName, objVal);
                     return cb(null, `{"versionId":"${objVal.versionId}"}`);
-                } else if (params && params.versionId) {
-                    objVal.versionId = params.versionId; // eslint-disable-line
-                    const mst = metadata.keyMaps.get(bucketName).get(objName);
-                    if (mst && mst.versionId === params.versionId) {
-                        metadata.keyMaps.get(bucketName).set(objName, objVal);
-                    }
-                    // eslint-disable-next-line
-                    objName = formatVersionKey(objName, params.versionId);
-                    metadata.keyMaps.get(bucketName).set(objName, objVal);
-                    return cb(null, `{"versionId":"${objVal.versionId}"}`);
                 }
                 metadata.keyMaps.get(bucketName).set(objName, objVal);
                 return cb(null);
@@ -205,7 +219,14 @@ const metastore = {
 
     listObject(bucketName, params, log, cb) {
         process.nextTick(() => {
-            const { prefix, marker, delimiter, maxKeys } = params;
+            const {
+                prefix,
+                marker,
+                delimiter,
+                maxKeys,
+                continuationToken,
+                startAfter,
+            } = params;
             if (prefix && typeof prefix !== 'string') {
                 return cb(errors.InvalidArgument);
             }
@@ -222,6 +243,14 @@ const metastore = {
                 return cb(errors.InvalidArgument);
             }
 
+            if (continuationToken && typeof continuationToken !== 'string') {
+                return cb(errors.InvalidArgument);
+            }
+
+            if (startAfter && typeof startAfter !== 'string') {
+                return cb(errors.InvalidArgument);
+            }
+
             // If paramMaxKeys is undefined, the default parameter will set it.
             // However, if it is null, the default parameter will not set it.
             let numKeys = maxKeys;
@@ -235,8 +264,8 @@ const metastore = {
 
             // If marker specified, edit the keys array so it
             // only contains keys that occur alphabetically after the marker
-            const listingType = params.listingType || 'Delimiter';
+            const listingType = params.listingType;
-            const extension = new algorithms.list[listingType](params, log);
+            const extension = new list[listingType](params, log);
             const listingParams = extension.genMDParams();
 
             const keys = [];
@@ -302,4 +331,4 @@ const metastore = {
     },
 };
 
-export default metastore;
+module.exports = metastore;

package.json

@@ -3,7 +3,7 @@
   "engines": {
     "node": ">=16"
   },
-  "version": "7.10.13",
+  "version": "7.10.14",
   "description": "Common utilities for the S3 project components",
   "main": "index.js",
   "repository": {

tests/unit/storage/metadata/in_memory/FakeBucketInfo.json

@@ -0,0 +1,26 @@
+{
+    "acl": {
+        "Canned": "private",
+        "FULL_CONTROL": [],
+        "WRITE": [],
+        "WRITE_ACP": [],
+        "READ": [],
+        "READ_ACP": []
+    },
+    "name": "BucketName",
+    "owner": "9d8fe19a78974c56dceb2ea4a8f01ed0f5fecb9d29f80e9e3b84104e4a3ea520",
+    "ownerDisplayName": "anonymousCoward",
+    "creationDate": "2018-06-04T17:45:42.592Z",
+    "mdBucketModelVersion": 8,
+    "transient": false,
+    "deleted": false,
+    "serverSideEncryption": null,
+    "versioningConfiguration": null,
+    "websiteConfiguration": null,
+    "locationConstraint": "us-east-1",
+    "readLocationConstraint": "us-east-1",
+    "cors": null,
+    "replicationConfiguration": null,
+    "lifecycleConfiguration": null,
+    "uid": "fea97818-6a9a-11e8-9777-e311618cc5d4"
+}

tests/unit/storage/metadata/in_memory/in_memory.spec.js

@@ -0,0 +1,59 @@
+const async = require('async');
+const assert = require('assert');
+const werelogs = require('werelogs');
+const MetadataWrapper = require('../../../../../lib/storage/metadata/MetadataWrapper');
+const fakeBucketInfo = require('./FakeBucketInfo.json');
+
+describe('InMemory', () => {
+    const fakeBucket = 'fake';
+    const logger = new werelogs.Logger('Injector');
+    const memBackend = new MetadataWrapper(
+        'mem', {}, null, logger);
+
+    before(done => {
+        memBackend.createBucket(fakeBucket, fakeBucketInfo, logger, done);
+    });
+
+    after(done => {
+        memBackend.deleteBucket(fakeBucket, logger, done);
+    });
+
+    it('basic', done => {
+        async.waterfall([
+            next => {
+                memBackend.putObjectMD(fakeBucket, 'foo', 'bar', {}, logger, err => {
+                    if (err) {
+                        return next(err);
+                    }
+                    return next();
+                });
+            },
+            next => {
+                memBackend.getObjectMD(fakeBucket, 'foo', {}, logger, (err, data) => {
+                    if (err) {
+                        return next(err);
+                    }
+                    assert.deepEqual(data, 'bar');
+                    return next();
+                });
+            },
+            next => {
+                memBackend.deleteObjectMD(fakeBucket, 'foo', {}, logger, err => {
+                    if (err) {
+                        return next(err);
+                    }
+                    return next();
+                });
+            },
+            next => {
+                memBackend.getObjectMD(fakeBucket, 'foo', {}, logger, err => {
+                    if (err) {
+                        assert.deepEqual(err.message, 'NoSuchKey');
+                        return next();
+                    }
+                    return next(new Error('unexpected success'));
+                });
+            },
+        ], done);
+    });
+});