index.js

@@ -106,13 +106,6 @@ module.exports = {
             require('./lib/storage/metadata/file/MetadataFileClient'),
             LogConsumer:
             require('./lib/storage/metadata/bucketclient/LogConsumer'),
-            inMemory: {
-                metastore:
-                require('./lib/storage/metadata/in_memory/metastore'),
-                metadata: require('./lib/storage/metadata/in_memory/metadata'),
-                bucketUtilities:
-                require('./lib/storage/metadata/in_memory/bucket_utilities'),
-            },
         },
         data: {
             file: {

lib/algos/list/exportAlgos.js

@@ -1,9 +0,0 @@
-module.exports = {
-    Basic: require('./basic').List,
-    Delimiter: require('./delimiter').Delimiter,
-    DelimiterVersions: require('./delimiterVersions')
-        .DelimiterVersions,
-    DelimiterMaster: require('./delimiterMaster')
-        .DelimiterMaster,
-    MPU: require('./MPU').MultipartUploads,
-};

lib/storage/metadata/MetadataWrapper.js

@@ -3,6 +3,8 @@ const errors = require('../../errors');
 const BucketInfo = require('../../models/BucketInfo');
 
 const BucketClientInterface = require('./bucketclient/BucketClientInterface');
+const BucketFileInterface = require('./file/BucketFileInterface');
+const MongoClientInterface = require('./mongoclient/MongoClientInterface');
 const metastore = require('./in_memory/metastore');
 
 let CdmiMetadata;
@@ -69,10 +71,25 @@ class MetadataWrapper {
         if (clientName === 'mem') {
             this.client = metastore;
             this.implName = 'memorybucket';
+        } else if (clientName === 'file') {
+            this.client = new BucketFileInterface(params, logger);
+            this.implName = 'bucketfile';
         } else if (clientName === 'scality') {
             this.client = new BucketClientInterface(params, bucketclient,
                 logger);
             this.implName = 'bucketclient';
+        } else if (clientName === 'mongodb') {
+            this.client = new MongoClientInterface({
+                replicaSetHosts: params.mongodb.replicaSetHosts,
+                writeConcern: params.mongodb.writeConcern,
+                replicaSet: params.mongodb.replicaSet,
+                readPreference: params.mongodb.readPreference,
+                database: params.mongodb.database,
+                replicationGroupId: params.replicationGroupId,
+                path: params.mongodb.path,
+                logger,
+            });
+            this.implName = 'mongoclient';
         } else if (clientName === 'cdmi') {
             if (!CdmiMetadata) {
                 throw new Error('Unauthorized backend');

lib/storage/metadata/in_memory/ListMultipartUploadsResult.js

@@ -1,6 +1,6 @@
-const ListResult = require('./ListResult');
+import ListResult from './ListResult';
 
-class ListMultipartUploadsResult extends ListResult {
+export class ListMultipartUploadsResult extends ListResult {
     constructor() {
         super();
         this.Uploads = [];
@@ -28,7 +28,3 @@ class ListMultipartUploadsResult extends ListResult {
         this.MaxKeys += 1;
     }
 }
-
-module.exports = {
-    ListMultipartUploadsResult,
-};

lib/storage/metadata/in_memory/ListResult.js

@@ -24,4 +24,4 @@ class ListResult {
     }
 }
 
-module.exports = ListResult;
+export default ListResult;

lib/storage/metadata/in_memory/backend.js

@@ -1,15 +1,16 @@
-const errors = require('../../../errors');
-const list = require('../../../algos/list/exportAlgos');
-const genVID =
-    require('../../../versioning/VersionID').generateVersionId;
-const getMultipartUploadListing = require('./getMultipartUploadListing');
-const { metadata } = require('./metadata');
+import { errors, algorithms, versioning } from 'arsenal';
+
+import getMultipartUploadListing from './getMultipartUploadListing';
+import { metadata } from './metadata';
+import config from '../../Config';
+
+const genVID = versioning.VersionID.generateVersionId;
 
 const defaultMaxKeys = 1000;
 let uidCounter = 0;
 
 function generateVersionId() {
-    return genVID(uidCounter++, undefined);
+    return genVID(uidCounter++, config.replicationGroupId);
 }
 
 function formatVersionKey(key, versionId) {
@@ -78,35 +79,10 @@ const metastore = {
 
     putObject: (bucketName, objName, objVal, params, log, cb) => {
         process.nextTick(() => {
-            // Ignore the PUT done by AbortMPU
-            if (params && params.isAbort) {
-                return cb(null);
-            }
-            return metastore.getBucketAttributes(bucketName, log, err => {
+            metastore.getBucketAttributes(bucketName, log, err => {
                 if (err) {
                     return cb(err);
                 }
-                /*
-                    valid combinations of versioning options:
-                    - !versioning && !versionId: normal non-versioning put
-                    - versioning && !versionId: create a new version
-                    - versionId: update (PUT/DELETE) an existing version,
-                        and also update master version in case the put
-                        version is newer or same version than master.
-                        if versionId === '' update master version
-                */
-
-                if (params && params.versionId) {
-                    objVal.versionId = params.versionId; // eslint-disable-line
-                    const mst = metadata.keyMaps.get(bucketName).get(objName);
-                    if (mst && mst.versionId === params.versionId || !mst) {
-                        metadata.keyMaps.get(bucketName).set(objName, objVal);
-                    }
-                    // eslint-disable-next-line
-                    objName = formatVersionKey(objName, params.versionId);
-                    metadata.keyMaps.get(bucketName).set(objName, objVal);
-                    return cb(null, `{"versionId":"${objVal.versionId}"}`);
-                }
                 if (params && params.versioning) {
                     const versionId = generateVersionId();
                     objVal.versionId = versionId; // eslint-disable-line
@@ -121,6 +97,16 @@ const metastore = {
                     objVal.versionId = versionId; // eslint-disable-line
                     metadata.keyMaps.get(bucketName).set(objName, objVal);
                     return cb(null, `{"versionId":"${objVal.versionId}"}`);
+                } else if (params && params.versionId) {
+                    objVal.versionId = params.versionId; // eslint-disable-line
+                    const mst = metadata.keyMaps.get(bucketName).get(objName);
+                    if (mst && mst.versionId === params.versionId) {
+                        metadata.keyMaps.get(bucketName).set(objName, objVal);
+                    }
+                    // eslint-disable-next-line
+                    objName = formatVersionKey(objName, params.versionId);
+                    metadata.keyMaps.get(bucketName).set(objName, objVal);
+                    return cb(null, `{"versionId":"${objVal.versionId}"}`);
                 }
                 metadata.keyMaps.get(bucketName).set(objName, objVal);
                 return cb(null);
@@ -219,14 +205,7 @@ const metastore = {
 
     listObject(bucketName, params, log, cb) {
         process.nextTick(() => {
-            const {
-                prefix,
-                marker,
-                delimiter,
-                maxKeys,
-                continuationToken,
-                startAfter,
-            } = params;
+            const { prefix, marker, delimiter, maxKeys } = params;
             if (prefix && typeof prefix !== 'string') {
                 return cb(errors.InvalidArgument);
             }
@@ -243,14 +222,6 @@ const metastore = {
                 return cb(errors.InvalidArgument);
             }
 
-            if (continuationToken && typeof continuationToken !== 'string') {
-                return cb(errors.InvalidArgument);
-            }
-
-            if (startAfter && typeof startAfter !== 'string') {
-                return cb(errors.InvalidArgument);
-            }
-
             // If paramMaxKeys is undefined, the default parameter will set it.
             // However, if it is null, the default parameter will not set it.
             let numKeys = maxKeys;
@@ -264,8 +235,8 @@ const metastore = {
 
             // If marker specified, edit the keys array so it
             // only contains keys that occur alphabetically after the marker
-            const listingType = params.listingType;
-            const extension = new list[listingType](params, log);
+            const listingType = params.listingType || 'Delimiter';
+            const extension = new algorithms.list[listingType](params, log);
             const listingParams = extension.genMDParams();
 
             const keys = [];
@@ -331,4 +302,4 @@ const metastore = {
     },
 };
 
-module.exports = metastore;
+export default metastore;

lib/storage/metadata/in_memory/bucket_utilities.js

@@ -1,4 +1,4 @@
-function markerFilterMPU(allMarkers, array) {
+export function markerFilterMPU(allMarkers, array) {
     const { keyMarker, uploadIdMarker } = allMarkers;
     for (let i = 0; i < array.length; i++) {
         // If the keyMarker is the same as the key,
@@ -30,7 +30,7 @@ function markerFilterMPU(allMarkers, array) {
     return array;
 }
 
-function prefixFilter(prefix, array) {
+export function prefixFilter(prefix, array) {
     for (let i = 0; i < array.length; i++) {
         if (array[i].indexOf(prefix) !== 0) {
             array.splice(i, 1);
@@ -40,12 +40,6 @@ function prefixFilter(prefix, array) {
     return array;
 }
 
-function isKeyInContents(responseObject, key) {
+export function isKeyInContents(responseObject, key) {
     return responseObject.Contents.some(val => val.key === key);
 }
-
-module.exports = {
-    markerFilterMPU,
-    prefixFilter,
-    isKeyInContents,
-};

lib/storage/metadata/in_memory/getMultipartUploadListing.js

@@ -1,11 +1,12 @@
-const errors = require('../../../errors');
+import { errors } from 'arsenal';
 
-const { markerFilterMPU, prefixFilter } = require('./bucket_utilities');
-const { ListMultipartUploadsResult } = require('./ListMultipartUploadsResult');
-const { metadata } = require('./metadata');
+import { markerFilterMPU, prefixFilter }
+    from './bucket_utilities';
+import { ListMultipartUploadsResult } from './ListMultipartUploadsResult';
+import { metadata } from './metadata';
 
 const defaultMaxKeys = 1000;
-function getMultipartUploadListing(bucket, params, callback) {
+export default function getMultipartUploadListing(bucket, params, callback) {
     const { delimiter, keyMarker,
         uploadIdMarker, prefix, queryPrefixLength, splitter } = params;
     const splitterLen = splitter.length;
@@ -144,5 +145,3 @@ function getMultipartUploadListing(bucket, params, callback) {
     response.IsTruncated = maxKeys === 0 ? false : response.IsTruncated;
     return callback(null, response);
 }
-
-module.exports = getMultipartUploadListing;

lib/storage/metadata/in_memory/metadata.js

@@ -3,6 +3,4 @@ const metadata = {
     keyMaps: new Map,
 };
 
-module.exports = {
-    metadata,
-};
+export { metadata };

package.json

@@ -3,7 +3,7 @@
   "engines": {
     "node": ">=16"
   },
-  "version": "7.10.14",
+  "version": "7.10.13",
   "description": "Common utilities for the S3 project components",
   "main": "index.js",
   "repository": {

tests/unit/storage/metadata/in_memory/FakeBucketInfo.json

@@ -1,26 +0,0 @@
-{
-    "acl": {
-        "Canned": "private",
-        "FULL_CONTROL": [],
-        "WRITE": [],
-        "WRITE_ACP": [],
-        "READ": [],
-        "READ_ACP": []
-    },
-    "name": "BucketName",
-    "owner": "9d8fe19a78974c56dceb2ea4a8f01ed0f5fecb9d29f80e9e3b84104e4a3ea520",
-    "ownerDisplayName": "anonymousCoward",
-    "creationDate": "2018-06-04T17:45:42.592Z",
-    "mdBucketModelVersion": 8,
-    "transient": false,
-    "deleted": false,
-    "serverSideEncryption": null,
-    "versioningConfiguration": null,
-    "websiteConfiguration": null,
-    "locationConstraint": "us-east-1",
-    "readLocationConstraint": "us-east-1",
-    "cors": null,
-    "replicationConfiguration": null,
-    "lifecycleConfiguration": null,
-    "uid": "fea97818-6a9a-11e8-9777-e311618cc5d4"
-}

tests/unit/storage/metadata/in_memory/in_memory.spec.js

@@ -1,59 +0,0 @@
-const async = require('async');
-const assert = require('assert');
-const werelogs = require('werelogs');
-const MetadataWrapper = require('../../../../../lib/storage/metadata/MetadataWrapper');
-const fakeBucketInfo = require('./FakeBucketInfo.json');
-
-describe('InMemory', () => {
-    const fakeBucket = 'fake';
-    const logger = new werelogs.Logger('Injector');
-    const memBackend = new MetadataWrapper(
-        'mem', {}, null, logger);
-
-    before(done => {
-        memBackend.createBucket(fakeBucket, fakeBucketInfo, logger, done);
-    });
-
-    after(done => {
-        memBackend.deleteBucket(fakeBucket, logger, done);
-    });
-
-    it('basic', done => {
-        async.waterfall([
-            next => {
-                memBackend.putObjectMD(fakeBucket, 'foo', 'bar', {}, logger, err => {
-                    if (err) {
-                        return next(err);
-                    }
-                    return next();
-                });
-            },
-            next => {
-                memBackend.getObjectMD(fakeBucket, 'foo', {}, logger, (err, data) => {
-                    if (err) {
-                        return next(err);
-                    }
-                    assert.deepEqual(data, 'bar');
-                    return next();
-                });
-            },
-            next => {
-                memBackend.deleteObjectMD(fakeBucket, 'foo', {}, logger, err => {
-                    if (err) {
-                        return next(err);
-                    }
-                    return next();
-                });
-            },
-            next => {
-                memBackend.getObjectMD(fakeBucket, 'foo', {}, logger, err => {
-                    if (err) {
-                        assert.deepEqual(err.message, 'NoSuchKey');
-                        return next();
-                    }
-                    return next(new Error('unexpected success'));
-                });
-            },
-        ], done);
-    });
-});