feature: ZENKO-1439 Conditional PUT logic

config.json

@@ -112,5 +112,16 @@
                 "maxSockets": null
             }
         }
+    },
+    "localCache": {
+        "host": "127.0.0.1",
+        "port": 6379
+    },
+    "utapi": {
+        "workers": 1
+    },
+    "redis": {
+        "host": "127.0.0.1",
+        "port": 6379
     }
 }

package.json

@@ -19,7 +19,7 @@
   },
   "homepage": "https://github.com/scality/S3#readme",
   "dependencies": {
-    "arsenal": "github:scality/arsenal#c657b4b",
+    "arsenal": "github:scality/arsenal#5b117d3",
     "async": "~2.5.0",
     "aws-sdk": "2.28.0",
     "azure-storage": "^2.1.0",
@@ -82,7 +82,8 @@
     "ft_s3cmd": "cd tests/functional/s3cmd && mocha -t 40000 *.js",
     "ft_s3curl": "cd tests/functional/s3curl && mocha -t 40000 *.js",
     "ft_util": "cd tests/functional/utilities && mocha -t 40000 *.js",
-    "ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management ft_util",
+    "ft_metadata": "cd tests/functional/metadata && mocha -t 40000 *.js",
+    "ft_test": "npm-run-all -s ft_awssdk ft_s3cmd ft_s3curl ft_node ft_healthchecks ft_management ft_util ft_metadata",
     "ft_search": "cd tests/functional/aws-node-sdk && mocha -t 90000 test/mdSearch",
     "install_ft_deps": "npm install aws-sdk@2.28.0 bluebird@3.3.1 mocha@2.3.4 mocha-junit-reporter@1.11.1 tv4@1.2.7",
     "lint": "eslint $(git ls-files '*.js')",

tests/functional/metadata/MetadataWrapper.js

@@ -0,0 +1,208 @@
+const assert = require('assert');
+const async = require('async');
+const uuid = require('uuid/v4');
+
+const MetadataWrapper = require('arsenal').storage.metadata.MetadataWrapper;
+
+const BucketInfo = require('arsenal').models.BucketInfo;
+const MongoClient = require('mongodb').MongoClient;
+const {
+    MongoClientInterface,
+} = require('arsenal').storage.metadata.mongoclient;
+
+const log = require('./utils/fakeLogger');
+
+const replicaSetHosts = 'localhost:27018,localhost:27019,localhost:27020';
+const writeConcern = 'majority';
+const replicaSet = 'rs0';
+const readPreference = 'primary';
+// const mongoUrl = `mongodb://${replicaSetHosts}/?w=${writeConcern}&` +
+//     `replicaSet=${replicaSet}&readPreference=${readPreference}`;
+const mongoUrl = 'mongodb://127.0.0.1:27017';
+
+const VID_SEP = '\0';
+const TEST_DB = 'test';
+// const TEST_COLLECTION = 'test-collection';
+const TEST_COLLECTION = 'test-collection';
+const BUCKET_NAME = 'test-bucket';
+const OBJECT_NAME = 'test-object';
+const VERSION_ID = '98451712418844999999RG001  22019.0';
+const TAG_1 = '557b9096-f3d9-4a70-bbb9-72edc757287f';
+const TAG_2 = '3d7383a8-3d43-4370-b276-66f14352140e';
+
+const mongoClientInterface = new MongoClientInterface({
+    replicaSetHosts,
+    writeConcern,
+    replicaSet,
+    readPreference,
+    replicationGroupId: 'RG001',
+    database: TEST_DB,
+    logger: log,
+});
+
+const metadata = new MetadataWrapper('mongodb', {
+    mongodb: {
+        replicaSetHosts,
+        writeConcern,
+        replicaSet,
+        readPreference,
+        replicationGroupId: 'RG001',
+        database: TEST_DB,    
+    },
+}, undefined, log);
+
+const tag = uuid();
+const objMD = { updated: false };
+const bucketInfo = new BucketInfo(BUCKET_NAME, 'foo', 'bar', `${new Date()}`);
+
+const runIfMongo =
+    process.env.S3METADATA === 'mongodb' ? describe : describe.skip;
+    
+function jobFunc(bucket, key, tag, objVal) {
+    // TODO: Update the tag from the next read.
+    
+    // Check if the update is relevant. If it's not, return null. For example,
+    // if you find yourself updating an overwritten object, there is no need to
+    // continue.
+    // if (tag !== TAG_1) {
+    //     return { retry: false };
+    // }
+    // Perform the operation. For example, updating the storage class.
+    const valueUpdate = { updated: true };
+    return { objVal: Object.assign({}, objVal, valueUpdate) };
+}
+
+runIfMongo('MongoClientInterface', () => {
+    let mongoClient;
+    let collection;
+    let db;
+    
+    before(done => metadata.setup(done));
+
+    beforeEach(done => {
+        async.series([
+            next => MongoClient.connect(mongoUrl, {}, (err, client) => {
+                if (err) {
+                    return next(err);
+                }
+                mongoClient = client;
+                return next();
+            }),
+            next => {
+                db = mongoClient.db(TEST_DB);
+                db.createCollection(TEST_COLLECTION, next);
+            },
+            next => metadata.createBucket(BUCKET_NAME, bucketInfo, log, next),
+        ], done);
+    });
+    
+    afterEach(done => {
+        const db = mongoClient.db(TEST_DB);
+        return db.dropDatabase(err => {
+            if (err) {
+                return done(err);
+            }
+            return mongoClient.close(true, done);
+        });
+    });
+
+    it('should put object metadata when jobFunc returns', done => 
+        async.series([
+            next => {
+                const collection = db.collection(BUCKET_NAME);
+                collection.insertOne({
+                    _id: OBJECT_NAME,
+                    tag: TAG_1,
+                    value: objMD,
+                }, next);
+            },
+            next => metadata.safePutObjectMD(BUCKET_NAME, OBJECT_NAME, 
+                { jobFunc, tag }, log, next),
+            next => metadata.getObjectMD(BUCKET_NAME, OBJECT_NAME, {}, log, 
+                (err, objMD) => {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.strictEqual(objMD.updated, true);
+                    done();
+                }),
+        ], done));
+
+    it('should not put object metadata', done => 
+        async.series([
+            next => {
+                const collection = db.collection(BUCKET_NAME);
+                collection.insertOne({
+                    _id: OBJECT_NAME,
+                    tag: TAG_2,
+                    value: objMD,
+                }, next);
+            },
+            next => metadata.safePutObjectMD(BUCKET_NAME, OBJECT_NAME, 
+                { jobFunc, tag: uuid() }, log, next),
+            next => metadata.getObjectMD(BUCKET_NAME, OBJECT_NAME, {}, log, 
+                (err, objMD) => {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.strictEqual(objMD.updated, false);
+                    done();
+                }),
+        ], done));
+        
+    it.only('should put object metadata when conflicting writes', function t(done) {
+        this.timeout(30000)
+        const collection = db.collection(BUCKET_NAME);
+        function iteratee(n, callback) {
+            async.parallel([
+                next => {
+                    collection.update(
+                        { 
+                            _id: OBJECT_NAME,
+                        },
+                        {
+                            _id: OBJECT_NAME,
+                            tag: uuid(),
+                            value: objMD,
+                        }, err => {
+                            console.log('updated...');
+                            next(err);
+                        });
+                },
+                next => {
+                    metadata.safePutObjectMD(BUCKET_NAME, OBJECT_NAME, { 
+                        jobFunc, 
+                        condPut: {
+                            tag
+                        },
+                    }, log, next);
+                },
+            ], callback);
+        }
+        collection.insertOne({
+            _id: OBJECT_NAME,
+            tag: TAG_1,
+            value: objMD,
+        }, err => {
+            if (err) {
+                return done(err);
+            }
+            async.times(10, iteratee, done);
+        });
+        
+        // async.series([
+        //     next => {
+        // 
+        //     },
+        // 
+        //     next => metadata.getObjectMD(BUCKET_NAME, OBJECT_NAME, {}, log, 
+        //         (err, objMD) => {
+        //             if (err) {
+        //                 return done(err);
+        //             }
+        //             assert.strictEqual(objMD.updated, true);
+        //             done();
+        //         }),
+        // ], done)    
+    });
+});

tests/functional/metadata/MongoClientInterface.js

@@ -0,0 +1,277 @@
+const assert = require('assert');
+const async = require('async');
+
+const MongoClient = require('mongodb').MongoClient;
+const {
+    MongoClientInterface,
+} = require('arsenal').storage.metadata.mongoclient;
+
+const log = require('./utils/fakeLogger');
+
+const replicaSetHosts = 'localhost:27018,localhost:27019,localhost:27020';
+const writeConcern = 'majority';
+const replicaSet = 'rs0';
+const readPreference = 'primary';
+// const mongoUrl = `mongodb://${replicaSetHosts}/?w=${writeConcern}&` +
+//     `replicaSet=${replicaSet}&readPreference=${readPreference}`;
+const mongoUrl = 'mongodb://127.0.0.1:27017';
+
+const VID_SEP = '\0';
+const TEST_DB = 'test';
+const TEST_COLLECTION = 'test-collection';
+const BUCKET_NAME = 'test-bucket';
+const OBJECT_NAME = 'test-object';
+const VERSION_ID = '98451712418844999999RG001  22019.0';
+const TAG_1 = '557b9096-f3d9-4a70-bbb9-72edc757287f';
+const TAG_2 = '3d7383a8-3d43-4370-b276-66f14352140e';
+
+const mongoClientInterface = new MongoClientInterface({
+    replicaSetHosts,
+    writeConcern,
+    replicaSet,
+    readPreference,
+    replicationGroupId: 'RG001',
+    database: TEST_DB,
+    logger: log,
+});
+
+const objVal = {
+    key: OBJECT_NAME,
+    versionId: VERSION_ID,
+    updated: false,
+};
+
+const updatedObjVal = {
+    updated: true,
+};
+
+const runIfMongo =
+    process.env.S3METADATA === 'mongodb' ? describe : describe.skip;
+
+runIfMongo('MongoClientInterface', () => {
+    let mongoClient;
+    let collection;
+
+    function checkTag({ shouldHaveUpdated }, cb) {
+        return collection.findOne({ _id: OBJECT_NAME }, (err, result) => {
+            if (err) {
+                return cb(err);
+            }
+            if (shouldHaveUpdated) {
+                assert(result.tag !== TAG_1);
+                assert(result.value.updated);
+            } else {
+                assert(result.tag === TAG_1);
+                assert.deepStrictEqual(result.value, objVal);
+            }
+            return cb();
+        });
+    }
+
+    beforeEach(done =>
+        MongoClient.connect(mongoUrl, {}, (err, client) => {
+            if (err) {
+                return done(err);
+            }
+            mongoClient = client;
+            const db = mongoClient.db(TEST_DB);
+            return db.createCollection(TEST_COLLECTION, (err, result) => {
+                if (err) {
+                    return done(err);
+                }
+                collection = result;
+                return done();
+            });
+        }));
+
+    afterEach(done => {
+        const db = mongoClient.db(TEST_DB);
+        return db.dropDatabase(err => {
+            if (err) {
+                return done(err);
+            }
+            return mongoClient.close(true, done);
+        });
+    });
+
+    describe('::putObjectNoVer', () => {
+        beforeEach(done =>
+            collection.insertOne({
+                _id: OBJECT_NAME,
+                tag: TAG_1,
+                value: objVal,
+            }, done));
+
+        function testPutMetadata({ params, shouldHaveUpdated }, cb) {
+            async.series([
+                next => mongoClientInterface.putObjectNoVer(collection,
+                    BUCKET_NAME, OBJECT_NAME, updatedObjVal, params, log, next),
+                next => checkTag({ shouldHaveUpdated }, next),
+            ], cb);
+        }
+
+        it('should update metadata when no tag is provided', done =>
+            testPutMetadata({
+                params: {},
+                shouldHaveUpdated: true,
+            }, done));
+
+        it('should update metadata when matching tag is provided', done =>
+            testPutMetadata({
+                params: {
+                    condPut: {
+                        tag: TAG_1,
+                    },
+                },
+                shouldHaveUpdated: true,
+            }, done));
+
+        it('should not update metadata when non-matching tag is provided',
+            done =>
+                testPutMetadata({
+                    params: {
+                        condPut: {
+                            tag: 'non-matching-tag',
+                        },
+                    },
+                    shouldHaveUpdated: false,
+                }, done));
+    });
+
+    describe('::putObjectVerCase2', () => {
+        beforeEach(done => {
+            collection.insertOne({
+                _id: OBJECT_NAME,
+                tag: TAG_1,
+                value: objVal,
+            }, done);
+        });
+
+        function testPutMetadata({ params, shouldHaveUpdated }, cb) {
+            async.series([
+                next => mongoClientInterface.putObjectVerCase2(collection,
+                    BUCKET_NAME, OBJECT_NAME, updatedObjVal, params, log, next),
+                next => checkTag({ shouldHaveUpdated }, next),
+            ], cb);
+        }
+
+        it('should update metadata when no tag is provided', done =>
+            testPutMetadata({
+                params: {},
+                shouldHaveUpdated: true,
+            }, done));
+
+        it('should update metadata when matching tag is provided', done =>
+            testPutMetadata({
+                params: {
+                    condPut: {
+                        tag: TAG_1,
+                    },
+                },
+                shouldHaveUpdated: true,
+            }, done));
+
+        it('should not update metadata when non-matching tag is provided',
+            done =>
+                testPutMetadata({
+                    params: {
+                        condPut: {
+                            tag: 'non-matching-tag',
+                        },
+                    },
+                    shouldHaveUpdated: false,
+                }, done));
+    });
+
+    describe('::putObjectVerCase3', () => {
+        const vObjName = `${OBJECT_NAME}${VID_SEP}${VERSION_ID}`;
+
+        beforeEach(done => {
+            async.series([
+                next => collection.insertOne({
+                    _id: vObjName,
+                    tag: TAG_1,
+                    value: objVal,
+                }, next),
+                next => collection.insertOne({
+                    _id: OBJECT_NAME,
+                    tag: TAG_2,
+                    value: objVal,
+                }, next),
+            ], done);
+        });
+
+        function testPutMetadata({ params, shouldHaveUpdated }, cb) {
+            async.series([
+                next => mongoClientInterface.putObjectVerCase3(collection,
+                    BUCKET_NAME, OBJECT_NAME, updatedObjVal, params, log, next),
+                next => async.series([
+                    done =>
+                        collection.findOne({
+                            _id: vObjName,
+                        }, (err, result) => {
+                            if (err) {
+                                return cb(err);
+                            }
+                            if (shouldHaveUpdated) {
+                                assert(result.tag !== TAG_1);
+                                assert(result.value.updated);
+                            } else {
+                                assert(result.tag === TAG_1);
+                                assert.deepStrictEqual(result.value, objVal);
+                            }
+                            return done();
+                        }),
+                    done =>
+                        collection.findOne({
+                            _id: OBJECT_NAME,
+                        }, (err, result) => {
+                            if (err) {
+                                return cb(err);
+                            }
+                            if (shouldHaveUpdated) {
+                                assert(result.tag !== TAG_2);
+                                assert(result.value.updated);
+                            } else {
+                                assert(result.tag === TAG_2);
+                                assert.deepStrictEqual(result.value, objVal);
+                            }
+                            return done();
+                        }),
+                ], next),
+            ], cb);
+        }
+
+        it('should update metadata when no tag is provided', done => {
+            testPutMetadata({
+                params: {
+                    versionId: VERSION_ID,
+                },
+                shouldHaveUpdated: true,
+            }, done);
+        });
+
+        it('should update metadata when matching tag is provided', done =>
+            testPutMetadata({
+                params: {
+                    condPut: {
+                        tag: TAG_2,
+                    },
+                    versionId: VERSION_ID,
+                },
+                shouldHaveUpdated: true,
+            }, done));
+
+        it('should not update metadata when non-matching tag is provided',
+            done =>
+                testPutMetadata({
+                    params: {
+                        condPut: {
+                            tag: 'non-matching-tag',
+                        },
+                        versionId: VERSION_ID,
+                    },
+                    shouldHaveUpdated: false,
+                }, done));
+    });
+});

tests/functional/metadata/utils/fakeLogger.js

@@ -0,0 +1,9 @@
+const fakeLogger = {
+    trace: () => {},
+    error: () => {},
+    info: () => {},
+    debug: () => {},
+    getSerializedUids: () => {},
+};
+
+module.exports = fakeLogger;