Compare commits
10 Commits
2ec6808e91
...
c55c4abe6c
Author | SHA1 | Date |
---|---|---|
Vianney Rancurel | c55c4abe6c | |
Taylor McKinnon | 13f33a81a6 | |
bbuchanan9 | 08c1a2046d | |
bbuchanan9 | ab239ffa54 | |
bbuchanan9 | ad1ee70c4e | |
bbuchanan9 | 61bb75b276 | |
Dora Korpar | 52bbe85463 | |
Rahul Padigela | 8921a0d9c7 | |
Electra Chong | 10a99a5f44 | |
Electra Chong | d053da3f6c |
7
index.js
7
index.js
|
@ -106,6 +106,13 @@ module.exports = {
|
|||
require('./lib/storage/metadata/file/MetadataFileClient'),
|
||||
LogConsumer:
|
||||
require('./lib/storage/metadata/bucketclient/LogConsumer'),
|
||||
inMemory: {
|
||||
metastore:
|
||||
require('./lib/storage/metadata/in_memory/metastore'),
|
||||
metadata: require('./lib/storage/metadata/in_memory/metadata'),
|
||||
bucketUtilities:
|
||||
require('./lib/storage/metadata/in_memory/bucket_utilities'),
|
||||
},
|
||||
},
|
||||
data: {
|
||||
file: {
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
module.exports = {
|
||||
Basic: require('./basic').List,
|
||||
Delimiter: require('./delimiter').Delimiter,
|
||||
DelimiterVersions: require('./delimiterVersions')
|
||||
.DelimiterVersions,
|
||||
DelimiterMaster: require('./delimiterMaster')
|
||||
.DelimiterMaster,
|
||||
MPU: require('./MPU').MultipartUploads,
|
||||
};
|
|
@ -3,8 +3,6 @@ const errors = require('../../errors');
|
|||
const BucketInfo = require('../../models/BucketInfo');
|
||||
|
||||
const BucketClientInterface = require('./bucketclient/BucketClientInterface');
|
||||
const BucketFileInterface = require('./file/BucketFileInterface');
|
||||
const MongoClientInterface = require('./mongoclient/MongoClientInterface');
|
||||
const metastore = require('./in_memory/metastore');
|
||||
|
||||
let CdmiMetadata;
|
||||
|
@ -71,25 +69,10 @@ class MetadataWrapper {
|
|||
if (clientName === 'mem') {
|
||||
this.client = metastore;
|
||||
this.implName = 'memorybucket';
|
||||
} else if (clientName === 'file') {
|
||||
this.client = new BucketFileInterface(params, logger);
|
||||
this.implName = 'bucketfile';
|
||||
} else if (clientName === 'scality') {
|
||||
this.client = new BucketClientInterface(params, bucketclient,
|
||||
logger);
|
||||
this.implName = 'bucketclient';
|
||||
} else if (clientName === 'mongodb') {
|
||||
this.client = new MongoClientInterface({
|
||||
replicaSetHosts: params.mongodb.replicaSetHosts,
|
||||
writeConcern: params.mongodb.writeConcern,
|
||||
replicaSet: params.mongodb.replicaSet,
|
||||
readPreference: params.mongodb.readPreference,
|
||||
database: params.mongodb.database,
|
||||
replicationGroupId: params.replicationGroupId,
|
||||
path: params.mongodb.path,
|
||||
logger,
|
||||
});
|
||||
this.implName = 'mongoclient';
|
||||
} else if (clientName === 'cdmi') {
|
||||
if (!CdmiMetadata) {
|
||||
throw new Error('Unauthorized backend');
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import ListResult from './ListResult';
|
||||
const ListResult = require('./ListResult');
|
||||
|
||||
export class ListMultipartUploadsResult extends ListResult {
|
||||
class ListMultipartUploadsResult extends ListResult {
|
||||
constructor() {
|
||||
super();
|
||||
this.Uploads = [];
|
||||
|
@ -28,3 +28,7 @@ export class ListMultipartUploadsResult extends ListResult {
|
|||
this.MaxKeys += 1;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
ListMultipartUploadsResult,
|
||||
};
|
||||
|
|
|
@ -24,4 +24,4 @@ class ListResult {
|
|||
}
|
||||
}
|
||||
|
||||
export default ListResult;
|
||||
module.exports = ListResult;
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
export function markerFilterMPU(allMarkers, array) {
|
||||
function markerFilterMPU(allMarkers, array) {
|
||||
const { keyMarker, uploadIdMarker } = allMarkers;
|
||||
for (let i = 0; i < array.length; i++) {
|
||||
// If the keyMarker is the same as the key,
|
||||
|
@ -30,7 +30,7 @@ export function markerFilterMPU(allMarkers, array) {
|
|||
return array;
|
||||
}
|
||||
|
||||
export function prefixFilter(prefix, array) {
|
||||
function prefixFilter(prefix, array) {
|
||||
for (let i = 0; i < array.length; i++) {
|
||||
if (array[i].indexOf(prefix) !== 0) {
|
||||
array.splice(i, 1);
|
||||
|
@ -40,6 +40,12 @@ export function prefixFilter(prefix, array) {
|
|||
return array;
|
||||
}
|
||||
|
||||
export function isKeyInContents(responseObject, key) {
|
||||
function isKeyInContents(responseObject, key) {
|
||||
return responseObject.Contents.some(val => val.key === key);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
markerFilterMPU,
|
||||
prefixFilter,
|
||||
isKeyInContents,
|
||||
};
|
||||
|
|
|
@ -1,12 +1,11 @@
|
|||
import { errors } from 'arsenal';
|
||||
const errors = require('../../../errors');
|
||||
|
||||
import { markerFilterMPU, prefixFilter }
|
||||
from './bucket_utilities';
|
||||
import { ListMultipartUploadsResult } from './ListMultipartUploadsResult';
|
||||
import { metadata } from './metadata';
|
||||
const { markerFilterMPU, prefixFilter } = require('./bucket_utilities');
|
||||
const { ListMultipartUploadsResult } = require('./ListMultipartUploadsResult');
|
||||
const { metadata } = require('./metadata');
|
||||
|
||||
const defaultMaxKeys = 1000;
|
||||
export default function getMultipartUploadListing(bucket, params, callback) {
|
||||
function getMultipartUploadListing(bucket, params, callback) {
|
||||
const { delimiter, keyMarker,
|
||||
uploadIdMarker, prefix, queryPrefixLength, splitter } = params;
|
||||
const splitterLen = splitter.length;
|
||||
|
@ -145,3 +144,5 @@ export default function getMultipartUploadListing(bucket, params, callback) {
|
|||
response.IsTruncated = maxKeys === 0 ? false : response.IsTruncated;
|
||||
return callback(null, response);
|
||||
}
|
||||
|
||||
module.exports = getMultipartUploadListing;
|
||||
|
|
|
@ -3,4 +3,6 @@ const metadata = {
|
|||
keyMaps: new Map,
|
||||
};
|
||||
|
||||
export { metadata };
|
||||
module.exports = {
|
||||
metadata,
|
||||
};
|
||||
|
|
|
@ -1,16 +1,15 @@
|
|||
import { errors, algorithms, versioning } from 'arsenal';
|
||||
|
||||
import getMultipartUploadListing from './getMultipartUploadListing';
|
||||
import { metadata } from './metadata';
|
||||
import config from '../../Config';
|
||||
|
||||
const genVID = versioning.VersionID.generateVersionId;
|
||||
const errors = require('../../../errors');
|
||||
const list = require('../../../algos/list/exportAlgos');
|
||||
const genVID =
|
||||
require('../../../versioning/VersionID').generateVersionId;
|
||||
const getMultipartUploadListing = require('./getMultipartUploadListing');
|
||||
const { metadata } = require('./metadata');
|
||||
|
||||
const defaultMaxKeys = 1000;
|
||||
let uidCounter = 0;
|
||||
|
||||
function generateVersionId() {
|
||||
return genVID(uidCounter++, config.replicationGroupId);
|
||||
return genVID(uidCounter++, undefined);
|
||||
}
|
||||
|
||||
function formatVersionKey(key, versionId) {
|
||||
|
@ -79,10 +78,35 @@ const metastore = {
|
|||
|
||||
putObject: (bucketName, objName, objVal, params, log, cb) => {
|
||||
process.nextTick(() => {
|
||||
metastore.getBucketAttributes(bucketName, log, err => {
|
||||
// Ignore the PUT done by AbortMPU
|
||||
if (params && params.isAbort) {
|
||||
return cb(null);
|
||||
}
|
||||
return metastore.getBucketAttributes(bucketName, log, err => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
/*
|
||||
valid combinations of versioning options:
|
||||
- !versioning && !versionId: normal non-versioning put
|
||||
- versioning && !versionId: create a new version
|
||||
- versionId: update (PUT/DELETE) an existing version,
|
||||
and also update master version in case the put
|
||||
version is newer or same version than master.
|
||||
if versionId === '' update master version
|
||||
*/
|
||||
|
||||
if (params && params.versionId) {
|
||||
objVal.versionId = params.versionId; // eslint-disable-line
|
||||
const mst = metadata.keyMaps.get(bucketName).get(objName);
|
||||
if (mst && mst.versionId === params.versionId || !mst) {
|
||||
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
||||
}
|
||||
// eslint-disable-next-line
|
||||
objName = formatVersionKey(objName, params.versionId);
|
||||
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
||||
return cb(null, `{"versionId":"${objVal.versionId}"}`);
|
||||
}
|
||||
if (params && params.versioning) {
|
||||
const versionId = generateVersionId();
|
||||
objVal.versionId = versionId; // eslint-disable-line
|
||||
|
@ -97,16 +121,6 @@ const metastore = {
|
|||
objVal.versionId = versionId; // eslint-disable-line
|
||||
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
||||
return cb(null, `{"versionId":"${objVal.versionId}"}`);
|
||||
} else if (params && params.versionId) {
|
||||
objVal.versionId = params.versionId; // eslint-disable-line
|
||||
const mst = metadata.keyMaps.get(bucketName).get(objName);
|
||||
if (mst && mst.versionId === params.versionId) {
|
||||
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
||||
}
|
||||
// eslint-disable-next-line
|
||||
objName = formatVersionKey(objName, params.versionId);
|
||||
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
||||
return cb(null, `{"versionId":"${objVal.versionId}"}`);
|
||||
}
|
||||
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
||||
return cb(null);
|
||||
|
@ -205,7 +219,14 @@ const metastore = {
|
|||
|
||||
listObject(bucketName, params, log, cb) {
|
||||
process.nextTick(() => {
|
||||
const { prefix, marker, delimiter, maxKeys } = params;
|
||||
const {
|
||||
prefix,
|
||||
marker,
|
||||
delimiter,
|
||||
maxKeys,
|
||||
continuationToken,
|
||||
startAfter,
|
||||
} = params;
|
||||
if (prefix && typeof prefix !== 'string') {
|
||||
return cb(errors.InvalidArgument);
|
||||
}
|
||||
|
@ -222,6 +243,14 @@ const metastore = {
|
|||
return cb(errors.InvalidArgument);
|
||||
}
|
||||
|
||||
if (continuationToken && typeof continuationToken !== 'string') {
|
||||
return cb(errors.InvalidArgument);
|
||||
}
|
||||
|
||||
if (startAfter && typeof startAfter !== 'string') {
|
||||
return cb(errors.InvalidArgument);
|
||||
}
|
||||
|
||||
// If paramMaxKeys is undefined, the default parameter will set it.
|
||||
// However, if it is null, the default parameter will not set it.
|
||||
let numKeys = maxKeys;
|
||||
|
@ -235,8 +264,8 @@ const metastore = {
|
|||
|
||||
// If marker specified, edit the keys array so it
|
||||
// only contains keys that occur alphabetically after the marker
|
||||
const listingType = params.listingType || 'Delimiter';
|
||||
const extension = new algorithms.list[listingType](params, log);
|
||||
const listingType = params.listingType;
|
||||
const extension = new list[listingType](params, log);
|
||||
const listingParams = extension.genMDParams();
|
||||
|
||||
const keys = [];
|
||||
|
@ -302,4 +331,4 @@ const metastore = {
|
|||
},
|
||||
};
|
||||
|
||||
export default metastore;
|
||||
module.exports = metastore;
|
|
@ -3,7 +3,7 @@
|
|||
"engines": {
|
||||
"node": ">=16"
|
||||
},
|
||||
"version": "7.10.13",
|
||||
"version": "7.10.14",
|
||||
"description": "Common utilities for the S3 project components",
|
||||
"main": "index.js",
|
||||
"repository": {
|
||||
|
|
|
@ -0,0 +1,26 @@
|
|||
{
|
||||
"acl": {
|
||||
"Canned": "private",
|
||||
"FULL_CONTROL": [],
|
||||
"WRITE": [],
|
||||
"WRITE_ACP": [],
|
||||
"READ": [],
|
||||
"READ_ACP": []
|
||||
},
|
||||
"name": "BucketName",
|
||||
"owner": "9d8fe19a78974c56dceb2ea4a8f01ed0f5fecb9d29f80e9e3b84104e4a3ea520",
|
||||
"ownerDisplayName": "anonymousCoward",
|
||||
"creationDate": "2018-06-04T17:45:42.592Z",
|
||||
"mdBucketModelVersion": 8,
|
||||
"transient": false,
|
||||
"deleted": false,
|
||||
"serverSideEncryption": null,
|
||||
"versioningConfiguration": null,
|
||||
"websiteConfiguration": null,
|
||||
"locationConstraint": "us-east-1",
|
||||
"readLocationConstraint": "us-east-1",
|
||||
"cors": null,
|
||||
"replicationConfiguration": null,
|
||||
"lifecycleConfiguration": null,
|
||||
"uid": "fea97818-6a9a-11e8-9777-e311618cc5d4"
|
||||
}
|
|
@ -0,0 +1,59 @@
|
|||
const async = require('async');
|
||||
const assert = require('assert');
|
||||
const werelogs = require('werelogs');
|
||||
const MetadataWrapper = require('../../../../../lib/storage/metadata/MetadataWrapper');
|
||||
const fakeBucketInfo = require('./FakeBucketInfo.json');
|
||||
|
||||
describe('InMemory', () => {
|
||||
const fakeBucket = 'fake';
|
||||
const logger = new werelogs.Logger('Injector');
|
||||
const memBackend = new MetadataWrapper(
|
||||
'mem', {}, null, logger);
|
||||
|
||||
before(done => {
|
||||
memBackend.createBucket(fakeBucket, fakeBucketInfo, logger, done);
|
||||
});
|
||||
|
||||
after(done => {
|
||||
memBackend.deleteBucket(fakeBucket, logger, done);
|
||||
});
|
||||
|
||||
it('basic', done => {
|
||||
async.waterfall([
|
||||
next => {
|
||||
memBackend.putObjectMD(fakeBucket, 'foo', 'bar', {}, logger, err => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
return next();
|
||||
});
|
||||
},
|
||||
next => {
|
||||
memBackend.getObjectMD(fakeBucket, 'foo', {}, logger, (err, data) => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
assert.deepEqual(data, 'bar');
|
||||
return next();
|
||||
});
|
||||
},
|
||||
next => {
|
||||
memBackend.deleteObjectMD(fakeBucket, 'foo', {}, logger, err => {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
return next();
|
||||
});
|
||||
},
|
||||
next => {
|
||||
memBackend.getObjectMD(fakeBucket, 'foo', {}, logger, err => {
|
||||
if (err) {
|
||||
assert.deepEqual(err.message, 'NoSuchKey');
|
||||
return next();
|
||||
}
|
||||
return next(new Error('unexpected success'));
|
||||
});
|
||||
},
|
||||
], done);
|
||||
});
|
||||
});
|
Loading…
Reference in New Issue