Compare commits
No commits in common. "c55c4abe6c2807356bcf563fc5fec01ffba89343" and "2ec6808e91218d5c320c3762350f7a57b68369f3" have entirely different histories.
c55c4abe6c
...
2ec6808e91
7
index.js
7
index.js
|
@ -106,13 +106,6 @@ module.exports = {
|
||||||
require('./lib/storage/metadata/file/MetadataFileClient'),
|
require('./lib/storage/metadata/file/MetadataFileClient'),
|
||||||
LogConsumer:
|
LogConsumer:
|
||||||
require('./lib/storage/metadata/bucketclient/LogConsumer'),
|
require('./lib/storage/metadata/bucketclient/LogConsumer'),
|
||||||
inMemory: {
|
|
||||||
metastore:
|
|
||||||
require('./lib/storage/metadata/in_memory/metastore'),
|
|
||||||
metadata: require('./lib/storage/metadata/in_memory/metadata'),
|
|
||||||
bucketUtilities:
|
|
||||||
require('./lib/storage/metadata/in_memory/bucket_utilities'),
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
data: {
|
data: {
|
||||||
file: {
|
file: {
|
||||||
|
|
|
@ -1,9 +0,0 @@
|
||||||
module.exports = {
|
|
||||||
Basic: require('./basic').List,
|
|
||||||
Delimiter: require('./delimiter').Delimiter,
|
|
||||||
DelimiterVersions: require('./delimiterVersions')
|
|
||||||
.DelimiterVersions,
|
|
||||||
DelimiterMaster: require('./delimiterMaster')
|
|
||||||
.DelimiterMaster,
|
|
||||||
MPU: require('./MPU').MultipartUploads,
|
|
||||||
};
|
|
|
@ -3,6 +3,8 @@ const errors = require('../../errors');
|
||||||
const BucketInfo = require('../../models/BucketInfo');
|
const BucketInfo = require('../../models/BucketInfo');
|
||||||
|
|
||||||
const BucketClientInterface = require('./bucketclient/BucketClientInterface');
|
const BucketClientInterface = require('./bucketclient/BucketClientInterface');
|
||||||
|
const BucketFileInterface = require('./file/BucketFileInterface');
|
||||||
|
const MongoClientInterface = require('./mongoclient/MongoClientInterface');
|
||||||
const metastore = require('./in_memory/metastore');
|
const metastore = require('./in_memory/metastore');
|
||||||
|
|
||||||
let CdmiMetadata;
|
let CdmiMetadata;
|
||||||
|
@ -69,10 +71,25 @@ class MetadataWrapper {
|
||||||
if (clientName === 'mem') {
|
if (clientName === 'mem') {
|
||||||
this.client = metastore;
|
this.client = metastore;
|
||||||
this.implName = 'memorybucket';
|
this.implName = 'memorybucket';
|
||||||
|
} else if (clientName === 'file') {
|
||||||
|
this.client = new BucketFileInterface(params, logger);
|
||||||
|
this.implName = 'bucketfile';
|
||||||
} else if (clientName === 'scality') {
|
} else if (clientName === 'scality') {
|
||||||
this.client = new BucketClientInterface(params, bucketclient,
|
this.client = new BucketClientInterface(params, bucketclient,
|
||||||
logger);
|
logger);
|
||||||
this.implName = 'bucketclient';
|
this.implName = 'bucketclient';
|
||||||
|
} else if (clientName === 'mongodb') {
|
||||||
|
this.client = new MongoClientInterface({
|
||||||
|
replicaSetHosts: params.mongodb.replicaSetHosts,
|
||||||
|
writeConcern: params.mongodb.writeConcern,
|
||||||
|
replicaSet: params.mongodb.replicaSet,
|
||||||
|
readPreference: params.mongodb.readPreference,
|
||||||
|
database: params.mongodb.database,
|
||||||
|
replicationGroupId: params.replicationGroupId,
|
||||||
|
path: params.mongodb.path,
|
||||||
|
logger,
|
||||||
|
});
|
||||||
|
this.implName = 'mongoclient';
|
||||||
} else if (clientName === 'cdmi') {
|
} else if (clientName === 'cdmi') {
|
||||||
if (!CdmiMetadata) {
|
if (!CdmiMetadata) {
|
||||||
throw new Error('Unauthorized backend');
|
throw new Error('Unauthorized backend');
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
const ListResult = require('./ListResult');
|
import ListResult from './ListResult';
|
||||||
|
|
||||||
class ListMultipartUploadsResult extends ListResult {
|
export class ListMultipartUploadsResult extends ListResult {
|
||||||
constructor() {
|
constructor() {
|
||||||
super();
|
super();
|
||||||
this.Uploads = [];
|
this.Uploads = [];
|
||||||
|
@ -28,7 +28,3 @@ class ListMultipartUploadsResult extends ListResult {
|
||||||
this.MaxKeys += 1;
|
this.MaxKeys += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
ListMultipartUploadsResult,
|
|
||||||
};
|
|
||||||
|
|
|
@ -24,4 +24,4 @@ class ListResult {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = ListResult;
|
export default ListResult;
|
||||||
|
|
|
@ -1,15 +1,16 @@
|
||||||
const errors = require('../../../errors');
|
import { errors, algorithms, versioning } from 'arsenal';
|
||||||
const list = require('../../../algos/list/exportAlgos');
|
|
||||||
const genVID =
|
import getMultipartUploadListing from './getMultipartUploadListing';
|
||||||
require('../../../versioning/VersionID').generateVersionId;
|
import { metadata } from './metadata';
|
||||||
const getMultipartUploadListing = require('./getMultipartUploadListing');
|
import config from '../../Config';
|
||||||
const { metadata } = require('./metadata');
|
|
||||||
|
const genVID = versioning.VersionID.generateVersionId;
|
||||||
|
|
||||||
const defaultMaxKeys = 1000;
|
const defaultMaxKeys = 1000;
|
||||||
let uidCounter = 0;
|
let uidCounter = 0;
|
||||||
|
|
||||||
function generateVersionId() {
|
function generateVersionId() {
|
||||||
return genVID(uidCounter++, undefined);
|
return genVID(uidCounter++, config.replicationGroupId);
|
||||||
}
|
}
|
||||||
|
|
||||||
function formatVersionKey(key, versionId) {
|
function formatVersionKey(key, versionId) {
|
||||||
|
@ -78,35 +79,10 @@ const metastore = {
|
||||||
|
|
||||||
putObject: (bucketName, objName, objVal, params, log, cb) => {
|
putObject: (bucketName, objName, objVal, params, log, cb) => {
|
||||||
process.nextTick(() => {
|
process.nextTick(() => {
|
||||||
// Ignore the PUT done by AbortMPU
|
metastore.getBucketAttributes(bucketName, log, err => {
|
||||||
if (params && params.isAbort) {
|
|
||||||
return cb(null);
|
|
||||||
}
|
|
||||||
return metastore.getBucketAttributes(bucketName, log, err => {
|
|
||||||
if (err) {
|
if (err) {
|
||||||
return cb(err);
|
return cb(err);
|
||||||
}
|
}
|
||||||
/*
|
|
||||||
valid combinations of versioning options:
|
|
||||||
- !versioning && !versionId: normal non-versioning put
|
|
||||||
- versioning && !versionId: create a new version
|
|
||||||
- versionId: update (PUT/DELETE) an existing version,
|
|
||||||
and also update master version in case the put
|
|
||||||
version is newer or same version than master.
|
|
||||||
if versionId === '' update master version
|
|
||||||
*/
|
|
||||||
|
|
||||||
if (params && params.versionId) {
|
|
||||||
objVal.versionId = params.versionId; // eslint-disable-line
|
|
||||||
const mst = metadata.keyMaps.get(bucketName).get(objName);
|
|
||||||
if (mst && mst.versionId === params.versionId || !mst) {
|
|
||||||
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
|
||||||
}
|
|
||||||
// eslint-disable-next-line
|
|
||||||
objName = formatVersionKey(objName, params.versionId);
|
|
||||||
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
|
||||||
return cb(null, `{"versionId":"${objVal.versionId}"}`);
|
|
||||||
}
|
|
||||||
if (params && params.versioning) {
|
if (params && params.versioning) {
|
||||||
const versionId = generateVersionId();
|
const versionId = generateVersionId();
|
||||||
objVal.versionId = versionId; // eslint-disable-line
|
objVal.versionId = versionId; // eslint-disable-line
|
||||||
|
@ -121,6 +97,16 @@ const metastore = {
|
||||||
objVal.versionId = versionId; // eslint-disable-line
|
objVal.versionId = versionId; // eslint-disable-line
|
||||||
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
||||||
return cb(null, `{"versionId":"${objVal.versionId}"}`);
|
return cb(null, `{"versionId":"${objVal.versionId}"}`);
|
||||||
|
} else if (params && params.versionId) {
|
||||||
|
objVal.versionId = params.versionId; // eslint-disable-line
|
||||||
|
const mst = metadata.keyMaps.get(bucketName).get(objName);
|
||||||
|
if (mst && mst.versionId === params.versionId) {
|
||||||
|
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
||||||
|
}
|
||||||
|
// eslint-disable-next-line
|
||||||
|
objName = formatVersionKey(objName, params.versionId);
|
||||||
|
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
||||||
|
return cb(null, `{"versionId":"${objVal.versionId}"}`);
|
||||||
}
|
}
|
||||||
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
metadata.keyMaps.get(bucketName).set(objName, objVal);
|
||||||
return cb(null);
|
return cb(null);
|
||||||
|
@ -219,14 +205,7 @@ const metastore = {
|
||||||
|
|
||||||
listObject(bucketName, params, log, cb) {
|
listObject(bucketName, params, log, cb) {
|
||||||
process.nextTick(() => {
|
process.nextTick(() => {
|
||||||
const {
|
const { prefix, marker, delimiter, maxKeys } = params;
|
||||||
prefix,
|
|
||||||
marker,
|
|
||||||
delimiter,
|
|
||||||
maxKeys,
|
|
||||||
continuationToken,
|
|
||||||
startAfter,
|
|
||||||
} = params;
|
|
||||||
if (prefix && typeof prefix !== 'string') {
|
if (prefix && typeof prefix !== 'string') {
|
||||||
return cb(errors.InvalidArgument);
|
return cb(errors.InvalidArgument);
|
||||||
}
|
}
|
||||||
|
@ -243,14 +222,6 @@ const metastore = {
|
||||||
return cb(errors.InvalidArgument);
|
return cb(errors.InvalidArgument);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (continuationToken && typeof continuationToken !== 'string') {
|
|
||||||
return cb(errors.InvalidArgument);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (startAfter && typeof startAfter !== 'string') {
|
|
||||||
return cb(errors.InvalidArgument);
|
|
||||||
}
|
|
||||||
|
|
||||||
// If paramMaxKeys is undefined, the default parameter will set it.
|
// If paramMaxKeys is undefined, the default parameter will set it.
|
||||||
// However, if it is null, the default parameter will not set it.
|
// However, if it is null, the default parameter will not set it.
|
||||||
let numKeys = maxKeys;
|
let numKeys = maxKeys;
|
||||||
|
@ -264,8 +235,8 @@ const metastore = {
|
||||||
|
|
||||||
// If marker specified, edit the keys array so it
|
// If marker specified, edit the keys array so it
|
||||||
// only contains keys that occur alphabetically after the marker
|
// only contains keys that occur alphabetically after the marker
|
||||||
const listingType = params.listingType;
|
const listingType = params.listingType || 'Delimiter';
|
||||||
const extension = new list[listingType](params, log);
|
const extension = new algorithms.list[listingType](params, log);
|
||||||
const listingParams = extension.genMDParams();
|
const listingParams = extension.genMDParams();
|
||||||
|
|
||||||
const keys = [];
|
const keys = [];
|
||||||
|
@ -331,4 +302,4 @@ const metastore = {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = metastore;
|
export default metastore;
|
|
@ -1,4 +1,4 @@
|
||||||
function markerFilterMPU(allMarkers, array) {
|
export function markerFilterMPU(allMarkers, array) {
|
||||||
const { keyMarker, uploadIdMarker } = allMarkers;
|
const { keyMarker, uploadIdMarker } = allMarkers;
|
||||||
for (let i = 0; i < array.length; i++) {
|
for (let i = 0; i < array.length; i++) {
|
||||||
// If the keyMarker is the same as the key,
|
// If the keyMarker is the same as the key,
|
||||||
|
@ -30,7 +30,7 @@ function markerFilterMPU(allMarkers, array) {
|
||||||
return array;
|
return array;
|
||||||
}
|
}
|
||||||
|
|
||||||
function prefixFilter(prefix, array) {
|
export function prefixFilter(prefix, array) {
|
||||||
for (let i = 0; i < array.length; i++) {
|
for (let i = 0; i < array.length; i++) {
|
||||||
if (array[i].indexOf(prefix) !== 0) {
|
if (array[i].indexOf(prefix) !== 0) {
|
||||||
array.splice(i, 1);
|
array.splice(i, 1);
|
||||||
|
@ -40,12 +40,6 @@ function prefixFilter(prefix, array) {
|
||||||
return array;
|
return array;
|
||||||
}
|
}
|
||||||
|
|
||||||
function isKeyInContents(responseObject, key) {
|
export function isKeyInContents(responseObject, key) {
|
||||||
return responseObject.Contents.some(val => val.key === key);
|
return responseObject.Contents.some(val => val.key === key);
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
markerFilterMPU,
|
|
||||||
prefixFilter,
|
|
||||||
isKeyInContents,
|
|
||||||
};
|
|
||||||
|
|
|
@ -1,11 +1,12 @@
|
||||||
const errors = require('../../../errors');
|
import { errors } from 'arsenal';
|
||||||
|
|
||||||
const { markerFilterMPU, prefixFilter } = require('./bucket_utilities');
|
import { markerFilterMPU, prefixFilter }
|
||||||
const { ListMultipartUploadsResult } = require('./ListMultipartUploadsResult');
|
from './bucket_utilities';
|
||||||
const { metadata } = require('./metadata');
|
import { ListMultipartUploadsResult } from './ListMultipartUploadsResult';
|
||||||
|
import { metadata } from './metadata';
|
||||||
|
|
||||||
const defaultMaxKeys = 1000;
|
const defaultMaxKeys = 1000;
|
||||||
function getMultipartUploadListing(bucket, params, callback) {
|
export default function getMultipartUploadListing(bucket, params, callback) {
|
||||||
const { delimiter, keyMarker,
|
const { delimiter, keyMarker,
|
||||||
uploadIdMarker, prefix, queryPrefixLength, splitter } = params;
|
uploadIdMarker, prefix, queryPrefixLength, splitter } = params;
|
||||||
const splitterLen = splitter.length;
|
const splitterLen = splitter.length;
|
||||||
|
@ -144,5 +145,3 @@ function getMultipartUploadListing(bucket, params, callback) {
|
||||||
response.IsTruncated = maxKeys === 0 ? false : response.IsTruncated;
|
response.IsTruncated = maxKeys === 0 ? false : response.IsTruncated;
|
||||||
return callback(null, response);
|
return callback(null, response);
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = getMultipartUploadListing;
|
|
||||||
|
|
|
@ -3,6 +3,4 @@ const metadata = {
|
||||||
keyMaps: new Map,
|
keyMaps: new Map,
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = {
|
export { metadata };
|
||||||
metadata,
|
|
||||||
};
|
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=16"
|
"node": ">=16"
|
||||||
},
|
},
|
||||||
"version": "7.10.14",
|
"version": "7.10.13",
|
||||||
"description": "Common utilities for the S3 project components",
|
"description": "Common utilities for the S3 project components",
|
||||||
"main": "index.js",
|
"main": "index.js",
|
||||||
"repository": {
|
"repository": {
|
||||||
|
|
|
@ -1,26 +0,0 @@
|
||||||
{
|
|
||||||
"acl": {
|
|
||||||
"Canned": "private",
|
|
||||||
"FULL_CONTROL": [],
|
|
||||||
"WRITE": [],
|
|
||||||
"WRITE_ACP": [],
|
|
||||||
"READ": [],
|
|
||||||
"READ_ACP": []
|
|
||||||
},
|
|
||||||
"name": "BucketName",
|
|
||||||
"owner": "9d8fe19a78974c56dceb2ea4a8f01ed0f5fecb9d29f80e9e3b84104e4a3ea520",
|
|
||||||
"ownerDisplayName": "anonymousCoward",
|
|
||||||
"creationDate": "2018-06-04T17:45:42.592Z",
|
|
||||||
"mdBucketModelVersion": 8,
|
|
||||||
"transient": false,
|
|
||||||
"deleted": false,
|
|
||||||
"serverSideEncryption": null,
|
|
||||||
"versioningConfiguration": null,
|
|
||||||
"websiteConfiguration": null,
|
|
||||||
"locationConstraint": "us-east-1",
|
|
||||||
"readLocationConstraint": "us-east-1",
|
|
||||||
"cors": null,
|
|
||||||
"replicationConfiguration": null,
|
|
||||||
"lifecycleConfiguration": null,
|
|
||||||
"uid": "fea97818-6a9a-11e8-9777-e311618cc5d4"
|
|
||||||
}
|
|
|
@ -1,59 +0,0 @@
|
||||||
const async = require('async');
|
|
||||||
const assert = require('assert');
|
|
||||||
const werelogs = require('werelogs');
|
|
||||||
const MetadataWrapper = require('../../../../../lib/storage/metadata/MetadataWrapper');
|
|
||||||
const fakeBucketInfo = require('./FakeBucketInfo.json');
|
|
||||||
|
|
||||||
describe('InMemory', () => {
|
|
||||||
const fakeBucket = 'fake';
|
|
||||||
const logger = new werelogs.Logger('Injector');
|
|
||||||
const memBackend = new MetadataWrapper(
|
|
||||||
'mem', {}, null, logger);
|
|
||||||
|
|
||||||
before(done => {
|
|
||||||
memBackend.createBucket(fakeBucket, fakeBucketInfo, logger, done);
|
|
||||||
});
|
|
||||||
|
|
||||||
after(done => {
|
|
||||||
memBackend.deleteBucket(fakeBucket, logger, done);
|
|
||||||
});
|
|
||||||
|
|
||||||
it('basic', done => {
|
|
||||||
async.waterfall([
|
|
||||||
next => {
|
|
||||||
memBackend.putObjectMD(fakeBucket, 'foo', 'bar', {}, logger, err => {
|
|
||||||
if (err) {
|
|
||||||
return next(err);
|
|
||||||
}
|
|
||||||
return next();
|
|
||||||
});
|
|
||||||
},
|
|
||||||
next => {
|
|
||||||
memBackend.getObjectMD(fakeBucket, 'foo', {}, logger, (err, data) => {
|
|
||||||
if (err) {
|
|
||||||
return next(err);
|
|
||||||
}
|
|
||||||
assert.deepEqual(data, 'bar');
|
|
||||||
return next();
|
|
||||||
});
|
|
||||||
},
|
|
||||||
next => {
|
|
||||||
memBackend.deleteObjectMD(fakeBucket, 'foo', {}, logger, err => {
|
|
||||||
if (err) {
|
|
||||||
return next(err);
|
|
||||||
}
|
|
||||||
return next();
|
|
||||||
});
|
|
||||||
},
|
|
||||||
next => {
|
|
||||||
memBackend.getObjectMD(fakeBucket, 'foo', {}, logger, err => {
|
|
||||||
if (err) {
|
|
||||||
assert.deepEqual(err.message, 'NoSuchKey');
|
|
||||||
return next();
|
|
||||||
}
|
|
||||||
return next(new Error('unexpected success'));
|
|
||||||
});
|
|
||||||
},
|
|
||||||
], done);
|
|
||||||
});
|
|
||||||
});
|
|
Loading…
Reference in New Issue