Compare commits

...

4 Commits

Author SHA1 Message Date
philipyoo 56aeb047c4 test change 2019-03-09 22:05:59 -08:00
philipyoo 66efbcdda4 test no change 2019-03-09 22:05:20 -08:00
philipyoo a31a944089 test package update 2019-03-06 17:20:27 -08:00
philipyoo 45321b0db7 test 2019-03-06 17:19:37 -08:00
3 changed files with 33 additions and 79 deletions

View File

@ -1,64 +1,8 @@
const s3config = require('../../../Config').config; const s3config = require('../../../Config').config;
const { isBackbeatUser } = require('../authorization/aclChecks'); const { isBackbeatUser } = require('../authorization/aclChecks');
const { replicationBackends } = require('arsenal').constants; // const { replicationBackends } = require('arsenal').constants;
const { getReplicationInfoObject } =
function _getBackend(objectMD, site) { require('arsenal').s3middleware.replicationInfo;
const backends = objectMD ? objectMD.replicationInfo.backends : [];
const backend = backends.find(o => o.site === site);
// If the backend already exists, just update the status.
if (backend) {
return Object.assign({}, backend, { status: 'PENDING' });
}
return {
site,
status: 'PENDING',
dataStoreVersionId: '',
};
}
function _getStorageClasses(rule) {
if (rule.storageClass) {
return rule.storageClass.split(',');
}
const { replicationEndpoints } = s3config;
// If no storage class, use the given default endpoint or the sole endpoint
if (replicationEndpoints.length > 1) {
const endPoint =
replicationEndpoints.find(endpoint => endpoint.default);
return [endPoint.site];
}
return [replicationEndpoints[0].site];
}
function _getReplicationInfo(rule, replicationConfig, content, operationType,
objectMD, bucketMD) {
const storageTypes = [];
const backends = [];
const storageClasses = _getStorageClasses(rule);
storageClasses.forEach(storageClass => {
const storageClassName =
storageClass.endsWith(':preferred_read') ?
storageClass.split(':')[0] : storageClass;
const location = s3config.locationConstraints[storageClassName];
if (location && replicationBackends[location.type]) {
storageTypes.push(location.type);
}
backends.push(_getBackend(objectMD, storageClassName));
});
if (storageTypes.length > 0 && operationType) {
content.push(operationType);
}
return {
status: 'PENDING',
backends,
content,
destination: replicationConfig.destination,
storageClass: storageClasses.join(','),
role: replicationConfig.role,
storageType: storageTypes.join(','),
isNFS: bucketMD.isNFS(),
};
}
/** /**
* Get the object replicationInfo to replicate data and metadata, or only * Get the object replicationInfo to replicate data and metadata, or only
@ -95,8 +39,8 @@ function getReplicationInfo(objKey, bucketMD, isMD, objSize, operationType,
const rule = config.rules.find(rule => const rule = config.rules.find(rule =>
(objKey.startsWith(rule.prefix) && rule.enabled)); (objKey.startsWith(rule.prefix) && rule.enabled));
if (rule) { if (rule) {
return _getReplicationInfo(rule, config, content, operationType, return getReplicationInfoObject(rule, config, content,
objectMD, bucketMD); operationType, objectMD, bucketMD, s3config);
} }
} }
return undefined; return undefined;

44
package-lock.json generated
View File

@ -120,7 +120,7 @@
}, },
"ajv": { "ajv": {
"version": "4.10.0", "version": "4.10.0",
"resolved": "http://registry.npmjs.org/ajv/-/ajv-4.10.0.tgz", "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.10.0.tgz",
"integrity": "sha1-euYWkYDrGZGSqLmhn9D0f8msh2Q=", "integrity": "sha1-euYWkYDrGZGSqLmhn9D0f8msh2Q=",
"requires": { "requires": {
"co": "^4.6.0", "co": "^4.6.0",
@ -225,8 +225,8 @@
"integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog==" "integrity": "sha512-wGUIVQXuehL5TCqQun8OW81jGzAWycqzFF8lFp+GOM5BXLYj3bKNsYC4daB7n6XjCqxQA/qgTJ+8ANR3acjrog=="
}, },
"arsenal": { "arsenal": {
"version": "github:scality/arsenal#c657b4b4691161a79773230d56958b816d9a740b", "version": "github:scality/arsenal#d460d8b9d54e6e5a1571268e95fa6a257f9629ab",
"from": "github:scality/arsenal#c657b4b", "from": "github:scality/arsenal#d460d8b",
"requires": { "requires": {
"JSONStream": "^1.0.0", "JSONStream": "^1.0.0",
"ajv": "4.10.0", "ajv": "4.10.0",
@ -238,6 +238,7 @@
"debug": "~4.1.0", "debug": "~4.1.0",
"diskusage": "^1.0.0", "diskusage": "^1.0.0",
"fcntl": "github:scality/node-fcntl#9108603d8881d7762dcadfde1db927a1653dfda5", "fcntl": "github:scality/node-fcntl#9108603d8881d7762dcadfde1db927a1653dfda5",
"https-proxy-agent": "^2.2.0",
"ioctl": "2.0.1", "ioctl": "2.0.1",
"ioredis": "4.2.0", "ioredis": "4.2.0",
"ipaddr.js": "1.8.1", "ipaddr.js": "1.8.1",
@ -249,6 +250,7 @@
"simple-glob": "^0.2.0", "simple-glob": "^0.2.0",
"socket.io": "~2.2.0", "socket.io": "~2.2.0",
"socket.io-client": "~2.2.0", "socket.io-client": "~2.2.0",
"sproxydclient": "github:scality/sproxydclient#45090b76b24ca1d05482bf151ba84ff6178423d1",
"utf8": "3.0.0", "utf8": "3.0.0",
"uuid": "^3.0.1", "uuid": "^3.0.1",
"werelogs": "github:scality/werelogs#0ff7ec82f0deb1e472d8285fb7cc9ebde72c5f61", "werelogs": "github:scality/werelogs#0ff7ec82f0deb1e472d8285fb7cc9ebde72c5f61",
@ -1366,7 +1368,7 @@
"resolved": "github:scality/cdmiclient#8f0c2e6331dfa905bfe269fb4e1558d65ca0b866", "resolved": "github:scality/cdmiclient#8f0c2e6331dfa905bfe269fb4e1558d65ca0b866",
"optional": true, "optional": true,
"requires": { "requires": {
"arsenal": "github:scality/Arsenal#9fe0ba5c8cd24dad3b6fea8b937e5616a04476bc", "arsenal": "github:scality/Arsenal#87103f83e10fc7cc5ce0143654a677c526545ab1",
"async": "~1.4.2", "async": "~1.4.2",
"werelogs": "github:scality/werelogs#1a6e052fb2bdfb1c4f6bb9467dc814e79abf6e46" "werelogs": "github:scality/werelogs#1a6e052fb2bdfb1c4f6bb9467dc814e79abf6e46"
}, },
@ -1395,7 +1397,7 @@
"integrity": "sha1-8zshWfBTKj8xB6JywMz70a0peco=" "integrity": "sha1-8zshWfBTKj8xB6JywMz70a0peco="
}, },
"arsenal": { "arsenal": {
"version": "github:scality/Arsenal#9fe0ba5c8cd24dad3b6fea8b937e5616a04476bc", "version": "github:scality/Arsenal#87103f83e10fc7cc5ce0143654a677c526545ab1",
"from": "github:scality/Arsenal#development/8.0", "from": "github:scality/Arsenal#development/8.0",
"optional": true, "optional": true,
"requires": { "requires": {
@ -1450,7 +1452,7 @@
}, },
"blob": { "blob": {
"version": "0.0.4", "version": "0.0.4",
"resolved": "http://registry.npmjs.org/blob/-/blob-0.0.4.tgz", "resolved": "https://registry.npmjs.org/blob/-/blob-0.0.4.tgz",
"integrity": "sha1-vPEwUspURj8w+fx+lbmkdjCpSSE=" "integrity": "sha1-vPEwUspURj8w+fx+lbmkdjCpSSE="
}, },
"bson": { "bson": {
@ -1515,7 +1517,7 @@
}, },
"engine.io-parser": { "engine.io-parser": {
"version": "1.3.2", "version": "1.3.2",
"resolved": "http://registry.npmjs.org/engine.io-parser/-/engine.io-parser-1.3.2.tgz", "resolved": "https://registry.npmjs.org/engine.io-parser/-/engine.io-parser-1.3.2.tgz",
"integrity": "sha1-k3sHnwAH0Ik+xW1GyyILjLQ1Igo=", "integrity": "sha1-k3sHnwAH0Ik+xW1GyyILjLQ1Igo=",
"requires": { "requires": {
"after": "0.8.2", "after": "0.8.2",
@ -1528,7 +1530,7 @@
}, },
"hoek": { "hoek": {
"version": "4.2.1", "version": "4.2.1",
"resolved": "http://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz", "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.1.tgz",
"integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA==" "integrity": "sha512-QLg82fGkfnJ/4iy1xZ81/9SIJiq1NGFUMGs6ParyjBZr6jW2Ufj/snDqTHixNlHdPNwN2RLVD0Pi3igeK9+JfA=="
}, },
"ioctl": { "ioctl": {
@ -1554,7 +1556,7 @@
}, },
"isemail": { "isemail": {
"version": "2.2.1", "version": "2.2.1",
"resolved": "http://registry.npmjs.org/isemail/-/isemail-2.2.1.tgz", "resolved": "https://registry.npmjs.org/isemail/-/isemail-2.2.1.tgz",
"integrity": "sha1-A1PT2aYpUQgMJiwqoKQrjqjp4qY=", "integrity": "sha1-A1PT2aYpUQgMJiwqoKQrjqjp4qY=",
"optional": true "optional": true
}, },
@ -1596,7 +1598,7 @@
}, },
"level-iterator-stream": { "level-iterator-stream": {
"version": "1.3.1", "version": "1.3.1",
"resolved": "http://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-1.3.1.tgz", "resolved": "https://registry.npmjs.org/level-iterator-stream/-/level-iterator-stream-1.3.1.tgz",
"integrity": "sha1-5Dt4sagUPm+pek9IXrjqUwNS8u0=", "integrity": "sha1-5Dt4sagUPm+pek9IXrjqUwNS8u0=",
"optional": true, "optional": true,
"requires": { "requires": {
@ -1699,7 +1701,7 @@
}, },
"ms": { "ms": {
"version": "0.7.1", "version": "0.7.1",
"resolved": "http://registry.npmjs.org/ms/-/ms-0.7.1.tgz", "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz",
"integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg=" "integrity": "sha1-nNE8A62/8ltl7/3nzoZO6VIBcJg="
}, },
"object-assign": { "object-assign": {
@ -1733,7 +1735,7 @@
}, },
"readable-stream": { "readable-stream": {
"version": "1.1.14", "version": "1.1.14",
"resolved": "http://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
"optional": true, "optional": true,
"requires": { "requires": {
@ -1814,7 +1816,7 @@
}, },
"socket.io-parser": { "socket.io-parser": {
"version": "2.3.1", "version": "2.3.1",
"resolved": "http://registry.npmjs.org/socket.io-parser/-/socket.io-parser-2.3.1.tgz", "resolved": "https://registry.npmjs.org/socket.io-parser/-/socket.io-parser-2.3.1.tgz",
"integrity": "sha1-3VMgJRA85Clpcya+/WQAX8/ltKA=", "integrity": "sha1-3VMgJRA85Clpcya+/WQAX8/ltKA=",
"requires": { "requires": {
"component-emitter": "1.1.2", "component-emitter": "1.1.2",
@ -1825,7 +1827,7 @@
"dependencies": { "dependencies": {
"debug": { "debug": {
"version": "2.2.0", "version": "2.2.0",
"resolved": "http://registry.npmjs.org/debug/-/debug-2.2.0.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
"integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=", "integrity": "sha1-+HBX6ZWxofauaklgZkE3vFbwOdo=",
"requires": { "requires": {
"ms": "0.7.1" "ms": "0.7.1"
@ -1835,7 +1837,7 @@
}, },
"topo": { "topo": {
"version": "2.0.2", "version": "2.0.2",
"resolved": "http://registry.npmjs.org/topo/-/topo-2.0.2.tgz", "resolved": "https://registry.npmjs.org/topo/-/topo-2.0.2.tgz",
"integrity": "sha1-zVYVdSU5BXwNwEkaYhw7xvvh0YI=", "integrity": "sha1-zVYVdSU5BXwNwEkaYhw7xvvh0YI=",
"optional": true, "optional": true,
"requires": { "requires": {
@ -6533,7 +6535,7 @@
"dependencies": { "dependencies": {
"werelogs": { "werelogs": {
"version": "github:scality/werelogs#0ff7ec82f0deb1e472d8285fb7cc9ebde72c5f61", "version": "github:scality/werelogs#0ff7ec82f0deb1e472d8285fb7cc9ebde72c5f61",
"from": "github:scality/werelogs#0ff7ec82", "from": "github:scality/werelogs#0ff7ec82f0deb1e472d8285fb7cc9ebde72c5f61",
"requires": { "requires": {
"safe-json-stringify": "1.0.3" "safe-json-stringify": "1.0.3"
} }
@ -6705,7 +6707,7 @@
"dependencies": { "dependencies": {
"werelogs": { "werelogs": {
"version": "github:scality/werelogs#0ff7ec82f0deb1e472d8285fb7cc9ebde72c5f61", "version": "github:scality/werelogs#0ff7ec82f0deb1e472d8285fb7cc9ebde72c5f61",
"from": "github:scality/werelogs#0ff7ec82", "from": "github:scality/werelogs#0ff7ec82f0deb1e472d8285fb7cc9ebde72c5f61",
"requires": { "requires": {
"safe-json-stringify": "1.0.3" "safe-json-stringify": "1.0.3"
} }
@ -6768,6 +6770,14 @@
"resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.6.tgz", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.6.tgz",
"integrity": "sha512-aRVgGdnmW2OiySVPUC9e6m+plolMAJKjZnQlCwNSuK5yQ0JN61DZSO1X1Ufd1foqWRAlig0rhduTCHe7sVtK5Q==" "integrity": "sha512-aRVgGdnmW2OiySVPUC9e6m+plolMAJKjZnQlCwNSuK5yQ0JN61DZSO1X1Ufd1foqWRAlig0rhduTCHe7sVtK5Q=="
}, },
"fcntl": {
"version": "github:scality/node-fcntl#9108603d8881d7762dcadfde1db927a1653dfda5",
"from": "github:scality/node-fcntl#9108603d8881d7762dcadfde1db927a1653dfda5",
"requires": {
"bindings": "^1.1.1",
"nan": "^2.3.2"
}
},
"ioredis": { "ioredis": {
"version": "4.2.0", "version": "4.2.0",
"resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.2.0.tgz", "resolved": "https://registry.npmjs.org/ioredis/-/ioredis-4.2.0.tgz",

View File

@ -19,7 +19,7 @@
}, },
"homepage": "https://github.com/scality/S3#readme", "homepage": "https://github.com/scality/S3#readme",
"dependencies": { "dependencies": {
"arsenal": "github:scality/arsenal#c479933", "arsenal": "github:scality/arsenal#d460d8b",
"async": "~2.5.0", "async": "~2.5.0",
"aws-sdk": "2.28.0", "aws-sdk": "2.28.0",
"azure-storage": "^2.1.0", "azure-storage": "^2.1.0",