Compare commits
10 Commits
9a1a56e7b9
...
a14037f518
Author | SHA1 | Date |
---|---|---|
Guillaume Hivert | a14037f518 | |
Guillaume Hivert | 9d7f041508 | |
Jordi Bertran de Balanda | 61da23c8d2 | |
Jordi Bertran de Balanda | 65c94012c6 | |
Jordi Bertran de Balanda | cb87471cc8 | |
Jordi Bertran de Balanda | 57e4ecc0ee | |
Jordi Bertran de Balanda | 7a390e684d | |
Jordi Bertran de Balanda | bbcd33a14d | |
Jordi Bertran de Balanda | 88311ffbd9 | |
Jordi Bertran de Balanda | be1e0fc56f |
|
@ -0,0 +1,7 @@
|
|||
{
|
||||
"extends": ["scality"],
|
||||
"plugins": ["jest"],
|
||||
"env": {
|
||||
"jest/globals": true
|
||||
}
|
||||
}
|
|
@ -0,0 +1,8 @@
|
|||
module.exports = {
|
||||
presets: [
|
||||
['@babel/preset-env', { targets: { node: 'current' } }],
|
||||
'@babel/preset-typescript',
|
||||
],
|
||||
plugins: ['add-module-exports'],
|
||||
};
|
||||
|
4
index.js
4
index.js
|
@ -76,10 +76,10 @@ module.exports = {
|
|||
tagging: require('./lib/s3middleware/tagging'),
|
||||
checkDateModifiedHeaders:
|
||||
require('./lib/s3middleware/validateConditionalHeaders')
|
||||
.checkDateModifiedHeaders,
|
||||
.checkDateModifiedHeaders,
|
||||
validateConditionalHeaders:
|
||||
require('./lib/s3middleware/validateConditionalHeaders')
|
||||
.validateConditionalHeaders,
|
||||
.validateConditionalHeaders,
|
||||
MD5Sum: require('./lib/s3middleware/MD5Sum'),
|
||||
NullStream: require('./lib/s3middleware/nullStream'),
|
||||
objectUtils: require('./lib/s3middleware/objectUtils'),
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const { inc, checkLimit, listingParamsMasterKeysV0ToV1,
|
||||
FILTER_END, FILTER_ACCEPT } = require('./tools');
|
||||
FILTER_END, FILTER_ACCEPT } = require('./tools');
|
||||
const DEFAULT_MAX_KEYS = 1000;
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
const Extension = require('./Extension').default;
|
||||
const { inc, listingParamsMasterKeysV0ToV1,
|
||||
FILTER_END, FILTER_ACCEPT, FILTER_SKIP } = require('./tools');
|
||||
FILTER_END, FILTER_ACCEPT, FILTER_SKIP } = require('./tools');
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
|
||||
|
||||
|
@ -66,7 +66,7 @@ class Delimiter extends Extension {
|
|||
this.continuationToken = parameters.continuationToken;
|
||||
this.alphabeticalOrder =
|
||||
typeof parameters.alphabeticalOrder !== 'undefined' ?
|
||||
parameters.alphabeticalOrder : true;
|
||||
parameters.alphabeticalOrder : true;
|
||||
|
||||
this.vFormat = vFormat || BucketVersioningKeyFormat.v0;
|
||||
// results
|
||||
|
@ -87,7 +87,7 @@ class Delimiter extends Extension {
|
|||
this[this.nextContinueMarker].startsWith(this.prefix || '')) {
|
||||
const nextDelimiterIndex =
|
||||
this[this.nextContinueMarker].indexOf(this.delimiter,
|
||||
this.prefix ? this.prefix.length : 0);
|
||||
this.prefix ? this.prefix.length : 0);
|
||||
this[this.nextContinueMarker] =
|
||||
this[this.nextContinueMarker].slice(0, nextDelimiterIndex +
|
||||
this.delimiter.length);
|
||||
|
|
|
@ -194,7 +194,7 @@ class DelimiterVersions extends Delimiter {
|
|||
// length is the same so we can remove their prefix without
|
||||
// looking at the type of key
|
||||
return this.filterCommon(obj.key.slice(DbPrefixes.Master.length),
|
||||
obj.value);
|
||||
obj.value);
|
||||
}
|
||||
|
||||
filterCommon(key, value) {
|
||||
|
@ -250,7 +250,7 @@ class DelimiterVersions extends Delimiter {
|
|||
}
|
||||
// skip to the same object key in both M and V range listings
|
||||
return [DbPrefixes.Master + skipV0,
|
||||
DbPrefixes.Version + skipV0];
|
||||
DbPrefixes.Version + skipV0];
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
const ArrayUtils = require('./ArrayUtils');
|
||||
|
||||
class SortedSet {
|
||||
|
||||
constructor(obj) {
|
||||
if (obj) {
|
||||
this.keys = obj.keys;
|
||||
|
|
|
@ -91,7 +91,7 @@ class Vault {
|
|||
requestContext: serializedRCsArr,
|
||||
},
|
||||
(err, userInfo) => vaultSignatureCb(err, userInfo,
|
||||
params.log, callback)
|
||||
params.log, callback),
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -146,7 +146,7 @@ class Vault {
|
|||
requestContext: serializedRCs,
|
||||
},
|
||||
(err, userInfo) => vaultSignatureCb(err, userInfo,
|
||||
params.log, callback, streamingV4Params)
|
||||
params.log, callback, streamingV4Params),
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -232,28 +232,28 @@ class Vault {
|
|||
*/
|
||||
getAccountIds(canonicalIDs, log, callback) {
|
||||
log.trace('getting accountIds from Vault based on canonicalIDs',
|
||||
{ canonicalIDs });
|
||||
{ canonicalIDs });
|
||||
this.client.getAccountIds(canonicalIDs,
|
||||
{ reqUid: log.getSerializedUids() },
|
||||
(err, info) => {
|
||||
if (err) {
|
||||
log.debug('received error message from vault',
|
||||
{ errorMessage: err });
|
||||
return callback(err);
|
||||
}
|
||||
const infoFromVault = info.message.body;
|
||||
log.trace('info received from vault', { infoFromVault });
|
||||
const result = {};
|
||||
/* If the accountId was not found in Vault, do not
|
||||
send the canonicalID back to the API */
|
||||
Object.keys(infoFromVault).forEach(key => {
|
||||
if (infoFromVault[key] !== 'NotFound' &&
|
||||
infoFromVault[key] !== 'WrongFormat') {
|
||||
result[key] = infoFromVault[key];
|
||||
{ reqUid: log.getSerializedUids() },
|
||||
(err, info) => {
|
||||
if (err) {
|
||||
log.debug('received error message from vault',
|
||||
{ errorMessage: err });
|
||||
return callback(err);
|
||||
}
|
||||
const infoFromVault = info.message.body;
|
||||
log.trace('info received from vault', { infoFromVault });
|
||||
const result = {};
|
||||
/* If the accountId was not found in Vault, do not
|
||||
send the canonicalID back to the API */
|
||||
Object.keys(infoFromVault).forEach(key => {
|
||||
if (infoFromVault[key] !== 'NotFound' &&
|
||||
infoFromVault[key] !== 'WrongFormat') {
|
||||
result[key] = infoFromVault[key];
|
||||
}
|
||||
});
|
||||
return callback(null, result);
|
||||
});
|
||||
return callback(null, result);
|
||||
});
|
||||
}
|
||||
|
||||
/** checkPolicies -- call Vault to evaluate policies
|
||||
|
|
|
@ -74,7 +74,7 @@ function extractParams(request, log, awsService, data) {
|
|||
version = 'v4';
|
||||
} else {
|
||||
log.trace('invalid authorization security header',
|
||||
{ header: authHeader });
|
||||
{ header: authHeader });
|
||||
return { err: errors.AccessDenied };
|
||||
}
|
||||
} else if (data.Signature) {
|
||||
|
@ -89,7 +89,7 @@ function extractParams(request, log, awsService, data) {
|
|||
if (version !== null && method !== null) {
|
||||
if (!checkFunctions[version] || !checkFunctions[version][method]) {
|
||||
log.trace('invalid auth version or method',
|
||||
{ version, authMethod: method });
|
||||
{ version, authMethod: method });
|
||||
return { err: errors.NotImplemented };
|
||||
}
|
||||
log.trace('identified auth method', { version, authMethod: method });
|
||||
|
@ -161,7 +161,7 @@ function doAuth(request, log, cb, awsService, requestContexts) {
|
|||
* @return {undefined}
|
||||
*/
|
||||
function generateV4Headers(request, data, accessKey, secretKeyValue,
|
||||
awsService, proxyPath, sessionToken) {
|
||||
awsService, proxyPath, sessionToken) {
|
||||
Object.assign(request, { headers: {} });
|
||||
const amzDate = convertUTCtoISO8601(Date.now());
|
||||
// get date without time
|
||||
|
@ -194,16 +194,16 @@ function generateV4Headers(request, data, accessKey, secretKeyValue,
|
|||
.filter(headerName =>
|
||||
headerName.startsWith('x-amz-')
|
||||
|| headerName.startsWith('x-scal-')
|
||||
|| headerName === 'host'
|
||||
|| headerName === 'host',
|
||||
).sort().join(';');
|
||||
const params = { request, signedHeaders, payloadChecksum,
|
||||
credentialScope, timestamp, query: data,
|
||||
awsService: service, proxyPath };
|
||||
const stringToSign = constructStringToSignV4(params);
|
||||
const signingKey = vaultUtilities.calculateSigningKey(secretKeyValue,
|
||||
region,
|
||||
scopeDate,
|
||||
service);
|
||||
region,
|
||||
scopeDate,
|
||||
service);
|
||||
const signature = crypto.createHmac('sha256', signingKey)
|
||||
.update(stringToSign, 'binary').digest('hex');
|
||||
const authorizationHeader = `${algorithm} Credential=${accessKey}` +
|
||||
|
|
|
@ -29,7 +29,7 @@ class ChainBackend extends BaseBackend {
|
|||
typeof client.getCanonicalIds === 'function' &&
|
||||
typeof client.getEmailAddresses === 'function' &&
|
||||
typeof client.checkPolicies === 'function' &&
|
||||
typeof client.healthcheck === 'function'
|
||||
typeof client.healthcheck === 'function',
|
||||
), 'invalid client: missing required auth backend methods');
|
||||
this._clients = clients;
|
||||
}
|
||||
|
@ -55,7 +55,7 @@ class ChainBackend extends BaseBackend {
|
|||
signatureFromRequest,
|
||||
accessKey,
|
||||
options,
|
||||
done
|
||||
done,
|
||||
), callback);
|
||||
}
|
||||
|
||||
|
@ -67,7 +67,7 @@ class ChainBackend extends BaseBackend {
|
|||
region,
|
||||
scopeDate,
|
||||
options,
|
||||
done
|
||||
done,
|
||||
), callback);
|
||||
}
|
||||
|
||||
|
@ -153,7 +153,7 @@ class ChainBackend extends BaseBackend {
|
|||
requestContextParams,
|
||||
userArn,
|
||||
options,
|
||||
done
|
||||
done,
|
||||
), (err, res) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
|
@ -171,8 +171,8 @@ class ChainBackend extends BaseBackend {
|
|||
client.healthcheck(reqUid, (err, res) => done(null, {
|
||||
error: !!err ? err : null,
|
||||
status: res,
|
||||
})
|
||||
), (err, res) => {
|
||||
}),
|
||||
), (err, res) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
|
|
@ -26,20 +26,20 @@ class AuthLoader {
|
|||
.required();
|
||||
|
||||
const accountsJoi = joi.array()
|
||||
.items({
|
||||
name: joi.string().required(),
|
||||
email: joi.string().email().required(),
|
||||
arn: joi.string().required(),
|
||||
canonicalID: joi.string().required(),
|
||||
shortid: joi.string().regex(/^[0-9]{12}$/).required(),
|
||||
keys: this._joiKeysValidator,
|
||||
// backward-compat
|
||||
users: joi.array(),
|
||||
})
|
||||
.required()
|
||||
.unique('arn')
|
||||
.unique('email')
|
||||
.unique('canonicalID');
|
||||
.items({
|
||||
name: joi.string().required(),
|
||||
email: joi.string().email().required(),
|
||||
arn: joi.string().required(),
|
||||
canonicalID: joi.string().required(),
|
||||
shortid: joi.string().regex(/^[0-9]{12}$/).required(),
|
||||
keys: this._joiKeysValidator,
|
||||
// backward-compat
|
||||
users: joi.array(),
|
||||
})
|
||||
.required()
|
||||
.unique('arn')
|
||||
.unique('email')
|
||||
.unique('canonicalID');
|
||||
this._joiValidator = joi.object({ accounts: accountsJoi });
|
||||
}
|
||||
|
||||
|
@ -136,7 +136,7 @@ class AuthLoader {
|
|||
|
||||
_validateData(authData, filePath) {
|
||||
const res = joi.validate(authData, this._joiValidator,
|
||||
{ abortEarly: false });
|
||||
{ abortEarly: false });
|
||||
if (res.error) {
|
||||
this._dumpJoiErrors(res.error.details, filePath);
|
||||
return false;
|
||||
|
@ -156,7 +156,7 @@ class AuthLoader {
|
|||
'master/conf/authdata.json). Also note that support ' +
|
||||
'for account users has been dropped.',
|
||||
{ accountName: account.name, accountArn: account.arn,
|
||||
filePath });
|
||||
filePath });
|
||||
arnError = true;
|
||||
return;
|
||||
}
|
||||
|
@ -167,7 +167,7 @@ class AuthLoader {
|
|||
'https://github.com/scality/S3/blob/master/conf/' +
|
||||
'authdata.json)',
|
||||
{ accountName: account.name, accountArn: account.arn,
|
||||
filePath });
|
||||
filePath });
|
||||
arnError = true;
|
||||
return;
|
||||
}
|
||||
|
@ -176,8 +176,8 @@ class AuthLoader {
|
|||
this._log.error(
|
||||
'authentication config validation error',
|
||||
{ reason: arnObj.error.description,
|
||||
accountName: account.name, accountArn: account.arn,
|
||||
filePath });
|
||||
accountName: account.name, accountArn: account.arn,
|
||||
filePath });
|
||||
arnError = true;
|
||||
return;
|
||||
}
|
||||
|
@ -185,8 +185,8 @@ class AuthLoader {
|
|||
this._log.error(
|
||||
'authentication config validation error',
|
||||
{ reason: 'not an IAM account ARN',
|
||||
accountName: account.name, accountArn: account.arn,
|
||||
filePath });
|
||||
accountName: account.name, accountArn: account.arn,
|
||||
filePath });
|
||||
arnError = true;
|
||||
return;
|
||||
}
|
||||
|
@ -215,7 +215,7 @@ class AuthLoader {
|
|||
logInfo.context = err.context;
|
||||
}
|
||||
this._log.error('authentication config validation error',
|
||||
logInfo);
|
||||
logInfo);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
@ -41,7 +41,7 @@ function getCanonicalizedAmzHeaders(headers, clientType) {
|
|||
// Build headerString
|
||||
return amzHeaders.reduce((headerStr, current) =>
|
||||
`${headerStr}${current[0]}:${current[1]}\n`,
|
||||
'');
|
||||
'');
|
||||
}
|
||||
|
||||
module.exports = getCanonicalizedAmzHeaders;
|
||||
|
|
|
@ -22,9 +22,9 @@ function check(request, log, data) {
|
|||
timestamp = Date.parse(timestamp);
|
||||
if (!timestamp) {
|
||||
log.debug('missing or invalid date header',
|
||||
{ method: 'auth/v2/headerAuthCheck.check' });
|
||||
{ method: 'auth/v2/headerAuthCheck.check' });
|
||||
return { err: errors.AccessDenied.
|
||||
customizeDescription('Authentication requires a valid Date or ' +
|
||||
customizeDescription('Authentication requires a valid Date or ' +
|
||||
'x-amz-date header') };
|
||||
}
|
||||
|
||||
|
|
|
@ -42,12 +42,12 @@ function check(request, log, data) {
|
|||
|
||||
if (expirationTime > currentTime + preSignedURLExpiry) {
|
||||
log.debug('expires parameter too far in future',
|
||||
{ expires: request.query.Expires });
|
||||
{ expires: request.query.Expires });
|
||||
return { err: errors.AccessDenied };
|
||||
}
|
||||
if (currentTime > expirationTime) {
|
||||
log.debug('current time exceeds expires time',
|
||||
{ expires: request.query.Expires });
|
||||
{ expires: request.query.Expires });
|
||||
return { err: errors.RequestTimeTooSkewed };
|
||||
}
|
||||
const accessKey = data.AWSAccessKeyId;
|
||||
|
|
|
@ -88,14 +88,14 @@ function check(request, log, data, awsService) {
|
|||
}
|
||||
if (!timestamp) {
|
||||
log.debug('missing or invalid date header',
|
||||
{ method: 'auth/v4/headerAuthCheck.check' });
|
||||
{ method: 'auth/v4/headerAuthCheck.check' });
|
||||
return { err: errors.AccessDenied.
|
||||
customizeDescription('Authentication requires a valid Date or ' +
|
||||
customizeDescription('Authentication requires a valid Date or ' +
|
||||
'x-amz-date header') };
|
||||
}
|
||||
|
||||
const validationResult = validateCredentials(credentialsArr, timestamp,
|
||||
log);
|
||||
log);
|
||||
if (validationResult instanceof Error) {
|
||||
log.debug('credentials in improper format', { credentialsArr,
|
||||
timestamp, validationResult });
|
||||
|
@ -134,7 +134,7 @@ function check(request, log, data, awsService) {
|
|||
} catch (err) {
|
||||
log.debug('invalid proxy_path header', { proxyPath, err });
|
||||
return { err: errors.InvalidArgument.customizeDescription(
|
||||
'invalid proxy_path header') };
|
||||
'invalid proxy_path header') };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -45,7 +45,7 @@ function check(request, log, data) {
|
|||
}
|
||||
|
||||
const validationResult = validateCredentials(credential, timestamp,
|
||||
log);
|
||||
log);
|
||||
if (validationResult instanceof Error) {
|
||||
log.debug('credentials in improper format', { credential,
|
||||
timestamp, validationResult });
|
||||
|
@ -69,7 +69,7 @@ function check(request, log, data) {
|
|||
} catch (err) {
|
||||
log.debug('invalid proxy_path header', { proxyPath });
|
||||
return { err: errors.InvalidArgument.customizeDescription(
|
||||
'invalid proxy_path header') };
|
||||
'invalid proxy_path header') };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -273,7 +273,7 @@ class V4Transform extends Transform {
|
|||
}
|
||||
// get next chunk
|
||||
return callback();
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -25,20 +25,20 @@ function validateCredentials(credentials, timestamp, log) {
|
|||
log.warn('accessKey provided is wrong format', { accessKey });
|
||||
return errors.InvalidArgument;
|
||||
}
|
||||
// The scope date (format YYYYMMDD) must be same date as the timestamp
|
||||
// on the request from the x-amz-date param (if queryAuthCheck)
|
||||
// or from the x-amz-date header or date header (if headerAuthCheck)
|
||||
// Format of timestamp is ISO 8601: YYYYMMDDTHHMMSSZ.
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/
|
||||
// sigv4-query-string-auth.html
|
||||
// http://docs.aws.amazon.com/general/latest/gr/
|
||||
// sigv4-date-handling.html
|
||||
// The scope date (format YYYYMMDD) must be same date as the timestamp
|
||||
// on the request from the x-amz-date param (if queryAuthCheck)
|
||||
// or from the x-amz-date header or date header (if headerAuthCheck)
|
||||
// Format of timestamp is ISO 8601: YYYYMMDDTHHMMSSZ.
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/
|
||||
// sigv4-query-string-auth.html
|
||||
// http://docs.aws.amazon.com/general/latest/gr/
|
||||
// sigv4-date-handling.html
|
||||
|
||||
// convert timestamp to format of scopeDate YYYYMMDD
|
||||
// convert timestamp to format of scopeDate YYYYMMDD
|
||||
const timestampDate = timestamp.split('T')[0];
|
||||
if (scopeDate.length !== 8 || scopeDate !== timestampDate) {
|
||||
log.warn('scope date must be the same date as the timestamp date',
|
||||
{ scopeDate, timestampDate });
|
||||
{ scopeDate, timestampDate });
|
||||
return errors.RequestTimeTooSkewed;
|
||||
}
|
||||
if (service !== 's3' && service !== 'iam' && service !== 'ring' &&
|
||||
|
@ -50,7 +50,7 @@ function validateCredentials(credentials, timestamp, log) {
|
|||
}
|
||||
if (requestType !== 'aws4_request') {
|
||||
log.warn('requestType contained in params is not aws4_request',
|
||||
{ requestType });
|
||||
{ requestType });
|
||||
return errors.InvalidArgument;
|
||||
}
|
||||
return {};
|
||||
|
@ -68,7 +68,7 @@ function extractQueryParams(queryObj, log) {
|
|||
// Do not need the algorithm sent back
|
||||
if (queryObj['X-Amz-Algorithm'] !== 'AWS4-HMAC-SHA256') {
|
||||
log.warn('algorithm param incorrect',
|
||||
{ algo: queryObj['X-Amz-Algorithm'] });
|
||||
{ algo: queryObj['X-Amz-Algorithm'] });
|
||||
return authParams;
|
||||
}
|
||||
|
||||
|
|
|
@ -103,11 +103,11 @@ module.exports = {
|
|||
gcpTaggingPrefix: 'aws-tag-',
|
||||
productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko',
|
||||
legacyLocations: ['sproxyd', 'legacy'],
|
||||
// healthcheck default call from nginx is every 2 seconds
|
||||
// healthcheck default call from nginx is every 2 seconds
|
||||
// for external backends, don't call unless at least 1 minute
|
||||
// (60,000 milliseconds) since last call
|
||||
externalBackendHealthCheckInterval: 60000,
|
||||
// some of the available data backends (if called directly rather
|
||||
// some of the available data backends (if called directly rather
|
||||
// than through the multiple backend gateway) need a key provided
|
||||
// as a string as first parameter of the get/delete methods.
|
||||
clientsRequireStringKey: { sproxyd: true, cdmi: true },
|
||||
|
|
10
lib/db.js
10
lib/db.js
|
@ -64,12 +64,12 @@ class IndexTransaction {
|
|||
push(op) {
|
||||
if (this.closed) {
|
||||
throw propError('pushOnCommittedTransaction',
|
||||
'can not add ops to already committed transaction');
|
||||
'can not add ops to already committed transaction');
|
||||
}
|
||||
|
||||
if (op.type !== 'put' && op.type !== 'del') {
|
||||
throw propError('invalidTransactionVerb',
|
||||
`unknown action type: ${op.type}`);
|
||||
`unknown action type: ${op.type}`);
|
||||
}
|
||||
|
||||
if (op.key === undefined) {
|
||||
|
@ -137,7 +137,7 @@ class IndexTransaction {
|
|||
addCondition(condition) {
|
||||
if (this.closed) {
|
||||
throw propError('pushOnCommittedTransaction',
|
||||
'can not add conditions to already committed transaction');
|
||||
'can not add conditions to already committed transaction');
|
||||
}
|
||||
if (condition === undefined || Object.keys(condition).length === 0) {
|
||||
throw propError('missingCondition', 'missing condition for conditional put');
|
||||
|
@ -159,12 +159,12 @@ class IndexTransaction {
|
|||
commit(cb) {
|
||||
if (this.closed) {
|
||||
return cb(propError('alreadyCommitted',
|
||||
'transaction was already committed'));
|
||||
'transaction was already committed'));
|
||||
}
|
||||
|
||||
if (this.operations.length === 0) {
|
||||
return cb(propError('emptyTransaction',
|
||||
'tried to commit an empty transaction'));
|
||||
'tried to commit an empty transaction'));
|
||||
}
|
||||
|
||||
this.closed = true;
|
||||
|
|
|
@ -76,11 +76,11 @@ function errorsGen() {
|
|||
const errorsObj = require('../errors/arsenalErrors.json');
|
||||
|
||||
Object.keys(errorsObj)
|
||||
.filter(index => index !== '_comment')
|
||||
.forEach(index => {
|
||||
errors[index] = new ArsenalError(index, errorsObj[index].code,
|
||||
errorsObj[index].description);
|
||||
});
|
||||
.filter(index => index !== '_comment')
|
||||
.forEach(index => {
|
||||
errors[index] = new ArsenalError(index, errorsObj[index].code,
|
||||
errorsObj[index].description);
|
||||
});
|
||||
return errors;
|
||||
}
|
||||
|
||||
|
|
|
@ -17,9 +17,9 @@ describe('decyrptSecret', () => {
|
|||
describe('parseServiceCredentials', () => {
|
||||
const conf = {
|
||||
users: [{ accessKey,
|
||||
accountType: 'service-clueso',
|
||||
secretKey,
|
||||
userName: 'Search Service Account' }],
|
||||
accountType: 'service-clueso',
|
||||
secretKey,
|
||||
userName: 'Search Service Account' }],
|
||||
};
|
||||
const auth = JSON.stringify({ privateKey });
|
||||
|
||||
|
|
|
@ -25,7 +25,7 @@ module.exports.once = function once(func) {
|
|||
state.res = func.apply(func, args);
|
||||
} else {
|
||||
debug('function already called:', func,
|
||||
'returning cached result:', state.res);
|
||||
'returning cached result:', state.res);
|
||||
}
|
||||
return state.res;
|
||||
};
|
||||
|
|
|
@ -17,7 +17,7 @@ class RedisClient {
|
|||
method: 'RedisClient.constructor',
|
||||
redisHost: config.host,
|
||||
redisPort: config.port,
|
||||
})
|
||||
}),
|
||||
);
|
||||
return this;
|
||||
}
|
||||
|
|
|
@ -9,7 +9,6 @@ const StatsClient = require('./StatsClient');
|
|||
* rather than by seconds
|
||||
*/
|
||||
class StatsModel extends StatsClient {
|
||||
|
||||
/**
|
||||
* Utility method to convert 2d array rows to columns, and vice versa
|
||||
* See also: https://docs.ruby-lang.org/en/2.0.0/Array.html#method-i-zip
|
||||
|
|
|
@ -2,8 +2,8 @@ const promClient = require('prom-client');
|
|||
|
||||
const collectDefaultMetricsIntervalMs =
|
||||
process.env.COLLECT_DEFAULT_METRICS_INTERVAL_MS !== undefined ?
|
||||
Number.parseInt(process.env.COLLECT_DEFAULT_METRICS_INTERVAL_MS, 10) :
|
||||
10000;
|
||||
Number.parseInt(process.env.COLLECT_DEFAULT_METRICS_INTERVAL_MS, 10) :
|
||||
10000;
|
||||
|
||||
promClient.collectDefaultMetrics({ timeout: collectDefaultMetricsIntervalMs });
|
||||
|
||||
|
|
|
@ -27,7 +27,7 @@ class ARN {
|
|||
|
||||
static createFromString(arnStr) {
|
||||
const [arn, partition, service, region, accountId,
|
||||
resourceType, resource] = arnStr.split(':');
|
||||
resourceType, resource] = arnStr.split(':');
|
||||
|
||||
if (arn !== 'arn') {
|
||||
return { error: errors.InvalidArgument.customizeDescription(
|
||||
|
@ -58,7 +58,7 @@ class ARN {
|
|||
'must be a 12-digit number or "*"') };
|
||||
}
|
||||
const fullResource = (resource !== undefined ?
|
||||
`${resourceType}:${resource}` : resourceType);
|
||||
`${resourceType}:${resource}` : resourceType);
|
||||
return new ARN(partition, service, region, accountId, fullResource);
|
||||
}
|
||||
|
||||
|
@ -98,7 +98,7 @@ class ARN {
|
|||
|
||||
toString() {
|
||||
return ['arn', this.getPartition(), this.getService(),
|
||||
this.getRegion(), this.getAccountId(), this.getResource()]
|
||||
this.getRegion(), this.getAccountId(), this.getResource()]
|
||||
.join(':');
|
||||
}
|
||||
}
|
||||
|
|
|
@ -52,9 +52,9 @@ class BackendInfo {
|
|||
*/
|
||||
static isRequestEndpointPresent(config, requestEndpoint, log) {
|
||||
if (Object.keys(config.restEndpoints).
|
||||
indexOf(requestEndpoint) < 0) {
|
||||
indexOf(requestEndpoint) < 0) {
|
||||
log.trace('requestEndpoint does not match config restEndpoints',
|
||||
{ requestEndpoint });
|
||||
{ requestEndpoint });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
@ -70,10 +70,10 @@ class BackendInfo {
|
|||
*/
|
||||
static isRequestEndpointValueValid(config, requestEndpoint, log) {
|
||||
if (Object.keys(config.locationConstraints).
|
||||
indexOf(config.restEndpoints[requestEndpoint]) < 0) {
|
||||
indexOf(config.restEndpoints[requestEndpoint]) < 0) {
|
||||
log.trace('the default locationConstraint for request' +
|
||||
'Endpoint does not match any config locationConstraint',
|
||||
{ requestEndpoint });
|
||||
{ requestEndpoint });
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
|
@ -110,7 +110,7 @@ class BackendInfo {
|
|||
*/
|
||||
static isValidRequestEndpointOrBackend(config, requestEndpoint, log) {
|
||||
if (!BackendInfo.isRequestEndpointPresent(config, requestEndpoint,
|
||||
log)) {
|
||||
log)) {
|
||||
return BackendInfo.isMemOrFileBackend(config, log);
|
||||
}
|
||||
return BackendInfo.isRequestEndpointValueValid(config, requestEndpoint,
|
||||
|
@ -132,7 +132,7 @@ class BackendInfo {
|
|||
bucketLocationConstraint, requestEndpoint, log) {
|
||||
if (objectLocationConstraint) {
|
||||
if (BackendInfo.isValidLocationConstraint(config,
|
||||
objectLocationConstraint, log)) {
|
||||
objectLocationConstraint, log)) {
|
||||
log.trace('objectLocationConstraint is valid');
|
||||
return { isValid: true };
|
||||
}
|
||||
|
@ -143,7 +143,7 @@ class BackendInfo {
|
|||
}
|
||||
if (bucketLocationConstraint) {
|
||||
if (BackendInfo.isValidLocationConstraint(config,
|
||||
bucketLocationConstraint, log)) {
|
||||
bucketLocationConstraint, log)) {
|
||||
log.trace('bucketLocationConstraint is valid');
|
||||
return { isValid: true };
|
||||
}
|
||||
|
@ -159,7 +159,7 @@ class BackendInfo {
|
|||
return { isValid: true, legacyLocationConstraint };
|
||||
}
|
||||
if (!BackendInfo.isValidRequestEndpointOrBackend(config,
|
||||
requestEndpoint, log)) {
|
||||
requestEndpoint, log)) {
|
||||
return { isValid: false, description: 'Endpoint Location Error - ' +
|
||||
`Your endpoint "${requestEndpoint}" is not in restEndpoints ` +
|
||||
'in your config OR the default location constraint for request ' +
|
||||
|
@ -167,7 +167,7 @@ class BackendInfo {
|
|||
'match any config locationConstraint - Please update.' };
|
||||
}
|
||||
if (BackendInfo.isRequestEndpointPresent(config, requestEndpoint,
|
||||
log)) {
|
||||
log)) {
|
||||
return { isValid: true };
|
||||
}
|
||||
return { isValid: true, defaultedToDataBackend: true };
|
||||
|
|
|
@ -69,13 +69,13 @@ class BucketInfo {
|
|||
* @param {object} [notificationConfiguration] - bucket notification configuration
|
||||
*/
|
||||
constructor(name, owner, ownerDisplayName, creationDate,
|
||||
mdBucketModelVersion, acl, transient, deleted,
|
||||
serverSideEncryption, versioningConfiguration,
|
||||
locationConstraint, websiteConfiguration, cors,
|
||||
replicationConfiguration, lifecycleConfiguration,
|
||||
bucketPolicy, uid, readLocationConstraint, isNFS,
|
||||
ingestionConfig, azureInfo, objectLockEnabled,
|
||||
objectLockConfiguration, notificationConfiguration) {
|
||||
mdBucketModelVersion, acl, transient, deleted,
|
||||
serverSideEncryption, versioningConfiguration,
|
||||
locationConstraint, websiteConfiguration, cors,
|
||||
replicationConfiguration, lifecycleConfiguration,
|
||||
bucketPolicy, uid, readLocationConstraint, isNFS,
|
||||
ingestionConfig, azureInfo, objectLockEnabled,
|
||||
objectLockConfiguration, notificationConfiguration) {
|
||||
assert.strictEqual(typeof name, 'string');
|
||||
assert.strictEqual(typeof owner, 'string');
|
||||
assert.strictEqual(typeof ownerDisplayName, 'string');
|
||||
|
@ -94,7 +94,7 @@ class BucketInfo {
|
|||
if (serverSideEncryption) {
|
||||
assert.strictEqual(typeof serverSideEncryption, 'object');
|
||||
const { cryptoScheme, algorithm, masterKeyId,
|
||||
configuredMasterKeyId, mandatory } = serverSideEncryption;
|
||||
configuredMasterKeyId, mandatory } = serverSideEncryption;
|
||||
assert.strictEqual(typeof cryptoScheme, 'number');
|
||||
assert.strictEqual(typeof algorithm, 'string');
|
||||
assert.strictEqual(typeof masterKeyId, 'string');
|
||||
|
|
|
@ -381,7 +381,7 @@ class LifecycleConfiguration {
|
|||
if (!tags[i].Key || !tags[i].Value) {
|
||||
tagObj.error =
|
||||
errors.MissingRequiredParameter.customizeDescription(
|
||||
'Tag XML does not contain both Key and Value');
|
||||
'Tag XML does not contain both Key and Value');
|
||||
break;
|
||||
}
|
||||
|
||||
|
@ -929,7 +929,7 @@ class LifecycleConfiguration {
|
|||
const daysInt = parseInt(subExp.Days[0], 10);
|
||||
if (daysInt < 1) {
|
||||
expObj.error = errors.InvalidArgument.customizeDescription(
|
||||
'Expiration days is not a positive integer');
|
||||
'Expiration days is not a positive integer');
|
||||
} else {
|
||||
expObj.days = daysInt;
|
||||
}
|
||||
|
@ -1125,10 +1125,10 @@ class LifecycleConfiguration {
|
|||
const { noncurrentDays, storageClass } = transition;
|
||||
xml.push(
|
||||
`<${actionName}>`,
|
||||
`<NoncurrentDays>${noncurrentDays}` +
|
||||
`<NoncurrentDays>${noncurrentDays}` +
|
||||
'</NoncurrentDays>',
|
||||
`<StorageClass>${storageClass}</StorageClass>`,
|
||||
`</${actionName}>`
|
||||
`<StorageClass>${storageClass}</StorageClass>`,
|
||||
`</${actionName}>`,
|
||||
);
|
||||
});
|
||||
Action = xml.join('');
|
||||
|
@ -1146,9 +1146,9 @@ class LifecycleConfiguration {
|
|||
}
|
||||
xml.push(
|
||||
`<${actionName}>`,
|
||||
element,
|
||||
`<StorageClass>${storageClass}</StorageClass>`,
|
||||
`</${actionName}>`
|
||||
element,
|
||||
`<StorageClass>${storageClass}</StorageClass>`,
|
||||
`</${actionName}>`,
|
||||
);
|
||||
});
|
||||
Action = xml.join('');
|
||||
|
|
|
@ -27,7 +27,7 @@ const errors = require('../errors');
|
|||
* </NotificationConfiguration>
|
||||
*/
|
||||
|
||||
/**
|
||||
/**
|
||||
* Format of config:
|
||||
*
|
||||
* config = {
|
||||
|
|
|
@ -17,7 +17,7 @@ const errors = require('../errors');
|
|||
* </ObjectLockConfiguration>
|
||||
*/
|
||||
|
||||
/**
|
||||
/**
|
||||
* Format of config:
|
||||
*
|
||||
* config = {
|
||||
|
|
|
@ -10,7 +10,6 @@ const ObjectMDLocation = require('./ObjectMDLocation');
|
|||
* mpuPart metadata for example)
|
||||
*/
|
||||
class ObjectMD {
|
||||
|
||||
/**
|
||||
* Create a new instance of ObjectMD. Parameter <tt>objMd</tt> is
|
||||
* reserved for internal use, users should call
|
||||
|
@ -149,7 +148,7 @@ class ObjectMD {
|
|||
|
||||
Object.assign(this._data, objMd._data);
|
||||
Object.assign(this._data.replicationInfo,
|
||||
objMd._data.replicationInfo);
|
||||
objMd._data.replicationInfo);
|
||||
}
|
||||
|
||||
_updateFromParsedJSON(objMd) {
|
||||
|
|
|
@ -3,7 +3,6 @@
|
|||
* 'location' array
|
||||
*/
|
||||
class ObjectMDLocation {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
* @param {object} locationObj - single data location info
|
||||
|
|
|
@ -111,7 +111,7 @@ class RoundRobin {
|
|||
pickHost() {
|
||||
if (this.logger) {
|
||||
this.logger.debug('pick host',
|
||||
{ host: this.getCurrentHost() });
|
||||
{ host: this.getCurrentHost() });
|
||||
}
|
||||
const curHost = this.getCurrentHost();
|
||||
++this.pickCount;
|
||||
|
@ -163,7 +163,7 @@ class RoundRobin {
|
|||
}
|
||||
if (this.logger) {
|
||||
this.logger.debug('round robin host',
|
||||
{ newHost: this.getCurrentHost() });
|
||||
{ newHost: this.getCurrentHost() });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -10,7 +10,6 @@ const { checkSupportIPv6 } = require('./utils');
|
|||
|
||||
|
||||
class Server {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
*
|
||||
|
@ -431,16 +430,16 @@ class Server {
|
|||
// Setting no delay of the socket to the value configured
|
||||
sock.setNoDelay(this.isNoDelay());
|
||||
sock.on('error', err => this._logger.info(
|
||||
'socket error - request rejected', { error: err }));
|
||||
'socket error - request rejected', { error: err }));
|
||||
});
|
||||
this._server.on('tlsClientError', (err, sock) =>
|
||||
this._onClientError(err, sock));
|
||||
this._onClientError(err, sock));
|
||||
this._server.on('clientError', (err, sock) =>
|
||||
this._onClientError(err, sock));
|
||||
this._onClientError(err, sock));
|
||||
this._server.on('checkContinue', (req, res) =>
|
||||
this._onCheckContinue(req, res));
|
||||
this._onCheckContinue(req, res));
|
||||
this._server.on('checkExpectation', (req, res) =>
|
||||
this._onCheckExpectation(req, res));
|
||||
this._onCheckExpectation(req, res));
|
||||
this._server.on('listening', () => this._onListening());
|
||||
}
|
||||
this._server.listen(this._port, this._address);
|
||||
|
|
|
@ -72,8 +72,8 @@ function getByteRangeFromSpec(rangeSpec, objectSize) {
|
|||
if (rangeSpec.start < objectSize) {
|
||||
// test is false if end is undefined
|
||||
return { range: [rangeSpec.start,
|
||||
(rangeSpec.end < objectSize ?
|
||||
rangeSpec.end : objectSize - 1)] };
|
||||
(rangeSpec.end < objectSize ?
|
||||
rangeSpec.end : objectSize - 1)] };
|
||||
}
|
||||
return { error: errors.InvalidRange };
|
||||
}
|
||||
|
|
|
@ -90,8 +90,8 @@ function _negotiateProtocolVersion(client, logger, cb) {
|
|||
if (err) {
|
||||
const error = _arsenalError(err);
|
||||
logger.error('KMIP::negotiateProtocolVersion',
|
||||
{ error,
|
||||
vendorIdentification: client.vendorIdentification });
|
||||
{ error,
|
||||
vendorIdentification: client.vendorIdentification });
|
||||
return cb(error);
|
||||
}
|
||||
const majorVersions =
|
||||
|
@ -102,8 +102,8 @@ function _negotiateProtocolVersion(client, logger, cb) {
|
|||
majorVersions.length !== minorVersions.length) {
|
||||
const error = _arsenalError('No suitable protocol version');
|
||||
logger.error('KMIP::negotiateProtocolVersion',
|
||||
{ error,
|
||||
vendorIdentification: client.vendorIdentification });
|
||||
{ error,
|
||||
vendorIdentification: client.vendorIdentification });
|
||||
return cb(error);
|
||||
}
|
||||
client.kmip.changeProtocolVersion(majorVersions[0], minorVersions[0]);
|
||||
|
@ -126,8 +126,8 @@ function _mapExtensions(client, logger, cb) {
|
|||
if (err) {
|
||||
const error = _arsenalError(err);
|
||||
logger.error('KMIP::mapExtensions',
|
||||
{ error,
|
||||
vendorIdentification: client.vendorIdentification });
|
||||
{ error,
|
||||
vendorIdentification: client.vendorIdentification });
|
||||
return cb(error);
|
||||
}
|
||||
const extensionNames = response.lookup(searchFilter.extensionName);
|
||||
|
@ -135,8 +135,8 @@ function _mapExtensions(client, logger, cb) {
|
|||
if (extensionNames.length !== extensionTags.length) {
|
||||
const error = _arsenalError('Inconsistent extension list');
|
||||
logger.error('KMIP::mapExtensions',
|
||||
{ error,
|
||||
vendorIdentification: client.vendorIdentification });
|
||||
{ error,
|
||||
vendorIdentification: client.vendorIdentification });
|
||||
return cb(error);
|
||||
}
|
||||
extensionNames.forEach((extensionName, idx) => {
|
||||
|
@ -160,7 +160,7 @@ function _queryServerInformation(client, logger, cb) {
|
|||
if (err) {
|
||||
const error = _arsenalError(err);
|
||||
logger.warn('KMIP::queryServerInformation',
|
||||
{ error });
|
||||
{ error });
|
||||
/* no error returned, caller can keep going */
|
||||
return cb();
|
||||
}
|
||||
|
@ -170,9 +170,9 @@ function _queryServerInformation(client, logger, cb) {
|
|||
JSON.stringify(response.lookup(searchFilter.serverInformation)[0]));
|
||||
|
||||
logger.info('KMIP Server identified',
|
||||
{ vendorIdentification: client.vendorIdentification,
|
||||
serverInformation: client.serverInformation,
|
||||
negotiatedProtocolVersion: client.kmip.protocolVersion });
|
||||
{ vendorIdentification: client.vendorIdentification,
|
||||
serverInformation: client.serverInformation,
|
||||
negotiatedProtocolVersion: client.kmip.protocolVersion });
|
||||
return cb();
|
||||
});
|
||||
}
|
||||
|
@ -196,8 +196,8 @@ function _queryOperationsAndObjects(client, logger, cb) {
|
|||
if (err) {
|
||||
const error = _arsenalError(err);
|
||||
logger.error('KMIP::queryOperationsAndObjects',
|
||||
{ error,
|
||||
vendorIdentification: client.vendorIdentification });
|
||||
{ error,
|
||||
vendorIdentification: client.vendorIdentification });
|
||||
return cb(error);
|
||||
}
|
||||
const supportedOperations = response.lookup(searchFilter.operation);
|
||||
|
@ -222,15 +222,15 @@ function _queryOperationsAndObjects(client, logger, cb) {
|
|||
logger.warn('KMIP::queryOperationsAndObjects: ' +
|
||||
'The KMIP Server announces that it ' +
|
||||
'does not support all of the required features',
|
||||
{ vendorIdentification: client.vendorIdentification,
|
||||
serverInformation: client.serverInformation,
|
||||
supportsEncrypt, supportsDecrypt,
|
||||
supportsActivate, supportsRevoke,
|
||||
supportsCreate, supportsDestroy,
|
||||
supportsQuery, supportsSymmetricKeys });
|
||||
{ vendorIdentification: client.vendorIdentification,
|
||||
serverInformation: client.serverInformation,
|
||||
supportsEncrypt, supportsDecrypt,
|
||||
supportsActivate, supportsRevoke,
|
||||
supportsCreate, supportsDestroy,
|
||||
supportsQuery, supportsSymmetricKeys });
|
||||
} else {
|
||||
logger.info('KMIP Server provides the necessary feature set',
|
||||
{ vendorIdentification: client.vendorIdentification });
|
||||
{ vendorIdentification: client.vendorIdentification });
|
||||
}
|
||||
return cb();
|
||||
});
|
||||
|
@ -264,8 +264,8 @@ class Client {
|
|||
this.vendorIdentification = '';
|
||||
this.serverInformation = [];
|
||||
this.kmip = new KMIP(CodecClass || TTLVCodec,
|
||||
TransportClass || TlsTransport,
|
||||
options);
|
||||
TransportClass || TlsTransport,
|
||||
options);
|
||||
this.kmip.registerHandshakeFunction((logger, cb) => {
|
||||
this._kmipHandshake(logger, cb);
|
||||
});
|
||||
|
@ -322,8 +322,8 @@ class Client {
|
|||
if (err) {
|
||||
const error = _arsenalError(err);
|
||||
logger.error('KMIP::_activateBucketKey',
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
return cb(error);
|
||||
}
|
||||
const uniqueIdentifier =
|
||||
|
@ -332,7 +332,7 @@ class Client {
|
|||
const error = _arsenalError(
|
||||
'Server did not return the expected identifier');
|
||||
logger.error('KMIP::cipherDataKey',
|
||||
{ error, uniqueIdentifier });
|
||||
{ error, uniqueIdentifier });
|
||||
return cb(error);
|
||||
}
|
||||
return cb(null, keyIdentifier);
|
||||
|
@ -351,20 +351,20 @@ class Client {
|
|||
const attributes = [];
|
||||
if (!!this.options.bucketNameAttributeName) {
|
||||
attributes.push(KMIP.Attribute('TextString',
|
||||
this.options.bucketNameAttributeName,
|
||||
bucketName));
|
||||
this.options.bucketNameAttributeName,
|
||||
bucketName));
|
||||
}
|
||||
attributes.push(...[
|
||||
KMIP.Attribute('Enumeration', 'Cryptographic Algorithm',
|
||||
CRYPTOGRAPHIC_ALGORITHM),
|
||||
CRYPTOGRAPHIC_ALGORITHM),
|
||||
KMIP.Attribute('Integer', 'Cryptographic Length',
|
||||
CRYPTOGRAPHIC_LENGTH),
|
||||
CRYPTOGRAPHIC_LENGTH),
|
||||
KMIP.Attribute('Integer', 'Cryptographic Usage Mask',
|
||||
this.kmip.encodeMask('Cryptographic Usage Mask',
|
||||
CRYPTOGRAPHIC_USAGE_MASK))]);
|
||||
this.kmip.encodeMask('Cryptographic Usage Mask',
|
||||
CRYPTOGRAPHIC_USAGE_MASK))]);
|
||||
if (this.options.compoundCreateActivate) {
|
||||
attributes.push(KMIP.Attribute('Date-Time', 'Activation Date',
|
||||
new Date(Date.UTC())));
|
||||
new Date(Date.UTC())));
|
||||
}
|
||||
|
||||
return this.kmip.request(logger, 'Create', [
|
||||
|
@ -374,8 +374,8 @@ class Client {
|
|||
if (err) {
|
||||
const error = _arsenalError(err);
|
||||
logger.error('KMIP::createBucketKey',
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
return cb(error);
|
||||
}
|
||||
const createdObjectType =
|
||||
|
@ -386,7 +386,7 @@ class Client {
|
|||
const error = _arsenalError(
|
||||
'Server created an object of wrong type');
|
||||
logger.error('KMIP::createBucketKey',
|
||||
{ error, createdObjectType });
|
||||
{ error, createdObjectType });
|
||||
return cb(error);
|
||||
}
|
||||
if (!this.options.compoundCreateActivate) {
|
||||
|
@ -411,16 +411,16 @@ class Client {
|
|||
KMIP.TextString('Unique Identifier', bucketKeyId),
|
||||
KMIP.Structure('Revocation Reason', [
|
||||
KMIP.Enumeration('Revocation Reason Code',
|
||||
'Cessation of Operation'),
|
||||
'Cessation of Operation'),
|
||||
KMIP.TextString('Revocation Message',
|
||||
'About to be deleted'),
|
||||
'About to be deleted'),
|
||||
]),
|
||||
], (err, response) => {
|
||||
if (err) {
|
||||
const error = _arsenalError(err);
|
||||
logger.error('KMIP::_revokeBucketKey',
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
return cb(error);
|
||||
}
|
||||
const uniqueIdentifier =
|
||||
|
@ -429,7 +429,7 @@ class Client {
|
|||
const error = _arsenalError(
|
||||
'Server did not return the expected identifier');
|
||||
logger.error('KMIP::_revokeBucketKey',
|
||||
{ error, uniqueIdentifier });
|
||||
{ error, uniqueIdentifier });
|
||||
return cb(error);
|
||||
}
|
||||
return cb();
|
||||
|
@ -448,8 +448,8 @@ class Client {
|
|||
if (err) {
|
||||
const error = _arsenalError(err);
|
||||
logger.error('KMIP::destroyBucketKey: revocation failed',
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
return cb(error);
|
||||
}
|
||||
return this.kmip.request(logger, 'Destroy', [
|
||||
|
@ -458,8 +458,8 @@ class Client {
|
|||
if (err) {
|
||||
const error = _arsenalError(err);
|
||||
logger.error('KMIP::destroyBucketKey',
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
return cb(error);
|
||||
}
|
||||
const uniqueIdentifier =
|
||||
|
@ -468,7 +468,7 @@ class Client {
|
|||
const error = _arsenalError(
|
||||
'Server did not return the expected identifier');
|
||||
logger.error('KMIP::destroyBucketKey',
|
||||
{ error, uniqueIdentifier });
|
||||
{ error, uniqueIdentifier });
|
||||
return cb(error);
|
||||
}
|
||||
return cb();
|
||||
|
@ -487,19 +487,19 @@ class Client {
|
|||
* @callback called with (err, cipheredDataKey: Buffer)
|
||||
*/
|
||||
cipherDataKey(cryptoScheme,
|
||||
masterKeyId,
|
||||
plainTextDataKey,
|
||||
logger,
|
||||
cb) {
|
||||
masterKeyId,
|
||||
plainTextDataKey,
|
||||
logger,
|
||||
cb) {
|
||||
return this.kmip.request(logger, 'Encrypt', [
|
||||
KMIP.TextString('Unique Identifier', masterKeyId),
|
||||
KMIP.Structure('Cryptographic Parameters', [
|
||||
KMIP.Enumeration('Block Cipher Mode',
|
||||
CRYPTOGRAPHIC_CIPHER_MODE),
|
||||
CRYPTOGRAPHIC_CIPHER_MODE),
|
||||
KMIP.Enumeration('Padding Method',
|
||||
CRYPTOGRAPHIC_PADDING_METHOD),
|
||||
CRYPTOGRAPHIC_PADDING_METHOD),
|
||||
KMIP.Enumeration('Cryptographic Algorithm',
|
||||
CRYPTOGRAPHIC_ALGORITHM),
|
||||
CRYPTOGRAPHIC_ALGORITHM),
|
||||
]),
|
||||
KMIP.ByteString('Data', plainTextDataKey),
|
||||
KMIP.ByteString('IV/Counter/Nonce', CRYPTOGRAPHIC_DEFAULT_IV),
|
||||
|
@ -507,8 +507,8 @@ class Client {
|
|||
if (err) {
|
||||
const error = _arsenalError(err);
|
||||
logger.error('KMIP::cipherDataKey',
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
return cb(error);
|
||||
}
|
||||
const uniqueIdentifier =
|
||||
|
@ -518,7 +518,7 @@ class Client {
|
|||
const error = _arsenalError(
|
||||
'Server did not return the expected identifier');
|
||||
logger.error('KMIP::cipherDataKey',
|
||||
{ error, uniqueIdentifier });
|
||||
{ error, uniqueIdentifier });
|
||||
return cb(error);
|
||||
}
|
||||
return cb(null, data);
|
||||
|
@ -536,19 +536,19 @@ class Client {
|
|||
* @callback called with (err, plainTextDataKey: Buffer)
|
||||
*/
|
||||
decipherDataKey(cryptoScheme,
|
||||
masterKeyId,
|
||||
cipheredDataKey,
|
||||
logger,
|
||||
cb) {
|
||||
masterKeyId,
|
||||
cipheredDataKey,
|
||||
logger,
|
||||
cb) {
|
||||
return this.kmip.request(logger, 'Decrypt', [
|
||||
KMIP.TextString('Unique Identifier', masterKeyId),
|
||||
KMIP.Structure('Cryptographic Parameters', [
|
||||
KMIP.Enumeration('Block Cipher Mode',
|
||||
CRYPTOGRAPHIC_CIPHER_MODE),
|
||||
CRYPTOGRAPHIC_CIPHER_MODE),
|
||||
KMIP.Enumeration('Padding Method',
|
||||
CRYPTOGRAPHIC_PADDING_METHOD),
|
||||
CRYPTOGRAPHIC_PADDING_METHOD),
|
||||
KMIP.Enumeration('Cryptographic Algorithm',
|
||||
CRYPTOGRAPHIC_ALGORITHM),
|
||||
CRYPTOGRAPHIC_ALGORITHM),
|
||||
]),
|
||||
KMIP.ByteString('Data', cipheredDataKey),
|
||||
KMIP.ByteString('IV/Counter/Nonce', CRYPTOGRAPHIC_DEFAULT_IV),
|
||||
|
@ -556,8 +556,8 @@ class Client {
|
|||
if (err) {
|
||||
const error = _arsenalError(err);
|
||||
logger.error('KMIP::decipherDataKey',
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
{ error,
|
||||
serverInformation: this.serverInformation });
|
||||
return cb(error);
|
||||
}
|
||||
const uniqueIdentifier =
|
||||
|
@ -567,7 +567,7 @@ class Client {
|
|||
const error = _arsenalError(
|
||||
'Server did not return the right identifier');
|
||||
logger.error('KMIP::decipherDataKey',
|
||||
{ error, uniqueIdentifier });
|
||||
{ error, uniqueIdentifier });
|
||||
return cb(error);
|
||||
}
|
||||
return cb(null, data);
|
||||
|
|
|
@ -55,15 +55,15 @@ function TTLVCodec() {
|
|||
const property = {};
|
||||
if (!TypeDecoder[elementType]) {
|
||||
_throwError(logger,
|
||||
'Unknown element type',
|
||||
{ funcName, elementTag, elementType });
|
||||
'Unknown element type',
|
||||
{ funcName, elementTag, elementType });
|
||||
}
|
||||
const elementValue = value.slice(i + 8,
|
||||
i + 8 + elementLength);
|
||||
i + 8 + elementLength);
|
||||
if (elementValue.length !== elementLength) {
|
||||
_throwError(logger, 'BUG: Wrong buffer size',
|
||||
{ funcName, elementLength,
|
||||
bufferLength: elementValue.length });
|
||||
{ funcName, elementLength,
|
||||
bufferLength: elementValue.length });
|
||||
}
|
||||
property.type = TypeDecoder[elementType].name;
|
||||
property.value = TypeDecoder[elementType]
|
||||
|
@ -75,7 +75,7 @@ function TTLVCodec() {
|
|||
const tagInfo = TagDecoder[elementTag];
|
||||
if (!tagInfo) {
|
||||
logger.debug('Unknown element tag',
|
||||
{ funcName, elementTag });
|
||||
{ funcName, elementTag });
|
||||
property.tag = elementTag;
|
||||
element['Unknown Tag'] = property;
|
||||
} else {
|
||||
|
@ -83,8 +83,8 @@ function TTLVCodec() {
|
|||
if (tagInfo.name === 'Attribute Name') {
|
||||
if (property.type !== 'TextString') {
|
||||
_throwError(logger,
|
||||
'Invalide type',
|
||||
{ funcName, type: property.type });
|
||||
'Invalide type',
|
||||
{ funcName, type: property.type });
|
||||
}
|
||||
diversion = property.value;
|
||||
}
|
||||
|
@ -114,8 +114,8 @@ function TTLVCodec() {
|
|||
}
|
||||
const itemResult =
|
||||
TypeEncoder[itemType].encode(itemTagName,
|
||||
itemValue,
|
||||
itemDiversion);
|
||||
itemValue,
|
||||
itemDiversion);
|
||||
encodedValue = encodedValue
|
||||
.concat(_ttlvPadVector(itemResult));
|
||||
});
|
||||
|
@ -133,9 +133,9 @@ function TTLVCodec() {
|
|||
const fixedLength = 4;
|
||||
if (fixedLength !== value.length) {
|
||||
_throwError(logger,
|
||||
'Length mismatch',
|
||||
{ funcName, fixedLength,
|
||||
bufferLength: value.length });
|
||||
'Length mismatch',
|
||||
{ funcName, fixedLength,
|
||||
bufferLength: value.length });
|
||||
}
|
||||
return value.readUInt32BE(0);
|
||||
},
|
||||
|
@ -156,16 +156,16 @@ function TTLVCodec() {
|
|||
const fixedLength = 8;
|
||||
if (fixedLength !== value.length) {
|
||||
_throwError(logger,
|
||||
'Length mismatch',
|
||||
{ funcName, fixedLength,
|
||||
bufferLength: value.length });
|
||||
'Length mismatch',
|
||||
{ funcName, fixedLength,
|
||||
bufferLength: value.length });
|
||||
}
|
||||
const longUInt = UINT32_MAX * value.readUInt32BE(0) +
|
||||
value.readUInt32BE(4);
|
||||
if (longUInt > Number.MAX_SAFE_INTEGER) {
|
||||
_throwError(logger,
|
||||
'53-bit overflow',
|
||||
{ funcName, longUInt });
|
||||
'53-bit overflow',
|
||||
{ funcName, longUInt });
|
||||
}
|
||||
return longUInt;
|
||||
},
|
||||
|
@ -200,9 +200,9 @@ function TTLVCodec() {
|
|||
const fixedLength = 4;
|
||||
if (fixedLength !== value.length) {
|
||||
_throwError(logger,
|
||||
'Length mismatch',
|
||||
{ funcName, fixedLength,
|
||||
bufferLength: value.length });
|
||||
'Length mismatch',
|
||||
{ funcName, fixedLength,
|
||||
bufferLength: value.length });
|
||||
}
|
||||
const enumValue = value.toString('hex');
|
||||
const actualTag = diversion ? TagEncoder[diversion].value : tag;
|
||||
|
@ -211,10 +211,10 @@ function TTLVCodec() {
|
|||
!enumInfo.enumeration ||
|
||||
!enumInfo.enumeration[enumValue]) {
|
||||
return { tag,
|
||||
value: enumValue,
|
||||
message: 'Unknown enumeration value',
|
||||
diversion,
|
||||
};
|
||||
value: enumValue,
|
||||
message: 'Unknown enumeration value',
|
||||
diversion,
|
||||
};
|
||||
}
|
||||
return enumInfo.enumeration[enumValue];
|
||||
},
|
||||
|
@ -227,7 +227,7 @@ function TTLVCodec() {
|
|||
const actualTag = diversion || tagName;
|
||||
const encodedValue =
|
||||
Buffer.from(TagEncoder[actualTag].enumeration[value],
|
||||
'hex');
|
||||
'hex');
|
||||
return _ttlvPadVector([tag, type, length, encodedValue]);
|
||||
},
|
||||
},
|
||||
|
@ -238,9 +238,9 @@ function TTLVCodec() {
|
|||
const fixedLength = 8;
|
||||
if (fixedLength !== value.length) {
|
||||
_throwError(logger,
|
||||
'Length mismatch',
|
||||
{ funcName, fixedLength,
|
||||
bufferLength: value.length });
|
||||
'Length mismatch',
|
||||
{ funcName, fixedLength,
|
||||
bufferLength: value.length });
|
||||
}
|
||||
const msUInt = value.readUInt32BE(0);
|
||||
const lsUInt = value.readUInt32BE(4);
|
||||
|
@ -267,7 +267,7 @@ function TTLVCodec() {
|
|||
const length = Buffer.alloc(4);
|
||||
length.writeUInt32BE(value.length);
|
||||
return _ttlvPadVector([tag, type, length,
|
||||
Buffer.from(value, 'utf8')]);
|
||||
Buffer.from(value, 'utf8')]);
|
||||
},
|
||||
},
|
||||
'08': {
|
||||
|
@ -289,17 +289,17 @@ function TTLVCodec() {
|
|||
const fixedLength = 8;
|
||||
if (fixedLength !== value.length) {
|
||||
_throwError(logger,
|
||||
'Length mismatch',
|
||||
{ funcName, fixedLength,
|
||||
bufferLength: value.length });
|
||||
'Length mismatch',
|
||||
{ funcName, fixedLength,
|
||||
bufferLength: value.length });
|
||||
}
|
||||
const d = new Date(0);
|
||||
const utcSeconds = UINT32_MAX * value.readUInt32BE(0) +
|
||||
value.readUInt32BE(4);
|
||||
if (utcSeconds > Number.MAX_SAFE_INTEGER) {
|
||||
_throwError(logger,
|
||||
'53-bit overflow',
|
||||
{ funcName, utcSeconds });
|
||||
'53-bit overflow',
|
||||
{ funcName, utcSeconds });
|
||||
}
|
||||
d.setUTCSeconds(utcSeconds);
|
||||
return d;
|
||||
|
@ -323,9 +323,9 @@ function TTLVCodec() {
|
|||
const fixedLength = 4;
|
||||
if (fixedLength !== value.length) {
|
||||
_throwError(logger,
|
||||
'Length mismatch',
|
||||
{ funcName, fixedLength,
|
||||
bufferLength: value.length });
|
||||
'Length mismatch',
|
||||
{ funcName, fixedLength,
|
||||
bufferLength: value.length });
|
||||
}
|
||||
return value.readInt32BE(0);
|
||||
},
|
||||
|
@ -415,8 +415,8 @@ function TTLVCodec() {
|
|||
throw Error(`Unknown Type '${type}'`);
|
||||
}
|
||||
const itemValue = TypeEncoder[type].encode(key,
|
||||
item[key].value,
|
||||
item[key].diversion);
|
||||
item[key].value,
|
||||
item[key].diversion);
|
||||
result = result.concat(_ttlvPadVector(itemValue));
|
||||
});
|
||||
});
|
||||
|
|
|
@ -275,11 +275,11 @@ class KMIP {
|
|||
KMIP.Structure('Request Header', [
|
||||
KMIP.Structure('Protocol Version', [
|
||||
KMIP.Integer('Protocol Version Major',
|
||||
this.protocolVersion.major),
|
||||
this.protocolVersion.major),
|
||||
KMIP.Integer('Protocol Version Minor',
|
||||
this.protocolVersion.minor)]),
|
||||
this.protocolVersion.minor)]),
|
||||
KMIP.Integer('Maximum Response Size',
|
||||
this.maximumResponseSize),
|
||||
this.maximumResponseSize),
|
||||
KMIP.Integer('Batch Count', 1)]),
|
||||
KMIP.Structure('Batch Item', [
|
||||
KMIP.Enumeration('Operation', operation),
|
||||
|
@ -292,7 +292,7 @@ class KMIP {
|
|||
(err, conversation, rawResponse) => {
|
||||
if (err) {
|
||||
logger.error('KMIP::request: Failed to send message',
|
||||
{ error: err });
|
||||
{ error: err });
|
||||
return cb(err);
|
||||
}
|
||||
const response = this._decodeMessage(logger, rawResponse);
|
||||
|
@ -311,16 +311,16 @@ class KMIP {
|
|||
this.transport.abortPipeline(conversation);
|
||||
const error = Error('Invalid batch item ID returned');
|
||||
logger.error('KMIP::request: failed',
|
||||
{ resultUniqueBatchItemID, uuid, error });
|
||||
{ resultUniqueBatchItemID, uuid, error });
|
||||
return cb(error);
|
||||
}
|
||||
if (performedOperation !== operation) {
|
||||
this.transport.abortPipeline(conversation);
|
||||
const error = Error('Operation mismatch',
|
||||
{ got: performedOperation,
|
||||
expected: operation });
|
||||
{ got: performedOperation,
|
||||
expected: operation });
|
||||
logger.error('KMIP::request: Operation mismatch',
|
||||
{ error });
|
||||
{ error });
|
||||
return cb(error);
|
||||
}
|
||||
if (resultStatus !== 'Success') {
|
||||
|
@ -331,19 +331,17 @@ class KMIP {
|
|||
response.lookup(
|
||||
'Response Message/Batch Item/Result Message')[0];
|
||||
const error = Error('KMIP request failure',
|
||||
{ resultStatus,
|
||||
resultReason,
|
||||
resultMessage });
|
||||
{ resultStatus,
|
||||
resultReason,
|
||||
resultMessage });
|
||||
logger.error('KMIP::request: request failed',
|
||||
{ error, resultStatus,
|
||||
resultReason, resultMessage });
|
||||
{ error, resultStatus,
|
||||
resultReason, resultMessage });
|
||||
return cb(error);
|
||||
}
|
||||
return cb(null, response);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -86,8 +86,8 @@ class TransportTemplate {
|
|||
const deferedRequest = this.deferedRequests.shift();
|
||||
process.nextTick(() => {
|
||||
this.send(logger,
|
||||
deferedRequest.encodedMessage,
|
||||
deferedRequest.cb);
|
||||
deferedRequest.encodedMessage,
|
||||
deferedRequest.cb);
|
||||
});
|
||||
} else if (this.callbackPipeline.length === 0 &&
|
||||
this.deferedRequests.length === 0 &&
|
||||
|
|
|
@ -26,7 +26,7 @@ function sendError(res, log, error, optMessage) {
|
|||
httpCode: error.code,
|
||||
errorType: error.message,
|
||||
error: message,
|
||||
}
|
||||
},
|
||||
);
|
||||
res.writeHead(error.code);
|
||||
res.end(JSON.stringify({
|
||||
|
|
|
@ -19,7 +19,7 @@ function setContentRange(response, byteRange, objectSize) {
|
|||
const [start, end] = byteRange;
|
||||
assert(start !== undefined && end !== undefined);
|
||||
response.setHeader('Content-Range',
|
||||
`bytes ${start}-${end}/${objectSize}`);
|
||||
`bytes ${start}-${end}/${objectSize}`);
|
||||
}
|
||||
|
||||
function sendError(res, log, error, optMessage) {
|
||||
|
@ -45,7 +45,6 @@ function sendError(res, log, error, optMessage) {
|
|||
* start() to start listening to the configured port.
|
||||
*/
|
||||
class RESTServer extends httpServer {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
* @param {Object} params - constructor params
|
||||
|
@ -227,7 +226,7 @@ class RESTServer extends httpServer {
|
|||
return sendError(res, log, err);
|
||||
}
|
||||
log.debug('sending back 200/206 response with contents',
|
||||
{ key: pathInfo.key });
|
||||
{ key: pathInfo.key });
|
||||
setContentLength(res, contentLength);
|
||||
res.setHeader('Accept-Ranges', 'bytes');
|
||||
if (byteRange) {
|
||||
|
@ -265,7 +264,7 @@ class RESTServer extends httpServer {
|
|||
return sendError(res, log, err);
|
||||
}
|
||||
log.debug('sending back 204 response to DELETE',
|
||||
{ key: pathInfo.key });
|
||||
{ key: pathInfo.key });
|
||||
res.writeHead(204);
|
||||
return res.end(() => {
|
||||
log.debug('DELETE response sent', { key: pathInfo.key });
|
||||
|
|
|
@ -19,7 +19,7 @@ function explodePath(path) {
|
|||
return {
|
||||
service: pathMatch[1],
|
||||
key: (pathMatch[3] !== undefined && pathMatch[3].length > 0 ?
|
||||
pathMatch[3] : undefined),
|
||||
pathMatch[3] : undefined),
|
||||
};
|
||||
}
|
||||
throw errors.InvalidURI.customizeDescription('malformed URI');
|
||||
|
|
|
@ -17,7 +17,6 @@ const rpc = require('./rpc.js');
|
|||
* RPC client object accessing the sub-level transparently.
|
||||
*/
|
||||
class LevelDbClient extends rpc.BaseClient {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
*
|
||||
|
@ -78,7 +77,6 @@ class LevelDbClient extends rpc.BaseClient {
|
|||
* env.subDb (env is passed as first parameter of received RPC calls).
|
||||
*/
|
||||
class LevelDbService extends rpc.BaseService {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
*
|
||||
|
|
|
@ -37,7 +37,6 @@ let streamRPCJSONObj;
|
|||
* an error occurred).
|
||||
*/
|
||||
class BaseClient extends EventEmitter {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
*
|
||||
|
@ -54,7 +53,7 @@ class BaseClient extends EventEmitter {
|
|||
*/
|
||||
constructor(params) {
|
||||
const { url, logger, callTimeoutMs,
|
||||
streamMaxPendingAck, streamAckTimeoutMs } = params;
|
||||
streamMaxPendingAck, streamAckTimeoutMs } = params;
|
||||
assert(url);
|
||||
assert(logger);
|
||||
|
||||
|
@ -82,11 +81,11 @@ class BaseClient extends EventEmitter {
|
|||
_call(remoteCall, args, cb) {
|
||||
const wrapCb = (err, data) => {
|
||||
cb(reconstructError(err),
|
||||
this.socketStreams.decodeStreams(data));
|
||||
this.socketStreams.decodeStreams(data));
|
||||
};
|
||||
this.logger.debug('remote call', { remoteCall, args });
|
||||
this.socket.emit('call', remoteCall,
|
||||
this.socketStreams.encodeStreams(args), wrapCb);
|
||||
this.socketStreams.encodeStreams(args), wrapCb);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
|
@ -113,8 +112,8 @@ class BaseClient extends EventEmitter {
|
|||
throw new Error(`argument cb=${cb} is not a callback`);
|
||||
}
|
||||
async.timeout(this._call.bind(this), timeoutMs,
|
||||
`operation ${remoteCall} timed out`)(remoteCall,
|
||||
args, cb);
|
||||
`operation ${remoteCall} timed out`)(remoteCall,
|
||||
args, cb);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
|
@ -142,7 +141,7 @@ class BaseClient extends EventEmitter {
|
|||
const url = this.url;
|
||||
this.socket.on('error', err => {
|
||||
this.logger.warn('connectivity error to the RPC service',
|
||||
{ url, error: err });
|
||||
{ url, error: err });
|
||||
});
|
||||
this.socket.on('connect', () => {
|
||||
this.emit('connect');
|
||||
|
@ -156,7 +155,7 @@ class BaseClient extends EventEmitter {
|
|||
this.getManifest((err, manifest) => {
|
||||
if (err) {
|
||||
this.logger.error('Error fetching manifest from RPC server',
|
||||
{ error: err });
|
||||
{ error: err });
|
||||
} else {
|
||||
manifest.api.forEach(apiItem => {
|
||||
this.createCall(apiItem.name);
|
||||
|
@ -251,7 +250,6 @@ class BaseClient extends EventEmitter {
|
|||
*
|
||||
*/
|
||||
class BaseService {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
*
|
||||
|
@ -497,7 +495,7 @@ function RPCServer(params) {
|
|||
|
||||
conn.on('error', err => {
|
||||
log.error('error on socket.io connection',
|
||||
{ namespace: service.namespace, error: err });
|
||||
{ namespace: service.namespace, error: err });
|
||||
});
|
||||
conn.on('call', (remoteCall, args, cb) => {
|
||||
const decodedArgs = streamsSocket.decodeStreams(args);
|
||||
|
@ -647,8 +645,8 @@ streamRPCJSONObj = function _streamRPCJSONObj(obj, wstream, cb) {
|
|||
// primitive types
|
||||
if (obj === undefined) {
|
||||
wstream.write('null'); // if undefined elements are present in
|
||||
// arrays, convert them to JSON null
|
||||
// objects
|
||||
// arrays, convert them to JSON null
|
||||
// objects
|
||||
} else {
|
||||
wstream.write(JSON.stringify(obj));
|
||||
}
|
||||
|
|
|
@ -16,7 +16,7 @@ class SIOOutputStream extends stream.Writable {
|
|||
constructor(socket, streamId, maxPendingAck, ackTimeoutMs) {
|
||||
super({ objectMode: true });
|
||||
this._initOutputStream(socket, streamId, maxPendingAck,
|
||||
ackTimeoutMs);
|
||||
ackTimeoutMs);
|
||||
}
|
||||
|
||||
_initOutputStream(socket, streamId, maxPendingAck, ackTimeoutMs) {
|
||||
|
@ -194,7 +194,7 @@ class SIOStreamSocket {
|
|||
this.socket.on('stream-data', (payload, cb) => {
|
||||
const { streamId, data } = payload;
|
||||
log.debug('received \'stream-data\' event',
|
||||
{ streamId, size: data.length });
|
||||
{ streamId, size: data.length });
|
||||
const stream = this.remoteStreams[streamId];
|
||||
if (!stream) {
|
||||
log.debug('no such remote stream registered', { streamId });
|
||||
|
@ -280,15 +280,15 @@ class SIOStreamSocket {
|
|||
let transportStream;
|
||||
if (isReadStream) {
|
||||
transportStream = new SIOOutputStream(this, streamId,
|
||||
this.maxPendingAck,
|
||||
this.ackTimeoutMs);
|
||||
this.maxPendingAck,
|
||||
this.ackTimeoutMs);
|
||||
} else {
|
||||
transportStream = new SIOInputStream(this, streamId);
|
||||
}
|
||||
this.localStreams[streamId] = arg;
|
||||
arg.once('close', () => {
|
||||
log.debug('stream closed, removing from local streams',
|
||||
{ streamId });
|
||||
{ streamId });
|
||||
delete this.localStreams[streamId];
|
||||
});
|
||||
arg.on('error', error => {
|
||||
|
@ -350,8 +350,8 @@ class SIOStreamSocket {
|
|||
stream = new SIOInputStream(this, streamId);
|
||||
} else if (arg.writable) {
|
||||
stream = new SIOOutputStream(this, streamId,
|
||||
this.maxPendingAck,
|
||||
this.ackTimeoutMs);
|
||||
this.maxPendingAck,
|
||||
this.ackTimeoutMs);
|
||||
} else {
|
||||
throw new Error('can\'t decode stream neither readable ' +
|
||||
'nor writable');
|
||||
|
@ -360,14 +360,14 @@ class SIOStreamSocket {
|
|||
if (arg.readable) {
|
||||
stream.once('close', () => {
|
||||
log.debug('stream closed, removing from remote streams',
|
||||
{ streamId });
|
||||
{ streamId });
|
||||
delete this.remoteStreams[streamId];
|
||||
});
|
||||
}
|
||||
if (arg.writable) {
|
||||
stream.once('finish', () => {
|
||||
log.debug('stream finished, removing from remote streams',
|
||||
{ streamId });
|
||||
{ streamId });
|
||||
delete this.remoteStreams[streamId];
|
||||
});
|
||||
}
|
||||
|
@ -399,7 +399,7 @@ class SIOStreamSocket {
|
|||
|
||||
_write(streamId, data, cb) {
|
||||
this.logger.debug('emit \'stream-data\' event',
|
||||
{ streamId, size: data.length });
|
||||
{ streamId, size: data.length });
|
||||
this.socket.emit('stream-data', { streamId, data }, cb);
|
||||
}
|
||||
|
||||
|
|
|
@ -50,7 +50,7 @@ evaluators.isResourceApplicable = (requestContext, statementResource, log) => {
|
|||
requestResourceArr, true);
|
||||
if (arnSegmentsMatch) {
|
||||
log.trace('policy resource is applicable to request',
|
||||
{ requestResource: resource, policyResource });
|
||||
{ requestResource: resource, policyResource });
|
||||
return true;
|
||||
}
|
||||
continue;
|
||||
|
@ -224,21 +224,21 @@ evaluators.evaluatePolicy = (requestContext, policy, log) => {
|
|||
// in policy, move on to next statement
|
||||
if (currentStatement.NotResource &&
|
||||
evaluators.isResourceApplicable(requestContext,
|
||||
currentStatement.NotResource, log)) {
|
||||
currentStatement.NotResource, log)) {
|
||||
continue;
|
||||
}
|
||||
// If affirmative action is in policy and request action is not
|
||||
// applicable, move on to next statement
|
||||
if (currentStatement.Action &&
|
||||
!evaluators.isActionApplicable(requestContext.getAction(),
|
||||
currentStatement.Action, log)) {
|
||||
currentStatement.Action, log)) {
|
||||
continue;
|
||||
}
|
||||
// If NotAction is in policy and action matches NotAction in policy,
|
||||
// move on to next statement
|
||||
if (currentStatement.NotAction &&
|
||||
evaluators.isActionApplicable(requestContext.getAction(),
|
||||
currentStatement.NotAction, log)) {
|
||||
currentStatement.NotAction, log)) {
|
||||
continue;
|
||||
}
|
||||
const conditionEval = currentStatement.Condition ?
|
||||
|
|
|
@ -39,11 +39,11 @@ conditions.findConditionKey = (key, requestContext) => {
|
|||
// (see Boolean Condition Operators).
|
||||
// Note: This key is only present if MFA was used. So, the following
|
||||
// will not work:
|
||||
// "Condition" :
|
||||
// { "Bool" : { "aws:MultiFactorAuthPresent" : false } }
|
||||
// "Condition" :
|
||||
// { "Bool" : { "aws:MultiFactorAuthPresent" : false } }
|
||||
// Instead use:
|
||||
// "Condition" :
|
||||
// { "Null" : { "aws:MultiFactorAuthPresent" : true } }
|
||||
// "Condition" :
|
||||
// { "Null" : { "aws:MultiFactorAuthPresent" : true } }
|
||||
map.set('aws:MultiFactorAuthPresent',
|
||||
requestContext.getMultiFactorAuthPresent());
|
||||
// aws:MultiFactorAuthAge – Used to check how many seconds since
|
||||
|
@ -166,8 +166,8 @@ conditions.findConditionKey = (key, requestContext) => {
|
|||
// so evaluation should be skipped
|
||||
map.set('s3:RequestObjectTagKeys',
|
||||
requestContext.getNeedTagEval() && requestContext.getRequestObjTags()
|
||||
? getTagKeys(requestContext.getRequestObjTags())
|
||||
: undefined);
|
||||
? getTagKeys(requestContext.getRequestObjTags())
|
||||
: undefined);
|
||||
return map.get(key);
|
||||
};
|
||||
|
||||
|
@ -191,7 +191,7 @@ function convertSpecialChars(string) {
|
|||
return map[char];
|
||||
}
|
||||
return string.replace(/(\$\{\*\})|(\$\{\?\})|(\$\{\$\})/g,
|
||||
characterMap);
|
||||
characterMap);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -425,10 +425,10 @@ conditions.convertConditionOperator = operator => {
|
|||
return !operatorMap.ArnLike(key, value);
|
||||
},
|
||||
Null: function nullOperator(key, value) {
|
||||
// Null is used to check if a condition key is present.
|
||||
// The policy statement value should be either true (the key doesn't
|
||||
// exist — it is null) or false (the key exists and its value is
|
||||
// not null).
|
||||
// Null is used to check if a condition key is present.
|
||||
// The policy statement value should be either true (the key doesn't
|
||||
// exist — it is null) or false (the key exists and its value is
|
||||
// not null).
|
||||
if ((key === undefined || key === null)
|
||||
&& value[0] === 'true' ||
|
||||
(key !== undefined && key !== null)
|
||||
|
|
|
@ -51,10 +51,10 @@ wildcards.handleWildcardInResource = arn => {
|
|||
// Wildcards can be part of the resource ARN.
|
||||
// Wildcards do NOT span segments of the ARN (separated by ":")
|
||||
|
||||
// Example: all elements in specific bucket:
|
||||
// "Resource": "arn:aws:s3:::my_corporate_bucket/*"
|
||||
// ARN format:
|
||||
// arn:partition:service:region:namespace:relative-id
|
||||
// Example: all elements in specific bucket:
|
||||
// "Resource": "arn:aws:s3:::my_corporate_bucket/*"
|
||||
// ARN format:
|
||||
// arn:partition:service:region:namespace:relative-id
|
||||
const arnArr = arn.split(':');
|
||||
return arnArr.map(portion => wildcards.handleWildcards(portion));
|
||||
};
|
||||
|
|
|
@ -6,7 +6,6 @@ const crypto = require('crypto');
|
|||
* data through a stream
|
||||
*/
|
||||
class MD5Sum extends Transform {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
*/
|
||||
|
@ -40,7 +39,6 @@ class MD5Sum extends Transform {
|
|||
this.emit('hashed');
|
||||
callback(null);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
module.exports = MD5Sum;
|
||||
|
|
|
@ -73,7 +73,7 @@ class ResultsCollector extends EventEmitter {
|
|||
* @property {Error} [results[].error] - error returned by Azure putting subpart
|
||||
* @property {number} results[].subPartIndex - index of the subpart
|
||||
*/
|
||||
/**
|
||||
/**
|
||||
* "error" event
|
||||
* @event ResultCollector#error
|
||||
* @type {(Error|undefined)} error - error returned by Azure last subpart
|
||||
|
|
|
@ -94,7 +94,7 @@ azureMpuUtils.getSubPartIds = (part, uploadId) =>
|
|||
azureMpuUtils.getBlockId(uploadId, part.partNumber, subPartIndex));
|
||||
|
||||
azureMpuUtils.putSinglePart = (errorWrapperFn, request, params, dataStoreName,
|
||||
log, cb) => {
|
||||
log, cb) => {
|
||||
const { bucketName, partNumber, size, objectKey, contentMD5, uploadId }
|
||||
= params;
|
||||
const blockId = azureMpuUtils.getBlockId(uploadId, partNumber, 0);
|
||||
|
@ -107,31 +107,31 @@ log, cb) => {
|
|||
request.pipe(passThrough);
|
||||
return errorWrapperFn('uploadPart', 'createBlockFromStream',
|
||||
[blockId, bucketName, objectKey, passThrough, size, options,
|
||||
(err, result) => {
|
||||
if (err) {
|
||||
log.error('Error from Azure data backend uploadPart',
|
||||
{ error: err.message, dataStoreName });
|
||||
if (err.code === 'ContainerNotFound') {
|
||||
return cb(errors.NoSuchBucket);
|
||||
(err, result) => {
|
||||
if (err) {
|
||||
log.error('Error from Azure data backend uploadPart',
|
||||
{ error: err.message, dataStoreName });
|
||||
if (err.code === 'ContainerNotFound') {
|
||||
return cb(errors.NoSuchBucket);
|
||||
}
|
||||
if (err.code === 'InvalidMd5') {
|
||||
return cb(errors.InvalidDigest);
|
||||
}
|
||||
if (err.code === 'Md5Mismatch') {
|
||||
return cb(errors.BadDigest);
|
||||
}
|
||||
return cb(errors.InternalError.customizeDescription(
|
||||
`Error returned from Azure: ${err.message}`),
|
||||
);
|
||||
}
|
||||
if (err.code === 'InvalidMd5') {
|
||||
return cb(errors.InvalidDigest);
|
||||
}
|
||||
if (err.code === 'Md5Mismatch') {
|
||||
return cb(errors.BadDigest);
|
||||
}
|
||||
return cb(errors.InternalError.customizeDescription(
|
||||
`Error returned from Azure: ${err.message}`)
|
||||
);
|
||||
}
|
||||
const md5 = result.headers['content-md5'] || '';
|
||||
const eTag = objectUtils.getHexMD5(md5);
|
||||
return cb(null, eTag, size);
|
||||
}], log, cb);
|
||||
const md5 = result.headers['content-md5'] || '';
|
||||
const eTag = objectUtils.getHexMD5(md5);
|
||||
return cb(null, eTag, size);
|
||||
}], log, cb);
|
||||
};
|
||||
|
||||
azureMpuUtils.putNextSubPart = (errorWrapperFn, partParams, subPartInfo,
|
||||
subPartStream, subPartIndex, resultsCollector, log, cb) => {
|
||||
subPartStream, subPartIndex, resultsCollector, log, cb) => {
|
||||
const { uploadId, partNumber, bucketName, objectKey } = partParams;
|
||||
const subPartSize = azureMpuUtils.getSubPartSize(
|
||||
subPartInfo, subPartIndex);
|
||||
|
@ -140,11 +140,11 @@ subPartStream, subPartIndex, resultsCollector, log, cb) => {
|
|||
resultsCollector.pushOp();
|
||||
errorWrapperFn('uploadPart', 'createBlockFromStream',
|
||||
[subPartId, bucketName, objectKey, subPartStream, subPartSize,
|
||||
{}, err => resultsCollector.pushResult(err, subPartIndex)], log, cb);
|
||||
{}, err => resultsCollector.pushResult(err, subPartIndex)], log, cb);
|
||||
};
|
||||
|
||||
azureMpuUtils.putSubParts = (errorWrapperFn, request, params,
|
||||
dataStoreName, log, cb) => {
|
||||
dataStoreName, log, cb) => {
|
||||
const subPartInfo = azureMpuUtils.getSubPartInfo(params.size);
|
||||
const resultsCollector = new ResultsCollector();
|
||||
const hashedStream = new MD5Sum();
|
||||
|
|
|
@ -31,9 +31,9 @@ convertMethods.listMultipartUploads = xmlParams => {
|
|||
const l = xmlParams.list;
|
||||
|
||||
xml.push('<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<ListMultipartUploadsResult ' +
|
||||
'<ListMultipartUploadsResult ' +
|
||||
'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">',
|
||||
`<Bucket>${escapeForXml(xmlParams.bucketName)}</Bucket>`
|
||||
`<Bucket>${escapeForXml(xmlParams.bucketName)}</Bucket>`,
|
||||
);
|
||||
|
||||
// For certain XML elements, if it is `undefined`, AWS returns either an
|
||||
|
@ -58,7 +58,7 @@ convertMethods.listMultipartUploads = xmlParams => {
|
|||
});
|
||||
|
||||
xml.push(`<MaxUploads>${escapeForXml(l.MaxKeys)}</MaxUploads>`,
|
||||
`<IsTruncated>${escapeForXml(l.IsTruncated)}</IsTruncated>`
|
||||
`<IsTruncated>${escapeForXml(l.IsTruncated)}</IsTruncated>`,
|
||||
);
|
||||
|
||||
l.Uploads.forEach(upload => {
|
||||
|
@ -69,29 +69,29 @@ convertMethods.listMultipartUploads = xmlParams => {
|
|||
}
|
||||
|
||||
xml.push('<Upload>',
|
||||
`<Key>${escapeForXml(key)}</Key>`,
|
||||
`<UploadId>${escapeForXml(val.UploadId)}</UploadId>`,
|
||||
'<Initiator>',
|
||||
`<ID>${escapeForXml(val.Initiator.ID)}</ID>`,
|
||||
`<DisplayName>${escapeForXml(val.Initiator.DisplayName)}` +
|
||||
`<Key>${escapeForXml(key)}</Key>`,
|
||||
`<UploadId>${escapeForXml(val.UploadId)}</UploadId>`,
|
||||
'<Initiator>',
|
||||
`<ID>${escapeForXml(val.Initiator.ID)}</ID>`,
|
||||
`<DisplayName>${escapeForXml(val.Initiator.DisplayName)}` +
|
||||
'</DisplayName>',
|
||||
'</Initiator>',
|
||||
'<Owner>',
|
||||
`<ID>${escapeForXml(val.Owner.ID)}</ID>`,
|
||||
`<DisplayName>${escapeForXml(val.Owner.DisplayName)}` +
|
||||
'</Initiator>',
|
||||
'<Owner>',
|
||||
`<ID>${escapeForXml(val.Owner.ID)}</ID>`,
|
||||
`<DisplayName>${escapeForXml(val.Owner.DisplayName)}` +
|
||||
'</DisplayName>',
|
||||
'</Owner>',
|
||||
`<StorageClass>${escapeForXml(val.StorageClass)}` +
|
||||
'</Owner>',
|
||||
`<StorageClass>${escapeForXml(val.StorageClass)}` +
|
||||
'</StorageClass>',
|
||||
`<Initiated>${escapeForXml(val.Initiated)}</Initiated>`,
|
||||
'</Upload>'
|
||||
`<Initiated>${escapeForXml(val.Initiated)}</Initiated>`,
|
||||
'</Upload>',
|
||||
);
|
||||
});
|
||||
|
||||
l.CommonPrefixes.forEach(prefix => {
|
||||
xml.push('<CommonPrefixes>',
|
||||
`<Prefix>${escapeForXml(prefix)}</Prefix>`,
|
||||
'</CommonPrefixes>'
|
||||
`<Prefix>${escapeForXml(prefix)}</Prefix>`,
|
||||
'</CommonPrefixes>',
|
||||
);
|
||||
});
|
||||
|
||||
|
|
|
@ -5,7 +5,6 @@ const Readable = require('stream').Readable;
|
|||
* This class is used to produce zeros filled buffers for a reader consumption
|
||||
*/
|
||||
class NullStream extends Readable {
|
||||
|
||||
/**
|
||||
* Construct a new zeros filled buffers producer that will
|
||||
* produce as much bytes as specified by the range parameter, or the size
|
||||
|
@ -32,8 +31,8 @@ class NullStream extends Readable {
|
|||
_read(size) {
|
||||
const toRead = Math.min(size, this.bytesToRead);
|
||||
const buffer = toRead > 0
|
||||
? Buffer.alloc(toRead, 0)
|
||||
: null;
|
||||
? Buffer.alloc(toRead, 0)
|
||||
: null;
|
||||
this.bytesToRead -= toRead;
|
||||
this.push(buffer);
|
||||
}
|
||||
|
|
|
@ -110,7 +110,7 @@ function generateMpuPartStorageInfo(filteredPartList) {
|
|||
* and extraPartLocations
|
||||
*/
|
||||
function validateAndFilterMpuParts(storedParts, jsonList, mpuOverviewKey,
|
||||
splitter, log) {
|
||||
splitter, log) {
|
||||
let storedPartsCopy = [];
|
||||
const filteredPartsObj = {};
|
||||
filteredPartsObj.partList = [];
|
||||
|
|
|
@ -4,11 +4,11 @@ const errors = require('../errors');
|
|||
const escapeForXml = require('./escapeForXml');
|
||||
|
||||
const errorInvalidArgument = errors.InvalidArgument
|
||||
.customizeDescription('The header \'x-amz-tagging\' shall be ' +
|
||||
.customizeDescription('The header \'x-amz-tagging\' shall be ' +
|
||||
'encoded as UTF-8 then URLEncoded URL query parameters without ' +
|
||||
'tag name duplicates.');
|
||||
const errorBadRequestLimit50 = errors.BadRequest
|
||||
.customizeDescription('Object tags cannot be greater than 50');
|
||||
.customizeDescription('Object tags cannot be greater than 50');
|
||||
|
||||
/*
|
||||
Format of xml request:
|
||||
|
@ -38,7 +38,7 @@ const _validator = {
|
|||
result.Tagging.TagSet &&
|
||||
result.Tagging.TagSet.length === 1 &&
|
||||
(
|
||||
result.Tagging.TagSet[0] === '' ||
|
||||
result.Tagging.TagSet[0] === '' ||
|
||||
result.Tagging.TagSet[0] &&
|
||||
Object.keys(result.Tagging.TagSet[0]).length === 1 &&
|
||||
result.Tagging.TagSet[0].Tag &&
|
||||
|
@ -155,7 +155,7 @@ function parseTagXml(xml, log, cb) {
|
|||
function convertToXml(objectTags) {
|
||||
const xml = [];
|
||||
xml.push('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>',
|
||||
'<Tagging> <TagSet>');
|
||||
'<Tagging> <TagSet>');
|
||||
if (objectTags && Object.keys(objectTags).length > 0) {
|
||||
Object.keys(objectTags).forEach(key => {
|
||||
xml.push(`<Tag><Key>${escapeForXml(key)}</Key>` +
|
||||
|
|
|
@ -43,7 +43,7 @@ function checkBucketAndKey(bucketName, objectKey, method, reqQuery,
|
|||
log.debug('empty bucket name', { method: 'routes' });
|
||||
return (method !== 'OPTIONS') ?
|
||||
errors.MethodNotAllowed : errors.AccessForbidden
|
||||
.customizeDescription('CORSResponse: Bucket not found');
|
||||
.customizeDescription('CORSResponse: Bucket not found');
|
||||
}
|
||||
if (bucketName !== undefined && routesUtils.isValidBucketName(bucketName,
|
||||
blacklistedPrefixes.bucket) === false) {
|
||||
|
@ -92,7 +92,7 @@ function checkTypes(req, res, params, logger, s3config) {
|
|||
'bad routes param: internalHandlers must be an object');
|
||||
if (params.statsClient) {
|
||||
assert.strictEqual(typeof params.statsClient, 'object',
|
||||
'bad routes param: statsClient must be an object');
|
||||
'bad routes param: statsClient must be an object');
|
||||
}
|
||||
assert(Array.isArray(params.allEndpoints),
|
||||
'bad routes param: allEndpoints must be an array');
|
||||
|
@ -100,13 +100,13 @@ function checkTypes(req, res, params, logger, s3config) {
|
|||
'bad routes param: allEndpoints must have at least one endpoint');
|
||||
params.allEndpoints.forEach(endpoint => {
|
||||
assert.strictEqual(typeof endpoint, 'string',
|
||||
'bad routes param: each item in allEndpoints must be a string');
|
||||
'bad routes param: each item in allEndpoints must be a string');
|
||||
});
|
||||
assert(Array.isArray(params.websiteEndpoints),
|
||||
'bad routes param: allEndpoints must be an array');
|
||||
params.websiteEndpoints.forEach(endpoint => {
|
||||
assert.strictEqual(typeof endpoint, 'string',
|
||||
'bad routes param: each item in websiteEndpoints must be a string');
|
||||
'bad routes param: each item in websiteEndpoints must be a string');
|
||||
});
|
||||
assert.strictEqual(typeof params.blacklistedPrefixes, 'object',
|
||||
'bad routes param: blacklistedPrefixes must be an object');
|
||||
|
@ -114,13 +114,13 @@ function checkTypes(req, res, params, logger, s3config) {
|
|||
'bad routes param: blacklistedPrefixes.bucket must be an array');
|
||||
params.blacklistedPrefixes.bucket.forEach(pre => {
|
||||
assert.strictEqual(typeof pre, 'string',
|
||||
'bad routes param: each blacklisted bucket prefix must be a string');
|
||||
'bad routes param: each blacklisted bucket prefix must be a string');
|
||||
});
|
||||
assert(Array.isArray(params.blacklistedPrefixes.object),
|
||||
'bad routes param: blacklistedPrefixes.object must be an array');
|
||||
params.blacklistedPrefixes.object.forEach(pre => {
|
||||
assert.strictEqual(typeof pre, 'string',
|
||||
'bad routes param: each blacklisted object prefix must be a string');
|
||||
'bad routes param: each blacklisted object prefix must be a string');
|
||||
});
|
||||
assert.strictEqual(typeof params.dataRetrievalParams, 'object',
|
||||
'bad routes param: dataRetrievalParams must be a defined object');
|
||||
|
@ -179,8 +179,8 @@ function routes(req, res, params, logger, s3config) {
|
|||
reqUids = undefined;
|
||||
}
|
||||
const log = (reqUids !== undefined ?
|
||||
logger.newRequestLoggerFromSerializedUids(reqUids) :
|
||||
logger.newRequestLogger());
|
||||
logger.newRequestLoggerFromSerializedUids(reqUids) :
|
||||
logger.newRequestLogger());
|
||||
|
||||
if (!req.url.startsWith('/_/healthcheck') &&
|
||||
!req.url.startsWith('/_/report')) {
|
||||
|
@ -216,7 +216,7 @@ function routes(req, res, params, logger, s3config) {
|
|||
return routesUtils.responseXMLBody(
|
||||
errors.InvalidURI.customizeDescription('Could not parse the ' +
|
||||
'specified URI. Check your restEndpoints configuration.'),
|
||||
undefined, res, log);
|
||||
undefined, res, log);
|
||||
}
|
||||
|
||||
log.addDefaultFields({
|
||||
|
@ -238,7 +238,7 @@ function routes(req, res, params, logger, s3config) {
|
|||
|
||||
if (bucketOrKeyError) {
|
||||
log.trace('error with bucket or key value',
|
||||
{ error: bucketOrKeyError });
|
||||
{ error: bucketOrKeyError });
|
||||
return routesUtils.responseXMLBody(bucketOrKeyError, null, res, log);
|
||||
}
|
||||
|
||||
|
|
|
@ -7,7 +7,7 @@ function routeDELETE(request, response, api, log, statsClient) {
|
|||
if (request.query.uploadId) {
|
||||
if (request.objectKey === undefined) {
|
||||
return routesUtils.responseNoBody(
|
||||
errors.InvalidRequest.customizeDescription('A key must be ' +
|
||||
errors.InvalidRequest.customizeDescription('A key must be ' +
|
||||
'specified'), null, response, 200, log);
|
||||
}
|
||||
api.callApiMethod('multipartDelete', request, response, log,
|
||||
|
@ -19,77 +19,77 @@ function routeDELETE(request, response, api, log, statsClient) {
|
|||
} else if (request.objectKey === undefined) {
|
||||
if (request.query.website !== undefined) {
|
||||
return api.callApiMethod('bucketDeleteWebsite', request,
|
||||
response, log, (err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
response, log, (err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
} else if (request.query.cors !== undefined) {
|
||||
return api.callApiMethod('bucketDeleteCors', request, response,
|
||||
log, (err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
log, (err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
} else if (request.query.replication !== undefined) {
|
||||
return api.callApiMethod('bucketDeleteReplication', request,
|
||||
response, log, (err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
response, log, (err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
} else if (request.query.lifecycle !== undefined) {
|
||||
return api.callApiMethod('bucketDeleteLifecycle', request,
|
||||
response, log, (err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
response, log, (err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
} else if (request.query.policy !== undefined) {
|
||||
return api.callApiMethod('bucketDeletePolicy', request,
|
||||
response, log, (err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
response, log, (err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
} else if (request.query.encryption !== undefined) {
|
||||
return api.callApiMethod('bucketDeleteEncryption', request,
|
||||
response, log, (err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
response, log, (err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
}
|
||||
api.callApiMethod('bucketDelete', request, response, log,
|
||||
(err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders, response,
|
||||
204, log);
|
||||
});
|
||||
(err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders, response,
|
||||
204, log);
|
||||
});
|
||||
} else {
|
||||
if (request.query.tagging !== undefined) {
|
||||
return api.callApiMethod('objectDeleteTagging', request,
|
||||
response, log, (err, resHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, resHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
response, log, (err, resHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, resHeaders,
|
||||
response, 204, log);
|
||||
});
|
||||
}
|
||||
api.callApiMethod('objectDelete', request, response, log,
|
||||
(err, corsHeaders) => {
|
||||
/*
|
||||
(err, corsHeaders) => {
|
||||
/*
|
||||
* Since AWS expects a 204 regardless of the existence of
|
||||
the object, the errors NoSuchKey and NoSuchVersion should not
|
||||
* be sent back as a response.
|
||||
*/
|
||||
if (err && !err.NoSuchKey && !err.NoSuchVersion) {
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, null, log);
|
||||
}
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(null, corsHeaders, response,
|
||||
204, log);
|
||||
});
|
||||
if (err && !err.NoSuchKey && !err.NoSuchVersion) {
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, null, log);
|
||||
}
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(null, corsHeaders, response,
|
||||
204, log);
|
||||
});
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
|
|
@ -2,7 +2,7 @@ const errors = require('../../errors');
|
|||
const routesUtils = require('../routesUtils');
|
||||
|
||||
function routerGET(request, response, api, log, statsClient,
|
||||
dataRetrievalParams) {
|
||||
dataRetrievalParams) {
|
||||
log.debug('routing request', { method: 'routerGET' });
|
||||
if (request.bucketName === undefined && request.objectKey !== undefined) {
|
||||
routesUtils.responseXMLBody(errors.NoSuchBucket, null, response, log);
|
||||
|
@ -17,18 +17,18 @@ dataRetrievalParams) {
|
|||
// GET bucket ACL
|
||||
if (request.query.acl !== undefined) {
|
||||
api.callApiMethod('bucketGetACL', request, response, log,
|
||||
(err, xml, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseXMLBody(err, xml, response, log,
|
||||
corsHeaders);
|
||||
});
|
||||
(err, xml, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseXMLBody(err, xml, response, log,
|
||||
corsHeaders);
|
||||
});
|
||||
} else if (request.query.replication !== undefined) {
|
||||
api.callApiMethod('bucketGetReplication', request, response, log,
|
||||
(err, xml, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseXMLBody(err, xml, response, log,
|
||||
corsHeaders);
|
||||
});
|
||||
(err, xml, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseXMLBody(err, xml, response, log,
|
||||
corsHeaders);
|
||||
});
|
||||
} else if (request.query.cors !== undefined) {
|
||||
api.callApiMethod('bucketGetCors', request, response, log,
|
||||
(err, xml, corsHeaders) => {
|
||||
|
@ -70,7 +70,7 @@ dataRetrievalParams) {
|
|||
(err, xml, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseXMLBody(err, xml, response, log,
|
||||
corsHeaders);
|
||||
corsHeaders);
|
||||
});
|
||||
} else if (request.query.policy !== undefined) {
|
||||
api.callApiMethod('bucketGetPolicy', request, response, log,
|
||||
|
@ -95,11 +95,11 @@ dataRetrievalParams) {
|
|||
});
|
||||
} else if (request.query.encryption !== undefined) {
|
||||
api.callApiMethod('bucketGetEncryption', request, response, log,
|
||||
(err, xml, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseXMLBody(err, xml, response,
|
||||
log, corsHeaders);
|
||||
});
|
||||
(err, xml, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseXMLBody(err, xml, response,
|
||||
log, corsHeaders);
|
||||
});
|
||||
} else if (request.query.search !== undefined) {
|
||||
api.callApiMethod('metadataSearch', request, response, log,
|
||||
(err, xml, corsHeaders) => {
|
||||
|
|
|
@ -21,11 +21,11 @@ function routeOPTIONS(request, response, api, log, statsClient) {
|
|||
}
|
||||
|
||||
return api.callApiMethod('corsPreflight', request, response, log,
|
||||
(err, resHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, resHeaders, response, 200,
|
||||
log);
|
||||
});
|
||||
(err, resHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, resHeaders, response, 200,
|
||||
log);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = routeOPTIONS;
|
||||
|
|
|
@ -27,28 +27,28 @@ function routePOST(request, response, api, log) {
|
|||
if (request.query.uploads !== undefined) {
|
||||
return api.callApiMethod('initiateMultipartUpload', request,
|
||||
response, log, (err, result, corsHeaders) =>
|
||||
routesUtils.responseXMLBody(err, result, response, log,
|
||||
corsHeaders));
|
||||
routesUtils.responseXMLBody(err, result, response, log,
|
||||
corsHeaders));
|
||||
}
|
||||
|
||||
// POST complete multipart upload
|
||||
if (request.query.uploadId !== undefined) {
|
||||
return api.callApiMethod('completeMultipartUpload', request,
|
||||
response, log, (err, result, resHeaders) =>
|
||||
routesUtils.responseXMLBody(err, result, response, log,
|
||||
resHeaders));
|
||||
routesUtils.responseXMLBody(err, result, response, log,
|
||||
resHeaders));
|
||||
}
|
||||
|
||||
// POST multiObjectDelete
|
||||
if (request.query.delete !== undefined) {
|
||||
return api.callApiMethod('multiObjectDelete', request, response,
|
||||
log, (err, xml, corsHeaders) =>
|
||||
routesUtils.responseXMLBody(err, xml, response, log,
|
||||
corsHeaders));
|
||||
routesUtils.responseXMLBody(err, xml, response, log,
|
||||
corsHeaders));
|
||||
}
|
||||
|
||||
return routesUtils.responseNoBody(errors.NotImplemented, null, response,
|
||||
200, log);
|
||||
200, log);
|
||||
}
|
||||
/* eslint-enable no-param-reassign */
|
||||
module.exports = routePOST;
|
||||
|
|
|
@ -14,16 +14,16 @@ function routePUT(request, response, api, log, statsClient) {
|
|||
|| contentLength < 0)) || contentLength === '') {
|
||||
log.debug('invalid content-length header');
|
||||
return routesUtils.responseNoBody(
|
||||
errors.BadRequest, null, response, null, log);
|
||||
errors.BadRequest, null, response, null, log);
|
||||
}
|
||||
// PUT bucket ACL
|
||||
if (request.query.acl !== undefined) {
|
||||
api.callApiMethod('bucketPutACL', request, response, log,
|
||||
(err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 200, log);
|
||||
});
|
||||
(err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 200, log);
|
||||
});
|
||||
} else if (request.query.versioning !== undefined) {
|
||||
api.callApiMethod('bucketPutVersioning', request, response, log,
|
||||
(err, corsHeaders) => {
|
||||
|
@ -82,11 +82,11 @@ function routePUT(request, response, api, log, statsClient) {
|
|||
});
|
||||
} else if (request.query.encryption !== undefined) {
|
||||
api.callApiMethod('bucketPutEncryption', request, response, log,
|
||||
(err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 200, log);
|
||||
});
|
||||
(err, corsHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, corsHeaders,
|
||||
response, 200, log);
|
||||
});
|
||||
} else {
|
||||
// PUT bucket
|
||||
return api.callApiMethod('bucketPut', request, response, log,
|
||||
|
@ -110,7 +110,7 @@ function routePUT(request, response, api, log, statsClient) {
|
|||
method: 'routePUT',
|
||||
});
|
||||
return routesUtils
|
||||
.responseNoBody(errors.InvalidDigest, null, response, 200, log);
|
||||
.responseNoBody(errors.InvalidDigest, null, response, 200, log);
|
||||
}
|
||||
if (request.headers['content-md5']) {
|
||||
request.contentMD5 = request.headers['content-md5'];
|
||||
|
@ -126,17 +126,17 @@ function routePUT(request, response, api, log, statsClient) {
|
|||
});
|
||||
return routesUtils
|
||||
.responseNoBody(errors.InvalidDigest, null, response, 200,
|
||||
log);
|
||||
log);
|
||||
}
|
||||
}
|
||||
if (request.query.partNumber) {
|
||||
if (request.headers['x-amz-copy-source']) {
|
||||
api.callApiMethod('objectPutCopyPart', request, response, log,
|
||||
(err, xml, additionalHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseXMLBody(err, xml, response, log,
|
||||
(err, xml, additionalHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseXMLBody(err, xml, response, log,
|
||||
additionalHeaders);
|
||||
});
|
||||
});
|
||||
} else {
|
||||
api.callApiMethod('objectPutPart', request, response, log,
|
||||
(err, calculatedHash, corsHeaders) => {
|
||||
|
@ -202,11 +202,11 @@ function routePUT(request, response, api, log, statsClient) {
|
|||
contentLength: request.parsedContentLength,
|
||||
});
|
||||
api.callApiMethod('objectPut', request, response, log,
|
||||
(err, resHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, resHeaders,
|
||||
response, 200, log);
|
||||
});
|
||||
(err, resHeaders) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
return routesUtils.responseNoBody(err, resHeaders,
|
||||
response, 200, log);
|
||||
});
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
|
|
|
@ -14,7 +14,7 @@ function routerWebsite(request, response, api, log, statsClient,
|
|||
if (request.method === 'GET') {
|
||||
return api.callApiMethod('websiteGet', request, response, log,
|
||||
(err, userErrorPageFailure, dataGetInfo, resMetaHeaders,
|
||||
redirectInfo, key) => {
|
||||
redirectInfo, key) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
// request being redirected
|
||||
if (redirectInfo) {
|
||||
|
@ -43,21 +43,21 @@ function routerWebsite(request, response, api, log, statsClient,
|
|||
}
|
||||
if (request.method === 'HEAD') {
|
||||
return api.callApiMethod('websiteHead', request, response, log,
|
||||
(err, resMetaHeaders, redirectInfo, key) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
if (redirectInfo) {
|
||||
return routesUtils.redirectRequest(redirectInfo,
|
||||
key, request.connection.encrypted,
|
||||
response, request.headers.host, resMetaHeaders, log);
|
||||
}
|
||||
// could redirect on err so check for redirectInfo first
|
||||
if (err) {
|
||||
return routesUtils.errorHeaderResponse(err, response,
|
||||
resMetaHeaders, log);
|
||||
}
|
||||
return routesUtils.responseContentHeaders(err, {}, resMetaHeaders,
|
||||
response, log);
|
||||
});
|
||||
(err, resMetaHeaders, redirectInfo, key) => {
|
||||
routesUtils.statsReport500(err, statsClient);
|
||||
if (redirectInfo) {
|
||||
return routesUtils.redirectRequest(redirectInfo,
|
||||
key, request.connection.encrypted,
|
||||
response, request.headers.host, resMetaHeaders, log);
|
||||
}
|
||||
// could redirect on err so check for redirectInfo first
|
||||
if (err) {
|
||||
return routesUtils.errorHeaderResponse(err, response,
|
||||
resMetaHeaders, log);
|
||||
}
|
||||
return routesUtils.responseContentHeaders(err, {}, resMetaHeaders,
|
||||
response, log);
|
||||
});
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
|
|
@ -28,7 +28,7 @@ function setCommonResponseHeaders(headers, response, log) {
|
|||
} catch (e) {
|
||||
log.debug('header can not be added ' +
|
||||
'to the response', { header: headers[key],
|
||||
error: e.stack, method: 'setCommonResponseHeaders' });
|
||||
error: e.stack, method: 'setCommonResponseHeaders' });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
@ -71,7 +71,7 @@ const XMLResponseBackend = {
|
|||
* @return {object} response - response object with additional headers
|
||||
*/
|
||||
okResponse: function okXMLResponse(xml, response, log,
|
||||
additionalHeaders) {
|
||||
additionalHeaders) {
|
||||
const bytesSent = Buffer.byteLength(xml);
|
||||
log.trace('sending success xml response');
|
||||
log.addDefaultFields({
|
||||
|
@ -118,7 +118,7 @@ const XMLResponseBackend = {
|
|||
`<Message>${errCode.description}</Message>`,
|
||||
'<Resource></Resource>',
|
||||
`<RequestId>${log.getSerializedUids()}</RequestId>`,
|
||||
'</Error>'
|
||||
'</Error>',
|
||||
);
|
||||
const xmlStr = xml.join('');
|
||||
const bytesSent = Buffer.byteLength(xmlStr);
|
||||
|
@ -148,7 +148,7 @@ const JSONResponseBackend = {
|
|||
* @return {object} response - response object with additional headers
|
||||
*/
|
||||
okResponse: function okJSONResponse(json, response, log,
|
||||
additionalHeaders) {
|
||||
additionalHeaders) {
|
||||
const bytesSent = Buffer.byteLength(json);
|
||||
log.trace('sending success json response');
|
||||
log.addDefaultFields({
|
||||
|
@ -166,7 +166,7 @@ const JSONResponseBackend = {
|
|||
},
|
||||
|
||||
errorResponse: function errorJSONResponse(errCode, response, log,
|
||||
corsHeaders) {
|
||||
corsHeaders) {
|
||||
log.trace('sending error json response', { errCode });
|
||||
/*
|
||||
{
|
||||
|
@ -368,27 +368,27 @@ function retrieveData(locations, retrieveDataParams, response, log) {
|
|||
currentStream = readable;
|
||||
return readable.pipe(response, { end: false });
|
||||
}), err => {
|
||||
currentStream = null;
|
||||
if (err) {
|
||||
log.debug('abort response due to error', {
|
||||
error: err.code, errMsg: err.message });
|
||||
}
|
||||
// call end for all cases (error/success) per node.js docs
|
||||
// recommendation
|
||||
response.end();
|
||||
currentStream = null;
|
||||
if (err) {
|
||||
log.debug('abort response due to error', {
|
||||
error: err.code, errMsg: err.message });
|
||||
}
|
||||
// call end for all cases (error/success) per node.js docs
|
||||
// recommendation
|
||||
response.end();
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function _responseBody(responseBackend, errCode, payload, response, log,
|
||||
additionalHeaders) {
|
||||
additionalHeaders) {
|
||||
if (errCode && !response.headersSent) {
|
||||
return responseBackend.errorResponse(errCode, response, log,
|
||||
additionalHeaders);
|
||||
additionalHeaders);
|
||||
}
|
||||
if (!response.headersSent) {
|
||||
return responseBackend.okResponse(payload, response, log,
|
||||
additionalHeaders);
|
||||
additionalHeaders);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
@ -397,8 +397,8 @@ function _computeContentLengthFromLocation(dataLocations) {
|
|||
return dataLocations.reduce(
|
||||
(sum, location) => (sum !== undefined &&
|
||||
(typeof location.size === 'number' || typeof location.size === 'string') ?
|
||||
sum + Number.parseInt(location.size, 10) :
|
||||
undefined), 0);
|
||||
sum + Number.parseInt(location.size, 10) :
|
||||
undefined), 0);
|
||||
}
|
||||
|
||||
function _contentLengthMatchesLocations(contentLength, dataLocations) {
|
||||
|
@ -419,7 +419,7 @@ const routesUtils = {
|
|||
*/
|
||||
responseXMLBody(errCode, xml, response, log, additionalHeaders) {
|
||||
return _responseBody(XMLResponseBackend, errCode, xml, response,
|
||||
log, additionalHeaders);
|
||||
log, additionalHeaders);
|
||||
},
|
||||
|
||||
/**
|
||||
|
@ -433,7 +433,7 @@ const routesUtils = {
|
|||
*/
|
||||
responseJSONBody(errCode, json, response, log, additionalHeaders) {
|
||||
return _responseBody(JSONResponseBackend, errCode, json, response,
|
||||
log, additionalHeaders);
|
||||
log, additionalHeaders);
|
||||
},
|
||||
|
||||
/**
|
||||
|
@ -448,7 +448,7 @@ const routesUtils = {
|
|||
responseNoBody(errCode, resHeaders, response, httpCode = 200, log) {
|
||||
if (errCode && !response.headersSent) {
|
||||
return XMLResponseBackend.errorResponse(errCode, response, log,
|
||||
resHeaders);
|
||||
resHeaders);
|
||||
}
|
||||
if (!response.headersSent) {
|
||||
return okHeaderResponse(resHeaders, response, httpCode, log);
|
||||
|
@ -466,10 +466,10 @@ const routesUtils = {
|
|||
* @return {object} - router's response object
|
||||
*/
|
||||
responseContentHeaders(errCode, overrideParams, resHeaders, response,
|
||||
log) {
|
||||
log) {
|
||||
if (errCode && !response.headersSent) {
|
||||
return XMLResponseBackend.errorResponse(errCode, response, log,
|
||||
resHeaders);
|
||||
resHeaders);
|
||||
}
|
||||
if (!response.headersSent) {
|
||||
// Undefined added as an argument since need to send range to
|
||||
|
@ -504,7 +504,7 @@ const routesUtils = {
|
|||
retrieveDataParams, response, range, log) {
|
||||
if (errCode && !response.headersSent) {
|
||||
return XMLResponseBackend.errorResponse(errCode, response, log,
|
||||
resHeaders);
|
||||
resHeaders);
|
||||
}
|
||||
if (dataLocations !== null && !response.headersSent) {
|
||||
// sanity check of content length against individual data
|
||||
|
@ -512,13 +512,13 @@ const routesUtils = {
|
|||
const contentLength = resHeaders && resHeaders['Content-Length'];
|
||||
if (contentLength !== undefined &&
|
||||
!_contentLengthMatchesLocations(contentLength,
|
||||
dataLocations)) {
|
||||
dataLocations)) {
|
||||
log.error('logic error: total length of fetched data ' +
|
||||
'locations does not match returned content-length',
|
||||
{ contentLength, dataLocations });
|
||||
{ contentLength, dataLocations });
|
||||
return XMLResponseBackend.errorResponse(errors.InternalError,
|
||||
response, log,
|
||||
resHeaders);
|
||||
response, log,
|
||||
resHeaders);
|
||||
}
|
||||
}
|
||||
if (!response.headersSent) {
|
||||
|
@ -591,7 +591,7 @@ const routesUtils = {
|
|||
`<h1>${err.code} ${response.statusMessage}</h1>`,
|
||||
'<ul>',
|
||||
`<li>Code: ${err.message}</li>`,
|
||||
`<li>Message: ${err.description}</li>`
|
||||
`<li>Message: ${err.description}</li>`,
|
||||
);
|
||||
|
||||
if (!userErrorPageFailure && bucketName) {
|
||||
|
@ -601,7 +601,7 @@ const routesUtils = {
|
|||
`<li>RequestId: ${log.getSerializedUids()}</li>`,
|
||||
// AWS response contains HostId here.
|
||||
// TODO: consider adding
|
||||
'</ul>'
|
||||
'</ul>',
|
||||
);
|
||||
if (userErrorPageFailure) {
|
||||
html.push(
|
||||
|
@ -611,13 +611,13 @@ const routesUtils = {
|
|||
'<ul>',
|
||||
`<li>Code: ${err.message}</li>`,
|
||||
`<li>Message: ${err.description}</li>`,
|
||||
'</ul>'
|
||||
'</ul>',
|
||||
);
|
||||
}
|
||||
html.push(
|
||||
'<hr/>',
|
||||
'</body>',
|
||||
'</html>'
|
||||
'</html>',
|
||||
);
|
||||
|
||||
return response.end(html.join(''), 'utf8', () => {
|
||||
|
@ -839,7 +839,7 @@ const routesUtils = {
|
|||
// most specific potential hostname
|
||||
bucketName =
|
||||
potentialBucketName.length < bucketName.length ?
|
||||
potentialBucketName : bucketName;
|
||||
potentialBucketName : bucketName;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -847,7 +847,7 @@ const routesUtils = {
|
|||
return bucketName;
|
||||
}
|
||||
throw new Error(
|
||||
`bad request: hostname ${host} is not in valid endpoints`
|
||||
`bad request: hostname ${host} is not in valid endpoints`,
|
||||
);
|
||||
},
|
||||
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -30,7 +30,7 @@ function parseLC(config, vault) {
|
|||
if (locationObj.details.connector.sproxyd) {
|
||||
clients[location] = new Sproxy({
|
||||
bootstrap: locationObj.details.connector
|
||||
.sproxyd.bootstrap,
|
||||
.sproxyd.bootstrap,
|
||||
// Might be undefined which is ok since there is a default
|
||||
// set in sproxydclient if chordCos is undefined
|
||||
chordCos: locationObj.details.connector.sproxyd.chordCos,
|
||||
|
@ -60,7 +60,7 @@ function parseLC(config, vault) {
|
|||
// keepalive config
|
||||
const httpAgentConfig =
|
||||
config.externalBackends[locationObj.type].httpAgent;
|
||||
// max sockets is infinity by default and expressed as null
|
||||
// max sockets is infinity by default and expressed as null
|
||||
if (httpAgentConfig.maxSockets === null) {
|
||||
httpAgentConfig.maxSockets = undefined;
|
||||
}
|
||||
|
@ -131,7 +131,7 @@ function parseLC(config, vault) {
|
|||
const azureStorageEndpoint = config.getAzureEndpoint(location);
|
||||
const proxyParams =
|
||||
backendUtils.proxyCompareUrl(azureStorageEndpoint) ?
|
||||
{} : config.outboundProxy;
|
||||
{} : config.outboundProxy;
|
||||
const azureStorageCredentials =
|
||||
config.getAzureStorageCredentials(location);
|
||||
clients[location] = new AzureClient({
|
||||
|
|
|
@ -52,26 +52,26 @@ class MultipleBackendGateway {
|
|||
}
|
||||
}
|
||||
return client.put(writeStream, size, keyContext, reqUids,
|
||||
(err, key, dataStoreVersionId, dataStoreSize, dataStoreMD5) => {
|
||||
const log = createLogger(reqUids);
|
||||
log.debug('put to location', { controllingLocationConstraint });
|
||||
if (err) {
|
||||
log.error('error from datastore',
|
||||
{ error: err, dataStoreType: client.clientType });
|
||||
return callback(errors.ServiceUnavailable);
|
||||
}
|
||||
const dataRetrievalInfo = {
|
||||
key,
|
||||
dataStoreName: controllingLocationConstraint,
|
||||
dataStoreType: client.clientType,
|
||||
dataStoreVersionId,
|
||||
dataStoreSize,
|
||||
dataStoreMD5,
|
||||
};
|
||||
return callback(null, dataRetrievalInfo);
|
||||
(err, key, dataStoreVersionId, dataStoreSize, dataStoreMD5) => {
|
||||
const log = createLogger(reqUids);
|
||||
log.debug('put to location', { controllingLocationConstraint });
|
||||
if (err) {
|
||||
log.error('error from datastore',
|
||||
{ error: err, dataStoreType: client.clientType });
|
||||
return callback(errors.ServiceUnavailable);
|
||||
}
|
||||
const dataRetrievalInfo = {
|
||||
key,
|
||||
dataStoreName: controllingLocationConstraint,
|
||||
dataStoreType: client.clientType,
|
||||
dataStoreVersionId,
|
||||
dataStoreSize,
|
||||
dataStoreMD5,
|
||||
};
|
||||
return callback(null, dataRetrievalInfo);
|
||||
// sproxyd accepts keyschema, send as null so sproxyd generates key
|
||||
// send metadata as param for AzureClient in Arsenal
|
||||
}, null, this.metadata);
|
||||
}, null, this.metadata);
|
||||
}
|
||||
|
||||
head(objectGetInfoArr, reqUids, callback) {
|
||||
|
@ -166,14 +166,14 @@ class MultipleBackendGateway {
|
|||
}, () => {
|
||||
async.parallel([
|
||||
next => checkExternalBackend(
|
||||
this.clients, awsArray, 'aws_s3', flightCheckOnStartUp,
|
||||
externalBackendHealthCheckInterval, next),
|
||||
this.clients, awsArray, 'aws_s3', flightCheckOnStartUp,
|
||||
externalBackendHealthCheckInterval, next),
|
||||
next => checkExternalBackend(
|
||||
this.clients, azureArray, 'azure', flightCheckOnStartUp,
|
||||
externalBackendHealthCheckInterval, next),
|
||||
this.clients, azureArray, 'azure', flightCheckOnStartUp,
|
||||
externalBackendHealthCheckInterval, next),
|
||||
next => checkExternalBackend(
|
||||
this.clients, gcpArray, 'gcp', flightCheckOnStartUp,
|
||||
externalBackendHealthCheckInterval, next),
|
||||
this.clients, gcpArray, 'gcp', flightCheckOnStartUp,
|
||||
externalBackendHealthCheckInterval, next),
|
||||
], (errNull, externalResp) => {
|
||||
const externalLocResults = [];
|
||||
externalResp.forEach(resp => externalLocResults.push(...resp));
|
||||
|
@ -185,19 +185,19 @@ class MultipleBackendGateway {
|
|||
}
|
||||
|
||||
createMPU(key, metaHeaders, bucketName, websiteRedirectHeader,
|
||||
location, contentType, cacheControl, contentDisposition,
|
||||
contentEncoding, tagging, log, cb) {
|
||||
location, contentType, cacheControl, contentDisposition,
|
||||
contentEncoding, tagging, log, cb) {
|
||||
const client = this.clients[location];
|
||||
if (client.clientType === 'aws_s3' || client.clientType === 'gcp') {
|
||||
return client.createMPU(key, metaHeaders, bucketName,
|
||||
websiteRedirectHeader, contentType, cacheControl,
|
||||
contentDisposition, contentEncoding, tagging, log, cb);
|
||||
websiteRedirectHeader, contentType, cacheControl,
|
||||
contentDisposition, contentEncoding, tagging, log, cb);
|
||||
}
|
||||
return cb();
|
||||
}
|
||||
|
||||
uploadPart(request, streamingV4Params, stream, size, location, key,
|
||||
uploadId, partNumber, bucketName, log, cb) {
|
||||
uploadId, partNumber, bucketName, log, cb) {
|
||||
const client = this.clients[location];
|
||||
|
||||
if (client.uploadPart) {
|
||||
|
@ -206,29 +206,29 @@ class MultipleBackendGateway {
|
|||
return cb(err);
|
||||
}
|
||||
return client.uploadPart(request, streamingV4Params, stream,
|
||||
size, key, uploadId, partNumber, bucketName, log,
|
||||
(err, partInfo) => {
|
||||
if (err) {
|
||||
size, key, uploadId, partNumber, bucketName, log,
|
||||
(err, partInfo) => {
|
||||
if (err) {
|
||||
// if error putting part, counter should be decremented
|
||||
return this.locStorageCheckFn(location, -size, log,
|
||||
error => {
|
||||
if (error) {
|
||||
log.error('Error decrementing location ' +
|
||||
return this.locStorageCheckFn(location, -size, log,
|
||||
error => {
|
||||
if (error) {
|
||||
log.error('Error decrementing location ' +
|
||||
'metric following object PUT failure',
|
||||
{ error: error.message });
|
||||
}
|
||||
return cb(err);
|
||||
});
|
||||
}
|
||||
return cb(null, partInfo);
|
||||
});
|
||||
{ error: error.message });
|
||||
}
|
||||
return cb(err);
|
||||
});
|
||||
}
|
||||
return cb(null, partInfo);
|
||||
});
|
||||
});
|
||||
}
|
||||
return cb();
|
||||
}
|
||||
|
||||
listParts(key, uploadId, location, bucketName, partNumberMarker, maxParts,
|
||||
log, cb) {
|
||||
log, cb) {
|
||||
const client = this.clients[location];
|
||||
|
||||
if (client.listParts) {
|
||||
|
@ -239,7 +239,7 @@ class MultipleBackendGateway {
|
|||
}
|
||||
|
||||
completeMPU(key, uploadId, location, jsonList, mdInfo, bucketName,
|
||||
userMetadata, contentSettings, tagging, log, cb) {
|
||||
userMetadata, contentSettings, tagging, log, cb) {
|
||||
const client = this.clients[location];
|
||||
if (client.completeMPU) {
|
||||
const args = [jsonList, mdInfo, key, uploadId, bucketName];
|
||||
|
@ -291,40 +291,40 @@ class MultipleBackendGateway {
|
|||
// NOTE: using copyObject only if copying object from one external
|
||||
// backend to the same external backend
|
||||
copyObject(request, destLocationConstraintName, externalSourceKey,
|
||||
sourceLocationConstraintName, storeMetadataParams, config, log, cb) {
|
||||
sourceLocationConstraintName, storeMetadataParams, config, log, cb) {
|
||||
const client = this.clients[destLocationConstraintName];
|
||||
if (client.copyObject) {
|
||||
return this.locStorageCheckFn(destLocationConstraintName,
|
||||
storeMetadataParams.size, log, err => {
|
||||
if (err) {
|
||||
cb(err);
|
||||
}
|
||||
return client.copyObject(request, destLocationConstraintName,
|
||||
externalSourceKey, sourceLocationConstraintName,
|
||||
storeMetadataParams, config, log,
|
||||
(err, key, dataStoreVersionId) => {
|
||||
const dataRetrievalInfo = {
|
||||
key,
|
||||
dataStoreName: destLocationConstraintName,
|
||||
dataStoreType: client.clientType,
|
||||
dataStoreVersionId,
|
||||
};
|
||||
storeMetadataParams.size, log, err => {
|
||||
if (err) {
|
||||
// if error copying obj, counter should be decremented
|
||||
return this.locStorageCheckFn(
|
||||
destLocationConstraintName, -storeMetadataParams.size,
|
||||
log, error => {
|
||||
if (error) {
|
||||
log.error('Error decrementing location ' +
|
||||
'metric following object PUT failure',
|
||||
{ error: error.message });
|
||||
}
|
||||
return cb(err);
|
||||
});
|
||||
cb(err);
|
||||
}
|
||||
return cb(null, dataRetrievalInfo);
|
||||
return client.copyObject(request, destLocationConstraintName,
|
||||
externalSourceKey, sourceLocationConstraintName,
|
||||
storeMetadataParams, config, log,
|
||||
(err, key, dataStoreVersionId) => {
|
||||
const dataRetrievalInfo = {
|
||||
key,
|
||||
dataStoreName: destLocationConstraintName,
|
||||
dataStoreType: client.clientType,
|
||||
dataStoreVersionId,
|
||||
};
|
||||
if (err) {
|
||||
// if error copying obj, counter should be decremented
|
||||
return this.locStorageCheckFn(
|
||||
destLocationConstraintName, -storeMetadataParams.size,
|
||||
log, error => {
|
||||
if (error) {
|
||||
log.error('Error decrementing location ' +
|
||||
'metric following object PUT failure',
|
||||
{ error: error.message });
|
||||
}
|
||||
return cb(err);
|
||||
});
|
||||
}
|
||||
return cb(null, dataRetrievalInfo);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
return cb(errors.NotImplemented
|
||||
.customizeDescription('Can not copy object from ' +
|
||||
|
@ -332,15 +332,15 @@ class MultipleBackendGateway {
|
|||
}
|
||||
|
||||
uploadPartCopy(request, location, awsSourceKey,
|
||||
sourceLocationConstraintName, config, log, cb) {
|
||||
sourceLocationConstraintName, config, log, cb) {
|
||||
const client = this.clients[location];
|
||||
if (client.uploadPartCopy) {
|
||||
return client.uploadPartCopy(request, awsSourceKey,
|
||||
sourceLocationConstraintName, config,
|
||||
log, cb);
|
||||
sourceLocationConstraintName, config,
|
||||
log, cb);
|
||||
}
|
||||
return cb(errors.NotImplemented.customizeDescription(
|
||||
'Can not copy object from ' +
|
||||
'Can not copy object from ' +
|
||||
`${client.clientType} to ${client.clientType}`));
|
||||
}
|
||||
|
||||
|
@ -348,7 +348,7 @@ class MultipleBackendGateway {
|
|||
const client = this.clients[location];
|
||||
if (client.protectAzureBlocks) {
|
||||
return client.protectAzureBlocks(this.metadata, bucketName,
|
||||
objectKey, location, log, cb);
|
||||
objectKey, location, log, cb);
|
||||
}
|
||||
return cb();
|
||||
}
|
||||
|
|
|
@ -11,7 +11,7 @@ const { createLogger, logHelper, removeQuotes, trimXMetaPrefix } =
|
|||
|
||||
const missingVerIdInternalError = errors.InternalError.customizeDescription(
|
||||
'Invalid state. Please ensure versioning is enabled ' +
|
||||
'in AWS for the location constraint and try again.'
|
||||
'in AWS for the location constraint and try again.',
|
||||
);
|
||||
|
||||
class AwsClient {
|
||||
|
@ -36,35 +36,35 @@ class AwsClient {
|
|||
// this request implicitly updates the endpoint for the location
|
||||
// the following code explcitly sets it to avoid surprises
|
||||
this._client.getBucketLocation({ Bucket: this._awsBucketName },
|
||||
(err, res) => {
|
||||
if (err && err.code !== 'AuthorizationHeaderMalformed') {
|
||||
this._logger.error('error during setup', {
|
||||
error: err,
|
||||
method: 'AwsClient.setup',
|
||||
});
|
||||
return cb(err);
|
||||
}
|
||||
let region;
|
||||
if (err && err.code === 'AuthorizationHeaderMalformed') {
|
||||
(err, res) => {
|
||||
if (err && err.code !== 'AuthorizationHeaderMalformed') {
|
||||
this._logger.error('error during setup', {
|
||||
error: err,
|
||||
method: 'AwsClient.setup',
|
||||
});
|
||||
return cb(err);
|
||||
}
|
||||
let region;
|
||||
if (err && err.code === 'AuthorizationHeaderMalformed') {
|
||||
// set regional endpoint
|
||||
region = err.region;
|
||||
} else if (res) {
|
||||
region = res.LocationConstraint;
|
||||
}
|
||||
this._client.config.update({ region });
|
||||
region = err.region;
|
||||
} else if (res) {
|
||||
region = res.LocationConstraint;
|
||||
}
|
||||
this._client.config.update({ region });
|
||||
|
||||
const isAWS = this._s3Params.endpoint.endsWith('amazonaws.com');
|
||||
if (region && isAWS) {
|
||||
const endpoint = `s3.${region}.amazonaws.com`;
|
||||
this._logger.debug('setting regional endpoint', {
|
||||
method: 'AwsClient.setup',
|
||||
region,
|
||||
endpoint,
|
||||
});
|
||||
this._client.endpoint = new AWS.Endpoint(endpoint);
|
||||
}
|
||||
return cb();
|
||||
});
|
||||
const isAWS = this._s3Params.endpoint.endsWith('amazonaws.com');
|
||||
if (region && isAWS) {
|
||||
const endpoint = `s3.${region}.amazonaws.com`;
|
||||
this._logger.debug('setting regional endpoint', {
|
||||
method: 'AwsClient.setup',
|
||||
region,
|
||||
endpoint,
|
||||
});
|
||||
this._client.endpoint = new AWS.Endpoint(endpoint);
|
||||
}
|
||||
return cb();
|
||||
});
|
||||
}
|
||||
|
||||
_createAwsKey(requestBucketName, requestObjectKey,
|
||||
|
@ -84,23 +84,23 @@ class AwsClient {
|
|||
|
||||
put(stream, size, keyContext, reqUids, callback) {
|
||||
const awsKey = this._createAwsKey(keyContext.bucketName,
|
||||
keyContext.objectKey, this._bucketMatch);
|
||||
keyContext.objectKey, this._bucketMatch);
|
||||
const metaHeaders = trimXMetaPrefix(keyContext.metaHeaders);
|
||||
const log = createLogger(reqUids);
|
||||
|
||||
const putCb = (err, data) => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from data backend',
|
||||
err, this._dataStoreName, this.clientType);
|
||||
err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`),
|
||||
);
|
||||
}
|
||||
if (!data.VersionId && this._supportsVersioning) {
|
||||
logHelper(log, 'error', 'missing version id for data ' +
|
||||
'backend object', missingVerIdInternalError,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(missingVerIdInternalError);
|
||||
}
|
||||
const dataStoreVersionId = data.VersionId;
|
||||
|
@ -180,8 +180,8 @@ class AwsClient {
|
|||
Range: range ? `bytes=${range[0]}-${range[1]}` : null,
|
||||
}).on('success', response => {
|
||||
log.trace(`${this.type} GET request response headers`,
|
||||
{ responseHeaders: response.httpResponse.headers,
|
||||
backendType: this.clientType });
|
||||
{ responseHeaders: response.httpResponse.headers,
|
||||
backendType: this.clientType });
|
||||
});
|
||||
const stream = request.createReadStream();
|
||||
|
||||
|
@ -202,8 +202,8 @@ class AwsClient {
|
|||
logLevel = 'error';
|
||||
}
|
||||
logHelper(log, logLevel,
|
||||
`error streaming data from ${this.type}`,
|
||||
err, this._dataStoreName, this.clientType);
|
||||
`error streaming data from ${this.type}`,
|
||||
err, this._dataStoreName, this.clientType);
|
||||
});
|
||||
// Always call the callback asynchronously: the caller may
|
||||
// destroy the stream with destroy(), which MUST be
|
||||
|
@ -232,8 +232,8 @@ class AwsClient {
|
|||
return callback();
|
||||
}
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`),
|
||||
);
|
||||
}
|
||||
return callback();
|
||||
|
@ -243,39 +243,39 @@ class AwsClient {
|
|||
healthcheck(location, callback) {
|
||||
const awsResp = {};
|
||||
this._client.headBucket({ Bucket: this._awsBucketName },
|
||||
err => {
|
||||
err => {
|
||||
/* eslint-disable no-param-reassign */
|
||||
if (err) {
|
||||
awsResp[location] = { error: err, external: true };
|
||||
return callback(null, awsResp);
|
||||
}
|
||||
if (!this._supportsVersioning) {
|
||||
awsResp[location] = {
|
||||
message: 'Congrats! You own the bucket',
|
||||
};
|
||||
return callback(null, awsResp);
|
||||
}
|
||||
return this._client.getBucketVersioning({
|
||||
Bucket: this._awsBucketName },
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
awsResp[location] = { error: err, external: true };
|
||||
} else if (!data.Status ||
|
||||
data.Status === 'Suspended') {
|
||||
return callback(null, awsResp);
|
||||
}
|
||||
if (!this._supportsVersioning) {
|
||||
awsResp[location] = {
|
||||
versioningStatus: data.Status,
|
||||
error: 'Versioning must be enabled',
|
||||
external: true,
|
||||
};
|
||||
} else {
|
||||
awsResp[location] = {
|
||||
versioningStatus: data.Status,
|
||||
message: 'Congrats! You own the bucket',
|
||||
};
|
||||
return callback(null, awsResp);
|
||||
}
|
||||
return callback(null, awsResp);
|
||||
return this._client.getBucketVersioning({
|
||||
Bucket: this._awsBucketName },
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
awsResp[location] = { error: err, external: true };
|
||||
} else if (!data.Status ||
|
||||
data.Status === 'Suspended') {
|
||||
awsResp[location] = {
|
||||
versioningStatus: data.Status,
|
||||
error: 'Versioning must be enabled',
|
||||
external: true,
|
||||
};
|
||||
} else {
|
||||
awsResp[location] = {
|
||||
versioningStatus: data.Status,
|
||||
message: 'Congrats! You own the bucket',
|
||||
};
|
||||
}
|
||||
return callback(null, awsResp);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
createMPU(key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
|
||||
|
@ -304,10 +304,10 @@ class AwsClient {
|
|||
return this._client.createMultipartUpload(params, (err, mpuResObj) => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from data backend',
|
||||
err, this._dataStoreName, this.clientType);
|
||||
err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`),
|
||||
);
|
||||
}
|
||||
return callback(null, mpuResObj);
|
||||
|
@ -315,7 +315,7 @@ class AwsClient {
|
|||
}
|
||||
|
||||
uploadPart(request, streamingV4Params, stream, size, key, uploadId,
|
||||
partNumber, bucketName, log, callback) {
|
||||
partNumber, bucketName, log, callback) {
|
||||
let hashedStream = stream;
|
||||
if (request) {
|
||||
const partStream = prepareStream(request, streamingV4Params,
|
||||
|
@ -335,7 +335,7 @@ class AwsClient {
|
|||
'on uploadPart', err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`)
|
||||
`${this.type}: ${err.message}`),
|
||||
);
|
||||
}
|
||||
// Because we manually add quotes to ETag later, remove quotes here
|
||||
|
@ -352,7 +352,7 @@ class AwsClient {
|
|||
}
|
||||
|
||||
listParts(key, uploadId, bucketName, partNumberMarker, maxParts, log,
|
||||
callback) {
|
||||
callback) {
|
||||
const awsBucket = this._awsBucketName;
|
||||
const awsKey = this._createAwsKey(bucketName, key, this._bucketMatch);
|
||||
const params = { Bucket: awsBucket, Key: awsKey, UploadId: uploadId,
|
||||
|
@ -360,10 +360,10 @@ class AwsClient {
|
|||
return this._client.listParts(params, (err, partList) => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from data backend on listPart',
|
||||
err, this._dataStoreName, this.clientType);
|
||||
err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`),
|
||||
);
|
||||
}
|
||||
// build storedParts object to mimic Scality S3 backend returns
|
||||
|
@ -424,47 +424,47 @@ class AwsClient {
|
|||
};
|
||||
const completeObjData = { key: awsKey };
|
||||
return this._client.completeMultipartUpload(mpuParams,
|
||||
(err, completeMpuRes) => {
|
||||
if (err) {
|
||||
if (mpuError[err.code]) {
|
||||
logHelper(log, 'trace', 'err from data backend on ' +
|
||||
'completeMPU', err, this._dataStoreName, this.clientType);
|
||||
return callback(errors[err.code]);
|
||||
}
|
||||
logHelper(log, 'error', 'err from data backend on ' +
|
||||
'completeMPU', err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`)
|
||||
);
|
||||
}
|
||||
if (!completeMpuRes.VersionId && this._supportsVersioning) {
|
||||
logHelper(log, 'error', 'missing version id for data ' +
|
||||
'backend object', missingVerIdInternalError,
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(missingVerIdInternalError);
|
||||
}
|
||||
// need to get content length of new object to store
|
||||
// in our metadata
|
||||
return this._client.headObject({ Bucket: awsBucket, Key: awsKey },
|
||||
(err, objHeaders) => {
|
||||
(err, completeMpuRes) => {
|
||||
if (err) {
|
||||
logHelper(log, 'trace', 'err from data backend on ' +
|
||||
'headObject', err, this._dataStoreName, this.clientType);
|
||||
if (mpuError[err.code]) {
|
||||
logHelper(log, 'trace', 'err from data backend on ' +
|
||||
'completeMPU', err, this._dataStoreName, this.clientType);
|
||||
return callback(errors[err.code]);
|
||||
}
|
||||
logHelper(log, 'error', 'err from data backend on ' +
|
||||
'completeMPU', err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`),
|
||||
);
|
||||
}
|
||||
// remove quotes from eTag because they're added later
|
||||
completeObjData.eTag = completeMpuRes.ETag
|
||||
.substring(1, completeMpuRes.ETag.length - 1);
|
||||
completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
|
||||
completeObjData.contentLength =
|
||||
if (!completeMpuRes.VersionId && this._supportsVersioning) {
|
||||
logHelper(log, 'error', 'missing version id for data ' +
|
||||
'backend object', missingVerIdInternalError,
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(missingVerIdInternalError);
|
||||
}
|
||||
// need to get content length of new object to store
|
||||
// in our metadata
|
||||
return this._client.headObject({ Bucket: awsBucket, Key: awsKey },
|
||||
(err, objHeaders) => {
|
||||
if (err) {
|
||||
logHelper(log, 'trace', 'err from data backend on ' +
|
||||
'headObject', err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`),
|
||||
);
|
||||
}
|
||||
// remove quotes from eTag because they're added later
|
||||
completeObjData.eTag = completeMpuRes.ETag
|
||||
.substring(1, completeMpuRes.ETag.length - 1);
|
||||
completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
|
||||
completeObjData.contentLength =
|
||||
Number.parseInt(objHeaders.ContentLength, 10);
|
||||
return callback(null, completeObjData);
|
||||
return callback(null, completeObjData);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
abortMPU(key, uploadId, bucketName, log, callback) {
|
||||
|
@ -480,8 +480,8 @@ class AwsClient {
|
|||
'using the same uploadId.', err, this._dataStoreName,
|
||||
this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`),
|
||||
);
|
||||
}
|
||||
return callback();
|
||||
|
@ -507,10 +507,10 @@ class AwsClient {
|
|||
if (err) {
|
||||
logHelper(log, 'error', 'error from data backend on ' +
|
||||
'putObjectTagging', err,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`),
|
||||
);
|
||||
}
|
||||
return callback();
|
||||
|
@ -532,19 +532,19 @@ class AwsClient {
|
|||
'deleteObjectTagging', err,
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`),
|
||||
);
|
||||
}
|
||||
return callback();
|
||||
});
|
||||
}
|
||||
copyObject(request, destLocationConstraintName, sourceKey,
|
||||
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
|
||||
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
|
||||
const destBucketName = request.bucketName;
|
||||
const destObjectKey = request.objectKey;
|
||||
const destAwsKey = this._createAwsKey(destBucketName, destObjectKey,
|
||||
this._bucketMatch);
|
||||
this._bucketMatch);
|
||||
|
||||
const sourceAwsBucketName =
|
||||
config.getAwsBucketName(sourceLocationConstraintName);
|
||||
|
@ -569,15 +569,15 @@ class AwsClient {
|
|||
`${sourceAwsBucketName} ${this.type} bucket`, err,
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.AccessDenied
|
||||
.customizeDescription('Error: Unable to access ' +
|
||||
`${sourceAwsBucketName} ${this.type} bucket`)
|
||||
.customizeDescription('Error: Unable to access ' +
|
||||
`${sourceAwsBucketName} ${this.type} bucket`),
|
||||
);
|
||||
}
|
||||
logHelper(log, 'error', 'error from data backend on ' +
|
||||
'copyObject', err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`),
|
||||
);
|
||||
}
|
||||
if (!copyResult.VersionId && this._supportsVersioning) {
|
||||
|
@ -590,7 +590,7 @@ class AwsClient {
|
|||
if (err || !data.VersionId) {
|
||||
logHelper(log, 'error', 'missing version id for data ' +
|
||||
'backend object', missingVerIdInternalError,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(missingVerIdInternalError);
|
||||
}
|
||||
return callback(null, destAwsKey, data.VersionId);
|
||||
|
@ -600,11 +600,11 @@ class AwsClient {
|
|||
});
|
||||
}
|
||||
uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName,
|
||||
config, log, callback) {
|
||||
config, log, callback) {
|
||||
const destBucketName = request.bucketName;
|
||||
const destObjectKey = request.objectKey;
|
||||
const destAwsKey = this._createAwsKey(destBucketName, destObjectKey,
|
||||
this._bucketMatch);
|
||||
this._bucketMatch);
|
||||
|
||||
const sourceAwsBucketName =
|
||||
config.getAwsBucketName(sourceLocationConstraintName);
|
||||
|
@ -628,15 +628,15 @@ class AwsClient {
|
|||
`${sourceAwsBucketName} AWS bucket`, err,
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.AccessDenied
|
||||
.customizeDescription('Error: Unable to access ' +
|
||||
`${sourceAwsBucketName} AWS bucket`)
|
||||
.customizeDescription('Error: Unable to access ' +
|
||||
`${sourceAwsBucketName} AWS bucket`),
|
||||
);
|
||||
}
|
||||
logHelper(log, 'error', 'error from data backend on ' +
|
||||
'uploadPartCopy', err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.type}: ${err.message}`),
|
||||
);
|
||||
}
|
||||
const eTag = removeQuotes(res.CopyPartResult.ETag);
|
||||
|
|
|
@ -51,7 +51,7 @@ class AzureClient {
|
|||
if (log) {
|
||||
log.error('error thrown by Azure Storage Client Library',
|
||||
{ error: err.message, stack: err.stack, s3Method,
|
||||
azureMethod, dataStoreName: this._dataStoreName });
|
||||
azureMethod, dataStoreName: this._dataStoreName });
|
||||
}
|
||||
cb(error.customizeDescription('Error from Azure ' +
|
||||
`method: ${azureMethod} on ${s3Method} S3 call: ` +
|
||||
|
@ -82,7 +82,7 @@ class AzureClient {
|
|||
// same key name. If it does, do not allow put or delete because Azure
|
||||
// will delete all blocks with same key name
|
||||
protectAzureBlocks(metadata, bucketName, objectKey, dataStoreName,
|
||||
log, cb) {
|
||||
log, cb) {
|
||||
const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`;
|
||||
const splitter = constants.splitter;
|
||||
const listingParams = {
|
||||
|
@ -93,23 +93,23 @@ class AzureClient {
|
|||
};
|
||||
|
||||
return metadata.listMultipartUploads(mpuBucketName, listingParams,
|
||||
log, (err, mpuList) => {
|
||||
if (err && !err.NoSuchBucket) {
|
||||
log.error('Error listing MPUs for Azure delete',
|
||||
{ error: err, dataStoreName });
|
||||
return cb(errors.ServiceUnavailable);
|
||||
}
|
||||
if (mpuList && mpuList.Uploads && mpuList.Uploads.length > 0) {
|
||||
const error = errors.MPUinProgress;
|
||||
log.error('Error: cannot put/delete object to Azure with ' +
|
||||
log, (err, mpuList) => {
|
||||
if (err && !err.NoSuchBucket) {
|
||||
log.error('Error listing MPUs for Azure delete',
|
||||
{ error: err, dataStoreName });
|
||||
return cb(errors.ServiceUnavailable);
|
||||
}
|
||||
if (mpuList && mpuList.Uploads && mpuList.Uploads.length > 0) {
|
||||
const error = errors.MPUinProgress;
|
||||
log.error('Error: cannot put/delete object to Azure with ' +
|
||||
'same key name as ongoing MPU on Azure',
|
||||
{ error, dataStoreName });
|
||||
return cb(error);
|
||||
}
|
||||
// If listMultipartUploads returns a NoSuchBucket error or the
|
||||
// mpu list is empty, there are no conflicting MPUs, so continue
|
||||
return cb();
|
||||
});
|
||||
return cb(error);
|
||||
}
|
||||
// If listMultipartUploads returns a NoSuchBucket error or the
|
||||
// mpu list is empty, there are no conflicting MPUs, so continue
|
||||
return cb();
|
||||
});
|
||||
}
|
||||
|
||||
toObjectGetInfo(objectKey, bucketName) {
|
||||
|
@ -123,52 +123,52 @@ class AzureClient {
|
|||
const log = createLogger(reqUids);
|
||||
// before blob is put, make sure there is no ongoing MPU with same key
|
||||
this.protectAzureBlocks(metadata, keyContext.bucketName,
|
||||
keyContext.objectKey, this._dataStoreName, log, err => {
|
||||
keyContext.objectKey, this._dataStoreName, log, err => {
|
||||
// if error returned, there is ongoing MPU, so do not put
|
||||
if (err) {
|
||||
return callback(err.customizeDescription(
|
||||
`Error putting object to Azure: ${err.message}`));
|
||||
}
|
||||
const azureKey = this._createAzureKey(keyContext.bucketName,
|
||||
keyContext.objectKey, this._bucketMatch);
|
||||
const options = {
|
||||
metadata: translateAzureMetaHeaders(keyContext.metaHeaders,
|
||||
keyContext.tagging),
|
||||
contentSettings: {
|
||||
contentType: keyContext.contentType || undefined,
|
||||
cacheControl: keyContext.cacheControl || undefined,
|
||||
contentDisposition: keyContext.contentDisposition ||
|
||||
if (err) {
|
||||
return callback(err.customizeDescription(
|
||||
`Error putting object to Azure: ${err.message}`));
|
||||
}
|
||||
const azureKey = this._createAzureKey(keyContext.bucketName,
|
||||
keyContext.objectKey, this._bucketMatch);
|
||||
const options = {
|
||||
metadata: translateAzureMetaHeaders(keyContext.metaHeaders,
|
||||
keyContext.tagging),
|
||||
contentSettings: {
|
||||
contentType: keyContext.contentType || undefined,
|
||||
cacheControl: keyContext.cacheControl || undefined,
|
||||
contentDisposition: keyContext.contentDisposition ||
|
||||
undefined,
|
||||
contentEncoding: keyContext.contentEncoding || undefined,
|
||||
},
|
||||
};
|
||||
if (size === 0) {
|
||||
return this._errorWrapper('put', 'createBlockBlobFromText',
|
||||
[this._azureContainerName, azureKey, '', options,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from Azure PUT data ' +
|
||||
contentEncoding: keyContext.contentEncoding || undefined,
|
||||
},
|
||||
};
|
||||
if (size === 0) {
|
||||
return this._errorWrapper('put', 'createBlockBlobFromText',
|
||||
[this._azureContainerName, azureKey, '', options,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from Azure PUT data ' +
|
||||
'backend', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`Azure: ${err.message}`));
|
||||
}
|
||||
return callback(null, azureKey);
|
||||
}], log, callback);
|
||||
}
|
||||
return this._errorWrapper('put', 'createBlockBlobFromStream',
|
||||
[this._azureContainerName, azureKey, stream, size, options,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from Azure PUT data ' +
|
||||
}
|
||||
return callback(null, azureKey);
|
||||
}], log, callback);
|
||||
}
|
||||
return this._errorWrapper('put', 'createBlockBlobFromStream',
|
||||
[this._azureContainerName, azureKey, stream, size, options,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from Azure PUT data ' +
|
||||
'backend', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`Azure: ${err.message}`));
|
||||
}
|
||||
return callback(null, azureKey);
|
||||
}], log, callback);
|
||||
});
|
||||
}
|
||||
return callback(null, azureKey);
|
||||
}], log, callback);
|
||||
});
|
||||
}
|
||||
|
||||
head(objectGetInfo, reqUids, callback) {
|
||||
|
@ -176,25 +176,25 @@ class AzureClient {
|
|||
const { key, azureStreamingOptions } = objectGetInfo;
|
||||
return this._errorWrapper('head', 'getBlobProperties',
|
||||
[this._azureContainerName, key, azureStreamingOptions,
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
let logLevel;
|
||||
let retError;
|
||||
if (err.code === 'NotFound') {
|
||||
logLevel = 'info';
|
||||
retError = errors.LocationNotFound;
|
||||
} else {
|
||||
logLevel = 'error';
|
||||
retError = errors.ServiceUnavailable
|
||||
.customizeDescription(
|
||||
`Error returned from Azure: ${err.message}`);
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
let logLevel;
|
||||
let retError;
|
||||
if (err.code === 'NotFound') {
|
||||
logLevel = 'info';
|
||||
retError = errors.LocationNotFound;
|
||||
} else {
|
||||
logLevel = 'error';
|
||||
retError = errors.ServiceUnavailable
|
||||
.customizeDescription(
|
||||
`Error returned from Azure: ${err.message}`);
|
||||
}
|
||||
logHelper(log, logLevel, 'err from Azure HEAD data backend',
|
||||
err, this._dataStoreName);
|
||||
return callback(retError);
|
||||
}
|
||||
logHelper(log, logLevel, 'err from Azure HEAD data backend',
|
||||
err, this._dataStoreName);
|
||||
return callback(retError);
|
||||
}
|
||||
return callback(null, data);
|
||||
}], log, callback);
|
||||
return callback(null, data);
|
||||
}], log, callback);
|
||||
}
|
||||
|
||||
get(objectGetInfo, range, reqUids, callback) {
|
||||
|
@ -213,14 +213,14 @@ class AzureClient {
|
|||
}
|
||||
this._errorWrapper('get', 'getBlobToStream',
|
||||
[this._azureContainerName, key, response, streamingOptions,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from Azure GET data backend',
|
||||
err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable);
|
||||
}
|
||||
return callback(null, response);
|
||||
}], log, callback);
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from Azure GET data backend',
|
||||
err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable);
|
||||
}
|
||||
return callback(null, response);
|
||||
}], log, callback);
|
||||
}
|
||||
|
||||
delete(objectGetInfo, reqUids, callback) {
|
||||
|
@ -234,20 +234,20 @@ class AzureClient {
|
|||
}
|
||||
return this._errorWrapper('delete', 'deleteBlobIfExists',
|
||||
[this._azureContainerName, key, options,
|
||||
err => {
|
||||
if (err && err.statusCode === 412) {
|
||||
return callback(errors.PreconditionFailed);
|
||||
}
|
||||
if (err) {
|
||||
const log = createLogger(reqUids);
|
||||
logHelper(log, 'error', 'error deleting object from ' +
|
||||
err => {
|
||||
if (err && err.statusCode === 412) {
|
||||
return callback(errors.PreconditionFailed);
|
||||
}
|
||||
if (err) {
|
||||
const log = createLogger(reqUids);
|
||||
logHelper(log, 'error', 'error deleting object from ' +
|
||||
'Azure datastore', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`Azure: ${err.message}`));
|
||||
}
|
||||
return callback();
|
||||
}], log, callback);
|
||||
}
|
||||
return callback();
|
||||
}], log, callback);
|
||||
}
|
||||
|
||||
healthcheck(location, callback, flightCheckOnStartUp) {
|
||||
|
@ -271,7 +271,7 @@ class AzureClient {
|
|||
}
|
||||
|
||||
uploadPart(request, streamingV4Params, partStream, size, key, uploadId,
|
||||
partNumber, bucket, log, callback) {
|
||||
partNumber, bucket, log, callback) {
|
||||
const azureKey = this._createAzureKey(bucket, key, this._bucketMatch);
|
||||
const params = { bucketName: this._azureContainerName,
|
||||
partNumber, size, objectKey: azureKey, uploadId };
|
||||
|
@ -299,27 +299,27 @@ class AzureClient {
|
|||
if (size <= azureMpuUtils.maxSubPartSize) {
|
||||
const errorWrapperFn = this._errorWrapper.bind(this);
|
||||
return azureMpuUtils.putSinglePart(errorWrapperFn,
|
||||
stream, params, this._dataStoreName, log, (err, dataStoreETag) => {
|
||||
stream, params, this._dataStoreName, log, (err, dataStoreETag) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
dataRetrievalInfo.dataStoreETag = dataStoreETag;
|
||||
return callback(null, dataRetrievalInfo);
|
||||
});
|
||||
}
|
||||
const errorWrapperFn = this._errorWrapper.bind(this);
|
||||
return azureMpuUtils.putSubParts(errorWrapperFn, stream,
|
||||
params, this._dataStoreName, log, (err, dataStoreETag) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
dataRetrievalInfo.dataStoreETag = dataStoreETag;
|
||||
return callback(null, dataRetrievalInfo);
|
||||
});
|
||||
}
|
||||
const errorWrapperFn = this._errorWrapper.bind(this);
|
||||
return azureMpuUtils.putSubParts(errorWrapperFn, stream,
|
||||
params, this._dataStoreName, log, (err, dataStoreETag) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
dataRetrievalInfo.dataStoreETag = dataStoreETag;
|
||||
return callback(null, dataRetrievalInfo);
|
||||
});
|
||||
}
|
||||
|
||||
completeMPU(jsonList, mdInfo, key, uploadId, bucket, metaHeaders,
|
||||
contentSettings, tagging, log, callback) {
|
||||
contentSettings, tagging, log, callback) {
|
||||
const azureKey = this._createAzureKey(bucket, key, this._bucketMatch);
|
||||
const commitList = {
|
||||
UncommittedBlocks: jsonList.uncommittedBlocks || [],
|
||||
|
@ -345,20 +345,20 @@ class AzureClient {
|
|||
};
|
||||
return this._errorWrapper('completeMPU', 'commitBlocks',
|
||||
[this._azureContainerName, azureKey, commitList, options,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err completing MPU on Azure ' +
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err completing MPU on Azure ' +
|
||||
'datastore', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`Azure: ${err.message}`));
|
||||
}
|
||||
const completeObjData = {
|
||||
key: azureKey,
|
||||
filteredPartsObj,
|
||||
};
|
||||
return callback(null, completeObjData);
|
||||
}], log, callback);
|
||||
}
|
||||
const completeObjData = {
|
||||
key: azureKey,
|
||||
filteredPartsObj,
|
||||
};
|
||||
return callback(null, completeObjData);
|
||||
}], log, callback);
|
||||
}
|
||||
|
||||
objectPutTagging(key, bucket, objectMD, log, callback) {
|
||||
|
@ -367,14 +367,14 @@ class AzureClient {
|
|||
azureMD.tags = JSON.stringify(objectMD.tags);
|
||||
this._errorWrapper('objectPutTagging', 'setBlobMetadata',
|
||||
[this._azureContainerName, azureKey, azureMD,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err putting object tags to ' +
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err putting object tags to ' +
|
||||
'Azure backend', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable);
|
||||
}
|
||||
return callback();
|
||||
}], log, callback);
|
||||
return callback(errors.ServiceUnavailable);
|
||||
}
|
||||
return callback();
|
||||
}], log, callback);
|
||||
}
|
||||
|
||||
objectDeleteTagging(key, bucketName, objectMD, log, callback) {
|
||||
|
@ -382,27 +382,27 @@ class AzureClient {
|
|||
const azureMD = this._getMetaHeaders(objectMD);
|
||||
this._errorWrapper('objectDeleteTagging', 'setBlobMetadata',
|
||||
[this._azureContainerName, azureKey, azureMD,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err putting object tags to ' +
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err putting object tags to ' +
|
||||
'Azure backend', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable);
|
||||
}
|
||||
return callback();
|
||||
}], log, callback);
|
||||
return callback(errors.ServiceUnavailable);
|
||||
}
|
||||
return callback();
|
||||
}], log, callback);
|
||||
}
|
||||
|
||||
copyObject(request, destLocationConstraintName, sourceKey,
|
||||
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
|
||||
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
|
||||
const destContainerName = request.bucketName;
|
||||
const destObjectKey = request.objectKey;
|
||||
|
||||
const destAzureKey = this._createAzureKey(destContainerName,
|
||||
destObjectKey, this._bucketMatch);
|
||||
destObjectKey, this._bucketMatch);
|
||||
|
||||
const sourceContainerName =
|
||||
config.locationConstraints[sourceLocationConstraintName]
|
||||
.details.azureContainerName;
|
||||
.details.azureContainerName;
|
||||
|
||||
let options;
|
||||
if (storeMetadataParams.metaHeaders) {
|
||||
|
@ -413,7 +413,7 @@ class AzureClient {
|
|||
this._errorWrapper('copyObject', 'startCopyBlob',
|
||||
[`${this._azureStorageEndpoint}` +
|
||||
`${sourceContainerName}/${sourceKey}`,
|
||||
this._azureContainerName, destAzureKey, options,
|
||||
this._azureContainerName, destAzureKey, options,
|
||||
(err, res) => {
|
||||
if (err) {
|
||||
if (err.code === 'CannotVerifyCopySource') {
|
||||
|
@ -421,36 +421,36 @@ class AzureClient {
|
|||
`${sourceContainerName} Azure Container`, err,
|
||||
this._dataStoreName);
|
||||
return callback(errors.AccessDenied
|
||||
.customizeDescription('Error: Unable to access ' +
|
||||
`${sourceContainerName} Azure Container`)
|
||||
.customizeDescription('Error: Unable to access ' +
|
||||
`${sourceContainerName} Azure Container`),
|
||||
);
|
||||
}
|
||||
logHelper(log, 'error', 'error from data backend on ' +
|
||||
'copyObject', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`AWS: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`AWS: ${err.message}`),
|
||||
);
|
||||
}
|
||||
if (res.copy.status === 'pending') {
|
||||
logHelper(log, 'error', 'Azure copy status is pending',
|
||||
err, this._dataStoreName);
|
||||
err, this._dataStoreName);
|
||||
const copyId = res.copy.id;
|
||||
this._client.abortCopyBlob(this._azureContainerName,
|
||||
destAzureKey, copyId, err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'error from data backend ' +
|
||||
destAzureKey, copyId, err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'error from data backend ' +
|
||||
'on abortCopyBlob', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`AWS on abortCopyBlob: ${err.message}`)
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`AWS on abortCopyBlob: ${err.message}`),
|
||||
);
|
||||
}
|
||||
return callback(errors.InvalidObjectState
|
||||
.customizeDescription('Error: Azure copy status was ' +
|
||||
'pending. It has been aborted successfully'),
|
||||
);
|
||||
}
|
||||
return callback(errors.InvalidObjectState
|
||||
.customizeDescription('Error: Azure copy status was ' +
|
||||
'pending. It has been aborted successfully')
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
return callback(null, destAzureKey);
|
||||
}], log, callback);
|
||||
|
|
|
@ -24,7 +24,7 @@ class MpuHelper {
|
|||
const handleFunc = (fnName, params, retry, callback) => {
|
||||
const timeout = backoff.duration();
|
||||
return setTimeout((params, cb) =>
|
||||
this.service[fnName](params, cb), timeout, params,
|
||||
this.service[fnName](params, cb), timeout, params,
|
||||
(err, res) => {
|
||||
if (err) {
|
||||
if (err.statusCode === 429 || err.code === 429) {
|
||||
|
@ -90,53 +90,53 @@ class MpuHelper {
|
|||
splitMerge(params, partList, level, callback) {
|
||||
// create composition of slices from the partList array
|
||||
return async.mapLimit(eachSlice.call(partList, 32),
|
||||
this.service._maxConcurrent,
|
||||
(infoParts, cb) => {
|
||||
const mpuPartList = infoParts.Parts.map(item =>
|
||||
({ PartName: item.PartName }));
|
||||
const partNumber = infoParts.PartNumber;
|
||||
const tmpKey =
|
||||
this.service._maxConcurrent,
|
||||
(infoParts, cb) => {
|
||||
const mpuPartList = infoParts.Parts.map(item =>
|
||||
({ PartName: item.PartName }));
|
||||
const partNumber = infoParts.PartNumber;
|
||||
const tmpKey =
|
||||
createMpuKey(params.Key, params.UploadId, partNumber, level);
|
||||
const mergedObject = { PartName: tmpKey };
|
||||
if (mpuPartList.length < 2) {
|
||||
logger.trace(
|
||||
'splitMerge: parts are fewer than 2, copy instead');
|
||||
// else just perform a copy
|
||||
const copyParams = {
|
||||
const mergedObject = { PartName: tmpKey };
|
||||
if (mpuPartList.length < 2) {
|
||||
logger.trace(
|
||||
'splitMerge: parts are fewer than 2, copy instead');
|
||||
// else just perform a copy
|
||||
const copyParams = {
|
||||
Bucket: params.MPU,
|
||||
Key: tmpKey,
|
||||
CopySource: `${params.MPU}/${mpuPartList[0].PartName}`,
|
||||
};
|
||||
return this.service.copyObject(copyParams, (err, res) => {
|
||||
if (err) {
|
||||
logHelper(logger, 'error',
|
||||
'error in splitMerge - copyObject', err);
|
||||
return cb(err);
|
||||
}
|
||||
mergedObject.VersionId = res.VersionId;
|
||||
mergedObject.ETag = res.ETag;
|
||||
return cb(null, mergedObject);
|
||||
});
|
||||
}
|
||||
const composeParams = {
|
||||
Bucket: params.MPU,
|
||||
Key: tmpKey,
|
||||
CopySource: `${params.MPU}/${mpuPartList[0].PartName}`,
|
||||
MultipartUpload: { Parts: mpuPartList },
|
||||
};
|
||||
return this.service.copyObject(copyParams, (err, res) => {
|
||||
return this.retryCompose(composeParams, (err, res) => {
|
||||
if (err) {
|
||||
logHelper(logger, 'error',
|
||||
'error in splitMerge - copyObject', err);
|
||||
return cb(err);
|
||||
}
|
||||
mergedObject.VersionId = res.VersionId;
|
||||
mergedObject.ETag = res.ETag;
|
||||
return cb(null, mergedObject);
|
||||
});
|
||||
}
|
||||
const composeParams = {
|
||||
Bucket: params.MPU,
|
||||
Key: tmpKey,
|
||||
MultipartUpload: { Parts: mpuPartList },
|
||||
};
|
||||
return this.retryCompose(composeParams, (err, res) => {
|
||||
}, (err, res) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
return callback(err);
|
||||
}
|
||||
mergedObject.VersionId = res.VersionId;
|
||||
mergedObject.ETag = res.ETag;
|
||||
return cb(null, mergedObject);
|
||||
return callback(null, res.length);
|
||||
});
|
||||
}, (err, res) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
return callback(null, res.length);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -284,7 +284,7 @@ class MpuHelper {
|
|||
if (err) {
|
||||
logHelper(logger, 'error', 'error in ' +
|
||||
'createMultipartUpload - final copyObject',
|
||||
err);
|
||||
err);
|
||||
return next(err);
|
||||
}
|
||||
const mpuResult = {
|
||||
|
@ -301,7 +301,7 @@ class MpuHelper {
|
|||
if (err) {
|
||||
logHelper(logger, 'error', 'error in ' +
|
||||
'createMultipartUpload - final head object',
|
||||
err);
|
||||
err);
|
||||
return next(err);
|
||||
}
|
||||
mpuResult.ContentLength = res.ContentLength;
|
||||
|
|
|
@ -70,7 +70,7 @@ class GcpManagedUpload {
|
|||
if (this.body instanceof stream) {
|
||||
assert.strictEqual(typeof this.totalBytes, 'number',
|
||||
errors.MissingContentLength.customizeDescription(
|
||||
'If body is a stream, ContentLength must be provided'));
|
||||
'If body is a stream, ContentLength must be provided'));
|
||||
} else {
|
||||
if (typeof this.body === 'string') {
|
||||
this.body = Buffer.from(this.body);
|
||||
|
@ -156,13 +156,13 @@ class GcpManagedUpload {
|
|||
.map(item =>
|
||||
Object.assign(item, { ETag: this.parts[item.PartNumber] }));
|
||||
return this.service.completeMultipartUpload(params,
|
||||
(err, res) => {
|
||||
if (err) {
|
||||
return this.cleanUp(err);
|
||||
}
|
||||
this.completed = true;
|
||||
return this.callback(null, res);
|
||||
});
|
||||
(err, res) => {
|
||||
if (err) {
|
||||
return this.cleanUp(err);
|
||||
}
|
||||
this.completed = true;
|
||||
return this.callback(null, res);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -187,16 +187,16 @@ class GcpManagedUpload {
|
|||
if (this.body instanceof stream) {
|
||||
// stream type
|
||||
this.body.on('error', err => this.cleanUp(err))
|
||||
.on('readable', () => this.chunkStream())
|
||||
.on('end', () => {
|
||||
this.isDoneChunking = true;
|
||||
this.chunkStream();
|
||||
.on('readable', () => this.chunkStream())
|
||||
.on('end', () => {
|
||||
this.isDoneChunking = true;
|
||||
this.chunkStream();
|
||||
|
||||
if (this.isDoneChunking && this.uploadedParts >= 1 &&
|
||||
if (this.isDoneChunking && this.uploadedParts >= 1 &&
|
||||
this.uploadedParts === this.totalParts) {
|
||||
this.completeUpload();
|
||||
}
|
||||
});
|
||||
this.completeUpload();
|
||||
}
|
||||
});
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
@ -251,22 +251,22 @@ class GcpManagedUpload {
|
|||
});
|
||||
},
|
||||
next => async.eachLimit(this.slicedParts, this.queueSize,
|
||||
(uploadPart, done) => {
|
||||
const params = {
|
||||
Bucket: this.mpuBucket,
|
||||
Key: this.params.Key,
|
||||
UploadId: this.uploadId,
|
||||
Body: uploadPart.Body,
|
||||
PartNumber: uploadPart.PartNumber,
|
||||
};
|
||||
this.service.uploadPart(params, (err, res) => {
|
||||
if (!err) {
|
||||
this.parts[uploadPart.PartNumber] = res.ETag;
|
||||
this.uploadedParts++;
|
||||
}
|
||||
return done(err);
|
||||
});
|
||||
}, next),
|
||||
(uploadPart, done) => {
|
||||
const params = {
|
||||
Bucket: this.mpuBucket,
|
||||
Key: this.params.Key,
|
||||
UploadId: this.uploadId,
|
||||
Body: uploadPart.Body,
|
||||
PartNumber: uploadPart.PartNumber,
|
||||
};
|
||||
this.service.uploadPart(params, (err, res) => {
|
||||
if (!err) {
|
||||
this.parts[uploadPart.PartNumber] = res.ETag;
|
||||
this.uploadedParts++;
|
||||
}
|
||||
return done(err);
|
||||
});
|
||||
}, next),
|
||||
], err => {
|
||||
if (err) {
|
||||
return this.cleanUp(new Error(
|
||||
|
|
|
@ -239,7 +239,7 @@ module.exports = {
|
|||
const requestId = resp.httpResponse.headers ?
|
||||
resp.httpResponse.headers['x-guploader-uploadid'] : null;
|
||||
if (resp.error) {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
resp.error.requestId = resp.requestId || requestId;
|
||||
}
|
||||
},
|
||||
|
|
|
@ -120,10 +120,10 @@ class GcpClient extends AwsClient {
|
|||
return this._client.createMultipartUpload(params, (err, mpuResObj) => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from data backend',
|
||||
err, this._dataStoreName, this.clientType);
|
||||
err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`GCP: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`GCP: ${err.message}`),
|
||||
);
|
||||
}
|
||||
return callback(null, mpuResObj);
|
||||
|
@ -162,32 +162,32 @@ class GcpClient extends AwsClient {
|
|||
};
|
||||
const completeObjData = { key: gcpKey };
|
||||
return this._client.completeMultipartUpload(mpuParams,
|
||||
(err, completeMpuRes) => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from data backend on ' +
|
||||
(err, completeMpuRes) => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from data backend on ' +
|
||||
'completeMPU', err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`GCP: ${err.message}`)
|
||||
);
|
||||
}
|
||||
if (!completeMpuRes.VersionId) {
|
||||
logHelper(log, 'error', 'missing version id for data ' +
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`GCP: ${err.message}`),
|
||||
);
|
||||
}
|
||||
if (!completeMpuRes.VersionId) {
|
||||
logHelper(log, 'error', 'missing version id for data ' +
|
||||
'backend object', missingVerIdInternalError,
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(missingVerIdInternalError);
|
||||
}
|
||||
// remove quotes from eTag because they're added later
|
||||
completeObjData.eTag = removeQuotes(completeMpuRes.ETag);
|
||||
completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
|
||||
completeObjData.contentLength =
|
||||
return callback(missingVerIdInternalError);
|
||||
}
|
||||
// remove quotes from eTag because they're added later
|
||||
completeObjData.eTag = removeQuotes(completeMpuRes.ETag);
|
||||
completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
|
||||
completeObjData.contentLength =
|
||||
Number.parseInt(completeMpuRes.ContentLength, 10);
|
||||
return callback(null, completeObjData);
|
||||
});
|
||||
return callback(null, completeObjData);
|
||||
});
|
||||
}
|
||||
|
||||
uploadPart(request, streamingV4Params, stream, size, key, uploadId,
|
||||
partNumber, bucketName, log, callback) {
|
||||
partNumber, bucketName, log, callback) {
|
||||
let hashedStream = stream;
|
||||
if (request) {
|
||||
const partStream = prepareStream(request, streamingV4Params,
|
||||
|
@ -209,8 +209,8 @@ class GcpClient extends AwsClient {
|
|||
logHelper(log, 'error', 'err from data backend ' +
|
||||
'on uploadPart', err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`GCP: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`GCP: ${err.message}`),
|
||||
);
|
||||
}
|
||||
// remove quotes from eTag because they're added later
|
||||
|
@ -226,11 +226,11 @@ class GcpClient extends AwsClient {
|
|||
}
|
||||
|
||||
uploadPartCopy(request, gcpSourceKey, sourceLocationConstraintName, config,
|
||||
log, callback) {
|
||||
log, callback) {
|
||||
const destBucketName = request.bucketName;
|
||||
const destObjectKey = request.objectKey;
|
||||
const destGcpKey = this._createGcpKey(destBucketName, destObjectKey,
|
||||
this._bucketMatch);
|
||||
this._bucketMatch);
|
||||
|
||||
const sourceGcpBucketName =
|
||||
config.getGcpBucketNames(sourceLocationConstraintName).bucketName;
|
||||
|
@ -241,8 +241,8 @@ class GcpClient extends AwsClient {
|
|||
|
||||
if (copySourceRange) {
|
||||
return callback(errors.NotImplemented
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.clientType}: copySourceRange not implemented`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`${this.clientType}: copySourceRange not implemented`),
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -260,15 +260,15 @@ class GcpClient extends AwsClient {
|
|||
`${sourceGcpBucketName} GCP bucket`, err,
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.AccessDenied
|
||||
.customizeDescription('Error: Unable to access ' +
|
||||
`${sourceGcpBucketName} GCP bucket`)
|
||||
.customizeDescription('Error: Unable to access ' +
|
||||
`${sourceGcpBucketName} GCP bucket`),
|
||||
);
|
||||
}
|
||||
logHelper(log, 'error', 'error from data backend on ' +
|
||||
'uploadPartCopy', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`GCP: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`GCP: ${err.message}`),
|
||||
);
|
||||
}
|
||||
// remove quotes from eTag because they're added later
|
||||
|
@ -290,8 +290,8 @@ class GcpClient extends AwsClient {
|
|||
logHelper(log, 'error', 'err from data backend ' +
|
||||
'on abortMPU', err, this._dataStoreName, this.clientType);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`GCP: ${err.message}`)
|
||||
.customizeDescription('Error returned from ' +
|
||||
`GCP: ${err.message}`),
|
||||
);
|
||||
}
|
||||
return callback();
|
||||
|
|
|
@ -45,7 +45,7 @@ class PfsClient {
|
|||
}
|
||||
return callback(null, keyContext.objectKey, '',
|
||||
keyContext.metaHeaders['x-amz-meta-size'],
|
||||
md5
|
||||
md5,
|
||||
);
|
||||
}
|
||||
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
|
||||
|
@ -72,7 +72,7 @@ class PfsClient {
|
|||
this._restClient.delete(key, reqUids, err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from data backend', err,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(err);
|
||||
}
|
||||
return callback();
|
||||
|
@ -86,61 +86,61 @@ class PfsClient {
|
|||
}
|
||||
|
||||
createMPU(key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
|
||||
cacheControl, contentDisposition, contentEncoding, log, callback) {
|
||||
cacheControl, contentDisposition, contentEncoding, log, callback) {
|
||||
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.NotImplemented);
|
||||
}
|
||||
|
||||
uploadPart(request, streamingV4Params, stream, size, key, uploadId,
|
||||
partNumber, bucketName, log, callback) {
|
||||
partNumber, bucketName, log, callback) {
|
||||
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.NotImplemented);
|
||||
}
|
||||
|
||||
listParts(key, uploadId, bucketName, partNumberMarker, maxParts, log,
|
||||
callback) {
|
||||
callback) {
|
||||
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.NotImplemented);
|
||||
}
|
||||
|
||||
completeMPU(jsonList, mdInfo, key, uploadId, bucketName, log, callback) {
|
||||
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.NotImplemented);
|
||||
}
|
||||
|
||||
abortMPU(key, uploadId, bucketName, log, callback) {
|
||||
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.NotImplemented);
|
||||
}
|
||||
|
||||
objectPutTagging(key, bucket, objectMD, log, callback) {
|
||||
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.NotImplemented);
|
||||
}
|
||||
|
||||
objectDeleteTagging(key, bucketName, objectMD, log, callback) {
|
||||
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.NotImplemented);
|
||||
}
|
||||
|
||||
copyObject(request, destLocationConstraintName, sourceKey,
|
||||
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
|
||||
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
|
||||
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.NotImplemented);
|
||||
}
|
||||
|
||||
uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName,
|
||||
config, log, callback) {
|
||||
config, log, callback) {
|
||||
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
|
||||
this._dataStoreName, this.clientType);
|
||||
this._dataStoreName, this.clientType);
|
||||
return callback(errors.NotImplemented);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -94,7 +94,7 @@ const utils = {
|
|||
* same account since Azure copy outside of an account is async
|
||||
*/
|
||||
externalBackendCopy(config, locationConstraintSrc, locationConstraintDest,
|
||||
sourceBucketMD, destBucketMD) {
|
||||
sourceBucketMD, destBucketMD) {
|
||||
const sourceBucketName = sourceBucketMD.getName();
|
||||
const destBucketName = destBucketMD.getName();
|
||||
const isSameBucket = sourceBucketName === destBucketName;
|
||||
|
@ -111,11 +111,11 @@ const utils = {
|
|||
sourceLocationConstraintType === 'gcp' ||
|
||||
(sourceLocationConstraintType === 'azure' &&
|
||||
config.isSameAzureAccount(locationConstraintSrc,
|
||||
locationConstraintDest)));
|
||||
locationConstraintDest)));
|
||||
},
|
||||
|
||||
checkExternalBackend(clients, locations, type, flightCheckOnStartUp,
|
||||
externalBackendHealthCheckInterval, cb) {
|
||||
externalBackendHealthCheckInterval, cb) {
|
||||
const checkStatus = backendHealth[type] || {};
|
||||
if (locations.length === 0) {
|
||||
return process.nextTick(cb, null, []);
|
||||
|
|
|
@ -35,7 +35,6 @@ const FOLDER_HASH = 3511;
|
|||
* directory hash structure under the configured dataPath.
|
||||
*/
|
||||
class DataFileStore {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
* @param {Object} dataConfig - configuration of the file backend
|
||||
|
@ -78,7 +77,7 @@ class DataFileStore {
|
|||
fs.access(this.dataPath, fs.F_OK | fs.R_OK | fs.W_OK, err => {
|
||||
if (err) {
|
||||
this.logger.error('Data path is not readable or writable',
|
||||
{ error: err });
|
||||
{ error: err });
|
||||
return callback(err);
|
||||
}
|
||||
if (this.isPassthrough) {
|
||||
|
@ -86,7 +85,7 @@ class DataFileStore {
|
|||
}
|
||||
// Create FOLDER_HASH subdirectories
|
||||
const subDirs = Array.from({ length: FOLDER_HASH },
|
||||
(v, k) => (k).toString());
|
||||
(v, k) => (k).toString());
|
||||
this.logger.info(`pre-creating ${subDirs.length} subdirs...`);
|
||||
if (!this.noSync) {
|
||||
storageUtils.setDirSyncFlag(this.dataPath, this.logger);
|
||||
|
@ -103,7 +102,7 @@ class DataFileStore {
|
|||
err => {
|
||||
if (err) {
|
||||
this.logger.error('Error creating subdirs',
|
||||
{ error: err });
|
||||
{ error: err });
|
||||
return callback(err);
|
||||
}
|
||||
this.logger.info('data file store init complete, ' +
|
||||
|
@ -167,7 +166,7 @@ class DataFileStore {
|
|||
fs.open(filePath, 'wx', (err, fd) => {
|
||||
if (err) {
|
||||
log.error('error opening filePath',
|
||||
{ method: 'put', key, filePath, error: err });
|
||||
{ method: 'put', key, filePath, error: err });
|
||||
return callback(errors.InternalError.customizeDescription(
|
||||
`filesystem error: open() returned ${err.code}`));
|
||||
}
|
||||
|
@ -181,7 +180,7 @@ class DataFileStore {
|
|||
fileStream.on('finish', () => {
|
||||
function ok() {
|
||||
log.debug('finished writing data',
|
||||
{ method: 'put', key, filePath });
|
||||
{ method: 'put', key, filePath });
|
||||
return cbOnce(null, key);
|
||||
}
|
||||
if (this.noSync) {
|
||||
|
@ -243,7 +242,7 @@ class DataFileStore {
|
|||
return undefined;
|
||||
}).on('error', err => {
|
||||
log.error('error streaming data on write',
|
||||
{ method: 'put', key, filePath, error: err });
|
||||
{ method: 'put', key, filePath, error: err });
|
||||
// destroying the write stream forces a close(fd)
|
||||
fileStream.destroy();
|
||||
return cbOnce(errors.InternalError.customizeDescription(
|
||||
|
@ -287,7 +286,7 @@ class DataFileStore {
|
|||
return callback(errors.ObjNotFound);
|
||||
}
|
||||
log.error('error on \'stat\' of file',
|
||||
{ key, filePath, error: err });
|
||||
{ key, filePath, error: err });
|
||||
return callback(errors.InternalError.customizeDescription(
|
||||
`filesystem error: stat() returned ${err.code}`));
|
||||
}
|
||||
|
@ -323,34 +322,34 @@ class DataFileStore {
|
|||
readStreamOptions.end = byteRange[1];
|
||||
}
|
||||
log.debug('opening readStream to get data',
|
||||
{ method: 'get', key, filePath, byteRange });
|
||||
{ method: 'get', key, filePath, byteRange });
|
||||
const cbOnce = jsutil.once(callback);
|
||||
const rs = fs.createReadStream(filePath, readStreamOptions)
|
||||
.on('error', err => {
|
||||
if (err.code === 'ENOENT') {
|
||||
return cbOnce(errors.ObjNotFound);
|
||||
.on('error', err => {
|
||||
if (err.code === 'ENOENT') {
|
||||
return cbOnce(errors.ObjNotFound);
|
||||
}
|
||||
log.error('error retrieving file',
|
||||
{ method: 'DataFileStore.get', key, filePath,
|
||||
error: err });
|
||||
return cbOnce(
|
||||
errors.InternalError.customizeDescription(
|
||||
`filesystem read error: ${err.code}`));
|
||||
})
|
||||
.on('open', () => { cbOnce(null, rs); })
|
||||
.on('end', () => {
|
||||
if (this.noCache) {
|
||||
releasePageCacheSync(filePath, rs.fd, log);
|
||||
}
|
||||
fs.close(rs.fd, err => {
|
||||
if (err) {
|
||||
log.error('unable to close file descriptor', {
|
||||
method: 'DataFileStore.get', key, filePath,
|
||||
error: err,
|
||||
});
|
||||
}
|
||||
log.error('error retrieving file',
|
||||
{ method: 'DataFileStore.get', key, filePath,
|
||||
error: err });
|
||||
return cbOnce(
|
||||
errors.InternalError.customizeDescription(
|
||||
`filesystem read error: ${err.code}`));
|
||||
})
|
||||
.on('open', () => { cbOnce(null, rs); })
|
||||
.on('end', () => {
|
||||
if (this.noCache) {
|
||||
releasePageCacheSync(filePath, rs.fd, log);
|
||||
}
|
||||
fs.close(rs.fd, err => {
|
||||
if (err) {
|
||||
log.error('unable to close file descriptor', {
|
||||
method: 'DataFileStore.get', key, filePath,
|
||||
error: err,
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -12,7 +12,7 @@ function releasePageCacheSync(filePath, fd, log) {
|
|||
const ret = posixFadvise(fd, 0, 0, 4);
|
||||
if (ret !== 0) {
|
||||
log.warning(
|
||||
`error fadv_dontneed ${filePath} returned ${ret}`);
|
||||
`error fadv_dontneed ${filePath} returned ${ret}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -37,16 +37,16 @@ const backend = {
|
|||
}
|
||||
cursor += data.length;
|
||||
})
|
||||
.on('end', () => {
|
||||
if (exceeded) {
|
||||
log.error('data stream exceed announced size',
|
||||
{ size, overflow: cursor });
|
||||
callback(errors.InternalError);
|
||||
} else {
|
||||
ds[count] = { value, keyContext };
|
||||
callback(null, count++);
|
||||
}
|
||||
});
|
||||
.on('end', () => {
|
||||
if (exceeded) {
|
||||
log.error('data stream exceed announced size',
|
||||
{ size, overflow: cursor });
|
||||
callback(errors.InternalError);
|
||||
} else {
|
||||
ds[count] = { value, keyContext };
|
||||
callback(null, count++);
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
get: function getMem(objectGetInfo, range, reqUids, callback) {
|
||||
|
|
|
@ -181,25 +181,25 @@ class MetadataWrapper {
|
|||
const value = typeof objVal.getValue === 'function' ?
|
||||
objVal.getValue() : objVal;
|
||||
this.client.putObject(bucketName, objName, value, params, log,
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
log.debug('error from metadata', { implName: this.implName,
|
||||
error: err });
|
||||
return cb(err);
|
||||
}
|
||||
if (data) {
|
||||
log.debug('object version successfully put in metadata',
|
||||
{ version: data });
|
||||
} else {
|
||||
log.debug('object successfully put in metadata');
|
||||
}
|
||||
return cb(err, data);
|
||||
});
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
log.debug('error from metadata', { implName: this.implName,
|
||||
error: err });
|
||||
return cb(err);
|
||||
}
|
||||
if (data) {
|
||||
log.debug('object version successfully put in metadata',
|
||||
{ version: data });
|
||||
} else {
|
||||
log.debug('object successfully put in metadata');
|
||||
}
|
||||
return cb(err, data);
|
||||
});
|
||||
}
|
||||
|
||||
getBucketAndObjectMD(bucketName, objName, params, log, cb) {
|
||||
log.debug('getting bucket and object from metadata',
|
||||
{ database: bucketName, object: objName });
|
||||
{ database: bucketName, object: objName });
|
||||
this.client.getBucketAndObject(bucketName, objName, params, log,
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
|
@ -208,7 +208,7 @@ class MetadataWrapper {
|
|||
return cb(err);
|
||||
}
|
||||
log.debug('bucket and object retrieved from metadata',
|
||||
{ database: bucketName, object: objName });
|
||||
{ database: bucketName, object: objName });
|
||||
return cb(err, data);
|
||||
});
|
||||
}
|
||||
|
|
|
@ -5,7 +5,7 @@ const BucketInfo = require('../../../models/BucketInfo');
|
|||
class BucketClientInterface {
|
||||
constructor(params, bucketclient, logger) {
|
||||
assert(params.bucketdBootstrap.length > 0,
|
||||
'bucketd bootstrap list is empty');
|
||||
'bucketd bootstrap list is empty');
|
||||
const bootstrap = params.bucketdBootstrap;
|
||||
const log = params.bucketdLog;
|
||||
if (params.https) {
|
||||
|
@ -29,7 +29,7 @@ class BucketClientInterface {
|
|||
|
||||
createBucket(bucketName, bucketMD, log, cb) {
|
||||
this.client.createBucket(bucketName, log.getSerializedUids(),
|
||||
bucketMD.serialize(), cb);
|
||||
bucketMD.serialize(), cb);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
@ -57,17 +57,17 @@ class BucketClientInterface {
|
|||
|
||||
getRaftBuckets(raftId, log, cb) {
|
||||
return this.client.getRaftBuckets(raftId, log.getSerializedUids(),
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
return cb(null, JSON.parse(data));
|
||||
});
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
return cb(null, JSON.parse(data));
|
||||
});
|
||||
}
|
||||
|
||||
putBucketAttributes(bucketName, bucketMD, log, cb) {
|
||||
this.client.putBucketAttributes(bucketName, log.getSerializedUids(),
|
||||
bucketMD.serialize(), cb);
|
||||
bucketMD.serialize(), cb);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
@ -95,7 +95,7 @@ class BucketClientInterface {
|
|||
|
||||
deleteObject(bucketName, objName, params, log, cb) {
|
||||
this.client.deleteObject(bucketName, objName, log.getSerializedUids(),
|
||||
cb, params);
|
||||
cb, params);
|
||||
return null;
|
||||
}
|
||||
|
||||
|
@ -183,8 +183,8 @@ class BucketClientInterface {
|
|||
reason.msg = undefined;
|
||||
respBody[implName] = {
|
||||
code: 200,
|
||||
message, // Provide interpreted reason msg
|
||||
body: reason, // Provide analysis data
|
||||
message, // Provide interpreted reason msg
|
||||
body: reason, // Provide analysis data
|
||||
};
|
||||
if (failure) {
|
||||
// Setting the `error` field is how the healthCheck
|
||||
|
|
|
@ -30,7 +30,6 @@ class ListRecordStream extends stream.Transform {
|
|||
* @classdesc Proxy object to access raft log API
|
||||
*/
|
||||
class LogConsumer {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
*
|
||||
|
@ -97,14 +96,14 @@ class LogConsumer {
|
|||
if (err.code === 404) {
|
||||
// no such raft session, log and ignore
|
||||
this.logger.warn('raft session does not exist yet',
|
||||
{ raftId: this.raftSession });
|
||||
{ raftId: this.raftSession });
|
||||
return cbOnce(null, { info: { start: null,
|
||||
end: null } });
|
||||
}
|
||||
if (err.code === 416) {
|
||||
// requested range not satisfiable
|
||||
this.logger.debug('no new log record to process',
|
||||
{ raftId: this.raftSession });
|
||||
{ raftId: this.raftSession });
|
||||
return cbOnce(null, { info: { start: null,
|
||||
end: null } });
|
||||
}
|
||||
|
@ -116,7 +115,7 @@ class LogConsumer {
|
|||
// is emitted
|
||||
recordStream.on('error', err => {
|
||||
this.logger.error('error receiving raft log',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cbOnce(errors.InternalError);
|
||||
});
|
||||
const jsonResponse = stream.pipe(jsonStream.parse('log.*'));
|
||||
|
@ -127,7 +126,7 @@ class LogConsumer {
|
|||
// remove temporary listener
|
||||
recordStream.removeAllListeners('error');
|
||||
return cbOnce(null, { info: header.info,
|
||||
log: recordStream });
|
||||
log: recordStream });
|
||||
})
|
||||
.on('error', err => recordStream.emit('error', err));
|
||||
return undefined;
|
||||
|
|
|
@ -14,13 +14,13 @@ const _operatorType1 = joi.string().valid(
|
|||
'$gt',
|
||||
'$gte',
|
||||
'$lt',
|
||||
'$lte'
|
||||
'$lte',
|
||||
);
|
||||
|
||||
// supports strings, numbers, and boolean
|
||||
const _operatorType2 = joi.string().valid(
|
||||
'$eq',
|
||||
'$ne'
|
||||
'$ne',
|
||||
);
|
||||
|
||||
const _valueType1 = joi.alternatives([
|
||||
|
|
|
@ -10,14 +10,13 @@ const list = require('../../../algos/list/exportAlgos');
|
|||
const MetadataFileClient = require('./MetadataFileClient');
|
||||
const versionSep =
|
||||
require('../../../versioning/constants')
|
||||
.VersioningConstants.VersionId.Separator;
|
||||
.VersioningConstants.VersionId.Separator;
|
||||
|
||||
const METASTORE = '__metastore';
|
||||
|
||||
const itemScanRefreshDelay = 1000 * 30 * 60; // 30 minutes
|
||||
|
||||
class BucketFileInterface {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
* @param {object} [params] - constructor params
|
||||
|
@ -65,7 +64,7 @@ class BucketFileInterface {
|
|||
if (err) {
|
||||
this.logger.fatal('error writing usersBucket ' +
|
||||
'attributes to metadata',
|
||||
{ error: err });
|
||||
{ error: err });
|
||||
throw (errors.InternalError);
|
||||
}
|
||||
return done();
|
||||
|
@ -104,8 +103,8 @@ class BucketFileInterface {
|
|||
}
|
||||
this.lastItemScanTime = null;
|
||||
this.putBucketAttributes(bucketName,
|
||||
bucketMD,
|
||||
log, cb);
|
||||
bucketMD,
|
||||
log, cb);
|
||||
return undefined;
|
||||
});
|
||||
}
|
||||
|
@ -191,7 +190,7 @@ class BucketFileInterface {
|
|||
errorStack: err.stack,
|
||||
};
|
||||
log.error('error deleting bucket',
|
||||
logObj);
|
||||
logObj);
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
this.lastItemScanTime = null;
|
||||
|
@ -392,16 +391,16 @@ class BucketFileInterface {
|
|||
cbDone = true;
|
||||
async.eachSeries(res.bucketList, (bucket, cb) => {
|
||||
this.getBucketAttributes(bucket.name, log,
|
||||
(err, bucketInfo) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
/* eslint-disable no-param-reassign */
|
||||
bucket.location =
|
||||
(err, bucketInfo) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
/* eslint-disable no-param-reassign */
|
||||
bucket.location =
|
||||
bucketInfo.getLocationConstraint();
|
||||
/* eslint-enable no-param-reassign */
|
||||
return cb();
|
||||
});
|
||||
/* eslint-enable no-param-reassign */
|
||||
return cb();
|
||||
});
|
||||
}, err => {
|
||||
if (!err) {
|
||||
this.lastItemScanTime = Date.now();
|
||||
|
|
|
@ -8,7 +8,6 @@ const { RecordLogProxy } = require('./RecordLog.js');
|
|||
const werelogs = require('werelogs');
|
||||
|
||||
class MetadataFileClient {
|
||||
|
||||
/**
|
||||
* Construct a metadata client
|
||||
*
|
||||
|
@ -86,7 +85,7 @@ class MetadataFileClient {
|
|||
logProxy.connect(err => {
|
||||
if (err) {
|
||||
this.logger.error('error connecting to record log service',
|
||||
{ url, error: err.stack });
|
||||
{ url, error: err.stack });
|
||||
return done(err);
|
||||
}
|
||||
this.logger.info('connected to record log service', { url });
|
||||
|
|
|
@ -25,7 +25,6 @@ const SYNC_OPTIONS = { sync: true };
|
|||
const SUBLEVEL_SEP = '::';
|
||||
|
||||
class MetadataFileServer {
|
||||
|
||||
/**
|
||||
* Construct a metadata server
|
||||
*
|
||||
|
@ -218,7 +217,7 @@ class MetadataFileServer {
|
|||
});
|
||||
} else {
|
||||
this.rootDb.batch(ops, SYNC_OPTIONS,
|
||||
err => callback(err));
|
||||
err => callback(err));
|
||||
}
|
||||
},
|
||||
};
|
||||
|
@ -235,17 +234,17 @@ class MetadataFileServer {
|
|||
put: (env, key, value, options, cb) => {
|
||||
const dbName = env.subLevel.join(SUBLEVEL_SEP);
|
||||
vrp.put({ db: dbName, key, value, options },
|
||||
env.requestLogger, cb);
|
||||
env.requestLogger, cb);
|
||||
},
|
||||
del: (env, key, options, cb) => {
|
||||
const dbName = env.subLevel.join(SUBLEVEL_SEP);
|
||||
vrp.del({ db: dbName, key, options },
|
||||
env.requestLogger, cb);
|
||||
env.requestLogger, cb);
|
||||
},
|
||||
get: (env, key, options, cb) => {
|
||||
const dbName = env.subLevel.join(SUBLEVEL_SEP);
|
||||
vrp.get({ db: dbName, key, options },
|
||||
env.requestLogger, cb);
|
||||
env.requestLogger, cb);
|
||||
},
|
||||
getDiskUsage: (env, cb) => diskusage.check(this.path, cb),
|
||||
});
|
||||
|
|
|
@ -18,7 +18,6 @@ const DEFAULT_RECORD_LOG_NAME = 's3-recordlog';
|
|||
* object.
|
||||
*/
|
||||
class RecordLogProxy extends rpc.BaseClient {
|
||||
|
||||
constructor(params) {
|
||||
super(params);
|
||||
|
||||
|
@ -102,7 +101,6 @@ class ListRecordStream extends stream.Transform {
|
|||
* updates can be transactional with each other.
|
||||
*/
|
||||
class RecordLogService extends rpc.BaseService {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
*
|
||||
|
@ -274,12 +272,12 @@ class RecordLogService extends rpc.BaseService {
|
|||
limit: _params.limit,
|
||||
};
|
||||
const userStream = new ListRecordStream(endSeq,
|
||||
_params.limit);
|
||||
_params.limit);
|
||||
const dbStream =
|
||||
openLog.logDb.createReadStream(queryParams);
|
||||
dbStream.pipe(userStream);
|
||||
dbStream.once('error',
|
||||
err => userStream.emit('error', err));
|
||||
err => userStream.emit('error', err));
|
||||
userStream.once('error', err => {
|
||||
userStream.removeAllListeners('info');
|
||||
cb(err);
|
||||
|
|
|
@ -8,8 +8,8 @@ function markerFilterMPU(allMarkers, array) {
|
|||
// in the array that is alphabetically after keyMarker
|
||||
const firstUnfilteredIndex = array.findIndex(
|
||||
item => (uploadIdMarker && item.key === keyMarker ?
|
||||
item.uploadId > uploadIdMarker :
|
||||
item.key > keyMarker));
|
||||
item.uploadId > uploadIdMarker :
|
||||
item.key > keyMarker));
|
||||
return firstUnfilteredIndex !== -1 ? array.slice(firstUnfilteredIndex) : [];
|
||||
}
|
||||
|
||||
|
|
|
@ -182,7 +182,7 @@ const metastore = {
|
|||
if (params && params.versionId) {
|
||||
const baseKey = inc(formatVersionKey(objName, ''));
|
||||
const vobjName = formatVersionKey(objName,
|
||||
params.versionId);
|
||||
params.versionId);
|
||||
metadata.keyMaps.get(bucketName).delete(vobjName);
|
||||
const mst = metadata.keyMaps.get(bucketName).get(objName);
|
||||
if (mst.versionId === params.versionId) {
|
||||
|
|
|
@ -79,9 +79,9 @@ class ListRecordStream extends stream.Readable {
|
|||
'did not encounter the last saved offset in oplog, ' +
|
||||
'resuming processing right after the latest record ' +
|
||||
'to date; some entries may have been skipped', {
|
||||
lastSavedID: this._lastSavedID,
|
||||
latestRecordID: this._latestOplogID,
|
||||
});
|
||||
lastSavedID: this._lastSavedID,
|
||||
latestRecordID: this._latestOplogID,
|
||||
});
|
||||
this._unpublishedListing = true;
|
||||
}
|
||||
++this._skipCount;
|
||||
|
@ -110,7 +110,7 @@ class ListRecordStream extends stream.Readable {
|
|||
}
|
||||
entry = {
|
||||
type: 'put', // updates overwrite the whole metadata,
|
||||
// so they are considered as puts
|
||||
// so they are considered as puts
|
||||
key: itemObj.o2._id,
|
||||
// updated value may be either stored directly in 'o'
|
||||
// attribute or in '$set' attribute (supposedly when
|
||||
|
@ -132,7 +132,7 @@ class ListRecordStream extends stream.Readable {
|
|||
}
|
||||
const streamObject = {
|
||||
timestamp: new Date((itemObj.ts ?
|
||||
itemObj.ts.toNumber() * 1000 : 0)),
|
||||
itemObj.ts.toNumber() * 1000 : 0)),
|
||||
db: dbName,
|
||||
entries: [entry],
|
||||
};
|
||||
|
|
|
@ -9,7 +9,6 @@ const MongoUtils = require('./utils');
|
|||
* @classdesc Class to consume mongo oplog
|
||||
*/
|
||||
class LogConsumer {
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
*
|
||||
|
@ -43,7 +42,7 @@ class LogConsumer {
|
|||
(err, client) => {
|
||||
if (err) {
|
||||
this._logger.error('Unable to connect to MongoDB',
|
||||
{ error: err });
|
||||
{ error: err });
|
||||
return done(err);
|
||||
}
|
||||
this._logger.info('connected to mongodb');
|
||||
|
|
|
@ -50,7 +50,7 @@ const VID_SEP = require('../../../versioning/constants')
|
|||
function generateVersionId(replicationGroupId) {
|
||||
// generate a unique number for each member of the nodejs cluster
|
||||
return genVID(`${process.pid}.${uidCounter++}`,
|
||||
replicationGroupId);
|
||||
replicationGroupId);
|
||||
}
|
||||
|
||||
function formatVersionKey(key, versionId) {
|
||||
|
@ -168,7 +168,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
this.logger.fatal('error writing usersBucket ' +
|
||||
'attributes to metastore',
|
||||
{ error: err });
|
||||
{ error: err });
|
||||
throw (errors.InternalError);
|
||||
}
|
||||
return cb();
|
||||
|
@ -185,7 +185,7 @@ class MongoClientInterface {
|
|||
getCollection(name) {
|
||||
/* mongo has a problem with .. in collection names */
|
||||
const newName = (name === constants.usersBucket) ?
|
||||
USERSBUCKET : name;
|
||||
USERSBUCKET : name;
|
||||
return this.db.collection(newName);
|
||||
}
|
||||
|
||||
|
@ -209,7 +209,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
log.error(
|
||||
'createBucket: error creating bucket',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
this.lastItemScanTime = null;
|
||||
|
@ -222,7 +222,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
log.error(
|
||||
'createBucket: error creating bucket',
|
||||
{ error: err });
|
||||
{ error: err });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
if (this.shardCollections) {
|
||||
|
@ -282,12 +282,12 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
if (err === errors.NoSuchKey) {
|
||||
return cb(null,
|
||||
{ bucket:
|
||||
{ bucket:
|
||||
BucketInfo.fromObj(bucket).serialize(),
|
||||
});
|
||||
});
|
||||
}
|
||||
log.error('getObject: error getting object',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(err);
|
||||
}
|
||||
return cb(null, {
|
||||
|
@ -317,7 +317,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
log.error(
|
||||
'putBucketAttributes: error putting bucket attributes',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return cb();
|
||||
|
@ -334,12 +334,12 @@ class MongoClientInterface {
|
|||
}, {}, (err, result) => {
|
||||
if (err) {
|
||||
log.error('deleteBucketStep2: error deleting bucket',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
if (result.ok !== 1) {
|
||||
log.error('deleteBucketStep2: failed deleting bucket',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return cb(null);
|
||||
|
@ -364,7 +364,7 @@ class MongoClientInterface {
|
|||
return this.deleteBucketStep2(bucketName, log, cb);
|
||||
}
|
||||
log.error('deleteBucket: error deleting bucket',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return this.deleteBucketStep2(bucketName, log, err => {
|
||||
|
@ -419,7 +419,7 @@ class MongoClientInterface {
|
|||
$gt: objVal.versionId,
|
||||
},
|
||||
},
|
||||
],
|
||||
],
|
||||
},
|
||||
update: {
|
||||
$set: { _id: objName, value: objVal },
|
||||
|
@ -494,7 +494,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
log.error(
|
||||
'putObjectVerCase2: error putting object version',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return cb(null, `{"versionId": "${objVal.versionId}"}`);
|
||||
|
@ -538,7 +538,7 @@ class MongoClientInterface {
|
|||
updateOne: {
|
||||
// eslint-disable-next-line
|
||||
filter: {
|
||||
_id: objName,
|
||||
'_id': objName,
|
||||
'value.versionId': params.versionId,
|
||||
},
|
||||
update: {
|
||||
|
@ -555,7 +555,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
log.error(
|
||||
'putObjectVerCase3: error putting object version',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
if (err.code === 11000) {
|
||||
// We want duplicate key error logged however in
|
||||
// case of the race condition mentioned above, the
|
||||
|
@ -601,13 +601,13 @@ class MongoClientInterface {
|
|||
this.getLatestVersion(c, objName, log, (err, mstObjVal) => {
|
||||
if (err) {
|
||||
log.error('getLatestVersion: getting latest version',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(err);
|
||||
}
|
||||
MongoUtils.serialize(mstObjVal);
|
||||
// eslint-disable-next-line
|
||||
c.update({
|
||||
_id: objName,
|
||||
'_id': objName,
|
||||
'value.versionId': {
|
||||
// We break the semantic correctness here with
|
||||
// $gte instead of $gt because we do not have
|
||||
|
@ -673,16 +673,16 @@ class MongoClientInterface {
|
|||
const c = this.getCollection(bucketName);
|
||||
if (params && params.versioning && !params.versionId) {
|
||||
return this.putObjectVerCase1(c, bucketName, objName, objVal,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
} else if (params && params.versionId === '') {
|
||||
return this.putObjectVerCase2(c, bucketName, objName, objVal,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
} else if (params && params.versionId && !params.repairMaster) {
|
||||
return this.putObjectVerCase3(c, bucketName, objName, objVal,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
} else if (params && params.versionId && params.repairMaster) {
|
||||
return this.putObjectVerCase4(c, bucketName, objName, objVal,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
}
|
||||
return this.putObjectNoVer(c, bucketName, objName, objVal,
|
||||
params, log, cb);
|
||||
|
@ -699,7 +699,7 @@ class MongoClientInterface {
|
|||
}, {}, (err, doc) => {
|
||||
if (err) {
|
||||
log.error('findOne: error getting object',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
if (!doc) {
|
||||
|
@ -709,7 +709,7 @@ class MongoClientInterface {
|
|||
this.getLatestVersion(c, objName, log, (err, value) => {
|
||||
if (err) {
|
||||
log.error('getLatestVersion: getting latest version',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(err);
|
||||
}
|
||||
return cb(null, value);
|
||||
|
@ -760,7 +760,7 @@ class MongoClientInterface {
|
|||
MongoUtils.serialize(objVal);
|
||||
// eslint-disable-next-line
|
||||
c.findOneAndReplace({
|
||||
_id: objName,
|
||||
'_id': objName,
|
||||
'value.isPHD': true,
|
||||
'value.versionId': mst.versionId,
|
||||
}, {
|
||||
|
@ -771,12 +771,12 @@ class MongoClientInterface {
|
|||
}, (err, result) => {
|
||||
if (err) {
|
||||
log.error('repair: error trying to repair value',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
if (result.ok !== 1) {
|
||||
log.error('repair: failed trying to repair value',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return cb(null);
|
||||
|
@ -791,7 +791,7 @@ class MongoClientInterface {
|
|||
this.getLatestVersion(c, objName, log, (err, value) => {
|
||||
if (err) {
|
||||
log.error('async-repair: getting latest version',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return undefined;
|
||||
}
|
||||
this.repair(c, bucketName, objName, value, mst, log, err => {
|
||||
|
@ -822,7 +822,7 @@ class MongoClientInterface {
|
|||
// version:
|
||||
// eslint-disable-next-line
|
||||
c.findOneAndDelete({
|
||||
_id: objName,
|
||||
'_id': objName,
|
||||
'value.isPHD': true,
|
||||
'value.versionId': mst.versionId,
|
||||
}, {}, err => {
|
||||
|
@ -839,7 +839,7 @@ class MongoClientInterface {
|
|||
return undefined;
|
||||
}
|
||||
log.error('getLatestVersion: error getting latest version',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(err);
|
||||
}
|
||||
// We have other versions available so repair:
|
||||
|
@ -883,7 +883,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
log.error(
|
||||
'deleteObjectVerMaster: error deleting object',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return this.deleteOrRepairPHD(c, bucketName, objName, mst, log, cb);
|
||||
|
@ -927,7 +927,7 @@ class MongoClientInterface {
|
|||
}, {}, (err, mst) => {
|
||||
if (err) {
|
||||
log.error('deleteObjectVer: error deleting versioned object',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
if (!mst) {
|
||||
|
@ -936,15 +936,15 @@ class MongoClientInterface {
|
|||
if (mst.value.isPHD ||
|
||||
mst.value.versionId === params.versionId) {
|
||||
return this.deleteObjectVerMaster(c, bucketName, objName,
|
||||
params, log, err => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
return cb();
|
||||
});
|
||||
params, log, err => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
return cb();
|
||||
});
|
||||
}
|
||||
return this.deleteObjectVerNotMaster(c, bucketName, objName,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -975,10 +975,10 @@ class MongoClientInterface {
|
|||
const c = this.getCollection(bucketName);
|
||||
if (params && params.versionId) {
|
||||
return this.deleteObjectVer(c, bucketName, objName,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
}
|
||||
return this.deleteObjectNoVer(c, bucketName, objName,
|
||||
params, log, cb);
|
||||
params, log, cb);
|
||||
}
|
||||
|
||||
internalListObject(bucketName, params, extension, log, cb) {
|
||||
|
@ -1041,7 +1041,7 @@ class MongoClientInterface {
|
|||
const internalParams = extension.genMDParams();
|
||||
internalParams.mongifiedSearch = params.mongifiedSearch;
|
||||
return this.internalListObject(bucketName, internalParams, extension,
|
||||
log, cb);
|
||||
log, cb);
|
||||
}
|
||||
|
||||
listMultipartUploads(bucketName, params, log, cb) {
|
||||
|
@ -1050,7 +1050,7 @@ class MongoClientInterface {
|
|||
const internalParams = extension.genMDParams();
|
||||
internalParams.mongifiedSearch = params.mongifiedSearch;
|
||||
return this.internalListObject(bucketName, internalParams, extension,
|
||||
log, cb);
|
||||
log, cb);
|
||||
}
|
||||
|
||||
checkHealth(implName, log, cb) {
|
||||
|
@ -1074,7 +1074,7 @@ class MongoClientInterface {
|
|||
}, {}, (err, doc) => {
|
||||
if (err) {
|
||||
log.error('readUUID: error reading UUID',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
if (!doc) {
|
||||
|
@ -1096,7 +1096,7 @@ class MongoClientInterface {
|
|||
return cb(errors.KeyAlreadyExists);
|
||||
}
|
||||
log.error('writeUUIDIfNotExists: error writing UUID',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
// FIXME: shoud we check for result.ok === 1 ?
|
||||
|
@ -1114,7 +1114,7 @@ class MongoClientInterface {
|
|||
if (err) {
|
||||
if (err === errors.InternalError) {
|
||||
log.error('getUUID: error getting UUID',
|
||||
{ error: err.message });
|
||||
{ error: err.message });
|
||||
return cb(err);
|
||||
}
|
||||
return this.readUUID(log, cb);
|
||||
|
@ -1129,7 +1129,7 @@ class MongoClientInterface {
|
|||
// For Kub/cluster deployments there should be a more sophisticated
|
||||
// way for guessing free space.
|
||||
diskusage.check(this.path !== undefined ?
|
||||
this.path : '/', cb);
|
||||
this.path : '/', cb);
|
||||
}
|
||||
|
||||
readCountItems(log, cb) {
|
||||
|
@ -1616,7 +1616,7 @@ class MongoClientInterface {
|
|||
const retResult = this._handleResults(collRes, isVer);
|
||||
retResult.stalled = stalledCount;
|
||||
return callback(null, retResult);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -1654,7 +1654,7 @@ class MongoClientInterface {
|
|||
const filter = { _id: objName };
|
||||
try {
|
||||
MongoUtils.translateConditions(0, 'value', filter,
|
||||
params.conditions);
|
||||
params.conditions);
|
||||
} catch (err) {
|
||||
log.error('error creating mongodb filter', {
|
||||
error: reshapeExceptionError(err),
|
||||
|
@ -1700,7 +1700,7 @@ class MongoClientInterface {
|
|||
const filter = { _id: objName };
|
||||
try {
|
||||
MongoUtils.translateConditions(0, 'value', filter,
|
||||
params.conditions);
|
||||
params.conditions);
|
||||
} catch (err) {
|
||||
log.error('error creating mongodb filter', {
|
||||
error: reshapeExceptionError(err),
|
||||
|
|
|
@ -8,8 +8,8 @@ function escape(obj) {
|
|||
const _obj = {};
|
||||
Object.keys(obj).forEach(prop => {
|
||||
const _prop = prop.
|
||||
replace(/\$/g, '\uFF04').
|
||||
replace(/\./g, '\uFF0E');
|
||||
replace(/\$/g, '\uFF04').
|
||||
replace(/\./g, '\uFF0E');
|
||||
_obj[_prop] = obj[prop];
|
||||
});
|
||||
return _obj;
|
||||
|
@ -19,8 +19,8 @@ function unescape(obj) {
|
|||
const _obj = {};
|
||||
Object.keys(obj).forEach(prop => {
|
||||
const _prop = prop.
|
||||
replace(/\uFF04/g, '$').
|
||||
replace(/\uFF0E/g, '.');
|
||||
replace(/\uFF04/g, '$').
|
||||
replace(/\uFF0E/g, '.');
|
||||
_obj[_prop] = obj[prop];
|
||||
});
|
||||
return _obj;
|
||||
|
|
|
@ -35,15 +35,15 @@ class BucketdRoutes {
|
|||
bucketName, logger, (err, data) => {
|
||||
if (err) {
|
||||
logger.error('Failed to get bucket attributes',
|
||||
{ bucket: bucketName, error: err });
|
||||
{ bucket: bucketName, error: err });
|
||||
return sendResponse(req, res, logger, err);
|
||||
}
|
||||
if (data === undefined) {
|
||||
return sendResponse(req, res, logger,
|
||||
errors.NoSuchBucket);
|
||||
errors.NoSuchBucket);
|
||||
}
|
||||
return sendResponse(req, res, logger, null,
|
||||
BucketInfo.fromObj(data).serialize());
|
||||
BucketInfo.fromObj(data).serialize());
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -112,7 +112,7 @@ class BucketdRoutes {
|
|||
return this._metadataWrapper.checkHealth(logger, (err, resp) => {
|
||||
if (err) {
|
||||
logger.error('Failed the health check',
|
||||
{ error: err, method: '_checkHealth' });
|
||||
{ error: err, method: '_checkHealth' });
|
||||
return sendResponse(req, res, logger, err);
|
||||
}
|
||||
return sendResponse(req, res, logger, undefined, resp);
|
||||
|
@ -122,8 +122,8 @@ class BucketdRoutes {
|
|||
_createRequestLogger(req) {
|
||||
const uids = req.headers['x-scal-request-uids'];
|
||||
const logger = uids === undefined ?
|
||||
this._logger.newRequestLogger() :
|
||||
this._logger.newRequestLoggerFromSerializedUids(uids);
|
||||
this._logger.newRequestLogger() :
|
||||
this._logger.newRequestLoggerFromSerializedUids(uids);
|
||||
logger.trace('new request', { method: req.method, url: req.url });
|
||||
return logger;
|
||||
}
|
||||
|
@ -150,7 +150,7 @@ class BucketdRoutes {
|
|||
_attributesRoutes(req, res, uriComponents, logger) {
|
||||
if (uriComponents.bucketName === undefined) {
|
||||
logger.error('Missing bucket name for attributes route',
|
||||
{ uriComponents });
|
||||
{ uriComponents });
|
||||
return sendResponse(req, res, logger, errors.BadRequest);
|
||||
}
|
||||
switch (req.method) {
|
||||
|
@ -195,26 +195,26 @@ class BucketdRoutes {
|
|||
_bucketRoutes(req, res, uriComponents, logger) {
|
||||
if (uriComponents.bucketName === undefined) {
|
||||
logger.error('Missing bucket name for bucket route',
|
||||
{ uriComponents });
|
||||
{ uriComponents });
|
||||
return sendResponse(req, res, logger, errors.BadRequest);
|
||||
}
|
||||
switch (req.method) {
|
||||
case 'GET':
|
||||
return this._listObject(req, res,
|
||||
uriComponents.bucketName,
|
||||
uriComponents.options,
|
||||
logger);
|
||||
uriComponents.bucketName,
|
||||
uriComponents.options,
|
||||
logger);
|
||||
case 'DELETE':
|
||||
return this._deleteBucket(req, res,
|
||||
uriComponents.bucketName, logger);
|
||||
uriComponents.bucketName, logger);
|
||||
case 'POST':
|
||||
return getRequestBody(logger, req, (err, body) => {
|
||||
if (err) {
|
||||
return sendResponse(req, res, logger, err);
|
||||
}
|
||||
return this._createBucket(req, res,
|
||||
uriComponents.bucketName,
|
||||
body, logger);
|
||||
uriComponents.bucketName,
|
||||
body, logger);
|
||||
});
|
||||
default:
|
||||
return sendResponse(req, res, logger, errors.RouteNotFound);
|
||||
|
@ -233,30 +233,30 @@ class BucketdRoutes {
|
|||
_objectRoutes(req, res, uriComponents, logger) {
|
||||
if (uriComponents.bucketName === undefined) {
|
||||
logger.error('Missing bucket name for object route',
|
||||
{ uriComponents });
|
||||
{ uriComponents });
|
||||
return sendResponse(req, res, logger, errors.BadRequest);
|
||||
}
|
||||
switch (req.method) {
|
||||
case 'GET':
|
||||
return this._getObject(req, res,
|
||||
uriComponents.bucketName,
|
||||
uriComponents.objectName,
|
||||
uriComponents.options,
|
||||
logger);
|
||||
uriComponents.bucketName,
|
||||
uriComponents.objectName,
|
||||
uriComponents.options,
|
||||
logger);
|
||||
case 'DELETE':
|
||||
return this._deleteObject(req, res,
|
||||
uriComponents.bucketName,
|
||||
uriComponents.objectName,
|
||||
uriComponents.options,
|
||||
logger);
|
||||
uriComponents.bucketName,
|
||||
uriComponents.objectName,
|
||||
uriComponents.options,
|
||||
logger);
|
||||
case 'POST':
|
||||
return getRequestBody(logger, req, (err, body) =>
|
||||
this._putObject(req, res,
|
||||
uriComponents.bucketName,
|
||||
uriComponents.objectName,
|
||||
body,
|
||||
uriComponents.options,
|
||||
logger));
|
||||
this._putObject(req, res,
|
||||
uriComponents.bucketName,
|
||||
uriComponents.objectName,
|
||||
body,
|
||||
uriComponents.options,
|
||||
logger));
|
||||
default:
|
||||
return sendResponse(req, res, logger, errors.RouteNotFound);
|
||||
}
|
||||
|
@ -274,16 +274,16 @@ class BucketdRoutes {
|
|||
_parallelRoutes(req, res, uriComponents, logger) {
|
||||
if (uriComponents.bucketName === undefined) {
|
||||
logger.error('Missing bucket name for parallel route',
|
||||
{ uriComponents });
|
||||
{ uriComponents });
|
||||
return sendResponse(req, res, logger, errors.BadRequest);
|
||||
}
|
||||
switch (req.method) {
|
||||
case 'GET':
|
||||
return this._getBucketAndObjectMD(req, res,
|
||||
uriComponents.bucketName,
|
||||
uriComponents.objectName,
|
||||
uriComponents.options,
|
||||
logger);
|
||||
uriComponents.bucketName,
|
||||
uriComponents.objectName,
|
||||
uriComponents.options,
|
||||
logger);
|
||||
default:
|
||||
return sendResponse(req, res, logger, errors.RouteNotFound);
|
||||
}
|
||||
|
@ -307,7 +307,7 @@ class BucketdRoutes {
|
|||
return sendResponse(req, res, logger, errors.NotImplemented);
|
||||
case 'metadataInformation':
|
||||
return sendResponse(req, res, logger, undefined,
|
||||
'{"metadataVersion":2}');
|
||||
'{"metadataVersion":2}');
|
||||
case 'parallel':
|
||||
logger.trace(`${uriComponents.context} operation`);
|
||||
if (uriComponents.objectName) {
|
||||
|
|
|
@ -11,7 +11,6 @@ const requiresOneWorker = {
|
|||
};
|
||||
|
||||
class Server {
|
||||
|
||||
/**
|
||||
* Create a new Metadata Proxy Server instance
|
||||
*
|
||||
|
@ -31,7 +30,7 @@ class Server {
|
|||
if (requiresOneWorker[metadataWrapper.implName] &&
|
||||
this._configuration.workers !== 1) {
|
||||
logger.warn('This metadata backend requires only one worker',
|
||||
{ metadataBackend: metadataWrapper.implName });
|
||||
{ metadataBackend: metadataWrapper.implName });
|
||||
this._configuration.workers = 1;
|
||||
}
|
||||
this._logger = logger;
|
||||
|
@ -79,7 +78,7 @@ class Server {
|
|||
});
|
||||
} else {
|
||||
this._httpServer = new HttpServer(this._configuration.port,
|
||||
this._logger);
|
||||
this._logger);
|
||||
if (this._configuration.bindAddress) {
|
||||
this._httpServer.setBindAddress(
|
||||
this._configuration.bindAddress);
|
||||
|
|
|
@ -67,8 +67,8 @@ function getURIComponents(uri, logger) {
|
|||
};
|
||||
} catch (ex) {
|
||||
logger.error('Invalid URI: failed to parse',
|
||||
{ uri, error: ex, errorStack: ex.stack,
|
||||
message: ex.message });
|
||||
{ uri, error: ex, errorStack: ex.stack,
|
||||
message: ex.message });
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -60,8 +60,8 @@ class TestMatrix {
|
|||
this.elementsToSpecialize = elementsToSpecialize;
|
||||
this.callback = callback;
|
||||
this.description = typeof description === 'undefined'
|
||||
? ''
|
||||
: description;
|
||||
? ''
|
||||
: description;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
@ -158,15 +158,15 @@ class TestMatrix {
|
|||
const callFunction = (matrixFather, matrixChild, callback,
|
||||
description) => {
|
||||
const result = Object.keys(matrixChild.params)
|
||||
.every(currentKey =>
|
||||
Object.prototype.toString.call(
|
||||
matrixChild.params[currentKey]
|
||||
.every(currentKey =>
|
||||
Object.prototype.toString.call(
|
||||
matrixChild.params[currentKey],
|
||||
).indexOf('Array') === -1);
|
||||
|
||||
if (result === true) {
|
||||
describe(matrixChild.serialize(), () => {
|
||||
it(description,
|
||||
done => callback(matrixChild, done));
|
||||
done => callback(matrixChild, done));
|
||||
});
|
||||
} else {
|
||||
describe(matrixChild.serialize(), () => {
|
||||
|
|
|
@ -247,7 +247,7 @@ function decode(str) {
|
|||
}
|
||||
|
||||
module.exports = { generateVersionId, getInfVid,
|
||||
hexEncode, hexDecode,
|
||||
base62Encode, base62Decode,
|
||||
encode, decode,
|
||||
ENC_TYPE_HEX, ENC_TYPE_BASE62 };
|
||||
hexEncode, hexDecode,
|
||||
base62Encode, base62Decode,
|
||||
encode, decode,
|
||||
ENC_TYPE_HEX, ENC_TYPE_BASE62 };
|
||||
|
|
|
@ -84,7 +84,7 @@ class VersioningRequestProcessor {
|
|||
return callback(null, data);
|
||||
}
|
||||
logger.debug('master version is a PHD, getting the latest version',
|
||||
{ db, key });
|
||||
{ db, key });
|
||||
// otherwise, need to search for the latest version
|
||||
return this.getByListing(request, logger, callback);
|
||||
});
|
||||
|
@ -187,7 +187,7 @@ class VersioningRequestProcessor {
|
|||
return entry.callback(err, value);
|
||||
}
|
||||
return this.wgm.get(entry.request, entry.logger,
|
||||
entry.callback);
|
||||
entry.callback);
|
||||
});
|
||||
delete this.queue[cacheKey];
|
||||
}
|
||||
|
@ -267,19 +267,19 @@ class VersioningRequestProcessor {
|
|||
return callback(err);
|
||||
}
|
||||
return this.writeCache.batch({ db, array, options },
|
||||
logger, err => callback(err, `{"versionId":"${vid}"}`));
|
||||
logger, err => callback(err, `{"versionId":"${vid}"}`));
|
||||
};
|
||||
|
||||
if (versionId) {
|
||||
return this.processVersionSpecificPut(request, logger,
|
||||
versioningCb);
|
||||
versioningCb);
|
||||
}
|
||||
if (versioning) {
|
||||
return this.processNewVersionPut(request, logger, versioningCb);
|
||||
}
|
||||
// no versioning or versioning configuration off
|
||||
return this.writeCache.batch({ db, array: [{ key, value }] },
|
||||
logger, callback);
|
||||
logger, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -353,7 +353,7 @@ class VersioningRequestProcessor {
|
|||
if (!(options && options.versionId)) {
|
||||
return this.writeCache.batch({ db,
|
||||
array: [{ key, type: 'del' }] },
|
||||
logger, callback);
|
||||
logger, callback);
|
||||
}
|
||||
// version specific DELETE
|
||||
return this.processVersionSpecificDelete(request, logger,
|
||||
|
@ -399,7 +399,7 @@ class VersioningRequestProcessor {
|
|||
const cacheKey = formatCacheKey(db, key);
|
||||
clearTimeout(this.repairing[cacheKey]);
|
||||
this.repairing[cacheKey] = setTimeout(() =>
|
||||
this.getByListing(request, logger, () => {}), 15000);
|
||||
this.getByListing(request, logger, () => {}), 15000);
|
||||
}
|
||||
return callback(null, ops, versionId);
|
||||
});
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue