Compare commits

..

No commits in common. "a14037f5183e656d7f63771dfb3f6f264cd489ed" and "9a1a56e7b957795eff7de20c48ff2dabc92da355" have entirely different histories.

182 changed files with 4787 additions and 6805 deletions

1
.eslintrc Normal file
View File

@ -0,0 +1 @@
{ "extends": "scality" }

View File

@ -1,7 +0,0 @@
{
"extends": ["scality"],
"plugins": ["jest"],
"env": {
"jest/globals": true
}
}

View File

@ -1,8 +0,0 @@
module.exports = {
presets: [
['@babel/preset-env', { targets: { node: 'current' } }],
'@babel/preset-typescript',
],
plugins: ['add-module-exports'],
};

View File

@ -76,10 +76,10 @@ module.exports = {
tagging: require('./lib/s3middleware/tagging'),
checkDateModifiedHeaders:
require('./lib/s3middleware/validateConditionalHeaders')
.checkDateModifiedHeaders,
.checkDateModifiedHeaders,
validateConditionalHeaders:
require('./lib/s3middleware/validateConditionalHeaders')
.validateConditionalHeaders,
.validateConditionalHeaders,
MD5Sum: require('./lib/s3middleware/MD5Sum'),
NullStream: require('./lib/s3middleware/nullStream'),
objectUtils: require('./lib/s3middleware/objectUtils'),

View File

@ -1,7 +1,7 @@
'use strict'; // eslint-disable-line strict
const { inc, checkLimit, listingParamsMasterKeysV0ToV1,
FILTER_END, FILTER_ACCEPT } = require('./tools');
FILTER_END, FILTER_ACCEPT } = require('./tools');
const DEFAULT_MAX_KEYS = 1000;
const VSConst = require('../../versioning/constants').VersioningConstants;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;

View File

@ -2,7 +2,7 @@
const Extension = require('./Extension').default;
const { inc, listingParamsMasterKeysV0ToV1,
FILTER_END, FILTER_ACCEPT, FILTER_SKIP } = require('./tools');
FILTER_END, FILTER_ACCEPT, FILTER_SKIP } = require('./tools');
const VSConst = require('../../versioning/constants').VersioningConstants;
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
@ -66,7 +66,7 @@ class Delimiter extends Extension {
this.continuationToken = parameters.continuationToken;
this.alphabeticalOrder =
typeof parameters.alphabeticalOrder !== 'undefined' ?
parameters.alphabeticalOrder : true;
parameters.alphabeticalOrder : true;
this.vFormat = vFormat || BucketVersioningKeyFormat.v0;
// results
@ -87,7 +87,7 @@ class Delimiter extends Extension {
this[this.nextContinueMarker].startsWith(this.prefix || '')) {
const nextDelimiterIndex =
this[this.nextContinueMarker].indexOf(this.delimiter,
this.prefix ? this.prefix.length : 0);
this.prefix ? this.prefix.length : 0);
this[this.nextContinueMarker] =
this[this.nextContinueMarker].slice(0, nextDelimiterIndex +
this.delimiter.length);

View File

@ -194,7 +194,7 @@ class DelimiterVersions extends Delimiter {
// length is the same so we can remove their prefix without
// looking at the type of key
return this.filterCommon(obj.key.slice(DbPrefixes.Master.length),
obj.value);
obj.value);
}
filterCommon(key, value) {
@ -250,7 +250,7 @@ class DelimiterVersions extends Delimiter {
}
// skip to the same object key in both M and V range listings
return [DbPrefixes.Master + skipV0,
DbPrefixes.Version + skipV0];
DbPrefixes.Version + skipV0];
}
/**

View File

@ -1,6 +1,7 @@
const ArrayUtils = require('./ArrayUtils');
class SortedSet {
constructor(obj) {
if (obj) {
this.keys = obj.keys;

View File

@ -91,7 +91,7 @@ class Vault {
requestContext: serializedRCsArr,
},
(err, userInfo) => vaultSignatureCb(err, userInfo,
params.log, callback),
params.log, callback)
);
}
@ -146,7 +146,7 @@ class Vault {
requestContext: serializedRCs,
},
(err, userInfo) => vaultSignatureCb(err, userInfo,
params.log, callback, streamingV4Params),
params.log, callback, streamingV4Params)
);
}
@ -232,28 +232,28 @@ class Vault {
*/
getAccountIds(canonicalIDs, log, callback) {
log.trace('getting accountIds from Vault based on canonicalIDs',
{ canonicalIDs });
{ canonicalIDs });
this.client.getAccountIds(canonicalIDs,
{ reqUid: log.getSerializedUids() },
(err, info) => {
if (err) {
log.debug('received error message from vault',
{ errorMessage: err });
return callback(err);
}
const infoFromVault = info.message.body;
log.trace('info received from vault', { infoFromVault });
const result = {};
/* If the accountId was not found in Vault, do not
{ reqUid: log.getSerializedUids() },
(err, info) => {
if (err) {
log.debug('received error message from vault',
{ errorMessage: err });
return callback(err);
}
const infoFromVault = info.message.body;
log.trace('info received from vault', { infoFromVault });
const result = {};
/* If the accountId was not found in Vault, do not
send the canonicalID back to the API */
Object.keys(infoFromVault).forEach(key => {
if (infoFromVault[key] !== 'NotFound' &&
Object.keys(infoFromVault).forEach(key => {
if (infoFromVault[key] !== 'NotFound' &&
infoFromVault[key] !== 'WrongFormat') {
result[key] = infoFromVault[key];
}
});
return callback(null, result);
result[key] = infoFromVault[key];
}
});
return callback(null, result);
});
}
/** checkPolicies -- call Vault to evaluate policies

View File

@ -74,7 +74,7 @@ function extractParams(request, log, awsService, data) {
version = 'v4';
} else {
log.trace('invalid authorization security header',
{ header: authHeader });
{ header: authHeader });
return { err: errors.AccessDenied };
}
} else if (data.Signature) {
@ -89,7 +89,7 @@ function extractParams(request, log, awsService, data) {
if (version !== null && method !== null) {
if (!checkFunctions[version] || !checkFunctions[version][method]) {
log.trace('invalid auth version or method',
{ version, authMethod: method });
{ version, authMethod: method });
return { err: errors.NotImplemented };
}
log.trace('identified auth method', { version, authMethod: method });
@ -161,7 +161,7 @@ function doAuth(request, log, cb, awsService, requestContexts) {
* @return {undefined}
*/
function generateV4Headers(request, data, accessKey, secretKeyValue,
awsService, proxyPath, sessionToken) {
awsService, proxyPath, sessionToken) {
Object.assign(request, { headers: {} });
const amzDate = convertUTCtoISO8601(Date.now());
// get date without time
@ -194,16 +194,16 @@ function generateV4Headers(request, data, accessKey, secretKeyValue,
.filter(headerName =>
headerName.startsWith('x-amz-')
|| headerName.startsWith('x-scal-')
|| headerName === 'host',
|| headerName === 'host'
).sort().join(';');
const params = { request, signedHeaders, payloadChecksum,
credentialScope, timestamp, query: data,
awsService: service, proxyPath };
const stringToSign = constructStringToSignV4(params);
const signingKey = vaultUtilities.calculateSigningKey(secretKeyValue,
region,
scopeDate,
service);
region,
scopeDate,
service);
const signature = crypto.createHmac('sha256', signingKey)
.update(stringToSign, 'binary').digest('hex');
const authorizationHeader = `${algorithm} Credential=${accessKey}` +

View File

@ -29,7 +29,7 @@ class ChainBackend extends BaseBackend {
typeof client.getCanonicalIds === 'function' &&
typeof client.getEmailAddresses === 'function' &&
typeof client.checkPolicies === 'function' &&
typeof client.healthcheck === 'function',
typeof client.healthcheck === 'function'
), 'invalid client: missing required auth backend methods');
this._clients = clients;
}
@ -55,7 +55,7 @@ class ChainBackend extends BaseBackend {
signatureFromRequest,
accessKey,
options,
done,
done
), callback);
}
@ -67,7 +67,7 @@ class ChainBackend extends BaseBackend {
region,
scopeDate,
options,
done,
done
), callback);
}
@ -153,7 +153,7 @@ class ChainBackend extends BaseBackend {
requestContextParams,
userArn,
options,
done,
done
), (err, res) => {
if (err) {
return callback(err);
@ -171,8 +171,8 @@ class ChainBackend extends BaseBackend {
client.healthcheck(reqUid, (err, res) => done(null, {
error: !!err ? err : null,
status: res,
}),
), (err, res) => {
})
), (err, res) => {
if (err) {
return callback(err);
}

View File

@ -26,20 +26,20 @@ class AuthLoader {
.required();
const accountsJoi = joi.array()
.items({
name: joi.string().required(),
email: joi.string().email().required(),
arn: joi.string().required(),
canonicalID: joi.string().required(),
shortid: joi.string().regex(/^[0-9]{12}$/).required(),
keys: this._joiKeysValidator,
// backward-compat
users: joi.array(),
})
.required()
.unique('arn')
.unique('email')
.unique('canonicalID');
.items({
name: joi.string().required(),
email: joi.string().email().required(),
arn: joi.string().required(),
canonicalID: joi.string().required(),
shortid: joi.string().regex(/^[0-9]{12}$/).required(),
keys: this._joiKeysValidator,
// backward-compat
users: joi.array(),
})
.required()
.unique('arn')
.unique('email')
.unique('canonicalID');
this._joiValidator = joi.object({ accounts: accountsJoi });
}
@ -136,7 +136,7 @@ class AuthLoader {
_validateData(authData, filePath) {
const res = joi.validate(authData, this._joiValidator,
{ abortEarly: false });
{ abortEarly: false });
if (res.error) {
this._dumpJoiErrors(res.error.details, filePath);
return false;
@ -156,7 +156,7 @@ class AuthLoader {
'master/conf/authdata.json). Also note that support ' +
'for account users has been dropped.',
{ accountName: account.name, accountArn: account.arn,
filePath });
filePath });
arnError = true;
return;
}
@ -167,7 +167,7 @@ class AuthLoader {
'https://github.com/scality/S3/blob/master/conf/' +
'authdata.json)',
{ accountName: account.name, accountArn: account.arn,
filePath });
filePath });
arnError = true;
return;
}
@ -176,8 +176,8 @@ class AuthLoader {
this._log.error(
'authentication config validation error',
{ reason: arnObj.error.description,
accountName: account.name, accountArn: account.arn,
filePath });
accountName: account.name, accountArn: account.arn,
filePath });
arnError = true;
return;
}
@ -185,8 +185,8 @@ class AuthLoader {
this._log.error(
'authentication config validation error',
{ reason: 'not an IAM account ARN',
accountName: account.name, accountArn: account.arn,
filePath });
accountName: account.name, accountArn: account.arn,
filePath });
arnError = true;
return;
}
@ -215,7 +215,7 @@ class AuthLoader {
logInfo.context = err.context;
}
this._log.error('authentication config validation error',
logInfo);
logInfo);
});
}
}

View File

@ -41,7 +41,7 @@ function getCanonicalizedAmzHeaders(headers, clientType) {
// Build headerString
return amzHeaders.reduce((headerStr, current) =>
`${headerStr}${current[0]}:${current[1]}\n`,
'');
'');
}
module.exports = getCanonicalizedAmzHeaders;

View File

@ -22,9 +22,9 @@ function check(request, log, data) {
timestamp = Date.parse(timestamp);
if (!timestamp) {
log.debug('missing or invalid date header',
{ method: 'auth/v2/headerAuthCheck.check' });
{ method: 'auth/v2/headerAuthCheck.check' });
return { err: errors.AccessDenied.
customizeDescription('Authentication requires a valid Date or ' +
customizeDescription('Authentication requires a valid Date or ' +
'x-amz-date header') };
}

View File

@ -42,12 +42,12 @@ function check(request, log, data) {
if (expirationTime > currentTime + preSignedURLExpiry) {
log.debug('expires parameter too far in future',
{ expires: request.query.Expires });
{ expires: request.query.Expires });
return { err: errors.AccessDenied };
}
if (currentTime > expirationTime) {
log.debug('current time exceeds expires time',
{ expires: request.query.Expires });
{ expires: request.query.Expires });
return { err: errors.RequestTimeTooSkewed };
}
const accessKey = data.AWSAccessKeyId;

View File

@ -88,14 +88,14 @@ function check(request, log, data, awsService) {
}
if (!timestamp) {
log.debug('missing or invalid date header',
{ method: 'auth/v4/headerAuthCheck.check' });
{ method: 'auth/v4/headerAuthCheck.check' });
return { err: errors.AccessDenied.
customizeDescription('Authentication requires a valid Date or ' +
customizeDescription('Authentication requires a valid Date or ' +
'x-amz-date header') };
}
const validationResult = validateCredentials(credentialsArr, timestamp,
log);
log);
if (validationResult instanceof Error) {
log.debug('credentials in improper format', { credentialsArr,
timestamp, validationResult });
@ -134,7 +134,7 @@ function check(request, log, data, awsService) {
} catch (err) {
log.debug('invalid proxy_path header', { proxyPath, err });
return { err: errors.InvalidArgument.customizeDescription(
'invalid proxy_path header') };
'invalid proxy_path header') };
}
}

View File

@ -45,7 +45,7 @@ function check(request, log, data) {
}
const validationResult = validateCredentials(credential, timestamp,
log);
log);
if (validationResult instanceof Error) {
log.debug('credentials in improper format', { credential,
timestamp, validationResult });
@ -69,7 +69,7 @@ function check(request, log, data) {
} catch (err) {
log.debug('invalid proxy_path header', { proxyPath });
return { err: errors.InvalidArgument.customizeDescription(
'invalid proxy_path header') };
'invalid proxy_path header') };
}
}

View File

@ -273,7 +273,7 @@ class V4Transform extends Transform {
}
// get next chunk
return callback();
},
}
);
}
}

View File

@ -25,20 +25,20 @@ function validateCredentials(credentials, timestamp, log) {
log.warn('accessKey provided is wrong format', { accessKey });
return errors.InvalidArgument;
}
// The scope date (format YYYYMMDD) must be same date as the timestamp
// on the request from the x-amz-date param (if queryAuthCheck)
// or from the x-amz-date header or date header (if headerAuthCheck)
// Format of timestamp is ISO 8601: YYYYMMDDTHHMMSSZ.
// http://docs.aws.amazon.com/AmazonS3/latest/API/
// sigv4-query-string-auth.html
// http://docs.aws.amazon.com/general/latest/gr/
// sigv4-date-handling.html
// The scope date (format YYYYMMDD) must be same date as the timestamp
// on the request from the x-amz-date param (if queryAuthCheck)
// or from the x-amz-date header or date header (if headerAuthCheck)
// Format of timestamp is ISO 8601: YYYYMMDDTHHMMSSZ.
// http://docs.aws.amazon.com/AmazonS3/latest/API/
// sigv4-query-string-auth.html
// http://docs.aws.amazon.com/general/latest/gr/
// sigv4-date-handling.html
// convert timestamp to format of scopeDate YYYYMMDD
// convert timestamp to format of scopeDate YYYYMMDD
const timestampDate = timestamp.split('T')[0];
if (scopeDate.length !== 8 || scopeDate !== timestampDate) {
log.warn('scope date must be the same date as the timestamp date',
{ scopeDate, timestampDate });
{ scopeDate, timestampDate });
return errors.RequestTimeTooSkewed;
}
if (service !== 's3' && service !== 'iam' && service !== 'ring' &&
@ -50,7 +50,7 @@ function validateCredentials(credentials, timestamp, log) {
}
if (requestType !== 'aws4_request') {
log.warn('requestType contained in params is not aws4_request',
{ requestType });
{ requestType });
return errors.InvalidArgument;
}
return {};
@ -68,7 +68,7 @@ function extractQueryParams(queryObj, log) {
// Do not need the algorithm sent back
if (queryObj['X-Amz-Algorithm'] !== 'AWS4-HMAC-SHA256') {
log.warn('algorithm param incorrect',
{ algo: queryObj['X-Amz-Algorithm'] });
{ algo: queryObj['X-Amz-Algorithm'] });
return authParams;
}

View File

@ -103,11 +103,11 @@ module.exports = {
gcpTaggingPrefix: 'aws-tag-',
productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko',
legacyLocations: ['sproxyd', 'legacy'],
// healthcheck default call from nginx is every 2 seconds
// healthcheck default call from nginx is every 2 seconds
// for external backends, don't call unless at least 1 minute
// (60,000 milliseconds) since last call
externalBackendHealthCheckInterval: 60000,
// some of the available data backends (if called directly rather
// some of the available data backends (if called directly rather
// than through the multiple backend gateway) need a key provided
// as a string as first parameter of the get/delete methods.
clientsRequireStringKey: { sproxyd: true, cdmi: true },

View File

@ -64,12 +64,12 @@ class IndexTransaction {
push(op) {
if (this.closed) {
throw propError('pushOnCommittedTransaction',
'can not add ops to already committed transaction');
'can not add ops to already committed transaction');
}
if (op.type !== 'put' && op.type !== 'del') {
throw propError('invalidTransactionVerb',
`unknown action type: ${op.type}`);
`unknown action type: ${op.type}`);
}
if (op.key === undefined) {
@ -137,7 +137,7 @@ class IndexTransaction {
addCondition(condition) {
if (this.closed) {
throw propError('pushOnCommittedTransaction',
'can not add conditions to already committed transaction');
'can not add conditions to already committed transaction');
}
if (condition === undefined || Object.keys(condition).length === 0) {
throw propError('missingCondition', 'missing condition for conditional put');
@ -159,12 +159,12 @@ class IndexTransaction {
commit(cb) {
if (this.closed) {
return cb(propError('alreadyCommitted',
'transaction was already committed'));
'transaction was already committed'));
}
if (this.operations.length === 0) {
return cb(propError('emptyTransaction',
'tried to commit an empty transaction'));
'tried to commit an empty transaction'));
}
this.closed = true;

View File

@ -76,11 +76,11 @@ function errorsGen() {
const errorsObj = require('../errors/arsenalErrors.json');
Object.keys(errorsObj)
.filter(index => index !== '_comment')
.forEach(index => {
errors[index] = new ArsenalError(index, errorsObj[index].code,
errorsObj[index].description);
});
.filter(index => index !== '_comment')
.forEach(index => {
errors[index] = new ArsenalError(index, errorsObj[index].code,
errorsObj[index].description);
});
return errors;
}

View File

@ -17,9 +17,9 @@ describe('decyrptSecret', () => {
describe('parseServiceCredentials', () => {
const conf = {
users: [{ accessKey,
accountType: 'service-clueso',
secretKey,
userName: 'Search Service Account' }],
accountType: 'service-clueso',
secretKey,
userName: 'Search Service Account' }],
};
const auth = JSON.stringify({ privateKey });

View File

@ -25,7 +25,7 @@ module.exports.once = function once(func) {
state.res = func.apply(func, args);
} else {
debug('function already called:', func,
'returning cached result:', state.res);
'returning cached result:', state.res);
}
return state.res;
};

View File

@ -17,7 +17,7 @@ class RedisClient {
method: 'RedisClient.constructor',
redisHost: config.host,
redisPort: config.port,
}),
})
);
return this;
}

View File

@ -9,6 +9,7 @@ const StatsClient = require('./StatsClient');
* rather than by seconds
*/
class StatsModel extends StatsClient {
/**
* Utility method to convert 2d array rows to columns, and vice versa
* See also: https://docs.ruby-lang.org/en/2.0.0/Array.html#method-i-zip

View File

@ -2,8 +2,8 @@ const promClient = require('prom-client');
const collectDefaultMetricsIntervalMs =
process.env.COLLECT_DEFAULT_METRICS_INTERVAL_MS !== undefined ?
Number.parseInt(process.env.COLLECT_DEFAULT_METRICS_INTERVAL_MS, 10) :
10000;
Number.parseInt(process.env.COLLECT_DEFAULT_METRICS_INTERVAL_MS, 10) :
10000;
promClient.collectDefaultMetrics({ timeout: collectDefaultMetricsIntervalMs });

View File

@ -27,7 +27,7 @@ class ARN {
static createFromString(arnStr) {
const [arn, partition, service, region, accountId,
resourceType, resource] = arnStr.split(':');
resourceType, resource] = arnStr.split(':');
if (arn !== 'arn') {
return { error: errors.InvalidArgument.customizeDescription(
@ -58,7 +58,7 @@ class ARN {
'must be a 12-digit number or "*"') };
}
const fullResource = (resource !== undefined ?
`${resourceType}:${resource}` : resourceType);
`${resourceType}:${resource}` : resourceType);
return new ARN(partition, service, region, accountId, fullResource);
}
@ -98,7 +98,7 @@ class ARN {
toString() {
return ['arn', this.getPartition(), this.getService(),
this.getRegion(), this.getAccountId(), this.getResource()]
this.getRegion(), this.getAccountId(), this.getResource()]
.join(':');
}
}

View File

@ -52,9 +52,9 @@ class BackendInfo {
*/
static isRequestEndpointPresent(config, requestEndpoint, log) {
if (Object.keys(config.restEndpoints).
indexOf(requestEndpoint) < 0) {
indexOf(requestEndpoint) < 0) {
log.trace('requestEndpoint does not match config restEndpoints',
{ requestEndpoint });
{ requestEndpoint });
return false;
}
return true;
@ -70,10 +70,10 @@ class BackendInfo {
*/
static isRequestEndpointValueValid(config, requestEndpoint, log) {
if (Object.keys(config.locationConstraints).
indexOf(config.restEndpoints[requestEndpoint]) < 0) {
indexOf(config.restEndpoints[requestEndpoint]) < 0) {
log.trace('the default locationConstraint for request' +
'Endpoint does not match any config locationConstraint',
{ requestEndpoint });
{ requestEndpoint });
return false;
}
return true;
@ -110,7 +110,7 @@ class BackendInfo {
*/
static isValidRequestEndpointOrBackend(config, requestEndpoint, log) {
if (!BackendInfo.isRequestEndpointPresent(config, requestEndpoint,
log)) {
log)) {
return BackendInfo.isMemOrFileBackend(config, log);
}
return BackendInfo.isRequestEndpointValueValid(config, requestEndpoint,
@ -132,7 +132,7 @@ class BackendInfo {
bucketLocationConstraint, requestEndpoint, log) {
if (objectLocationConstraint) {
if (BackendInfo.isValidLocationConstraint(config,
objectLocationConstraint, log)) {
objectLocationConstraint, log)) {
log.trace('objectLocationConstraint is valid');
return { isValid: true };
}
@ -143,7 +143,7 @@ class BackendInfo {
}
if (bucketLocationConstraint) {
if (BackendInfo.isValidLocationConstraint(config,
bucketLocationConstraint, log)) {
bucketLocationConstraint, log)) {
log.trace('bucketLocationConstraint is valid');
return { isValid: true };
}
@ -159,7 +159,7 @@ class BackendInfo {
return { isValid: true, legacyLocationConstraint };
}
if (!BackendInfo.isValidRequestEndpointOrBackend(config,
requestEndpoint, log)) {
requestEndpoint, log)) {
return { isValid: false, description: 'Endpoint Location Error - ' +
`Your endpoint "${requestEndpoint}" is not in restEndpoints ` +
'in your config OR the default location constraint for request ' +
@ -167,7 +167,7 @@ class BackendInfo {
'match any config locationConstraint - Please update.' };
}
if (BackendInfo.isRequestEndpointPresent(config, requestEndpoint,
log)) {
log)) {
return { isValid: true };
}
return { isValid: true, defaultedToDataBackend: true };

View File

@ -69,13 +69,13 @@ class BucketInfo {
* @param {object} [notificationConfiguration] - bucket notification configuration
*/
constructor(name, owner, ownerDisplayName, creationDate,
mdBucketModelVersion, acl, transient, deleted,
serverSideEncryption, versioningConfiguration,
locationConstraint, websiteConfiguration, cors,
replicationConfiguration, lifecycleConfiguration,
bucketPolicy, uid, readLocationConstraint, isNFS,
ingestionConfig, azureInfo, objectLockEnabled,
objectLockConfiguration, notificationConfiguration) {
mdBucketModelVersion, acl, transient, deleted,
serverSideEncryption, versioningConfiguration,
locationConstraint, websiteConfiguration, cors,
replicationConfiguration, lifecycleConfiguration,
bucketPolicy, uid, readLocationConstraint, isNFS,
ingestionConfig, azureInfo, objectLockEnabled,
objectLockConfiguration, notificationConfiguration) {
assert.strictEqual(typeof name, 'string');
assert.strictEqual(typeof owner, 'string');
assert.strictEqual(typeof ownerDisplayName, 'string');
@ -94,7 +94,7 @@ class BucketInfo {
if (serverSideEncryption) {
assert.strictEqual(typeof serverSideEncryption, 'object');
const { cryptoScheme, algorithm, masterKeyId,
configuredMasterKeyId, mandatory } = serverSideEncryption;
configuredMasterKeyId, mandatory } = serverSideEncryption;
assert.strictEqual(typeof cryptoScheme, 'number');
assert.strictEqual(typeof algorithm, 'string');
assert.strictEqual(typeof masterKeyId, 'string');

View File

@ -381,7 +381,7 @@ class LifecycleConfiguration {
if (!tags[i].Key || !tags[i].Value) {
tagObj.error =
errors.MissingRequiredParameter.customizeDescription(
'Tag XML does not contain both Key and Value');
'Tag XML does not contain both Key and Value');
break;
}
@ -929,7 +929,7 @@ class LifecycleConfiguration {
const daysInt = parseInt(subExp.Days[0], 10);
if (daysInt < 1) {
expObj.error = errors.InvalidArgument.customizeDescription(
'Expiration days is not a positive integer');
'Expiration days is not a positive integer');
} else {
expObj.days = daysInt;
}
@ -1125,10 +1125,10 @@ class LifecycleConfiguration {
const { noncurrentDays, storageClass } = transition;
xml.push(
`<${actionName}>`,
`<NoncurrentDays>${noncurrentDays}` +
`<NoncurrentDays>${noncurrentDays}` +
'</NoncurrentDays>',
`<StorageClass>${storageClass}</StorageClass>`,
`</${actionName}>`,
`<StorageClass>${storageClass}</StorageClass>`,
`</${actionName}>`
);
});
Action = xml.join('');
@ -1146,9 +1146,9 @@ class LifecycleConfiguration {
}
xml.push(
`<${actionName}>`,
element,
`<StorageClass>${storageClass}</StorageClass>`,
`</${actionName}>`,
element,
`<StorageClass>${storageClass}</StorageClass>`,
`</${actionName}>`
);
});
Action = xml.join('');

View File

@ -27,7 +27,7 @@ const errors = require('../errors');
* </NotificationConfiguration>
*/
/**
/**
* Format of config:
*
* config = {

View File

@ -17,7 +17,7 @@ const errors = require('../errors');
* </ObjectLockConfiguration>
*/
/**
/**
* Format of config:
*
* config = {

View File

@ -10,6 +10,7 @@ const ObjectMDLocation = require('./ObjectMDLocation');
* mpuPart metadata for example)
*/
class ObjectMD {
/**
* Create a new instance of ObjectMD. Parameter <tt>objMd</tt> is
* reserved for internal use, users should call
@ -148,7 +149,7 @@ class ObjectMD {
Object.assign(this._data, objMd._data);
Object.assign(this._data.replicationInfo,
objMd._data.replicationInfo);
objMd._data.replicationInfo);
}
_updateFromParsedJSON(objMd) {

View File

@ -3,6 +3,7 @@
* 'location' array
*/
class ObjectMDLocation {
/**
* @constructor
* @param {object} locationObj - single data location info

View File

@ -111,7 +111,7 @@ class RoundRobin {
pickHost() {
if (this.logger) {
this.logger.debug('pick host',
{ host: this.getCurrentHost() });
{ host: this.getCurrentHost() });
}
const curHost = this.getCurrentHost();
++this.pickCount;
@ -163,7 +163,7 @@ class RoundRobin {
}
if (this.logger) {
this.logger.debug('round robin host',
{ newHost: this.getCurrentHost() });
{ newHost: this.getCurrentHost() });
}
}
}

View File

@ -10,6 +10,7 @@ const { checkSupportIPv6 } = require('./utils');
class Server {
/**
* @constructor
*
@ -430,16 +431,16 @@ class Server {
// Setting no delay of the socket to the value configured
sock.setNoDelay(this.isNoDelay());
sock.on('error', err => this._logger.info(
'socket error - request rejected', { error: err }));
'socket error - request rejected', { error: err }));
});
this._server.on('tlsClientError', (err, sock) =>
this._onClientError(err, sock));
this._onClientError(err, sock));
this._server.on('clientError', (err, sock) =>
this._onClientError(err, sock));
this._onClientError(err, sock));
this._server.on('checkContinue', (req, res) =>
this._onCheckContinue(req, res));
this._onCheckContinue(req, res));
this._server.on('checkExpectation', (req, res) =>
this._onCheckExpectation(req, res));
this._onCheckExpectation(req, res));
this._server.on('listening', () => this._onListening());
}
this._server.listen(this._port, this._address);

View File

@ -72,8 +72,8 @@ function getByteRangeFromSpec(rangeSpec, objectSize) {
if (rangeSpec.start < objectSize) {
// test is false if end is undefined
return { range: [rangeSpec.start,
(rangeSpec.end < objectSize ?
rangeSpec.end : objectSize - 1)] };
(rangeSpec.end < objectSize ?
rangeSpec.end : objectSize - 1)] };
}
return { error: errors.InvalidRange };
}

View File

@ -90,8 +90,8 @@ function _negotiateProtocolVersion(client, logger, cb) {
if (err) {
const error = _arsenalError(err);
logger.error('KMIP::negotiateProtocolVersion',
{ error,
vendorIdentification: client.vendorIdentification });
{ error,
vendorIdentification: client.vendorIdentification });
return cb(error);
}
const majorVersions =
@ -102,8 +102,8 @@ function _negotiateProtocolVersion(client, logger, cb) {
majorVersions.length !== minorVersions.length) {
const error = _arsenalError('No suitable protocol version');
logger.error('KMIP::negotiateProtocolVersion',
{ error,
vendorIdentification: client.vendorIdentification });
{ error,
vendorIdentification: client.vendorIdentification });
return cb(error);
}
client.kmip.changeProtocolVersion(majorVersions[0], minorVersions[0]);
@ -126,8 +126,8 @@ function _mapExtensions(client, logger, cb) {
if (err) {
const error = _arsenalError(err);
logger.error('KMIP::mapExtensions',
{ error,
vendorIdentification: client.vendorIdentification });
{ error,
vendorIdentification: client.vendorIdentification });
return cb(error);
}
const extensionNames = response.lookup(searchFilter.extensionName);
@ -135,8 +135,8 @@ function _mapExtensions(client, logger, cb) {
if (extensionNames.length !== extensionTags.length) {
const error = _arsenalError('Inconsistent extension list');
logger.error('KMIP::mapExtensions',
{ error,
vendorIdentification: client.vendorIdentification });
{ error,
vendorIdentification: client.vendorIdentification });
return cb(error);
}
extensionNames.forEach((extensionName, idx) => {
@ -160,7 +160,7 @@ function _queryServerInformation(client, logger, cb) {
if (err) {
const error = _arsenalError(err);
logger.warn('KMIP::queryServerInformation',
{ error });
{ error });
/* no error returned, caller can keep going */
return cb();
}
@ -170,9 +170,9 @@ function _queryServerInformation(client, logger, cb) {
JSON.stringify(response.lookup(searchFilter.serverInformation)[0]));
logger.info('KMIP Server identified',
{ vendorIdentification: client.vendorIdentification,
serverInformation: client.serverInformation,
negotiatedProtocolVersion: client.kmip.protocolVersion });
{ vendorIdentification: client.vendorIdentification,
serverInformation: client.serverInformation,
negotiatedProtocolVersion: client.kmip.protocolVersion });
return cb();
});
}
@ -196,8 +196,8 @@ function _queryOperationsAndObjects(client, logger, cb) {
if (err) {
const error = _arsenalError(err);
logger.error('KMIP::queryOperationsAndObjects',
{ error,
vendorIdentification: client.vendorIdentification });
{ error,
vendorIdentification: client.vendorIdentification });
return cb(error);
}
const supportedOperations = response.lookup(searchFilter.operation);
@ -222,15 +222,15 @@ function _queryOperationsAndObjects(client, logger, cb) {
logger.warn('KMIP::queryOperationsAndObjects: ' +
'The KMIP Server announces that it ' +
'does not support all of the required features',
{ vendorIdentification: client.vendorIdentification,
serverInformation: client.serverInformation,
supportsEncrypt, supportsDecrypt,
supportsActivate, supportsRevoke,
supportsCreate, supportsDestroy,
supportsQuery, supportsSymmetricKeys });
{ vendorIdentification: client.vendorIdentification,
serverInformation: client.serverInformation,
supportsEncrypt, supportsDecrypt,
supportsActivate, supportsRevoke,
supportsCreate, supportsDestroy,
supportsQuery, supportsSymmetricKeys });
} else {
logger.info('KMIP Server provides the necessary feature set',
{ vendorIdentification: client.vendorIdentification });
{ vendorIdentification: client.vendorIdentification });
}
return cb();
});
@ -264,8 +264,8 @@ class Client {
this.vendorIdentification = '';
this.serverInformation = [];
this.kmip = new KMIP(CodecClass || TTLVCodec,
TransportClass || TlsTransport,
options);
TransportClass || TlsTransport,
options);
this.kmip.registerHandshakeFunction((logger, cb) => {
this._kmipHandshake(logger, cb);
});
@ -322,8 +322,8 @@ class Client {
if (err) {
const error = _arsenalError(err);
logger.error('KMIP::_activateBucketKey',
{ error,
serverInformation: this.serverInformation });
{ error,
serverInformation: this.serverInformation });
return cb(error);
}
const uniqueIdentifier =
@ -332,7 +332,7 @@ class Client {
const error = _arsenalError(
'Server did not return the expected identifier');
logger.error('KMIP::cipherDataKey',
{ error, uniqueIdentifier });
{ error, uniqueIdentifier });
return cb(error);
}
return cb(null, keyIdentifier);
@ -351,20 +351,20 @@ class Client {
const attributes = [];
if (!!this.options.bucketNameAttributeName) {
attributes.push(KMIP.Attribute('TextString',
this.options.bucketNameAttributeName,
bucketName));
this.options.bucketNameAttributeName,
bucketName));
}
attributes.push(...[
KMIP.Attribute('Enumeration', 'Cryptographic Algorithm',
CRYPTOGRAPHIC_ALGORITHM),
CRYPTOGRAPHIC_ALGORITHM),
KMIP.Attribute('Integer', 'Cryptographic Length',
CRYPTOGRAPHIC_LENGTH),
CRYPTOGRAPHIC_LENGTH),
KMIP.Attribute('Integer', 'Cryptographic Usage Mask',
this.kmip.encodeMask('Cryptographic Usage Mask',
CRYPTOGRAPHIC_USAGE_MASK))]);
this.kmip.encodeMask('Cryptographic Usage Mask',
CRYPTOGRAPHIC_USAGE_MASK))]);
if (this.options.compoundCreateActivate) {
attributes.push(KMIP.Attribute('Date-Time', 'Activation Date',
new Date(Date.UTC())));
new Date(Date.UTC())));
}
return this.kmip.request(logger, 'Create', [
@ -374,8 +374,8 @@ class Client {
if (err) {
const error = _arsenalError(err);
logger.error('KMIP::createBucketKey',
{ error,
serverInformation: this.serverInformation });
{ error,
serverInformation: this.serverInformation });
return cb(error);
}
const createdObjectType =
@ -386,7 +386,7 @@ class Client {
const error = _arsenalError(
'Server created an object of wrong type');
logger.error('KMIP::createBucketKey',
{ error, createdObjectType });
{ error, createdObjectType });
return cb(error);
}
if (!this.options.compoundCreateActivate) {
@ -411,16 +411,16 @@ class Client {
KMIP.TextString('Unique Identifier', bucketKeyId),
KMIP.Structure('Revocation Reason', [
KMIP.Enumeration('Revocation Reason Code',
'Cessation of Operation'),
'Cessation of Operation'),
KMIP.TextString('Revocation Message',
'About to be deleted'),
'About to be deleted'),
]),
], (err, response) => {
if (err) {
const error = _arsenalError(err);
logger.error('KMIP::_revokeBucketKey',
{ error,
serverInformation: this.serverInformation });
{ error,
serverInformation: this.serverInformation });
return cb(error);
}
const uniqueIdentifier =
@ -429,7 +429,7 @@ class Client {
const error = _arsenalError(
'Server did not return the expected identifier');
logger.error('KMIP::_revokeBucketKey',
{ error, uniqueIdentifier });
{ error, uniqueIdentifier });
return cb(error);
}
return cb();
@ -448,8 +448,8 @@ class Client {
if (err) {
const error = _arsenalError(err);
logger.error('KMIP::destroyBucketKey: revocation failed',
{ error,
serverInformation: this.serverInformation });
{ error,
serverInformation: this.serverInformation });
return cb(error);
}
return this.kmip.request(logger, 'Destroy', [
@ -458,8 +458,8 @@ class Client {
if (err) {
const error = _arsenalError(err);
logger.error('KMIP::destroyBucketKey',
{ error,
serverInformation: this.serverInformation });
{ error,
serverInformation: this.serverInformation });
return cb(error);
}
const uniqueIdentifier =
@ -468,7 +468,7 @@ class Client {
const error = _arsenalError(
'Server did not return the expected identifier');
logger.error('KMIP::destroyBucketKey',
{ error, uniqueIdentifier });
{ error, uniqueIdentifier });
return cb(error);
}
return cb();
@ -487,19 +487,19 @@ class Client {
* @callback called with (err, cipheredDataKey: Buffer)
*/
cipherDataKey(cryptoScheme,
masterKeyId,
plainTextDataKey,
logger,
cb) {
masterKeyId,
plainTextDataKey,
logger,
cb) {
return this.kmip.request(logger, 'Encrypt', [
KMIP.TextString('Unique Identifier', masterKeyId),
KMIP.Structure('Cryptographic Parameters', [
KMIP.Enumeration('Block Cipher Mode',
CRYPTOGRAPHIC_CIPHER_MODE),
CRYPTOGRAPHIC_CIPHER_MODE),
KMIP.Enumeration('Padding Method',
CRYPTOGRAPHIC_PADDING_METHOD),
CRYPTOGRAPHIC_PADDING_METHOD),
KMIP.Enumeration('Cryptographic Algorithm',
CRYPTOGRAPHIC_ALGORITHM),
CRYPTOGRAPHIC_ALGORITHM),
]),
KMIP.ByteString('Data', plainTextDataKey),
KMIP.ByteString('IV/Counter/Nonce', CRYPTOGRAPHIC_DEFAULT_IV),
@ -507,8 +507,8 @@ class Client {
if (err) {
const error = _arsenalError(err);
logger.error('KMIP::cipherDataKey',
{ error,
serverInformation: this.serverInformation });
{ error,
serverInformation: this.serverInformation });
return cb(error);
}
const uniqueIdentifier =
@ -518,7 +518,7 @@ class Client {
const error = _arsenalError(
'Server did not return the expected identifier');
logger.error('KMIP::cipherDataKey',
{ error, uniqueIdentifier });
{ error, uniqueIdentifier });
return cb(error);
}
return cb(null, data);
@ -536,19 +536,19 @@ class Client {
* @callback called with (err, plainTextDataKey: Buffer)
*/
decipherDataKey(cryptoScheme,
masterKeyId,
cipheredDataKey,
logger,
cb) {
masterKeyId,
cipheredDataKey,
logger,
cb) {
return this.kmip.request(logger, 'Decrypt', [
KMIP.TextString('Unique Identifier', masterKeyId),
KMIP.Structure('Cryptographic Parameters', [
KMIP.Enumeration('Block Cipher Mode',
CRYPTOGRAPHIC_CIPHER_MODE),
CRYPTOGRAPHIC_CIPHER_MODE),
KMIP.Enumeration('Padding Method',
CRYPTOGRAPHIC_PADDING_METHOD),
CRYPTOGRAPHIC_PADDING_METHOD),
KMIP.Enumeration('Cryptographic Algorithm',
CRYPTOGRAPHIC_ALGORITHM),
CRYPTOGRAPHIC_ALGORITHM),
]),
KMIP.ByteString('Data', cipheredDataKey),
KMIP.ByteString('IV/Counter/Nonce', CRYPTOGRAPHIC_DEFAULT_IV),
@ -556,8 +556,8 @@ class Client {
if (err) {
const error = _arsenalError(err);
logger.error('KMIP::decipherDataKey',
{ error,
serverInformation: this.serverInformation });
{ error,
serverInformation: this.serverInformation });
return cb(error);
}
const uniqueIdentifier =
@ -567,7 +567,7 @@ class Client {
const error = _arsenalError(
'Server did not return the right identifier');
logger.error('KMIP::decipherDataKey',
{ error, uniqueIdentifier });
{ error, uniqueIdentifier });
return cb(error);
}
return cb(null, data);

View File

@ -55,15 +55,15 @@ function TTLVCodec() {
const property = {};
if (!TypeDecoder[elementType]) {
_throwError(logger,
'Unknown element type',
{ funcName, elementTag, elementType });
'Unknown element type',
{ funcName, elementTag, elementType });
}
const elementValue = value.slice(i + 8,
i + 8 + elementLength);
i + 8 + elementLength);
if (elementValue.length !== elementLength) {
_throwError(logger, 'BUG: Wrong buffer size',
{ funcName, elementLength,
bufferLength: elementValue.length });
{ funcName, elementLength,
bufferLength: elementValue.length });
}
property.type = TypeDecoder[elementType].name;
property.value = TypeDecoder[elementType]
@ -75,7 +75,7 @@ function TTLVCodec() {
const tagInfo = TagDecoder[elementTag];
if (!tagInfo) {
logger.debug('Unknown element tag',
{ funcName, elementTag });
{ funcName, elementTag });
property.tag = elementTag;
element['Unknown Tag'] = property;
} else {
@ -83,8 +83,8 @@ function TTLVCodec() {
if (tagInfo.name === 'Attribute Name') {
if (property.type !== 'TextString') {
_throwError(logger,
'Invalide type',
{ funcName, type: property.type });
'Invalide type',
{ funcName, type: property.type });
}
diversion = property.value;
}
@ -114,8 +114,8 @@ function TTLVCodec() {
}
const itemResult =
TypeEncoder[itemType].encode(itemTagName,
itemValue,
itemDiversion);
itemValue,
itemDiversion);
encodedValue = encodedValue
.concat(_ttlvPadVector(itemResult));
});
@ -133,9 +133,9 @@ function TTLVCodec() {
const fixedLength = 4;
if (fixedLength !== value.length) {
_throwError(logger,
'Length mismatch',
{ funcName, fixedLength,
bufferLength: value.length });
'Length mismatch',
{ funcName, fixedLength,
bufferLength: value.length });
}
return value.readUInt32BE(0);
},
@ -156,16 +156,16 @@ function TTLVCodec() {
const fixedLength = 8;
if (fixedLength !== value.length) {
_throwError(logger,
'Length mismatch',
{ funcName, fixedLength,
bufferLength: value.length });
'Length mismatch',
{ funcName, fixedLength,
bufferLength: value.length });
}
const longUInt = UINT32_MAX * value.readUInt32BE(0) +
value.readUInt32BE(4);
if (longUInt > Number.MAX_SAFE_INTEGER) {
_throwError(logger,
'53-bit overflow',
{ funcName, longUInt });
'53-bit overflow',
{ funcName, longUInt });
}
return longUInt;
},
@ -200,9 +200,9 @@ function TTLVCodec() {
const fixedLength = 4;
if (fixedLength !== value.length) {
_throwError(logger,
'Length mismatch',
{ funcName, fixedLength,
bufferLength: value.length });
'Length mismatch',
{ funcName, fixedLength,
bufferLength: value.length });
}
const enumValue = value.toString('hex');
const actualTag = diversion ? TagEncoder[diversion].value : tag;
@ -211,10 +211,10 @@ function TTLVCodec() {
!enumInfo.enumeration ||
!enumInfo.enumeration[enumValue]) {
return { tag,
value: enumValue,
message: 'Unknown enumeration value',
diversion,
};
value: enumValue,
message: 'Unknown enumeration value',
diversion,
};
}
return enumInfo.enumeration[enumValue];
},
@ -227,7 +227,7 @@ function TTLVCodec() {
const actualTag = diversion || tagName;
const encodedValue =
Buffer.from(TagEncoder[actualTag].enumeration[value],
'hex');
'hex');
return _ttlvPadVector([tag, type, length, encodedValue]);
},
},
@ -238,9 +238,9 @@ function TTLVCodec() {
const fixedLength = 8;
if (fixedLength !== value.length) {
_throwError(logger,
'Length mismatch',
{ funcName, fixedLength,
bufferLength: value.length });
'Length mismatch',
{ funcName, fixedLength,
bufferLength: value.length });
}
const msUInt = value.readUInt32BE(0);
const lsUInt = value.readUInt32BE(4);
@ -267,7 +267,7 @@ function TTLVCodec() {
const length = Buffer.alloc(4);
length.writeUInt32BE(value.length);
return _ttlvPadVector([tag, type, length,
Buffer.from(value, 'utf8')]);
Buffer.from(value, 'utf8')]);
},
},
'08': {
@ -289,17 +289,17 @@ function TTLVCodec() {
const fixedLength = 8;
if (fixedLength !== value.length) {
_throwError(logger,
'Length mismatch',
{ funcName, fixedLength,
bufferLength: value.length });
'Length mismatch',
{ funcName, fixedLength,
bufferLength: value.length });
}
const d = new Date(0);
const utcSeconds = UINT32_MAX * value.readUInt32BE(0) +
value.readUInt32BE(4);
if (utcSeconds > Number.MAX_SAFE_INTEGER) {
_throwError(logger,
'53-bit overflow',
{ funcName, utcSeconds });
'53-bit overflow',
{ funcName, utcSeconds });
}
d.setUTCSeconds(utcSeconds);
return d;
@ -323,9 +323,9 @@ function TTLVCodec() {
const fixedLength = 4;
if (fixedLength !== value.length) {
_throwError(logger,
'Length mismatch',
{ funcName, fixedLength,
bufferLength: value.length });
'Length mismatch',
{ funcName, fixedLength,
bufferLength: value.length });
}
return value.readInt32BE(0);
},
@ -415,8 +415,8 @@ function TTLVCodec() {
throw Error(`Unknown Type '${type}'`);
}
const itemValue = TypeEncoder[type].encode(key,
item[key].value,
item[key].diversion);
item[key].value,
item[key].diversion);
result = result.concat(_ttlvPadVector(itemValue));
});
});

View File

@ -275,11 +275,11 @@ class KMIP {
KMIP.Structure('Request Header', [
KMIP.Structure('Protocol Version', [
KMIP.Integer('Protocol Version Major',
this.protocolVersion.major),
this.protocolVersion.major),
KMIP.Integer('Protocol Version Minor',
this.protocolVersion.minor)]),
this.protocolVersion.minor)]),
KMIP.Integer('Maximum Response Size',
this.maximumResponseSize),
this.maximumResponseSize),
KMIP.Integer('Batch Count', 1)]),
KMIP.Structure('Batch Item', [
KMIP.Enumeration('Operation', operation),
@ -292,7 +292,7 @@ class KMIP {
(err, conversation, rawResponse) => {
if (err) {
logger.error('KMIP::request: Failed to send message',
{ error: err });
{ error: err });
return cb(err);
}
const response = this._decodeMessage(logger, rawResponse);
@ -311,16 +311,16 @@ class KMIP {
this.transport.abortPipeline(conversation);
const error = Error('Invalid batch item ID returned');
logger.error('KMIP::request: failed',
{ resultUniqueBatchItemID, uuid, error });
{ resultUniqueBatchItemID, uuid, error });
return cb(error);
}
if (performedOperation !== operation) {
this.transport.abortPipeline(conversation);
const error = Error('Operation mismatch',
{ got: performedOperation,
expected: operation });
{ got: performedOperation,
expected: operation });
logger.error('KMIP::request: Operation mismatch',
{ error });
{ error });
return cb(error);
}
if (resultStatus !== 'Success') {
@ -331,17 +331,19 @@ class KMIP {
response.lookup(
'Response Message/Batch Item/Result Message')[0];
const error = Error('KMIP request failure',
{ resultStatus,
resultReason,
resultMessage });
{ resultStatus,
resultReason,
resultMessage });
logger.error('KMIP::request: request failed',
{ error, resultStatus,
resultReason, resultMessage });
{ error, resultStatus,
resultReason, resultMessage });
return cb(error);
}
return cb(null, response);
});
}
}

View File

@ -86,8 +86,8 @@ class TransportTemplate {
const deferedRequest = this.deferedRequests.shift();
process.nextTick(() => {
this.send(logger,
deferedRequest.encodedMessage,
deferedRequest.cb);
deferedRequest.encodedMessage,
deferedRequest.cb);
});
} else if (this.callbackPipeline.length === 0 &&
this.deferedRequests.length === 0 &&

View File

@ -26,7 +26,7 @@ function sendError(res, log, error, optMessage) {
httpCode: error.code,
errorType: error.message,
error: message,
},
}
);
res.writeHead(error.code);
res.end(JSON.stringify({

View File

@ -19,7 +19,7 @@ function setContentRange(response, byteRange, objectSize) {
const [start, end] = byteRange;
assert(start !== undefined && end !== undefined);
response.setHeader('Content-Range',
`bytes ${start}-${end}/${objectSize}`);
`bytes ${start}-${end}/${objectSize}`);
}
function sendError(res, log, error, optMessage) {
@ -45,6 +45,7 @@ function sendError(res, log, error, optMessage) {
* start() to start listening to the configured port.
*/
class RESTServer extends httpServer {
/**
* @constructor
* @param {Object} params - constructor params
@ -226,7 +227,7 @@ class RESTServer extends httpServer {
return sendError(res, log, err);
}
log.debug('sending back 200/206 response with contents',
{ key: pathInfo.key });
{ key: pathInfo.key });
setContentLength(res, contentLength);
res.setHeader('Accept-Ranges', 'bytes');
if (byteRange) {
@ -264,7 +265,7 @@ class RESTServer extends httpServer {
return sendError(res, log, err);
}
log.debug('sending back 204 response to DELETE',
{ key: pathInfo.key });
{ key: pathInfo.key });
res.writeHead(204);
return res.end(() => {
log.debug('DELETE response sent', { key: pathInfo.key });

View File

@ -19,7 +19,7 @@ function explodePath(path) {
return {
service: pathMatch[1],
key: (pathMatch[3] !== undefined && pathMatch[3].length > 0 ?
pathMatch[3] : undefined),
pathMatch[3] : undefined),
};
}
throw errors.InvalidURI.customizeDescription('malformed URI');

View File

@ -17,6 +17,7 @@ const rpc = require('./rpc.js');
* RPC client object accessing the sub-level transparently.
*/
class LevelDbClient extends rpc.BaseClient {
/**
* @constructor
*
@ -77,6 +78,7 @@ class LevelDbClient extends rpc.BaseClient {
* env.subDb (env is passed as first parameter of received RPC calls).
*/
class LevelDbService extends rpc.BaseService {
/**
* @constructor
*

View File

@ -37,6 +37,7 @@ let streamRPCJSONObj;
* an error occurred).
*/
class BaseClient extends EventEmitter {
/**
* @constructor
*
@ -53,7 +54,7 @@ class BaseClient extends EventEmitter {
*/
constructor(params) {
const { url, logger, callTimeoutMs,
streamMaxPendingAck, streamAckTimeoutMs } = params;
streamMaxPendingAck, streamAckTimeoutMs } = params;
assert(url);
assert(logger);
@ -81,11 +82,11 @@ class BaseClient extends EventEmitter {
_call(remoteCall, args, cb) {
const wrapCb = (err, data) => {
cb(reconstructError(err),
this.socketStreams.decodeStreams(data));
this.socketStreams.decodeStreams(data));
};
this.logger.debug('remote call', { remoteCall, args });
this.socket.emit('call', remoteCall,
this.socketStreams.encodeStreams(args), wrapCb);
this.socketStreams.encodeStreams(args), wrapCb);
return undefined;
}
@ -112,8 +113,8 @@ class BaseClient extends EventEmitter {
throw new Error(`argument cb=${cb} is not a callback`);
}
async.timeout(this._call.bind(this), timeoutMs,
`operation ${remoteCall} timed out`)(remoteCall,
args, cb);
`operation ${remoteCall} timed out`)(remoteCall,
args, cb);
return undefined;
}
@ -141,7 +142,7 @@ class BaseClient extends EventEmitter {
const url = this.url;
this.socket.on('error', err => {
this.logger.warn('connectivity error to the RPC service',
{ url, error: err });
{ url, error: err });
});
this.socket.on('connect', () => {
this.emit('connect');
@ -155,7 +156,7 @@ class BaseClient extends EventEmitter {
this.getManifest((err, manifest) => {
if (err) {
this.logger.error('Error fetching manifest from RPC server',
{ error: err });
{ error: err });
} else {
manifest.api.forEach(apiItem => {
this.createCall(apiItem.name);
@ -250,6 +251,7 @@ class BaseClient extends EventEmitter {
*
*/
class BaseService {
/**
* @constructor
*
@ -495,7 +497,7 @@ function RPCServer(params) {
conn.on('error', err => {
log.error('error on socket.io connection',
{ namespace: service.namespace, error: err });
{ namespace: service.namespace, error: err });
});
conn.on('call', (remoteCall, args, cb) => {
const decodedArgs = streamsSocket.decodeStreams(args);
@ -645,8 +647,8 @@ streamRPCJSONObj = function _streamRPCJSONObj(obj, wstream, cb) {
// primitive types
if (obj === undefined) {
wstream.write('null'); // if undefined elements are present in
// arrays, convert them to JSON null
// objects
// arrays, convert them to JSON null
// objects
} else {
wstream.write(JSON.stringify(obj));
}

View File

@ -16,7 +16,7 @@ class SIOOutputStream extends stream.Writable {
constructor(socket, streamId, maxPendingAck, ackTimeoutMs) {
super({ objectMode: true });
this._initOutputStream(socket, streamId, maxPendingAck,
ackTimeoutMs);
ackTimeoutMs);
}
_initOutputStream(socket, streamId, maxPendingAck, ackTimeoutMs) {
@ -194,7 +194,7 @@ class SIOStreamSocket {
this.socket.on('stream-data', (payload, cb) => {
const { streamId, data } = payload;
log.debug('received \'stream-data\' event',
{ streamId, size: data.length });
{ streamId, size: data.length });
const stream = this.remoteStreams[streamId];
if (!stream) {
log.debug('no such remote stream registered', { streamId });
@ -280,15 +280,15 @@ class SIOStreamSocket {
let transportStream;
if (isReadStream) {
transportStream = new SIOOutputStream(this, streamId,
this.maxPendingAck,
this.ackTimeoutMs);
this.maxPendingAck,
this.ackTimeoutMs);
} else {
transportStream = new SIOInputStream(this, streamId);
}
this.localStreams[streamId] = arg;
arg.once('close', () => {
log.debug('stream closed, removing from local streams',
{ streamId });
{ streamId });
delete this.localStreams[streamId];
});
arg.on('error', error => {
@ -350,8 +350,8 @@ class SIOStreamSocket {
stream = new SIOInputStream(this, streamId);
} else if (arg.writable) {
stream = new SIOOutputStream(this, streamId,
this.maxPendingAck,
this.ackTimeoutMs);
this.maxPendingAck,
this.ackTimeoutMs);
} else {
throw new Error('can\'t decode stream neither readable ' +
'nor writable');
@ -360,14 +360,14 @@ class SIOStreamSocket {
if (arg.readable) {
stream.once('close', () => {
log.debug('stream closed, removing from remote streams',
{ streamId });
{ streamId });
delete this.remoteStreams[streamId];
});
}
if (arg.writable) {
stream.once('finish', () => {
log.debug('stream finished, removing from remote streams',
{ streamId });
{ streamId });
delete this.remoteStreams[streamId];
});
}
@ -399,7 +399,7 @@ class SIOStreamSocket {
_write(streamId, data, cb) {
this.logger.debug('emit \'stream-data\' event',
{ streamId, size: data.length });
{ streamId, size: data.length });
this.socket.emit('stream-data', { streamId, data }, cb);
}

View File

@ -50,7 +50,7 @@ evaluators.isResourceApplicable = (requestContext, statementResource, log) => {
requestResourceArr, true);
if (arnSegmentsMatch) {
log.trace('policy resource is applicable to request',
{ requestResource: resource, policyResource });
{ requestResource: resource, policyResource });
return true;
}
continue;
@ -224,21 +224,21 @@ evaluators.evaluatePolicy = (requestContext, policy, log) => {
// in policy, move on to next statement
if (currentStatement.NotResource &&
evaluators.isResourceApplicable(requestContext,
currentStatement.NotResource, log)) {
currentStatement.NotResource, log)) {
continue;
}
// If affirmative action is in policy and request action is not
// applicable, move on to next statement
if (currentStatement.Action &&
!evaluators.isActionApplicable(requestContext.getAction(),
currentStatement.Action, log)) {
currentStatement.Action, log)) {
continue;
}
// If NotAction is in policy and action matches NotAction in policy,
// move on to next statement
if (currentStatement.NotAction &&
evaluators.isActionApplicable(requestContext.getAction(),
currentStatement.NotAction, log)) {
currentStatement.NotAction, log)) {
continue;
}
const conditionEval = currentStatement.Condition ?

View File

@ -39,11 +39,11 @@ conditions.findConditionKey = (key, requestContext) => {
// (see Boolean Condition Operators).
// Note: This key is only present if MFA was used. So, the following
// will not work:
// "Condition" :
// { "Bool" : { "aws:MultiFactorAuthPresent" : false } }
// "Condition" :
// { "Bool" : { "aws:MultiFactorAuthPresent" : false } }
// Instead use:
// "Condition" :
// { "Null" : { "aws:MultiFactorAuthPresent" : true } }
// "Condition" :
// { "Null" : { "aws:MultiFactorAuthPresent" : true } }
map.set('aws:MultiFactorAuthPresent',
requestContext.getMultiFactorAuthPresent());
// aws:MultiFactorAuthAge Used to check how many seconds since
@ -166,8 +166,8 @@ conditions.findConditionKey = (key, requestContext) => {
// so evaluation should be skipped
map.set('s3:RequestObjectTagKeys',
requestContext.getNeedTagEval() && requestContext.getRequestObjTags()
? getTagKeys(requestContext.getRequestObjTags())
: undefined);
? getTagKeys(requestContext.getRequestObjTags())
: undefined);
return map.get(key);
};
@ -191,7 +191,7 @@ function convertSpecialChars(string) {
return map[char];
}
return string.replace(/(\$\{\*\})|(\$\{\?\})|(\$\{\$\})/g,
characterMap);
characterMap);
}
/**
@ -425,10 +425,10 @@ conditions.convertConditionOperator = operator => {
return !operatorMap.ArnLike(key, value);
},
Null: function nullOperator(key, value) {
// Null is used to check if a condition key is present.
// The policy statement value should be either true (the key doesn't
// exist — it is null) or false (the key exists and its value is
// not null).
// Null is used to check if a condition key is present.
// The policy statement value should be either true (the key doesn't
// exist — it is null) or false (the key exists and its value is
// not null).
if ((key === undefined || key === null)
&& value[0] === 'true' ||
(key !== undefined && key !== null)

View File

@ -51,10 +51,10 @@ wildcards.handleWildcardInResource = arn => {
// Wildcards can be part of the resource ARN.
// Wildcards do NOT span segments of the ARN (separated by ":")
// Example: all elements in specific bucket:
// "Resource": "arn:aws:s3:::my_corporate_bucket/*"
// ARN format:
// arn:partition:service:region:namespace:relative-id
// Example: all elements in specific bucket:
// "Resource": "arn:aws:s3:::my_corporate_bucket/*"
// ARN format:
// arn:partition:service:region:namespace:relative-id
const arnArr = arn.split(':');
return arnArr.map(portion => wildcards.handleWildcards(portion));
};

View File

@ -6,6 +6,7 @@ const crypto = require('crypto');
* data through a stream
*/
class MD5Sum extends Transform {
/**
* @constructor
*/
@ -39,6 +40,7 @@ class MD5Sum extends Transform {
this.emit('hashed');
callback(null);
}
}
module.exports = MD5Sum;

View File

@ -73,7 +73,7 @@ class ResultsCollector extends EventEmitter {
* @property {Error} [results[].error] - error returned by Azure putting subpart
* @property {number} results[].subPartIndex - index of the subpart
*/
/**
/**
* "error" event
* @event ResultCollector#error
* @type {(Error|undefined)} error - error returned by Azure last subpart

View File

@ -94,7 +94,7 @@ azureMpuUtils.getSubPartIds = (part, uploadId) =>
azureMpuUtils.getBlockId(uploadId, part.partNumber, subPartIndex));
azureMpuUtils.putSinglePart = (errorWrapperFn, request, params, dataStoreName,
log, cb) => {
log, cb) => {
const { bucketName, partNumber, size, objectKey, contentMD5, uploadId }
= params;
const blockId = azureMpuUtils.getBlockId(uploadId, partNumber, 0);
@ -107,31 +107,31 @@ azureMpuUtils.putSinglePart = (errorWrapperFn, request, params, dataStoreName,
request.pipe(passThrough);
return errorWrapperFn('uploadPart', 'createBlockFromStream',
[blockId, bucketName, objectKey, passThrough, size, options,
(err, result) => {
if (err) {
log.error('Error from Azure data backend uploadPart',
{ error: err.message, dataStoreName });
if (err.code === 'ContainerNotFound') {
return cb(errors.NoSuchBucket);
}
if (err.code === 'InvalidMd5') {
return cb(errors.InvalidDigest);
}
if (err.code === 'Md5Mismatch') {
return cb(errors.BadDigest);
}
return cb(errors.InternalError.customizeDescription(
`Error returned from Azure: ${err.message}`),
);
(err, result) => {
if (err) {
log.error('Error from Azure data backend uploadPart',
{ error: err.message, dataStoreName });
if (err.code === 'ContainerNotFound') {
return cb(errors.NoSuchBucket);
}
const md5 = result.headers['content-md5'] || '';
const eTag = objectUtils.getHexMD5(md5);
return cb(null, eTag, size);
}], log, cb);
if (err.code === 'InvalidMd5') {
return cb(errors.InvalidDigest);
}
if (err.code === 'Md5Mismatch') {
return cb(errors.BadDigest);
}
return cb(errors.InternalError.customizeDescription(
`Error returned from Azure: ${err.message}`)
);
}
const md5 = result.headers['content-md5'] || '';
const eTag = objectUtils.getHexMD5(md5);
return cb(null, eTag, size);
}], log, cb);
};
azureMpuUtils.putNextSubPart = (errorWrapperFn, partParams, subPartInfo,
subPartStream, subPartIndex, resultsCollector, log, cb) => {
subPartStream, subPartIndex, resultsCollector, log, cb) => {
const { uploadId, partNumber, bucketName, objectKey } = partParams;
const subPartSize = azureMpuUtils.getSubPartSize(
subPartInfo, subPartIndex);
@ -140,11 +140,11 @@ azureMpuUtils.putNextSubPart = (errorWrapperFn, partParams, subPartInfo,
resultsCollector.pushOp();
errorWrapperFn('uploadPart', 'createBlockFromStream',
[subPartId, bucketName, objectKey, subPartStream, subPartSize,
{}, err => resultsCollector.pushResult(err, subPartIndex)], log, cb);
{}, err => resultsCollector.pushResult(err, subPartIndex)], log, cb);
};
azureMpuUtils.putSubParts = (errorWrapperFn, request, params,
dataStoreName, log, cb) => {
dataStoreName, log, cb) => {
const subPartInfo = azureMpuUtils.getSubPartInfo(params.size);
const resultsCollector = new ResultsCollector();
const hashedStream = new MD5Sum();

View File

@ -31,9 +31,9 @@ convertMethods.listMultipartUploads = xmlParams => {
const l = xmlParams.list;
xml.push('<?xml version="1.0" encoding="UTF-8"?>',
'<ListMultipartUploadsResult ' +
'<ListMultipartUploadsResult ' +
'xmlns="http://s3.amazonaws.com/doc/2006-03-01/">',
`<Bucket>${escapeForXml(xmlParams.bucketName)}</Bucket>`,
`<Bucket>${escapeForXml(xmlParams.bucketName)}</Bucket>`
);
// For certain XML elements, if it is `undefined`, AWS returns either an
@ -58,7 +58,7 @@ convertMethods.listMultipartUploads = xmlParams => {
});
xml.push(`<MaxUploads>${escapeForXml(l.MaxKeys)}</MaxUploads>`,
`<IsTruncated>${escapeForXml(l.IsTruncated)}</IsTruncated>`,
`<IsTruncated>${escapeForXml(l.IsTruncated)}</IsTruncated>`
);
l.Uploads.forEach(upload => {
@ -69,29 +69,29 @@ convertMethods.listMultipartUploads = xmlParams => {
}
xml.push('<Upload>',
`<Key>${escapeForXml(key)}</Key>`,
`<UploadId>${escapeForXml(val.UploadId)}</UploadId>`,
'<Initiator>',
`<ID>${escapeForXml(val.Initiator.ID)}</ID>`,
`<DisplayName>${escapeForXml(val.Initiator.DisplayName)}` +
`<Key>${escapeForXml(key)}</Key>`,
`<UploadId>${escapeForXml(val.UploadId)}</UploadId>`,
'<Initiator>',
`<ID>${escapeForXml(val.Initiator.ID)}</ID>`,
`<DisplayName>${escapeForXml(val.Initiator.DisplayName)}` +
'</DisplayName>',
'</Initiator>',
'<Owner>',
`<ID>${escapeForXml(val.Owner.ID)}</ID>`,
`<DisplayName>${escapeForXml(val.Owner.DisplayName)}` +
'</Initiator>',
'<Owner>',
`<ID>${escapeForXml(val.Owner.ID)}</ID>`,
`<DisplayName>${escapeForXml(val.Owner.DisplayName)}` +
'</DisplayName>',
'</Owner>',
`<StorageClass>${escapeForXml(val.StorageClass)}` +
'</Owner>',
`<StorageClass>${escapeForXml(val.StorageClass)}` +
'</StorageClass>',
`<Initiated>${escapeForXml(val.Initiated)}</Initiated>`,
'</Upload>',
`<Initiated>${escapeForXml(val.Initiated)}</Initiated>`,
'</Upload>'
);
});
l.CommonPrefixes.forEach(prefix => {
xml.push('<CommonPrefixes>',
`<Prefix>${escapeForXml(prefix)}</Prefix>`,
'</CommonPrefixes>',
`<Prefix>${escapeForXml(prefix)}</Prefix>`,
'</CommonPrefixes>'
);
});

View File

@ -5,6 +5,7 @@ const Readable = require('stream').Readable;
* This class is used to produce zeros filled buffers for a reader consumption
*/
class NullStream extends Readable {
/**
* Construct a new zeros filled buffers producer that will
* produce as much bytes as specified by the range parameter, or the size
@ -31,8 +32,8 @@ class NullStream extends Readable {
_read(size) {
const toRead = Math.min(size, this.bytesToRead);
const buffer = toRead > 0
? Buffer.alloc(toRead, 0)
: null;
? Buffer.alloc(toRead, 0)
: null;
this.bytesToRead -= toRead;
this.push(buffer);
}

View File

@ -110,7 +110,7 @@ function generateMpuPartStorageInfo(filteredPartList) {
* and extraPartLocations
*/
function validateAndFilterMpuParts(storedParts, jsonList, mpuOverviewKey,
splitter, log) {
splitter, log) {
let storedPartsCopy = [];
const filteredPartsObj = {};
filteredPartsObj.partList = [];

View File

@ -4,11 +4,11 @@ const errors = require('../errors');
const escapeForXml = require('./escapeForXml');
const errorInvalidArgument = errors.InvalidArgument
.customizeDescription('The header \'x-amz-tagging\' shall be ' +
.customizeDescription('The header \'x-amz-tagging\' shall be ' +
'encoded as UTF-8 then URLEncoded URL query parameters without ' +
'tag name duplicates.');
const errorBadRequestLimit50 = errors.BadRequest
.customizeDescription('Object tags cannot be greater than 50');
.customizeDescription('Object tags cannot be greater than 50');
/*
Format of xml request:
@ -38,7 +38,7 @@ const _validator = {
result.Tagging.TagSet &&
result.Tagging.TagSet.length === 1 &&
(
result.Tagging.TagSet[0] === '' ||
result.Tagging.TagSet[0] === '' ||
result.Tagging.TagSet[0] &&
Object.keys(result.Tagging.TagSet[0]).length === 1 &&
result.Tagging.TagSet[0].Tag &&
@ -155,7 +155,7 @@ function parseTagXml(xml, log, cb) {
function convertToXml(objectTags) {
const xml = [];
xml.push('<?xml version="1.0" encoding="UTF-8" standalone="yes"?>',
'<Tagging> <TagSet>');
'<Tagging> <TagSet>');
if (objectTags && Object.keys(objectTags).length > 0) {
Object.keys(objectTags).forEach(key => {
xml.push(`<Tag><Key>${escapeForXml(key)}</Key>` +

View File

@ -43,7 +43,7 @@ function checkBucketAndKey(bucketName, objectKey, method, reqQuery,
log.debug('empty bucket name', { method: 'routes' });
return (method !== 'OPTIONS') ?
errors.MethodNotAllowed : errors.AccessForbidden
.customizeDescription('CORSResponse: Bucket not found');
.customizeDescription('CORSResponse: Bucket not found');
}
if (bucketName !== undefined && routesUtils.isValidBucketName(bucketName,
blacklistedPrefixes.bucket) === false) {
@ -92,7 +92,7 @@ function checkTypes(req, res, params, logger, s3config) {
'bad routes param: internalHandlers must be an object');
if (params.statsClient) {
assert.strictEqual(typeof params.statsClient, 'object',
'bad routes param: statsClient must be an object');
'bad routes param: statsClient must be an object');
}
assert(Array.isArray(params.allEndpoints),
'bad routes param: allEndpoints must be an array');
@ -100,13 +100,13 @@ function checkTypes(req, res, params, logger, s3config) {
'bad routes param: allEndpoints must have at least one endpoint');
params.allEndpoints.forEach(endpoint => {
assert.strictEqual(typeof endpoint, 'string',
'bad routes param: each item in allEndpoints must be a string');
'bad routes param: each item in allEndpoints must be a string');
});
assert(Array.isArray(params.websiteEndpoints),
'bad routes param: allEndpoints must be an array');
params.websiteEndpoints.forEach(endpoint => {
assert.strictEqual(typeof endpoint, 'string',
'bad routes param: each item in websiteEndpoints must be a string');
'bad routes param: each item in websiteEndpoints must be a string');
});
assert.strictEqual(typeof params.blacklistedPrefixes, 'object',
'bad routes param: blacklistedPrefixes must be an object');
@ -114,13 +114,13 @@ function checkTypes(req, res, params, logger, s3config) {
'bad routes param: blacklistedPrefixes.bucket must be an array');
params.blacklistedPrefixes.bucket.forEach(pre => {
assert.strictEqual(typeof pre, 'string',
'bad routes param: each blacklisted bucket prefix must be a string');
'bad routes param: each blacklisted bucket prefix must be a string');
});
assert(Array.isArray(params.blacklistedPrefixes.object),
'bad routes param: blacklistedPrefixes.object must be an array');
params.blacklistedPrefixes.object.forEach(pre => {
assert.strictEqual(typeof pre, 'string',
'bad routes param: each blacklisted object prefix must be a string');
'bad routes param: each blacklisted object prefix must be a string');
});
assert.strictEqual(typeof params.dataRetrievalParams, 'object',
'bad routes param: dataRetrievalParams must be a defined object');
@ -179,8 +179,8 @@ function routes(req, res, params, logger, s3config) {
reqUids = undefined;
}
const log = (reqUids !== undefined ?
logger.newRequestLoggerFromSerializedUids(reqUids) :
logger.newRequestLogger());
logger.newRequestLoggerFromSerializedUids(reqUids) :
logger.newRequestLogger());
if (!req.url.startsWith('/_/healthcheck') &&
!req.url.startsWith('/_/report')) {
@ -216,7 +216,7 @@ function routes(req, res, params, logger, s3config) {
return routesUtils.responseXMLBody(
errors.InvalidURI.customizeDescription('Could not parse the ' +
'specified URI. Check your restEndpoints configuration.'),
undefined, res, log);
undefined, res, log);
}
log.addDefaultFields({
@ -238,7 +238,7 @@ function routes(req, res, params, logger, s3config) {
if (bucketOrKeyError) {
log.trace('error with bucket or key value',
{ error: bucketOrKeyError });
{ error: bucketOrKeyError });
return routesUtils.responseXMLBody(bucketOrKeyError, null, res, log);
}

View File

@ -7,7 +7,7 @@ function routeDELETE(request, response, api, log, statsClient) {
if (request.query.uploadId) {
if (request.objectKey === undefined) {
return routesUtils.responseNoBody(
errors.InvalidRequest.customizeDescription('A key must be ' +
errors.InvalidRequest.customizeDescription('A key must be ' +
'specified'), null, response, 200, log);
}
api.callApiMethod('multipartDelete', request, response, log,
@ -19,77 +19,77 @@ function routeDELETE(request, response, api, log, statsClient) {
} else if (request.objectKey === undefined) {
if (request.query.website !== undefined) {
return api.callApiMethod('bucketDeleteWebsite', request,
response, log, (err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
response, log, (err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
} else if (request.query.cors !== undefined) {
return api.callApiMethod('bucketDeleteCors', request, response,
log, (err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
log, (err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
} else if (request.query.replication !== undefined) {
return api.callApiMethod('bucketDeleteReplication', request,
response, log, (err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
response, log, (err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
} else if (request.query.lifecycle !== undefined) {
return api.callApiMethod('bucketDeleteLifecycle', request,
response, log, (err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
response, log, (err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
} else if (request.query.policy !== undefined) {
return api.callApiMethod('bucketDeletePolicy', request,
response, log, (err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
response, log, (err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
} else if (request.query.encryption !== undefined) {
return api.callApiMethod('bucketDeleteEncryption', request,
response, log, (err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
response, log, (err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 204, log);
});
}
api.callApiMethod('bucketDelete', request, response, log,
(err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders, response,
204, log);
});
(err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders, response,
204, log);
});
} else {
if (request.query.tagging !== undefined) {
return api.callApiMethod('objectDeleteTagging', request,
response, log, (err, resHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, resHeaders,
response, 204, log);
});
response, log, (err, resHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, resHeaders,
response, 204, log);
});
}
api.callApiMethod('objectDelete', request, response, log,
(err, corsHeaders) => {
/*
(err, corsHeaders) => {
/*
* Since AWS expects a 204 regardless of the existence of
the object, the errors NoSuchKey and NoSuchVersion should not
* be sent back as a response.
*/
if (err && !err.NoSuchKey && !err.NoSuchVersion) {
return routesUtils.responseNoBody(err, corsHeaders,
response, null, log);
}
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(null, corsHeaders, response,
204, log);
});
if (err && !err.NoSuchKey && !err.NoSuchVersion) {
return routesUtils.responseNoBody(err, corsHeaders,
response, null, log);
}
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(null, corsHeaders, response,
204, log);
});
}
return undefined;
}

View File

@ -2,7 +2,7 @@ const errors = require('../../errors');
const routesUtils = require('../routesUtils');
function routerGET(request, response, api, log, statsClient,
dataRetrievalParams) {
dataRetrievalParams) {
log.debug('routing request', { method: 'routerGET' });
if (request.bucketName === undefined && request.objectKey !== undefined) {
routesUtils.responseXMLBody(errors.NoSuchBucket, null, response, log);
@ -17,18 +17,18 @@ function routerGET(request, response, api, log, statsClient,
// GET bucket ACL
if (request.query.acl !== undefined) {
api.callApiMethod('bucketGetACL', request, response, log,
(err, xml, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseXMLBody(err, xml, response, log,
corsHeaders);
});
(err, xml, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseXMLBody(err, xml, response, log,
corsHeaders);
});
} else if (request.query.replication !== undefined) {
api.callApiMethod('bucketGetReplication', request, response, log,
(err, xml, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseXMLBody(err, xml, response, log,
corsHeaders);
});
(err, xml, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseXMLBody(err, xml, response, log,
corsHeaders);
});
} else if (request.query.cors !== undefined) {
api.callApiMethod('bucketGetCors', request, response, log,
(err, xml, corsHeaders) => {
@ -70,7 +70,7 @@ function routerGET(request, response, api, log, statsClient,
(err, xml, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseXMLBody(err, xml, response, log,
corsHeaders);
corsHeaders);
});
} else if (request.query.policy !== undefined) {
api.callApiMethod('bucketGetPolicy', request, response, log,
@ -95,11 +95,11 @@ function routerGET(request, response, api, log, statsClient,
});
} else if (request.query.encryption !== undefined) {
api.callApiMethod('bucketGetEncryption', request, response, log,
(err, xml, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseXMLBody(err, xml, response,
log, corsHeaders);
});
(err, xml, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseXMLBody(err, xml, response,
log, corsHeaders);
});
} else if (request.query.search !== undefined) {
api.callApiMethod('metadataSearch', request, response, log,
(err, xml, corsHeaders) => {

View File

@ -21,11 +21,11 @@ function routeOPTIONS(request, response, api, log, statsClient) {
}
return api.callApiMethod('corsPreflight', request, response, log,
(err, resHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, resHeaders, response, 200,
log);
});
(err, resHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, resHeaders, response, 200,
log);
});
}
module.exports = routeOPTIONS;

View File

@ -27,28 +27,28 @@ function routePOST(request, response, api, log) {
if (request.query.uploads !== undefined) {
return api.callApiMethod('initiateMultipartUpload', request,
response, log, (err, result, corsHeaders) =>
routesUtils.responseXMLBody(err, result, response, log,
corsHeaders));
routesUtils.responseXMLBody(err, result, response, log,
corsHeaders));
}
// POST complete multipart upload
if (request.query.uploadId !== undefined) {
return api.callApiMethod('completeMultipartUpload', request,
response, log, (err, result, resHeaders) =>
routesUtils.responseXMLBody(err, result, response, log,
resHeaders));
routesUtils.responseXMLBody(err, result, response, log,
resHeaders));
}
// POST multiObjectDelete
if (request.query.delete !== undefined) {
return api.callApiMethod('multiObjectDelete', request, response,
log, (err, xml, corsHeaders) =>
routesUtils.responseXMLBody(err, xml, response, log,
corsHeaders));
routesUtils.responseXMLBody(err, xml, response, log,
corsHeaders));
}
return routesUtils.responseNoBody(errors.NotImplemented, null, response,
200, log);
200, log);
}
/* eslint-enable no-param-reassign */
module.exports = routePOST;

View File

@ -14,16 +14,16 @@ function routePUT(request, response, api, log, statsClient) {
|| contentLength < 0)) || contentLength === '') {
log.debug('invalid content-length header');
return routesUtils.responseNoBody(
errors.BadRequest, null, response, null, log);
errors.BadRequest, null, response, null, log);
}
// PUT bucket ACL
if (request.query.acl !== undefined) {
api.callApiMethod('bucketPutACL', request, response, log,
(err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 200, log);
});
(err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 200, log);
});
} else if (request.query.versioning !== undefined) {
api.callApiMethod('bucketPutVersioning', request, response, log,
(err, corsHeaders) => {
@ -82,11 +82,11 @@ function routePUT(request, response, api, log, statsClient) {
});
} else if (request.query.encryption !== undefined) {
api.callApiMethod('bucketPutEncryption', request, response, log,
(err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 200, log);
});
(err, corsHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, corsHeaders,
response, 200, log);
});
} else {
// PUT bucket
return api.callApiMethod('bucketPut', request, response, log,
@ -110,7 +110,7 @@ function routePUT(request, response, api, log, statsClient) {
method: 'routePUT',
});
return routesUtils
.responseNoBody(errors.InvalidDigest, null, response, 200, log);
.responseNoBody(errors.InvalidDigest, null, response, 200, log);
}
if (request.headers['content-md5']) {
request.contentMD5 = request.headers['content-md5'];
@ -126,17 +126,17 @@ function routePUT(request, response, api, log, statsClient) {
});
return routesUtils
.responseNoBody(errors.InvalidDigest, null, response, 200,
log);
log);
}
}
if (request.query.partNumber) {
if (request.headers['x-amz-copy-source']) {
api.callApiMethod('objectPutCopyPart', request, response, log,
(err, xml, additionalHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseXMLBody(err, xml, response, log,
(err, xml, additionalHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseXMLBody(err, xml, response, log,
additionalHeaders);
});
});
} else {
api.callApiMethod('objectPutPart', request, response, log,
(err, calculatedHash, corsHeaders) => {
@ -202,11 +202,11 @@ function routePUT(request, response, api, log, statsClient) {
contentLength: request.parsedContentLength,
});
api.callApiMethod('objectPut', request, response, log,
(err, resHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, resHeaders,
response, 200, log);
});
(err, resHeaders) => {
routesUtils.statsReport500(err, statsClient);
return routesUtils.responseNoBody(err, resHeaders,
response, 200, log);
});
}
}
return undefined;

View File

@ -14,7 +14,7 @@ function routerWebsite(request, response, api, log, statsClient,
if (request.method === 'GET') {
return api.callApiMethod('websiteGet', request, response, log,
(err, userErrorPageFailure, dataGetInfo, resMetaHeaders,
redirectInfo, key) => {
redirectInfo, key) => {
routesUtils.statsReport500(err, statsClient);
// request being redirected
if (redirectInfo) {
@ -43,21 +43,21 @@ function routerWebsite(request, response, api, log, statsClient,
}
if (request.method === 'HEAD') {
return api.callApiMethod('websiteHead', request, response, log,
(err, resMetaHeaders, redirectInfo, key) => {
routesUtils.statsReport500(err, statsClient);
if (redirectInfo) {
return routesUtils.redirectRequest(redirectInfo,
key, request.connection.encrypted,
response, request.headers.host, resMetaHeaders, log);
}
// could redirect on err so check for redirectInfo first
if (err) {
return routesUtils.errorHeaderResponse(err, response,
resMetaHeaders, log);
}
return routesUtils.responseContentHeaders(err, {}, resMetaHeaders,
response, log);
});
(err, resMetaHeaders, redirectInfo, key) => {
routesUtils.statsReport500(err, statsClient);
if (redirectInfo) {
return routesUtils.redirectRequest(redirectInfo,
key, request.connection.encrypted,
response, request.headers.host, resMetaHeaders, log);
}
// could redirect on err so check for redirectInfo first
if (err) {
return routesUtils.errorHeaderResponse(err, response,
resMetaHeaders, log);
}
return routesUtils.responseContentHeaders(err, {}, resMetaHeaders,
response, log);
});
}
return undefined;
}

View File

@ -28,7 +28,7 @@ function setCommonResponseHeaders(headers, response, log) {
} catch (e) {
log.debug('header can not be added ' +
'to the response', { header: headers[key],
error: e.stack, method: 'setCommonResponseHeaders' });
error: e.stack, method: 'setCommonResponseHeaders' });
}
}
});
@ -71,7 +71,7 @@ const XMLResponseBackend = {
* @return {object} response - response object with additional headers
*/
okResponse: function okXMLResponse(xml, response, log,
additionalHeaders) {
additionalHeaders) {
const bytesSent = Buffer.byteLength(xml);
log.trace('sending success xml response');
log.addDefaultFields({
@ -118,7 +118,7 @@ const XMLResponseBackend = {
`<Message>${errCode.description}</Message>`,
'<Resource></Resource>',
`<RequestId>${log.getSerializedUids()}</RequestId>`,
'</Error>',
'</Error>'
);
const xmlStr = xml.join('');
const bytesSent = Buffer.byteLength(xmlStr);
@ -148,7 +148,7 @@ const JSONResponseBackend = {
* @return {object} response - response object with additional headers
*/
okResponse: function okJSONResponse(json, response, log,
additionalHeaders) {
additionalHeaders) {
const bytesSent = Buffer.byteLength(json);
log.trace('sending success json response');
log.addDefaultFields({
@ -166,7 +166,7 @@ const JSONResponseBackend = {
},
errorResponse: function errorJSONResponse(errCode, response, log,
corsHeaders) {
corsHeaders) {
log.trace('sending error json response', { errCode });
/*
{
@ -368,27 +368,27 @@ function retrieveData(locations, retrieveDataParams, response, log) {
currentStream = readable;
return readable.pipe(response, { end: false });
}), err => {
currentStream = null;
if (err) {
log.debug('abort response due to error', {
error: err.code, errMsg: err.message });
currentStream = null;
if (err) {
log.debug('abort response due to error', {
error: err.code, errMsg: err.message });
}
// call end for all cases (error/success) per node.js docs
// recommendation
response.end();
}
// call end for all cases (error/success) per node.js docs
// recommendation
response.end();
},
);
}
function _responseBody(responseBackend, errCode, payload, response, log,
additionalHeaders) {
additionalHeaders) {
if (errCode && !response.headersSent) {
return responseBackend.errorResponse(errCode, response, log,
additionalHeaders);
additionalHeaders);
}
if (!response.headersSent) {
return responseBackend.okResponse(payload, response, log,
additionalHeaders);
additionalHeaders);
}
return undefined;
}
@ -397,8 +397,8 @@ function _computeContentLengthFromLocation(dataLocations) {
return dataLocations.reduce(
(sum, location) => (sum !== undefined &&
(typeof location.size === 'number' || typeof location.size === 'string') ?
sum + Number.parseInt(location.size, 10) :
undefined), 0);
sum + Number.parseInt(location.size, 10) :
undefined), 0);
}
function _contentLengthMatchesLocations(contentLength, dataLocations) {
@ -419,7 +419,7 @@ const routesUtils = {
*/
responseXMLBody(errCode, xml, response, log, additionalHeaders) {
return _responseBody(XMLResponseBackend, errCode, xml, response,
log, additionalHeaders);
log, additionalHeaders);
},
/**
@ -433,7 +433,7 @@ const routesUtils = {
*/
responseJSONBody(errCode, json, response, log, additionalHeaders) {
return _responseBody(JSONResponseBackend, errCode, json, response,
log, additionalHeaders);
log, additionalHeaders);
},
/**
@ -448,7 +448,7 @@ const routesUtils = {
responseNoBody(errCode, resHeaders, response, httpCode = 200, log) {
if (errCode && !response.headersSent) {
return XMLResponseBackend.errorResponse(errCode, response, log,
resHeaders);
resHeaders);
}
if (!response.headersSent) {
return okHeaderResponse(resHeaders, response, httpCode, log);
@ -466,10 +466,10 @@ const routesUtils = {
* @return {object} - router's response object
*/
responseContentHeaders(errCode, overrideParams, resHeaders, response,
log) {
log) {
if (errCode && !response.headersSent) {
return XMLResponseBackend.errorResponse(errCode, response, log,
resHeaders);
resHeaders);
}
if (!response.headersSent) {
// Undefined added as an argument since need to send range to
@ -504,7 +504,7 @@ const routesUtils = {
retrieveDataParams, response, range, log) {
if (errCode && !response.headersSent) {
return XMLResponseBackend.errorResponse(errCode, response, log,
resHeaders);
resHeaders);
}
if (dataLocations !== null && !response.headersSent) {
// sanity check of content length against individual data
@ -512,13 +512,13 @@ const routesUtils = {
const contentLength = resHeaders && resHeaders['Content-Length'];
if (contentLength !== undefined &&
!_contentLengthMatchesLocations(contentLength,
dataLocations)) {
dataLocations)) {
log.error('logic error: total length of fetched data ' +
'locations does not match returned content-length',
{ contentLength, dataLocations });
{ contentLength, dataLocations });
return XMLResponseBackend.errorResponse(errors.InternalError,
response, log,
resHeaders);
response, log,
resHeaders);
}
}
if (!response.headersSent) {
@ -591,7 +591,7 @@ const routesUtils = {
`<h1>${err.code} ${response.statusMessage}</h1>`,
'<ul>',
`<li>Code: ${err.message}</li>`,
`<li>Message: ${err.description}</li>`,
`<li>Message: ${err.description}</li>`
);
if (!userErrorPageFailure && bucketName) {
@ -601,7 +601,7 @@ const routesUtils = {
`<li>RequestId: ${log.getSerializedUids()}</li>`,
// AWS response contains HostId here.
// TODO: consider adding
'</ul>',
'</ul>'
);
if (userErrorPageFailure) {
html.push(
@ -611,13 +611,13 @@ const routesUtils = {
'<ul>',
`<li>Code: ${err.message}</li>`,
`<li>Message: ${err.description}</li>`,
'</ul>',
'</ul>'
);
}
html.push(
'<hr/>',
'</body>',
'</html>',
'</html>'
);
return response.end(html.join(''), 'utf8', () => {
@ -839,7 +839,7 @@ const routesUtils = {
// most specific potential hostname
bucketName =
potentialBucketName.length < bucketName.length ?
potentialBucketName : bucketName;
potentialBucketName : bucketName;
}
}
}
@ -847,7 +847,7 @@ const routesUtils = {
return bucketName;
}
throw new Error(
`bad request: hostname ${host} is not in valid endpoints`,
`bad request: hostname ${host} is not in valid endpoints`
);
},

View File

@ -19,7 +19,7 @@ const externalVersioningErrorMessage = 'We do not currently support putting ' +
class DataWrapper {
constructor(client, implName, config, kms, metadata, locStorageCheckFn,
vault) {
vault) {
this.client = client;
this.implName = implName;
this.config = config;
@ -36,31 +36,31 @@ class DataWrapper {
return cb(err);
}
return this._put(cipherBundle, value, valueSize, keyContext,
backendInfo, log, (err, dataRetrievalInfo, hashedStream) => {
if (err) {
backendInfo, log, (err, dataRetrievalInfo, hashedStream) => {
if (err) {
// if error putting object, counter should be decremented
return this.locStorageCheckFn(location, -valueSize, log,
error => {
if (error) {
log.error('Error decrementing location metric ' +
return this.locStorageCheckFn(location, -valueSize, log,
error => {
if (error) {
log.error('Error decrementing location metric ' +
'following object PUT failure',
{ error: error.message });
}
return cb(err);
});
}
if (hashedStream) {
if (hashedStream.completedHash) {
return cb(null, dataRetrievalInfo, hashedStream);
{ error: error.message });
}
hashedStream.on('hashed', () => {
hashedStream.removeAllListeners('hashed');
return cb(null, dataRetrievalInfo, hashedStream);
});
return undefined;
return cb(err);
});
}
if (hashedStream) {
if (hashedStream.completedHash) {
return cb(null, dataRetrievalInfo, hashedStream);
}
return cb(null, dataRetrievalInfo);
});
hashedStream.on('hashed', () => {
hashedStream.removeAllListeners('hashed');
return cb(null, dataRetrievalInfo, hashedStream);
});
return undefined;
}
return cb(null, dataRetrievalInfo);
});
});
}
@ -97,33 +97,33 @@ class DataWrapper {
clientGetInfo.response = response;
}
this.client.get(clientGetInfo, range, log.getSerializedUids(),
(err, stream) => {
if (err) {
log.error('get error from datastore',
{ error: err, implName: this.implName });
return cb(errors.ServiceUnavailable);
}
if (objectGetInfo.cipheredDataKey) {
const serverSideEncryption = {
cryptoScheme: objectGetInfo.cryptoScheme,
masterKeyId: objectGetInfo.masterKeyId,
cipheredDataKey: Buffer.from(
objectGetInfo.cipheredDataKey, 'base64'),
};
const offset = objectGetInfo.range ? objectGetInfo.range[0] : 0;
return this.kms.createDecipherBundle(serverSideEncryption,
offset, log, (err, decipherBundle) => {
if (err) {
log.error('cannot get decipher bundle from kms',
{ method: 'data.wrapper.data.get' });
return cb(err);
}
stream.pipe(decipherBundle.decipher);
return cb(null, decipherBundle.decipher);
});
}
return cb(null, stream);
});
(err, stream) => {
if (err) {
log.error('get error from datastore',
{ error: err, implName: this.implName });
return cb(errors.ServiceUnavailable);
}
if (objectGetInfo.cipheredDataKey) {
const serverSideEncryption = {
cryptoScheme: objectGetInfo.cryptoScheme,
masterKeyId: objectGetInfo.masterKeyId,
cipheredDataKey: Buffer.from(
objectGetInfo.cipheredDataKey, 'base64'),
};
const offset = objectGetInfo.range ? objectGetInfo.range[0] : 0;
return this.kms.createDecipherBundle(serverSideEncryption,
offset, log, (err, decipherBundle) => {
if (err) {
log.error('cannot get decipher bundle from kms',
{ method: 'data.wrapper.data.get' });
return cb(err);
}
stream.pipe(decipherBundle.decipher);
return cb(null, decipherBundle.decipher);
});
}
return cb(null, stream);
});
}
delete(objectGetInfo, log, cb) {
@ -153,15 +153,15 @@ class DataWrapper {
if (!err) {
// pass size as negative so location metric is decremented
return this.locStorageCheckFn(objectGetInfo.dataStoreName,
-objectGetInfo.size, log, err => {
if (err) {
log.error('Utapi error pushing location metric', {
error: err,
key: objectGetInfo.key,
method: 'locationStorageCheck' });
}
return callback(err);
});
-objectGetInfo.size, log, err => {
if (err) {
log.error('Utapi error pushing location metric', {
error: err,
key: objectGetInfo.key,
method: 'locationStorageCheck' });
}
return callback(err);
});
}
return callback(err);
});
@ -173,7 +173,7 @@ class DataWrapper {
// meantime, we at least log the location of the data we are
// about to delete before attempting its deletion.
if (this._shouldSkipDelete(locations, requestMethod,
newObjDataStoreName)) {
newObjDataStoreName)) {
return process.nextTick(cb);
}
log.trace('initiating batch delete', {
@ -210,7 +210,7 @@ class DataWrapper {
err => {
if (err) {
log.end().error('batch delete failed', { error: err });
// deletion of non-existing objects result in 204
// deletion of non-existing objects result in 204
if (err.code === 404) {
return cb();
}
@ -233,27 +233,27 @@ class DataWrapper {
return cb(null, defResp);
}
return this.client.healthcheck(flightCheckOnStartUp, log,
(err, result) => {
let respBody = {};
if (err) {
log.error(`error from ${this.implName}`, { error: err });
respBody[this.implName] = {
error: err,
};
// error returned as null so async parallel doesn't return
// before all backends are checked
return cb(null, respBody);
}
if (this.implName === 'multipleBackends') {
respBody = result;
return cb(null, respBody);
}
(err, result) => {
let respBody = {};
if (err) {
log.error(`error from ${this.implName}`, { error: err });
respBody[this.implName] = {
code: result.statusCode,
message: result.statusMessage,
error: err,
};
// error returned as null so async parallel doesn't return
// before all backends are checked
return cb(null, respBody);
});
}
if (this.implName === 'multipleBackends') {
respBody = result;
return cb(null, respBody);
}
respBody[this.implName] = {
code: result.statusCode,
message: result.statusMessage,
};
return cb(null, respBody);
});
}
getDiskUsage(log, cb) {
@ -291,36 +291,36 @@ class DataWrapper {
* @returns {function} cb - callback
*/
copyObject(request, sourceLocationConstraintName, storeMetadataParams,
dataLocator, dataStoreContext, destBackendInfo, sourceBucketMD,
destBucketMD, serverSideEncryption, log, cb) {
dataLocator, dataStoreContext, destBackendInfo, sourceBucketMD,
destBucketMD, serverSideEncryption, log, cb) {
if (this.config.backends.data === 'multiple' &&
backendUtils.externalBackendCopy(this.config,
sourceLocationConstraintName, storeMetadataParams.dataStoreName,
sourceBucketMD, destBucketMD)
sourceLocationConstraintName, storeMetadataParams.dataStoreName,
sourceBucketMD, destBucketMD)
&& serverSideEncryption === null) {
const destLocationConstraintName =
storeMetadataParams.dataStoreName;
const objectGetInfo = dataLocator[0];
const externalSourceKey = objectGetInfo.key;
return this.client.copyObject(request, destLocationConstraintName,
externalSourceKey, sourceLocationConstraintName,
storeMetadataParams, this.config, log,
(error, objectRetrievalInfo) => {
if (error) {
return cb(error);
}
const putResult = {
key: objectRetrievalInfo.key,
dataStoreName: objectRetrievalInfo.dataStoreName,
dataStoreType: objectRetrievalInfo.dataStoreType,
dataStoreVersionId: objectRetrievalInfo.dataStoreVersionId,
size: storeMetadataParams.size,
dataStoreETag: objectGetInfo.dataStoreETag,
start: objectGetInfo.start,
};
const putResultArr = [putResult];
return cb(null, putResultArr);
});
externalSourceKey, sourceLocationConstraintName,
storeMetadataParams, this.config, log,
(error, objectRetrievalInfo) => {
if (error) {
return cb(error);
}
const putResult = {
key: objectRetrievalInfo.key,
dataStoreName: objectRetrievalInfo.dataStoreName,
dataStoreType: objectRetrievalInfo.dataStoreType,
dataStoreVersionId: objectRetrievalInfo.dataStoreVersionId,
size: storeMetadataParams.size,
dataStoreETag: objectGetInfo.dataStoreETag,
start: objectGetInfo.start,
};
const putResultArr = [putResult];
return cb(null, putResultArr);
});
}
const that = this;
@ -330,37 +330,37 @@ class DataWrapper {
// copied at once.
return async.mapLimit(dataLocator, 1,
// eslint-disable-next-line prefer-arrow-callback
function copyPart(part, copyCb) {
if (part.dataStoreType === 'azure') {
const passThrough = new PassThrough();
return async.parallel([
parallelCb => that.get(part, passThrough, log, err =>
parallelCb(err)),
parallelCb => that._dataCopyPut(serverSideEncryption,
passThrough, part, dataStoreContext,
destBackendInfo, log, parallelCb),
], (err, res) => {
if (err) {
return copyCb(err);
}
return copyCb(null, res[1]);
});
}
return that.get(part, null, log, (err, stream) => {
function copyPart(part, copyCb) {
if (part.dataStoreType === 'azure') {
const passThrough = new PassThrough();
return async.parallel([
parallelCb => that.get(part, passThrough, log, err =>
parallelCb(err)),
parallelCb => that._dataCopyPut(serverSideEncryption,
passThrough, part, dataStoreContext,
destBackendInfo, log, parallelCb),
], (err, res) => {
if (err) {
return copyCb(err);
}
return that._dataCopyPut(serverSideEncryption, stream,
part, dataStoreContext, destBackendInfo, log, copyCb);
return copyCb(null, res[1]);
});
}, (err, results) => {
}
return that.get(part, null, log, (err, stream) => {
if (err) {
log.debug('error transferring data from source',
{ error: err });
return cb(err);
return copyCb(err);
}
return cb(null, results);
return that._dataCopyPut(serverSideEncryption, stream,
part, dataStoreContext, destBackendInfo, log, copyCb);
});
}, (err, results) => {
if (err) {
log.debug('error transferring data from source',
{ error: err });
return cb(err);
}
return cb(null, results);
});
}
/**
@ -386,8 +386,8 @@ class DataWrapper {
* @returns {function} cb - callback
*/
uploadPartCopy(request, log, destBucketMD, sourceLocationConstraintName,
destLocationConstraintName, dataLocator, dataStoreContext, lcCheckFn,
callback) {
destLocationConstraintName, dataLocator, dataStoreContext, lcCheckFn,
callback) {
const serverSideEncryption = destBucketMD.getServerSideEncryption();
const lastModified = new Date().toJSON();
@ -425,15 +425,15 @@ class DataWrapper {
if (locationTypeMatch && dataLocator.length === 1) {
const sourceKey = dataLocator[0].key;
return this.client.uploadPartCopy(request,
destLocationConstraintName, sourceKey, sourceLocationConstraintName,
this.config, log, (error, eTag) => {
if (error) {
return callback(error);
}
const doSkip = srcType === 'aws' ? skipError : null;
return callback(doSkip, eTag, lastModified,
serverSideEncryption);
});
destLocationConstraintName, sourceKey, sourceLocationConstraintName,
this.config, log, (error, eTag) => {
if (error) {
return callback(error);
}
const doSkip = srcType === 'aws' ? skipError : null;
return callback(doSkip, eTag, lastModified,
serverSideEncryption);
});
}
const backendInfo = new BackendInfo(this._config,
@ -448,72 +448,72 @@ class DataWrapper {
// in order so can get the ETag of full object
return async.forEachOfSeries(dataLocator,
// eslint-disable-next-line prefer-arrow-callback
function copyPart(part, index, cb) {
if (part.dataStoreType === 'azure') {
const passThrough = new PassThrough();
return async.parallel([
next => that.get(part, passThrough, log, err => {
if (err) {
log.error('error getting data part from Azure',
{
error: err,
method: 'objectPutCopyPart::' +
'multipleBackendGateway.copyPart',
});
return next(err);
}
return next();
}),
next => that._dataCopyPutPart(request,
serverSideEncryption, passThrough, part,
dataStoreContext, backendInfo, locations, log, next),
], err => {
function copyPart(part, index, cb) {
if (part.dataStoreType === 'azure') {
const passThrough = new PassThrough();
return async.parallel([
next => that.get(part, passThrough, log, err => {
if (err) {
return cb(err);
log.error('error getting data part from Azure',
{
error: err,
method: 'objectPutCopyPart::' +
'multipleBackendGateway.copyPart',
});
return next(err);
}
return cb();
});
}
return that.get(part, null, log, (err, stream) => {
return next();
}),
next => that._dataCopyPutPart(request,
serverSideEncryption, passThrough, part,
dataStoreContext, backendInfo, locations, log, next),
], err => {
if (err) {
log.debug('error getting object part', { error: err });
return cb(err);
}
const hashedStream =
return cb();
});
}
return that.get(part, null, log, (err, stream) => {
if (err) {
log.debug('error getting object part', { error: err });
return cb(err);
}
const hashedStream =
new RelayMD5Sum(totalHash, updatedHash => {
totalHash = updatedHash;
});
stream.pipe(hashedStream);
stream.pipe(hashedStream);
// destLocationConstraintName is location of the
// destination MPU object
return that._dataCopyPutPart(request, serverSideEncryption,
hashedStream, part, dataStoreContext, backendInfo,
locations, log, cb);
});
}, err => {
// Digest the final combination of all of the part streams
if (err && err !== skipError) {
log.debug('error transferring data from source',
{ error: err, method: 'goGetData' });
return callback(err);
}
if (totalHash) {
totalHash = totalHash.digest('hex');
} else {
totalHash = locations[0].dataStoreETag;
}
if (err && err === skipError) {
return callback(skipError, totalHash, lastModified,
serverSideEncryption);
}
return callback(null, totalHash, lastModified,
serverSideEncryption, locations);
// destLocationConstraintName is location of the
// destination MPU object
return that._dataCopyPutPart(request, serverSideEncryption,
hashedStream, part, dataStoreContext, backendInfo,
locations, log, cb);
});
}, err => {
// Digest the final combination of all of the part streams
if (err && err !== skipError) {
log.debug('error transferring data from source',
{ error: err, method: 'goGetData' });
return callback(err);
}
if (totalHash) {
totalHash = totalHash.digest('hex');
} else {
totalHash = locations[0].dataStoreETag;
}
if (err && err === skipError) {
return callback(skipError, totalHash, lastModified,
serverSideEncryption);
}
return callback(null, totalHash, lastModified,
serverSideEncryption, locations);
});
}
abortMPU(objectKey, uploadId, location, bucketName, request, destBucket,
lcCheckFn, log, callback) {
lcCheckFn, log, callback) {
if (this.config.backends.data === 'multiple') {
// if controlling location constraint is not stored in object
// metadata, mpu was initiated in legacy S3C, so need to
@ -531,13 +531,13 @@ class DataWrapper {
}
return this.client.abortMPU(objectKey, uploadId, location,
bucketName, log, callback);
bucketName, log, callback);
}
return callback(null, false);
}
completeMPU(request, mpuInfo, mdInfo, location, userMetadata,
contentSettings, tagging, lcCheckFn, log, callback) {
contentSettings, tagging, lcCheckFn, log, callback) {
const { objectKey, uploadId, jsonList, bucketName, destBucket } =
mpuInfo;
if (this.config.backends.data === 'multiple') {
@ -576,30 +576,30 @@ class DataWrapper {
} = mpuInfo;
if (this.config.backends.data === 'multiple') {
return this.client.createMPU(objectKey, metaHeaders,
bucketName, websiteRedirectHeader, locConstraint, contentType,
cacheControl, contentDisposition, contentEncoding, tagging, log,
(err, dataBackendResObj) => {
if (err) {
return callback(err);
}
const configLoc =
bucketName, websiteRedirectHeader, locConstraint, contentType,
cacheControl, contentDisposition, contentEncoding, tagging, log,
(err, dataBackendResObj) => {
if (err) {
return callback(err);
}
const configLoc =
this.config.locationConstraints[locConstraint];
if (locConstraint && configLoc && configLoc.type &&
if (locConstraint && configLoc && configLoc.type &&
constants.versioningNotImplBackends[configLoc.type]) {
const vcfg = destinationBucket.getVersioningConfiguration();
const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
if (isVersionedObj) {
log.debug(externalVersioningErrorMessage,
{ method: 'initiateMultipartUpload',
error: errors.NotImplemented });
return callback(errors.NotImplemented
.customizeDescription(
externalVersioningErrorMessage),
const vcfg = destinationBucket.getVersioningConfiguration();
const isVersionedObj = vcfg && vcfg.Status === 'Enabled';
if (isVersionedObj) {
log.debug(externalVersioningErrorMessage,
{ method: 'initiateMultipartUpload',
error: errors.NotImplemented });
return callback(errors.NotImplemented
.customizeDescription(
externalVersioningErrorMessage),
null, isVersionedObj);
}
}
return callback(null, dataBackendResObj);
});
}
return callback(null, dataBackendResObj);
});
}
return callback();
}
@ -632,19 +632,19 @@ class DataWrapper {
location = mpuOverviewObj.controllingLocationConstraint;
}
return this.client.listParts(objectKey, uploadId,
location, bucketName, partNumberMarker, maxParts, log,
(err, backendPartList) => {
if (err) {
return callback(err);
}
return callback(null, backendPartList);
});
location, bucketName, partNumberMarker, maxParts, log,
(err, backendPartList) => {
if (err) {
return callback(err);
}
return callback(null, backendPartList);
});
}
return callback();
}
putPart(request, mpuInfo, streamingV4Params, objectLocationConstraint,
lcCheckFn, log, callback) {
lcCheckFn, log, callback) {
const {
stream,
destinationBucket,
@ -668,19 +668,19 @@ class DataWrapper {
objectLocationConstraint = backendInfoObj.controllingLC;
}
return this.client.uploadPart(request,
streamingV4Params, stream, size, objectLocationConstraint,
objectKey, uploadId, partNumber, bucketName, log,
(err, partInfo) => {
if (err) {
log.error('error putting part to data backend', {
error: err,
method:
streamingV4Params, stream, size, objectLocationConstraint,
objectKey, uploadId, partNumber, bucketName, log,
(err, partInfo) => {
if (err) {
log.error('error putting part to data backend', {
error: err,
method:
'objectPutPart::multipleBackendGateway.uploadPart',
});
return callback(err);
}
return callback(null, partInfo, objectLocationConstraint);
});
});
return callback(err);
}
return callback(null, partInfo, objectLocationConstraint);
});
}
return callback();
}
@ -688,7 +688,7 @@ class DataWrapper {
objectTagging(method, objectKey, bucket, objectMD, log, callback) {
if (this.config.backends.data === 'multiple') {
return this.client.objectTagging(method, objectKey,
bucket, objectMD, log, err => callback(err));
bucket, objectMD, log, err => callback(err));
}
return callback();
}
@ -716,7 +716,7 @@ class DataWrapper {
return callback();
}
/**
/**
* _putForCopy - put used for copying object
* @param {object} cipherBundle - cipher bundle that encrypt the data
* @param {object} stream - stream containing the data
@ -735,34 +735,34 @@ class DataWrapper {
* @returns {function} cb - callback
*/
_putForCopy(cipherBundle, stream, part, dataStoreContext, destBackendInfo,
log, cb) {
log, cb) {
return this.put(cipherBundle, stream, part.size, dataStoreContext,
destBackendInfo, log, (error, partRetrievalInfo) => {
if (error) {
return cb(error);
}
const partResult = {
key: partRetrievalInfo.key,
dataStoreName: partRetrievalInfo
.dataStoreName,
dataStoreType: partRetrievalInfo
.dataStoreType,
start: part.start,
size: part.size,
};
if (cipherBundle) {
partResult.cryptoScheme = cipherBundle.cryptoScheme;
partResult.cipheredDataKey = cipherBundle.cipheredDataKey;
}
if (part.dataStoreETag) {
partResult.dataStoreETag = part.dataStoreETag;
}
if (partRetrievalInfo.dataStoreVersionId) {
partResult.dataStoreVersionId =
destBackendInfo, log, (error, partRetrievalInfo) => {
if (error) {
return cb(error);
}
const partResult = {
key: partRetrievalInfo.key,
dataStoreName: partRetrievalInfo
.dataStoreName,
dataStoreType: partRetrievalInfo
.dataStoreType,
start: part.start,
size: part.size,
};
if (cipherBundle) {
partResult.cryptoScheme = cipherBundle.cryptoScheme;
partResult.cipheredDataKey = cipherBundle.cipheredDataKey;
}
if (part.dataStoreETag) {
partResult.dataStoreETag = part.dataStoreETag;
}
if (partRetrievalInfo.dataStoreVersionId) {
partResult.dataStoreVersionId =
partRetrievalInfo.dataStoreVersionId;
}
return cb(null, partResult);
});
}
return cb(null, partResult);
});
}
/**
@ -783,17 +783,17 @@ class DataWrapper {
* @returns {function} cb - callback
*/
_dataCopyPut(serverSideEncryption, stream, part, dataStoreContext,
destBackendInfo, log, cb) {
destBackendInfo, log, cb) {
if (serverSideEncryption) {
return this.kms.createCipherBundle(serverSideEncryption, log,
(err, cipherBundle) => {
if (err) {
log.debug('error getting cipherBundle');
return cb(errors.InternalError);
}
return this._putForCopy(cipherBundle, stream, part,
dataStoreContext, destBackendInfo, log, cb);
});
(err, cipherBundle) => {
if (err) {
log.debug('error getting cipherBundle');
return cb(errors.InternalError);
}
return this._putForCopy(cipherBundle, stream, part,
dataStoreContext, destBackendInfo, log, cb);
});
}
// Copied object is not encrypted so just put it
// without a cipherBundle
@ -802,7 +802,7 @@ class DataWrapper {
}
_dataCopyPutPart(request, serverSideEncryption, stream, part,
dataStoreContext, destBackendInfo, locations, log, cb) {
dataStoreContext, destBackendInfo, locations, log, cb) {
const numberPartSize = Number.parseInt(part.size, 10);
const partNumber = Number.parseInt(request.query.partNumber, 10);
const uploadId = request.query.uploadId;
@ -811,100 +811,100 @@ class DataWrapper {
const destLocationConstraintName = destBackendInfo
.getControllingLocationConstraint();
if (externalBackends[this.config.locationConstraints
[destLocationConstraintName].type]) {
[destLocationConstraintName].type]) {
return this.client.uploadPart(null, null, stream,
numberPartSize, destLocationConstraintName, destObjectKey, uploadId,
partNumber, destBucketName, log, (err, partInfo) => {
if (err) {
log.error('error putting part to AWS', {
error: err,
method: 'objectPutCopyPart::' +
numberPartSize, destLocationConstraintName, destObjectKey, uploadId,
partNumber, destBucketName, log, (err, partInfo) => {
if (err) {
log.error('error putting part to AWS', {
error: err,
method: 'objectPutCopyPart::' +
'multipleBackendGateway.uploadPart',
});
return cb(errors.ServiceUnavailable);
}
// skip to end of waterfall because don't need to store
// part metadata
if (partInfo && partInfo.dataStoreType === 'aws_s3') {
});
return cb(errors.ServiceUnavailable);
}
// skip to end of waterfall because don't need to store
// part metadata
if (partInfo && partInfo.dataStoreType === 'aws_s3') {
// if data backend handles MPU, skip to end of waterfall
const partResult = {
dataStoreETag: partInfo.dataStoreETag,
};
locations.push(partResult);
return cb(skipError, partInfo.dataStoreETag);
} else if (partInfo && partInfo.dataStoreType === 'azure') {
const partResult = {
key: partInfo.key,
dataStoreName: partInfo.dataStoreName,
dataStoreETag: partInfo.dataStoreETag,
size: numberPartSize,
numberSubParts: partInfo.numberSubParts,
partNumber: partInfo.partNumber,
};
locations.push(partResult);
return cb();
} else if (partInfo && partInfo.dataStoreType === 'gcp') {
const partResult = {
key: partInfo.key,
dataStoreName: partInfo.dataStoreName,
dataStoreETag: partInfo.dataStoreETag,
size: numberPartSize,
partNumber: partInfo.partNumber,
};
locations.push(partResult);
return cb();
}
return cb(skipError);
});
const partResult = {
dataStoreETag: partInfo.dataStoreETag,
};
locations.push(partResult);
return cb(skipError, partInfo.dataStoreETag);
} else if (partInfo && partInfo.dataStoreType === 'azure') {
const partResult = {
key: partInfo.key,
dataStoreName: partInfo.dataStoreName,
dataStoreETag: partInfo.dataStoreETag,
size: numberPartSize,
numberSubParts: partInfo.numberSubParts,
partNumber: partInfo.partNumber,
};
locations.push(partResult);
return cb();
} else if (partInfo && partInfo.dataStoreType === 'gcp') {
const partResult = {
key: partInfo.key,
dataStoreName: partInfo.dataStoreName,
dataStoreETag: partInfo.dataStoreETag,
size: numberPartSize,
partNumber: partInfo.partNumber,
};
locations.push(partResult);
return cb();
}
return cb(skipError);
});
}
if (serverSideEncryption) {
return this.kms.createCipherBundle(serverSideEncryption, log,
(err, cipherBundle) => {
if (err) {
log.debug('error getting cipherBundle', { error: err });
return cb(errors.InternalError);
(err, cipherBundle) => {
if (err) {
log.debug('error getting cipherBundle', { error: err });
return cb(errors.InternalError);
}
return this.put(cipherBundle, stream, numberPartSize,
dataStoreContext, destBackendInfo, log,
(error, partRetrievalInfo, hashedStream) => {
if (error) {
log.debug('error putting encrypted part', { error });
return cb(error);
}
return this.put(cipherBundle, stream, numberPartSize,
dataStoreContext, destBackendInfo, log,
(error, partRetrievalInfo, hashedStream) => {
if (error) {
log.debug('error putting encrypted part', { error });
return cb(error);
}
const partResult = {
key: partRetrievalInfo.key,
dataStoreName: partRetrievalInfo.dataStoreName,
dataStoreETag: hashedStream.completedHash,
// Do not include part start here since will change in
// final MPU object
size: numberPartSize,
sseCryptoScheme: cipherBundle.cryptoScheme,
sseCipheredDataKey: cipherBundle.cipheredDataKey,
sseAlgorithm: cipherBundle.algorithm,
sseMasterKeyId: cipherBundle.masterKeyId,
};
locations.push(partResult);
return cb();
});
const partResult = {
key: partRetrievalInfo.key,
dataStoreName: partRetrievalInfo.dataStoreName,
dataStoreETag: hashedStream.completedHash,
// Do not include part start here since will change in
// final MPU object
size: numberPartSize,
sseCryptoScheme: cipherBundle.cryptoScheme,
sseCipheredDataKey: cipherBundle.cipheredDataKey,
sseAlgorithm: cipherBundle.algorithm,
sseMasterKeyId: cipherBundle.masterKeyId,
};
locations.push(partResult);
return cb();
});
});
}
// Copied object is not encrypted so just put it
// without a cipherBundle
return this.put(null, stream, numberPartSize, dataStoreContext,
destBackendInfo, log, (error, partRetrievalInfo, hashedStream) => {
if (error) {
log.debug('error putting object part', { error });
return cb(error);
}
const partResult = {
key: partRetrievalInfo.key,
dataStoreName: partRetrievalInfo.dataStoreName,
dataStoreETag: hashedStream.completedHash,
size: numberPartSize,
};
locations.push(partResult);
return cb();
});
destBackendInfo, log, (error, partRetrievalInfo, hashedStream) => {
if (error) {
log.debug('error putting object part', { error });
return cb(error);
}
const partResult = {
key: partRetrievalInfo.key,
dataStoreName: partRetrievalInfo.dataStoreName,
dataStoreETag: hashedStream.completedHash,
size: numberPartSize,
};
locations.push(partResult);
return cb();
});
}
_put(cipherBundle, value, valueSize, keyContext, backendInfo, log, cb) {
@ -930,17 +930,17 @@ class DataWrapper {
/* eslint-disable no-param-reassign */
keyContext.cipherBundle = cipherBundle;
return this.client.put(hashedStream, valueSize, keyContext,
backendInfo, log.getSerializedUids(), (err, dataRetrievalInfo) => {
if (err) {
log.error('put error from datastore',
backendInfo, log.getSerializedUids(), (err, dataRetrievalInfo) => {
if (err) {
log.error('put error from datastore',
{ error: err, implName: this.implName });
if (err.httpCode === 408) {
return cb(errors.IncompleteBody);
}
return cb(errors.ServiceUnavailable);
if (err.httpCode === 408) {
return cb(errors.IncompleteBody);
}
return cb(null, dataRetrievalInfo, hashedStream);
});
return cb(errors.ServiceUnavailable);
}
return cb(null, dataRetrievalInfo, hashedStream);
});
}
/* eslint-enable no-param-reassign */
let writeStream = hashedStream;
@ -950,21 +950,21 @@ class DataWrapper {
}
return this.client.put(writeStream, valueSize, keyContext,
log.getSerializedUids(), (err, key) => {
if (err) {
log.error('put error from datastore',
log.getSerializedUids(), (err, key) => {
if (err) {
log.error('put error from datastore',
{ error: err, implName: this.implName });
if (err.httpCode === 408) {
return cb(errors.IncompleteBody);
}
return cb(errors.InternalError);
if (err.httpCode === 408) {
return cb(errors.IncompleteBody);
}
const dataRetrievalInfo = {
key,
dataStoreName: this.implName,
};
return cb(null, dataRetrievalInfo, hashedStream);
});
return cb(errors.InternalError);
}
const dataRetrievalInfo = {
key,
dataStoreName: this.implName,
};
return cb(null, dataRetrievalInfo, hashedStream);
});
}
/**
@ -982,25 +982,25 @@ class DataWrapper {
return cb(errors.InternalError);
}
return this.client.delete(objectGetInfo, log.getSerializedUids(),
err => {
if (err) {
if (err.ObjNotFound) {
log.info('no such key in datastore', {
objectGetInfo,
implName: this.implName,
moreRetries: 'no',
});
return cb(err);
}
log.error('delete error from datastore', {
error: err,
err => {
if (err) {
if (err.ObjNotFound) {
log.info('no such key in datastore', {
objectGetInfo,
implName: this.implName,
moreRetries: 'yes',
moreRetries: 'no',
});
return this._retryDelete(objectGetInfo, log, count + 1, cb);
return cb(err);
}
return cb();
});
log.error('delete error from datastore', {
error: err,
implName: this.implName,
moreRetries: 'yes',
});
return this._retryDelete(objectGetInfo, log, count + 1, cb);
}
return cb();
});
}
// This check is done because on a put, complete mpu or copy request to

View File

@ -30,7 +30,7 @@ function parseLC(config, vault) {
if (locationObj.details.connector.sproxyd) {
clients[location] = new Sproxy({
bootstrap: locationObj.details.connector
.sproxyd.bootstrap,
.sproxyd.bootstrap,
// Might be undefined which is ok since there is a default
// set in sproxydclient if chordCos is undefined
chordCos: locationObj.details.connector.sproxyd.chordCos,
@ -60,7 +60,7 @@ function parseLC(config, vault) {
// keepalive config
const httpAgentConfig =
config.externalBackends[locationObj.type].httpAgent;
// max sockets is infinity by default and expressed as null
// max sockets is infinity by default and expressed as null
if (httpAgentConfig.maxSockets === null) {
httpAgentConfig.maxSockets = undefined;
}
@ -131,7 +131,7 @@ function parseLC(config, vault) {
const azureStorageEndpoint = config.getAzureEndpoint(location);
const proxyParams =
backendUtils.proxyCompareUrl(azureStorageEndpoint) ?
{} : config.outboundProxy;
{} : config.outboundProxy;
const azureStorageCredentials =
config.getAzureStorageCredentials(location);
clients[location] = new AzureClient({

View File

@ -52,26 +52,26 @@ class MultipleBackendGateway {
}
}
return client.put(writeStream, size, keyContext, reqUids,
(err, key, dataStoreVersionId, dataStoreSize, dataStoreMD5) => {
const log = createLogger(reqUids);
log.debug('put to location', { controllingLocationConstraint });
if (err) {
log.error('error from datastore',
{ error: err, dataStoreType: client.clientType });
return callback(errors.ServiceUnavailable);
}
const dataRetrievalInfo = {
key,
dataStoreName: controllingLocationConstraint,
dataStoreType: client.clientType,
dataStoreVersionId,
dataStoreSize,
dataStoreMD5,
};
return callback(null, dataRetrievalInfo);
(err, key, dataStoreVersionId, dataStoreSize, dataStoreMD5) => {
const log = createLogger(reqUids);
log.debug('put to location', { controllingLocationConstraint });
if (err) {
log.error('error from datastore',
{ error: err, dataStoreType: client.clientType });
return callback(errors.ServiceUnavailable);
}
const dataRetrievalInfo = {
key,
dataStoreName: controllingLocationConstraint,
dataStoreType: client.clientType,
dataStoreVersionId,
dataStoreSize,
dataStoreMD5,
};
return callback(null, dataRetrievalInfo);
// sproxyd accepts keyschema, send as null so sproxyd generates key
// send metadata as param for AzureClient in Arsenal
}, null, this.metadata);
}, null, this.metadata);
}
head(objectGetInfoArr, reqUids, callback) {
@ -166,14 +166,14 @@ class MultipleBackendGateway {
}, () => {
async.parallel([
next => checkExternalBackend(
this.clients, awsArray, 'aws_s3', flightCheckOnStartUp,
externalBackendHealthCheckInterval, next),
this.clients, awsArray, 'aws_s3', flightCheckOnStartUp,
externalBackendHealthCheckInterval, next),
next => checkExternalBackend(
this.clients, azureArray, 'azure', flightCheckOnStartUp,
externalBackendHealthCheckInterval, next),
this.clients, azureArray, 'azure', flightCheckOnStartUp,
externalBackendHealthCheckInterval, next),
next => checkExternalBackend(
this.clients, gcpArray, 'gcp', flightCheckOnStartUp,
externalBackendHealthCheckInterval, next),
this.clients, gcpArray, 'gcp', flightCheckOnStartUp,
externalBackendHealthCheckInterval, next),
], (errNull, externalResp) => {
const externalLocResults = [];
externalResp.forEach(resp => externalLocResults.push(...resp));
@ -185,19 +185,19 @@ class MultipleBackendGateway {
}
createMPU(key, metaHeaders, bucketName, websiteRedirectHeader,
location, contentType, cacheControl, contentDisposition,
contentEncoding, tagging, log, cb) {
location, contentType, cacheControl, contentDisposition,
contentEncoding, tagging, log, cb) {
const client = this.clients[location];
if (client.clientType === 'aws_s3' || client.clientType === 'gcp') {
return client.createMPU(key, metaHeaders, bucketName,
websiteRedirectHeader, contentType, cacheControl,
contentDisposition, contentEncoding, tagging, log, cb);
websiteRedirectHeader, contentType, cacheControl,
contentDisposition, contentEncoding, tagging, log, cb);
}
return cb();
}
uploadPart(request, streamingV4Params, stream, size, location, key,
uploadId, partNumber, bucketName, log, cb) {
uploadId, partNumber, bucketName, log, cb) {
const client = this.clients[location];
if (client.uploadPart) {
@ -206,29 +206,29 @@ class MultipleBackendGateway {
return cb(err);
}
return client.uploadPart(request, streamingV4Params, stream,
size, key, uploadId, partNumber, bucketName, log,
(err, partInfo) => {
if (err) {
size, key, uploadId, partNumber, bucketName, log,
(err, partInfo) => {
if (err) {
// if error putting part, counter should be decremented
return this.locStorageCheckFn(location, -size, log,
error => {
if (error) {
log.error('Error decrementing location ' +
return this.locStorageCheckFn(location, -size, log,
error => {
if (error) {
log.error('Error decrementing location ' +
'metric following object PUT failure',
{ error: error.message });
}
return cb(err);
});
}
return cb(null, partInfo);
});
{ error: error.message });
}
return cb(err);
});
}
return cb(null, partInfo);
});
});
}
return cb();
}
listParts(key, uploadId, location, bucketName, partNumberMarker, maxParts,
log, cb) {
log, cb) {
const client = this.clients[location];
if (client.listParts) {
@ -239,7 +239,7 @@ class MultipleBackendGateway {
}
completeMPU(key, uploadId, location, jsonList, mdInfo, bucketName,
userMetadata, contentSettings, tagging, log, cb) {
userMetadata, contentSettings, tagging, log, cb) {
const client = this.clients[location];
if (client.completeMPU) {
const args = [jsonList, mdInfo, key, uploadId, bucketName];
@ -291,40 +291,40 @@ class MultipleBackendGateway {
// NOTE: using copyObject only if copying object from one external
// backend to the same external backend
copyObject(request, destLocationConstraintName, externalSourceKey,
sourceLocationConstraintName, storeMetadataParams, config, log, cb) {
sourceLocationConstraintName, storeMetadataParams, config, log, cb) {
const client = this.clients[destLocationConstraintName];
if (client.copyObject) {
return this.locStorageCheckFn(destLocationConstraintName,
storeMetadataParams.size, log, err => {
storeMetadataParams.size, log, err => {
if (err) {
cb(err);
}
return client.copyObject(request, destLocationConstraintName,
externalSourceKey, sourceLocationConstraintName,
storeMetadataParams, config, log,
(err, key, dataStoreVersionId) => {
const dataRetrievalInfo = {
key,
dataStoreName: destLocationConstraintName,
dataStoreType: client.clientType,
dataStoreVersionId,
};
if (err) {
cb(err);
}
return client.copyObject(request, destLocationConstraintName,
externalSourceKey, sourceLocationConstraintName,
storeMetadataParams, config, log,
(err, key, dataStoreVersionId) => {
const dataRetrievalInfo = {
key,
dataStoreName: destLocationConstraintName,
dataStoreType: client.clientType,
dataStoreVersionId,
};
if (err) {
// if error copying obj, counter should be decremented
return this.locStorageCheckFn(
destLocationConstraintName, -storeMetadataParams.size,
log, error => {
if (error) {
log.error('Error decrementing location ' +
// if error copying obj, counter should be decremented
return this.locStorageCheckFn(
destLocationConstraintName, -storeMetadataParams.size,
log, error => {
if (error) {
log.error('Error decrementing location ' +
'metric following object PUT failure',
{ error: error.message });
}
return cb(err);
});
{ error: error.message });
}
return cb(null, dataRetrievalInfo);
return cb(err);
});
}
return cb(null, dataRetrievalInfo);
});
});
}
return cb(errors.NotImplemented
.customizeDescription('Can not copy object from ' +
@ -332,15 +332,15 @@ class MultipleBackendGateway {
}
uploadPartCopy(request, location, awsSourceKey,
sourceLocationConstraintName, config, log, cb) {
sourceLocationConstraintName, config, log, cb) {
const client = this.clients[location];
if (client.uploadPartCopy) {
return client.uploadPartCopy(request, awsSourceKey,
sourceLocationConstraintName, config,
log, cb);
sourceLocationConstraintName, config,
log, cb);
}
return cb(errors.NotImplemented.customizeDescription(
'Can not copy object from ' +
'Can not copy object from ' +
`${client.clientType} to ${client.clientType}`));
}
@ -348,7 +348,7 @@ class MultipleBackendGateway {
const client = this.clients[location];
if (client.protectAzureBlocks) {
return client.protectAzureBlocks(this.metadata, bucketName,
objectKey, location, log, cb);
objectKey, location, log, cb);
}
return cb();
}

View File

@ -11,7 +11,7 @@ const { createLogger, logHelper, removeQuotes, trimXMetaPrefix } =
const missingVerIdInternalError = errors.InternalError.customizeDescription(
'Invalid state. Please ensure versioning is enabled ' +
'in AWS for the location constraint and try again.',
'in AWS for the location constraint and try again.'
);
class AwsClient {
@ -36,35 +36,35 @@ class AwsClient {
// this request implicitly updates the endpoint for the location
// the following code explcitly sets it to avoid surprises
this._client.getBucketLocation({ Bucket: this._awsBucketName },
(err, res) => {
if (err && err.code !== 'AuthorizationHeaderMalformed') {
this._logger.error('error during setup', {
error: err,
method: 'AwsClient.setup',
});
return cb(err);
}
let region;
if (err && err.code === 'AuthorizationHeaderMalformed') {
(err, res) => {
if (err && err.code !== 'AuthorizationHeaderMalformed') {
this._logger.error('error during setup', {
error: err,
method: 'AwsClient.setup',
});
return cb(err);
}
let region;
if (err && err.code === 'AuthorizationHeaderMalformed') {
// set regional endpoint
region = err.region;
} else if (res) {
region = res.LocationConstraint;
}
this._client.config.update({ region });
region = err.region;
} else if (res) {
region = res.LocationConstraint;
}
this._client.config.update({ region });
const isAWS = this._s3Params.endpoint.endsWith('amazonaws.com');
if (region && isAWS) {
const endpoint = `s3.${region}.amazonaws.com`;
this._logger.debug('setting regional endpoint', {
method: 'AwsClient.setup',
region,
endpoint,
});
this._client.endpoint = new AWS.Endpoint(endpoint);
}
return cb();
});
const isAWS = this._s3Params.endpoint.endsWith('amazonaws.com');
if (region && isAWS) {
const endpoint = `s3.${region}.amazonaws.com`;
this._logger.debug('setting regional endpoint', {
method: 'AwsClient.setup',
region,
endpoint,
});
this._client.endpoint = new AWS.Endpoint(endpoint);
}
return cb();
});
}
_createAwsKey(requestBucketName, requestObjectKey,
@ -84,23 +84,23 @@ class AwsClient {
put(stream, size, keyContext, reqUids, callback) {
const awsKey = this._createAwsKey(keyContext.bucketName,
keyContext.objectKey, this._bucketMatch);
keyContext.objectKey, this._bucketMatch);
const metaHeaders = trimXMetaPrefix(keyContext.metaHeaders);
const log = createLogger(reqUids);
const putCb = (err, data) => {
if (err) {
logHelper(log, 'error', 'err from data backend',
err, this._dataStoreName, this.clientType);
err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`),
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
if (!data.VersionId && this._supportsVersioning) {
logHelper(log, 'error', 'missing version id for data ' +
'backend object', missingVerIdInternalError,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(missingVerIdInternalError);
}
const dataStoreVersionId = data.VersionId;
@ -180,8 +180,8 @@ class AwsClient {
Range: range ? `bytes=${range[0]}-${range[1]}` : null,
}).on('success', response => {
log.trace(`${this.type} GET request response headers`,
{ responseHeaders: response.httpResponse.headers,
backendType: this.clientType });
{ responseHeaders: response.httpResponse.headers,
backendType: this.clientType });
});
const stream = request.createReadStream();
@ -202,8 +202,8 @@ class AwsClient {
logLevel = 'error';
}
logHelper(log, logLevel,
`error streaming data from ${this.type}`,
err, this._dataStoreName, this.clientType);
`error streaming data from ${this.type}`,
err, this._dataStoreName, this.clientType);
});
// Always call the callback asynchronously: the caller may
// destroy the stream with destroy(), which MUST be
@ -232,8 +232,8 @@ class AwsClient {
return callback();
}
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`),
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
return callback();
@ -243,39 +243,39 @@ class AwsClient {
healthcheck(location, callback) {
const awsResp = {};
this._client.headBucket({ Bucket: this._awsBucketName },
err => {
err => {
/* eslint-disable no-param-reassign */
if (err) {
awsResp[location] = { error: err, external: true };
return callback(null, awsResp);
}
if (!this._supportsVersioning) {
awsResp[location] = {
message: 'Congrats! You own the bucket',
};
return callback(null, awsResp);
}
return this._client.getBucketVersioning({
Bucket: this._awsBucketName },
(err, data) => {
if (err) {
awsResp[location] = { error: err, external: true };
return callback(null, awsResp);
}
if (!this._supportsVersioning) {
} else if (!data.Status ||
data.Status === 'Suspended') {
awsResp[location] = {
versioningStatus: data.Status,
error: 'Versioning must be enabled',
external: true,
};
} else {
awsResp[location] = {
versioningStatus: data.Status,
message: 'Congrats! You own the bucket',
};
return callback(null, awsResp);
}
return this._client.getBucketVersioning({
Bucket: this._awsBucketName },
(err, data) => {
if (err) {
awsResp[location] = { error: err, external: true };
} else if (!data.Status ||
data.Status === 'Suspended') {
awsResp[location] = {
versioningStatus: data.Status,
error: 'Versioning must be enabled',
external: true,
};
} else {
awsResp[location] = {
versioningStatus: data.Status,
message: 'Congrats! You own the bucket',
};
}
return callback(null, awsResp);
});
return callback(null, awsResp);
});
});
}
createMPU(key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
@ -304,10 +304,10 @@ class AwsClient {
return this._client.createMultipartUpload(params, (err, mpuResObj) => {
if (err) {
logHelper(log, 'error', 'err from data backend',
err, this._dataStoreName, this.clientType);
err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`),
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
return callback(null, mpuResObj);
@ -315,7 +315,7 @@ class AwsClient {
}
uploadPart(request, streamingV4Params, stream, size, key, uploadId,
partNumber, bucketName, log, callback) {
partNumber, bucketName, log, callback) {
let hashedStream = stream;
if (request) {
const partStream = prepareStream(request, streamingV4Params,
@ -335,7 +335,7 @@ class AwsClient {
'on uploadPart', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`),
`${this.type}: ${err.message}`)
);
}
// Because we manually add quotes to ETag later, remove quotes here
@ -352,7 +352,7 @@ class AwsClient {
}
listParts(key, uploadId, bucketName, partNumberMarker, maxParts, log,
callback) {
callback) {
const awsBucket = this._awsBucketName;
const awsKey = this._createAwsKey(bucketName, key, this._bucketMatch);
const params = { Bucket: awsBucket, Key: awsKey, UploadId: uploadId,
@ -360,10 +360,10 @@ class AwsClient {
return this._client.listParts(params, (err, partList) => {
if (err) {
logHelper(log, 'error', 'err from data backend on listPart',
err, this._dataStoreName, this.clientType);
err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`),
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
// build storedParts object to mimic Scality S3 backend returns
@ -424,47 +424,47 @@ class AwsClient {
};
const completeObjData = { key: awsKey };
return this._client.completeMultipartUpload(mpuParams,
(err, completeMpuRes) => {
if (err) {
if (mpuError[err.code]) {
logHelper(log, 'trace', 'err from data backend on ' +
(err, completeMpuRes) => {
if (err) {
if (mpuError[err.code]) {
logHelper(log, 'trace', 'err from data backend on ' +
'completeMPU', err, this._dataStoreName, this.clientType);
return callback(errors[err.code]);
}
logHelper(log, 'error', 'err from data backend on ' +
'completeMPU', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`),
);
return callback(errors[err.code]);
}
if (!completeMpuRes.VersionId && this._supportsVersioning) {
logHelper(log, 'error', 'missing version id for data ' +
logHelper(log, 'error', 'err from data backend on ' +
'completeMPU', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
if (!completeMpuRes.VersionId && this._supportsVersioning) {
logHelper(log, 'error', 'missing version id for data ' +
'backend object', missingVerIdInternalError,
this._dataStoreName, this.clientType);
return callback(missingVerIdInternalError);
}
// need to get content length of new object to store
// in our metadata
return this._client.headObject({ Bucket: awsBucket, Key: awsKey },
(err, objHeaders) => {
if (err) {
logHelper(log, 'trace', 'err from data backend on ' +
return callback(missingVerIdInternalError);
}
// need to get content length of new object to store
// in our metadata
return this._client.headObject({ Bucket: awsBucket, Key: awsKey },
(err, objHeaders) => {
if (err) {
logHelper(log, 'trace', 'err from data backend on ' +
'headObject', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`),
);
}
// remove quotes from eTag because they're added later
completeObjData.eTag = completeMpuRes.ETag
.substring(1, completeMpuRes.ETag.length - 1);
completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
completeObjData.contentLength =
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
// remove quotes from eTag because they're added later
completeObjData.eTag = completeMpuRes.ETag
.substring(1, completeMpuRes.ETag.length - 1);
completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
completeObjData.contentLength =
Number.parseInt(objHeaders.ContentLength, 10);
return callback(null, completeObjData);
});
return callback(null, completeObjData);
});
});
}
abortMPU(key, uploadId, bucketName, log, callback) {
@ -480,8 +480,8 @@ class AwsClient {
'using the same uploadId.', err, this._dataStoreName,
this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`),
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
return callback();
@ -507,10 +507,10 @@ class AwsClient {
if (err) {
logHelper(log, 'error', 'error from data backend on ' +
'putObjectTagging', err,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`),
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
return callback();
@ -532,19 +532,19 @@ class AwsClient {
'deleteObjectTagging', err,
this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`),
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
return callback();
});
}
copyObject(request, destLocationConstraintName, sourceKey,
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
const destBucketName = request.bucketName;
const destObjectKey = request.objectKey;
const destAwsKey = this._createAwsKey(destBucketName, destObjectKey,
this._bucketMatch);
this._bucketMatch);
const sourceAwsBucketName =
config.getAwsBucketName(sourceLocationConstraintName);
@ -569,15 +569,15 @@ class AwsClient {
`${sourceAwsBucketName} ${this.type} bucket`, err,
this._dataStoreName, this.clientType);
return callback(errors.AccessDenied
.customizeDescription('Error: Unable to access ' +
`${sourceAwsBucketName} ${this.type} bucket`),
.customizeDescription('Error: Unable to access ' +
`${sourceAwsBucketName} ${this.type} bucket`)
);
}
logHelper(log, 'error', 'error from data backend on ' +
'copyObject', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`),
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
if (!copyResult.VersionId && this._supportsVersioning) {
@ -590,7 +590,7 @@ class AwsClient {
if (err || !data.VersionId) {
logHelper(log, 'error', 'missing version id for data ' +
'backend object', missingVerIdInternalError,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(missingVerIdInternalError);
}
return callback(null, destAwsKey, data.VersionId);
@ -600,11 +600,11 @@ class AwsClient {
});
}
uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName,
config, log, callback) {
config, log, callback) {
const destBucketName = request.bucketName;
const destObjectKey = request.objectKey;
const destAwsKey = this._createAwsKey(destBucketName, destObjectKey,
this._bucketMatch);
this._bucketMatch);
const sourceAwsBucketName =
config.getAwsBucketName(sourceLocationConstraintName);
@ -628,15 +628,15 @@ class AwsClient {
`${sourceAwsBucketName} AWS bucket`, err,
this._dataStoreName, this.clientType);
return callback(errors.AccessDenied
.customizeDescription('Error: Unable to access ' +
`${sourceAwsBucketName} AWS bucket`),
.customizeDescription('Error: Unable to access ' +
`${sourceAwsBucketName} AWS bucket`)
);
}
logHelper(log, 'error', 'error from data backend on ' +
'uploadPartCopy', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`),
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
const eTag = removeQuotes(res.CopyPartResult.ETag);

View File

@ -51,7 +51,7 @@ class AzureClient {
if (log) {
log.error('error thrown by Azure Storage Client Library',
{ error: err.message, stack: err.stack, s3Method,
azureMethod, dataStoreName: this._dataStoreName });
azureMethod, dataStoreName: this._dataStoreName });
}
cb(error.customizeDescription('Error from Azure ' +
`method: ${azureMethod} on ${s3Method} S3 call: ` +
@ -82,7 +82,7 @@ class AzureClient {
// same key name. If it does, do not allow put or delete because Azure
// will delete all blocks with same key name
protectAzureBlocks(metadata, bucketName, objectKey, dataStoreName,
log, cb) {
log, cb) {
const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`;
const splitter = constants.splitter;
const listingParams = {
@ -93,23 +93,23 @@ class AzureClient {
};
return metadata.listMultipartUploads(mpuBucketName, listingParams,
log, (err, mpuList) => {
if (err && !err.NoSuchBucket) {
log.error('Error listing MPUs for Azure delete',
{ error: err, dataStoreName });
return cb(errors.ServiceUnavailable);
}
if (mpuList && mpuList.Uploads && mpuList.Uploads.length > 0) {
const error = errors.MPUinProgress;
log.error('Error: cannot put/delete object to Azure with ' +
log, (err, mpuList) => {
if (err && !err.NoSuchBucket) {
log.error('Error listing MPUs for Azure delete',
{ error: err, dataStoreName });
return cb(errors.ServiceUnavailable);
}
if (mpuList && mpuList.Uploads && mpuList.Uploads.length > 0) {
const error = errors.MPUinProgress;
log.error('Error: cannot put/delete object to Azure with ' +
'same key name as ongoing MPU on Azure',
{ error, dataStoreName });
return cb(error);
}
// If listMultipartUploads returns a NoSuchBucket error or the
// mpu list is empty, there are no conflicting MPUs, so continue
return cb();
});
return cb(error);
}
// If listMultipartUploads returns a NoSuchBucket error or the
// mpu list is empty, there are no conflicting MPUs, so continue
return cb();
});
}
toObjectGetInfo(objectKey, bucketName) {
@ -123,52 +123,52 @@ class AzureClient {
const log = createLogger(reqUids);
// before blob is put, make sure there is no ongoing MPU with same key
this.protectAzureBlocks(metadata, keyContext.bucketName,
keyContext.objectKey, this._dataStoreName, log, err => {
keyContext.objectKey, this._dataStoreName, log, err => {
// if error returned, there is ongoing MPU, so do not put
if (err) {
return callback(err.customizeDescription(
`Error putting object to Azure: ${err.message}`));
}
const azureKey = this._createAzureKey(keyContext.bucketName,
keyContext.objectKey, this._bucketMatch);
const options = {
metadata: translateAzureMetaHeaders(keyContext.metaHeaders,
keyContext.tagging),
contentSettings: {
contentType: keyContext.contentType || undefined,
cacheControl: keyContext.cacheControl || undefined,
contentDisposition: keyContext.contentDisposition ||
if (err) {
return callback(err.customizeDescription(
`Error putting object to Azure: ${err.message}`));
}
const azureKey = this._createAzureKey(keyContext.bucketName,
keyContext.objectKey, this._bucketMatch);
const options = {
metadata: translateAzureMetaHeaders(keyContext.metaHeaders,
keyContext.tagging),
contentSettings: {
contentType: keyContext.contentType || undefined,
cacheControl: keyContext.cacheControl || undefined,
contentDisposition: keyContext.contentDisposition ||
undefined,
contentEncoding: keyContext.contentEncoding || undefined,
},
};
if (size === 0) {
return this._errorWrapper('put', 'createBlockBlobFromText',
[this._azureContainerName, azureKey, '', options,
err => {
if (err) {
logHelper(log, 'error', 'err from Azure PUT data ' +
contentEncoding: keyContext.contentEncoding || undefined,
},
};
if (size === 0) {
return this._errorWrapper('put', 'createBlockBlobFromText',
[this._azureContainerName, azureKey, '', options,
err => {
if (err) {
logHelper(log, 'error', 'err from Azure PUT data ' +
'backend', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`Azure: ${err.message}`));
}
return callback(null, azureKey);
}], log, callback);
}
return this._errorWrapper('put', 'createBlockBlobFromStream',
[this._azureContainerName, azureKey, stream, size, options,
err => {
if (err) {
logHelper(log, 'error', 'err from Azure PUT data ' +
}
return callback(null, azureKey);
}], log, callback);
}
return this._errorWrapper('put', 'createBlockBlobFromStream',
[this._azureContainerName, azureKey, stream, size, options,
err => {
if (err) {
logHelper(log, 'error', 'err from Azure PUT data ' +
'backend', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`Azure: ${err.message}`));
}
return callback(null, azureKey);
}], log, callback);
});
}
return callback(null, azureKey);
}], log, callback);
});
}
head(objectGetInfo, reqUids, callback) {
@ -176,25 +176,25 @@ class AzureClient {
const { key, azureStreamingOptions } = objectGetInfo;
return this._errorWrapper('head', 'getBlobProperties',
[this._azureContainerName, key, azureStreamingOptions,
(err, data) => {
if (err) {
let logLevel;
let retError;
if (err.code === 'NotFound') {
logLevel = 'info';
retError = errors.LocationNotFound;
} else {
logLevel = 'error';
retError = errors.ServiceUnavailable
.customizeDescription(
`Error returned from Azure: ${err.message}`);
}
logHelper(log, logLevel, 'err from Azure HEAD data backend',
err, this._dataStoreName);
return callback(retError);
(err, data) => {
if (err) {
let logLevel;
let retError;
if (err.code === 'NotFound') {
logLevel = 'info';
retError = errors.LocationNotFound;
} else {
logLevel = 'error';
retError = errors.ServiceUnavailable
.customizeDescription(
`Error returned from Azure: ${err.message}`);
}
return callback(null, data);
}], log, callback);
logHelper(log, logLevel, 'err from Azure HEAD data backend',
err, this._dataStoreName);
return callback(retError);
}
return callback(null, data);
}], log, callback);
}
get(objectGetInfo, range, reqUids, callback) {
@ -213,14 +213,14 @@ class AzureClient {
}
this._errorWrapper('get', 'getBlobToStream',
[this._azureContainerName, key, response, streamingOptions,
err => {
if (err) {
logHelper(log, 'error', 'err from Azure GET data backend',
err, this._dataStoreName);
return callback(errors.ServiceUnavailable);
}
return callback(null, response);
}], log, callback);
err => {
if (err) {
logHelper(log, 'error', 'err from Azure GET data backend',
err, this._dataStoreName);
return callback(errors.ServiceUnavailable);
}
return callback(null, response);
}], log, callback);
}
delete(objectGetInfo, reqUids, callback) {
@ -234,20 +234,20 @@ class AzureClient {
}
return this._errorWrapper('delete', 'deleteBlobIfExists',
[this._azureContainerName, key, options,
err => {
if (err && err.statusCode === 412) {
return callback(errors.PreconditionFailed);
}
if (err) {
const log = createLogger(reqUids);
logHelper(log, 'error', 'error deleting object from ' +
err => {
if (err && err.statusCode === 412) {
return callback(errors.PreconditionFailed);
}
if (err) {
const log = createLogger(reqUids);
logHelper(log, 'error', 'error deleting object from ' +
'Azure datastore', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`Azure: ${err.message}`));
}
return callback();
}], log, callback);
}
return callback();
}], log, callback);
}
healthcheck(location, callback, flightCheckOnStartUp) {
@ -271,7 +271,7 @@ class AzureClient {
}
uploadPart(request, streamingV4Params, partStream, size, key, uploadId,
partNumber, bucket, log, callback) {
partNumber, bucket, log, callback) {
const azureKey = this._createAzureKey(bucket, key, this._bucketMatch);
const params = { bucketName: this._azureContainerName,
partNumber, size, objectKey: azureKey, uploadId };
@ -299,27 +299,27 @@ class AzureClient {
if (size <= azureMpuUtils.maxSubPartSize) {
const errorWrapperFn = this._errorWrapper.bind(this);
return azureMpuUtils.putSinglePart(errorWrapperFn,
stream, params, this._dataStoreName, log, (err, dataStoreETag) => {
if (err) {
return callback(err);
}
dataRetrievalInfo.dataStoreETag = dataStoreETag;
return callback(null, dataRetrievalInfo);
});
}
const errorWrapperFn = this._errorWrapper.bind(this);
return azureMpuUtils.putSubParts(errorWrapperFn, stream,
params, this._dataStoreName, log, (err, dataStoreETag) => {
stream, params, this._dataStoreName, log, (err, dataStoreETag) => {
if (err) {
return callback(err);
}
dataRetrievalInfo.dataStoreETag = dataStoreETag;
return callback(null, dataRetrievalInfo);
});
}
const errorWrapperFn = this._errorWrapper.bind(this);
return azureMpuUtils.putSubParts(errorWrapperFn, stream,
params, this._dataStoreName, log, (err, dataStoreETag) => {
if (err) {
return callback(err);
}
dataRetrievalInfo.dataStoreETag = dataStoreETag;
return callback(null, dataRetrievalInfo);
});
}
completeMPU(jsonList, mdInfo, key, uploadId, bucket, metaHeaders,
contentSettings, tagging, log, callback) {
contentSettings, tagging, log, callback) {
const azureKey = this._createAzureKey(bucket, key, this._bucketMatch);
const commitList = {
UncommittedBlocks: jsonList.uncommittedBlocks || [],
@ -345,20 +345,20 @@ class AzureClient {
};
return this._errorWrapper('completeMPU', 'commitBlocks',
[this._azureContainerName, azureKey, commitList, options,
err => {
if (err) {
logHelper(log, 'error', 'err completing MPU on Azure ' +
err => {
if (err) {
logHelper(log, 'error', 'err completing MPU on Azure ' +
'datastore', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`Azure: ${err.message}`));
}
const completeObjData = {
key: azureKey,
filteredPartsObj,
};
return callback(null, completeObjData);
}], log, callback);
}
const completeObjData = {
key: azureKey,
filteredPartsObj,
};
return callback(null, completeObjData);
}], log, callback);
}
objectPutTagging(key, bucket, objectMD, log, callback) {
@ -367,14 +367,14 @@ class AzureClient {
azureMD.tags = JSON.stringify(objectMD.tags);
this._errorWrapper('objectPutTagging', 'setBlobMetadata',
[this._azureContainerName, azureKey, azureMD,
err => {
if (err) {
logHelper(log, 'error', 'err putting object tags to ' +
err => {
if (err) {
logHelper(log, 'error', 'err putting object tags to ' +
'Azure backend', err, this._dataStoreName);
return callback(errors.ServiceUnavailable);
}
return callback();
}], log, callback);
return callback(errors.ServiceUnavailable);
}
return callback();
}], log, callback);
}
objectDeleteTagging(key, bucketName, objectMD, log, callback) {
@ -382,27 +382,27 @@ class AzureClient {
const azureMD = this._getMetaHeaders(objectMD);
this._errorWrapper('objectDeleteTagging', 'setBlobMetadata',
[this._azureContainerName, azureKey, azureMD,
err => {
if (err) {
logHelper(log, 'error', 'err putting object tags to ' +
err => {
if (err) {
logHelper(log, 'error', 'err putting object tags to ' +
'Azure backend', err, this._dataStoreName);
return callback(errors.ServiceUnavailable);
}
return callback();
}], log, callback);
return callback(errors.ServiceUnavailable);
}
return callback();
}], log, callback);
}
copyObject(request, destLocationConstraintName, sourceKey,
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
const destContainerName = request.bucketName;
const destObjectKey = request.objectKey;
const destAzureKey = this._createAzureKey(destContainerName,
destObjectKey, this._bucketMatch);
destObjectKey, this._bucketMatch);
const sourceContainerName =
config.locationConstraints[sourceLocationConstraintName]
.details.azureContainerName;
.details.azureContainerName;
let options;
if (storeMetadataParams.metaHeaders) {
@ -413,7 +413,7 @@ class AzureClient {
this._errorWrapper('copyObject', 'startCopyBlob',
[`${this._azureStorageEndpoint}` +
`${sourceContainerName}/${sourceKey}`,
this._azureContainerName, destAzureKey, options,
this._azureContainerName, destAzureKey, options,
(err, res) => {
if (err) {
if (err.code === 'CannotVerifyCopySource') {
@ -421,36 +421,36 @@ class AzureClient {
`${sourceContainerName} Azure Container`, err,
this._dataStoreName);
return callback(errors.AccessDenied
.customizeDescription('Error: Unable to access ' +
`${sourceContainerName} Azure Container`),
.customizeDescription('Error: Unable to access ' +
`${sourceContainerName} Azure Container`)
);
}
logHelper(log, 'error', 'error from data backend on ' +
'copyObject', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`AWS: ${err.message}`),
.customizeDescription('Error returned from ' +
`AWS: ${err.message}`)
);
}
if (res.copy.status === 'pending') {
logHelper(log, 'error', 'Azure copy status is pending',
err, this._dataStoreName);
err, this._dataStoreName);
const copyId = res.copy.id;
this._client.abortCopyBlob(this._azureContainerName,
destAzureKey, copyId, err => {
if (err) {
logHelper(log, 'error', 'error from data backend ' +
destAzureKey, copyId, err => {
if (err) {
logHelper(log, 'error', 'error from data backend ' +
'on abortCopyBlob', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`AWS on abortCopyBlob: ${err.message}`),
);
}
return callback(errors.InvalidObjectState
.customizeDescription('Error: Azure copy status was ' +
'pending. It has been aborted successfully'),
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`AWS on abortCopyBlob: ${err.message}`)
);
});
}
return callback(errors.InvalidObjectState
.customizeDescription('Error: Azure copy status was ' +
'pending. It has been aborted successfully')
);
});
}
return callback(null, destAzureKey);
}], log, callback);

View File

@ -24,7 +24,7 @@ class MpuHelper {
const handleFunc = (fnName, params, retry, callback) => {
const timeout = backoff.duration();
return setTimeout((params, cb) =>
this.service[fnName](params, cb), timeout, params,
this.service[fnName](params, cb), timeout, params,
(err, res) => {
if (err) {
if (err.statusCode === 429 || err.code === 429) {
@ -90,53 +90,53 @@ class MpuHelper {
splitMerge(params, partList, level, callback) {
// create composition of slices from the partList array
return async.mapLimit(eachSlice.call(partList, 32),
this.service._maxConcurrent,
(infoParts, cb) => {
const mpuPartList = infoParts.Parts.map(item =>
({ PartName: item.PartName }));
const partNumber = infoParts.PartNumber;
const tmpKey =
this.service._maxConcurrent,
(infoParts, cb) => {
const mpuPartList = infoParts.Parts.map(item =>
({ PartName: item.PartName }));
const partNumber = infoParts.PartNumber;
const tmpKey =
createMpuKey(params.Key, params.UploadId, partNumber, level);
const mergedObject = { PartName: tmpKey };
if (mpuPartList.length < 2) {
logger.trace(
'splitMerge: parts are fewer than 2, copy instead');
// else just perform a copy
const copyParams = {
Bucket: params.MPU,
Key: tmpKey,
CopySource: `${params.MPU}/${mpuPartList[0].PartName}`,
};
return this.service.copyObject(copyParams, (err, res) => {
if (err) {
logHelper(logger, 'error',
'error in splitMerge - copyObject', err);
return cb(err);
}
mergedObject.VersionId = res.VersionId;
mergedObject.ETag = res.ETag;
return cb(null, mergedObject);
});
}
const composeParams = {
const mergedObject = { PartName: tmpKey };
if (mpuPartList.length < 2) {
logger.trace(
'splitMerge: parts are fewer than 2, copy instead');
// else just perform a copy
const copyParams = {
Bucket: params.MPU,
Key: tmpKey,
MultipartUpload: { Parts: mpuPartList },
CopySource: `${params.MPU}/${mpuPartList[0].PartName}`,
};
return this.retryCompose(composeParams, (err, res) => {
return this.service.copyObject(copyParams, (err, res) => {
if (err) {
logHelper(logger, 'error',
'error in splitMerge - copyObject', err);
return cb(err);
}
mergedObject.VersionId = res.VersionId;
mergedObject.ETag = res.ETag;
return cb(null, mergedObject);
});
}, (err, res) => {
}
const composeParams = {
Bucket: params.MPU,
Key: tmpKey,
MultipartUpload: { Parts: mpuPartList },
};
return this.retryCompose(composeParams, (err, res) => {
if (err) {
return callback(err);
return cb(err);
}
return callback(null, res.length);
mergedObject.VersionId = res.VersionId;
mergedObject.ETag = res.ETag;
return cb(null, mergedObject);
});
}, (err, res) => {
if (err) {
return callback(err);
}
return callback(null, res.length);
});
}
/**
@ -284,7 +284,7 @@ class MpuHelper {
if (err) {
logHelper(logger, 'error', 'error in ' +
'createMultipartUpload - final copyObject',
err);
err);
return next(err);
}
const mpuResult = {
@ -301,7 +301,7 @@ class MpuHelper {
if (err) {
logHelper(logger, 'error', 'error in ' +
'createMultipartUpload - final head object',
err);
err);
return next(err);
}
mpuResult.ContentLength = res.ContentLength;

View File

@ -70,7 +70,7 @@ class GcpManagedUpload {
if (this.body instanceof stream) {
assert.strictEqual(typeof this.totalBytes, 'number',
errors.MissingContentLength.customizeDescription(
'If body is a stream, ContentLength must be provided'));
'If body is a stream, ContentLength must be provided'));
} else {
if (typeof this.body === 'string') {
this.body = Buffer.from(this.body);
@ -156,13 +156,13 @@ class GcpManagedUpload {
.map(item =>
Object.assign(item, { ETag: this.parts[item.PartNumber] }));
return this.service.completeMultipartUpload(params,
(err, res) => {
if (err) {
return this.cleanUp(err);
}
this.completed = true;
return this.callback(null, res);
});
(err, res) => {
if (err) {
return this.cleanUp(err);
}
this.completed = true;
return this.callback(null, res);
});
}
/**
@ -187,16 +187,16 @@ class GcpManagedUpload {
if (this.body instanceof stream) {
// stream type
this.body.on('error', err => this.cleanUp(err))
.on('readable', () => this.chunkStream())
.on('end', () => {
this.isDoneChunking = true;
this.chunkStream();
.on('readable', () => this.chunkStream())
.on('end', () => {
this.isDoneChunking = true;
this.chunkStream();
if (this.isDoneChunking && this.uploadedParts >= 1 &&
if (this.isDoneChunking && this.uploadedParts >= 1 &&
this.uploadedParts === this.totalParts) {
this.completeUpload();
}
});
this.completeUpload();
}
});
}
return undefined;
}
@ -251,22 +251,22 @@ class GcpManagedUpload {
});
},
next => async.eachLimit(this.slicedParts, this.queueSize,
(uploadPart, done) => {
const params = {
Bucket: this.mpuBucket,
Key: this.params.Key,
UploadId: this.uploadId,
Body: uploadPart.Body,
PartNumber: uploadPart.PartNumber,
};
this.service.uploadPart(params, (err, res) => {
if (!err) {
this.parts[uploadPart.PartNumber] = res.ETag;
this.uploadedParts++;
}
return done(err);
});
}, next),
(uploadPart, done) => {
const params = {
Bucket: this.mpuBucket,
Key: this.params.Key,
UploadId: this.uploadId,
Body: uploadPart.Body,
PartNumber: uploadPart.PartNumber,
};
this.service.uploadPart(params, (err, res) => {
if (!err) {
this.parts[uploadPart.PartNumber] = res.ETag;
this.uploadedParts++;
}
return done(err);
});
}, next),
], err => {
if (err) {
return this.cleanUp(new Error(

View File

@ -239,7 +239,7 @@ module.exports = {
const requestId = resp.httpResponse.headers ?
resp.httpResponse.headers['x-guploader-uploadid'] : null;
if (resp.error) {
// eslint-disable-next-line no-param-reassign
// eslint-disable-next-line no-param-reassign
resp.error.requestId = resp.requestId || requestId;
}
},

View File

@ -120,10 +120,10 @@ class GcpClient extends AwsClient {
return this._client.createMultipartUpload(params, (err, mpuResObj) => {
if (err) {
logHelper(log, 'error', 'err from data backend',
err, this._dataStoreName, this.clientType);
err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`),
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`)
);
}
return callback(null, mpuResObj);
@ -162,32 +162,32 @@ class GcpClient extends AwsClient {
};
const completeObjData = { key: gcpKey };
return this._client.completeMultipartUpload(mpuParams,
(err, completeMpuRes) => {
if (err) {
logHelper(log, 'error', 'err from data backend on ' +
(err, completeMpuRes) => {
if (err) {
logHelper(log, 'error', 'err from data backend on ' +
'completeMPU', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`),
);
}
if (!completeMpuRes.VersionId) {
logHelper(log, 'error', 'missing version id for data ' +
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`)
);
}
if (!completeMpuRes.VersionId) {
logHelper(log, 'error', 'missing version id for data ' +
'backend object', missingVerIdInternalError,
this._dataStoreName, this.clientType);
return callback(missingVerIdInternalError);
}
// remove quotes from eTag because they're added later
completeObjData.eTag = removeQuotes(completeMpuRes.ETag);
completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
completeObjData.contentLength =
return callback(missingVerIdInternalError);
}
// remove quotes from eTag because they're added later
completeObjData.eTag = removeQuotes(completeMpuRes.ETag);
completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
completeObjData.contentLength =
Number.parseInt(completeMpuRes.ContentLength, 10);
return callback(null, completeObjData);
});
return callback(null, completeObjData);
});
}
uploadPart(request, streamingV4Params, stream, size, key, uploadId,
partNumber, bucketName, log, callback) {
partNumber, bucketName, log, callback) {
let hashedStream = stream;
if (request) {
const partStream = prepareStream(request, streamingV4Params,
@ -209,8 +209,8 @@ class GcpClient extends AwsClient {
logHelper(log, 'error', 'err from data backend ' +
'on uploadPart', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`),
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`)
);
}
// remove quotes from eTag because they're added later
@ -226,11 +226,11 @@ class GcpClient extends AwsClient {
}
uploadPartCopy(request, gcpSourceKey, sourceLocationConstraintName, config,
log, callback) {
log, callback) {
const destBucketName = request.bucketName;
const destObjectKey = request.objectKey;
const destGcpKey = this._createGcpKey(destBucketName, destObjectKey,
this._bucketMatch);
this._bucketMatch);
const sourceGcpBucketName =
config.getGcpBucketNames(sourceLocationConstraintName).bucketName;
@ -241,8 +241,8 @@ class GcpClient extends AwsClient {
if (copySourceRange) {
return callback(errors.NotImplemented
.customizeDescription('Error returned from ' +
`${this.clientType}: copySourceRange not implemented`),
.customizeDescription('Error returned from ' +
`${this.clientType}: copySourceRange not implemented`)
);
}
@ -260,15 +260,15 @@ class GcpClient extends AwsClient {
`${sourceGcpBucketName} GCP bucket`, err,
this._dataStoreName, this.clientType);
return callback(errors.AccessDenied
.customizeDescription('Error: Unable to access ' +
`${sourceGcpBucketName} GCP bucket`),
.customizeDescription('Error: Unable to access ' +
`${sourceGcpBucketName} GCP bucket`)
);
}
logHelper(log, 'error', 'error from data backend on ' +
'uploadPartCopy', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`),
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`)
);
}
// remove quotes from eTag because they're added later
@ -290,8 +290,8 @@ class GcpClient extends AwsClient {
logHelper(log, 'error', 'err from data backend ' +
'on abortMPU', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`),
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`)
);
}
return callback();

View File

@ -45,7 +45,7 @@ class PfsClient {
}
return callback(null, keyContext.objectKey, '',
keyContext.metaHeaders['x-amz-meta-size'],
md5,
md5
);
}
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
@ -72,7 +72,7 @@ class PfsClient {
this._restClient.delete(key, reqUids, err => {
if (err) {
logHelper(log, 'error', 'err from data backend', err,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(err);
}
return callback();
@ -86,61 +86,61 @@ class PfsClient {
}
createMPU(key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
cacheControl, contentDisposition, contentEncoding, log, callback) {
cacheControl, contentDisposition, contentEncoding, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
uploadPart(request, streamingV4Params, stream, size, key, uploadId,
partNumber, bucketName, log, callback) {
partNumber, bucketName, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
listParts(key, uploadId, bucketName, partNumberMarker, maxParts, log,
callback) {
callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
completeMPU(jsonList, mdInfo, key, uploadId, bucketName, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
abortMPU(key, uploadId, bucketName, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
objectPutTagging(key, bucket, objectMD, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
objectDeleteTagging(key, bucketName, objectMD, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
copyObject(request, destLocationConstraintName, sourceKey,
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName,
config, log, callback) {
config, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
}

View File

@ -94,7 +94,7 @@ const utils = {
* same account since Azure copy outside of an account is async
*/
externalBackendCopy(config, locationConstraintSrc, locationConstraintDest,
sourceBucketMD, destBucketMD) {
sourceBucketMD, destBucketMD) {
const sourceBucketName = sourceBucketMD.getName();
const destBucketName = destBucketMD.getName();
const isSameBucket = sourceBucketName === destBucketName;
@ -111,11 +111,11 @@ const utils = {
sourceLocationConstraintType === 'gcp' ||
(sourceLocationConstraintType === 'azure' &&
config.isSameAzureAccount(locationConstraintSrc,
locationConstraintDest)));
locationConstraintDest)));
},
checkExternalBackend(clients, locations, type, flightCheckOnStartUp,
externalBackendHealthCheckInterval, cb) {
externalBackendHealthCheckInterval, cb) {
const checkStatus = backendHealth[type] || {};
if (locations.length === 0) {
return process.nextTick(cb, null, []);

View File

@ -35,6 +35,7 @@ const FOLDER_HASH = 3511;
* directory hash structure under the configured dataPath.
*/
class DataFileStore {
/**
* @constructor
* @param {Object} dataConfig - configuration of the file backend
@ -77,7 +78,7 @@ class DataFileStore {
fs.access(this.dataPath, fs.F_OK | fs.R_OK | fs.W_OK, err => {
if (err) {
this.logger.error('Data path is not readable or writable',
{ error: err });
{ error: err });
return callback(err);
}
if (this.isPassthrough) {
@ -85,7 +86,7 @@ class DataFileStore {
}
// Create FOLDER_HASH subdirectories
const subDirs = Array.from({ length: FOLDER_HASH },
(v, k) => (k).toString());
(v, k) => (k).toString());
this.logger.info(`pre-creating ${subDirs.length} subdirs...`);
if (!this.noSync) {
storageUtils.setDirSyncFlag(this.dataPath, this.logger);
@ -102,7 +103,7 @@ class DataFileStore {
err => {
if (err) {
this.logger.error('Error creating subdirs',
{ error: err });
{ error: err });
return callback(err);
}
this.logger.info('data file store init complete, ' +
@ -166,7 +167,7 @@ class DataFileStore {
fs.open(filePath, 'wx', (err, fd) => {
if (err) {
log.error('error opening filePath',
{ method: 'put', key, filePath, error: err });
{ method: 'put', key, filePath, error: err });
return callback(errors.InternalError.customizeDescription(
`filesystem error: open() returned ${err.code}`));
}
@ -180,7 +181,7 @@ class DataFileStore {
fileStream.on('finish', () => {
function ok() {
log.debug('finished writing data',
{ method: 'put', key, filePath });
{ method: 'put', key, filePath });
return cbOnce(null, key);
}
if (this.noSync) {
@ -242,7 +243,7 @@ class DataFileStore {
return undefined;
}).on('error', err => {
log.error('error streaming data on write',
{ method: 'put', key, filePath, error: err });
{ method: 'put', key, filePath, error: err });
// destroying the write stream forces a close(fd)
fileStream.destroy();
return cbOnce(errors.InternalError.customizeDescription(
@ -286,7 +287,7 @@ class DataFileStore {
return callback(errors.ObjNotFound);
}
log.error('error on \'stat\' of file',
{ key, filePath, error: err });
{ key, filePath, error: err });
return callback(errors.InternalError.customizeDescription(
`filesystem error: stat() returned ${err.code}`));
}
@ -322,34 +323,34 @@ class DataFileStore {
readStreamOptions.end = byteRange[1];
}
log.debug('opening readStream to get data',
{ method: 'get', key, filePath, byteRange });
{ method: 'get', key, filePath, byteRange });
const cbOnce = jsutil.once(callback);
const rs = fs.createReadStream(filePath, readStreamOptions)
.on('error', err => {
if (err.code === 'ENOENT') {
return cbOnce(errors.ObjNotFound);
}
log.error('error retrieving file',
{ method: 'DataFileStore.get', key, filePath,
error: err });
return cbOnce(
errors.InternalError.customizeDescription(
`filesystem read error: ${err.code}`));
})
.on('open', () => { cbOnce(null, rs); })
.on('end', () => {
if (this.noCache) {
releasePageCacheSync(filePath, rs.fd, log);
}
fs.close(rs.fd, err => {
if (err) {
log.error('unable to close file descriptor', {
method: 'DataFileStore.get', key, filePath,
error: err,
});
.on('error', err => {
if (err.code === 'ENOENT') {
return cbOnce(errors.ObjNotFound);
}
log.error('error retrieving file',
{ method: 'DataFileStore.get', key, filePath,
error: err });
return cbOnce(
errors.InternalError.customizeDescription(
`filesystem read error: ${err.code}`));
})
.on('open', () => { cbOnce(null, rs); })
.on('end', () => {
if (this.noCache) {
releasePageCacheSync(filePath, rs.fd, log);
}
fs.close(rs.fd, err => {
if (err) {
log.error('unable to close file descriptor', {
method: 'DataFileStore.get', key, filePath,
error: err,
});
}
});
});
});
}
/**

View File

@ -12,7 +12,7 @@ function releasePageCacheSync(filePath, fd, log) {
const ret = posixFadvise(fd, 0, 0, 4);
if (ret !== 0) {
log.warning(
`error fadv_dontneed ${filePath} returned ${ret}`);
`error fadv_dontneed ${filePath} returned ${ret}`);
}
}

View File

@ -37,16 +37,16 @@ const backend = {
}
cursor += data.length;
})
.on('end', () => {
if (exceeded) {
log.error('data stream exceed announced size',
{ size, overflow: cursor });
callback(errors.InternalError);
} else {
ds[count] = { value, keyContext };
callback(null, count++);
}
});
.on('end', () => {
if (exceeded) {
log.error('data stream exceed announced size',
{ size, overflow: cursor });
callback(errors.InternalError);
} else {
ds[count] = { value, keyContext };
callback(null, count++);
}
});
},
get: function getMem(objectGetInfo, range, reqUids, callback) {

View File

@ -181,25 +181,25 @@ class MetadataWrapper {
const value = typeof objVal.getValue === 'function' ?
objVal.getValue() : objVal;
this.client.putObject(bucketName, objName, value, params, log,
(err, data) => {
if (err) {
log.debug('error from metadata', { implName: this.implName,
error: err });
return cb(err);
}
if (data) {
log.debug('object version successfully put in metadata',
{ version: data });
} else {
log.debug('object successfully put in metadata');
}
return cb(err, data);
});
(err, data) => {
if (err) {
log.debug('error from metadata', { implName: this.implName,
error: err });
return cb(err);
}
if (data) {
log.debug('object version successfully put in metadata',
{ version: data });
} else {
log.debug('object successfully put in metadata');
}
return cb(err, data);
});
}
getBucketAndObjectMD(bucketName, objName, params, log, cb) {
log.debug('getting bucket and object from metadata',
{ database: bucketName, object: objName });
{ database: bucketName, object: objName });
this.client.getBucketAndObject(bucketName, objName, params, log,
(err, data) => {
if (err) {
@ -208,7 +208,7 @@ class MetadataWrapper {
return cb(err);
}
log.debug('bucket and object retrieved from metadata',
{ database: bucketName, object: objName });
{ database: bucketName, object: objName });
return cb(err, data);
});
}

View File

@ -5,7 +5,7 @@ const BucketInfo = require('../../../models/BucketInfo');
class BucketClientInterface {
constructor(params, bucketclient, logger) {
assert(params.bucketdBootstrap.length > 0,
'bucketd bootstrap list is empty');
'bucketd bootstrap list is empty');
const bootstrap = params.bucketdBootstrap;
const log = params.bucketdLog;
if (params.https) {
@ -29,7 +29,7 @@ class BucketClientInterface {
createBucket(bucketName, bucketMD, log, cb) {
this.client.createBucket(bucketName, log.getSerializedUids(),
bucketMD.serialize(), cb);
bucketMD.serialize(), cb);
return null;
}
@ -57,17 +57,17 @@ class BucketClientInterface {
getRaftBuckets(raftId, log, cb) {
return this.client.getRaftBuckets(raftId, log.getSerializedUids(),
(err, data) => {
if (err) {
return cb(err);
}
return cb(null, JSON.parse(data));
});
(err, data) => {
if (err) {
return cb(err);
}
return cb(null, JSON.parse(data));
});
}
putBucketAttributes(bucketName, bucketMD, log, cb) {
this.client.putBucketAttributes(bucketName, log.getSerializedUids(),
bucketMD.serialize(), cb);
bucketMD.serialize(), cb);
return null;
}
@ -95,7 +95,7 @@ class BucketClientInterface {
deleteObject(bucketName, objName, params, log, cb) {
this.client.deleteObject(bucketName, objName, log.getSerializedUids(),
cb, params);
cb, params);
return null;
}
@ -183,8 +183,8 @@ class BucketClientInterface {
reason.msg = undefined;
respBody[implName] = {
code: 200,
message, // Provide interpreted reason msg
body: reason, // Provide analysis data
message, // Provide interpreted reason msg
body: reason, // Provide analysis data
};
if (failure) {
// Setting the `error` field is how the healthCheck

View File

@ -30,6 +30,7 @@ class ListRecordStream extends stream.Transform {
* @classdesc Proxy object to access raft log API
*/
class LogConsumer {
/**
* @constructor
*
@ -96,14 +97,14 @@ class LogConsumer {
if (err.code === 404) {
// no such raft session, log and ignore
this.logger.warn('raft session does not exist yet',
{ raftId: this.raftSession });
{ raftId: this.raftSession });
return cbOnce(null, { info: { start: null,
end: null } });
}
if (err.code === 416) {
// requested range not satisfiable
this.logger.debug('no new log record to process',
{ raftId: this.raftSession });
{ raftId: this.raftSession });
return cbOnce(null, { info: { start: null,
end: null } });
}
@ -115,7 +116,7 @@ class LogConsumer {
// is emitted
recordStream.on('error', err => {
this.logger.error('error receiving raft log',
{ error: err.message });
{ error: err.message });
return cbOnce(errors.InternalError);
});
const jsonResponse = stream.pipe(jsonStream.parse('log.*'));
@ -126,7 +127,7 @@ class LogConsumer {
// remove temporary listener
recordStream.removeAllListeners('error');
return cbOnce(null, { info: header.info,
log: recordStream });
log: recordStream });
})
.on('error', err => recordStream.emit('error', err));
return undefined;

View File

@ -14,13 +14,13 @@ const _operatorType1 = joi.string().valid(
'$gt',
'$gte',
'$lt',
'$lte',
'$lte'
);
// supports strings, numbers, and boolean
const _operatorType2 = joi.string().valid(
'$eq',
'$ne',
'$ne'
);
const _valueType1 = joi.alternatives([

View File

@ -10,13 +10,14 @@ const list = require('../../../algos/list/exportAlgos');
const MetadataFileClient = require('./MetadataFileClient');
const versionSep =
require('../../../versioning/constants')
.VersioningConstants.VersionId.Separator;
.VersioningConstants.VersionId.Separator;
const METASTORE = '__metastore';
const itemScanRefreshDelay = 1000 * 30 * 60; // 30 minutes
class BucketFileInterface {
/**
* @constructor
* @param {object} [params] - constructor params
@ -64,7 +65,7 @@ class BucketFileInterface {
if (err) {
this.logger.fatal('error writing usersBucket ' +
'attributes to metadata',
{ error: err });
{ error: err });
throw (errors.InternalError);
}
return done();
@ -103,8 +104,8 @@ class BucketFileInterface {
}
this.lastItemScanTime = null;
this.putBucketAttributes(bucketName,
bucketMD,
log, cb);
bucketMD,
log, cb);
return undefined;
});
}
@ -190,7 +191,7 @@ class BucketFileInterface {
errorStack: err.stack,
};
log.error('error deleting bucket',
logObj);
logObj);
return cb(errors.InternalError);
}
this.lastItemScanTime = null;
@ -391,16 +392,16 @@ class BucketFileInterface {
cbDone = true;
async.eachSeries(res.bucketList, (bucket, cb) => {
this.getBucketAttributes(bucket.name, log,
(err, bucketInfo) => {
if (err) {
return cb(err);
}
/* eslint-disable no-param-reassign */
bucket.location =
(err, bucketInfo) => {
if (err) {
return cb(err);
}
/* eslint-disable no-param-reassign */
bucket.location =
bucketInfo.getLocationConstraint();
/* eslint-enable no-param-reassign */
return cb();
});
/* eslint-enable no-param-reassign */
return cb();
});
}, err => {
if (!err) {
this.lastItemScanTime = Date.now();

View File

@ -8,6 +8,7 @@ const { RecordLogProxy } = require('./RecordLog.js');
const werelogs = require('werelogs');
class MetadataFileClient {
/**
* Construct a metadata client
*
@ -85,7 +86,7 @@ class MetadataFileClient {
logProxy.connect(err => {
if (err) {
this.logger.error('error connecting to record log service',
{ url, error: err.stack });
{ url, error: err.stack });
return done(err);
}
this.logger.info('connected to record log service', { url });

View File

@ -25,6 +25,7 @@ const SYNC_OPTIONS = { sync: true };
const SUBLEVEL_SEP = '::';
class MetadataFileServer {
/**
* Construct a metadata server
*
@ -217,7 +218,7 @@ class MetadataFileServer {
});
} else {
this.rootDb.batch(ops, SYNC_OPTIONS,
err => callback(err));
err => callback(err));
}
},
};
@ -234,17 +235,17 @@ class MetadataFileServer {
put: (env, key, value, options, cb) => {
const dbName = env.subLevel.join(SUBLEVEL_SEP);
vrp.put({ db: dbName, key, value, options },
env.requestLogger, cb);
env.requestLogger, cb);
},
del: (env, key, options, cb) => {
const dbName = env.subLevel.join(SUBLEVEL_SEP);
vrp.del({ db: dbName, key, options },
env.requestLogger, cb);
env.requestLogger, cb);
},
get: (env, key, options, cb) => {
const dbName = env.subLevel.join(SUBLEVEL_SEP);
vrp.get({ db: dbName, key, options },
env.requestLogger, cb);
env.requestLogger, cb);
},
getDiskUsage: (env, cb) => diskusage.check(this.path, cb),
});

View File

@ -18,6 +18,7 @@ const DEFAULT_RECORD_LOG_NAME = 's3-recordlog';
* object.
*/
class RecordLogProxy extends rpc.BaseClient {
constructor(params) {
super(params);
@ -101,6 +102,7 @@ class ListRecordStream extends stream.Transform {
* updates can be transactional with each other.
*/
class RecordLogService extends rpc.BaseService {
/**
* @constructor
*
@ -272,12 +274,12 @@ class RecordLogService extends rpc.BaseService {
limit: _params.limit,
};
const userStream = new ListRecordStream(endSeq,
_params.limit);
_params.limit);
const dbStream =
openLog.logDb.createReadStream(queryParams);
dbStream.pipe(userStream);
dbStream.once('error',
err => userStream.emit('error', err));
err => userStream.emit('error', err));
userStream.once('error', err => {
userStream.removeAllListeners('info');
cb(err);

View File

@ -8,8 +8,8 @@ function markerFilterMPU(allMarkers, array) {
// in the array that is alphabetically after keyMarker
const firstUnfilteredIndex = array.findIndex(
item => (uploadIdMarker && item.key === keyMarker ?
item.uploadId > uploadIdMarker :
item.key > keyMarker));
item.uploadId > uploadIdMarker :
item.key > keyMarker));
return firstUnfilteredIndex !== -1 ? array.slice(firstUnfilteredIndex) : [];
}

View File

@ -182,7 +182,7 @@ const metastore = {
if (params && params.versionId) {
const baseKey = inc(formatVersionKey(objName, ''));
const vobjName = formatVersionKey(objName,
params.versionId);
params.versionId);
metadata.keyMaps.get(bucketName).delete(vobjName);
const mst = metadata.keyMaps.get(bucketName).get(objName);
if (mst.versionId === params.versionId) {

View File

@ -79,9 +79,9 @@ class ListRecordStream extends stream.Readable {
'did not encounter the last saved offset in oplog, ' +
'resuming processing right after the latest record ' +
'to date; some entries may have been skipped', {
lastSavedID: this._lastSavedID,
latestRecordID: this._latestOplogID,
});
lastSavedID: this._lastSavedID,
latestRecordID: this._latestOplogID,
});
this._unpublishedListing = true;
}
++this._skipCount;
@ -110,7 +110,7 @@ class ListRecordStream extends stream.Readable {
}
entry = {
type: 'put', // updates overwrite the whole metadata,
// so they are considered as puts
// so they are considered as puts
key: itemObj.o2._id,
// updated value may be either stored directly in 'o'
// attribute or in '$set' attribute (supposedly when
@ -132,7 +132,7 @@ class ListRecordStream extends stream.Readable {
}
const streamObject = {
timestamp: new Date((itemObj.ts ?
itemObj.ts.toNumber() * 1000 : 0)),
itemObj.ts.toNumber() * 1000 : 0)),
db: dbName,
entries: [entry],
};

View File

@ -9,6 +9,7 @@ const MongoUtils = require('./utils');
* @classdesc Class to consume mongo oplog
*/
class LogConsumer {
/**
* @constructor
*
@ -42,7 +43,7 @@ class LogConsumer {
(err, client) => {
if (err) {
this._logger.error('Unable to connect to MongoDB',
{ error: err });
{ error: err });
return done(err);
}
this._logger.info('connected to mongodb');

View File

@ -50,7 +50,7 @@ const VID_SEP = require('../../../versioning/constants')
function generateVersionId(replicationGroupId) {
// generate a unique number for each member of the nodejs cluster
return genVID(`${process.pid}.${uidCounter++}`,
replicationGroupId);
replicationGroupId);
}
function formatVersionKey(key, versionId) {
@ -168,7 +168,7 @@ class MongoClientInterface {
if (err) {
this.logger.fatal('error writing usersBucket ' +
'attributes to metastore',
{ error: err });
{ error: err });
throw (errors.InternalError);
}
return cb();
@ -185,7 +185,7 @@ class MongoClientInterface {
getCollection(name) {
/* mongo has a problem with .. in collection names */
const newName = (name === constants.usersBucket) ?
USERSBUCKET : name;
USERSBUCKET : name;
return this.db.collection(newName);
}
@ -209,7 +209,7 @@ class MongoClientInterface {
if (err) {
log.error(
'createBucket: error creating bucket',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
this.lastItemScanTime = null;
@ -222,7 +222,7 @@ class MongoClientInterface {
if (err) {
log.error(
'createBucket: error creating bucket',
{ error: err });
{ error: err });
return cb(errors.InternalError);
}
if (this.shardCollections) {
@ -282,12 +282,12 @@ class MongoClientInterface {
if (err) {
if (err === errors.NoSuchKey) {
return cb(null,
{ bucket:
{ bucket:
BucketInfo.fromObj(bucket).serialize(),
});
});
}
log.error('getObject: error getting object',
{ error: err.message });
{ error: err.message });
return cb(err);
}
return cb(null, {
@ -317,7 +317,7 @@ class MongoClientInterface {
if (err) {
log.error(
'putBucketAttributes: error putting bucket attributes',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
return cb();
@ -334,12 +334,12 @@ class MongoClientInterface {
}, {}, (err, result) => {
if (err) {
log.error('deleteBucketStep2: error deleting bucket',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
if (result.ok !== 1) {
log.error('deleteBucketStep2: failed deleting bucket',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
return cb(null);
@ -364,7 +364,7 @@ class MongoClientInterface {
return this.deleteBucketStep2(bucketName, log, cb);
}
log.error('deleteBucket: error deleting bucket',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
return this.deleteBucketStep2(bucketName, log, err => {
@ -419,7 +419,7 @@ class MongoClientInterface {
$gt: objVal.versionId,
},
},
],
],
},
update: {
$set: { _id: objName, value: objVal },
@ -494,7 +494,7 @@ class MongoClientInterface {
if (err) {
log.error(
'putObjectVerCase2: error putting object version',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
return cb(null, `{"versionId": "${objVal.versionId}"}`);
@ -538,7 +538,7 @@ class MongoClientInterface {
updateOne: {
// eslint-disable-next-line
filter: {
'_id': objName,
_id: objName,
'value.versionId': params.versionId,
},
update: {
@ -555,7 +555,7 @@ class MongoClientInterface {
if (err) {
log.error(
'putObjectVerCase3: error putting object version',
{ error: err.message });
{ error: err.message });
if (err.code === 11000) {
// We want duplicate key error logged however in
// case of the race condition mentioned above, the
@ -601,13 +601,13 @@ class MongoClientInterface {
this.getLatestVersion(c, objName, log, (err, mstObjVal) => {
if (err) {
log.error('getLatestVersion: getting latest version',
{ error: err.message });
{ error: err.message });
return cb(err);
}
MongoUtils.serialize(mstObjVal);
// eslint-disable-next-line
c.update({
'_id': objName,
_id: objName,
'value.versionId': {
// We break the semantic correctness here with
// $gte instead of $gt because we do not have
@ -673,16 +673,16 @@ class MongoClientInterface {
const c = this.getCollection(bucketName);
if (params && params.versioning && !params.versionId) {
return this.putObjectVerCase1(c, bucketName, objName, objVal,
params, log, cb);
params, log, cb);
} else if (params && params.versionId === '') {
return this.putObjectVerCase2(c, bucketName, objName, objVal,
params, log, cb);
params, log, cb);
} else if (params && params.versionId && !params.repairMaster) {
return this.putObjectVerCase3(c, bucketName, objName, objVal,
params, log, cb);
params, log, cb);
} else if (params && params.versionId && params.repairMaster) {
return this.putObjectVerCase4(c, bucketName, objName, objVal,
params, log, cb);
params, log, cb);
}
return this.putObjectNoVer(c, bucketName, objName, objVal,
params, log, cb);
@ -699,7 +699,7 @@ class MongoClientInterface {
}, {}, (err, doc) => {
if (err) {
log.error('findOne: error getting object',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
if (!doc) {
@ -709,7 +709,7 @@ class MongoClientInterface {
this.getLatestVersion(c, objName, log, (err, value) => {
if (err) {
log.error('getLatestVersion: getting latest version',
{ error: err.message });
{ error: err.message });
return cb(err);
}
return cb(null, value);
@ -760,7 +760,7 @@ class MongoClientInterface {
MongoUtils.serialize(objVal);
// eslint-disable-next-line
c.findOneAndReplace({
'_id': objName,
_id: objName,
'value.isPHD': true,
'value.versionId': mst.versionId,
}, {
@ -771,12 +771,12 @@ class MongoClientInterface {
}, (err, result) => {
if (err) {
log.error('repair: error trying to repair value',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
if (result.ok !== 1) {
log.error('repair: failed trying to repair value',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
return cb(null);
@ -791,7 +791,7 @@ class MongoClientInterface {
this.getLatestVersion(c, objName, log, (err, value) => {
if (err) {
log.error('async-repair: getting latest version',
{ error: err.message });
{ error: err.message });
return undefined;
}
this.repair(c, bucketName, objName, value, mst, log, err => {
@ -822,7 +822,7 @@ class MongoClientInterface {
// version:
// eslint-disable-next-line
c.findOneAndDelete({
'_id': objName,
_id: objName,
'value.isPHD': true,
'value.versionId': mst.versionId,
}, {}, err => {
@ -839,7 +839,7 @@ class MongoClientInterface {
return undefined;
}
log.error('getLatestVersion: error getting latest version',
{ error: err.message });
{ error: err.message });
return cb(err);
}
// We have other versions available so repair:
@ -883,7 +883,7 @@ class MongoClientInterface {
if (err) {
log.error(
'deleteObjectVerMaster: error deleting object',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
return this.deleteOrRepairPHD(c, bucketName, objName, mst, log, cb);
@ -927,7 +927,7 @@ class MongoClientInterface {
}, {}, (err, mst) => {
if (err) {
log.error('deleteObjectVer: error deleting versioned object',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
if (!mst) {
@ -936,15 +936,15 @@ class MongoClientInterface {
if (mst.value.isPHD ||
mst.value.versionId === params.versionId) {
return this.deleteObjectVerMaster(c, bucketName, objName,
params, log, err => {
if (err) {
return cb(err);
}
return cb();
});
params, log, err => {
if (err) {
return cb(err);
}
return cb();
});
}
return this.deleteObjectVerNotMaster(c, bucketName, objName,
params, log, cb);
params, log, cb);
});
}
@ -975,10 +975,10 @@ class MongoClientInterface {
const c = this.getCollection(bucketName);
if (params && params.versionId) {
return this.deleteObjectVer(c, bucketName, objName,
params, log, cb);
params, log, cb);
}
return this.deleteObjectNoVer(c, bucketName, objName,
params, log, cb);
params, log, cb);
}
internalListObject(bucketName, params, extension, log, cb) {
@ -1041,7 +1041,7 @@ class MongoClientInterface {
const internalParams = extension.genMDParams();
internalParams.mongifiedSearch = params.mongifiedSearch;
return this.internalListObject(bucketName, internalParams, extension,
log, cb);
log, cb);
}
listMultipartUploads(bucketName, params, log, cb) {
@ -1050,7 +1050,7 @@ class MongoClientInterface {
const internalParams = extension.genMDParams();
internalParams.mongifiedSearch = params.mongifiedSearch;
return this.internalListObject(bucketName, internalParams, extension,
log, cb);
log, cb);
}
checkHealth(implName, log, cb) {
@ -1074,7 +1074,7 @@ class MongoClientInterface {
}, {}, (err, doc) => {
if (err) {
log.error('readUUID: error reading UUID',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
if (!doc) {
@ -1096,7 +1096,7 @@ class MongoClientInterface {
return cb(errors.KeyAlreadyExists);
}
log.error('writeUUIDIfNotExists: error writing UUID',
{ error: err.message });
{ error: err.message });
return cb(errors.InternalError);
}
// FIXME: shoud we check for result.ok === 1 ?
@ -1114,7 +1114,7 @@ class MongoClientInterface {
if (err) {
if (err === errors.InternalError) {
log.error('getUUID: error getting UUID',
{ error: err.message });
{ error: err.message });
return cb(err);
}
return this.readUUID(log, cb);
@ -1129,7 +1129,7 @@ class MongoClientInterface {
// For Kub/cluster deployments there should be a more sophisticated
// way for guessing free space.
diskusage.check(this.path !== undefined ?
this.path : '/', cb);
this.path : '/', cb);
}
readCountItems(log, cb) {
@ -1616,7 +1616,7 @@ class MongoClientInterface {
const retResult = this._handleResults(collRes, isVer);
retResult.stalled = stalledCount;
return callback(null, retResult);
},
}
);
}
@ -1654,7 +1654,7 @@ class MongoClientInterface {
const filter = { _id: objName };
try {
MongoUtils.translateConditions(0, 'value', filter,
params.conditions);
params.conditions);
} catch (err) {
log.error('error creating mongodb filter', {
error: reshapeExceptionError(err),
@ -1700,7 +1700,7 @@ class MongoClientInterface {
const filter = { _id: objName };
try {
MongoUtils.translateConditions(0, 'value', filter,
params.conditions);
params.conditions);
} catch (err) {
log.error('error creating mongodb filter', {
error: reshapeExceptionError(err),

View File

@ -8,8 +8,8 @@ function escape(obj) {
const _obj = {};
Object.keys(obj).forEach(prop => {
const _prop = prop.
replace(/\$/g, '\uFF04').
replace(/\./g, '\uFF0E');
replace(/\$/g, '\uFF04').
replace(/\./g, '\uFF0E');
_obj[_prop] = obj[prop];
});
return _obj;
@ -19,8 +19,8 @@ function unescape(obj) {
const _obj = {};
Object.keys(obj).forEach(prop => {
const _prop = prop.
replace(/\uFF04/g, '$').
replace(/\uFF0E/g, '.');
replace(/\uFF04/g, '$').
replace(/\uFF0E/g, '.');
_obj[_prop] = obj[prop];
});
return _obj;

View File

@ -35,15 +35,15 @@ class BucketdRoutes {
bucketName, logger, (err, data) => {
if (err) {
logger.error('Failed to get bucket attributes',
{ bucket: bucketName, error: err });
{ bucket: bucketName, error: err });
return sendResponse(req, res, logger, err);
}
if (data === undefined) {
return sendResponse(req, res, logger,
errors.NoSuchBucket);
errors.NoSuchBucket);
}
return sendResponse(req, res, logger, null,
BucketInfo.fromObj(data).serialize());
BucketInfo.fromObj(data).serialize());
});
}
@ -112,7 +112,7 @@ class BucketdRoutes {
return this._metadataWrapper.checkHealth(logger, (err, resp) => {
if (err) {
logger.error('Failed the health check',
{ error: err, method: '_checkHealth' });
{ error: err, method: '_checkHealth' });
return sendResponse(req, res, logger, err);
}
return sendResponse(req, res, logger, undefined, resp);
@ -122,8 +122,8 @@ class BucketdRoutes {
_createRequestLogger(req) {
const uids = req.headers['x-scal-request-uids'];
const logger = uids === undefined ?
this._logger.newRequestLogger() :
this._logger.newRequestLoggerFromSerializedUids(uids);
this._logger.newRequestLogger() :
this._logger.newRequestLoggerFromSerializedUids(uids);
logger.trace('new request', { method: req.method, url: req.url });
return logger;
}
@ -150,7 +150,7 @@ class BucketdRoutes {
_attributesRoutes(req, res, uriComponents, logger) {
if (uriComponents.bucketName === undefined) {
logger.error('Missing bucket name for attributes route',
{ uriComponents });
{ uriComponents });
return sendResponse(req, res, logger, errors.BadRequest);
}
switch (req.method) {
@ -195,26 +195,26 @@ class BucketdRoutes {
_bucketRoutes(req, res, uriComponents, logger) {
if (uriComponents.bucketName === undefined) {
logger.error('Missing bucket name for bucket route',
{ uriComponents });
{ uriComponents });
return sendResponse(req, res, logger, errors.BadRequest);
}
switch (req.method) {
case 'GET':
return this._listObject(req, res,
uriComponents.bucketName,
uriComponents.options,
logger);
uriComponents.bucketName,
uriComponents.options,
logger);
case 'DELETE':
return this._deleteBucket(req, res,
uriComponents.bucketName, logger);
uriComponents.bucketName, logger);
case 'POST':
return getRequestBody(logger, req, (err, body) => {
if (err) {
return sendResponse(req, res, logger, err);
}
return this._createBucket(req, res,
uriComponents.bucketName,
body, logger);
uriComponents.bucketName,
body, logger);
});
default:
return sendResponse(req, res, logger, errors.RouteNotFound);
@ -233,30 +233,30 @@ class BucketdRoutes {
_objectRoutes(req, res, uriComponents, logger) {
if (uriComponents.bucketName === undefined) {
logger.error('Missing bucket name for object route',
{ uriComponents });
{ uriComponents });
return sendResponse(req, res, logger, errors.BadRequest);
}
switch (req.method) {
case 'GET':
return this._getObject(req, res,
uriComponents.bucketName,
uriComponents.objectName,
uriComponents.options,
logger);
uriComponents.bucketName,
uriComponents.objectName,
uriComponents.options,
logger);
case 'DELETE':
return this._deleteObject(req, res,
uriComponents.bucketName,
uriComponents.objectName,
uriComponents.options,
logger);
uriComponents.bucketName,
uriComponents.objectName,
uriComponents.options,
logger);
case 'POST':
return getRequestBody(logger, req, (err, body) =>
this._putObject(req, res,
uriComponents.bucketName,
uriComponents.objectName,
body,
uriComponents.options,
logger));
this._putObject(req, res,
uriComponents.bucketName,
uriComponents.objectName,
body,
uriComponents.options,
logger));
default:
return sendResponse(req, res, logger, errors.RouteNotFound);
}
@ -274,16 +274,16 @@ class BucketdRoutes {
_parallelRoutes(req, res, uriComponents, logger) {
if (uriComponents.bucketName === undefined) {
logger.error('Missing bucket name for parallel route',
{ uriComponents });
{ uriComponents });
return sendResponse(req, res, logger, errors.BadRequest);
}
switch (req.method) {
case 'GET':
return this._getBucketAndObjectMD(req, res,
uriComponents.bucketName,
uriComponents.objectName,
uriComponents.options,
logger);
uriComponents.bucketName,
uriComponents.objectName,
uriComponents.options,
logger);
default:
return sendResponse(req, res, logger, errors.RouteNotFound);
}
@ -307,7 +307,7 @@ class BucketdRoutes {
return sendResponse(req, res, logger, errors.NotImplemented);
case 'metadataInformation':
return sendResponse(req, res, logger, undefined,
'{"metadataVersion":2}');
'{"metadataVersion":2}');
case 'parallel':
logger.trace(`${uriComponents.context} operation`);
if (uriComponents.objectName) {

View File

@ -11,6 +11,7 @@ const requiresOneWorker = {
};
class Server {
/**
* Create a new Metadata Proxy Server instance
*
@ -30,7 +31,7 @@ class Server {
if (requiresOneWorker[metadataWrapper.implName] &&
this._configuration.workers !== 1) {
logger.warn('This metadata backend requires only one worker',
{ metadataBackend: metadataWrapper.implName });
{ metadataBackend: metadataWrapper.implName });
this._configuration.workers = 1;
}
this._logger = logger;
@ -78,7 +79,7 @@ class Server {
});
} else {
this._httpServer = new HttpServer(this._configuration.port,
this._logger);
this._logger);
if (this._configuration.bindAddress) {
this._httpServer.setBindAddress(
this._configuration.bindAddress);

View File

@ -67,8 +67,8 @@ function getURIComponents(uri, logger) {
};
} catch (ex) {
logger.error('Invalid URI: failed to parse',
{ uri, error: ex, errorStack: ex.stack,
message: ex.message });
{ uri, error: ex, errorStack: ex.stack,
message: ex.message });
return null;
}
}

View File

@ -60,8 +60,8 @@ class TestMatrix {
this.elementsToSpecialize = elementsToSpecialize;
this.callback = callback;
this.description = typeof description === 'undefined'
? ''
: description;
? ''
: description;
return this;
}
@ -158,15 +158,15 @@ class TestMatrix {
const callFunction = (matrixFather, matrixChild, callback,
description) => {
const result = Object.keys(matrixChild.params)
.every(currentKey =>
Object.prototype.toString.call(
matrixChild.params[currentKey],
.every(currentKey =>
Object.prototype.toString.call(
matrixChild.params[currentKey]
).indexOf('Array') === -1);
if (result === true) {
describe(matrixChild.serialize(), () => {
it(description,
done => callback(matrixChild, done));
done => callback(matrixChild, done));
});
} else {
describe(matrixChild.serialize(), () => {

View File

@ -247,7 +247,7 @@ function decode(str) {
}
module.exports = { generateVersionId, getInfVid,
hexEncode, hexDecode,
base62Encode, base62Decode,
encode, decode,
ENC_TYPE_HEX, ENC_TYPE_BASE62 };
hexEncode, hexDecode,
base62Encode, base62Decode,
encode, decode,
ENC_TYPE_HEX, ENC_TYPE_BASE62 };

View File

@ -84,7 +84,7 @@ class VersioningRequestProcessor {
return callback(null, data);
}
logger.debug('master version is a PHD, getting the latest version',
{ db, key });
{ db, key });
// otherwise, need to search for the latest version
return this.getByListing(request, logger, callback);
});
@ -187,7 +187,7 @@ class VersioningRequestProcessor {
return entry.callback(err, value);
}
return this.wgm.get(entry.request, entry.logger,
entry.callback);
entry.callback);
});
delete this.queue[cacheKey];
}
@ -267,19 +267,19 @@ class VersioningRequestProcessor {
return callback(err);
}
return this.writeCache.batch({ db, array, options },
logger, err => callback(err, `{"versionId":"${vid}"}`));
logger, err => callback(err, `{"versionId":"${vid}"}`));
};
if (versionId) {
return this.processVersionSpecificPut(request, logger,
versioningCb);
versioningCb);
}
if (versioning) {
return this.processNewVersionPut(request, logger, versioningCb);
}
// no versioning or versioning configuration off
return this.writeCache.batch({ db, array: [{ key, value }] },
logger, callback);
logger, callback);
}
/**
@ -353,7 +353,7 @@ class VersioningRequestProcessor {
if (!(options && options.versionId)) {
return this.writeCache.batch({ db,
array: [{ key, type: 'del' }] },
logger, callback);
logger, callback);
}
// version specific DELETE
return this.processVersionSpecificDelete(request, logger,
@ -399,7 +399,7 @@ class VersioningRequestProcessor {
const cacheKey = formatCacheKey(db, key);
clearTimeout(this.repairing[cacheKey]);
this.repairing[cacheKey] = setTimeout(() =>
this.getByListing(request, logger, () => {}), 15000);
this.getByListing(request, logger, () => {}), 15000);
}
return callback(null, ops, versionId);
});

Some files were not shown because too many files have changed in this diff Show More