Compare commits

...

4 Commits

Author SHA1 Message Date
Dora Korpar d7169662e2 ft: ZENKO 1402 data backend tests 2019-01-09 17:28:05 -08:00
Dora Korpar 801d0490cf move ext. backend related unit tests 2019-01-09 12:44:38 -08:00
Dora Korpar 86ba6b2e22 update Arsenal version 2019-01-08 16:39:11 -08:00
Dora Korpar ef256edccb ft: ZENKO 1402 move data backends 2019-01-08 16:39:11 -08:00
45 changed files with 6180 additions and 2 deletions

View File

@ -31,6 +31,11 @@ stages:
- ShellCommand:
name: run lint_md
command: npm run --silent lint_md
- ShellCommand:
name: add hostname
command: sudo sh -c "echo '127.0.0.1 testrequestbucket.localhost' \
>> /etc/hosts"
- ShellCommand:
name: run test
command: npm run --silent test

View File

@ -77,6 +77,8 @@ module.exports = {
SubStreamInterface:
require('./lib/s3middleware/azureHelpers/SubStreamInterface'),
},
prepareStream: require('./lib/s3middleware/prepareStream'),
processMpuParts: require('./lib/s3middleware/processMpuParts'),
},
storage: {
metadata: {
@ -119,6 +121,16 @@ module.exports = {
DataFileStore:
require('./lib/storage/data/file/DataFileStore'),
},
external: {
AwsClient: require('./lib/storage/data/external/AwsClient'),
AzureClient: require('./lib/storage/data/external/AzureClient'),
GcpClient: require('./lib/storage/data/external/GcpClient'),
GCP: require('./lib/storage/data/external/GCP/GcpService'),
GcpUtils: require('./lib/storage/data/external/GCP/GcpUtils'),
GcpSigner: require('./lib/storage/data/external/GCP/GcpSigner'),
PfsClient: require('./lib/storage/data/external/PfsClient'),
backendUtils: require('./lib/storage/data/external/utils'),
},
},
utils: require('./lib/storage/utils'),
},

47
lib/auth/v4/getVault.js Normal file
View File

@ -0,0 +1,47 @@
const vaultclient = require('vaultclient');
const Vault = require('../Vault');
const Backend = require('../in_memory/Backend').s3;
function getVault(config, logger) {
const backend = new Backend(config.authData);
let client;
let implName;
if (config.backends.auth === 'mem') {
config.on('authdata-update', () => {
backend.refreshAuthData(config.authData);
});
client = backend;
implName = 'vaultMem';
} else {
const { host, port } = config.vaultd;
implName = 'vault';
if (config.https) {
const { key, cert, ca } = config.https;
logger.info('vaultclient configuration', {
host,
port,
https: true,
});
client = new vaultclient.Client(host, port, true, key, cert, ca);
} else {
logger.info('vaultclient configuration', {
host,
port,
https: false,
});
client = new vaultclient.Client(host, port);
}
if (config.log) {
client.setLoggerConfig({
level: config.log.logLevel,
dump: config.log.dumpLevel,
});
}
}
return new Vault(client, implName);
}
module.exports = {
getVault,
};

View File

@ -0,0 +1,282 @@
const { Transform } = require('stream');
const async = require('async');
const errors = require('../../../errors');
const { getVault } = require('../getVault');
const constructChunkStringToSign = require('./constructChunkStringToSign');
/**
* This class is designed to handle the chunks sent in a streaming
* v4 Auth request
*/
class V4Transform extends Transform {
/**
* @constructor
* @param {object} streamingV4Params - info for chunk authentication
* @param {string} streamingV4Params.accessKey - requester's accessKey
* @param {string} streamingV4Params.signatureFromRequest - signature
* sent with headers
* @param {string} streamingV4Params.region - region sent with auth header
* @param {string} streamingV4Params.scopeDate - date sent with auth header
* @param {string} streamingV4Params.timestamp - date parsed from headers
* in ISO 8601 format: YYYYMMDDTHHMMSSZ
* @param {string} streamingV4Params.credentialScope - items from auth
* header plus the string 'aws4_request' joined with '/':
* timestamp/region/aws-service/aws4_request
* @param {object} config - CloudServer config for vault configuration
* @param {object} log - logger object
* @param {function} cb - callback to api
*/
constructor(streamingV4Params, config, log, cb) {
const { accessKey, signatureFromRequest, region, scopeDate, timestamp,
credentialScope } = streamingV4Params;
super({});
this.log = log;
this.cb = cb;
this.accessKey = accessKey;
this.region = region;
this.scopeDate = scopeDate;
this.timestamp = timestamp;
this.credentialScope = credentialScope;
this.lastSignature = signatureFromRequest;
this.currentSignature = undefined;
this.haveMetadata = false;
// keep this as -1 to start since a seekingDataSize of 0
// means that chunk is just metadata (as is the case with the
// last chunk)
this.seekingDataSize = -1;
this.currentData = undefined;
this.dataCursor = 0;
this.currentMetadata = [];
this.lastPieceDone = false;
this.lastChunk = false;
this.vault = getVault(config, this.log);
}
/**
* This function will parse the metadata portion of the chunk
* @param {Buffer} remainingChunk - chunk sent from _transform
* @return {object} response - if error, will return 'err' key with
* arsenal error value.
* if incomplete metadata, will return 'completeMetadata' key with
* value false
* if complete metadata received, will return 'completeMetadata' key with
* value true and the key 'unparsedChunk' with the remaining chunk without
* the parsed metadata piece
*/
_parseMetadata(remainingChunk) {
let remainingPlusStoredMetadata = remainingChunk;
// have metadata pieces so need to add to the front of
// remainingChunk
if (this.currentMetadata.length > 0) {
this.currentMetadata.push(remainingChunk);
remainingPlusStoredMetadata = Buffer.concat(this.currentMetadata);
// zero out stored metadata
this.currentMetadata.length = 0;
}
let lineBreakIndex = remainingPlusStoredMetadata.indexOf('\r\n');
if (lineBreakIndex < 0) {
this.currentMetadata.push(remainingPlusStoredMetadata);
return { completeMetadata: false };
}
let fullMetadata = remainingPlusStoredMetadata.slice(0,
lineBreakIndex);
// handle extra line break on end of data chunk
if (fullMetadata.length === 0) {
const chunkWithoutLeadingLineBreak = remainingPlusStoredMetadata
.slice(2);
// find second line break
lineBreakIndex = chunkWithoutLeadingLineBreak.indexOf('\r\n');
if (lineBreakIndex < 0) {
this.currentMetadata.push(chunkWithoutLeadingLineBreak);
return { completeMetadata: false };
}
fullMetadata = chunkWithoutLeadingLineBreak.slice(0,
lineBreakIndex);
}
const splitMeta = fullMetadata.toString().split(';');
this.log.trace('parsed full metadata for chunk', { splitMeta });
if (splitMeta.length !== 2) {
this.log.trace('chunk body did not contain correct ' +
'metadata format');
return { err: errors.InvalidArgument };
}
let dataSize = splitMeta[0];
// chunk-size is sent in hex
dataSize = Number.parseInt(dataSize, 16);
if (Number.isNaN(dataSize)) {
this.log.trace('chunk body did not contain valid size');
return { err: errors.InvalidArgument };
}
let chunkSig = splitMeta[1];
if (!chunkSig || chunkSig.indexOf('chunk-signature=') < 0) {
this.log.trace('chunk body did not contain correct sig format');
return { err: errors.InvalidArgument };
}
chunkSig = chunkSig.replace('chunk-signature=', '');
this.currentSignature = chunkSig;
this.haveMetadata = true;
if (dataSize === 0) {
this.lastChunk = true;
return {
completeMetadata: true,
};
}
// + 2 to get \r\n at end
this.seekingDataSize = dataSize + 2;
this.currentData = Buffer.alloc(dataSize);
return {
completeMetadata: true,
// start slice at lineBreak plus 2 to remove line break at end of
// metadata piece since length of '\r\n' is 2
unparsedChunk: remainingPlusStoredMetadata
.slice(lineBreakIndex + 2),
};
}
/**
* Build the stringToSign and authenticate the chunk
* @param {Buffer} dataToSend - chunk sent from _transform or null
* if last chunk without data
* @param {function} done - callback to _transform
* @return {function} executes callback with err if applicable
*/
_authenticate(dataToSend, done) {
// use prior sig to construct new string to sign
const stringToSign = constructChunkStringToSign(this.timestamp,
this.credentialScope, this.lastSignature, dataToSend);
this.log.trace('constructed chunk string to sign',
{ stringToSign });
// once used prior sig to construct string to sign, reassign
// lastSignature to current signature
this.lastSignature = this.currentSignature;
const vaultParams = {
log: this.log,
data: {
accessKey: this.accessKey,
signatureFromRequest: this.currentSignature,
region: this.region,
scopeDate: this.scopeDate,
stringToSign,
timestamp: this.timestamp,
credentialScope: this.credentialScope,
},
};
return this.vault.authenticateV4Request(vaultParams, null, err => {
if (err) {
this.log.trace('err from vault on streaming v4 auth',
{ error: err, paramsSentToVault: vaultParams.data });
return done(err);
}
return done();
});
}
/**
* This function will parse the chunk into metadata and data,
* use the metadata to authenticate with vault and send the
* data on to be stored if authentication passes
*
* @param {Buffer} chunk - chunk from request body
* @param {string} encoding - Data encoding
* @param {function} callback - Callback(err, justDataChunk, encoding)
* @return {function }executes callback with err if applicable
*/
_transform(chunk, encoding, callback) {
// 'chunk' here is the node streaming chunk
// transfer-encoding chunks should be of the format:
// string(IntHexBase(chunk-size)) + ";chunk-signature=" +
// signature + \r\n + chunk-data + \r\n
// Last transfer-encoding chunk will have size 0 and no chunk-data.
if (this.lastPieceDone) {
const slice = chunk.slice(0, 10);
this.log.trace('received chunk after end.' +
'See first 10 bytes of chunk',
{ chunk: slice.toString() });
return callback();
}
let unparsedChunk = chunk;
let chunkLeftToEvaluate = true;
return async.whilst(
// test function
() => chunkLeftToEvaluate,
// async function
done => {
if (!this.haveMetadata) {
this.log.trace('do not have metadata so calling ' +
'_parseMetadata');
// need to parse our metadata
const parsedMetadataResults =
this._parseMetadata(unparsedChunk);
if (parsedMetadataResults.err) {
return done(parsedMetadataResults.err);
}
// if do not have full metadata get next chunk
if (!parsedMetadataResults.completeMetadata) {
chunkLeftToEvaluate = false;
return done();
}
// have metadata so reset unparsedChunk to remaining
// without metadata piece
unparsedChunk = parsedMetadataResults.unparsedChunk;
}
if (this.lastChunk) {
this.log.trace('authenticating final chunk with no data');
return this._authenticate(null, err => {
if (err) {
return done(err);
}
chunkLeftToEvaluate = false;
this.lastPieceDone = true;
return done();
});
}
if (unparsedChunk.length < this.seekingDataSize) {
// add chunk to currentData and get next chunk
unparsedChunk.copy(this.currentData, this.dataCursor);
this.dataCursor += unparsedChunk.length;
this.seekingDataSize -= unparsedChunk.length;
chunkLeftToEvaluate = false;
return done();
}
// parse just the next data piece without \r\n at the end
// (therefore, minus 2)
const nextDataPiece =
unparsedChunk.slice(0, this.seekingDataSize - 2);
// add parsed data piece to other currentData pieces
// so that this.currentData is the full data piece
nextDataPiece.copy(this.currentData, this.dataCursor);
return this._authenticate(this.currentData, err => {
if (err) {
return done(err);
}
unparsedChunk =
unparsedChunk.slice(this.seekingDataSize);
this.push(this.currentData);
this.haveMetadata = false;
this.seekingDataSize = -1;
this.currentData = undefined;
this.dataCursor = 0;
chunkLeftToEvaluate = unparsedChunk.length > 0;
return done();
});
},
// final callback
err => {
if (err) {
return this.cb(err);
}
// get next chunk
return callback();
}
);
}
}
module.exports = V4Transform;

View File

@ -0,0 +1,32 @@
const crypto = require('crypto');
const constants = require('../../../constants');
/**
* Constructs stringToSign for chunk
* @param {string} timestamp - date parsed from headers
* in ISO 8601 format: YYYYMMDDTHHMMSSZ
* @param {string} credentialScope - items from auth
* header plus the string 'aws4_request' joined with '/':
* timestamp/region/aws-service/aws4_request
* @param {string} lastSignature - signature from headers or prior chunk
* @param {string} justDataChunk - data portion of chunk
* @returns {string} stringToSign
*/
function constructChunkStringToSign(timestamp,
credentialScope, lastSignature, justDataChunk) {
let currentChunkHash;
// for last chunk, there will be no data, so use emptyStringHash
if (!justDataChunk) {
currentChunkHash = constants.emptyStringHash;
} else {
currentChunkHash = crypto.createHash('sha256');
currentChunkHash = currentChunkHash
.update(justDataChunk, 'binary').digest('hex');
}
return `AWS4-HMAC-SHA256-PAYLOAD\n${timestamp}\n` +
`${credentialScope}\n${lastSignature}\n` +
`${constants.emptyStringHash}\n${currentChunkHash}`;
}
module.exports = constructChunkStringToSign;

View File

@ -1,4 +1,5 @@
'use strict'; // eslint-disable-line strict
const crypto = require('crypto');
// The min value here is to manage further backward compat if we
// need it
@ -85,4 +86,16 @@ module.exports = {
zenkoSeparator: ':',
/* eslint-disable camelcase */
replicationBackends: { aws_s3: true, azure: true, gcp: true },
// hex digest of sha256 hash of empty string:
emptyStringHash: crypto.createHash('sha256')
.update('', 'binary').digest('hex'),
mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
// AWS sets a minimum size limit for parts except for the last part.
// http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
minimumAllowedPartSize: 5242880,
gcpMaximumAllowedPartCount: 1024,
// GCP Object Tagging Prefix
gcpTaggingPrefix: 'aws-tag-',
productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko',
};

View File

@ -0,0 +1,26 @@
const V4Transform = require('../auth/v4/streamingV4/V4Transform');
/**
* Prepares the stream if the chunks are sent in a v4 Auth request
* @param {object} stream - stream containing the data
* @param {object | null } streamingV4Params - if v4 auth, object containing
* accessKey, signatureFromRequest, region, scopeDate, timestamp, and
* credentialScope (to be used for streaming v4 auth if applicable)
* @param {object} config - CloudServer config for Vault configuration
* @param {RequestLogger} log - the current request logger
* @param {function} cb - callback containing the result for V4Transform
* @return {object} - V4Transform object if v4 Auth request, or else the stream
*/
function prepareStream(stream, streamingV4Params, config, log, cb) {
if (stream.headers['x-amz-content-sha256'] ===
'STREAMING-AWS4-HMAC-SHA256-PAYLOAD') {
const v4Transform = new V4Transform(streamingV4Params, config, log, cb);
stream.pipe(v4Transform);
return v4Transform;
}
return stream;
}
module.exports = {
prepareStream,
};

View File

@ -0,0 +1,242 @@
const errors = require('../errors');
const crypto = require('crypto');
const constants = require('../constants');
/**
* createAggregateETag - creates ETag from concatenated MPU part ETags to
* mimic AWS
* @param {string} concatETags - string of concatenated MPU part ETags
* @param {array} partList - list of parts to complete MPU with
* @return {string} aggregateETag - final complete MPU obj ETag
*/
function createAggregateETag(concatETags, partList) {
// AWS documentation is unclear on what the MD5 is that it returns
// in the response for a complete multipart upload request.
// The docs state that they might or might not
// return the MD5 of the complete object. It appears
// they are returning the MD5 of the parts' MD5s so that is
// what we have done here. We:
// 1) concatenate the hex version of the
// individual ETags
// 2) convert the concatenated hex to binary
// 3) take the md5 of the binary
// 4) create the hex digest of the md5
// 5) add '-' plus the number of parts at the end
// Convert the concatenated hex ETags to binary
const bufferedHex = Buffer.from(concatETags, 'hex');
// Convert the buffer to a binary string
const binaryString = bufferedHex.toString('binary');
// Get the md5 of the binary string
const md5Hash = crypto.createHash('md5');
md5Hash.update(binaryString, 'binary');
// Get the hex digest of the md5
let aggregateETag = md5Hash.digest('hex');
// Add the number of parts at the end
aggregateETag = `${aggregateETag}-${partList.length}`;
return aggregateETag;
}
/**
* generateMpuPartStorageInfo - generates info needed for storage of
* completed MPU object
* @param {array} filteredPartList - list of parts filtered from metadata
* @return {object} partsInfo - contains three keys: aggregateETag,
* dataLocations, and calculatedSize
*/
function generateMpuPartStorageInfo(filteredPartList) {
// Assemble array of part locations, aggregate size
// and build string to create aggregate ETag
let calculatedSize = 0;
const dataLocations = [];
let concatETags = '';
const partsInfo = {};
filteredPartList.forEach((storedPart, index) => {
const partETagWithoutQuotes =
storedPart.ETag.slice(1, -1);
const dataStoreETag = `${index + 1}:${partETagWithoutQuotes}`;
concatETags += partETagWithoutQuotes;
// If part was put by a regular put part rather than a
// copy it is always one location. With a put part
// copy, could be multiple locations so loop over array
// of locations.
for (let j = 0; j < storedPart.locations.length; j++) {
// If the piece has parts (was a put part object
// copy) each piece will have a size attribute.
// Otherwise, the piece was put by a regular put
// part and the size the of the piece is the full
// part size.
const location = storedPart.locations[j];
// If there is no location, move on
if (!location || typeof location !== 'object') {
continue;
}
let pieceSize = Number.parseInt(storedPart.size, 10);
if (location.size) {
pieceSize = Number.parseInt(location.size, 10);
}
const pieceRetrievalInfo = {
key: location.key,
size: pieceSize,
start: calculatedSize,
dataStoreName: location.dataStoreName,
dataStoreETag,
cryptoScheme: location.sseCryptoScheme,
cipheredDataKey: location.sseCipheredDataKey,
};
dataLocations.push(pieceRetrievalInfo);
// eslint-disable-next-line no-param-reassign
calculatedSize += pieceSize;
}
});
partsInfo.aggregateETag =
createAggregateETag(concatETags, filteredPartList);
partsInfo.dataLocations = dataLocations;
partsInfo.calculatedSize = calculatedSize;
return partsInfo;
}
/**
* validateAndFilterMpuParts - validates part list sent by user and filters
* parts stored in metadata against user part list
* @param {array} storedParts - array of parts stored in metadata
* @param {array} jsonList - array of parts sent by user for completion
* @param {string} mpuOverviewKey - metadata mpu key
* @param {string} splitter - mpu key divider
* @param {object} log - Werelogs instance
* @return {object} filtersPartsObj - contains 3 keys: partList, keysToDelete,
* and extraPartLocations
*/
function validateAndFilterMpuParts(storedParts, jsonList, mpuOverviewKey,
splitter, log) {
let storedPartsCopy = [];
const filteredPartsObj = {};
filteredPartsObj.partList = [];
const keysToDelete = [];
storedParts.forEach(item => {
keysToDelete.push(item.key);
storedPartsCopy.push({
// In order to delete the part listing in the shadow
// bucket, need the full key
key: item.key,
ETag: `"${item.value.ETag}"`,
size: item.value.Size,
locations: Array.isArray(item.value.partLocations) ?
item.value.partLocations : [item.value.partLocations],
});
});
keysToDelete.push(mpuOverviewKey);
// Check list sent to make sure valid
const partLength = jsonList.Part.length;
// A user can put more parts than they end up including
// in the completed MPU but there cannot be more
// parts in the complete message than were already put
if (partLength > storedPartsCopy.length) {
filteredPartsObj.error = errors.InvalidPart;
return filteredPartsObj;
}
let extraParts = [];
const extraPartLocations = [];
for (let i = 0; i < partLength; i++) {
const part = jsonList.Part[i];
const partNumber = Number.parseInt(part.PartNumber[0], 10);
// If the complete list of parts sent with
// the complete multipart upload request is not
// in ascending order return an error
if (i > 0) {
const previousPartNumber =
Number.parseInt(jsonList.Part[i - 1].PartNumber[0], 10);
if (partNumber <= previousPartNumber) {
filteredPartsObj.error = errors.InvalidPartOrder;
return filteredPartsObj;
}
}
let isPartUploaded = false;
while (storedPartsCopy.length > 0 && !isPartUploaded) {
const storedPart = storedPartsCopy[0];
const storedPartNumber =
Number.parseInt(storedPart.key.split(splitter)[1], 10);
if (storedPartNumber === partNumber) {
isPartUploaded = true;
filteredPartsObj.partList.push(storedPart);
let partETag = part.ETag[0].replace(/['"]/g, '');
// some clients send base64, convert to hex
// 32 chars = 16 bytes(2 chars-per-byte) = 128 bits of
// MD5 hex
if (partETag.length !== 32) {
const buffered = Buffer.from(part.ETag[0], 'base64')
.toString('hex');
partETag = `${buffered}`;
}
partETag = `"${partETag}"`;
// If list of parts sent with complete mpu request contains
// a part ETag that does not match the ETag for the part
// stored in metadata, return an error
if (partETag !== storedPart.ETag) {
filteredPartsObj.error = errors.InvalidPart;
return filteredPartsObj;
}
// If any part other than the last part is less than
// 5MB, return an error
const storedPartSize =
Number.parseInt(storedPart.size, 10);
// allow smaller parts for testing
if (process.env.MPU_TESTING) {
log.info('MPU_TESTING env variable setting',
{ setting: process.env.MPU_TESTING });
}
if (process.env.MPU_TESTING !== 'yes' &&
i < jsonList.Part.length - 1 &&
storedPartSize < constants.minimumAllowedPartSize) {
log.debug('part too small on complete mpu');
filteredPartsObj.error = errors.EntityTooSmall;
return filteredPartsObj;
}
storedPartsCopy = storedPartsCopy.splice(1);
} else {
extraParts.push(storedPart);
storedPartsCopy = storedPartsCopy.splice(1);
}
}
if (!isPartUploaded) {
filteredPartsObj.error = errors.InvalidPart;
return filteredPartsObj;
}
}
extraParts = extraParts.concat(storedPartsCopy);
// if extra parts, need to delete the data when done with completing
// mpu so extract the info to delete here
if (extraParts.length > 0) {
extraParts.forEach(part => {
const locations = part.locations;
locations.forEach(location => {
if (!location || typeof location !== 'object') {
return;
}
extraPartLocations.push(location);
});
});
}
filteredPartsObj.keysToDelete = keysToDelete;
filteredPartsObj.extraPartLocations = extraPartLocations;
return filteredPartsObj;
}
module.exports = {
generateMpuPartStorageInfo,
validateAndFilterMpuParts,
createAggregateETag,
};

622
lib/storage/data/external/AwsClient.js vendored Normal file
View File

@ -0,0 +1,622 @@
const AWS = require('aws-sdk');
const werelogs = require('werelogs');
const errors = require('../../../errors');
const MD5Sum = require('../../../s3middleware/MD5Sum');
const getMetaHeaders =
require('../../../s3middleware/userMetadata').getMetaHeaders;
const { prepareStream } = require('../../../s3middleware/prepareStream');
const { createLogger, logHelper, removeQuotes, trimXMetaPrefix } =
require('./utils');
const missingVerIdInternalError = errors.InternalError.customizeDescription(
'Invalid state. Please ensure versioning is enabled ' +
'in AWS for the location constraint and try again.'
);
class AwsClient {
constructor(config) {
this.clientType = 'aws_s3';
this.type = 'AWS';
this._s3Params = config.s3Params;
this._awsBucketName = config.bucketName;
this._bucketMatch = config.bucketMatch;
this._dataStoreName = config.dataStoreName;
this._serverSideEncryption = config.serverSideEncryption;
this._supportsVersioning = config.supportsVersioning;
this._client = new AWS.S3(this._s3Params);
this._logger = new werelogs.Logger('AwsClient');
}
setup(cb) {
// this request implicitly updates the endpoint for the location
// the following code explcitly sets it to avoid surprises
this._client.getBucketLocation({ Bucket: this._awsBucketName },
(err, res) => {
if (err && err.code !== 'AuthorizationHeaderMalformed') {
this._logger.error('error during setup', {
error: err,
method: 'AwsClient.setup',
});
return cb(err);
}
let region;
if (err && err.code === 'AuthorizationHeaderMalformed') {
// set regional endpoint
region = err.region;
} else if (res) {
region = res.LocationConstraint;
}
const isAWS = this._s3Params.endpoint.endsWith('amazonaws.com');
if (region && isAWS) {
const endpoint = `s3.${region}.amazonaws.com`;
this._logger.debug('setting regional endpoint', {
method: 'AwsClient.setup',
region,
endpoint,
});
this._client.endpoint = new AWS.Endpoint(endpoint);
}
return cb();
});
}
_createAwsKey(requestBucketName, requestObjectKey,
bucketMatch) {
if (bucketMatch) {
return requestObjectKey;
}
return `${requestBucketName}/${requestObjectKey}`;
}
toObjectGetInfo(objectKey, bucketName) {
return {
key: this._createAwsKey(bucketName, objectKey, this._bucketMatch),
dataStoreName: this._dataStoreName,
};
}
put(stream, size, keyContext, reqUids, callback) {
const awsKey = this._createAwsKey(keyContext.bucketName,
keyContext.objectKey, this._bucketMatch);
const metaHeaders = trimXMetaPrefix(keyContext.metaHeaders);
const log = createLogger(reqUids);
const putCb = (err, data) => {
if (err) {
logHelper(log, 'error', 'err from data backend',
err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
if (!data.VersionId && this._supportsVersioning) {
logHelper(log, 'error', 'missing version id for data ' +
'backend object', missingVerIdInternalError,
this._dataStoreName, this.clientType);
return callback(missingVerIdInternalError);
}
const dataStoreVersionId = data.VersionId;
return callback(null, awsKey, dataStoreVersionId);
};
const params = {
Bucket: this._awsBucketName,
Key: awsKey,
};
// we call data.put to create a delete marker, but it's actually a
// delete request in call to AWS
if (keyContext.isDeleteMarker) {
return this._client.deleteObject(params, putCb);
}
const uploadParams = params;
uploadParams.Metadata = metaHeaders;
uploadParams.ContentLength = size;
if (this._serverSideEncryption) {
uploadParams.ServerSideEncryption = 'AES256';
}
if (keyContext.tagging) {
uploadParams.Tagging = keyContext.tagging;
}
if (keyContext.contentType !== undefined) {
uploadParams.ContentType = keyContext.contentType;
}
if (keyContext.cacheControl !== undefined) {
uploadParams.CacheControl = keyContext.cacheControl;
}
if (keyContext.contentDisposition !== undefined) {
uploadParams.ContentDisposition = keyContext.contentDisposition;
}
if (keyContext.contentEncoding !== undefined) {
uploadParams.ContentEncoding = keyContext.contentEncoding;
}
if (!stream) {
return this._client.putObject(uploadParams, putCb);
}
uploadParams.Body = stream;
return this._client.upload(uploadParams, putCb);
}
head(objectGetInfo, reqUids, callback) {
const log = createLogger(reqUids);
const { key, dataStoreVersionId } = objectGetInfo;
return this._client.headObject({
Bucket: this._awsBucketName,
Key: key,
VersionId: dataStoreVersionId,
}, err => {
if (err) {
logHelper(log, 'error', 'error heading object ' +
'from datastore', err, this._dataStoreName);
if (err.code === 'NotFound') {
const error = errors.ServiceUnavailable
.customizeDescription(
`Unexpected error from ${this.type}: ` +
`"NotFound". Data on ${this.type} ` +
'may have been altered outside of CloudServer.'
);
return callback(error);
}
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
return callback();
});
}
get(objectGetInfo, range, reqUids, callback) {
const log = createLogger(reqUids);
const { key, dataStoreVersionId } = objectGetInfo;
const request = this._client.getObject({
Bucket: this._awsBucketName,
Key: key,
VersionId: dataStoreVersionId,
Range: range ? `bytes=${range[0]}-${range[1]}` : null,
}).on('success', response => {
log.trace(`${this.type} GET request response headers`,
{ responseHeaders: response.httpResponse.headers,
backendType: this.clientType });
});
const stream = request.createReadStream().on('error', err => {
logHelper(log, 'error',
`error streaming data from ${this.type}`,
err, this._dataStoreName, this.clientType);
return callback(err);
});
return callback(null, stream);
}
delete(objectGetInfo, reqUids, callback) {
const { key, dataStoreVersionId, deleteVersion } = objectGetInfo;
const log = createLogger(reqUids);
const params = {
Bucket: this._awsBucketName,
Key: key,
};
if (deleteVersion) {
params.VersionId = dataStoreVersionId;
}
return this._client.deleteObject(params, err => {
if (err) {
logHelper(log, 'error', 'error deleting object from ' +
'datastore', err, this._dataStoreName, this.clientType);
if (err.code === 'NoSuchVersion' || err.code === 'NoSuchKey') {
// data may have been deleted directly from the AWS backend
// don't want to retry the delete and errors are not
// sent back to client anyway, so no need to return err
return callback();
}
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
return callback();
});
}
healthcheck(location, callback) {
const awsResp = {};
this._client.headBucket({ Bucket: this._awsBucketName },
err => {
/* eslint-disable no-param-reassign */
if (err) {
awsResp[location] = { error: err, external: true };
return callback(null, awsResp);
}
if (!this._supportsVersioning) {
awsResp[location] = {
message: 'Congrats! You own the bucket',
};
return callback(null, awsResp);
}
return this._client.getBucketVersioning({
Bucket: this._awsBucketName },
(err, data) => {
if (err) {
awsResp[location] = { error: err, external: true };
} else if (!data.Status ||
data.Status === 'Suspended') {
awsResp[location] = {
versioningStatus: data.Status,
error: 'Versioning must be enabled',
external: true,
};
} else {
awsResp[location] = {
versioningStatus: data.Status,
message: 'Congrats! You own the bucket',
};
}
return callback(null, awsResp);
});
});
}
createMPU(key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
cacheControl, contentDisposition, contentEncoding, log, callback) {
const metaHeadersTrimmed = {};
Object.keys(metaHeaders).forEach(header => {
if (header.startsWith('x-amz-meta-')) {
const headerKey = header.substring(11);
metaHeadersTrimmed[headerKey] = metaHeaders[header];
}
});
const awsBucket = this._awsBucketName;
const awsKey = this._createAwsKey(bucketName, key, this._bucketMatch);
const params = {
Bucket: awsBucket,
Key: awsKey,
WebsiteRedirectLocation: websiteRedirectHeader,
Metadata: metaHeadersTrimmed,
ContentType: contentType,
CacheControl: cacheControl,
ContentDisposition: contentDisposition,
ContentEncoding: contentEncoding,
};
return this._client.createMultipartUpload(params, (err, mpuResObj) => {
if (err) {
logHelper(log, 'error', 'err from data backend',
err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
return callback(null, mpuResObj);
});
}
uploadPart(request, streamingV4Params, stream, size, key, uploadId,
partNumber, bucketName, config, log, callback) {
let hashedStream = stream;
if (request) {
const partStream = prepareStream(request, streamingV4Params,
config, log, callback);
hashedStream = new MD5Sum();
partStream.pipe(hashedStream);
}
const awsBucket = this._awsBucketName;
const awsKey = this._createAwsKey(bucketName, key, this._bucketMatch);
const params = { Bucket: awsBucket, Key: awsKey, UploadId: uploadId,
Body: hashedStream, ContentLength: size,
PartNumber: partNumber };
return this._client.uploadPart(params, (err, partResObj) => {
if (err) {
logHelper(log, 'error', 'err from data backend ' +
'on uploadPart', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
// Because we manually add quotes to ETag later, remove quotes here
const noQuotesETag =
partResObj.ETag.substring(1, partResObj.ETag.length - 1);
const dataRetrievalInfo = {
key: awsKey,
dataStoreType: 'aws_s3',
dataStoreName: this._dataStoreName,
dataStoreETag: noQuotesETag,
};
return callback(null, dataRetrievalInfo);
});
}
listParts(key, uploadId, bucketName, partNumberMarker, maxParts, log,
callback) {
const awsBucket = this._awsBucketName;
const awsKey = this._createAwsKey(bucketName, key, this._bucketMatch);
const params = { Bucket: awsBucket, Key: awsKey, UploadId: uploadId,
PartNumberMarker: partNumberMarker, MaxParts: maxParts };
return this._client.listParts(params, (err, partList) => {
if (err) {
logHelper(log, 'error', 'err from data backend on listPart',
err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
// build storedParts object to mimic Scality S3 backend returns
const storedParts = {};
storedParts.IsTruncated = partList.IsTruncated;
storedParts.Contents = [];
storedParts.Contents = partList.Parts.map(item => {
// We manually add quotes to ETag later, so remove quotes here
const noQuotesETag =
item.ETag.substring(1, item.ETag.length - 1);
return {
partNumber: item.PartNumber,
value: {
Size: item.Size,
ETag: noQuotesETag,
LastModified: item.LastModified,
},
};
});
return callback(null, storedParts);
});
}
/**
* completeMPU - complete multipart upload on AWS backend
* @param {object} jsonList - user-sent list of parts to include in
* final mpu object
* @param {object} mdInfo - object containing 3 keys: storedParts,
* mpuOverviewKey, and splitter
* @param {string} key - object key
* @param {string} uploadId - multipart upload id string
* @param {string} bucketName - name of bucket
* @param {RequestLogger} log - logger instance
* @param {function} callback - callback function
* @return {(Error|object)} - return Error if complete MPU fails, otherwise
* object containing completed object key, eTag, and contentLength
*/
completeMPU(jsonList, mdInfo, key, uploadId, bucketName, log, callback) {
const awsBucket = this._awsBucketName;
const awsKey = this._createAwsKey(bucketName, key, this._bucketMatch);
const mpuError = {
InvalidPart: true,
InvalidPartOrder: true,
EntityTooSmall: true,
};
const partArray = [];
const partList = jsonList.Part;
partList.forEach(partObj => {
const partParams = { PartNumber: partObj.PartNumber[0],
ETag: partObj.ETag[0] };
partArray.push(partParams);
});
const mpuParams = {
Bucket: awsBucket, Key: awsKey, UploadId: uploadId,
MultipartUpload: {
Parts: partArray,
},
};
const completeObjData = { key: awsKey };
return this._client.completeMultipartUpload(mpuParams,
(err, completeMpuRes) => {
if (err) {
if (mpuError[err.code]) {
logHelper(log, 'trace', 'err from data backend on ' +
'completeMPU', err, this._dataStoreName, this.clientType);
return callback(errors[err.code]);
}
logHelper(log, 'error', 'err from data backend on ' +
'completeMPU', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
if (!completeMpuRes.VersionId && this._supportsVersioning) {
logHelper(log, 'error', 'missing version id for data ' +
'backend object', missingVerIdInternalError,
this._dataStoreName, this.clientType);
return callback(missingVerIdInternalError);
}
// need to get content length of new object to store
// in our metadata
return this._client.headObject({ Bucket: awsBucket, Key: awsKey },
(err, objHeaders) => {
if (err) {
logHelper(log, 'trace', 'err from data backend on ' +
'headObject', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
// remove quotes from eTag because they're added later
completeObjData.eTag = completeMpuRes.ETag
.substring(1, completeMpuRes.ETag.length - 1);
completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
completeObjData.contentLength =
Number.parseInt(objHeaders.ContentLength, 10);
return callback(null, completeObjData);
});
});
}
abortMPU(key, uploadId, bucketName, log, callback) {
const awsBucket = this._awsBucketName;
const awsKey = this._createAwsKey(bucketName, key, this._bucketMatch);
const abortParams = {
Bucket: awsBucket, Key: awsKey, UploadId: uploadId,
};
return this._client.abortMultipartUpload(abortParams, err => {
if (err) {
logHelper(log, 'error', 'There was an error aborting ' +
'the MPU on AWS S3. You should abort directly on AWS S3 ' +
'using the same uploadId.', err, this._dataStoreName,
this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
return callback();
});
}
objectPutTagging(key, bucket, objectMD, log, callback) {
const awsBucket = this._awsBucketName;
const awsKey = this._createAwsKey(bucket, key, this._bucketMatch);
const dataStoreVersionId = objectMD.location[0].dataStoreVersionId;
const tagParams = {
Bucket: awsBucket,
Key: awsKey,
VersionId: dataStoreVersionId,
};
const keyArray = Object.keys(objectMD.tags);
tagParams.Tagging = {};
tagParams.Tagging.TagSet = keyArray.map(key => {
const value = objectMD.tags[key];
return { Key: key, Value: value };
});
return this._client.putObjectTagging(tagParams, err => {
if (err) {
logHelper(log, 'error', 'error from data backend on ' +
'putObjectTagging', err,
this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
return callback();
});
}
objectDeleteTagging(key, bucket, objectMD, log, callback) {
const awsBucket = this._awsBucketName;
const awsKey = this._createAwsKey(bucket, key, this._bucketMatch);
const dataStoreVersionId = objectMD.location[0].dataStoreVersionId;
const tagParams = {
Bucket: awsBucket,
Key: awsKey,
VersionId: dataStoreVersionId,
};
return this._client.deleteObjectTagging(tagParams, err => {
if (err) {
logHelper(log, 'error', 'error from data backend on ' +
'deleteObjectTagging', err,
this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
return callback();
});
}
copyObject(request, destLocationConstraintName, sourceKey,
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
const destBucketName = request.bucketName;
const destObjectKey = request.objectKey;
const destAwsKey = this._createAwsKey(destBucketName, destObjectKey,
this._bucketMatch);
const sourceAwsBucketName =
config.getAwsBucketName(sourceLocationConstraintName);
const metadataDirective = request.headers['x-amz-metadata-directive'];
const metaHeaders = trimXMetaPrefix(getMetaHeaders(request.headers));
const awsParams = {
Bucket: this._awsBucketName,
Key: destAwsKey,
CopySource: `${sourceAwsBucketName}/${sourceKey}`,
Metadata: metaHeaders,
MetadataDirective: metadataDirective,
};
if (destLocationConstraintName &&
config.isAWSServerSideEncrytion(destLocationConstraintName)) {
awsParams.ServerSideEncryption = 'AES256';
}
this._client.copyObject(awsParams, (err, copyResult) => {
if (err) {
if (err.code === 'AccessDenied') {
logHelper(log, 'error', 'Unable to access ' +
`${sourceAwsBucketName} ${this.type} bucket`, err,
this._dataStoreName, this.clientType);
return callback(errors.AccessDenied
.customizeDescription('Error: Unable to access ' +
`${sourceAwsBucketName} ${this.type} bucket`)
);
}
logHelper(log, 'error', 'error from data backend on ' +
'copyObject', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
if (!copyResult.VersionId && this._supportsVersioning) {
this._logger.debug('No VersionId found in response, ' +
'calling headObject to resolve');
return this._client.headObject({
Bucket: this._awsBucketName,
Key: destAwsKey,
}, (err, data) => {
if (err || !data.VersionId) {
logHelper(log, 'error', 'missing version id for data ' +
'backend object', missingVerIdInternalError,
this._dataStoreName, this.clientType);
return callback(missingVerIdInternalError);
}
return callback(null, destAwsKey, data.VersionId);
});
}
return callback(null, destAwsKey, copyResult.VersionId);
});
}
uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName,
config, log, callback) {
const destBucketName = request.bucketName;
const destObjectKey = request.objectKey;
const destAwsKey = this._createAwsKey(destBucketName, destObjectKey,
this._bucketMatch);
const sourceAwsBucketName =
config.getAwsBucketName(sourceLocationConstraintName);
const uploadId = request.query.uploadId;
const partNumber = request.query.partNumber;
const copySourceRange = request.headers['x-amz-copy-source-range'];
const params = {
Bucket: this._awsBucketName,
CopySource: `${sourceAwsBucketName}/${awsSourceKey}`,
CopySourceRange: copySourceRange,
Key: destAwsKey,
PartNumber: partNumber,
UploadId: uploadId,
};
return this._client.uploadPartCopy(params, (err, res) => {
if (err) {
if (err.code === 'AccessDenied') {
logHelper(log, 'error', 'Unable to access ' +
`${sourceAwsBucketName} AWS bucket`, err,
this._dataStoreName, this.clientType);
return callback(errors.AccessDenied
.customizeDescription('Error: Unable to access ' +
`${sourceAwsBucketName} AWS bucket`)
);
}
logHelper(log, 'error', 'error from data backend on ' +
'uploadPartCopy', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`${this.type}: ${err.message}`)
);
}
const eTag = removeQuotes(res.CopyPartResult.ETag);
return callback(null, eTag);
});
}
}
module.exports = AwsClient;

450
lib/storage/data/external/AzureClient.js vendored Normal file
View File

@ -0,0 +1,450 @@
const url = require('url');
const azure = require('azure-storage');
const errors = require('../../../errors');
const azureMpuUtils = require('../../../s3middleware/azureHelpers/mpuUtils');
const { validateAndFilterMpuParts } =
require('../../../s3middleware/processMpuParts');
const { createLogger, logHelper, translateAzureMetaHeaders } =
require('./utils');
const constants = require('../../../constants');
const packageVersion = require('../../../../package.json').version;
azure.Constants.USER_AGENT_PRODUCT_NAME = constants.productName;
azure.Constants.USER_AGENT_PRODUCT_VERSION = packageVersion;
class AzureClient {
constructor(config) {
this._azureStorageEndpoint = config.azureStorageEndpoint;
this._azureStorageCredentials = config.azureStorageCredentials;
this._azureContainerName = config.azureContainerName;
this._client = azure.createBlobService(
this._azureStorageCredentials.storageAccountName,
this._azureStorageCredentials.storageAccessKey,
this._azureStorageEndpoint);
this._dataStoreName = config.dataStoreName;
this._bucketMatch = config.bucketMatch;
if (config.proxy && config.proxy.url) {
const parsedUrl = url.parse(config.proxy.url);
if (!parsedUrl.port) {
parsedUrl.port = 80;
}
const proxyParams = parsedUrl;
if (config.proxy.certs) {
Object.assign(proxyParams, config.proxy.certs);
}
this._client.setProxy(proxyParams);
}
}
_errorWrapper(s3Method, azureMethod, args, log, cb) {
if (log) {
log.info(`calling azure ${azureMethod}`);
}
try {
this._client[azureMethod].apply(this._client, args);
} catch (err) {
const error = errors.ServiceUnavailable;
if (log) {
log.error('error thrown by Azure Storage Client Library',
{ error: err.message, stack: err.stack, s3Method,
azureMethod, dataStoreName: this._dataStoreName });
}
cb(error.customizeDescription('Error from Azure ' +
`method: ${azureMethod} on ${s3Method} S3 call: ` +
`${err.message}`));
}
}
_createAzureKey(requestBucketName, requestObjectKey,
bucketMatch) {
if (bucketMatch) {
return requestObjectKey;
}
return `${requestBucketName}/${requestObjectKey}`;
}
_getMetaHeaders(objectMD) {
const metaHeaders = {};
Object.keys(objectMD).forEach(mdKey => {
const isMetaHeader = mdKey.startsWith('x-amz-meta-');
if (isMetaHeader) {
metaHeaders[mdKey] = objectMD[mdKey];
}
});
return translateAzureMetaHeaders(metaHeaders);
}
// Before putting or deleting object on Azure, check if MPU exists with
// same key name. If it does, do not allow put or delete because Azure
// will delete all blocks with same key name
protectAzureBlocks(metadata, bucketName, objectKey, dataStoreName,
log, cb) {
const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`;
const splitter = constants.splitter;
const listingParams = {
prefix: `overview${splitter}${objectKey}`,
listingType: 'MPU',
splitter,
maxKeys: 1,
};
return metadata.listMultipartUploads(mpuBucketName, listingParams,
log, (err, mpuList) => {
if (err && !err.NoSuchBucket) {
log.error('Error listing MPUs for Azure delete',
{ error: err, dataStoreName });
return cb(errors.ServiceUnavailable);
}
if (mpuList && mpuList.Uploads && mpuList.Uploads.length > 0) {
const error = errors.MPUinProgress;
log.error('Error: cannot put/delete object to Azure with ' +
'same key name as ongoing MPU on Azure',
{ error, dataStoreName });
return cb(error);
}
// If listMultipartUploads returns a NoSuchBucket error or the
// mpu list is empty, there are no conflicting MPUs, so continue
return cb();
});
}
toObjectGetInfo(objectKey, bucketName) {
return {
key: this._createAzureKey(bucketName, objectKey, this._bucketMatch),
dataStoreName: this._dataStoreName,
};
}
put(stream, size, keyContext, reqUids, callback, metadata) {
const log = createLogger(reqUids);
// before blob is put, make sure there is no ongoing MPU with same key
this.protectAzureBlocks(metadata, keyContext.bucketName,
keyContext.objectKey, this._dataStoreName, log, err => {
// if error returned, there is ongoing MPU, so do not put
if (err) {
return callback(err.customizeDescription(
`Error putting object to Azure: ${err.message}`));
}
const azureKey = this._createAzureKey(keyContext.bucketName,
keyContext.objectKey, this._bucketMatch);
const options = {
metadata: translateAzureMetaHeaders(keyContext.metaHeaders,
keyContext.tagging),
contentSettings: {
contentType: keyContext.contentType || undefined,
cacheControl: keyContext.cacheControl || undefined,
contentDisposition: keyContext.contentDisposition ||
undefined,
contentEncoding: keyContext.contentEncoding || undefined,
},
};
if (size === 0) {
return this._errorWrapper('put', 'createBlockBlobFromText',
[this._azureContainerName, azureKey, '', options,
err => {
if (err) {
logHelper(log, 'error', 'err from Azure PUT data ' +
'backend', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`Azure: ${err.message}`));
}
return callback(null, azureKey);
}], log, callback);
}
return this._errorWrapper('put', 'createBlockBlobFromStream',
[this._azureContainerName, azureKey, stream, size, options,
err => {
if (err) {
logHelper(log, 'error', 'err from Azure PUT data ' +
'backend', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`Azure: ${err.message}`));
}
return callback(null, azureKey);
}], log, callback);
});
}
head(objectGetInfo, reqUids, callback) {
const log = createLogger(reqUids);
const { key, azureStreamingOptions } = objectGetInfo;
return this._errorWrapper('head', 'getBlobProperties',
[this._azureContainerName, key, azureStreamingOptions,
err => {
if (err) {
logHelper(log, 'error', 'err from Azure HEAD data backend',
err, this._dataStoreName);
if (err.code === 'NotFound') {
const error = errors.ServiceUnavailable
.customizeDescription(
'Unexpected error from Azure: "NotFound". Data ' +
'on Azure may have been altered outside of ' +
'CloudServer.');
return callback(error);
}
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`Azure: ${err.message}`));
}
return callback();
}], log, callback);
}
get(objectGetInfo, range, reqUids, callback) {
const log = createLogger(reqUids);
// for backwards compatibility
const { key, response, azureStreamingOptions } = objectGetInfo;
let streamingOptions;
if (azureStreamingOptions) {
// option coming from api.get()
streamingOptions = azureStreamingOptions;
} else if (range) {
// option coming from multipleBackend.upload()
const rangeStart = range[0] ? range[0].toString() : undefined;
const rangeEnd = range[1] ? range[1].toString() : undefined;
streamingOptions = { rangeStart, rangeEnd };
}
this._errorWrapper('get', 'getBlobToStream',
[this._azureContainerName, key, response, streamingOptions,
err => {
if (err) {
logHelper(log, 'error', 'err from Azure GET data backend',
err, this._dataStoreName);
return callback(errors.ServiceUnavailable);
}
return callback(null, response);
}], log, callback);
}
delete(objectGetInfo, reqUids, callback) {
const log = createLogger(reqUids);
// for backwards compatibility
const key = typeof objectGetInfo === 'string' ? objectGetInfo :
objectGetInfo.key;
return this._errorWrapper('delete', 'deleteBlobIfExists',
[this._azureContainerName, key,
err => {
if (err) {
const log = createLogger(reqUids);
logHelper(log, 'error', 'error deleting object from ' +
'Azure datastore', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`Azure: ${err.message}`));
}
return callback();
}], log, callback);
}
healthcheck(location, callback, flightCheckOnStartUp) {
const azureResp = {};
const healthCheckAction = flightCheckOnStartUp ?
'createContainerIfNotExists' : 'doesContainerExist';
this._errorWrapper('checkAzureHealth', healthCheckAction,
[this._azureContainerName, err => {
/* eslint-disable no-param-reassign */
if (err) {
azureResp[location] = { error: err.message,
external: true };
return callback(null, azureResp);
}
azureResp[location] = {
message:
'Congrats! You can access the Azure storage account',
};
return callback(null, azureResp);
}], null, callback);
}
uploadPart(request, streamingV4Params, partStream, size, key, uploadId,
partNumber, bucket, config, log, callback) {
const azureKey = this._createAzureKey(bucket, key, this._bucketMatch);
const params = { bucketName: this._azureContainerName,
partNumber, size, objectKey: azureKey, uploadId };
const stream = request || partStream;
if (request && request.headers && request.headers['content-md5']) {
params.contentMD5 = request.headers['content-md5'];
}
const dataRetrievalInfo = {
key: azureKey,
partNumber,
dataStoreName: this._dataStoreName,
dataStoreType: 'azure',
numberSubParts: azureMpuUtils.getSubPartInfo(size)
.expectedNumberSubParts,
};
if (size === 0) {
log.debug('0-byte part does not store data',
{ method: 'uploadPart' });
dataRetrievalInfo.dataStoreETag = azureMpuUtils.zeroByteETag;
dataRetrievalInfo.numberSubParts = 0;
return callback(null, dataRetrievalInfo);
}
if (size <= azureMpuUtils.maxSubPartSize) {
const errorWrapperFn = this._errorWrapper.bind(this);
return azureMpuUtils.putSinglePart(errorWrapperFn,
stream, params, this._dataStoreName, log, (err, dataStoreETag) => {
if (err) {
return callback(err);
}
dataRetrievalInfo.dataStoreETag = dataStoreETag;
return callback(null, dataRetrievalInfo);
});
}
const errorWrapperFn = this._errorWrapper.bind(this);
return azureMpuUtils.putSubParts(errorWrapperFn, stream,
params, this._dataStoreName, log, (err, dataStoreETag) => {
if (err) {
return callback(err);
}
dataRetrievalInfo.dataStoreETag = dataStoreETag;
return callback(null, dataRetrievalInfo);
});
}
completeMPU(jsonList, mdInfo, key, uploadId, bucket, metaHeaders,
contentSettings, log, callback) {
const azureKey = this._createAzureKey(bucket, key, this._bucketMatch);
const commitList = {
UncommittedBlocks: jsonList.uncommittedBlocks || [],
};
let filteredPartsObj;
if (!jsonList.uncommittedBlocks) {
const { storedParts, mpuOverviewKey, splitter } = mdInfo;
filteredPartsObj = validateAndFilterMpuParts(storedParts, jsonList,
mpuOverviewKey, splitter, log);
filteredPartsObj.partList.forEach(part => {
// part.locations is always array of 1, which contains data info
const subPartIds =
azureMpuUtils.getSubPartIds(part.locations[0], uploadId);
commitList.UncommittedBlocks.push(...subPartIds);
});
}
const options = {
contentSettings,
metadata: metaHeaders ? translateAzureMetaHeaders(metaHeaders) :
undefined,
};
this._errorWrapper('completeMPU', 'commitBlocks',
[this._azureContainerName, azureKey, commitList, options,
err => {
if (err) {
logHelper(log, 'error', 'err completing MPU on Azure ' +
'datastore', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`Azure: ${err.message}`));
}
const completeObjData = {
key: azureKey,
filteredPartsObj,
};
return callback(null, completeObjData);
}], log, callback);
}
objectPutTagging(key, bucket, objectMD, log, callback) {
const azureKey = this._createAzureKey(bucket, key, this._bucketMatch);
const azureMD = this._getMetaHeaders(objectMD);
azureMD.tags = JSON.stringify(objectMD.tags);
this._errorWrapper('objectPutTagging', 'setBlobMetadata',
[this._azureContainerName, azureKey, azureMD,
err => {
if (err) {
logHelper(log, 'error', 'err putting object tags to ' +
'Azure backend', err, this._dataStoreName);
return callback(errors.ServiceUnavailable);
}
return callback();
}], log, callback);
}
objectDeleteTagging(key, bucket, objectMD, log, callback) {
const azureKey = this._createAzureKey(bucket, key, this._bucketMatch);
const azureMD = this._getMetaHeaders(objectMD);
this._errorWrapper('objectDeleteTagging', 'setBlobMetadata',
[this._azureContainerName, azureKey, azureMD,
err => {
if (err) {
logHelper(log, 'error', 'err putting object tags to ' +
'Azure backend', err, this._dataStoreName);
return callback(errors.ServiceUnavailable);
}
return callback();
}], log, callback);
}
copyObject(request, destLocationConstraintName, sourceKey,
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
const destContainerName = request.bucketName;
const destObjectKey = request.objectKey;
const destAzureKey = this._createAzureKey(destContainerName,
destObjectKey, this._bucketMatch);
const sourceContainerName =
config.locationConstraints[sourceLocationConstraintName]
.details.azureContainerName;
let options;
if (storeMetadataParams.metaHeaders) {
options = { metadata:
translateAzureMetaHeaders(storeMetadataParams.metaHeaders) };
}
this._errorWrapper('copyObject', 'startCopyBlob',
[`${this._azureStorageEndpoint}` +
`${sourceContainerName}/${sourceKey}`,
this._azureContainerName, destAzureKey, options,
(err, res) => {
if (err) {
if (err.code === 'CannotVerifyCopySource') {
logHelper(log, 'error', 'Unable to access ' +
`${sourceContainerName} Azure Container`, err,
this._dataStoreName);
return callback(errors.AccessDenied
.customizeDescription('Error: Unable to access ' +
`${sourceContainerName} Azure Container`)
);
}
logHelper(log, 'error', 'error from data backend on ' +
'copyObject', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`AWS: ${err.message}`)
);
}
if (res.copy.status === 'pending') {
logHelper(log, 'error', 'Azure copy status is pending',
err, this._dataStoreName);
const copyId = res.copy.id;
this._client.abortCopyBlob(this._azureContainerName,
destAzureKey, copyId, err => {
if (err) {
logHelper(log, 'error', 'error from data backend ' +
'on abortCopyBlob', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`AWS on abortCopyBlob: ${err.message}`)
);
}
return callback(errors.InvalidObjectState
.customizeDescription('Error: Azure copy status was ' +
'pending. It has been aborted successfully')
);
});
}
return callback(null, destAzureKey);
}], log, callback);
}
}
module.exports = AzureClient;

View File

@ -0,0 +1,33 @@
const errors = require('../../../../../errors');
const MpuHelper = require('./mpuHelper');
const { createMpuKey, logger } = require('../GcpUtils');
const { logHelper } = require('../../utils');
/**
* abortMPU - remove all objects of a GCP Multipart Upload
* @param {object} params - abortMPU params
* @param {string} params.Bucket - bucket name
* @param {string} params.MPU - mpu bucket name
* @param {string} params.Key - object key
* @param {number} params.UploadId - MPU upload id
* @param {function} callback - callback function to call
* @return {undefined}
*/
function abortMPU(params, callback) {
if (!params || !params.Key || !params.UploadId ||
!params.Bucket || !params.MPU) {
const error = errors.InvalidRequest
.customizeDescription('Missing required parameter');
logHelper(logger, 'error', 'error in abortMultipartUpload', error);
return callback(error);
}
const mpuHelper = new MpuHelper(this);
const delParams = {
Bucket: params.Bucket,
MPU: params.MPU,
Prefix: createMpuKey(params.Key, params.UploadId),
};
return mpuHelper.removeParts(delParams, callback);
}
module.exports = abortMPU;

View File

@ -0,0 +1,69 @@
const async = require('async');
const errors = require('../../../../../errors');
const MpuHelper = require('./mpuHelper');
const { createMpuKey, logger } = require('../GcpUtils');
const { logHelper } = require('../../utils');
/**
* completeMPU - merges a list of parts into a single object
* @param {object} params - completeMPU params
* @param {string} params.Bucket - bucket name
* @param {string} params.MPU - mpu bucket name
* @param {string} params.Key - object key
* @param {number} params.UploadId - MPU upload id
* @param {Object} params.MultipartUpload - MPU upload object
* @param {Object[]} param.MultipartUpload.Parts - a list of parts to merge
* @param {function} callback - callback function to call with MPU result
* @return {undefined}
*/
function completeMPU(params, callback) {
if (!params || !params.MultipartUpload ||
!params.MultipartUpload.Parts || !params.UploadId ||
!params.Bucket || !params.Key) {
const error = errors.InvalidRequest
.customizeDescription('Missing required parameter');
logHelper(logger, 'error', 'error in completeMultipartUpload', error);
return callback(error);
}
const partList = params.MultipartUpload.Parts;
// verify that the part list is in order
if (params.MultipartUpload.Parts.length === 0) {
const error = errors.InvalidRequest
.customizeDescription('You must specify at least one part');
logHelper(logger, 'error', 'error in completeMultipartUpload', error);
return callback(error);
}
for (let ind = 1; ind < partList.length; ++ind) {
if (partList[ind - 1].PartNumber >= partList[ind].PartNumber) {
logHelper(logger, 'error', 'error in completeMultipartUpload',
errors.InvalidPartOrder);
return callback(errors.InvalidPartOrder);
}
}
const mpuHelper = new MpuHelper(this); // this === GcpClient
return async.waterfall([
next => {
// first compose: in mpu bucket
// max 10,000 => 313 parts
// max component count per object 32
logger.trace('completeMultipartUpload: compose',
{ partCount: partList.length });
mpuHelper.splitMerge(params, partList, 'compose', next);
},
(numParts, next) => mpuHelper.composeFinal(numParts, params, next),
(result, next) => mpuHelper.generateMpuResult(result, partList, next),
(result, aggregateETag, next) =>
mpuHelper.copyToMain(result, aggregateETag, params, next),
(mpuResult, next) => {
const delParams = {
Bucket: params.Bucket,
MPU: params.MPU,
Prefix: createMpuKey(params.Key, params.UploadId),
};
return mpuHelper.removeParts(delParams,
err => next(err, mpuResult));
},
], callback);
}
module.exports = completeMPU;

View File

@ -0,0 +1,51 @@
const uuid = require('uuid/v4');
const errors = require('../../../../../errors');
const { createMpuKey, logger, getPutTagsMetadata } = require('../GcpUtils');
const { logHelper } = require('../../utils');
/**
* createMPU - creates a MPU upload on GCP (sets a 0-byte object placeholder
* with for the final composed object)
* @param {object} params - createMPU param
* @param {string} params.Bucket - bucket name
* @param {string} params.Key - object key
* @param {string} params.Metadata - object Metadata
* @param {string} params.ContentType - Content-Type header
* @param {string} params.CacheControl - Cache-Control header
* @param {string} params.ContentDisposition - Content-Disposition header
* @param {string} params.ContentEncoding - Content-Encoding header
* @param {function} callback - callback function to call with the generated
* upload-id for MPU operations
* @return {undefined}
*/
function createMPU(params, callback) {
// As google cloud does not have a create MPU function,
// create an empty 'init' object that will temporarily store the
// object metadata and return an upload ID to mimic an AWS MPU
if (!params || !params.Bucket || !params.Key) {
const error = errors.InvalidRequest
.customizeDescription('Missing required parameter');
logHelper(logger, 'error', 'error in createMultipartUpload', error);
return callback(error);
}
const uploadId = uuid().replace(/-/g, '');
const mpuParams = {
Bucket: params.Bucket,
Key: createMpuKey(params.Key, uploadId, 'init'),
Metadata: params.Metadata,
ContentType: params.ContentType,
CacheControl: params.CacheControl,
ContentDisposition: params.ContentDisposition,
ContentEncoding: params.ContentEncoding,
};
mpuParams.Metadata = getPutTagsMetadata(mpuParams.Metadata, params.Tagging);
return this.putObject(mpuParams, err => {
if (err) {
logHelper(logger, 'error', 'error in createMPU - putObject', err);
return callback(err);
}
return callback(null, { UploadId: uploadId });
});
}
module.exports = createMPU;

View File

@ -0,0 +1,24 @@
const async = require('async');
const { stripTags } = require('../GcpUtils');
function deleteObjectTagging(params, callback) {
return async.waterfall([
next => this.headObject({
Bucket: params.Bucket,
Key: params.Key,
VersionId: params.VersionId,
}, next),
(resObj, next) => {
const completeMD = stripTags(resObj.Metadata);
this.copyObject({
Bucket: params.Bucket,
Key: params.Key,
CopySource: `${params.Bucket}/${params.Key}`,
Metadata: completeMD,
MetadataDirective: 'REPLACE',
}, next);
},
], callback);
}
module.exports = deleteObjectTagging;

View File

@ -0,0 +1,19 @@
const { retrieveTags } = require('../GcpUtils');
function getObjectTagging(params, callback) {
const headParams = {
Bucket: params.Bucket,
Key: params.Key,
VersionId: params.VersionId,
};
this.headObject(headParams, (err, res) => {
const TagSet = retrieveTags(res.Metadata);
const retObj = {
VersionId: res.VersionId,
TagSet,
};
return callback(null, retObj);
});
}
module.exports = getObjectTagging;

View File

@ -0,0 +1,14 @@
module.exports = {
// mpu functions
abortMultipartUpload: require('./abortMPU'),
completeMultipartUpload: require('./completeMPU'),
createMultipartUpload: require('./createMPU'),
listParts: require('./listParts'),
uploadPart: require('./uploadPart'),
uploadPartCopy: require('./uploadPartCopy'),
// object tagging
putObject: require('./putObject'),
putObjectTagging: require('./putTagging'),
getObjectTagging: require('./getTagging'),
deleteObjectTagging: require('./deleteTagging'),
};

View File

@ -0,0 +1,42 @@
const errors = require('../../../../../errors');
const { createMpuKey, logger } = require('../GcpUtils');
const { logHelper } = require('../../utils');
/**
* listParts - list uploaded MPU parts
* @param {object} params - listParts param
* @param {string} params.Bucket - bucket name
* @param {string} params.Key - object key
* @param {string} params.UploadId - MPU upload id
* @param {function} callback - callback function to call with the list of parts
* @return {undefined}
*/
function listParts(params, callback) {
if (!params || !params.UploadId || !params.Bucket || !params.Key) {
const error = errors.InvalidRequest
.customizeDescription('Missing required parameter');
logHelper(logger, 'error', 'error in listParts', error);
return callback(error);
}
if (params.PartNumberMarker && params.PartNumberMarker < 0) {
return callback(errors.InvalidArgument
.customizeDescription('The request specified an invalid marker'));
}
const mpuParams = {
Bucket: params.Bucket,
Prefix: createMpuKey(params.Key, params.UploadId, 'parts'),
Marker: createMpuKey(params.Key, params.UploadId,
params.PartNumberMarker, 'parts'),
MaxKeys: params.MaxParts,
};
return this.listObjects(mpuParams, (err, res) => {
if (err) {
logHelper(logger, 'error',
'error in listParts - listObjects', err);
return callback(err);
}
return callback(null, res);
});
}
module.exports = listParts;

View File

@ -0,0 +1,316 @@
const async = require('async');
const Backoff = require('backo');
const { eachSlice, createMpuKey, createMpuList, logger } =
require('../GcpUtils');
const { logHelper } = require('../../utils');
const { createAggregateETag } =
require('../../../../../s3middleware/processMpuParts');
const BACKOFF_PARAMS = { min: 1000, max: 300000, jitter: 0.1, factor: 1.5 };
class MpuHelper {
constructor(service, options = {}) {
this.service = service;
this.backoffParams = {
min: options.min || BACKOFF_PARAMS.min,
max: options.max || BACKOFF_PARAMS.max,
jitter: options.jitter || BACKOFF_PARAMS.jitter,
factor: options.factor || BACKOFF_PARAMS.factor,
};
}
_retry(fnName, params, callback) {
const backoff = new Backoff(this.backoffParams);
const handleFunc = (fnName, params, retry, callback) => {
const timeout = backoff.duration();
return setTimeout((params, cb) =>
this.service[fnName](params, cb), timeout, params,
(err, res) => {
if (err) {
if (err.statusCode === 429 || err.code === 429) {
if (fnName === 'composeObject') {
logger.trace('composeObject: slow down request',
{ retryCount: retry, timeout });
} else if (fnName === 'copyObject') {
logger.trace('copyObject: slow down request',
{ retryCount: retry, timeout });
}
return handleFunc(
fnName, params, retry + 1, callback);
}
logHelper(logger, 'error', `${fnName} failed`, err);
return callback(err);
}
backoff.reset();
return callback(null, res);
});
};
handleFunc(fnName, params, 0, callback);
}
/**
* retryCompose - exponential backoff retry implementation for the compose
* operation
* @param {object} params - compose object params
* @param {function} callback - callback function to call with the result
* of the compose operation
* @return {undefined}
*/
retryCompose(params, callback) {
this._retry('composeObject', params, callback);
}
/**
* retryCopy - exponential backoff retry implementation for the copy
* operation
* @param {object} params - copy object params
* @param {function} callback - callback function to call with the result
* of the copy operation
* @return {undefined}
*/
retryCopy(params, callback) {
this._retry('copyObject', params, callback);
}
/**
* splitMerge - breaks down the MPU list of parts to be compose on GCP;
* splits partList into chunks of 32 objects, the limit of each compose
* operation.
* @param {object} params - complete MPU params
* @param {string} params.Bucket - bucket name
* @param {string} params.MPU - mpu bucket name
* @param {string} params.Key - object key
* @param {string} params.UploadId - MPU upload id
* @param {object[]} partList - list of parts for complete multipart upload
* @param {string} level - the phase name of the MPU process
* @param {function} callback - the callback function to call
* @return {undefined}
*/
splitMerge(params, partList, level, callback) {
// create composition of slices from the partList array
return async.mapLimit(eachSlice.call(partList, 32),
this.service._maxConcurrent,
(infoParts, cb) => {
const mpuPartList = infoParts.Parts.map(item =>
({ PartName: item.PartName }));
const partNumber = infoParts.PartNumber;
const tmpKey =
createMpuKey(params.Key, params.UploadId, partNumber, level);
const mergedObject = { PartName: tmpKey };
if (mpuPartList.length < 2) {
logger.trace(
'splitMerge: parts are fewer than 2, copy instead');
// else just perform a copy
const copyParams = {
Bucket: params.MPU,
Key: tmpKey,
CopySource: `${params.MPU}/${mpuPartList[0].PartName}`,
};
return this.service.copyObject(copyParams, (err, res) => {
if (err) {
logHelper(logger, 'error',
'error in splitMerge - copyObject', err);
return cb(err);
}
mergedObject.VersionId = res.VersionId;
mergedObject.ETag = res.ETag;
return cb(null, mergedObject);
});
}
const composeParams = {
Bucket: params.MPU,
Key: tmpKey,
MultipartUpload: { Parts: mpuPartList },
};
return this.retryCompose(composeParams, (err, res) => {
if (err) {
return cb(err);
}
mergedObject.VersionId = res.VersionId;
mergedObject.ETag = res.ETag;
return cb(null, mergedObject);
});
}, (err, res) => {
if (err) {
return callback(err);
}
return callback(null, res.length);
});
}
/**
* removeParts - remove all objects created to perform a multipart upload
* @param {object} params - remove parts params
* @param {string} params.Bucket - bucket name
* @param {string} params.MPU - mpu bucket name
* @param {string} params.Key - object key
* @param {string} params.UploadId - MPU upload id
* @param {function} callback - callback function to call
* @return {undefined}
*/
removeParts(params, callback) {
const _getObjectVersions = callback => {
logger.trace('remove all parts from mpu bucket');
let partsList = [];
let isTruncated = true;
let nextMarker;
return async.whilst(() => isTruncated, next => {
const listParams = {
Bucket: params.MPU,
Prefix: params.Prefix,
Marker: nextMarker,
};
return this.service.listVersions(listParams, (err, res) => {
if (err) {
logHelper(logger, 'error', 'error in ' +
'removeParts - listVersions', err);
return next(err);
}
nextMarker = res.NextMarker;
isTruncated = res.IsTruncated;
partsList = partsList.concat(res.Versions);
return next();
});
}, err => callback(err, partsList));
};
const _deleteObjects = (partsList, callback) => {
logger.trace('successfully listed mpu parts', {
objectCount: partsList.length,
});
return async.eachLimit(partsList, 10, (obj, next) => {
const delParams = {
Bucket: params.MPU,
Key: obj.Key,
VersionId: obj.VersionId,
};
this.service.deleteObject(delParams, err => {
if (err) {
logHelper(logger, 'error',
'error deleting object', err);
return next(err);
}
return next();
});
}, err => callback(err));
};
return async.waterfall([
_getObjectVersions,
_deleteObjects,
], err => callback(err));
}
composeFinal(numParts, params, callback) {
// final compose:
// number of parts to compose <= 10
// perform final compose in mpu bucket
logger.trace('completeMultipartUpload: final compose');
const parts = createMpuList(params, 'compose', numParts);
const partList = parts.map(item => (
{ PartName: item.PartName }));
if (partList.length < 2) {
logger.trace(
'fewer than 2 parts, skip to copy phase');
return callback(null, partList[0].PartName);
}
const composeParams = {
Bucket: params.MPU,
Key: createMpuKey(params.Key, params.UploadId, 'final'),
MultipartUpload: { Parts: partList },
};
return this.retryCompose(composeParams, err => {
if (err) {
return callback(err);
}
return callback(null, null);
});
}
/*
* Create MPU Aggregate ETag
*/
generateMpuResult(res, partList, callback) {
const concatETag = partList.reduce((prev, curr) =>
prev + curr.ETag.substring(1, curr.ETag.length - 1), '');
const aggregateETag = createAggregateETag(concatETag, partList);
return callback(null, res, aggregateETag);
}
copyToMain(res, aggregateETag, params, callback) {
// move object from mpu bucket into the main bucket
// retrieve initial metadata then compose the object
const copySource = res ||
createMpuKey(params.Key, params.UploadId, 'final');
return async.waterfall([
next => {
// retrieve metadata from init object in mpu bucket
const headParams = {
Bucket: params.MPU,
Key: createMpuKey(params.Key, params.UploadId,
'init'),
};
logger.trace('retrieving object metadata');
return this.service.headObject(headParams, (err, res) => {
if (err) {
logHelper(logger, 'error',
'error in createMultipartUpload - headObject',
err);
return next(err);
}
return next(null, res);
});
},
(res, next) => {
const metadata = res.Metadata;
// copy the final object into the main bucket
const copyMetadata = Object.assign({}, metadata);
copyMetadata['scal-etag'] = aggregateETag;
const copyParams = {
Bucket: params.Bucket,
Key: params.Key,
Metadata: copyMetadata,
MetadataDirective: 'REPLACE',
CopySource: `${params.MPU}/${copySource}`,
ContentType: res.ContentType,
CacheControl: res.CacheControl,
ContentEncoding: res.ContentEncoding,
ContentDisposition: res.ContentDisposition,
ContentLanguage: res.ContentLanguage,
};
logger.trace('copyParams', { copyParams });
this.retryCopy(copyParams, (err, res) => {
if (err) {
logHelper(logger, 'error', 'error in ' +
'createMultipartUpload - final copyObject',
err);
return next(err);
}
const mpuResult = {
Bucket: params.Bucket,
Key: params.Key,
VersionId: res.VersionId,
ETag: `"${aggregateETag}"`,
};
return this.service.headObject({
Bucket: params.Bucket,
Key: params.Key,
VersionId: res.VersionId,
}, (err, res) => {
if (err) {
logHelper(logger, 'error', 'error in ' +
'createMultipartUpload - final head object',
err);
return next(err);
}
mpuResult.ContentLength = res.ContentLength;
return next(null, mpuResult);
});
});
},
], callback);
}
}
module.exports = MpuHelper;

View File

@ -0,0 +1,11 @@
const { getPutTagsMetadata } = require('../GcpUtils');
function putObject(params, callback) {
const putParams = Object.assign({}, params);
putParams.Metadata = getPutTagsMetadata(putParams.Metadata, params.Tagging);
delete putParams.Tagging;
// error handling will be by the actual putObject request
return this.putObjectReq(putParams, callback);
}
module.exports = putObject;

View File

@ -0,0 +1,33 @@
const async = require('async');
const errors = require('../../../../../errors');
const { processTagSet } = require('../GcpUtils');
function putObjectTagging(params, callback) {
if (!params.Tagging || !params.Tagging.TagSet) {
return callback(errors.MissingParameter);
}
const tagRes = processTagSet(params.Tagging.TagSet);
if (tagRes instanceof Error) {
return callback(tagRes);
}
return async.waterfall([
next => this.headObject({
Bucket: params.Bucket,
Key: params.Key,
VersionId: params.VersionId,
}, next),
(resObj, next) => {
const completeMD = Object.assign({}, resObj.Metadata, tagRes);
this.copyObject({
Bucket: params.Bucket,
Key: params.Key,
CopySource: `${params.Bucket}/${params.Key}`,
Metadata: completeMD,
MetadataDirective: 'REPLACE',
}, next);
},
], callback);
}
module.exports = putObjectTagging;

View File

@ -0,0 +1,43 @@
const errors = require('../../../../../errors');
const { getPartNumber, createMpuKey, logger } = require('../GcpUtils');
const { logHelper } = require('../../utils');
/**
* uploadPart - upload part
* @param {object} params - upload part params
* @param {string} params.Bucket - bucket name
* @param {string} params.Key - object key
* @param {function} callback - callback function to call
* @return {undefined}
*/
function uploadPart(params, callback) {
if (!params || !params.UploadId || !params.Bucket || !params.Key) {
const error = errors.InvalidRequest
.customizeDescription('Missing required parameter');
logHelper(logger, 'error', 'error in uploadPart', error);
return callback(error);
}
const partNumber = getPartNumber(params.PartNumber);
if (!partNumber) {
const error = errors.InvalidArgument
.customizeDescription('PartNumber is invalid');
logHelper(logger, 'debug', 'error in uploadPart', error);
return callback(error);
}
const mpuParams = {
Bucket: params.Bucket,
Key: createMpuKey(params.Key, params.UploadId, partNumber),
Body: params.Body,
ContentLength: params.ContentLength,
};
return this.putObjectReq(mpuParams, (err, res) => {
if (err) {
logHelper(logger, 'error',
'error in uploadPart - putObject', err);
return callback(err);
}
return callback(null, res);
});
}
module.exports = uploadPart;

View File

@ -0,0 +1,37 @@
const errors = require('../../../../../errors');
const { getPartNumber, createMpuKey, logger } = require('../GcpUtils');
const { logHelper } = require('../../utils');
/**
* uploadPartCopy - upload part copy
* @param {object} params - upload part copy params
* @param {string} params.Bucket - bucket name
* @param {string} params.Key - object key
* @param {string} params.CopySource - source object to copy
* @param {function} callback - callback function to call
* @return {undefined}
*/
function uploadPartCopy(params, callback) {
if (!params || !params.UploadId || !params.Bucket || !params.Key ||
!params.CopySource) {
const error = errors.InvalidRequest
.customizeDescription('Missing required parameter');
logHelper(logger, 'error', 'error in uploadPartCopy', error);
return callback(error);
}
const partNumber = getPartNumber(params.PartNumber);
if (!partNumber) {
const error = errors.InvalidArgument
.customizeDescription('PartNumber is not a number');
logHelper(logger, 'debug', 'error in uploadPartCopy', error);
return callback(error);
}
const mpuParams = {
Bucket: params.Bucket,
Key: createMpuKey(params.Key, params.UploadId, partNumber),
CopySource: params.CopySource,
};
return this.copyObject(mpuParams, callback);
}
module.exports = uploadPartCopy;

View File

@ -0,0 +1,402 @@
const async = require('async');
const assert = require('assert');
const stream = require('stream');
const errors = require('../../../../errors');
const { minimumAllowedPartSize, gcpMaximumAllowedPartCount } =
require('../../../../constants');
const { createMpuList, logger } = require('./GcpUtils');
const { logHelper } = require('../utils');
function sliceFn(body, size) {
const array = [];
let partNumber = 1;
for (let ind = 0; ind < body.length; ind += size) {
array.push({
Body: body.slice(ind, ind + size),
PartNumber: partNumber++,
});
}
return array;
}
class GcpManagedUpload {
/**
* GcpMangedUpload - class to mimic the upload method in AWS-SDK
* To-Do: implement retry on failure like S3's upload
* @param {GcpService} service - client object
* @param {object} params - upload params
* @param {string} params.Bucket - bucket name
* @param {string} params.MPU - mpu bucket name
* @param {string} params.Key - object key
* @param {object} options - config setting for GcpManagedUpload object
* @param {number} options.partSize - set object chunk size
* @param {number} options.queueSize - set the number of concurrent upload
* @return {object} - return an GcpManagedUpload object
*/
constructor(service, params, options = {}) {
this.service = service;
this.params = params;
this.mainBucket =
this.params.Bucket || this.service.config.mainBucket;
this.mpuBucket =
this.params.MPU || this.service.config.mpuBucket;
this.partSize = minimumAllowedPartSize;
this.queueSize = options.queueSize || 4;
this.validateBody();
this.setPartSize();
// multipart information
this.parts = {};
this.uploadedParts = 0;
this.activeParts = 0;
this.partBuffers = [];
this.partQueue = [];
this.partBufferLength = 0;
this.totalChunkedBytes = 0;
this.partNumber = 0;
}
/**
* validateBody - validate that body contains data to upload. If body is not
* of type stream, it must then be of either string or buffer. If string,
* convert to a Buffer type and split into chunks if body is large enough
* @return {undefined}
*/
validateBody() {
this.body = this.params.Body;
assert(this.body, errors.MissingRequestBodyError.customizeDescription(
'Missing request body'));
this.totalBytes = this.params.ContentLength;
if (this.body instanceof stream) {
assert.strictEqual(typeof this.totalBytes, 'number',
errors.MissingContentLength.customizeDescription(
'If body is a stream, ContentLength must be provided'));
} else {
if (typeof this.body === 'string') {
this.body = Buffer.from(this.body);
}
this.totalBytes = this.body.byteLength;
assert(this.totalBytes, errors.InternalError.customizeDescription(
'Unable to perform upload'));
}
}
setPartSize() {
const newPartSize =
Math.ceil(this.totalBytes / gcpMaximumAllowedPartCount);
if (newPartSize > this.partSize) this.partSize = newPartSize;
this.totalParts = Math.ceil(this.totalBytes / this.partSize);
if (this.body instanceof Buffer && this.totalParts > 1) {
this.slicedParts = sliceFn(this.body, this.partSize);
}
}
/**
* cleanUp - function that is called if GcpManagedUpload fails at any point,
* perform clean up of used resources. Ends the request by calling an
* internal callback function
* @param {Error} err - Error object
* @return {undefined}
*/
cleanUp(err) {
// is only called when an error happens
if (this.failed || this.completed) {
return undefined;
}
this.failed = true;
if (this.uploadId) {
// if MPU was successfuly created
return this.abortMPU(mpuErr => {
if (mpuErr) {
logHelper(logger, 'error',
'GcpMangedUpload: abortMPU failed in cleanup', mpuErr);
}
return this.callback(err);
});
}
return this.callback(err);
}
/**
* abortMPU - function that is called to remove a multipart upload
* @param {function} callback - callback function to call to complete the
* upload
* @return {undefined}
*/
abortMPU(callback) {
const params = {
Bucket: this.mainBucket,
MPU: this.mpuBucket,
UploadId: this.uploadId,
Key: this.params.Key,
};
this.service.abortMultipartUpload(params, callback);
}
/**
* completeUpload - function that is called to to complete a multipart
* upload
* @param {function} callback - callback function to call to complete the
* upload
* @return {undefined}
*/
completeUpload() {
if (this.failed || this.completed) {
return undefined;
}
const params = {
Bucket: this.mainBucket,
MPU: this.mpuBucket,
Key: this.params.Key,
UploadId: this.uploadId,
MultipartUpload: {},
};
params.MultipartUpload.Parts =
createMpuList(params, 'parts', this.uploadedParts)
.map(item =>
Object.assign(item, { ETag: this.parts[item.PartNumber] }));
return this.service.completeMultipartUpload(params,
(err, res) => {
if (err) {
return this.cleanUp(err);
}
this.completed = true;
return this.callback(null, res);
});
}
/**
* send - function that is called to execute the method request
* @param {function} callback - callback function to be called and stored
* at the completion of the method
* @return {undefined}
*/
send(callback) {
if (this.called || this.callback) {
return undefined;
}
this.failed = false;
this.called = true;
this.callback = callback;
if (this.totalBytes <= this.partSize) {
return this.uploadSingle();
}
if (this.slicedParts) {
return this.uploadBufferSlices();
}
if (this.body instanceof stream) {
// stream type
this.body.on('error', err => this.cleanUp(err))
.on('readable', () => this.chunkStream())
.on('end', () => {
this.isDoneChunking = true;
this.chunkStream();
if (this.isDoneChunking && this.uploadedParts >= 1 &&
this.uploadedParts === this.totalParts) {
this.completeUpload();
}
});
}
return undefined;
}
/**
* uploadSingle - perform a regular put object upload if the object is
* small enough
* @return {undefined}
*/
uploadSingle() {
if (this.failed || this.completed) {
return undefined;
}
// use putObject to upload the single part object
const params = Object.assign({}, this.params);
params.Bucket = this.mainBucket;
delete params.MPU;
return this.service.putObject(params, (err, res) => {
if (err) {
return this.cleanUp(err);
}
// return results from a putObject request
this.completed = true;
return this.callback(null, res);
});
}
/**
* uploadBufferSlices - perform a multipart upload for body of type string
* or Buffer.
* @return {undefined}
*/
uploadBufferSlices() {
if (this.failed || this.completed) {
return undefined;
}
if (this.slicedParts.length <= 1 && this.totalParts) {
// there is only one part
return this.uploadSingle();
}
// multiple slices
return async.series([
// createMultipartUpload
next => {
const params = this.params;
params.Bucket = this.mpuBucket;
this.service.createMultipartUpload(params, (err, res) => {
if (!err) {
this.uploadId = res.UploadId;
}
return next(err);
});
},
next => async.eachLimit(this.slicedParts, this.queueSize,
(uploadPart, done) => {
const params = {
Bucket: this.mpuBucket,
Key: this.params.Key,
UploadId: this.uploadId,
Body: uploadPart.Body,
PartNumber: uploadPart.PartNumber,
};
this.service.uploadPart(params, (err, res) => {
if (!err) {
this.parts[uploadPart.PartNumber] = res.ETag;
this.uploadedParts++;
}
return done(err);
});
}, next),
], err => {
if (err) {
return this.cleanUp(new Error(
'GcpManagedUpload: unable to complete upload'));
}
return this.completeUpload();
});
}
/**
* chunkStream - read stream up until the max chunk size then call an
* uploadPart method on that chunk. If more than chunk size has be read,
* move the extra bytes into a queue for the next read.
* @return {undefined}
*/
chunkStream() {
const buf = this.body.read(this.partSize - this.partBufferLength) ||
this.body.read();
if (buf) {
this.partBuffers.push(buf);
this.partBufferLength += buf.length;
this.totalChunkedBytes += buf.length;
}
let pbuf;
if (this.partBufferLength >= this.partSize) {
pbuf = Buffer.concat(this.partBuffers);
this.partBuffers = [];
this.partBufferLength = 0;
if (pbuf.length > this.partSize) {
const rest = pbuf.slice(this.partSize);
this.partBuffers.push(rest);
this.partBufferLength += rest.length;
pbuf = pbuf.slice(0, this.partSize);
}
this.processChunk(pbuf);
}
// when chunking the last part
if (this.isDoneChunking && !this.completeChunking) {
this.completeChunking = true;
pbuf = Buffer.concat(this.partBuffers);
this.partBuffers = [];
this.partBufferLength = 0;
if (pbuf.length > 0) {
this.processChunk(pbuf);
} else {
if (this.uploadedParts === 0) {
// this is a 0-byte object
this.uploadSingle();
}
}
}
this.body.read(0);
}
/**
* processChunk - create a multipart request if one does not exist;
* otherwise, call uploadChunk to upload a chunk
* @param {Buffer} chunk - bytes to be uploaded
* @return {undefined}
*/
processChunk(chunk) {
const partNumber = ++this.partNumber;
if (!this.uploadId) {
// if multipart upload does not exist
if (!this.multipartReq) {
const params = this.params;
params.Bucket = this.mpuBucket;
this.multipartReq =
this.service.createMultipartUpload(params, (err, res) => {
if (err) {
return this.cleanUp(err);
}
this.uploadId = res.UploadId;
this.uploadChunk(chunk, partNumber);
if (this.partQueue.length > 0) {
this.partQueue.forEach(
part => this.uploadChunk(...part));
}
return undefined;
});
} else {
this.partQueue.push([chunk, partNumber]);
}
} else {
// queues chunks for upload
this.uploadChunk(chunk, partNumber);
this.activeParts++;
if (this.activeParts < this.queueSize) {
this.chunkStream();
}
}
}
/**
* uploadChunk - perform the partUpload
* @param {Buffer} chunk - bytes to be uploaded
* @param {number} partNumber - upload object part number
* @return {undefined}
*/
uploadChunk(chunk, partNumber) {
if (this.failed || this.completed) {
return undefined;
}
const params = {
Bucket: this.mpuBucket,
Key: this.params.Key,
UploadId: this.uploadId,
PartNumber: partNumber,
Body: chunk,
ContentLength: chunk.length,
};
return this.service.uploadPart(params, (err, res) => {
if (err) {
return this.cleanUp(err);
}
this.parts[partNumber] = res.ETag;
this.uploadedParts++;
this.activeParts--;
if (this.totalParts === this.uploadedParts &&
this.isDoneChunking) {
return this.completeUpload();
}
return this.chunkStream();
});
}
}
module.exports = GcpManagedUpload;

View File

@ -0,0 +1,141 @@
const AWS = require('aws-sdk');
const Service = AWS.Service;
const errors = require('../../../../errors');
const GcpApis = require('./GcpApis');
const GcpServiceSetup = require('./GcpServiceSetup');
const GcpManagedUpload = require('./GcpManagedUpload');
AWS.apiLoader.services.gcp = {};
const GCP = Service.defineService('gcp', ['2017-11-01']);
Object.assign(GCP.prototype, GcpServiceSetup, {
_maxConcurrent: 5,
// Implemented APIs
// Bucket API
getBucket(params, callback) {
return this.listObjects(params, callback);
},
// Object APIs
upload(params, callback) {
try {
const uploader = new GcpManagedUpload(this, params);
return uploader.send(callback);
} catch (err) {
return callback(err);
}
},
putObjectCopy(params, callback) {
return this.copyObject(params, callback);
},
// TO-DO: Implement the following APIs
// Service API
listBuckets(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: listBuckets not implemented'));
},
// Bucket APIs
getBucketLocation(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: getBucketLocation not implemented'));
},
deleteBucket(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: deleteBucket not implemented'));
},
listObjectVersions(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: listObjectVersions not implemented'));
},
createBucket(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: createBucket not implemented'));
},
putBucket(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: putBucket not implemented'));
},
getBucketAcl(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: getBucketAcl not implemented'));
},
putBucketAcl(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: putBucketAcl not implemented'));
},
putBucketWebsite(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: putBucketWebsite not implemented'));
},
getBucketWebsite(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: getBucketWebsite not implemented'));
},
deleteBucketWebsite(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: deleteBucketWebsite not implemented'));
},
putBucketCors(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: putBucketCors not implemented'));
},
getBucketCors(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: getBucketCors not implemented'));
},
deleteBucketCors(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: deleteBucketCors not implemented'));
},
// Object APIs
putObjectTagging(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: putObjectTagging not implemented'));
},
deleteObjectTagging(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: deleteObjectTagging not implemented'));
},
putObjectAcl(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: putObjectAcl not implemented'));
},
getObjectAcl(params, callback) {
return callback(errors.NotImplemented
.customizeDescription('GCP: getObjectAcl not implemented'));
},
});
Object.assign(GCP.prototype, GcpApis);
Object.defineProperty(AWS.apiLoader.services.gcp, '2017-11-01', {
get: function get() {
const model = require('./gcp-2017-11-01.api.json');
return model;
},
enumerable: true,
configurable: true,
});
module.exports = GCP;

View File

@ -0,0 +1,246 @@
/**
* This file contains the private methods for the GCP service to form/sign
* Google Cloud Storage requests.
*
* It uses a modified version of the S3 service private methods to form/sign
* requests compatible with the Google Cloud Storage XML api.
*/
const AWS = require('aws-sdk');
const GcpSigner = require('./GcpSigner');
module.exports = {
/**
* GCP compatible request signer
* @return {AWS.Signer} AWS Signer
*/
getSignerClass() {
return GcpSigner;
},
/**
* set service region
* @return {undefined}
*/
validateService() {
if (!this.config.region) {
this.config.region = 'us-east-1';
}
},
/**
* setup listeners for building requests
* @param {AWS.Request} request - AWS request object
* @return {undefined}
*/
setupRequestListeners(request) {
request.addListener('validate', this.validateBucketEndpoint);
request.addListener('build', this.addContentType);
request.addListener('build', this.populateURI);
request.addListener('build', this.computeContentMd5);
request.addListener('extractError', this.extractRequestIds);
},
/**
* validate that when bucket endpoitn flag is set, root level apis are
* inaccessible
* @param {AWS.Request} req - AWS request object
* @returns {undefined}
* @api private
*/
validateBucketEndpoint(req) {
if (!req.params.Bucket && req.service.config.s3BucketEndpoint) {
const msg =
'Cannot send requests to root API with `s3BucketEndpoint` set.';
throw AWS.util.error(new Error(),
{ code: 'ConfigError', message: msg });
}
},
/**
* S3 prefers dns-compatible bucket names to be moved from the uri path
* to the hostname as a sub-domain. This is not possible, even for
* dns-compat buckets when using SSL and the bucket name contains a dot.
* The ssl wildcard certificate is only 1-level deep.
* @param {AWS.Request} req - AWS request object
* @returns {undefined}
*
* @api private
*/
populateURI(req) {
const httpRequest = req.httpRequest;
const b = req.params.Bucket;
const service = req.service;
const endpoint = httpRequest.endpoint;
if (b) {
if (!service.pathStyleBucketName(b)) {
if (!service.config.s3BucketEndpoint) {
endpoint.hostname = `${b}.${endpoint.hostname}`;
}
const port = endpoint.port;
if (port !== 80 && port !== 443) {
endpoint.host = `${endpoint.hostname}:${endpoint.port}`;
} else {
endpoint.host = endpoint.hostname;
}
// needed for signing the request
httpRequest.virtualHostedBucket = b;
service.removeVirtualHostedBucketFromPath(req);
}
}
},
/**
* Takes the bucket name out of the path if bucket is virtual-hosted
* @param {AWS.Request} req - AWS request object
* @returns {undefined}
*
* @api private
*/
removeVirtualHostedBucketFromPath(req) {
const httpRequest = req.httpRequest;
const bucket = httpRequest.virtualHostedBucket;
if (bucket && httpRequest.path) {
httpRequest.path =
httpRequest.path.replace(new RegExp(`/${bucket}`), '');
if (httpRequest.path[0] !== '/') {
httpRequest.path = `/${httpRequest.path}`;
}
}
},
/**
* Adds a default content type if none is supplied.
* @param {AWS.Request} req - AWS request object
* @returns {undefined}
*
* @api private
*/
addContentType(req) {
const httpRequest = req.httpRequest;
if (httpRequest.method === 'GET' || httpRequest.method === 'HEAD') {
// Content-Type is not set in GET/HEAD requests
delete httpRequest.headers['Content-Type'];
return;
}
// always have a Content-Type
if (!httpRequest.headers['Content-Type']) {
httpRequest.headers['Content-Type'] = 'application/octet-stream';
}
const contentType = httpRequest.headers['Content-Type'];
if (AWS.util.isBrowser()) {
if (typeof httpRequest.body === 'string' &&
!contentType.match(/;\s*charset=/)) {
const charset = '; charset=UTF-8';
httpRequest.headers['Content-Type'] += charset;
} else {
const replaceFn = (_, prefix, charsetName) =>
prefix + charsetName.toUpperCase();
httpRequest.headers['Content-Type'] =
contentType.replace(/(;\s*charset=)(.+)$/, replaceFn);
}
}
},
computableChecksumOperations: {
putBucketCors: true,
putBucketLifecycle: true,
putBucketLifecycleConfiguration: true,
putBucketTagging: true,
deleteObjects: true,
putBucketReplication: true,
},
/**
* Checks whether checksums should be computed for the request.
* If the request requires checksums to be computed, this will always
* return true, otherwise it depends on whether
* {AWS.Config.computeChecksums} is set.
* @param {AWS.Request} req - the request to check against
* @return {Boolean} whether to compute checksums for a request.
*
* @api private
*/
willComputeChecksums(req) {
if (this.computableChecksumOperations[req.operation]) return true;
if (!this.config.computeChecksums) return false;
// TODO: compute checksums for Stream objects
if (!AWS.util.Buffer.isBuffer(req.httpRequest.body) &&
typeof req.httpRequest.body !== 'string') {
return false;
}
return false;
},
/**
* A listener that computes the Content-MD5 and sets it in the header.
* @param {AWS.Request} req - AWS request object
* @returns {undefined}
*
* @see AWS.S3.willComputeChecksums
* @api private
*/
computeContentMd5(req) {
if (req.service.willComputeChecksums(req)) {
const md5 = AWS.util.crypto.md5(req.httpRequest.body, 'base64');
// eslint-disable-next-line no-param-reassign
req.httpRequest.headers['Content-MD5'] = md5;
}
},
/**
* Returns true if the bucket name should be left in the URI path for
* a request to S3. This function takes into account the current
* endpoint protocol (e.g. http or https).
* @param {string} bucketName - bucket name
* @returns {Boolean} whether request should use path style
*
* @api private
*/
pathStyleBucketName(bucketName) {
// user can force path style requests via the configuration
if (this.config.s3ForcePathStyle) return true;
if (this.config.s3BucketEndpoint) return false;
if (this.dnsCompatibleBucketName(bucketName)) {
return this.config.sslEnabled && bucketName.match(/\./);
}
return true; // not dns compatible names must always use path style
},
/**
* Returns true if the bucket name is DNS compatible. Buckets created
* outside of the classic region MUST be DNS compatible.
* @param {string} bucketName - bucket name
* @returns {Boolean} whether bucket name is dns compatible
*
* @api private
*/
dnsCompatibleBucketName(bucketName) {
const b = bucketName;
const domain = new RegExp(/^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$/);
const ipAddress = new RegExp(/(\d+\.){3}\d+/);
const dots = new RegExp(/\.\./);
return b.match(domain) && !b.match(ipAddress) && !b.match(dots);
},
/**
* Extracts GCP specific request ids from the http response.
* @param {object} resp - response object
* @returns {undefined}
* @api private
*/
extractRequestIds(resp) {
const requestId = resp.httpResponse.headers ?
resp.httpResponse.headers['x-guploader-uploadid'] : null;
if (resp.error) {
// eslint-disable-next-line no-param-reassign
resp.error.requestId = resp.requestId || requestId;
}
},
};

View File

@ -0,0 +1,51 @@
const url = require('url');
const qs = require('querystring');
const AWS = require('aws-sdk');
const werelogs = require('werelogs');
const { constructStringToSignV2 } = require('../../../../auth/auth').client;
const logger = new werelogs.Logger('GcpSigner');
function genQueryObject(uri) {
const queryString = url.parse(uri).query;
return qs.parse(queryString);
}
const GcpSigner = AWS.util.inherit(AWS.Signers.RequestSigner, {
constructor: function GcpSigner(request) {
AWS.Signers.RequestSigner.call(this, request);
},
addAuthorization: function addAuthorization(credentials, date) {
if (!this.request.headers['presigned-expires']) {
this.request.headers['x-goog-date'] = AWS.util.date.rfc822(date);
}
const signature =
this.sign(credentials.secretAccessKey, this.stringToSign());
const auth = `GOOG1 ${credentials.accessKeyId}: ${signature}`;
this.request.headers.Authorization = auth;
},
stringToSign: function stringToSign() {
const requestObject = {
url: this.request.path,
method: this.request.method,
host: this.request.endpoint.host,
headers: this.request.headers,
bucketName: this.request.virtualHostedBucket,
query: genQueryObject(this.request.path) || {},
};
requestObject.gotBucketNameFromHost =
requestObject.host.indexOf(this.request.virtualHostedBucket) >= 0;
const data = Object.assign({}, this.request.headers);
return constructStringToSignV2(requestObject, data, logger, 'GCP');
},
sign: function sign(secret, string) {
return AWS.util.crypto.hmac(secret, string, 'base64', 'sha1');
},
});
module.exports = GcpSigner;

View File

@ -0,0 +1,171 @@
const werelogs = require('werelogs');
const errors = require('../../../../errors');
const { parseTagFromQuery } = require('../../../../s3middleware/tagging');
const { gcpTaggingPrefix } = require('../../../../constants');
const gcpLogLevel = 'info';
const gcpDumpLevel = 'error';
werelogs.configure({
level: gcpLogLevel,
dump: gcpDumpLevel,
});
const logger = new werelogs.Logger('gcpUtil');
function eachSlice(size) {
this.array = [];
let partNumber = 1;
for (let ind = 0; ind < this.length; ind += size) {
this.array.push({
Parts: this.slice(ind, ind + size),
PartNumber: partNumber++,
});
}
return this.array;
}
function getSourceInfo(CopySource) {
const source =
CopySource.startsWith('/') ? CopySource.slice(1) : CopySource;
const sourceArray = source.split(/\/(.+)/);
const sourceBucket = sourceArray[0];
const sourceObject = sourceArray[1];
return { sourceBucket, sourceObject };
}
function getPaddedPartNumber(number) {
return `000000${number}`.substr(-5);
}
function getPartNumber(number) {
if (isNaN(number)) {
return undefined;
}
if (typeof number === 'string') {
return parseInt(number, 10);
}
return number;
}
function createMpuKey(key, uploadId, partNumberArg, fileNameArg) {
let partNumber = partNumberArg;
let fileName = fileNameArg;
if (typeof partNumber === 'string' && fileName === undefined) {
fileName = partNumber;
partNumber = null;
}
const paddedNumber = getPaddedPartNumber(partNumber);
if (fileName && typeof fileName === 'string') {
// if partNumber is given, return a "full file path"
// else return a "directory path"
return partNumber ? `${key}-${uploadId}/${fileName}/${paddedNumber}` :
`${key}-${uploadId}/${fileName}`;
}
if (partNumber && typeof partNumber === 'number') {
// filename wasn't passed as an argument. Create default
return `${key}-${uploadId}/parts/${paddedNumber}`;
}
// returns a "directory path"
return `${key}-${uploadId}/`;
}
function createMpuList(params, level, size) {
// populate and return a parts list for compose
const retList = [];
for (let i = 1; i <= size; ++i) {
retList.push({
PartName: createMpuKey(params.Key, params.UploadId, i, level),
PartNumber: i,
});
}
return retList;
}
function processTagSet(tagSet = []) {
if (tagSet.length > 10) {
return errors.BadRequest
.customizeDescription('Object tags cannot be greater than 10');
}
let error = undefined;
const tagAsMeta = {};
const taggingDict = {};
tagSet.every(tag => {
const { Key: key, Value: value } = tag;
if (key.length > 128) {
error = errors.InvalidTag
.customizeDescription(
`The TagKey provided is too long, ${key.length}`);
return false;
}
if (value.length > 256) {
error = errors.InvalidTag
.customizeDescription(
`The TagValue provided is too long, ${value.length}`);
return false;
}
if (taggingDict[key]) {
error = errors.InvalidTag
.customizeDescription(
'Cannot provide multiple Tags with the same key');
return false;
}
tagAsMeta[`${gcpTaggingPrefix}${key}`] = value;
taggingDict[key] = true;
return true;
});
if (error) {
return error;
}
return tagAsMeta;
}
function stripTags(metadata = {}) {
const retMD = Object.assign({}, metadata);
Object.keys(retMD).forEach(key => {
if (key.startsWith(gcpTaggingPrefix)) {
delete retMD[key];
}
});
return retMD;
}
function retrieveTags(metadata = {}) {
const retTagSet = [];
Object.keys(metadata).forEach(key => {
if (key.startsWith(gcpTaggingPrefix)) {
retTagSet.push({
Key: key.slice(gcpTaggingPrefix.length),
Value: metadata[key],
});
}
});
return retTagSet;
}
function getPutTagsMetadata(metadata, tagging = '') {
let retMetadata = metadata || {};
retMetadata = stripTags(retMetadata);
const tagObj = parseTagFromQuery(tagging);
Object.keys(tagObj).forEach(header => {
const prefixed = `${gcpTaggingPrefix}${header}`.toLowerCase();
retMetadata[prefixed] = tagObj[header];
});
return retMetadata;
}
module.exports = {
// functions
eachSlice,
createMpuKey,
createMpuList,
getSourceInfo,
processTagSet,
stripTags,
retrieveTags,
getPutTagsMetadata,
getPartNumber,
// util objects
logger,
};

View File

@ -0,0 +1,932 @@
{
"version": "1.0",
"metadata": {
"apiVersion": "2017-11-01",
"checksumFormat": "md5",
"endpointPrefix": "storage",
"globalEndpoint": "storage.googleapi.com",
"protocol": "rest-xml",
"serviceAbbreviation": "GCP",
"serviceFullName": "Google Cloud Storage",
"signatureVersion": "s3",
"timestampFormat": "rfc822",
"uid": "gcp-2017-11-01"
},
"operations": {
"HeadBucket": {
"http": {
"method": "HEAD",
"requestUri": "/{Bucket}"
},
"input": {
"type": "structure",
"required": [
"Bucket"
],
"members": {
"Bucket": {
"location": "uri",
"locationName": "Bucket"
},
"ProjectId": {
"location": "header",
"locationName": "x-goog-project-id"
}
}
},
"output": {
"type": "structure",
"members": {
"MetaVersionId": {
"location": "header",
"locationName": "x-goog-metageneration"
}
}
}
},
"listObjects": {
"http": {
"method": "GET",
"requestUri": "/{Bucket}"
},
"input": {
"type": "structure",
"required": [
"Bucket"
],
"members": {
"Bucket": {
"location": "uri",
"locationName": "Bucket"
},
"Delimiter": {
"location": "querystring",
"locationName": "delimiter"
},
"Marker": {
"location": "querystring",
"locationName": "marker"
},
"MaxKeys": {
"location": "querystring",
"locationName": "max-keys",
"type": "integer"
},
"Prefix": {
"location": "querystring",
"locationName": "prefix"
},
"ProjectId": {
"location": "header",
"locationName": "x-goog-project-id"
}
}
},
"output": {
"type": "structure",
"members": {
"IsTruncated": {
"type": "boolean"
},
"Marker": {},
"NextMarker": {},
"Contents": {
"shape": "ContentsShape"
},
"Name": {},
"Prefix": {},
"Delimiter": {},
"MaxKeys": {
"type": "integer"
},
"CommonPrefixes": {
"shape": "CommonPrefixShape"
}
}
}
},
"listVersions": {
"http": {
"method": "GET",
"requestUri": "/{Bucket}?versions"
},
"input": {
"type": "structure",
"required": [
"Bucket"
],
"members": {
"Bucket": {
"location": "uri",
"locationName": "Bucket"
},
"Delimiter": {
"location": "querystring",
"locationName": "delimiter"
},
"Marker": {
"location": "querystring",
"locationName": "marker"
},
"MaxKeys": {
"location": "querystring",
"locationName": "max-keys",
"type": "integer"
},
"Prefix": {
"location": "querystring",
"locationName": "prefix"
},
"ProjectId": {
"location": "header",
"locationName": "x-goog-project-id"
}
}
},
"output": {
"type": "structure",
"members": {
"IsTruncated": {
"type": "boolean"
},
"Marker": {},
"NextMarker": {},
"Versions": {
"locationName": "Version",
"shape": "ContentsShape"
},
"Name": {},
"Prefix": {},
"Delimiter": {},
"MaxKeys": {
"type": "integer"
},
"CommonPrefixes": {
"shape": "CommonPrefixShape"
}
}
}
},
"PutBucketVersioning": {
"http": {
"method": "PUT",
"requestUri": "/{Bucket}?versioning"
},
"input": {
"type": "structure",
"required": [
"Bucket",
"VersioningConfiguration"
],
"members": {
"Bucket": {
"location": "uri",
"locationName": "Bucket"
},
"ContentMD5": {
"location": "header",
"locationName": "Content-MD5"
},
"VersioningConfiguration": {
"locationName": "VersioningConfiguration",
"type": "structure",
"members": {
"Status": {}
}
}
},
"payload": "VersioningConfiguration"
}
},
"GetBucketVersioning": {
"http": {
"method": "GET",
"requestUri": "/{Bucket}?versioning"
},
"input": {
"type": "structure",
"required": [
"Bucket"
],
"members": {
"Bucket": {
"location": "uri",
"locationName": "Bucket"
}
}
},
"output": {
"type": "structure",
"members": {
"Status": {}
}
}
},
"HeadObject": {
"http": {
"method": "HEAD",
"requestUri": "/{Bucket}/{Key+}"
},
"input": {
"type": "structure",
"required": [
"Bucket",
"Key"
],
"members": {
"Date": {
"location": "header",
"locationName": "Date",
"type": "timestamp"
},
"Bucket": {
"location": "uri",
"locationName": "Bucket"
},
"IfMatch": {
"location": "header",
"locationName": "If-Match"
},
"IfModifiedSince": {
"location": "header",
"locationName": "If-Modified-Since",
"type": "timestamp"
},
"IfNoneMatch": {
"location": "header",
"locationName": "If-None-Match"
},
"IfUnmodifiedSince": {
"location": "header",
"locationName": "If-Unmodified-Since",
"type": "timestamp"
},
"Range": {
"location": "header",
"locationName": "Range"
},
"Key": {
"location": "uri",
"locationName": "Key"
},
"Range": {
"location": "header",
"locationName": "Range"
},
"VersionId": {
"location": "querystring",
"locationName": "generation"
},
"ProjectId": {
"location": "header",
"locationName": "x-goog-project-id"
}
}
},
"output": {
"type": "structure",
"members": {
"Date": {
"location": "header",
"locationName": "Date",
"type": "timestamp"
},
"AcceptRanges": {
"location": "header",
"locationName": "accept-ranges"
},
"Expiration": {
"location": "header",
"locationName": "x-goog-expiration"
},
"LastModified": {
"location": "header",
"locationName": "Last-Modified",
"type": "timestamp"
},
"ContentLength": {
"location": "header",
"locationName": "Content-Length",
"type": "long"
},
"ContentHash": {
"location": "header",
"locationName": "x-goog-hash"
},
"ETag": {
"location": "header",
"locationName": "ETag"
},
"VersionId": {
"location": "header",
"locationName": "x-goog-generation"
},
"MetaVersionId": {
"location": "header",
"locationName": "x-goog-metageneration"
},
"CacheControl": {
"location": "header",
"locationName": "Cache-Control"
},
"ContentDisposition": {
"location": "header",
"locationName": "Content-Disposition"
},
"ContentEncoding": {
"location": "header",
"locationName": "Content-Encoding"
},
"ContentLanguage": {
"location": "header",
"locationName": "Content-Language"
},
"ContentType": {
"location": "header",
"locationName": "Content-Type"
},
"Expires": {
"location": "header",
"locationName": "Expires",
"type": "timestamp"
},
"Metadata": {
"shape": "MetadataShape",
"location": "headers",
"locationName": "x-goog-meta-"
},
"StorageClass": {
"location": "headers",
"locationName": "x-goog-storage-class"
}
}
}
},
"PutObjectReq": {
"http": {
"method": "PUT",
"requestUri": "/{Bucket}/{Key+}"
},
"input": {
"type": "structure",
"required": [
"Bucket",
"Key"
],
"members": {
"Date": {
"location": "header",
"locationName": "Date",
"type": "timestamp"
},
"ACL": {
"location": "header",
"locationName": "x-goog-acl"
},
"Body": {
"streaming": true,
"type": "blob"
},
"Bucket": {
"location": "uri",
"locationName": "Bucket"
},
"CacheControl": {
"location": "header",
"locationName": "Cache-Control"
},
"ContentDisposition": {
"location": "header",
"locationName": "Content-Disposition"
},
"ContentEncoding": {
"location": "header",
"locationName": "Content-Encoding"
},
"ContentLanguage": {
"location": "header",
"locationName": "Content-Language"
},
"ContentLength": {
"location": "header",
"locationName": "Content-Length",
"type": "long"
},
"ContentMD5": {
"location": "header",
"locationName": "Content-MD5"
},
"ContentType": {
"location": "header",
"locationName": "Content-Type"
},
"Expires": {
"location": "header",
"locationName": "Expires",
"type": "timestamp"
},
"Key": {
"location": "uri",
"locationName": "Key"
},
"Metadata": {
"shape": "MetadataShape",
"location": "headers",
"locationName": "x-goog-meta-"
},
"ProjectId": {
"location": "header",
"locationName": "x-goog-project-id"
}
},
"payload": "Body"
},
"output": {
"type": "structure",
"members": {
"Expiration": {
"location": "header",
"locationName": "x-goog-expiration"
},
"ETag": {
"location": "header",
"locationName": "ETag"
},
"ContentHash": {
"location": "header",
"locationName": "x-goog-hash"
},
"VersionId": {
"location": "header",
"locationName": "x-goog-generation"
},
"MetaVersionId": {
"location": "header",
"locationName": "x-goog-metageneration"
}
}
}
},
"GetObject": {
"http": {
"method": "GET",
"requestUri": "/{Bucket}/{Key+}"
},
"input": {
"type": "structure",
"required": [
"Bucket",
"Key"
],
"members": {
"Bucket": {
"location": "uri",
"locationName": "Bucket"
},
"IfMatch": {
"location": "header",
"locationName": "If-Match"
},
"IfModifiedSince": {
"location": "header",
"locationName": "If-Modified-Since",
"type": "timestamp"
},
"IfNoneMatch": {
"location": "header",
"locationName": "If-None-Match"
},
"IfUnmodifiedSince": {
"location": "header",
"locationName": "If-Unmodified-Since",
"type": "timestamp"
},
"Key": {
"location": "uri",
"locationName": "Key"
},
"Range": {
"location": "header",
"locationName": "Range"
},
"ResponseCacheControl": {
"location": "querystring",
"locationName": "response-cache-control"
},
"ResponseContentDisposition": {
"location": "querystring",
"locationName": "response-content-disposition"
},
"ResponseContentEncoding": {
"location": "querystring",
"locationName": "response-content-encoding"
},
"ResponseContentLanguage": {
"location": "querystring",
"locationName": "response-content-language"
},
"ResponseContentType": {
"location": "querystring",
"locationName": "response-content-type"
},
"ResponseExpires": {
"location": "querystring",
"locationName": "response-expires",
"type": "timestamp"
},
"VersionId": {
"location": "querystring",
"locationName": "generation"
},
"ProjectId": {
"location": "header",
"locationName": "x-goog-project-id"
}
}
},
"output": {
"type": "structure",
"members": {
"Body": {
"streaming": true,
"type": "blob"
},
"AcceptRanges": {
"location": "header",
"locationName": "accept-ranges"
},
"Expiration": {
"location": "header",
"locationName": "x-goog-expiration"
},
"LastModified": {
"location": "header",
"locationName": "Last-Modified",
"type": "timestamp"
},
"ContentLength": {
"location": "header",
"locationName": "Content-Length",
"type": "long"
},
"ETag": {
"location": "header",
"locationName": "ETag"
},
"VersionId": {
"location": "header",
"locationName": "x-goog-generation"
},
"MetaVersionId": {
"location": "header",
"locationName": "x-goog-metageneration"
},
"CacheControl": {
"location": "header",
"locationName": "Cache-Control"
},
"ContentDisposition": {
"location": "header",
"locationName": "Content-Disposition"
},
"ContentEncoding": {
"location": "header",
"locationName": "Content-Encoding"
},
"ContentLanguage": {
"location": "header",
"locationName": "Content-Language"
},
"ContentRange": {
"location": "header",
"locationName": "Content-Range"
},
"ContentType": {
"location": "header",
"locationName": "Content-Type"
},
"ContentHash": {
"location": "header",
"locationName": "x-goog-hash"
},
"Expires": {
"location": "header",
"locationName": "Expires",
"type": "timestamp"
},
"WebsiteRedirectLocation": {
"location": "header",
"locationName": "x-goog-website-redirect-location"
},
"ServerSideEncryption": {
"location": "header",
"locationName": "x-goog-server-side-encryption"
},
"Metadata": {
"shape": "MetadataShape",
"location": "headers",
"locationName": "x-goog-meta-"
},
"StorageClass": {
"location": "header",
"locationName": "x-goog-storage-class"
}
},
"payload": "Body"
}
},
"DeleteObject": {
"http": {
"method": "DELETE",
"requestUri": "/{Bucket}/{Key+}"
},
"input": {
"type": "structure",
"required": [
"Bucket",
"Key"
],
"members": {
"Bucket": {
"location": "uri",
"locationName": "Bucket"
},
"Key": {
"location": "uri",
"locationName": "Key"
},
"VersionId": {
"location": "querystring",
"locationName": "generation"
},
"ProjectId": {
"location": "header",
"locationName": "x-goog-project-id"
}
}
},
"output": {
"type": "structure",
"members": {
"VersionId": {
"location": "header",
"locationName": "x-goog-generation"
}
}
}
},
"ComposeObject": {
"http": {
"method": "PUT",
"requestUri": "/{Bucket}/{Key+}?compose"
},
"input": {
"type": "structure",
"required": [
"Bucket",
"Key"
],
"members": {
"Bucket": {
"location": "uri",
"locationName": "Bucket"
},
"Source": {
"location": "header",
"locationName": "x-goog-copy-source"
},
"Key": {
"location": "uri",
"locationName": "Key"
},
"MetadataDirective": {
"location": "header",
"locationName": "x-goog-metadata-directive"
},
"ContentDisposition": {
"location": "header",
"locationName": "Content-Disposition"
},
"Content-Encoding": {
"location": "header",
"locationName": "Content-Encoding"
},
"MultipartUpload": {
"locationName": "ComposeRequest",
"type": "structure",
"members": {
"Parts": {
"locationName": "Component",
"type": "list",
"member": {
"type": "structure",
"members": {
"PartName": {
"locationName": "Name"
}
}
},
"flattened": true
}
}
},
"Metadata": {
"shape": "MetadataShape",
"location": "headers",
"locationName": "x-goog-meta-"
},
"ProjectId": {
"location": "header",
"locationName": "x-goog-project-id"
}
},
"payload": "MultipartUpload"
},
"output": {
"type": "structure",
"members": {
"Expiration": {
"location": "header",
"locationName": "x-goog-expiration"
},
"ETag": {},
"VersioId": {
"location": "header",
"locationName": "x-goog-generation"
},
"MetaVersionId": {
"location": "header",
"locationName": "x-goog-metageneration"
}
}
}
},
"CopyObject": {
"http": {
"method": "PUT",
"requestUri": "/{Bucket}/{Key+}"
},
"input": {
"type": "structure",
"required": [
"Bucket",
"CopySource",
"Key"
],
"members": {
"ACL": {
"location": "header",
"locationName": "x-goog-acl"
},
"Bucket": {
"location": "uri",
"locationName": "Bucket"
},
"CacheControl": {
"location": "header",
"locationName": "Cache-Control"
},
"ContentDisposition": {
"location": "header",
"locationName": "Content-Disposition"
},
"ContentEncoding": {
"location": "header",
"locationName": "Content-Encoding"
},
"ContentLanguage": {
"location": "header",
"locationName": "Content-Language"
},
"ContentType": {
"location": "header",
"locationName": "Content-Type"
},
"CopySource": {
"location": "header",
"locationName": "x-goog-copy-source"
},
"CopySourceIfMatch": {
"location": "header",
"locationName": "x-goog-copy-source-if-match"
},
"CopySourceIfModifiedSince": {
"location": "header",
"locationName": "x-goog-copy-source-if-modified-since"
},
"CopySourceIfNoneMatch": {
"location": "header",
"locationName": "x-goog-copy-source-if-none-match"
},
"CopySourceIfUnmodifiedSince": {
"location": "header",
"locationName": "x-goog-copy-source-if-unmodified-since",
"type": "timestamp"
},
"Expires": {
"location": "header",
"locationName": "Expires",
"type": "timestamp"
},
"Key": {
"location": "uri",
"locationName": "Key"
},
"Metadata": {
"shape": "MetadataShape",
"location": "headers",
"locationName": "x-goog-meta-"
},
"MetadataDirective": {
"location": "header",
"locationName": "x-goog-metadata-directive"
},
"ProjectId": {
"location": "header",
"locationName": "x-goog-project-id"
}
}
},
"output": {
"type": "structure",
"members": {
"CopyObjectResult": {
"type": "structure",
"members": {
"ETag": {},
"LastModified": {
"type": "timestamp"
}
}
},
"Expiration": {
"location": "header",
"locationName": "x-goog-expiration"
},
"ContentHash": {
"location": "header",
"locationName": "x-goog-hash"
},
"VersionId": {
"location": "header",
"locationName": "x-goog-generation"
},
"MetaVersionId": {
"location": "header",
"locationName": "x-goog-metageneration"
}
},
"payload": "CopyObjectResult"
}
}
},
"shapes": {
"MetadataShape": {
"type": "map",
"key": {},
"value": {}
},
"OwnerShape": {
"locationName": "Owner",
"type": "structure",
"members": {
"ID": {},
"Name": {}
}
},
"ContentsShape": {
"type": "list",
"member": {
"type": "structure",
"members": {
"Key": {},
"LastModified": {
"type": "timestamp"
},
"ETag": {},
"Size": {
"type": "integer"
},
"StorageClass": {},
"Owner": {
"shape": "OwnerShape"
},
"VersionId": {
"locationName": "Generation"
}
}
},
"flattened": true
},
"CommonPrefixShape": {
"type": "list",
"member": {
"type": "structure",
"members": {
"Prefix": {}
}
},
"flattened": true
}
}
}

View File

@ -0,0 +1,5 @@
module.exports = {
GCP: require('./GcpService'),
GcpSigner: require('./GcpSigner'),
GcpUtils: require('./GcpUtils'),
};

301
lib/storage/data/external/GcpClient.js vendored Normal file
View File

@ -0,0 +1,301 @@
const async = require('async');
const errors = require('../../../errors');
const MD5Sum = require('../../../s3middleware/MD5Sum');
const { GCP, GcpUtils } = require('./GCP');
const { createMpuKey } = GcpUtils;
const AwsClient = require('./AwsClient');
const { prepareStream } = require('../../../s3middleware/prepareStream');
const { logHelper, removeQuotes } = require('./utils');
const missingVerIdInternalError = errors.InternalError.customizeDescription(
'Invalid state. Please ensure versioning is enabled ' +
'in GCP for the location constraint and try again.');
/**
* Class representing a Google Cloud Storage backend object
* @extends AwsClient
*/
class GcpClient extends AwsClient {
/**
* constructor - creates a Gcp backend client object (inherits )
* @param {object} config - configuration object for Gcp Backend up
* @param {object} config.s3params - S3 configuration
* @param {string} config.bucketName - GCP bucket name
* @param {string} config.mpuBucket - GCP mpu bucket name
* @param {boolean} config.bucketMatch - bucket match flag
* @param {object} config.authParams - GCP service credentials
* @param {string} config.dataStoreName - locationConstraint name
* @param {booblean} config.serverSideEncryption - server side encryption
* flag
* @return {object} - returns a GcpClient object
*/
constructor(config) {
super(config);
this.clientType = 'gcp';
this.type = 'GCP';
this._gcpBucketName = config.bucketName;
this._mpuBucketName = config.mpuBucket;
this._createGcpKey = this._createAwsKey;
this._gcpParams = Object.assign(this._s3Params, {
mainBucket: this._gcpBucketName,
mpuBucket: this._mpuBucketName,
});
this._client = new GCP(this._gcpParams);
// reassign inherited list parts method from AWS to trigger
// listing using S3 metadata part list instead of request to GCP
this.listParts = undefined;
}
toObjectGetInfo(objectKey, bucketName) {
return {
key: this._createGcpKey(bucketName, objectKey, this._bucketMatch),
dataStoreName: this._dataStoreName,
};
}
/**
* healthcheck - the gcp health requires checking multiple buckets:
* main and mpu buckets
* @param {string} location - location name
* @param {function} callback - callback function to call with the bucket
* statuses
* @return {undefined}
*/
healthcheck(location, callback) {
const checkBucketHealth = (bucket, cb) => {
let bucketResp;
this._client.headBucket({ Bucket: bucket }, err => {
if (err) {
bucketResp = {
gcpBucket: bucket,
error: err };
} else {
bucketResp = {
gcpBucket: bucket,
message: 'Congrats! You own the bucket',
};
}
return cb(null, bucketResp);
});
};
const gcpResp = {};
async.parallel({
main: done => checkBucketHealth(this._gcpBucketName, done),
mpu: done => checkBucketHealth(this._mpuBucketName, done),
}, (err, result) => {
if (err) {
return callback(errors.InternalFailure
.customizeDescription('Unable to perform health check'));
}
gcpResp[location] = result.main.error || result.mpu.error ?
{ error: true, external: true } : {};
Object.assign(gcpResp[location], result);
return callback(null, gcpResp);
});
}
createMPU(key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
cacheControl, contentDisposition, contentEncoding, log, callback) {
const metaHeadersTrimmed = {};
Object.keys(metaHeaders).forEach(header => {
if (header.startsWith('x-amz-meta-')) {
const headerKey = header.substring(11);
metaHeadersTrimmed[headerKey] = metaHeaders[header];
}
});
const gcpKey = this._createGcpKey(bucketName, key, this._bucketMatch);
const params = {
Bucket: this._mpuBucketName,
Key: gcpKey,
Metadata: metaHeadersTrimmed,
ContentType: contentType,
CacheControl: cacheControl,
ContentDisposition: contentDisposition,
ContentEncoding: contentEncoding,
};
return this._client.createMultipartUpload(params, (err, mpuResObj) => {
if (err) {
logHelper(log, 'error', 'err from data backend',
err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`)
);
}
return callback(null, mpuResObj);
});
}
completeMPU(jsonList, mdInfo, key, uploadId, bucketName, log, callback) {
const gcpKey = this._createGcpKey(bucketName, key, this._bucketMatch);
const partArray = [];
const partList = jsonList.Part;
for (let i = 0; i < partList.length; ++i) {
const partObj = partList[i];
if (!partObj.PartNumber || !partObj.ETag) {
return callback(errors.MalformedXML);
}
const number = partObj.PartNumber[0];
// check if the partNumber is an actual number throw an error
// otherwise
if (isNaN(number)) {
return callback(errors.MalformedXML);
}
const partNumber = parseInt(number, 10);
const partParams = {
PartName: createMpuKey(gcpKey, uploadId, partNumber),
PartNumber: partNumber,
ETag: partObj.ETag[0],
};
partArray.push(partParams);
}
const mpuParams = {
Bucket: this._gcpBucketName,
MPU: this._mpuBucketName,
Key: gcpKey,
UploadId: uploadId,
MultipartUpload: { Parts: partArray },
};
const completeObjData = { key: gcpKey };
return this._client.completeMultipartUpload(mpuParams,
(err, completeMpuRes) => {
if (err) {
logHelper(log, 'error', 'err from data backend on ' +
'completeMPU', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`)
);
}
if (!completeMpuRes.VersionId) {
logHelper(log, 'error', 'missing version id for data ' +
'backend object', missingVerIdInternalError,
this._dataStoreName, this.clientType);
return callback(missingVerIdInternalError);
}
// remove quotes from eTag because they're added later
completeObjData.eTag = removeQuotes(completeMpuRes.ETag);
completeObjData.dataStoreVersionId = completeMpuRes.VersionId;
completeObjData.contentLength =
Number.parseInt(completeMpuRes.ContentLength, 10);
return callback(null, completeObjData);
});
}
uploadPart(request, streamingV4Params, stream, size, key, uploadId,
partNumber, bucketName, config, log, callback) {
let hashedStream = stream;
if (request) {
const partStream = prepareStream(request, streamingV4Params,
log, callback);
hashedStream = new MD5Sum();
partStream.pipe(hashedStream);
}
const gcpKey = this._createGcpKey(bucketName, key, this._bucketMatch);
const params = {
Bucket: this._mpuBucketName,
Key: gcpKey,
UploadId: uploadId,
Body: hashedStream,
ContentLength: size,
PartNumber: partNumber };
return this._client.uploadPart(params, (err, partResObj) => {
if (err) {
logHelper(log, 'error', 'err from data backend ' +
'on uploadPart', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`)
);
}
// remove quotes from eTag because they're added later
const noQuotesETag = removeQuotes(partResObj.ETag);
const dataRetrievalInfo = {
key: gcpKey,
dataStoreType: 'gcp',
dataStoreName: this._dataStoreName,
dataStoreETag: noQuotesETag,
};
return callback(null, dataRetrievalInfo);
});
}
uploadPartCopy(request, gcpSourceKey, sourceLocationConstraintName, config,
log, callback) {
const destBucketName = request.bucketName;
const destObjectKey = request.objectKey;
const destGcpKey = this._createGcpKey(destBucketName, destObjectKey,
this._bucketMatch);
const sourceGcpBucketName =
config.getGcpBucketNames(sourceLocationConstraintName).bucketName;
const uploadId = request.query.uploadId;
const partNumber = request.query.partNumber;
const copySourceRange = request.headers['x-amz-copy-source-range'];
if (copySourceRange) {
return callback(errors.NotImplemented
.customizeDescription('Error returned from ' +
`${this.clientType}: copySourceRange not implemented`)
);
}
const params = {
Bucket: this._mpuBucketName,
CopySource: `${sourceGcpBucketName}/${gcpSourceKey}`,
Key: destGcpKey,
UploadId: uploadId,
PartNumber: partNumber,
};
return this._client.uploadPartCopy(params, (err, res) => {
if (err) {
if (err.code === 'AccesssDenied') {
logHelper(log, 'error', 'Unable to access ' +
`${sourceGcpBucketName} GCP bucket`, err,
this._dataStoreName, this.clientType);
return callback(errors.AccessDenied
.customizeDescription('Error: Unable to access ' +
`${sourceGcpBucketName} GCP bucket`)
);
}
logHelper(log, 'error', 'error from data backend on ' +
'uploadPartCopy', err, this._dataStoreName);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`)
);
}
// remove quotes from eTag because they're added later
const eTag = removeQuotes(res.CopyObjectResult.ETag);
return callback(null, eTag);
});
}
abortMPU(key, uploadId, bucketName, log, callback) {
const gcpKey = this._createGcpKey(bucketName, key, this._bucketMatch);
const getParams = {
Bucket: this._gcpBucketName,
MPU: this._mpuBucketName,
Key: gcpKey,
UploadId: uploadId,
};
return this._client.abortMultipartUpload(getParams, err => {
if (err) {
logHelper(log, 'error', 'err from data backend ' +
'on abortMPU', err, this._dataStoreName, this.clientType);
return callback(errors.ServiceUnavailable
.customizeDescription('Error returned from ' +
`GCP: ${err.message}`)
);
}
return callback();
});
}
}
module.exports = GcpClient;

147
lib/storage/data/external/PfsClient.js vendored Normal file
View File

@ -0,0 +1,147 @@
const errors = require('../../../errors');
const { createLogger, logHelper } = require('./utils');
const RESTClient = require('../../../network/rest/RESTClient');
class PfsClient {
constructor(config) {
const { host, port } = config.endpoint;
this.clientType = 'pfs';
this._bucketMatch = config.bucketMatch;
this._dataStoreName = config.dataStoreName;
this._restClient = new RESTClient({
host,
port,
isPassthrough: true,
});
}
setup(cb) {
return cb();
}
_createFsKey(requestBucketName, requestObjectKey, bucketMatch) {
if (bucketMatch) {
return requestObjectKey;
}
return `${requestBucketName}/${requestObjectKey}`;
}
toObjectGetInfo(objectKey, bucketName) {
return {
key: this._createFsKey(bucketName, objectKey, this._bucketMatch),
dataStoreName: this._dataStoreName,
};
}
put(stream, size, keyContext, reqUids, callback) {
const log = createLogger(reqUids);
if (keyContext.metaHeaders['x-amz-meta-mdonly'] === 'true') {
const b64 = keyContext.metaHeaders['x-amz-meta-md5chksum'];
let md5 = null;
if (b64) {
md5 = new Buffer(b64, 'base64').toString('hex');
}
return callback(null, keyContext.objectKey, '',
keyContext.metaHeaders['x-amz-meta-size'],
md5
);
}
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
get(objectGetInfo, range, reqUids, callback) {
const log = createLogger(reqUids);
this._restClient.get(objectGetInfo.key, range, reqUids, (err, rs) => {
if (err) {
logHelper(log, 'error', 'err from data backend', err,
this._dataStoreName, this.clientType);
return callback(err, null);
}
return callback(null, rs);
});
}
delete(objectGetInfo, reqUids, callback) {
const log = createLogger(reqUids);
const key = typeof objectGetInfo === 'string' ? objectGetInfo :
objectGetInfo.key;
this._restClient.delete(key, reqUids, err => {
if (err) {
logHelper(log, 'error', 'err from data backend', err,
this._dataStoreName, this.clientType);
return callback(err);
}
return callback();
});
}
// TODO: Implement a healthcheck
healthcheck(location, callback) {
const fsResp = {};
return callback(null, fsResp);
}
createMPU(key, metaHeaders, bucketName, websiteRedirectHeader, contentType,
cacheControl, contentDisposition, contentEncoding, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
uploadPart(request, streamingV4Params, stream, size, key, uploadId,
partNumber, bucketName, config, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
listParts(key, uploadId, bucketName, partNumberMarker, maxParts, log,
callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
completeMPU(jsonList, mdInfo, key, uploadId, bucketName, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
abortMPU(key, uploadId, bucketName, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
objectPutTagging(key, bucket, objectMD, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
objectDeleteTagging(key, bucket, objectMD, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
copyObject(request, destLocationConstraintName, sourceKey,
sourceLocationConstraintName, storeMetadataParams, config, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
uploadPartCopy(request, awsSourceKey, sourceLocationConstraintName,
config, log, callback) {
logHelper(log, 'error', 'Not implemented', errors.NotImplemented,
this._dataStoreName, this.clientType);
return callback(errors.NotImplemented);
}
}
module.exports = PfsClient;

222
lib/storage/data/external/utils.js vendored Normal file
View File

@ -0,0 +1,222 @@
const async = require('async');
const werelogs = require('werelogs');
const constants = require('../../../constants');
/* eslint-disable camelcase */
const backendHealth = {
aws_s3: {
response: undefined,
time: 0,
},
azure: {
response: undefined,
time: 0,
},
gcp: {
reponse: undefined,
time: 0,
},
};
/* eslint-enable camelcase */
const logger = new werelogs.Logger('MultipleBackendGateway');
const utils = {
createLogger(reqUids) {
return reqUids ?
logger.newRequestLoggerFromSerializedUids(reqUids) :
logger.newRequestLogger();
},
logHelper(log, level, description, error, dataStoreName, backendType) {
const { message, name, requestId, extendedRequestId } = error;
log[level](description, {
error: message,
errorName: name,
dataStoreName,
backendType,
extRequestId: requestId,
extExtendedRequestId: extendedRequestId,
});
},
// take off the 'x-amz-meta-'
trimXMetaPrefix(obj) {
const newObj = {};
const metaObj = obj || {};
Object.keys(metaObj).forEach(key => {
const newKey = key.substring(11);
newObj[newKey] = metaObj[key];
});
return newObj;
},
removeQuotes(word) {
return word.slice(1, -1);
},
skipMpuPartProcessing(completeObjData) {
const backendType = completeObjData.dataStoreType;
if (constants.mpuMDStoredExternallyBackend[backendType]) {
return true;
}
return false;
},
/**
* checkAzureBackendMatch - checks that the external backend location for
* two data objects is the same and is Azure
* @param {array} objectDataOne - data of first object to compare
* @param {object} objectDataTwo - data of second object to compare
* @return {boolean} - true if both data backends are Azure, false if not
*/
checkAzureBackendMatch(objectDataOne, objectDataTwo) {
if (objectDataOne.dataStoreType === 'azure' &&
objectDataTwo.dataStoreType === 'azure') {
return true;
}
return false;
},
/**
* externalBackendCopy - Server side copy should only be allowed:
* 1) if source object and destination object are both on aws, both
* on azure, or both on gcp
* 2) if azure to azure, must be the same storage account since Azure
* copy outside of an account is async
* 3) if the source bucket is not an encrypted bucket and the
* destination bucket is not an encrypted bucket (unless the copy
* is all within the same bucket).
* @param {object} config - CloudServer config
* @param {string} locationConstraintSrc - location constraint of the source
* @param {string} locationConstraintDest - location constraint of the
* destination
* @param {object} sourceBucketMD - metadata of the source bucket
* @param {object} destBucketMD - metadata of the destination bucket
* @return {boolean} - true if copying object from one
* externalbackend to the same external backend and for Azure if it is the
* same account since Azure copy outside of an account is async
*/
externalBackendCopy(config, locationConstraintSrc, locationConstraintDest,
sourceBucketMD, destBucketMD) {
const sourceBucketName = sourceBucketMD.getName();
const destBucketName = destBucketMD.getName();
const isSameBucket = sourceBucketName === destBucketName;
const bucketsNotEncrypted = destBucketMD.getServerSideEncryption()
=== sourceBucketMD.getServerSideEncryption() &&
sourceBucketMD.getServerSideEncryption() === null;
const sourceLocationConstraintType =
config.getLocationConstraintType(locationConstraintSrc);
const locationTypeMatch =
config.getLocationConstraintType(locationConstraintSrc) ===
config.getLocationConstraintType(locationConstraintDest);
return locationTypeMatch && (isSameBucket || bucketsNotEncrypted) &&
(sourceLocationConstraintType === 'aws_s3' ||
sourceLocationConstraintType === 'gcp' ||
(sourceLocationConstraintType === 'azure' &&
config.isSameAzureAccount(locationConstraintSrc,
locationConstraintDest)));
},
checkExternalBackend(clients, locations, type, flightCheckOnStartUp,
externalBackendHealthCheckInterval, cb) {
const checkStatus = backendHealth[type] || {};
if (locations.length === 0) {
return process.nextTick(cb, null, []);
}
if (!flightCheckOnStartUp && checkStatus.response &&
Date.now() - checkStatus.time < externalBackendHealthCheckInterval) {
return process.nextTick(cb, null, checkStatus.response);
}
let locationsToCheck;
if (flightCheckOnStartUp) {
// check all locations if flight check on start up
locationsToCheck = locations;
} else {
const randomLocation = locations[Math.floor(Math.random() *
locations.length)];
locationsToCheck = [randomLocation];
}
return async.mapLimit(locationsToCheck, 5, (location, next) => {
const client = clients[location];
client.healthcheck(location, next, flightCheckOnStartUp);
}, (err, results) => {
if (err) {
return cb(err);
}
if (!flightCheckOnStartUp) {
checkStatus.response = results;
checkStatus.time = Date.now();
}
return cb(null, results);
});
},
translateAzureMetaHeaders(metaHeaders, tags) {
const translatedMetaHeaders = {};
if (tags) {
// tags are passed as string of format 'key1=value1&key2=value2'
const tagObj = {};
const tagArr = tags.split('&');
tagArr.forEach(keypair => {
const equalIndex = keypair.indexOf('=');
const key = keypair.substring(0, equalIndex);
tagObj[key] = keypair.substring(equalIndex + 1);
});
translatedMetaHeaders.tags = JSON.stringify(tagObj);
}
Object.keys(metaHeaders).forEach(headerName => {
const translated = headerName.substring(11).replace(/-/g, '_');
translatedMetaHeaders[translated] = metaHeaders[headerName];
});
return translatedMetaHeaders;
},
/**
* proxyCompareUrl - compares request endpoint to urls in NO_PROXY env var
* @param {string} endpoint - url of request
* @return {bool} true if request endpoint matches no proxy, false if not
*/
proxyCompareUrl(endpoint) {
const noProxy = process.env.NO_PROXY || process.env.no_proxy;
if (!noProxy) {
return false;
}
// noProxy env var is a comma separated list of urls not to proxy
const noProxyList = noProxy.split(',');
if (noProxyList.includes(endpoint)) {
return true;
}
const epArr = endpoint.split('.');
// reverse array to make comparison easier
epArr.reverse();
let match = false;
for (let j = 0; j < noProxyList.length; j++) {
const urlArr = noProxyList[j].split('.');
urlArr.reverse();
for (let i = 0; i < epArr.length; i++) {
if (epArr[i] === urlArr[i]) {
match = true;
} else if (urlArr[i] === '*' && i === (urlArr.length - 1)) {
// if first char of url is '*', remaining endpoint matches
match = true;
break;
} else if (urlArr[i] === '' && i === (urlArr.length - 1)) {
// if first char of url is '.', it is treated as wildcard
match = true;
break;
} else if (urlArr[i] === '*') {
match = true;
} else if (epArr[i] !== urlArr[i]) {
match = false;
break;
}
}
// if endpoint matches noProxy element, stop checking
if (match) {
break;
}
}
// if endpoint matches, request should not be proxied
if (match) {
return true;
}
return false;
},
};
module.exports = utils;

View File

@ -3,7 +3,7 @@
"engines": {
"node": ">=8"
},
"version": "8.1.3",
"version": "8.1.4",
"description": "Common utilities for the S3 project components",
"main": "index.js",
"repository": {
@ -20,6 +20,9 @@
"JSONStream": "^1.0.0",
"ajv": "4.10.0",
"async": "~2.6.1",
"aws-sdk": "2.28.0",
"azure-storage": "^2.1.0",
"backo": "^1.1.0",
"bson": "4.0.0",
"debug": "~4.1.0",
"diskusage": "^1.0.0",
@ -36,6 +39,7 @@
"socket.io-client": "~2.2.0",
"utf8": "3.0.0",
"uuid": "^3.0.1",
"vaultclient": "scality/vaultclient#754b6e1",
"werelogs": "scality/werelogs#0ff7ec82",
"xml2js": "~0.4.16"
},

View File

@ -0,0 +1,153 @@
const async = require('async');
const assert = require('assert');
const uuid = require('uuid/v4');
const { makeGcpRequest } = require('./makeRequest');
const genUniqID = () => uuid().replace(/-/g, '');
function gcpRequestRetry(params, retry, callback) {
const maxRetries = 4;
const timeout = Math.pow(2, retry) * 1000;
return setTimeout(makeGcpRequest, timeout, params, (err, res) => {
if (err) {
if (retry <= maxRetries && err.statusCode === 429) {
return gcpRequestRetry(params, retry + 1, callback);
}
return callback(err);
}
return callback(null, res);
});
}
function gcpClientRetry(fn, params, callback, retry = 0) {
const maxRetries = 4;
const timeout = Math.pow(2, retry) * 1000;
return setTimeout(fn, timeout, params, (err, res) => {
if (err) {
if (retry <= maxRetries && err.statusCode === 429) {
return gcpClientRetry(fn, params, callback, retry + 1);
}
return callback(err);
}
return callback(null, res);
});
}
// mpu test helpers
function gcpMpuSetup(params, callback) {
const { gcpClient, bucketNames, key, partCount, partSize } = params;
return async.waterfall([
next => gcpClient.createMultipartUpload({
Bucket: bucketNames.mpu.Name,
Key: key,
}, (err, res) => {
assert.equal(err, null,
`Expected success, but got error ${err}`);
return next(null, res.UploadId);
}),
(uploadId, next) => {
if (partCount <= 0) {
return next('SkipPutPart', { uploadId });
}
const arrayData = Array.from(Array(partCount).keys());
const etagList = Array(partCount);
let count = 0;
return async.eachLimit(arrayData, 10,
(info, moveOn) => {
gcpClient.uploadPart({
Bucket: bucketNames.mpu.Name,
Key: key,
UploadId: uploadId,
PartNumber: info + 1,
Body: Buffer.alloc(partSize),
ContentLength: partSize,
}, (err, res) => {
if (err) {
return moveOn(err);
}
if (!(++count % 100)) {
process.stdout.write(`Uploaded Parts: ${count}\n`);
}
etagList[info] = res.ETag;
return moveOn(null);
});
}, err => {
next(err, { uploadId, etagList });
});
},
], (err, result) => {
if (err) {
if (err === 'SkipPutPart') {
return callback(null, result);
}
return callback(err);
}
return callback(null, result);
});
}
function genPutTagObj(size, duplicate) {
const retTagSet = [];
Array.from(Array(size).keys()).forEach(ind => {
retTagSet.push({
Key: duplicate ? 'dupeKey' : `key${ind}`,
Value: `Value${ind}`,
});
});
return retTagSet;
}
function genGetTagObj(size, tagPrefix) {
const retObj = {};
const expectedTagObj = [];
for (let i = 1; i <= size; ++i) {
retObj[`${tagPrefix}testtag${i}`] = `testtag${i}`;
expectedTagObj.push({
Key: `testtag${i}`,
Value: `testtag${i}`,
});
}
return { tagHeader: retObj, expectedTagObj };
}
function genDelTagObj(size, tagPrefix) {
const headers = {};
const expectedTagObj = {};
const expectedMetaObj = {};
for (let i = 1; i <= size; ++i) {
headers[`${tagPrefix}testtag${i}`] = `testtag${i}`;
expectedTagObj[`${tagPrefix}testtag${i}`] = `testtag${i}`;
headers[`x-goog-meta-testmeta${i}`] = `testmeta${i}`;
expectedMetaObj[`x-goog-meta-testmeta${i}`] = `testmeta${i}`;
}
return { headers, expectedTagObj, expectedMetaObj };
}
/*
<CreateBucketConfiguration>
<LocationConstraint><location></LocationConstraint>
<StorageClass><storage class></StorageClass>
</CreateBucketConfiguration>
*/
const regionalLoc = 'us-west1';
const multiRegionalLoc = 'us';
function setBucketClass(storageClass) {
const locationConstraint =
storageClass === 'REGIONAL' ? regionalLoc : multiRegionalLoc;
return '<CreateBucketConfiguration>' +
`<LocationConstraint>${locationConstraint}</LocationConstraint>` +
`<StorageClass>${storageClass}</StorageClass>` +
'</CreateBucketConfiguration>';
}
module.exports = {
gcpRequestRetry,
gcpClientRetry,
setBucketClass,
gcpMpuSetup,
genPutTagObj,
genGetTagObj,
genDelTagObj,
genUniqID,
};

View File

@ -0,0 +1,163 @@
const http = require('http');
const querystring = require('querystring');
const auth = require('../../../../lib/auth/auth');
const GcpSigner =
require('../../../../lib/storage/data/external/GCP/GcpSigner');
function _parseError(responseBody, statusCode, jsonResponse) {
if (jsonResponse && statusCode !== 200) {
return JSON.parse(responseBody);
}
if (responseBody.indexOf('<Error>') > -1) {
const error = {};
const codeStartIndex = responseBody.indexOf('<Code>') + 6;
const codeEndIndex = responseBody.indexOf('</Code>');
error.code = responseBody.slice(codeStartIndex, codeEndIndex);
const msgStartIndex = responseBody.indexOf('<Message>') + 9;
const msgEndIndex = responseBody.indexOf('</Message>');
error.message = responseBody.slice(msgStartIndex, msgEndIndex);
return error;
}
return null;
}
function _decodeURI(uri) {
// do the same decoding than in S3 server
return decodeURIComponent(uri.replace(/\+/g, ' '));
}
/** makeRequest - utility function to generate a request
* @param {object} params - params for making request
* @param {string} params.hostname - request hostname
* @param {number} [params.port] - request port
* @param {string} params.method - request method
* @param {object} [params.queryObj] - query fields and their string values
* @param {object} [params.headers] - headers and their string values
* @param {string} [params.path] - URL-encoded request path
* @param {object} [params.authCredentials] - authentication credentials
* @param {object} params.authCredentials.accessKey - access key
* @param {object} params.authCredentials.secretKey - secret key
* @param {object} params.GCP - flag to setup for GCP request
* @param {string} [params.requestBody] - request body contents
* @param {boolean} [params.jsonResponse] - if true, response is
* expected to be received in JSON format (including errors)
* @param {function} callback - with error and response parameters
* @return {undefined} - and call callback
*/
function makeRequest(params, callback) {
const { hostname, port, method, queryObj, headers, path,
authCredentials, requestBody, jsonResponse } = params;
const options = {
hostname,
port,
method,
headers,
path: path || '/',
rejectUnauthorized: false,
};
const qs = querystring.stringify(queryObj);
if (params.GCP && authCredentials) {
const gcpPath = queryObj ? `${options.path}?${qs}` : options.path;
const getAuthObject = {
endpoint: { host: hostname },
method,
path: gcpPath || '/',
headers,
};
const signer = new GcpSigner(getAuthObject);
signer.addAuthorization(authCredentials, new Date());
Object.assign(options.headers, {
Authorization: getAuthObject.headers.Authorization,
Date: getAuthObject.headers['x-goog-date'],
});
}
const req = http.request(options, res => {
const body = [];
res.on('data', chunk => {
body.push(chunk);
});
res.on('error', err => {
process.stdout.write('err receiving response');
return callback(err);
});
res.on('end', () => {
const total = body.join('');
const data = {
headers: res.headers,
statusCode: res.statusCode,
body: total,
};
const err = _parseError(total, res.statusCode, jsonResponse);
if (err) {
err.statusCode = res.statusCode;
}
return callback(err, data);
});
});
req.on('error', err => {
process.stdout.write('err sending request');
return callback(err);
});
// generate v4 headers if authentication credentials are provided
const encodedPath = req.path;
// decode path because signing code re-encodes it
req.path = _decodeURI(encodedPath);
if (authCredentials && !params.GCP) {
if (queryObj) {
auth.client.generateV4Headers(req, queryObj,
authCredentials.accessKey, authCredentials.secretKey, 's3');
// may update later if request may contain POST body
} else {
auth.client.generateV4Headers(req, '', authCredentials.accessKey,
authCredentials.secretKey, 's3');
}
}
// restore original URL-encoded path
req.path = encodedPath;
req.path = queryObj ? `${options.path}?${qs}` : req.path;
if (requestBody) {
req.write(requestBody);
}
req.end();
}
/** makeGcpRequest - utility function to generate a request against GCP
* @param {object} params - params for making request
* @param {string} params.method - request method
* @param {object} [params.queryObj] - query fields and their string values
* @param {object} [params.headers] - headers and their string values
* @param {string} [params.bucket] - bucket name
* @param {string} [params.objectKey] - object key name
* @param {object} [params.authCredentials] - authentication credentials
* @param {object} params.authCredentials.accessKey - access key
* @param {object} params.authCredentials.secretKey - secret key
* @param {function} callback - with error and response parameters
* @return {undefined} - and call callback
*/
function makeGcpRequest(params, callback) {
const { method, queryObj, headers, bucket, objectKey, authCredentials,
requestBody } = params;
const options = {
authCredentials,
requestBody,
hostname: 'storage.googleapis.com',
port: 80,
method,
queryObj,
headers: headers || {},
path: bucket ? `/${bucket}/` : '/',
GCP: true,
};
if (objectKey) {
options.path = `${options.path}${objectKey}`;
}
makeRequest(options, callback);
}
module.exports = {
makeRequest,
makeGcpRequest,
};

View File

@ -0,0 +1,78 @@
const assert = require('assert');
const { Readable } = require('stream');
const V4Transform =
require('../../../../../lib/auth/v4/streamingV4/V4Transform');
const { DummyRequestLogger, DummyAuthConfig } = require('../../../helpers');
const log = new DummyRequestLogger();
const streamingV4Params = {
accessKey: 'accessKey1',
signatureFromRequest: '2b8637632a997e06ee7b6c85d7' +
'147d2025e8f04d4374f4d7d7320de1618c7509',
region: 'us-east-1',
scopeDate: '20170516',
timestamp: '20170516T204738Z',
credentialScope: '20170516/us-east-1/s3/aws4_request',
};
const cloudserverConfig = new DummyAuthConfig();
class AuthMe extends Readable {
constructor(chunks) {
super();
this._parts = chunks;
this._index = 0;
}
_read() {
this.push(this._parts[this._index]);
this._index++;
}
}
describe('V4Transform class', () => {
it('should authenticate successfully', done => {
const v4Transform = new V4Transform(streamingV4Params,
cloudserverConfig, log, err => {
assert.strictEqual(err, null);
});
const filler1 = '8;chunk-signature=51d2511f7c6887907dff20474d8db6' +
'7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n';
const filler2 = '0;chunk-signature=c0eac24b7ce72141ec077df9753db' +
'4cc8b7991491806689da0395c8bd0231e48\r\n';
const chunks = [
Buffer.from(filler1),
Buffer.from(filler2),
null,
];
const authMe = new AuthMe(chunks);
authMe.pipe(v4Transform);
v4Transform.on('finish', () => {
done();
});
});
it('should ignore data sent after final chunk', done => {
const v4Transform = new V4Transform(streamingV4Params,
cloudserverConfig, log, err => {
assert.strictEqual(err, null);
done();
});
const filler1 = '8;chunk-signature=51d2511f7c6887907dff20474d8db6' +
'7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n';
const filler2 = '0;chunk-signature=c0eac24b7ce72141ec077df9753db' +
'4cc8b7991491806689da0395c8bd0231e48\r\n';
const filler3 = '\r\n';
const chunks = [
Buffer.from(filler1),
Buffer.from(filler2),
Buffer.from(filler3),
null,
];
const authMe = new AuthMe(chunks);
authMe.pipe(v4Transform);
v4Transform.on('finish', () => {
done();
});
});
});

View File

@ -1,4 +1,5 @@
'use strict'; // eslint-disable-line strict
const { EventEmitter } = require('events');
const AuthInfo = require('../../lib/auth/AuthInfo');
const constants = require('../../lib/constants');
@ -110,5 +111,39 @@ class DummyRequestLogger {
}
}
class DummyAuthConfig extends EventEmitter {
constructor() {
super();
this.backends = { auth: 'mem' };
this.authData = {
accounts:
[{ name: 'Bart',
email: 'sampleaccount1@sampling.com',
arn: 'arn:aws:iam::123456789012:root',
canonicalID:
'79a59df900b949e55d96a1e698fbacedf' +
'd6e09d98eacf8f8d5218e7cd47ef2be',
shortid: '123456789012',
keys: [{ access: 'accessKey1', secret: 'verySecretKey1' }] },
{ name: 'Lisa',
email: 'sampleaccount2@sampling.com',
arn: 'arn:aws:iam::123456789013:root',
canonicalID:
'79a59df900b949e55d96a1e698fbacedf' +
'd6e09d98eacf8f8d5218e7cd47ef2bf',
shortid: '123456789013',
keys: [{ access: 'accessKey2', secret: 'verySecretKey2' }] },
],
};
}
setAuthDataAccounts(accounts) {
this.authData.accounts = accounts;
this.emit('authdata-update');
}
}
module.exports = { makeid, timeDiff, makeAuthInfo,
createAlteredRequest, zpad, DummyRequestLogger };
createAlteredRequest, zpad, DummyRequestLogger,
DummyAuthConfig };

View File

@ -0,0 +1,62 @@
const uuid = require('uuid/v4');
class DummyService {
constructor(config = {}) {
this.versioning = config.versioning;
}
headBucket(params, callback) {
return callback();
}
getBucketVersioning(params, callback) {
if (this.versioning) {
return callback(null, { Status: 'Enabled' });
}
return callback(null, {});
}
headObject(params, callback) {
const retObj = {
ContentLength: `${1024 * 1024 * 1024}`,
};
return callback(null, retObj);
}
completeMultipartUpload(params, callback) {
const retObj = {
Bucket: params.Bucket,
Key: params.Key,
ETag: `"${uuid().replace(/-/g, '')}"`,
ContentLength: `${1024 * 1024 * 1024}`,
};
if (this.versioning) {
retObj.VersionId = uuid().replace(/-/g, '');
}
return callback(null, retObj);
}
upload(params, callback) {
this.putObject(params, callback);
}
putObject(params, callback) {
const retObj = {
ETag: `"${uuid().replace(/-/g, '')}"`,
};
if (this.versioning) {
retObj.VersionId = uuid().replace(/-/g, '');
}
return callback(null, retObj);
}
copyObject(params, callback) {
const retObj = {
CopyObjectResult: {
ETag: `"${uuid().replace(/-/g, '')}"`,
LastModified: new Date().toISOString(),
},
VersionId: null,
};
if (this.versioning) {
retObj.VersionId = uuid().replace(/-/g, '');
}
return callback(null, retObj);
}
// To-Do: add tests for other methods
}
module.exports = DummyService;

View File

@ -0,0 +1,107 @@
const assert = require('assert');
const AwsClient = require('../../../../../lib/storage/data/external/AwsClient');
const GcpClient = require('../../../../../lib/storage/data/external/GcpClient');
const AzureClient =
require('../../../../../lib/storage/data/external/AzureClient');
const DummyService = require('../DummyService');
const { DummyRequestLogger } = require('../../../helpers');
const backendClients = [
{
Class: AwsClient,
name: 'AwsClient',
config: {
s3Params: {},
bucketName: 'awsTestBucketName',
dataStoreName: 'awsDataStore',
serverSideEncryption: false,
type: 'aws',
},
},
{
Class: GcpClient,
name: 'GcpClient',
config: {
s3Params: {},
bucketName: 'gcpTestBucketName',
mpuBucket: 'gcpTestMpuBucketName',
dataStoreName: 'gcpDataStore',
type: 'gcp',
},
},
{
Class: AzureClient,
name: 'AzureClient',
config: {
azureStorageEndpoint: '',
azureStorageCredentials: {
storageAccountName: 'scality',
storageAccessKey: 'Zm9vCg==',
},
azureContainerName: 'azureTestBucketName',
dataStoreName: 'azureDataStore',
type: 'azure',
},
},
];
const log = new DummyRequestLogger();
describe('external backend clients', () => {
backendClients.forEach(backend => {
let testClient;
before(() => {
testClient = new backend.Class(backend.config);
testClient._client = new DummyService({ versioning: true });
});
if (backend.config.type !== 'azure') {
it(`${backend.name} completeMPU should return correctly ` +
'typed mpu results', done => {
const jsonList = {
Part: [
{
PartNumber: [1],
ETag: ['testpart0001etag'],
},
{
PartNumber: [2],
ETag: ['testpart0002etag'],
},
{
PartNumber: [3],
ETag: ['testpart0003etag'],
},
],
};
const key = 'externalBackendTestKey';
const bucketName = 'externalBackendTestBucket';
const uploadId = 'externalBackendTestUploadId';
testClient.completeMPU(jsonList, null, key,
uploadId, bucketName, log, (err, res) => {
assert.strictEqual(typeof res.key, 'string');
assert.strictEqual(typeof res.eTag, 'string');
assert.strictEqual(typeof res.dataStoreVersionId,
'string');
assert.strictEqual(typeof res.contentLength, 'number');
return done();
});
});
}
it(`${backend.name} toObjectGetInfo should return correct ` +
'objectGetInfo object', () => {
const key = 'externalBackendTestKey';
const bucketName = 'externalBackendTestBucket';
const objectGetInfo = testClient.toObjectGetInfo(key, bucketName);
assert.deepStrictEqual(objectGetInfo, {
// bucketMatch === false => expect bucket name to be
// prefixed to the backend key
key: 'externalBackendTestBucket/externalBackendTestKey',
dataStoreName: backend.config.dataStoreName,
});
});
// To-Do: test the other external client methods
});
});

View File

@ -0,0 +1,194 @@
const assert = require('assert');
const http = require('http');
const { GCP } = require('../../../../../lib/storage/data/external/GCP');
const httpPort = 8888;
// test values
const host = 'localhost:8888';
const Bucket = 'testrequestbucket';
const Key = 'testRequestKey';
const MultipartUpload = { Parts: [{ PartName: 'part' }] };
const CopySource = 'copyBucket/copyKey';
const accessKeyId = 'accesskey';
const secretAccessKey = 'secretaccesskey';
function handler(isPathStyle) {
return (req, res) => {
if (isPathStyle) {
assert(req.headers.host, host);
assert(req.url.includes(Bucket));
} else {
assert(req.headers.host, `${Bucket}.${host}`);
assert(!req.url.includes(Bucket));
}
res.end();
};
}
const invalidBucketNames = [
'..',
'.bucketname',
'bucketname.',
'bucketName.',
'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',
'256.256.256.256',
'',
];
function badBucketNameHandler(req, res) {
assert(req.headers.host, host);
const bucketFromUrl = req.url.split('/')[1];
assert.strictEqual(typeof bucketFromUrl, 'string');
assert(invalidBucketNames.includes(bucketFromUrl));
res.end();
}
const operations = [
{
op: 'headBucket',
params: { Bucket },
},
{
op: 'listObjects',
params: { Bucket },
},
{
op: 'listVersions',
params: { Bucket },
},
{
op: 'getBucketVersioning',
params: { Bucket },
},
{
op: 'headObject',
params: { Bucket, Key },
},
{
op: 'putObject',
params: { Bucket, Key },
},
{
op: 'getObject',
params: { Bucket, Key },
},
{
op: 'deleteObject',
params: { Bucket, Key },
},
{
op: 'composeObject',
params: { Bucket, Key, MultipartUpload },
},
{
op: 'copyObject',
params: { Bucket, Key, CopySource },
},
];
describe('GcpService request behavior', function testSuite() {
this.timeout(120000);
let httpServer;
let client;
before(done => {
client = new GCP({
endpoint: `http://${host}`,
maxRetries: 0,
s3ForcePathStyle: false,
accessKeyId,
secretAccessKey,
});
httpServer =
http.createServer(badBucketNameHandler).listen(httpPort);
httpServer.on('listening', done);
httpServer.on('error', err => {
process.stdout.write(`https server: ${err.stack}\n`);
process.exit(1);
});
});
after('Terminating Server', () => {
httpServer.close();
});
invalidBucketNames.forEach(bucket => {
it(`should not use dns-style if bucket isn't dns compatible: ${bucket}`,
done => {
client.headBucket({ Bucket: bucket }, err => {
assert.ifError(err);
done();
});
});
});
});
describe('GcpService pathStyle tests', function testSuite() {
this.timeout(120000);
let httpServer;
let client;
before(done => {
client = new GCP({
endpoint: `http://${host}`,
maxRetries: 0,
s3ForcePathStyle: true,
accessKeyId,
secretAccessKey,
});
httpServer =
http.createServer(handler(true)).listen(httpPort);
httpServer.on('listening', done);
httpServer.on('error', err => {
process.stdout.write(`https server: ${err.stack}\n`);
process.exit(1);
});
});
after('Terminating Server', () => {
httpServer.close();
});
operations.forEach(test => it(`GCP::${test.op}`, done => {
client[test.op](test.params, err => {
assert.ifError(err);
done();
});
}));
});
describe('GcpService dnsStyle tests', function testSuite() {
this.timeout(120000);
let httpServer;
let client;
before(done => {
client = new GCP({
endpoint: `http://${host}`,
maxRetries: 0,
s3ForcePathStyle: false,
accessKeyId,
secretAccessKey,
});
httpServer =
http.createServer(handler(false)).listen(httpPort);
httpServer.on('listening', done);
httpServer.on('error', err => {
process.stdout.write(`https server: ${err.stack}\n`);
process.exit(1);
});
});
after('Terminating Server', () => {
httpServer.close();
});
operations.forEach(test => it(`GCP::${test.op}`, done => {
client[test.op](test.params, err => {
assert.ifError(err);
done();
});
}));
});

View File

@ -0,0 +1,64 @@
const assert = require('assert');
const { checkExternalBackend } =
require('../../../../../lib/storage/data/external/utils');
const awsLocations = [
'awsbackend',
];
const statusSuccess = {
versioningStatus: 'Enabled',
message: 'Congrats! You own the bucket',
};
const statusFailure = {
versioningStatus: 'Suspended',
error: 'Versioning must be enabled',
external: true,
};
const externalBackendHealthCheckInterval = 10000;
function getClients(isSuccess) {
const status = isSuccess ? statusSuccess : statusFailure;
return {
awsbackend: {
healthcheck: (location, cb) => cb(null, { awsbackend: status }),
},
};
}
describe('Testing _checkExternalBackend', function describeF() {
this.timeout(50000);
beforeEach(done => {
const clients = getClients(true);
return checkExternalBackend(clients, awsLocations, 'aws_s3', false,
externalBackendHealthCheckInterval, done);
});
it('should not refresh response before externalBackendHealthCheckInterval',
done => {
const clients = getClients(false);
return checkExternalBackend(clients, awsLocations, 'aws_s3',
false, externalBackendHealthCheckInterval, (err, res) => {
if (err) {
return done(err);
}
assert.strictEqual(res[0].awsbackend, statusSuccess);
return done();
});
});
it('should refresh response after externalBackendHealthCheckInterval',
done => {
const clients = getClients(false);
setTimeout(() => {
checkExternalBackend(clients, awsLocations, 'aws_s3',
false, externalBackendHealthCheckInterval, (err, res) => {
if (err) {
return done(err);
}
assert.strictEqual(res[0].awsbackend, statusFailure);
return done();
});
}, externalBackendHealthCheckInterval + 1);
});
});

View File

@ -0,0 +1,65 @@
const assert = require('assert');
const uuid = require('uuid/v4');
const { createMpuKey, createMpuList } =
require('../../../../../lib/storage/data/external/GCP/GcpUtils');
const key = `somekey${Date.now()}`;
const uploadId = uuid().replace(/-/g, '');
const phase = 'createMpulist';
const size = 2;
const correctMpuList = [
{ PartName: `${key}-${uploadId}/${phase}/00001`, PartNumber: 1 },
{ PartName: `${key}-${uploadId}/${phase}/00002`, PartNumber: 2 },
];
describe('GcpUtils MPU Helper Functions:', () => {
describe('createMpuKey', () => {
const tests = [
{
it: 'if phase and part number are given',
input: { phase: 'test', partNumber: 1 },
output: `${key}-${uploadId}/test/00001`,
},
{
it: 'if only phase is given',
input: { phase: 'test' },
output: `${key}-${uploadId}/test`,
},
{
it: 'if part number is given',
input: { partNumber: 1 },
output: `${key}-${uploadId}/parts/00001`,
},
{
it: 'if phase and part number aren not given',
input: {},
output: `${key}-${uploadId}/`,
},
];
tests.forEach(test => {
it(test.it, () => {
const { partNumber, phase } = test.input;
assert.strictEqual(createMpuKey(
key, uploadId, partNumber, phase), test.output);
});
});
});
describe('createMpuList', () => {
const tests = [
{
it: 'should create valid mpu list',
input: { phase, size },
output: correctMpuList,
},
];
tests.forEach(test => {
it(test.it, () => {
const { phase, size } = test.input;
assert.deepStrictEqual(createMpuList(
{ Key: key, UploadId: uploadId }, phase, size),
test.output);
});
});
});
});

View File

@ -0,0 +1,156 @@
const assert = require('assert');
const errors = require('../../../../../lib/errors');
const { gcpTaggingPrefix } = require('../../../../../lib/constants');
const { genPutTagObj } =
require('../../../../functional/externalBackends/utils/gcpUtils');
const { processTagSet, stripTags, retrieveTags, getPutTagsMetadata } =
require('../../../../../lib/storage/data/external/GCP/GcpUtils');
const maxTagSize = 10;
const validTagSet = genPutTagObj(2);
const validTagObj = {};
validTagObj[`${gcpTaggingPrefix}key0`] = 'Value0';
validTagObj[`${gcpTaggingPrefix}key1`] = 'Value1';
const tagQuery = 'key0=Value0&key1=Value1';
const invalidSizeTagSet = genPutTagObj(maxTagSize + 1);
const invalidDuplicateTagSet = genPutTagObj(maxTagSize, true);
const invalidKeyTagSet = [{ Key: Buffer.alloc(129, 'a'), Value: 'value' }];
const invalidValueTagSet = [{ Key: 'key', Value: Buffer.alloc(257, 'a') }];
const onlyMetadata = {
metadata1: 'metadatavalue1',
metadata2: 'metadatavalue2',
};
const tagMetadata = Object.assign({}, validTagObj, onlyMetadata);
const oldTagMetadata = {};
oldTagMetadata[`${gcpTaggingPrefix}Old`] = 'OldValue0';
const withPriorTags = Object.assign({}, onlyMetadata, oldTagMetadata);
describe('GcpUtils Tagging Helper Functions:', () => {
describe('processTagSet', () => {
const tests = [
{
it: 'should return tag object as metadata for valid tag set',
input: validTagSet,
output: validTagObj,
},
{
it: 'should return error for invalid tag set size',
input: invalidSizeTagSet,
output: errors.BadRequest.customizeDescription(
'Object tags cannot be greater than 10'),
},
{
it: 'should return error for duplicate tag keys',
input: invalidDuplicateTagSet,
output: errors.InvalidTag.customizeDescription(
'Cannot provide multiple Tags with the same key'),
},
{
it: 'should return error for invalid "key" value',
input: invalidKeyTagSet,
output: errors.InvalidTag.customizeDescription(
'The TagKey provided is too long, 129'),
},
{
it: 'should return error for invalid "value" value',
input: invalidValueTagSet,
output: errors.InvalidTag.customizeDescription(
'The TagValue provided is too long, 257'),
},
{
it: 'should return empty tag object when input is undefined',
input: undefined,
output: {},
},
];
tests.forEach(test => {
it(test.it, () => {
assert.deepStrictEqual(processTagSet(test.input), test.output);
});
});
});
describe('stripTags', () => {
const tests = [
{
it: 'should return metadata without tag',
input: tagMetadata,
output: onlyMetadata,
},
{
it: 'should return empty object if metadata only has tags',
input: validTagObj,
output: {},
},
{
it: 'should return empty object if input is undefined',
input: undefined,
output: {},
},
];
tests.forEach(test => {
it(test.it, () => {
assert.deepStrictEqual(stripTags(test.input), test.output);
});
});
});
describe('retrieveTags', () => {
const tests = [
{
it: 'should return tagSet from given input metadata',
input: tagMetadata,
output: validTagSet,
},
{
it: 'should return empty when metadata does not have tags',
input: onlyMetadata,
output: [],
},
{
it: 'should return empty if input is undefined',
input: undefined,
output: [],
},
];
tests.forEach(test => {
it(test.it, () => {
assert.deepStrictEqual(retrieveTags(test.input), test.output);
});
});
});
describe('getPutTagsMetadata', () => {
const tests = [
{
it: 'should return correct object when' +
' given a tag query string and a metadata obj',
input: { metadata: Object.assign({}, onlyMetadata), tagQuery },
output: tagMetadata,
},
{
it: 'should return correct object when given only query string',
input: { tagQuery },
output: validTagObj,
},
{
it: 'should return correct object when only metadata is given',
input: { metadata: onlyMetadata },
output: onlyMetadata,
},
{
it: 'should return metadata with correct tag properties ' +
'if given a metdata with prior tags and query string',
input: { metadata: Object.assign({}, withPriorTags), tagQuery },
output: tagMetadata,
},
];
tests.forEach(test => {
it(test.it, () => {
const { metadata, tagQuery } = test.input;
assert.deepStrictEqual(
getPutTagsMetadata(metadata, tagQuery), test.output);
});
});
});
});

View File

@ -0,0 +1,56 @@
const assert = require('assert');
const { proxyCompareUrl } =
require('../../../../../lib/storage/data/external/utils');
const testCases = [
{
endpoint: 'test.scality.com',
noProxy: '',
expRes: false,
desc: 'no NO_PROXY env var set',
},
{
endpoint: 'test.scality.com',
noProxy: 'test.*.com',
expRes: true,
desc: 'NO_PROXY matches with middle wildcard',
},
{
endpoint: 'test.scality.com',
noProxy: '*.com',
expRes: true,
desc: 'NO_PROXY matches with beginning wildcard',
},
{
endpoint: 'test.scality.com',
noProxy: '.scality.com',
expRes: true,
desc: 'NO_PROXY matches with beginning period',
},
{
endpoint: 'test.scality.com',
noProxy: 'test.nomatch,test.scality.*',
expRes: true,
desc: 'match with wildcard',
},
{
endpoint: 'test.scality.com',
noProxy: 'test.nomatch,no.scality.no,no.*.com,scality.com',
expRes: false,
desc: 'no match',
},
];
describe('proxyCompareURL util function', () => {
testCases.forEach(test => {
it(`should return ${test.expRes} if ${test.desc}`, () => {
process.env.NO_PROXY = test.noProxy;
const proxyMatch = proxyCompareUrl(test.endpoint);
assert.strictEqual(test.expRes, proxyMatch);
});
});
after(() => {
process.env.NO_PROXY = '';
});
});