Compare commits
2 Commits
developmen
...
dev/bennet
Author | SHA1 | Date |
---|---|---|
Jonathan Gramain | 3920d3c207 | |
bbuchanan9 | 4879912bc8 |
|
@ -5,9 +5,10 @@ const httpProxy = require('http-proxy');
|
||||||
const backbeatProxy = httpProxy.createProxyServer({
|
const backbeatProxy = httpProxy.createProxyServer({
|
||||||
ignorePath: true,
|
ignorePath: true,
|
||||||
});
|
});
|
||||||
const { auth, errors, s3middleware, models } = require('arsenal');
|
const { auth, errors, s3middleware, models, storage } = require('arsenal');
|
||||||
const { responseJSONBody } = require('arsenal').s3routes.routesUtils;
|
const { responseJSONBody } = require('arsenal').s3routes.routesUtils;
|
||||||
const { getSubPartIds } = s3middleware.azureHelper.mpuUtils;
|
const { getSubPartIds } = s3middleware.azureHelper.mpuUtils;
|
||||||
|
const { skipMpuPartProcessing } = storage.data.external.backendUtils;
|
||||||
const vault = require('../auth/vault');
|
const vault = require('../auth/vault');
|
||||||
const { data } = require('../data/wrapper');
|
const { data } = require('../data/wrapper');
|
||||||
const metadata = require('../metadata/wrapper');
|
const metadata = require('../metadata/wrapper');
|
||||||
|
@ -183,9 +184,64 @@ function getPartList(parts, objectKey, uploadId, storageLocation) {
|
||||||
} else {
|
} else {
|
||||||
partList.Part = parts;
|
partList.Part = parts;
|
||||||
}
|
}
|
||||||
|
|
||||||
return partList;
|
return partList;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function generateMpuAggregateInfo(parts) {
|
||||||
|
let aggregateSize;
|
||||||
|
|
||||||
|
// CopyLocationTask does transmit a size for each part,
|
||||||
|
// MultipleBackendTask does not, so check if size is defined in
|
||||||
|
// the first part.
|
||||||
|
if (parts[0] && parts[0].Size) {
|
||||||
|
aggregateSize = parts.reduce(
|
||||||
|
(agg, part) => agg + Number.parseInt(part.Size[0], 10), 0);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
aggregateSize,
|
||||||
|
aggregateETag: s3middleware.processMpuParts.createAggregateETag(
|
||||||
|
parts.map(part => part.ETag[0])),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Helper to create the response object for putObject and completeMPU
|
||||||
|
*
|
||||||
|
* @param {object} params - response info
|
||||||
|
* @param {string} params.dataStoreName - name of location
|
||||||
|
* @param {string} params.dataStoreType - location type (e.g. "aws_s3")
|
||||||
|
* @param {string} params.key - object key
|
||||||
|
* @param {number} params.size - total byte length
|
||||||
|
* @param {string} params.dataStoreETag - object ETag
|
||||||
|
* @param {string} [params.dataStoreVersionId] - object version ID, if
|
||||||
|
* versioned
|
||||||
|
* @return {object} - the response object to serialize and send back
|
||||||
|
*/
|
||||||
|
function constructPutResponse(params) {
|
||||||
|
// FIXME: The main data locations array may eventually resemble
|
||||||
|
// locations stored in replication info object, i.e. without
|
||||||
|
// size/start for cloud locations, which could ease passing
|
||||||
|
// standard location objects across services. For now let's just
|
||||||
|
// create the location as they are usually stored in the
|
||||||
|
// "locations" attribute, with size/start info.
|
||||||
|
|
||||||
|
const location = [{
|
||||||
|
dataStoreName: params.dataStoreName,
|
||||||
|
dataStoreType: params.dataStoreType,
|
||||||
|
key: params.key,
|
||||||
|
start: 0,
|
||||||
|
size: params.size,
|
||||||
|
dataStoreETag: params.dataStoreETag,
|
||||||
|
dataStoreVersionId: params.dataStoreVersionId,
|
||||||
|
}];
|
||||||
|
return {
|
||||||
|
// TODO: Remove '' when versioning implemented for Azure.
|
||||||
|
versionId: params.dataStoreVersionId || '',
|
||||||
|
location,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function handleTaggingOperation(request, response, type, dataStoreVersionId,
|
function handleTaggingOperation(request, response, type, dataStoreVersionId,
|
||||||
log, callback) {
|
log, callback) {
|
||||||
const storageLocation = request.headers['x-scal-storage-class'];
|
const storageLocation = request.headers['x-scal-storage-class'];
|
||||||
|
@ -401,11 +457,16 @@ function putObject(request, response, log, callback) {
|
||||||
if (contentMD5 !== md5) {
|
if (contentMD5 !== md5) {
|
||||||
return callback(errors.BadDigest);
|
return callback(errors.BadDigest);
|
||||||
}
|
}
|
||||||
const dataRetrievalInfo = {
|
const responsePayload = constructPutResponse({
|
||||||
// TODO: Remove '' when versioning implemented for Azure.
|
dataStoreName: retrievalInfo.dataStoreName,
|
||||||
versionId: retrievalInfo.dataStoreVersionId || '',
|
dataStoreType: retrievalInfo.dataStoreType,
|
||||||
};
|
key: retrievalInfo.key,
|
||||||
return _respond(response, dataRetrievalInfo, log, callback);
|
size: payloadLen,
|
||||||
|
dataStoreETag: retrievalInfo.dataStoreETag ?
|
||||||
|
`1:${retrievalInfo.dataStoreETag}` : `1:${md5}`,
|
||||||
|
dataStoreVersionId: retrievalInfo.dataStoreVersionId,
|
||||||
|
});
|
||||||
|
return _respond(response, responsePayload, log, callback);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -577,8 +638,8 @@ function completeMultipartUpload(request, response, log, callback) {
|
||||||
// FIXME: add error type MalformedJSON
|
// FIXME: add error type MalformedJSON
|
||||||
return callback(errors.MalformedPOSTRequest);
|
return callback(errors.MalformedPOSTRequest);
|
||||||
}
|
}
|
||||||
const partList =
|
const partList = getPartList(
|
||||||
getPartList(parts, request.objectKey, uploadId, storageLocation);
|
parts, request.objectKey, uploadId, storageLocation);
|
||||||
// Azure client will set user metadata at this point.
|
// Azure client will set user metadata at this point.
|
||||||
const metaHeaders = { 'x-amz-meta-scal-replication-status': 'REPLICA' };
|
const metaHeaders = { 'x-amz-meta-scal-replication-status': 'REPLICA' };
|
||||||
if (sourceVersionId) {
|
if (sourceVersionId) {
|
||||||
|
@ -615,11 +676,31 @@ function completeMultipartUpload(request, response, log, callback) {
|
||||||
});
|
});
|
||||||
return callback(err);
|
return callback(err);
|
||||||
}
|
}
|
||||||
const dataRetrievalInfo = {
|
// The logic here is an aggregate of code coming from
|
||||||
// TODO: Remove '' when versioning implemented for Azure.
|
// lib/api/completeMultipartUpload.js.
|
||||||
versionId: retrievalInfo.dataStoreVersionId || '',
|
|
||||||
};
|
const { key, dataStoreType, dataStoreVersionId } =
|
||||||
return _respond(response, dataRetrievalInfo, log, callback);
|
retrievalInfo;
|
||||||
|
let size;
|
||||||
|
let dataStoreETag;
|
||||||
|
if (skipMpuPartProcessing(retrievalInfo)) {
|
||||||
|
size = retrievalInfo.contentLength;
|
||||||
|
dataStoreETag = retrievalInfo.eTag;
|
||||||
|
} else {
|
||||||
|
const { aggregateSize, aggregateETag } =
|
||||||
|
generateMpuAggregateInfo(parts);
|
||||||
|
size = aggregateSize;
|
||||||
|
dataStoreETag = aggregateETag;
|
||||||
|
}
|
||||||
|
const responsePayload = constructPutResponse({
|
||||||
|
dataStoreName: storageLocation,
|
||||||
|
dataStoreType,
|
||||||
|
key,
|
||||||
|
size,
|
||||||
|
dataStoreETag,
|
||||||
|
dataStoreVersionId,
|
||||||
|
});
|
||||||
|
return _respond(response, responsePayload, log, callback);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
return undefined;
|
return undefined;
|
||||||
|
|
|
@ -38,6 +38,7 @@ function getCapabilities() {
|
||||||
locationTypeCephRadosGW: true,
|
locationTypeCephRadosGW: true,
|
||||||
preferredReadLocation: true,
|
preferredReadLocation: true,
|
||||||
managedLifecycle: true,
|
managedLifecycle: true,
|
||||||
|
managedLifecycleTransition: true,
|
||||||
secureChannelOptimizedPath: hasWSOptionalDependencies(),
|
secureChannelOptimizedPath: hasWSOptionalDependencies(),
|
||||||
s3cIngestLocation: true,
|
s3cIngestLocation: true,
|
||||||
};
|
};
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -19,7 +19,7 @@
|
||||||
},
|
},
|
||||||
"homepage": "https://github.com/scality/S3#readme",
|
"homepage": "https://github.com/scality/S3#readme",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"arsenal": "github:scality/arsenal#f199d52",
|
"arsenal": "github:scality/arsenal#6a04e6c",
|
||||||
"async": "~2.5.0",
|
"async": "~2.5.0",
|
||||||
"aws-sdk": "2.28.0",
|
"aws-sdk": "2.28.0",
|
||||||
"azure-storage": "^2.1.0",
|
"azure-storage": "^2.1.0",
|
||||||
|
|
Loading…
Reference in New Issue