Merge pull request #3003 from yutaka-oishi/ft/restore-object

FT: Provide RestoreObject API to support cold storages

lib/api/api.js

@@ -52,6 +52,7 @@ const objectPutTagging = require('./objectPutTagging');
 const objectPutPart = require('./objectPutPart');
 const objectPutCopyPart = require('./objectPutCopyPart');
 const objectPutRetention = require('./objectPutRetention');
+const objectRestore = require('./objectRestore');
 const prepareRequestContexts
     = require('./apiUtils/authorization/prepareRequestContexts');
 const serviceGet = require('./serviceGet');
@@ -267,6 +268,7 @@ const api = {
     serviceGet,
     websiteGet,
     websiteHead,
+    objectRestore,
 };
 
 module.exports = api;

lib/api/apiUtils/object/coldStorage.js

@@ -0,0 +1,185 @@
+const coldstorage = require('../../../coldstorage/wrapper');
+const { metadataGetObject } = require('../../../metadata/metadataUtils');
+const errors = require('arsenal').errors;
+
+/**
+ * Get response header "x-amz-restore"
+ * Be called by objectHead.js
+ * @param {object} objMD - object's metadata
+ * @returns {string} x-amz-restore
+ */
+function getAmzRestoreResHeader(objMD){
+
+    let value;
+    if(objMD['x-amz-restore']){
+        if(objMD['x-amz-restore']['ongoing-request']){
+            value = `ongoing-request="${objMD['x-amz-restore']['ongoing-request']}"`;
+        }
+        
+        // expiry-date is transformed to format of RFC2822
+        if (objMD['x-amz-restore']['expiry-date']) {
+            const utcDateTime = alDateUtils.toUTCString(new Date(objMD['x-amz-restore']['expiry-date']));
+            value = `${value}, ${expiry-date}="${utcDateTime}"`;
+        }
+    }
+
+    return value;
+
+}
+
+/**
+ * Check object metadata if GET request is possible
+ * Be called by objectGet.js
+ * @param {object} objMD - object's metadata
+ * @return {boolean} true if the GET request is accepted, false if not
+ */
+function validateAmzRestoreForGet(objMD){
+
+    if(!objMD) {
+        return false;
+    }
+
+    if(objMD['x-amz-storage-class'] === 'GLACIER'){
+        return false;
+    }
+
+    if(objMD['x-amz-restore']['ongoing-request']){
+        return false;
+    }
+
+    return true;
+
+}
+
+/**
+ * Start to archive to GLACIER
+ * ( Be called by Lifecycle batch? ) 
+ */
+function startGlacier(bucketName, objName, versionId, log, cb){
+    
+    return completeGlacier(bucketName, objName, versionId, log, cb);
+}
+
+/**
+ * Complete to archive to GLACIER
+ * ( Be called by Lifecycle batch? ) 
+ * update x-amz-storage-class to "GLACIER".
+ */
+function completeGlacier(bucketName, objName, versionId, log, cb){
+
+    metadataGetObject(bucketName, objectKey, versionId, log,
+        (err, objMD) => {
+            if(err){
+                log.trace('error processing get metadata', {
+                    error: err,
+                    method: 'metadataGetObject',
+                });
+                return cb(err);
+            }
+
+            const storageClass = 'GLACIER';
+
+            // FIXME: return error NotImplemented when using "ColdStorageFileInterface"
+            coldstorage.updateAmzStorageClass(bucketName, objName, objMD, storageClass, log, cb);
+        }
+    );
+
+
+
+}
+
+/**
+ * start to restore object.
+ * If not exist x-amz-restore, add it to objectMD.(x-amz-restore = false)
+ * calculate restore expiry-date and add it to objectMD.
+ * Be called by objectRestore.js
+ * 
+ * FIXME: After restore is started, there is no timing to update restore parameter to the content of complete restore.
+ */
+function startRestore(bucketName, objName, objectMD, restoreParam, cb){
+
+    let checkResult = _validateStartRestore(objectMD);
+    if(checkResult instanceof errors){
+        return cb(checkResult);
+    };
+
+    // update restore parameter to the content of doing restore.
+    _updateRestoreExpiration(bucketName, objName, objMD, restoreParam, log, cb);
+
+
+    return cb(objectMD, restoreParam);
+    
+}
+
+/**
+ * complete to restore object.
+ * Update restore-ongoing to false.
+ * ( Be called by batch to check if the restore is complete? )
+ * 
+ */
+function completeRestore(bucketName, objName, objMD){
+
+    const updateParam = false;
+
+    // FIXME: return error NotImplemented when using "ColdStorageFileInterface"
+    return coldstorage.updateRestoreOngoing(bucketName, objName, objMD, updateParam, log, cb);
+}
+
+
+/**
+ * expire to restore object.
+ * Delete x-amz-restore.
+ * ( Be called by batch to check if the restore is expire? )
+ */
+function expireRestore(bucketName, objName, objMD){
+
+    // FIXME: return error NotImplemented when using "ColdStorageFileInterface"
+    return coldstorage.deleteAmzRestore(bucketName, objName, objMD, log, cb);
+}
+
+
+
+/**
+ * Check if restore has already started.
+ */
+function _validateStartRestore(objectMD){
+    
+    if(objectMD['x-amz-restore'] && objMD['x-amz-restore']['ongoing-request']){
+        return errors.RestoreAlreadyInProgress;
+    }
+    else{
+        return undefined;
+    }
+
+}
+
+/**
+ * update restore expiration date.
+ */
+function _updateRestoreExpiration(bucketName, objName, objMD, restoreParam, log, cb){
+
+    if(objMD['x-amz-restore'] && !objMD['x-amz-restore']['ongoing-request']){
+
+        // FIXME: return error NotImplemented when using "ColdStorageFileInterface"
+        return coldstorage.updateRestoreExpiration(bucketName, objName, objMD, restoreParam, log, cb);
+    }
+    else{
+        log.debug('do not updateRestoreExpiration', { method: '_updateRestoreExpiration' });
+        return undefined;
+    }
+
+}
+
+
+
+
+
+module.exports = {
+    getAmzRestoreResHeader,
+    validateAmzRestoreForGet,
+    startGlacier,
+    completeGlacier,
+    startRestore,
+    completeRestore,
+    expireRestore,
+};

lib/api/apiUtils/object/objectRestore.js

@@ -0,0 +1,316 @@
+
+const async = require('async');
+
+const { errors } = require('arsenal');
+
+const ObjectMD = require('arsenal').models.ObjectMD;
+const coldStorage = require('./coldStorage');
+
+
+const METHOD = 'objectRestore';
+
+
+/**
+ * POST Object restore process
+ *
+ * @param {MetadataWrapper} metadata metadata wrapper
+ * @param {object} mdUtils utility object to treat metadata
+ * @param {object} func object with a reference to each function of cloudserver
+ * @param {function(object):string|Error} func.decodeVersionId 
+ * @param {function(object, string, BucketInfo):object} func.collectCorsHeaders 
+ * @param {function(object, object):string} func.getVersionIdResHeader
+ * @param {AuthInfo} userInfo Instance of AuthInfo class with requester's info
+ * @param {IncomingMessage} request request info
+ * @param {werelogs.Logger} log Werelogs instance
+ * @param {module:api/objectRestore~NoBodyResultCallback} callback callback function
+ * @return {void}
+ */
+function objectRestore(metadata, mdUtils, func, userInfo, request, log, callback) {
+
+    const { bucketName, objectKey } = request;
+    const requestedAt = request['x-sdt-requested-at'];
+
+    log.debug('processing request', { method: METHOD });
+
+    const decodedVidResult = func.decodeVersionId(request.query);
+    if (decodedVidResult instanceof Error) {
+        log.trace('invalid versionId query',
+            { method: METHOD, versionId: request.query.versionId, error: decodedVidResult });
+        return callback(decodedVidResult, decodedVidResult.code);
+    }
+
+    const reqVersionId = decodedVidResult;
+
+    const mdValueParams = {
+        authInfo: userInfo,
+        bucketName,
+        objectKey,
+        versionId: reqVersionId,
+        requestType: 'bucketOwnerAction',
+    };
+
+    return async.waterfall([
+
+        // get metadata of bucket and object
+        function validateBucketAndObject(next) {
+
+            return mdUtils.metadataValidateBucketAndObj(mdValueParams, log, (err, bucketMD, objectMD) => {
+
+                if (err) {
+                    log.trace('request authorization failed', { method: METHOD, error: err });
+                    return next(err);
+                }
+
+                // Call back error if object metadata could not be obtained
+                if (!objectMD) {
+                    const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey;
+                    log.trace('error no object metadata found', { method: METHOD, error: err });
+                    return next(err, bucketMD);
+                }
+
+                const instance = new ObjectMD(objectMD);
+
+                // If object metadata is delete marker,
+                // call back NoSuchKey or MethodNotAllowed depending on specifying versionId
+                if (objectMD.isDeleteMarker) {
+                    let err = errors.NoSuchKey;
+                    if (reqVersionId) {
+                        err = errors.MethodNotAllowed;
+                    }
+                    log.trace('version is a delete marker', { method: METHOD, error: err });
+                    return next(err, bucketMD, instance);
+                }
+
+                log.info('it acquired the object metadata.', {
+                    'method': METHOD,
+                    'x-coldstorage-uuid': instance.getColdstorageUuid(),
+                    'x-coldstorage-zenko-id': instance.getColdstorageZenkoId(),
+                });
+
+                return next(null, bucketMD, instance);
+            });
+        },
+
+        // generate restore param obj from xml of request body
+        function parseRequestXml(bucketMD, objectMD, next) {
+
+            return parsePostObjectRestoreXml(request.post, log, (err, params) => {
+
+                if (err) {
+                    return next(err, bucketMD, objectMD);
+                }
+
+                log.info('it parsed xml of the request body.', { method: METHOD, value: params });
+
+                return next(null, bucketMD, objectMD, params);
+            });
+        },
+
+        // start restore process
+        function startRestore(bucketMD, objectMD, next) {
+            return coldStorage.startRestore(bucketName, objectKey, objectMD, params, 
+                (err, result) => next(err, bucketMD, objectMD, result));
+        },
+    ],
+    (err, bucketMD, objectMD, result) => {
+
+        // generate CORS response header
+        const responseHeaders = func.collectCorsHeaders(request.headers.origin, request.method, bucketMD);
+
+        if (err) {
+            log.trace('error processing request', { method: METHOD, error: err });
+
+            // If object metadata is delete marker and error is MethodNotAllowed,
+            // set response header of x-amz-delete-marker and x-amz-version-id (S3 API compliant)
+            if (objectMD && objectMD.getIsDeleteMarker() && err.MethodNotAllowed) {
+                const vConfig = bucketMD.getVersioningConfiguration();
+                responseHeaders['x-amz-delete-marker'] = true;
+                responseHeaders['x-amz-version-id'] = func.getVersionIdResHeader(vConfig, objectMD.getValue());
+            }
+
+            return callback(err, err.code, responseHeaders);
+        }
+
+        // If versioning configuration is setting, set response header of x-amz-version-id
+        const vConfig = bucketMD.getVersioningConfiguration();
+        responseHeaders['x-amz-version-id'] = func.getVersionIdResHeader(vConfig, objectMD.getValue());
+
+        return callback(null, result.statusCode, responseHeaders);
+    });
+
+    /**
+     * Generate request parameter object by parsing XML ofrequest body
+     *
+     * @param {convertableToString} xml XML of request body
+     * @param {werelogs.Logger} log logger
+     * @param {module:api/utils~ObjectResultCallback} callback callback function
+     * @returns {void}
+     */
+    function parsePostObjectRestoreXml(xml, log, callback) {
+
+        log.debug('parsing xml string of request body.', alCreateLogParams(
+            this, this.parsePostObjectRestoreXml, {
+                xmlString: xml,
+            // eslint-disable-next-line comma-dangle
+            }
+        ));
+
+        return xml2js.parseString(xml, { explicitArray: false }, (err, result) => {
+
+            // If cause an error, callback MalformedXML
+            if (err) {
+                log.info('parse xml string of request body was failed.', { error: err });
+                return callback(errors.MalformedXML);
+            }
+
+            // If restore parameter is invalid, callback MalformedXML
+            const validateResult = validateRestoreRequestParameters(result);
+            if (validateResult) {
+                log.info('invalid restore parameters.', { error: validateResult.message });
+                return callback(errors.MalformedXML);
+            }
+
+            // normalize restore request parameters
+            const normalizedResult = normalizeRestoreRequestParameters(result);
+
+            log.debug('parse xml string of request body.', alCreateLogParams(
+                this, this.parsePostObjectRestoreXml, {
+                    resultObject: normalizedResult,
+                // eslint-disable-next-line comma-dangle
+                }
+            ));
+
+            return callback(null, normalizedResult);
+        });
+    };
+
+
+    /**
+     * validate restore parameter object
+     *
+     * @private
+     * @param {object} params restore parameter object
+     * @returns {Error} Error instance
+     */
+    function validateRestoreRequestParameters(params) {
+
+        if (!params) {
+            return new Error('request body is required.');
+        }
+
+        const rootElem = getSafeValue(params, 'RestoreRequest');
+        if (!rootElem) {
+            return new Error('RestoreRequest element is required.');
+        }
+
+        if (!rootElem['Days']) {
+            return new Error('RestoreRequest.Days element is required.');
+        }
+
+        // RestoreRequest.Days must be greater than or equal to 1
+        const daysValue = Number.parseInt(rootElem['Days'], 10);
+        if (Number.isNaN(daysValue)) {
+            return new Error(`RestoreRequest.Days is invalid type. [${rootElem['Days']}]`);
+        }
+        if (daysValue < 1) {
+            return new Error(`RestoreRequest.Days must be greater than 0. [${rootElem['Days']}]`);
+        }
+
+        if (daysValue > 2147483647) {
+            return new Error(`RestoreRequest.Days must be less than 2147483648. [${rootElem['Days']}]`);
+        }
+
+        // If RestoreRequest.GlacierJobParameters.Tier is specified,
+        // Must be "Expedited" or "Standard" or "Bulk"
+        const tierValue = getSafeValue(rootElem,
+            'GlacierJobParameters', 'Tier');
+        const tierList = {
+            EXPEDITED: 'Expedited',
+            STANDARD: 'Standard',
+            BULK: 'Bulk',
+        }
+        const tierConstants = getValues(tierList);
+        if (tierValue && !tierConstants.includes(tierValue)) {
+            return new Error(`RestoreRequest.GlacierJobParameters.Tier is invalid value. [${tierValue}]`);
+        }
+
+        return undefined;
+    }
+
+    /**
+     * Normalize restore request parameters.
+     *
+     * @private
+     * @param {object} params restore request parameters object
+     * @return {object} restore request parameters object(normalized)
+     */
+    function normalizeRestoreRequestParameters(params) {
+
+        const normalizedParams = {};
+
+        // set RestoreRequest.Days
+        const rootElem = getSafeValue(params, 'RestoreRequest');
+        const daysValue = Number.parseInt(rootElem['Days'], 10);
+        setSafeValue(normalizedParams, daysValue, 'Days');
+
+        // set RestoreRequest.GlacierJobParameters.Tier
+        // If do not specify, set "Standard"
+        const tierValue = getSafeValue(rootElem,
+            'GlacierJobParameters', 'Tier')
+            || 'Standard';
+        setSafeValue(normalizedParams, tierValue,
+            'GlacierJobParameters', 'Tier');
+
+        return normalizedParams;
+    }
+
+    /**
+     * Attribute values ​​that the object has are returned as an array.
+     * Node v6 does not support Object.values, so prepare a function with the same result.
+     *
+     * @param {object} obj object
+     * @returns {Array<object>} UTC date infomation(string)
+     */
+    function getValues(obj) {
+
+        const results = [];
+
+        Object.keys(obj).forEach(key => {
+            results.push(obj[key]);
+        });
+
+        return results;
+    }
+
+
+
+    /**
+     * For layered objects, safely get the value corresponding to the key passed in the variable length argument.
+     *
+     * @param {object} obj object
+     * @param  {...string} args array of keys
+     * @returns {object} 
+     */
+    function getSafeValue(obj, ...args) {
+
+        let result = obj;
+
+        if (!result || !Array.isArray(args) || args.length === 0) {
+            return result;
+        }
+
+        args.some(value => {
+            result = result[value];
+            return !result;
+        });
+
+        return result;
+    }
+
+
+}
+
+
+module.exports = {
+    objectRestore,
+};

lib/api/objectGet.js

@@ -19,6 +19,8 @@ const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
 const { config } = require('../Config');
 const monitoring = require('../utilities/monitoringHandler');
 
+const { getAmzRestoreResHeader, validateAmzRestoreForGet }  = require('./apiUtils/object/coldStorage');
+
 const validateHeaders = s3middleware.validateConditionalHeaders;
 
 /**
@@ -106,9 +108,18 @@ function objectGet(authInfo, request, returnTagCount, log, callback) {
         if (headerValResult.error) {
             return callback(headerValResult.error, null, corsHeaders);
         }
+
         const responseMetaHeaders = collectResponseHeaders(objMD,
             corsHeaders, verCfg, returnTagCount);
 
+        if (!validateAmzRestoreForGet(objMD)){
+            monitoring.promMetrics(
+                'GET', bucketName, 403, 'getObject');
+            return callback(errors.InvalidObjectState, null, responseMetaHeaders);
+
+        }
+
+        responseMetaHeaders['x-amz-restore'] = getAmzRestoreResHeader(objMD);
         const objLength = (objMD.location === null ?
                            0 : parseInt(objMD['content-length'], 10));
         let byteRange;

lib/api/objectHead.js

@@ -12,6 +12,8 @@ const { getPartNumber, getPartSize } = require('./apiUtils/object/partInfo');
 const { metadataValidateBucketAndObj } = require('../metadata/metadataUtils');
 const { maximumAllowedPartCount } = require('../../constants');
 
+const { getAmzRestoreResHeader }  = require('./apiUtils/object/coldStorage');
+
 /**
  * HEAD Object - Same as Get Object but only respond with headers
  *(no actual body)
@@ -130,6 +132,11 @@ function objectHead(authInfo, request, log, callback) {
                 }
                 responseHeaders['content-length'] = partSize;
             }
+            const responseHeaders =
+                collectResponseHeaders(objMD, corsHeaders, verCfg);
+            
+            responseHeaders['x-amz-restore'] = getAmzRestoreResHeader(objMD);
+            
             pushMetric('headObject', log, {
                 authInfo,
                 bucket: bucketName,

lib/api/objectRestore.js

@@ -0,0 +1,43 @@
+/**
+ * This module handles POST Object restore.
+ *
+ * @module api/objectRestore
+ */
+
+const collectCorsHeaders = require('../utilities/collectCorsHeaders');
+const metadata = require('../metadata/wrapper');
+const metadataUtils = require('../metadata/metadataUtils');
+
+const { decodeVersionId, getVersionIdResHeader } =
+    require('./apiUtils/object/versioning');
+
+const sdtObjectRestore = require('./apiUtils/object/objectRestore');
+
+/**
+ * Process POST Object restore request.
+ *
+ * @param {AuthInfo} userInfo Instance of AuthInfo class with requester's info
+ * @param {IncomingMessage} request normalized request object
+ * @param {werelogs.Logger} log werelogs request instance
+ * @param {module:api/objectRestore~NoBodyResultCallback} callback
+ * callback to function in route
+ * @return {void}
+ */
+function objectRestore(userInfo, request, log, callback) {
+    const func = {
+        decodeVersionId,
+        collectCorsHeaders,
+        getVersionIdResHeader,
+    };
+
+    return sdtObjectRestore(metadata, metadataUtils, func, userInfo, request,
+        log, callback);
+}
+
+/**
+ * @callback module:api/objectRestore~NoBodyResultCallback
+ * @param {ArsenalError} error ArsenalError instance in case of error
+ * @param {object} responseHeaders Response header object
+ */
+
+module.exports = objectRestore;

lib/coldstorage/wrapper.js

@@ -0,0 +1,12 @@
+const ColdStorageWrapper =
+    require('arsenal').storage.coldstorage.ColdStorageWrapper;
+const logger = require('../utilities/logger');
+
+const clientName = 'file';
+let params;
+if (clientName === 'file') {
+    params = {};
+}
+
+const coldstorage = new ColdStorageWrapper(clientName, params, logger);
+module.exports = coldstorage;