Compare commits

...

1 Commits

Author SHA1 Message Date
Alexander Chan 7c61b4dec8 ARSN-218: support lifecycle noncurrent version transition 2022-06-06 09:11:58 -07:00
5 changed files with 457 additions and 3 deletions

View File

@ -150,7 +150,7 @@ export const supportedLifecycleRules = [
'noncurrentVersionExpiration', 'noncurrentVersionExpiration',
'abortIncompleteMultipartUpload', 'abortIncompleteMultipartUpload',
'transitions', 'transitions',
'noncurrentVersionTransition', 'noncurrentVersionTransitions',
]; ];
// Maximum number of buckets to cache (bucket metadata) // Maximum number of buckets to cache (bucket metadata)
export const maxCachedBuckets = process.env.METADATA_MAX_CACHED_BUCKETS ? export const maxCachedBuckets = process.env.METADATA_MAX_CACHED_BUCKETS ?

View File

@ -31,6 +31,9 @@ class LifecycleRule {
if (this.transitions) { if (this.transitions) {
rule.Transitions = this.transitions; rule.Transitions = this.transitions;
} }
if (this.ncvTransitions) {
rule.NoncurrentVersionTransitions = this.ncvTransitions;
}
const filter = {}; const filter = {};
@ -133,6 +136,11 @@ class LifecycleRule {
this.transitions = transitions; this.transitions = transitions;
return this; return this;
} }
addNCVTransitions(nvcTransitions) {
this.ncvTransitions = nvcTransitions;
return this;
}
} }
module.exports = LifecycleRule; module.exports = LifecycleRule;

View File

@ -44,7 +44,7 @@ export default class LifecycleDateTime {
* @return - The normalized transition timestamp * @return - The normalized transition timestamp
*/ */
getTransitionTimestamp( getTransitionTimestamp(
transition: { Date?: string; Days?: number }, transition: { Date?: string; Days?: number, NoncurrentDays?: number },
lastModified: string, lastModified: string,
) { ) {
if (transition.Date !== undefined) { if (transition.Date !== undefined) {
@ -55,5 +55,10 @@ export default class LifecycleDateTime {
const timeTravel = this._transitionOneDayEarlier ? -oneDay : 0; const timeTravel = this._transitionOneDayEarlier ? -oneDay : 0;
return lastModifiedTime + (transition.Days * oneDay) + timeTravel; return lastModifiedTime + (transition.Days * oneDay) + timeTravel;
} }
if (transition.NoncurrentDays !== undefined) {
const lastModifiedTime = this.getTimestamp(lastModified);
const timeTravel = this._transitionOneDayEarlier ? -oneDay : 0;
return lastModifiedTime + (transition.NoncurrentDays * oneDay) + timeTravel;
}
} }
} }

View File

@ -98,6 +98,46 @@ export default class LifecycleUtils {
}); });
} }
/**
* Find the most relevant trantition rule for the given transitions array
* and any previously stored transition from another rule.
* @param params.noncurrentTransitions - Array of lifecycle rule noncurrent
* transitions
* @param params.lastModified - The object's last modified
* @param params.currentDate - current date
* @param params.store - object containing applicable rules
* date
* @return The most applicable transition rule
*/
getApplicableNoncurrentVersionTransition(params: {
store: any;
currentDate: Date;
noncurrentTransitions: any[];
lastModified: string;
}) {
const { noncurrentTransitions, store, lastModified, currentDate } = params;
const ncvt = noncurrentTransitions.reduce((result, ncvt) => {
const isApplicable = // Is the transition time in the past?
this._datetime.getTimestamp(currentDate) >=
this._datetime.getTransitionTimestamp(ncvt, lastModified)!;
if (!isApplicable) {
return result;
}
return this.compareTransitions({
transition1: ncvt,
transition2: result,
lastModified,
});
}, undefined);
return this.compareTransitions({
transition1: ncvt,
transition2: store.NoncurrentVersionTransition,
lastModified,
});
}
// TODO // TODO
/** /**
* Filter out all rules based on `Status` and `Filter` (Prefix and Tags) * Filter out all rules based on `Status` and `Filter` (Prefix and Tags)
@ -239,7 +279,18 @@ export default class LifecycleUtils {
currentDate, currentDate,
}); });
} }
// TODO: Add support for NoncurrentVersionTransitions.
const ncvt = 'NoncurrentVersionTransitions';
const hasNoncurrentTransitions = Array.isArray(rule[ncvt]) && rule[ncvt].length > 0;
if (hasNoncurrentTransitions && this._supportedRules.includes('noncurrentVersionTransitions')) {
store.NoncurrentVersionTransition = this.getApplicableNoncurrentVersionTransition({
noncurrentTransitions: rule.NoncurrentVersionTransitions,
lastModified: metadata.LastModified,
store,
currentDate,
});
}
return store; return store;
}, {}); }, {});
// Do not transition to a location where the object is already stored. // Do not transition to a location where the object is already stored.
@ -247,6 +298,12 @@ export default class LifecycleUtils {
&& applicableRules.Transition.StorageClass === metadata.StorageClass) { && applicableRules.Transition.StorageClass === metadata.StorageClass) {
applicableRules.Transition = undefined; applicableRules.Transition = undefined;
} }
if (applicableRules.NoncurrentVersionTransition
&& applicableRules.NoncurrentVersionTransition.StorageClass === metadata.StorageClass) {
applicableRules.NoncurrentVersionTransition = undefined;
}
return applicableRules; return applicableRules;
/* eslint-enable no-param-reassign */ /* eslint-enable no-param-reassign */
} }

View File

@ -40,6 +40,7 @@ describe('LifecycleUtils::getApplicableRules', () => {
'noncurrentVersionExpiration', 'noncurrentVersionExpiration',
'abortIncompleteMultipartUpload', 'abortIncompleteMultipartUpload',
'transitions', 'transitions',
'noncurrentVersionTransitions',
]); ]);
}); });
@ -383,6 +384,208 @@ describe('LifecycleUtils::getApplicableRules', () => {
assert.strictEqual(rules.Transition, undefined); assert.strictEqual(rules.Transition, undefined);
}); });
it('should return NoncurrentVersionTransition with Days', () => {
const applicableRules = [
new LifecycleRule()
.addNCVTransitions([
{
NoncurrentDays: 1,
StorageClass: 'zenko',
},
])
.build(),
];
const lastModified = getDate({ numberOfDaysFromNow: -2 });
const object = getMetadataObject(lastModified);
const rules = lutils.getApplicableRules(applicableRules, object);
assert.deepStrictEqual(rules, {
NoncurrentVersionTransition: {
NoncurrentDays: 1,
StorageClass: 'zenko',
},
});
});
it('should return Transition when multiple rule transitions', () => {
const applicableRules = [
new LifecycleRule()
.addNCVTransitions([
{
NoncurrentDays: 1,
StorageClass: 'zenko-1',
},
{
NoncurrentDays: 3,
StorageClass: 'zenko-3',
},
])
.build(),
];
const lastModified = getDate({ numberOfDaysFromNow: -4 });
const object = getMetadataObject(lastModified);
const rules = lutils.getApplicableRules(applicableRules, object);
assert.deepStrictEqual(rules, {
NoncurrentVersionTransition: {
NoncurrentDays: 3,
StorageClass: 'zenko-3',
},
});
});
it('should return Transition across many rules: first rule', () => {
const applicableRules = [
new LifecycleRule()
.addNCVTransitions([{
NoncurrentDays: 1,
StorageClass: 'zenko-1',
}])
.build(),
new LifecycleRule()
.addNCVTransitions([{
NoncurrentDays: 3,
StorageClass: 'zenko-3',
}])
.build(),
];
const lastModified = getDate({ numberOfDaysFromNow: -2 });
const object = getMetadataObject(lastModified);
const rules = lutils.getApplicableRules(applicableRules, object);
assert.deepStrictEqual(rules, {
NoncurrentVersionTransition: {
NoncurrentDays: 1,
StorageClass: 'zenko-1',
},
});
});
it('should return Transition across many rules: second rule', () => {
const applicableRules = [
new LifecycleRule()
.addNCVTransitions([{
NoncurrentDays: 1,
StorageClass: 'zenko-1',
}])
.build(),
new LifecycleRule()
.addNCVTransitions([{
NoncurrentDays: 3,
StorageClass: 'zenko-3',
}])
.build(),
];
const lastModified = getDate({ numberOfDaysFromNow: -4 });
const object = getMetadataObject(lastModified);
const rules = lutils.getApplicableRules(applicableRules, object);
assert.deepStrictEqual(rules, {
NoncurrentVersionTransition: {
NoncurrentDays: 3,
StorageClass: 'zenko-3',
},
});
});
it('should return Transition across many rules: first rule with ' +
'multiple transitions', () => {
const applicableRules = [
new LifecycleRule()
.addNCVTransitions([{
NoncurrentDays: 1,
StorageClass: 'zenko-1',
}, {
NoncurrentDays: 3,
StorageClass: 'zenko-3',
}])
.build(),
new LifecycleRule()
.addNCVTransitions([{
NoncurrentDays: 4,
StorageClass: 'zenko-4',
}])
.build(),
];
const lastModified = getDate({ numberOfDaysFromNow: -2 });
const object = getMetadataObject(lastModified);
const rules = lutils.getApplicableRules(applicableRules, object);
assert.deepStrictEqual(rules, {
NoncurrentVersionTransition: {
NoncurrentDays: 1,
StorageClass: 'zenko-1',
},
});
});
it('should return Transition across many rules: second rule with ' +
'multiple transitions', () => {
const applicableRules = [
new LifecycleRule()
.addNCVTransitions([{
NoncurrentDays: 1,
StorageClass: 'zenko-1',
}, {
NoncurrentDays: 3,
StorageClass: 'zenko-3',
}])
.build(),
new LifecycleRule()
.addNCVTransitions([{
NoncurrentDays: 4,
StorageClass: 'zenko-4',
}, {
NoncurrentDays: 6,
StorageClass: 'zenko-6',
}])
.build(),
];
const lastModified = getDate({ numberOfDaysFromNow: -5 });
const object = getMetadataObject(lastModified);
const rules = lutils.getApplicableRules(applicableRules, object);
assert.deepStrictEqual(rules, {
NoncurrentVersionTransition: {
NoncurrentDays: 4,
StorageClass: 'zenko-4',
},
});
});
it('should not return transition when Transitions has no applicable ' +
'rule: Days', () => {
const applicableRules = [
new LifecycleRule()
.addNCVTransitions([
{
NoncurrentDays: 3,
StorageClass: 'zenko',
},
])
.build(),
];
const lastModified = getDate({ numberOfDaysFromNow: -2 });
const object = getMetadataObject(lastModified);
const rules = lutils.getApplicableRules(applicableRules, object);
assert.strictEqual(rules.Transition, undefined);
});
it('should not return transition when Transitions is an empty ' +
'array', () => {
const applicableRules = [
new LifecycleRule()
.addNCVTransitions([])
.build(),
];
const rules = lutils.getApplicableRules(applicableRules, {});
assert.strictEqual(rules.Transition, undefined);
});
it('should not return noncurrentTransition when undefined', () => {
const applicableRules = [
new LifecycleRule()
.addExpiration('Days', 1)
.build(),
];
const rules = lutils.getApplicableRules(applicableRules, {});
assert.strictEqual(rules.Transition, undefined);
});
describe('transitioning to the same storage class', () => { describe('transitioning to the same storage class', () => {
it('should not return transition when applicable transition is ' + it('should not return transition when applicable transition is ' +
'already stored at the destination', () => { 'already stored at the destination', () => {
@ -427,6 +630,50 @@ describe('LifecycleUtils::getApplicableRules', () => {
const rules = lutils.getApplicableRules(applicableRules, object); const rules = lutils.getApplicableRules(applicableRules, object);
assert.strictEqual(rules.Transition, undefined); assert.strictEqual(rules.Transition, undefined);
}); });
it('should not return transition when applicable transition is ' +
'already stored at the destination', () => {
const applicableRules = [
new LifecycleRule()
.addNCVTransitions([
{
NoncurrentDays: 1,
StorageClass: 'zenko',
},
])
.build(),
];
const lastModified = getDate({ numberOfDaysFromNow: -2 });
const object = getMetadataObject(lastModified, 'zenko');
const rules = lutils.getApplicableRules(applicableRules, object);
assert.strictEqual(rules.Transition, undefined);
});
it('should not return transition when applicable transition is ' +
'already stored at the destination: multiple rules', () => {
const applicableRules = [
new LifecycleRule()
.addNCVTransitions([
{
Days: 2,
StorageClass: 'zenko',
},
])
.build(),
new LifecycleRule()
.addNCVTransitions([
{
Days: 1,
StorageClass: 'STANDARD',
},
])
.build(),
];
const lastModified = getDate({ numberOfDaysFromNow: -3 });
const object = getMetadataObject(lastModified, 'zenko');
const rules = lutils.getApplicableRules(applicableRules, object);
assert.strictEqual(rules.Transition, undefined);
});
}); });
}); });
@ -852,5 +1099,142 @@ describe('LifecycleUtils::compareTransitions', () => {
}); });
assert.deepStrictEqual(result, transition2); assert.deepStrictEqual(result, transition2);
}); });
it('should return the first rule if older than the second rule (noncurrent)', () => {
const transition1 = {
NoncurrentDays: 2,
StorageClass: 'zenko',
};
const transition2 = {
NoncurrentDays: 1,
StorageClass: 'zenko',
};
const result = lutils.compareTransitions({
transition1,
transition2,
lastModified: '1970-01-01T00:00:00.000Z',
});
assert.deepStrictEqual(result, transition1);
}); });
it('should return the second rule if older than the first rule (noncurrent)', () => {
const transition1 = {
NoncurrentDays: 1,
StorageClass: 'zenko',
};
const transition2 = {
NoncurrentDays: 2,
StorageClass: 'zenko',
};
const result = lutils.compareTransitions({
transition1,
transition2,
lastModified: '1970-01-01T00:00:00.000Z',
});
assert.deepStrictEqual(result, transition2);
});
});
describe('LifecycleUtils::getApplicableNoncurrentVersionTransition', () => {
let lutils;
beforeAll(() => {
lutils = new LifecycleUtils();
});
describe('using NoncurrentDays time type', () => {
it('should return undefined if no rules given', () => {
const result = lutils.getApplicableNoncurrentVersionTransition({
noncurrentTransitions: [],
currentDate: '1970-01-03T00:00:00.000Z',
lastModified: '1970-01-01T00:00:00.000Z',
store: {},
});
assert.deepStrictEqual(result, undefined);
});
it('should return undefined when no rule applies', () => {
const result = lutils.getApplicableNoncurrentVersionTransition({
noncurrentTransitions: [
{
NoncurrentDays: 1,
StorageClass: 'zenko',
},
],
currentDate: '1970-01-01T23:59:59.999Z',
lastModified: '1970-01-01T00:00:00.000Z',
store: {},
});
assert.deepStrictEqual(result, undefined);
});
it('should return a single rule if it applies', () => {
const result = lutils.getApplicableNoncurrentVersionTransition({
noncurrentTransitions: [
{
NoncurrentDays: 1,
StorageClass: 'zenko',
},
],
currentDate: '1970-01-02T00:00:00.000Z',
lastModified: '1970-01-01T00:00:00.000Z',
store: {},
});
const expected = {
NoncurrentDays: 1,
StorageClass: 'zenko',
};
assert.deepStrictEqual(result, expected);
});
it('should return the most applicable rule: last rule', () => {
const result = lutils.getApplicableNoncurrentVersionTransition({
noncurrentTransitions: [
{
NoncurrentDays: 1,
StorageClass: 'zenko',
},
{
NoncurrentDays: 10,
StorageClass: 'zenko',
},
],
currentDate: '1970-01-11T00:00:00.000Z',
lastModified: '1970-01-01T00:00:00.000Z',
store: {},
});
const expected = {
NoncurrentDays: 10,
StorageClass: 'zenko',
};
assert.deepStrictEqual(result, expected);
});
it('should return the most applicable rule: middle rule', () => {
const result = lutils.getApplicableNoncurrentVersionTransition({
noncurrentTransitions: [
{
NoncurrentDays: 1,
StorageClass: 'zenko',
},
{
NoncurrentDays: 4,
StorageClass: 'zenko',
},
{
NoncurrentDays: 10,
StorageClass: 'zenko',
},
],
currentDate: '1970-01-05T00:00:00.000Z',
lastModified: '1970-01-01T00:00:00.000Z',
store: {},
});
const expected = {
NoncurrentDays: 4,
StorageClass: 'zenko',
};
assert.deepStrictEqual(result, expected);
});
});
});