Compare commits
1 Commits
developmen
...
feature/ma
Author | SHA1 | Date |
---|---|---|
Taylor McKinnon | 4187bb0f12 |
|
@ -20,6 +20,11 @@ class ReindexTask extends BaseTask {
|
||||||
this._defaultLag = 0;
|
this._defaultLag = 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async _setup(includeDefaultOpts = true) {
|
||||||
|
await super._setup(includeDefaultOpts);
|
||||||
|
this._program.option('--bucket <bucket>', 'Specify a single bucket to reindex.');
|
||||||
|
}
|
||||||
|
|
||||||
static async _indexBucket(bucket) {
|
static async _indexBucket(bucket) {
|
||||||
let size = 0;
|
let size = 0;
|
||||||
let count = 0;
|
let count = 0;
|
||||||
|
@ -119,13 +124,20 @@ class ReindexTask extends BaseTask {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
get targetBuckets() {
|
||||||
|
if (this._program.bucket) {
|
||||||
|
return [{ name: this._program.bucket }];
|
||||||
|
}
|
||||||
|
|
||||||
|
return metadata.listBuckets();
|
||||||
|
}
|
||||||
|
|
||||||
async _execute() {
|
async _execute() {
|
||||||
logger.info('started reindex task');
|
logger.info('started reindex task');
|
||||||
|
|
||||||
const accountTotals = {};
|
const accountTotals = {};
|
||||||
const ignoredAccounts = new Set();
|
const ignoredAccounts = new Set();
|
||||||
|
await async.eachLimit(this.targetBuckets, 5, async bucket => {
|
||||||
await async.eachLimit(metadata.listBuckets(), 5, async bucket => {
|
|
||||||
logger.info('started bucket reindex', { bucket: bucket.name });
|
logger.info('started bucket reindex', { bucket: bucket.name });
|
||||||
|
|
||||||
const mpuBucket = `${mpuBucketPrefix}${bucket.name}`;
|
const mpuBucket = `${mpuBucketPrefix}${bucket.name}`;
|
||||||
|
@ -136,8 +148,14 @@ class ReindexTask extends BaseTask {
|
||||||
bktTotal = await async.retryable(ReindexTask._indexBucket)(bucket.name);
|
bktTotal = await async.retryable(ReindexTask._indexBucket)(bucket.name);
|
||||||
mpuTotal = await async.retryable(ReindexTask._indexMpuBucket)(mpuBucket);
|
mpuTotal = await async.retryable(ReindexTask._indexMpuBucket)(mpuBucket);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('failed bucket reindex. associated account skipped', { error, bucket: bucket.name });
|
logger.error(
|
||||||
|
'failed bucket reindex. any associated account will be skipped',
|
||||||
|
{ error, bucket: bucket.name }
|
||||||
|
);
|
||||||
|
// buckets passed with `--bucket` won't have an account property
|
||||||
|
if (bucket.account) {
|
||||||
ignoredAccounts.add(bucket.account);
|
ignoredAccounts.add(bucket.account);
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -146,12 +164,15 @@ class ReindexTask extends BaseTask {
|
||||||
count: bktTotal.count,
|
count: bktTotal.count,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// buckets passed with `--bucket` won't have an account property
|
||||||
|
if (bucket.account) {
|
||||||
if (accountTotals[bucket.account]) {
|
if (accountTotals[bucket.account]) {
|
||||||
accountTotals[bucket.account].size += total.size;
|
accountTotals[bucket.account].size += total.size;
|
||||||
accountTotals[bucket.account].count += total.count;
|
accountTotals[bucket.account].count += total.count;
|
||||||
} else {
|
} else {
|
||||||
accountTotals[bucket.account] = { ...total };
|
accountTotals[bucket.account] = { ...total };
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
logger.info('finished bucket reindex', { bucket: bucket.name });
|
logger.info('finished bucket reindex', { bucket: bucket.name });
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue