Compare commits

...

2 Commits

Author SHA1 Message Date
Taylor McKinnon f464930b21 stash 2020-12-15 15:08:11 -08:00
Taylor McKinnon 5e1c753986 stash 2020-12-14 14:52:52 -08:00
9 changed files with 156 additions and 11 deletions

14
bin/diskUsage.js Normal file
View File

@ -0,0 +1,14 @@
const { tasks } = require('..');
const { LoggerContext } = require('../libV2/utils');
const logger = new LoggerContext({
task: 'MonitorDiskUsage',
});
const task = new tasks.MonitorDiskUsage();
task.setup()
.then(() => logger.info('Starting disk usage monitor'))
.then(() => task.start())
.then(() => logger.info('Disk usage monitor started'));

View File

@ -5,6 +5,8 @@ const assert = require('assert');
const { truthy, envNamespace } = require('../constants'); const { truthy, envNamespace } = require('../constants');
const configSchema = require('./schema'); const configSchema = require('./schema');
// We need to require the specific file rather than the parent module to avoid a circular require
const { parseDiskSizeSpec } = require('../utils/disk');
function _splitServer(text) { function _splitServer(text) {
assert.notStrictEqual(text.indexOf(':'), -1); assert.notStrictEqual(text.indexOf(':'), -1);
@ -20,9 +22,9 @@ const _typeCasts = {
int: val => parseInt(val, 10), int: val => parseInt(val, 10),
list: val => val.split(',').map(v => v.trim()), list: val => val.split(',').map(v => v.trim()),
serverList: val => val.split(',').map(v => v.trim()).map(_splitServer), serverList: val => val.split(',').map(v => v.trim()).map(_splitServer),
diskSize: parseDiskSizeSpec,
}; };
function _definedInEnv(key) { function _definedInEnv(key) {
return process.env[`${envNamespace}_${key}`] !== undefined; return process.env[`${envNamespace}_${key}`] !== undefined;
} }
@ -280,6 +282,7 @@ class Config {
parsedConfig.snapshotSchedule = _loadFromEnv('SNAPSHOT_SCHEDULE', config.snapshotSchedule); parsedConfig.snapshotSchedule = _loadFromEnv('SNAPSHOT_SCHEDULE', config.snapshotSchedule);
parsedConfig.repairSchedule = _loadFromEnv('REPAIR_SCHEDULE', config.repairSchedule); parsedConfig.repairSchedule = _loadFromEnv('REPAIR_SCHEDULE', config.repairSchedule);
parsedConfig.reindexSchedule = _loadFromEnv('REINDEX_SCHEDULE', config.reindexSchedule); parsedConfig.reindexSchedule = _loadFromEnv('REINDEX_SCHEDULE', config.reindexSchedule);
parsedConfig.diskUsageSchedule = _loadFromEnv('DISK_USAGE_SCHEDULE', config.diskUsageSchedule);
parsedConfig.ingestionLagSeconds = _loadFromEnv( parsedConfig.ingestionLagSeconds = _loadFromEnv(
'INGESTION_LAG_SECONDS', 'INGESTION_LAG_SECONDS',
@ -292,6 +295,21 @@ class Config {
_typeCasts.int, _typeCasts.int,
); );
const diskUsage = {
path: _loadFromEnv('DISK_USAGE_PATH', (config.diskUsage || {}).path),
softLimit: _loadFromEnv('DISK_USAGE_SOFT_LIMIT', (config.diskUsage || {}).softLimit, _typeCasts.diskSize),
hardLimit: _loadFromEnv('DISK_USAGE_HARD_LIMIT', (config.diskUsage || {}).hardLimit, _typeCasts.diskSize),
};
if (!diskUsage.path && (diskUsage.softLimit !== undefined || diskUsage.hardLimit !== undefined)) {
throw Error('You must specify diskUsage.path to monitor for disk usage');
} else if (diskUsage.path && (diskUsage.softLimit === undefined && diskUsage.hardLimit === undefined)) {
throw Error('One of diskUsage.softLimit or diskUsage.hardLimit must be specified');
}
diskUsage.enabled = diskUsage.path !== undefined;
parsedConfig.diskUsage = diskUsage;
parsedConfig.vaultd = { parsedConfig.vaultd = {
host: _loadFromEnv('VAULT_HOST', config.vaultd.host), host: _loadFromEnv('VAULT_HOST', config.vaultd.host),
port: _loadFromEnv('VAULT_PORT', config.vaultd.port), port: _loadFromEnv('VAULT_PORT', config.vaultd.port),

View File

@ -77,6 +77,12 @@ const schema = Joi.object({
snapshotSchedule: Joi.string(), snapshotSchedule: Joi.string(),
repairSchedule: Joi.string(), repairSchedule: Joi.string(),
reindexSchedule: Joi.string(), reindexSchedule: Joi.string(),
diskUsageSchedule: Joi.string(),
diskUsage: Joi.object({
path: Joi.string(),
softLimit: Joi.string(),
hardLimit: Joi.string(),
}),
}); });
module.exports = schema; module.exports = schema;

32
libV2/tasks/DiskUsage.js Normal file
View File

@ -0,0 +1,32 @@
const BaseTask = require('./BaseTask');
const config = require('../config');
const { LoggerContext, getFolderSize, formatDiskSize } = require('../utils');
const moduleLogger = new LoggerContext({
module: 'MonitorDiskUsage',
});
class MonitorDiskUsage extends BaseTask {
constructor(options) {
super(
options,
);
this._defaultSchedule = config.diskUsageSchedule;
this._defaultLag = 0;
}
// eslint-disable-next-line class-methods-use-this
async _execute() {
if (!config.diskUsage.enabled) {
moduleLogger.trace('disk usage monitoring not enabled, skipping check');
return;
}
const logger = moduleLogger.with({ path: config.diskUsage.path });
logger.trace('checking disk usage');
const size = await getFolderSize(config.diskUsage.path);
logger.trace(`using ${formatDiskSize(size)}`);
}
}
module.exports = MonitorDiskUsage;

View File

@ -5,6 +5,7 @@ const CreateSnapshot = require('./CreateSnapshot');
const RepairTask = require('./Repair'); const RepairTask = require('./Repair');
const ReindexTask = require('./Reindex'); const ReindexTask = require('./Reindex');
const MigrateTask = require('./Migrate'); const MigrateTask = require('./Migrate');
const MonitorDiskUsage = require('./DiskUsage');
module.exports = { module.exports = {
IngestShard, IngestShard,
@ -14,4 +15,5 @@ module.exports = {
RepairTask, RepairTask,
ReindexTask, ReindexTask,
MigrateTask, MigrateTask,
MonitorDiskUsage,
}; };

52
libV2/utils/disk.js Normal file
View File

@ -0,0 +1,52 @@
const { promisify } = require('util');
const getFolderSize = require('get-folder-size');
const byteSize = require('byte-size');
const diskSpecRegex = /(\d+)([bkmgtxz])(i?b)?/;
const suffixToExp = {
b: 0,
k: 1,
m: 2,
g: 3,
t: 4,
x: 5,
z: 6,
};
/**
* Converts a string specifying disk size into its value in bytes
* Supported formats:
* 1b/1B - Directly specify a byte size
* 1K/1MB/1GiB - Specify a number of bytes using IEC or common suffixes
*
* Suffixes are case insensitive.
* All suffixes are considered IEC standard with 1 kibibyte being 2^10 bytes.
*
* @param {String} spec - string for conversion
* @returns {Integer} - disk size in bytes
*/
function parseDiskSizeSpec(spec) {
const normalized = spec.toLowerCase();
if (!diskSpecRegex.test(normalized)) {
throw Error('Format does not match a known suffix');
}
const match = diskSpecRegex.exec(normalized);
const size = parseInt(match[1], 10);
const exponent = suffixToExp[match[2]];
return size * (1024 ** exponent);
}
function _formatFunc() {
return `${this.value}${this.unit}`;
}
function formatDiskSize(value) {
return byteSize(value, { units: 'iec', toStringFn: _formatFunc }).toString();
}
module.exports = {
parseDiskSizeSpec,
getFolderSize: promisify(getFolderSize),
formatDiskSize,
};

View File

@ -2,10 +2,12 @@ const log = require('./log');
const shard = require('./shard'); const shard = require('./shard');
const timestamp = require('./timestamp'); const timestamp = require('./timestamp');
const func = require('./func'); const func = require('./func');
const disk = require('./disk');
module.exports = { module.exports = {
...log, ...log,
...shard, ...shard,
...timestamp, ...timestamp,
...func, ...func,
...disk,
}; };

View File

@ -24,9 +24,12 @@
"aws4": "^1.8.0", "aws4": "^1.8.0",
"body-parser": "^1.19.0", "body-parser": "^1.19.0",
"bucketclient": "scality/bucketclient", "bucketclient": "scality/bucketclient",
"byte-size": "^7.0.0",
"commander": "^5.1.0", "commander": "^5.1.0",
"cron-parser": "^2.15.0", "cron-parser": "^2.15.0",
"diskusage": "^1.1.3",
"express": "^4.17.1", "express": "^4.17.1",
"get-folder-size": "^2.0.1",
"ioredis": "^4.9.5", "ioredis": "^4.9.5",
"js-yaml": "^3.14.0", "js-yaml": "^3.14.0",
"level-mem": "^5.0.1", "level-mem": "^5.0.1",
@ -64,6 +67,7 @@
"start_v2:task:repair": "ENABLE_UTAPI_V2=1 node bin/repair.js", "start_v2:task:repair": "ENABLE_UTAPI_V2=1 node bin/repair.js",
"start_v2:task:reindex": "ENABLE_UTAPI_V2=1 node bin/reindex.js", "start_v2:task:reindex": "ENABLE_UTAPI_V2=1 node bin/reindex.js",
"start_v2:task:migrate": "ENABLE_UTAPI_V2=1 node bin/migrate.js", "start_v2:task:migrate": "ENABLE_UTAPI_V2=1 node bin/migrate.js",
"start_v2:task:disk": "ENABLE_UTAPI_V2=1 node bin/diskUsage.js",
"start_v2:server": "ENABLE_UTAPI_V2=1 node bin/server.js", "start_v2:server": "ENABLE_UTAPI_V2=1 node bin/server.js",
"start_v2:server:dev": "UTAPI_DEV_MODE=t ENABLE_UTAPI_V2=t yarn nodemon --watch './**/*.js' --watch './**/*.json' --watch './**/*.yaml' --exec node bin/server.js" "start_v2:server:dev": "UTAPI_DEV_MODE=t ENABLE_UTAPI_V2=t yarn nodemon --watch './**/*.js' --watch './**/*.json' --watch './**/*.yaml' --exec node bin/server.js"
} }

View File

@ -1081,6 +1081,11 @@ busboy@^0.2.11:
dicer "0.2.5" dicer "0.2.5"
readable-stream "1.1.x" readable-stream "1.1.x"
byte-size@^7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/byte-size/-/byte-size-7.0.0.tgz#36528cd1ca87d39bd9abd51f5715dc93b6ceb032"
integrity sha512-NNiBxKgxybMBtWdmvx7ZITJi4ZG+CYUgwOSZTfqB1qogkRHrhbQE/R2r5Fh94X+InN5MCYz6SvB/ejHMj/HbsQ==
bytes@3.1.0: bytes@3.1.0:
version "3.1.0" version "3.1.0"
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.0.tgz#f6cf7933a360e0588fa9fde85651cdc7f805d1f6"
@ -1458,15 +1463,7 @@ core-util-is@1.0.2, core-util-is@~1.0.0:
resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
cron-parser@^2.15.0: cron-parser@^2.15.0, cron-parser@^2.7.3:
version "2.16.3"
resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.16.3.tgz#acb8e405eed1733aac542fdf604cb7c1daf0204a"
integrity sha512-XNJBD1QLFeAMUkZtZQuncAAOgJFWNhBdIbwgD22hZxrcWOImBFMKgPC66GzaXpyoJs7UvYLLgPH/8BRk/7gbZg==
dependencies:
is-nan "^1.3.0"
moment-timezone "^0.5.31"
cron-parser@^2.7.3:
version "2.16.3" version "2.16.3"
resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.16.3.tgz#acb8e405eed1733aac542fdf604cb7c1daf0204a" resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.16.3.tgz#acb8e405eed1733aac542fdf604cb7c1daf0204a"
integrity sha512-XNJBD1QLFeAMUkZtZQuncAAOgJFWNhBdIbwgD22hZxrcWOImBFMKgPC66GzaXpyoJs7UvYLLgPH/8BRk/7gbZg== integrity sha512-XNJBD1QLFeAMUkZtZQuncAAOgJFWNhBdIbwgD22hZxrcWOImBFMKgPC66GzaXpyoJs7UvYLLgPH/8BRk/7gbZg==
@ -1674,7 +1671,7 @@ diff@^4.0.1, diff@^4.0.2:
resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d"
integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==
diskusage@^1.1.1: diskusage@^1.1.1, diskusage@^1.1.3:
version "1.1.3" version "1.1.3"
resolved "https://registry.yarnpkg.com/diskusage/-/diskusage-1.1.3.tgz#680d7dbf1b679168a195c9240eb3552cbd2c067b" resolved "https://registry.yarnpkg.com/diskusage/-/diskusage-1.1.3.tgz#680d7dbf1b679168a195c9240eb3552cbd2c067b"
integrity sha512-EAyaxl8hy4Ph07kzlzGTfpbZMNAAAHXSZtNEMwdlnSd1noHzvA6HsgKt4fEMSvaEXQYLSphe5rPMxN4WOj0hcQ== integrity sha512-EAyaxl8hy4Ph07kzlzGTfpbZMNAAAHXSZtNEMwdlnSd1noHzvA6HsgKt4fEMSvaEXQYLSphe5rPMxN4WOj0hcQ==
@ -2406,6 +2403,19 @@ functional-red-black-tree@^1.0.1, functional-red-black-tree@~1.0.1:
resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327" resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327"
integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc= integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=
gar@^1.0.4:
version "1.0.4"
resolved "https://registry.yarnpkg.com/gar/-/gar-1.0.4.tgz#f777bc7db425c0572fdeb52676172ca1ae9888b8"
integrity sha512-w4n9cPWyP7aHxKxYHFQMegj7WIAsL/YX/C4Bs5Rr8s1H9M1rNtRWRsw+ovYMkXDQ5S4ZbYHsHAPmevPjPgw44w==
get-folder-size@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/get-folder-size/-/get-folder-size-2.0.1.tgz#3fe0524dd3bad05257ef1311331417bcd020a497"
integrity sha512-+CEb+GDCM7tkOS2wdMKTn9vU7DgnKUTuDlehkNJKNSovdCOVxs14OfKCk4cvSaR3za4gj+OBdl9opPN9xrJ0zA==
dependencies:
gar "^1.0.4"
tiny-each-async "2.0.3"
get-stream@^4.0.0, get-stream@^4.1.0: get-stream@^4.0.0, get-stream@^4.1.0:
version "4.1.0" version "4.1.0"
resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-4.1.0.tgz#c1b255575f3dc21d59bfc79cd3d2b46b1c3a54b5"
@ -6033,6 +6043,11 @@ thunkify@^2.1.2:
resolved "https://registry.yarnpkg.com/thunkify/-/thunkify-2.1.2.tgz#faa0e9d230c51acc95ca13a361ac05ca7e04553d" resolved "https://registry.yarnpkg.com/thunkify/-/thunkify-2.1.2.tgz#faa0e9d230c51acc95ca13a361ac05ca7e04553d"
integrity sha1-+qDp0jDFGsyVyhOjYawFyn4EVT0= integrity sha1-+qDp0jDFGsyVyhOjYawFyn4EVT0=
tiny-each-async@2.0.3:
version "2.0.3"
resolved "https://registry.yarnpkg.com/tiny-each-async/-/tiny-each-async-2.0.3.tgz#8ebbbfd6d6295f1370003fbb37162afe5a0a51d1"
integrity sha1-jru/1tYpXxNwAD+7NxYq/loKUdE=
tmp@0.0.33, tmp@^0.0.33: tmp@0.0.33, tmp@^0.0.33:
version "0.0.33" version "0.0.33"
resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9" resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"