Compare commits

...

5 Commits

Author SHA1 Message Date
Bennett Buchanan 529da4315e [squash] Revert to original config 2017-04-04 11:16:36 -07:00
Bennett Buchanan fef6da1445 [drop] Add utapi dependency 2017-04-03 10:39:43 -07:00
Bennett Buchanan a97971f448 [squash] Update config for utapi 2017-04-03 10:38:41 -07:00
Bennett Buchanan 32aae80c0b [squash] Remove utapi command line util 2017-04-01 15:38:39 -07:00
Bennett Buchanan 53bef46a42 DEV: Remove Utapi 2017-04-01 15:37:46 -07:00
9 changed files with 2 additions and 244 deletions

View File

@ -1,5 +0,0 @@
#!/usr/bin/env node
'use strict'; // eslint-disable-line strict
require('babel-core/register');
require('../lib/utapi/utilities.js').listMetrics('buckets');

View File

@ -1,5 +0,0 @@
#!/usr/bin/env node
'use strict'; // eslint-disable-line strict
require('babel-core/register');
require('../lib/utapi/utilities.js').listMetrics();

View File

@ -294,12 +294,7 @@ class Config {
'bad config: utapi port must be a positive integer');
this.utapi.port = config.utapi.port;
}
if (config.utapi.workers !== undefined) {
assert(Number.isInteger(config.utapi.workers)
&& config.utapi.workers > 0,
'bad config: utapi workers must be a positive integer');
this.utapi.workers = config.utapi.workers;
}
// Utapi uses the same localCache config defined for S3 to avoid
// config duplication.
assert(config.localCache, 'missing required property of utapi ' +
@ -340,22 +335,6 @@ class Config {
if (config.utapi.component) {
this.utapi.component = config.utapi.component;
}
// (optional) The value of the replay schedule should be cron-style
// scheduling. For example, every five minutes: '*/5 * * * *'.
if (config.utapi.replaySchedule) {
assert(typeof config.utapi.replaySchedule === 'string', 'bad' +
'config: utapi.replaySchedule must be a string');
this.utapi.replaySchedule = config.utapi.replaySchedule;
}
// (optional) The number of elements processed by each call to the
// Redis local cache during a replay. For example, 50.
if (config.utapi.batchSize) {
assert(typeof config.utapi.batchSize === 'number', 'bad' +
'config: utapi.batchSize must be a number');
assert(config.utapi.batchSize > 0, 'bad config:' +
'utapi.batchSize must be a number greater than 0');
this.utapi.batchSize = config.utapi.batchSize;
}
}
this.log = { logLevel: 'debug', dumpLevel: 'error' };

View File

@ -1,21 +0,0 @@
const utapiServer = require('utapi').UtapiServer;
const _config = require('../Config').default;
// start utapi server
export default function main() {
if (_config.utapi) {
const fullConfig = Object.assign({}, _config.utapi);
if (_config.vaultd) {
Object.assign(fullConfig, { vaultd: _config.vaultd });
}
if (_config.https) {
Object.assign(fullConfig, { https: _config.https });
}
// copy healthcheck IPs
if (_config.healthChecks) {
Object.assign(fullConfig, { healthChecks: _config.healthChecks });
}
utapiServer(fullConfig);
}
}

View File

@ -1,8 +0,0 @@
const UtapiReplay = require('utapi').UtapiReplay;
const _config = require('../Config').default;
// start utapi server
export default function main() {
const replay = new UtapiReplay(_config.utapi);
replay.start();
}

View File

@ -1,180 +1,8 @@
import { auth } from 'arsenal';
import { UtapiClient } from 'utapi';
import commander from 'commander';
import http from 'http';
import https from 'https';
import { logger } from '../utilities/logger';
import _config from '../Config';
// setup utapi client
const utapi = new UtapiClient(_config.utapi);
function _listMetrics(host,
port,
metric,
metricType,
timeRange,
accessKey,
secretKey,
verbose,
recent,
ssl) {
const listAction = recent ? 'ListRecentMetrics' : 'ListMetrics';
const options = {
host,
port,
method: 'POST',
path: `/${metric}?Action=${listAction}`,
headers: {
'content-type': 'application/json',
'cache-control': 'no-cache',
},
rejectUnauthorized: false,
};
const transport = ssl ? https : http;
const request = transport.request(options, response => {
if (verbose) {
logger.info('response status code', {
statusCode: response.statusCode,
});
logger.info('response headers', { headers: response.headers });
}
const body = [];
response.setEncoding('utf8');
response.on('data', chunk => body.push(chunk));
response.on('end', () => {
const responseBody = JSON.parse(body.join(''));
if (response.statusCode >= 200 && response.statusCode < 300) {
process.stdout.write(JSON.stringify(responseBody, null, 2));
process.stdout.write('\n');
process.exit(0);
} else {
logger.error('request failed with HTTP Status ', {
statusCode: response.statusCode,
body: responseBody,
});
process.exit(1);
}
});
});
// TODO: cleanup with refactor of generateV4Headers
request.path = `/${metric}`;
auth.client.generateV4Headers(request, { Action: listAction },
accessKey, secretKey, 's3');
request.path = `/${metric}?Action=${listAction}`;
if (verbose) {
logger.info('request headers', { headers: request._headers });
}
// If recent listing, we do not provide `timeRange` in the request
const requestObj = recent ? {} : { timeRange };
requestObj[metric] = metricType;
request.write(JSON.stringify(requestObj));
request.end();
}
/**
* This function is used as a binary to send a request to utapi server
* to list metrics for buckets or accounts
* @param {string} [metricType] - (optional) Defined as 'buckets' if old style
* bucket metrics listing
* @return {undefined}
*/
export function listMetrics(metricType) {
commander
.version('0.0.1')
.option('-a, --access-key <accessKey>', 'Access key id')
.option('-k, --secret-key <secretKey>', 'Secret access key');
// We want to continue support of previous bucket listing. Hence the ability
// to specify `metricType`. Remove `if` statement and
// bin/list_bucket_metrics.js when prior method of listing bucket metrics is
// no longer supported.
if (metricType === 'buckets') {
commander
.option('-b, --buckets <buckets>', 'Name of bucket(s) with ' +
'a comma separator if more than one');
} else {
commander
.option('-m, --metric <metric>', 'Metric type')
.option('--buckets <buckets>', 'Name of bucket(s) with a comma ' +
'separator if more than one')
.option('--accounts <accounts>', 'Account ID(s) with a comma ' +
'separator if more than one')
.option('--users <users>', 'User ID(s) with a comma separator if ' +
'more than one')
.option('--service <service>', 'Name of service');
}
commander
.option('-s, --start <start>', 'Start of time range')
.option('-r, --recent', 'List metrics including the previous and ' +
'current 15 minute interval')
.option('-e --end <end>', 'End of time range')
.option('-h, --host <host>', 'Host of the server')
.option('-p, --port <port>', 'Port of the server')
.option('--ssl', 'Enable ssl')
.option('-v, --verbose')
.parse(process.argv);
const { host, port, accessKey, secretKey, start, end, verbose, recent,
ssl } =
commander;
const requiredOptions = { host, port, accessKey, secretKey };
// If not old style bucket metrics, we require usage of the metric option
if (metricType !== 'buckets') {
requiredOptions.metric = commander.metric;
const validMetrics = ['buckets', 'accounts', 'users', 'service'];
if (validMetrics.indexOf(commander.metric) < 0) {
logger.error('metric must be \'buckets\', \'accounts\', ' +
'\'users\', or \'service\'');
commander.outputHelp();
process.exit(1);
return;
}
}
// If old style bucket metrics, `metricType` will be 'buckets'. Otherwise,
// `commander.metric` should be defined.
const metric = metricType === 'buckets' ? 'buckets' : commander.metric;
requiredOptions[metric] = commander[metric];
// If not recent listing, the start option must be provided
if (!recent) {
requiredOptions.start = commander.start;
}
Object.keys(requiredOptions).forEach(option => {
if (!requiredOptions[option]) {
logger.error(`missing required option: ${option}`);
commander.outputHelp();
process.exit(1);
return;
}
});
const timeRange = [];
// If recent listing, we disregard any start or end option given
if (!recent) {
const numStart = Number.parseInt(start, 10);
if (!numStart) {
logger.error('start must be a number');
commander.outputHelp();
process.exit(1);
return;
}
timeRange.push(numStart);
if (end) {
const numEnd = Number.parseInt(end, 10);
if (!numEnd) {
logger.error('end must be a number');
commander.outputHelp();
process.exit(1);
return;
}
timeRange.push(numEnd);
}
}
// The string `commander[metric]` is a comma-separated list of resources
// given by the user.
const resources = commander[metric].split(',');
_listMetrics(host, port, metric, resources, timeRange, accessKey, secretKey,
verbose, recent, ssl);
}
/**
* Call the Utapi Client `pushMetric` method with the associated parameters
* @param {string} action - the metric action to push a metric for

View File

@ -34,7 +34,7 @@
"node-uuid": "^1.4.3",
"ready-set-stream": "1.0.7",
"sproxydclient": "scality/sproxydclient",
"utapi": "scality/utapi",
"utapi": "scality/utapi#S3C-150/ft-separate-utapi-from-s3",
"utf8": "~2.1.1",
"vaultclient": "scality/vaultclient",
"werelogs": "scality/werelogs",
@ -73,8 +73,6 @@
"mem_backend": "S3BACKEND=mem node index.js",
"perf": "mocha --compilers js:babel-core/register tests/performance/s3standard.js",
"start": "node init.js && node index.js",
"start_utapi": "node utapiServer.js",
"utapi_replay": "node utapiReplay.js",
"test": "S3BACKEND=mem mocha --compilers js:babel-core/register --recursive tests/unit",
"multiple_backend_test": "node init.js && S3BACKEND=mem S3DATA=multiple mocha --compilers js:babel-core/register --recursive tests/multipleBackend",
"unit_coverage": "mkdir -p coverage/unit/ && S3BACKEND=mem MOCHA_FILE=$CIRCLE_TEST_REPORTS/unit/unit.xml istanbul cover --dir coverage/unit _mocha -- --compilers js:babel-core/register --reporter mocha-junit-reporter --recursive tests/unit"

View File

@ -1,4 +0,0 @@
'use strict'; // eslint-disable-line strict
require('babel-core/register')();
require('./lib/utapi/utapiReplay.js').default();

View File

@ -1,4 +0,0 @@
'use strict'; // eslint-disable-line strict
require('babel-core/register')();
require('./lib/utapi/utapi.js').default();