Compare commits
3 Commits
developmen
...
wip/ingest
Author | SHA1 | Date |
---|---|---|
philipyoo | f87c27791c | |
philipyoo | 8d92c8747b | |
philipyoo | 4c469809a0 |
|
@ -10,13 +10,13 @@ const isTest = process.env.CI === 'true';
|
|||
|
||||
class Metrics {
|
||||
constructor(config, logger) {
|
||||
const { redisConfig, validSites, internalStart } = config;
|
||||
const { redisConfig, crrSites, internalStart } = config;
|
||||
this._logger = logger;
|
||||
this._redisClient = new RedisClient(redisConfig, this._logger);
|
||||
// Redis expiry increased by an additional interval so we can reference
|
||||
// the immediate older data for average throughput calculation
|
||||
this._statsClient = new StatsModel(this._redisClient, INTERVAL, EXPIRY);
|
||||
this._validSites = validSites;
|
||||
this._crrSites = crrSites;
|
||||
this._internalStart = internalStart;
|
||||
}
|
||||
|
||||
|
@ -34,7 +34,7 @@ class Metrics {
|
|||
return async.map(ops, (op, done) => {
|
||||
const hasGlobalKey = this._hasGlobalKey(op);
|
||||
if (site === 'all') {
|
||||
const queryStrings = this._validSites.map(s => {
|
||||
const queryStrings = this._crrSites.map(s => {
|
||||
if (bucketName && objectKey && versionId) {
|
||||
return `${s}:${bucketName}:${objectKey}:` +
|
||||
`${versionId}:${op}`;
|
||||
|
@ -50,7 +50,7 @@ class Metrics {
|
|||
}
|
||||
// Query only a single given site or storage class
|
||||
// First, validate the site or storage class
|
||||
if (!this._validSites.includes(site)) {
|
||||
if (!this._crrSites.includes(site)) {
|
||||
// escalate error to log later
|
||||
return done({
|
||||
message: 'invalid site name provided',
|
||||
|
|
|
@ -5,10 +5,17 @@
|
|||
/**
|
||||
* The metrics route model.
|
||||
* @param {Object} redisKeys - The Redis keys used for Backbeat metrics
|
||||
* @param {Array} allLocations - The list of replication location names
|
||||
* @param {Object} locations - Locations by service
|
||||
* @param {Array} locations.crr - The list of replication location names
|
||||
* @param {Array} locations.ingestion - The list of ingestion location names
|
||||
* @return {Array} The array of route objects
|
||||
*/
|
||||
function routes(redisKeys, allLocations) {
|
||||
function routes(redisKeys, locations) {
|
||||
/* eslint-disable no-param-reassign */
|
||||
locations.crr = locations.crr || [];
|
||||
locations.ingestion = locations.ingestion || [];
|
||||
/* eslint-enable no-param-reassign */
|
||||
|
||||
return [
|
||||
// Route: /_/healthcheck
|
||||
{
|
||||
|
@ -23,7 +30,7 @@ function routes(redisKeys, allLocations) {
|
|||
httpMethod: 'GET',
|
||||
category: 'metrics',
|
||||
type: 'pending',
|
||||
extensions: { crr: [...allLocations, 'all'] },
|
||||
extensions: { crr: [...locations.crr, 'all'] },
|
||||
method: 'getPending',
|
||||
dataPoints: [redisKeys.opsPending, redisKeys.bytesPending],
|
||||
},
|
||||
|
@ -32,7 +39,7 @@ function routes(redisKeys, allLocations) {
|
|||
httpMethod: 'GET',
|
||||
category: 'metrics',
|
||||
type: 'backlog',
|
||||
extensions: { crr: [...allLocations, 'all'] },
|
||||
extensions: { crr: [...locations.crr, 'all'] },
|
||||
method: 'getBacklog',
|
||||
dataPoints: [redisKeys.opsPending, redisKeys.bytesPending],
|
||||
},
|
||||
|
@ -41,7 +48,7 @@ function routes(redisKeys, allLocations) {
|
|||
httpMethod: 'GET',
|
||||
category: 'metrics',
|
||||
type: 'completions',
|
||||
extensions: { crr: [...allLocations, 'all'] },
|
||||
extensions: { crr: [...locations.crr, 'all'] },
|
||||
method: 'getCompletions',
|
||||
dataPoints: [redisKeys.opsDone, redisKeys.bytesDone],
|
||||
},
|
||||
|
@ -50,7 +57,7 @@ function routes(redisKeys, allLocations) {
|
|||
httpMethod: 'GET',
|
||||
category: 'metrics',
|
||||
type: 'failures',
|
||||
extensions: { crr: [...allLocations, 'all'] },
|
||||
extensions: { crr: [...locations.crr, 'all'] },
|
||||
method: 'getFailedMetrics',
|
||||
dataPoints: [redisKeys.opsFail, redisKeys.bytesFail],
|
||||
},
|
||||
|
@ -59,7 +66,7 @@ function routes(redisKeys, allLocations) {
|
|||
httpMethod: 'GET',
|
||||
category: 'metrics',
|
||||
type: 'throughput',
|
||||
extensions: { crr: [...allLocations, 'all'] },
|
||||
extensions: { crr: [...locations.crr, 'all'] },
|
||||
method: 'getThroughput',
|
||||
dataPoints: [redisKeys.opsDone, redisKeys.bytesDone],
|
||||
},
|
||||
|
@ -68,7 +75,7 @@ function routes(redisKeys, allLocations) {
|
|||
httpMethod: 'GET',
|
||||
category: 'metrics',
|
||||
type: 'all',
|
||||
extensions: { crr: [...allLocations, 'all'] },
|
||||
extensions: { crr: [...locations.crr, 'all'] },
|
||||
method: 'getAllMetrics',
|
||||
dataPoints: [redisKeys.ops, redisKeys.opsDone, redisKeys.opsFail,
|
||||
redisKeys.bytes, redisKeys.bytesDone, redisKeys.bytesFail,
|
||||
|
@ -80,7 +87,7 @@ function routes(redisKeys, allLocations) {
|
|||
category: 'metrics',
|
||||
type: 'progress',
|
||||
level: 'object',
|
||||
extensions: { crr: [...allLocations] },
|
||||
extensions: { crr: [...locations.crr] },
|
||||
method: 'getObjectProgress',
|
||||
dataPoints: [redisKeys.objectBytes, redisKeys.objectBytesDone],
|
||||
},
|
||||
|
@ -90,7 +97,7 @@ function routes(redisKeys, allLocations) {
|
|||
category: 'metrics',
|
||||
type: 'throughput',
|
||||
level: 'object',
|
||||
extensions: { crr: [...allLocations] },
|
||||
extensions: { crr: [...locations.crr] },
|
||||
method: 'getObjectThroughput',
|
||||
dataPoints: [redisKeys.objectBytesDone],
|
||||
},
|
||||
|
@ -124,42 +131,54 @@ function routes(redisKeys, allLocations) {
|
|||
method: 'monitoringHandler',
|
||||
},
|
||||
// Route: /_/crr/pause/<location>
|
||||
// Route: /_/ingestion/pause/<location>
|
||||
// Where <location> is an optional field
|
||||
{
|
||||
httpMethod: 'POST',
|
||||
type: 'pause',
|
||||
extensions: { crr: [...allLocations, 'all'] },
|
||||
method: 'pauseCRRService',
|
||||
extensions: {
|
||||
crr: [...locations.crr, 'all'],
|
||||
ingestion: [...locations.ingestion, 'all'],
|
||||
},
|
||||
method: 'pauseService',
|
||||
},
|
||||
// Route: /_/crr/resume/<location>
|
||||
// Route: /_/ingestion/resume/<location>
|
||||
// Route: /_/crr/resume/<location>/schedule
|
||||
// Where <location> is an optional field unless "schedule" route
|
||||
{
|
||||
httpMethod: 'POST',
|
||||
type: 'resume',
|
||||
extensions: { crr: [...allLocations, 'all'] },
|
||||
method: 'resumeCRRService',
|
||||
extensions: {
|
||||
crr: [...locations.crr, 'all'],
|
||||
ingestion: [...locations.ingestion, 'all'],
|
||||
},
|
||||
method: 'resumeService',
|
||||
},
|
||||
{
|
||||
httpMethod: 'DELETE',
|
||||
type: 'resume',
|
||||
extensions: { crr: [...allLocations, 'all'] },
|
||||
extensions: { crr: [...locations.crr, 'all'] },
|
||||
method: 'deleteScheduledResumeService',
|
||||
},
|
||||
// Route: /_/crr/resume/<location>
|
||||
{
|
||||
httpMethod: 'GET',
|
||||
type: 'resume',
|
||||
extensions: { crr: [...allLocations, 'all'] },
|
||||
extensions: { crr: [...locations.crr, 'all'] },
|
||||
method: 'getResumeCRRSchedule',
|
||||
},
|
||||
// Route: /_/crr/status/<location>
|
||||
// Route: /_/ingestion/status/<location>
|
||||
// Where <location> is an optional field
|
||||
{
|
||||
httpMethod: 'GET',
|
||||
type: 'status',
|
||||
extensions: { crr: [...allLocations, 'all'] },
|
||||
method: 'getCRRServiceStatus',
|
||||
extensions: {
|
||||
crr: [...locations.crr, 'all'],
|
||||
ingestion: [...locations.ingestion, 'all'],
|
||||
},
|
||||
method: 'getServiceStatus',
|
||||
},
|
||||
];
|
||||
}
|
||||
|
|
|
@ -25,7 +25,7 @@ const redisClient = new RedisClient(config, fakeLogger);
|
|||
const sites = ['site1', 'site2'];
|
||||
const metrics = new backbeat.Metrics({
|
||||
redisConfig: config,
|
||||
validSites: ['site1', 'site2', 'all'],
|
||||
crrSites: ['site1', 'site2', 'all'],
|
||||
internalStart: Date.now() - (EXPIRY * 1000), // 24 hours ago.
|
||||
}, fakeLogger);
|
||||
|
||||
|
@ -46,7 +46,7 @@ describe('Metrics class', () => {
|
|||
opsPending: 'bb:crr:bytespending',
|
||||
bytesPending: 'bb:crr:opspending',
|
||||
};
|
||||
const routes = backbeat.routes(redisKeys, sites);
|
||||
const routes = backbeat.routes(redisKeys, { crr: sites });
|
||||
const details = routes.find(route =>
|
||||
route.category === 'metrics' && route.type === 'all');
|
||||
details.site = 'all';
|
||||
|
|
Loading…
Reference in New Issue