Compare commits
3 Commits
developmen
...
wip/ingest
Author | SHA1 | Date |
---|---|---|
philipyoo | f87c27791c | |
philipyoo | 8d92c8747b | |
philipyoo | 4c469809a0 |
|
@ -10,13 +10,13 @@ const isTest = process.env.CI === 'true';
|
||||||
|
|
||||||
class Metrics {
|
class Metrics {
|
||||||
constructor(config, logger) {
|
constructor(config, logger) {
|
||||||
const { redisConfig, validSites, internalStart } = config;
|
const { redisConfig, crrSites, internalStart } = config;
|
||||||
this._logger = logger;
|
this._logger = logger;
|
||||||
this._redisClient = new RedisClient(redisConfig, this._logger);
|
this._redisClient = new RedisClient(redisConfig, this._logger);
|
||||||
// Redis expiry increased by an additional interval so we can reference
|
// Redis expiry increased by an additional interval so we can reference
|
||||||
// the immediate older data for average throughput calculation
|
// the immediate older data for average throughput calculation
|
||||||
this._statsClient = new StatsModel(this._redisClient, INTERVAL, EXPIRY);
|
this._statsClient = new StatsModel(this._redisClient, INTERVAL, EXPIRY);
|
||||||
this._validSites = validSites;
|
this._crrSites = crrSites;
|
||||||
this._internalStart = internalStart;
|
this._internalStart = internalStart;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -34,7 +34,7 @@ class Metrics {
|
||||||
return async.map(ops, (op, done) => {
|
return async.map(ops, (op, done) => {
|
||||||
const hasGlobalKey = this._hasGlobalKey(op);
|
const hasGlobalKey = this._hasGlobalKey(op);
|
||||||
if (site === 'all') {
|
if (site === 'all') {
|
||||||
const queryStrings = this._validSites.map(s => {
|
const queryStrings = this._crrSites.map(s => {
|
||||||
if (bucketName && objectKey && versionId) {
|
if (bucketName && objectKey && versionId) {
|
||||||
return `${s}:${bucketName}:${objectKey}:` +
|
return `${s}:${bucketName}:${objectKey}:` +
|
||||||
`${versionId}:${op}`;
|
`${versionId}:${op}`;
|
||||||
|
@ -50,7 +50,7 @@ class Metrics {
|
||||||
}
|
}
|
||||||
// Query only a single given site or storage class
|
// Query only a single given site or storage class
|
||||||
// First, validate the site or storage class
|
// First, validate the site or storage class
|
||||||
if (!this._validSites.includes(site)) {
|
if (!this._crrSites.includes(site)) {
|
||||||
// escalate error to log later
|
// escalate error to log later
|
||||||
return done({
|
return done({
|
||||||
message: 'invalid site name provided',
|
message: 'invalid site name provided',
|
||||||
|
|
|
@ -5,10 +5,17 @@
|
||||||
/**
|
/**
|
||||||
* The metrics route model.
|
* The metrics route model.
|
||||||
* @param {Object} redisKeys - The Redis keys used for Backbeat metrics
|
* @param {Object} redisKeys - The Redis keys used for Backbeat metrics
|
||||||
* @param {Array} allLocations - The list of replication location names
|
* @param {Object} locations - Locations by service
|
||||||
|
* @param {Array} locations.crr - The list of replication location names
|
||||||
|
* @param {Array} locations.ingestion - The list of ingestion location names
|
||||||
* @return {Array} The array of route objects
|
* @return {Array} The array of route objects
|
||||||
*/
|
*/
|
||||||
function routes(redisKeys, allLocations) {
|
function routes(redisKeys, locations) {
|
||||||
|
/* eslint-disable no-param-reassign */
|
||||||
|
locations.crr = locations.crr || [];
|
||||||
|
locations.ingestion = locations.ingestion || [];
|
||||||
|
/* eslint-enable no-param-reassign */
|
||||||
|
|
||||||
return [
|
return [
|
||||||
// Route: /_/healthcheck
|
// Route: /_/healthcheck
|
||||||
{
|
{
|
||||||
|
@ -23,7 +30,7 @@ function routes(redisKeys, allLocations) {
|
||||||
httpMethod: 'GET',
|
httpMethod: 'GET',
|
||||||
category: 'metrics',
|
category: 'metrics',
|
||||||
type: 'pending',
|
type: 'pending',
|
||||||
extensions: { crr: [...allLocations, 'all'] },
|
extensions: { crr: [...locations.crr, 'all'] },
|
||||||
method: 'getPending',
|
method: 'getPending',
|
||||||
dataPoints: [redisKeys.opsPending, redisKeys.bytesPending],
|
dataPoints: [redisKeys.opsPending, redisKeys.bytesPending],
|
||||||
},
|
},
|
||||||
|
@ -32,7 +39,7 @@ function routes(redisKeys, allLocations) {
|
||||||
httpMethod: 'GET',
|
httpMethod: 'GET',
|
||||||
category: 'metrics',
|
category: 'metrics',
|
||||||
type: 'backlog',
|
type: 'backlog',
|
||||||
extensions: { crr: [...allLocations, 'all'] },
|
extensions: { crr: [...locations.crr, 'all'] },
|
||||||
method: 'getBacklog',
|
method: 'getBacklog',
|
||||||
dataPoints: [redisKeys.opsPending, redisKeys.bytesPending],
|
dataPoints: [redisKeys.opsPending, redisKeys.bytesPending],
|
||||||
},
|
},
|
||||||
|
@ -41,7 +48,7 @@ function routes(redisKeys, allLocations) {
|
||||||
httpMethod: 'GET',
|
httpMethod: 'GET',
|
||||||
category: 'metrics',
|
category: 'metrics',
|
||||||
type: 'completions',
|
type: 'completions',
|
||||||
extensions: { crr: [...allLocations, 'all'] },
|
extensions: { crr: [...locations.crr, 'all'] },
|
||||||
method: 'getCompletions',
|
method: 'getCompletions',
|
||||||
dataPoints: [redisKeys.opsDone, redisKeys.bytesDone],
|
dataPoints: [redisKeys.opsDone, redisKeys.bytesDone],
|
||||||
},
|
},
|
||||||
|
@ -50,7 +57,7 @@ function routes(redisKeys, allLocations) {
|
||||||
httpMethod: 'GET',
|
httpMethod: 'GET',
|
||||||
category: 'metrics',
|
category: 'metrics',
|
||||||
type: 'failures',
|
type: 'failures',
|
||||||
extensions: { crr: [...allLocations, 'all'] },
|
extensions: { crr: [...locations.crr, 'all'] },
|
||||||
method: 'getFailedMetrics',
|
method: 'getFailedMetrics',
|
||||||
dataPoints: [redisKeys.opsFail, redisKeys.bytesFail],
|
dataPoints: [redisKeys.opsFail, redisKeys.bytesFail],
|
||||||
},
|
},
|
||||||
|
@ -59,7 +66,7 @@ function routes(redisKeys, allLocations) {
|
||||||
httpMethod: 'GET',
|
httpMethod: 'GET',
|
||||||
category: 'metrics',
|
category: 'metrics',
|
||||||
type: 'throughput',
|
type: 'throughput',
|
||||||
extensions: { crr: [...allLocations, 'all'] },
|
extensions: { crr: [...locations.crr, 'all'] },
|
||||||
method: 'getThroughput',
|
method: 'getThroughput',
|
||||||
dataPoints: [redisKeys.opsDone, redisKeys.bytesDone],
|
dataPoints: [redisKeys.opsDone, redisKeys.bytesDone],
|
||||||
},
|
},
|
||||||
|
@ -68,7 +75,7 @@ function routes(redisKeys, allLocations) {
|
||||||
httpMethod: 'GET',
|
httpMethod: 'GET',
|
||||||
category: 'metrics',
|
category: 'metrics',
|
||||||
type: 'all',
|
type: 'all',
|
||||||
extensions: { crr: [...allLocations, 'all'] },
|
extensions: { crr: [...locations.crr, 'all'] },
|
||||||
method: 'getAllMetrics',
|
method: 'getAllMetrics',
|
||||||
dataPoints: [redisKeys.ops, redisKeys.opsDone, redisKeys.opsFail,
|
dataPoints: [redisKeys.ops, redisKeys.opsDone, redisKeys.opsFail,
|
||||||
redisKeys.bytes, redisKeys.bytesDone, redisKeys.bytesFail,
|
redisKeys.bytes, redisKeys.bytesDone, redisKeys.bytesFail,
|
||||||
|
@ -80,7 +87,7 @@ function routes(redisKeys, allLocations) {
|
||||||
category: 'metrics',
|
category: 'metrics',
|
||||||
type: 'progress',
|
type: 'progress',
|
||||||
level: 'object',
|
level: 'object',
|
||||||
extensions: { crr: [...allLocations] },
|
extensions: { crr: [...locations.crr] },
|
||||||
method: 'getObjectProgress',
|
method: 'getObjectProgress',
|
||||||
dataPoints: [redisKeys.objectBytes, redisKeys.objectBytesDone],
|
dataPoints: [redisKeys.objectBytes, redisKeys.objectBytesDone],
|
||||||
},
|
},
|
||||||
|
@ -90,7 +97,7 @@ function routes(redisKeys, allLocations) {
|
||||||
category: 'metrics',
|
category: 'metrics',
|
||||||
type: 'throughput',
|
type: 'throughput',
|
||||||
level: 'object',
|
level: 'object',
|
||||||
extensions: { crr: [...allLocations] },
|
extensions: { crr: [...locations.crr] },
|
||||||
method: 'getObjectThroughput',
|
method: 'getObjectThroughput',
|
||||||
dataPoints: [redisKeys.objectBytesDone],
|
dataPoints: [redisKeys.objectBytesDone],
|
||||||
},
|
},
|
||||||
|
@ -124,42 +131,54 @@ function routes(redisKeys, allLocations) {
|
||||||
method: 'monitoringHandler',
|
method: 'monitoringHandler',
|
||||||
},
|
},
|
||||||
// Route: /_/crr/pause/<location>
|
// Route: /_/crr/pause/<location>
|
||||||
|
// Route: /_/ingestion/pause/<location>
|
||||||
// Where <location> is an optional field
|
// Where <location> is an optional field
|
||||||
{
|
{
|
||||||
httpMethod: 'POST',
|
httpMethod: 'POST',
|
||||||
type: 'pause',
|
type: 'pause',
|
||||||
extensions: { crr: [...allLocations, 'all'] },
|
extensions: {
|
||||||
method: 'pauseCRRService',
|
crr: [...locations.crr, 'all'],
|
||||||
|
ingestion: [...locations.ingestion, 'all'],
|
||||||
|
},
|
||||||
|
method: 'pauseService',
|
||||||
},
|
},
|
||||||
// Route: /_/crr/resume/<location>
|
// Route: /_/crr/resume/<location>
|
||||||
|
// Route: /_/ingestion/resume/<location>
|
||||||
// Route: /_/crr/resume/<location>/schedule
|
// Route: /_/crr/resume/<location>/schedule
|
||||||
// Where <location> is an optional field unless "schedule" route
|
// Where <location> is an optional field unless "schedule" route
|
||||||
{
|
{
|
||||||
httpMethod: 'POST',
|
httpMethod: 'POST',
|
||||||
type: 'resume',
|
type: 'resume',
|
||||||
extensions: { crr: [...allLocations, 'all'] },
|
extensions: {
|
||||||
method: 'resumeCRRService',
|
crr: [...locations.crr, 'all'],
|
||||||
|
ingestion: [...locations.ingestion, 'all'],
|
||||||
|
},
|
||||||
|
method: 'resumeService',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
httpMethod: 'DELETE',
|
httpMethod: 'DELETE',
|
||||||
type: 'resume',
|
type: 'resume',
|
||||||
extensions: { crr: [...allLocations, 'all'] },
|
extensions: { crr: [...locations.crr, 'all'] },
|
||||||
method: 'deleteScheduledResumeService',
|
method: 'deleteScheduledResumeService',
|
||||||
},
|
},
|
||||||
// Route: /_/crr/resume/<location>
|
// Route: /_/crr/resume/<location>
|
||||||
{
|
{
|
||||||
httpMethod: 'GET',
|
httpMethod: 'GET',
|
||||||
type: 'resume',
|
type: 'resume',
|
||||||
extensions: { crr: [...allLocations, 'all'] },
|
extensions: { crr: [...locations.crr, 'all'] },
|
||||||
method: 'getResumeCRRSchedule',
|
method: 'getResumeCRRSchedule',
|
||||||
},
|
},
|
||||||
// Route: /_/crr/status/<location>
|
// Route: /_/crr/status/<location>
|
||||||
|
// Route: /_/ingestion/status/<location>
|
||||||
// Where <location> is an optional field
|
// Where <location> is an optional field
|
||||||
{
|
{
|
||||||
httpMethod: 'GET',
|
httpMethod: 'GET',
|
||||||
type: 'status',
|
type: 'status',
|
||||||
extensions: { crr: [...allLocations, 'all'] },
|
extensions: {
|
||||||
method: 'getCRRServiceStatus',
|
crr: [...locations.crr, 'all'],
|
||||||
|
ingestion: [...locations.ingestion, 'all'],
|
||||||
|
},
|
||||||
|
method: 'getServiceStatus',
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
|
@ -25,7 +25,7 @@ const redisClient = new RedisClient(config, fakeLogger);
|
||||||
const sites = ['site1', 'site2'];
|
const sites = ['site1', 'site2'];
|
||||||
const metrics = new backbeat.Metrics({
|
const metrics = new backbeat.Metrics({
|
||||||
redisConfig: config,
|
redisConfig: config,
|
||||||
validSites: ['site1', 'site2', 'all'],
|
crrSites: ['site1', 'site2', 'all'],
|
||||||
internalStart: Date.now() - (EXPIRY * 1000), // 24 hours ago.
|
internalStart: Date.now() - (EXPIRY * 1000), // 24 hours ago.
|
||||||
}, fakeLogger);
|
}, fakeLogger);
|
||||||
|
|
||||||
|
@ -46,7 +46,7 @@ describe('Metrics class', () => {
|
||||||
opsPending: 'bb:crr:bytespending',
|
opsPending: 'bb:crr:bytespending',
|
||||||
bytesPending: 'bb:crr:opspending',
|
bytesPending: 'bb:crr:opspending',
|
||||||
};
|
};
|
||||||
const routes = backbeat.routes(redisKeys, sites);
|
const routes = backbeat.routes(redisKeys, { crr: sites });
|
||||||
const details = routes.find(route =>
|
const details = routes.find(route =>
|
||||||
route.category === 'metrics' && route.type === 'all');
|
route.category === 'metrics' && route.type === 'all');
|
||||||
details.site = 'all';
|
details.site = 'all';
|
||||||
|
|
Loading…
Reference in New Issue