Compare commits
No commits in common. "0b79f1e72636337b52794e057020352594f91e1d" and "c2f121d0d3a2fca0dc22627483d9f161f061684b" have entirely different histories.
0b79f1e726
...
c2f121d0d3
|
|
@ -103,8 +103,7 @@ jobs:
|
|||
test:
|
||||
- name: run unit tests
|
||||
command: yarn test
|
||||
env:
|
||||
UTAPI_METRICS_ENABLED: 'true'
|
||||
env: {}
|
||||
- name: run client tests
|
||||
command: bash ./.github/scripts/run_ft_tests.bash false ft_test:client
|
||||
env: {}
|
||||
|
|
|
|||
|
|
@ -19,10 +19,7 @@ class RedisCache {
|
|||
moduleLogger.debug('Connecting to redis...');
|
||||
this._redis = new RedisClient(this._options);
|
||||
this._redis.connect();
|
||||
return new Promise((resolve, reject) => {
|
||||
this._redis.once('ready', () => resolve(true));
|
||||
this._redis.once('error', error => reject(error));
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
async disconnect() {
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ const Joi = require('@hapi/joi');
|
|||
const { buildModel } = require('./Base');
|
||||
const { apiOperations } = require('../server/spec');
|
||||
const ResponseContainer = require('./ResponseContainer');
|
||||
const { httpRequestDurationSeconds } = require('../server/metrics');
|
||||
|
||||
const apiTags = Object.keys(apiOperations);
|
||||
const apiOperationIds = Object.values(apiOperations)
|
||||
|
|
@ -22,7 +21,6 @@ const contextSchema = {
|
|||
logger: Joi.any(),
|
||||
request: Joi.any(),
|
||||
results: Joi.any(),
|
||||
requestTimer: Joi.any(),
|
||||
};
|
||||
|
||||
const RequestContextModel = buildModel('RequestContext', contextSchema);
|
||||
|
|
@ -36,10 +34,6 @@ class RequestContext extends RequestContextModel {
|
|||
const tag = request.swagger.operation['x-router-controller'];
|
||||
const { operationId } = request.swagger.operation;
|
||||
|
||||
const requestTimer = tag !== 'internal'
|
||||
? httpRequestDurationSeconds.startTimer({ action: operationId })
|
||||
: null;
|
||||
|
||||
request.logger.logger.addDefaultFields({
|
||||
tag,
|
||||
operationId,
|
||||
|
|
@ -56,7 +50,6 @@ class RequestContext extends RequestContextModel {
|
|||
encrypted,
|
||||
results: new ResponseContainer(),
|
||||
logger: request.logger,
|
||||
requestTimer,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ class UtapiServer extends Process {
|
|||
app.use(middleware.loggerMiddleware);
|
||||
await initializeOasTools(spec, app);
|
||||
app.use(middleware.errorMiddleware);
|
||||
app.use(middleware.httpMetricsMiddleware);
|
||||
app.use(middleware.responseLoggerMiddleware);
|
||||
return app;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,20 +0,0 @@
|
|||
const promClient = require('prom-client');
|
||||
|
||||
const httpRequestsTotal = new promClient.Counter({
|
||||
name: 'utapi_http_requests_total',
|
||||
help: 'Total number of HTTP requests',
|
||||
labelNames: ['action', 'code'],
|
||||
});
|
||||
|
||||
const httpRequestDurationSeconds = new promClient.Histogram({
|
||||
name: 'utapi_http_request_duration_seconds',
|
||||
help: 'Duration of HTTP requests in seconds',
|
||||
labelNames: ['action', 'code'],
|
||||
// buckets for response time from 0.1ms to 60s
|
||||
buckets: [0.0001, 0.005, 0.015, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 1.0, 5.0, 15.0, 30.0, 60.0],
|
||||
});
|
||||
|
||||
module.exports = {
|
||||
httpRequestDurationSeconds,
|
||||
httpRequestsTotal,
|
||||
};
|
||||
|
|
@ -6,7 +6,6 @@ const config = require('../config');
|
|||
const { logger, buildRequestLogger } = require('../utils');
|
||||
const errors = require('../errors');
|
||||
const { translateAndAuthorize } = require('../vault');
|
||||
const metricHandlers = require('./metrics');
|
||||
|
||||
const oasOptions = {
|
||||
controllers: path.join(__dirname, './API/'),
|
||||
|
|
@ -56,23 +55,6 @@ function responseLoggerMiddleware(req, res, next) {
|
|||
}
|
||||
}
|
||||
|
||||
function httpMetricsMiddleware(request, response, next) {
|
||||
// If the request.ctx is undefined then this is an internal oasTools request (/_/docs)
|
||||
// No metrics should be pushed
|
||||
if (config.metrics.enabled && request.ctx && request.ctx.tag !== 'internal') {
|
||||
metricHandlers.httpRequestsTotal
|
||||
.labels({
|
||||
action: request.ctx.operationId,
|
||||
code: response.statusCode,
|
||||
}).inc(1);
|
||||
request.ctx.requestTimer({ code: response.statusCode });
|
||||
}
|
||||
|
||||
if (next) {
|
||||
next();
|
||||
}
|
||||
}
|
||||
|
||||
// next is purposely not called as all error responses are handled here
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
function errorMiddleware(err, req, res, next) {
|
||||
|
|
@ -100,7 +82,7 @@ function errorMiddleware(err, req, res, next) {
|
|||
code,
|
||||
message,
|
||||
});
|
||||
responseLoggerMiddleware(req, res, () => httpMetricsMiddleware(req, res));
|
||||
responseLoggerMiddleware(req, res);
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
|
|
@ -176,6 +158,5 @@ module.exports = {
|
|||
responseLoggerMiddleware,
|
||||
authV4Middleware,
|
||||
clientIpLimitMiddleware,
|
||||
httpMetricsMiddleware,
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -48,11 +48,6 @@ class BaseTask extends Process {
|
|||
}
|
||||
|
||||
if (this._enableMetrics) {
|
||||
promClient.collectDefaultMetrics({
|
||||
timeout: 10000,
|
||||
gcDurationBuckets: [0.001, 0.01, 0.1, 1, 2, 5],
|
||||
});
|
||||
|
||||
this._metricsHandlers = {
|
||||
...this._registerDefaultMetricHandlers(),
|
||||
...this._registerMetricHandlers(),
|
||||
|
|
|
|||
|
|
@ -29,26 +29,8 @@ class CreateCheckpoint extends BaseTask {
|
|||
labelNames: ['origin', 'containerName'],
|
||||
});
|
||||
|
||||
const getLastCheckpoint = this._getLastCheckpoint.bind(this);
|
||||
const lastCheckpoint = new promClient.Gauge({
|
||||
name: 'utapi_create_checkpoint_last_checkpoint_seconds',
|
||||
help: 'Timestamp of the last successfully created checkpoint',
|
||||
labelNames: ['origin', 'containerName'],
|
||||
async collect() {
|
||||
try {
|
||||
const timestamp = await getLastCheckpoint();
|
||||
if (timestamp !== null) {
|
||||
this.set(timestamp);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('error during metric collection', { error });
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
created,
|
||||
lastCheckpoint,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -73,25 +55,6 @@ class CreateCheckpoint extends BaseTask {
|
|||
}
|
||||
}
|
||||
|
||||
async _getLastCheckpoint() {
|
||||
const resp = await this.withWarp10(async warp10 => warp10.fetch({
|
||||
className: 'utapi.checkpoint.master',
|
||||
labels: {
|
||||
node: warp10.nodeId,
|
||||
},
|
||||
start: 'now',
|
||||
stop: -1,
|
||||
}));
|
||||
|
||||
if (resp.result && (resp.result.length === 0 || resp.result[0] === '' || resp.result[0] === '[]')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const result = JSON.parse(resp.result[0])[0];
|
||||
const timestamp = result.v[0][0];
|
||||
return timestamp / 1000000;// Convert timestamp from microseconds to seconds
|
||||
}
|
||||
|
||||
async _execute(timestamp) {
|
||||
logger.debug('creating checkpoints', { checkpointTimestamp: timestamp });
|
||||
const status = await this.withWarp10(async warp10 => {
|
||||
|
|
|
|||
|
|
@ -29,26 +29,8 @@ class CreateSnapshot extends BaseTask {
|
|||
labelNames: ['origin', 'containerName'],
|
||||
});
|
||||
|
||||
const getLastSnapshot = this._getLastSnapshot.bind(this);
|
||||
const lastSnapshot = new promClient.Gauge({
|
||||
name: 'utapi_create_snapshot_last_snapshot_seconds',
|
||||
help: 'Timestamp of the last successfully created snapshot',
|
||||
labelNames: ['origin', 'containerName'],
|
||||
async collect() {
|
||||
try {
|
||||
const timestamp = await getLastSnapshot();
|
||||
if (timestamp !== null) {
|
||||
this.set(timestamp);
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error('error during metric collection', { error });
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
created,
|
||||
lastSnapshot,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -73,25 +55,6 @@ class CreateSnapshot extends BaseTask {
|
|||
}
|
||||
}
|
||||
|
||||
async _getLastSnapshot() {
|
||||
const resp = await this.withWarp10(async warp10 => warp10.fetch({
|
||||
className: 'utapi.snapshot.master',
|
||||
labels: {
|
||||
node: warp10.nodeId,
|
||||
},
|
||||
start: 'now',
|
||||
stop: -1,
|
||||
}));
|
||||
|
||||
if (resp.result && (resp.result.length === 0 || resp.result[0] === '' || resp.result[0] === '[]')) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const result = JSON.parse(resp.result[0])[0];
|
||||
const timestamp = result.v[0][0];
|
||||
return timestamp / 1000000;// Convert timestamp from microseconds to seconds
|
||||
}
|
||||
|
||||
async _execute(timestamp) {
|
||||
logger.debug('creating snapshots', { snapshotTimestamp: timestamp });
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,4 @@
|
|||
const async = require('async');
|
||||
const Path = require('path');
|
||||
const fs = require('fs');
|
||||
const promClient = require('prom-client');
|
||||
const BaseTask = require('./BaseTask');
|
||||
const config = require('../config');
|
||||
|
|
@ -57,15 +55,9 @@ class MonitorDiskUsage extends BaseTask {
|
|||
labelNames: ['origin', 'containerName'],
|
||||
});
|
||||
|
||||
const leveldbBytes = new promClient.Gauge({
|
||||
name: 'utapi_monitor_disk_usage_leveldb_bytes',
|
||||
help: 'Total bytes used by warp 10 leveldb',
|
||||
labelNames: ['origin', 'containerName'],
|
||||
});
|
||||
|
||||
const datalogBytes = new promClient.Gauge({
|
||||
name: 'utapi_monitor_disk_usage_datalog_bytes',
|
||||
help: 'Total bytes used by warp 10 datalog',
|
||||
const diskUsage = new promClient.Gauge({
|
||||
name: 'utapi_monitor_disk_usage_bytes',
|
||||
help: 'Total bytes used by warp 10',
|
||||
labelNames: ['origin', 'containerName'],
|
||||
});
|
||||
|
||||
|
|
@ -83,8 +75,7 @@ class MonitorDiskUsage extends BaseTask {
|
|||
|
||||
return {
|
||||
isLocked,
|
||||
leveldbBytes,
|
||||
datalogBytes,
|
||||
diskUsage,
|
||||
hardLimitRatio,
|
||||
hardLimitSetting,
|
||||
};
|
||||
|
|
@ -94,8 +85,7 @@ class MonitorDiskUsage extends BaseTask {
|
|||
* Metrics for MonitorDiskUsage
|
||||
* @typedef {Object} MonitorDiskUsageMetrics
|
||||
* @property {boolean} isLocked - Indicates if writes have been disabled for the monitored warp10
|
||||
* @property {number} leveldbBytes - Total bytes used by warp 10 leveldb
|
||||
* @property {number} datalogBytes - Total bytes used by warp 10 datalog
|
||||
* @property {number} diskUsage - Total bytes used by warp 10
|
||||
* @property {number} hardLimitRatio - Percent of the hard limit used by warp 10
|
||||
* @property {number} hardLimitSetting - The hard limit setting in bytes
|
||||
*/
|
||||
|
|
@ -114,12 +104,8 @@ class MonitorDiskUsage extends BaseTask {
|
|||
this._metricsHandlers.isLocked.set(metrics.isLocked ? 1 : 0);
|
||||
}
|
||||
|
||||
if (metrics.leveldbBytes !== undefined) {
|
||||
this._metricsHandlers.leveldbBytes.set(metrics.leveldbBytes);
|
||||
}
|
||||
|
||||
if (metrics.datalogBytes !== undefined) {
|
||||
this._metricsHandlers.datalogBytes.set(metrics.datalogBytes);
|
||||
if (metrics.diskUsage !== undefined) {
|
||||
this._metricsHandlers.diskUsage.set(metrics.diskUsage);
|
||||
}
|
||||
|
||||
if (metrics.hardLimitRatio !== undefined) {
|
||||
|
|
@ -143,13 +129,9 @@ class MonitorDiskUsage extends BaseTask {
|
|||
return this._program.lock !== undefined;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line class-methods-use-this
|
||||
async _getUsage(path) {
|
||||
moduleLogger.debug(`calculating disk usage for ${path}`);
|
||||
if (!fs.existsSync(path)) {
|
||||
throw Error(`failed to calculate usage for non-existent path ${path}`);
|
||||
}
|
||||
return getFolderSize(path);
|
||||
_getUsage() {
|
||||
moduleLogger.debug(`calculating disk usage for ${this._path}`);
|
||||
return getFolderSize(this._path);
|
||||
}
|
||||
|
||||
async _expireMetrics(timestamp) {
|
||||
|
|
@ -267,19 +249,16 @@ class MonitorDiskUsage extends BaseTask {
|
|||
return;
|
||||
}
|
||||
|
||||
let leveldbBytes = null;
|
||||
let datalogBytes = null;
|
||||
let size = null;
|
||||
try {
|
||||
leveldbBytes = await this._getUsage(Path.join(this._path, 'leveldb'));
|
||||
datalogBytes = await this._getUsage(Path.join(this._path, 'datalog'));
|
||||
size = await this._getUsage();
|
||||
} catch (error) {
|
||||
moduleLogger.error(`error calculating disk usage for ${this._path}`, { error });
|
||||
return;
|
||||
}
|
||||
|
||||
this._pushMetrics({ leveldbBytes, datalogBytes });
|
||||
this._pushMetrics({ diskUsage: size });
|
||||
|
||||
const size = leveldbBytes + datalogBytes;
|
||||
if (this._hardLimit !== null) {
|
||||
moduleLogger.info(`warp 10 leveldb using ${formatDiskSize(size)} of disk space`, { usage: size });
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
const assert = require('assert');
|
||||
const fs = require('fs');
|
||||
const sinon = require('sinon');
|
||||
const uuid = require('uuid');
|
||||
const promClient = require('prom-client');
|
||||
|
|
@ -18,7 +17,6 @@ describe('Test MonitorDiskUsage hard limit', function () {
|
|||
|
||||
beforeEach(async () => {
|
||||
path = `/tmp/diskusage-${uuid.v4()}`;
|
||||
fs.mkdirSync(`${path}/datalog`, { recursive: true });
|
||||
promClient.register.clear();
|
||||
task = new MonitorDiskUsage({ warp10: warp10Clients, enableMetrics: true });
|
||||
await task.setup();
|
||||
|
|
@ -29,7 +27,7 @@ describe('Test MonitorDiskUsage hard limit', function () {
|
|||
afterEach(async () => task.join());
|
||||
|
||||
it('should trigger a database lock if above the limit', async () => {
|
||||
fillDir(`${path}/leveldb`, { count: 1, size: 100 });
|
||||
fillDir(path, { count: 1, size: 100 });
|
||||
task._hardLimit = 1;
|
||||
const checkSpy = sinon.spy(task, '_checkHardLimit');
|
||||
const lockSpy = sinon.spy(task, '_disableWarp10Updates');
|
||||
|
|
@ -45,7 +43,7 @@ describe('Test MonitorDiskUsage hard limit', function () {
|
|||
});
|
||||
|
||||
it('should trigger a database unlock if below the limit', async () => {
|
||||
fillDir(`${path}/leveldb`, { count: 1, size: 100 });
|
||||
fillDir(path, { count: 1, size: 100 });
|
||||
task._hardLimit = 10240;
|
||||
const checkSpy = sinon.spy(task, '_checkHardLimit');
|
||||
const lockSpy = sinon.spy(task, '_disableWarp10Updates');
|
||||
|
|
@ -58,7 +56,7 @@ describe('Test MonitorDiskUsage hard limit', function () {
|
|||
});
|
||||
|
||||
it('should not throw when failing to calculate disk usage', async () => {
|
||||
fillDir(`${path}/leveldb`, { count: 1, size: 100 });
|
||||
fillDir(path, { count: 1, size: 100 });
|
||||
task._hardLimit = 1;
|
||||
sinon.stub(task, '_getUsage').throws();
|
||||
const _task = task.execute();
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
const assert = require('assert');
|
||||
const fs = require('fs');
|
||||
const promClient = require('prom-client');
|
||||
const uuid = require('uuid');
|
||||
|
||||
|
|
@ -10,10 +9,9 @@ const { fillDir } = require('../../../utils/v2Data');
|
|||
const { assertMetricValue } = require('../../../utils/prom');
|
||||
|
||||
class MonitorDiskUsageShim extends MonitorDiskUsage {
|
||||
async _getUsage(path) {
|
||||
const usage = await super._getUsage(path);
|
||||
this.usage = (this.usage || 0) + usage;
|
||||
return usage;
|
||||
async _getUsage() {
|
||||
this.usage = await super._getUsage();
|
||||
return this.usage;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -47,7 +45,6 @@ describe('Test MonitorDiskUsage', () => {
|
|||
|
||||
beforeEach(async () => {
|
||||
path = `/tmp/diskusage-${uuid.v4()}`;
|
||||
fs.mkdirSync(path);
|
||||
task = new MonitorDiskUsageShim({ warp10: [], enableMetrics: true });
|
||||
task._path = path;
|
||||
task._enabled = true;
|
||||
|
|
@ -59,23 +56,12 @@ describe('Test MonitorDiskUsage', () => {
|
|||
promClient.register.clear();
|
||||
});
|
||||
|
||||
testCases.map(testCase => {
|
||||
testCases.map(testCase =>
|
||||
it(`should calculate disk usage for ${testCase.count} files of ${testCase.size} bytes each`,
|
||||
async () => {
|
||||
fillDir(`${path}/leveldb`, testCase);
|
||||
fillDir(`${path}/datalog`, testCase);
|
||||
fillDir(path, testCase);
|
||||
await task._execute();
|
||||
const expectedSingleSize = emptyDirSize + testCase.expected + (emptyFileSize * testCase.count);
|
||||
const expectedTotalSize = expectedSingleSize * 2;
|
||||
assert.strictEqual(task.usage, expectedTotalSize);
|
||||
// Should equal the usage minus the empty datalog
|
||||
await assertMetricValue('utapi_monitor_disk_usage_leveldb_bytes', expectedSingleSize);
|
||||
await assertMetricValue('utapi_monitor_disk_usage_datalog_bytes', expectedSingleSize);
|
||||
});
|
||||
});
|
||||
|
||||
it('should fail if a subpath does not exists', () => {
|
||||
assert.doesNotThrow(() => task._execute());
|
||||
assert.strictEqual(task.usage, undefined);
|
||||
});
|
||||
assert.strictEqual(task.usage, testCase.expected + emptyDirSize + (emptyFileSize * testCase.count));
|
||||
await assertMetricValue('utapi_monitor_disk_usage_bytes', task.usage);
|
||||
}));
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,7 +10,6 @@ const templateExpected = opts => ({
|
|||
operationId: 'healthcheck',
|
||||
tag: 'internal',
|
||||
encrypted: false,
|
||||
requestTimer: null,
|
||||
...(opts || {}),
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,8 @@
|
|||
const assert = require('assert');
|
||||
const sinon = require('sinon');
|
||||
const promClient = require('prom-client');
|
||||
|
||||
const { middleware } = require('../../../../libV2/server/middleware');
|
||||
const { templateRequest, ExpressResponseStub } = require('../../../utils/v2Data');
|
||||
const RequestContext = require('../../../../libV2/models/RequestContext');
|
||||
const { getMetricValues, assertMetricValue } = require('../../../utils/prom');
|
||||
|
||||
describe('Test middleware', () => {
|
||||
it('should build a request logger', next => {
|
||||
|
|
@ -89,55 +86,4 @@ describe('Test middleware', () => {
|
|||
}));
|
||||
});
|
||||
});
|
||||
|
||||
describe('test httpMetricsMiddleware', () => {
|
||||
let resp;
|
||||
|
||||
beforeEach(() => {
|
||||
resp = new ExpressResponseStub();
|
||||
resp.status(200);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
promClient.register.clear();
|
||||
});
|
||||
|
||||
it('should increment the counter if not an internal route', async () => {
|
||||
const req = templateRequest({
|
||||
swagger: {
|
||||
operation: {
|
||||
'x-router-controller': 'metrics',
|
||||
'operationId': 'listMetrics',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
req.ctx = new RequestContext(req);
|
||||
middleware.httpMetricsMiddleware(req, resp);
|
||||
await assertMetricValue('utapi_http_requests_total', 1);
|
||||
const duration = await getMetricValues('utapi_http_request_duration_seconds');
|
||||
// 14 defined buckets + 1 for Infinity
|
||||
assert.strictEqual(
|
||||
duration.filter(metric => metric.metricName === 'utapi_http_request_duration_seconds_bucket').length,
|
||||
15,
|
||||
);
|
||||
const count = duration.filter(metric => metric.metricName === 'utapi_http_request_duration_seconds_count');
|
||||
assert.deepStrictEqual(count, [{
|
||||
labels: {
|
||||
action: 'listMetrics',
|
||||
code: 200,
|
||||
},
|
||||
metricName: 'utapi_http_request_duration_seconds_count',
|
||||
value: 1,
|
||||
}]);
|
||||
assert.strictEqual(count[0].value, 1);
|
||||
});
|
||||
|
||||
it('should not increment the counter if an internal route', async () => {
|
||||
const req = templateRequest();
|
||||
req.ctx = new RequestContext(req);
|
||||
middleware.httpMetricsMiddleware(req, resp);
|
||||
assert.rejects(() => getMetricValues('utapi_http_requests_total'));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -115,16 +115,7 @@
|
|||
%> FOREACH
|
||||
%>
|
||||
<%
|
||||
// If no new events were found
|
||||
// - drop the empty list
|
||||
// - write a new master checkpoint
|
||||
// - return 0
|
||||
DROP
|
||||
NEWGTS $master_checkpoint_class RENAME
|
||||
$endTimestamp NaN NaN NaN 0 ADDVALUE
|
||||
{ 'node' $nodeId } RELABEL
|
||||
$write_token UPDATE
|
||||
0 STOP
|
||||
DROP 0 STOP
|
||||
%> IFTE
|
||||
|
||||
0 'checkpoints' STORE
|
||||
|
|
|
|||
Loading…
Reference in New Issue