Compare commits
6 Commits
developmen
...
feature/AR
Author | SHA1 | Date |
---|---|---|
Nicolas Humbert | 4c3098c13e | |
Ronnie Smith | 7f29e03e4a | |
Ronnie Smith | 3c6742b607 | |
Ronnie Smith | 1f1fd401c1 | |
Ronnie Smith | dab7f54dc5 | |
Ronnie Smith | d6fcbdb4f6 |
|
@ -1,6 +1 @@
|
|||
{
|
||||
"extends": "scality",
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 2020
|
||||
}
|
||||
}
|
||||
{ "extends": "scality" }
|
||||
|
|
|
@ -1,25 +0,0 @@
|
|||
---
|
||||
name: codeQL
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [development/*, stabilization/*, hotfix/*]
|
||||
pull_request:
|
||||
branches: [development/*, stabilization/*, hotfix/*]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Static analysis with CodeQL
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v3
|
||||
with:
|
||||
languages: javascript, typescript
|
||||
|
||||
- name: Build and analyze
|
||||
uses: github/codeql-action/analyze@v3
|
|
@ -1,16 +0,0 @@
|
|||
---
|
||||
name: dependency review
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [development/*, stabilization/*, hotfix/*]
|
||||
|
||||
jobs:
|
||||
dependency-review:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@v4
|
|
@ -25,30 +25,24 @@ jobs:
|
|||
- 6379:6379
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
uses: actions/checkout@v2
|
||||
- uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: '16'
|
||||
cache: 'yarn'
|
||||
- name: install dependencies
|
||||
run: yarn install --frozen-lockfile --prefer-offline --network-concurrency 1
|
||||
run: yarn install --frozen-lockfile --prefer-offline
|
||||
continue-on-error: true # TODO ARSN-97 Remove it when no errors in TS
|
||||
- name: lint yaml
|
||||
run: yarn --silent lint_yml
|
||||
- name: lint javascript
|
||||
run: yarn --silent lint --max-warnings 0
|
||||
run: yarn --silent lint -- --max-warnings 0
|
||||
- name: lint markdown
|
||||
run: yarn --silent lint_md
|
||||
- name: add hostname
|
||||
run: |
|
||||
sudo sh -c "echo '127.0.0.1 testrequestbucket.localhost' >> /etc/hosts"
|
||||
- name: test and coverage
|
||||
run: yarn --silent coverage
|
||||
- name: run unit tests
|
||||
run: yarn test
|
||||
- name: run functional tests
|
||||
run: yarn ft_test
|
||||
- uses: codecov/codecov-action@v4
|
||||
with:
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
- name: run executables tests
|
||||
run: yarn install && yarn test
|
||||
working-directory: 'lib/executables/pensieveCreds/'
|
||||
|
@ -59,9 +53,9 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
uses: actions/checkout@v2
|
||||
- name: Install NodeJS
|
||||
uses: actions/setup-node@v4
|
||||
uses: actions/setup-node@v2
|
||||
with:
|
||||
node-version: '16'
|
||||
cache: yarn
|
||||
|
@ -72,7 +66,7 @@ jobs:
|
|||
run: yarn build
|
||||
continue-on-error: true # TODO ARSN-97 Remove it when no errors in TS
|
||||
- name: Upload artifacts
|
||||
uses: scality/action-artifacts@v4
|
||||
uses: scality/action-artifacts@v2
|
||||
with:
|
||||
url: https://artifacts.scality.net
|
||||
user: ${{ secrets.ARTIFACTS_USER }}
|
||||
|
|
12
.swcrc
12
.swcrc
|
@ -1,12 +0,0 @@
|
|||
{
|
||||
"$schema": "https://swc.rs/schema.json",
|
||||
"jsc": {
|
||||
"parser": {
|
||||
"syntax": "typescript"
|
||||
},
|
||||
"target": "es2017"
|
||||
},
|
||||
"module": {
|
||||
"type": "commonjs"
|
||||
}
|
||||
}
|
|
@ -1,7 +1,5 @@
|
|||
# Arsenal
|
||||
|
||||
[![codecov](https://codecov.io/gh/scality/Arsenal/branch/development/8.1/graph/badge.svg?token=X0esXhJSwb)](https://codecov.io/gh/scality/Arsenal)
|
||||
|
||||
Common utilities for the S3 project components
|
||||
|
||||
Within this repository, you will be able to find the shared libraries for the
|
||||
|
|
|
@ -85,66 +85,6 @@ Used to store the bucket lifecycle configuration info
|
|||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._uid = uid || uuid();
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to set a unique identifier on a bucket
|
||||
|
||||
## Model version 8
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._readLocationConstraint = readLocationConstraint || null;
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to store default read location of the bucket
|
||||
|
||||
## Model version 9
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._isNFS = isNFS || null;
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to determine whether the bucket may be accessed through NFS
|
||||
|
||||
## Model version 10
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._ingestion = ingestionConfig || null;
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to store the ingestion status of a bucket
|
||||
|
||||
## Model version 11
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._azureInfo = azureInfo || null;
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to store Azure storage account specific information
|
||||
|
||||
## Model version 12
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._objectLockEnabled = objectLockEnabled || false;
|
||||
this._objectLockConfiguration = objectLockConfiguration || null;
|
||||
|
@ -155,7 +95,7 @@ this._objectLockConfiguration = objectLockConfiguration || null;
|
|||
Used to determine whether object lock capabilities are enabled on a bucket and
|
||||
to store the object lock configuration of the bucket
|
||||
|
||||
## Model version 13
|
||||
## Model version 8
|
||||
|
||||
### Properties Added
|
||||
|
||||
|
@ -167,7 +107,7 @@ this._notificationConfiguration = notificationConfiguration || null;
|
|||
|
||||
Used to store the bucket notification configuration info
|
||||
|
||||
## Model version 14
|
||||
## Model version 9
|
||||
|
||||
### Properties Added
|
||||
|
||||
|
@ -179,7 +119,19 @@ this._serverSideEncryption.configuredMasterKeyId = configuredMasterKeyId || unde
|
|||
|
||||
Used to store the users configured KMS key id
|
||||
|
||||
## Model version 15
|
||||
## Model version 10
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._uid = uid || uuid();
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to set a unique identifier on a bucket
|
||||
|
||||
## Model version 11
|
||||
|
||||
### Properties Added
|
||||
|
||||
|
@ -187,74 +139,6 @@ Used to store the users configured KMS key id
|
|||
this._tags = tags || null;
|
||||
```
|
||||
|
||||
The Tag Set of a bucket is an array of objects with Key and Value:
|
||||
|
||||
```javascript
|
||||
[
|
||||
{
|
||||
Key: 'something',
|
||||
Value: 'some_data'
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
## Model version 16
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._capabilities = capabilities || undefined;
|
||||
```
|
||||
|
||||
For capacity-enabled buckets, contains the following data:
|
||||
|
||||
```javascript
|
||||
{
|
||||
_capabilities: {
|
||||
VeeamSOSApi?: {
|
||||
SystemInfo?: {
|
||||
ProtocolVersion: String,
|
||||
ModelName: String,
|
||||
ProtocolCapabilities: {
|
||||
CapacityInfo: Boolean,
|
||||
UploadSessions: Boolean,
|
||||
IAMSTS: Boolean,
|
||||
},
|
||||
APIEndpoints: {
|
||||
IAMEndpoint: String,
|
||||
STSEndpoint: String,
|
||||
},
|
||||
SystemRecommendations?: {
|
||||
S3ConcurrentTaskLimit: Number,
|
||||
S3MultiObjectDelete: Number,
|
||||
StorageCurrentTasksLimit: Number,
|
||||
KbBlockSize: Number,
|
||||
}
|
||||
LastModified?: String,
|
||||
},
|
||||
CapacityInfo?: {
|
||||
Capacity: Number,
|
||||
Available: Number,
|
||||
Used: Number,
|
||||
LastModified?: String,
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to store bucket tagging
|
||||
|
||||
## Model version 17
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._quotaMax = quotaMax || 0;
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to store bucket quota
|
|
@ -1,27 +0,0 @@
|
|||
# Delimiter
|
||||
|
||||
The Delimiter class handles raw listings from the database with an
|
||||
optional delimiter, and fills in a curated listing with "Contents" and
|
||||
"CommonPrefixes" as a result.
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
- only lists keys belonging to the given **prefix** (if provided)
|
||||
|
||||
- groups listed keys that have a common prefix ending with a delimiter
|
||||
inside CommonPrefixes
|
||||
|
||||
- can take a **marker** or **continuationToken** to list from a specific key
|
||||
|
||||
- can take a **maxKeys** parameter to limit how many keys can be returned
|
||||
|
||||
## State Chart
|
||||
|
||||
- States with grey background are *Idle* states, which are waiting for
|
||||
a new listing key
|
||||
|
||||
- States with blue background are *Processing* states, which are
|
||||
actively processing a new listing key passed by the filter()
|
||||
function
|
||||
|
||||
![Delimiter State Chart](./pics/delimiterStateChart.svg)
|
|
@ -1,45 +0,0 @@
|
|||
# DelimiterMaster
|
||||
|
||||
The DelimiterMaster class handles raw listings from the database of a
|
||||
versioned or non-versioned bucket with an optional delimiter, and
|
||||
fills in a curated listing with "Contents" and "CommonPrefixes" as a
|
||||
result.
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
- only lists latest versions of versioned buckets
|
||||
|
||||
- only lists keys belonging to the given **prefix** (if provided)
|
||||
|
||||
- does not list latest versions that are delete markers
|
||||
|
||||
- groups listed keys that have a common prefix ending with a delimiter
|
||||
inside CommonPrefixes
|
||||
|
||||
- can take a **marker** or **continuationToken** to list from a specific key
|
||||
|
||||
- can take a **maxKeys** parameter to limit how many keys can be returned
|
||||
|
||||
- reconciles internal PHD keys with the next version (those are
|
||||
created when a specific version that is the latest version is
|
||||
deleted)
|
||||
|
||||
- skips internal keys like replay keys
|
||||
|
||||
## State Chart
|
||||
|
||||
- States with grey background are *Idle* states, which are waiting for
|
||||
a new listing key
|
||||
|
||||
- States with blue background are *Processing* states, which are
|
||||
actively processing a new listing key passed by the filter()
|
||||
function
|
||||
|
||||
### Bucket Vformat=v0
|
||||
|
||||
![DelimiterMaster State Chart for v0 format](./pics/delimiterMasterV0StateChart.svg)
|
||||
|
||||
### Bucket Vformat=v1
|
||||
|
||||
For buckets in versioning key format **v1**, the algorithm used is the
|
||||
one from [Delimiter](delimiter.md).
|
|
@ -1,33 +0,0 @@
|
|||
# DelimiterVersions
|
||||
|
||||
The DelimiterVersions class handles raw listings from the database of a
|
||||
versioned or non-versioned bucket with an optional delimiter, and
|
||||
fills in a curated listing with "Versions" and "CommonPrefixes" as a
|
||||
result.
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
- lists individual distinct versions of versioned buckets
|
||||
|
||||
- only lists keys belonging to the given **prefix** (if provided)
|
||||
|
||||
- groups listed keys that have a common prefix ending with a delimiter
|
||||
inside CommonPrefixes
|
||||
|
||||
- can take a **keyMarker** and optionally a **versionIdMarker** to
|
||||
list from a specific key or version
|
||||
|
||||
- can take a **maxKeys** parameter to limit how many keys can be returned
|
||||
|
||||
- skips internal keys like replay keys
|
||||
|
||||
## State Chart
|
||||
|
||||
- States with grey background are *Idle* states, which are waiting for
|
||||
a new listing key
|
||||
|
||||
- States with blue background are *Processing* states, which are
|
||||
actively processing a new listing key passed by the filter()
|
||||
function
|
||||
|
||||
![DelimiterVersions State Chart](./pics/delimiterVersionsStateChart.svg)
|
|
@ -1,45 +0,0 @@
|
|||
digraph {
|
||||
node [shape="box",style="filled,rounded",fontsize=16,fixedsize=true,width=3];
|
||||
edge [fontsize=14];
|
||||
rankdir=TB;
|
||||
|
||||
START [shape="circle",width=0.2,label="",style="filled",fillcolor="black"]
|
||||
END [shape="circle",width=0.2,label="",style="filled",fillcolor="black",peripheries=2]
|
||||
|
||||
node [fillcolor="lightgrey"];
|
||||
"NotSkippingPrefixNorVersions.Idle" [label="NotSkippingPrefixNorVersions",group="NotSkippingPrefixNorVersions",width=4];
|
||||
"SkippingPrefix.Idle" [label="SkippingPrefix",group="SkippingPrefix"];
|
||||
"SkippingVersions.Idle" [label="SkippingVersions",group="SkippingVersions"];
|
||||
"WaitVersionAfterPHD.Idle" [label="WaitVersionAfterPHD",group="WaitVersionAfterPHD"];
|
||||
|
||||
node [fillcolor="lightblue"];
|
||||
"NotSkippingPrefixNorVersions.Processing" [label="NotSkippingPrefixNorVersions",group="NotSkippingPrefixNorVersions",width=4];
|
||||
"SkippingPrefix.Processing" [label="SkippingPrefix",group="SkippingPrefix"];
|
||||
"SkippingVersions.Processing" [label="SkippingVersions",group="SkippingVersions"];
|
||||
"WaitVersionAfterPHD.Processing" [label="WaitVersionAfterPHD",group="WaitVersionAfterPHD"];
|
||||
|
||||
START -> "SkippingVersions.Idle" [label="[marker != undefined]"]
|
||||
START -> "NotSkippingPrefixNorVersions.Idle" [label="[marker == undefined]"]
|
||||
|
||||
"NotSkippingPrefixNorVersions.Idle" -> "NotSkippingPrefixNorVersions.Processing" [label="filter(key, value)"]
|
||||
"SkippingPrefix.Idle" -> "SkippingPrefix.Processing" [label="filter(key, value)"]
|
||||
"SkippingVersions.Idle" -> "SkippingVersions.Processing" [label="filter(key, value)"]
|
||||
"WaitVersionAfterPHD.Idle" -> "WaitVersionAfterPHD.Processing" [label="filter(key, value)"]
|
||||
|
||||
|
||||
"NotSkippingPrefixNorVersions.Processing" -> "SkippingVersions.Idle" [label="[Version.isDeleteMarker(value)]\n-> FILTER_ACCEPT"]
|
||||
"NotSkippingPrefixNorVersions.Processing" -> "WaitVersionAfterPHD.Idle" [label="[Version.isPHD(value)]\n-> FILTER_ACCEPT"]
|
||||
"NotSkippingPrefixNorVersions.Processing" -> "SkippingPrefix.Idle" [label="[key.startsWith(<ReplayPrefix>)]\n/ prefix <- <ReplayPrefix>\n-> FILTER_SKIP"]
|
||||
"NotSkippingPrefixNorVersions.Processing" -> END [label="[isListableKey(key, value) and\nKeys == maxKeys]\n-> FILTER_END"]
|
||||
"NotSkippingPrefixNorVersions.Processing" -> "SkippingPrefix.Idle" [label="[isListableKey(key, value) and\nnKeys < maxKeys and\nhasDelimiter(key)]\n/ prefix <- prefixOf(key)\n/ CommonPrefixes.append(prefixOf(key))\n-> FILTER_ACCEPT"]
|
||||
"NotSkippingPrefixNorVersions.Processing" -> "SkippingVersions.Idle" [label="[isListableKey(key, value) and\nnKeys < maxKeys and\nnot hasDelimiter(key)]\n/ Contents.append(key, value)\n-> FILTER_ACCEPT"]
|
||||
|
||||
"SkippingPrefix.Processing" -> "SkippingPrefix.Idle" [label="[key.startsWith(prefix)]\n-> FILTER_SKIP"]
|
||||
"SkippingPrefix.Processing" -> "NotSkippingPrefixNorVersions.Processing" [label="[not key.startsWith(prefix)]"]
|
||||
|
||||
"SkippingVersions.Processing" -> "SkippingVersions.Idle" [label="[isVersionKey(key)]\n-> FILTER_SKIP"]
|
||||
"SkippingVersions.Processing" -> "NotSkippingPrefixNorVersions.Processing" [label="[not isVersionKey(key)]"]
|
||||
|
||||
"WaitVersionAfterPHD.Processing" -> "NotSkippingPrefixNorVersions.Processing" [label="[isVersionKey(key) and master(key) == PHDkey]\n/ key <- master(key)"]
|
||||
"WaitVersionAfterPHD.Processing" -> "NotSkippingPrefixNorVersions.Processing" [label="[not isVersionKey(key) or master(key) != PHDkey]"]
|
||||
}
|
|
@ -1,216 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.43.0 (0)
|
||||
-->
|
||||
<!-- Title: %3 Pages: 1 -->
|
||||
<svg width="2313pt" height="460pt"
|
||||
viewBox="0.00 0.00 2313.37 460.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 456)">
|
||||
<title>%3</title>
|
||||
<polygon fill="white" stroke="transparent" points="-4,4 -4,-456 2309.37,-456 2309.37,4 -4,4"/>
|
||||
<!-- START -->
|
||||
<g id="node1" class="node">
|
||||
<title>START</title>
|
||||
<ellipse fill="black" stroke="black" cx="35.37" cy="-445" rx="7" ry="7"/>
|
||||
</g>
|
||||
<!-- NotSkippingPrefixNorVersions.Idle -->
|
||||
<g id="node3" class="node">
|
||||
<title>NotSkippingPrefixNorVersions.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M925.37,-387C925.37,-387 661.37,-387 661.37,-387 655.37,-387 649.37,-381 649.37,-375 649.37,-375 649.37,-363 649.37,-363 649.37,-357 655.37,-351 661.37,-351 661.37,-351 925.37,-351 925.37,-351 931.37,-351 937.37,-357 937.37,-363 937.37,-363 937.37,-375 937.37,-375 937.37,-381 931.37,-387 925.37,-387"/>
|
||||
<text text-anchor="middle" x="793.37" y="-365.2" font-family="Times,serif" font-size="16.00">NotSkippingPrefixNorVersions</text>
|
||||
</g>
|
||||
<!-- START->NotSkippingPrefixNorVersions.Idle -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>START->NotSkippingPrefixNorVersions.Idle</title>
|
||||
<path fill="none" stroke="black" d="M42.39,-443.31C95.3,-438.15 434.98,-404.99 638.94,-385.08"/>
|
||||
<polygon fill="black" stroke="black" points="639.54,-388.53 649.15,-384.08 638.86,-381.57 639.54,-388.53"/>
|
||||
<text text-anchor="middle" x="497.87" y="-408.8" font-family="Times,serif" font-size="14.00">[marker == undefined]</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Idle -->
|
||||
<g id="node5" class="node">
|
||||
<title>SkippingVersions.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M242.37,-138C242.37,-138 50.37,-138 50.37,-138 44.37,-138 38.37,-132 38.37,-126 38.37,-126 38.37,-114 38.37,-114 38.37,-108 44.37,-102 50.37,-102 50.37,-102 242.37,-102 242.37,-102 248.37,-102 254.37,-108 254.37,-114 254.37,-114 254.37,-126 254.37,-126 254.37,-132 248.37,-138 242.37,-138"/>
|
||||
<text text-anchor="middle" x="146.37" y="-116.2" font-family="Times,serif" font-size="16.00">SkippingVersions</text>
|
||||
</g>
|
||||
<!-- START->SkippingVersions.Idle -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>START->SkippingVersions.Idle</title>
|
||||
<path fill="none" stroke="black" d="M33.04,-438.14C20.64,-405.9 -34.57,-248.17 33.37,-156 36.76,-151.4 40.74,-147.39 45.16,-143.89"/>
|
||||
<polygon fill="black" stroke="black" points="47.27,-146.68 53.53,-138.13 43.3,-140.92 47.27,-146.68"/>
|
||||
<text text-anchor="middle" x="85.87" y="-321.8" font-family="Times,serif" font-size="14.00">[marker != undefined]</text>
|
||||
</g>
|
||||
<!-- END -->
|
||||
<g id="node2" class="node">
|
||||
<title>END</title>
|
||||
<ellipse fill="black" stroke="black" cx="727.37" cy="-120" rx="7" ry="7"/>
|
||||
<ellipse fill="none" stroke="black" cx="727.37" cy="-120" rx="11" ry="11"/>
|
||||
</g>
|
||||
<!-- NotSkippingPrefixNorVersions.Processing -->
|
||||
<g id="node7" class="node">
|
||||
<title>NotSkippingPrefixNorVersions.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M925.37,-300C925.37,-300 661.37,-300 661.37,-300 655.37,-300 649.37,-294 649.37,-288 649.37,-288 649.37,-276 649.37,-276 649.37,-270 655.37,-264 661.37,-264 661.37,-264 925.37,-264 925.37,-264 931.37,-264 937.37,-270 937.37,-276 937.37,-276 937.37,-288 937.37,-288 937.37,-294 931.37,-300 925.37,-300"/>
|
||||
<text text-anchor="middle" x="793.37" y="-278.2" font-family="Times,serif" font-size="16.00">NotSkippingPrefixNorVersions</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefixNorVersions.Idle->NotSkippingPrefixNorVersions.Processing -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>NotSkippingPrefixNorVersions.Idle->NotSkippingPrefixNorVersions.Processing</title>
|
||||
<path fill="none" stroke="black" d="M793.37,-350.8C793.37,-339.16 793.37,-323.55 793.37,-310.24"/>
|
||||
<polygon fill="black" stroke="black" points="796.87,-310.18 793.37,-300.18 789.87,-310.18 796.87,-310.18"/>
|
||||
<text text-anchor="middle" x="851.37" y="-321.8" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Idle -->
|
||||
<g id="node4" class="node">
|
||||
<title>SkippingPrefix.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M1209.37,-138C1209.37,-138 1017.37,-138 1017.37,-138 1011.37,-138 1005.37,-132 1005.37,-126 1005.37,-126 1005.37,-114 1005.37,-114 1005.37,-108 1011.37,-102 1017.37,-102 1017.37,-102 1209.37,-102 1209.37,-102 1215.37,-102 1221.37,-108 1221.37,-114 1221.37,-114 1221.37,-126 1221.37,-126 1221.37,-132 1215.37,-138 1209.37,-138"/>
|
||||
<text text-anchor="middle" x="1113.37" y="-116.2" font-family="Times,serif" font-size="16.00">SkippingPrefix</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Processing -->
|
||||
<g id="node8" class="node">
|
||||
<title>SkippingPrefix.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M1070.37,-36C1070.37,-36 878.37,-36 878.37,-36 872.37,-36 866.37,-30 866.37,-24 866.37,-24 866.37,-12 866.37,-12 866.37,-6 872.37,0 878.37,0 878.37,0 1070.37,0 1070.37,0 1076.37,0 1082.37,-6 1082.37,-12 1082.37,-12 1082.37,-24 1082.37,-24 1082.37,-30 1076.37,-36 1070.37,-36"/>
|
||||
<text text-anchor="middle" x="974.37" y="-14.2" font-family="Times,serif" font-size="16.00">SkippingPrefix</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Idle->SkippingPrefix.Processing -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>SkippingPrefix.Idle->SkippingPrefix.Processing</title>
|
||||
<path fill="none" stroke="black" d="M1011.89,-101.96C994.96,-97.13 981.04,-91.17 975.37,-84 967.11,-73.56 966.25,-58.93 967.72,-46.2"/>
|
||||
<polygon fill="black" stroke="black" points="971.22,-46.52 969.4,-36.09 964.31,-45.38 971.22,-46.52"/>
|
||||
<text text-anchor="middle" x="1033.37" y="-65.3" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Processing -->
|
||||
<g id="node9" class="node">
|
||||
<title>SkippingVersions.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M381.37,-36C381.37,-36 189.37,-36 189.37,-36 183.37,-36 177.37,-30 177.37,-24 177.37,-24 177.37,-12 177.37,-12 177.37,-6 183.37,0 189.37,0 189.37,0 381.37,0 381.37,0 387.37,0 393.37,-6 393.37,-12 393.37,-12 393.37,-24 393.37,-24 393.37,-30 387.37,-36 381.37,-36"/>
|
||||
<text text-anchor="middle" x="285.37" y="-14.2" font-family="Times,serif" font-size="16.00">SkippingVersions</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Idle->SkippingVersions.Processing -->
|
||||
<g id="edge5" class="edge">
|
||||
<title>SkippingVersions.Idle->SkippingVersions.Processing</title>
|
||||
<path fill="none" stroke="black" d="M141.4,-101.91C138.35,-87.58 136.8,-67.37 147.37,-54 151.89,-48.28 161.64,-43.34 173.99,-39.12"/>
|
||||
<polygon fill="black" stroke="black" points="175.39,-42.36 183.89,-36.04 173.3,-35.67 175.39,-42.36"/>
|
||||
<text text-anchor="middle" x="205.37" y="-65.3" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- WaitVersionAfterPHD.Idle -->
|
||||
<g id="node6" class="node">
|
||||
<title>WaitVersionAfterPHD.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M1534.37,-138C1534.37,-138 1342.37,-138 1342.37,-138 1336.37,-138 1330.37,-132 1330.37,-126 1330.37,-126 1330.37,-114 1330.37,-114 1330.37,-108 1336.37,-102 1342.37,-102 1342.37,-102 1534.37,-102 1534.37,-102 1540.37,-102 1546.37,-108 1546.37,-114 1546.37,-114 1546.37,-126 1546.37,-126 1546.37,-132 1540.37,-138 1534.37,-138"/>
|
||||
<text text-anchor="middle" x="1438.37" y="-116.2" font-family="Times,serif" font-size="16.00">WaitVersionAfterPHD</text>
|
||||
</g>
|
||||
<!-- WaitVersionAfterPHD.Processing -->
|
||||
<g id="node10" class="node">
|
||||
<title>WaitVersionAfterPHD.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M1534.37,-36C1534.37,-36 1342.37,-36 1342.37,-36 1336.37,-36 1330.37,-30 1330.37,-24 1330.37,-24 1330.37,-12 1330.37,-12 1330.37,-6 1336.37,0 1342.37,0 1342.37,0 1534.37,0 1534.37,0 1540.37,0 1546.37,-6 1546.37,-12 1546.37,-12 1546.37,-24 1546.37,-24 1546.37,-30 1540.37,-36 1534.37,-36"/>
|
||||
<text text-anchor="middle" x="1438.37" y="-14.2" font-family="Times,serif" font-size="16.00">WaitVersionAfterPHD</text>
|
||||
</g>
|
||||
<!-- WaitVersionAfterPHD.Idle->WaitVersionAfterPHD.Processing -->
|
||||
<g id="edge6" class="edge">
|
||||
<title>WaitVersionAfterPHD.Idle->WaitVersionAfterPHD.Processing</title>
|
||||
<path fill="none" stroke="black" d="M1438.37,-101.58C1438.37,-86.38 1438.37,-64.07 1438.37,-46.46"/>
|
||||
<polygon fill="black" stroke="black" points="1441.87,-46.22 1438.37,-36.22 1434.87,-46.22 1441.87,-46.22"/>
|
||||
<text text-anchor="middle" x="1496.37" y="-65.3" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefixNorVersions.Processing->END -->
|
||||
<g id="edge10" class="edge">
|
||||
<title>NotSkippingPrefixNorVersions.Processing->END</title>
|
||||
<path fill="none" stroke="black" d="M649.15,-273.62C611.7,-268.54 578.44,-260.07 566.37,-246 540.33,-215.64 540,-186.08 566.37,-156 586.46,-133.07 673.88,-148.86 702.37,-138 705.22,-136.91 708.06,-135.44 710.76,-133.82"/>
|
||||
<polygon fill="black" stroke="black" points="712.88,-136.61 719.13,-128.05 708.91,-130.84 712.88,-136.61"/>
|
||||
<text text-anchor="middle" x="672.87" y="-212.3" font-family="Times,serif" font-size="14.00">[isListableKey(key, value) and</text>
|
||||
<text text-anchor="middle" x="672.87" y="-197.3" font-family="Times,serif" font-size="14.00">Keys == maxKeys]</text>
|
||||
<text text-anchor="middle" x="672.87" y="-182.3" font-family="Times,serif" font-size="14.00">-> FILTER_END</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefixNorVersions.Processing->SkippingPrefix.Idle -->
|
||||
<g id="edge9" class="edge">
|
||||
<title>NotSkippingPrefixNorVersions.Processing->SkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M937.6,-274.31C1018.89,-269.01 1106.69,-260.11 1119.37,-246 1143.16,-219.51 1134.03,-175.72 1124.38,-147.62"/>
|
||||
<polygon fill="black" stroke="black" points="1127.6,-146.22 1120.86,-138.04 1121.03,-148.64 1127.6,-146.22"/>
|
||||
<text text-anchor="middle" x="1254.37" y="-212.3" font-family="Times,serif" font-size="14.00">[key.startsWith(<ReplayPrefix>)]</text>
|
||||
<text text-anchor="middle" x="1254.37" y="-197.3" font-family="Times,serif" font-size="14.00">/ prefix <- <ReplayPrefix></text>
|
||||
<text text-anchor="middle" x="1254.37" y="-182.3" font-family="Times,serif" font-size="14.00">-> FILTER_SKIP</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefixNorVersions.Processing->SkippingPrefix.Idle -->
|
||||
<g id="edge11" class="edge">
|
||||
<title>NotSkippingPrefixNorVersions.Processing->SkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M799.18,-263.65C800.96,-258.05 802.85,-251.79 804.37,-246 814.73,-206.45 793.03,-183.41 823.37,-156 851.23,-130.83 954.1,-142.59 991.37,-138 992.65,-137.84 993.94,-137.68 995.24,-137.52"/>
|
||||
<polygon fill="black" stroke="black" points="995.81,-140.98 1005.29,-136.25 994.93,-134.03 995.81,-140.98"/>
|
||||
<text text-anchor="middle" x="969.37" y="-234.8" font-family="Times,serif" font-size="14.00">[isListableKey(key, value) and</text>
|
||||
<text text-anchor="middle" x="969.37" y="-219.8" font-family="Times,serif" font-size="14.00">nKeys < maxKeys and</text>
|
||||
<text text-anchor="middle" x="969.37" y="-204.8" font-family="Times,serif" font-size="14.00">hasDelimiter(key)]</text>
|
||||
<text text-anchor="middle" x="969.37" y="-189.8" font-family="Times,serif" font-size="14.00">/ prefix <- prefixOf(key)</text>
|
||||
<text text-anchor="middle" x="969.37" y="-174.8" font-family="Times,serif" font-size="14.00">/ CommonPrefixes.append(prefixOf(key))</text>
|
||||
<text text-anchor="middle" x="969.37" y="-159.8" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefixNorVersions.Processing->SkippingVersions.Idle -->
|
||||
<g id="edge7" class="edge">
|
||||
<title>NotSkippingPrefixNorVersions.Processing->SkippingVersions.Idle</title>
|
||||
<path fill="none" stroke="black" d="M649.11,-279.23C439.56,-275.94 73.58,-267.19 53.37,-246 25.76,-217.06 30.6,-188.89 53.37,-156 56.56,-151.39 60.44,-147.39 64.78,-143.91"/>
|
||||
<polygon fill="black" stroke="black" points="66.8,-146.76 73.04,-138.2 62.83,-141 66.8,-146.76"/>
|
||||
<text text-anchor="middle" x="167.87" y="-204.8" font-family="Times,serif" font-size="14.00">[Version.isDeleteMarker(value)]</text>
|
||||
<text text-anchor="middle" x="167.87" y="-189.8" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefixNorVersions.Processing->SkippingVersions.Idle -->
|
||||
<g id="edge12" class="edge">
|
||||
<title>NotSkippingPrefixNorVersions.Processing->SkippingVersions.Idle</title>
|
||||
<path fill="none" stroke="black" d="M649.33,-279.1C514.97,-275.99 331.4,-267.75 305.37,-246 273.69,-219.53 311.53,-185.22 282.37,-156 276.73,-150.36 270.32,-145.59 263.42,-141.56"/>
|
||||
<polygon fill="black" stroke="black" points="264.92,-138.39 254.44,-136.84 261.67,-144.59 264.92,-138.39"/>
|
||||
<text text-anchor="middle" x="411.87" y="-227.3" font-family="Times,serif" font-size="14.00">[isListableKey(key, value) and</text>
|
||||
<text text-anchor="middle" x="411.87" y="-212.3" font-family="Times,serif" font-size="14.00">nKeys < maxKeys and</text>
|
||||
<text text-anchor="middle" x="411.87" y="-197.3" font-family="Times,serif" font-size="14.00">not hasDelimiter(key)]</text>
|
||||
<text text-anchor="middle" x="411.87" y="-182.3" font-family="Times,serif" font-size="14.00">/ Contents.append(key, value)</text>
|
||||
<text text-anchor="middle" x="411.87" y="-167.3" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefixNorVersions.Processing->WaitVersionAfterPHD.Idle -->
|
||||
<g id="edge8" class="edge">
|
||||
<title>NotSkippingPrefixNorVersions.Processing->WaitVersionAfterPHD.Idle</title>
|
||||
<path fill="none" stroke="black" d="M937.38,-280.87C1099.43,-279.42 1344.59,-272.74 1378.37,-246 1411.11,-220.08 1384.48,-192.16 1405.37,-156 1407.38,-152.52 1409.8,-149.11 1412.4,-145.87"/>
|
||||
<polygon fill="black" stroke="black" points="1415.16,-148.04 1419.13,-138.21 1409.9,-143.41 1415.16,-148.04"/>
|
||||
<text text-anchor="middle" x="1486.87" y="-204.8" font-family="Times,serif" font-size="14.00">[Version.isPHD(value)]</text>
|
||||
<text text-anchor="middle" x="1486.87" y="-189.8" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Processing->SkippingPrefix.Idle -->
|
||||
<g id="edge13" class="edge">
|
||||
<title>SkippingPrefix.Processing->SkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M1064.61,-36.08C1074.44,-40.7 1083.66,-46.57 1091.37,-54 1101.65,-63.92 1107.13,-78.81 1110.04,-91.84"/>
|
||||
<polygon fill="black" stroke="black" points="1106.62,-92.56 1111.88,-101.76 1113.5,-91.29 1106.62,-92.56"/>
|
||||
<text text-anchor="middle" x="1190.37" y="-72.8" font-family="Times,serif" font-size="14.00">[key.startsWith(prefix)]</text>
|
||||
<text text-anchor="middle" x="1190.37" y="-57.8" font-family="Times,serif" font-size="14.00">-> FILTER_SKIP</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Processing->NotSkippingPrefixNorVersions.Processing -->
|
||||
<g id="edge14" class="edge">
|
||||
<title>SkippingPrefix.Processing->NotSkippingPrefixNorVersions.Processing</title>
|
||||
<path fill="none" stroke="black" d="M899.82,-36.01C864.18,-48.2 824.54,-68.57 802.37,-102 771.84,-148.02 779.31,-216.26 786.77,-253.8"/>
|
||||
<polygon fill="black" stroke="black" points="783.43,-254.92 788.94,-263.97 790.28,-253.46 783.43,-254.92"/>
|
||||
<text text-anchor="middle" x="899.37" y="-116.3" font-family="Times,serif" font-size="14.00">[not key.startsWith(prefix)]</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Processing->SkippingVersions.Idle -->
|
||||
<g id="edge15" class="edge">
|
||||
<title>SkippingVersions.Processing->SkippingVersions.Idle</title>
|
||||
<path fill="none" stroke="black" d="M283.88,-36.24C281.71,-50.87 276.4,-71.43 263.37,-84 258.07,-89.11 252.06,-93.48 245.62,-97.21"/>
|
||||
<polygon fill="black" stroke="black" points="243.85,-94.19 236.61,-101.92 247.09,-100.39 243.85,-94.19"/>
|
||||
<text text-anchor="middle" x="349.87" y="-72.8" font-family="Times,serif" font-size="14.00">[isVersionKey(key)]</text>
|
||||
<text text-anchor="middle" x="349.87" y="-57.8" font-family="Times,serif" font-size="14.00">-> FILTER_SKIP</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Processing->NotSkippingPrefixNorVersions.Processing -->
|
||||
<g id="edge16" class="edge">
|
||||
<title>SkippingVersions.Processing->NotSkippingPrefixNorVersions.Processing</title>
|
||||
<path fill="none" stroke="black" d="M382.46,-36.08C396.72,-40.7 410.82,-46.57 423.37,-54 476.67,-85.57 487.28,-102.42 518.37,-156 539.39,-192.23 514.46,-218.85 546.37,-246 561.72,-259.06 598.56,-267.25 639.23,-272.39"/>
|
||||
<polygon fill="black" stroke="black" points="639.01,-275.89 649.36,-273.59 639.84,-268.93 639.01,-275.89"/>
|
||||
<text text-anchor="middle" x="590.37" y="-116.3" font-family="Times,serif" font-size="14.00">[not isVersionKey(key)]</text>
|
||||
</g>
|
||||
<!-- WaitVersionAfterPHD.Processing->NotSkippingPrefixNorVersions.Processing -->
|
||||
<g id="edge17" class="edge">
|
||||
<title>WaitVersionAfterPHD.Processing->NotSkippingPrefixNorVersions.Processing</title>
|
||||
<path fill="none" stroke="black" d="M1536.41,-36.13C1544.73,-40.79 1552.27,-46.65 1558.37,-54 1585.64,-86.89 1597.89,-215.12 1568.37,-246 1547.29,-268.05 1167.71,-276.42 947.74,-279.43"/>
|
||||
<polygon fill="black" stroke="black" points="947.67,-275.93 937.71,-279.57 947.76,-282.93 947.67,-275.93"/>
|
||||
<text text-anchor="middle" x="1758.37" y="-123.8" font-family="Times,serif" font-size="14.00">[isVersionKey(key) and master(key) == PHDkey]</text>
|
||||
<text text-anchor="middle" x="1758.37" y="-108.8" font-family="Times,serif" font-size="14.00">/ key <- master(key)</text>
|
||||
</g>
|
||||
<!-- WaitVersionAfterPHD.Processing->NotSkippingPrefixNorVersions.Processing -->
|
||||
<g id="edge18" class="edge">
|
||||
<title>WaitVersionAfterPHD.Processing->NotSkippingPrefixNorVersions.Processing</title>
|
||||
<path fill="none" stroke="black" d="M1546.51,-21.25C1677.94,-26.54 1888.29,-44.09 1937.37,-102 1947.71,-114.21 1946.85,-125.11 1937.37,-138 1841.62,-268.08 1749.48,-218.23 1590.37,-246 1471.26,-266.79 1143.92,-275.5 947.77,-278.94"/>
|
||||
<polygon fill="black" stroke="black" points="947.6,-275.44 937.66,-279.11 947.72,-282.44 947.6,-275.44"/>
|
||||
<text text-anchor="middle" x="2124.87" y="-116.3" font-family="Times,serif" font-size="14.00">[not isVersionKey(key) or master(key) != PHDkey]</text>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
Before Width: | Height: | Size: 18 KiB |
|
@ -1,35 +0,0 @@
|
|||
digraph {
|
||||
node [shape="box",style="filled,rounded",fontsize=16,fixedsize=true,width=3];
|
||||
edge [fontsize=14];
|
||||
rankdir=TB;
|
||||
|
||||
START [shape="circle",width=0.2,label="",style="filled",fillcolor="black"]
|
||||
END [shape="circle",width=0.2,label="",style="filled",fillcolor="black",peripheries=2]
|
||||
|
||||
node [fillcolor="lightgrey"];
|
||||
"NotSkipping.Idle" [label="NotSkipping",group="NotSkipping"];
|
||||
"NeverSkipping.Idle" [label="NeverSkipping",group="NeverSkipping"];
|
||||
"NotSkippingPrefix.Idle" [label="NotSkippingPrefix",group="NotSkippingPrefix"];
|
||||
"SkippingPrefix.Idle" [label="SkippingPrefix",group="SkippingPrefix"];
|
||||
|
||||
node [fillcolor="lightblue"];
|
||||
"NeverSkipping.Processing" [label="NeverSkipping",group="NeverSkipping"];
|
||||
"NotSkippingPrefix.Processing" [label="NotSkippingPrefix",group="NotSkippingPrefix"];
|
||||
"SkippingPrefix.Processing" [label="SkippingPrefix",group="SkippingPrefix"];
|
||||
|
||||
START -> "NotSkipping.Idle"
|
||||
"NotSkipping.Idle" -> "NeverSkipping.Idle" [label="[delimiter == undefined]"]
|
||||
"NotSkipping.Idle" -> "NotSkippingPrefix.Idle" [label="[delimiter == '/']"]
|
||||
|
||||
"NeverSkipping.Idle" -> "NeverSkipping.Processing" [label="filter(key, value)"]
|
||||
"NotSkippingPrefix.Idle" -> "NotSkippingPrefix.Processing" [label="filter(key, value)"]
|
||||
"SkippingPrefix.Idle" -> "SkippingPrefix.Processing" [label="filter(key, value)"]
|
||||
|
||||
"NeverSkipping.Processing" -> END [label="[nKeys == maxKeys]\n-> FILTER_END"]
|
||||
"NeverSkipping.Processing" -> "NeverSkipping.Idle" [label="[nKeys < maxKeys]\n/ Contents.append(key, value)\n -> FILTER_ACCEPT"]
|
||||
"NotSkippingPrefix.Processing" -> END [label="[nKeys == maxKeys]\n -> FILTER_END"]
|
||||
"NotSkippingPrefix.Processing" -> "SkippingPrefix.Idle" [label="[nKeys < maxKeys and hasDelimiter(key)]\n/ prefix <- prefixOf(key)\n/ CommonPrefixes.append(prefixOf(key))\n-> FILTER_ACCEPT"]
|
||||
"NotSkippingPrefix.Processing" -> "NotSkippingPrefix.Idle" [label="[nKeys < maxKeys and not hasDelimiter(key)]\n/ Contents.append(key, value)\n -> FILTER_ACCEPT"]
|
||||
"SkippingPrefix.Processing" -> "SkippingPrefix.Idle" [label="[key.startsWith(prefix)]\n-> FILTER_SKIP"]
|
||||
"SkippingPrefix.Processing" -> "NotSkippingPrefix.Processing" [label="[not key.startsWith(prefix)]"]
|
||||
}
|
|
@ -1,166 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.43.0 (0)
|
||||
-->
|
||||
<!-- Title: %3 Pages: 1 -->
|
||||
<svg width="975pt" height="533pt"
|
||||
viewBox="0.00 0.00 975.00 533.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 529)">
|
||||
<title>%3</title>
|
||||
<polygon fill="white" stroke="transparent" points="-4,4 -4,-529 971,-529 971,4 -4,4"/>
|
||||
<!-- START -->
|
||||
<g id="node1" class="node">
|
||||
<title>START</title>
|
||||
<ellipse fill="black" stroke="black" cx="283" cy="-518" rx="7" ry="7"/>
|
||||
</g>
|
||||
<!-- NotSkipping.Idle -->
|
||||
<g id="node3" class="node">
|
||||
<title>NotSkipping.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M379,-474C379,-474 187,-474 187,-474 181,-474 175,-468 175,-462 175,-462 175,-450 175,-450 175,-444 181,-438 187,-438 187,-438 379,-438 379,-438 385,-438 391,-444 391,-450 391,-450 391,-462 391,-462 391,-468 385,-474 379,-474"/>
|
||||
<text text-anchor="middle" x="283" y="-452.2" font-family="Times,serif" font-size="16.00">NotSkipping</text>
|
||||
</g>
|
||||
<!-- START->NotSkipping.Idle -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>START->NotSkipping.Idle</title>
|
||||
<path fill="none" stroke="black" d="M283,-510.58C283,-504.23 283,-494.07 283,-484.3"/>
|
||||
<polygon fill="black" stroke="black" points="286.5,-484.05 283,-474.05 279.5,-484.05 286.5,-484.05"/>
|
||||
</g>
|
||||
<!-- END -->
|
||||
<g id="node2" class="node">
|
||||
<title>END</title>
|
||||
<ellipse fill="black" stroke="black" cx="196" cy="-120" rx="7" ry="7"/>
|
||||
<ellipse fill="none" stroke="black" cx="196" cy="-120" rx="11" ry="11"/>
|
||||
</g>
|
||||
<!-- NeverSkipping.Idle -->
|
||||
<g id="node4" class="node">
|
||||
<title>NeverSkipping.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M262,-387C262,-387 70,-387 70,-387 64,-387 58,-381 58,-375 58,-375 58,-363 58,-363 58,-357 64,-351 70,-351 70,-351 262,-351 262,-351 268,-351 274,-357 274,-363 274,-363 274,-375 274,-375 274,-381 268,-387 262,-387"/>
|
||||
<text text-anchor="middle" x="166" y="-365.2" font-family="Times,serif" font-size="16.00">NeverSkipping</text>
|
||||
</g>
|
||||
<!-- NotSkipping.Idle->NeverSkipping.Idle -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>NotSkipping.Idle->NeverSkipping.Idle</title>
|
||||
<path fill="none" stroke="black" d="M216.5,-437.82C206.51,-433.18 196.91,-427.34 189,-420 182.25,-413.74 177.33,-405.11 173.81,-396.79"/>
|
||||
<polygon fill="black" stroke="black" points="177.05,-395.47 170.3,-387.31 170.49,-397.9 177.05,-395.47"/>
|
||||
<text text-anchor="middle" x="279.5" y="-408.8" font-family="Times,serif" font-size="14.00">[delimiter == undefined]</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefix.Idle -->
|
||||
<g id="node5" class="node">
|
||||
<title>NotSkippingPrefix.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M496,-387C496,-387 304,-387 304,-387 298,-387 292,-381 292,-375 292,-375 292,-363 292,-363 292,-357 298,-351 304,-351 304,-351 496,-351 496,-351 502,-351 508,-357 508,-363 508,-363 508,-375 508,-375 508,-381 502,-387 496,-387"/>
|
||||
<text text-anchor="middle" x="400" y="-365.2" font-family="Times,serif" font-size="16.00">NotSkippingPrefix</text>
|
||||
</g>
|
||||
<!-- NotSkipping.Idle->NotSkippingPrefix.Idle -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>NotSkipping.Idle->NotSkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M340.77,-437.93C351.2,-433.2 361.45,-427.29 370,-420 377.58,-413.53 383.76,-404.65 388.51,-396.16"/>
|
||||
<polygon fill="black" stroke="black" points="391.63,-397.74 393.08,-387.24 385.4,-394.54 391.63,-397.74"/>
|
||||
<text text-anchor="middle" x="442.5" y="-408.8" font-family="Times,serif" font-size="14.00">[delimiter == '/']</text>
|
||||
</g>
|
||||
<!-- NeverSkipping.Processing -->
|
||||
<g id="node7" class="node">
|
||||
<title>NeverSkipping.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M204,-270C204,-270 12,-270 12,-270 6,-270 0,-264 0,-258 0,-258 0,-246 0,-246 0,-240 6,-234 12,-234 12,-234 204,-234 204,-234 210,-234 216,-240 216,-246 216,-246 216,-258 216,-258 216,-264 210,-270 204,-270"/>
|
||||
<text text-anchor="middle" x="108" y="-248.2" font-family="Times,serif" font-size="16.00">NeverSkipping</text>
|
||||
</g>
|
||||
<!-- NeverSkipping.Idle->NeverSkipping.Processing -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>NeverSkipping.Idle->NeverSkipping.Processing</title>
|
||||
<path fill="none" stroke="black" d="M64.1,-350.93C47.33,-346.11 33.58,-340.17 28,-333 15.72,-317.21 17.05,-304.74 28,-288 30.93,-283.52 34.58,-279.6 38.69,-276.19"/>
|
||||
<polygon fill="black" stroke="black" points="40.97,-278.86 47.1,-270.22 36.92,-273.16 40.97,-278.86"/>
|
||||
<text text-anchor="middle" x="86" y="-306.8" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefix.Processing -->
|
||||
<g id="node8" class="node">
|
||||
<title>NotSkippingPrefix.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M554,-270C554,-270 362,-270 362,-270 356,-270 350,-264 350,-258 350,-258 350,-246 350,-246 350,-240 356,-234 362,-234 362,-234 554,-234 554,-234 560,-234 566,-240 566,-246 566,-246 566,-258 566,-258 566,-264 560,-270 554,-270"/>
|
||||
<text text-anchor="middle" x="458" y="-248.2" font-family="Times,serif" font-size="16.00">NotSkippingPrefix</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefix.Idle->NotSkippingPrefix.Processing -->
|
||||
<g id="edge5" class="edge">
|
||||
<title>NotSkippingPrefix.Idle->NotSkippingPrefix.Processing</title>
|
||||
<path fill="none" stroke="black" d="M395.69,-350.84C392.38,-333.75 390.03,-307.33 401,-288 403.42,-283.74 406.58,-279.94 410.19,-276.55"/>
|
||||
<polygon fill="black" stroke="black" points="412.5,-279.18 418.1,-270.18 408.11,-273.73 412.5,-279.18"/>
|
||||
<text text-anchor="middle" x="459" y="-306.8" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Idle -->
|
||||
<g id="node6" class="node">
|
||||
<title>SkippingPrefix.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M554,-138C554,-138 362,-138 362,-138 356,-138 350,-132 350,-126 350,-126 350,-114 350,-114 350,-108 356,-102 362,-102 362,-102 554,-102 554,-102 560,-102 566,-108 566,-114 566,-114 566,-126 566,-126 566,-132 560,-138 554,-138"/>
|
||||
<text text-anchor="middle" x="458" y="-116.2" font-family="Times,serif" font-size="16.00">SkippingPrefix</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Processing -->
|
||||
<g id="node9" class="node">
|
||||
<title>SkippingPrefix.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M691,-36C691,-36 499,-36 499,-36 493,-36 487,-30 487,-24 487,-24 487,-12 487,-12 487,-6 493,0 499,0 499,0 691,0 691,0 697,0 703,-6 703,-12 703,-12 703,-24 703,-24 703,-30 697,-36 691,-36"/>
|
||||
<text text-anchor="middle" x="595" y="-14.2" font-family="Times,serif" font-size="16.00">SkippingPrefix</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Idle->SkippingPrefix.Processing -->
|
||||
<g id="edge6" class="edge">
|
||||
<title>SkippingPrefix.Idle->SkippingPrefix.Processing</title>
|
||||
<path fill="none" stroke="black" d="M452.35,-101.95C448.76,-87.65 446.54,-67.45 457,-54 461.44,-48.29 471.08,-43.36 483.3,-39.15"/>
|
||||
<polygon fill="black" stroke="black" points="484.61,-42.41 493.1,-36.07 482.51,-35.73 484.61,-42.41"/>
|
||||
<text text-anchor="middle" x="515" y="-65.3" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- NeverSkipping.Processing->END -->
|
||||
<g id="edge7" class="edge">
|
||||
<title>NeverSkipping.Processing->END</title>
|
||||
<path fill="none" stroke="black" d="M102.91,-233.88C97.93,-213.45 93.18,-179.15 109,-156 123.79,-134.35 154.41,-126.09 175.08,-122.94"/>
|
||||
<polygon fill="black" stroke="black" points="175.62,-126.4 185.11,-121.69 174.76,-119.45 175.62,-126.4"/>
|
||||
<text text-anchor="middle" x="185" y="-189.8" font-family="Times,serif" font-size="14.00">[nKeys == maxKeys]</text>
|
||||
<text text-anchor="middle" x="185" y="-174.8" font-family="Times,serif" font-size="14.00">-> FILTER_END</text>
|
||||
</g>
|
||||
<!-- NeverSkipping.Processing->NeverSkipping.Idle -->
|
||||
<g id="edge8" class="edge">
|
||||
<title>NeverSkipping.Processing->NeverSkipping.Idle</title>
|
||||
<path fill="none" stroke="black" d="M129.49,-270.27C134.87,-275.48 140.18,-281.55 144,-288 153.56,-304.17 159.09,-324.63 162.21,-340.81"/>
|
||||
<polygon fill="black" stroke="black" points="158.78,-341.49 163.94,-350.74 165.68,-340.29 158.78,-341.49"/>
|
||||
<text text-anchor="middle" x="265.5" y="-321.8" font-family="Times,serif" font-size="14.00">[nKeys < maxKeys]</text>
|
||||
<text text-anchor="middle" x="265.5" y="-306.8" font-family="Times,serif" font-size="14.00">/ Contents.append(key, value)</text>
|
||||
<text text-anchor="middle" x="265.5" y="-291.8" font-family="Times,serif" font-size="14.00"> -> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefix.Processing->END -->
|
||||
<g id="edge9" class="edge">
|
||||
<title>NotSkippingPrefix.Processing->END</title>
|
||||
<path fill="none" stroke="black" d="M349.96,-237.93C333,-232.81 316.36,-225.74 302,-216 275.27,-197.87 285.01,-177.6 261,-156 247.64,-143.98 229.41,-134.62 215.65,-128.62"/>
|
||||
<polygon fill="black" stroke="black" points="216.74,-125.28 206.16,-124.7 214.07,-131.75 216.74,-125.28"/>
|
||||
<text text-anchor="middle" x="378" y="-189.8" font-family="Times,serif" font-size="14.00">[nKeys == maxKeys]</text>
|
||||
<text text-anchor="middle" x="378" y="-174.8" font-family="Times,serif" font-size="14.00"> -> FILTER_END</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefix.Processing->NotSkippingPrefix.Idle -->
|
||||
<g id="edge11" class="edge">
|
||||
<title>NotSkippingPrefix.Processing->NotSkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M499.64,-270.11C506.59,-274.86 512.87,-280.76 517,-288 526.9,-305.38 528.94,-316.96 517,-333 513.56,-337.62 509.53,-341.66 505.07,-345.18"/>
|
||||
<polygon fill="black" stroke="black" points="502.89,-342.43 496.63,-350.98 506.85,-348.2 502.89,-342.43"/>
|
||||
<text text-anchor="middle" x="690.5" y="-321.8" font-family="Times,serif" font-size="14.00">[nKeys < maxKeys and not hasDelimiter(key)]</text>
|
||||
<text text-anchor="middle" x="690.5" y="-306.8" font-family="Times,serif" font-size="14.00">/ Contents.append(key, value)</text>
|
||||
<text text-anchor="middle" x="690.5" y="-291.8" font-family="Times,serif" font-size="14.00"> -> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- NotSkippingPrefix.Processing->SkippingPrefix.Idle -->
|
||||
<g id="edge10" class="edge">
|
||||
<title>NotSkippingPrefix.Processing->SkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M458,-233.74C458,-211.98 458,-174.32 458,-148.56"/>
|
||||
<polygon fill="black" stroke="black" points="461.5,-148.33 458,-138.33 454.5,-148.33 461.5,-148.33"/>
|
||||
<text text-anchor="middle" x="609.5" y="-204.8" font-family="Times,serif" font-size="14.00">[nKeys < maxKeys and hasDelimiter(key)]</text>
|
||||
<text text-anchor="middle" x="609.5" y="-189.8" font-family="Times,serif" font-size="14.00">/ prefix <- prefixOf(key)</text>
|
||||
<text text-anchor="middle" x="609.5" y="-174.8" font-family="Times,serif" font-size="14.00">/ CommonPrefixes.append(prefixOf(key))</text>
|
||||
<text text-anchor="middle" x="609.5" y="-159.8" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Processing->SkippingPrefix.Idle -->
|
||||
<g id="edge12" class="edge">
|
||||
<title>SkippingPrefix.Processing->SkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M593.49,-36.23C591.32,-50.84 586,-71.39 573,-84 567.75,-89.09 561.77,-93.45 555.38,-97.17"/>
|
||||
<polygon fill="black" stroke="black" points="553.66,-94.12 546.43,-101.87 556.91,-100.32 553.66,-94.12"/>
|
||||
<text text-anchor="middle" x="672" y="-72.8" font-family="Times,serif" font-size="14.00">[key.startsWith(prefix)]</text>
|
||||
<text text-anchor="middle" x="672" y="-57.8" font-family="Times,serif" font-size="14.00">-> FILTER_SKIP</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Processing->NotSkippingPrefix.Processing -->
|
||||
<g id="edge13" class="edge">
|
||||
<title>SkippingPrefix.Processing->NotSkippingPrefix.Processing</title>
|
||||
<path fill="none" stroke="black" d="M703.16,-31.64C728.6,-36.87 750.75,-44.11 759,-54 778.46,-77.34 776.26,-200.01 762,-216 749.37,-230.17 656.13,-239.42 576.2,-244.84"/>
|
||||
<polygon fill="black" stroke="black" points="575.77,-241.36 566.03,-245.51 576.24,-248.34 575.77,-241.36"/>
|
||||
<text text-anchor="middle" x="870" y="-116.3" font-family="Times,serif" font-size="14.00">[not key.startsWith(prefix)]</text>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
Before Width: | Height: | Size: 12 KiB |
|
@ -1,50 +0,0 @@
|
|||
digraph {
|
||||
node [shape="box",style="filled,rounded",fontsize=16,fixedsize=true,width=3];
|
||||
edge [fontsize=14];
|
||||
rankdir=TB;
|
||||
|
||||
START [shape="circle",width=0.2,label="",style="filled",fillcolor="black"]
|
||||
END [shape="circle",width=0.2,label="",style="filled",fillcolor="black",peripheries=2]
|
||||
|
||||
node [fillcolor="lightgrey"];
|
||||
"NotSkipping.Idle" [label="NotSkipping",group="NotSkipping",width=4];
|
||||
"SkippingPrefix.Idle" [label="SkippingPrefix",group="SkippingPrefix"];
|
||||
"WaitForNullKey.Idle" [label="WaitForNullKey",group="WaitForNullKey"];
|
||||
"SkippingVersions.Idle" [label="SkippingVersions",group="SkippingVersions"];
|
||||
|
||||
node [fillcolor="lightblue"];
|
||||
"NotSkipping.Processing" [label="NotSkipping",group="NotSkipping",width=4];
|
||||
"NotSkippingV0.Processing" [label="NotSkippingV0",group="NotSkipping",width=4];
|
||||
"NotSkippingV1.Processing" [label="NotSkippingV1",group="NotSkipping",width=4];
|
||||
"NotSkippingCommon.Processing" [label="NotSkippingCommon",group="NotSkipping",width=4];
|
||||
"SkippingPrefix.Processing" [label="SkippingPrefix",group="SkippingPrefix"];
|
||||
"WaitForNullKey.Processing" [label="WaitForNullKey",group="WaitForNullKey"];
|
||||
"SkippingVersions.Processing" [label="SkippingVersions",group="SkippingVersions"];
|
||||
|
||||
START -> "WaitForNullKey.Idle" [label="[versionIdMarker != undefined]"]
|
||||
START -> "NotSkipping.Idle" [label="[versionIdMarker == undefined]"]
|
||||
|
||||
"NotSkipping.Idle" -> "NotSkipping.Processing" [label="filter(key, value)"]
|
||||
"SkippingPrefix.Idle" -> "SkippingPrefix.Processing" [label="filter(key, value)"]
|
||||
"WaitForNullKey.Idle" -> "WaitForNullKey.Processing" [label="filter(key, value)"]
|
||||
"SkippingVersions.Idle" -> "SkippingVersions.Processing" [label="filter(key, value)"]
|
||||
|
||||
"NotSkipping.Processing" -> "NotSkippingV0.Processing" [label="vFormat='v0'"]
|
||||
"NotSkipping.Processing" -> "NotSkippingV1.Processing" [label="vFormat='v1'"]
|
||||
|
||||
"WaitForNullKey.Processing" -> "NotSkipping.Processing" [label="master(key) != keyMarker"]
|
||||
"WaitForNullKey.Processing" -> "SkippingVersions.Processing" [label="master(key) == keyMarker"]
|
||||
"NotSkippingV0.Processing" -> "SkippingPrefix.Idle" [label="[key.startsWith(<ReplayPrefix>)]\n/ prefix <- <ReplayPrefix>\n-> FILTER_SKIP"]
|
||||
"NotSkippingV0.Processing" -> "NotSkipping.Idle" [label="[Version.isPHD(value)]\n-> FILTER_ACCEPT"]
|
||||
"NotSkippingV0.Processing" -> "NotSkippingCommon.Processing" [label="[not key.startsWith(<ReplayPrefix>)\nand not Version.isPHD(value)]"]
|
||||
"NotSkippingV1.Processing" -> "NotSkippingCommon.Processing" [label="[always]"]
|
||||
"NotSkippingCommon.Processing" -> END [label="[isListableKey(key, value) and\nKeys == maxKeys]\n-> FILTER_END"]
|
||||
"NotSkippingCommon.Processing" -> "SkippingPrefix.Idle" [label="[isListableKey(key, value) and\nnKeys < maxKeys and\nhasDelimiter(key)]\n/ prefix <- prefixOf(key)\n/ CommonPrefixes.append(prefixOf(key))\n-> FILTER_ACCEPT"]
|
||||
"NotSkippingCommon.Processing" -> "NotSkipping.Idle" [label="[isListableKey(key, value) and\nnKeys < maxKeys and\nnot hasDelimiter(key)]\n/ Contents.append(key, versionId, value)\n-> FILTER_ACCEPT"]
|
||||
|
||||
"SkippingPrefix.Processing" -> "SkippingPrefix.Idle" [label="[key.startsWith(prefix)]\n-> FILTER_SKIP"]
|
||||
"SkippingPrefix.Processing" -> "NotSkipping.Processing" [label="[not key.startsWith(prefix)]"]
|
||||
"SkippingVersions.Processing" -> "NotSkipping.Processing" [label="master(key) !== keyMarker or \nversionId > versionIdMarker"]
|
||||
"SkippingVersions.Processing" -> "SkippingVersions.Idle" [label="master(key) === keyMarker and \nversionId < versionIdMarker\n-> FILTER_SKIP"]
|
||||
"SkippingVersions.Processing" -> "SkippingVersions.Idle" [label="master(key) === keyMarker and \nversionId == versionIdMarker\n-> FILTER_ACCEPT"]
|
||||
}
|
|
@ -1,265 +0,0 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.43.0 (0)
|
||||
-->
|
||||
<!-- Title: %3 Pages: 1 -->
|
||||
<svg width="1522pt" height="922pt"
|
||||
viewBox="0.00 0.00 1522.26 922.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 918)">
|
||||
<title>%3</title>
|
||||
<polygon fill="white" stroke="transparent" points="-4,4 -4,-918 1518.26,-918 1518.26,4 -4,4"/>
|
||||
<!-- START -->
|
||||
<g id="node1" class="node">
|
||||
<title>START</title>
|
||||
<ellipse fill="black" stroke="black" cx="393.26" cy="-907" rx="7" ry="7"/>
|
||||
</g>
|
||||
<!-- NotSkipping.Idle -->
|
||||
<g id="node3" class="node">
|
||||
<title>NotSkipping.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M436.26,-675C436.26,-675 172.26,-675 172.26,-675 166.26,-675 160.26,-669 160.26,-663 160.26,-663 160.26,-651 160.26,-651 160.26,-645 166.26,-639 172.26,-639 172.26,-639 436.26,-639 436.26,-639 442.26,-639 448.26,-645 448.26,-651 448.26,-651 448.26,-663 448.26,-663 448.26,-669 442.26,-675 436.26,-675"/>
|
||||
<text text-anchor="middle" x="304.26" y="-653.2" font-family="Times,serif" font-size="16.00">NotSkipping</text>
|
||||
</g>
|
||||
<!-- START->NotSkipping.Idle -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>START->NotSkipping.Idle</title>
|
||||
<path fill="none" stroke="black" d="M391.06,-899.87C380.45,-870.31 334.26,-741.58 313.93,-684.93"/>
|
||||
<polygon fill="black" stroke="black" points="317.12,-683.46 310.45,-675.23 310.53,-685.82 317.12,-683.46"/>
|
||||
<text text-anchor="middle" x="470.76" y="-783.8" font-family="Times,serif" font-size="14.00">[versionIdMarker == undefined]</text>
|
||||
</g>
|
||||
<!-- WaitForNullKey.Idle -->
|
||||
<g id="node5" class="node">
|
||||
<title>WaitForNullKey.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M692.26,-849C692.26,-849 500.26,-849 500.26,-849 494.26,-849 488.26,-843 488.26,-837 488.26,-837 488.26,-825 488.26,-825 488.26,-819 494.26,-813 500.26,-813 500.26,-813 692.26,-813 692.26,-813 698.26,-813 704.26,-819 704.26,-825 704.26,-825 704.26,-837 704.26,-837 704.26,-843 698.26,-849 692.26,-849"/>
|
||||
<text text-anchor="middle" x="596.26" y="-827.2" font-family="Times,serif" font-size="16.00">WaitForNullKey</text>
|
||||
</g>
|
||||
<!-- START->WaitForNullKey.Idle -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>START->WaitForNullKey.Idle</title>
|
||||
<path fill="none" stroke="black" d="M399.56,-903.7C420.56,-896.05 489.7,-870.85 540.08,-852.48"/>
|
||||
<polygon fill="black" stroke="black" points="541.38,-855.73 549.57,-849.02 538.98,-849.16 541.38,-855.73"/>
|
||||
<text text-anchor="middle" x="608.76" y="-870.8" font-family="Times,serif" font-size="14.00">[versionIdMarker != undefined]</text>
|
||||
</g>
|
||||
<!-- END -->
|
||||
<g id="node2" class="node">
|
||||
<title>END</title>
|
||||
<ellipse fill="black" stroke="black" cx="45.26" cy="-120" rx="7" ry="7"/>
|
||||
<ellipse fill="none" stroke="black" cx="45.26" cy="-120" rx="11" ry="11"/>
|
||||
</g>
|
||||
<!-- NotSkipping.Processing -->
|
||||
<g id="node7" class="node">
|
||||
<title>NotSkipping.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M761.26,-558C761.26,-558 497.26,-558 497.26,-558 491.26,-558 485.26,-552 485.26,-546 485.26,-546 485.26,-534 485.26,-534 485.26,-528 491.26,-522 497.26,-522 497.26,-522 761.26,-522 761.26,-522 767.26,-522 773.26,-528 773.26,-534 773.26,-534 773.26,-546 773.26,-546 773.26,-552 767.26,-558 761.26,-558"/>
|
||||
<text text-anchor="middle" x="629.26" y="-536.2" font-family="Times,serif" font-size="16.00">NotSkipping</text>
|
||||
</g>
|
||||
<!-- NotSkipping.Idle->NotSkipping.Processing -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>NotSkipping.Idle->NotSkipping.Processing</title>
|
||||
<path fill="none" stroke="black" d="M333.17,-638.98C364.86,-620.99 417.68,-592.92 466.26,-576 483.64,-569.95 502.44,-564.74 520.88,-560.34"/>
|
||||
<polygon fill="black" stroke="black" points="521.83,-563.71 530.78,-558.04 520.25,-556.89 521.83,-563.71"/>
|
||||
<text text-anchor="middle" x="524.26" y="-594.8" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Idle -->
|
||||
<g id="node4" class="node">
|
||||
<title>SkippingPrefix.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M662.26,-138C662.26,-138 470.26,-138 470.26,-138 464.26,-138 458.26,-132 458.26,-126 458.26,-126 458.26,-114 458.26,-114 458.26,-108 464.26,-102 470.26,-102 470.26,-102 662.26,-102 662.26,-102 668.26,-102 674.26,-108 674.26,-114 674.26,-114 674.26,-126 674.26,-126 674.26,-132 668.26,-138 662.26,-138"/>
|
||||
<text text-anchor="middle" x="566.26" y="-116.2" font-family="Times,serif" font-size="16.00">SkippingPrefix</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Processing -->
|
||||
<g id="node11" class="node">
|
||||
<title>SkippingPrefix.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M779.26,-36C779.26,-36 587.26,-36 587.26,-36 581.26,-36 575.26,-30 575.26,-24 575.26,-24 575.26,-12 575.26,-12 575.26,-6 581.26,0 587.26,0 587.26,0 779.26,0 779.26,0 785.26,0 791.26,-6 791.26,-12 791.26,-12 791.26,-24 791.26,-24 791.26,-30 785.26,-36 779.26,-36"/>
|
||||
<text text-anchor="middle" x="683.26" y="-14.2" font-family="Times,serif" font-size="16.00">SkippingPrefix</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Idle->SkippingPrefix.Processing -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>SkippingPrefix.Idle->SkippingPrefix.Processing</title>
|
||||
<path fill="none" stroke="black" d="M552.64,-101.74C543.31,-87.68 534.41,-67.95 545.26,-54 549.71,-48.29 559.34,-43.36 571.56,-39.15"/>
|
||||
<polygon fill="black" stroke="black" points="572.87,-42.41 581.36,-36.07 570.77,-35.73 572.87,-42.41"/>
|
||||
<text text-anchor="middle" x="603.26" y="-65.3" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- WaitForNullKey.Processing -->
|
||||
<g id="node12" class="node">
|
||||
<title>WaitForNullKey.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M692.26,-762C692.26,-762 500.26,-762 500.26,-762 494.26,-762 488.26,-756 488.26,-750 488.26,-750 488.26,-738 488.26,-738 488.26,-732 494.26,-726 500.26,-726 500.26,-726 692.26,-726 692.26,-726 698.26,-726 704.26,-732 704.26,-738 704.26,-738 704.26,-750 704.26,-750 704.26,-756 698.26,-762 692.26,-762"/>
|
||||
<text text-anchor="middle" x="596.26" y="-740.2" font-family="Times,serif" font-size="16.00">WaitForNullKey</text>
|
||||
</g>
|
||||
<!-- WaitForNullKey.Idle->WaitForNullKey.Processing -->
|
||||
<g id="edge5" class="edge">
|
||||
<title>WaitForNullKey.Idle->WaitForNullKey.Processing</title>
|
||||
<path fill="none" stroke="black" d="M596.26,-812.8C596.26,-801.16 596.26,-785.55 596.26,-772.24"/>
|
||||
<polygon fill="black" stroke="black" points="599.76,-772.18 596.26,-762.18 592.76,-772.18 599.76,-772.18"/>
|
||||
<text text-anchor="middle" x="654.26" y="-783.8" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Idle -->
|
||||
<g id="node6" class="node">
|
||||
<title>SkippingVersions.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M1241.26,-558C1241.26,-558 1049.26,-558 1049.26,-558 1043.26,-558 1037.26,-552 1037.26,-546 1037.26,-546 1037.26,-534 1037.26,-534 1037.26,-528 1043.26,-522 1049.26,-522 1049.26,-522 1241.26,-522 1241.26,-522 1247.26,-522 1253.26,-528 1253.26,-534 1253.26,-534 1253.26,-546 1253.26,-546 1253.26,-552 1247.26,-558 1241.26,-558"/>
|
||||
<text text-anchor="middle" x="1145.26" y="-536.2" font-family="Times,serif" font-size="16.00">SkippingVersions</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Processing -->
|
||||
<g id="node13" class="node">
|
||||
<title>SkippingVersions.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M1241.26,-675C1241.26,-675 1049.26,-675 1049.26,-675 1043.26,-675 1037.26,-669 1037.26,-663 1037.26,-663 1037.26,-651 1037.26,-651 1037.26,-645 1043.26,-639 1049.26,-639 1049.26,-639 1241.26,-639 1241.26,-639 1247.26,-639 1253.26,-645 1253.26,-651 1253.26,-651 1253.26,-663 1253.26,-663 1253.26,-669 1247.26,-675 1241.26,-675"/>
|
||||
<text text-anchor="middle" x="1145.26" y="-653.2" font-family="Times,serif" font-size="16.00">SkippingVersions</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Idle->SkippingVersions.Processing -->
|
||||
<g id="edge6" class="edge">
|
||||
<title>SkippingVersions.Idle->SkippingVersions.Processing</title>
|
||||
<path fill="none" stroke="black" d="M1145.26,-558.25C1145.26,-576.77 1145.26,-606.45 1145.26,-628.25"/>
|
||||
<polygon fill="black" stroke="black" points="1141.76,-628.53 1145.26,-638.53 1148.76,-628.53 1141.76,-628.53"/>
|
||||
<text text-anchor="middle" x="1203.26" y="-594.8" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- NotSkippingV0.Processing -->
|
||||
<g id="node8" class="node">
|
||||
<title>NotSkippingV0.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M436.26,-411C436.26,-411 172.26,-411 172.26,-411 166.26,-411 160.26,-405 160.26,-399 160.26,-399 160.26,-387 160.26,-387 160.26,-381 166.26,-375 172.26,-375 172.26,-375 436.26,-375 436.26,-375 442.26,-375 448.26,-381 448.26,-387 448.26,-387 448.26,-399 448.26,-399 448.26,-405 442.26,-411 436.26,-411"/>
|
||||
<text text-anchor="middle" x="304.26" y="-389.2" font-family="Times,serif" font-size="16.00">NotSkippingV0</text>
|
||||
</g>
|
||||
<!-- NotSkipping.Processing->NotSkippingV0.Processing -->
|
||||
<g id="edge7" class="edge">
|
||||
<title>NotSkipping.Processing->NotSkippingV0.Processing</title>
|
||||
<path fill="none" stroke="black" d="M573.96,-521.95C558.07,-516.64 540.84,-510.46 525.26,-504 460.22,-477.02 387.62,-439.36 343.97,-415.84"/>
|
||||
<polygon fill="black" stroke="black" points="345.57,-412.72 335.11,-411.04 342.24,-418.88 345.57,-412.72"/>
|
||||
<text text-anchor="middle" x="573.76" y="-462.8" font-family="Times,serif" font-size="14.00">vFormat='v0'</text>
|
||||
</g>
|
||||
<!-- NotSkippingV1.Processing -->
|
||||
<g id="node9" class="node">
|
||||
<title>NotSkippingV1.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M758.26,-411C758.26,-411 494.26,-411 494.26,-411 488.26,-411 482.26,-405 482.26,-399 482.26,-399 482.26,-387 482.26,-387 482.26,-381 488.26,-375 494.26,-375 494.26,-375 758.26,-375 758.26,-375 764.26,-375 770.26,-381 770.26,-387 770.26,-387 770.26,-399 770.26,-399 770.26,-405 764.26,-411 758.26,-411"/>
|
||||
<text text-anchor="middle" x="626.26" y="-389.2" font-family="Times,serif" font-size="16.00">NotSkippingV1</text>
|
||||
</g>
|
||||
<!-- NotSkipping.Processing->NotSkippingV1.Processing -->
|
||||
<g id="edge8" class="edge">
|
||||
<title>NotSkipping.Processing->NotSkippingV1.Processing</title>
|
||||
<path fill="none" stroke="black" d="M628.91,-521.8C628.39,-496.94 627.44,-450.74 626.83,-421.23"/>
|
||||
<polygon fill="black" stroke="black" points="630.32,-421.11 626.62,-411.18 623.33,-421.25 630.32,-421.11"/>
|
||||
<text text-anchor="middle" x="676.76" y="-462.8" font-family="Times,serif" font-size="14.00">vFormat='v1'</text>
|
||||
</g>
|
||||
<!-- NotSkippingV0.Processing->NotSkipping.Idle -->
|
||||
<g id="edge12" class="edge">
|
||||
<title>NotSkippingV0.Processing->NotSkipping.Idle</title>
|
||||
<path fill="none" stroke="black" d="M304.26,-411.25C304.26,-455.74 304.26,-574.61 304.26,-628.62"/>
|
||||
<polygon fill="black" stroke="black" points="300.76,-628.81 304.26,-638.81 307.76,-628.81 300.76,-628.81"/>
|
||||
<text text-anchor="middle" x="385.76" y="-543.8" font-family="Times,serif" font-size="14.00">[Version.isPHD(value)]</text>
|
||||
<text text-anchor="middle" x="385.76" y="-528.8" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- NotSkippingV0.Processing->SkippingPrefix.Idle -->
|
||||
<g id="edge11" class="edge">
|
||||
<title>NotSkippingV0.Processing->SkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M448.41,-376.93C508.52,-369.95 565.63,-362.09 570.26,-357 622.9,-299.12 594.8,-196.31 577.11,-147.78"/>
|
||||
<polygon fill="black" stroke="black" points="580.33,-146.4 573.53,-138.28 573.78,-148.87 580.33,-146.4"/>
|
||||
<text text-anchor="middle" x="720.26" y="-297.8" font-family="Times,serif" font-size="14.00">[key.startsWith(<ReplayPrefix>)]</text>
|
||||
<text text-anchor="middle" x="720.26" y="-282.8" font-family="Times,serif" font-size="14.00">/ prefix <- <ReplayPrefix></text>
|
||||
<text text-anchor="middle" x="720.26" y="-267.8" font-family="Times,serif" font-size="14.00">-> FILTER_SKIP</text>
|
||||
</g>
|
||||
<!-- NotSkippingCommon.Processing -->
|
||||
<g id="node10" class="node">
|
||||
<title>NotSkippingCommon.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M436.26,-304.5C436.26,-304.5 172.26,-304.5 172.26,-304.5 166.26,-304.5 160.26,-298.5 160.26,-292.5 160.26,-292.5 160.26,-280.5 160.26,-280.5 160.26,-274.5 166.26,-268.5 172.26,-268.5 172.26,-268.5 436.26,-268.5 436.26,-268.5 442.26,-268.5 448.26,-274.5 448.26,-280.5 448.26,-280.5 448.26,-292.5 448.26,-292.5 448.26,-298.5 442.26,-304.5 436.26,-304.5"/>
|
||||
<text text-anchor="middle" x="304.26" y="-282.7" font-family="Times,serif" font-size="16.00">NotSkippingCommon</text>
|
||||
</g>
|
||||
<!-- NotSkippingV0.Processing->NotSkippingCommon.Processing -->
|
||||
<g id="edge13" class="edge">
|
||||
<title>NotSkippingV0.Processing->NotSkippingCommon.Processing</title>
|
||||
<path fill="none" stroke="black" d="M304.26,-374.74C304.26,-358.48 304.26,-333.85 304.26,-314.9"/>
|
||||
<polygon fill="black" stroke="black" points="307.76,-314.78 304.26,-304.78 300.76,-314.78 307.76,-314.78"/>
|
||||
<text text-anchor="middle" x="435.26" y="-345.8" font-family="Times,serif" font-size="14.00">[not key.startsWith(<ReplayPrefix>)</text>
|
||||
<text text-anchor="middle" x="435.26" y="-330.8" font-family="Times,serif" font-size="14.00">and not Version.isPHD(value)]</text>
|
||||
</g>
|
||||
<!-- NotSkippingV1.Processing->NotSkippingCommon.Processing -->
|
||||
<g id="edge14" class="edge">
|
||||
<title>NotSkippingV1.Processing->NotSkippingCommon.Processing</title>
|
||||
<path fill="none" stroke="black" d="M616.43,-374.83C606.75,-359.62 590.48,-338.14 570.26,-327 549.98,-315.83 505.48,-307.38 458.57,-301.23"/>
|
||||
<polygon fill="black" stroke="black" points="458.9,-297.74 448.53,-299.95 458.01,-304.69 458.9,-297.74"/>
|
||||
<text text-anchor="middle" x="632.26" y="-338.3" font-family="Times,serif" font-size="14.00">[always]</text>
|
||||
</g>
|
||||
<!-- NotSkippingCommon.Processing->END -->
|
||||
<g id="edge15" class="edge">
|
||||
<title>NotSkippingCommon.Processing->END</title>
|
||||
<path fill="none" stroke="black" d="M159.92,-279.56C109.8,-274.24 62.13,-264.33 46.26,-246 20.92,-216.72 30.42,-167.54 38.5,-140.42"/>
|
||||
<polygon fill="black" stroke="black" points="41.94,-141.16 41.67,-130.57 35.27,-139.02 41.94,-141.16"/>
|
||||
<text text-anchor="middle" x="152.76" y="-212.3" font-family="Times,serif" font-size="14.00">[isListableKey(key, value) and</text>
|
||||
<text text-anchor="middle" x="152.76" y="-197.3" font-family="Times,serif" font-size="14.00">Keys == maxKeys]</text>
|
||||
<text text-anchor="middle" x="152.76" y="-182.3" font-family="Times,serif" font-size="14.00">-> FILTER_END</text>
|
||||
</g>
|
||||
<!-- NotSkippingCommon.Processing->NotSkipping.Idle -->
|
||||
<g id="edge17" class="edge">
|
||||
<title>NotSkippingCommon.Processing->NotSkipping.Idle</title>
|
||||
<path fill="none" stroke="black" d="M214.74,-304.54C146.51,-322.73 57.06,-358.99 13.26,-429 -49.27,-528.95 128.43,-602.49 233.32,-635.95"/>
|
||||
<polygon fill="black" stroke="black" points="232.34,-639.31 242.93,-638.97 234.43,-632.63 232.34,-639.31"/>
|
||||
<text text-anchor="middle" x="156.76" y="-492.8" font-family="Times,serif" font-size="14.00">[isListableKey(key, value) and</text>
|
||||
<text text-anchor="middle" x="156.76" y="-477.8" font-family="Times,serif" font-size="14.00">nKeys < maxKeys and</text>
|
||||
<text text-anchor="middle" x="156.76" y="-462.8" font-family="Times,serif" font-size="14.00">not hasDelimiter(key)]</text>
|
||||
<text text-anchor="middle" x="156.76" y="-447.8" font-family="Times,serif" font-size="14.00">/ Contents.append(key, versionId, value)</text>
|
||||
<text text-anchor="middle" x="156.76" y="-432.8" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- NotSkippingCommon.Processing->SkippingPrefix.Idle -->
|
||||
<g id="edge16" class="edge">
|
||||
<title>NotSkippingCommon.Processing->SkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M292.14,-268.23C288.18,-261.59 284.27,-253.75 282.26,-246 272.21,-207.28 255.76,-185.96 282.26,-156 293.6,-143.18 374.98,-134.02 447.74,-128.3"/>
|
||||
<polygon fill="black" stroke="black" points="448.24,-131.77 457.94,-127.51 447.7,-124.79 448.24,-131.77"/>
|
||||
<text text-anchor="middle" x="428.26" y="-234.8" font-family="Times,serif" font-size="14.00">[isListableKey(key, value) and</text>
|
||||
<text text-anchor="middle" x="428.26" y="-219.8" font-family="Times,serif" font-size="14.00">nKeys < maxKeys and</text>
|
||||
<text text-anchor="middle" x="428.26" y="-204.8" font-family="Times,serif" font-size="14.00">hasDelimiter(key)]</text>
|
||||
<text text-anchor="middle" x="428.26" y="-189.8" font-family="Times,serif" font-size="14.00">/ prefix <- prefixOf(key)</text>
|
||||
<text text-anchor="middle" x="428.26" y="-174.8" font-family="Times,serif" font-size="14.00">/ CommonPrefixes.append(prefixOf(key))</text>
|
||||
<text text-anchor="middle" x="428.26" y="-159.8" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Processing->SkippingPrefix.Idle -->
|
||||
<g id="edge18" class="edge">
|
||||
<title>SkippingPrefix.Processing->SkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M681.57,-36.04C679.28,-50.54 673.9,-71.03 661.26,-84 656.4,-88.99 650.77,-93.28 644.72,-96.95"/>
|
||||
<polygon fill="black" stroke="black" points="642.71,-94.06 635.6,-101.92 646.05,-100.21 642.71,-94.06"/>
|
||||
<text text-anchor="middle" x="759.26" y="-72.8" font-family="Times,serif" font-size="14.00">[key.startsWith(prefix)]</text>
|
||||
<text text-anchor="middle" x="759.26" y="-57.8" font-family="Times,serif" font-size="14.00">-> FILTER_SKIP</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Processing->NotSkipping.Processing -->
|
||||
<g id="edge19" class="edge">
|
||||
<title>SkippingPrefix.Processing->NotSkipping.Processing</title>
|
||||
<path fill="none" stroke="black" d="M791.46,-33.51C815.84,-38.71 837.21,-45.46 846.26,-54 868.07,-74.57 864.26,-89.02 864.26,-119 864.26,-394 864.26,-394 864.26,-394 864.26,-462.4 791.27,-499.6 726.64,-519.12"/>
|
||||
<polygon fill="black" stroke="black" points="725.39,-515.84 716.77,-521.99 727.35,-522.56 725.39,-515.84"/>
|
||||
<text text-anchor="middle" x="961.26" y="-282.8" font-family="Times,serif" font-size="14.00">[not key.startsWith(prefix)]</text>
|
||||
</g>
|
||||
<!-- WaitForNullKey.Processing->NotSkipping.Processing -->
|
||||
<g id="edge9" class="edge">
|
||||
<title>WaitForNullKey.Processing->NotSkipping.Processing</title>
|
||||
<path fill="none" stroke="black" d="M599.08,-725.78C604.81,-690.67 617.89,-610.59 624.8,-568.31"/>
|
||||
<polygon fill="black" stroke="black" points="628.3,-568.61 626.46,-558.18 621.39,-567.48 628.3,-568.61"/>
|
||||
<text text-anchor="middle" x="707.26" y="-653.3" font-family="Times,serif" font-size="14.00">master(key) != keyMarker</text>
|
||||
</g>
|
||||
<!-- WaitForNullKey.Processing->SkippingVersions.Processing -->
|
||||
<g id="edge10" class="edge">
|
||||
<title>WaitForNullKey.Processing->SkippingVersions.Processing</title>
|
||||
<path fill="none" stroke="black" d="M704.4,-726.26C797.32,-711.87 931.09,-691.16 1026.87,-676.33"/>
|
||||
<polygon fill="black" stroke="black" points="1027.55,-679.77 1036.89,-674.78 1026.47,-672.85 1027.55,-679.77"/>
|
||||
<text text-anchor="middle" x="1001.26" y="-696.8" font-family="Times,serif" font-size="14.00">master(key) == keyMarker</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Processing->SkippingVersions.Idle -->
|
||||
<g id="edge21" class="edge">
|
||||
<title>SkippingVersions.Processing->SkippingVersions.Idle</title>
|
||||
<path fill="none" stroke="black" d="M1241.89,-638.98C1249.74,-634.29 1256.75,-628.4 1262.26,-621 1274.21,-604.96 1274.21,-592.04 1262.26,-576 1258.82,-571.38 1254.79,-567.34 1250.33,-563.82"/>
|
||||
<polygon fill="black" stroke="black" points="1252.11,-560.8 1241.89,-558.02 1248.15,-566.57 1252.11,-560.8"/>
|
||||
<text text-anchor="middle" x="1392.26" y="-609.8" font-family="Times,serif" font-size="14.00">master(key) === keyMarker and </text>
|
||||
<text text-anchor="middle" x="1392.26" y="-594.8" font-family="Times,serif" font-size="14.00">versionId < versionIdMarker</text>
|
||||
<text text-anchor="middle" x="1392.26" y="-579.8" font-family="Times,serif" font-size="14.00">-> FILTER_SKIP</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Processing->SkippingVersions.Idle -->
|
||||
<g id="edge22" class="edge">
|
||||
<title>SkippingVersions.Processing->SkippingVersions.Idle</title>
|
||||
<path fill="none" stroke="black" d="M1036.97,-654.38C978.97,-650.96 915.73,-642.25 897.26,-621 884.15,-605.9 884.15,-591.1 897.26,-576 914.65,-555.99 971.71,-547.1 1026.73,-543.28"/>
|
||||
<polygon fill="black" stroke="black" points="1027.21,-546.76 1036.97,-542.62 1026.76,-539.77 1027.21,-546.76"/>
|
||||
<text text-anchor="middle" x="1019.26" y="-609.8" font-family="Times,serif" font-size="14.00">master(key) === keyMarker and </text>
|
||||
<text text-anchor="middle" x="1019.26" y="-594.8" font-family="Times,serif" font-size="14.00">versionId == versionIdMarker</text>
|
||||
<text text-anchor="middle" x="1019.26" y="-579.8" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Processing->NotSkipping.Processing -->
|
||||
<g id="edge20" class="edge">
|
||||
<title>SkippingVersions.Processing->NotSkipping.Processing</title>
|
||||
<path fill="none" stroke="black" d="M1037.02,-651.24C897.84,-644.67 672.13,-632.37 657.26,-621 641.04,-608.6 634.18,-586.13 631.3,-568.16"/>
|
||||
<polygon fill="black" stroke="black" points="634.76,-567.68 630.02,-558.21 627.82,-568.57 634.76,-567.68"/>
|
||||
<text text-anchor="middle" x="770.26" y="-602.3" font-family="Times,serif" font-size="14.00">master(key) !== keyMarker or </text>
|
||||
<text text-anchor="middle" x="770.26" y="-587.3" font-family="Times,serif" font-size="14.00">versionId > versionIdMarker</text>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
Before Width: | Height: | Size: 21 KiB |
|
@ -1,28 +0,0 @@
|
|||
{
|
||||
"groups": {
|
||||
"default": {
|
||||
"packages": [
|
||||
"lib/executables/pensieveCreds/package.json",
|
||||
"package.json"
|
||||
]
|
||||
}
|
||||
},
|
||||
"branchPrefix": "improvement/greenkeeper.io/",
|
||||
"commitMessages": {
|
||||
"initialBadge": "docs(readme): add Greenkeeper badge",
|
||||
"initialDependencies": "chore(package): update dependencies",
|
||||
"initialBranches": "chore(bert-e): whitelist greenkeeper branches",
|
||||
"dependencyUpdate": "fix(package): update ${dependency} to version ${version}",
|
||||
"devDependencyUpdate": "chore(package): update ${dependency} to version ${version}",
|
||||
"dependencyPin": "fix: pin ${dependency} to ${oldVersionResolved}",
|
||||
"devDependencyPin": "chore: pin ${dependency} to ${oldVersionResolved}",
|
||||
"closes": "\n\nCloses #${number}"
|
||||
},
|
||||
"ignore": [
|
||||
"ajv",
|
||||
"eslint",
|
||||
"eslint-plugin-react",
|
||||
"eslint-config-airbnb",
|
||||
"eslint-config-scality"
|
||||
]
|
||||
}
|
182
index.ts
182
index.ts
|
@ -1,105 +1,104 @@
|
|||
import * as evaluators from './lib/policyEvaluator/evaluator';
|
||||
import evaluatePrincipal from './lib/policyEvaluator/principal';
|
||||
import RequestContext, {
|
||||
actionNeedQuotaCheck,
|
||||
actionNeedQuotaCheckCopy,
|
||||
actionWithDataDeletion } from './lib/policyEvaluator/RequestContext';
|
||||
import * as requestUtils from './lib/policyEvaluator/requestUtils';
|
||||
import * as actionMaps from './lib/policyEvaluator/utils/actionMaps';
|
||||
import { validateUserPolicy } from './lib/policy/policyValidator'
|
||||
import * as locationConstraints from './lib/patches/locationConstraints';
|
||||
import * as userMetadata from './lib/s3middleware/userMetadata';
|
||||
import convertToXml from './lib/s3middleware/convertToXml';
|
||||
import escapeForXml from './lib/s3middleware/escapeForXml';
|
||||
import * as objectLegalHold from './lib/s3middleware/objectLegalHold';
|
||||
import * as tagging from './lib/s3middleware/tagging';
|
||||
import { checkDateModifiedHeaders } from './lib/s3middleware/validateConditionalHeaders';
|
||||
import { validateConditionalHeaders } from './lib/s3middleware/validateConditionalHeaders';
|
||||
import MD5Sum from './lib/s3middleware/MD5Sum';
|
||||
import NullStream from './lib/s3middleware/nullStream';
|
||||
import * as objectUtils from './lib/s3middleware/objectUtils';
|
||||
import * as mpuUtils from './lib/s3middleware/azureHelpers/mpuUtils';
|
||||
import ResultsCollector from './lib/s3middleware/azureHelpers/ResultsCollector';
|
||||
import SubStreamInterface from './lib/s3middleware/azureHelpers/SubStreamInterface';
|
||||
import { prepareStream } from './lib/s3middleware/prepareStream';
|
||||
import * as processMpuParts from './lib/s3middleware/processMpuParts';
|
||||
import * as retention from './lib/s3middleware/objectRetention';
|
||||
import * as objectRestore from './lib/s3middleware/objectRestore';
|
||||
import * as lifecycleHelpers from './lib/s3middleware/lifecycleHelpers';
|
||||
export { default as errors } from './lib/errors';
|
||||
export { default as Clustering } from './lib/Clustering';
|
||||
export * as ClusterRPC from './lib/clustering/ClusterRPC';
|
||||
export * as ipCheck from './lib/ipCheck';
|
||||
export * as auth from './lib/auth/auth';
|
||||
export * as constants from './lib/constants';
|
||||
export * as https from './lib/https';
|
||||
export * as metrics from './lib/metrics';
|
||||
export * as network from './lib/network';
|
||||
export * as s3routes from './lib/s3routes';
|
||||
export * as versioning from './lib/versioning';
|
||||
export * as stream from './lib/stream';
|
||||
export * as jsutil from './lib/jsutil';
|
||||
export { default as stringHash } from './lib/stringHash';
|
||||
export * as db from './lib/db';
|
||||
export * as errorUtils from './lib/errorUtils';
|
||||
export { default as shuffle } from './lib/shuffle';
|
||||
export * as models from './lib/models';
|
||||
|
||||
export const auth = require('./lib/auth/auth');
|
||||
export const constants = require('./lib/constants');
|
||||
export const db = require('./lib/db');
|
||||
export const shuffle = require('./lib/shuffle');
|
||||
export const stringHash = require('./lib/stringHash');
|
||||
export const ipCheck = require('./lib/ipCheck');
|
||||
export const jsutil = require('./lib/jsutil');
|
||||
export const Clustering = require('./lib/Clustering');
|
||||
|
||||
export const https = {
|
||||
ciphers: require('./lib/https/ciphers.js'),
|
||||
dhparam: require('./lib/https/dh2048.js'),
|
||||
};
|
||||
|
||||
export const algorithms = {
|
||||
list: require('./lib/algos/list/exportAlgos'),
|
||||
list: {
|
||||
Basic: require('./lib/algos/list/basic').List,
|
||||
Delimiter: require('./lib/algos/list/delimiter').Delimiter,
|
||||
DelimiterVersions: require('./lib/algos/list/delimiterVersions').DelimiterVersions,
|
||||
DelimiterMaster: require('./lib/algos/list/delimiterMaster').DelimiterMaster,
|
||||
MPU: require('./lib/algos/list/MPU').MultipartUploads,
|
||||
},
|
||||
listTools: {
|
||||
DelimiterTools: require('./lib/algos/list/tools'),
|
||||
Skip: require('./lib/algos/list/skip'),
|
||||
},
|
||||
cache: {
|
||||
GapSet: require('./lib/algos/cache/GapSet'),
|
||||
GapCache: require('./lib/algos/cache/GapCache'),
|
||||
LRUCache: require('./lib/algos/cache/LRUCache'),
|
||||
},
|
||||
stream: {
|
||||
MergeStream: require('./lib/algos/stream/MergeStream'),
|
||||
},
|
||||
SortedSet: require('./lib/algos/set/SortedSet'),
|
||||
Heap: require('./lib/algos/heap/Heap'),
|
||||
};
|
||||
|
||||
export const policies = {
|
||||
evaluators,
|
||||
validateUserPolicy,
|
||||
evaluatePrincipal,
|
||||
RequestContext,
|
||||
requestUtils,
|
||||
actionMaps,
|
||||
actionNeedQuotaCheck,
|
||||
actionWithDataDeletion,
|
||||
actionNeedQuotaCheckCopy,
|
||||
evaluators: require('./lib/policyEvaluator/evaluator.js'),
|
||||
validateUserPolicy: require('./lib/policy/policyValidator').validateUserPolicy,
|
||||
evaluatePrincipal: require('./lib/policyEvaluator/principal'),
|
||||
RequestContext: require('./lib/policyEvaluator/RequestContext.js'),
|
||||
requestUtils: require('./lib/policyEvaluator/requestUtils'),
|
||||
actionMaps: require('./lib/policyEvaluator/utils/actionMaps'),
|
||||
};
|
||||
|
||||
export const testing = {
|
||||
matrix: require('./lib/testing/matrix.js'),
|
||||
};
|
||||
|
||||
export const s3middleware = {
|
||||
userMetadata,
|
||||
convertToXml,
|
||||
escapeForXml,
|
||||
objectLegalHold,
|
||||
tagging,
|
||||
checkDateModifiedHeaders,
|
||||
validateConditionalHeaders,
|
||||
MD5Sum,
|
||||
NullStream,
|
||||
objectUtils,
|
||||
azureHelper: {
|
||||
mpuUtils,
|
||||
ResultsCollector,
|
||||
SubStreamInterface,
|
||||
export const versioning = {
|
||||
VersioningConstants: require('./lib/versioning/constants.js').VersioningConstants,
|
||||
Version: require('./lib/versioning/Version.js').Version,
|
||||
VersionID: require('./lib/versioning/VersionID.js'),
|
||||
WriteGatheringManager: require('./lib/versioning/WriteGatheringManager.js'),
|
||||
WriteCache: require('./lib/versioning/WriteCache.js'),
|
||||
VersioningRequestProcessor: require('./lib/versioning/VersioningRequestProcessor.js'),
|
||||
};
|
||||
|
||||
export const network = {
|
||||
http: {
|
||||
server: require('./lib/network/http/server'),
|
||||
utils: require('./lib/network/http/utils'),
|
||||
},
|
||||
prepareStream,
|
||||
processMpuParts,
|
||||
retention,
|
||||
objectRestore,
|
||||
lifecycleHelpers,
|
||||
rpc: require('./lib/network/rpc/rpc'),
|
||||
level: require('./lib/network/rpc/level-net'),
|
||||
rest: {
|
||||
RESTServer: require('./lib/network/rest/RESTServer'),
|
||||
RESTClient: require('./lib/network/rest/RESTClient'),
|
||||
},
|
||||
probe: {
|
||||
ProbeServer: require('./lib/network/probe/ProbeServer'),
|
||||
},
|
||||
RoundRobin: require('./lib/network/RoundRobin'),
|
||||
kmip: require('./lib/network/kmip'),
|
||||
kmipClient: require('./lib/network/kmip/Client'),
|
||||
};
|
||||
|
||||
export const s3routes = {
|
||||
routes: require('./lib/s3routes/routes'),
|
||||
routesUtils: require('./lib/s3routes/routesUtils'),
|
||||
};
|
||||
|
||||
export const s3middleware = {
|
||||
userMetadata: require('./lib/s3middleware/userMetadata'),
|
||||
convertToXml: require('./lib/s3middleware/convertToXml'),
|
||||
escapeForXml: require('./lib/s3middleware/escapeForXml'),
|
||||
objectLegalHold: require('./lib/s3middleware/objectLegalHold'),
|
||||
tagging: require('./lib/s3middleware/tagging'),
|
||||
validateConditionalHeaders:
|
||||
require('./lib/s3middleware/validateConditionalHeaders').validateConditionalHeaders,
|
||||
MD5Sum: require('./lib/s3middleware/MD5Sum'),
|
||||
NullStream: require('./lib/s3middleware/nullStream'),
|
||||
objectUtils: require('./lib/s3middleware/objectUtils'),
|
||||
azureHelper: {
|
||||
mpuUtils: require('./lib/s3middleware/azureHelpers/mpuUtils'),
|
||||
ResultsCollector: require('./lib/s3middleware/azureHelpers/ResultsCollector'),
|
||||
SubStreamInterface: require('./lib/s3middleware/azureHelpers/SubStreamInterface'),
|
||||
},
|
||||
processMpuParts: require('./lib/s3middleware/processMpuParts'),
|
||||
retention: require('./lib/s3middleware/objectRetention'),
|
||||
lifecycleHelpers: require('./lib/s3middleware/lifecycleHelpers'),
|
||||
};
|
||||
|
||||
export const storage = {
|
||||
|
@ -166,10 +165,31 @@ export const storage = {
|
|||
utils: require('./lib/storage/utils'),
|
||||
};
|
||||
|
||||
export const models = {
|
||||
BucketInfo: require('./lib/models/BucketInfo'),
|
||||
ObjectMD: require('./lib/models/ObjectMD'),
|
||||
ObjectMDLocation: require('./lib/models/ObjectMDLocation'),
|
||||
ARN: require('./lib/models/ARN'),
|
||||
WebsiteConfiguration: require('./lib/models/WebsiteConfiguration'),
|
||||
ReplicationConfiguration: require('./lib/models/ReplicationConfiguration'),
|
||||
LifecycleConfiguration: require('./lib/models/LifecycleConfiguration'),
|
||||
LifecycleRule: require('./lib/models/LifecycleRule'),
|
||||
BucketPolicy: require('./lib/models/BucketPolicy'),
|
||||
ObjectLockConfiguration: require('./lib/models/ObjectLockConfiguration'),
|
||||
NotificationConfiguration: require('./lib/models/NotificationConfiguration'),
|
||||
};
|
||||
|
||||
export const metrics = {
|
||||
StatsClient: require('./lib/metrics/StatsClient'),
|
||||
StatsModel: require('./lib/metrics/StatsModel'),
|
||||
RedisClient: require('./lib/metrics/RedisClient'),
|
||||
ZenkoMetrics: require('./lib/metrics/ZenkoMetrics'),
|
||||
};
|
||||
|
||||
export const pensieve = {
|
||||
credentialUtils: require('./lib/executables/pensieveCreds/utils'),
|
||||
};
|
||||
|
||||
export const patches = {
|
||||
locationConstraints,
|
||||
export const stream = {
|
||||
readJSONStreamObject: require('./lib/stream/readJSONStreamObject'),
|
||||
};
|
||||
|
|
|
@ -1,28 +1,18 @@
|
|||
import cluster, { Worker } from 'cluster';
|
||||
import * as werelogs from 'werelogs';
|
||||
'use strict'; // eslint-disable-line
|
||||
|
||||
export default class Clustering {
|
||||
_size: number;
|
||||
_shutdownTimeout: number;
|
||||
_logger: werelogs.Logger;
|
||||
_shutdown: boolean;
|
||||
_workers: (Worker | undefined)[];
|
||||
_workersTimeout: (NodeJS.Timeout | undefined)[];
|
||||
_workersStatus: (number | string | undefined)[];
|
||||
_status: number;
|
||||
_exitCb?: (clustering: Clustering, exitSignal?: string) => void;
|
||||
_index?: number;
|
||||
const cluster = require('cluster');
|
||||
|
||||
class Clustering {
|
||||
/**
|
||||
* Constructor
|
||||
*
|
||||
* @param size Cluster size
|
||||
* @param logger Logger object
|
||||
* @param [shutdownTimeout=5000] Change default shutdown timeout
|
||||
* @param {number} size Cluster size
|
||||
* @param {Logger} logger Logger object
|
||||
* @param {number} [shutdownTimeout=5000] Change default shutdown timeout
|
||||
* releasing ressources
|
||||
* @return itself
|
||||
* @return {Clustering} itself
|
||||
*/
|
||||
constructor(size: number, logger: werelogs.Logger, shutdownTimeout?: number) {
|
||||
constructor(size, logger, shutdownTimeout) {
|
||||
this._size = size;
|
||||
if (size < 1) {
|
||||
throw new Error('Cluster size must be greater than or equal to 1');
|
||||
|
@ -42,6 +32,7 @@ export default class Clustering {
|
|||
* Method called after a stop() call
|
||||
*
|
||||
* @private
|
||||
* @return {undefined}
|
||||
*/
|
||||
_afterStop() {
|
||||
// Asuming all workers shutdown gracefully
|
||||
|
@ -50,11 +41,10 @@ export default class Clustering {
|
|||
for (let i = 0; i < size; ++i) {
|
||||
// If the process return an error code or killed by a signal,
|
||||
// set the status
|
||||
const status = this._workersStatus[i];
|
||||
if (typeof status === 'number') {
|
||||
this._status = status;
|
||||
if (typeof this._workersStatus[i] === 'number') {
|
||||
this._status = this._workersStatus[i];
|
||||
break;
|
||||
} else if (typeof status === 'string') {
|
||||
} else if (typeof this._workersStatus[i] === 'string') {
|
||||
this._status = 1;
|
||||
break;
|
||||
}
|
||||
|
@ -68,17 +58,13 @@ export default class Clustering {
|
|||
/**
|
||||
* Method called when a worker exited
|
||||
*
|
||||
* @param worker - Current worker
|
||||
* @param i - Worker index
|
||||
* @param code - Exit code
|
||||
* @param signal - Exit signal
|
||||
* @param {Cluster.worker} worker - Current worker
|
||||
* @param {number} i - Worker index
|
||||
* @param {number} code - Exit code
|
||||
* @param {string} signal - Exit signal
|
||||
* @return {undefined}
|
||||
*/
|
||||
_workerExited(
|
||||
worker: Worker,
|
||||
i: number,
|
||||
code: number,
|
||||
signal: string,
|
||||
) {
|
||||
_workerExited(worker, i, code, signal) {
|
||||
// If the worker:
|
||||
// - was killed by a signal
|
||||
// - return an error code
|
||||
|
@ -105,9 +91,8 @@ export default class Clustering {
|
|||
this._workersStatus[i] = undefined;
|
||||
}
|
||||
this._workers[i] = undefined;
|
||||
const timeout = this._workersTimeout[i];
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
if (this._workersTimeout[i]) {
|
||||
clearTimeout(this._workersTimeout[i]);
|
||||
this._workersTimeout[i] = undefined;
|
||||
}
|
||||
// If we don't trigger the stop method, the watchdog
|
||||
|
@ -125,28 +110,29 @@ export default class Clustering {
|
|||
/**
|
||||
* Method to start a worker
|
||||
*
|
||||
* @param i Index of the starting worker
|
||||
* @param {number} i Index of the starting worker
|
||||
* @return {undefined}
|
||||
*/
|
||||
startWorker(i: number) {
|
||||
if (!cluster.isPrimary) {
|
||||
startWorker(i) {
|
||||
if (!cluster.isMaster) {
|
||||
return;
|
||||
}
|
||||
// Fork a new worker
|
||||
this._workers[i] = cluster.fork();
|
||||
// Listen for message from the worker
|
||||
this._workers[i]!.on('message', msg => {
|
||||
this._workers[i].on('message', msg => {
|
||||
// If the worker is ready, send him his id
|
||||
if (msg === 'ready') {
|
||||
this._workers[i]!.send({ msg: 'setup', id: i });
|
||||
this._workers[i].send({ msg: 'setup', id: i });
|
||||
}
|
||||
});
|
||||
this._workers[i]!.on('exit', (code, signal) =>
|
||||
this._workerExited(this._workers[i]!, i, code, signal));
|
||||
this._workers[i].on('exit', (code, signal) =>
|
||||
this._workerExited(this._workers[i], i, code, signal));
|
||||
// Trigger when the worker was started
|
||||
this._workers[i]!.on('online', () => {
|
||||
this._workers[i].on('online', () => {
|
||||
this._logger.info('Worker started', {
|
||||
id: i,
|
||||
childPid: this._workers[i]!.process.pid,
|
||||
childPid: this._workers[i].process.pid,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
@ -154,10 +140,10 @@ export default class Clustering {
|
|||
/**
|
||||
* Method to put handler on cluster exit
|
||||
*
|
||||
* @param cb - Callback(Clustering, [exitSignal])
|
||||
* @return Itself
|
||||
* @param {function} cb - Callback(Clustering, [exitSignal])
|
||||
* @return {Clustering} Itself
|
||||
*/
|
||||
onExit(cb: (clustering: Clustering, exitSignal?: string) => void) {
|
||||
onExit(cb) {
|
||||
this._exitCb = cb;
|
||||
return this;
|
||||
}
|
||||
|
@ -166,33 +152,33 @@ export default class Clustering {
|
|||
* Method to start the cluster (if master) or to start the callback
|
||||
* (worker)
|
||||
*
|
||||
* @param cb - Callback to run the worker
|
||||
* @return itself
|
||||
* @param {function} cb - Callback to run the worker
|
||||
* @return {Clustering} itself
|
||||
*/
|
||||
start(cb: (clustering: Clustering) => void) {
|
||||
start(cb) {
|
||||
process.on('SIGINT', () => this.stop('SIGINT'));
|
||||
process.on('SIGHUP', () => this.stop('SIGHUP'));
|
||||
process.on('SIGQUIT', () => this.stop('SIGQUIT'));
|
||||
process.on('SIGTERM', () => this.stop('SIGTERM'));
|
||||
process.on('SIGPIPE', () => {});
|
||||
process.on('exit', (code?: number, signal?: string) => {
|
||||
process.on('exit', (code, signal) => {
|
||||
if (this._exitCb) {
|
||||
this._status = code || 0;
|
||||
return this._exitCb(this, signal);
|
||||
}
|
||||
return process.exit(code || 0);
|
||||
});
|
||||
process.on('uncaughtException', (err: Error) => {
|
||||
process.on('uncaughtException', err => {
|
||||
this._logger.fatal('caught error', {
|
||||
error: err.message,
|
||||
stack: err.stack?.split('\n')?.map(str => str.trim()),
|
||||
stack: err.stack.split('\n').map(str => str.trim()),
|
||||
});
|
||||
process.exit(1);
|
||||
});
|
||||
if (!cluster.isPrimary) {
|
||||
if (!cluster.isMaster) {
|
||||
// Waiting for message from master to
|
||||
// know the id of the slave cluster
|
||||
process.on('message', (msg: any) => {
|
||||
process.on('message', msg => {
|
||||
if (msg.msg === 'setup') {
|
||||
this._index = msg.id;
|
||||
cb(this);
|
||||
|
@ -200,7 +186,7 @@ export default class Clustering {
|
|||
});
|
||||
// Send message to the master, to let him know
|
||||
// the worker has started
|
||||
process.send?.('ready');
|
||||
process.send('ready');
|
||||
} else {
|
||||
for (let i = 0; i < this._size; ++i) {
|
||||
this.startWorker(i);
|
||||
|
@ -212,7 +198,7 @@ export default class Clustering {
|
|||
/**
|
||||
* Method to get workers
|
||||
*
|
||||
* @return Workers
|
||||
* @return {Cluster.Worker[]} Workers
|
||||
*/
|
||||
getWorkers() {
|
||||
return this._workers;
|
||||
|
@ -221,7 +207,7 @@ export default class Clustering {
|
|||
/**
|
||||
* Method to get the status of the cluster
|
||||
*
|
||||
* @return Status code
|
||||
* @return {number} Status code
|
||||
*/
|
||||
getStatus() {
|
||||
return this._status;
|
||||
|
@ -230,7 +216,7 @@ export default class Clustering {
|
|||
/**
|
||||
* Method to return if it's the master process
|
||||
*
|
||||
* @return - True if master, false otherwise
|
||||
* @return {boolean} - True if master, false otherwise
|
||||
*/
|
||||
isMaster() {
|
||||
return this._index === undefined;
|
||||
|
@ -239,7 +225,7 @@ export default class Clustering {
|
|||
/**
|
||||
* Method to get index of the worker
|
||||
*
|
||||
* @return Worker index, undefined if it's master
|
||||
* @return {number|undefined} Worker index, undefined if it's master
|
||||
*/
|
||||
getIndex() {
|
||||
return this._index;
|
||||
|
@ -248,10 +234,11 @@ export default class Clustering {
|
|||
/**
|
||||
* Method to stop the cluster
|
||||
*
|
||||
* @param signal - Set internally when processes killed by signal
|
||||
* @param {string} signal - Set internally when processes killed by signal
|
||||
* @return {undefined}
|
||||
*/
|
||||
stop(signal?: string) {
|
||||
if (!cluster.isPrimary) {
|
||||
stop(signal) {
|
||||
if (!cluster.isMaster) {
|
||||
if (this._exitCb) {
|
||||
return this._exitCb(this, signal);
|
||||
}
|
||||
|
@ -264,17 +251,13 @@ export default class Clustering {
|
|||
}
|
||||
this._workersTimeout[i] = setTimeout(() => {
|
||||
// Kill the worker if the sigterm was ignored or take too long
|
||||
if (worker.process.pid) {
|
||||
process.kill(worker.process.pid, 'SIGKILL');
|
||||
}
|
||||
process.kill(worker.process.pid, 'SIGKILL');
|
||||
}, this._shutdownTimeout);
|
||||
// Send sigterm to the process, allowing to release ressources
|
||||
// and save some states
|
||||
if (worker.process.pid) {
|
||||
return process.kill(worker.process.pid, 'SIGTERM');
|
||||
} else {
|
||||
return true;
|
||||
}
|
||||
return process.kill(worker.process.pid, 'SIGTERM');
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Clustering;
|
|
@ -1,363 +0,0 @@
|
|||
import { OrderedSet } from '@js-sdsl/ordered-set';
|
||||
import {
|
||||
default as GapSet,
|
||||
GapSetEntry,
|
||||
} from './GapSet';
|
||||
|
||||
// the API is similar but is not strictly a superset of GapSetInterface
|
||||
// so we don't extend from it
|
||||
export interface GapCacheInterface {
|
||||
exposureDelayMs: number;
|
||||
maxGapWeight: number;
|
||||
size: number;
|
||||
|
||||
setGap: (firstKey: string, lastKey: string, weight: number) => void;
|
||||
removeOverlappingGaps: (overlappingKeys: string[]) => number;
|
||||
lookupGap: (minKey: string, maxKey?: string) => Promise<GapSetEntry | null>;
|
||||
[Symbol.iterator]: () => Iterator<GapSetEntry>;
|
||||
toArray: () => GapSetEntry[];
|
||||
};
|
||||
|
||||
class GapCacheUpdateSet {
|
||||
newGaps: GapSet;
|
||||
updatedKeys: OrderedSet<string>;
|
||||
|
||||
constructor(maxGapWeight: number) {
|
||||
this.newGaps = new GapSet(maxGapWeight);
|
||||
this.updatedKeys = new OrderedSet();
|
||||
}
|
||||
|
||||
addUpdateBatch(updatedKeys: OrderedSet<string>): void {
|
||||
this.updatedKeys.union(updatedKeys);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Cache of listing "gaps" i.e. ranges of keys that can be skipped
|
||||
* over during listing (because they only contain delete markers as
|
||||
* latest versions).
|
||||
*
|
||||
* Typically, a single GapCache instance would be attached to a raft session.
|
||||
*
|
||||
* The API usage is as follows:
|
||||
*
|
||||
* - Initialize a GapCache instance by calling start() (this starts an internal timer)
|
||||
*
|
||||
* - Insert a gap or update an existing one via setGap()
|
||||
*
|
||||
* - Lookup existing gaps via lookupGap()
|
||||
*
|
||||
* - Invalidate gaps that overlap a specific set of keys via removeOverlappingGaps()
|
||||
*
|
||||
* - Shut down a GapCache instance by calling stop() (this stops the internal timer)
|
||||
*
|
||||
* Gaps inserted via setGap() are not exposed immediately to lookupGap(), but only:
|
||||
*
|
||||
* - after a certain delay always larger than 'exposureDelayMs' and usually shorter
|
||||
* than twice this value (but might be slightly longer in rare cases)
|
||||
*
|
||||
* - and only if they haven't been invalidated by a recent call to removeOverlappingGaps()
|
||||
*
|
||||
* This ensures atomicity between gap creation and invalidation from updates under
|
||||
* the condition that a gap is created from first key to last key within the time defined
|
||||
* by 'exposureDelayMs'.
|
||||
*
|
||||
* The implementation is based on two extra temporary "update sets" on top of the main
|
||||
* exposed gap set, one called "staging" and the other "frozen", each containing a
|
||||
* temporary updated gap set and a list of updated keys to invalidate gaps with (coming
|
||||
* from calls to removeOverlappingGaps()). Every "exposureDelayMs" milliseconds, the frozen
|
||||
* gaps are invalidated by all key updates coming from either of the "staging" or "frozen"
|
||||
* update set, then merged into the exposed gaps set, after which the staging updates become
|
||||
* the frozen updates and won't receive any new gap until the next cycle.
|
||||
*/
|
||||
export default class GapCache implements GapCacheInterface {
|
||||
_exposureDelayMs: number;
|
||||
maxGaps: number;
|
||||
|
||||
_stagingUpdates: GapCacheUpdateSet;
|
||||
_frozenUpdates: GapCacheUpdateSet;
|
||||
_exposedGaps: GapSet;
|
||||
_exposeFrozenInterval: NodeJS.Timeout | null;
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
*
|
||||
* @param {number} exposureDelayMs - minimum delay between
|
||||
* insertion of a gap via setGap() and its exposure via
|
||||
* lookupGap()
|
||||
* @param {number} maxGaps - maximum number of cached gaps, after
|
||||
* which no new gap can be added by setGap(). (Note: a future
|
||||
* improvement could replace this by an eviction strategy)
|
||||
* @param {number} maxGapWeight - maximum "weight" of individual
|
||||
* cached gaps, which is also the granularity for
|
||||
* invalidation. Individual gaps can be chained together,
|
||||
* which lookupGap() transparently consolidates in the response
|
||||
* into a single large gap.
|
||||
*/
|
||||
constructor(exposureDelayMs: number, maxGaps: number, maxGapWeight: number) {
|
||||
this._exposureDelayMs = exposureDelayMs;
|
||||
this.maxGaps = maxGaps;
|
||||
|
||||
this._stagingUpdates = new GapCacheUpdateSet(maxGapWeight);
|
||||
this._frozenUpdates = new GapCacheUpdateSet(maxGapWeight);
|
||||
this._exposedGaps = new GapSet(maxGapWeight);
|
||||
this._exposeFrozenInterval = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a GapCache from an array of exposed gap entries (used in tests)
|
||||
*
|
||||
* @return {GapCache} - a new GapCache instance
|
||||
*/
|
||||
static createFromArray(
|
||||
gaps: GapSetEntry[],
|
||||
exposureDelayMs: number,
|
||||
maxGaps: number,
|
||||
maxGapWeight: number
|
||||
): GapCache {
|
||||
const gapCache = new GapCache(exposureDelayMs, maxGaps, maxGapWeight);
|
||||
gapCache._exposedGaps = GapSet.createFromArray(gaps, maxGapWeight)
|
||||
return gapCache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal helper to remove gaps in the staging and frozen sets
|
||||
* overlapping with previously updated keys, right before the
|
||||
* frozen gaps get exposed.
|
||||
*
|
||||
* @return {undefined}
|
||||
*/
|
||||
_removeOverlappingGapsBeforeExpose(): void {
|
||||
for (const { updatedKeys } of [this._stagingUpdates, this._frozenUpdates]) {
|
||||
if (updatedKeys.size() === 0) {
|
||||
continue;
|
||||
}
|
||||
for (const { newGaps } of [this._stagingUpdates, this._frozenUpdates]) {
|
||||
if (newGaps.size === 0) {
|
||||
continue;
|
||||
}
|
||||
newGaps.removeOverlappingGaps(updatedKeys);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This function is the core mechanism that updates the exposed gaps in the
|
||||
* cache. It is called on a regular interval defined by 'exposureDelayMs'.
|
||||
*
|
||||
* It does the following in order:
|
||||
*
|
||||
* - remove gaps from the frozen set that overlap with any key present in a
|
||||
* batch passed to removeOverlappingGaps() since the last two triggers of
|
||||
* _exposeFrozen()
|
||||
*
|
||||
* - merge the remaining gaps from the frozen set to the exposed set, which
|
||||
* makes them visible from calls to lookupGap()
|
||||
*
|
||||
* - rotate by freezing the currently staging updates and initiating a new
|
||||
* staging updates set
|
||||
*
|
||||
* @return {undefined}
|
||||
*/
|
||||
_exposeFrozen(): void {
|
||||
this._removeOverlappingGapsBeforeExpose();
|
||||
for (const gap of this._frozenUpdates.newGaps) {
|
||||
// Use a trivial strategy to keep the cache size within
|
||||
// limits: refuse to add new gaps when the size is above
|
||||
// the 'maxGaps' threshold. We solely rely on
|
||||
// removeOverlappingGaps() to make space for new gaps.
|
||||
if (this._exposedGaps.size < this.maxGaps) {
|
||||
this._exposedGaps.setGap(gap.firstKey, gap.lastKey, gap.weight);
|
||||
}
|
||||
}
|
||||
this._frozenUpdates = this._stagingUpdates;
|
||||
this._stagingUpdates = new GapCacheUpdateSet(this.maxGapWeight);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the internal GapCache timer
|
||||
*
|
||||
* @return {undefined}
|
||||
*/
|
||||
start(): void {
|
||||
if (this._exposeFrozenInterval) {
|
||||
return;
|
||||
}
|
||||
this._exposeFrozenInterval = setInterval(
|
||||
() => this._exposeFrozen(),
|
||||
this._exposureDelayMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the internal GapCache timer
|
||||
*
|
||||
* @return {undefined}
|
||||
*/
|
||||
stop(): void {
|
||||
if (this._exposeFrozenInterval) {
|
||||
clearInterval(this._exposeFrozenInterval);
|
||||
this._exposeFrozenInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a gap between two keys, associated with a weight to
|
||||
* limit individual gap's spanning ranges in the cache, for a more
|
||||
* granular invalidation.
|
||||
*
|
||||
* The function handles splitting and merging existing gaps to
|
||||
* maintain an optimal weight of cache entries.
|
||||
*
|
||||
* NOTE 1: the caller must ensure that the full length of the gap
|
||||
* between 'firstKey' and 'lastKey' has been built from a listing
|
||||
* snapshot that is more recent than 'exposureDelayMs' milliseconds,
|
||||
* in order to guarantee that the exposed gap will be fully
|
||||
* covered (and potentially invalidated) from recent calls to
|
||||
* removeOverlappingGaps().
|
||||
*
|
||||
* NOTE 2: a usual pattern when building a large gap from multiple
|
||||
* calls to setGap() is to start the next gap from 'lastKey',
|
||||
* which will be passed as 'firstKey' in the next call, so that
|
||||
* gaps can be chained together and consolidated by lookupGap().
|
||||
*
|
||||
* @param {string} firstKey - first key of the gap
|
||||
* @param {string} lastKey - last key of the gap, must be greater
|
||||
* or equal than 'firstKey'
|
||||
* @param {number} weight - total weight between 'firstKey' and 'lastKey'
|
||||
* @return {undefined}
|
||||
*/
|
||||
setGap(firstKey: string, lastKey: string, weight: number): void {
|
||||
this._stagingUpdates.newGaps.setGap(firstKey, lastKey, weight);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove gaps that overlap with a given set of keys. Used to
|
||||
* invalidate gaps when keys are inserted or deleted.
|
||||
*
|
||||
* @param {OrderedSet<string> | string[]} overlappingKeys - remove gaps that
|
||||
* overlap with any of this set of keys
|
||||
* @return {number} - how many gaps were removed from the exposed
|
||||
* gaps only (overlapping gaps not yet exposed are also invalidated
|
||||
* but are not accounted for in the returned value)
|
||||
*/
|
||||
removeOverlappingGaps(overlappingKeys: OrderedSet<string> | string[]): number {
|
||||
let overlappingKeysSet;
|
||||
if (Array.isArray(overlappingKeys)) {
|
||||
overlappingKeysSet = new OrderedSet(overlappingKeys);
|
||||
} else {
|
||||
overlappingKeysSet = overlappingKeys;
|
||||
}
|
||||
this._stagingUpdates.addUpdateBatch(overlappingKeysSet);
|
||||
return this._exposedGaps.removeOverlappingGaps(overlappingKeysSet);
|
||||
}
|
||||
|
||||
/**
|
||||
* Lookup the next exposed gap that overlaps with [minKey, maxKey]. Internally
|
||||
* chained gaps are coalesced in the response into a single contiguous large gap.
|
||||
*
|
||||
* @param {string} minKey - minimum key overlapping with the returned gap
|
||||
* @param {string} [maxKey] - maximum key overlapping with the returned gap
|
||||
* @return {Promise<GapSetEntry | null>} - result of the lookup if a gap
|
||||
* was found, null otherwise, as a Promise
|
||||
*/
|
||||
lookupGap(minKey: string, maxKey?: string): Promise<GapSetEntry | null> {
|
||||
return this._exposedGaps.lookupGap(minKey, maxKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the maximum weight setting for individual gaps.
|
||||
*
|
||||
* @return {number} - maximum weight of individual gaps
|
||||
*/
|
||||
get maxGapWeight(): number {
|
||||
return this._exposedGaps.maxWeight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the maximum weight setting for individual gaps.
|
||||
*
|
||||
* @param {number} gapWeight - maximum weight of individual gaps
|
||||
*/
|
||||
set maxGapWeight(gapWeight: number) {
|
||||
this._exposedGaps.maxWeight = gapWeight;
|
||||
// also update transient gap sets
|
||||
this._stagingUpdates.newGaps.maxWeight = gapWeight;
|
||||
this._frozenUpdates.newGaps.maxWeight = gapWeight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the exposure delay in milliseconds, which is the minimum
|
||||
* time after which newly cached gaps will be exposed by
|
||||
* lookupGap().
|
||||
*
|
||||
* @return {number} - exposure delay in milliseconds
|
||||
*/
|
||||
get exposureDelayMs(): number {
|
||||
return this._exposureDelayMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the exposure delay in milliseconds, which is the minimum
|
||||
* time after which newly cached gaps will be exposed by
|
||||
* lookupGap(). Setting this attribute automatically updates the
|
||||
* internal state to honor the new value.
|
||||
*
|
||||
* @param {number} - exposure delay in milliseconds
|
||||
*/
|
||||
set exposureDelayMs(exposureDelayMs: number) {
|
||||
if (exposureDelayMs !== this._exposureDelayMs) {
|
||||
this._exposureDelayMs = exposureDelayMs;
|
||||
if (this._exposeFrozenInterval) {
|
||||
// invalidate all pending gap updates, as the new interval may not be
|
||||
// safe for them
|
||||
this._stagingUpdates = new GapCacheUpdateSet(this.maxGapWeight);
|
||||
this._frozenUpdates = new GapCacheUpdateSet(this.maxGapWeight);
|
||||
|
||||
// reinitialize the _exposeFrozenInterval timer with the updated delay
|
||||
this.stop();
|
||||
this.start();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of exposed gaps
|
||||
*
|
||||
* @return {number} number of exposed gaps
|
||||
*/
|
||||
get size(): number {
|
||||
return this._exposedGaps.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate over exposed gaps
|
||||
*
|
||||
* @return {Iterator<GapSetEntry>} an iterator over exposed gaps
|
||||
*/
|
||||
[Symbol.iterator](): Iterator<GapSetEntry> {
|
||||
return this._exposedGaps[Symbol.iterator]();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an array of all exposed gaps
|
||||
*
|
||||
* @return {GapSetEntry[]} array of exposed gaps
|
||||
*/
|
||||
toArray(): GapSetEntry[] {
|
||||
return this._exposedGaps.toArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all exposed and staging gaps from the cache.
|
||||
*
|
||||
* Note: retains invalidating updates from removeOverlappingGaps()
|
||||
* for correctness of gaps inserted afterwards.
|
||||
*
|
||||
* @return {undefined}
|
||||
*/
|
||||
clear(): void {
|
||||
this._stagingUpdates.newGaps = new GapSet(this.maxGapWeight);
|
||||
this._frozenUpdates.newGaps = new GapSet(this.maxGapWeight);
|
||||
this._exposedGaps = new GapSet(this.maxGapWeight);
|
||||
}
|
||||
}
|
|
@ -1,366 +0,0 @@
|
|||
import assert from 'assert';
|
||||
import { OrderedSet } from '@js-sdsl/ordered-set';
|
||||
|
||||
import errors from '../../errors';
|
||||
|
||||
export type GapSetEntry = {
|
||||
firstKey: string,
|
||||
lastKey: string,
|
||||
weight: number,
|
||||
};
|
||||
|
||||
export interface GapSetInterface {
|
||||
maxWeight: number;
|
||||
size: number;
|
||||
|
||||
setGap: (firstKey: string, lastKey: string, weight: number) => GapSetEntry;
|
||||
removeOverlappingGaps: (overlappingKeys: string[]) => number;
|
||||
lookupGap: (minKey: string, maxKey?: string) => Promise<GapSetEntry | null>;
|
||||
[Symbol.iterator]: () => Iterator<GapSetEntry>;
|
||||
toArray: () => GapSetEntry[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Specialized data structure to support caching of listing "gaps",
|
||||
* i.e. ranges of keys that can be skipped over during listing
|
||||
* (because they only contain delete markers as latest versions)
|
||||
*/
|
||||
export default class GapSet implements GapSetInterface, Iterable<GapSetEntry> {
|
||||
_gaps: OrderedSet<GapSetEntry>;
|
||||
_maxWeight: number;
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
|
||||
* @param {number} maxWeight - weight threshold for each cached
|
||||
* gap (unitless). Triggers splitting gaps when reached
|
||||
*/
|
||||
constructor(maxWeight: number) {
|
||||
this._gaps = new OrderedSet(
|
||||
[],
|
||||
(left: GapSetEntry, right: GapSetEntry) => (
|
||||
left.firstKey < right.firstKey ? -1 :
|
||||
left.firstKey > right.firstKey ? 1 : 0
|
||||
)
|
||||
);
|
||||
this._maxWeight = maxWeight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a GapSet from an array of gap entries (used in tests)
|
||||
*/
|
||||
static createFromArray(gaps: GapSetEntry[], maxWeight: number): GapSet {
|
||||
const gapSet = new GapSet(maxWeight);
|
||||
for (const gap of gaps) {
|
||||
gapSet._gaps.insert(gap);
|
||||
}
|
||||
return gapSet;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a gap between two keys, associated with a weight to limit
|
||||
* individual gap sizes in the cache.
|
||||
*
|
||||
* The function handles splitting and merging existing gaps to
|
||||
* maintain an optimal weight of cache entries.
|
||||
*
|
||||
* @param {string} firstKey - first key of the gap
|
||||
* @param {string} lastKey - last key of the gap, must be greater
|
||||
* or equal than 'firstKey'
|
||||
* @param {number} weight - total weight between 'firstKey' and 'lastKey'
|
||||
* @return {GapSetEntry} - existing or new gap entry
|
||||
*/
|
||||
setGap(firstKey: string, lastKey: string, weight: number): GapSetEntry {
|
||||
assert(lastKey >= firstKey);
|
||||
|
||||
// Step 1/4: Find the closest left-overlapping gap, and either re-use it
|
||||
// or chain it with a new gap depending on the weights if it exists (otherwise
|
||||
// just creates a new gap).
|
||||
const curGapIt = this._gaps.reverseLowerBound(<GapSetEntry>{ firstKey });
|
||||
let curGap;
|
||||
if (curGapIt.isAccessible()) {
|
||||
curGap = curGapIt.pointer;
|
||||
if (curGap.lastKey >= lastKey) {
|
||||
// return fully overlapping gap already cached
|
||||
return curGap;
|
||||
}
|
||||
}
|
||||
let remainingWeight = weight;
|
||||
if (!curGap // no previous gap
|
||||
|| curGap.lastKey < firstKey // previous gap not overlapping
|
||||
|| (curGap.lastKey === firstKey // previous gap overlapping by one key...
|
||||
&& curGap.weight + weight > this._maxWeight) // ...but we can't extend it
|
||||
) {
|
||||
// create a new gap indexed by 'firstKey'
|
||||
curGap = { firstKey, lastKey: firstKey, weight: 0 };
|
||||
this._gaps.insert(curGap);
|
||||
} else if (curGap.lastKey > firstKey && weight > this._maxWeight) {
|
||||
// previous gap is either fully or partially contained in the new gap
|
||||
// and cannot be extended: substract its weight from the total (heuristic
|
||||
// in case the previous gap doesn't start at 'firstKey', which is the
|
||||
// uncommon case)
|
||||
remainingWeight -= curGap.weight;
|
||||
|
||||
// there may be an existing chained gap starting with the previous gap's
|
||||
// 'lastKey': use it if it exists
|
||||
const chainedGapIt = this._gaps.find(<GapSetEntry>{ firstKey: curGap.lastKey });
|
||||
if (chainedGapIt.isAccessible()) {
|
||||
curGap = chainedGapIt.pointer;
|
||||
} else {
|
||||
// no existing chained gap: chain a new gap to the previous gap
|
||||
curGap = {
|
||||
firstKey: curGap.lastKey,
|
||||
lastKey: curGap.lastKey,
|
||||
weight: 0,
|
||||
};
|
||||
this._gaps.insert(curGap);
|
||||
}
|
||||
}
|
||||
// Step 2/4: Cleanup existing gaps fully included in firstKey -> lastKey, and
|
||||
// aggregate their weights in curGap to define the minimum weight up to the
|
||||
// last merged gap.
|
||||
let nextGap;
|
||||
while (true) {
|
||||
const nextGapIt = this._gaps.upperBound(<GapSetEntry>{ firstKey: curGap.firstKey });
|
||||
nextGap = nextGapIt.isAccessible() && nextGapIt.pointer;
|
||||
// stop the cleanup when no more gap or if the next gap is not fully
|
||||
// included in curGap
|
||||
if (!nextGap || nextGap.lastKey > lastKey) {
|
||||
break;
|
||||
}
|
||||
this._gaps.eraseElementByIterator(nextGapIt);
|
||||
curGap.lastKey = nextGap.lastKey;
|
||||
curGap.weight += nextGap.weight;
|
||||
}
|
||||
|
||||
// Step 3/4: Extend curGap to lastKey, adjusting the weight.
|
||||
// At this point, curGap weight is the minimum weight of the finished gap, save it
|
||||
// for step 4.
|
||||
let minMergedWeight = curGap.weight;
|
||||
if (curGap.lastKey === firstKey && firstKey !== lastKey) {
|
||||
// extend the existing gap by the full amount 'firstKey -> lastKey'
|
||||
curGap.lastKey = lastKey;
|
||||
curGap.weight += remainingWeight;
|
||||
} else if (curGap.lastKey <= lastKey) {
|
||||
curGap.lastKey = lastKey;
|
||||
curGap.weight = remainingWeight;
|
||||
}
|
||||
|
||||
// Step 4/4: Find the closest right-overlapping gap, and if it exists, either merge
|
||||
// it or chain it with curGap depending on the weights.
|
||||
if (nextGap && nextGap.firstKey <= lastKey) {
|
||||
// nextGap overlaps with the new gap: check if we can merge it
|
||||
minMergedWeight += nextGap.weight;
|
||||
let mergedWeight;
|
||||
if (lastKey === nextGap.firstKey) {
|
||||
// nextGap is chained with curGap: add the full weight of nextGap
|
||||
mergedWeight = curGap.weight + nextGap.weight;
|
||||
} else {
|
||||
// strict overlap: don't add nextGap's weight unless
|
||||
// it's larger than the sum of merged ranges (as it is
|
||||
// then included in `minMergedWeight`)
|
||||
mergedWeight = Math.max(curGap.weight, minMergedWeight);
|
||||
}
|
||||
if (mergedWeight <= this._maxWeight) {
|
||||
// merge nextGap into curGap
|
||||
curGap.lastKey = nextGap.lastKey;
|
||||
curGap.weight = mergedWeight;
|
||||
this._gaps.eraseElementByKey(nextGap);
|
||||
} else {
|
||||
// adjust the last key to chain with nextGap and substract the next
|
||||
// gap's weight from curGap (heuristic)
|
||||
curGap.lastKey = nextGap.firstKey;
|
||||
curGap.weight = Math.max(mergedWeight - nextGap.weight, 0);
|
||||
curGap = nextGap;
|
||||
}
|
||||
}
|
||||
// return a copy of curGap
|
||||
return Object.assign({}, curGap);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove gaps that overlap with one or more keys in a given array or
|
||||
* OrderedSet. Used to invalidate gaps when keys are inserted or deleted.
|
||||
*
|
||||
* @param {OrderedSet<string> | string[]} overlappingKeys - remove gaps that overlap
|
||||
* with any of this set of keys
|
||||
* @return {number} - how many gaps were removed
|
||||
*/
|
||||
removeOverlappingGaps(overlappingKeys: OrderedSet<string> | string[]): number {
|
||||
// To optimize processing with a large number of keys and/or gaps, this function:
|
||||
//
|
||||
// 1. converts the overlappingKeys array to a OrderedSet (if not already a OrderedSet)
|
||||
// 2. queries both the gaps set and the overlapping keys set in a loop, which allows:
|
||||
// - skipping ranges of overlapping keys at once when there is no new overlapping gap
|
||||
// - skipping ranges of gaps at once when there is no overlapping key
|
||||
//
|
||||
// This way, it is efficient when the number of non-overlapping gaps is large
|
||||
// (which is the most common case in practice).
|
||||
|
||||
let overlappingKeysSet;
|
||||
if (Array.isArray(overlappingKeys)) {
|
||||
overlappingKeysSet = new OrderedSet(overlappingKeys);
|
||||
} else {
|
||||
overlappingKeysSet = overlappingKeys;
|
||||
}
|
||||
const firstKeyIt = overlappingKeysSet.begin();
|
||||
let currentKey = firstKeyIt.isAccessible() && firstKeyIt.pointer;
|
||||
let nRemoved = 0;
|
||||
while (currentKey) {
|
||||
const closestGapIt = this._gaps.reverseUpperBound(<GapSetEntry>{ firstKey: currentKey });
|
||||
if (closestGapIt.isAccessible()) {
|
||||
const closestGap = closestGapIt.pointer;
|
||||
if (currentKey <= closestGap.lastKey) {
|
||||
// currentKey overlaps closestGap: remove the gap
|
||||
this._gaps.eraseElementByIterator(closestGapIt);
|
||||
nRemoved += 1;
|
||||
}
|
||||
}
|
||||
const nextGapIt = this._gaps.lowerBound(<GapSetEntry>{ firstKey: currentKey });
|
||||
if (!nextGapIt.isAccessible()) {
|
||||
// no more gap: we're done
|
||||
return nRemoved;
|
||||
}
|
||||
const nextGap = nextGapIt.pointer;
|
||||
// advance to the last key potentially overlapping with nextGap
|
||||
let currentKeyIt = overlappingKeysSet.reverseLowerBound(nextGap.lastKey);
|
||||
if (currentKeyIt.isAccessible()) {
|
||||
currentKey = currentKeyIt.pointer;
|
||||
if (currentKey >= nextGap.firstKey) {
|
||||
// currentKey overlaps nextGap: remove the gap
|
||||
this._gaps.eraseElementByIterator(nextGapIt);
|
||||
nRemoved += 1;
|
||||
}
|
||||
}
|
||||
// advance to the first key potentially overlapping with another gap
|
||||
currentKeyIt = overlappingKeysSet.lowerBound(nextGap.lastKey);
|
||||
currentKey = currentKeyIt.isAccessible() && currentKeyIt.pointer;
|
||||
}
|
||||
return nRemoved;
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal helper to coalesce multiple chained gaps into a single gap.
|
||||
*
|
||||
* It is only used to construct lookupGap() return values and
|
||||
* doesn't modify the GapSet.
|
||||
*
|
||||
* NOTE: The function may take a noticeable amount of time and CPU
|
||||
* to execute if a large number of chained gaps have to be
|
||||
* coalesced, but it should never take more than a few seconds. In
|
||||
* most cases it should take less than a millisecond. It regularly
|
||||
* yields to the nodejs event loop to avoid blocking it during a
|
||||
* long execution.
|
||||
*
|
||||
* @param {GapSetEntry} firstGap - first gap of the chain to coalesce with
|
||||
* the next ones in the chain
|
||||
* @return {Promise<GapSetEntry>} - a new coalesced entry, as a Promise
|
||||
*/
|
||||
_coalesceGapChain(firstGap: GapSetEntry): Promise<GapSetEntry> {
|
||||
return new Promise(resolve => {
|
||||
const coalescedGap: GapSetEntry = Object.assign({}, firstGap);
|
||||
const coalesceGapChainIteration = () => {
|
||||
// efficiency trade-off: 100 iterations of log(N) complexity lookups should
|
||||
// not block the event loop for too long
|
||||
for (let opCounter = 0; opCounter < 100; ++opCounter) {
|
||||
const chainedGapIt = this._gaps.find(
|
||||
<GapSetEntry>{ firstKey: coalescedGap.lastKey });
|
||||
if (!chainedGapIt.isAccessible()) {
|
||||
// chain is complete
|
||||
return resolve(coalescedGap);
|
||||
}
|
||||
const chainedGap = chainedGapIt.pointer;
|
||||
if (chainedGap.firstKey === chainedGap.lastKey) {
|
||||
// found a single-key gap: chain is complete
|
||||
return resolve(coalescedGap);
|
||||
}
|
||||
coalescedGap.lastKey = chainedGap.lastKey;
|
||||
coalescedGap.weight += chainedGap.weight;
|
||||
}
|
||||
// yield to the event loop before continuing the process
|
||||
// of coalescing the gap chain
|
||||
return process.nextTick(coalesceGapChainIteration);
|
||||
};
|
||||
coalesceGapChainIteration();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Lookup the next gap that overlaps with [minKey, maxKey]. Internally chained
|
||||
* gaps are coalesced in the response into a single contiguous large gap.
|
||||
*
|
||||
* @param {string} minKey - minimum key overlapping with the returned gap
|
||||
* @param {string} [maxKey] - maximum key overlapping with the returned gap
|
||||
* @return {Promise<GapSetEntry | null>} - result of the lookup if a gap
|
||||
* was found, null otherwise, as a Promise
|
||||
*/
|
||||
async lookupGap(minKey: string, maxKey?: string): Promise<GapSetEntry | null> {
|
||||
let firstGap: GapSetEntry | null = null;
|
||||
const minGapIt = this._gaps.reverseLowerBound(<GapSetEntry>{ firstKey: minKey });
|
||||
const minGap = minGapIt.isAccessible() && minGapIt.pointer;
|
||||
if (minGap && minGap.lastKey >= minKey) {
|
||||
firstGap = minGap;
|
||||
} else {
|
||||
const maxGapIt = this._gaps.upperBound(<GapSetEntry>{ firstKey: minKey });
|
||||
const maxGap = maxGapIt.isAccessible() && maxGapIt.pointer;
|
||||
if (maxGap && (maxKey === undefined || maxGap.firstKey <= maxKey)) {
|
||||
firstGap = maxGap;
|
||||
}
|
||||
}
|
||||
if (!firstGap) {
|
||||
return null;
|
||||
}
|
||||
return this._coalesceGapChain(firstGap);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the maximum weight setting for individual gaps.
|
||||
*
|
||||
* @return {number} - maximum weight of individual gaps
|
||||
*/
|
||||
get maxWeight(): number {
|
||||
return this._maxWeight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the maximum weight setting for individual gaps.
|
||||
*
|
||||
* @param {number} gapWeight - maximum weight of individual gaps
|
||||
*/
|
||||
set maxWeight(gapWeight: number) {
|
||||
this._maxWeight = gapWeight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of gaps stored in this set.
|
||||
*
|
||||
* @return {number} - number of gaps stored in this set
|
||||
*/
|
||||
get size(): number {
|
||||
return this._gaps.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate over each gap of the set, ordered by first key
|
||||
*
|
||||
* @return {Iterator<GapSetEntry>} - an iterator over all gaps
|
||||
* Example:
|
||||
* for (const gap of myGapSet) { ... }
|
||||
*/
|
||||
[Symbol.iterator](): Iterator<GapSetEntry> {
|
||||
return this._gaps[Symbol.iterator]();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an array containing all gaps, ordered by first key
|
||||
*
|
||||
* NOTE: there is a toArray() method in the OrderedSet implementation
|
||||
* but it does not scale well and overflows the stack quickly. This is
|
||||
* why we provide an implementation based on an iterator.
|
||||
*
|
||||
* @return {GapSetEntry[]} - an array containing all gaps
|
||||
*/
|
||||
toArray(): GapSetEntry[] {
|
||||
return [...this];
|
||||
}
|
||||
}
|
|
@ -1,124 +0,0 @@
|
|||
export enum HeapOrder {
|
||||
Min = -1,
|
||||
Max = 1,
|
||||
}
|
||||
|
||||
export enum CompareResult {
|
||||
LT = -1,
|
||||
EQ = 0,
|
||||
GT = 1,
|
||||
}
|
||||
|
||||
export type CompareFunction = (x: any, y: any) => CompareResult;
|
||||
|
||||
export class Heap {
|
||||
size: number;
|
||||
_maxSize: number;
|
||||
_order: HeapOrder;
|
||||
_heap: any[];
|
||||
_cmpFn: CompareFunction;
|
||||
|
||||
constructor(size: number, order: HeapOrder, cmpFn: CompareFunction) {
|
||||
this.size = 0;
|
||||
this._maxSize = size;
|
||||
this._order = order;
|
||||
this._cmpFn = cmpFn;
|
||||
this._heap = new Array<any>(this._maxSize);
|
||||
}
|
||||
|
||||
_parent(i: number): number {
|
||||
return Math.floor((i - 1) / 2);
|
||||
}
|
||||
|
||||
_left(i: number): number {
|
||||
return Math.floor((2 * i) + 1);
|
||||
}
|
||||
|
||||
_right(i: number): number {
|
||||
return Math.floor((2 * i) + 2);
|
||||
}
|
||||
|
||||
_shouldSwap(childIdx: number, parentIdx: number): boolean {
|
||||
return this._cmpFn(this._heap[childIdx], this._heap[parentIdx]) as number === this._order as number;
|
||||
}
|
||||
|
||||
_swap(i: number, j: number) {
|
||||
const tmp = this._heap[i];
|
||||
this._heap[i] = this._heap[j];
|
||||
this._heap[j] = tmp;
|
||||
}
|
||||
|
||||
_heapify(i: number) {
|
||||
const l = this._left(i);
|
||||
const r = this._right(i);
|
||||
let c = i;
|
||||
|
||||
if (l < this.size && this._shouldSwap(l, c)) {
|
||||
c = l;
|
||||
}
|
||||
|
||||
if (r < this.size && this._shouldSwap(r, c)) {
|
||||
c = r;
|
||||
}
|
||||
|
||||
if (c != i) {
|
||||
this._swap(c, i);
|
||||
this._heapify(c);
|
||||
}
|
||||
}
|
||||
|
||||
add(item: any): any {
|
||||
if (this.size >= this._maxSize) {
|
||||
return new Error('Max heap size reached');
|
||||
}
|
||||
|
||||
++this.size;
|
||||
let c = this.size - 1;
|
||||
this._heap[c] = item;
|
||||
|
||||
while (c > 0) {
|
||||
if (!this._shouldSwap(c, this._parent(c))) {
|
||||
return null;
|
||||
}
|
||||
|
||||
this._swap(c, this._parent(c));
|
||||
c = this._parent(c);
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
remove(): any {
|
||||
if (this.size <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const ret = this._heap[0];
|
||||
this._heap[0] = this._heap[this.size - 1];
|
||||
this._heapify(0);
|
||||
--this.size;
|
||||
|
||||
return ret;
|
||||
};
|
||||
|
||||
peek(): any {
|
||||
if (this.size <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return this._heap[0];
|
||||
};
|
||||
}
|
||||
|
||||
export class MinHeap extends Heap {
|
||||
constructor(size: number, cmpFn: CompareFunction) {
|
||||
super(size, HeapOrder.Min, cmpFn);
|
||||
}
|
||||
}
|
||||
|
||||
export class MaxHeap extends Heap {
|
||||
constructor(size: number, cmpFn: CompareFunction) {
|
||||
super(size, HeapOrder.Max, cmpFn);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const { FILTER_ACCEPT, SKIP_NONE } = require('./tools');
|
||||
const { FILTER_SKIP, SKIP_NONE } = require('./tools');
|
||||
|
||||
// Use a heuristic to amortize the cost of JSON
|
||||
// serialization/deserialization only on largest metadata where the
|
||||
|
@ -92,26 +92,21 @@ class Extension {
|
|||
* @param {object} entry - a listing entry from metadata
|
||||
* expected format: { key, value }
|
||||
* @return {number} - result of filtering the entry:
|
||||
* FILTER_ACCEPT: entry is accepted and may or not be included
|
||||
* in the result
|
||||
* FILTER_SKIP: listing may skip directly (with "gte" param) to
|
||||
* the key returned by the skipping() method
|
||||
* FILTER_END: the results are complete, listing can be stopped
|
||||
* > 0: entry is accepted and included in the result
|
||||
* = 0: entry is accepted but not included (skipping)
|
||||
* < 0: entry is not accepted, listing should finish
|
||||
*/
|
||||
filter(/* entry: { key, value } */) {
|
||||
return FILTER_ACCEPT;
|
||||
filter(entry) {
|
||||
return entry ? FILTER_SKIP : FILTER_SKIP;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides the next key at which the listing task is allowed to skip to.
|
||||
* This could allow to skip over:
|
||||
* - a key prefix ending with the delimiter
|
||||
* - all remaining versions of an object when doing a current
|
||||
* versions listing in v0 format
|
||||
* - a cached "gap" of deleted objects when doing a current
|
||||
* versions listing in v0 format
|
||||
* Provides the insight into why filter is skipping an entry. This could be
|
||||
* because it is skipping a range of delimited keys or a range of specific
|
||||
* version when doing master version listing.
|
||||
*
|
||||
* @return {string} - the next key at which the listing task is allowed to skip to
|
||||
* @return {string} - the insight: a common prefix or a master key,
|
||||
* or SKIP_NONE if there is no insight
|
||||
*/
|
||||
skipping() {
|
||||
return SKIP_NONE;
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const { inc, checkLimit, listingParamsMasterKeysV0ToV1,
|
||||
FILTER_END, FILTER_ACCEPT, SKIP_NONE } = require('./tools');
|
||||
FILTER_END, FILTER_ACCEPT } = require('./tools');
|
||||
const DEFAULT_MAX_KEYS = 1000;
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
|
||||
|
@ -163,7 +163,7 @@ class MultipartUploads {
|
|||
}
|
||||
|
||||
skipping() {
|
||||
return SKIP_NONE;
|
||||
return '';
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
const Extension = require('./Extension').default;
|
||||
|
||||
const { checkLimit, FILTER_END, FILTER_ACCEPT } = require('./tools');
|
||||
const { checkLimit, FILTER_END, FILTER_ACCEPT, FILTER_SKIP } = require('./tools');
|
||||
const DEFAULT_MAX_KEYS = 10000;
|
||||
|
||||
/**
|
||||
|
@ -91,7 +91,7 @@ class List extends Extension {
|
|||
* < 0 : listing done
|
||||
*/
|
||||
filter(elem) {
|
||||
// Check if the result array is full
|
||||
// Check first in case of maxkeys <= 0
|
||||
if (this.keys >= this.maxKeys) {
|
||||
return FILTER_END;
|
||||
}
|
||||
|
@ -99,7 +99,7 @@ class List extends Extension {
|
|||
this.filterKeyStartsWith !== undefined) &&
|
||||
typeof elem === 'object' &&
|
||||
!this.customFilter(elem.value)) {
|
||||
return FILTER_ACCEPT;
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
if (typeof elem === 'object') {
|
||||
this.res.push({
|
||||
|
|
|
@ -0,0 +1,274 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const Extension = require('./Extension').default;
|
||||
const { inc, listingParamsMasterKeysV0ToV1,
|
||||
FILTER_END, FILTER_ACCEPT, FILTER_SKIP } = require('./tools');
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
|
||||
|
||||
/**
|
||||
* Find the common prefix in the path
|
||||
*
|
||||
* @param {String} key - path of the object
|
||||
* @param {String} delimiter - separator
|
||||
* @param {Number} delimiterIndex - 'folder' index in the path
|
||||
* @return {String} - CommonPrefix
|
||||
*/
|
||||
function getCommonPrefix(key, delimiter, delimiterIndex) {
|
||||
return key.substring(0, delimiterIndex + delimiter.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters
|
||||
*
|
||||
* @prop {String[]} CommonPrefixes - 'folders' defined by the delimiter
|
||||
* @prop {String[]} Contents - 'files' to list
|
||||
* @prop {Boolean} IsTruncated - truncated listing flag
|
||||
* @prop {String|undefined} NextMarker - marker per amazon format
|
||||
* @prop {Number} keys - count of listed keys
|
||||
* @prop {String|undefined} delimiter - separator per amazon format
|
||||
* @prop {String|undefined} prefix - prefix per amazon format
|
||||
* @prop {Number} maxKeys - number of keys to list
|
||||
*/
|
||||
class Delimiter extends Extension {
|
||||
/**
|
||||
* Create a new Delimiter instance
|
||||
* @constructor
|
||||
* @param {Object} parameters - listing parameters
|
||||
* @param {String} [parameters.delimiter] - delimiter per amazon
|
||||
* format
|
||||
* @param {String} [parameters.prefix] - prefix per amazon
|
||||
* format
|
||||
* @param {String} [parameters.marker] - marker per amazon
|
||||
* format
|
||||
* @param {Number} [parameters.maxKeys] - number of keys to list
|
||||
* @param {Boolean} [parameters.v2] - indicates whether v2
|
||||
* format
|
||||
* @param {String} [parameters.startAfter] - marker per amazon
|
||||
* format
|
||||
* @param {String} [parameters.continuationToken] - obfuscated amazon
|
||||
* token
|
||||
* @param {Boolean} [parameters.alphabeticalOrder] - Either the result is
|
||||
* alphabetically ordered
|
||||
* or not
|
||||
* @param {RequestLogger} logger - The logger of the
|
||||
* request
|
||||
* @param {String} [vFormat] - versioning key format
|
||||
*/
|
||||
constructor(parameters, logger, vFormat) {
|
||||
super(parameters, logger);
|
||||
// original listing parameters
|
||||
this.delimiter = parameters.delimiter;
|
||||
this.prefix = parameters.prefix;
|
||||
this.marker = parameters.marker;
|
||||
this.maxKeys = parameters.maxKeys || 1000;
|
||||
this.startAfter = parameters.startAfter;
|
||||
this.continuationToken = parameters.continuationToken;
|
||||
this.alphabeticalOrder =
|
||||
typeof parameters.alphabeticalOrder !== 'undefined' ?
|
||||
parameters.alphabeticalOrder : true;
|
||||
|
||||
this.vFormat = vFormat || BucketVersioningKeyFormat.v0;
|
||||
// results
|
||||
this.CommonPrefixes = [];
|
||||
this.Contents = [];
|
||||
this.IsTruncated = false;
|
||||
this.NextMarker = parameters.marker;
|
||||
this.NextContinuationToken =
|
||||
parameters.continuationToken || parameters.startAfter;
|
||||
|
||||
this.startMarker = parameters.v2 ? 'startAfter' : 'marker';
|
||||
this.continueMarker = parameters.v2 ? 'continuationToken' : 'marker';
|
||||
this.nextContinueMarker = parameters.v2 ?
|
||||
'NextContinuationToken' : 'NextMarker';
|
||||
|
||||
if (this.delimiter !== undefined &&
|
||||
this[this.nextContinueMarker] !== undefined &&
|
||||
this[this.nextContinueMarker].startsWith(this.prefix || '')) {
|
||||
const nextDelimiterIndex =
|
||||
this[this.nextContinueMarker].indexOf(this.delimiter,
|
||||
this.prefix ? this.prefix.length : 0);
|
||||
this[this.nextContinueMarker] =
|
||||
this[this.nextContinueMarker].slice(0, nextDelimiterIndex +
|
||||
this.delimiter.length);
|
||||
}
|
||||
|
||||
Object.assign(this, {
|
||||
[BucketVersioningKeyFormat.v0]: {
|
||||
genMDParams: this.genMDParamsV0,
|
||||
getObjectKey: this.getObjectKeyV0,
|
||||
skipping: this.skippingV0,
|
||||
},
|
||||
[BucketVersioningKeyFormat.v1]: {
|
||||
genMDParams: this.genMDParamsV1,
|
||||
getObjectKey: this.getObjectKeyV1,
|
||||
skipping: this.skippingV1,
|
||||
},
|
||||
}[this.vFormat]);
|
||||
}
|
||||
|
||||
genMDParamsV0() {
|
||||
const params = {};
|
||||
if (this.prefix) {
|
||||
params.gte = this.prefix;
|
||||
params.lt = inc(this.prefix);
|
||||
}
|
||||
const startVal = this[this.continueMarker] || this[this.startMarker];
|
||||
if (startVal) {
|
||||
if (params.gte && params.gte > startVal) {
|
||||
return params;
|
||||
}
|
||||
delete params.gte;
|
||||
params.gt = startVal;
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
genMDParamsV1() {
|
||||
const params = this.genMDParamsV0();
|
||||
return listingParamsMasterKeysV0ToV1(params);
|
||||
}
|
||||
|
||||
/**
|
||||
* check if the max keys count has been reached and set the
|
||||
* final state of the result if it is the case
|
||||
* @return {Boolean} - indicates if the iteration has to stop
|
||||
*/
|
||||
_reachedMaxKeys() {
|
||||
if (this.keys >= this.maxKeys) {
|
||||
// In cases of maxKeys <= 0 -> IsTruncated = false
|
||||
this.IsTruncated = this.maxKeys > 0;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a (key, value) tuple to the listing
|
||||
* Set the NextMarker to the current key
|
||||
* Increment the keys counter
|
||||
* @param {String} key - The key to add
|
||||
* @param {String} value - The value of the key
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
addContents(key, value) {
|
||||
if (this._reachedMaxKeys()) {
|
||||
return FILTER_END;
|
||||
}
|
||||
this.Contents.push({ key, value: this.trimMetadata(value) });
|
||||
this[this.nextContinueMarker] = key;
|
||||
++this.keys;
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
|
||||
getObjectKeyV0(obj) {
|
||||
return obj.key;
|
||||
}
|
||||
|
||||
getObjectKeyV1(obj) {
|
||||
return obj.key.slice(DbPrefixes.Master.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter to apply on each iteration, based on:
|
||||
* - prefix
|
||||
* - delimiter
|
||||
* - maxKeys
|
||||
* The marker is being handled directly by levelDB
|
||||
* @param {Object} obj - The key and value of the element
|
||||
* @param {String} obj.key - The key of the element
|
||||
* @param {String} obj.value - The value of the element
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
filter(obj) {
|
||||
const key = this.getObjectKey(obj);
|
||||
const value = obj.value;
|
||||
if ((this.prefix && !key.startsWith(this.prefix))
|
||||
|| (this.alphabeticalOrder
|
||||
&& typeof this[this.nextContinueMarker] === 'string'
|
||||
&& key <= this[this.nextContinueMarker])) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
if (this.delimiter) {
|
||||
const baseIndex = this.prefix ? this.prefix.length : 0;
|
||||
const delimiterIndex = key.indexOf(this.delimiter, baseIndex);
|
||||
if (delimiterIndex === -1) {
|
||||
return this.addContents(key, value);
|
||||
}
|
||||
return this.addCommonPrefix(key, delimiterIndex);
|
||||
}
|
||||
return this.addContents(key, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a Common Prefix in the list
|
||||
* @param {String} key - object name
|
||||
* @param {Number} index - after prefix starting point
|
||||
* @return {Boolean} - indicates if iteration should continue
|
||||
*/
|
||||
addCommonPrefix(key, index) {
|
||||
const commonPrefix = getCommonPrefix(key, this.delimiter, index);
|
||||
if (this.CommonPrefixes.indexOf(commonPrefix) === -1
|
||||
&& this[this.nextContinueMarker] !== commonPrefix) {
|
||||
if (this._reachedMaxKeys()) {
|
||||
return FILTER_END;
|
||||
}
|
||||
this.CommonPrefixes.push(commonPrefix);
|
||||
this[this.nextContinueMarker] = commonPrefix;
|
||||
++this.keys;
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
|
||||
/**
|
||||
* If repd happens to want to skip listing on a bucket in v0
|
||||
* versioning key format, here is an idea.
|
||||
*
|
||||
* @return {string} - the present range (NextMarker) if repd believes
|
||||
* that it's enough and should move on
|
||||
*/
|
||||
skippingV0() {
|
||||
return this[this.nextContinueMarker];
|
||||
}
|
||||
|
||||
/**
|
||||
* If repd happens to want to skip listing on a bucket in v1
|
||||
* versioning key format, here is an idea.
|
||||
*
|
||||
* @return {string} - the present range (NextMarker) if repd believes
|
||||
* that it's enough and should move on
|
||||
*/
|
||||
skippingV1() {
|
||||
return DbPrefixes.Master + this[this.nextContinueMarker];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an object containing all mandatory fields to use once the
|
||||
* iteration is done, doesn't show a NextMarker field if the output
|
||||
* isn't truncated
|
||||
* @return {Object} - following amazon format
|
||||
*/
|
||||
result() {
|
||||
/* NextMarker is only provided when delimiter is used.
|
||||
* specified in v1 listing documentation
|
||||
* http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
|
||||
*/
|
||||
const result = {
|
||||
CommonPrefixes: this.CommonPrefixes,
|
||||
Contents: this.Contents,
|
||||
IsTruncated: this.IsTruncated,
|
||||
Delimiter: this.delimiter,
|
||||
};
|
||||
if (this.parameters.v2) {
|
||||
result.NextContinuationToken = this.IsTruncated
|
||||
? this.NextContinuationToken : undefined;
|
||||
} else {
|
||||
result.NextMarker = (this.IsTruncated && this.delimiter)
|
||||
? this.NextMarker : undefined;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { Delimiter };
|
|
@ -1,356 +0,0 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const Extension = require('./Extension').default;
|
||||
const { inc, listingParamsMasterKeysV0ToV1,
|
||||
FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } = require('./tools');
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
|
||||
|
||||
export interface FilterState {
|
||||
id: number,
|
||||
};
|
||||
|
||||
export interface FilterReturnValue {
|
||||
FILTER_ACCEPT,
|
||||
FILTER_SKIP,
|
||||
FILTER_END,
|
||||
};
|
||||
|
||||
export const enum DelimiterFilterStateId {
|
||||
NotSkipping = 1,
|
||||
SkippingPrefix = 2,
|
||||
};
|
||||
|
||||
export interface DelimiterFilterState_NotSkipping extends FilterState {
|
||||
id: DelimiterFilterStateId.NotSkipping,
|
||||
};
|
||||
|
||||
export interface DelimiterFilterState_SkippingPrefix extends FilterState {
|
||||
id: DelimiterFilterStateId.SkippingPrefix,
|
||||
prefix: string;
|
||||
};
|
||||
|
||||
type KeyHandler = (key: string, value: string) => FilterReturnValue;
|
||||
|
||||
export type ResultObject = {
|
||||
CommonPrefixes: string[];
|
||||
Contents: {
|
||||
key: string;
|
||||
value: string;
|
||||
}[];
|
||||
IsTruncated: boolean;
|
||||
Delimiter ?: string;
|
||||
NextMarker ?: string;
|
||||
NextContinuationToken ?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters
|
||||
*
|
||||
* @prop {String[]} CommonPrefixes - 'folders' defined by the delimiter
|
||||
* @prop {String[]} Contents - 'files' to list
|
||||
* @prop {Boolean} IsTruncated - truncated listing flag
|
||||
* @prop {String|undefined} NextMarker - marker per amazon format
|
||||
* @prop {Number} keys - count of listed keys
|
||||
* @prop {String|undefined} delimiter - separator per amazon format
|
||||
* @prop {String|undefined} prefix - prefix per amazon format
|
||||
* @prop {Number} maxKeys - number of keys to list
|
||||
*/
|
||||
export class Delimiter extends Extension {
|
||||
|
||||
state: FilterState;
|
||||
keyHandlers: { [id: number]: KeyHandler };
|
||||
|
||||
/**
|
||||
* Create a new Delimiter instance
|
||||
* @constructor
|
||||
* @param {Object} parameters - listing parameters
|
||||
* @param {String} [parameters.delimiter] - delimiter per amazon
|
||||
* format
|
||||
* @param {String} [parameters.prefix] - prefix per amazon
|
||||
* format
|
||||
* @param {String} [parameters.marker] - marker per amazon
|
||||
* format
|
||||
* @param {Number} [parameters.maxKeys] - number of keys to list
|
||||
* @param {Boolean} [parameters.v2] - indicates whether v2
|
||||
* format
|
||||
* @param {String} [parameters.startAfter] - marker per amazon
|
||||
* format
|
||||
* @param {String} [parameters.continuationToken] - obfuscated amazon
|
||||
* token
|
||||
* @param {RequestLogger} logger - The logger of the
|
||||
* request
|
||||
* @param {String} [vFormat] - versioning key format
|
||||
*/
|
||||
constructor(parameters, logger, vFormat) {
|
||||
super(parameters, logger);
|
||||
// original listing parameters
|
||||
this.delimiter = parameters.delimiter;
|
||||
this.prefix = parameters.prefix;
|
||||
this.maxKeys = parameters.maxKeys || 1000;
|
||||
|
||||
if (parameters.v2) {
|
||||
this.marker = parameters.continuationToken || parameters.startAfter;
|
||||
} else {
|
||||
this.marker = parameters.marker;
|
||||
}
|
||||
this.nextMarker = this.marker;
|
||||
|
||||
this.vFormat = vFormat || BucketVersioningKeyFormat.v0;
|
||||
// results
|
||||
this.CommonPrefixes = [];
|
||||
this.Contents = [];
|
||||
this.IsTruncated = false;
|
||||
this.keyHandlers = {};
|
||||
|
||||
Object.assign(this, {
|
||||
[BucketVersioningKeyFormat.v0]: {
|
||||
genMDParams: this.genMDParamsV0,
|
||||
getObjectKey: this.getObjectKeyV0,
|
||||
skipping: this.skippingV0,
|
||||
},
|
||||
[BucketVersioningKeyFormat.v1]: {
|
||||
genMDParams: this.genMDParamsV1,
|
||||
getObjectKey: this.getObjectKeyV1,
|
||||
skipping: this.skippingV1,
|
||||
},
|
||||
}[this.vFormat]);
|
||||
|
||||
// if there is a delimiter, we may skip ranges by prefix,
|
||||
// hence using the NotSkippingPrefix flavor that checks the
|
||||
// subprefix up to the delimiter for the NotSkipping state
|
||||
if (this.delimiter) {
|
||||
this.setKeyHandler(
|
||||
DelimiterFilterStateId.NotSkipping,
|
||||
this.keyHandler_NotSkippingPrefix.bind(this));
|
||||
} else {
|
||||
// listing without a delimiter never has to skip over any
|
||||
// prefix -> use NeverSkipping flavor for the NotSkipping
|
||||
// state
|
||||
this.setKeyHandler(
|
||||
DelimiterFilterStateId.NotSkipping,
|
||||
this.keyHandler_NeverSkipping.bind(this));
|
||||
}
|
||||
this.setKeyHandler(
|
||||
DelimiterFilterStateId.SkippingPrefix,
|
||||
this.keyHandler_SkippingPrefix.bind(this));
|
||||
|
||||
this.state = <DelimiterFilterState_NotSkipping> {
|
||||
id: DelimiterFilterStateId.NotSkipping,
|
||||
};
|
||||
}
|
||||
|
||||
genMDParamsV0() {
|
||||
const params: { gt ?: string, gte ?: string, lt ?: string } = {};
|
||||
if (this.prefix) {
|
||||
params.gte = this.prefix;
|
||||
params.lt = inc(this.prefix);
|
||||
}
|
||||
if (this.marker && this.delimiter) {
|
||||
const commonPrefix = this.getCommonPrefix(this.marker);
|
||||
if (commonPrefix) {
|
||||
const afterPrefix = inc(commonPrefix);
|
||||
if (!params.gte || afterPrefix > params.gte) {
|
||||
params.gte = afterPrefix;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.marker && (!params.gte || this.marker >= params.gte)) {
|
||||
delete params.gte;
|
||||
params.gt = this.marker;
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
genMDParamsV1() {
|
||||
const params = this.genMDParamsV0();
|
||||
return listingParamsMasterKeysV0ToV1(params);
|
||||
}
|
||||
|
||||
/**
|
||||
* check if the max keys count has been reached and set the
|
||||
* final state of the result if it is the case
|
||||
* @return {Boolean} - indicates if the iteration has to stop
|
||||
*/
|
||||
_reachedMaxKeys(): boolean {
|
||||
if (this.keys >= this.maxKeys) {
|
||||
// In cases of maxKeys <= 0 -> IsTruncated = false
|
||||
this.IsTruncated = this.maxKeys > 0;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a (key, value) tuple to the listing
|
||||
* Set the NextMarker to the current key
|
||||
* Increment the keys counter
|
||||
* @param {String} key - The key to add
|
||||
* @param {String} value - The value of the key
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
addContents(key: string, value: string): void {
|
||||
this.Contents.push({ key, value: this.trimMetadata(value) });
|
||||
++this.keys;
|
||||
this.nextMarker = key;
|
||||
}
|
||||
|
||||
getCommonPrefix(key: string): string | undefined {
|
||||
if (!this.delimiter) {
|
||||
return undefined;
|
||||
}
|
||||
const baseIndex = this.prefix ? this.prefix.length : 0;
|
||||
const delimiterIndex = key.indexOf(this.delimiter, baseIndex);
|
||||
if (delimiterIndex === -1) {
|
||||
return undefined;
|
||||
}
|
||||
return key.substring(0, delimiterIndex + this.delimiter.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a Common Prefix in the list
|
||||
* @param {String} commonPrefix - common prefix to add
|
||||
* @param {String} key - full key starting with commonPrefix
|
||||
* @return {Boolean} - indicates if iteration should continue
|
||||
*/
|
||||
addCommonPrefix(commonPrefix: string, key: string): void {
|
||||
// add the new prefix to the list
|
||||
this.CommonPrefixes.push(commonPrefix);
|
||||
++this.keys;
|
||||
this.nextMarker = commonPrefix;
|
||||
}
|
||||
|
||||
addCommonPrefixOrContents(key: string, value: string): string | undefined {
|
||||
// add the subprefix to the common prefixes if the key has the delimiter
|
||||
const commonPrefix = this.getCommonPrefix(key);
|
||||
if (commonPrefix) {
|
||||
this.addCommonPrefix(commonPrefix, key);
|
||||
return commonPrefix;
|
||||
}
|
||||
this.addContents(key, value);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
getObjectKeyV0(obj: { key: string }): string {
|
||||
return obj.key;
|
||||
}
|
||||
|
||||
getObjectKeyV1(obj: { key: string }): string {
|
||||
return obj.key.slice(DbPrefixes.Master.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter to apply on each iteration, based on:
|
||||
* - prefix
|
||||
* - delimiter
|
||||
* - maxKeys
|
||||
* The marker is being handled directly by levelDB
|
||||
* @param {Object} obj - The key and value of the element
|
||||
* @param {String} obj.key - The key of the element
|
||||
* @param {String} obj.value - The value of the element
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
filter(obj: { key: string, value: string }): FilterReturnValue {
|
||||
const key = this.getObjectKey(obj);
|
||||
const value = obj.value;
|
||||
|
||||
return this.handleKey(key, value);
|
||||
}
|
||||
|
||||
setState(state: FilterState): void {
|
||||
this.state = state;
|
||||
}
|
||||
|
||||
setKeyHandler(stateId: number, keyHandler: KeyHandler): void {
|
||||
this.keyHandlers[stateId] = keyHandler;
|
||||
}
|
||||
|
||||
handleKey(key: string, value: string): FilterReturnValue {
|
||||
return this.keyHandlers[this.state.id](key, value);
|
||||
}
|
||||
|
||||
keyHandler_NeverSkipping(key: string, value: string): FilterReturnValue {
|
||||
if (this._reachedMaxKeys()) {
|
||||
return FILTER_END;
|
||||
}
|
||||
this.addContents(key, value);
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
|
||||
keyHandler_NotSkippingPrefix(key: string, value: string): FilterReturnValue {
|
||||
if (this._reachedMaxKeys()) {
|
||||
return FILTER_END;
|
||||
}
|
||||
const commonPrefix = this.addCommonPrefixOrContents(key, value);
|
||||
if (commonPrefix) {
|
||||
// transition into SkippingPrefix state to skip all following keys
|
||||
// while they start with the same prefix
|
||||
this.setState(<DelimiterFilterState_SkippingPrefix> {
|
||||
id: DelimiterFilterStateId.SkippingPrefix,
|
||||
prefix: commonPrefix,
|
||||
});
|
||||
}
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
|
||||
keyHandler_SkippingPrefix(key: string, value: string): FilterReturnValue {
|
||||
const { prefix } = <DelimiterFilterState_SkippingPrefix> this.state;
|
||||
if (key.startsWith(prefix)) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
this.setState(<DelimiterFilterState_NotSkipping> {
|
||||
id: DelimiterFilterStateId.NotSkipping,
|
||||
});
|
||||
return this.handleKey(key, value);
|
||||
}
|
||||
|
||||
skippingBase(): string | undefined {
|
||||
switch (this.state.id) {
|
||||
case DelimiterFilterStateId.SkippingPrefix:
|
||||
const { prefix } = <DelimiterFilterState_SkippingPrefix> this.state;
|
||||
return inc(prefix);
|
||||
|
||||
default:
|
||||
return SKIP_NONE;
|
||||
}
|
||||
}
|
||||
|
||||
skippingV0() {
|
||||
return this.skippingBase();
|
||||
}
|
||||
|
||||
skippingV1() {
|
||||
const skipTo = this.skippingBase();
|
||||
if (skipTo === SKIP_NONE) {
|
||||
return SKIP_NONE;
|
||||
}
|
||||
return DbPrefixes.Master + skipTo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an object containing all mandatory fields to use once the
|
||||
* iteration is done, doesn't show a NextMarker field if the output
|
||||
* isn't truncated
|
||||
* @return {Object} - following amazon format
|
||||
*/
|
||||
result(): ResultObject {
|
||||
/* NextMarker is only provided when delimiter is used.
|
||||
* specified in v1 listing documentation
|
||||
* http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
|
||||
*/
|
||||
const result: ResultObject = {
|
||||
CommonPrefixes: this.CommonPrefixes,
|
||||
Contents: this.Contents,
|
||||
IsTruncated: this.IsTruncated,
|
||||
Delimiter: this.delimiter,
|
||||
};
|
||||
if (this.parameters.v2) {
|
||||
result.NextContinuationToken = this.IsTruncated
|
||||
? this.nextMarker : undefined;
|
||||
} else {
|
||||
result.NextMarker = (this.IsTruncated && this.delimiter)
|
||||
? this.nextMarker : undefined;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
|
@ -1,127 +0,0 @@
|
|||
const { DelimiterMaster } = require('./delimiterMaster');
|
||||
const { FILTER_ACCEPT, FILTER_END } = require('./tools');
|
||||
|
||||
type ResultObject = {
|
||||
Contents: {
|
||||
key: string;
|
||||
value: string;
|
||||
}[];
|
||||
IsTruncated: boolean;
|
||||
NextMarker ?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters. This extends the base class DelimiterMaster
|
||||
* to return the master/current versions.
|
||||
*/
|
||||
class DelimiterCurrent extends DelimiterMaster {
|
||||
/**
|
||||
* Delimiter listing of current versions.
|
||||
* @param {Object} parameters - listing parameters
|
||||
* @param {String} parameters.beforeDate - limit the response to keys older than beforeDate
|
||||
* @param {String} parameters.excludedDataStoreName - excluded datatore name
|
||||
* @param {Number} parameters.maxScannedLifecycleListingEntries - max number of entries to be scanned
|
||||
* @param {RequestLogger} logger - The logger of the request
|
||||
* @param {String} [vFormat] - versioning key format
|
||||
*/
|
||||
constructor(parameters, logger, vFormat) {
|
||||
super(parameters, logger, vFormat);
|
||||
|
||||
this.beforeDate = parameters.beforeDate;
|
||||
this.excludedDataStoreName = parameters.excludedDataStoreName;
|
||||
this.maxScannedLifecycleListingEntries = parameters.maxScannedLifecycleListingEntries;
|
||||
this.scannedKeys = 0;
|
||||
}
|
||||
|
||||
genMDParamsV0() {
|
||||
const params = super.genMDParamsV0();
|
||||
// lastModified and dataStoreName parameters are used by metadata that enables built-in filtering,
|
||||
// a feature currently exclusive to MongoDB
|
||||
if (this.beforeDate) {
|
||||
params.lastModified = {
|
||||
lt: this.beforeDate,
|
||||
};
|
||||
}
|
||||
|
||||
if (this.excludedDataStoreName) {
|
||||
params.dataStoreName = {
|
||||
ne: this.excludedDataStoreName,
|
||||
}
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the stringified entry's value.
|
||||
* @param s - sringified value
|
||||
* @return - undefined if parsing fails, otherwise it contains the parsed value.
|
||||
*/
|
||||
_parse(s) {
|
||||
let p;
|
||||
try {
|
||||
p = JSON.parse(s);
|
||||
} catch (e: any) {
|
||||
this.logger.warn(
|
||||
'Could not parse Object Metadata while listing',
|
||||
{ err: e.toString() });
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
/**
|
||||
* check if the max keys count has been reached and set the
|
||||
* final state of the result if it is the case
|
||||
*
|
||||
* specialized implementation on DelimiterCurrent to also check
|
||||
* the number of scanned keys
|
||||
*
|
||||
* @return {Boolean} - indicates if the iteration has to stop
|
||||
*/
|
||||
_reachedMaxKeys(): boolean {
|
||||
if (this.maxScannedLifecycleListingEntries && this.scannedKeys >= this.maxScannedLifecycleListingEntries) {
|
||||
this.IsTruncated = true;
|
||||
this.logger.info('listing stopped due to reaching the maximum scanned entries limit',
|
||||
{
|
||||
maxScannedLifecycleListingEntries: this.maxScannedLifecycleListingEntries,
|
||||
scannedKeys: this.scannedKeys,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
return super._reachedMaxKeys();
|
||||
}
|
||||
|
||||
addContents(key, value) {
|
||||
++this.scannedKeys;
|
||||
const parsedValue = this._parse(value);
|
||||
// if parsing fails, skip the key.
|
||||
if (parsedValue) {
|
||||
const lastModified = parsedValue['last-modified'];
|
||||
const dataStoreName = parsedValue.dataStoreName;
|
||||
// We then check if the current version is older than the "beforeDate" and
|
||||
// "excludedDataStoreName" is not specified or if specified and the data store name is different.
|
||||
if ((!this.beforeDate || (lastModified && lastModified < this.beforeDate)) &&
|
||||
(!this.excludedDataStoreName || dataStoreName !== this.excludedDataStoreName)) {
|
||||
super.addContents(key, value);
|
||||
}
|
||||
// In the event of a timeout occurring before any content is added,
|
||||
// NextMarker is updated even if the object is not eligible.
|
||||
// It minimizes the amount of data that the client needs to re-process if the request times out.
|
||||
this.nextMarker = key;
|
||||
}
|
||||
}
|
||||
|
||||
result(): object {
|
||||
const result: ResultObject = {
|
||||
Contents: this.Contents,
|
||||
IsTruncated: this.IsTruncated,
|
||||
};
|
||||
|
||||
if (this.IsTruncated) {
|
||||
result.NextMarker = this.nextMarker;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
module.exports = { DelimiterCurrent };
|
|
@ -0,0 +1,196 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const Delimiter = require('./delimiter').Delimiter;
|
||||
const Version = require('../../versioning/Version').Version;
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { BucketVersioningKeyFormat } = VSConst;
|
||||
const { FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } = require('./tools');
|
||||
|
||||
const VID_SEP = VSConst.VersionId.Separator;
|
||||
const { DbPrefixes } = VSConst;
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters. This extends the base class Delimiter
|
||||
* to return the raw master versions of existing objects.
|
||||
*/
|
||||
class DelimiterMaster extends Delimiter {
|
||||
/**
|
||||
* Delimiter listing of master versions.
|
||||
* @param {Object} parameters - listing parameters
|
||||
* @param {String} parameters.delimiter - delimiter per amazon format
|
||||
* @param {String} parameters.prefix - prefix per amazon format
|
||||
* @param {String} parameters.marker - marker per amazon format
|
||||
* @param {Number} parameters.maxKeys - number of keys to list
|
||||
* @param {Boolean} parameters.v2 - indicates whether v2 format
|
||||
* @param {String} parameters.startAfter - marker per amazon v2 format
|
||||
* @param {String} parameters.continuationToken - obfuscated amazon token
|
||||
* @param {RequestLogger} logger - The logger of the request
|
||||
* @param {String} [vFormat] - versioning key format
|
||||
*/
|
||||
constructor(parameters, logger, vFormat) {
|
||||
super(parameters, logger, vFormat);
|
||||
// non-PHD master version or a version whose master is a PHD version
|
||||
this.prvKey = undefined;
|
||||
this.prvPHDKey = undefined;
|
||||
this.inReplayPrefix = false;
|
||||
|
||||
Object.assign(this, {
|
||||
[BucketVersioningKeyFormat.v0]: {
|
||||
filter: this.filterV0,
|
||||
skipping: this.skippingV0,
|
||||
},
|
||||
[BucketVersioningKeyFormat.v1]: {
|
||||
filter: this.filterV1,
|
||||
skipping: this.skippingV1,
|
||||
},
|
||||
}[this.vFormat]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter to apply on each iteration for buckets in v0 format,
|
||||
* based on:
|
||||
* - prefix
|
||||
* - delimiter
|
||||
* - maxKeys
|
||||
* The marker is being handled directly by levelDB
|
||||
* @param {Object} obj - The key and value of the element
|
||||
* @param {String} obj.key - The key of the element
|
||||
* @param {String} obj.value - The value of the element
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
filterV0(obj) {
|
||||
let key = obj.key;
|
||||
const value = obj.value;
|
||||
|
||||
if (key.startsWith(DbPrefixes.Replay)) {
|
||||
this.inReplayPrefix = true;
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
this.inReplayPrefix = false;
|
||||
|
||||
/* Skip keys not starting with the prefix or not alphabetically
|
||||
* ordered. */
|
||||
if ((this.prefix && !key.startsWith(this.prefix))
|
||||
|| (typeof this[this.nextContinueMarker] === 'string' &&
|
||||
key <= this[this.nextContinueMarker])) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
|
||||
/* Skip version keys (<key><versionIdSeparator><version>) if we already
|
||||
* have a master version. */
|
||||
const versionIdIndex = key.indexOf(VID_SEP);
|
||||
if (versionIdIndex >= 0) {
|
||||
key = key.slice(0, versionIdIndex);
|
||||
/* - key === this.prvKey is triggered when a master version has
|
||||
* been accepted for this key,
|
||||
* - key === this.NextMarker or this.NextContinueToken is triggered
|
||||
* when a listing page ends on an accepted obj and the next page
|
||||
* starts with a version of this object.
|
||||
* In that case prvKey is default set to undefined
|
||||
* in the constructor and comparing to NextMarker is the only
|
||||
* way to know we should not accept this version. This test is
|
||||
* not redundant with the one at the beginning of this function,
|
||||
* we are comparing here the key without the version suffix,
|
||||
* - key startsWith the previous NextMarker happens because we set
|
||||
* NextMarker to the common prefix instead of the whole key
|
||||
* value. (TODO: remove this test once ZENKO-1048 is fixed)
|
||||
* */
|
||||
if (key === this.prvKey || key === this[this.nextContinueMarker] ||
|
||||
(this.delimiter &&
|
||||
key.startsWith(this[this.nextContinueMarker]))) {
|
||||
/* master version already filtered */
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
}
|
||||
if (Version.isPHD(value)) {
|
||||
/* master version is a PHD version, we want to wait for the next
|
||||
* one:
|
||||
* - Set the prvKey to undefined to not skip the next version,
|
||||
* - return accept to avoid users to skip the next values in range
|
||||
* (skip scan mechanism in metadata backend like Metadata or
|
||||
* MongoClient). */
|
||||
this.prvKey = undefined;
|
||||
this.prvPHDKey = key;
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
if (Version.isDeleteMarker(value)) {
|
||||
/* This entry is a deleteMarker which has not been filtered by the
|
||||
* version test. Either :
|
||||
* - it is a deleteMarker on the master version, we want to SKIP
|
||||
* all the following entries with this key (no master version),
|
||||
* - or a deleteMarker following a PHD (setting prvKey to undefined
|
||||
* when an entry is a PHD avoids the skip on version for the
|
||||
* next entry). In that case we expect the master version to
|
||||
* follow. */
|
||||
if (key === this.prvPHDKey) {
|
||||
this.prvKey = undefined;
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
this.prvKey = key;
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
|
||||
this.prvKey = key;
|
||||
if (this.delimiter) {
|
||||
// check if the key has the delimiter
|
||||
const baseIndex = this.prefix ? this.prefix.length : 0;
|
||||
const delimiterIndex = key.indexOf(this.delimiter, baseIndex);
|
||||
if (delimiterIndex >= 0) {
|
||||
// try to add the prefix to the list
|
||||
return this.addCommonPrefix(key, delimiterIndex);
|
||||
}
|
||||
}
|
||||
return this.addContents(key, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter to apply on each iteration for buckets in v1 format,
|
||||
* based on:
|
||||
* - prefix
|
||||
* - delimiter
|
||||
* - maxKeys
|
||||
* The marker is being handled directly by levelDB
|
||||
* @param {Object} obj - The key and value of the element
|
||||
* @param {String} obj.key - The key of the element
|
||||
* @param {String} obj.value - The value of the element
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
filterV1(obj) {
|
||||
// Filtering master keys in v1 is simply listing the master
|
||||
// keys, as the state of version keys do not change the
|
||||
// result, so we can use Delimiter method directly.
|
||||
return super.filter(obj);
|
||||
}
|
||||
|
||||
skippingBase() {
|
||||
if (this[this.nextContinueMarker]) {
|
||||
// next marker or next continuation token:
|
||||
// - foo/ : skipping foo/
|
||||
// - foo : skipping foo.
|
||||
const index = this[this.nextContinueMarker].
|
||||
lastIndexOf(this.delimiter);
|
||||
if (index === this[this.nextContinueMarker].length - 1) {
|
||||
return this[this.nextContinueMarker];
|
||||
}
|
||||
return this[this.nextContinueMarker] + VID_SEP;
|
||||
}
|
||||
return SKIP_NONE;
|
||||
}
|
||||
|
||||
skippingV0() {
|
||||
if (this.inReplayPrefix) {
|
||||
return DbPrefixes.Replay;
|
||||
}
|
||||
return this.skippingBase();
|
||||
}
|
||||
|
||||
skippingV1() {
|
||||
const skipTo = this.skippingBase();
|
||||
if (skipTo === SKIP_NONE) {
|
||||
return SKIP_NONE;
|
||||
}
|
||||
return DbPrefixes.Master + skipTo;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { DelimiterMaster };
|
|
@ -1,620 +0,0 @@
|
|||
import {
|
||||
Delimiter,
|
||||
FilterState,
|
||||
FilterReturnValue,
|
||||
DelimiterFilterStateId,
|
||||
DelimiterFilterState_NotSkipping,
|
||||
DelimiterFilterState_SkippingPrefix,
|
||||
ResultObject,
|
||||
} from './delimiter';
|
||||
const Version = require('../../versioning/Version').Version;
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { BucketVersioningKeyFormat } = VSConst;
|
||||
const { FILTER_ACCEPT, FILTER_SKIP, FILTER_END, SKIP_NONE, inc } = require('./tools');
|
||||
|
||||
import { GapSetEntry } from '../cache/GapSet';
|
||||
import { GapCacheInterface } from '../cache/GapCache';
|
||||
|
||||
const VID_SEP = VSConst.VersionId.Separator;
|
||||
const { DbPrefixes } = VSConst;
|
||||
|
||||
export const enum DelimiterMasterFilterStateId {
|
||||
SkippingVersionsV0 = 101,
|
||||
WaitVersionAfterPHDV0 = 102,
|
||||
SkippingGapV0 = 103,
|
||||
};
|
||||
|
||||
interface DelimiterMasterFilterState_SkippingVersionsV0 extends FilterState {
|
||||
id: DelimiterMasterFilterStateId.SkippingVersionsV0,
|
||||
masterKey: string,
|
||||
};
|
||||
|
||||
interface DelimiterMasterFilterState_WaitVersionAfterPHDV0 extends FilterState {
|
||||
id: DelimiterMasterFilterStateId.WaitVersionAfterPHDV0,
|
||||
masterKey: string,
|
||||
};
|
||||
|
||||
interface DelimiterMasterFilterState_SkippingGapV0 extends FilterState {
|
||||
id: DelimiterMasterFilterStateId.SkippingGapV0,
|
||||
};
|
||||
|
||||
export const enum GapCachingState {
|
||||
NoGapCache = 0, // there is no gap cache
|
||||
UnknownGap = 1, // waiting for a cache lookup
|
||||
GapLookupInProgress = 2, // asynchronous gap lookup in progress
|
||||
GapCached = 3, // an upcoming or already skippable gap is cached
|
||||
NoMoreGap = 4, // the cache doesn't have any more gaps inside the listed range
|
||||
};
|
||||
|
||||
type GapCachingInfo_NoGapCache = {
|
||||
state: GapCachingState.NoGapCache;
|
||||
};
|
||||
|
||||
type GapCachingInfo_NoCachedGap = {
|
||||
state: GapCachingState.UnknownGap
|
||||
| GapCachingState.GapLookupInProgress
|
||||
gapCache: GapCacheInterface;
|
||||
};
|
||||
|
||||
type GapCachingInfo_GapCached = {
|
||||
state: GapCachingState.GapCached;
|
||||
gapCache: GapCacheInterface;
|
||||
gapCached: GapSetEntry;
|
||||
};
|
||||
|
||||
type GapCachingInfo_NoMoreGap = {
|
||||
state: GapCachingState.NoMoreGap;
|
||||
};
|
||||
|
||||
type GapCachingInfo = GapCachingInfo_NoGapCache
|
||||
| GapCachingInfo_NoCachedGap
|
||||
| GapCachingInfo_GapCached
|
||||
| GapCachingInfo_NoMoreGap;
|
||||
|
||||
|
||||
export const enum GapBuildingState {
|
||||
Disabled = 0, // no gap cache or no gap building needed (e.g. in V1 versioning format)
|
||||
NotBuilding = 1, // not currently building a gap (i.e. not listing within a gap)
|
||||
Building = 2, // currently building a gap (i.e. listing within a gap)
|
||||
Expired = 3, // not allowed to build due to exposure delay timeout
|
||||
};
|
||||
|
||||
type GapBuildingInfo_NothingToBuild = {
|
||||
state: GapBuildingState.Disabled | GapBuildingState.Expired;
|
||||
};
|
||||
|
||||
type GapBuildingParams = {
|
||||
/**
|
||||
* minimum weight for a gap to be created in the cache
|
||||
*/
|
||||
minGapWeight: number;
|
||||
/**
|
||||
* trigger a cache setGap() call every N skippable keys
|
||||
*/
|
||||
triggerSaveGapWeight: number;
|
||||
/**
|
||||
* timestamp to assess whether we're still inside the validity period to
|
||||
* be allowed to build gaps
|
||||
*/
|
||||
initTimestamp: number;
|
||||
};
|
||||
|
||||
type GapBuildingInfo_NotBuilding = {
|
||||
state: GapBuildingState.NotBuilding;
|
||||
gapCache: GapCacheInterface;
|
||||
params: GapBuildingParams;
|
||||
};
|
||||
|
||||
type GapBuildingInfo_Building = {
|
||||
state: GapBuildingState.Building;
|
||||
gapCache: GapCacheInterface;
|
||||
params: GapBuildingParams;
|
||||
/**
|
||||
* Gap currently being created
|
||||
*/
|
||||
gap: GapSetEntry;
|
||||
/**
|
||||
* total current weight of the gap being created
|
||||
*/
|
||||
gapWeight: number;
|
||||
};
|
||||
|
||||
type GapBuildingInfo = GapBuildingInfo_NothingToBuild
|
||||
| GapBuildingInfo_NotBuilding
|
||||
| GapBuildingInfo_Building;
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters. This extends the base class Delimiter
|
||||
* to return the raw master versions of existing objects.
|
||||
*/
|
||||
export class DelimiterMaster extends Delimiter {
|
||||
|
||||
_gapCaching: GapCachingInfo;
|
||||
_gapBuilding: GapBuildingInfo;
|
||||
_refreshedBuildingParams: GapBuildingParams | null;
|
||||
|
||||
/**
|
||||
* Delimiter listing of master versions.
|
||||
* @param {Object} parameters - listing parameters
|
||||
* @param {String} [parameters.delimiter] - delimiter per amazon format
|
||||
* @param {String} [parameters.prefix] - prefix per amazon format
|
||||
* @param {String} [parameters.marker] - marker per amazon format
|
||||
* @param {Number} [parameters.maxKeys] - number of keys to list
|
||||
* @param {Boolean} [parameters.v2] - indicates whether v2 format
|
||||
* @param {String} [parameters.startAfter] - marker per amazon v2 format
|
||||
* @param {String} [parameters.continuationToken] - obfuscated amazon token
|
||||
* @param {RequestLogger} logger - The logger of the request
|
||||
* @param {String} [vFormat="v0"] - versioning key format
|
||||
*/
|
||||
constructor(parameters, logger, vFormat?: string) {
|
||||
super(parameters, logger, vFormat);
|
||||
|
||||
if (this.vFormat === BucketVersioningKeyFormat.v0) {
|
||||
// override Delimiter's implementation of NotSkipping for
|
||||
// DelimiterMaster logic (skipping versions and special
|
||||
// handling of delete markers and PHDs)
|
||||
this.setKeyHandler(
|
||||
DelimiterFilterStateId.NotSkipping,
|
||||
this.keyHandler_NotSkippingPrefixNorVersionsV0.bind(this));
|
||||
|
||||
// add extra state handlers specific to DelimiterMaster with v0 format
|
||||
this.setKeyHandler(
|
||||
DelimiterMasterFilterStateId.SkippingVersionsV0,
|
||||
this.keyHandler_SkippingVersionsV0.bind(this));
|
||||
|
||||
this.setKeyHandler(
|
||||
DelimiterMasterFilterStateId.WaitVersionAfterPHDV0,
|
||||
this.keyHandler_WaitVersionAfterPHDV0.bind(this));
|
||||
|
||||
this.setKeyHandler(
|
||||
DelimiterMasterFilterStateId.SkippingGapV0,
|
||||
this.keyHandler_SkippingGapV0.bind(this));
|
||||
|
||||
if (this.marker) {
|
||||
// distinct initial state to include some special logic
|
||||
// before the first master key is found that does not have
|
||||
// to be checked afterwards
|
||||
this.state = <DelimiterMasterFilterState_SkippingVersionsV0> {
|
||||
id: DelimiterMasterFilterStateId.SkippingVersionsV0,
|
||||
masterKey: this.marker,
|
||||
};
|
||||
} else {
|
||||
this.state = <DelimiterFilterState_NotSkipping> {
|
||||
id: DelimiterFilterStateId.NotSkipping,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
// save base implementation of the `NotSkipping` state in
|
||||
// Delimiter before overriding it with ours, to be able to call it from there
|
||||
this.keyHandler_NotSkipping_Delimiter = this.keyHandlers[DelimiterFilterStateId.NotSkipping];
|
||||
this.setKeyHandler(
|
||||
DelimiterFilterStateId.NotSkipping,
|
||||
this.keyHandler_NotSkippingPrefixNorVersionsV1.bind(this));
|
||||
}
|
||||
// in v1, we can directly use Delimiter's implementation,
|
||||
// which is already set to the proper state
|
||||
|
||||
// default initialization of the gap cache and building states, can be
|
||||
// set by refreshGapCache()
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.NoGapCache,
|
||||
};
|
||||
this._gapBuilding = {
|
||||
state: GapBuildingState.Disabled,
|
||||
};
|
||||
this._refreshedBuildingParams = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the validity period left before a refresh of the gap cache is needed
|
||||
* to continue building new gaps.
|
||||
*
|
||||
* @return {number|null} one of:
|
||||
* - the remaining time in milliseconds in which gaps can be added to the
|
||||
* cache before a call to refreshGapCache() is required
|
||||
* - or 0 if there is no time left and a call to refreshGapCache() is required
|
||||
* to resume caching gaps
|
||||
* - or null if refreshing the cache is never needed (because the gap cache
|
||||
* is either not available or not used)
|
||||
*/
|
||||
getGapBuildingValidityPeriodMs(): number | null {
|
||||
let gapBuilding;
|
||||
switch (this._gapBuilding.state) {
|
||||
case GapBuildingState.Disabled:
|
||||
return null;
|
||||
case GapBuildingState.Expired:
|
||||
return 0;
|
||||
case GapBuildingState.NotBuilding:
|
||||
gapBuilding = <GapBuildingInfo_NotBuilding> this._gapBuilding;
|
||||
break;
|
||||
case GapBuildingState.Building:
|
||||
gapBuilding = <GapBuildingInfo_Building> this._gapBuilding;
|
||||
break;
|
||||
}
|
||||
const { gapCache, params } = gapBuilding;
|
||||
const elapsedTime = Date.now() - params.initTimestamp;
|
||||
return Math.max(gapCache.exposureDelayMs - elapsedTime, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh the gaps caching logic (gaps are series of current delete markers
|
||||
* in V0 bucket metadata format). It has two effects:
|
||||
*
|
||||
* - starts exposing existing and future gaps from the cache to efficiently
|
||||
* skip over series of current delete markers that have been seen and cached
|
||||
* earlier
|
||||
*
|
||||
* - enables building and caching new gaps (or extend existing ones), for a
|
||||
* limited time period defined by the `gapCacheProxy.exposureDelayMs` value
|
||||
* in milliseconds. To refresh the validity period and resume building and
|
||||
* caching new gaps, one must restart a new listing from the database (starting
|
||||
* at the current listing key, included), then call refreshGapCache() again.
|
||||
*
|
||||
* @param {GapCacheInterface} gapCacheProxy - API proxy to the gaps cache
|
||||
* (the proxy should handle prefixing object keys with the bucket name)
|
||||
* @param {number} [minGapWeight=100] - minimum weight of a gap for it to be
|
||||
* added in the cache
|
||||
* @param {number} [triggerSaveGapWeight] - cumulative weight to wait for
|
||||
* before saving the current building gap. Cannot be greater than
|
||||
* `gapCacheProxy.maxGapWeight` (the value is thresholded to `maxGapWeight`
|
||||
* otherwise). Defaults to `gapCacheProxy.maxGapWeight / 2`.
|
||||
* @return {undefined}
|
||||
*/
|
||||
refreshGapCache(
|
||||
gapCacheProxy: GapCacheInterface,
|
||||
minGapWeight?: number,
|
||||
triggerSaveGapWeight?: number
|
||||
): void {
|
||||
if (this.vFormat !== BucketVersioningKeyFormat.v0) {
|
||||
return;
|
||||
}
|
||||
if (this._gapCaching.state === GapCachingState.NoGapCache) {
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.UnknownGap,
|
||||
gapCache: gapCacheProxy,
|
||||
};
|
||||
}
|
||||
const refreshedBuildingParams: GapBuildingParams = {
|
||||
minGapWeight: minGapWeight || 100,
|
||||
triggerSaveGapWeight: triggerSaveGapWeight
|
||||
|| Math.trunc(gapCacheProxy.maxGapWeight / 2),
|
||||
initTimestamp: Date.now(),
|
||||
};
|
||||
if (this._gapBuilding.state === GapBuildingState.Building) {
|
||||
// refreshed params will be applied as soon as the current building gap is saved
|
||||
this._refreshedBuildingParams = refreshedBuildingParams;
|
||||
} else {
|
||||
this._gapBuilding = {
|
||||
state: GapBuildingState.NotBuilding,
|
||||
gapCache: gapCacheProxy,
|
||||
params: refreshedBuildingParams,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger a lookup of the closest upcoming or already skippable gap.
|
||||
*
|
||||
* @param {string} fromKey - lookup a gap not before 'fromKey'
|
||||
* @return {undefined} - the lookup is asynchronous and its
|
||||
* response is handled inside this function
|
||||
*/
|
||||
_triggerGapLookup(gapCaching: GapCachingInfo_NoCachedGap, fromKey: string): void {
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.GapLookupInProgress,
|
||||
gapCache: gapCaching.gapCache,
|
||||
};
|
||||
const maxKey = this.prefix ? inc(this.prefix) : undefined;
|
||||
gapCaching.gapCache.lookupGap(fromKey, maxKey).then(_gap => {
|
||||
const gap = <GapSetEntry | null> _gap;
|
||||
if (gap) {
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.GapCached,
|
||||
gapCache: gapCaching.gapCache,
|
||||
gapCached: gap,
|
||||
};
|
||||
} else {
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.NoMoreGap,
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
_checkGapOnMasterDeleteMarker(key: string): FilterReturnValue {
|
||||
switch (this._gapBuilding.state) {
|
||||
case GapBuildingState.Disabled:
|
||||
case GapBuildingState.Expired:
|
||||
break;
|
||||
case GapBuildingState.NotBuilding:
|
||||
this._createBuildingGap(key, 1);
|
||||
break;
|
||||
case GapBuildingState.Building:
|
||||
this._updateBuildingGap(key);
|
||||
break;
|
||||
}
|
||||
if (this._gapCaching.state === GapCachingState.GapCached) {
|
||||
const { gapCached } = this._gapCaching;
|
||||
if (key >= gapCached.firstKey) {
|
||||
if (key <= gapCached.lastKey) {
|
||||
// we are inside the last looked up cached gap: transition to
|
||||
// 'SkippingGapV0' state
|
||||
this.setState(<DelimiterMasterFilterState_SkippingGapV0> {
|
||||
id: DelimiterMasterFilterStateId.SkippingGapV0,
|
||||
});
|
||||
// cut the current gap before skipping, it will be merged or
|
||||
// chained with the existing one (depending on its weight)
|
||||
if (this._gapBuilding.state === GapBuildingState.Building) {
|
||||
// substract 1 from the weight because we are going to chain this gap,
|
||||
// which has an overlap of one key.
|
||||
this._gapBuilding.gap.weight -= 1;
|
||||
this._cutBuildingGap();
|
||||
}
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
// as we are past the cached gap, we will need another lookup
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.UnknownGap,
|
||||
gapCache: this._gapCaching.gapCache,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (this._gapCaching.state === GapCachingState.UnknownGap) {
|
||||
this._triggerGapLookup(this._gapCaching, key);
|
||||
}
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
|
||||
filter_onNewMasterKeyV0(key: string, value: string): FilterReturnValue {
|
||||
// if this master key is a delete marker, accept it without
|
||||
// adding the version to the contents
|
||||
if (Version.isDeleteMarker(value)) {
|
||||
// update the state to start skipping versions of the new master key
|
||||
this.setState(<DelimiterMasterFilterState_SkippingVersionsV0> {
|
||||
id: DelimiterMasterFilterStateId.SkippingVersionsV0,
|
||||
masterKey: key,
|
||||
});
|
||||
return this._checkGapOnMasterDeleteMarker(key);
|
||||
}
|
||||
if (Version.isPHD(value)) {
|
||||
// master version is a PHD version: wait for the first
|
||||
// following version that will be considered as the actual
|
||||
// master key
|
||||
this.setState(<DelimiterMasterFilterState_WaitVersionAfterPHDV0> {
|
||||
id: DelimiterMasterFilterStateId.WaitVersionAfterPHDV0,
|
||||
masterKey: key,
|
||||
});
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
// cut the current gap as soon as a non-deleted entry is seen
|
||||
this._cutBuildingGap();
|
||||
|
||||
if (key.startsWith(DbPrefixes.Replay)) {
|
||||
// skip internal replay prefix entirely
|
||||
this.setState(<DelimiterFilterState_SkippingPrefix> {
|
||||
id: DelimiterFilterStateId.SkippingPrefix,
|
||||
prefix: DbPrefixes.Replay,
|
||||
});
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
if (this._reachedMaxKeys()) {
|
||||
return FILTER_END;
|
||||
}
|
||||
|
||||
const commonPrefix = this.addCommonPrefixOrContents(key, value);
|
||||
if (commonPrefix) {
|
||||
// transition into SkippingPrefix state to skip all following keys
|
||||
// while they start with the same prefix
|
||||
this.setState(<DelimiterFilterState_SkippingPrefix> {
|
||||
id: DelimiterFilterStateId.SkippingPrefix,
|
||||
prefix: commonPrefix,
|
||||
});
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
// update the state to start skipping versions of the new master key
|
||||
this.setState(<DelimiterMasterFilterState_SkippingVersionsV0> {
|
||||
id: DelimiterMasterFilterStateId.SkippingVersionsV0,
|
||||
masterKey: key,
|
||||
});
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
|
||||
keyHandler_NotSkippingPrefixNorVersionsV0(key: string, value: string): FilterReturnValue {
|
||||
return this.filter_onNewMasterKeyV0(key, value);
|
||||
}
|
||||
|
||||
filter_onNewMasterKeyV1(key: string, value: string): FilterReturnValue {
|
||||
// if this master key is a delete marker, accept it without
|
||||
// adding the version to the contents
|
||||
if (Version.isDeleteMarker(value)) {
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
// use base Delimiter's implementation
|
||||
return this.keyHandler_NotSkipping_Delimiter(key, value);
|
||||
}
|
||||
|
||||
keyHandler_NotSkippingPrefixNorVersionsV1(key: string, value: string): FilterReturnValue {
|
||||
return this.filter_onNewMasterKeyV1(key, value);
|
||||
}
|
||||
|
||||
keyHandler_SkippingVersionsV0(key: string, value: string): FilterReturnValue {
|
||||
/* In the SkippingVersionsV0 state, skip all version keys
|
||||
* (<key><versionIdSeparator><version>) */
|
||||
const versionIdIndex = key.indexOf(VID_SEP);
|
||||
if (versionIdIndex !== -1) {
|
||||
// version keys count in the building gap weight because they must
|
||||
// also be listed until skipped
|
||||
if (this._gapBuilding.state === GapBuildingState.Building) {
|
||||
this._updateBuildingGap(key);
|
||||
}
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
return this.filter_onNewMasterKeyV0(key, value);
|
||||
}
|
||||
|
||||
keyHandler_WaitVersionAfterPHDV0(key: string, value: string): FilterReturnValue {
|
||||
// After a PHD key is encountered, the next version key of the
|
||||
// same object if it exists is the new master key, hence
|
||||
// consider it as such and call 'onNewMasterKeyV0' (the test
|
||||
// 'masterKey == phdKey' is probably redundant when we already
|
||||
// know we have a versioned key, since all objects in v0 have
|
||||
// a master key, but keeping it in doubt)
|
||||
const { masterKey: phdKey } = <DelimiterMasterFilterState_WaitVersionAfterPHDV0> this.state;
|
||||
const versionIdIndex = key.indexOf(VID_SEP);
|
||||
if (versionIdIndex !== -1) {
|
||||
const masterKey = key.slice(0, versionIdIndex);
|
||||
if (masterKey === phdKey) {
|
||||
return this.filter_onNewMasterKeyV0(masterKey, value);
|
||||
}
|
||||
}
|
||||
return this.filter_onNewMasterKeyV0(key, value);
|
||||
}
|
||||
|
||||
keyHandler_SkippingGapV0(key: string, value: string): FilterReturnValue {
|
||||
const { gapCache, gapCached } = <GapCachingInfo_GapCached> this._gapCaching;
|
||||
if (key <= gapCached.lastKey) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.UnknownGap,
|
||||
gapCache,
|
||||
};
|
||||
this.setState(<DelimiterMasterFilterState_SkippingVersionsV0> {
|
||||
id: DelimiterMasterFilterStateId.SkippingVersionsV0,
|
||||
});
|
||||
// Start a gap with weight=0 from the latest skippable key. This will
|
||||
// allow to extend the gap just skipped with a chained gap in case
|
||||
// other delete markers are seen after the existing gap is skipped.
|
||||
this._createBuildingGap(gapCached.lastKey, 0, gapCached.weight);
|
||||
|
||||
return this.handleKey(key, value);
|
||||
}
|
||||
|
||||
skippingBase(): string | undefined {
|
||||
switch (this.state.id) {
|
||||
case DelimiterMasterFilterStateId.SkippingVersionsV0:
|
||||
const { masterKey } = <DelimiterMasterFilterState_SkippingVersionsV0> this.state;
|
||||
return masterKey + inc(VID_SEP);
|
||||
|
||||
case DelimiterMasterFilterStateId.SkippingGapV0:
|
||||
const { gapCached } = <GapCachingInfo_GapCached> this._gapCaching;
|
||||
return gapCached.lastKey;
|
||||
|
||||
default:
|
||||
return super.skippingBase();
|
||||
}
|
||||
}
|
||||
|
||||
result(): ResultObject {
|
||||
this._cutBuildingGap();
|
||||
return super.result();
|
||||
}
|
||||
|
||||
_checkRefreshedBuildingParams(params: GapBuildingParams): GapBuildingParams {
|
||||
if (this._refreshedBuildingParams) {
|
||||
const newParams = this._refreshedBuildingParams;
|
||||
this._refreshedBuildingParams = null;
|
||||
return newParams;
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the gap being built if allowed (i.e. still within the
|
||||
* allocated exposure time window).
|
||||
*
|
||||
* @return {boolean} - true if the gap was saved, false if we are
|
||||
* outside the allocated exposure time window.
|
||||
*/
|
||||
_saveBuildingGap(): boolean {
|
||||
const { gapCache, params, gap, gapWeight } =
|
||||
<GapBuildingInfo_Building> this._gapBuilding;
|
||||
const totalElapsed = Date.now() - params.initTimestamp;
|
||||
if (totalElapsed >= gapCache.exposureDelayMs) {
|
||||
this._gapBuilding = {
|
||||
state: GapBuildingState.Expired,
|
||||
};
|
||||
this._refreshedBuildingParams = null;
|
||||
return false;
|
||||
}
|
||||
const { firstKey, lastKey, weight } = gap;
|
||||
gapCache.setGap(firstKey, lastKey, weight);
|
||||
this._gapBuilding = {
|
||||
state: GapBuildingState.Building,
|
||||
gapCache,
|
||||
params: this._checkRefreshedBuildingParams(params),
|
||||
gap: {
|
||||
firstKey: gap.lastKey,
|
||||
lastKey: gap.lastKey,
|
||||
weight: 0,
|
||||
},
|
||||
gapWeight,
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new gap to be extended afterwards
|
||||
*
|
||||
* @param {string} newKey - gap's first key
|
||||
* @param {number} startWeight - initial weight of the building gap (usually 0 or 1)
|
||||
* @param {number} [cachedWeight] - if continuing a cached gap, weight of the existing
|
||||
* cached portion
|
||||
* @return {undefined}
|
||||
*/
|
||||
_createBuildingGap(newKey: string, startWeight: number, cachedWeight?: number): void {
|
||||
if (this._gapBuilding.state === GapBuildingState.NotBuilding) {
|
||||
const { gapCache, params } = <GapBuildingInfo_NotBuilding> this._gapBuilding;
|
||||
this._gapBuilding = {
|
||||
state: GapBuildingState.Building,
|
||||
gapCache,
|
||||
params: this._checkRefreshedBuildingParams(params),
|
||||
gap: {
|
||||
firstKey: newKey,
|
||||
lastKey: newKey,
|
||||
weight: startWeight,
|
||||
},
|
||||
gapWeight: (cachedWeight || 0) + startWeight,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
_updateBuildingGap(newKey: string): void {
|
||||
const gapBuilding = <GapBuildingInfo_Building> this._gapBuilding;
|
||||
const { params, gap } = gapBuilding;
|
||||
gap.lastKey = newKey;
|
||||
gap.weight += 1;
|
||||
gapBuilding.gapWeight += 1;
|
||||
// the GapCache API requires updating a gap regularly because it can only split
|
||||
// it once per update, by the known last key. In practice the default behavior
|
||||
// is to trigger an update after a number of keys that is half the maximum weight.
|
||||
// It is also useful for other listings to benefit from the cache sooner.
|
||||
if (gapBuilding.gapWeight >= params.minGapWeight &&
|
||||
gap.weight >= params.triggerSaveGapWeight) {
|
||||
this._saveBuildingGap();
|
||||
}
|
||||
}
|
||||
|
||||
_cutBuildingGap(): void {
|
||||
if (this._gapBuilding.state === GapBuildingState.Building) {
|
||||
let gapBuilding = <GapBuildingInfo_Building> this._gapBuilding;
|
||||
let { gapCache, params, gap, gapWeight } = gapBuilding;
|
||||
// only set gaps that are significant enough in weight and
|
||||
// with a non-empty extension
|
||||
if (gapWeight >= params.minGapWeight && gap.weight > 0) {
|
||||
// we're done if we were not allowed to save the gap
|
||||
if (!this._saveBuildingGap()) {
|
||||
return;
|
||||
}
|
||||
// params may have been refreshed, reload them
|
||||
gapBuilding = <GapBuildingInfo_Building> this._gapBuilding;
|
||||
params = gapBuilding.params;
|
||||
}
|
||||
this._gapBuilding = {
|
||||
state: GapBuildingState.NotBuilding,
|
||||
gapCache,
|
||||
params,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,202 +0,0 @@
|
|||
const { DelimiterVersions } = require('./delimiterVersions');
|
||||
const { FILTER_END, FILTER_SKIP } = require('./tools');
|
||||
|
||||
const TRIM_METADATA_MIN_BLOB_SIZE = 10000;
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters. This extends the base class DelimiterVersions
|
||||
* to return the raw non-current versions objects.
|
||||
*/
|
||||
class DelimiterNonCurrent extends DelimiterVersions {
|
||||
/**
|
||||
* Delimiter listing of non-current versions.
|
||||
* @param {Object} parameters - listing parameters
|
||||
* @param {String} parameters.keyMarker - key marker
|
||||
* @param {String} parameters.versionIdMarker - version id marker
|
||||
* @param {String} parameters.beforeDate - limit the response to keys with stale date older than beforeDate.
|
||||
* “stale date” is the date on when a version becomes non-current.
|
||||
* @param {Number} parameters.maxScannedLifecycleListingEntries - max number of entries to be scanned
|
||||
* @param {String} parameters.excludedDataStoreName - exclude dataStoreName matches from the versions
|
||||
* @param {RequestLogger} logger - The logger of the request
|
||||
* @param {String} [vFormat] - versioning key format
|
||||
*/
|
||||
constructor(parameters, logger, vFormat) {
|
||||
super(parameters, logger, vFormat);
|
||||
|
||||
this.beforeDate = parameters.beforeDate;
|
||||
this.excludedDataStoreName = parameters.excludedDataStoreName;
|
||||
this.maxScannedLifecycleListingEntries = parameters.maxScannedLifecycleListingEntries;
|
||||
|
||||
// internal state
|
||||
this.prevKey = null;
|
||||
this.staleDate = null;
|
||||
|
||||
this.scannedKeys = 0;
|
||||
}
|
||||
|
||||
getLastModified(value) {
|
||||
let lastModified;
|
||||
try {
|
||||
const v = JSON.parse(value);
|
||||
lastModified = v['last-modified'];
|
||||
} catch (e) {
|
||||
this.logger.warn('could not parse Object Metadata while listing',
|
||||
{
|
||||
method: 'getLastModified',
|
||||
err: e.toString(),
|
||||
});
|
||||
}
|
||||
return lastModified;
|
||||
}
|
||||
|
||||
// Overwrite keyHandler_SkippingVersions to include the last version from the previous listing.
|
||||
// The creation (last-modified) date of this version will be the stale date for the following version.
|
||||
// eslint-disable-next-line camelcase
|
||||
keyHandler_SkippingVersions(key, versionId, value) {
|
||||
if (key === this.keyMarker) {
|
||||
// since the nonversioned key equals the marker, there is
|
||||
// necessarily a versionId in this key
|
||||
const _versionId = versionId;
|
||||
if (_versionId < this.versionIdMarker) {
|
||||
// skip all versions until marker
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
}
|
||||
this.setState({
|
||||
id: 1 /* NotSkipping */,
|
||||
});
|
||||
return this.handleKey(key, versionId, value);
|
||||
}
|
||||
|
||||
filter(obj) {
|
||||
if (this.maxScannedLifecycleListingEntries && this.scannedKeys >= this.maxScannedLifecycleListingEntries) {
|
||||
this.IsTruncated = true;
|
||||
this.logger.info('listing stopped due to reaching the maximum scanned entries limit',
|
||||
{
|
||||
maxScannedLifecycleListingEntries: this.maxScannedLifecycleListingEntries,
|
||||
scannedKeys: this.scannedKeys,
|
||||
});
|
||||
return FILTER_END;
|
||||
}
|
||||
++this.scannedKeys;
|
||||
return super.filter(obj);
|
||||
}
|
||||
|
||||
/**
|
||||
* NOTE: Each version of a specific key is sorted from the latest to the oldest
|
||||
* thanks to the way version ids are generated.
|
||||
* DESCRIPTION: Skip the version if it represents the master key, but keep its last-modified date in memory,
|
||||
* which will be the stale date of the following version.
|
||||
* The following version is pushed only:
|
||||
* - if the "stale date" (picked up from the previous version) is available (JSON.parse has not failed),
|
||||
* - if "beforeDate" is not specified or if specified and the "stale date" is older.
|
||||
* - if "excludedDataStoreName" is not specified or if specified and the data store name is different
|
||||
* The in-memory "stale date" is then updated with the version's last-modified date to be used for
|
||||
* the following version.
|
||||
* The process stops and returns the available results if either:
|
||||
* - no more metadata key is left to be processed
|
||||
* - the listing reaches the maximum number of key to be returned
|
||||
* - the internal timeout is reached
|
||||
* @param {String} key - The key to add
|
||||
* @param {String} versionId - The version id
|
||||
* @param {String} value - The value of the key
|
||||
* @return {undefined}
|
||||
*/
|
||||
addVersion(key, versionId, value) {
|
||||
this.nextKeyMarker = key;
|
||||
this.nextVersionIdMarker = versionId;
|
||||
|
||||
// Skip the version if it represents the non-current version, but keep its last-modified date,
|
||||
// which will be the stale date of the following version.
|
||||
const isCurrentVersion = key !== this.prevKey;
|
||||
if (isCurrentVersion) {
|
||||
this.staleDate = this.getLastModified(value);
|
||||
this.prevKey = key;
|
||||
return;
|
||||
}
|
||||
|
||||
// The following version is pushed only:
|
||||
// - if the "stale date" (picked up from the previous version) is available (JSON.parse has not failed),
|
||||
// - if "beforeDate" is not specified or if specified and the "stale date" is older.
|
||||
// - if "excludedDataStoreName" is not specified or if specified and the data store name is different
|
||||
let lastModified;
|
||||
if (this.staleDate && (!this.beforeDate || this.staleDate < this.beforeDate)) {
|
||||
const parsedValue = this._parse(value);
|
||||
// if parsing fails, skip the key.
|
||||
if (parsedValue) {
|
||||
const dataStoreName = parsedValue.dataStoreName;
|
||||
lastModified = parsedValue['last-modified'];
|
||||
if (!this.excludedDataStoreName || dataStoreName !== this.excludedDataStoreName) {
|
||||
const s = this._stringify(parsedValue, this.staleDate);
|
||||
// check that _stringify succeeds to only push objects with a defined staleDate.
|
||||
if (s) {
|
||||
this.Versions.push({ key, value: s });
|
||||
++this.keys;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The in-memory "stale date" is then updated with the version's last-modified date to be used for
|
||||
// the following version.
|
||||
this.staleDate = lastModified || this.getLastModified(value);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the stringified entry's value and remove the location property if too large.
|
||||
* @param {string} s - sringified value
|
||||
* @return {object} p - undefined if parsing fails, otherwise it contains the parsed value.
|
||||
*/
|
||||
_parse(s) {
|
||||
let p;
|
||||
try {
|
||||
p = JSON.parse(s);
|
||||
if (s.length >= TRIM_METADATA_MIN_BLOB_SIZE) {
|
||||
delete p.location;
|
||||
}
|
||||
} catch (e) {
|
||||
this.logger.warn('Could not parse Object Metadata while listing', {
|
||||
method: 'DelimiterNonCurrent._parse',
|
||||
err: e.toString(),
|
||||
});
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
_stringify(parsedMD, staleDate) {
|
||||
const p = parsedMD;
|
||||
let s = undefined;
|
||||
p.staleDate = staleDate;
|
||||
try {
|
||||
s = JSON.stringify(p);
|
||||
} catch (e) {
|
||||
this.logger.warn('could not stringify Object Metadata while listing', {
|
||||
method: 'DelimiterNonCurrent._stringify',
|
||||
err: e.toString(),
|
||||
});
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
result() {
|
||||
const { Versions, IsTruncated, NextKeyMarker, NextVersionIdMarker } = super.result();
|
||||
|
||||
const result = {
|
||||
Contents: Versions,
|
||||
IsTruncated,
|
||||
};
|
||||
|
||||
if (NextKeyMarker) {
|
||||
result.NextKeyMarker = NextKeyMarker;
|
||||
}
|
||||
|
||||
if (NextVersionIdMarker) {
|
||||
result.NextVersionIdMarker = NextVersionIdMarker;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
module.exports = { DelimiterNonCurrent };
|
|
@ -1,204 +0,0 @@
|
|||
const DelimiterVersions = require('./delimiterVersions').DelimiterVersions;
|
||||
const { FILTER_END } = require('./tools');
|
||||
const TRIM_METADATA_MIN_BLOB_SIZE = 10000;
|
||||
/**
|
||||
* Handle object listing with parameters. This extends the base class DelimiterVersions
|
||||
* to return the orphan delete markers. Orphan delete markers are also
|
||||
* refered as expired object delete marker.
|
||||
* They are delete marker with zero noncurrent versions.
|
||||
*/
|
||||
class DelimiterOrphanDeleteMarker extends DelimiterVersions {
|
||||
/**
|
||||
* Delimiter listing of orphan delete markers.
|
||||
* @param {Object} parameters - listing parameters
|
||||
* @param {String} parameters.beforeDate - limit the response to keys older than beforeDate
|
||||
* @param {Number} parameters.maxScannedLifecycleListingEntries - max number of entries to be scanned
|
||||
* @param {RequestLogger} logger - The logger of the request
|
||||
* @param {String} [vFormat] - versioning key format
|
||||
*/
|
||||
constructor(parameters, logger, vFormat) {
|
||||
const {
|
||||
marker,
|
||||
maxKeys,
|
||||
prefix,
|
||||
beforeDate,
|
||||
maxScannedLifecycleListingEntries,
|
||||
} = parameters;
|
||||
|
||||
const versionParams = {
|
||||
// The orphan delete marker logic uses the term 'marker' instead of 'keyMarker',
|
||||
// as the latter could suggest the presence of a 'versionIdMarker'.
|
||||
keyMarker: marker,
|
||||
maxKeys,
|
||||
prefix,
|
||||
};
|
||||
super(versionParams, logger, vFormat);
|
||||
|
||||
this.maxScannedLifecycleListingEntries = maxScannedLifecycleListingEntries;
|
||||
this.beforeDate = beforeDate;
|
||||
// this.prevKeyName is used as a marker for the next listing when the current one reaches its entry limit.
|
||||
// We cannot rely on this.keyName, as it contains the name of the current key.
|
||||
// In the event of a listing interruption due to reaching the maximum scanned entries,
|
||||
// relying on this.keyName would cause the next listing to skip the current key because S3 starts
|
||||
// listing after the marker.
|
||||
this.prevKeyName = null;
|
||||
this.keyName = null;
|
||||
this.value = null;
|
||||
this.scannedKeys = 0;
|
||||
}
|
||||
|
||||
_reachedMaxKeys() {
|
||||
if (this.keys >= this.maxKeys) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
_addOrphan() {
|
||||
const parsedValue = this._parse(this.value);
|
||||
// if parsing fails, skip the key.
|
||||
if (parsedValue) {
|
||||
const lastModified = parsedValue['last-modified'];
|
||||
const isDeleteMarker = parsedValue.isDeleteMarker;
|
||||
// We then check if the orphan version is a delete marker and if it is older than the "beforeDate"
|
||||
if ((!this.beforeDate || (lastModified && lastModified < this.beforeDate)) && isDeleteMarker) {
|
||||
// Prefer returning an untrimmed data rather than stopping the service in case of parsing failure.
|
||||
const s = this._stringify(parsedValue) || this.value;
|
||||
this.Versions.push({ key: this.keyName, value: s });
|
||||
this.nextKeyMarker = this.keyName;
|
||||
++this.keys;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the stringified entry's value and remove the location property if too large.
|
||||
* @param {string} s - sringified value
|
||||
* @return {object} p - undefined if parsing fails, otherwise it contains the parsed value.
|
||||
*/
|
||||
_parse(s) {
|
||||
let p;
|
||||
try {
|
||||
p = JSON.parse(s);
|
||||
if (s.length >= TRIM_METADATA_MIN_BLOB_SIZE) {
|
||||
delete p.location;
|
||||
}
|
||||
} catch (e) {
|
||||
this.logger.warn('Could not parse Object Metadata while listing', {
|
||||
method: 'DelimiterOrphanDeleteMarker._parse',
|
||||
err: e.toString(),
|
||||
});
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
_stringify(value) {
|
||||
const p = value;
|
||||
let s = undefined;
|
||||
try {
|
||||
s = JSON.stringify(p);
|
||||
} catch (e) {
|
||||
this.logger.warn('could not stringify Object Metadata while listing',
|
||||
{
|
||||
method: 'DelimiterOrphanDeleteMarker._stringify',
|
||||
err: e.toString(),
|
||||
});
|
||||
}
|
||||
return s;
|
||||
}
|
||||
/**
|
||||
* The purpose of _isMaxScannedEntriesReached is to restrict the number of scanned entries,
|
||||
* thus controlling resource overhead (CPU...).
|
||||
* @return {boolean} isMaxScannedEntriesReached - true if the maximum limit on the number
|
||||
* of entries scanned has been reached, false otherwise.
|
||||
*/
|
||||
_isMaxScannedEntriesReached() {
|
||||
return this.maxScannedLifecycleListingEntries && this.scannedKeys >= this.maxScannedLifecycleListingEntries;
|
||||
}
|
||||
|
||||
filter(obj) {
|
||||
if (this._isMaxScannedEntriesReached()) {
|
||||
this.nextKeyMarker = this.prevKeyName;
|
||||
this.IsTruncated = true;
|
||||
this.logger.info('listing stopped due to reaching the maximum scanned entries limit',
|
||||
{
|
||||
maxScannedLifecycleListingEntries: this.maxScannedLifecycleListingEntries,
|
||||
scannedKeys: this.scannedKeys,
|
||||
});
|
||||
return FILTER_END;
|
||||
}
|
||||
++this.scannedKeys;
|
||||
return super.filter(obj);
|
||||
}
|
||||
|
||||
/**
|
||||
* NOTE: Each version of a specific key is sorted from the latest to the oldest
|
||||
* thanks to the way version ids are generated.
|
||||
* DESCRIPTION: For a given key, the latest version is kept in memory since it is the current version.
|
||||
* If the following version reference a new key, it means that the previous one was an orphan version.
|
||||
* We then check if the orphan version is a delete marker and if it is older than the "beforeDate"
|
||||
* The process stops and returns the available results if either:
|
||||
* - no more metadata key is left to be processed
|
||||
* - the listing reaches the maximum number of key to be returned
|
||||
* - the internal timeout is reached
|
||||
* NOTE: we cannot leverage MongoDB to list keys older than "beforeDate"
|
||||
* because then we will not be able to assess its orphanage.
|
||||
* @param {String} key - The object key.
|
||||
* @param {String} versionId - The object version id.
|
||||
* @param {String} value - The value of the key
|
||||
* @return {undefined}
|
||||
*/
|
||||
addVersion(key, versionId, value) {
|
||||
// For a given key, the youngest version is kept in memory since it represents the current version.
|
||||
if (key !== this.keyName) {
|
||||
// If this.value is defined, it means that <this.keyName, this.value> pair is "allowed" to be an orphan.
|
||||
if (this.value) {
|
||||
this._addOrphan();
|
||||
}
|
||||
this.prevKeyName = this.keyName;
|
||||
this.keyName = key;
|
||||
this.value = value;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// If the key is not the current version, we can skip it in the next listing
|
||||
// in the case where the current listing is interrupted due to reaching the maximum scanned entries.
|
||||
this.prevKeyName = key;
|
||||
this.keyName = key;
|
||||
this.value = null;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
result() {
|
||||
// Only check for remaining last orphan delete marker if the listing is not interrupted.
|
||||
// This will help avoid false positives.
|
||||
if (!this._isMaxScannedEntriesReached()) {
|
||||
// The following check makes sure the last orphan delete marker is not forgotten.
|
||||
if (this.keys < this.maxKeys) {
|
||||
if (this.value) {
|
||||
this._addOrphan();
|
||||
}
|
||||
// The following make sure that if makeKeys is reached, isTruncated is set to true.
|
||||
// We moved the "isTruncated" from _reachedMaxKeys to make sure we take into account the last entity
|
||||
// if listing is truncated right before the last entity and the last entity is a orphan delete marker.
|
||||
} else {
|
||||
this.IsTruncated = this.maxKeys > 0;
|
||||
}
|
||||
}
|
||||
|
||||
const result = {
|
||||
Contents: this.Versions,
|
||||
IsTruncated: this.IsTruncated,
|
||||
};
|
||||
|
||||
if (this.IsTruncated) {
|
||||
result.NextMarker = this.nextKeyMarker;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { DelimiterOrphanDeleteMarker };
|
|
@ -0,0 +1,283 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const Delimiter = require('./delimiter').Delimiter;
|
||||
const Version = require('../../versioning/Version').Version;
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { inc, FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } =
|
||||
require('./tools');
|
||||
|
||||
const VID_SEP = VSConst.VersionId.Separator;
|
||||
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters
|
||||
*
|
||||
* @prop {String[]} CommonPrefixes - 'folders' defined by the delimiter
|
||||
* @prop {String[]} Contents - 'files' to list
|
||||
* @prop {Boolean} IsTruncated - truncated listing flag
|
||||
* @prop {String|undefined} NextMarker - marker per amazon format
|
||||
* @prop {Number} keys - count of listed keys
|
||||
* @prop {String|undefined} delimiter - separator per amazon format
|
||||
* @prop {String|undefined} prefix - prefix per amazon format
|
||||
* @prop {Number} maxKeys - number of keys to list
|
||||
*/
|
||||
class DelimiterVersions extends Delimiter {
|
||||
constructor(parameters, logger, vFormat) {
|
||||
super(parameters, logger, vFormat);
|
||||
// specific to version listing
|
||||
this.keyMarker = parameters.keyMarker;
|
||||
this.versionIdMarker = parameters.versionIdMarker;
|
||||
// internal state
|
||||
this.masterKey = undefined;
|
||||
this.masterVersionId = undefined;
|
||||
// listing results
|
||||
this.NextMarker = parameters.keyMarker;
|
||||
this.NextVersionIdMarker = undefined;
|
||||
this.inReplayPrefix = false;
|
||||
|
||||
Object.assign(this, {
|
||||
[BucketVersioningKeyFormat.v0]: {
|
||||
genMDParams: this.genMDParamsV0,
|
||||
filter: this.filterV0,
|
||||
skipping: this.skippingV0,
|
||||
},
|
||||
[BucketVersioningKeyFormat.v1]: {
|
||||
genMDParams: this.genMDParamsV1,
|
||||
filter: this.filterV1,
|
||||
skipping: this.skippingV1,
|
||||
},
|
||||
}[this.vFormat]);
|
||||
}
|
||||
|
||||
genMDParamsV0() {
|
||||
const params = {};
|
||||
if (this.parameters.prefix) {
|
||||
params.gte = this.parameters.prefix;
|
||||
params.lt = inc(this.parameters.prefix);
|
||||
}
|
||||
if (this.parameters.keyMarker) {
|
||||
if (params.gte && params.gte > this.parameters.keyMarker) {
|
||||
return params;
|
||||
}
|
||||
delete params.gte;
|
||||
if (this.parameters.versionIdMarker) {
|
||||
// versionIdMarker should always come with keyMarker
|
||||
// but may not be the other way around
|
||||
params.gt = this.parameters.keyMarker
|
||||
+ VID_SEP
|
||||
+ this.parameters.versionIdMarker;
|
||||
} else {
|
||||
params.gt = inc(this.parameters.keyMarker + VID_SEP);
|
||||
}
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
genMDParamsV1() {
|
||||
// return an array of two listing params sets to ask for
|
||||
// synchronized listing of M and V ranges
|
||||
const params = [{}, {}];
|
||||
if (this.parameters.prefix) {
|
||||
params[0].gte = DbPrefixes.Master + this.parameters.prefix;
|
||||
params[0].lt = DbPrefixes.Master + inc(this.parameters.prefix);
|
||||
params[1].gte = DbPrefixes.Version + this.parameters.prefix;
|
||||
params[1].lt = DbPrefixes.Version + inc(this.parameters.prefix);
|
||||
} else {
|
||||
params[0].gte = DbPrefixes.Master;
|
||||
params[0].lt = inc(DbPrefixes.Master); // stop after the last master key
|
||||
params[1].gte = DbPrefixes.Version;
|
||||
params[1].lt = inc(DbPrefixes.Version); // stop after the last version key
|
||||
}
|
||||
if (this.parameters.keyMarker) {
|
||||
if (params[1].gte <= DbPrefixes.Version + this.parameters.keyMarker) {
|
||||
delete params[0].gte;
|
||||
delete params[1].gte;
|
||||
params[0].gt = DbPrefixes.Master + inc(this.parameters.keyMarker + VID_SEP);
|
||||
if (this.parameters.versionIdMarker) {
|
||||
// versionIdMarker should always come with keyMarker
|
||||
// but may not be the other way around
|
||||
params[1].gt = DbPrefixes.Version
|
||||
+ this.parameters.keyMarker
|
||||
+ VID_SEP
|
||||
+ this.parameters.versionIdMarker;
|
||||
} else {
|
||||
params[1].gt = DbPrefixes.Version
|
||||
+ inc(this.parameters.keyMarker + VID_SEP);
|
||||
}
|
||||
}
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Used to synchronize listing of M and V prefixes by object key
|
||||
*
|
||||
* @param {object} masterObj object listed from first range
|
||||
* returned by genMDParamsV1() (the master keys range)
|
||||
* @param {object} versionObj object listed from second range
|
||||
* returned by genMDParamsV1() (the version keys range)
|
||||
* @return {number} comparison result:
|
||||
* * -1 if master key < version key
|
||||
* * 1 if master key > version key
|
||||
*/
|
||||
compareObjects(masterObj, versionObj) {
|
||||
const masterKey = masterObj.key.slice(DbPrefixes.Master.length);
|
||||
const versionKey = versionObj.key.slice(DbPrefixes.Version.length);
|
||||
return masterKey < versionKey ? -1 : 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a (key, versionId, value) tuple to the listing.
|
||||
* Set the NextMarker to the current key
|
||||
* Increment the keys counter
|
||||
* @param {object} obj - the entry to add to the listing result
|
||||
* @param {String} obj.key - The key to add
|
||||
* @param {String} obj.versionId - versionId
|
||||
* @param {String} obj.value - The value of the key
|
||||
* @return {Boolean} - indicates if iteration should continue
|
||||
*/
|
||||
addContents(obj) {
|
||||
if (this._reachedMaxKeys()) {
|
||||
return FILTER_END;
|
||||
}
|
||||
this.Contents.push({
|
||||
key: obj.key,
|
||||
value: this.trimMetadata(obj.value),
|
||||
versionId: obj.versionId,
|
||||
});
|
||||
this.NextMarker = obj.key;
|
||||
this.NextVersionIdMarker = obj.versionId;
|
||||
++this.keys;
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter to apply on each iteration if bucket is in v0
|
||||
* versioning key format, based on:
|
||||
* - prefix
|
||||
* - delimiter
|
||||
* - maxKeys
|
||||
* The marker is being handled directly by levelDB
|
||||
* @param {Object} obj - The key and value of the element
|
||||
* @param {String} obj.key - The key of the element
|
||||
* @param {String} obj.value - The value of the element
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
filterV0(obj) {
|
||||
if (obj.key.startsWith(DbPrefixes.Replay)) {
|
||||
this.inReplayPrefix = true;
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
this.inReplayPrefix = false;
|
||||
|
||||
if (Version.isPHD(obj.value)) {
|
||||
// return accept to avoid skipping the next values in range
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
return this.filterCommon(obj.key, obj.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter to apply on each iteration if bucket is in v1
|
||||
* versioning key format, based on:
|
||||
* - prefix
|
||||
* - delimiter
|
||||
* - maxKeys
|
||||
* The marker is being handled directly by levelDB
|
||||
* @param {Object} obj - The key and value of the element
|
||||
* @param {String} obj.key - The key of the element
|
||||
* @param {String} obj.value - The value of the element
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
filterV1(obj) {
|
||||
if (Version.isPHD(obj.value)) {
|
||||
// return accept to avoid skipping the next values in range
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
// this function receives both M and V keys, but their prefix
|
||||
// length is the same so we can remove their prefix without
|
||||
// looking at the type of key
|
||||
return this.filterCommon(obj.key.slice(DbPrefixes.Master.length),
|
||||
obj.value);
|
||||
}
|
||||
|
||||
filterCommon(key, value) {
|
||||
if (this.prefix && !key.startsWith(this.prefix)) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
let nonversionedKey;
|
||||
let versionId = undefined;
|
||||
const versionIdIndex = key.indexOf(VID_SEP);
|
||||
if (versionIdIndex < 0) {
|
||||
nonversionedKey = key;
|
||||
this.masterKey = key;
|
||||
this.masterVersionId =
|
||||
Version.from(value).getVersionId() || 'null';
|
||||
versionId = this.masterVersionId;
|
||||
} else {
|
||||
nonversionedKey = key.slice(0, versionIdIndex);
|
||||
versionId = key.slice(versionIdIndex + 1);
|
||||
// skip a version key if it is the master version
|
||||
if (this.masterKey === nonversionedKey && this.masterVersionId === versionId) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
this.masterKey = undefined;
|
||||
this.masterVersionId = undefined;
|
||||
}
|
||||
if (this.delimiter) {
|
||||
const baseIndex = this.prefix ? this.prefix.length : 0;
|
||||
const delimiterIndex = nonversionedKey.indexOf(this.delimiter, baseIndex);
|
||||
if (delimiterIndex >= 0) {
|
||||
return this.addCommonPrefix(nonversionedKey, delimiterIndex);
|
||||
}
|
||||
}
|
||||
return this.addContents({ key: nonversionedKey, value, versionId });
|
||||
}
|
||||
|
||||
skippingV0() {
|
||||
if (this.inReplayPrefix) {
|
||||
return DbPrefixes.Replay;
|
||||
}
|
||||
if (this.NextMarker) {
|
||||
const index = this.NextMarker.lastIndexOf(this.delimiter);
|
||||
if (index === this.NextMarker.length - 1) {
|
||||
return this.NextMarker;
|
||||
}
|
||||
}
|
||||
return SKIP_NONE;
|
||||
}
|
||||
|
||||
skippingV1() {
|
||||
const skipV0 = this.skippingV0();
|
||||
if (skipV0 === SKIP_NONE) {
|
||||
return SKIP_NONE;
|
||||
}
|
||||
// skip to the same object key in both M and V range listings
|
||||
return [DbPrefixes.Master + skipV0,
|
||||
DbPrefixes.Version + skipV0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an object containing all mandatory fields to use once the
|
||||
* iteration is done, doesn't show a NextMarker field if the output
|
||||
* isn't truncated
|
||||
* @return {Object} - following amazon format
|
||||
*/
|
||||
result() {
|
||||
/* NextMarker is only provided when delimiter is used.
|
||||
* specified in v1 listing documentation
|
||||
* http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
|
||||
*/
|
||||
return {
|
||||
CommonPrefixes: this.CommonPrefixes,
|
||||
Versions: this.Contents,
|
||||
IsTruncated: this.IsTruncated,
|
||||
NextKeyMarker: this.IsTruncated ? this.NextMarker : undefined,
|
||||
NextVersionIdMarker: this.IsTruncated ?
|
||||
this.NextVersionIdMarker : undefined,
|
||||
Delimiter: this.delimiter,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { DelimiterVersions };
|
|
@ -1,535 +0,0 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const Extension = require('./Extension').default;
|
||||
|
||||
import {
|
||||
FilterState,
|
||||
FilterReturnValue,
|
||||
} from './delimiter';
|
||||
|
||||
const Version = require('../../versioning/Version').Version;
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { inc, FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } =
|
||||
require('./tools');
|
||||
|
||||
const VID_SEP = VSConst.VersionId.Separator;
|
||||
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
|
||||
|
||||
export const enum DelimiterVersionsFilterStateId {
|
||||
NotSkipping = 1,
|
||||
SkippingPrefix = 2,
|
||||
SkippingVersions = 3,
|
||||
};
|
||||
|
||||
export interface DelimiterVersionsFilterState_NotSkipping extends FilterState {
|
||||
id: DelimiterVersionsFilterStateId.NotSkipping,
|
||||
};
|
||||
|
||||
export interface DelimiterVersionsFilterState_SkippingPrefix extends FilterState {
|
||||
id: DelimiterVersionsFilterStateId.SkippingPrefix,
|
||||
prefix: string;
|
||||
};
|
||||
|
||||
export interface DelimiterVersionsFilterState_SkippingVersions extends FilterState {
|
||||
id: DelimiterVersionsFilterStateId.SkippingVersions,
|
||||
gt: string;
|
||||
};
|
||||
|
||||
type KeyHandler = (key: string, versionId: string | undefined, value: string) => FilterReturnValue;
|
||||
|
||||
type ResultObject = {
|
||||
CommonPrefixes: string[],
|
||||
Versions: {
|
||||
key: string;
|
||||
value: string;
|
||||
versionId: string;
|
||||
}[];
|
||||
IsTruncated: boolean;
|
||||
Delimiter ?: string;
|
||||
NextKeyMarker ?: string;
|
||||
NextVersionIdMarker ?: string;
|
||||
};
|
||||
|
||||
type GenMDParamsItem = {
|
||||
gt ?: string,
|
||||
gte ?: string,
|
||||
lt ?: string,
|
||||
};
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters
|
||||
*
|
||||
* @prop {String[]} CommonPrefixes - 'folders' defined by the delimiter
|
||||
* @prop {String[]} Contents - 'files' to list
|
||||
* @prop {Boolean} IsTruncated - truncated listing flag
|
||||
* @prop {String|undefined} NextMarker - marker per amazon format
|
||||
* @prop {Number} keys - count of listed keys
|
||||
* @prop {String|undefined} delimiter - separator per amazon format
|
||||
* @prop {String|undefined} prefix - prefix per amazon format
|
||||
* @prop {Number} maxKeys - number of keys to list
|
||||
*/
|
||||
export class DelimiterVersions extends Extension {
|
||||
|
||||
state: FilterState;
|
||||
keyHandlers: { [id: number]: KeyHandler };
|
||||
|
||||
constructor(parameters, logger, vFormat) {
|
||||
super(parameters, logger);
|
||||
// original listing parameters
|
||||
this.delimiter = parameters.delimiter;
|
||||
this.prefix = parameters.prefix;
|
||||
this.maxKeys = parameters.maxKeys || 1000;
|
||||
// specific to version listing
|
||||
this.keyMarker = parameters.keyMarker;
|
||||
this.versionIdMarker = parameters.versionIdMarker;
|
||||
// internal state
|
||||
this.masterKey = undefined;
|
||||
this.masterVersionId = undefined;
|
||||
this.nullKey = null;
|
||||
this.vFormat = vFormat || BucketVersioningKeyFormat.v0;
|
||||
// listing results
|
||||
this.CommonPrefixes = [];
|
||||
this.Versions = [];
|
||||
this.IsTruncated = false;
|
||||
this.nextKeyMarker = parameters.keyMarker;
|
||||
this.nextVersionIdMarker = undefined;
|
||||
|
||||
this.keyHandlers = {};
|
||||
|
||||
Object.assign(this, {
|
||||
[BucketVersioningKeyFormat.v0]: {
|
||||
genMDParams: this.genMDParamsV0,
|
||||
getObjectKey: this.getObjectKeyV0,
|
||||
skipping: this.skippingV0,
|
||||
},
|
||||
[BucketVersioningKeyFormat.v1]: {
|
||||
genMDParams: this.genMDParamsV1,
|
||||
getObjectKey: this.getObjectKeyV1,
|
||||
skipping: this.skippingV1,
|
||||
},
|
||||
}[this.vFormat]);
|
||||
|
||||
if (this.vFormat === BucketVersioningKeyFormat.v0) {
|
||||
this.setKeyHandler(
|
||||
DelimiterVersionsFilterStateId.NotSkipping,
|
||||
this.keyHandler_NotSkippingV0.bind(this));
|
||||
} else {
|
||||
this.setKeyHandler(
|
||||
DelimiterVersionsFilterStateId.NotSkipping,
|
||||
this.keyHandler_NotSkippingV1.bind(this));
|
||||
}
|
||||
this.setKeyHandler(
|
||||
DelimiterVersionsFilterStateId.SkippingPrefix,
|
||||
this.keyHandler_SkippingPrefix.bind(this));
|
||||
|
||||
this.setKeyHandler(
|
||||
DelimiterVersionsFilterStateId.SkippingVersions,
|
||||
this.keyHandler_SkippingVersions.bind(this));
|
||||
|
||||
if (this.versionIdMarker) {
|
||||
this.state = <DelimiterVersionsFilterState_SkippingVersions> {
|
||||
id: DelimiterVersionsFilterStateId.SkippingVersions,
|
||||
gt: `${this.keyMarker}${VID_SEP}${this.versionIdMarker}`,
|
||||
};
|
||||
} else {
|
||||
this.state = <DelimiterVersionsFilterState_NotSkipping> {
|
||||
id: DelimiterVersionsFilterStateId.NotSkipping,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
genMDParamsV0() {
|
||||
const params: GenMDParamsItem = {};
|
||||
if (this.prefix) {
|
||||
params.gte = this.prefix;
|
||||
params.lt = inc(this.prefix);
|
||||
}
|
||||
if (this.keyMarker && this.delimiter) {
|
||||
const commonPrefix = this.getCommonPrefix(this.keyMarker);
|
||||
if (commonPrefix) {
|
||||
const afterPrefix = inc(commonPrefix);
|
||||
if (!params.gte || afterPrefix > params.gte) {
|
||||
params.gte = afterPrefix;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.keyMarker && (!params.gte || this.keyMarker >= params.gte)) {
|
||||
delete params.gte;
|
||||
if (this.versionIdMarker) {
|
||||
// start from the beginning of versions so we can
|
||||
// check if there's a null key and fetch it
|
||||
// (afterwards, we can skip the rest of versions until
|
||||
// we reach versionIdMarker)
|
||||
params.gte = `${this.keyMarker}${VID_SEP}`;
|
||||
} else {
|
||||
params.gt = `${this.keyMarker}${inc(VID_SEP)}`;
|
||||
}
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
genMDParamsV1() {
|
||||
// return an array of two listing params sets to ask for
|
||||
// synchronized listing of M and V ranges
|
||||
const v0Params: GenMDParamsItem = this.genMDParamsV0();
|
||||
const mParams: GenMDParamsItem = {};
|
||||
const vParams: GenMDParamsItem = {};
|
||||
if (v0Params.gt) {
|
||||
mParams.gt = `${DbPrefixes.Master}${v0Params.gt}`;
|
||||
vParams.gt = `${DbPrefixes.Version}${v0Params.gt}`;
|
||||
} else if (v0Params.gte) {
|
||||
mParams.gte = `${DbPrefixes.Master}${v0Params.gte}`;
|
||||
vParams.gte = `${DbPrefixes.Version}${v0Params.gte}`;
|
||||
} else {
|
||||
mParams.gte = DbPrefixes.Master;
|
||||
vParams.gte = DbPrefixes.Version;
|
||||
}
|
||||
if (v0Params.lt) {
|
||||
mParams.lt = `${DbPrefixes.Master}${v0Params.lt}`;
|
||||
vParams.lt = `${DbPrefixes.Version}${v0Params.lt}`;
|
||||
} else {
|
||||
mParams.lt = inc(DbPrefixes.Master);
|
||||
vParams.lt = inc(DbPrefixes.Version);
|
||||
}
|
||||
return [mParams, vParams];
|
||||
}
|
||||
|
||||
/**
|
||||
* check if the max keys count has been reached and set the
|
||||
* final state of the result if it is the case
|
||||
* @return {Boolean} - indicates if the iteration has to stop
|
||||
*/
|
||||
_reachedMaxKeys(): boolean {
|
||||
if (this.keys >= this.maxKeys) {
|
||||
// In cases of maxKeys <= 0 -> IsTruncated = false
|
||||
this.IsTruncated = this.maxKeys > 0;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Used to synchronize listing of M and V prefixes by object key
|
||||
*
|
||||
* @param {object} masterObj object listed from first range
|
||||
* returned by genMDParamsV1() (the master keys range)
|
||||
* @param {object} versionObj object listed from second range
|
||||
* returned by genMDParamsV1() (the version keys range)
|
||||
* @return {number} comparison result:
|
||||
* * -1 if master key < version key
|
||||
* * 1 if master key > version key
|
||||
*/
|
||||
compareObjects(masterObj, versionObj) {
|
||||
const masterKey = masterObj.key.slice(DbPrefixes.Master.length);
|
||||
const versionKey = versionObj.key.slice(DbPrefixes.Version.length);
|
||||
return masterKey < versionKey ? -1 : 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a listing key into its nonversioned key and version ID components
|
||||
*
|
||||
* @param {string} key - full listing key
|
||||
* @return {object} obj
|
||||
* @return {string} obj.key - nonversioned part of key
|
||||
* @return {string} [obj.versionId] - version ID in the key
|
||||
*/
|
||||
parseKey(fullKey: string): { key: string, versionId ?: string } {
|
||||
const versionIdIndex = fullKey.indexOf(VID_SEP);
|
||||
if (versionIdIndex === -1) {
|
||||
return { key: fullKey };
|
||||
}
|
||||
const nonversionedKey: string = fullKey.slice(0, versionIdIndex);
|
||||
let versionId: string = fullKey.slice(versionIdIndex + 1);
|
||||
return { key: nonversionedKey, versionId };
|
||||
}
|
||||
|
||||
/**
|
||||
* Include a key in the listing output, in the Versions or CommonPrefix result
|
||||
*
|
||||
* @param {string} key - key (without version ID)
|
||||
* @param {string} versionId - version ID
|
||||
* @param {string} value - metadata value
|
||||
* @return {undefined}
|
||||
*/
|
||||
addKey(key: string, versionId: string, value: string) {
|
||||
// add the subprefix to the common prefixes if the key has the delimiter
|
||||
const commonPrefix = this.getCommonPrefix(key);
|
||||
if (commonPrefix) {
|
||||
this.addCommonPrefix(commonPrefix);
|
||||
// transition into SkippingPrefix state to skip all following keys
|
||||
// while they start with the same prefix
|
||||
this.setState(<DelimiterVersionsFilterState_SkippingPrefix> {
|
||||
id: DelimiterVersionsFilterStateId.SkippingPrefix,
|
||||
prefix: commonPrefix,
|
||||
});
|
||||
} else {
|
||||
this.addVersion(key, versionId, value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a (key, versionId, value) tuple to the listing.
|
||||
* Set the NextMarker to the current key
|
||||
* Increment the keys counter
|
||||
* @param {String} key - The key to add
|
||||
* @param {String} versionId - versionId
|
||||
* @param {String} value - The value of the key
|
||||
* @return {undefined}
|
||||
*/
|
||||
addVersion(key: string, versionId: string, value: string) {
|
||||
this.Versions.push({
|
||||
key,
|
||||
versionId,
|
||||
value: this.trimMetadata(value),
|
||||
});
|
||||
this.nextKeyMarker = key;
|
||||
this.nextVersionIdMarker = versionId;
|
||||
++this.keys;
|
||||
}
|
||||
|
||||
getCommonPrefix(key: string): string | undefined {
|
||||
if (!this.delimiter) {
|
||||
return undefined;
|
||||
}
|
||||
const baseIndex = this.prefix ? this.prefix.length : 0;
|
||||
const delimiterIndex = key.indexOf(this.delimiter, baseIndex);
|
||||
if (delimiterIndex === -1) {
|
||||
return undefined;
|
||||
}
|
||||
return key.substring(0, delimiterIndex + this.delimiter.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a Common Prefix in the list
|
||||
* @param {String} commonPrefix - common prefix to add
|
||||
* @return {undefined}
|
||||
*/
|
||||
addCommonPrefix(commonPrefix: string): void {
|
||||
// add the new prefix to the list
|
||||
this.CommonPrefixes.push(commonPrefix);
|
||||
++this.keys;
|
||||
this.nextKeyMarker = commonPrefix;
|
||||
this.nextVersionIdMarker = undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache the current null key, to save it for outputting it later at
|
||||
* the correct position
|
||||
*
|
||||
* @param {String} key - nonversioned key of the null key
|
||||
* @param {String} versionId - real version ID of the null key
|
||||
* @param {String} value - value of the null key
|
||||
* @return {undefined}
|
||||
*/
|
||||
cacheNullKey(key: string, versionId: string, value: string): void {
|
||||
this.nullKey = { key, versionId, value };
|
||||
}
|
||||
|
||||
getObjectKeyV0(obj: { key: string }): string {
|
||||
return obj.key;
|
||||
}
|
||||
|
||||
getObjectKeyV1(obj: { key: string }): string {
|
||||
return obj.key.slice(DbPrefixes.Master.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter to apply on each iteration, based on:
|
||||
* - prefix
|
||||
* - delimiter
|
||||
* - maxKeys
|
||||
* The marker is being handled directly by levelDB
|
||||
* @param {Object} obj - The key and value of the element
|
||||
* @param {String} obj.key - The key of the element
|
||||
* @param {String} obj.value - The value of the element
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
filter(obj: { key: string, value: string }): FilterReturnValue {
|
||||
const key = this.getObjectKey(obj);
|
||||
const value = obj.value;
|
||||
|
||||
const { key: nonversionedKey, versionId: keyVersionId } = this.parseKey(key);
|
||||
if (this.nullKey) {
|
||||
if (this.nullKey.key !== nonversionedKey
|
||||
|| this.nullKey.versionId < <string> keyVersionId) {
|
||||
this.handleKey(
|
||||
this.nullKey.key, this.nullKey.versionId, this.nullKey.value);
|
||||
this.nullKey = null;
|
||||
}
|
||||
}
|
||||
if (keyVersionId === '') {
|
||||
// null key
|
||||
this.cacheNullKey(nonversionedKey, Version.from(value).getVersionId(), value);
|
||||
if (this.state.id === DelimiterVersionsFilterStateId.SkippingVersions) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
return this.handleKey(nonversionedKey, keyVersionId, value);
|
||||
}
|
||||
|
||||
setState(state: FilterState): void {
|
||||
this.state = state;
|
||||
}
|
||||
|
||||
setKeyHandler(stateId: number, keyHandler: KeyHandler): void {
|
||||
this.keyHandlers[stateId] = keyHandler;
|
||||
}
|
||||
|
||||
handleKey(key: string, versionId: string | undefined, value: string): FilterReturnValue {
|
||||
return this.keyHandlers[this.state.id](key, versionId, value);
|
||||
}
|
||||
|
||||
keyHandler_NotSkippingV0(key: string, versionId: string | undefined, value: string): FilterReturnValue {
|
||||
if (key.startsWith(DbPrefixes.Replay)) {
|
||||
// skip internal replay prefix entirely
|
||||
this.setState(<DelimiterVersionsFilterState_SkippingPrefix> {
|
||||
id: DelimiterVersionsFilterStateId.SkippingPrefix,
|
||||
prefix: DbPrefixes.Replay,
|
||||
});
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
if (Version.isPHD(value)) {
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
return this.filter_onNewKey(key, versionId, value);
|
||||
}
|
||||
|
||||
keyHandler_NotSkippingV1(key: string, versionId: string | undefined, value: string): FilterReturnValue {
|
||||
// NOTE: this check on PHD is only useful for Artesca, S3C
|
||||
// does not use PHDs in V1 format
|
||||
if (Version.isPHD(value)) {
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
return this.filter_onNewKey(key, versionId, value);
|
||||
}
|
||||
|
||||
filter_onNewKey(key: string, versionId: string | undefined, value: string): FilterReturnValue {
|
||||
if (this._reachedMaxKeys()) {
|
||||
return FILTER_END;
|
||||
}
|
||||
if (versionId === undefined) {
|
||||
this.masterKey = key;
|
||||
this.masterVersionId = Version.from(value).getVersionId() || 'null';
|
||||
this.addKey(this.masterKey, this.masterVersionId, value);
|
||||
} else {
|
||||
if (this.masterKey === key && this.masterVersionId === versionId) {
|
||||
// do not add a version key if it is the master version
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
this.addKey(key, versionId, value);
|
||||
}
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
|
||||
keyHandler_SkippingPrefix(key: string, versionId: string | undefined, value: string): FilterReturnValue {
|
||||
const { prefix } = <DelimiterVersionsFilterState_SkippingPrefix> this.state;
|
||||
if (key.startsWith(prefix)) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
this.setState(<DelimiterVersionsFilterState_NotSkipping> {
|
||||
id: DelimiterVersionsFilterStateId.NotSkipping,
|
||||
});
|
||||
return this.handleKey(key, versionId, value);
|
||||
}
|
||||
|
||||
keyHandler_SkippingVersions(key: string, versionId: string | undefined, value: string): FilterReturnValue {
|
||||
if (key === this.keyMarker) {
|
||||
// since the nonversioned key equals the marker, there is
|
||||
// necessarily a versionId in this key
|
||||
const _versionId = <string> versionId;
|
||||
if (_versionId < this.versionIdMarker) {
|
||||
// skip all versions until marker
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
if (_versionId === this.versionIdMarker) {
|
||||
// nothing left to skip, so return ACCEPT, but don't add this version
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
}
|
||||
this.setState(<DelimiterVersionsFilterState_NotSkipping> {
|
||||
id: DelimiterVersionsFilterStateId.NotSkipping,
|
||||
});
|
||||
return this.handleKey(key, versionId, value);
|
||||
}
|
||||
|
||||
skippingBase(): string | undefined {
|
||||
switch (this.state.id) {
|
||||
case DelimiterVersionsFilterStateId.SkippingPrefix:
|
||||
const { prefix } = <DelimiterVersionsFilterState_SkippingPrefix> this.state;
|
||||
return inc(prefix);
|
||||
|
||||
case DelimiterVersionsFilterStateId.SkippingVersions:
|
||||
const { gt } = <DelimiterVersionsFilterState_SkippingVersions> this.state;
|
||||
// the contract of skipping() is to return the first key
|
||||
// that can be skipped to, so adding a null byte to skip
|
||||
// over the existing versioned key set in 'gt'
|
||||
return `${gt}\0`;
|
||||
|
||||
default:
|
||||
return SKIP_NONE;
|
||||
}
|
||||
}
|
||||
|
||||
skippingV0() {
|
||||
return this.skippingBase();
|
||||
}
|
||||
|
||||
skippingV1() {
|
||||
const skipTo = this.skippingBase();
|
||||
if (skipTo === SKIP_NONE) {
|
||||
return SKIP_NONE;
|
||||
}
|
||||
// skip to the same object key in both M and V range listings
|
||||
return [
|
||||
`${DbPrefixes.Master}${skipTo}`,
|
||||
`${DbPrefixes.Version}${skipTo}`,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an object containing all mandatory fields to use once the
|
||||
* iteration is done, doesn't show a NextMarker field if the output
|
||||
* isn't truncated
|
||||
* @return {Object} - following amazon format
|
||||
*/
|
||||
result() {
|
||||
// Add the last null key if still in cache (when it is the
|
||||
// last version of the last key)
|
||||
//
|
||||
// NOTE: _reachedMaxKeys sets IsTruncated to true when it
|
||||
// returns true. Here we want this because either:
|
||||
//
|
||||
// - we did not reach the max keys yet so the result is not
|
||||
// - truncated, and there is still room for the null key in
|
||||
// - the results
|
||||
//
|
||||
// - OR we reached it already while having to process a new
|
||||
// key (so the result is truncated even without the null key)
|
||||
//
|
||||
// - OR we are *just* below the limit but the null key to add
|
||||
// does not fit, so we know the result is now truncated
|
||||
// because there remains the null key to be output.
|
||||
//
|
||||
if (this.nullKey) {
|
||||
this.handleKey(this.nullKey.key, this.nullKey.versionId, this.nullKey.value);
|
||||
}
|
||||
const result: ResultObject = {
|
||||
CommonPrefixes: this.CommonPrefixes,
|
||||
Versions: this.Versions,
|
||||
IsTruncated: this.IsTruncated,
|
||||
};
|
||||
if (this.delimiter) {
|
||||
result.Delimiter = this.delimiter;
|
||||
}
|
||||
if (this.IsTruncated) {
|
||||
result.NextKeyMarker = this.nextKeyMarker;
|
||||
if (this.nextVersionIdMarker) {
|
||||
result.NextVersionIdMarker = this.nextVersionIdMarker;
|
||||
}
|
||||
};
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { DelimiterVersions };
|
|
@ -6,7 +6,4 @@ module.exports = {
|
|||
DelimiterMaster: require('./delimiterMaster')
|
||||
.DelimiterMaster,
|
||||
MPU: require('./MPU').MultipartUploads,
|
||||
DelimiterCurrent: require('./delimiterCurrent').DelimiterCurrent,
|
||||
DelimiterNonCurrent: require('./delimiterNonCurrent').DelimiterNonCurrent,
|
||||
DelimiterOrphanDeleteMarker: require('./delimiterOrphanDeleteMarker').DelimiterOrphanDeleteMarker,
|
||||
};
|
||||
|
|
|
@ -52,21 +52,21 @@ class Skip {
|
|||
assert(this.skipRangeCb);
|
||||
|
||||
const filteringResult = this.extension.filter(entry);
|
||||
const skipTo = this.extension.skipping();
|
||||
const skippingRange = this.extension.skipping();
|
||||
|
||||
if (filteringResult === FILTER_END) {
|
||||
this.listingEndCb();
|
||||
} else if (filteringResult === FILTER_SKIP
|
||||
&& skipTo !== SKIP_NONE) {
|
||||
&& skippingRange !== SKIP_NONE) {
|
||||
if (++this.streakLength >= MAX_STREAK_LENGTH) {
|
||||
let newRange;
|
||||
if (Array.isArray(skipTo)) {
|
||||
if (Array.isArray(skippingRange)) {
|
||||
newRange = [];
|
||||
for (let i = 0; i < skipTo.length; ++i) {
|
||||
newRange.push(skipTo[i]);
|
||||
for (let i = 0; i < skippingRange.length; ++i) {
|
||||
newRange.push(this._inc(skippingRange[i]));
|
||||
}
|
||||
} else {
|
||||
newRange = skipTo;
|
||||
newRange = this._inc(skippingRange);
|
||||
}
|
||||
/* Avoid to loop on the same range again and again. */
|
||||
if (newRange === this.gteParams) {
|
||||
|
@ -79,6 +79,16 @@ class Skip {
|
|||
this.streakLength = 0;
|
||||
}
|
||||
}
|
||||
|
||||
_inc(str) {
|
||||
if (!str) {
|
||||
return str;
|
||||
}
|
||||
const lastCharValue = str.charCodeAt(str.length - 1);
|
||||
const lastCharNewValue = String.fromCharCode(lastCharValue + 1);
|
||||
|
||||
return `${str.slice(0, str.length - 1)}${lastCharNewValue}`;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -1,4 +1,6 @@
|
|||
import * as constants from '../constants';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const constants = require('../constants');
|
||||
|
||||
/**
|
||||
* Class containing requester's information received from Vault
|
||||
|
@ -6,15 +8,9 @@ import * as constants from '../constants';
|
|||
* shortid, email, accountDisplayName and IAMdisplayName (if applicable)
|
||||
* @return {AuthInfo} an AuthInfo instance
|
||||
*/
|
||||
export default class AuthInfo {
|
||||
arn: string;
|
||||
canonicalID: string;
|
||||
shortid: string;
|
||||
email: string;
|
||||
accountDisplayName: string;
|
||||
IAMdisplayName: string;
|
||||
|
||||
constructor(objectFromVault: any) {
|
||||
class AuthInfo {
|
||||
constructor(objectFromVault) {
|
||||
// amazon resource name for IAM user (if applicable)
|
||||
this.arn = objectFromVault.arn;
|
||||
// account canonicalID
|
||||
|
@ -57,8 +53,10 @@ export default class AuthInfo {
|
|||
return this.canonicalID.startsWith(
|
||||
`${constants.zenkoServiceAccount}/`);
|
||||
}
|
||||
isRequesterThisServiceAccount(serviceName: string) {
|
||||
const computedCanonicalID = `${constants.zenkoServiceAccount}/${serviceName}`;
|
||||
return this.canonicalID === computedCanonicalID;
|
||||
isRequesterThisServiceAccount(serviceName) {
|
||||
return this.canonicalID ===
|
||||
`${constants.zenkoServiceAccount}/${serviceName}`;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = AuthInfo;
|
|
@ -1,22 +1,16 @@
|
|||
import { Logger } from 'werelogs';
|
||||
import errors from '../errors';
|
||||
import AuthInfo from './AuthInfo';
|
||||
const errors = require('../errors').default;
|
||||
const AuthInfo = require('./AuthInfo');
|
||||
|
||||
/** vaultSignatureCb parses message from Vault and instantiates
|
||||
* @param err - error from vault
|
||||
* @param authInfo - info from vault
|
||||
* @param log - log for request
|
||||
* @param callback - callback to authCheck functions
|
||||
* @param [streamingV4Params] - present if v4 signature;
|
||||
* @param {object} err - error from vault
|
||||
* @param {object} authInfo - info from vault
|
||||
* @param {object} log - log for request
|
||||
* @param {function} callback - callback to authCheck functions
|
||||
* @param {object} [streamingV4Params] - present if v4 signature;
|
||||
* items used to calculate signature on chunks if streaming auth
|
||||
* @return {undefined}
|
||||
*/
|
||||
function vaultSignatureCb(
|
||||
err: Error | null,
|
||||
authInfo: { message: { body: any } },
|
||||
log: Logger,
|
||||
callback: (err: Error | null, data?: any, results?: any, params?: any, infos?: any) => void,
|
||||
streamingV4Params?: any
|
||||
) {
|
||||
function vaultSignatureCb(err, authInfo, log, callback, streamingV4Params) {
|
||||
// vaultclient API guarantees that it returns:
|
||||
// - either `err`, an Error object with `code` and `message` properties set
|
||||
// - or `err == null` and `info` is an object with `message.code` and
|
||||
|
@ -30,101 +24,58 @@ function vaultSignatureCb(
|
|||
const info = authInfo.message.body;
|
||||
const userInfo = new AuthInfo(info.userInfo);
|
||||
const authorizationResults = info.authorizationResults;
|
||||
const auditLog: { accountDisplayName: string, IAMdisplayName?: string } =
|
||||
{ accountDisplayName: userInfo.getAccountDisplayName() };
|
||||
const auditLog = { accountDisplayName: userInfo.getAccountDisplayName() };
|
||||
const iamDisplayName = userInfo.getIAMdisplayName();
|
||||
if (iamDisplayName) {
|
||||
auditLog.IAMdisplayName = iamDisplayName;
|
||||
}
|
||||
// @ts-ignore
|
||||
log.addDefaultFields(auditLog);
|
||||
return callback(null, userInfo, authorizationResults, streamingV4Params, {
|
||||
accountQuota: info.accountQuota || {},
|
||||
});
|
||||
return callback(null, userInfo, authorizationResults, streamingV4Params);
|
||||
}
|
||||
|
||||
export type AuthV4RequestParams = {
|
||||
version: 4;
|
||||
log: Logger;
|
||||
data: {
|
||||
accessKey: string;
|
||||
signatureFromRequest: string;
|
||||
region: string;
|
||||
stringToSign: string;
|
||||
scopeDate: string;
|
||||
authType: 'query' | 'header';
|
||||
signatureVersion: string;
|
||||
signatureAge?: number;
|
||||
timestamp: number;
|
||||
credentialScope: string;
|
||||
securityToken: string;
|
||||
algo: string;
|
||||
log: Logger;
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Class that provides common authentication methods against different
|
||||
* authentication backends.
|
||||
* @class Vault
|
||||
*/
|
||||
export default class Vault {
|
||||
client: any;
|
||||
implName: string;
|
||||
|
||||
class Vault {
|
||||
/**
|
||||
* @constructor
|
||||
* @param {object} client - authentication backend or vault client
|
||||
* @param {string} implName - implementation name for auth backend
|
||||
*/
|
||||
constructor(client: any, implName: string) {
|
||||
constructor(client, implName) {
|
||||
this.client = client;
|
||||
this.implName = implName;
|
||||
}
|
||||
/**
|
||||
* authenticateV2Request
|
||||
*
|
||||
* @param params - the authentication parameters as returned by
|
||||
* @param {string} params - the authentication parameters as returned by
|
||||
* auth.extractParams
|
||||
* @param params.version - shall equal 2
|
||||
* @param params.data.accessKey - the user's accessKey
|
||||
* @param params.data.signatureFromRequest - the signature read
|
||||
* @param {number} params.version - shall equal 2
|
||||
* @param {string} params.data.accessKey - the user's accessKey
|
||||
* @param {string} params.data.signatureFromRequest - the signature read
|
||||
* from the request
|
||||
* @param params.data.stringToSign - the stringToSign
|
||||
* @param params.data.algo - the hashing algorithm used for the
|
||||
* @param {string} params.data.stringToSign - the stringToSign
|
||||
* @param {string} params.data.algo - the hashing algorithm used for the
|
||||
* signature
|
||||
* @param params.data.authType - the type of authentication (query
|
||||
* @param {string} params.data.authType - the type of authentication (query
|
||||
* or header)
|
||||
* @param params.data.signatureVersion - the version of the
|
||||
* @param {string} params.data.signatureVersion - the version of the
|
||||
* signature (AWS or AWS4)
|
||||
* @param [params.data.signatureAge] - the age of the signature in
|
||||
* @param {number} [params.data.signatureAge] - the age of the signature in
|
||||
* ms
|
||||
* @param params.data.log - the logger object
|
||||
* @param {string} params.data.log - the logger object
|
||||
* @param {RequestContext []} requestContexts - an array of RequestContext
|
||||
* instances which contain information for policy authorization check
|
||||
* @param callback - callback with either error or user info
|
||||
* @param {function} callback - callback with either error or user info
|
||||
* @returns {undefined}
|
||||
*/
|
||||
authenticateV2Request(
|
||||
params: {
|
||||
version: 2;
|
||||
log: Logger;
|
||||
data: {
|
||||
securityToken: string;
|
||||
accessKey: string;
|
||||
signatureFromRequest: string;
|
||||
stringToSign: string;
|
||||
algo: string;
|
||||
authType: 'query' | 'header';
|
||||
signatureVersion: string;
|
||||
signatureAge?: number;
|
||||
log: Logger;
|
||||
};
|
||||
},
|
||||
requestContexts: any[],
|
||||
callback: (err: Error | null, data?: any) => void
|
||||
) {
|
||||
authenticateV2Request(params, requestContexts, callback) {
|
||||
params.log.debug('authenticating V2 request');
|
||||
let serializedRCsArr: any;
|
||||
let serializedRCsArr;
|
||||
if (requestContexts) {
|
||||
serializedRCsArr = requestContexts.map(rc => rc.serialize());
|
||||
}
|
||||
|
@ -134,48 +85,44 @@ export default class Vault {
|
|||
params.data.accessKey,
|
||||
{
|
||||
algo: params.data.algo,
|
||||
// @ts-ignore
|
||||
reqUid: params.log.getSerializedUids(),
|
||||
logger: params.log,
|
||||
securityToken: params.data.securityToken,
|
||||
requestContext: serializedRCsArr,
|
||||
},
|
||||
(err: Error | null, userInfo?: any) => vaultSignatureCb(err, userInfo,
|
||||
params.log, callback),
|
||||
(err, userInfo) => vaultSignatureCb(err, userInfo,
|
||||
params.log, callback)
|
||||
);
|
||||
}
|
||||
|
||||
/** authenticateV4Request
|
||||
* @param params - the authentication parameters as returned by
|
||||
* @param {object} params - the authentication parameters as returned by
|
||||
* auth.extractParams
|
||||
* @param params.version - shall equal 4
|
||||
* @param params.data.log - the logger object
|
||||
* @param params.data.accessKey - the user's accessKey
|
||||
* @param params.data.signatureFromRequest - the signature read
|
||||
* @param {number} params.version - shall equal 4
|
||||
* @param {string} params.data.log - the logger object
|
||||
* @param {string} params.data.accessKey - the user's accessKey
|
||||
* @param {string} params.data.signatureFromRequest - the signature read
|
||||
* from the request
|
||||
* @param params.data.region - the AWS region
|
||||
* @param params.data.stringToSign - the stringToSign
|
||||
* @param params.data.scopeDate - the timespan to allow the request
|
||||
* @param params.data.authType - the type of authentication (query
|
||||
* @param {string} params.data.region - the AWS region
|
||||
* @param {string} params.data.stringToSign - the stringToSign
|
||||
* @param {string} params.data.scopeDate - the timespan to allow the request
|
||||
* @param {string} params.data.authType - the type of authentication (query
|
||||
* or header)
|
||||
* @param params.data.signatureVersion - the version of the
|
||||
* @param {string} params.data.signatureVersion - the version of the
|
||||
* signature (AWS or AWS4)
|
||||
* @param params.data.signatureAge - the age of the signature in ms
|
||||
* @param params.data.timestamp - signaure timestamp
|
||||
* @param params.credentialScope - credentialScope for signature
|
||||
* @param {number} params.data.signatureAge - the age of the signature in ms
|
||||
* @param {number} params.data.timestamp - signaure timestamp
|
||||
* @param {string} params.credentialScope - credentialScope for signature
|
||||
* @param {RequestContext [] | null} requestContexts -
|
||||
* an array of RequestContext or null if authenticaiton of a chunk
|
||||
* in streamingv4 auth
|
||||
* instances which contain information for policy authorization check
|
||||
* @param callback - callback with either error or user info
|
||||
* @param {function} callback - callback with either error or user info
|
||||
* @return {undefined}
|
||||
*/
|
||||
authenticateV4Request(
|
||||
params: AuthV4RequestParams,
|
||||
requestContexts: any[] | null,
|
||||
callback: (err: Error | null, data?: any) => void
|
||||
) {
|
||||
authenticateV4Request(params, requestContexts, callback) {
|
||||
params.log.debug('authenticating V4 request');
|
||||
let serializedRCs: any;
|
||||
let serializedRCs;
|
||||
if (requestContexts) {
|
||||
serializedRCs = requestContexts.map(rc => rc.serialize());
|
||||
}
|
||||
|
@ -193,39 +140,31 @@ export default class Vault {
|
|||
params.data.region,
|
||||
params.data.scopeDate,
|
||||
{
|
||||
// @ts-ignore
|
||||
reqUid: params.log.getSerializedUids(),
|
||||
logger: params.log,
|
||||
securityToken: params.data.securityToken,
|
||||
requestContext: serializedRCs,
|
||||
},
|
||||
(err: Error | null, userInfo?: any) => vaultSignatureCb(err, userInfo,
|
||||
params.log, callback, streamingV4Params),
|
||||
(err, userInfo) => vaultSignatureCb(err, userInfo,
|
||||
params.log, callback, streamingV4Params)
|
||||
);
|
||||
}
|
||||
|
||||
/** getCanonicalIds -- call Vault to get canonicalIDs based on email
|
||||
* addresses
|
||||
* @param emailAddresses - list of emailAddresses
|
||||
* @param log - log object
|
||||
* @param callback - callback with either error or an array
|
||||
* @param {array} emailAddresses - list of emailAddresses
|
||||
* @param {object} log - log object
|
||||
* @param {function} callback - callback with either error or an array
|
||||
* of objects with each object containing the canonicalID and emailAddress
|
||||
* of an account as properties
|
||||
* @return {undefined}
|
||||
*/
|
||||
getCanonicalIds(
|
||||
emailAddresses: string[],
|
||||
log: Logger,
|
||||
callback: (
|
||||
err: Error | null,
|
||||
data?: { canonicalID: string; email: string }[]
|
||||
) => void
|
||||
) {
|
||||
getCanonicalIds(emailAddresses, log, callback) {
|
||||
log.trace('getting canonicalIDs from Vault based on emailAddresses',
|
||||
{ emailAddresses });
|
||||
this.client.getCanonicalIds(emailAddresses,
|
||||
// @ts-ignore
|
||||
{ reqUid: log.getSerializedUids() },
|
||||
(err: Error | null, info?: any) => {
|
||||
(err, info) => {
|
||||
if (err) {
|
||||
log.debug('received error message from auth provider',
|
||||
{ errorMessage: err });
|
||||
|
@ -233,17 +172,17 @@ export default class Vault {
|
|||
}
|
||||
const infoFromVault = info.message.body;
|
||||
log.trace('info received from vault', { infoFromVault });
|
||||
const foundIds: { canonicalID: string; email: string }[] = [];
|
||||
const foundIds = [];
|
||||
for (let i = 0; i < Object.keys(infoFromVault).length; i++) {
|
||||
const key = Object.keys(infoFromVault)[i];
|
||||
if (infoFromVault[key] === 'WrongFormat'
|
||||
|| infoFromVault[key] === 'NotFound') {
|
||||
return callback(errors.UnresolvableGrantByEmailAddress);
|
||||
}
|
||||
foundIds.push({
|
||||
email: key,
|
||||
canonicalID: infoFromVault[key],
|
||||
})
|
||||
const obj = {};
|
||||
obj.email = key;
|
||||
obj.canonicalID = infoFromVault[key];
|
||||
foundIds.push(obj);
|
||||
}
|
||||
return callback(null, foundIds);
|
||||
});
|
||||
|
@ -251,22 +190,18 @@ export default class Vault {
|
|||
|
||||
/** getEmailAddresses -- call Vault to get email addresses based on
|
||||
* canonicalIDs
|
||||
* @param canonicalIDs - list of canonicalIDs
|
||||
* @param log - log object
|
||||
* @param callback - callback with either error or an object
|
||||
* @param {array} canonicalIDs - list of canonicalIDs
|
||||
* @param {object} log - log object
|
||||
* @param {function} callback - callback with either error or an object
|
||||
* with canonicalID keys and email address values
|
||||
* @return {undefined}
|
||||
*/
|
||||
getEmailAddresses(
|
||||
canonicalIDs: string[],
|
||||
log: Logger,
|
||||
callback: (err: Error | null, data?: { [key: string]: any }) => void
|
||||
) {
|
||||
getEmailAddresses(canonicalIDs, log, callback) {
|
||||
log.trace('getting emailAddresses from Vault based on canonicalIDs',
|
||||
{ canonicalIDs });
|
||||
this.client.getEmailAddresses(canonicalIDs,
|
||||
// @ts-ignore
|
||||
{ reqUid: log.getSerializedUids() },
|
||||
(err: Error | null, info?: any) => {
|
||||
(err, info) => {
|
||||
if (err) {
|
||||
log.debug('received error message from vault',
|
||||
{ errorMessage: err });
|
||||
|
@ -289,40 +224,36 @@ export default class Vault {
|
|||
|
||||
/** getAccountIds -- call Vault to get accountIds based on
|
||||
* canonicalIDs
|
||||
* @param canonicalIDs - list of canonicalIDs
|
||||
* @param log - log object
|
||||
* @param callback - callback with either error or an object
|
||||
* @param {array} canonicalIDs - list of canonicalIDs
|
||||
* @param {object} log - log object
|
||||
* @param {function} callback - callback with either error or an object
|
||||
* with canonicalID keys and accountId values
|
||||
* @return {undefined}
|
||||
*/
|
||||
getAccountIds(
|
||||
canonicalIDs: string[],
|
||||
log: Logger,
|
||||
callback: (err: Error | null, data?: { [key: string]: string }) => void
|
||||
) {
|
||||
getAccountIds(canonicalIDs, log, callback) {
|
||||
log.trace('getting accountIds from Vault based on canonicalIDs',
|
||||
{ canonicalIDs });
|
||||
{ canonicalIDs });
|
||||
this.client.getAccountIds(canonicalIDs,
|
||||
// @ts-expect-error
|
||||
{ reqUid: log.getSerializedUids() },
|
||||
(err: Error | null, info?: any) => {
|
||||
if (err) {
|
||||
log.debug('received error message from vault',
|
||||
{ errorMessage: err });
|
||||
return callback(err);
|
||||
}
|
||||
const infoFromVault = info.message.body;
|
||||
log.trace('info received from vault', { infoFromVault });
|
||||
const result = {};
|
||||
/* If the accountId was not found in Vault, do not
|
||||
{ reqUid: log.getSerializedUids() },
|
||||
(err, info) => {
|
||||
if (err) {
|
||||
log.debug('received error message from vault',
|
||||
{ errorMessage: err });
|
||||
return callback(err);
|
||||
}
|
||||
const infoFromVault = info.message.body;
|
||||
log.trace('info received from vault', { infoFromVault });
|
||||
const result = {};
|
||||
/* If the accountId was not found in Vault, do not
|
||||
send the canonicalID back to the API */
|
||||
Object.keys(infoFromVault).forEach(key => {
|
||||
if (infoFromVault[key] !== 'NotFound' &&
|
||||
Object.keys(infoFromVault).forEach(key => {
|
||||
if (infoFromVault[key] !== 'NotFound' &&
|
||||
infoFromVault[key] !== 'WrongFormat') {
|
||||
result[key] = infoFromVault[key];
|
||||
}
|
||||
});
|
||||
return callback(null, result);
|
||||
result[key] = infoFromVault[key];
|
||||
}
|
||||
});
|
||||
return callback(null, result);
|
||||
});
|
||||
}
|
||||
|
||||
/** checkPolicies -- call Vault to evaluate policies
|
||||
|
@ -337,19 +268,14 @@ export default class Vault {
|
|||
* @param {object} log - log object
|
||||
* @param {function} callback - callback with either error or an array
|
||||
* of authorization results
|
||||
* @return {undefined}
|
||||
*/
|
||||
checkPolicies(
|
||||
requestContextParams: any[],
|
||||
userArn: string,
|
||||
log: Logger,
|
||||
callback: (err: Error | null, data?: any[]) => void
|
||||
) {
|
||||
checkPolicies(requestContextParams, userArn, log, callback) {
|
||||
log.trace('sending request context params to vault to evaluate' +
|
||||
'policies');
|
||||
this.client.checkPolicies(requestContextParams, userArn, {
|
||||
// @ts-ignore
|
||||
reqUid: log.getSerializedUids(),
|
||||
}, (err: Error | null, info?: any) => {
|
||||
}, (err, info) => {
|
||||
if (err) {
|
||||
log.debug('received error message from auth provider',
|
||||
{ error: err });
|
||||
|
@ -360,14 +286,13 @@ export default class Vault {
|
|||
});
|
||||
}
|
||||
|
||||
checkHealth(log: Logger, callback: (err: Error | null, data?: any) => void) {
|
||||
checkHealth(log, callback) {
|
||||
if (!this.client.healthcheck) {
|
||||
const defResp = {};
|
||||
defResp[this.implName] = { code: 200, message: 'OK' };
|
||||
return callback(null, defResp);
|
||||
}
|
||||
// @ts-ignore
|
||||
return this.client.healthcheck(log.getSerializedUids(), (err: Error | null, obj?: any) => {
|
||||
return this.client.healthcheck(log.getSerializedUids(), (err, obj) => {
|
||||
const respBody = {};
|
||||
if (err) {
|
||||
log.debug(`error from ${this.implName}`, { error: err });
|
||||
|
@ -386,19 +311,6 @@ export default class Vault {
|
|||
return callback(null, respBody);
|
||||
});
|
||||
}
|
||||
|
||||
report(log: Logger, callback: (err: Error | null, data?: any) => void) {
|
||||
// call the report function of the client
|
||||
if (!this.client.report) {
|
||||
return callback(null, {});
|
||||
}
|
||||
// @ts-ignore
|
||||
return this.client.report(log.getSerializedUids(), (err: Error | null, obj?: any) => {
|
||||
if (err) {
|
||||
log.debug(`error from ${this.implName}`, { error: err });
|
||||
return callback(err);
|
||||
}
|
||||
return callback(null, obj);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Vault;
|
|
@ -1,23 +1,22 @@
|
|||
import * as crypto from 'crypto';
|
||||
import { Logger } from 'werelogs';
|
||||
import errors from '../errors';
|
||||
import * as queryString from 'querystring';
|
||||
import AuthInfo from './AuthInfo';
|
||||
import * as v2 from './v2/authV2';
|
||||
import * as v4 from './v4/authV4';
|
||||
import * as constants from '../constants';
|
||||
import constructStringToSignV2 from './v2/constructStringToSign';
|
||||
import constructStringToSignV4 from './v4/constructStringToSign';
|
||||
import { convertUTCtoISO8601 } from './v4/timeUtils';
|
||||
import * as vaultUtilities from './backends/in_memory/vaultUtilities';
|
||||
import * as inMemoryBackend from './backends/in_memory/Backend';
|
||||
import baseBackend from './backends/base';
|
||||
import chainBackend from './backends/ChainBackend';
|
||||
import validateAuthConfig from './backends/in_memory/validateAuthConfig';
|
||||
import AuthLoader from './backends/in_memory/AuthLoader';
|
||||
import Vault from './Vault';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
let vault: Vault | null = null;
|
||||
const crypto = require('crypto');
|
||||
const errors = require('../errors').default;
|
||||
const queryString = require('querystring');
|
||||
const AuthInfo = require('./AuthInfo');
|
||||
const v2 = require('./v2/authV2');
|
||||
const v4 = require('./v4/authV4');
|
||||
const constants = require('../constants');
|
||||
const constructStringToSignV2 = require('./v2/constructStringToSign');
|
||||
const constructStringToSignV4 = require('./v4/constructStringToSign');
|
||||
const convertUTCtoISO8601 = require('./v4/timeUtils').convertUTCtoISO8601;
|
||||
const vaultUtilities = require('./in_memory/vaultUtilities');
|
||||
const backend = require('./in_memory/Backend');
|
||||
const validateAuthConfig = require('./in_memory/validateAuthConfig');
|
||||
const AuthLoader = require('./in_memory/AuthLoader');
|
||||
const Vault = require('./Vault');
|
||||
|
||||
let vault = null;
|
||||
const auth = {};
|
||||
const checkFunctions = {
|
||||
v2: {
|
||||
|
@ -34,7 +33,7 @@ const checkFunctions = {
|
|||
// 'All Users Group' so use this group as the canonicalID for the publicUser
|
||||
const publicUserInfo = new AuthInfo({ canonicalID: constants.publicId });
|
||||
|
||||
function setAuthHandler(handler: Vault) {
|
||||
function setAuthHandler(handler) {
|
||||
vault = handler;
|
||||
return auth;
|
||||
}
|
||||
|
@ -42,30 +41,25 @@ function setAuthHandler(handler: Vault) {
|
|||
/**
|
||||
* This function will check validity of request parameters to authenticate
|
||||
*
|
||||
* @param request - Http request object
|
||||
* @param log - Logger object
|
||||
* @param awsService - Aws service related
|
||||
* @param data - Parameters from queryString parsing or body of
|
||||
* @param {Http.Request} request - Http request object
|
||||
* @param {object} log - Logger object
|
||||
* @param {string} awsService - Aws service related
|
||||
* @param {object} data - Parameters from queryString parsing or body of
|
||||
* POST request
|
||||
*
|
||||
* @return ret
|
||||
* @return ret.err - arsenal.errors object if any error was found
|
||||
* @return ret.params - auth parameters to use later on for signature
|
||||
* @return {object} ret
|
||||
* @return {object} ret.err - arsenal.errors object if any error was found
|
||||
* @return {object} ret.params - auth parameters to use later on for signature
|
||||
* computation and check
|
||||
* @return ret.params.version - the auth scheme version
|
||||
* @return {object} ret.params.version - the auth scheme version
|
||||
* (undefined, 2, 4)
|
||||
* @return ret.params.data - the auth scheme's specific data
|
||||
* @return {object} ret.params.data - the auth scheme's specific data
|
||||
*/
|
||||
function extractParams(
|
||||
request: any,
|
||||
log: Logger,
|
||||
awsService: string,
|
||||
data: { [key: string]: string }
|
||||
) {
|
||||
function extractParams(request, log, awsService, data) {
|
||||
log.trace('entered', { method: 'Arsenal.auth.server.extractParams' });
|
||||
const authHeader = request.headers.authorization;
|
||||
let version: 'v2' |'v4' | null = null;
|
||||
let method: 'query' | 'headers' | null = null;
|
||||
let version = null;
|
||||
let method = null;
|
||||
|
||||
// Identify auth version and method to dispatch to the right check function
|
||||
if (authHeader) {
|
||||
|
@ -78,7 +72,7 @@ function extractParams(
|
|||
version = 'v4';
|
||||
} else {
|
||||
log.trace('invalid authorization security header',
|
||||
{ header: authHeader });
|
||||
{ header: authHeader });
|
||||
return { err: errors.AccessDenied };
|
||||
}
|
||||
} else if (data.Signature) {
|
||||
|
@ -93,7 +87,7 @@ function extractParams(
|
|||
if (version !== null && method !== null) {
|
||||
if (!checkFunctions[version] || !checkFunctions[version][method]) {
|
||||
log.trace('invalid auth version or method',
|
||||
{ version, authMethod: method });
|
||||
{ version, authMethod: method });
|
||||
return { err: errors.NotImplemented };
|
||||
}
|
||||
log.trace('identified auth method', { version, authMethod: method });
|
||||
|
@ -108,21 +102,16 @@ function extractParams(
|
|||
/**
|
||||
* This function will check validity of request parameters to authenticate
|
||||
*
|
||||
* @param request - Http request object
|
||||
* @param log - Logger object
|
||||
* @param cb - the callback
|
||||
* @param awsService - Aws service related
|
||||
* @param {Http.Request} request - Http request object
|
||||
* @param {object} log - Logger object
|
||||
* @param {function} cb - the callback
|
||||
* @param {string} awsService - Aws service related
|
||||
* @param {RequestContext[] | null} requestContexts - array of RequestContext
|
||||
* or null if no requestContexts to be sent to Vault (for instance,
|
||||
* in multi-object delete request)
|
||||
* @return {undefined}
|
||||
*/
|
||||
function doAuth(
|
||||
request: any,
|
||||
log: Logger,
|
||||
cb: (err: Error | null, data?: any) => void,
|
||||
awsService: string,
|
||||
requestContexts: any[] | null
|
||||
) {
|
||||
function doAuth(request, log, cb, awsService, requestContexts) {
|
||||
const res = extractParams(request, log, awsService, request.query);
|
||||
if (res.err) {
|
||||
return cb(res.err);
|
||||
|
@ -130,31 +119,23 @@ function doAuth(
|
|||
return cb(null, res.params);
|
||||
}
|
||||
if (requestContexts) {
|
||||
requestContexts.forEach((requestContext) => {
|
||||
const { params } = res
|
||||
if ('data' in params) {
|
||||
const { data } = params
|
||||
requestContext.setAuthType(data.authType);
|
||||
requestContext.setSignatureVersion(data.signatureVersion);
|
||||
requestContext.setSecurityToken(data.securityToken);
|
||||
if ('signatureAge' in data) {
|
||||
requestContext.setSignatureAge(data.signatureAge);
|
||||
}
|
||||
}
|
||||
requestContexts.forEach(requestContext => {
|
||||
requestContext.setAuthType(res.params.data.authType);
|
||||
requestContext.setSignatureVersion(res.params
|
||||
.data.signatureVersion);
|
||||
requestContext.setSignatureAge(res.params.data.signatureAge);
|
||||
requestContext.setSecurityToken(res.params.data.securityToken);
|
||||
});
|
||||
}
|
||||
|
||||
// Corner cases managed, we're left with normal auth
|
||||
// TODO What's happening here?
|
||||
// @ts-ignore
|
||||
res.params.log = log;
|
||||
if (res.params.version === 2) {
|
||||
// @ts-ignore
|
||||
return vault!.authenticateV2Request(res.params, requestContexts, cb);
|
||||
return vault.authenticateV2Request(res.params, requestContexts, cb);
|
||||
}
|
||||
if (res.params.version === 4) {
|
||||
// @ts-ignore
|
||||
return vault!.authenticateV4Request(res.params, requestContexts, cb);
|
||||
return vault.authenticateV4Request(res.params, requestContexts, cb,
|
||||
awsService);
|
||||
}
|
||||
|
||||
log.error('authentication method not found', {
|
||||
|
@ -163,44 +144,22 @@ function doAuth(
|
|||
return cb(errors.InternalError);
|
||||
}
|
||||
|
||||
/**
|
||||
* This function will generate a version 4 content-md5 header
|
||||
* It looks at the request path to determine what kind of header encoding is required
|
||||
*
|
||||
* @param path - the request path
|
||||
* @param payload - the request payload to hash
|
||||
*/
|
||||
function generateContentMD5Header(
|
||||
path: string,
|
||||
payload: string,
|
||||
) {
|
||||
const encoding = path && path.startsWith('/_/backbeat/') ? 'hex' : 'base64';
|
||||
return crypto.createHash('md5').update(payload, 'binary').digest(encoding);
|
||||
}
|
||||
/**
|
||||
* This function will generate a version 4 header
|
||||
*
|
||||
* @param request - Http request object
|
||||
* @param data - Parameters from queryString parsing or body of
|
||||
* @param {Http.Request} request - Http request object
|
||||
* @param {object} data - Parameters from queryString parsing or body of
|
||||
* POST request
|
||||
* @param accessKey - the accessKey
|
||||
* @param secretKeyValue - the secretKey
|
||||
* @param awsService - Aws service related
|
||||
* @param [proxyPath] - path that gets proxied by reverse proxy
|
||||
* @param [sessionToken] - security token if the access/secret keys
|
||||
* @param {string} accessKey - the accessKey
|
||||
* @param {string} secretKeyValue - the secretKey
|
||||
* @param {string} awsService - Aws service related
|
||||
* @param {sting} [proxyPath] - path that gets proxied by reverse proxy
|
||||
* @param {string} [sessionToken] - security token if the access/secret keys
|
||||
* are temporary credentials from STS
|
||||
* @param [payload] - body of the request if any
|
||||
* @return {undefined}
|
||||
*/
|
||||
function generateV4Headers(
|
||||
request: any,
|
||||
data: { [key: string]: string },
|
||||
accessKey: string,
|
||||
secretKeyValue: string,
|
||||
awsService: string,
|
||||
proxyPath?: string,
|
||||
sessionToken?: string,
|
||||
payload?: string,
|
||||
) {
|
||||
function generateV4Headers(request, data, accessKey, secretKeyValue,
|
||||
awsService, proxyPath, sessionToken) {
|
||||
Object.assign(request, { headers: {} });
|
||||
const amzDate = convertUTCtoISO8601(Date.now());
|
||||
// get date without time
|
||||
|
@ -212,9 +171,9 @@ function generateV4Headers(
|
|||
const timestamp = amzDate;
|
||||
const algorithm = 'AWS4-HMAC-SHA256';
|
||||
|
||||
payload = payload || '';
|
||||
let payload = '';
|
||||
if (request.method === 'POST') {
|
||||
payload = queryString.stringify(data, undefined, undefined, {
|
||||
payload = queryString.stringify(data, null, null, {
|
||||
encodeURIComponent,
|
||||
});
|
||||
}
|
||||
|
@ -223,7 +182,6 @@ function generateV4Headers(
|
|||
request.setHeader('host', request._headers.host);
|
||||
request.setHeader('x-amz-date', amzDate);
|
||||
request.setHeader('x-amz-content-sha256', payloadChecksum);
|
||||
request.setHeader('content-md5', generateContentMD5Header(request.path, payload));
|
||||
|
||||
if (sessionToken) {
|
||||
request.setHeader('x-amz-security-token', sessionToken);
|
||||
|
@ -234,19 +192,18 @@ function generateV4Headers(
|
|||
.filter(headerName =>
|
||||
headerName.startsWith('x-amz-')
|
||||
|| headerName.startsWith('x-scal-')
|
||||
|| headerName === 'content-md5'
|
||||
|| headerName === 'host',
|
||||
|| headerName === 'host'
|
||||
).sort().join(';');
|
||||
const params = { request, signedHeaders, payloadChecksum,
|
||||
credentialScope, timestamp, query: data,
|
||||
awsService: service, proxyPath };
|
||||
const stringToSign = constructStringToSignV4(params);
|
||||
const signingKey = vaultUtilities.calculateSigningKey(secretKeyValue,
|
||||
region,
|
||||
scopeDate,
|
||||
service);
|
||||
region,
|
||||
scopeDate,
|
||||
service);
|
||||
const signature = crypto.createHmac('sha256', signingKey)
|
||||
.update(stringToSign as string, 'binary').digest('hex');
|
||||
.update(stringToSign, 'binary').digest('hex');
|
||||
const authorizationHeader = `${algorithm} Credential=${accessKey}` +
|
||||
`/${credentialScope}, SignedHeaders=${signedHeaders}, ` +
|
||||
`Signature=${signature}`;
|
||||
|
@ -254,12 +211,21 @@ function generateV4Headers(
|
|||
Object.assign(request, { headers: {} });
|
||||
}
|
||||
|
||||
export const server = { extractParams, doAuth }
|
||||
export const client = { generateV4Headers, constructStringToSignV2 }
|
||||
export const inMemory = { backend: inMemoryBackend, validateAuthConfig, AuthLoader }
|
||||
export const backends = { baseBackend, chainBackend }
|
||||
export {
|
||||
setAuthHandler as setHandler,
|
||||
module.exports = {
|
||||
setHandler: setAuthHandler,
|
||||
server: {
|
||||
extractParams,
|
||||
doAuth,
|
||||
},
|
||||
client: {
|
||||
generateV4Headers,
|
||||
constructStringToSignV2,
|
||||
},
|
||||
inMemory: {
|
||||
backend,
|
||||
validateAuthConfig,
|
||||
AuthLoader,
|
||||
},
|
||||
AuthInfo,
|
||||
Vault
|
||||
}
|
||||
Vault,
|
||||
};
|
|
@ -1,233 +0,0 @@
|
|||
import assert from 'assert';
|
||||
import async from 'async';
|
||||
import errors from '../../errors';
|
||||
import BaseBackend from './base';
|
||||
|
||||
/**
|
||||
* Class that provides an authentication backend that will verify signatures
|
||||
* and retrieve emails and canonical ids associated with an account using a
|
||||
* given list of authentication backends and vault clients.
|
||||
*
|
||||
* @class ChainBackend
|
||||
*/
|
||||
export default class ChainBackend extends BaseBackend {
|
||||
_clients: any[];
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
* @param {string} service - service id
|
||||
* @param {object[]} clients - list of authentication backends or vault clients
|
||||
*/
|
||||
constructor(service: string, clients: any[]) {
|
||||
super(service);
|
||||
|
||||
assert(Array.isArray(clients) && clients.length > 0, 'invalid client list');
|
||||
assert(clients.every(client =>
|
||||
typeof client.verifySignatureV4 === 'function' &&
|
||||
typeof client.verifySignatureV2 === 'function' &&
|
||||
typeof client.getCanonicalIds === 'function' &&
|
||||
typeof client.getEmailAddresses === 'function' &&
|
||||
typeof client.checkPolicies === 'function' &&
|
||||
typeof client.healthcheck === 'function',
|
||||
), 'invalid client: missing required auth backend methods');
|
||||
this._clients = clients;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* try task against each client for one to be successful
|
||||
*/
|
||||
_tryEachClient(task: any, cb: any) {
|
||||
// @ts-ignore
|
||||
async.tryEach(this._clients.map(client => done => task(client, done)), cb);
|
||||
}
|
||||
|
||||
/*
|
||||
* apply task to all clients
|
||||
*/
|
||||
_forEachClient(task: any, cb: any) {
|
||||
async.map(this._clients, task, cb);
|
||||
}
|
||||
|
||||
verifySignatureV2(
|
||||
stringToSign: string,
|
||||
signatureFromRequest: string,
|
||||
accessKey: string,
|
||||
options: any,
|
||||
callback: any,
|
||||
) {
|
||||
this._tryEachClient((client, done) => client.verifySignatureV2(
|
||||
stringToSign,
|
||||
signatureFromRequest,
|
||||
accessKey,
|
||||
options,
|
||||
done,
|
||||
), callback);
|
||||
}
|
||||
|
||||
verifySignatureV4(
|
||||
stringToSign: string,
|
||||
signatureFromRequest: string,
|
||||
accessKey: string,
|
||||
region: string,
|
||||
scopeDate: string,
|
||||
options: any,
|
||||
callback: any,
|
||||
) {
|
||||
this._tryEachClient((client, done) => client.verifySignatureV4(
|
||||
stringToSign,
|
||||
signatureFromRequest,
|
||||
accessKey,
|
||||
region,
|
||||
scopeDate,
|
||||
options,
|
||||
done,
|
||||
), callback);
|
||||
}
|
||||
|
||||
static _mergeObjects(objectResponses: any) {
|
||||
return objectResponses.reduce(
|
||||
(retObj, resObj) => Object.assign(retObj, resObj.message.body),
|
||||
{});
|
||||
}
|
||||
|
||||
getCanonicalIds(emailAddresses: string[], options: any, callback: any) {
|
||||
this._forEachClient(
|
||||
(client, done) => client.getCanonicalIds(emailAddresses, options, done),
|
||||
(err, res) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
// TODO: atm naive merge, better handling of conflicting email results
|
||||
return callback(null, {
|
||||
message: {
|
||||
body: ChainBackend._mergeObjects(res),
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
getEmailAddresses(canonicalIDs: string[], options: any, callback: any) {
|
||||
this._forEachClient(
|
||||
(client, done) => client.getEmailAddresses(canonicalIDs, options, done),
|
||||
(err, res) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
return callback(null, {
|
||||
message: {
|
||||
body: ChainBackend._mergeObjects(res),
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/*
|
||||
* merge policy responses into a single message
|
||||
*/
|
||||
static _mergePolicies(policyResponses: any) {
|
||||
const policyMap: any = {};
|
||||
|
||||
policyResponses.forEach(resp => {
|
||||
if (!resp.message || !Array.isArray(resp.message.body)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const check = (policy) => {
|
||||
const key = (policy.arn || '') + (policy.versionId || '') + (policy.action || '');
|
||||
if (!policyMap[key] || !policyMap[key].isAllowed) {
|
||||
policyMap[key] = policy;
|
||||
}
|
||||
// else is duplicate policy
|
||||
};
|
||||
|
||||
resp.message.body.forEach(policy => {
|
||||
if (Array.isArray(policy)) {
|
||||
policy.forEach(authResult => check(authResult));
|
||||
} else {
|
||||
check(policy);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return Object.keys(policyMap).map(key => {
|
||||
const policyRes: any = { isAllowed: policyMap[key].isAllowed };
|
||||
if (policyMap[key].arn !== '') {
|
||||
policyRes.arn = policyMap[key].arn;
|
||||
}
|
||||
if (policyMap[key].versionId) {
|
||||
policyRes.versionId = policyMap[key].versionId;
|
||||
}
|
||||
if (policyMap[key].isImplicit !== undefined) {
|
||||
policyRes.isImplicit = policyMap[key].isImplicit;
|
||||
}
|
||||
if (policyMap[key].action) {
|
||||
policyRes.action = policyMap[key].action;
|
||||
}
|
||||
return policyRes;
|
||||
});
|
||||
}
|
||||
|
||||
/*
|
||||
response format:
|
||||
{ message: {
|
||||
body: [{}],
|
||||
code: number,
|
||||
message: string,
|
||||
} }
|
||||
*/
|
||||
checkPolicies(requestContextParams: any, userArn: string, options: any, callback: any) {
|
||||
this._forEachClient((client, done) => client.checkPolicies(
|
||||
requestContextParams,
|
||||
userArn,
|
||||
options,
|
||||
done,
|
||||
), (err, res) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
return callback(null, {
|
||||
message: {
|
||||
body: ChainBackend._mergePolicies(res),
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
healthcheck(reqUid: string, callback: any) {
|
||||
this._forEachClient((client, done) =>
|
||||
client.healthcheck(reqUid, (err, res) => done(null, {
|
||||
error: !!err ? err : null,
|
||||
status: res,
|
||||
}),
|
||||
), (err, res) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
const isError = res.some(results => !!results.error);
|
||||
if (isError) {
|
||||
return callback(errors.InternalError, res);
|
||||
}
|
||||
return callback(null, res);
|
||||
});
|
||||
}
|
||||
|
||||
report(reqUid: string, callback: any) {
|
||||
this._forEachClient((client, done) =>
|
||||
client.report(reqUid, done),
|
||||
(err, res) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
const mergedRes = res.reduce((acc, val) => {
|
||||
Object.keys(val).forEach(k => {
|
||||
acc[k] = val[k];
|
||||
});
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
return callback(null, mergedRes);
|
||||
});
|
||||
}
|
||||
}
|
|
@ -1,96 +0,0 @@
|
|||
import errors from '../../errors';
|
||||
|
||||
/**
|
||||
* Base backend class
|
||||
*
|
||||
* @class BaseBackend
|
||||
*/
|
||||
export default class BaseBackend {
|
||||
service: string;
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
* @param {string} service - service identifer for construction arn
|
||||
*/
|
||||
constructor(service: string) {
|
||||
this.service = service;
|
||||
}
|
||||
|
||||
/** verifySignatureV2
|
||||
* @param stringToSign - string to sign built per AWS rules
|
||||
* @param signatureFromRequest - signature sent with request
|
||||
* @param accessKey - account accessKey
|
||||
* @param options - contains algorithm (SHA1 or SHA256)
|
||||
* @param callback - callback with either error or user info
|
||||
* @return calls callback
|
||||
*/
|
||||
verifySignatureV2(
|
||||
stringToSign: string,
|
||||
signatureFromRequest: string,
|
||||
accessKey: string,
|
||||
options: any,
|
||||
callback: any
|
||||
) {
|
||||
return callback(errors.AuthMethodNotImplemented);
|
||||
}
|
||||
|
||||
|
||||
/** verifySignatureV4
|
||||
* @param stringToSign - string to sign built per AWS rules
|
||||
* @param signatureFromRequest - signature sent with request
|
||||
* @param accessKey - account accessKey
|
||||
* @param region - region specified in request credential
|
||||
* @param scopeDate - date specified in request credential
|
||||
* @param options - options to send to Vault
|
||||
* (just contains reqUid for logging in Vault)
|
||||
* @param callback - callback with either error or user info
|
||||
* @return calls callback
|
||||
*/
|
||||
verifySignatureV4(
|
||||
stringToSign: string,
|
||||
signatureFromRequest: string,
|
||||
accessKey: string,
|
||||
region: string,
|
||||
scopeDate: string,
|
||||
options: any,
|
||||
callback: any
|
||||
) {
|
||||
return callback(errors.AuthMethodNotImplemented);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets canonical ID's for a list of accounts
|
||||
* based on email associated with account
|
||||
* @param emails - list of email addresses
|
||||
* @param options - to send log id to vault
|
||||
* @param callback - callback to calling function
|
||||
* @returns callback with either error or
|
||||
* object with email addresses as keys and canonical IDs
|
||||
* as values
|
||||
*/
|
||||
getCanonicalIds(emails: string[], options: any, callback: any) {
|
||||
return callback(errors.AuthMethodNotImplemented);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets email addresses (referred to as diplay names for getACL's)
|
||||
* for a list of accounts based on canonical IDs associated with account
|
||||
* @param canonicalIDs - list of canonicalIDs
|
||||
* @param options - to send log id to vault
|
||||
* @param callback - callback to calling function
|
||||
* @returns callback with either error or
|
||||
* an object from Vault containing account canonicalID
|
||||
* as each object key and an email address as the value (or "NotFound")
|
||||
*/
|
||||
getEmailAddresses(canonicalIDs: string[], options: any, callback: any) {
|
||||
return callback(errors.AuthMethodNotImplemented);
|
||||
}
|
||||
|
||||
checkPolicies(requestContextParams: any, userArn: string, options: any, callback: any) {
|
||||
return callback(null, { message: { body: [] } });
|
||||
}
|
||||
|
||||
healthcheck(reqUid: string, callback: any) {
|
||||
return callback(null, { code: 200, message: 'OK' });
|
||||
}
|
||||
}
|
|
@ -1,204 +0,0 @@
|
|||
import * as fs from 'fs';
|
||||
import glob from 'simple-glob';
|
||||
import joi from 'joi';
|
||||
import werelogs from 'werelogs';
|
||||
import * as types from './types';
|
||||
import { Account, Accounts } from './types';
|
||||
import ARN from '../../../models/ARN';
|
||||
|
||||
/** Load authentication information from files or pre-loaded account objects */
|
||||
export default class AuthLoader {
|
||||
#log: werelogs.Logger;
|
||||
#authData: Accounts;
|
||||
#isValid: 'waiting-for-validation' | 'valid' | 'invalid';
|
||||
|
||||
constructor(logApi: { Logger: typeof werelogs.Logger } = werelogs) {
|
||||
this.#log = new logApi.Logger('S3');
|
||||
this.#authData = { accounts: [] };
|
||||
this.#isValid = 'waiting-for-validation';
|
||||
}
|
||||
|
||||
/** Add one or more accounts to the authentication info */
|
||||
addAccounts(authData: Accounts, filePath?: string) {
|
||||
const isValid = this.#isAuthDataValid(authData, filePath);
|
||||
if (isValid) {
|
||||
this.#authData.accounts = [
|
||||
...this.#authData.accounts,
|
||||
...authData.accounts,
|
||||
];
|
||||
// defer validity checking when getting data to avoid
|
||||
// logging multiple times the errors (we need to validate
|
||||
// all accounts at once to detect duplicate values)
|
||||
if (this.#isValid === 'valid') {
|
||||
this.#isValid = 'waiting-for-validation';
|
||||
}
|
||||
} else {
|
||||
this.#isValid = 'invalid';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add account information from a file. Use { legacy: false } as an option
|
||||
* to use the new, Promise-based version.
|
||||
*
|
||||
* @param filePath - file path containing JSON
|
||||
* authentication info (see {@link addAccounts()} for format)
|
||||
*/
|
||||
addFile(filePath: string, options: { legacy: false }): Promise<void>;
|
||||
/** @deprecated Please use Promise-version instead. */
|
||||
addFile(filePath: string, options?: { legacy: true }): void;
|
||||
addFile(filePath: string, options = { legacy: true }) {
|
||||
// On deprecation, remove the legacy part and keep the promises.
|
||||
const readFunc: any = options.legacy ? fs.readFileSync : fs.promises.readFile;
|
||||
const readResult = readFunc(filePath, 'utf8') as Promise<string> | string;
|
||||
const prom = Promise.resolve(readResult).then((data) => {
|
||||
const authData = JSON.parse(data);
|
||||
this.addAccounts(authData, filePath);
|
||||
});
|
||||
return options.legacy ? undefined : prom;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add account information from a filesystem path
|
||||
*
|
||||
* @param globPattern - filesystem glob pattern,
|
||||
* can be a single string or an array of glob patterns. Globs
|
||||
* can be simple file paths or can contain glob matching
|
||||
* characters, like '/a/b/*.json'. The matching files are
|
||||
* individually loaded as JSON and accounts are added. See
|
||||
* {@link addAccounts()} for JSON format.
|
||||
*/
|
||||
addFilesByGlob(globPattern: string | string[]) {
|
||||
// FIXME switch glob to async version
|
||||
const files = glob(globPattern);
|
||||
files.forEach((filePath) => this.addFile(filePath));
|
||||
}
|
||||
|
||||
/**
|
||||
* Perform validation on authentication info previously
|
||||
* loaded. Note that it has to be done on the entire set after an
|
||||
* update to catch duplicate account IDs or access keys.
|
||||
*/
|
||||
validate() {
|
||||
if (this.#isValid === 'waiting-for-validation') {
|
||||
const isValid = this.#isAuthDataValid(this.#authData);
|
||||
this.#isValid = isValid ? 'valid' : 'invalid';
|
||||
}
|
||||
return this.#isValid === 'valid';
|
||||
}
|
||||
|
||||
/**
|
||||
* Get authentication info as a plain JS object containing all accounts
|
||||
* under the "accounts" attribute, with validation.
|
||||
*/
|
||||
get data() {
|
||||
return this.validate() ? this.#authData : null;
|
||||
}
|
||||
|
||||
/** backward-compat: ignore arn if starts with 'aws:' and log a warning */
|
||||
#isNotLegacyAWSARN(account: Account, filePath?: string) {
|
||||
if (account.arn.startsWith('aws:')) {
|
||||
const { name: accountName, arn: accountArn } = account;
|
||||
this.#log.error(
|
||||
'account must have a valid AWS ARN, legacy examples ' +
|
||||
"starting with 'aws:' are not supported anymore. " +
|
||||
'Please convert to a proper account entry (see ' +
|
||||
'examples at https://github.com/scality/S3/blob/' +
|
||||
'master/conf/authdata.json). Also note that support ' +
|
||||
'for account users has been dropped.',
|
||||
{ accountName, accountArn, filePath }
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#isValidUsers(account: Account, filePath?: string) {
|
||||
if (account.users) {
|
||||
const { name: accountName, arn: accountArn } = account;
|
||||
this.#log.error(
|
||||
'support for account users has been dropped, consider ' +
|
||||
'turning users into account entries (see examples at ' +
|
||||
'https://github.com/scality/S3/blob/master/conf/' +
|
||||
'authdata.json)',
|
||||
{ accountName, accountArn, filePath }
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#isValidARN(account: Account, filePath?: string) {
|
||||
const arnObj = ARN.createFromString(account.arn);
|
||||
const { name: accountName, arn: accountArn } = account;
|
||||
if (arnObj instanceof ARN) {
|
||||
if (!arnObj.isIAMAccount()) {
|
||||
this.#log.error('authentication config validation error', {
|
||||
reason: 'not an IAM account ARN',
|
||||
accountName,
|
||||
accountArn,
|
||||
filePath,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
this.#log.error('authentication config validation error', {
|
||||
reason: arnObj.error.description,
|
||||
accountName,
|
||||
accountArn,
|
||||
filePath,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#isAuthDataValid(authData: any, filePath?: string) {
|
||||
const options = { abortEarly: true };
|
||||
const response = types.validators.accounts.validate(authData, options);
|
||||
if (response.error) {
|
||||
this.#dumpJoiErrors(response.error.details, filePath);
|
||||
return false;
|
||||
}
|
||||
const validAccounts = response.value.accounts.filter(
|
||||
(account: Account) =>
|
||||
this.#isNotLegacyAWSARN(account, filePath) &&
|
||||
this.#isValidUsers(account, filePath) &&
|
||||
this.#isValidARN(account, filePath)
|
||||
);
|
||||
const areSomeInvalidAccounts =
|
||||
validAccounts.length !== response.value.accounts.length;
|
||||
if (areSomeInvalidAccounts) {
|
||||
return false;
|
||||
}
|
||||
const keys = validAccounts.flatMap((account) => account.keys);
|
||||
const uniqueKeysValidator = types.validators.keys.unique('access');
|
||||
const areKeysUnique = uniqueKeysValidator.validate(keys);
|
||||
if (areKeysUnique.error) {
|
||||
this.#dumpJoiErrors(areKeysUnique.error.details, filePath);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#dumpJoiErrors(errors: joi.ValidationErrorItem[], filePath?: string) {
|
||||
errors.forEach((err) => {
|
||||
const baseLogInfo = { item: err.path, filePath };
|
||||
const logInfo = () => {
|
||||
if (err.type === 'array.unique') {
|
||||
const reason = `duplicate value '${err.context?.path}'`;
|
||||
const dupValue = err.context?.value[err.context.path];
|
||||
return { ...baseLogInfo, reason, dupValue };
|
||||
} else {
|
||||
const reason = err.message;
|
||||
const context = err.context;
|
||||
return { ...baseLogInfo, reason, context };
|
||||
}
|
||||
};
|
||||
this.#log.error(
|
||||
'authentication config validation error',
|
||||
logInfo()
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
|
@ -1,93 +0,0 @@
|
|||
import { Accounts, Account, Entity } from './types';
|
||||
|
||||
/**
|
||||
* Class that provides an internal indexing over the simple data provided by
|
||||
* the authentication configuration file for the memory backend. This allows
|
||||
* accessing the different authentication entities through various types of
|
||||
* keys.
|
||||
*/
|
||||
export default class Indexer {
|
||||
accountsBy: {
|
||||
canId: { [id: string]: Entity | undefined },
|
||||
accessKey: { [id: string]: Entity | undefined },
|
||||
email: { [id: string]: Entity | undefined },
|
||||
}
|
||||
|
||||
constructor(authdata?: Accounts) {
|
||||
this.accountsBy = {
|
||||
canId: {},
|
||||
accessKey: {},
|
||||
email: {},
|
||||
};
|
||||
|
||||
/*
|
||||
* This may happen if the application is configured to use another
|
||||
* authentication backend than in-memory.
|
||||
* As such, we're managing the error here to avoid screwing up there.
|
||||
*/
|
||||
if (!authdata) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.#build(authdata);
|
||||
}
|
||||
|
||||
#indexAccount(account: Account) {
|
||||
const accountData: Entity = {
|
||||
arn: account.arn,
|
||||
canonicalID: account.canonicalID,
|
||||
shortid: account.shortid,
|
||||
accountDisplayName: account.name,
|
||||
email: account.email.toLowerCase(),
|
||||
keys: [],
|
||||
};
|
||||
this.accountsBy.canId[accountData.canonicalID] = accountData;
|
||||
this.accountsBy.email[accountData.email] = accountData;
|
||||
if (account.keys !== undefined) {
|
||||
account.keys.forEach(key => {
|
||||
accountData.keys.push(key);
|
||||
this.accountsBy.accessKey[key.access] = accountData;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#build(authdata: Accounts) {
|
||||
authdata.accounts.forEach(account => {
|
||||
this.#indexAccount(account);
|
||||
});
|
||||
}
|
||||
|
||||
/** This method returns the account associated to a canonical ID. */
|
||||
getEntityByCanId(canId: string): Entity | undefined {
|
||||
return this.accountsBy.canId[canId];
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns the entity (either an account or a user) associated
|
||||
* to a canonical ID.
|
||||
* @param {string} key - The accessKey of the entity
|
||||
*/
|
||||
getEntityByKey(key: string): Entity | undefined {
|
||||
return this.accountsBy.accessKey[key];
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns the entity (either an account or a user) associated
|
||||
* to an email address.
|
||||
*/
|
||||
getEntityByEmail(email: string): Entity | undefined {
|
||||
const lowerCasedEmail = email.toLowerCase();
|
||||
return this.accountsBy.email[lowerCasedEmail];
|
||||
}
|
||||
|
||||
/** This method returns the secret key associated with the entity. */
|
||||
getSecretKey(entity: Entity, accessKey: string) {
|
||||
const keys = entity.keys.filter(kv => kv.access === accessKey);
|
||||
return keys[0].secret;
|
||||
}
|
||||
|
||||
/** This method returns the account display name associated with the entity. */
|
||||
getAcctDisplayName(entity: Entity) {
|
||||
return entity.accountDisplayName;
|
||||
}
|
||||
}
|
|
@ -1,51 +0,0 @@
|
|||
import joi from 'joi';
|
||||
|
||||
export type Callback<Data = any> = (err?: Error | null | undefined, data?: Data) => void;
|
||||
|
||||
export type Credentials = { access: string; secret: string };
|
||||
export type Base = {
|
||||
arn: string;
|
||||
canonicalID: string;
|
||||
shortid: string;
|
||||
email: string;
|
||||
keys: Credentials[];
|
||||
};
|
||||
export type Account = Base & { name: string; users: any[] };
|
||||
export type Accounts = { accounts: Account[] };
|
||||
export type Entity = Base & { accountDisplayName: string };
|
||||
|
||||
const keys = ((): joi.ArraySchema => {
|
||||
const str = joi.string().required();
|
||||
const items = { access: str, secret: str };
|
||||
return joi.array().items(items).required();
|
||||
})();
|
||||
|
||||
const account = (() => {
|
||||
return joi.object<Account>({
|
||||
name: joi.string().required(),
|
||||
email: joi.string().email().required(),
|
||||
arn: joi.string().required(),
|
||||
canonicalID: joi.string().required(),
|
||||
shortid: joi
|
||||
.string()
|
||||
.regex(/^[0-9]{12}$/)
|
||||
.required(),
|
||||
keys: keys,
|
||||
// backward-compat
|
||||
users: joi.array(),
|
||||
});
|
||||
})();
|
||||
|
||||
const accounts = (() => {
|
||||
return joi.object<Accounts>({
|
||||
accounts: joi
|
||||
.array()
|
||||
.items(account)
|
||||
.required()
|
||||
.unique('arn')
|
||||
.unique('email')
|
||||
.unique('canonicalID'),
|
||||
});
|
||||
})();
|
||||
|
||||
export const validators = { keys, account, accounts };
|
|
@ -1,16 +0,0 @@
|
|||
import { Logger } from 'werelogs';
|
||||
import AuthLoader from './AuthLoader';
|
||||
import { Accounts } from './types';
|
||||
|
||||
/**
|
||||
* @deprecated please use {@link AuthLoader} class instead
|
||||
* @return true on erroneous data false on success
|
||||
*/
|
||||
export default function validateAuthConfig(
|
||||
authdata: Accounts,
|
||||
logApi?: { Logger: typeof Logger }
|
||||
) {
|
||||
const authLoader = new AuthLoader(logApi);
|
||||
authLoader.addAccounts(authdata);
|
||||
return !authLoader.validate();
|
||||
}
|
|
@ -0,0 +1,223 @@
|
|||
const fs = require('fs');
|
||||
const glob = require('simple-glob');
|
||||
const joi = require('@hapi/joi');
|
||||
const werelogs = require('werelogs');
|
||||
|
||||
const ARN = require('../../models/ARN');
|
||||
|
||||
/**
|
||||
* Load authentication information from files or pre-loaded account
|
||||
* objects
|
||||
*
|
||||
* @class AuthLoader
|
||||
*/
|
||||
class AuthLoader {
|
||||
constructor(logApi) {
|
||||
this._log = new (logApi || werelogs).Logger('S3');
|
||||
this._authData = { accounts: [] };
|
||||
// null: unknown validity, true/false: valid or invalid
|
||||
this._isValid = null;
|
||||
|
||||
this._joiKeysValidator = joi.array()
|
||||
.items({
|
||||
access: joi.string().required(),
|
||||
secret: joi.string().required(),
|
||||
})
|
||||
.required();
|
||||
|
||||
const accountsJoi = joi.array()
|
||||
.items({
|
||||
name: joi.string().required(),
|
||||
email: joi.string().email().required(),
|
||||
arn: joi.string().required(),
|
||||
canonicalID: joi.string().required(),
|
||||
shortid: joi.string().regex(/^[0-9]{12}$/).required(),
|
||||
keys: this._joiKeysValidator,
|
||||
// backward-compat
|
||||
users: joi.array(),
|
||||
})
|
||||
.required()
|
||||
.unique('arn')
|
||||
.unique('email')
|
||||
.unique('canonicalID');
|
||||
this._joiValidator = joi.object({ accounts: accountsJoi });
|
||||
}
|
||||
|
||||
/**
|
||||
* add one or more accounts to the authentication info
|
||||
*
|
||||
* @param {object} authData - authentication data
|
||||
* @param {object[]} authData.accounts - array of account data
|
||||
* @param {string} authData.accounts[].name - account name
|
||||
* @param {string} authData.accounts[].email: email address
|
||||
* @param {string} authData.accounts[].arn: account ARN,
|
||||
* e.g. 'arn:aws:iam::123456789012:root'
|
||||
* @param {string} authData.accounts[].canonicalID account
|
||||
* canonical ID
|
||||
* @param {string} authData.accounts[].shortid account ID number,
|
||||
* e.g. '123456789012'
|
||||
* @param {object[]} authData.accounts[].keys array of
|
||||
* access/secret keys
|
||||
* @param {object[]} authData.accounts[].keys[].access access key
|
||||
* @param {object[]} authData.accounts[].keys[].secret secret key
|
||||
* @param {string} [filePath] - optional file path info for
|
||||
* logging purpose
|
||||
* @return {undefined}
|
||||
*/
|
||||
addAccounts(authData, filePath) {
|
||||
const isValid = this._validateData(authData, filePath);
|
||||
if (isValid) {
|
||||
this._authData.accounts =
|
||||
this._authData.accounts.concat(authData.accounts);
|
||||
// defer validity checking when getting data to avoid
|
||||
// logging multiple times the errors (we need to validate
|
||||
// all accounts at once to detect duplicate values)
|
||||
if (this._isValid) {
|
||||
this._isValid = null;
|
||||
}
|
||||
} else {
|
||||
this._isValid = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* add account information from a file
|
||||
*
|
||||
* @param {string} filePath - file path containing JSON
|
||||
* authentication info (see {@link addAccounts()} for format)
|
||||
* @return {undefined}
|
||||
*/
|
||||
addFile(filePath) {
|
||||
const authData = JSON.parse(fs.readFileSync(filePath));
|
||||
this.addAccounts(authData, filePath);
|
||||
}
|
||||
|
||||
/**
|
||||
* add account information from a filesystem path
|
||||
*
|
||||
* @param {string|string[]} globPattern - filesystem glob pattern,
|
||||
* can be a single string or an array of glob patterns. Globs
|
||||
* can be simple file paths or can contain glob matching
|
||||
* characters, like '/a/b/*.json'. The matching files are
|
||||
* individually loaded as JSON and accounts are added. See
|
||||
* {@link addAccounts()} for JSON format.
|
||||
* @return {undefined}
|
||||
*/
|
||||
addFilesByGlob(globPattern) {
|
||||
const files = glob(globPattern);
|
||||
files.forEach(filePath => this.addFile(filePath));
|
||||
}
|
||||
|
||||
/**
|
||||
* perform validation on authentication info previously
|
||||
* loaded. Note that it has to be done on the entire set after an
|
||||
* update to catch duplicate account IDs or access keys.
|
||||
*
|
||||
* @return {boolean} true if authentication info is valid
|
||||
* false otherwise
|
||||
*/
|
||||
validate() {
|
||||
if (this._isValid === null) {
|
||||
this._isValid = this._validateData(this._authData);
|
||||
}
|
||||
return this._isValid;
|
||||
}
|
||||
|
||||
/**
|
||||
* get authentication info as a plain JS object containing all accounts
|
||||
* under the "accounts" attribute, with validation.
|
||||
*
|
||||
* @return {object|null} the validated authentication data
|
||||
* null if invalid
|
||||
*/
|
||||
getData() {
|
||||
return this.validate() ? this._authData : null;
|
||||
}
|
||||
|
||||
_validateData(authData, filePath) {
|
||||
const res = joi.validate(authData, this._joiValidator,
|
||||
{ abortEarly: false });
|
||||
if (res.error) {
|
||||
this._dumpJoiErrors(res.error.details, filePath);
|
||||
return false;
|
||||
}
|
||||
let allKeys = [];
|
||||
let arnError = false;
|
||||
const validatedAuth = res.value;
|
||||
validatedAuth.accounts.forEach(account => {
|
||||
// backward-compat: ignore arn if starts with 'aws:' and log a
|
||||
// warning
|
||||
if (account.arn.startsWith('aws:')) {
|
||||
this._log.error(
|
||||
'account must have a valid AWS ARN, legacy examples ' +
|
||||
'starting with \'aws:\' are not supported anymore. ' +
|
||||
'Please convert to a proper account entry (see ' +
|
||||
'examples at https://github.com/scality/S3/blob/' +
|
||||
'master/conf/authdata.json). Also note that support ' +
|
||||
'for account users has been dropped.',
|
||||
{ accountName: account.name, accountArn: account.arn,
|
||||
filePath });
|
||||
arnError = true;
|
||||
return;
|
||||
}
|
||||
if (account.users) {
|
||||
this._log.error(
|
||||
'support for account users has been dropped, consider ' +
|
||||
'turning users into account entries (see examples at ' +
|
||||
'https://github.com/scality/S3/blob/master/conf/' +
|
||||
'authdata.json)',
|
||||
{ accountName: account.name, accountArn: account.arn,
|
||||
filePath });
|
||||
arnError = true;
|
||||
return;
|
||||
}
|
||||
const arnObj = ARN.createFromString(account.arn);
|
||||
if (arnObj.error) {
|
||||
this._log.error(
|
||||
'authentication config validation error',
|
||||
{ reason: arnObj.error.description,
|
||||
accountName: account.name, accountArn: account.arn,
|
||||
filePath });
|
||||
arnError = true;
|
||||
return;
|
||||
}
|
||||
if (!arnObj.isIAMAccount()) {
|
||||
this._log.error(
|
||||
'authentication config validation error',
|
||||
{ reason: 'not an IAM account ARN',
|
||||
accountName: account.name, accountArn: account.arn,
|
||||
filePath });
|
||||
arnError = true;
|
||||
return;
|
||||
}
|
||||
allKeys = allKeys.concat(account.keys);
|
||||
});
|
||||
if (arnError) {
|
||||
return false;
|
||||
}
|
||||
const uniqueKeysRes = joi.validate(
|
||||
allKeys, this._joiKeysValidator.unique('access'));
|
||||
if (uniqueKeysRes.error) {
|
||||
this._dumpJoiErrors(uniqueKeysRes.error.details, filePath);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
_dumpJoiErrors(errors, filePath) {
|
||||
errors.forEach(err => {
|
||||
const logInfo = { item: err.path, filePath };
|
||||
if (err.type === 'array.unique') {
|
||||
logInfo.reason = `duplicate value '${err.context.path}'`;
|
||||
logInfo.dupValue = err.context.value[err.context.path];
|
||||
} else {
|
||||
logInfo.reason = err.message;
|
||||
logInfo.context = err.context;
|
||||
}
|
||||
this._log.error('authentication config validation error',
|
||||
logInfo);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = AuthLoader;
|
|
@ -1,12 +1,13 @@
|
|||
import crypto from 'crypto';
|
||||
import { Logger } from 'werelogs';
|
||||
import errors from '../../../errors';
|
||||
import { calculateSigningKey, hashSignature } from './vaultUtilities';
|
||||
import Indexer from './Indexer';
|
||||
import BaseBackend from '../base';
|
||||
import { Accounts } from './types';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
function _formatResponse(userInfoToSend: any) {
|
||||
const crypto = require('crypto');
|
||||
|
||||
const errors = require('../../errors').default;
|
||||
const calculateSigningKey = require('./vaultUtilities').calculateSigningKey;
|
||||
const hashSignature = require('./vaultUtilities').hashSignature;
|
||||
const Indexer = require('./Indexer');
|
||||
|
||||
function _formatResponse(userInfoToSend) {
|
||||
return {
|
||||
message: {
|
||||
body: { userInfo: userInfoToSend },
|
||||
|
@ -18,32 +19,32 @@ function _formatResponse(userInfoToSend: any) {
|
|||
* Class that provides a memory backend for verifying signatures and getting
|
||||
* emails and canonical ids associated with an account.
|
||||
*
|
||||
* @class InMemoryBackend
|
||||
* @class Backend
|
||||
*/
|
||||
class InMemoryBackend extends BaseBackend {
|
||||
indexer: Indexer;
|
||||
formatResponse: any;
|
||||
|
||||
class Backend {
|
||||
/**
|
||||
* @constructor
|
||||
* @param service - service identifer for construction arn
|
||||
* @param indexer - indexer instance for retrieving account info
|
||||
* @param formatter - function which accepts user info to send
|
||||
* @param {string} service - service identifer for construction arn
|
||||
* @param {Indexer} indexer - indexer instance for retrieving account info
|
||||
* @param {function} formatter - function which accepts user info to send
|
||||
* back and returns it in an object
|
||||
*/
|
||||
constructor(service: string, indexer: Indexer, formatter: typeof _formatResponse) {
|
||||
super(service);
|
||||
constructor(service, indexer, formatter) {
|
||||
this.service = service;
|
||||
this.indexer = indexer;
|
||||
this.formatResponse = formatter;
|
||||
}
|
||||
|
||||
verifySignatureV2(
|
||||
stringToSign: string,
|
||||
signatureFromRequest: string,
|
||||
accessKey: string,
|
||||
options: any,
|
||||
callback: any,
|
||||
) {
|
||||
/** verifySignatureV2
|
||||
* @param {string} stringToSign - string to sign built per AWS rules
|
||||
* @param {string} signatureFromRequest - signature sent with request
|
||||
* @param {string} accessKey - account accessKey
|
||||
* @param {object} options - contains algorithm (SHA1 or SHA256)
|
||||
* @param {function} callback - callback with either error or user info
|
||||
* @return {function} calls callback
|
||||
*/
|
||||
verifySignatureV2(stringToSign, signatureFromRequest,
|
||||
accessKey, options, callback) {
|
||||
const entity = this.indexer.getEntityByKey(accessKey);
|
||||
if (!entity) {
|
||||
return callback(errors.InvalidAccessKeyId);
|
||||
|
@ -58,22 +59,26 @@ class InMemoryBackend extends BaseBackend {
|
|||
accountDisplayName: this.indexer.getAcctDisplayName(entity),
|
||||
canonicalID: entity.canonicalID,
|
||||
arn: entity.arn,
|
||||
// @ts-ignore
|
||||
IAMdisplayName: entity.IAMdisplayName,
|
||||
};
|
||||
const vaultReturnObject = this.formatResponse(userInfoToSend);
|
||||
return callback(null, vaultReturnObject);
|
||||
}
|
||||
|
||||
verifySignatureV4(
|
||||
stringToSign: string,
|
||||
signatureFromRequest: string,
|
||||
accessKey: string,
|
||||
region: string,
|
||||
scopeDate: string,
|
||||
options: any,
|
||||
callback: any,
|
||||
) {
|
||||
|
||||
/** verifySignatureV4
|
||||
* @param {string} stringToSign - string to sign built per AWS rules
|
||||
* @param {string} signatureFromRequest - signature sent with request
|
||||
* @param {string} accessKey - account accessKey
|
||||
* @param {string} region - region specified in request credential
|
||||
* @param {string} scopeDate - date specified in request credential
|
||||
* @param {object} options - options to send to Vault
|
||||
* (just contains reqUid for logging in Vault)
|
||||
* @param {function} callback - callback with either error or user info
|
||||
* @return {function} calls callback
|
||||
*/
|
||||
verifySignatureV4(stringToSign, signatureFromRequest, accessKey,
|
||||
region, scopeDate, options, callback) {
|
||||
const entity = this.indexer.getEntityByKey(accessKey);
|
||||
if (!entity) {
|
||||
return callback(errors.InvalidAccessKeyId);
|
||||
|
@ -89,14 +94,23 @@ class InMemoryBackend extends BaseBackend {
|
|||
accountDisplayName: this.indexer.getAcctDisplayName(entity),
|
||||
canonicalID: entity.canonicalID,
|
||||
arn: entity.arn,
|
||||
// @ts-ignore
|
||||
IAMdisplayName: entity.IAMdisplayName,
|
||||
};
|
||||
const vaultReturnObject = this.formatResponse(userInfoToSend);
|
||||
return callback(null, vaultReturnObject);
|
||||
}
|
||||
|
||||
getCanonicalIds(emails: string[], log: Logger, cb: any) {
|
||||
/**
|
||||
* Gets canonical ID's for a list of accounts
|
||||
* based on email associated with account
|
||||
* @param {array} emails - list of email addresses
|
||||
* @param {object} log - log object
|
||||
* @param {function} cb - callback to calling function
|
||||
* @returns {function} callback with either error or
|
||||
* object with email addresses as keys and canonical IDs
|
||||
* as values
|
||||
*/
|
||||
getCanonicalIds(emails, log, cb) {
|
||||
const results = {};
|
||||
emails.forEach(email => {
|
||||
const lowercasedEmail = email.toLowerCase();
|
||||
|
@ -116,7 +130,17 @@ class InMemoryBackend extends BaseBackend {
|
|||
return cb(null, vaultReturnObject);
|
||||
}
|
||||
|
||||
getEmailAddresses(canonicalIDs: string[], options: any, cb: any) {
|
||||
/**
|
||||
* Gets email addresses (referred to as diplay names for getACL's)
|
||||
* for a list of accounts based on canonical IDs associated with account
|
||||
* @param {array} canonicalIDs - list of canonicalIDs
|
||||
* @param {object} options - to send log id to vault
|
||||
* @param {function} cb - callback to calling function
|
||||
* @returns {function} callback with either error or
|
||||
* an object from Vault containing account canonicalID
|
||||
* as each object key and an email address as the value (or "NotFound")
|
||||
*/
|
||||
getEmailAddresses(canonicalIDs, options, cb) {
|
||||
const results = {};
|
||||
canonicalIDs.forEach(canonicalId => {
|
||||
const foundEntity = this.indexer.getEntityByCanId(canonicalId);
|
||||
|
@ -137,14 +161,14 @@ class InMemoryBackend extends BaseBackend {
|
|||
/**
|
||||
* Gets accountIds for a list of accounts based on
|
||||
* the canonical IDs associated with the account
|
||||
* @param canonicalIDs - list of canonicalIDs
|
||||
* @param options - to send log id to vault
|
||||
* @param cb - callback to calling function
|
||||
* @returns callback with either error or
|
||||
* @param {array} canonicalIDs - list of canonicalIDs
|
||||
* @param {object} options - to send log id to vault
|
||||
* @param {function} cb - callback to calling function
|
||||
* @returns {function} callback with either error or
|
||||
* an object from Vault containing account canonicalID
|
||||
* as each object key and an accountId as the value (or "NotFound")
|
||||
*/
|
||||
getAccountIds(canonicalIDs: string[], options: any, cb: any) {
|
||||
getAccountIds(canonicalIDs, options, cb) {
|
||||
const results = {};
|
||||
canonicalIDs.forEach(canonicalID => {
|
||||
const foundEntity = this.indexer.getEntityByCanId(canonicalID);
|
||||
|
@ -161,34 +185,33 @@ class InMemoryBackend extends BaseBackend {
|
|||
};
|
||||
return cb(null, vaultReturnObject);
|
||||
}
|
||||
|
||||
report(log: Logger, callback: any) {
|
||||
return callback(null, {});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class S3AuthBackend extends InMemoryBackend {
|
||||
class S3AuthBackend extends Backend {
|
||||
/**
|
||||
* @constructor
|
||||
* @param authdata - the authentication config file's data
|
||||
* @param authdata.accounts - array of account objects
|
||||
* @param authdata.accounts[].name - account name
|
||||
* @param authdata.accounts[].email - account email
|
||||
* @param authdata.accounts[].arn - IAM resource name
|
||||
* @param authdata.accounts[].canonicalID - account canonical ID
|
||||
* @param authdata.accounts[].shortid - short account ID
|
||||
* @param authdata.accounts[].keys - array of key objects
|
||||
* @param authdata.accounts[].keys[].access - access key
|
||||
* @param authdata.accounts[].keys[].secret - secret key
|
||||
* @param {object} authdata - the authentication config file's data
|
||||
* @param {object[]} authdata.accounts - array of account objects
|
||||
* @param {string=} authdata.accounts[].name - account name
|
||||
* @param {string} authdata.accounts[].email - account email
|
||||
* @param {string} authdata.accounts[].arn - IAM resource name
|
||||
* @param {string} authdata.accounts[].canonicalID - account canonical ID
|
||||
* @param {string} authdata.accounts[].shortid - short account ID
|
||||
* @param {object[]=} authdata.accounts[].keys - array of key objects
|
||||
* @param {string} authdata.accounts[].keys[].access - access key
|
||||
* @param {string} authdata.accounts[].keys[].secret - secret key
|
||||
* @return {undefined}
|
||||
*/
|
||||
constructor(authdata?: Accounts) {
|
||||
constructor(authdata) {
|
||||
super('s3', new Indexer(authdata), _formatResponse);
|
||||
}
|
||||
|
||||
refreshAuthData(authData?: Accounts) {
|
||||
refreshAuthData(authData) {
|
||||
this.indexer = new Indexer(authData);
|
||||
}
|
||||
}
|
||||
|
||||
export { S3AuthBackend as s3 }
|
||||
module.exports = {
|
||||
s3: S3AuthBackend,
|
||||
};
|
|
@ -0,0 +1,145 @@
|
|||
/**
|
||||
* Class that provides an internal indexing over the simple data provided by
|
||||
* the authentication configuration file for the memory backend. This allows
|
||||
* accessing the different authentication entities through various types of
|
||||
* keys.
|
||||
*
|
||||
* @class Indexer
|
||||
*/
|
||||
class Indexer {
|
||||
/**
|
||||
* @constructor
|
||||
* @param {object} authdata - the authentication config file's data
|
||||
* @param {object[]} authdata.accounts - array of account objects
|
||||
* @param {string=} authdata.accounts[].name - account name
|
||||
* @param {string} authdata.accounts[].email - account email
|
||||
* @param {string} authdata.accounts[].arn - IAM resource name
|
||||
* @param {string} authdata.accounts[].canonicalID - account canonical ID
|
||||
* @param {string} authdata.accounts[].shortid - short account ID
|
||||
* @param {object[]=} authdata.accounts[].keys - array of key objects
|
||||
* @param {string} authdata.accounts[].keys[].access - access key
|
||||
* @param {string} authdata.accounts[].keys[].secret - secret key
|
||||
* @return {undefined}
|
||||
*/
|
||||
constructor(authdata) {
|
||||
this.accountsBy = {
|
||||
canId: {},
|
||||
accessKey: {},
|
||||
email: {},
|
||||
};
|
||||
|
||||
/*
|
||||
* This may happen if the application is configured to use another
|
||||
* authentication backend than in-memory.
|
||||
* As such, we're managing the error here to avoid screwing up there.
|
||||
*/
|
||||
if (!authdata) {
|
||||
return;
|
||||
}
|
||||
|
||||
this._build(authdata);
|
||||
}
|
||||
|
||||
_indexAccount(account) {
|
||||
const accountData = {
|
||||
arn: account.arn,
|
||||
canonicalID: account.canonicalID,
|
||||
shortid: account.shortid,
|
||||
accountDisplayName: account.name,
|
||||
email: account.email.toLowerCase(),
|
||||
keys: [],
|
||||
};
|
||||
this.accountsBy.canId[accountData.canonicalID] = accountData;
|
||||
this.accountsBy.email[accountData.email] = accountData;
|
||||
if (account.keys !== undefined) {
|
||||
account.keys.forEach(key => {
|
||||
accountData.keys.push(key);
|
||||
this.accountsBy.accessKey[key.access] = accountData;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
_build(authdata) {
|
||||
authdata.accounts.forEach(account => {
|
||||
this._indexAccount(account);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns the account associated to a canonical ID.
|
||||
*
|
||||
* @param {string} canId - The canonicalId of the account
|
||||
* @return {Object} account - The account object
|
||||
* @return {Object} account.arn - The account's ARN
|
||||
* @return {Object} account.canonicalID - The account's canonical ID
|
||||
* @return {Object} account.shortid - The account's internal shortid
|
||||
* @return {Object} account.accountDisplayName - The account's display name
|
||||
* @return {Object} account.email - The account's lowercased email
|
||||
*/
|
||||
getEntityByCanId(canId) {
|
||||
return this.accountsBy.canId[canId];
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns the entity (either an account or a user) associated
|
||||
* to a canonical ID.
|
||||
*
|
||||
* @param {string} key - The accessKey of the entity
|
||||
* @return {Object} entity - The entity object
|
||||
* @return {Object} entity.arn - The entity's ARN
|
||||
* @return {Object} entity.canonicalID - The canonical ID for the entity's
|
||||
* account
|
||||
* @return {Object} entity.shortid - The entity's internal shortid
|
||||
* @return {Object} entity.accountDisplayName - The entity's account
|
||||
* display name
|
||||
* @return {Object} entity.IAMDisplayName - The user's display name
|
||||
* (if the entity is an user)
|
||||
* @return {Object} entity.email - The entity's lowercased email
|
||||
*/
|
||||
getEntityByKey(key) {
|
||||
return this.accountsBy.accessKey[key];
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns the entity (either an account or a user) associated
|
||||
* to an email address.
|
||||
*
|
||||
* @param {string} email - The email address
|
||||
* @return {Object} entity - The entity object
|
||||
* @return {Object} entity.arn - The entity's ARN
|
||||
* @return {Object} entity.canonicalID - The canonical ID for the entity's
|
||||
* account
|
||||
* @return {Object} entity.shortid - The entity's internal shortid
|
||||
* @return {Object} entity.accountDisplayName - The entity's account
|
||||
* display name
|
||||
* @return {Object} entity.IAMDisplayName - The user's display name
|
||||
* (if the entity is an user)
|
||||
* @return {Object} entity.email - The entity's lowercased email
|
||||
*/
|
||||
getEntityByEmail(email) {
|
||||
const lowerCasedEmail = email.toLowerCase();
|
||||
return this.accountsBy.email[lowerCasedEmail];
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns the secret key associated with the entity.
|
||||
* @param {Object} entity - the entity object
|
||||
* @param {string} accessKey - access key
|
||||
* @returns {string} secret key
|
||||
*/
|
||||
getSecretKey(entity, accessKey) {
|
||||
return entity.keys
|
||||
.filter(kv => kv.access === accessKey)[0].secret;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method returns the account display name associated with the entity.
|
||||
* @param {Object} entity - the entity object
|
||||
* @returns {string} account display name
|
||||
*/
|
||||
getAcctDisplayName(entity) {
|
||||
return entity.accountDisplayName;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = Indexer;
|
|
@ -0,0 +1,18 @@
|
|||
const AuthLoader = require('./AuthLoader');
|
||||
|
||||
/**
|
||||
* @deprecated please use {@link AuthLoader} class instead
|
||||
*
|
||||
* @param {object} authdata - the authentication config file's data
|
||||
* @param {werelogs.API} logApi - object providing a constructor function
|
||||
* for the Logger object
|
||||
* @return {boolean} true on erroneous data
|
||||
* false on success
|
||||
*/
|
||||
function validateAuthConfig(authdata, logApi) {
|
||||
const authLoader = new AuthLoader(logApi);
|
||||
authLoader.addAccounts(authdata);
|
||||
return !authLoader.validate();
|
||||
}
|
||||
|
||||
module.exports = validateAuthConfig;
|
|
@ -1,4 +1,6 @@
|
|||
import * as crypto from 'crypto';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const crypto = require('crypto');
|
||||
|
||||
/** hashSignature for v2 Auth
|
||||
* @param {string} stringToSign - built string to sign per AWS rules
|
||||
|
@ -6,19 +8,11 @@ import * as crypto from 'crypto';
|
|||
* @param {string} algorithm - either SHA256 or SHA1
|
||||
* @return {string} reconstructed signature
|
||||
*/
|
||||
export function hashSignature(
|
||||
stringToSign: string,
|
||||
secretKey: string,
|
||||
algorithm: 'SHA256' | 'SHA1'
|
||||
): string {
|
||||
function hashSignature(stringToSign, secretKey, algorithm) {
|
||||
const hmacObject = crypto.createHmac(algorithm, secretKey);
|
||||
return hmacObject.update(stringToSign, 'binary').digest('base64');
|
||||
}
|
||||
|
||||
const sha256Digest = (key: string | Buffer, data: string) => {
|
||||
return crypto.createHmac('sha256', key).update(data, 'binary').digest();
|
||||
};
|
||||
|
||||
/** calculateSigningKey for v4 Auth
|
||||
* @param {string} secretKey - requester's secretKey
|
||||
* @param {string} region - region included in request
|
||||
|
@ -26,15 +20,16 @@ const sha256Digest = (key: string | Buffer, data: string) => {
|
|||
* @param {string} [service] - To specify another service than s3
|
||||
* @return {string} signingKey - signingKey to calculate signature
|
||||
*/
|
||||
export function calculateSigningKey(
|
||||
secretKey: string,
|
||||
region: string,
|
||||
scopeDate: string,
|
||||
service?: string
|
||||
): Buffer {
|
||||
const dateKey = sha256Digest(`AWS4${secretKey}`, scopeDate);
|
||||
const dateRegionKey = sha256Digest(dateKey, region);
|
||||
const dateRegionServiceKey = sha256Digest(dateRegionKey, service || 's3');
|
||||
const signingKey = sha256Digest(dateRegionServiceKey, 'aws4_request');
|
||||
function calculateSigningKey(secretKey, region, scopeDate, service) {
|
||||
const dateKey = crypto.createHmac('sha256', `AWS4${secretKey}`)
|
||||
.update(scopeDate, 'binary').digest();
|
||||
const dateRegionKey = crypto.createHmac('sha256', dateKey)
|
||||
.update(region, 'binary').digest();
|
||||
const dateRegionServiceKey = crypto.createHmac('sha256', dateRegionKey)
|
||||
.update(service || 's3', 'binary').digest();
|
||||
const signingKey = crypto.createHmac('sha256', dateRegionServiceKey)
|
||||
.update('aws4_request', 'binary').digest();
|
||||
return signingKey;
|
||||
}
|
||||
|
||||
module.exports = { hashSignature, calculateSigningKey };
|
|
@ -1,5 +1,7 @@
|
|||
export default function algoCheck(signatureLength: number) {
|
||||
let algo: 'sha256' | 'sha1';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
function algoCheck(signatureLength) {
|
||||
let algo;
|
||||
// If the signature sent is 44 characters,
|
||||
// this means that sha256 was used:
|
||||
// 44 characters in base64
|
||||
|
@ -11,6 +13,7 @@ export default function algoCheck(signatureLength: number) {
|
|||
if (signatureLength === SHA1LEN) {
|
||||
algo = 'sha1';
|
||||
}
|
||||
// @ts-ignore
|
||||
return algo;
|
||||
}
|
||||
|
||||
module.exports = algoCheck;
|
|
@ -0,0 +1,11 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const headerAuthCheck = require('./headerAuthCheck');
|
||||
const queryAuthCheck = require('./queryAuthCheck');
|
||||
|
||||
const authV2 = {
|
||||
header: headerAuthCheck,
|
||||
query: queryAuthCheck,
|
||||
};
|
||||
|
||||
module.exports = authV2;
|
|
@ -1,2 +0,0 @@
|
|||
export * as header from './headerAuthCheck';
|
||||
export * as query from './queryAuthCheck';
|
|
@ -1,9 +1,9 @@
|
|||
import { Logger } from 'werelogs';
|
||||
import errors from '../../errors';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
const errors = require('../../errors').default;
|
||||
|
||||
const epochTime = new Date('1970-01-01').getTime();
|
||||
|
||||
export default function checkRequestExpiry(timestamp: number, log: Logger) {
|
||||
function checkRequestExpiry(timestamp, log) {
|
||||
// If timestamp is before epochTime, the request is invalid and return
|
||||
// errors.AccessDenied
|
||||
if (timestamp < epochTime) {
|
||||
|
@ -32,3 +32,5 @@ export default function checkRequestExpiry(timestamp: number, log: Logger) {
|
|||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
module.exports = checkRequestExpiry;
|
|
@ -1,14 +1,11 @@
|
|||
import { Logger } from 'werelogs';
|
||||
import utf8 from 'utf8';
|
||||
import getCanonicalizedAmzHeaders from './getCanonicalizedAmzHeaders';
|
||||
import getCanonicalizedResource from './getCanonicalizedResource';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
export default function constructStringToSign(
|
||||
request: any,
|
||||
data: { [key: string]: string },
|
||||
log: Logger,
|
||||
clientType?: any
|
||||
) {
|
||||
const utf8 = require('utf8');
|
||||
|
||||
const getCanonicalizedAmzHeaders = require('./getCanonicalizedAmzHeaders');
|
||||
const getCanonicalizedResource = require('./getCanonicalizedResource');
|
||||
|
||||
function constructStringToSign(request, data, log, clientType) {
|
||||
/*
|
||||
Build signature per AWS requirements:
|
||||
StringToSign = HTTP-Verb + '\n' +
|
||||
|
@ -45,3 +42,5 @@ export default function constructStringToSign(
|
|||
+ getCanonicalizedResource(request, clientType);
|
||||
return utf8.encode(stringToSign);
|
||||
}
|
||||
|
||||
module.exports = constructStringToSign;
|
|
@ -1,12 +1,14 @@
|
|||
export default function getCanonicalizedAmzHeaders(headers: Headers, clientType: string) {
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
function getCanonicalizedAmzHeaders(headers, clientType) {
|
||||
/*
|
||||
Iterate through headers and pull any headers that are x-amz headers.
|
||||
Need to include 'x-amz-date' here even though AWS docs
|
||||
ambiguous on this.
|
||||
*/
|
||||
const filterFn = clientType === 'GCP' ?
|
||||
(val: string) => val.substr(0, 7) === 'x-goog-' :
|
||||
(val: string) => val.substr(0, 6) === 'x-amz-';
|
||||
val => val.substr(0, 7) === 'x-goog-' :
|
||||
val => val.substr(0, 6) === 'x-amz-';
|
||||
const amzHeaders = Object.keys(headers)
|
||||
.filter(filterFn)
|
||||
.map(val => [val.trim(), headers[val].trim()]);
|
||||
|
@ -39,5 +41,7 @@ export default function getCanonicalizedAmzHeaders(headers: Headers, clientType:
|
|||
// Build headerString
|
||||
return amzHeaders.reduce((headerStr, current) =>
|
||||
`${headerStr}${current[0]}:${current[1]}\n`,
|
||||
'');
|
||||
'');
|
||||
}
|
||||
|
||||
module.exports = getCanonicalizedAmzHeaders;
|
|
@ -1,4 +1,6 @@
|
|||
import * as url from 'url';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const url = require('url');
|
||||
|
||||
const gcpSubresources = [
|
||||
'acl',
|
||||
|
@ -39,7 +41,7 @@ const awsSubresources = [
|
|||
'website',
|
||||
];
|
||||
|
||||
export default function getCanonicalizedResource(request: any, clientType: string) {
|
||||
function getCanonicalizedResource(request, clientType) {
|
||||
/*
|
||||
This variable is used to determine whether to insert
|
||||
a '?' or '&'. Once a query parameter is added to the resourceString,
|
||||
|
@ -115,3 +117,5 @@ export default function getCanonicalizedResource(request: any, clientType: strin
|
|||
}
|
||||
return resourceString;
|
||||
}
|
||||
|
||||
module.exports = getCanonicalizedResource;
|
|
@ -1,11 +1,12 @@
|
|||
import { Logger } from 'werelogs';
|
||||
import errors from '../../errors';
|
||||
import * as constants from '../../constants';
|
||||
import constructStringToSign from './constructStringToSign';
|
||||
import checkRequestExpiry from './checkRequestExpiry';
|
||||
import algoCheck from './algoCheck';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
export function check(request: any, log: Logger, data: { [key: string]: string }) {
|
||||
const errors = require('../../errors').default;
|
||||
const constants = require('../../constants');
|
||||
const constructStringToSign = require('./constructStringToSign');
|
||||
const checkRequestExpiry = require('./checkRequestExpiry');
|
||||
const algoCheck = require('./algoCheck');
|
||||
|
||||
function check(request, log, data) {
|
||||
log.trace('running header auth check');
|
||||
const headers = request.headers;
|
||||
|
||||
|
@ -21,9 +22,9 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
timestamp = Date.parse(timestamp);
|
||||
if (!timestamp) {
|
||||
log.debug('missing or invalid date header',
|
||||
{ method: 'auth/v2/headerAuthCheck.check' });
|
||||
{ method: 'auth/v2/headerAuthCheck.check' });
|
||||
return { err: errors.AccessDenied.
|
||||
customizeDescription('Authentication requires a valid Date or ' +
|
||||
customizeDescription('Authentication requires a valid Date or ' +
|
||||
'x-amz-date header') };
|
||||
}
|
||||
|
||||
|
@ -51,7 +52,6 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
log.trace('invalid authorization header', { authInfo });
|
||||
return { err: errors.MissingSecurityHeader };
|
||||
}
|
||||
// @ts-ignore
|
||||
log.addDefaultFields({ accessKey });
|
||||
|
||||
const signatureFromRequest = authInfo.substring(semicolonIndex + 1).trim();
|
||||
|
@ -80,3 +80,5 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
},
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { check };
|
|
@ -1,10 +1,11 @@
|
|||
import { Logger } from 'werelogs';
|
||||
import errors from '../../errors';
|
||||
import * as constants from '../../constants';
|
||||
import algoCheck from './algoCheck';
|
||||
import constructStringToSign from './constructStringToSign';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
export function check(request: any, log: Logger, data: { [key: string]: string }) {
|
||||
const errors = require('../../errors').default;
|
||||
const constants = require('../../constants');
|
||||
const algoCheck = require('./algoCheck');
|
||||
const constructStringToSign = require('./constructStringToSign');
|
||||
|
||||
function check(request, log, data) {
|
||||
log.trace('running query auth check');
|
||||
if (request.method === 'POST') {
|
||||
log.debug('query string auth not supported for post requests');
|
||||
|
@ -41,16 +42,15 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
|
||||
if (expirationTime > currentTime + preSignedURLExpiry) {
|
||||
log.debug('expires parameter too far in future',
|
||||
{ expires: request.query.Expires });
|
||||
{ expires: request.query.Expires });
|
||||
return { err: errors.AccessDenied };
|
||||
}
|
||||
if (currentTime > expirationTime) {
|
||||
log.debug('current time exceeds expires time',
|
||||
{ expires: request.query.Expires });
|
||||
{ expires: request.query.Expires });
|
||||
return { err: errors.RequestTimeTooSkewed };
|
||||
}
|
||||
const accessKey = data.AWSAccessKeyId;
|
||||
// @ts-ignore
|
||||
log.addDefaultFields({ accessKey });
|
||||
|
||||
const signatureFromRequest = decodeURIComponent(data.Signature);
|
||||
|
@ -82,3 +82,5 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
},
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { check };
|
|
@ -0,0 +1,11 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const headerAuthCheck = require('./headerAuthCheck');
|
||||
const queryAuthCheck = require('./queryAuthCheck');
|
||||
|
||||
const authV4 = {
|
||||
header: headerAuthCheck,
|
||||
query: queryAuthCheck,
|
||||
};
|
||||
|
||||
module.exports = authV4;
|
|
@ -1,2 +0,0 @@
|
|||
export * as header from './headerAuthCheck';
|
||||
export * as query from './queryAuthCheck';
|
|
@ -1,3 +1,5 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
/*
|
||||
AWS's URI encoding rules:
|
||||
URI encode every byte. Uri-Encode() must enforce the following rules:
|
||||
|
@ -17,7 +19,7 @@ See http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
|
|||
*/
|
||||
|
||||
// converts utf8 character to hex and pads "%" before every two hex digits
|
||||
function _toHexUTF8(char: string) {
|
||||
function _toHexUTF8(char) {
|
||||
const hexRep = Buffer.from(char, 'utf8').toString('hex').toUpperCase();
|
||||
let res = '';
|
||||
hexRep.split('').forEach((v, n) => {
|
||||
|
@ -30,23 +32,18 @@ function _toHexUTF8(char: string) {
|
|||
return res;
|
||||
}
|
||||
|
||||
export default function awsURIencode(
|
||||
input: string,
|
||||
encodeSlash?: boolean,
|
||||
noEncodeStar?: boolean
|
||||
) {
|
||||
function awsURIencode(input, encodeSlash, noEncodeStar) {
|
||||
const encSlash = encodeSlash === undefined ? true : encodeSlash;
|
||||
let encoded = '';
|
||||
/**
|
||||
* Duplicate query params are not suppported by AWS S3 APIs. These params
|
||||
* are parsed as Arrays by Node.js HTTP parser which breaks this method
|
||||
*/
|
||||
if (typeof input !== 'string') {
|
||||
return '';
|
||||
*/
|
||||
if (typeof input !== 'string') {
|
||||
return encoded;
|
||||
}
|
||||
let encoded = "";
|
||||
const slash = encodeSlash === undefined || encodeSlash ? '%2F' : '/';
|
||||
const star = noEncodeStar !== undefined && noEncodeStar ? '*' : '%2A';
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
let ch = input.charAt(i);
|
||||
const ch = input.charAt(i);
|
||||
if ((ch >= 'A' && ch <= 'Z') ||
|
||||
(ch >= 'a' && ch <= 'z') ||
|
||||
(ch >= '0' && ch <= '9') ||
|
||||
|
@ -56,26 +53,14 @@ export default function awsURIencode(
|
|||
} else if (ch === ' ') {
|
||||
encoded = encoded.concat('%20');
|
||||
} else if (ch === '/') {
|
||||
encoded = encoded.concat(slash);
|
||||
encoded = encoded.concat(encSlash ? '%2F' : ch);
|
||||
} else if (ch === '*') {
|
||||
encoded = encoded.concat(star);
|
||||
encoded = encoded.concat(noEncodeStar ? '*' : '%2A');
|
||||
} else {
|
||||
if (ch >= '\uD800' && ch <= '\uDBFF') {
|
||||
// If this character is a high surrogate peek the next character
|
||||
// and join it with this one if the next character is a low
|
||||
// surrogate.
|
||||
// Otherwise the encoded URI will contain the two surrogates as
|
||||
// two distinct UTF-8 sequences which is not valid UTF-8.
|
||||
if (i + 1 < input.length) {
|
||||
const ch2 = input.charAt(i + 1);
|
||||
if (ch2 >= '\uDC00' && ch2 <= '\uDFFF') {
|
||||
i++;
|
||||
ch += ch2;
|
||||
}
|
||||
}
|
||||
}
|
||||
encoded = encoded.concat(_toHexUTF8(ch));
|
||||
}
|
||||
}
|
||||
return encoded;
|
||||
}
|
||||
|
||||
module.exports = awsURIencode;
|
|
@ -1,33 +1,17 @@
|
|||
import * as crypto from 'crypto';
|
||||
import { Logger } from 'werelogs';
|
||||
import createCanonicalRequest from './createCanonicalRequest';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const crypto = require('crypto');
|
||||
|
||||
const createCanonicalRequest = require('./createCanonicalRequest');
|
||||
|
||||
/**
|
||||
* constructStringToSign - creates V4 stringToSign
|
||||
* @param {object} params - params object
|
||||
* @returns {string} - stringToSign
|
||||
*/
|
||||
export default function constructStringToSign(params: {
|
||||
request: any;
|
||||
signedHeaders: any;
|
||||
payloadChecksum: any;
|
||||
credentialScope: string;
|
||||
timestamp: string;
|
||||
query: { [key: string]: string };
|
||||
log?: Logger;
|
||||
proxyPath?: string;
|
||||
awsService: string;
|
||||
}): string | Error {
|
||||
const {
|
||||
request,
|
||||
signedHeaders,
|
||||
payloadChecksum,
|
||||
credentialScope,
|
||||
timestamp,
|
||||
query,
|
||||
log,
|
||||
proxyPath,
|
||||
} = params;
|
||||
function constructStringToSign(params) {
|
||||
const { request, signedHeaders, payloadChecksum, credentialScope, timestamp,
|
||||
query, log, proxyPath } = params;
|
||||
const path = proxyPath || request.path;
|
||||
|
||||
const canonicalReqResult = createCanonicalRequest({
|
||||
|
@ -40,8 +24,6 @@ export default function constructStringToSign(params: {
|
|||
service: params.awsService,
|
||||
});
|
||||
|
||||
// TODO Why that line?
|
||||
// @ts-ignore
|
||||
if (canonicalReqResult instanceof Error) {
|
||||
if (log) {
|
||||
log.error('error creating canonicalRequest');
|
||||
|
@ -58,3 +40,5 @@ export default function constructStringToSign(params: {
|
|||
`${credentialScope}\n${canonicalHex}`;
|
||||
return stringToSign;
|
||||
}
|
||||
|
||||
module.exports = constructStringToSign;
|
|
@ -1,33 +1,27 @@
|
|||
import * as crypto from 'crypto';
|
||||
import * as queryString from 'querystring';
|
||||
import awsURIencode from './awsURIencode';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const awsURIencode = require('./awsURIencode');
|
||||
const crypto = require('crypto');
|
||||
const queryString = require('querystring');
|
||||
|
||||
/**
|
||||
* createCanonicalRequest - creates V4 canonical request
|
||||
* @param params - contains pHttpVerb (request type),
|
||||
* @param {object} params - contains pHttpVerb (request type),
|
||||
* pResource (parsed from URL), pQuery (request query),
|
||||
* pHeaders (request headers), pSignedHeaders (signed headers from request),
|
||||
* payloadChecksum (from request)
|
||||
* @returns - canonicalRequest
|
||||
* @returns {string} - canonicalRequest
|
||||
*/
|
||||
export default function createCanonicalRequest(
|
||||
params: {
|
||||
pHttpVerb: string;
|
||||
pResource: string;
|
||||
pQuery: { [key: string]: string };
|
||||
pHeaders: any;
|
||||
pSignedHeaders: any;
|
||||
service: string;
|
||||
payloadChecksum: string;
|
||||
}
|
||||
) {
|
||||
function createCanonicalRequest(params) {
|
||||
const pHttpVerb = params.pHttpVerb;
|
||||
const pResource = params.pResource;
|
||||
const pQuery = params.pQuery;
|
||||
const pHeaders = params.pHeaders;
|
||||
const pSignedHeaders = params.pSignedHeaders;
|
||||
const service = params.service;
|
||||
|
||||
let payloadChecksum = params.payloadChecksum;
|
||||
|
||||
if (!payloadChecksum) {
|
||||
if (pHttpVerb === 'GET') {
|
||||
payloadChecksum = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b' +
|
||||
|
@ -40,7 +34,7 @@ export default function createCanonicalRequest(
|
|||
if (/aws-sdk-java\/[0-9.]+/.test(pHeaders['user-agent'])) {
|
||||
notEncodeStar = true;
|
||||
}
|
||||
let payload = queryString.stringify(pQuery, undefined, undefined, {
|
||||
let payload = queryString.stringify(pQuery, null, null, {
|
||||
encodeURIComponent: input => awsURIencode(input, true,
|
||||
notEncodeStar),
|
||||
});
|
||||
|
@ -67,11 +61,11 @@ export default function createCanonicalRequest(
|
|||
|
||||
// signed headers
|
||||
const signedHeadersList = pSignedHeaders.split(';');
|
||||
signedHeadersList.sort((a: any, b: any) => a.localeCompare(b));
|
||||
signedHeadersList.sort((a, b) => a.localeCompare(b));
|
||||
const signedHeaders = signedHeadersList.join(';');
|
||||
|
||||
// canonical headers
|
||||
const canonicalHeadersList = signedHeadersList.map((signedHeader: any) => {
|
||||
const canonicalHeadersList = signedHeadersList.map(signedHeader => {
|
||||
if (pHeaders[signedHeader] !== undefined) {
|
||||
const trimmedHeader = pHeaders[signedHeader]
|
||||
.trim().replace(/\s+/g, ' ');
|
||||
|
@ -93,3 +87,5 @@ export default function createCanonicalRequest(
|
|||
`${signedHeaders}\n${payloadChecksum}`;
|
||||
return canonicalRequest;
|
||||
}
|
||||
|
||||
module.exports = createCanonicalRequest;
|
|
@ -1,32 +1,27 @@
|
|||
import { Logger } from 'werelogs';
|
||||
import errors from '../../../lib/errors';
|
||||
import * as constants from '../../constants';
|
||||
import constructStringToSign from './constructStringToSign';
|
||||
import {
|
||||
checkTimeSkew,
|
||||
convertUTCtoISO8601,
|
||||
convertAmzTimeToMs,
|
||||
} from './timeUtils';
|
||||
import {
|
||||
extractAuthItems,
|
||||
validateCredentials,
|
||||
areSignedHeadersComplete,
|
||||
} from './validateInputs';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const errors = require('../../../lib/errors').default;
|
||||
const constants = require('../../constants');
|
||||
|
||||
const constructStringToSign = require('./constructStringToSign');
|
||||
const checkTimeSkew = require('./timeUtils').checkTimeSkew;
|
||||
const convertUTCtoISO8601 = require('./timeUtils').convertUTCtoISO8601;
|
||||
const convertAmzTimeToMs = require('./timeUtils').convertAmzTimeToMs;
|
||||
const extractAuthItems = require('./validateInputs').extractAuthItems;
|
||||
const validateCredentials = require('./validateInputs').validateCredentials;
|
||||
const areSignedHeadersComplete =
|
||||
require('./validateInputs').areSignedHeadersComplete;
|
||||
|
||||
/**
|
||||
* V4 header auth check
|
||||
* @param request - HTTP request object
|
||||
* @param log - logging object
|
||||
* @param data - Parameters from queryString parsing or body of
|
||||
* @param {object} request - HTTP request object
|
||||
* @param {object} log - logging object
|
||||
* @param {object} data - Parameters from queryString parsing or body of
|
||||
* POST request
|
||||
* @param awsService - Aws service ('iam' or 's3')
|
||||
* @param {string} awsService - Aws service ('iam' or 's3')
|
||||
* @return {callback} calls callback
|
||||
*/
|
||||
export function check(
|
||||
request: any,
|
||||
log: Logger,
|
||||
data: { [key: string]: string },
|
||||
awsService: string
|
||||
) {
|
||||
function check(request, log, data, awsService) {
|
||||
log.trace('running header auth check');
|
||||
|
||||
const token = request.headers['x-amz-security-token'];
|
||||
|
@ -67,16 +62,16 @@ export function check(
|
|||
|
||||
log.trace('authorization header from request', { authHeader });
|
||||
|
||||
const signatureFromRequest = authHeaderItems.signatureFromRequest!;
|
||||
const credentialsArr = authHeaderItems.credentialsArr!;
|
||||
const signedHeaders = authHeaderItems.signedHeaders!;
|
||||
const signatureFromRequest = authHeaderItems.signatureFromRequest;
|
||||
const credentialsArr = authHeaderItems.credentialsArr;
|
||||
const signedHeaders = authHeaderItems.signedHeaders;
|
||||
|
||||
if (!areSignedHeadersComplete(signedHeaders, request.headers)) {
|
||||
log.debug('signedHeaders are incomplete', { signedHeaders });
|
||||
return { err: errors.AccessDenied };
|
||||
}
|
||||
|
||||
let timestamp: string | undefined;
|
||||
let timestamp;
|
||||
// check request timestamp
|
||||
const xAmzDate = request.headers['x-amz-date'];
|
||||
if (xAmzDate) {
|
||||
|
@ -93,14 +88,14 @@ export function check(
|
|||
}
|
||||
if (!timestamp) {
|
||||
log.debug('missing or invalid date header',
|
||||
{ method: 'auth/v4/headerAuthCheck.check' });
|
||||
{ method: 'auth/v4/headerAuthCheck.check' });
|
||||
return { err: errors.AccessDenied.
|
||||
customizeDescription('Authentication requires a valid Date or ' +
|
||||
customizeDescription('Authentication requires a valid Date or ' +
|
||||
'x-amz-date header') };
|
||||
}
|
||||
|
||||
const validationResult = validateCredentials(credentialsArr, timestamp,
|
||||
log);
|
||||
log);
|
||||
if (validationResult instanceof Error) {
|
||||
log.debug('credentials in improper format', { credentialsArr,
|
||||
timestamp, validationResult });
|
||||
|
@ -132,17 +127,6 @@ export function check(
|
|||
return { err: errors.RequestTimeTooSkewed };
|
||||
}
|
||||
|
||||
let proxyPath: string | undefined;
|
||||
if (request.headers.proxy_path) {
|
||||
try {
|
||||
proxyPath = decodeURIComponent(request.headers.proxy_path);
|
||||
} catch (err) {
|
||||
log.debug('invalid proxy_path header', { proxyPath, err });
|
||||
return { err: errors.InvalidArgument.customizeDescription(
|
||||
'invalid proxy_path header') };
|
||||
}
|
||||
}
|
||||
|
||||
const stringToSign = constructStringToSign({
|
||||
log,
|
||||
request,
|
||||
|
@ -152,7 +136,6 @@ export function check(
|
|||
timestamp,
|
||||
payloadChecksum,
|
||||
awsService: service,
|
||||
proxyPath,
|
||||
});
|
||||
log.trace('constructed stringToSign', { stringToSign });
|
||||
if (stringToSign instanceof Error) {
|
||||
|
@ -183,3 +166,5 @@ export function check(
|
|||
},
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { check };
|
|
@ -1,18 +1,24 @@
|
|||
import { Logger } from 'werelogs';
|
||||
import * as constants from '../../constants';
|
||||
import errors from '../../errors';
|
||||
import constructStringToSign from './constructStringToSign';
|
||||
import { checkTimeSkew, convertAmzTimeToMs } from './timeUtils';
|
||||
import { validateCredentials, extractQueryParams } from './validateInputs';
|
||||
import { areSignedHeadersComplete } from './validateInputs';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const constants = require('../../constants');
|
||||
const errors = require('../../errors').default;
|
||||
|
||||
const constructStringToSign = require('./constructStringToSign');
|
||||
const checkTimeSkew = require('./timeUtils').checkTimeSkew;
|
||||
const convertAmzTimeToMs = require('./timeUtils').convertAmzTimeToMs;
|
||||
const validateCredentials = require('./validateInputs').validateCredentials;
|
||||
const extractQueryParams = require('./validateInputs').extractQueryParams;
|
||||
const areSignedHeadersComplete =
|
||||
require('./validateInputs').areSignedHeadersComplete;
|
||||
|
||||
/**
|
||||
* V4 query auth check
|
||||
* @param request - HTTP request object
|
||||
* @param log - logging object
|
||||
* @param data - Contain authentification params (GET or POST data)
|
||||
* @param {object} request - HTTP request object
|
||||
* @param {object} log - logging object
|
||||
* @param {object} data - Contain authentification params (GET or POST data)
|
||||
* @return {callback} calls callback
|
||||
*/
|
||||
export function check(request: any, log: Logger, data: { [key: string]: string }) {
|
||||
function check(request, log, data) {
|
||||
const authParams = extractQueryParams(data, log);
|
||||
|
||||
if (Object.keys(authParams).length !== 5) {
|
||||
|
@ -27,11 +33,11 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
return { err: errors.InvalidToken };
|
||||
}
|
||||
|
||||
const signedHeaders = authParams.signedHeaders!;
|
||||
const signatureFromRequest = authParams.signatureFromRequest!;
|
||||
const timestamp = authParams.timestamp!;
|
||||
const expiry = authParams.expiry!;
|
||||
const credential = authParams.credential!;
|
||||
const signedHeaders = authParams.signedHeaders;
|
||||
const signatureFromRequest = authParams.signatureFromRequest;
|
||||
const timestamp = authParams.timestamp;
|
||||
const expiry = authParams.expiry;
|
||||
const credential = authParams.credential;
|
||||
|
||||
if (!areSignedHeadersComplete(signedHeaders, request.headers)) {
|
||||
log.debug('signedHeaders are incomplete', { signedHeaders });
|
||||
|
@ -39,7 +45,7 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
}
|
||||
|
||||
const validationResult = validateCredentials(credential, timestamp,
|
||||
log);
|
||||
log);
|
||||
if (validationResult instanceof Error) {
|
||||
log.debug('credentials in improper format', { credential,
|
||||
timestamp, validationResult });
|
||||
|
@ -56,17 +62,6 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
return { err: errors.RequestTimeTooSkewed };
|
||||
}
|
||||
|
||||
let proxyPath: string | undefined;
|
||||
if (request.headers.proxy_path) {
|
||||
try {
|
||||
proxyPath = decodeURIComponent(request.headers.proxy_path);
|
||||
} catch (err) {
|
||||
log.debug('invalid proxy_path header', { proxyPath });
|
||||
return { err: errors.InvalidArgument.customizeDescription(
|
||||
'invalid proxy_path header') };
|
||||
}
|
||||
}
|
||||
|
||||
// In query v4 auth, the canonical request needs
|
||||
// to include the query params OTHER THAN
|
||||
// the signature so create a
|
||||
|
@ -92,7 +87,6 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
credentialScope:
|
||||
`${scopeDate}/${region}/${service}/${requestType}`,
|
||||
awsService: service,
|
||||
proxyPath,
|
||||
});
|
||||
if (stringToSign instanceof Error) {
|
||||
return { err: stringToSign };
|
||||
|
@ -116,3 +110,5 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
},
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { check };
|
|
@ -1,67 +1,33 @@
|
|||
import { Transform } from 'stream';
|
||||
import async from 'async';
|
||||
import errors from '../../../errors';
|
||||
import { Logger } from 'werelogs';
|
||||
import Vault, { AuthV4RequestParams } from '../../Vault';
|
||||
import { Callback } from '../../backends/in_memory/types';
|
||||
const { Transform } = require('stream');
|
||||
|
||||
import constructChunkStringToSign from './constructChunkStringToSign';
|
||||
const async = require('async');
|
||||
const errors = require('../../../errors').default;
|
||||
|
||||
export type TransformParams = {
|
||||
accessKey: string;
|
||||
signatureFromRequest: string;
|
||||
region: string;
|
||||
scopeDate: string;
|
||||
timestamp: string;
|
||||
credentialScope: string;
|
||||
};
|
||||
const constructChunkStringToSign = require('./constructChunkStringToSign');
|
||||
|
||||
/**
|
||||
* This class is designed to handle the chunks sent in a streaming
|
||||
* v4 Auth request
|
||||
*/
|
||||
export default class V4Transform extends Transform {
|
||||
log: Logger;
|
||||
cb: Callback;
|
||||
accessKey: string;
|
||||
region: string;
|
||||
scopeDate: string;
|
||||
timestamp: string;
|
||||
credentialScope: string;
|
||||
lastSignature: string;
|
||||
currentSignature?: string;
|
||||
haveMetadata: boolean;
|
||||
seekingDataSize: number;
|
||||
currentData?: any;
|
||||
dataCursor: number;
|
||||
currentMetadata: any[];
|
||||
lastPieceDone: boolean;
|
||||
lastChunk: boolean;
|
||||
vault: Vault;
|
||||
|
||||
class V4Transform extends Transform {
|
||||
/**
|
||||
* @constructor
|
||||
* @param streamingV4Params - info for chunk authentication
|
||||
* @param streamingV4Params.accessKey - requester's accessKey
|
||||
* @param streamingV4Params.signatureFromRequest - signature
|
||||
* @param {object} streamingV4Params - info for chunk authentication
|
||||
* @param {string} streamingV4Params.accessKey - requester's accessKey
|
||||
* @param {string} streamingV4Params.signatureFromRequest - signature
|
||||
* sent with headers
|
||||
* @param streamingV4Params.region - region sent with auth header
|
||||
* @param streamingV4Params.scopeDate - date sent with auth header
|
||||
* @param streamingV4Params.timestamp - date parsed from headers
|
||||
* @param {string} streamingV4Params.region - region sent with auth header
|
||||
* @param {string} streamingV4Params.scopeDate - date sent with auth header
|
||||
* @param {string} streamingV4Params.timestamp - date parsed from headers
|
||||
* in ISO 8601 format: YYYYMMDDTHHMMSSZ
|
||||
* @param streamingV4Params.credentialScope - items from auth
|
||||
* @param {string} streamingV4Params.credentialScope - items from auth
|
||||
* header plus the string 'aws4_request' joined with '/':
|
||||
* timestamp/region/aws-service/aws4_request
|
||||
* @param vault - Vault instance passed from CloudServer
|
||||
* @param log - logger object
|
||||
* @param cb - callback to api
|
||||
* @param {object} vault - Vault instance passed from CloudServer
|
||||
* @param {object} log - logger object
|
||||
* @param {function} cb - callback to api
|
||||
*/
|
||||
constructor(
|
||||
streamingV4Params: TransformParams,
|
||||
vault: Vault,
|
||||
log: Logger,
|
||||
cb: Callback,
|
||||
) {
|
||||
constructor(streamingV4Params, vault, log, cb) {
|
||||
const { accessKey, signatureFromRequest, region, scopeDate, timestamp,
|
||||
credentialScope } = streamingV4Params;
|
||||
super({});
|
||||
|
@ -89,8 +55,8 @@ export default class V4Transform extends Transform {
|
|||
|
||||
/**
|
||||
* This function will parse the metadata portion of the chunk
|
||||
* @param remainingChunk - chunk sent from _transform
|
||||
* @return response - if error, will return 'err' key with
|
||||
* @param {Buffer} remainingChunk - chunk sent from _transform
|
||||
* @return {object} response - if error, will return 'err' key with
|
||||
* arsenal error value.
|
||||
* if incomplete metadata, will return 'completeMetadata' key with
|
||||
* value false
|
||||
|
@ -98,7 +64,7 @@ export default class V4Transform extends Transform {
|
|||
* value true and the key 'unparsedChunk' with the remaining chunk without
|
||||
* the parsed metadata piece
|
||||
*/
|
||||
_parseMetadata(remainingChunk: Buffer) {
|
||||
_parseMetadata(remainingChunk) {
|
||||
let remainingPlusStoredMetadata = remainingChunk;
|
||||
// have metadata pieces so need to add to the front of
|
||||
// remainingChunk
|
||||
|
@ -137,8 +103,9 @@ export default class V4Transform extends Transform {
|
|||
'metadata format');
|
||||
return { err: errors.InvalidArgument };
|
||||
}
|
||||
let dataSize = splitMeta[0];
|
||||
// chunk-size is sent in hex
|
||||
const dataSize = Number.parseInt(splitMeta[0], 16);
|
||||
dataSize = Number.parseInt(dataSize, 16);
|
||||
if (Number.isNaN(dataSize)) {
|
||||
this.log.trace('chunk body did not contain valid size');
|
||||
return { err: errors.InvalidArgument };
|
||||
|
@ -172,30 +139,28 @@ export default class V4Transform extends Transform {
|
|||
|
||||
/**
|
||||
* Build the stringToSign and authenticate the chunk
|
||||
* @param dataToSend - chunk sent from _transform or null
|
||||
* @param {Buffer} dataToSend - chunk sent from _transform or null
|
||||
* if last chunk without data
|
||||
* @param done - callback to _transform
|
||||
* @return executes callback with err if applicable
|
||||
* @param {function} done - callback to _transform
|
||||
* @return {function} executes callback with err if applicable
|
||||
*/
|
||||
_authenticate(dataToSend: Buffer | null, done: Callback) {
|
||||
_authenticate(dataToSend, done) {
|
||||
// use prior sig to construct new string to sign
|
||||
const stringToSign = constructChunkStringToSign(this.timestamp,
|
||||
this.credentialScope, this.lastSignature, dataToSend ?? undefined);
|
||||
this.credentialScope, this.lastSignature, dataToSend);
|
||||
this.log.trace('constructed chunk string to sign',
|
||||
{ stringToSign });
|
||||
// once used prior sig to construct string to sign, reassign
|
||||
// lastSignature to current signature
|
||||
this.lastSignature = this.currentSignature!;
|
||||
const vaultParams: AuthV4RequestParams = {
|
||||
this.lastSignature = this.currentSignature;
|
||||
const vaultParams = {
|
||||
log: this.log,
|
||||
data: {
|
||||
accessKey: this.accessKey,
|
||||
signatureFromRequest: this.currentSignature!,
|
||||
signatureFromRequest: this.currentSignature,
|
||||
region: this.region,
|
||||
scopeDate: this.scopeDate,
|
||||
stringToSign,
|
||||
// TODO FIXME This can not work
|
||||
// @ts-expect-errors
|
||||
timestamp: this.timestamp,
|
||||
credentialScope: this.credentialScope,
|
||||
},
|
||||
|
@ -216,12 +181,12 @@ export default class V4Transform extends Transform {
|
|||
* use the metadata to authenticate with vault and send the
|
||||
* data on to be stored if authentication passes
|
||||
*
|
||||
* @param chunk - chunk from request body
|
||||
* @param _encoding - Data encoding unused
|
||||
* @param callback - Callback(err, justDataChunk, encoding)
|
||||
* @return executes callback with err if applicable
|
||||
* @param {Buffer} chunk - chunk from request body
|
||||
* @param {string} encoding - Data encoding
|
||||
* @param {function} callback - Callback(err, justDataChunk, encoding)
|
||||
* @return {function }executes callback with err if applicable
|
||||
*/
|
||||
_transform(chunk: Buffer, _encoding: string, callback: Callback) {
|
||||
_transform(chunk, encoding, callback) {
|
||||
// 'chunk' here is the node streaming chunk
|
||||
// transfer-encoding chunks should be of the format:
|
||||
// string(IntHexBase(chunk-size)) + ";chunk-signature=" +
|
||||
|
@ -258,8 +223,6 @@ export default class V4Transform extends Transform {
|
|||
}
|
||||
// have metadata so reset unparsedChunk to remaining
|
||||
// without metadata piece
|
||||
// TODO Is that okay?
|
||||
// @ts-expect-errors
|
||||
unparsedChunk = parsedMetadataResults.unparsedChunk;
|
||||
}
|
||||
if (this.lastChunk) {
|
||||
|
@ -306,7 +269,7 @@ export default class V4Transform extends Transform {
|
|||
// final callback
|
||||
err => {
|
||||
if (err) {
|
||||
return this.cb(err as any);
|
||||
return this.cb(err);
|
||||
}
|
||||
// get next chunk
|
||||
return callback();
|
||||
|
@ -314,3 +277,5 @@ export default class V4Transform extends Transform {
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = V4Transform;
|
|
@ -0,0 +1,32 @@
|
|||
const crypto = require('crypto');
|
||||
|
||||
const constants = require('../../../constants');
|
||||
|
||||
/**
|
||||
* Constructs stringToSign for chunk
|
||||
* @param {string} timestamp - date parsed from headers
|
||||
* in ISO 8601 format: YYYYMMDDTHHMMSSZ
|
||||
* @param {string} credentialScope - items from auth
|
||||
* header plus the string 'aws4_request' joined with '/':
|
||||
* timestamp/region/aws-service/aws4_request
|
||||
* @param {string} lastSignature - signature from headers or prior chunk
|
||||
* @param {string} justDataChunk - data portion of chunk
|
||||
* @returns {string} stringToSign
|
||||
*/
|
||||
function constructChunkStringToSign(timestamp,
|
||||
credentialScope, lastSignature, justDataChunk) {
|
||||
let currentChunkHash;
|
||||
// for last chunk, there will be no data, so use emptyStringHash
|
||||
if (!justDataChunk) {
|
||||
currentChunkHash = constants.emptyStringHash;
|
||||
} else {
|
||||
currentChunkHash = crypto.createHash('sha256');
|
||||
currentChunkHash = currentChunkHash
|
||||
.update(justDataChunk, 'binary').digest('hex');
|
||||
}
|
||||
return `AWS4-HMAC-SHA256-PAYLOAD\n${timestamp}\n` +
|
||||
`${credentialScope}\n${lastSignature}\n` +
|
||||
`${constants.emptyStringHash}\n${currentChunkHash}`;
|
||||
}
|
||||
|
||||
module.exports = constructChunkStringToSign;
|
|
@ -1,35 +0,0 @@
|
|||
import * as crypto from 'crypto';
|
||||
import * as constants from '../../../constants';
|
||||
|
||||
/**
|
||||
* Constructs stringToSign for chunk
|
||||
* @param timestamp - date parsed from headers
|
||||
* in ISO 8601 format: YYYYMMDDTHHMMSSZ
|
||||
* @param credentialScope - items from auth
|
||||
* header plus the string 'aws4_request' joined with '/':
|
||||
* timestamp/region/aws-service/aws4_request
|
||||
* @param lastSignature - signature from headers or prior chunk
|
||||
* @param justDataChunk - data portion of chunk
|
||||
* @returns stringToSign
|
||||
*/
|
||||
export default function constructChunkStringToSign(
|
||||
timestamp: string,
|
||||
credentialScope: string,
|
||||
lastSignature: string,
|
||||
justDataChunk?: Buffer | string,
|
||||
) {
|
||||
let currentChunkHash: string;
|
||||
// for last chunk, there will be no data, so use emptyStringHash
|
||||
if (!justDataChunk) {
|
||||
currentChunkHash = constants.emptyStringHash;
|
||||
} else {
|
||||
const hash = crypto.createHash('sha256');
|
||||
const temp = justDataChunk instanceof Buffer
|
||||
? hash.update(justDataChunk)
|
||||
: hash.update(justDataChunk, 'binary');
|
||||
currentChunkHash = temp.digest('hex');
|
||||
}
|
||||
return `AWS4-HMAC-SHA256-PAYLOAD\n${timestamp}\n` +
|
||||
`${credentialScope}\n${lastSignature}\n` +
|
||||
`${constants.emptyStringHash}\n${currentChunkHash}`;
|
||||
}
|
|
@ -1,11 +1,12 @@
|
|||
import { Logger } from 'werelogs';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
/**
|
||||
* Convert timestamp to milliseconds since Unix Epoch
|
||||
* @param timestamp of ISO8601Timestamp format without
|
||||
* @param {string} timestamp of ISO8601Timestamp format without
|
||||
* dashes or colons, e.g. 20160202T220410Z
|
||||
* @return {number} number of milliseconds since Unix Epoch
|
||||
*/
|
||||
export function convertAmzTimeToMs(timestamp: string) {
|
||||
function convertAmzTimeToMs(timestamp) {
|
||||
const arr = timestamp.split('');
|
||||
// Convert to YYYY-MM-DDTHH:mm:ss.sssZ
|
||||
const ISO8601time = `${arr.slice(0, 4).join('')}-${arr[4]}${arr[5]}` +
|
||||
|
@ -14,12 +15,13 @@ export function convertAmzTimeToMs(timestamp: string) {
|
|||
return Date.parse(ISO8601time);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Convert UTC timestamp to ISO 8601 timestamp
|
||||
* @param timestamp of UTC form: Fri, 10 Feb 2012 21:34:55 GMT
|
||||
* @return ISO8601 timestamp of form: YYYYMMDDTHHMMSSZ
|
||||
* @param {string} timestamp of UTC form: Fri, 10 Feb 2012 21:34:55 GMT
|
||||
* @return {string} ISO8601 timestamp of form: YYYYMMDDTHHMMSSZ
|
||||
*/
|
||||
export function convertUTCtoISO8601(timestamp: string | number) {
|
||||
function convertUTCtoISO8601(timestamp) {
|
||||
// convert to ISO string: YYYY-MM-DDTHH:mm:ss.sssZ.
|
||||
const converted = new Date(timestamp).toISOString();
|
||||
// Remove "-"s and "."s and milliseconds
|
||||
|
@ -28,13 +30,13 @@ export function convertUTCtoISO8601(timestamp: string | number) {
|
|||
|
||||
/**
|
||||
* Check whether timestamp predates request or is too old
|
||||
* @param timestamp of ISO8601Timestamp format without
|
||||
* @param {string} timestamp of ISO8601Timestamp format without
|
||||
* dashes or colons, e.g. 20160202T220410Z
|
||||
* @param expiry - number of seconds signature should be valid
|
||||
* @param log - log for request
|
||||
* @return true if there is a time problem
|
||||
* @param {number} expiry - number of seconds signature should be valid
|
||||
* @param {object} log - log for request
|
||||
* @return {boolean} true if there is a time problem
|
||||
*/
|
||||
export function checkTimeSkew(timestamp: string, expiry: number, log: Logger) {
|
||||
function checkTimeSkew(timestamp, expiry, log) {
|
||||
const currentTime = Date.now();
|
||||
const fifteenMinutes = (15 * 60 * 1000);
|
||||
const parsedTimestamp = convertAmzTimeToMs(timestamp);
|
||||
|
@ -54,3 +56,5 @@ export function checkTimeSkew(timestamp: string, expiry: number, log: Logger) {
|
|||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
module.exports = { convertAmzTimeToMs, convertUTCtoISO8601, checkTimeSkew };
|
|
@ -1,19 +1,17 @@
|
|||
import { Logger } from 'werelogs';
|
||||
import errors from '../../../lib/errors';
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const errors = require('../../../lib/errors').default;
|
||||
|
||||
/**
|
||||
* Validate Credentials
|
||||
* @param credentials - contains accessKey, scopeDate,
|
||||
* @param {array} credentials - contains accessKey, scopeDate,
|
||||
* region, service, requestType
|
||||
* @param timestamp - timestamp from request in
|
||||
* @param {string} timestamp - timestamp from request in
|
||||
* the format of ISO 8601: YYYYMMDDTHHMMSSZ
|
||||
* @param log - logging object
|
||||
* @param {object} log - logging object
|
||||
* @return {boolean} true if credentials are correct format, false if not
|
||||
*/
|
||||
export function validateCredentials(
|
||||
credentials: [string, string, string, string, string],
|
||||
timestamp: string,
|
||||
log: Logger
|
||||
): Error | {} {
|
||||
function validateCredentials(credentials, timestamp, log) {
|
||||
if (!Array.isArray(credentials) || credentials.length !== 5) {
|
||||
log.warn('credentials in improper format', { credentials });
|
||||
return errors.InvalidArgument;
|
||||
|
@ -27,20 +25,20 @@ export function validateCredentials(
|
|||
log.warn('accessKey provided is wrong format', { accessKey });
|
||||
return errors.InvalidArgument;
|
||||
}
|
||||
// The scope date (format YYYYMMDD) must be same date as the timestamp
|
||||
// on the request from the x-amz-date param (if queryAuthCheck)
|
||||
// or from the x-amz-date header or date header (if headerAuthCheck)
|
||||
// Format of timestamp is ISO 8601: YYYYMMDDTHHMMSSZ.
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/
|
||||
// sigv4-query-string-auth.html
|
||||
// http://docs.aws.amazon.com/general/latest/gr/
|
||||
// sigv4-date-handling.html
|
||||
// The scope date (format YYYYMMDD) must be same date as the timestamp
|
||||
// on the request from the x-amz-date param (if queryAuthCheck)
|
||||
// or from the x-amz-date header or date header (if headerAuthCheck)
|
||||
// Format of timestamp is ISO 8601: YYYYMMDDTHHMMSSZ.
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/
|
||||
// sigv4-query-string-auth.html
|
||||
// http://docs.aws.amazon.com/general/latest/gr/
|
||||
// sigv4-date-handling.html
|
||||
|
||||
// convert timestamp to format of scopeDate YYYYMMDD
|
||||
// convert timestamp to format of scopeDate YYYYMMDD
|
||||
const timestampDate = timestamp.split('T')[0];
|
||||
if (scopeDate.length !== 8 || scopeDate !== timestampDate) {
|
||||
log.warn('scope date must be the same date as the timestamp date',
|
||||
{ scopeDate, timestampDate });
|
||||
{ scopeDate, timestampDate });
|
||||
return errors.RequestTimeTooSkewed;
|
||||
}
|
||||
if (service !== 's3' && service !== 'iam' && service !== 'ring' &&
|
||||
|
@ -52,7 +50,7 @@ export function validateCredentials(
|
|||
}
|
||||
if (requestType !== 'aws4_request') {
|
||||
log.warn('requestType contained in params is not aws4_request',
|
||||
{ requestType });
|
||||
{ requestType });
|
||||
return errors.InvalidArgument;
|
||||
}
|
||||
return {};
|
||||
|
@ -60,26 +58,17 @@ export function validateCredentials(
|
|||
|
||||
/**
|
||||
* Extract and validate components from query object
|
||||
* @param queryObj - query object from request
|
||||
* @param log - logging object
|
||||
* @return object containing extracted query params for authV4
|
||||
* @param {object} queryObj - query object from request
|
||||
* @param {object} log - logging object
|
||||
* @return {object} object containing extracted query params for authV4
|
||||
*/
|
||||
export function extractQueryParams(
|
||||
queryObj: { [key: string]: string | undefined },
|
||||
log: Logger
|
||||
) {
|
||||
const authParams: {
|
||||
signedHeaders?: string;
|
||||
signatureFromRequest?: string;
|
||||
timestamp?: string;
|
||||
expiry?: number;
|
||||
credential?: [string, string, string, string, string];
|
||||
} = {};
|
||||
function extractQueryParams(queryObj, log) {
|
||||
const authParams = {};
|
||||
|
||||
// Do not need the algorithm sent back
|
||||
if (queryObj['X-Amz-Algorithm'] !== 'AWS4-HMAC-SHA256') {
|
||||
log.warn('algorithm param incorrect',
|
||||
{ algo: queryObj['X-Amz-Algorithm'] });
|
||||
{ algo: queryObj['X-Amz-Algorithm'] });
|
||||
return authParams;
|
||||
}
|
||||
|
||||
|
@ -110,7 +99,7 @@ export function extractQueryParams(
|
|||
return authParams;
|
||||
}
|
||||
|
||||
const expiry = Number.parseInt(queryObj['X-Amz-Expires'] ?? 'nope', 10);
|
||||
const expiry = Number.parseInt(queryObj['X-Amz-Expires'], 10);
|
||||
const sevenDays = 604800;
|
||||
if (expiry && (expiry > 0 && expiry <= sevenDays)) {
|
||||
authParams.expiry = expiry;
|
||||
|
@ -121,7 +110,6 @@ export function extractQueryParams(
|
|||
|
||||
const credential = queryObj['X-Amz-Credential'];
|
||||
if (credential && credential.length > 28 && credential.indexOf('/') > -1) {
|
||||
// @ts-ignore
|
||||
authParams.credential = credential.split('/');
|
||||
} else {
|
||||
log.warn('invalid credential param', { credential });
|
||||
|
@ -133,17 +121,14 @@ export function extractQueryParams(
|
|||
|
||||
/**
|
||||
* Extract and validate components from auth header
|
||||
* @param authHeader - authorization header from request
|
||||
* @param log - logging object
|
||||
* @return object containing extracted auth header items for authV4
|
||||
* @param {string} authHeader - authorization header from request
|
||||
* @param {object} log - logging object
|
||||
* @return {object} object containing extracted auth header items for authV4
|
||||
*/
|
||||
export function extractAuthItems(authHeader: string, log: Logger) {
|
||||
const authItems: {
|
||||
credentialsArr?: [string, string, string, string, string];
|
||||
signedHeaders?: string;
|
||||
signatureFromRequest?: string;
|
||||
} = {};
|
||||
const authArray = authHeader.replace('AWS4-HMAC-SHA256 ', '').split(',');
|
||||
function extractAuthItems(authHeader, log) {
|
||||
const authItems = {};
|
||||
const authArray = authHeader
|
||||
.replace('AWS4-HMAC-SHA256 ', '').split(',');
|
||||
|
||||
if (authArray.length < 3) {
|
||||
return authItems;
|
||||
|
@ -153,12 +138,8 @@ export function extractAuthItems(authHeader: string, log: Logger) {
|
|||
const signedHeadersStr = authArray[1];
|
||||
const signatureStr = authArray[2];
|
||||
log.trace('credentials from request', { credentialStr });
|
||||
if (
|
||||
credentialStr &&
|
||||
credentialStr.trim().startsWith('Credential=') &&
|
||||
credentialStr.indexOf('/') > -1
|
||||
) {
|
||||
// @ts-ignore
|
||||
if (credentialStr && credentialStr.trim().startsWith('Credential=')
|
||||
&& credentialStr.indexOf('/') > -1) {
|
||||
authItems.credentialsArr = credentialStr
|
||||
.trim().replace('Credential=', '').split('/');
|
||||
} else {
|
||||
|
@ -185,11 +166,11 @@ export function extractAuthItems(authHeader: string, log: Logger) {
|
|||
/**
|
||||
* Checks whether the signed headers include the host header
|
||||
* and all x-amz- and x-scal- headers in request
|
||||
* @param signedHeaders - signed headers sent with request
|
||||
* @param allHeaders - request.headers
|
||||
* @return true if all x-amz-headers included and false if not
|
||||
* @param {string} signedHeaders - signed headers sent with request
|
||||
* @param {object} allHeaders - request.headers
|
||||
* @return {boolean} true if all x-amz-headers included and false if not
|
||||
*/
|
||||
export function areSignedHeadersComplete(signedHeaders: string, allHeaders: Headers) {
|
||||
function areSignedHeadersComplete(signedHeaders, allHeaders) {
|
||||
const signedHeadersList = signedHeaders.split(';');
|
||||
if (signedHeadersList.indexOf('host') === -1) {
|
||||
return false;
|
||||
|
@ -204,3 +185,6 @@ export function areSignedHeadersComplete(signedHeaders: string, allHeaders: Head
|
|||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
module.exports = { validateCredentials, extractQueryParams,
|
||||
areSignedHeadersComplete, extractAuthItems };
|
|
@ -1,569 +0,0 @@
|
|||
import cluster, { Worker } from 'cluster';
|
||||
import * as werelogs from 'werelogs';
|
||||
|
||||
import { default as errors } from '../../lib/errors';
|
||||
|
||||
const rpcLogger = new werelogs.Logger('ClusterRPC');
|
||||
|
||||
/**
|
||||
* Remote procedure calls support between cluster workers.
|
||||
*
|
||||
* When using the cluster module, new processes are forked and are
|
||||
* dispatched workloads, usually HTTP requests. The ClusterRPC module
|
||||
* implements a RPC system to send commands to all cluster worker
|
||||
* processes at once from any particular worker, and retrieve their
|
||||
* individual command results, like a distributed map operation.
|
||||
*
|
||||
* The existing nodejs cluster IPC channel is setup from the primary
|
||||
* to each worker, but not between workers, so there has to be a hop
|
||||
* by the primary.
|
||||
*
|
||||
* How a command is treated:
|
||||
*
|
||||
* - a worker sends a command message to the primary
|
||||
*
|
||||
* - the primary then forwards that command to each existing worker
|
||||
* (including the requestor)
|
||||
*
|
||||
* - each worker then executes the command and returns a result or an
|
||||
* error
|
||||
*
|
||||
* - the primary gathers all workers results into an array
|
||||
*
|
||||
* - finally, the primary dispatches the results array to the original
|
||||
* requesting worker
|
||||
*
|
||||
*
|
||||
* Limitations:
|
||||
*
|
||||
* - The command payload must be serializable, which means that:
|
||||
* - it should not contain circular references
|
||||
* - it should be of a reasonable size to be sent in a single RPC message
|
||||
*
|
||||
* - The "toWorkers" parameter of value "*" targets the set of workers
|
||||
* that are available at the time the command is dispatched. Any new
|
||||
* worker spawned after the command has been dispatched for
|
||||
* processing, but before the command completes, don't execute
|
||||
* the command and hence are not part of the results array.
|
||||
*
|
||||
*
|
||||
* To set it up:
|
||||
*
|
||||
* - On the primary:
|
||||
* if (cluster.isPrimary) {
|
||||
* setupRPCPrimary();
|
||||
* }
|
||||
*
|
||||
* - On the workers:
|
||||
* if (!cluster.isPrimary) {
|
||||
* setupRPCWorker({
|
||||
* handler1: (payload: object, uids: string, callback: HandlerCallback) => void,
|
||||
* handler2: ...
|
||||
* });
|
||||
* }
|
||||
* Handler functions will be passed the command payload, request
|
||||
* serialized uids, and must call the callback when the worker is done
|
||||
* processing the command:
|
||||
* callback(error: Error | null | undefined, result?: any)
|
||||
*
|
||||
* When this setup is done, any worker can start sending commands by calling
|
||||
* the async function sendWorkerCommand().
|
||||
*/
|
||||
|
||||
// exported types
|
||||
|
||||
export type ResultObject = {
|
||||
error: Error | null;
|
||||
result: any;
|
||||
};
|
||||
|
||||
/**
|
||||
* saved Promise for sendWorkerCommand
|
||||
*/
|
||||
export type CommandPromise = {
|
||||
resolve: (results?: ResultObject[]) => void;
|
||||
reject: (error: Error) => void;
|
||||
timeout: NodeJS.Timeout | null;
|
||||
};
|
||||
export type HandlerCallback = (error: (Error & { code?: number }) | null | undefined, result?: any) => void;
|
||||
export type HandlerFunction = (payload: object, uids: string, callback: HandlerCallback) => void;
|
||||
export type HandlersMap = {
|
||||
[index: string]: HandlerFunction;
|
||||
};
|
||||
export type PrimaryHandlerFunction = (worker: Worker, payload: object, uids: string, callback: HandlerCallback) => void;
|
||||
export type PrimaryHandlersMap = Record<string, PrimaryHandlerFunction>;
|
||||
|
||||
// private types
|
||||
|
||||
type RPCMessage<T extends string, P> = {
|
||||
type: T;
|
||||
uids: string;
|
||||
payload: P;
|
||||
};
|
||||
|
||||
type RPCCommandMessage = RPCMessage<'cluster-rpc:command', any> & {
|
||||
toWorkers: string;
|
||||
toHandler: string;
|
||||
};
|
||||
|
||||
type MarshalledResultObject = {
|
||||
error: string | null;
|
||||
errorCode?: number;
|
||||
result: any;
|
||||
};
|
||||
|
||||
type RPCCommandResultMessage = RPCMessage<'cluster-rpc:commandResult', MarshalledResultObject>;
|
||||
|
||||
type RPCCommandResultsMessage = RPCMessage<'cluster-rpc:commandResults', {
|
||||
results: MarshalledResultObject[];
|
||||
}>;
|
||||
|
||||
type RPCCommandErrorMessage = RPCMessage<'cluster-rpc:commandError', {
|
||||
error: string;
|
||||
}>;
|
||||
|
||||
interface RPCSetupOptions {
|
||||
/**
|
||||
* As werelogs is not a peerDependency, arsenal and a parent project
|
||||
* might have their own separate versions duplicated in dependencies.
|
||||
* The config are therefore not shared.
|
||||
* Use this to propagate werelogs config to arsenal's ClusterRPC.
|
||||
*/
|
||||
werelogsConfig?: Parameters<typeof werelogs.configure>[0];
|
||||
};
|
||||
|
||||
/**
|
||||
* In primary: store worker IDs that are waiting to be dispatched
|
||||
* their command's results, as a mapping.
|
||||
*/
|
||||
const uidsToWorkerId: {
|
||||
[index: string]: number;
|
||||
} = {};
|
||||
|
||||
|
||||
/**
|
||||
* In primary: store worker responses for commands in progress as a
|
||||
* mapping.
|
||||
*
|
||||
* Result objects are 'null' while the worker is still processing the
|
||||
* command. When a worker finishes processing it stores the result as:
|
||||
* {
|
||||
* error: string | null,
|
||||
* result: any
|
||||
* }
|
||||
*/
|
||||
const uidsToCommandResults: {
|
||||
[index: string]: {
|
||||
[index: number]: MarshalledResultObject | null;
|
||||
};
|
||||
} = {};
|
||||
|
||||
/**
|
||||
* In workers: store promise callbacks for commands waiting to be
|
||||
* dispatched, as a mapping.
|
||||
*/
|
||||
const uidsToCommandPromise: {
|
||||
[index: string]: CommandPromise;
|
||||
} = {};
|
||||
|
||||
|
||||
function _isRpcMessage(message) {
|
||||
return (message !== null &&
|
||||
typeof message === 'object' &&
|
||||
typeof message.type === 'string' &&
|
||||
message.type.startsWith('cluster-rpc:'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup cluster RPC system on the primary
|
||||
*
|
||||
* @param {object} [handlers] - mapping of handler names to handler functions
|
||||
* handler function:
|
||||
* `handler({Worker} worker, {object} payload, {string} uids, {function} callback)`
|
||||
* handler callback must be called when worker is done with the command:
|
||||
* `callback({Error|null} error, {any} [result])`
|
||||
* @return {undefined}
|
||||
*/
|
||||
export function setupRPCPrimary(handlers?: PrimaryHandlersMap, options?: RPCSetupOptions) {
|
||||
if (options?.werelogsConfig) {
|
||||
werelogs.configure(options.werelogsConfig);
|
||||
}
|
||||
cluster.on('message', (worker, message) => {
|
||||
if (_isRpcMessage(message)) {
|
||||
_handlePrimaryMessage(worker, message, handlers);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup RPCs on a cluster worker process
|
||||
*
|
||||
* @param {object} handlers - mapping of handler names to handler functions
|
||||
* handler function:
|
||||
* handler({object} payload, {string} uids, {function} callback)
|
||||
* handler callback must be called when worker is done with the command:
|
||||
* callback({Error|null} error, {any} [result])
|
||||
* @return {undefined}
|
||||
* }
|
||||
*/
|
||||
export function setupRPCWorker(handlers: HandlersMap, options?: RPCSetupOptions) {
|
||||
if (!process.send) {
|
||||
throw new Error('fatal: cannot setup cluster RPC: "process.send" is not available');
|
||||
}
|
||||
if (options?.werelogsConfig) {
|
||||
werelogs.configure(options.werelogsConfig);
|
||||
}
|
||||
process.on('message', (message: RPCCommandMessage | RPCCommandResultsMessage) => {
|
||||
if (_isRpcMessage(message)) {
|
||||
_handleWorkerMessage(message, handlers);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a command for workers to execute in parallel, and wait for results
|
||||
*
|
||||
* @param {string} toWorkers - which workers should execute the command
|
||||
* Currently the supported values are:
|
||||
* - "*", meaning all workers will execute the command
|
||||
* - "PRIMARY", meaning primary process will execute the command
|
||||
* @param {string} toHandler - name of handler that will execute the
|
||||
* command in workers, as declared in setupRPCWorker() parameter object
|
||||
* @param {string} uids - unique identifier of the command, must be
|
||||
* unique across all commands in progress
|
||||
* @param {object} payload - message payload, sent as-is to the handler
|
||||
* @param {number} [timeoutMs=60000] - timeout the command with a
|
||||
* "RequestTimeout" error after this number of milliseconds - set to 0
|
||||
* to disable timeouts (the command may then hang forever)
|
||||
* @returns {Promise}
|
||||
*/
|
||||
export async function sendWorkerCommand(
|
||||
toWorkers: string,
|
||||
toHandler: string,
|
||||
uids: string,
|
||||
payload: object,
|
||||
timeoutMs: number = 60000
|
||||
) {
|
||||
if (typeof uids !== 'string') {
|
||||
rpcLogger.error('missing or invalid "uids" field', { uids });
|
||||
throw errors.MissingParameter;
|
||||
}
|
||||
if (uidsToCommandPromise[uids] !== undefined) {
|
||||
rpcLogger.error('a command is already in progress with same uids', { uids });
|
||||
throw errors.OperationAborted;
|
||||
}
|
||||
rpcLogger.info('sending command', { toWorkers, toHandler, uids, payload });
|
||||
return new Promise((resolve, reject) => {
|
||||
let timeout: NodeJS.Timeout | null = null;
|
||||
if (timeoutMs) {
|
||||
timeout = setTimeout(() => {
|
||||
delete uidsToCommandPromise[uids];
|
||||
reject(errors.RequestTimeout);
|
||||
}, timeoutMs);
|
||||
}
|
||||
uidsToCommandPromise[uids] = { resolve, reject, timeout };
|
||||
const message: RPCCommandMessage = {
|
||||
type: 'cluster-rpc:command',
|
||||
toWorkers,
|
||||
toHandler,
|
||||
uids,
|
||||
payload,
|
||||
};
|
||||
return process.send?.(message);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of commands in flight
|
||||
* @returns {number}
|
||||
*/
|
||||
export function getPendingCommandsCount() {
|
||||
return Object.keys(uidsToCommandPromise).length;
|
||||
}
|
||||
|
||||
|
||||
function _dispatchCommandResultsToWorker(
|
||||
worker: Worker,
|
||||
uids: string,
|
||||
resultsArray: MarshalledResultObject[]
|
||||
): void {
|
||||
const message: RPCCommandResultsMessage = {
|
||||
type: 'cluster-rpc:commandResults',
|
||||
uids,
|
||||
payload: {
|
||||
results: resultsArray,
|
||||
},
|
||||
};
|
||||
worker.send(message);
|
||||
}
|
||||
|
||||
function _dispatchCommandErrorToWorker(
|
||||
worker: Worker,
|
||||
uids: string,
|
||||
error: Error,
|
||||
): void {
|
||||
const message: RPCCommandErrorMessage = {
|
||||
type: 'cluster-rpc:commandError',
|
||||
uids,
|
||||
payload: {
|
||||
error: error.message,
|
||||
},
|
||||
};
|
||||
worker.send(message);
|
||||
}
|
||||
|
||||
function _sendPrimaryCommandResult(
|
||||
worker: Worker,
|
||||
uids: string,
|
||||
error: (Error & { code?: number }) | null | undefined,
|
||||
result?: any
|
||||
): void {
|
||||
const message: RPCCommandResultsMessage = {
|
||||
type: 'cluster-rpc:commandResults',
|
||||
uids,
|
||||
payload: {
|
||||
results: [{ error: error?.message || null, errorCode: error?.code, result }],
|
||||
},
|
||||
};
|
||||
worker.send?.(message);
|
||||
}
|
||||
|
||||
function _handlePrimaryCommandMessage(
|
||||
fromWorker: Worker,
|
||||
logger: any,
|
||||
message: RPCCommandMessage,
|
||||
handlers?: PrimaryHandlersMap
|
||||
): void {
|
||||
const { toWorkers, toHandler, uids, payload } = message;
|
||||
if (toWorkers === '*') {
|
||||
if (uidsToWorkerId[uids] !== undefined) {
|
||||
logger.warn('new command already has a waiting worker with same uids', {
|
||||
uids, workerId: uidsToWorkerId[uids],
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
const commandResults = {};
|
||||
for (const workerId of Object.keys(cluster.workers || {})) {
|
||||
commandResults[workerId] = null;
|
||||
}
|
||||
uidsToWorkerId[uids] = fromWorker?.id;
|
||||
uidsToCommandResults[uids] = commandResults;
|
||||
|
||||
for (const [workerId, worker] of Object.entries(cluster.workers || {})) {
|
||||
logger.debug('sending command message to worker', {
|
||||
workerId, toHandler, payload,
|
||||
});
|
||||
if (worker) {
|
||||
worker.send(message);
|
||||
}
|
||||
}
|
||||
} else if (toWorkers === 'PRIMARY') {
|
||||
const { toHandler, uids, payload } = message;
|
||||
const cb: HandlerCallback = (err, result) => _sendPrimaryCommandResult(fromWorker, uids, err, result);
|
||||
|
||||
if (toHandler in (handlers || {})) {
|
||||
return handlers![toHandler](fromWorker, payload, uids, cb);
|
||||
}
|
||||
logger.error('no such handler in "toHandler" field from worker command message', {
|
||||
toHandler,
|
||||
});
|
||||
return cb(errors.NotImplemented);
|
||||
} else {
|
||||
logger.error('unsupported "toWorkers" field from worker command message', {
|
||||
toWorkers,
|
||||
});
|
||||
if (fromWorker) {
|
||||
_dispatchCommandErrorToWorker(fromWorker, uids, errors.NotImplemented);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function _handlePrimaryCommandResultMessage(
|
||||
fromWorkerId: number,
|
||||
logger: any,
|
||||
message: RPCCommandResultMessage
|
||||
): void {
|
||||
const { uids, payload } = message;
|
||||
const commandResults = uidsToCommandResults[uids];
|
||||
if (!commandResults) {
|
||||
logger.warn('received command response message from worker for command not in flight', {
|
||||
workerId: fromWorkerId,
|
||||
uids,
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
if (commandResults[fromWorkerId] === undefined) {
|
||||
logger.warn('received command response message with unexpected worker ID', {
|
||||
workerId: fromWorkerId,
|
||||
uids,
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
if (commandResults[fromWorkerId] !== null) {
|
||||
logger.warn('ignoring duplicate command response from worker', {
|
||||
workerId: fromWorkerId,
|
||||
uids,
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
commandResults[fromWorkerId] = payload;
|
||||
const commandResultsArray = Object.values(commandResults);
|
||||
if (commandResultsArray.every(response => response !== null)) {
|
||||
logger.debug('all workers responded to command', { uids });
|
||||
const completeCommandResultsArray = <MarshalledResultObject[]> commandResultsArray;
|
||||
const toWorkerId = uidsToWorkerId[uids];
|
||||
const toWorker = cluster.workers?.[toWorkerId];
|
||||
|
||||
delete uidsToCommandResults[uids];
|
||||
delete uidsToWorkerId[uids];
|
||||
|
||||
if (!toWorker) {
|
||||
logger.warn('worker shut down while its command was executing', {
|
||||
workerId: toWorkerId, uids,
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
// send back response to original worker
|
||||
_dispatchCommandResultsToWorker(toWorker, uids, completeCommandResultsArray);
|
||||
}
|
||||
}
|
||||
|
||||
function _handlePrimaryMessage(
|
||||
fromWorker: Worker,
|
||||
message: RPCCommandMessage | RPCCommandResultMessage,
|
||||
handlers?: PrimaryHandlersMap
|
||||
): void {
|
||||
const { type: messageType, uids } = message;
|
||||
const logger = rpcLogger.newRequestLoggerFromSerializedUids(uids);
|
||||
logger.debug('primary received message from worker', {
|
||||
workerId: fromWorker?.id, rpcMessage: message,
|
||||
});
|
||||
if (messageType === 'cluster-rpc:command') {
|
||||
return _handlePrimaryCommandMessage(fromWorker, logger, message, handlers);
|
||||
}
|
||||
if (messageType === 'cluster-rpc:commandResult') {
|
||||
return _handlePrimaryCommandResultMessage(fromWorker?.id, logger, message);
|
||||
}
|
||||
logger.error('unsupported message type', {
|
||||
workerId: fromWorker?.id, messageType, uids,
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function _sendWorkerCommandResult(
|
||||
uids: string,
|
||||
error: Error | null | undefined,
|
||||
result?: any
|
||||
): void {
|
||||
const message: RPCCommandResultMessage = {
|
||||
type: 'cluster-rpc:commandResult',
|
||||
uids,
|
||||
payload: {
|
||||
error: error ? error.message : null,
|
||||
result,
|
||||
},
|
||||
};
|
||||
process.send?.(message);
|
||||
}
|
||||
|
||||
function _handleWorkerCommandMessage(
|
||||
logger: any,
|
||||
message: RPCCommandMessage,
|
||||
handlers: HandlersMap
|
||||
): void {
|
||||
const { toHandler, uids, payload } = message;
|
||||
const cb: HandlerCallback = (err, result) => _sendWorkerCommandResult(uids, err, result);
|
||||
|
||||
if (toHandler in handlers) {
|
||||
return handlers[toHandler](payload, uids, cb);
|
||||
}
|
||||
logger.error('no such handler in "toHandler" field from worker command message', {
|
||||
toHandler,
|
||||
});
|
||||
return cb(errors.NotImplemented);
|
||||
}
|
||||
|
||||
function _handleWorkerCommandResultsMessage(
|
||||
logger: any,
|
||||
message: RPCCommandResultsMessage,
|
||||
): void {
|
||||
const { uids, payload } = message;
|
||||
const { results } = payload;
|
||||
const commandPromise: CommandPromise = uidsToCommandPromise[uids];
|
||||
if (commandPromise === undefined) {
|
||||
logger.error('missing promise for command results', { uids, payload });
|
||||
return undefined;
|
||||
}
|
||||
if (commandPromise.timeout) {
|
||||
clearTimeout(commandPromise.timeout);
|
||||
}
|
||||
delete uidsToCommandPromise[uids];
|
||||
const unmarshalledResults = results.map(workerResult => {
|
||||
let workerError: Error | null = null;
|
||||
if (workerResult.error) {
|
||||
if (workerResult.error in errors) {
|
||||
workerError = errors[workerResult.error];
|
||||
} else {
|
||||
workerError = new Error(workerResult.error);
|
||||
}
|
||||
}
|
||||
if (workerError && workerResult.errorCode) {
|
||||
(workerError as Error & { code: number }).code = workerResult.errorCode;
|
||||
}
|
||||
const unmarshalledResult: ResultObject = {
|
||||
error: workerError,
|
||||
result: workerResult.result,
|
||||
};
|
||||
return unmarshalledResult;
|
||||
});
|
||||
return commandPromise.resolve(unmarshalledResults);
|
||||
}
|
||||
|
||||
function _handleWorkerCommandErrorMessage(
|
||||
logger: any,
|
||||
message: RPCCommandErrorMessage,
|
||||
): void {
|
||||
const { uids, payload } = message;
|
||||
const { error } = payload;
|
||||
const commandPromise: CommandPromise = uidsToCommandPromise[uids];
|
||||
if (commandPromise === undefined) {
|
||||
logger.error('missing promise for command results', { uids, payload });
|
||||
return undefined;
|
||||
}
|
||||
if (commandPromise.timeout) {
|
||||
clearTimeout(commandPromise.timeout);
|
||||
}
|
||||
delete uidsToCommandPromise[uids];
|
||||
let commandError: Error | null = null;
|
||||
if (error in errors) {
|
||||
commandError = errors[error];
|
||||
} else {
|
||||
commandError = new Error(error);
|
||||
}
|
||||
return commandPromise.reject(<Error> commandError);
|
||||
}
|
||||
|
||||
function _handleWorkerMessage(
|
||||
message: RPCCommandMessage | RPCCommandResultsMessage | RPCCommandErrorMessage,
|
||||
handlers: HandlersMap
|
||||
): void {
|
||||
const { type: messageType, uids } = message;
|
||||
const workerId = cluster.worker?.id;
|
||||
const logger = rpcLogger.newRequestLoggerFromSerializedUids(uids);
|
||||
logger.debug('worker received message from primary', {
|
||||
workerId, rpcMessage: message,
|
||||
});
|
||||
if (messageType === 'cluster-rpc:command') {
|
||||
return _handleWorkerCommandMessage(logger, message, handlers);
|
||||
}
|
||||
if (messageType === 'cluster-rpc:commandResults') {
|
||||
return _handleWorkerCommandResultsMessage(logger, message);
|
||||
}
|
||||
if (messageType === 'cluster-rpc:commandError') {
|
||||
return _handleWorkerCommandErrorMessage(logger, message);
|
||||
}
|
||||
logger.error('unsupported message type', {
|
||||
workerId, messageType,
|
||||
});
|
||||
return undefined;
|
||||
}
|
|
@ -0,0 +1,121 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
// The min value here is to manage further backward compat if we
|
||||
// need it
|
||||
const iamSecurityTokenSizeMin = 128;
|
||||
const iamSecurityTokenSizeMax = 128;
|
||||
// Security token is an hex string (no real format from amazon)
|
||||
const iamSecurityTokenPattern =
|
||||
new RegExp(`^[a-f0-9]{${iamSecurityTokenSizeMin},` +
|
||||
`${iamSecurityTokenSizeMax}}$`);
|
||||
|
||||
module.exports = {
|
||||
// info about the iam security token
|
||||
iamSecurityToken: {
|
||||
min: iamSecurityTokenSizeMin,
|
||||
max: iamSecurityTokenSizeMax,
|
||||
pattern: iamSecurityTokenPattern,
|
||||
},
|
||||
// PublicId is used as the canonicalID for a request that contains
|
||||
// no authentication information. Requestor can access
|
||||
// only public resources
|
||||
publicId: 'http://acs.amazonaws.com/groups/global/AllUsers',
|
||||
zenkoServiceAccount: 'http://acs.zenko.io/accounts/service',
|
||||
metadataFileNamespace: '/MDFile',
|
||||
dataFileURL: '/DataFile',
|
||||
// AWS states max size for user-defined metadata
|
||||
// (x-amz-meta- headers) is 2 KB:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
|
||||
// In testing, AWS seems to allow up to 88 more bytes,
|
||||
// so we do the same.
|
||||
maximumMetaHeadersSize: 2136,
|
||||
emptyFileMd5: 'd41d8cd98f00b204e9800998ecf8427e',
|
||||
// Version 2 changes the format of the data location property
|
||||
// Version 3 adds the dataStoreName attribute
|
||||
mdModelVersion: 3,
|
||||
/*
|
||||
* Splitter is used to build the object name for the overview of a
|
||||
* multipart upload and to build the object names for each part of a
|
||||
* multipart upload. These objects with large names are then stored in
|
||||
* metadata in a "shadow bucket" to a real bucket. The shadow bucket
|
||||
* contains all ongoing multipart uploads. We include in the object
|
||||
* name some of the info we might need to pull about an open multipart
|
||||
* upload or about an individual part with each piece of info separated
|
||||
* by the splitter. We can then extract each piece of info by splitting
|
||||
* the object name string with this splitter.
|
||||
* For instance, assuming a splitter of '...!*!',
|
||||
* the name of the upload overview would be:
|
||||
* overview...!*!objectKey...!*!uploadId
|
||||
* For instance, the name of a part would be:
|
||||
* uploadId...!*!partNumber
|
||||
*
|
||||
* The sequence of characters used in the splitter should not occur
|
||||
* elsewhere in the pieces of info to avoid splitting where not
|
||||
* intended.
|
||||
*
|
||||
* Splitter is also used in adding bucketnames to the
|
||||
* namespacerusersbucket. The object names added to the
|
||||
* namespaceusersbucket are of the form:
|
||||
* canonicalID...!*!bucketname
|
||||
*/
|
||||
|
||||
splitter: '..|..',
|
||||
usersBucket: 'users..bucket',
|
||||
// MPU Bucket Prefix is used to create the name of the shadow
|
||||
// bucket used for multipart uploads. There is one shadow mpu
|
||||
// bucket per bucket and its name is the mpuBucketPrefix followed
|
||||
// by the name of the final destination bucket for the object
|
||||
// once the multipart upload is complete.
|
||||
mpuBucketPrefix: 'mpuShadowBucket',
|
||||
// since aws s3 does not allow capitalized buckets, these may be
|
||||
// used for special internal purposes
|
||||
permittedCapitalizedBuckets: {
|
||||
METADATA: true,
|
||||
},
|
||||
/* eslint-disable camelcase */
|
||||
externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true },
|
||||
hasCopyPartBackends: { aws_s3: true, gcp: true },
|
||||
versioningNotImplBackends: { azure: true, gcp: true },
|
||||
mpuMDStoredExternallyBackend: { aws_s3: true, gcp: true },
|
||||
/* eslint-enable camelcase */
|
||||
// Default expiration value of the S3 pre-signed URL duration
|
||||
// 604800 seconds (seven days).
|
||||
defaultPreSignedURLExpiry: 7 * 24 * 60 * 60,
|
||||
// Regex for ISO-8601 formatted date
|
||||
shortIso8601Regex: /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/,
|
||||
longIso8601Regex: /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/,
|
||||
supportedNotificationEvents: new Set([
|
||||
's3:ObjectCreated:*',
|
||||
's3:ObjectCreated:Put',
|
||||
's3:ObjectCreated:Copy',
|
||||
's3:ObjectCreated:CompleteMultipartUpload',
|
||||
's3:ObjectRemoved:*',
|
||||
's3:ObjectRemoved:Delete',
|
||||
's3:ObjectRemoved:DeleteMarkerCreated',
|
||||
]),
|
||||
notificationArnPrefix: 'arn:scality:bucketnotif',
|
||||
// some of the available data backends (if called directly rather
|
||||
// than through the multiple backend gateway) need a key provided
|
||||
// as a string as first parameter of the get/delete methods.
|
||||
clientsRequireStringKey: { sproxyd: true, cdmi: true },
|
||||
// HTTP server keep-alive timeout is set to a higher value than
|
||||
// client's free sockets timeout to avoid the risk of triggering
|
||||
// ECONNRESET errors if the server closes the connection at the
|
||||
// exact moment clients attempt to reuse an established connection
|
||||
// for a new request.
|
||||
//
|
||||
// Note: the ability to close inactive connections on the client
|
||||
// after httpClientFreeSocketsTimeout milliseconds requires the
|
||||
// use of "agentkeepalive" module instead of the regular node.js
|
||||
// http.Agent.
|
||||
httpServerKeepAliveTimeout: 60000,
|
||||
httpClientFreeSocketTimeout: 55000,
|
||||
supportedLifecycleRules: [
|
||||
'expiration',
|
||||
'noncurrentVersionExpiration',
|
||||
'abortIncompleteMultipartUpload',
|
||||
],
|
||||
// Maximum number of buckets to cache (bucket metadata)
|
||||
maxCachedBuckets: process.env.METADATA_MAX_CACHED_BUCKETS ?
|
||||
Number(process.env.METADATA_MAX_CACHED_BUCKETS) : 1000,
|
||||
};
|
177
lib/constants.ts
177
lib/constants.ts
|
@ -1,177 +0,0 @@
|
|||
import * as crypto from 'crypto';
|
||||
|
||||
// The min value here is to manage further backward compat if we
|
||||
// need it
|
||||
// Default value
|
||||
export const vaultGeneratedIamSecurityTokenSizeMin = 128;
|
||||
// Safe to assume that a typical token size is less than 8192 bytes
|
||||
export const vaultGeneratedIamSecurityTokenSizeMax = 8192;
|
||||
// Base-64
|
||||
export const vaultGeneratedIamSecurityTokenPattern = /^[A-Za-z0-9/+=]*$/;
|
||||
|
||||
// info about the iam security token
|
||||
export const iamSecurityToken = {
|
||||
min: vaultGeneratedIamSecurityTokenSizeMin,
|
||||
max: vaultGeneratedIamSecurityTokenSizeMax,
|
||||
pattern: vaultGeneratedIamSecurityTokenPattern,
|
||||
};
|
||||
// PublicId is used as the canonicalID for a request that contains
|
||||
// no authentication information. Requestor can access
|
||||
// only public resources
|
||||
export const publicId = 'http://acs.amazonaws.com/groups/global/AllUsers';
|
||||
export const zenkoServiceAccount = 'http://acs.zenko.io/accounts/service';
|
||||
export const metadataFileNamespace = '/MDFile';
|
||||
export const dataFileURL = '/DataFile';
|
||||
export const passthroughFileURL = '/PassthroughFile';
|
||||
// AWS states max size for user-defined metadata
|
||||
// (x-amz-meta- headers) is 2 KB:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
|
||||
// In testing, AWS seems to allow up to 88 more bytes,
|
||||
// so we do the same.
|
||||
export const maximumMetaHeadersSize = 2136;
|
||||
export const emptyFileMd5 = 'd41d8cd98f00b204e9800998ecf8427e';
|
||||
// Version 2 changes the format of the data location property
|
||||
// Version 3 adds the dataStoreName attribute
|
||||
// Version 4 add the Creation-Time and Content-Language attributes,
|
||||
// and add support for x-ms-meta-* headers in UserMetadata
|
||||
// Version 5 adds the azureInfo structure
|
||||
// Version 6 adds a "deleted" flag that is updated to true before
|
||||
// the object gets deleted. This is done to keep object metadata in the
|
||||
// oplog when deleting the object, as oplog deletion events don't contain
|
||||
// any metadata of the object.
|
||||
// version 6 also adds the "isPHD" flag that is used to indicate that the master
|
||||
// object is a placeholder and is not up to date.
|
||||
export const mdModelVersion = 6;
|
||||
/*
|
||||
* Splitter is used to build the object name for the overview of a
|
||||
* multipart upload and to build the object names for each part of a
|
||||
* multipart upload. These objects with large names are then stored in
|
||||
* metadata in a "shadow bucket" to a real bucket. The shadow bucket
|
||||
* contains all ongoing multipart uploads. We include in the object
|
||||
* name some of the info we might need to pull about an open multipart
|
||||
* upload or about an individual part with each piece of info separated
|
||||
* by the splitter. We can then extract each piece of info by splitting
|
||||
* the object name string with this splitter.
|
||||
* For instance, assuming a splitter of '...!*!',
|
||||
* the name of the upload overview would be:
|
||||
* overview...!*!objectKey...!*!uploadId
|
||||
* For instance, the name of a part would be:
|
||||
* uploadId...!*!partNumber
|
||||
*
|
||||
* The sequence of characters used in the splitter should not occur
|
||||
* elsewhere in the pieces of info to avoid splitting where not
|
||||
* intended.
|
||||
*
|
||||
* Splitter is also used in adding bucketnames to the
|
||||
* namespacerusersbucket. The object names added to the
|
||||
* namespaceusersbucket are of the form:
|
||||
* canonicalID...!*!bucketname
|
||||
*/
|
||||
|
||||
export const splitter = '..|..';
|
||||
export const usersBucket = 'users..bucket';
|
||||
// MPU Bucket Prefix is used to create the name of the shadow
|
||||
// bucket used for multipart uploads. There is one shadow mpu
|
||||
// bucket per bucket and its name is the mpuBucketPrefix followed
|
||||
// by the name of the final destination bucket for the object
|
||||
// once the multipart upload is complete.
|
||||
export const mpuBucketPrefix = 'mpuShadowBucket';
|
||||
// since aws s3 does not allow capitalized buckets, these may be
|
||||
// used for special internal purposes
|
||||
export const permittedCapitalizedBuckets = {
|
||||
METADATA: true,
|
||||
};
|
||||
// Setting a lower object key limit to account for:
|
||||
// - Mongo key limit of 1012 bytes
|
||||
// - Version ID in Mongo Key if versioned of 33
|
||||
// - Max bucket name length if bucket match false of 63
|
||||
// - Extra prefix slash for bucket prefix if bucket match of 1
|
||||
export const objectKeyByteLimit = 915;
|
||||
/* delimiter for location-constraint. The location constraint will be able
|
||||
* to include the ingestion flag
|
||||
*/
|
||||
export const zenkoSeparator = ':';
|
||||
/* eslint-disable camelcase */
|
||||
export const externalBackends = { aws_s3: true, azure: true, gcp: true, pfs: true };
|
||||
export const replicationBackends = { aws_s3: true, azure: true, gcp: true };
|
||||
// hex digest of sha256 hash of empty string:
|
||||
export const emptyStringHash = crypto.createHash('sha256')
|
||||
.update('', 'binary').digest('hex');
|
||||
export const mpuMDStoredExternallyBackend = { aws_s3: true, gcp: true };
|
||||
// AWS sets a minimum size limit for parts except for the last part.
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
|
||||
export const minimumAllowedPartSize = 5242880;
|
||||
export const gcpMaximumAllowedPartCount = 1024;
|
||||
// GCP Object Tagging Prefix
|
||||
export const gcpTaggingPrefix = 'aws-tag-';
|
||||
export const productName = 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko';
|
||||
export const legacyLocations = ['sproxyd', 'legacy'];
|
||||
// healthcheck default call from nginx is every 2 seconds
|
||||
// for external backends, don't call unless at least 1 minute
|
||||
// (60,000 milliseconds) since last call
|
||||
export const externalBackendHealthCheckInterval = 60000;
|
||||
// some of the available data backends (if called directly rather
|
||||
// than through the multiple backend gateway) need a key provided
|
||||
// as a string as first parameter of the get/delete methods.
|
||||
export const clientsRequireStringKey = { sproxyd: true, cdmi: true };
|
||||
export const hasCopyPartBackends = { aws_s3: true, gcp: true };
|
||||
export const versioningNotImplBackends = { azure: true, gcp: true };
|
||||
// user metadata applied on zenko-created objects
|
||||
export const zenkoIDHeader = 'x-amz-meta-zenko-instance-id';
|
||||
// Default expiration value of the S3 pre-signed URL duration
|
||||
// 604800 seconds (seven days).
|
||||
export const defaultPreSignedURLExpiry = 7 * 24 * 60 * 60;
|
||||
// Regex for ISO-8601 formatted date
|
||||
export const shortIso8601Regex = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/;
|
||||
export const longIso8601Regex = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/;
|
||||
export const supportedNotificationEvents = new Set([
|
||||
's3:ObjectCreated:*',
|
||||
's3:ObjectCreated:Put',
|
||||
's3:ObjectCreated:Copy',
|
||||
's3:ObjectCreated:CompleteMultipartUpload',
|
||||
's3:ObjectRemoved:*',
|
||||
's3:ObjectRemoved:Delete',
|
||||
's3:ObjectRemoved:DeleteMarkerCreated',
|
||||
's3:Replication:OperationFailedReplication',
|
||||
's3:ObjectTagging:*',
|
||||
's3:ObjectTagging:Put',
|
||||
's3:ObjectTagging:Delete',
|
||||
's3:ObjectAcl:Put',
|
||||
's3:ObjectRestore:*',
|
||||
's3:ObjectRestore:Post',
|
||||
's3:ObjectRestore:Completed',
|
||||
's3:ObjectRestore:Delete',
|
||||
's3:LifecycleTransition',
|
||||
's3:LifecycleExpiration:*',
|
||||
's3:LifecycleExpiration:DeleteMarkerCreated',
|
||||
's3:LifecycleExpiration:Delete',
|
||||
]);
|
||||
export const notificationArnPrefix = 'arn:scality:bucketnotif';
|
||||
// HTTP server keep-alive timeout is set to a higher value than
|
||||
// client's free sockets timeout to avoid the risk of triggering
|
||||
// ECONNRESET errors if the server closes the connection at the
|
||||
// exact moment clients attempt to reuse an established connection
|
||||
// for a new request.
|
||||
//
|
||||
// Note: the ability to close inactive connections on the client
|
||||
// after httpClientFreeSocketsTimeout milliseconds requires the
|
||||
// use of "agentkeepalive" module instead of the regular node.js
|
||||
// http.Agent.
|
||||
export const httpServerKeepAliveTimeout = 60000;
|
||||
export const httpClientFreeSocketTimeout = 55000;
|
||||
export const supportedLifecycleRules = [
|
||||
'expiration',
|
||||
'noncurrentVersionExpiration',
|
||||
'abortIncompleteMultipartUpload',
|
||||
'transitions',
|
||||
'noncurrentVersionTransition',
|
||||
];
|
||||
// Maximum number of buckets to cache (bucket metadata)
|
||||
export const maxCachedBuckets = process.env.METADATA_MAX_CACHED_BUCKETS ?
|
||||
Number(process.env.METADATA_MAX_CACHED_BUCKETS) : 1000;
|
||||
|
||||
export const validRestoreObjectTiers = new Set(['Expedited', 'Standard', 'Bulk']);
|
||||
export const maxBatchingConcurrentOperations = 5;
|
||||
|
||||
/** For policy resource arn check we allow empty account ID to not break compatibility */
|
||||
export const policyArnAllowedEmptyAccountId = ['utapi', 'scuba'];
|
|
@ -1,3 +1,7 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const writeOptions = { sync: true };
|
||||
|
||||
/**
|
||||
* Like Error, but with a property set to true.
|
||||
* TODO: this is copied from kineticlib, should consolidate with the
|
||||
|
@ -10,36 +14,29 @@
|
|||
* use:
|
||||
* throw propError("badTypeInput", "input is not a buffer");
|
||||
*
|
||||
* @param propName - the property name.
|
||||
* @param message - the Error message.
|
||||
* @returns the Error object.
|
||||
* @param {String} propName - the property name.
|
||||
* @param {String} message - the Error message.
|
||||
* @returns {Error} the Error object.
|
||||
*/
|
||||
function propError(propName: string, message: string): Error {
|
||||
function propError(propName, message) {
|
||||
const err = new Error(message);
|
||||
err[propName] = true;
|
||||
// @ts-ignore
|
||||
err.is = { [propName]: true };
|
||||
return err;
|
||||
}
|
||||
|
||||
/**
|
||||
* Running transaction with multiple updates to be committed atomically
|
||||
*/
|
||||
export class IndexTransaction {
|
||||
operations: { type: 'put' | 'del'; key: string; value?: any }[];
|
||||
db: any;
|
||||
closed: boolean;
|
||||
conditions: { [key: string]: string }[];
|
||||
|
||||
class IndexTransaction {
|
||||
/**
|
||||
* Builds a new transaction
|
||||
*
|
||||
* @argument {Leveldb} db an open database to which the updates
|
||||
* will be applied
|
||||
*
|
||||
* @returns a new empty transaction
|
||||
* @returns {IndexTransaction} a new empty transaction
|
||||
*/
|
||||
constructor(db: any) {
|
||||
constructor(db) {
|
||||
this.operations = [];
|
||||
this.db = db;
|
||||
this.closed = false;
|
||||
|
@ -49,34 +46,30 @@ export class IndexTransaction {
|
|||
/**
|
||||
* Adds a new operation to participate in this running transaction
|
||||
*
|
||||
* @argument op an object with the following attributes:
|
||||
* @argument {object} op an object with the following attributes:
|
||||
* {
|
||||
* type: 'put' or 'del',
|
||||
* key: the object key,
|
||||
* value: (optional for del) the value to store,
|
||||
* }
|
||||
*
|
||||
* @throws an error described by the following properties
|
||||
* @throws {Error} an error described by the following properties
|
||||
* - invalidTransactionVerb if op is not put or del
|
||||
* - pushOnCommittedTransaction if already committed
|
||||
* - missingKey if the key is missing from the op
|
||||
* - missingValue if putting without a value
|
||||
*
|
||||
* @returns {undefined}
|
||||
*/
|
||||
push(op: { type: 'put'; key: string; value: any }): void;
|
||||
push(op: { type: 'del'; key: string }): void;
|
||||
push(op: { type: 'put' | 'del'; key: string; value?: any }): void {
|
||||
push(op) {
|
||||
if (this.closed) {
|
||||
throw propError(
|
||||
'pushOnCommittedTransaction',
|
||||
'can not add ops to already committed transaction'
|
||||
);
|
||||
throw propError('pushOnCommittedTransaction',
|
||||
'can not add ops to already committed transaction');
|
||||
}
|
||||
|
||||
if (op.type !== 'put' && op.type !== 'del') {
|
||||
throw propError(
|
||||
'invalidTransactionVerb',
|
||||
`unknown action type: ${op.type}`
|
||||
);
|
||||
throw propError('invalidTransactionVerb',
|
||||
`unknown action type: ${op.type}`);
|
||||
}
|
||||
|
||||
if (op.key === undefined) {
|
||||
|
@ -100,59 +93,57 @@ export class IndexTransaction {
|
|||
* - pushOnCommittedTransaction if already committed
|
||||
* - missingKey if the key is missing from the op
|
||||
* - missingValue if putting without a value
|
||||
*
|
||||
* @returns {undefined}
|
||||
*
|
||||
* @see push
|
||||
*/
|
||||
put(key: string, value: any) {
|
||||
put(key, value) {
|
||||
this.push({ type: 'put', key, value });
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a new del operation to this running transaction
|
||||
*
|
||||
* @argument key - the key of the object to delete
|
||||
* @argument {string} key - the key of the object to delete
|
||||
*
|
||||
* @throws an error described by the following properties
|
||||
* @throws {Error} an error described by the following properties
|
||||
* - pushOnCommittedTransaction if already committed
|
||||
* - missingKey if the key is missing from the op
|
||||
*
|
||||
* @returns {undefined}
|
||||
*
|
||||
* @see push
|
||||
*/
|
||||
del(key: string) {
|
||||
del(key) {
|
||||
this.push({ type: 'del', key });
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a condition for the transaction
|
||||
*
|
||||
* @argument condition an object with the following attributes:
|
||||
* @argument {object} condition an object with the following attributes:
|
||||
* {
|
||||
* <condition>: the object key
|
||||
* }
|
||||
* example: { notExists: 'key1' }
|
||||
*
|
||||
* @throws an error described by the following properties
|
||||
* @throws {Error} an error described by the following properties
|
||||
* - pushOnCommittedTransaction if already committed
|
||||
* - missingCondition if the condition is empty
|
||||
*
|
||||
* @returns {undefined}
|
||||
*/
|
||||
addCondition(condition: { [key: string]: string }) {
|
||||
addCondition(condition) {
|
||||
if (this.closed) {
|
||||
throw propError(
|
||||
'pushOnCommittedTransaction',
|
||||
'can not add conditions to already committed transaction'
|
||||
);
|
||||
throw propError('pushOnCommittedTransaction',
|
||||
'can not add conditions to already committed transaction');
|
||||
}
|
||||
if (condition === undefined || Object.keys(condition).length === 0) {
|
||||
throw propError(
|
||||
'missingCondition',
|
||||
'missing condition for conditional put'
|
||||
);
|
||||
throw propError('missingCondition', 'missing condition for conditional put');
|
||||
}
|
||||
if (typeof condition.notExists !== 'string' && typeof condition.exists !== 'string') {
|
||||
throw propError(
|
||||
'unsupportedConditionalOperation',
|
||||
'missing key or supported condition'
|
||||
);
|
||||
if (typeof (condition.notExists) !== 'string') {
|
||||
throw propError('unsupportedConditionalOperation', 'missing key or supported condition');
|
||||
}
|
||||
this.conditions.push(condition);
|
||||
}
|
||||
|
@ -160,35 +151,32 @@ export class IndexTransaction {
|
|||
/**
|
||||
* Applies the queued updates in this transaction atomically.
|
||||
*
|
||||
* @argument cb function to be called when the commit
|
||||
* @argument {function} cb function to be called when the commit
|
||||
* finishes, taking an optional error argument
|
||||
*
|
||||
* @returns {undefined}
|
||||
*/
|
||||
commit(cb: (error: Error | null, data?: any) => void) {
|
||||
commit(cb) {
|
||||
if (this.closed) {
|
||||
return cb(
|
||||
propError(
|
||||
'alreadyCommitted',
|
||||
'transaction was already committed'
|
||||
)
|
||||
);
|
||||
return cb(propError('alreadyCommitted',
|
||||
'transaction was already committed'));
|
||||
}
|
||||
|
||||
if (this.operations.length === 0) {
|
||||
return cb(
|
||||
propError(
|
||||
'emptyTransaction',
|
||||
'tried to commit an empty transaction'
|
||||
)
|
||||
);
|
||||
return cb(propError('emptyTransaction',
|
||||
'tried to commit an empty transaction'));
|
||||
}
|
||||
|
||||
this.closed = true;
|
||||
const options = { sync: true, conditions: this.conditions };
|
||||
writeOptions.conditions = this.conditions;
|
||||
|
||||
// The array-of-operations variant of the `batch` method
|
||||
// allows passing options such has `sync: true` whereas the
|
||||
// chained form does not.
|
||||
return this.db.batch(this.operations, options, cb);
|
||||
return this.db.batch(this.operations, writeOptions, cb);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
IndexTransaction,
|
||||
};
|
|
@ -0,0 +1,13 @@
|
|||
function reshapeExceptionError(error) {
|
||||
const { message, code, stack, name } = error;
|
||||
return {
|
||||
message,
|
||||
code,
|
||||
stack,
|
||||
name,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
reshapeExceptionError,
|
||||
};
|
|
@ -1,11 +0,0 @@
|
|||
export interface ErrorLike {
|
||||
message: any;
|
||||
code: any;
|
||||
stack: any;
|
||||
name: any;
|
||||
}
|
||||
|
||||
export function reshapeExceptionError(error: ErrorLike) {
|
||||
const { message, code, stack, name } = error;
|
||||
return { message, code, stack, name };
|
||||
}
|
|
@ -42,7 +42,7 @@ export const BucketAlreadyOwnedByYou: ErrorFormat = {
|
|||
code: 409,
|
||||
|
||||
description:
|
||||
'A bucket with this name exists and is already owned by you',
|
||||
'Your previous request to create the named bucket succeeded and you already own it. You get this error in all AWS regions except US Standard, us-east-1. In us-east-1 region, you will get 200 OK, but it is no-op (if bucket exists S3 will not do anything).',
|
||||
};
|
||||
|
||||
export const BucketNotEmpty: ErrorFormat = {
|
||||
|
@ -365,11 +365,6 @@ export const NoSuchWebsiteConfiguration: ErrorFormat = {
|
|||
description: 'The specified bucket does not have a website configuration',
|
||||
};
|
||||
|
||||
export const NoSuchTagSet: ErrorFormat = {
|
||||
code: 404,
|
||||
description: 'The TagSet does not exist',
|
||||
};
|
||||
|
||||
export const NoSuchUpload: ErrorFormat = {
|
||||
code: 404,
|
||||
description:
|
||||
|
@ -690,11 +685,6 @@ export const ReportNotPresent: ErrorFormat = {
|
|||
'The request was rejected because the credential report does not exist. To generate a credential report, use GenerateCredentialReport.',
|
||||
};
|
||||
|
||||
export const Found: ErrorFormat = {
|
||||
code: 302,
|
||||
description: 'Resource Found'
|
||||
};
|
||||
|
||||
// ------------- Special non-AWS S3 errors -------------
|
||||
|
||||
export const MPUinProgress: ErrorFormat = {
|
||||
|
@ -1042,15 +1032,3 @@ export const AuthMethodNotImplemented: ErrorFormat = {
|
|||
description: 'AuthMethodNotImplemented',
|
||||
code: 501,
|
||||
};
|
||||
|
||||
// --------------------- quotaErros ---------------------
|
||||
|
||||
export const NoSuchQuota: ErrorFormat = {
|
||||
code: 404,
|
||||
description: 'The specified resource does not have a quota.',
|
||||
};
|
||||
|
||||
export const QuotaExceeded: ErrorFormat = {
|
||||
code: 429,
|
||||
description: 'The quota set for the resource is exceeded.',
|
||||
};
|
||||
|
|
|
@ -2,7 +2,7 @@ import type { ServerResponse } from 'http';
|
|||
import * as rawErrors from './arsenalErrors';
|
||||
|
||||
/** All possible errors names. */
|
||||
export type Name = keyof typeof rawErrors;
|
||||
export type Name = keyof typeof rawErrors
|
||||
/** Object containing all errors names. It has the format { [Name]: "Name" } */
|
||||
export type Names = { [Name_ in Name]: Name_ };
|
||||
/** Mapping used to determine an error type. It has the format { [Name]: boolean } */
|
||||
|
@ -13,16 +13,9 @@ export type Errors = { [_ in Name]: ArsenalError };
|
|||
// This object is reused constantly through createIs, we store it there
|
||||
// to avoid recomputation.
|
||||
const isBase = Object.fromEntries(
|
||||
Object.keys(rawErrors).map((key) => [key, false])
|
||||
Object.keys(rawErrors).map(key => [key, false])
|
||||
) as Is;
|
||||
|
||||
// This allows to conditionally add the old behavior of errors to properly
|
||||
// test migration.
|
||||
// Activate CI tests with `ALLOW_UNSAFE_ERROR_COMPARISON=false yarn test`.
|
||||
// Remove this mechanism in ARSN-176.
|
||||
export const allowUnsafeErrComp = (
|
||||
process.env.ALLOW_UNSAFE_ERROR_COMPARISON ?? 'true') === 'true'
|
||||
|
||||
// This contains some metaprog. Be careful.
|
||||
// Proxy can be found on MDN.
|
||||
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Proxy
|
||||
|
@ -32,7 +25,7 @@ export const allowUnsafeErrComp = (
|
|||
// the Proxy will return false.
|
||||
const createIs = (type: Name): Is => {
|
||||
const get = (is: Is, value: string | symbol) => is[value] ?? false;
|
||||
const final = Object.freeze({ ...isBase, [type]: true });
|
||||
const final = Object.freeze({ ...isBase, [type]: true })
|
||||
return new Proxy(final, { get });
|
||||
};
|
||||
|
||||
|
@ -46,27 +39,13 @@ export class ArsenalError extends Error {
|
|||
/** Object used to determine the error type.
|
||||
* Example: error.is.InternalError */
|
||||
#is: Is;
|
||||
/** A map of error metadata (can be extra fields
|
||||
* that only show in debug mode) */
|
||||
#metadata: Map<string, Object[]>;
|
||||
|
||||
private constructor(type: Name, code: number, description: string,
|
||||
metadata?: Map<string, Object[]>) {
|
||||
private constructor(type: Name, code: number, description: string) {
|
||||
super(type);
|
||||
this.#code = code;
|
||||
this.#description = description;
|
||||
this.#type = type;
|
||||
this.#is = createIs(type);
|
||||
this.#metadata = metadata ?? new Map<string, Object[]>();
|
||||
|
||||
// This restores the old behavior of errors, to make sure they're now
|
||||
// backward-compatible. Fortunately it's handled by TS, but it cannot
|
||||
// be type-checked. This means we have to be extremely careful about
|
||||
// what we're doing when using errors.
|
||||
// Disables the feature when in CI tests but not in production.
|
||||
if (allowUnsafeErrComp) {
|
||||
this[type] = true;
|
||||
}
|
||||
}
|
||||
|
||||
/** Output the error as a JSON string */
|
||||
|
@ -76,30 +55,6 @@ export class ArsenalError extends Error {
|
|||
return JSON.stringify({ errorType, errorMessage });
|
||||
}
|
||||
|
||||
flatten() {
|
||||
return {
|
||||
is_arsenal_error: true,
|
||||
code: this.#code,
|
||||
description: this.#description,
|
||||
type: this.#type,
|
||||
stack: this.stack
|
||||
}
|
||||
}
|
||||
|
||||
static unflatten(flat_obj) {
|
||||
if (!flat_obj.is_arsenal_error) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const err = new ArsenalError(
|
||||
flat_obj.type,
|
||||
flat_obj.code,
|
||||
flat_obj.description
|
||||
)
|
||||
err.stack = flat_obj.stack
|
||||
return err;
|
||||
}
|
||||
|
||||
/** Write the error in an HTTP response */
|
||||
writeResponse(res: ServerResponse) {
|
||||
res.writeHead(this.#code);
|
||||
|
@ -111,22 +66,7 @@ export class ArsenalError extends Error {
|
|||
customizeDescription(description: string): ArsenalError {
|
||||
const type = this.#type;
|
||||
const code = this.#code;
|
||||
const metadata = new Map(this.#metadata);
|
||||
const err = new ArsenalError(type, code, description, metadata);
|
||||
err.stack = this.stack;
|
||||
return err;
|
||||
}
|
||||
|
||||
/** Clone the error with a new metadata field */
|
||||
addMetadataEntry(key: string, value: Object[]): ArsenalError {
|
||||
const type = this.#type;
|
||||
const code = this.#code;
|
||||
const description = this.#description;
|
||||
const metadata = new Map(this.#metadata);
|
||||
metadata.set(key, value);
|
||||
const err = new ArsenalError(type, code, description, metadata);
|
||||
err.stack = this.stack;
|
||||
return err;
|
||||
return new ArsenalError(type, code, description);
|
||||
}
|
||||
|
||||
/** Used to determine the error type. Example: error.is.InternalError */
|
||||
|
@ -151,14 +91,9 @@ export class ArsenalError extends Error {
|
|||
return this.#type;
|
||||
}
|
||||
|
||||
/** A map of error metadata */
|
||||
get metadata() {
|
||||
return this.#metadata;
|
||||
}
|
||||
|
||||
/** Generate all possible errors. An instance is created by default. */
|
||||
static errors() {
|
||||
const errors = {};
|
||||
const errors = {}
|
||||
Object.entries(rawErrors).forEach((value) => {
|
||||
const name = value[0] as Name;
|
||||
const error = value[1];
|
||||
|
@ -166,7 +101,7 @@ export class ArsenalError extends Error {
|
|||
const get = () => new ArsenalError(name, code, description);
|
||||
Object.defineProperty(errors, name, { get });
|
||||
});
|
||||
return errors as Errors;
|
||||
return errors as Errors
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -7,8 +7,8 @@
|
|||
"test": "mocha --recursive --timeout 5500 tests/unit"
|
||||
},
|
||||
"dependencies": {
|
||||
"mocha": "5.2.0",
|
||||
"async": "~2.6.1",
|
||||
"mocha": "2.5.3",
|
||||
"async": "^2.6.0",
|
||||
"node-forge": "^0.7.1"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -17,9 +17,9 @@ describe('decyrptSecret', () => {
|
|||
describe('parseServiceCredentials', () => {
|
||||
const conf = {
|
||||
users: [{ accessKey,
|
||||
accountType: 'service-clueso',
|
||||
secretKey,
|
||||
userName: 'Search Service Account' }],
|
||||
accountType: 'service-clueso',
|
||||
secretKey,
|
||||
userName: 'Search Service Account' }],
|
||||
};
|
||||
const auth = JSON.stringify({ privateKey });
|
||||
|
||||
|
|
|
@ -1,4 +1,6 @@
|
|||
export const ciphers = [
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const ciphers = [
|
||||
'DHE-RSA-AES128-GCM-SHA256',
|
||||
'ECDHE-ECDSA-AES128-GCM-SHA256',
|
||||
'ECDHE-RSA-AES256-GCM-SHA384',
|
||||
|
@ -26,3 +28,7 @@ export const ciphers = [
|
|||
'!EDH-RSA-DES-CBC3-SHA',
|
||||
'!KRB5-DES-CBC3-SHA',
|
||||
].join(':');
|
||||
|
||||
module.exports = {
|
||||
ciphers,
|
||||
};
|
|
@ -29,11 +29,16 @@ c2CNfUEqyRbJF4pE9ZcdQReT5p/llmyhQdvq6cHH+cKJk63C6DHRVoStsnsUcvKe
|
|||
bLxKsygK77ttjr61cxLoDJeGd5L5h1CPmwIBAg==
|
||||
-----END DH PARAMETERS-----
|
||||
*/
|
||||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
export const dhparam =
|
||||
const dhparam =
|
||||
'MIIBCAKCAQEAh99T77KGNuiY9N6xrCJ3QNv4SFADTa3CD+1VMTAdRJLHUNpglB+i' +
|
||||
'AoTYiLDFZgtTCpx0ZZUD+JM3qiCZy0OK5/ZGlVD7sZmxjRtdpVK4qIPtwav8t0J7' +
|
||||
'c2CNfUEqyRbJF4pE9ZcdQReT5p/llmyhQdvq6cHH+cKJk63C6DHRVoStsnsUcvKe' +
|
||||
'23PLGZulKg8H3eRBxHamHkmyuEVDtoNhMIoJONsdXSpo5GgcD4EQMM8xb/qsnCxn' +
|
||||
'6QIGTBvcHskxtlTZOfUPk4XQ6Yb3tQi2TurzkQHLln4U7p/GZs+D+6D3SgSPqr6P' +
|
||||
'bLxKsygK77ttjr61cxLoDJeGd5L5h1CPmwIBAg==';
|
||||
|
||||
module.exports = {
|
||||
dhparam,
|
||||
};
|
|
@ -1,2 +0,0 @@
|
|||
export * as ciphers from './ciphers'
|
||||
export * as dhparam from './dh2048'
|
|
@ -0,0 +1,83 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const ipaddr = require('ipaddr.js');
|
||||
|
||||
/**
|
||||
* checkIPinRangeOrMatch checks whether a given ip address is in an ip address
|
||||
* range or matches the given ip address
|
||||
* @param {string} cidr - ip address range or ip address
|
||||
* @param {object} ip - parsed ip address
|
||||
* @return {boolean} true if in range, false if not
|
||||
*/
|
||||
function checkIPinRangeOrMatch(cidr, ip) {
|
||||
// If there is an exact match of the ip address, no need to check ranges
|
||||
if (ip.toString() === cidr) {
|
||||
return true;
|
||||
}
|
||||
let range;
|
||||
|
||||
try {
|
||||
range = ipaddr.IPv4.parseCIDR(cidr);
|
||||
} catch (err) {
|
||||
try {
|
||||
// not ipv4 so try ipv6
|
||||
range = ipaddr.IPv6.parseCIDR(cidr);
|
||||
} catch (err) {
|
||||
// range is not valid ipv4 or ipv6
|
||||
return false;
|
||||
}
|
||||
}
|
||||
try {
|
||||
return ip.match(range);
|
||||
} catch (err) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse IP address into object representation
|
||||
* @param {string} ip - IPV4/IPV6/IPV4-mapped IPV6 address
|
||||
* @return {object} parsedIp - Object representation of parsed IP
|
||||
*/
|
||||
function parseIp(ip) {
|
||||
if (ipaddr.IPv4.isValid(ip)) {
|
||||
return ipaddr.parse(ip);
|
||||
}
|
||||
if (ipaddr.IPv6.isValid(ip)) {
|
||||
// also parses IPv6 mapped IPv4 addresses into IPv4 representation
|
||||
return ipaddr.process(ip);
|
||||
}
|
||||
// not valid ip address according to module, so return empty object
|
||||
// which will obviously not match a range of ip addresses that the parsedIp
|
||||
// is being tested against
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Checks if an IP adress matches a given list of CIDR ranges
|
||||
* @param {string[]} cidrList - List of CIDR ranges
|
||||
* @param {string} ip - IP address
|
||||
* @return {boolean} - true if there is match or false for no match
|
||||
*/
|
||||
function ipMatchCidrList(cidrList, ip) {
|
||||
const parsedIp = parseIp(ip);
|
||||
return cidrList.some(item => {
|
||||
let cidr;
|
||||
// patch the cidr if range is not specified
|
||||
if (item.indexOf('/') === -1) {
|
||||
if (item.startsWith('127.')) {
|
||||
cidr = `${item}/8`;
|
||||
} else if (ipaddr.IPv4.isValid(item)) {
|
||||
cidr = `${item}/32`;
|
||||
}
|
||||
}
|
||||
return checkIPinRangeOrMatch(cidr || item, parsedIp);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
checkIPinRangeOrMatch,
|
||||
ipMatchCidrList,
|
||||
parseIp,
|
||||
};
|
|
@ -1,71 +0,0 @@
|
|||
import ipaddr from 'ipaddr.js';
|
||||
|
||||
/**
|
||||
* checkIPinRangeOrMatch checks whether a given ip address is in an ip address
|
||||
* range or matches the given ip address
|
||||
* @param cidr - ip address range or ip address
|
||||
* @param ip - parsed ip address
|
||||
* @return true if in range, false if not
|
||||
*/
|
||||
export function checkIPinRangeOrMatch(
|
||||
cidr: string,
|
||||
ip: ipaddr.IPv4 | ipaddr.IPv6,
|
||||
): boolean {
|
||||
// If there is an exact match of the ip address, no need to check ranges
|
||||
if (ip.toString() === cidr) {
|
||||
return true;
|
||||
}
|
||||
try {
|
||||
if (ip instanceof ipaddr.IPv6) {
|
||||
const range = ipaddr.IPv6.parseCIDR(cidr);
|
||||
return ip.match(range);
|
||||
} else {
|
||||
const range = ipaddr.IPv4.parseCIDR(cidr);
|
||||
return ip.match(range);
|
||||
}
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse IP address into object representation
|
||||
* @param ip - IPV4/IPV6/IPV4-mapped IPV6 address
|
||||
* @return parsedIp - Object representation of parsed IP
|
||||
*/
|
||||
export function parseIp(ip: string): ipaddr.IPv4 | ipaddr.IPv6 | {} {
|
||||
if (ipaddr.IPv4.isValid(ip)) {
|
||||
return ipaddr.parse(ip);
|
||||
}
|
||||
if (ipaddr.IPv6.isValid(ip)) {
|
||||
// also parses IPv6 mapped IPv4 addresses into IPv4 representation
|
||||
return ipaddr.process(ip);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if an IP adress matches a given list of CIDR ranges
|
||||
* @param cidrList - List of CIDR ranges
|
||||
* @param ip - IP address
|
||||
* @return - true if there is match or false for no match
|
||||
*/
|
||||
export function ipMatchCidrList(cidrList: string[], ip: string): boolean {
|
||||
const parsedIp = parseIp(ip);
|
||||
return cidrList.some((item) => {
|
||||
let cidr: string | undefined;
|
||||
// patch the cidr if range is not specified
|
||||
if (item.indexOf('/') === -1) {
|
||||
if (item.startsWith('127.')) {
|
||||
cidr = `${item}/8`;
|
||||
} else if (ipaddr.IPv4.isValid(item)) {
|
||||
cidr = `${item}/32`;
|
||||
}
|
||||
}
|
||||
return (
|
||||
(parsedIp instanceof ipaddr.IPv4 ||
|
||||
parsedIp instanceof ipaddr.IPv6) &&
|
||||
checkIPinRangeOrMatch(cidr || item, parsedIp)
|
||||
);
|
||||
});
|
||||
}
|
|
@ -0,0 +1,32 @@
|
|||
'use strict'; // eslint-disable-line
|
||||
|
||||
const debug = require('util').debuglog('jsutil');
|
||||
|
||||
// JavaScript utility functions
|
||||
|
||||
/**
|
||||
* force <tt>func</tt> to be called only once, even if actually called
|
||||
* multiple times. The cached result of the first call is then
|
||||
* returned (if any).
|
||||
*
|
||||
* @note underscore.js provides this functionality but not worth
|
||||
* adding a new dependency for such a small use case.
|
||||
*
|
||||
* @param {function} func function to call at most once
|
||||
|
||||
* @return {function} a callable wrapper mirroring <tt>func</tt> but
|
||||
* only calls <tt>func</tt> at first invocation.
|
||||
*/
|
||||
module.exports.once = function once(func) {
|
||||
const state = { called: false, res: undefined };
|
||||
return function wrapper(...args) {
|
||||
if (!state.called) {
|
||||
state.called = true;
|
||||
state.res = func.apply(func, args);
|
||||
} else {
|
||||
debug('function already called:', func,
|
||||
'returning cached result:', state.res);
|
||||
}
|
||||
return state.res;
|
||||
};
|
||||
};
|
|
@ -1,33 +0,0 @@
|
|||
import * as util from 'util';
|
||||
const debug = util.debuglog('jsutil');
|
||||
|
||||
// JavaScript utility functions
|
||||
|
||||
/**
|
||||
* force <tt>func</tt> to be called only once, even if actually called
|
||||
* multiple times. The cached result of the first call is then
|
||||
* returned (if any).
|
||||
*
|
||||
* @note underscore.js provides this functionality but not worth
|
||||
* adding a new dependency for such a small use case.
|
||||
*
|
||||
* @param func function to call at most once
|
||||
|
||||
* @return a callable wrapper mirroring <tt>func</tt> but
|
||||
* only calls <tt>func</tt> at first invocation.
|
||||
*/
|
||||
export function once<T>(func: (...args: any[]) => T): (...args: any[]) => T {
|
||||
type State = { called: boolean; res: any };
|
||||
const state: State = { called: false, res: undefined };
|
||||
return function wrapper(...args: any[]) {
|
||||
if (!state.called) {
|
||||
state.called = true;
|
||||
state.res = func.apply(func, args);
|
||||
} else {
|
||||
const m1 = 'function already called:';
|
||||
const m2 = 'returning cached result:';
|
||||
debug(m1, func, m2, state.res);
|
||||
}
|
||||
return state.res;
|
||||
};
|
||||
}
|
|
@ -0,0 +1,162 @@
|
|||
const Redis = require('ioredis');
|
||||
|
||||
class RedisClient {
|
||||
/**
|
||||
* @constructor
|
||||
* @param {Object} config - config
|
||||
* @param {string} config.host - Redis host
|
||||
* @param {number} config.port - Redis port
|
||||
* @param {string} config.password - Redis password
|
||||
* @param {werelogs.Logger} logger - logger instance
|
||||
*/
|
||||
constructor(config, logger) {
|
||||
this._client = new Redis(config);
|
||||
this._client.on('error', err =>
|
||||
logger.trace('error from redis', {
|
||||
error: err,
|
||||
method: 'RedisClient.constructor',
|
||||
redisHost: config.host,
|
||||
redisPort: config.port,
|
||||
})
|
||||
);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* increment value of a key by 1 and set a ttl
|
||||
* @param {string} key - key holding the value
|
||||
* @param {number} expiry - expiry in seconds
|
||||
* @param {callback} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
incrEx(key, expiry, cb) {
|
||||
return this._client
|
||||
.multi([['incr', key], ['expire', key, expiry]])
|
||||
.exec(cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* increment value of a key by a given amount and set a ttl
|
||||
* @param {string} key - key holding the value
|
||||
* @param {number} amount - amount to increase by
|
||||
* @param {number} expiry - expiry in seconds
|
||||
* @param {callback} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
incrbyEx(key, amount, expiry, cb) {
|
||||
return this._client
|
||||
.multi([['incrby', key, amount], ['expire', key, expiry]])
|
||||
.exec(cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* execute a batch of commands
|
||||
* @param {string[]} cmds - list of commands
|
||||
* @param {callback} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
batch(cmds, cb) {
|
||||
return this._client.pipeline(cmds).exec(cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a key exists
|
||||
* @param {string} key - name of key
|
||||
* @param {function} cb - callback
|
||||
* If cb response returns 0, key does not exist.
|
||||
* If cb response returns 1, key exists.
|
||||
* @return {undefined}
|
||||
*/
|
||||
exists(key, cb) {
|
||||
return this._client.exists(key, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a value and its score to a sorted set. If no sorted set exists, this
|
||||
* will create a new one for the given key.
|
||||
* @param {string} key - name of key
|
||||
* @param {integer} score - score used to order set
|
||||
* @param {string} value - value to store
|
||||
* @param {callback} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
zadd(key, score, value, cb) {
|
||||
return this._client.zadd(key, score, value, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get number of elements in a sorted set.
|
||||
* Note: using this on a key that does not exist will return 0.
|
||||
* Note: using this on an existing key that isn't a sorted set will
|
||||
* return an error WRONGTYPE.
|
||||
* @param {string} key - name of key
|
||||
* @param {function} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
zcard(key, cb) {
|
||||
return this._client.zcard(key, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the score for given value in a sorted set
|
||||
* Note: using this on a key that does not exist will return nil.
|
||||
* Note: using this on a value that does not exist in a valid sorted set key
|
||||
* will return nil.
|
||||
* @param {string} key - name of key
|
||||
* @param {string} value - value within sorted set
|
||||
* @param {function} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
zscore(key, value, cb) {
|
||||
return this._client.zscore(key, value, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a value from a sorted set
|
||||
* @param {string} key - name of key
|
||||
* @param {string|array} value - value within sorted set. Can specify
|
||||
* multiple values within an array
|
||||
* @param {function} cb - callback
|
||||
* The cb response returns number of values removed
|
||||
* @return {undefined}
|
||||
*/
|
||||
zrem(key, value, cb) {
|
||||
return this._client.zrem(key, value, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get specified range of elements in a sorted set
|
||||
* @param {string} key - name of key
|
||||
* @param {integer} start - start index (inclusive)
|
||||
* @param {integer} end - end index (inclusive) (can use -1)
|
||||
* @param {function} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
zrange(key, start, end, cb) {
|
||||
return this._client.zrange(key, start, end, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get range of elements in a sorted set based off score
|
||||
* @param {string} key - name of key
|
||||
* @param {integer|string} min - min score value (inclusive)
|
||||
* (can use "-inf")
|
||||
* @param {integer|string} max - max score value (inclusive)
|
||||
* (can use "+inf")
|
||||
* @param {function} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
zrangebyscore(key, min, max, cb) {
|
||||
return this._client.zrangebyscore(key, min, max, cb);
|
||||
}
|
||||
|
||||
clear(cb) {
|
||||
return this._client.flushdb(cb);
|
||||
}
|
||||
|
||||
disconnect() {
|
||||
this._client.disconnect();
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = RedisClient;
|
|
@ -1,218 +0,0 @@
|
|||
import Redis from 'ioredis';
|
||||
import { Logger } from 'werelogs';
|
||||
|
||||
export type Config = { host: string; port: number; password: string };
|
||||
export type Callback = (error: Error | null, value?: any) => void;
|
||||
|
||||
export default class RedisClient {
|
||||
_client: Redis.Redis;
|
||||
|
||||
constructor(config: Config, logger: Logger) {
|
||||
this._client = new Redis(config);
|
||||
this._client.on('error', err =>
|
||||
logger.trace('error from redis', {
|
||||
error: err,
|
||||
method: 'RedisClient.constructor',
|
||||
redisHost: config.host,
|
||||
redisPort: config.port,
|
||||
})
|
||||
);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* scan a pattern and return matching keys
|
||||
* @param pattern - string pattern to match with all existing keys
|
||||
* @param [count=10] - scan count
|
||||
* @param cb - callback (error, result)
|
||||
*/
|
||||
scan(pattern: string, count = 10, cb: Callback) {
|
||||
const params = { match: pattern, count };
|
||||
const keys: any[] = [];
|
||||
|
||||
const stream = this._client.scanStream(params);
|
||||
stream.on('data', resultKeys => {
|
||||
for (let i = 0; i < resultKeys.length; i++) {
|
||||
keys.push(resultKeys[i]);
|
||||
}
|
||||
});
|
||||
stream.on('end', () => {
|
||||
cb(null, keys);
|
||||
});
|
||||
}
|
||||
|
||||
/** increment value of a key by 1 and set a ttl
|
||||
* @param key - key holding the value
|
||||
* @param expiry - expiry in seconds
|
||||
* @param cb - callback
|
||||
*/
|
||||
incrEx(key: string, expiry: number, cb: Callback) {
|
||||
const exp = expiry.toString();
|
||||
return this._client
|
||||
.multi([['incr', key], ['expire', key, exp]])
|
||||
.exec(cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* increment value of a key by a given amount
|
||||
* @param key - key holding the value
|
||||
* @param amount - amount to increase by
|
||||
* @param cb - callback
|
||||
*/
|
||||
incrby(key: string, amount: number, cb: Callback) {
|
||||
return this._client.incrby(key, amount, cb);
|
||||
}
|
||||
|
||||
/** increment value of a key by a given amount and set a ttl
|
||||
* @param key - key holding the value
|
||||
* @param amount - amount to increase by
|
||||
* @param expiry - expiry in seconds
|
||||
* @param cb - callback
|
||||
*/
|
||||
incrbyEx(key: string, amount: number, expiry: number, cb: Callback) {
|
||||
const am = amount.toString();
|
||||
const exp = expiry.toString();
|
||||
return this._client
|
||||
.multi([['incrby', key, am], ['expire', key, exp]])
|
||||
.exec(cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* decrement value of a key by a given amount
|
||||
* @param key - key holding the value
|
||||
* @param amount - amount to increase by
|
||||
* @param cb - callback
|
||||
*/
|
||||
decrby(key: string, amount: number, cb: Callback) {
|
||||
return this._client.decrby(key, amount, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* execute a batch of commands
|
||||
* @param cmds - list of commands
|
||||
* @param cb - callback
|
||||
* @return
|
||||
*/
|
||||
batch(cmds: string[][], cb: Callback) {
|
||||
return this._client.pipeline(cmds).exec(cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a key exists
|
||||
* @param key - name of key
|
||||
* @param cb - callback
|
||||
* If cb response returns 0, key does not exist.
|
||||
* If cb response returns 1, key exists.
|
||||
*/
|
||||
exists(key: string, cb: Callback) {
|
||||
return this._client.exists(key, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* get value stored at key
|
||||
* @param key - key holding the value
|
||||
* @param cb - callback
|
||||
*/
|
||||
get(key: string, cb: Callback) {
|
||||
return this._client.get(key, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a value and its score to a sorted set. If no sorted set exists, this
|
||||
* will create a new one for the given key.
|
||||
* @param key - name of key
|
||||
* @param score - score used to order set
|
||||
* @param value - value to store
|
||||
* @param cb - callback
|
||||
*/
|
||||
zadd(key: string, score: number, value: string, cb: Callback) {
|
||||
return this._client.zadd(key, score, value, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get number of elements in a sorted set.
|
||||
* Note: using this on a key that does not exist will return 0.
|
||||
* Note: using this on an existing key that isn't a sorted set will
|
||||
* return an error WRONGTYPE.
|
||||
* @param key - name of key
|
||||
* @param cb - callback
|
||||
*/
|
||||
zcard(key: string, cb: Callback) {
|
||||
return this._client.zcard(key, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the score for given value in a sorted set
|
||||
* Note: using this on a key that does not exist will return nil.
|
||||
* Note: using this on a value that does not exist in a valid sorted set key
|
||||
* will return nil.
|
||||
* @param key - name of key
|
||||
* @param value - value within sorted set
|
||||
* @param cb - callback
|
||||
*/
|
||||
zscore(key: string, value: string, cb: Callback) {
|
||||
return this._client.zscore(key, value, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a value from a sorted set
|
||||
* @param key - name of key
|
||||
* @param value - value within sorted set. Can specify
|
||||
* multiple values within an array
|
||||
* @param cb - callback
|
||||
* The cb response returns number of values removed
|
||||
*/
|
||||
zrem(key: string, value: string | string[], cb: Callback) {
|
||||
return this._client.zrem(key, value, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get specified range of elements in a sorted set
|
||||
* @param key - name of key
|
||||
* @param start - start index (inclusive)
|
||||
* @param end - end index (inclusive) (can use -1)
|
||||
* @param cb - callback
|
||||
*/
|
||||
zrange(key: string, start: number, end: number, cb: Callback) {
|
||||
return this._client.zrange(key, start, end, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get range of elements in a sorted set based off score
|
||||
* @param key - name of key
|
||||
* @param min - min score value (inclusive)
|
||||
* (can use "-inf")
|
||||
* @param max - max score value (inclusive)
|
||||
* (can use "+inf")
|
||||
* @param cb - callback
|
||||
*/
|
||||
zrangebyscore(
|
||||
key: string,
|
||||
min: number | string,
|
||||
max: number | string,
|
||||
cb: Callback,
|
||||
) {
|
||||
return this._client.zrangebyscore(key, min, max, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* get TTL or expiration in seconds
|
||||
* @param key - name of key
|
||||
* @param cb - callback
|
||||
*/
|
||||
ttl(key: string, cb: Callback) {
|
||||
return this._client.ttl(key, cb);
|
||||
}
|
||||
|
||||
clear(cb: Callback) {
|
||||
return this._client.flushdb(cb);
|
||||
}
|
||||
|
||||
disconnect() {
|
||||
this._client.disconnect();
|
||||
}
|
||||
|
||||
listClients(cb: Callback) {
|
||||
return this._client.client('list', cb);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,163 @@
|
|||
const async = require('async');
|
||||
|
||||
class StatsClient {
|
||||
/**
|
||||
* @constructor
|
||||
* @param {object} redisClient - RedisClient instance
|
||||
* @param {number} interval - sampling interval by seconds
|
||||
* @param {number} expiry - sampling duration by seconds
|
||||
*/
|
||||
constructor(redisClient, interval, expiry) {
|
||||
this._redis = redisClient;
|
||||
this._interval = interval;
|
||||
this._expiry = expiry;
|
||||
return this;
|
||||
}
|
||||
|
||||
/*
|
||||
* Utility function to use when callback is undefined
|
||||
*/
|
||||
_noop() {}
|
||||
|
||||
/**
|
||||
* normalize to the nearest interval
|
||||
* @param {object} d - Date instance
|
||||
* @return {number} timestamp - normalized to the nearest interval
|
||||
*/
|
||||
_normalizeTimestamp(d) {
|
||||
const s = d.getSeconds();
|
||||
return d.setSeconds(s - s % this._interval, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* set timestamp to the previous interval
|
||||
* @param {object} d - Date instance
|
||||
* @return {number} timestamp - set to the previous interval
|
||||
*/
|
||||
_setPrevInterval(d) {
|
||||
return d.setSeconds(d.getSeconds() - this._interval);
|
||||
}
|
||||
|
||||
/**
|
||||
* build redis key to get total number of occurrences on the server
|
||||
* @param {string} name - key name identifier
|
||||
* @param {object} d - Date instance
|
||||
* @return {string} key - key for redis
|
||||
*/
|
||||
_buildKey(name, d) {
|
||||
return `${name}:${this._normalizeTimestamp(d)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* reduce the array of values to a single value
|
||||
* typical input looks like [[null, '1'], [null, '2'], [null, null]...]
|
||||
* @param {array} arr - Date instance
|
||||
* @return {string} key - key for redis
|
||||
*/
|
||||
_getCount(arr) {
|
||||
return arr.reduce((prev, a) => {
|
||||
let num = parseInt(a[1], 10);
|
||||
num = Number.isNaN(num) ? 0 : num;
|
||||
return prev + num;
|
||||
}, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* report/record a new request received on the server
|
||||
* @param {string} id - service identifier
|
||||
* @param {number} incr - optional param increment
|
||||
* @param {function} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
reportNewRequest(id, incr, cb) {
|
||||
if (!this._redis) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
let callback;
|
||||
let amount;
|
||||
if (typeof incr === 'function') {
|
||||
// In case where optional `incr` is not passed, but `cb` is passed
|
||||
callback = incr;
|
||||
amount = 1;
|
||||
} else {
|
||||
callback = (cb && typeof cb === 'function') ? cb : this._noop;
|
||||
amount = (typeof incr === 'number') ? incr : 1;
|
||||
}
|
||||
|
||||
const key = this._buildKey(`${id}:requests`, new Date());
|
||||
|
||||
return this._redis.incrbyEx(key, amount, this._expiry, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* report/record a request that ended up being a 500 on the server
|
||||
* @param {string} id - service identifier
|
||||
* @param {callback} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
report500(id, cb) {
|
||||
if (!this._redis) {
|
||||
return undefined;
|
||||
}
|
||||
const callback = cb || this._noop;
|
||||
const key = this._buildKey(`${id}:500s`, new Date());
|
||||
return this._redis.incrEx(key, this._expiry, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* get stats for the last x seconds, x being the sampling duration
|
||||
* @param {object} log - Werelogs request logger
|
||||
* @param {string} id - service identifier
|
||||
* @param {callback} cb - callback to call with the err/result
|
||||
* @return {undefined}
|
||||
*/
|
||||
getStats(log, id, cb) {
|
||||
if (!this._redis) {
|
||||
return cb(null, {});
|
||||
}
|
||||
const d = new Date();
|
||||
const totalKeys = Math.floor(this._expiry / this._interval);
|
||||
const reqsKeys = [];
|
||||
const req500sKeys = [];
|
||||
for (let i = 0; i < totalKeys; i++) {
|
||||
reqsKeys.push(['get', this._buildKey(`${id}:requests`, d)]);
|
||||
req500sKeys.push(['get', this._buildKey(`${id}:500s`, d)]);
|
||||
this._setPrevInterval(d);
|
||||
}
|
||||
return async.parallel([
|
||||
next => this._redis.batch(reqsKeys, next),
|
||||
next => this._redis.batch(req500sKeys, next),
|
||||
], (err, results) => {
|
||||
/**
|
||||
* Batch result is of the format
|
||||
* [ [null, '1'], [null, '2'], [null, '3'] ] where each
|
||||
* item is the result of the each batch command
|
||||
* Foreach item in the result, index 0 signifies the error and
|
||||
* index 1 contains the result
|
||||
*/
|
||||
const statsRes = {
|
||||
'requests': 0,
|
||||
'500s': 0,
|
||||
'sampleDuration': this._expiry,
|
||||
};
|
||||
if (err) {
|
||||
log.error('error getting stats', {
|
||||
error: err,
|
||||
method: 'StatsClient.getStats',
|
||||
});
|
||||
/**
|
||||
* Redis for stats is not a critial component, ignoring
|
||||
* any error here as returning an InternalError
|
||||
* would be confused with the health of the service
|
||||
*/
|
||||
return cb(null, statsRes);
|
||||
}
|
||||
statsRes.requests = this._getCount(results[0]);
|
||||
statsRes['500s'] = this._getCount(results[1]);
|
||||
return cb(null, statsRes);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = StatsClient;
|
|
@ -1,230 +0,0 @@
|
|||
import async from 'async';
|
||||
import RedisClient from './RedisClient';
|
||||
import { Logger } from 'werelogs';
|
||||
|
||||
export type Callback = (error: Error | null, value?: any) => void;
|
||||
|
||||
export default class StatsClient {
|
||||
_redis: RedisClient;
|
||||
_interval: number;
|
||||
_expiry: number;
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
* @param redisClient - RedisClient instance
|
||||
* @param interval - sampling interval by seconds
|
||||
* @param expiry - sampling duration by seconds
|
||||
*/
|
||||
constructor(redisClient: RedisClient, interval: number, expiry: number) {
|
||||
this._redis = redisClient;
|
||||
this._interval = interval;
|
||||
this._expiry = expiry;
|
||||
return this;
|
||||
}
|
||||
|
||||
/** Utility function to use when callback is undefined */
|
||||
_noop() {}
|
||||
|
||||
/**
|
||||
* normalize to the nearest interval
|
||||
* @param d - Date instance
|
||||
* @return timestamp - normalized to the nearest interval
|
||||
*/
|
||||
_normalizeTimestamp(d: Date): number {
|
||||
const s = d.getSeconds();
|
||||
return d.setSeconds(s - s % this._interval, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* set timestamp to the previous interval
|
||||
* @param d - Date instance
|
||||
* @return timestamp - set to the previous interval
|
||||
*/
|
||||
_setPrevInterval(d: Date): number {
|
||||
return d.setSeconds(d.getSeconds() - this._interval);
|
||||
}
|
||||
|
||||
/**
|
||||
* build redis key to get total number of occurrences on the server
|
||||
* @param name - key name identifier
|
||||
* @param d - Date instance
|
||||
* @return key - key for redis
|
||||
*/
|
||||
buildKey(name: string, d: Date): string {
|
||||
return `${name}:${this._normalizeTimestamp(d)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* reduce the array of values to a single value
|
||||
* typical input looks like [[null, '1'], [null, '2'], [null, null]...]
|
||||
* @param arr - Date instance
|
||||
* @return key - key for redis
|
||||
*/
|
||||
_getCount(arr: [any, string | null][]): number {
|
||||
return arr.reduce((prev, a) => {
|
||||
let num = parseInt(a[1] ?? '', 10);
|
||||
num = Number.isNaN(num) ? 0 : num;
|
||||
return prev + num;
|
||||
}, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* report/record a new request received on the server
|
||||
* @param id - service identifier
|
||||
* @param incr - optional param increment
|
||||
*/
|
||||
reportNewRequest(
|
||||
id: string,
|
||||
incr?: number | ((error: Error | null, value?: any) => void),
|
||||
cb?: (error: Error | null, value?: any) => void,
|
||||
) {
|
||||
if (!this._redis) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
let callback: (error: Error | null, value?: any) => void;
|
||||
let amount: number;
|
||||
if (typeof incr === 'function') {
|
||||
// In case where optional `incr` is not passed, but `cb` is passed
|
||||
callback = incr;
|
||||
amount = 1;
|
||||
} else {
|
||||
callback = (cb && typeof cb === 'function') ? cb : this._noop;
|
||||
amount = (typeof incr === 'number') ? incr : 1;
|
||||
}
|
||||
|
||||
const key = this.buildKey(`${id}:requests`, new Date());
|
||||
|
||||
return this._redis.incrbyEx(key, amount, this._expiry, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment the given key by the given value.
|
||||
* @param key - The Redis key to increment
|
||||
* @param incr - The value to increment by
|
||||
* @param [cb] - callback
|
||||
*/
|
||||
incrementKey(key: string, incr: number, cb: Callback) {
|
||||
const callback = cb || this._noop;
|
||||
return this._redis.incrby(key, incr, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decrement the given key by the given value.
|
||||
* @param key - The Redis key to decrement
|
||||
* @param decr - The value to decrement by
|
||||
* @param [cb] - callback
|
||||
*/
|
||||
decrementKey(key: string, decr: number, cb: Callback) {
|
||||
const callback = cb || this._noop;
|
||||
return this._redis.decrby(key, decr, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* report/record a request that ended up being a 500 on the server
|
||||
* @param id - service identifier
|
||||
*/
|
||||
report500(id: string, cb?: (error: Error | null, value?: any) => void) {
|
||||
if (!this._redis) {
|
||||
return undefined;
|
||||
}
|
||||
const callback = cb || this._noop;
|
||||
const key = this.buildKey(`${id}:500s`, new Date());
|
||||
return this._redis.incrEx(key, this._expiry, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* wrapper on `getStats` that handles a list of keys
|
||||
* @param log - Werelogs request logger
|
||||
* @param ids - service identifiers
|
||||
* @param cb - callback to call with the err/result
|
||||
*/
|
||||
getAllStats(log: Logger, ids: string[], cb: Callback) {
|
||||
if (!this._redis) {
|
||||
return cb(null, {});
|
||||
}
|
||||
|
||||
const statsRes = {
|
||||
'requests': 0,
|
||||
'500s': 0,
|
||||
'sampleDuration': this._expiry,
|
||||
};
|
||||
let requests = 0;
|
||||
let errors = 0;
|
||||
|
||||
// for now set concurrency to default of 10
|
||||
return async.eachLimit(ids, 10, (id: string, done) => {
|
||||
this.getStats(log, id, (err, res) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
requests += res.requests;
|
||||
errors += res['500s'];
|
||||
return done();
|
||||
});
|
||||
}, error => {
|
||||
if (error) {
|
||||
log.error('error getting stats', {
|
||||
error,
|
||||
method: 'StatsClient.getAllStats',
|
||||
});
|
||||
return cb(null, statsRes);
|
||||
}
|
||||
statsRes.requests = requests;
|
||||
statsRes['500s'] = errors;
|
||||
return cb(null, statsRes);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* get stats for the last x seconds, x being the sampling duration
|
||||
* @param log - Werelogs request logger
|
||||
* @param id - service identifier
|
||||
*/
|
||||
getStats(log: Logger, id: string, cb: (error: Error | null, value?: any) => void) {
|
||||
if (!this._redis) {
|
||||
return cb(null, {});
|
||||
}
|
||||
const d = new Date();
|
||||
const totalKeys = Math.floor(this._expiry / this._interval);
|
||||
const reqsKeys: ['get', string][] = [];
|
||||
const req500sKeys: ['get', string][] = [];
|
||||
for (let i = 0; i < totalKeys; i++) {
|
||||
reqsKeys.push(['get', this.buildKey(`${id}:requests`, d)]);
|
||||
req500sKeys.push(['get', this.buildKey(`${id}:500s`, d)]);
|
||||
this._setPrevInterval(d);
|
||||
}
|
||||
return async.parallel([
|
||||
next => this._redis.batch(reqsKeys, next),
|
||||
next => this._redis.batch(req500sKeys, next),
|
||||
], (err, results) => {
|
||||
/**
|
||||
* Batch result is of the format
|
||||
* [ [null, '1'], [null, '2'], [null, '3'] ] where each
|
||||
* item is the result of the each batch command
|
||||
* Foreach item in the result, index 0 signifies the error and
|
||||
* index 1 contains the result
|
||||
*/
|
||||
const statsRes = {
|
||||
'requests': 0,
|
||||
'500s': 0,
|
||||
'sampleDuration': this._expiry,
|
||||
};
|
||||
if (err) {
|
||||
log.error('error getting stats', {
|
||||
error: err,
|
||||
method: 'StatsClient.getStats',
|
||||
});
|
||||
/**
|
||||
* Redis for stats is not a critial component, ignoring
|
||||
* any error here as returning an InternalError
|
||||
* would be confused with the health of the service
|
||||
*/
|
||||
return cb(null, statsRes);
|
||||
}
|
||||
statsRes.requests = this._getCount((results as any)[0]);
|
||||
statsRes['500s'] = this._getCount((results as any)[1]);
|
||||
return cb(null, statsRes);
|
||||
});
|
||||
}
|
||||
}
|
|
@ -0,0 +1,120 @@
|
|||
const StatsClient = require('./StatsClient');
|
||||
/**
|
||||
* @class StatsModel
|
||||
*
|
||||
* @classdesc Extend and overwrite how timestamps are normalized by minutes
|
||||
* rather than by seconds
|
||||
*/
|
||||
class StatsModel extends StatsClient {
|
||||
/**
|
||||
* normalize date timestamp to the nearest hour
|
||||
* @param {Date} d - Date instance
|
||||
* @return {number} timestamp - normalized to the nearest hour
|
||||
*/
|
||||
normalizeTimestampByHour(d) {
|
||||
return d.setMinutes(0, 0, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* get previous hour to date given
|
||||
* @param {Date} d - Date instance
|
||||
* @return {number} timestamp - one hour prior to date passed
|
||||
*/
|
||||
_getDatePreviousHour(d) {
|
||||
return d.setHours(d.getHours() - 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* normalize to the nearest interval
|
||||
* @param {object} d - Date instance
|
||||
* @return {number} timestamp - normalized to the nearest interval
|
||||
*/
|
||||
_normalizeTimestamp(d) {
|
||||
const m = d.getMinutes();
|
||||
return d.setMinutes(m - m % (Math.floor(this._interval / 60)), 0, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* override the method to get the result as an array of integers separated
|
||||
* by each interval
|
||||
* typical input looks like [[null, '1'], [null, '2'], [null, null]...]
|
||||
* @param {array} arr - each index contains the result of each batch command
|
||||
* where index 0 signifies the error and index 1 contains the result
|
||||
* @return {array} array of integers, ordered from most recent interval to
|
||||
* oldest interval
|
||||
*/
|
||||
_getCount(arr) {
|
||||
return arr.reduce((store, i) => {
|
||||
let num = parseInt(i[1], 10);
|
||||
num = Number.isNaN(num) ? 0 : num;
|
||||
store.push(num);
|
||||
return store;
|
||||
}, []);
|
||||
}
|
||||
|
||||
/**
|
||||
* get list of sorted set key timestamps
|
||||
* @param {number} epoch - epoch time
|
||||
* @return {array} array of sorted set key timestamps
|
||||
*/
|
||||
getSortedSetHours(epoch) {
|
||||
const timestamps = [];
|
||||
let date = this.normalizeTimestampByHour(new Date(epoch));
|
||||
while (timestamps.length < 24) {
|
||||
timestamps.push(date);
|
||||
date = this._getDatePreviousHour(new Date(date));
|
||||
}
|
||||
return timestamps;
|
||||
}
|
||||
|
||||
/**
|
||||
* get the normalized hour timestamp for given epoch time
|
||||
* @param {number} epoch - epoch time
|
||||
* @return {string} normalized hour timestamp for given time
|
||||
*/
|
||||
getSortedSetCurrentHour(epoch) {
|
||||
return this.normalizeTimestampByHour(new Date(epoch));
|
||||
}
|
||||
|
||||
/**
|
||||
* helper method to add element to a sorted set, applying TTL if new set
|
||||
* @param {string} key - name of key
|
||||
* @param {integer} score - score used to order set
|
||||
* @param {string} value - value to store
|
||||
* @param {callback} cb - callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
addToSortedSet(key, score, value, cb) {
|
||||
this._redis.exists(key, (err, resCode) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
if (resCode === 0) {
|
||||
// milliseconds in a day
|
||||
const msInADay = 24 * 60 * 60 * 1000;
|
||||
const nearestHour = this.normalizeTimestampByHour(new Date());
|
||||
// in seconds
|
||||
const ttl = Math.ceil(
|
||||
(msInADay - (Date.now() - nearestHour)) / 1000);
|
||||
const cmds = [
|
||||
['zadd', key, score, value],
|
||||
['expire', key, ttl],
|
||||
];
|
||||
return this._redis.batch(cmds, (err, res) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
const cmdErr = res.find(r => r[0] !== null);
|
||||
if (cmdErr) {
|
||||
return cb(cmdErr);
|
||||
}
|
||||
const successResponse = res[0][1];
|
||||
return cb(null, successResponse);
|
||||
});
|
||||
}
|
||||
return this._redis.zadd(key, score, value, cb);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = StatsModel;
|
|
@ -1,232 +0,0 @@
|
|||
import StatsClient from './StatsClient';
|
||||
import { Logger } from 'werelogs';
|
||||
import async from 'async';
|
||||
|
||||
export type Callback = (error: Error | null, value?: any) => void;
|
||||
|
||||
/**
|
||||
* @class StatsModel
|
||||
*
|
||||
* @classdesc Extend and overwrite how timestamps are normalized by minutes
|
||||
* rather than by seconds
|
||||
*/
|
||||
export default class StatsModel extends StatsClient {
|
||||
/**
|
||||
* Utility method to convert 2d array rows to columns, and vice versa
|
||||
* See also: https://docs.ruby-lang.org/en/2.0.0/Array.html#method-i-zip
|
||||
* @param arrays - 2d array of integers
|
||||
* @return converted array
|
||||
*/
|
||||
_zip(arrays: number[][]) {
|
||||
if (arrays.length > 0 && arrays.every(a => Array.isArray(a))) {
|
||||
return arrays[0].map((_, i) => arrays.map(a => a[i]));
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* normalize to the nearest interval
|
||||
* @param d - Date instance
|
||||
* @return timestamp - normalized to the nearest interval
|
||||
*/
|
||||
_normalizeTimestamp(d: Date) {
|
||||
const m = d.getMinutes();
|
||||
return d.setMinutes(m - m % (Math.floor(this._interval / 60)), 0, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* override the method to get the count as an array of integers separated
|
||||
* by each interval
|
||||
* typical input looks like [[null, '1'], [null, '2'], [null, null]...]
|
||||
* @param arr - each index contains the result of each batch command
|
||||
* where index 0 signifies the error and index 1 contains the result
|
||||
* @return array of integers, ordered from most recent interval to
|
||||
* oldest interval with length of (expiry / interval)
|
||||
*/
|
||||
// @ts-expect-errors
|
||||
_getCount(arr: [any, string | null][]): number[] {
|
||||
const size = Math.floor(this._expiry / this._interval);
|
||||
const array = arr.reduce((store, i) => {
|
||||
let num = parseInt(i[1] ?? '', 10);
|
||||
num = Number.isNaN(num) ? 0 : num;
|
||||
store.push(num);
|
||||
return store;
|
||||
}, [] as number[]);
|
||||
|
||||
if (array.length < size) {
|
||||
array.push(...Array(size - array.length).fill(0));
|
||||
}
|
||||
return array;
|
||||
}
|
||||
|
||||
/**
|
||||
* wrapper on `getStats` that handles a list of keys
|
||||
* override the method to reduce the returned 2d array from `_getCount`
|
||||
* @param log - Werelogs request logger
|
||||
* @param ids - service identifiers
|
||||
* @param cb - callback to call with the err/result
|
||||
*/
|
||||
getAllStats(log: Logger, ids: string[], cb: Callback) {
|
||||
if (!this._redis) {
|
||||
return cb(null, {});
|
||||
}
|
||||
|
||||
const size = Math.floor(this._expiry / this._interval);
|
||||
const statsRes = {
|
||||
'requests': Array(size).fill(0),
|
||||
'500s': Array(size).fill(0),
|
||||
'sampleDuration': this._expiry,
|
||||
};
|
||||
const requests: any[] = [];
|
||||
const errors: any[] = [];
|
||||
|
||||
if (ids.length === 0) {
|
||||
return cb(null, statsRes);
|
||||
}
|
||||
|
||||
// for now set concurrency to default of 10
|
||||
return async.eachLimit(ids, 10, (id, done) => {
|
||||
this.getStats(log, id, (err, res) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
requests.push(res.requests);
|
||||
errors.push(res['500s']);
|
||||
return done();
|
||||
});
|
||||
}, error => {
|
||||
if (error) {
|
||||
log.error('error getting stats', {
|
||||
error,
|
||||
method: 'StatsModel.getAllStats',
|
||||
});
|
||||
return cb(null, statsRes);
|
||||
}
|
||||
|
||||
statsRes.requests = this._zip(requests).map(arr =>
|
||||
arr.reduce((acc, i) => acc + i), 0);
|
||||
statsRes['500s'] = this._zip(errors).map(arr =>
|
||||
arr.reduce((acc, i) => acc + i), 0);
|
||||
|
||||
return cb(null, statsRes);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles getting a list of global keys.
|
||||
* @param ids - Service identifiers
|
||||
* @param log - Werelogs request logger
|
||||
* @param cb - Callback
|
||||
*/
|
||||
getAllGlobalStats(ids: string[], log: Logger, cb: Callback) {
|
||||
const reqsKeys = ids.map(key => (['get', key]));
|
||||
return this._redis.batch(reqsKeys, (err, res) => {
|
||||
const statsRes = { requests: 0 };
|
||||
if (err) {
|
||||
log.error('error getting metrics', {
|
||||
error: err,
|
||||
method: 'StatsClient.getAllGlobalStats',
|
||||
});
|
||||
return cb(null, statsRes);
|
||||
}
|
||||
statsRes.requests = res.reduce((sum, curr) => {
|
||||
const [cmdErr, val] = curr;
|
||||
if (cmdErr) {
|
||||
// Log any individual request errors from the batch request.
|
||||
log.error('error getting metrics', {
|
||||
error: cmdErr,
|
||||
method: 'StatsClient.getAllGlobalStats',
|
||||
});
|
||||
}
|
||||
return sum + (Number.parseInt(val, 10) || 0);
|
||||
}, 0);
|
||||
return cb(null, statsRes);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* normalize date timestamp to the nearest hour
|
||||
* @param d - Date instance
|
||||
* @return timestamp - normalized to the nearest hour
|
||||
*/
|
||||
normalizeTimestampByHour(d: Date) {
|
||||
return d.setMinutes(0, 0, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* get previous hour to date given
|
||||
* @param d - Date instance
|
||||
* @return timestamp - one hour prior to date passed
|
||||
*/
|
||||
_getDatePreviousHour(d: Date) {
|
||||
return d.setHours(d.getHours() - 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* get list of sorted set key timestamps
|
||||
* @param epoch - epoch time
|
||||
* @return array of sorted set key timestamps
|
||||
*/
|
||||
getSortedSetHours(epoch: number) {
|
||||
const timestamps: number[] = [];
|
||||
let date = this.normalizeTimestampByHour(new Date(epoch));
|
||||
while (timestamps.length < 24) {
|
||||
timestamps.push(date);
|
||||
date = this._getDatePreviousHour(new Date(date));
|
||||
}
|
||||
return timestamps;
|
||||
}
|
||||
|
||||
/**
|
||||
* get the normalized hour timestamp for given epoch time
|
||||
* @param epoch - epoch time
|
||||
* @return normalized hour timestamp for given time
|
||||
*/
|
||||
getSortedSetCurrentHour(epoch: number) {
|
||||
return this.normalizeTimestampByHour(new Date(epoch));
|
||||
}
|
||||
|
||||
/**
|
||||
* helper method to add element to a sorted set, applying TTL if new set
|
||||
* @param key - name of key
|
||||
* @param score - score used to order set
|
||||
* @param value - value to store
|
||||
* @param cb - callback
|
||||
*/
|
||||
addToSortedSet(
|
||||
key: string,
|
||||
score: number,
|
||||
value: string,
|
||||
cb: (error: Error | null, value?: any) => void,
|
||||
) {
|
||||
this._redis.exists(key, (err, resCode) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
if (resCode === 0) {
|
||||
// milliseconds in a day
|
||||
const msInADay = 24 * 60 * 60 * 1000;
|
||||
const nearestHour = this.normalizeTimestampByHour(new Date());
|
||||
// in seconds
|
||||
const ttl = Math.ceil(
|
||||
(msInADay - (Date.now() - nearestHour)) / 1000);
|
||||
const cmds = [
|
||||
['zadd', key, score.toString(), value],
|
||||
['expire', key, ttl.toString()],
|
||||
];
|
||||
return this._redis.batch(cmds, (err, res) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
const cmdErr = res.find((r: any) => r[0] !== null);
|
||||
if (cmdErr) {
|
||||
return cb(cmdErr);
|
||||
}
|
||||
const successResponse = res[0][1];
|
||||
return cb(null, successResponse);
|
||||
});
|
||||
}
|
||||
return this._redis.zadd(key, score, value, cb);
|
||||
});
|
||||
}
|
||||
}
|
|
@ -0,0 +1,40 @@
|
|||
const promClient = require('prom-client');
|
||||
|
||||
const collectDefaultMetricsIntervalMs =
|
||||
process.env.COLLECT_DEFAULT_METRICS_INTERVAL_MS !== undefined ?
|
||||
Number.parseInt(process.env.COLLECT_DEFAULT_METRICS_INTERVAL_MS, 10) :
|
||||
10000;
|
||||
|
||||
promClient.collectDefaultMetrics({ timeout: collectDefaultMetricsIntervalMs });
|
||||
|
||||
class ZenkoMetrics {
|
||||
static createCounter(params) {
|
||||
return new promClient.Counter(params);
|
||||
}
|
||||
|
||||
static createGauge(params) {
|
||||
return new promClient.Gauge(params);
|
||||
}
|
||||
|
||||
static createHistogram(params) {
|
||||
return new promClient.Histogram(params);
|
||||
}
|
||||
|
||||
static createSummary(params) {
|
||||
return new promClient.Summary(params);
|
||||
}
|
||||
|
||||
static getMetric(name) {
|
||||
return promClient.register.getSingleMetric(name);
|
||||
}
|
||||
|
||||
static asPrometheus() {
|
||||
return promClient.register.metrics();
|
||||
}
|
||||
|
||||
static asPrometheusContentType() {
|
||||
return promClient.register.contentType;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = ZenkoMetrics;
|
|
@ -1,35 +0,0 @@
|
|||
import promClient from 'prom-client';
|
||||
|
||||
export default class ZenkoMetrics {
|
||||
static createCounter(params: promClient.CounterConfiguration<string>) {
|
||||
return new promClient.Counter(params);
|
||||
}
|
||||
|
||||
static createGauge(params: promClient.GaugeConfiguration<string>) {
|
||||
return new promClient.Gauge(params);
|
||||
}
|
||||
|
||||
static createHistogram(params: promClient.HistogramConfiguration<string>) {
|
||||
return new promClient.Histogram(params);
|
||||
}
|
||||
|
||||
static createSummary(params: promClient.SummaryConfiguration<string>) {
|
||||
return new promClient.Summary(params);
|
||||
}
|
||||
|
||||
static getMetric(name: string) {
|
||||
return promClient.register.getSingleMetric(name);
|
||||
}
|
||||
|
||||
static async asPrometheus() {
|
||||
return promClient.register.metrics();
|
||||
}
|
||||
|
||||
static asPrometheusContentType() {
|
||||
return promClient.register.contentType;
|
||||
}
|
||||
|
||||
static collectDefaultMetrics() {
|
||||
return promClient.collectDefaultMetrics();
|
||||
}
|
||||
}
|
|
@ -1,4 +0,0 @@
|
|||
export { default as StatsClient } from './StatsClient';
|
||||
export { default as StatsModel } from './StatsModel';
|
||||
export { default as RedisClient } from './RedisClient';
|
||||
export { default as ZenkoMetrics } from './ZenkoMetrics';
|
|
@ -1,35 +1,23 @@
|
|||
import errors from '../errors'
|
||||
const errors = require('../errors').default;
|
||||
|
||||
const validServices = {
|
||||
aws: ['s3', 'iam', 'sts', 'ring'],
|
||||
scality: ['utapi', 'sso'],
|
||||
};
|
||||
|
||||
export default class ARN {
|
||||
_partition: string;
|
||||
_service: string;
|
||||
_region: string | null;
|
||||
_accountId?: string | null;
|
||||
_resource: string;
|
||||
|
||||
class ARN {
|
||||
/**
|
||||
*
|
||||
* Create an ARN object from its individual components
|
||||
*
|
||||
* @constructor
|
||||
* @param partition - ARN partition (e.g. 'aws')
|
||||
* @param service - service name in partition (e.g. 's3')
|
||||
* @param [region] - AWS region
|
||||
* @param [accountId] - AWS 12-digit account ID
|
||||
* @param resource - AWS resource path (e.g. 'foo/bar')
|
||||
* @param {string} partition - ARN partition (e.g. 'aws')
|
||||
* @param {string} service - service name in partition (e.g. 's3')
|
||||
* @param {string} [region] - AWS region
|
||||
* @param {string} [accountId] - AWS 12-digit account ID
|
||||
* @param {string} resource - AWS resource path (e.g. 'foo/bar')
|
||||
*/
|
||||
constructor(
|
||||
partition: string,
|
||||
service: string,
|
||||
region: string | undefined | null,
|
||||
accountId: string | undefined | null,
|
||||
resource: string,
|
||||
) {
|
||||
constructor(partition, service, region, accountId, resource) {
|
||||
this._partition = partition;
|
||||
this._service = service;
|
||||
this._region = region || null;
|
||||
|
@ -37,9 +25,9 @@ export default class ARN {
|
|||
this._resource = resource;
|
||||
}
|
||||
|
||||
static createFromString(arnStr: string) {
|
||||
static createFromString(arnStr) {
|
||||
const [arn, partition, service, region, accountId,
|
||||
resourceType, resource] = arnStr.split(':');
|
||||
resourceType, resource] = arnStr.split(':');
|
||||
|
||||
if (arn !== 'arn') {
|
||||
return { error: errors.InvalidArgument.customizeDescription(
|
||||
|
@ -70,7 +58,7 @@ export default class ARN {
|
|||
'must be a 12-digit number or "*"') };
|
||||
}
|
||||
const fullResource = (resource !== undefined ?
|
||||
`${resourceType}:${resource}` : resourceType);
|
||||
`${resourceType}:${resource}` : resourceType);
|
||||
return new ARN(partition, service, region, accountId, fullResource);
|
||||
}
|
||||
|
||||
|
@ -110,7 +98,9 @@ export default class ARN {
|
|||
|
||||
toString() {
|
||||
return ['arn', this.getPartition(), this.getService(),
|
||||
this.getRegion(), this.getAccountId(), this.getResource()]
|
||||
this.getRegion(), this.getAccountId(), this.getResource()]
|
||||
.join(':');
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = ARN;
|
|
@ -1,36 +1,22 @@
|
|||
import { RequestLogger } from 'werelogs';
|
||||
|
||||
import { legacyLocations } from '../constants';
|
||||
import escapeForXml from '../s3middleware/escapeForXml';
|
||||
|
||||
type CloudServerConfig = any;
|
||||
|
||||
export default class BackendInfo {
|
||||
_config: CloudServerConfig;
|
||||
_requestEndpoint: string;
|
||||
_objectLocationConstraint?: string;
|
||||
_bucketLocationConstraint?: string;
|
||||
_legacyLocationConstraint?: string;
|
||||
const { legacyLocations } = require('../constants');
|
||||
const escapeForXml = require('../s3middleware/escapeForXml');
|
||||
|
||||
class BackendInfo {
|
||||
/**
|
||||
* Represents the info necessary to evaluate which data backend to use
|
||||
* on a data put call.
|
||||
* @constructor
|
||||
* @param config - CloudServer config containing list of locations
|
||||
* @param objectLocationConstraint - location constraint
|
||||
* @param {object} config - CloudServer config containing list of locations
|
||||
* @param {string | undefined} objectLocationConstraint - location constraint
|
||||
* for object based on user meta header
|
||||
* @param bucketLocationConstraint - location
|
||||
* @param {string | undefined } bucketLocationConstraint - location
|
||||
* constraint for bucket based on bucket metadata
|
||||
* @param requestEndpoint - endpoint to which request was made
|
||||
* @param legacyLocationConstraint - legacy location constraint
|
||||
* @param {string} requestEndpoint - endpoint to which request was made
|
||||
* @param {string | undefined } legacyLocationConstraint - legacy location
|
||||
* constraint
|
||||
*/
|
||||
constructor(
|
||||
config: CloudServerConfig,
|
||||
objectLocationConstraint: string | undefined,
|
||||
bucketLocationConstraint: string | undefined,
|
||||
requestEndpoint: string,
|
||||
legacyLocationConstraint: string | undefined,
|
||||
) {
|
||||
constructor(config, objectLocationConstraint, bucketLocationConstraint,
|
||||
requestEndpoint, legacyLocationConstraint) {
|
||||
this._config = config;
|
||||
this._objectLocationConstraint = objectLocationConstraint;
|
||||
this._bucketLocationConstraint = bucketLocationConstraint;
|
||||
|
@ -41,18 +27,15 @@ export default class BackendInfo {
|
|||
|
||||
/**
|
||||
* validate proposed location constraint against config
|
||||
* @param config - CloudServer config
|
||||
* @param locationConstraint - value of user
|
||||
* @param {object} config - CloudServer config
|
||||
* @param {string | undefined} locationConstraint - value of user
|
||||
* metadata location constraint header or bucket location constraint
|
||||
* @param log - werelogs logger
|
||||
* @return - true if valid, false if not
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {boolean} - true if valid, false if not
|
||||
*/
|
||||
static isValidLocationConstraint(
|
||||
config: CloudServerConfig,
|
||||
locationConstraint: string | undefined,
|
||||
log: RequestLogger,
|
||||
) {
|
||||
if (!locationConstraint || !(locationConstraint in config.locationConstraints)) {
|
||||
static isValidLocationConstraint(config, locationConstraint, log) {
|
||||
if (Object.keys(config.locationConstraints).
|
||||
indexOf(locationConstraint) < 0) {
|
||||
log.trace('proposed locationConstraint is invalid',
|
||||
{ locationConstraint });
|
||||
return false;
|
||||
|
@ -62,17 +45,14 @@ export default class BackendInfo {
|
|||
|
||||
/**
|
||||
* validate that request endpoint is listed in the restEndpoint config
|
||||
* @param config - CloudServer config
|
||||
* @param requestEndpoint - request endpoint
|
||||
* @param log - werelogs logger
|
||||
* @return true if present, false if not
|
||||
* @param {object} config - CloudServer config
|
||||
* @param {string} requestEndpoint - request endpoint
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {boolean} - true if present, false if not
|
||||
*/
|
||||
static isRequestEndpointPresent(
|
||||
config: CloudServerConfig,
|
||||
requestEndpoint: string,
|
||||
log: RequestLogger,
|
||||
) {
|
||||
if (!(requestEndpoint in config.restEndpoints)) {
|
||||
static isRequestEndpointPresent(config, requestEndpoint, log) {
|
||||
if (Object.keys(config.restEndpoints).
|
||||
indexOf(requestEndpoint) < 0) {
|
||||
log.trace('requestEndpoint does not match config restEndpoints',
|
||||
{ requestEndpoint });
|
||||
return false;
|
||||
|
@ -83,18 +63,14 @@ export default class BackendInfo {
|
|||
/**
|
||||
* validate that locationConstraint for request Endpoint matches
|
||||
* one config locationConstraint
|
||||
* @param config - CloudServer config
|
||||
* @param requestEndpoint - request endpoint
|
||||
* @param log - werelogs logger
|
||||
* @return - true if matches, false if not
|
||||
* @param {object} config - CloudServer config
|
||||
* @param {string} requestEndpoint - request endpoint
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {boolean} - true if matches, false if not
|
||||
*/
|
||||
static isRequestEndpointValueValid(
|
||||
config: CloudServerConfig,
|
||||
requestEndpoint: string,
|
||||
log: RequestLogger,
|
||||
) {
|
||||
const restEndpoint = config.restEndpoints[requestEndpoint];
|
||||
if (!(restEndpoint in config.locationConstraints)) {
|
||||
static isRequestEndpointValueValid(config, requestEndpoint, log) {
|
||||
if (Object.keys(config.locationConstraints).
|
||||
indexOf(config.restEndpoints[requestEndpoint]) < 0) {
|
||||
log.trace('the default locationConstraint for request' +
|
||||
'Endpoint does not match any config locationConstraint',
|
||||
{ requestEndpoint });
|
||||
|
@ -105,11 +81,11 @@ export default class BackendInfo {
|
|||
|
||||
/**
|
||||
* validate that s3 server is running with a file or memory backend
|
||||
* @param config - CloudServer config
|
||||
* @param log - werelogs logger
|
||||
* @return - true if running with file/mem backend, false if not
|
||||
* @param {object} config - CloudServer config
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {boolean} - true if running with file/mem backend, false if not
|
||||
*/
|
||||
static isMemOrFileBackend(config: CloudServerConfig, log: RequestLogger) {
|
||||
static isMemOrFileBackend(config, log) {
|
||||
if (config.backends.data === 'mem' || config.backends.data === 'file') {
|
||||
log.trace('use data backend for the location', {
|
||||
dataBackend: config.backends.data,
|
||||
|
@ -127,16 +103,12 @@ export default class BackendInfo {
|
|||
* data backend for the location.
|
||||
* - if locationConstraint for request Endpoint does not match
|
||||
* any config locationConstraint, we will return an error
|
||||
* @param config - CloudServer config
|
||||
* @param requestEndpoint - request endpoint
|
||||
* @param log - werelogs logger
|
||||
* @return - true if valid, false if not
|
||||
* @param {object} config - CloudServer config
|
||||
* @param {string} requestEndpoint - request endpoint
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {boolean} - true if valid, false if not
|
||||
*/
|
||||
static isValidRequestEndpointOrBackend(
|
||||
config: CloudServerConfig,
|
||||
requestEndpoint: string,
|
||||
log: RequestLogger,
|
||||
) {
|
||||
static isValidRequestEndpointOrBackend(config, requestEndpoint, log) {
|
||||
if (!BackendInfo.isRequestEndpointPresent(config, requestEndpoint,
|
||||
log)) {
|
||||
return BackendInfo.isMemOrFileBackend(config, log);
|
||||
|
@ -147,22 +119,17 @@ export default class BackendInfo {
|
|||
|
||||
/**
|
||||
* validate controlling BackendInfo Parameter
|
||||
* @param config - CloudServer config
|
||||
* @param objectLocationConstraint - value of user
|
||||
* @param {object} config - CloudServer config
|
||||
* @param {string | undefined} objectLocationConstraint - value of user
|
||||
* metadata location constraint header
|
||||
* @param bucketLocationConstraint - location
|
||||
* @param {string | null} bucketLocationConstraint - location
|
||||
* constraint from bucket metadata
|
||||
* @param requestEndpoint - endpoint of request
|
||||
* @param log - werelogs logger
|
||||
* @return - location constraint validity
|
||||
* @param {string} requestEndpoint - endpoint of request
|
||||
* @param {object} log - werelogs logger
|
||||
* @return {object} - location constraint validity
|
||||
*/
|
||||
static controllingBackendParam(
|
||||
config: CloudServerConfig,
|
||||
objectLocationConstraint: string | undefined,
|
||||
bucketLocationConstraint: string | null,
|
||||
requestEndpoint: string,
|
||||
log: RequestLogger,
|
||||
) {
|
||||
static controllingBackendParam(config, objectLocationConstraint,
|
||||
bucketLocationConstraint, requestEndpoint, log) {
|
||||
if (objectLocationConstraint) {
|
||||
if (BackendInfo.isValidLocationConstraint(config,
|
||||
objectLocationConstraint, log)) {
|
||||
|
@ -208,16 +175,16 @@ export default class BackendInfo {
|
|||
|
||||
/**
|
||||
* Return legacyLocationConstraint
|
||||
* @param config CloudServer config
|
||||
* @return legacyLocationConstraint;
|
||||
* @param {object} config CloudServer config
|
||||
* @return {string | undefined} legacyLocationConstraint;
|
||||
*/
|
||||
static getLegacyLocationConstraint(config: CloudServerConfig) {
|
||||
static getLegacyLocationConstraint(config) {
|
||||
return legacyLocations.find(ll => config.locationConstraints[ll]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return objectLocationConstraint
|
||||
* @return objectLocationConstraint;
|
||||
* @return {string | undefined} objectLocationConstraint;
|
||||
*/
|
||||
getObjectLocationConstraint() {
|
||||
return this._objectLocationConstraint;
|
||||
|
@ -225,7 +192,7 @@ export default class BackendInfo {
|
|||
|
||||
/**
|
||||
* Return bucketLocationConstraint
|
||||
* @return bucketLocationConstraint;
|
||||
* @return {string | undefined} bucketLocationConstraint;
|
||||
*/
|
||||
getBucketLocationConstraint() {
|
||||
return this._bucketLocationConstraint;
|
||||
|
@ -233,7 +200,7 @@ export default class BackendInfo {
|
|||
|
||||
/**
|
||||
* Return requestEndpoint
|
||||
* @return requestEndpoint;
|
||||
* @return {string} requestEndpoint;
|
||||
*/
|
||||
getRequestEndpoint() {
|
||||
return this._requestEndpoint;
|
||||
|
@ -248,9 +215,9 @@ export default class BackendInfo {
|
|||
* (4) default locationConstraint for requestEndpoint if requestEndpoint
|
||||
* is listed in restEndpoints in config.json
|
||||
* (5) default data backend
|
||||
* @return locationConstraint;
|
||||
* @return {string} locationConstraint;
|
||||
*/
|
||||
getControllingLocationConstraint(): string {
|
||||
getControllingLocationConstraint() {
|
||||
const objectLC = this.getObjectLocationConstraint();
|
||||
const bucketLC = this.getBucketLocationConstraint();
|
||||
const reqEndpoint = this.getRequestEndpoint();
|
||||
|
@ -269,3 +236,5 @@ export default class BackendInfo {
|
|||
return this._config.backends.data;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = BackendInfo;
|
|
@ -1,281 +0,0 @@
|
|||
export type DeleteRetentionPolicy = {
|
||||
enabled: boolean;
|
||||
days: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Helper class to ease access to the Azure specific information for
|
||||
* storage accounts mapped to buckets.
|
||||
*/
|
||||
export default class BucketAzureInfo {
|
||||
_data: {
|
||||
sku: string;
|
||||
accessTier: string;
|
||||
kind: string;
|
||||
systemKeys: string[];
|
||||
tenantKeys: string[];
|
||||
subscriptionId: string;
|
||||
resourceGroup: string;
|
||||
deleteRetentionPolicy: DeleteRetentionPolicy;
|
||||
managementPolicies: any[];
|
||||
httpsOnly: boolean;
|
||||
tags: any;
|
||||
networkACL: any[];
|
||||
cname: string;
|
||||
azureFilesAADIntegration: boolean;
|
||||
hnsEnabled: boolean;
|
||||
logging: any;
|
||||
hourMetrics: any;
|
||||
minuteMetrics: any;
|
||||
serviceVersion: string;
|
||||
}
|
||||
/**
|
||||
* @constructor
|
||||
* @param obj - Raw structure for the Azure info on storage account
|
||||
* @param obj.sku - SKU name of this storage account
|
||||
* @param obj.accessTier - Access Tier name of this storage account
|
||||
* @param obj.kind - Kind name of this storage account
|
||||
* @param obj.systemKeys - pair of shared keys for the system
|
||||
* @param obj.tenantKeys - pair of shared keys for the tenant
|
||||
* @param obj.subscriptionId - subscription ID the storage account
|
||||
* belongs to
|
||||
* @param obj.resourceGroup - Resource group name the storage
|
||||
* account belongs to
|
||||
* @param obj.deleteRetentionPolicy - Delete retention policy
|
||||
* @param obj.deleteRetentionPolicy.enabled -
|
||||
* @param obj.deleteRetentionPolicy.days -
|
||||
* @param obj.managementPolicies - Management policies for this
|
||||
* storage account
|
||||
* @param obj.httpsOnly - Server the content of this storage
|
||||
* account through HTTPS only
|
||||
* @param obj.tags - Set of tags applied on this storage account
|
||||
* @param obj.networkACL - Network ACL of this storage account
|
||||
* @param obj.cname - CNAME of this storage account
|
||||
* @param obj.azureFilesAADIntegration - whether or not Azure
|
||||
* Files AAD Integration is enabled for this storage account
|
||||
* @param obj.hnsEnabled - whether or not a hierarchical namespace
|
||||
* is enabled for this storage account
|
||||
* @param obj.logging - service properties: logging
|
||||
* @param obj.hourMetrics - service properties: hourMetrics
|
||||
* @param obj.minuteMetrics - service properties: minuteMetrics
|
||||
* @param obj.serviceVersion - service properties: serviceVersion
|
||||
*/
|
||||
constructor(obj: {
|
||||
sku: string;
|
||||
accessTier: string;
|
||||
kind: string;
|
||||
systemKeys: string[];
|
||||
tenantKeys: string[];
|
||||
subscriptionId: string;
|
||||
resourceGroup: string;
|
||||
deleteRetentionPolicy: DeleteRetentionPolicy;
|
||||
managementPolicies: any[];
|
||||
httpsOnly: boolean;
|
||||
tags: any;
|
||||
networkACL: any[];
|
||||
cname: string;
|
||||
azureFilesAADIntegration: boolean;
|
||||
hnsEnabled: boolean;
|
||||
logging: any;
|
||||
hourMetrics: any;
|
||||
minuteMetrics: any;
|
||||
serviceVersion: string;
|
||||
}) {
|
||||
this._data = {
|
||||
sku: obj.sku,
|
||||
accessTier: obj.accessTier,
|
||||
kind: obj.kind,
|
||||
systemKeys: obj.systemKeys,
|
||||
tenantKeys: obj.tenantKeys,
|
||||
subscriptionId: obj.subscriptionId,
|
||||
resourceGroup: obj.resourceGroup,
|
||||
deleteRetentionPolicy: obj.deleteRetentionPolicy,
|
||||
managementPolicies: obj.managementPolicies,
|
||||
httpsOnly: obj.httpsOnly,
|
||||
tags: obj.tags,
|
||||
networkACL: obj.networkACL,
|
||||
cname: obj.cname,
|
||||
azureFilesAADIntegration: obj.azureFilesAADIntegration,
|
||||
hnsEnabled: obj.hnsEnabled,
|
||||
logging: obj.logging,
|
||||
hourMetrics: obj.hourMetrics,
|
||||
minuteMetrics: obj.minuteMetrics,
|
||||
serviceVersion: obj.serviceVersion,
|
||||
};
|
||||
}
|
||||
|
||||
getSku() {
|
||||
return this._data.sku;
|
||||
}
|
||||
|
||||
setSku(sku: string) {
|
||||
this._data.sku = sku;
|
||||
return this;
|
||||
}
|
||||
|
||||
getAccessTier() {
|
||||
return this._data.accessTier;
|
||||
}
|
||||
|
||||
setAccessTier(accessTier: string) {
|
||||
this._data.accessTier = accessTier;
|
||||
return this;
|
||||
}
|
||||
|
||||
getKind() {
|
||||
return this._data.kind;
|
||||
}
|
||||
|
||||
setKind(kind: string) {
|
||||
this._data.kind = kind;
|
||||
return this;
|
||||
}
|
||||
|
||||
getSystemKeys() {
|
||||
return this._data.systemKeys;
|
||||
}
|
||||
|
||||
setSystemKeys(systemKeys: string[]) {
|
||||
this._data.systemKeys = systemKeys;
|
||||
return this;
|
||||
}
|
||||
|
||||
getTenantKeys() {
|
||||
return this._data.tenantKeys;
|
||||
}
|
||||
|
||||
setTenantKeys(tenantKeys: string[]) {
|
||||
this._data.tenantKeys = tenantKeys;
|
||||
return this;
|
||||
}
|
||||
|
||||
getSubscriptionId() {
|
||||
return this._data.subscriptionId;
|
||||
}
|
||||
|
||||
setSubscriptionId(subscriptionId: string) {
|
||||
this._data.subscriptionId = subscriptionId;
|
||||
return this;
|
||||
}
|
||||
|
||||
getResourceGroup() {
|
||||
return this._data.resourceGroup;
|
||||
}
|
||||
|
||||
setResourceGroup(resourceGroup: string) {
|
||||
this._data.resourceGroup = resourceGroup;
|
||||
return this;
|
||||
}
|
||||
|
||||
getDeleteRetentionPolicy() {
|
||||
return this._data.deleteRetentionPolicy;
|
||||
}
|
||||
|
||||
setDeleteRetentionPolicy(deleteRetentionPolicy: DeleteRetentionPolicy) {
|
||||
this._data.deleteRetentionPolicy = deleteRetentionPolicy;
|
||||
return this;
|
||||
}
|
||||
|
||||
getManagementPolicies() {
|
||||
return this._data.managementPolicies;
|
||||
}
|
||||
|
||||
setManagementPolicies(managementPolicies: any[]) {
|
||||
this._data.managementPolicies = managementPolicies;
|
||||
return this;
|
||||
}
|
||||
|
||||
getHttpsOnly() {
|
||||
return this._data.httpsOnly;
|
||||
}
|
||||
|
||||
setHttpsOnly(httpsOnly: boolean) {
|
||||
this._data.httpsOnly = httpsOnly;
|
||||
return this;
|
||||
}
|
||||
|
||||
getTags() {
|
||||
return this._data.tags;
|
||||
}
|
||||
|
||||
setTags(tags: any) {
|
||||
this._data.tags = tags;
|
||||
return this;
|
||||
}
|
||||
|
||||
getNetworkACL() {
|
||||
return this._data.networkACL;
|
||||
}
|
||||
|
||||
setNetworkACL(networkACL: any[]) {
|
||||
this._data.networkACL = networkACL;
|
||||
return this;
|
||||
}
|
||||
|
||||
getCname() {
|
||||
return this._data.cname;
|
||||
}
|
||||
|
||||
setCname(cname: string) {
|
||||
this._data.cname = cname;
|
||||
return this;
|
||||
}
|
||||
|
||||
getAzureFilesAADIntegration() {
|
||||
return this._data.azureFilesAADIntegration;
|
||||
}
|
||||
|
||||
setAzureFilesAADIntegration(azureFilesAADIntegration: boolean) {
|
||||
this._data.azureFilesAADIntegration = azureFilesAADIntegration;
|
||||
return this;
|
||||
}
|
||||
|
||||
getHnsEnabled() {
|
||||
return this._data.hnsEnabled;
|
||||
}
|
||||
|
||||
setHnsEnabled(hnsEnabled: boolean) {
|
||||
this._data.hnsEnabled = hnsEnabled;
|
||||
return this;
|
||||
}
|
||||
|
||||
getLogging() {
|
||||
return this._data.logging;
|
||||
}
|
||||
|
||||
setLogging(logging: any) {
|
||||
this._data.logging = logging;
|
||||
return this;
|
||||
}
|
||||
|
||||
getHourMetrics() {
|
||||
return this._data.hourMetrics;
|
||||
}
|
||||
|
||||
setHourMetrics(hourMetrics: any) {
|
||||
this._data.hourMetrics = hourMetrics;
|
||||
return this;
|
||||
}
|
||||
|
||||
getMinuteMetrics() {
|
||||
return this._data.minuteMetrics;
|
||||
}
|
||||
|
||||
setMinuteMetrics(minuteMetrics: any) {
|
||||
this._data.minuteMetrics = minuteMetrics;
|
||||
return this;
|
||||
}
|
||||
|
||||
getServiceVersion() {
|
||||
return this._data.serviceVersion;
|
||||
}
|
||||
|
||||
setServiceVersion(serviceVersion: any) {
|
||||
this._data.serviceVersion = serviceVersion;
|
||||
return this;
|
||||
}
|
||||
|
||||
getValue() {
|
||||
return this._data;
|
||||
}
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue