Compare commits
13 Commits
developmen
...
hotfix/7.1
Author | SHA1 | Date |
---|---|---|
Maha Benzekri | c1e10cd662 | |
Maha Benzekri | c48a037116 | |
Taylor McKinnon | 71e373a13e | |
Taylor McKinnon | 50ce8e959f | |
Taylor McKinnon | 5dacc8998d | |
williamlardier | 52e08d30f3 | |
williamlardier | 4261a01c16 | |
williamlardier | df29f82812 | |
williamlardier | 47ca8c055d | |
gaspardmoindrot | 1af74fb6cb | |
Nicolas Humbert | 6642e3c93c | |
Nicolas Humbert | ebaf54f267 | |
Dimitrios Vasilas | 6345b1b915 |
|
@ -0,0 +1,25 @@
|
|||
---
|
||||
name: codeQL
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [development/*, stabilization/*, hotfix/*]
|
||||
pull_request:
|
||||
branches: [development/*, stabilization/*, hotfix/*]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
analyze:
|
||||
name: Static analysis with CodeQL
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
with:
|
||||
languages: javascript, python, ruby
|
||||
|
||||
- name: Build and analyze
|
||||
uses: github/codeql-action/analyze@v2
|
|
@ -0,0 +1,16 @@
|
|||
---
|
||||
name: dependency review
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [development/*, stabilization/*, hotfix/*]
|
||||
|
||||
jobs:
|
||||
dependency-review:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@v3
|
|
@ -217,18 +217,13 @@ jobs:
|
|||
uses: actions/checkout@v3
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: |
|
||||
2.7
|
||||
3.9
|
||||
python-version: 3.9
|
||||
- name: Setup CI environment
|
||||
uses: ./.github/actions/setup-ci
|
||||
- name: Setup python2 test environment
|
||||
- name: Setup python test environment
|
||||
run: |
|
||||
sudo apt-get install -y libdigest-hmac-perl
|
||||
pip install virtualenv
|
||||
virtualenv -p $(which python2) ~/.virtualenv/py2
|
||||
source ~/.virtualenv/py2/bin/activate
|
||||
pip install 's3cmd==1.6.1'
|
||||
pip install 's3cmd==2.3.0'
|
||||
- name: Setup CI services
|
||||
run: docker-compose up -d
|
||||
working-directory: .github/docker
|
||||
|
@ -236,7 +231,6 @@ jobs:
|
|||
run: |-
|
||||
set -o pipefail;
|
||||
bash wait_for_local_port.bash 8000 40
|
||||
source ~/.virtualenv/py2/bin/activate
|
||||
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
|
||||
- name: Upload logs to artifacts
|
||||
uses: scality/action-artifacts@v3
|
||||
|
|
|
@ -472,22 +472,68 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
|
|||
omVal[headerName] = objMd[headerName];
|
||||
});
|
||||
}
|
||||
const versionId = decodeVersionId(request.query);
|
||||
// specify both 'versioning' and 'versionId' to create a "new"
|
||||
// version (updating master as well) but with specified
|
||||
// versionId
|
||||
const options = {
|
||||
versioning: bucketInfo.isVersioningEnabled(),
|
||||
versionId,
|
||||
};
|
||||
|
||||
let versionId = decodeVersionId(request.query);
|
||||
let versioning = bucketInfo.isVersioningEnabled();
|
||||
let isNull = false;
|
||||
|
||||
if (versionId === 'null') {
|
||||
isNull = true;
|
||||
// Retrieve the null version id from the object metadata.
|
||||
versionId = objMd && objMd.versionId;
|
||||
if (!versionId) {
|
||||
// Set isNull in the object metadata to be written.
|
||||
// Since metadata will generate a versionId for the null version,
|
||||
// the flag is needed to allow cloudserver to know that the version
|
||||
// is a null version and allow access to it using the "null" versionId.
|
||||
omVal.isNull = true;
|
||||
if (versioning) {
|
||||
// If the null version does not have a version id, it is a current null version.
|
||||
// To update the metadata of a current version, versioning is set to false.
|
||||
|
||||
// This condition is to handle the case where a single null version looks like a master
|
||||
// key and will not have a duplicate versioned key and no version ID.
|
||||
// They are created when you have a non-versioned bucket with objects,
|
||||
// and then convert bucket to versioned.
|
||||
// If no new versioned objects are added for given object(s), they look like
|
||||
// standalone master keys.
|
||||
versioning = false;
|
||||
} else {
|
||||
const versioningConf = bucketInfo.getVersioningConfiguration();
|
||||
// The purpose of this condition is to address situations in which
|
||||
// - versioning is "suspended" and
|
||||
// - no existing object or no null version.
|
||||
// In such scenarios, we generate a new null version and designate it as the master version.
|
||||
if (versioningConf && versioningConf.Status === 'Suspended') {
|
||||
versionId = '';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If the object is from a source bucket without versioning (i.e. NFS),
|
||||
// then we want to create a version for the replica object even though
|
||||
// none was provided in the object metadata value.
|
||||
if (omVal.replicationInfo.isNFS) {
|
||||
const isReplica = omVal.replicationInfo.status === 'REPLICA';
|
||||
options.versioning = isReplica;
|
||||
versioning = isReplica;
|
||||
omVal.replicationInfo.isNFS = !isReplica;
|
||||
}
|
||||
|
||||
const options = {
|
||||
versionId,
|
||||
isNull,
|
||||
};
|
||||
|
||||
// NOTE: When 'versioning' is set to true and no 'versionId' is specified,
|
||||
// it results in the creation of a "new" version, which also updates the master.
|
||||
// NOTE: Since option fields are converted to strings when they're sent to Metadata via the query string,
|
||||
// Metadata interprets the value "false" as if it were true.
|
||||
// Therefore, to avoid this confusion, we don't pass the versioning parameter at all if its value is false.
|
||||
if (versioning) {
|
||||
options.versioning = true;
|
||||
}
|
||||
|
||||
log.trace('putting object version', {
|
||||
objectKey: request.objectKey, omVal, options });
|
||||
return metadata.putObjectMD(bucketName, objectKey, omVal, options, log,
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "s3",
|
||||
"version": "7.10.27",
|
||||
"version": "7.10.27-2",
|
||||
"description": "S3 connector",
|
||||
"main": "index.js",
|
||||
"engines": {
|
||||
|
@ -20,7 +20,7 @@
|
|||
"homepage": "https://github.com/scality/S3#readme",
|
||||
"dependencies": {
|
||||
"@hapi/joi": "^17.1.0",
|
||||
"arsenal": "git+https://github.com/scality/arsenal#7.10.43",
|
||||
"arsenal": "git+https://github.com/scality/arsenal#7.10.43-1",
|
||||
"async": "~2.5.0",
|
||||
"aws-sdk": "2.905.0",
|
||||
"azure-storage": "^2.1.0",
|
||||
|
|
|
@ -14,6 +14,18 @@ class BucketUtility {
|
|||
});
|
||||
}
|
||||
|
||||
bucketExists(bucketName) {
|
||||
return this.s3
|
||||
.headBucket({ Bucket: bucketName }).promise()
|
||||
.then(() => true)
|
||||
.catch(err => {
|
||||
if (err.code === 'NotFound') {
|
||||
return false;
|
||||
}
|
||||
throw err;
|
||||
});
|
||||
}
|
||||
|
||||
createOne(bucketName) {
|
||||
return this.s3
|
||||
.createBucket({ Bucket: bucketName }).promise()
|
||||
|
@ -121,6 +133,24 @@ class BucketUtility {
|
|||
return Promise.all(promises);
|
||||
}
|
||||
|
||||
emptyIfExists(bucketName) {
|
||||
return this.bucketExists(bucketName)
|
||||
.then(exists => {
|
||||
if (exists) {
|
||||
return this.empty(bucketName);
|
||||
}
|
||||
return undefined;
|
||||
});
|
||||
}
|
||||
|
||||
emptyManyIfExists(bucketNames) {
|
||||
const promises = bucketNames.map(
|
||||
bucketName => this.emptyIfExists(bucketName)
|
||||
);
|
||||
|
||||
return Promise.all(promises);
|
||||
}
|
||||
|
||||
getOwner() {
|
||||
return this.s3
|
||||
.listBuckets().promise()
|
||||
|
|
|
@ -30,6 +30,33 @@ function getPolicyParams(paramToChange) {
|
|||
};
|
||||
}
|
||||
|
||||
function getPolicyParamsWithId(paramToChange, policyId) {
|
||||
const newParam = {};
|
||||
const bucketPolicy = {
|
||||
Version: '2012-10-17',
|
||||
Id: policyId,
|
||||
Statement: [basicStatement],
|
||||
};
|
||||
if (paramToChange) {
|
||||
newParam[paramToChange.key] = paramToChange.value;
|
||||
bucketPolicy.Statement[0] = Object.assign({}, basicStatement, newParam);
|
||||
}
|
||||
return {
|
||||
Bucket: bucket,
|
||||
Policy: JSON.stringify(bucketPolicy),
|
||||
};
|
||||
}
|
||||
|
||||
function generateRandomString(length) {
|
||||
// All allowed characters matching the regex in arsenal
|
||||
const allowedCharacters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+=,.@ -/';
|
||||
const allowedCharactersLength = allowedCharacters.length;
|
||||
|
||||
return [...Array(length)]
|
||||
.map(() => allowedCharacters[~~(Math.random() * allowedCharactersLength)])
|
||||
.join('');
|
||||
}
|
||||
|
||||
// Check for the expected error response code and status code.
|
||||
function assertError(err, expectedErr, cb) {
|
||||
if (expectedErr === null) {
|
||||
|
@ -44,6 +71,7 @@ function assertError(err, expectedErr, cb) {
|
|||
cb();
|
||||
}
|
||||
|
||||
|
||||
describe('aws-sdk test put bucket policy', () => {
|
||||
let s3;
|
||||
let otherAccountS3;
|
||||
|
@ -102,5 +130,31 @@ describe('aws-sdk test put bucket policy', () => {
|
|||
s3.putBucketPolicy(params, err =>
|
||||
assertError(err, 'MalformedPolicy', done));
|
||||
});
|
||||
|
||||
it('should return MalformedPolicy because Id is not a string',
|
||||
done => {
|
||||
const params = getPolicyParamsWithId(null, 59);
|
||||
s3.putBucketPolicy(params, err =>
|
||||
assertError(err, 'MalformedPolicy', done));
|
||||
});
|
||||
|
||||
it('should put a bucket policy on bucket since Id is a string',
|
||||
done => {
|
||||
const params = getPolicyParamsWithId(null, 'cd3ad3d9-2776-4ef1-a904-4c229d1642e');
|
||||
s3.putBucketPolicy(params, err =>
|
||||
assertError(err, null, done));
|
||||
});
|
||||
|
||||
it('should allow bucket policy with pincipal arn less than 2048 characters', done => {
|
||||
const params = getPolicyParams({ key: 'Principal', value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(150)}` } }); // eslint-disable-line max-len
|
||||
s3.putBucketPolicy(params, err =>
|
||||
assertError(err, null, done));
|
||||
});
|
||||
|
||||
it('should not allow bucket policy with pincipal arn more than 2048 characters', done => {
|
||||
const params = getPolicyParams({ key: 'Principal', value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(2020)}` } }); // eslint-disable-line max-len
|
||||
s3.putBucketPolicy(params, err =>
|
||||
assertError(err, 'MalformedPolicy', done));
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -165,7 +165,9 @@ function readJsonFromChild(child, lineFinder, cb) {
|
|||
const findBrace = data.indexOf('{', findLine);
|
||||
const findEnd = findEndJson(data, findBrace);
|
||||
const endJson = data.substring(findBrace, findEnd + 1)
|
||||
.replace(/"/g, '\\"').replace(/'/g, '"');
|
||||
.replace(/"/g, '\\"').replace(/'/g, '"')
|
||||
.replace(/b'/g, '\'')
|
||||
.replace(/b"/g, '"');
|
||||
return cb(JSON.parse(endJson));
|
||||
});
|
||||
}
|
||||
|
@ -344,18 +346,18 @@ describe('s3cmd getService', () => {
|
|||
|
||||
it("should have response headers matching AWS's response headers",
|
||||
done => {
|
||||
provideLineOfInterest(['ls', '--debug'], 'DEBUG: Response: {',
|
||||
provideLineOfInterest(['ls', '--debug'], '\'headers\': {',
|
||||
parsedObject => {
|
||||
assert(parsedObject.headers['x-amz-id-2']);
|
||||
assert(parsedObject.headers['transfer-encoding']);
|
||||
assert(parsedObject.headers['x-amz-request-id']);
|
||||
const gmtDate = new Date(parsedObject.headers.date)
|
||||
assert(parsedObject['x-amz-id-2']);
|
||||
assert(parsedObject['transfer-encoding']);
|
||||
assert(parsedObject['x-amz-request-id']);
|
||||
const gmtDate = new Date(parsedObject.date)
|
||||
.toUTCString();
|
||||
assert.strictEqual(parsedObject.headers.date, gmtDate);
|
||||
assert.strictEqual(parsedObject.date, gmtDate);
|
||||
assert.strictEqual(parsedObject
|
||||
.headers['content-type'], 'application/xml');
|
||||
['content-type'], 'application/xml');
|
||||
assert.strictEqual(parsedObject
|
||||
.headers['set-cookie'], undefined);
|
||||
['set-cookie'], undefined);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
@ -395,11 +397,11 @@ describe('s3cmd getObject', function toto() {
|
|||
});
|
||||
|
||||
it('get non existing file in existing bucket, should fail', done => {
|
||||
exec(['get', `s3://${bucket}/${nonexist}`, 'fail'], done, 12);
|
||||
exec(['get', `s3://${bucket}/${nonexist}`, 'fail'], done, 64);
|
||||
});
|
||||
|
||||
it('get file in non existing bucket, should fail', done => {
|
||||
exec(['get', `s3://${nonexist}/${nonexist}`, 'fail2'], done, 12);
|
||||
exec(['get', `s3://${nonexist}/${nonexist}`, 'fail2'], done, 64);
|
||||
});
|
||||
});
|
||||
|
||||
|
@ -511,7 +513,7 @@ describe('s3cmd delObject', () => {
|
|||
|
||||
it('delete an already deleted object, should return a 204', done => {
|
||||
provideLineOfInterest(['rm', `s3://${bucket}/${upload}`, '--debug'],
|
||||
'DEBUG: Response: {', parsedObject => {
|
||||
'DEBUG: Response:\n{', parsedObject => {
|
||||
assert.strictEqual(parsedObject.status, 204);
|
||||
done();
|
||||
});
|
||||
|
@ -519,14 +521,14 @@ describe('s3cmd delObject', () => {
|
|||
|
||||
it('delete non-existing object, should return a 204', done => {
|
||||
provideLineOfInterest(['rm', `s3://${bucket}/${nonexist}`, '--debug'],
|
||||
'DEBUG: Response: {', parsedObject => {
|
||||
'DEBUG: Response:\n{', parsedObject => {
|
||||
assert.strictEqual(parsedObject.status, 204);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('try to get the deleted object, should fail', done => {
|
||||
exec(['get', `s3://${bucket}/${upload}`, download], done, 12);
|
||||
exec(['get', `s3://${bucket}/${upload}`, download], done, 64);
|
||||
});
|
||||
});
|
||||
|
||||
|
@ -621,7 +623,7 @@ describe('s3cmd multipart upload', function titi() {
|
|||
});
|
||||
|
||||
it('should not be able to get deleted object', done => {
|
||||
exec(['get', `s3://${bucket}/${MPUpload}`, download], done, 12);
|
||||
exec(['get', `s3://${bucket}/${MPUpload}`, download], done, 64);
|
||||
});
|
||||
});
|
||||
|
||||
|
@ -660,7 +662,7 @@ MPUploadSplitter.forEach(file => {
|
|||
});
|
||||
|
||||
it('should not be able to get deleted object', done => {
|
||||
exec(['get', `s3://${bucket}/${file}`, download], done, 12);
|
||||
exec(['get', `s3://${bucket}/${file}`, download], done, 64);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
@ -728,7 +730,7 @@ describe('s3cmd info', () => {
|
|||
|
||||
// test that POLICY and CORS are returned as 'none'
|
||||
it('should find that policy has a value of none', done => {
|
||||
checkRawOutput(['info', `s3://${bucket}`], 'policy', 'none',
|
||||
checkRawOutput(['info', `s3://${bucket}`], 'Policy', 'none',
|
||||
'stdout', foundIt => {
|
||||
assert(foundIt);
|
||||
done();
|
||||
|
@ -736,7 +738,7 @@ describe('s3cmd info', () => {
|
|||
});
|
||||
|
||||
it('should find that cors has a value of none', done => {
|
||||
checkRawOutput(['info', `s3://${bucket}`], 'cors', 'none',
|
||||
checkRawOutput(['info', `s3://${bucket}`], 'CORS', 'none',
|
||||
'stdout', foundIt => {
|
||||
assert(foundIt);
|
||||
done();
|
||||
|
@ -762,7 +764,7 @@ describe('s3cmd info', () => {
|
|||
});
|
||||
|
||||
it('should find that cors has a value', done => {
|
||||
checkRawOutput(['info', `s3://${bucket}`], 'cors', corsConfig,
|
||||
checkRawOutput(['info', `s3://${bucket}`], 'CORS', corsConfig,
|
||||
'stdout', foundIt => {
|
||||
assert(foundIt, 'Did not find value for cors');
|
||||
done();
|
||||
|
|
16
yarn.lock
16
yarn.lock
|
@ -466,9 +466,9 @@ arraybuffer.slice@~0.0.7:
|
|||
optionalDependencies:
|
||||
ioctl "^2.0.2"
|
||||
|
||||
"arsenal@git+https://github.com/scality/arsenal#7.10.43":
|
||||
version "7.10.43"
|
||||
resolved "git+https://github.com/scality/arsenal#054f61d6c1b3c9bdef0ad7a98bb4703b5acacad4"
|
||||
"arsenal@git+https://github.com/scality/arsenal#7.10.43-1":
|
||||
version "7.10.43-1"
|
||||
resolved "git+https://github.com/scality/arsenal#b30d1a23a13c54351eea9efb72f83a69d51020ac"
|
||||
dependencies:
|
||||
"@types/async" "^3.2.12"
|
||||
"@types/utf8" "^3.0.1"
|
||||
|
@ -484,7 +484,7 @@ arraybuffer.slice@~0.0.7:
|
|||
bson "4.0.0"
|
||||
debug "~2.6.9"
|
||||
diskusage "^1.1.1"
|
||||
fcntl "github:scality/node-fcntl#0.2.0"
|
||||
fcntl "github:scality/node-fcntl#0.2.2"
|
||||
hdclient scality/hdclient#1.1.0
|
||||
https-proxy-agent "^2.2.0"
|
||||
ioredis "^4.28.5"
|
||||
|
@ -1851,6 +1851,14 @@ fast-levenshtein@~2.0.6:
|
|||
nan "^2.3.2"
|
||||
node-gyp "^8.0.0"
|
||||
|
||||
"fcntl@github:scality/node-fcntl#0.2.2":
|
||||
version "0.2.1"
|
||||
resolved "https://codeload.github.com/scality/node-fcntl/tar.gz/b1335ca204c6265cedc50c26020c4d63aabe920e"
|
||||
dependencies:
|
||||
bindings "^1.1.1"
|
||||
nan "^2.3.2"
|
||||
node-gyp "^8.0.0"
|
||||
|
||||
fecha@^4.2.0:
|
||||
version "4.2.3"
|
||||
resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd"
|
||||
|
|
Loading…
Reference in New Issue