Compare commits

...

3461 Commits

Author SHA1 Message Date
Vitaliy Filippov b5711e9cbf Use fs.readFileSync to read config file instead of require 2024-08-13 11:19:38 +03:00
Vitaliy Filippov 36dc6298d2 Use webpack to pack 2024-08-13 02:20:08 +03:00
Vitaliy Filippov bc2d637578 Add installation instructions for Vitastor backend 2024-08-12 01:36:42 +03:00
Vitaliy Filippov b543695048 Add example Vitastor backend configs 2024-08-11 17:24:05 +03:00
Vitaliy Filippov 90024d044d Configure "legacy" werelogs because otherwise MultipleBackendGateway was skipping messages 2024-08-04 01:22:48 +03:00
Vitaliy Filippov 451ab33f68 Use config.workers instead of config.clusters 2024-08-03 14:10:39 +03:00
Vitaliy Filippov c86107e912 Add authdata config file reference to config.json 2024-08-03 01:36:01 +03:00
Vitaliy Filippov 0a5962f256 Require scality kms only if kms backend is scality 2024-08-03 01:29:04 +03:00
Vitaliy Filippov 0e292791c6 Setup backends in config.json 2024-08-02 01:45:38 +03:00
Vitaliy Filippov fc07729bd0 Use ^versions 2024-08-02 01:44:13 +03:00
Vitaliy Filippov 4527dd6795 Do not store actual configs in git 2024-08-01 15:52:02 +03:00
Vitaliy Filippov 05fb581023 Use x-amz-storage-class instead of x-amz-meta-scal-location-constraint
FIXME: Ideally, both locations and storage classes should be supported
2024-07-28 02:00:38 +03:00
Vitaliy Filippov 956739a04e Use internal vaultclient for utapi server 2024-07-23 16:32:48 +03:00
Vitaliy Filippov 7ad0888a66 Change git dependency URLs 2024-07-21 17:36:47 +03:00
Vitaliy Filippov bf01ba4ed1 Change git dependency URLs 2024-07-21 15:26:06 +03:00
Vitaliy Filippov ab019e7e50 Make vaultclient dependency optional 2024-07-21 14:19:54 +03:00
Vitaliy Filippov 3797695e74 Make bucketclient dependency optional 2024-07-18 11:17:05 +03:00
Vitaliy Filippov c8084196c4 Remove remote management 2024-07-16 20:34:11 +03:00
bert-e b72e918ff9 Merge branch 'w/8.7/bugfix/CLDSRV-555-deleteObjects-policy-eval-fix' into tmp/octopus/w/8.8/bugfix/CLDSRV-555-deleteObjects-policy-eval-fix 2024-07-15 12:20:52 +00:00
bert-e 22887f47d8 Merge branch 'w/8.6/bugfix/CLDSRV-555-deleteObjects-policy-eval-fix' into tmp/octopus/w/8.7/bugfix/CLDSRV-555-deleteObjects-policy-eval-fix 2024-07-15 12:20:52 +00:00
bert-e 0cd10a73f3 Merge branch 'w/7.70/bugfix/CLDSRV-555-deleteObjects-policy-eval-fix' into tmp/octopus/w/8.6/bugfix/CLDSRV-555-deleteObjects-policy-eval-fix 2024-07-15 12:20:51 +00:00
bert-e e139406612 Merge branch 'bugfix/CLDSRV-555-deleteObjects-policy-eval-fix' into tmp/octopus/w/7.70/bugfix/CLDSRV-555-deleteObjects-policy-eval-fix 2024-07-15 12:20:51 +00:00
Maha Benzekri d91853a38b
processBucketPolicy fixup for objectDelete
Introduced by https://github.com/scality/cloudserver/pull/5580
we now do send a requestContext with no specific resource instead
of "null", which results in a policy evaluation error.
As we get an implicit deny for the requestType "objectDelete",
cause the processed result to be false , thus sending an empty
array of objects to vault , resulting in a deny even when the policy
allows the action on specific objects.

Linked Issue : https://scality.atlassian.net/browse/CLDSRV-555
2024-07-15 14:20:08 +02:00
Mickael Bourgois a7e798f909
CLDSRV-544: bump version 8.8.27 2024-07-03 19:08:02 +02:00
Mickael Bourgois 3a1ba29869
Merge remote-tracking branch 'origin/w/8.7/improvement/CLDSRV-544-stderr' into w/8.8/improvement/CLDSRV-544-stderr 2024-07-03 19:07:41 +02:00
Mickael Bourgois dbb9b6d787
CLDSRV-544: bump version 8.7.48 2024-07-03 18:52:35 +02:00
Mickael Bourgois fce76f0934
Merge remote-tracking branch 'origin/w/8.6/improvement/CLDSRV-544-stderr' into w/8.7/improvement/CLDSRV-544-stderr 2024-07-03 18:52:20 +02:00
Mickael Bourgois 0e39aaac09
CLDSRV: bump version 8.6.27 2024-07-03 18:48:28 +02:00
Mickael Bourgois 0b14c93fac
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-544-stderr' into w/8.6/improvement/CLDSRV-544-stderr 2024-07-03 18:48:12 +02:00
Mickael Bourgois ab2960bbf4
CLDSRV-544: bump version 2024-07-01 12:28:23 +02:00
Mickael Bourgois 7305b112e2
Merge remote-tracking branch 'origin/improvement/CLDSRV-544-stderr' into w/7.70/improvement/CLDSRV-544-stderr 2024-07-01 12:28:07 +02:00
Mickael Bourgois cd9e2e757b
CLDSRV-544: bump version 2024-06-30 21:15:52 +02:00
Mickael Bourgois ca0904f584
CLDSRV-544 Add timestamp on stderr utapi v1 2024-06-30 21:15:52 +02:00
Mickael Bourgois 0dd3dd35e6
CLDSRV-544: Add timestamp on stderr
The previous version would not exit the master of the cluster
Now it exits as it should do
2024-06-30 21:15:52 +02:00
bert-e bf7e4b7e23 Merge branch 'w/8.7/bugfix/CLDSRV-547-fixup-version' into tmp/octopus/w/8.8/bugfix/CLDSRV-547-fixup-version 2024-06-27 21:23:30 +00:00
bert-e 92f4794727 Merge branch 'w/8.6/bugfix/CLDSRV-547-fixup-version' into tmp/octopus/w/8.7/bugfix/CLDSRV-547-fixup-version 2024-06-27 21:23:29 +00:00
Jonathan Gramain c6ef85e3a1 Merge remote-tracking branch 'origin/bugfix/CLDSRV-547-fixup-version' into w/8.6/bugfix/CLDSRV-547-fixup-version 2024-06-27 14:05:27 -07:00
Jonathan Gramain c0fe0cfbcf CLDSRV-547 [fixup] bump version to 7.70.49
Fixup the version, as 7.70.48 was already tagged
2024-06-27 11:42:37 -07:00
bert-e 9c936f2b83 Merge branch 'w/8.7/bugfix/CLDSRV-547-updateRedisConfigForUtapiReindex' into tmp/octopus/w/8.8/bugfix/CLDSRV-547-updateRedisConfigForUtapiReindex 2024-06-27 18:17:08 +00:00
bert-e d26bac2ebc Merge branch 'w/8.6/bugfix/CLDSRV-547-updateRedisConfigForUtapiReindex' into tmp/octopus/w/8.7/bugfix/CLDSRV-547-updateRedisConfigForUtapiReindex 2024-06-27 18:17:08 +00:00
Jonathan Gramain cfb9db5178 Merge branch 'w/7.70/bugfix/CLDSRV-547-updateRedisConfigForUtapiReindex' into w/8.6/bugfix/CLDSRV-547-updateRedisConfigForUtapiReindex 2024-06-27 10:53:41 -07:00
Jonathan Gramain 2ce004751a Merge remote-tracking branch 'origin/bugfix/CLDSRV-547-updateRedisConfigForUtapiReindex' into w/7.70/bugfix/CLDSRV-547-updateRedisConfigForUtapiReindex 2024-06-27 10:32:45 -07:00
Jonathan Gramain 539219e046 CLDSRV-547 bump cloudserver version 2024-06-27 10:27:45 -07:00
Jonathan Gramain be49e55db5 bf: CLDSRV-547 update redis config for utapi reindex
Update the redis configuration of utapi reindex to include a list of
sentinels, rather than a single sentinel (previously set to
"localhost" in Federation).

I took this opportunity to cleanup tech debt related to parsing redis
configuration, using "joi" for validation instead and making it common
across the three different places where redis config is parsed. Not
doing so would have required yet another copy-paste of dumb and
error-prone validation code. Added unit tests for the new validation.
2024-06-27 10:25:10 -07:00
bert-e e6b240421b Merge branch 'w/8.7/bugfix/CLDSRV-549-restoreGitCommitShaImageLabel' into tmp/octopus/w/8.8/bugfix/CLDSRV-549-restoreGitCommitShaImageLabel 2024-06-26 01:47:54 +00:00
bert-e 81739e3ecf Merge branch 'w/8.6/bugfix/CLDSRV-549-restoreGitCommitShaImageLabel' into tmp/octopus/w/8.7/bugfix/CLDSRV-549-restoreGitCommitShaImageLabel 2024-06-26 01:47:54 +00:00
Jonathan Gramain c475503248 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-549-restoreGitCommitShaImageLabel' into w/8.6/bugfix/CLDSRV-549-restoreGitCommitShaImageLabel 2024-06-25 18:40:18 -07:00
bert-e 7acbd5d2fb Merge branch 'bugfix/CLDSRV-549-restoreGitCommitShaImageLabel' into tmp/octopus/w/7.70/bugfix/CLDSRV-549-restoreGitCommitShaImageLabel 2024-06-26 01:39:02 +00:00
Jonathan Gramain 8d726322e5 CLDSRV-549 restore 'git.commit-sha' and 'git.repository' labels
Add back the 'git.commit-sha' and 'git.repository' labels to pushed
images, which were not attached anymore after the change of registry.
2024-06-25 18:26:54 -07:00
williamlardier 4f7aa54886 CLDSRV-541: bump project version 2024-06-13 13:58:54 +02:00
williamlardier 0117a5b0b4 CLDSRV-541: add unit test for deleteobjects authz 2024-06-13 13:58:54 +02:00
williamlardier f679831ba2 CLDSRV-541: update unit tests 2024-06-13 13:56:18 +02:00
williamlardier bb162ca7d3 CLDSRV-541: send request context in deleteobjects to get quota information 2024-06-13 11:58:33 +02:00
williamlardier 0c6dfc7b6e CLDSRV-537: bump project version 2024-05-31 13:47:26 +02:00
williamlardier d608d849df CLDSRV-537: bump checkout version for alerts 2024-05-31 13:47:26 +02:00
williamlardier 2cb63f58d4 CLDSRV-537: bump action-prom-render-test version 2024-05-31 13:44:05 +02:00
williamlardier 51585712f4 CLDSRV-537: do not raise quota error if no quota is defined
This ensures fresh installs, or buckets that get empty-ed are
not triggering the alert by mistake
2024-05-31 13:44:05 +02:00
bert-e 61eb24e46f Merge branch 'w/8.6/bugfix/CLDSRV-534/disable_git_clone_protection' into tmp/octopus/w/8.7/bugfix/CLDSRV-534/disable_git_clone_protection 2024-05-22 17:13:02 +00:00
bert-e a34b162782 Merge branch 'w/8.7/bugfix/CLDSRV-534/disable_git_clone_protection' into tmp/octopus/w/8.8/bugfix/CLDSRV-534/disable_git_clone_protection 2024-05-22 17:13:02 +00:00
bert-e a9e50fe046 Merge branch 'w/7.70/bugfix/CLDSRV-534/disable_git_clone_protection' into tmp/octopus/w/8.6/bugfix/CLDSRV-534/disable_git_clone_protection 2024-05-22 17:13:01 +00:00
bert-e 4150a8432e Merge branch 'bugfix/CLDSRV-534/disable_git_clone_protection' into tmp/octopus/w/7.70/bugfix/CLDSRV-534/disable_git_clone_protection 2024-05-22 17:13:01 +00:00
Taylor McKinnon 7e70ff9cbc Disable git clone protection to work around git bug affecting git-lfs 2024-05-22 10:05:17 -07:00
bert-e 09dc45289c Merge branches 'development/8.8' and 'w/8.7/bugfix/CLDSRV-529/bump_utapi' into tmp/octopus/w/8.8/bugfix/CLDSRV-529/bump_utapi 2024-05-17 13:21:31 +00:00
bert-e 47c628e0e1 Merge branch 'w/8.6/bugfix/CLDSRV-529/bump_utapi' into tmp/octopus/w/8.7/bugfix/CLDSRV-529/bump_utapi 2024-05-17 13:21:30 +00:00
Nicolas Humbert a1f4d3fe8a CLDSRV-529 use shorthand utapi dependency format 2024-05-17 15:10:40 +02:00
williamlardier 926242b077 CLDSRV-553: bump project version 2024-05-17 12:35:59 +02:00
williamlardier aa2aac5db3 CLDSRV-553: functional restore test to simulate cold backend calls 2024-05-17 12:35:59 +02:00
williamlardier f2e2d82e51 CLDSRV-553: unit test the onlyCheckQuota flag 2024-05-17 12:35:59 +02:00
williamlardier 88ad86b0c6 CLDSRV-553: adapt calls to quota evaluation
When the API is being called by a cold backend, the
x-scal-s3-version-id header is set. In this case, the quotas must
be evaluated with a 0 inflight.
2024-05-17 12:35:59 +02:00
bert-e 8f25892247 Merge branch 'w/8.7/bugfix/CLDSRV-529/bump_utapi' into tmp/octopus/w/8.8/bugfix/CLDSRV-529/bump_utapi 2024-05-17 08:40:32 +00:00
bert-e 9ac207187b Merge branch 'w/8.6/bugfix/CLDSRV-529/bump_utapi' into tmp/octopus/w/8.7/bugfix/CLDSRV-529/bump_utapi 2024-05-17 08:40:31 +00:00
Anurag Mittal 624a04805f
Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-529/bump_utapi' into w/8.6/bugfix/CLDSRV-529/bump_utapi 2024-05-17 10:40:00 +02:00
Anurag Mittal ba99933765
Merge remote-tracking branch 'origin/bugfix/CLDSRV-529/bump_utapi' into w/7.70/bugfix/CLDSRV-529/bump_utapi 2024-05-17 10:36:36 +02:00
williamlardier 38d1ac1d2c CLDSRV-553: conditionnaly force evaluating quotas with 0 inflight
A corner case was found, where any PUT from the cold backend would
fail if the quota is already exceeded, as the storage was reserved
for the restore, but the restore itself requires some more bytes
as inflights when evaluating quotas. By passing a flag in the quota
evaluation function, we ensure that we can, in these cases,
evaluate the quotas with 0 inflight.
2024-05-17 08:06:35 +02:00
Taylor McKinnon 4f34a34a11 bf(CLDSRV-529): Bump version 2024-05-16 12:19:45 -07:00
Taylor McKinnon 53f2a159fa bf(CLDSRV-529): Bump utapi 2024-05-16 12:18:24 -07:00
Maha Benzekri 63f6a75a86
CLDSRV-530: bump project version 2024-05-10 18:36:01 +02:00
Maha Benzekri 41acc7968e
CLDSRV-530: from accountwithQuota to accountWithQuotaCount 2024-05-10 18:32:07 +02:00
williamlardier c98c5207fc CLDSRV-520: bump project version 2024-05-10 09:51:02 +02:00
williamlardier 615ee393a4 CLDSRV-520: fix federation image with tsc 2024-05-10 09:51:02 +02:00
williamlardier 97dfc699aa CLDSRV-520: bump cloudserver version 2024-05-10 08:12:35 +02:00
williamlardier 76786282d1 CLDSRV-520: deduplicate quota logic 2024-05-10 08:12:35 +02:00
williamlardier a19d6524be CLDSRV-520: generic quota retrieval latency dashboard 2024-05-10 08:12:35 +02:00
williamlardier bbf6dfba22 CLDSRV-520: monitor quota cleanup
The finalization of quota logic will always be executed. Some tests
are added to ensure the inflights are only cleaned when they are
enabled, and an error happened in the API.
In any case, this ensures we monitor quotas in a single place,
for each of the executed action, and compute correctly the total
duration of the quota impact on the API.
2024-05-10 08:11:27 +02:00
williamlardier f0663fd507 CLDSRV-520: add dashboards 2024-05-10 08:11:27 +02:00
williamlardier d4decbbd6c CLDSRV-520: add alerts 2024-05-10 08:11:27 +02:00
williamlardier 288b2b7b87 CLDSRV-520: observe number of buckets and accounts with quota 2024-05-10 08:11:27 +02:00
williamlardier ccf9b62e59 CLDSRV-520: observe metrics during quota evaluations 2024-05-10 08:11:27 +02:00
williamlardier 9fc2d552ae CLDSRV-520: add metrics for quota 2024-05-07 17:56:24 +02:00
williamlardier d7cc4cf7d5 CLDSRV-515: adapt dockerfile for scubaclient 2024-05-07 16:24:25 +02:00
williamlardier 334d33ef44 CLDSRV-515: unit testing 2024-05-07 16:24:25 +02:00
williamlardier 989b0214d9 CLDSRV-515: functional testing 2024-05-07 16:21:13 +02:00
williamlardier 04d0730f97 CLDSRV-515: clear inflights in case of quota exceeded
- If the quotas are evaluated with success and inflights are
  enabled, it means the quota service will store the information
  and persist it till the next update of the utilization metrics.
  In this case, aany API that will fail after authorization would
  still mean that the bytes are considered, even if nothing was
  written. To overcome that, we call a function from the quota
  evaluation logic to erase anything that we wrote during the
  authorization.
2024-05-07 16:21:13 +02:00
williamlardier fbc642c022 CLDSRV-515: evaluate quotas
Quotas are evaluated:
- As part of the authorization process, after both the bucket and
  the object are authorized. The checks are skipped if the API does
  not need any quota evaluation, if the inflight bytes are 0 (i.e.,
  no data added, so no need to check the quota).
- The Copy APIs will evaluate the quotas when the source object is
  checked. In this particular case, the action is objectGet, so a
  flag is passed to force the quota evaluation logic. A subsequent
  check is done in the logic.
- The restoreObject API has a special case where the extension of
  the restoration duration would still cause the evaluation of the
  quotas, causing a potential increase in the inflights stored. We
  detect this case and remove any added inflight.
2024-05-07 16:21:13 +02:00
williamlardier 104435f0b6 CLDSRV-515: implement the quota logic as an helper file 2024-05-07 16:21:13 +02:00
williamlardier a362ac202e CLDSRV-515: bootstrap scuba on startup 2024-05-07 16:21:13 +02:00
williamlardier 1277e58150 CLDSRV-515: create a wrapper for scubaclient and quota service 2024-05-07 16:21:13 +02:00
williamlardier 7727ccf5f0 CLDSRV-515: add configuration for quotas
- Quota service is generic. We only support scuba backend now,
  but we can add others later, if needed, as long as they share
  the same implementation as the scuba client.
- Scuba configuration is passed for the scubaclient tool.
- Ability to disable the inflights is provided. This changes the
  behavior of the quota checks, so that the inflights won't be
  part of the request to the utilization metrics services. This
  reduces the complexity of the quota evaluation logic in case
  of error, as no cleanup will be needed in this case. This,
  however, requires a backend that can provide up to date metrics
  (i.e., <2s).
2024-05-05 15:31:34 +02:00
williamlardier 71860fc90c CLDSRV-515: do not recreate variable at every authz 2024-05-05 15:31:04 +02:00
williamlardier e504b52de7 CLDSRV-515: bump arsenal and vaultclient, introduce scubaclient 2024-05-02 15:09:23 +02:00
Maha Benzekri b369a47c4d CLDSRV-516: add tests 2024-05-02 14:44:31 +02:00
Maha Benzekri b4fa81e832 CLDSRV-516: implement BucketDeleteQuota API 2024-05-02 14:44:31 +02:00
Maha Benzekri 1e03d53879 CLDSRV-516: implement BucketGetQuota API 2024-05-02 14:44:31 +02:00
Maha Benzekri 63e502d419 CLDSRV-516: implement UpdateBucketQuota API 2024-05-02 14:44:31 +02:00
Maha Benzekri d2a31dc20a CLDSRV-516: specify the signature version of old auth tests
This is unrelated to the quotas, but blocks the CI.
2024-05-02 14:44:28 +02:00
Maha Benzekri f24411875f CLDSRV-516: introduce quota APIs in router 2024-05-02 14:28:56 +02:00
Maha Benzekri 4fd7faa6a3 CLDSRV-516: bump arsenal version 2024-05-02 14:27:44 +02:00
Francois Ferrand 118aaba702
Use sproxyd from ghcr
Issue: CLDSRV-524
2024-04-18 20:38:37 +02:00
Francois Ferrand e4442fdc52
Merge branch 'w/8.7/improvement/CLDSRV-524' into w/8.8/improvement/CLDSRV-524 2024-04-16 18:36:03 +02:00
Francois Ferrand 7fa199741f
Merge branch 'w/8.6/improvement/CLDSRV-524' into w/8.7/improvement/CLDSRV-524 2024-04-16 18:35:32 +02:00
Francois Ferrand f7f95af78f
Migrate to ghcr
Issue: CLDSRV-524
2024-04-16 18:34:49 +02:00
Francois Ferrand 2dc053a784
Merge branch 'w/7.70/improvement/CLDSRV-524' into w/8.6/improvement/CLDSRV-524 2024-04-16 17:57:54 +02:00
Francois Ferrand cc9bb9047e
Merge branch 'improvement/CLDSRV-524' into w/7.70/improvement/CLDSRV-524 2024-04-16 16:58:57 +02:00
Francois Ferrand b824fc0828
Use official docker build steps
The docker-build step from `scality/workflows/` fails to login to
 ghcr, as it picks up the old registry creds.

Issue: CLDSRV-524
2024-04-16 16:54:51 +02:00
Francois Ferrand a2e6d91cf2
Build pykmip image
Issue: CLDSRV-524
2024-04-16 16:54:41 +02:00
Francois Ferrand c1060853dd
Upgrade actions
- artifacts@v4
- cache@v4
- checkout@v4
- codeql@v3
- dependency-review@v4
- login@v3
- setup-buildx@v3
- setup-node@v4
- setup-python@v5

Issue: CLDSRV-524
2024-04-16 16:54:23 +02:00
Francois Ferrand 227d6edd09
Migrate to ghcr
Issue: CLDSRV-524
2024-04-16 16:54:02 +02:00
bert-e b4754c68ea Merge branches 'w/8.8/bugfix/CLDSRV-518/duplication' and 'q/5548/8.7/bugfix/CLDSRV-518/duplication' into tmp/octopus/q/8.8 2024-03-25 12:56:17 +00:00
bert-e 11aea5d93b Merge branches 'w/8.7/bugfix/CLDSRV-518/duplication' and 'q/5548/8.6/bugfix/CLDSRV-518/duplication' into tmp/octopus/q/8.7 2024-03-25 12:56:17 +00:00
bert-e 0c50a5952f Merge branches 'w/8.6/bugfix/CLDSRV-518/duplication' and 'q/5548/7.70/bugfix/CLDSRV-518/duplication' into tmp/octopus/q/8.6 2024-03-25 12:56:16 +00:00
bert-e 4a32e05855 Merge branches 'w/7.70/bugfix/CLDSRV-518/duplication' and 'q/5548/7.10/bugfix/CLDSRV-518/duplication' into tmp/octopus/q/7.70 2024-03-25 12:56:16 +00:00
bert-e 402ed21b14 Merge branch 'bugfix/CLDSRV-518/duplication' into q/7.10 2024-03-25 12:56:16 +00:00
Nicolas Humbert a22719ed47 Merge remote-tracking branch 'origin/w/8.7/bugfix/CLDSRV-518/duplication' into w/8.8/bugfix/CLDSRV-518/duplication 2024-03-20 08:48:00 +01:00
Nicolas Humbert 41975d539d Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-518/duplication' into w/8.7/bugfix/CLDSRV-518/duplication 2024-03-19 18:12:42 +01:00
Nicolas Humbert c6724eb811 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-518/duplication' into w/8.6/bugfix/CLDSRV-518/duplication 2024-03-19 05:54:35 +01:00
Nicolas Humbert d027006938 Merge remote-tracking branch 'origin/bugfix/CLDSRV-518/duplication' into w/7.70/bugfix/CLDSRV-518/duplication 2024-03-14 20:50:08 +01:00
Nicolas Humbert 92cfd47572 CLDSRV-518 Duplication of version ID in metadata 2024-03-14 16:33:25 +01:00
bert-e 8796bf0f44 Merge branch 'w/8.7/bugfix/CLDSRV-513-batchDeleteRequestLogger' into tmp/octopus/w/8.8/bugfix/CLDSRV-513-batchDeleteRequestLogger 2024-03-13 21:04:57 +00:00
bert-e 735fcd04ef Merge branch 'w/8.6/bugfix/CLDSRV-513-batchDeleteRequestLogger' into tmp/octopus/w/8.7/bugfix/CLDSRV-513-batchDeleteRequestLogger 2024-03-13 21:04:57 +00:00
Jonathan Gramain c5522685b2 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-513-batchDeleteRequestLogger' into w/8.6/bugfix/CLDSRV-513-batchDeleteRequestLogger 2024-03-13 14:04:21 -07:00
Jonathan Gramain 48df7df271 Merge remote-tracking branch 'origin/bugfix/CLDSRV-513-batchDeleteRequestLogger' into w/7.70/bugfix/CLDSRV-513-batchDeleteRequestLogger 2024-03-13 14:02:52 -07:00
Jonathan Gramain e028eb227f CLDSRV-513 bump cloudserver 2024-03-13 14:00:55 -07:00
Nicolas Humbert caf3146662 CLDSRV-518 fix Ruby dependency: excon
(cherry picked from commit cc1607eaaecb97ab5c48da15f1b1449fe7a4680f)
2024-03-13 13:58:41 -07:00
bert-e 1dee707eb8 Merge branch 'w/8.7/bugfix/CLDSRV-513-batchDeleteRequestLogger' into tmp/octopus/w/8.8/bugfix/CLDSRV-513-batchDeleteRequestLogger 2024-03-13 17:36:39 +00:00
Jonathan Gramain 2c8d69c20a Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-513-batchDeleteRequestLogger' into w/8.7/bugfix/CLDSRV-513-batchDeleteRequestLogger 2024-03-13 10:18:39 -07:00
Jonathan Gramain 0b2b6ceeb5 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-513-batchDeleteRequestLogger' into w/8.6/bugfix/CLDSRV-513-batchDeleteRequestLogger 2024-03-13 09:46:43 -07:00
Jonathan Gramain f4b3f39dc6 Merge remote-tracking branch 'origin/bugfix/CLDSRV-513-batchDeleteRequestLogger' into w/7.70/bugfix/CLDSRV-513-batchDeleteRequestLogger 2024-03-13 09:39:49 -07:00
Jonathan Gramain 84260340d0 CLDSRV-513 bump arsenal dependency 2024-03-13 09:34:30 -07:00
Jonathan Gramain e531abc346 bf: CLDSRV-513 fix request logger for batchDelete
Arsenal's `DataWrapper.batchDelete()` now already creates a request
logger on which it calls `end()` to get the elapsed time. So as
there's no need to create one before the call, remove the
corresponding code.

Note that the main fix is the arsenal version bump which, by creating
a request logger, fixes naturally the forgotten case in
`checkHashMatchMD5`.
2024-03-13 09:31:10 -07:00
Jonathan Gramain 20f6e3089b CLDSRV-513 bump werelogs dependency 2024-03-13 09:31:10 -07:00
bert-e 9dc34f2155 Merge branch 'w/8.7/bugfix/CLDSRV-501/putmetadata' into tmp/octopus/w/8.8/bugfix/CLDSRV-501/putmetadata 2024-03-07 10:05:14 +00:00
bert-e 08a4c3ade3 Merge branch 'w/8.6/bugfix/CLDSRV-501/putmetadata' into tmp/octopus/w/8.7/bugfix/CLDSRV-501/putmetadata 2024-03-07 10:05:13 +00:00
Nicolas Humbert d5c731856b Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-501/putmetadata' into w/8.6/bugfix/CLDSRV-501/putmetadata 2024-03-07 10:51:36 +01:00
Nicolas Humbert 584c94692b Merge remote-tracking branch 'origin/bugfix/CLDSRV-501/putmetadata' into w/7.70/bugfix/CLDSRV-501/putmetadata 2024-03-07 10:16:03 +01:00
Nicolas Humbert a0e5257c75 CLDSRV-501 bump arsenal 2024-03-07 10:09:28 +01:00
bert-e 5435c14116 Merge branch 'w/8.7/bugfix/CLDSRV-501/putmetadata' into tmp/octopus/w/8.8/bugfix/CLDSRV-501/putmetadata 2024-03-01 16:31:40 +00:00
bert-e 38c44ea874 Merge branch 'w/8.6/bugfix/CLDSRV-501/putmetadata' into tmp/octopus/w/8.7/bugfix/CLDSRV-501/putmetadata 2024-03-01 16:31:39 +00:00
Nicolas Humbert 4200346dd2 CLDSRV-501 skip tests related to Backbeat routes for replication 2024-03-01 17:16:36 +01:00
bert-e 5472d0da59 Merge branch 'w/8.7/bugfix/CLDSRV-501/putmetadata' into tmp/octopus/w/8.8/bugfix/CLDSRV-501/putmetadata 2024-02-29 10:03:38 +00:00
bert-e cdc0bb1128 Merge branch 'w/8.6/bugfix/CLDSRV-501/putmetadata' into tmp/octopus/w/8.7/bugfix/CLDSRV-501/putmetadata 2024-02-29 10:03:38 +00:00
Nicolas Humbert 795f8bcf1c Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-501/putmetadata' into w/8.6/bugfix/CLDSRV-501/putmetadata 2024-02-29 10:44:42 +01:00
Nicolas Humbert 9371d8d734 Merge remote-tracking branch 'origin/bugfix/CLDSRV-501/putmetadata' into w/7.70/bugfix/CLDSRV-501/putmetadata 2024-02-29 08:56:30 +01:00
Nicolas Humbert 3f31c7f3a1 CLDSRV-501 PutMetadata should write metadata on top of a null version 2024-02-27 14:29:35 +01:00
KillianG 39cba3ee6c
Merge remote-tracking branch 'origin/improvement/CLDSRV-512-use-TPF-variable-when-restore-adjust' into w/8.8/improvement/CLDSRV-512-use-TPF-variable-when-restore-adjust 2024-02-27 11:31:55 +01:00
KillianG a00952712f
Bump 8.7.47
Issue: CLDSRV-512
2024-02-27 10:41:34 +01:00
KillianG a246e18e17
Updatest test for startRestore
Issue: CLDSRV-512
2024-02-27 10:26:19 +01:00
KillianG 3bb3a4d161
Use scaledMsPerDay when restore-adjust
Use scaledMsPerday when restoring an object that has already been restored to be able to make the time go faster for testing purpose

Issue: CLDSRV-512
2024-02-27 10:26:11 +01:00
bert-e c6ba7f981e Merge branches 'w/8.8/bugfix/CLDSRV-498/null' and 'q/5526/8.7/bugfix/CLDSRV-498/null' into tmp/octopus/q/8.8 2024-02-21 13:57:14 +00:00
bert-e 69c82da878 Merge branches 'w/8.6/bugfix/CLDSRV-498/null' and 'q/5526/7.70/bugfix/CLDSRV-498/null' into tmp/octopus/q/8.6 2024-02-21 13:57:13 +00:00
bert-e 762ae5a0ff Merge branches 'w/8.7/bugfix/CLDSRV-498/null' and 'q/5526/8.6/bugfix/CLDSRV-498/null' into tmp/octopus/q/8.7 2024-02-21 13:57:13 +00:00
bert-e 89dfc794a6 Merge branch 'w/7.70/bugfix/CLDSRV-498/null' into tmp/octopus/q/7.70 2024-02-21 13:57:12 +00:00
bert-e 3205d117f5 Merge branches 'w/8.8/bugfix/CLDSRV-508-fix-bucket-tagging' and 'q/5539/8.7/bugfix/CLDSRV-508-fix-bucket-tagging' into tmp/octopus/q/8.8 2024-02-20 13:05:07 +00:00
bert-e 4eafae44d8 Merge branches 'w/8.6/bugfix/CLDSRV-508-fix-bucket-tagging' and 'q/5539/7.70/bugfix/CLDSRV-508-fix-bucket-tagging' into tmp/octopus/q/8.6 2024-02-20 13:05:06 +00:00
bert-e 4cab3c84f3 Merge branches 'w/8.7/bugfix/CLDSRV-508-fix-bucket-tagging' and 'q/5539/8.6/bugfix/CLDSRV-508-fix-bucket-tagging' into tmp/octopus/q/8.7 2024-02-20 13:05:06 +00:00
bert-e e3301a2db9 Merge branch 'bugfix/CLDSRV-508-fix-bucket-tagging' into q/7.70 2024-02-20 13:05:05 +00:00
williamlardier 0dcc93cdbe Merge remote-tracking branch 'origin/w/8.7/bugfix/CLDSRV-508-fix-bucket-tagging' into w/8.8/bugfix/CLDSRV-508-fix-bucket-tagging 2024-02-20 13:49:56 +01:00
williamlardier 2f2f91d6e8 Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-508-fix-bucket-tagging' into w/8.7/bugfix/CLDSRV-508-fix-bucket-tagging 2024-02-20 13:48:05 +01:00
williamlardier a28b141dfb Merge remote-tracking branch 'origin/bugfix/CLDSRV-508-fix-bucket-tagging' into w/8.6/bugfix/CLDSRV-508-fix-bucket-tagging 2024-02-20 13:43:22 +01:00
williamlardier 46fe061895 CLDSRV-508: bump project version 2024-02-20 12:44:23 +01:00
williamlardier 34202eaa62 CLDSRV-508: add tests for bucket tagging APIs 2024-02-20 12:44:07 +01:00
williamlardier 4d343fe468 CLDSRV-508: standardize XML with object tagging API 2024-02-20 12:42:34 +01:00
williamlardier 229e641f88 CLDSRV-508: add missing parameters in buckjet tagging APIs 2024-02-20 12:42:18 +01:00
bert-e 1433973e5c Merge branch 'w/8.7/bugfix/CLDSRV-498/null' into tmp/octopus/w/8.8/bugfix/CLDSRV-498/null 2024-02-20 11:24:08 +00:00
bert-e 201170b1ed Merge branch 'w/8.6/bugfix/CLDSRV-498/null' into tmp/octopus/w/8.7/bugfix/CLDSRV-498/null 2024-02-20 11:24:08 +00:00
bert-e f13985094e Merge branch 'w/7.70/bugfix/CLDSRV-498/null' into tmp/octopus/w/8.6/bugfix/CLDSRV-498/null 2024-02-20 11:24:07 +00:00
Nicolas Humbert 395033acd2 Merge remote-tracking branch 'origin/bugfix/CLDSRV-498/null' into w/7.70/bugfix/CLDSRV-498/null 2024-02-20 12:13:38 +01:00
Nicolas Humbert 632ef26826 CLDSRV-498 Handling isNull master version with no versionId
In certain cases, a master version may not have a versionId and be set as null (isNull:true). For instance, this occurs when a customer:

Create a bucket.

Put an object to it.

Put bucket versioning.

Put metadata (BackbeatClient.putMetadata), which results in the master version being set to null (isNull:true) with no versionId.

Currently, if an object is put after these steps, CloudServer fails to appropriately generate a null version. This is because CloudServer doesn't handle situations where the master version is set to isNull:true with no versionId.

The correct approach when an object is put should be to:

Create the new version key.

Create a new null version key, assigning it a “default non-version version id”.

Update this “default non-version version id” to the `nullVersionId` field of the master key.
2024-02-20 12:04:53 +01:00
bert-e 242b2ec85a Merge branches 'w/8.8/bugfix/CLDSRV-505-ip-handling-fix' and 'q/5534/8.7/bugfix/CLDSRV-505-ip-handling-fix' into tmp/octopus/q/8.8 2024-02-19 15:00:59 +00:00
bert-e 3186a97113 Merge branches 'w/8.7/bugfix/CLDSRV-505-ip-handling-fix' and 'q/5534/8.6/bugfix/CLDSRV-505-ip-handling-fix' into tmp/octopus/q/8.7 2024-02-19 15:00:59 +00:00
bert-e 3861b8d317 Merge branch 'q/5534/7.10/bugfix/CLDSRV-505-ip-handling-fix' into tmp/normal/q/7.70 2024-02-19 15:00:58 +00:00
bert-e bb278f7d7e Merge branches 'w/8.6/bugfix/CLDSRV-505-ip-handling-fix' and 'q/5534/7.70/bugfix/CLDSRV-505-ip-handling-fix' into tmp/octopus/q/8.6 2024-02-19 15:00:58 +00:00
bert-e 3b9309490d Merge branch 'bugfix/CLDSRV-505-ip-handling-fix' into q/7.10 2024-02-19 15:00:57 +00:00
Will Toozs 0118dfabbb
Merge remote-tracking branch 'origin/w/8.7/bugfix/CLDSRV-505-ip-handling-fix' into w/8.8/bugfix/CLDSRV-505-ip-handling-fix 2024-02-19 15:40:58 +01:00
Will Toozs ff40dfaadf
Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-505-ip-handling-fix' into w/8.7/bugfix/CLDSRV-505-ip-handling-fix 2024-02-19 15:25:18 +01:00
Will Toozs 9a31236da0
Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-505-ip-handling-fix' into w/8.6/bugfix/CLDSRV-505-ip-handling-fix 2024-02-19 15:22:08 +01:00
Will Toozs 61ebacfbf3
Merge remote-tracking branch 'origin/bugfix/CLDSRV-505-ip-handling-fix' into w/7.70/bugfix/CLDSRV-505-ip-handling-fix 2024-02-19 14:26:43 +01:00
Will Toozs aa646ced28
CLDSRV-505: bump CS version 2024-02-19 12:00:41 +01:00
Will Toozs f2ca37b5fb
CLDSRV-505: update ip check tests for arrays 2024-02-19 12:00:41 +01:00
Will Toozs 9d74cedde8
CLDSRV-505: update ip check for arrays 2024-02-19 12:00:41 +01:00
bert-e 9c99a6980f Merge branches 'w/8.8/bugfix/CLDSRV-507-bp-fixes' and 'q/5530/8.7/bugfix/CLDSRV-507-bp-fixes' into tmp/octopus/q/8.8 2024-02-19 10:16:26 +00:00
bert-e d4e255781b Merge branches 'w/8.7/bugfix/CLDSRV-507-bp-fixes' and 'q/5530/8.6/bugfix/CLDSRV-507-bp-fixes' into tmp/octopus/q/8.7 2024-02-19 10:16:26 +00:00
bert-e f5763d012e Merge branches 'w/8.6/bugfix/CLDSRV-507-bp-fixes' and 'q/5530/7.70/bugfix/CLDSRV-507-bp-fixes' into tmp/octopus/q/8.6 2024-02-19 10:16:24 +00:00
bert-e 8fb740cf09 Merge branch 'bugfix/CLDSRV-507-bp-fixes' into q/7.10 2024-02-19 10:16:23 +00:00
bert-e 55c8d89de2 Merge branches 'w/7.70/bugfix/CLDSRV-507-bp-fixes' and 'q/5530/7.10/bugfix/CLDSRV-507-bp-fixes' into tmp/octopus/q/7.70 2024-02-19 10:16:23 +00:00
bert-e 1afaaec0ac Merge branch 'w/8.7/bugfix/CLDSRV-507-bp-fixes' into tmp/octopus/w/8.8/bugfix/CLDSRV-507-bp-fixes 2024-02-19 09:13:24 +00:00
bert-e e20e458971 Merge branch 'w/8.6/bugfix/CLDSRV-507-bp-fixes' into tmp/octopus/w/8.7/bugfix/CLDSRV-507-bp-fixes 2024-02-19 09:13:23 +00:00
williamlardier 56e52de056 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-507-bp-fixes' into w/8.6/bugfix/CLDSRV-507-bp-fixes 2024-02-19 10:01:09 +01:00
williamlardier d9fc4aae50 Merge remote-tracking branch 'origin/bugfix/CLDSRV-507-bp-fixes' into w/7.70/bugfix/CLDSRV-507-bp-fixes 2024-02-19 09:54:06 +01:00
williamlardier 08de09a2ab CLDSRV-507: bump arsenal version 2024-02-19 09:48:13 +01:00
bert-e bef9220032 Merge branches 'w/8.8/bugfix/CLDSRV-497/putmetadata' and 'q/5525/8.7/bugfix/CLDSRV-497/putmetadata' into tmp/octopus/q/8.8 2024-02-15 18:43:31 +00:00
bert-e de20f1efdc Merge branches 'w/8.7/bugfix/CLDSRV-497/putmetadata' and 'q/5525/8.6/bugfix/CLDSRV-497/putmetadata' into tmp/octopus/q/8.7 2024-02-15 18:43:31 +00:00
bert-e 4817f11f36 Merge branches 'w/8.6/bugfix/CLDSRV-497/putmetadata' and 'q/5525/7.70/bugfix/CLDSRV-497/putmetadata' into tmp/octopus/q/8.6 2024-02-15 18:43:30 +00:00
bert-e a6b283f5a2 Merge branch 'bugfix/CLDSRV-497/putmetadata' into q/7.10 2024-02-15 18:43:29 +00:00
bert-e 3f810a7596 Merge branches 'w/7.70/bugfix/CLDSRV-497/putmetadata' and 'q/5525/7.10/bugfix/CLDSRV-497/putmetadata' into tmp/octopus/q/7.70 2024-02-15 18:43:29 +00:00
bert-e b89d19c9f8 Merge branch 'w/8.7/bugfix/CLDSRV-497/putmetadata' into tmp/octopus/w/8.8/bugfix/CLDSRV-497/putmetadata 2024-02-15 17:58:27 +00:00
Nicolas Humbert 4dc9788629 Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-497/putmetadata' into w/8.7/bugfix/CLDSRV-497/putmetadata 2024-02-15 18:43:28 +01:00
Nicolas Humbert 65a891d6f8 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-497/putmetadata' into w/8.6/bugfix/CLDSRV-497/putmetadata 2024-02-15 17:51:48 +01:00
bert-e 2ecca4feef Merge branch 'bugfix/CLDSRV-497/putmetadata' into tmp/octopus/w/7.70/bugfix/CLDSRV-497/putmetadata 2024-02-15 16:34:04 +00:00
Nicolas Humbert c52a3a6e44 CLDSRV-497 Fix BackbeatClient.putMetadata with versionID
Issue: When Cloudserver BackbeatClient.putMetadata() option fields are sent to Metadata through the query string, they are converted to strings. As a result, Metadata interprets the value undefined in the versionId field as an empty string ('').

Background: Previously, the 'crrExistingObject' script used this bug/behavior as a workaround to generate an internal version ID to replicate null version (= objects created before versioning was enabled). However, this approach has led to inconsistencies, occasionally resulting in the creation of multiple null internal versions.

Resolution: To address this issue, the 'crrExistingObject' workaround will be deprecated. Instead, Backbeat will be enhanced to support the replication of null versions directly, thereby ensuring more reliable and consistent behavior in handling versioning.
2024-02-15 17:31:23 +01:00
williamlardier d82965ff78 CLDSRV-507: normalize request types 2024-02-15 09:26:45 +01:00
williamlardier f488a65f15 CLDSRV-507: support no object metadata for MPU APIs resource authz
The MPU APIs are dealing with object resources. At the time the
Bucket Policies and ACLs were only evaluated when there was an
Allow from IAM, there was no need to handle this case.
However now, these APIs are evaluating the bucket policies and
ACLs and because there is no object metadata associated, we
end up allowing requests without any permission by relying
on the existing code, where the permission is changed and becomes
"bucketGet". We must treat MPU APIs as different APIs and check
the right permission. For that, we rely on the updated bucket
policy action map in arsenal wth these APIs, and ensure that we
properly map that to the existing logic where we only checked the
"objectPut" permission to handle these 3 specific APIs:

- initiate MPU
- upload part
- complete MPU
2024-02-14 15:16:48 +01:00
williamlardier 40a575a717 CLDSRV-507: use correct action for put part APIs 2024-02-14 15:16:48 +01:00
williamlardier fea82f15ea CLDSRV-507: use correct action for MPU 2024-02-14 15:16:48 +01:00
bert-e 06dc042154 Merge branches 'w/8.8/improvement/CLDSRV-502' and 'q/5528/8.7/improvement/CLDSRV-502' into tmp/octopus/q/8.8 2024-02-08 13:49:18 +00:00
bert-e aa4643644a Merge branches 'w/8.7/improvement/CLDSRV-502' and 'q/5528/8.6/improvement/CLDSRV-502' into tmp/octopus/q/8.7 2024-02-08 13:49:18 +00:00
bert-e 89edf7e3d0 Merge branch 'w/8.6/improvement/CLDSRV-502' into tmp/octopus/q/8.6 2024-02-08 13:49:18 +00:00
Francois Ferrand 4c7d3ae4bc
Merge branch 'w/8.7/improvement/CLDSRV-502' into w/8.8/improvement/CLDSRV-502 2024-02-05 18:50:27 +01:00
Francois Ferrand 23883dae8b
Merge branch 'w/8.6/improvement/CLDSRV-502' into w/8.7/improvement/CLDSRV-502 2024-02-05 18:50:12 +01:00
Francois Ferrand e616ffa374
gha: fix test alert trigger to match other premerge build
Issue: CLDSRV-502
2024-02-05 18:49:31 +01:00
Francois Ferrand 515c20e4cf
Merge branch 'w/7.70/improvement/CLDSRV-502' into w/8.6/improvement/CLDSRV-502 2024-02-05 18:48:18 +01:00
Francois Ferrand f8eedddebf
Merge branch 'improvement/CLDSRV-502' into w/7.70/improvement/CLDSRV-502 2024-02-05 18:48:01 +01:00
Francois Ferrand f3654e4fb8
Fix trigger for codeql jobs
Build on pull request & bert-e queue build, and skip rebuild when PR
lands on development branch.

Issue: CLDSRV-502
2024-02-05 18:47:30 +01:00
Francois Ferrand 517fb99190
gha: add release name to release job
Issue: CLDSRV-502
2024-02-05 18:46:02 +01:00
Francois Ferrand 531c83a359
Release 8.8.17
Issue: CLDSRV-500
2024-02-05 17:35:43 +01:00
Francois Ferrand b84fa851f7
Merge branch 'w/8.7/bugfix/CLDSRV-500' into w/8.8/bugfix/CLDSRV-500 2024-02-05 17:35:20 +01:00
Francois Ferrand 4cb1a879f7
Release 8.7.44
Issue: CLDSRV-500
2024-02-05 17:34:45 +01:00
Francois Ferrand 7ae55b20e7
Merge branch 'bugfix/CLDSRV-500' into w/8.7/bugfix/CLDSRV-500 2024-02-05 17:32:53 +01:00
Francois Ferrand d0a6fa17a5
Release 8.6.24
Issue: CLDSRV-500
2024-02-05 17:31:36 +01:00
Francois Ferrand 7275459f70
Use rate interval in `Request time` panel
- Should use $__rate_interval, which handles small time range.
- Regenerating the dashboard also fixes the 'latency per s3 action'
  panel.

Issue: CLDSRV-500
2024-02-01 15:49:29 +01:00
Hervé Dombya 363afcd17f CLDSRV-473: fix cors issues in getVeeamFile 2024-01-26 15:59:10 +01:00
Frédéric Meinnel 1cf0250ce9 Merge remote-tracking branch 'origin/w/8.7/bugfix/CLDSRV-494/fix-generate-v4-headers-for-put-with-body-requests' into w/8.8/bugfix/CLDSRV-494/fix-generate-v4-headers-for-put-with-body-requests 2024-01-23 15:06:05 +01:00
Frédéric Meinnel 20d0b38d0b Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-494/fix-generate-v4-headers-for-put-with-body-requests' into w/8.7/bugfix/CLDSRV-494/fix-generate-v4-headers-for-put-with-body-requests 2024-01-23 15:05:39 +01:00
Frédéric Meinnel 9988a8327a Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-494/fix-generate-v4-headers-for-put-with-body-requests' into w/8.6/bugfix/CLDSRV-494/fix-generate-v4-headers-for-put-with-body-requests 2024-01-23 14:06:31 +01:00
Frédéric Meinnel b481d24637 Merge remote-tracking branch 'origin/bugfix/CLDSRV-494/fix-generate-v4-headers-for-put-with-body-requests' into w/7.70/bugfix/CLDSRV-494/fix-generate-v4-headers-for-put-with-body-requests 2024-01-23 14:01:59 +01:00
Frédéric Meinnel 71625774c1 CLDSRV-494: version bump 2024-01-23 13:42:36 +01:00
Frédéric Meinnel 9b9338f2b8 CLDSRV-494: Fix generateV4Headers for HTTP PUT with body 2024-01-23 13:42:31 +01:00
Frédéric Meinnel 601619f200 Merge remote-tracking branch 'origin/w/8.7/bugfix/CLDSRV-493/fully-align-with-aws-on-lifecycle-configuration-dates' into w/8.8/bugfix/CLDSRV-493/fully-align-with-aws-on-lifecycle-configuration-dates 2024-01-17 13:24:05 +01:00
Frédéric Meinnel a92e71fd50 Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-493/fully-align-with-aws-on-lifecycle-configuration-dates' into w/8.7/bugfix/CLDSRV-493/fully-align-with-aws-on-lifecycle-configuration-dates 2024-01-17 13:22:55 +01:00
Frédéric Meinnel 8802ea0617 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-493/fully-align-with-aws-on-lifecycle-configuration-dates' into w/8.6/bugfix/CLDSRV-493/fully-align-with-aws-on-lifecycle-configuration-dates 2024-01-17 13:21:42 +01:00
Frédéric Meinnel acc5f74787 Merge remote-tracking branch 'origin/bugfix/CLDSRV-493/fully-align-with-aws-on-lifecycle-configuration-dates' into w/7.70/bugfix/CLDSRV-493/fully-align-with-aws-on-lifecycle-configuration-dates 2024-01-17 13:20:10 +01:00
Frédéric Meinnel e3c093f352 CLDSRV-493: Version bump 2024-01-17 13:18:32 +01:00
Frédéric Meinnel e17383a678 CLDSRV-493: Fix dates accepted in lifecycle configuration 2024-01-17 13:18:32 +01:00
bert-e 43f62b847c Merge branch 'w/8.7/bugfix/CLDSRV-492-head-monitoring' into tmp/octopus/w/8.8/bugfix/CLDSRV-492-head-monitoring 2024-01-16 20:57:39 +00:00
bert-e a031905bba Merge branch 'w/8.6/bugfix/CLDSRV-492-head-monitoring' into tmp/octopus/w/8.7/bugfix/CLDSRV-492-head-monitoring 2024-01-16 20:57:39 +00:00
bert-e 13ad6881f4 Merge branch 'bugfix/CLDSRV-492-head-monitoring' into tmp/octopus/w/8.6/bugfix/CLDSRV-492-head-monitoring 2024-01-16 20:57:38 +00:00
Mickael Bourgois dea5173075
CLDSRV-492: remove duplicate error monitoring 2024-01-16 21:34:26 +01:00
Mickael Bourgois b3f96198fe
CLDSRV-492: update monitoring head 2024-01-15 14:48:08 +01:00
Mickael Bourgois 5e2dd8cccb
Merge remote-tracking branch 'origin/development/7.70' into bugfix/CLDSRV-492-head-monitoring 2024-01-15 11:56:50 +01:00
bert-e cd2406b827 Merge branches 'w/8.8/bugfix/CLDSRV-489-redirect-folder-index' and 'q/5520/8.7/bugfix/CLDSRV-489-redirect-folder-index' into tmp/octopus/q/8.8 2024-01-15 09:47:24 +00:00
bert-e 62f707caff Merge branches 'w/8.7/bugfix/CLDSRV-489-redirect-folder-index' and 'q/5520/8.6/bugfix/CLDSRV-489-redirect-folder-index' into tmp/octopus/q/8.7 2024-01-15 09:47:23 +00:00
bert-e f01ef00a52 Merge branches 'w/8.6/bugfix/CLDSRV-489-redirect-folder-index' and 'q/5520/7.70/bugfix/CLDSRV-489-redirect-folder-index' into tmp/octopus/q/8.6 2024-01-15 09:47:23 +00:00
bert-e 30fb64e443 Merge branch 'bugfix/CLDSRV-489-redirect-folder-index' into q/7.10 2024-01-15 09:47:22 +00:00
bert-e 054107d8fb Merge branches 'w/7.70/bugfix/CLDSRV-489-redirect-folder-index' and 'q/5520/7.10/bugfix/CLDSRV-489-redirect-folder-index' into tmp/octopus/q/7.70 2024-01-15 09:47:22 +00:00
bert-e 848bf318fe Merge branches 'development/8.8' and 'w/8.7/bugfix/CLDSRV-489-redirect-folder-index' into tmp/octopus/w/8.8/bugfix/CLDSRV-489-redirect-folder-index 2024-01-15 09:07:46 +00:00
bert-e 0beb48a1fd Merge branches 'development/8.7' and 'w/8.6/bugfix/CLDSRV-489-redirect-folder-index' into tmp/octopus/w/8.7/bugfix/CLDSRV-489-redirect-folder-index 2024-01-15 09:07:45 +00:00
bert-e 618d4dffc7 Merge branches 'development/8.6' and 'w/7.70/bugfix/CLDSRV-489-redirect-folder-index' into tmp/octopus/w/8.6/bugfix/CLDSRV-489-redirect-folder-index 2024-01-15 09:07:44 +00:00
bert-e b5aae192f7 Merge branches 'development/7.70' and 'bugfix/CLDSRV-489-redirect-folder-index' into tmp/octopus/w/7.70/bugfix/CLDSRV-489-redirect-folder-index 2024-01-15 09:07:43 +00:00
Mickael Bourgois 557f3dcde6
CLDSRV-489: fix lint indentation 2024-01-12 10:07:39 +01:00
Mickael Bourgois 3291af36bb
CLDSRV-489: Apply style suggestions
Co-authored-by: Jonathan Gramain <jonathan.gramain@scality.com>
2024-01-12 09:53:57 +01:00
Will Toozs d274acd8ed
Merge remote-tracking branch 'origin/w/8.7/improvement/CLDSRV-436-bump-version' into w/8.8/improvement/CLDSRV-436-bump-version 2024-01-11 13:10:57 +01:00
Will Toozs e6d9e8fc35
Merge remote-tracking branch 'origin/w/8.6/improvement/CLDSRV-436-bump-version' into w/8.7/improvement/CLDSRV-436-bump-version 2024-01-11 11:50:25 +01:00
Will Toozs b08edefad6
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-436-bump-version' into w/8.6/improvement/CLDSRV-436-bump-version 2024-01-11 11:24:50 +01:00
Will Toozs e9c353d62a
Merge remote-tracking branch 'origin/improvement/CLDSRV-436-bump-version' into w/7.70/improvement/CLDSRV-436-bump-version 2024-01-11 11:04:53 +01:00
Will Toozs c7c55451a1
CLDSRV-436: bump package version 2024-01-11 10:45:47 +01:00
bert-e 7bb004586d Merge branch 'w/8.7/improvement/CLDSRV-436-bp-conds' into tmp/octopus/w/8.8/improvement/CLDSRV-436-bp-conds 2024-01-10 21:13:49 +00:00
bert-e d48de67723 Merge branch 'w/8.6/improvement/CLDSRV-436-bp-conds' into tmp/octopus/w/8.7/improvement/CLDSRV-436-bp-conds 2024-01-10 21:13:48 +00:00
Will Toozs fa4dec01cb
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-436-bp-conds' into w/8.6/improvement/CLDSRV-436-bp-conds 2024-01-10 22:00:37 +01:00
Will Toozs 4f79a9c59c
Merge remote-tracking branch 'origin/improvement/CLDSRV-436-bp-conds' into w/7.70/improvement/CLDSRV-436-bp-conds 2024-01-10 21:43:08 +01:00
Will Toozs 05c759110b
CLDSRV-436: update dummyRequest of unit tests 2024-01-10 21:02:15 +01:00
Will Toozs deae294a81
CLDSRV-436: unit test policy condition validation 2024-01-10 21:02:15 +01:00
Will Toozs ab587385e6
CLDSRV-436: add functional test cases for conditions 2024-01-10 21:01:44 +01:00
Will Toozs 6243911072
CLDSRV-436: update tests 2024-01-10 20:59:26 +01:00
Will Toozs da804054e5
CLDSRV-436: update put retention logic 2024-01-10 20:57:38 +01:00
Will Toozs 493a6da773
CLDSRV-436: update put policy logic 2024-01-10 20:57:38 +01:00
Will Toozs 7ecdd11783
CLDSRV-436: add conditions logic 2024-01-10 20:57:37 +01:00
Mickael Bourgois 7e53b67c90
CLDSRV-492: fix monitoring for website head
Match head before the merging in CLDSRV-482
2024-01-10 20:29:20 +01:00
bert-e b141c59bb7 Merge branch 'w/8.7/bugfix/CLDSRV-489-redirect-folder-index' into tmp/octopus/w/8.8/bugfix/CLDSRV-489-redirect-folder-index 2024-01-10 18:44:57 +00:00
bert-e 0b79ecd942 Merge branch 'w/8.6/bugfix/CLDSRV-489-redirect-folder-index' into tmp/octopus/w/8.7/bugfix/CLDSRV-489-redirect-folder-index 2024-01-10 18:44:57 +00:00
bert-e 86ece5c264 Merge branch 'w/7.70/bugfix/CLDSRV-489-redirect-folder-index' into tmp/octopus/w/8.6/bugfix/CLDSRV-489-redirect-folder-index 2024-01-10 18:44:56 +00:00
Mickael Bourgois 0b79cd6af6
Merge remote-tracking branch 'origin/bugfix/CLDSRV-489-redirect-folder-index' into w/7.70/bugfix/CLDSRV-489-redirect-folder-index 2024-01-10 19:32:46 +01:00
Mickael Bourgois a51b5e0af3
CLDSRV-489: test redirect 302 on folder without / 2024-01-10 19:10:57 +01:00
bert-e 10ca6b98fa Merge branch 'w/8.7/improvement/CLDSRV-475/add_overhead_fields_for_suspended_versioning' into tmp/octopus/w/8.8/improvement/CLDSRV-475/add_overhead_fields_for_suspended_versioning 2024-01-10 18:05:23 +00:00
bert-e 171925732f Merge branch 'w/8.6/improvement/CLDSRV-475/add_overhead_fields_for_suspended_versioning' into tmp/octopus/w/8.7/improvement/CLDSRV-475/add_overhead_fields_for_suspended_versioning 2024-01-10 18:05:23 +00:00
Taylor McKinnon 6d36f9c867 Merge remote-tracking branch 'origin/improvement/CLDSRV-475/add_overhead_fields_for_suspended_versioning' into w/8.6/improvement/CLDSRV-475/add_overhead_fields_for_suspended_versioning 2024-01-10 10:04:49 -08:00
Taylor McKinnon 1a21c4f867 impr(CLDSRV-475): Bump version to 7.70.41 2024-01-10 10:02:36 -08:00
Taylor McKinnon 866dec1b81 impr(CLDSRV-475): Add isDeleteMarker to overhead fields 2024-01-10 10:02:15 -08:00
Mickael Bourgois 9491e82235
CLDSRV-489: redirect 302 on folder without /
If a key is not found, we must check if key/index.html
is accessible to redirect to append a trailing /
to the key

@see https://docs.aws.amazon.com/AmazonS3/latest/userguide/IndexDocumentSupport.html#IndexDocumentsandFolders
2024-01-10 17:39:13 +01:00
bert-e 70e8b20af9 Merge branch 'w/8.7/bugfix/CLDSRV-485-custom-err-redirect' into tmp/octopus/w/8.8/bugfix/CLDSRV-485-custom-err-redirect 2024-01-10 12:51:20 +00:00
bert-e 0ec5f4fee5 Merge branch 'w/8.6/bugfix/CLDSRV-485-custom-err-redirect' into tmp/octopus/w/8.7/bugfix/CLDSRV-485-custom-err-redirect 2024-01-10 12:51:20 +00:00
bert-e 6c468a01d9 Merge branch 'w/7.70/bugfix/CLDSRV-485-custom-err-redirect' into tmp/octopus/w/8.6/bugfix/CLDSRV-485-custom-err-redirect 2024-01-10 12:51:19 +00:00
bert-e 3d2b75f344 Merge branch 'bugfix/CLDSRV-485-custom-err-redirect' into tmp/octopus/w/7.70/bugfix/CLDSRV-485-custom-err-redirect 2024-01-10 12:51:19 +00:00
Mickael Bourgois 5811fa5326
CLDSRV-485: fix linter in tests for 8.6 2024-01-10 13:50:11 +01:00
bert-e e600677545 Merge branch 'w/8.7/bugfix/CLDSRV-485-custom-err-redirect' into tmp/octopus/w/8.8/bugfix/CLDSRV-485-custom-err-redirect 2024-01-10 12:34:50 +00:00
bert-e 72e5da10b7 Merge branch 'w/8.6/bugfix/CLDSRV-485-custom-err-redirect' into tmp/octopus/w/8.7/bugfix/CLDSRV-485-custom-err-redirect 2024-01-10 12:34:50 +00:00
Mickael Bourgois de0e7e6449
Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-485-custom-err-redirect' into w/8.6/bugfix/CLDSRV-485-custom-err-redirect 2024-01-10 13:15:29 +01:00
Mickael Bourgois 97b5ed6dd3
Merge remote-tracking branch 'origin/bugfix/CLDSRV-485-custom-err-redirect' into w/7.70/bugfix/CLDSRV-485-custom-err-redirect 2024-01-10 12:09:35 +01:00
Mickael Bourgois dad8a3ee37
Merge remote-tracking branch 'origin/development/7.10' into bugfix/CLDSRV-485-custom-err-redirect 2024-01-10 12:02:54 +01:00
Mickael Bourgois 8aca658c5c
CLDSRV-485: bump arsenal 2024-01-10 11:52:27 +01:00
bert-e 759817c5a0 Merge branch 'w/8.7/bugfix/CLDSRV-482-head-redirect-index' into tmp/octopus/w/8.8/bugfix/CLDSRV-482-head-redirect-index 2024-01-10 10:37:52 +00:00
bert-e 035c7e8d7f Merge branch 'w/8.6/bugfix/CLDSRV-482-head-redirect-index' into tmp/octopus/w/8.7/bugfix/CLDSRV-482-head-redirect-index 2024-01-10 10:37:52 +00:00
Mickael Bourgois b8af1225d5
Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-482-head-redirect-index' into w/8.6/bugfix/CLDSRV-482-head-redirect-index 2024-01-10 11:28:13 +01:00
Mickael Bourgois 40faa5f3fa
Merge remote-tracking branch 'origin/bugfix/CLDSRV-482-head-redirect-index' into w/7.70/bugfix/CLDSRV-482-head-redirect-index 2024-01-10 11:19:09 +01:00
Mickael Bourgois 1fc8622614
Merge remote-tracking branch 'origin/development/7.10' into bugfix/CLDSRV-482-head-redirect-index 2024-01-10 11:01:51 +01:00
Mickael Bourgois a0acefb4a8
CLDSRV-482: apply style suggestion
Co-authored-by: William <91462779+williamlardier@users.noreply.github.com>
2024-01-10 10:13:08 +01:00
bert-e de27a5b88e Merge branch 'w/8.7/bugfix/CLDSRV-488-error-type-bp' into tmp/octopus/w/8.8/bugfix/CLDSRV-488-error-type-bp 2024-01-10 08:56:49 +00:00
bert-e a4cc5e45f3 Merge branch 'w/8.6/bugfix/CLDSRV-488-error-type-bp' into tmp/octopus/w/8.7/bugfix/CLDSRV-488-error-type-bp 2024-01-10 08:56:49 +00:00
bert-e 621cb33680 Merge branch 'w/7.70/bugfix/CLDSRV-488-error-type-bp' into tmp/octopus/w/8.6/bugfix/CLDSRV-488-error-type-bp 2024-01-10 08:56:48 +00:00
bert-e b025443d21 Merge branch 'bugfix/CLDSRV-488-error-type-bp' into tmp/octopus/w/7.70/bugfix/CLDSRV-488-error-type-bp 2024-01-10 08:56:48 +00:00
Mickael Bourgois d502a81284
CLDSRV-488: fix lint 2024-01-10 09:56:27 +01:00
bert-e 9a8b707e82 Merge branch 'w/8.7/bugfix/CLDSRV-488-error-type-bp' into tmp/octopus/w/8.8/bugfix/CLDSRV-488-error-type-bp 2024-01-10 08:54:32 +00:00
bert-e 002dbe0019 Merge branch 'w/8.6/bugfix/CLDSRV-488-error-type-bp' into tmp/octopus/w/8.7/bugfix/CLDSRV-488-error-type-bp 2024-01-10 08:54:31 +00:00
bert-e 59e52f6df2 Merge branch 'w/7.70/bugfix/CLDSRV-488-error-type-bp' into tmp/octopus/w/8.6/bugfix/CLDSRV-488-error-type-bp 2024-01-10 08:54:31 +00:00
bert-e b52f2356ba Merge branch 'bugfix/CLDSRV-488-error-type-bp' into tmp/octopus/w/7.70/bugfix/CLDSRV-488-error-type-bp 2024-01-10 08:54:30 +00:00
Mickael Bourgois 60679495b6
CLDSRV-488: apply review suggestion
Co-authored-by: Jonathan Gramain <jonathan.gramain@scality.com>
2024-01-10 09:53:24 +01:00
Mickael Bourgois 9dfacd0827
CLDSRV-482: factorize website GET and HEAD 2024-01-09 18:45:02 +01:00
Mickael Bourgois 485ef1e9bb
CLDSRV-482: test routing and implicit index 2024-01-09 17:18:07 +01:00
Mickael Bourgois 5e041ca5e7
CLDSRV-482: fix head implicit index
Routing check must be performed before added index prefix
To prevent matching a routing rule on the index
2024-01-09 17:18:07 +01:00
Mickael Bourgois 52137772d9
Merge branch 'development/7.10' into bugfix/CLDSRV-488-error-type-bp 2024-01-09 16:44:18 +01:00
Mickael Bourgois fcf193d033
CLDSRV-488: move website condition, replace flag 2024-01-09 16:40:55 +01:00
Mickael Bourgois fb61cad786
CLDSRV-485: test website redirect custom error 2024-01-08 18:00:32 +01:00
Mickael Bourgois b6367eb2b8
CLDSRV-485: website redirect from custom error doc 2024-01-08 17:58:09 +01:00
bert-e d803bdcadc Merge branch 'w/8.7/bugfix/CLDSRV-477-putobj-perm-check' into tmp/octopus/w/8.8/bugfix/CLDSRV-477-putobj-perm-check 2024-01-08 13:49:57 +00:00
bert-e 4f1b8f25b7 Merge branch 'w/8.6/bugfix/CLDSRV-477-putobj-perm-check' into tmp/octopus/w/8.7/bugfix/CLDSRV-477-putobj-perm-check 2024-01-08 13:49:56 +00:00
bert-e 94363482c3 Merge branch 'w/7.70/bugfix/CLDSRV-477-putobj-perm-check' into tmp/octopus/w/8.6/bugfix/CLDSRV-477-putobj-perm-check 2024-01-08 13:49:56 +00:00
bert-e 6b0a8cb9ed Merge branch 'bugfix/CLDSRV-477-putobj-perm-check' into tmp/octopus/w/7.70/bugfix/CLDSRV-477-putobj-perm-check 2024-01-08 13:49:55 +00:00
Will Toozs 5dbf5d965f
CLDSRV-477: add tests 2024-01-08 14:43:41 +01:00
Will Toozs ebefc4b5b0
CLDSRV-477: change position of ACL check 2024-01-08 14:43:40 +01:00
Mickael Bourgois ac1c75e414
CLDSRV-488: test website 404 with bucket policy 2024-01-05 12:52:51 +01:00
Mickael Bourgois fee4f3a96e
CLDSRV-488: fix website 404 with bucket policy
If bucket policy authorize access to a
non existant object, there should be a 404
and not a 403
2024-01-05 12:52:50 +01:00
bert-e e969eeaa20 Merge branches 'w/8.8/bugfix/CLDSRV-490-bucket-policy-resource' and 'q/5516/8.7/bugfix/CLDSRV-490-bucket-policy-resource' into tmp/octopus/q/8.8 2024-01-05 11:24:59 +00:00
bert-e 2ee78bcf6a Merge branches 'w/8.7/bugfix/CLDSRV-490-bucket-policy-resource' and 'q/5516/8.6/bugfix/CLDSRV-490-bucket-policy-resource' into tmp/octopus/q/8.7 2024-01-05 11:24:58 +00:00
bert-e 64273365d5 Merge branches 'w/8.6/bugfix/CLDSRV-490-bucket-policy-resource' and 'q/5516/7.70/bugfix/CLDSRV-490-bucket-policy-resource' into tmp/octopus/q/8.6 2024-01-05 11:24:58 +00:00
bert-e 65c6bacd34 Merge branches 'w/7.70/bugfix/CLDSRV-490-bucket-policy-resource' and 'q/5516/7.10/bugfix/CLDSRV-490-bucket-policy-resource' into tmp/octopus/q/7.70 2024-01-05 11:24:57 +00:00
bert-e d60d252eaf Merge branch 'bugfix/CLDSRV-490-bucket-policy-resource' into q/7.10 2024-01-05 11:24:57 +00:00
bert-e f31fe2f2bf Merge branch 'w/8.7/bugfix/CLDSRV-490-bucket-policy-resource' into tmp/octopus/w/8.8/bugfix/CLDSRV-490-bucket-policy-resource 2024-01-05 11:10:28 +00:00
bert-e ee47cece90 Merge branch 'w/8.6/bugfix/CLDSRV-490-bucket-policy-resource' into tmp/octopus/w/8.7/bugfix/CLDSRV-490-bucket-policy-resource 2024-01-05 11:10:28 +00:00
Mickael Bourgois 7a5cddacbc
Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-490-bucket-policy-resource' into w/8.6/bugfix/CLDSRV-490-bucket-policy-resource 2024-01-05 12:08:54 +01:00
Mickael Bourgois baa6203b57
Merge remote-tracking branch 'origin/bugfix/CLDSRV-490-bucket-policy-resource' into w/7.70/bugfix/CLDSRV-490-bucket-policy-resource 2024-01-05 12:04:25 +01:00
Mickael Bourgois 141056637b
CLDSRV-490: bump version 2024-01-05 11:51:49 +01:00
Mickael Bourgois 0f007e0489
CLDSRV-490: fix linting in tests for 8.6 2024-01-05 11:51:48 +01:00
Mickael Bourgois 2d50a76923
Merge remote-tracking branch 'origin/w/8.7/bugfix/CLDSRV-486-object-redirect-root' into w/8.8/bugfix/CLDSRV-486-object-redirect-root 2024-01-04 16:59:20 +01:00
Mickael Bourgois 6b4f10ae56
Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-486-object-redirect-root' into w/8.7/bugfix/CLDSRV-486-object-redirect-root 2024-01-04 16:57:36 +01:00
Mickael Bourgois 23eaf89cc3
Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-486-object-redirect-root' into w/8.6/bugfix/CLDSRV-486-object-redirect-root 2024-01-04 16:55:48 +01:00
Mickael Bourgois d6a2144508
Merge remote-tracking branch 'origin/bugfix/CLDSRV-486-object-redirect-root' into w/7.70/bugfix/CLDSRV-486-object-redirect-root 2024-01-04 16:49:20 +01:00
Mickael Bourgois 40dd3f37a4
Merge branch 'development/7.10' into bugfix/CLDSRV-486-object-redirect-root 2024-01-04 16:36:03 +01:00
Mickael Bourgois d3307654a6
CLDSRV-486: bump cloudserver version 2024-01-04 16:34:10 +01:00
Mickael Bourgois e342a90b48
CLDSRV-486: bump arsenal version 2024-01-04 16:29:04 +01:00
williamlardier dbda5f16a6 CLDSRV-407: bump mongodb to v5.0 in CI 2024-01-04 14:04:20 +01:00
Mickael Bourgois d4a4825668
CLDSRV-490: test bucket policy with request 2024-01-04 10:18:36 +01:00
Mickael Bourgois 83b9e9a775
CLDSRV-490: fix missing request for bucket policy
If request is missing, bucket policy ignore resource
and apply effect to any matching principal and action
2024-01-03 18:24:54 +01:00
Maha Benzekri 2959c950dd
Merge remote-tracking branch 'origin/w/8.7/bugfix/CLDSRV-480-ByPassGovernance-bucket-policy-tests' into w/8.8/bugfix/CLDSRV-480-ByPassGovernance-bucket-policy-tests 2024-01-03 10:36:20 +01:00
Maha Benzekri 462ddf7ef1
Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-480-ByPassGovernance-bucket-policy-tests' into w/8.7/bugfix/CLDSRV-480-ByPassGovernance-bucket-policy-tests 2024-01-03 10:34:44 +01:00
Maha Benzekri fda42e7399
Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-480-ByPassGovernance-bucket-policy-tests' into w/8.6/bugfix/CLDSRV-480-ByPassGovernance-bucket-policy-tests 2024-01-03 10:32:41 +01:00
Maha Benzekri edbd6caeb4
Merge remote-tracking branch 'origin/bugfix/CLDSRV-480-ByPassGovernance-bucket-policy-tests' into w/7.70/bugfix/CLDSRV-480-ByPassGovernance-bucket-policy-tests 2024-01-03 09:38:25 +01:00
Maha Benzekri 1befaa1f28
CLDSRV-480: CLDSRV version bump 2024-01-03 09:35:19 +01:00
Maha Benzekri 0cefca831d
CLDSRV-480: condition check fix for isImplicit 2024-01-03 09:34:19 +01:00
Jonathan Gramain ea7b69e313 Merge remote-tracking branch 'origin/w/8.7/bugfix/CLDSRV-478-bump-arsenal-dep' into w/8.8/bugfix/CLDSRV-478-bump-arsenal-dep 2024-01-02 15:26:27 -08:00
Jonathan Gramain 8ec1c2f2db Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-478-bump-arsenal-dep' into w/8.7/bugfix/CLDSRV-478-bump-arsenal-dep 2024-01-02 15:08:40 -08:00
Jonathan Gramain 3af6ca5f6d Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-478-bump-arsenal-dep' into w/8.6/bugfix/CLDSRV-478-bump-arsenal-dep 2024-01-02 15:06:45 -08:00
Jonathan Gramain 997d71df08 Merge remote-tracking branch 'origin/bugfix/CLDSRV-478-bump-arsenal-dep' into w/7.70/bugfix/CLDSRV-478-bump-arsenal-dep 2024-01-02 14:49:08 -08:00
Jonathan Gramain 275ebcec5c CLDSRV-478 bump cloudserver version 2024-01-02 14:45:56 -08:00
Mickael Bourgois 8b77530b2b
CLDSRV-486: fix object redirect to root / 2024-01-02 19:16:32 +01:00
bert-e 43f9606598 Merge branch 'w/8.7/bugfix/CLDSRV-478-bump-arsenal-dep' into tmp/octopus/w/8.8/bugfix/CLDSRV-478-bump-arsenal-dep 2024-01-02 17:45:02 +00:00
bert-e be34e5ad59 Merge branch 'w/8.6/bugfix/CLDSRV-478-bump-arsenal-dep' into tmp/octopus/w/8.7/bugfix/CLDSRV-478-bump-arsenal-dep 2024-01-02 17:45:01 +00:00
Jonathan Gramain 5bc64ede43 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-478-bump-arsenal-dep' into w/8.6/bugfix/CLDSRV-478-bump-arsenal-dep 2024-01-02 09:41:03 -08:00
Jonathan Gramain 911010376e Merge remote-tracking branch 'origin/bugfix/CLDSRV-478-bump-arsenal-dep' into w/7.70/bugfix/CLDSRV-478-bump-arsenal-dep 2024-01-02 09:26:30 -08:00
Jonathan Gramain b5ec37b38b bf: CLDSRV-478 bump arsenal dependency 2024-01-02 09:19:15 -08:00
Mickael Bourgois 3ce869cea3
Merge remote-tracking branch 'origin/w/8.7/bugfix/CLDSRV-479-website-fqdn-index' into w/8.8/bugfix/CLDSRV-479-website-fqdn-index
# Conflicts:
#	package.json
2024-01-02 11:40:28 +01:00
Mickael Bourgois b7960784db
Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-479-website-fqdn-index' into w/8.7/bugfix/CLDSRV-479-website-fqdn-index 2024-01-02 11:35:36 +01:00
Mickael Bourgois 5ac10cefa8
Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-479-website-fqdn-index' into w/8.6/bugfix/CLDSRV-479-website-fqdn-index 2024-01-02 11:33:49 +01:00
Mickael Bourgois 2dafefd77f
Merge remote-tracking branch 'origin/bugfix/CLDSRV-479-website-fqdn-index' into w/7.70/bugfix/CLDSRV-479-website-fqdn-index 2024-01-02 11:29:47 +01:00
Mickael Bourgois 36f147b441
CLDSRV-479: update test bucket policy index 2024-01-02 11:13:40 +01:00
Mickael Bourgois 8ed447ba63
CLDSRV-479: helper function for index append 2024-01-02 10:27:38 +01:00
bert-e bf235f3335 Merge branch 'w/8.7/bugfix/CLDSRV-483/ceph-tests' into tmp/octopus/w/8.8/bugfix/CLDSRV-483/ceph-tests 2023-12-31 10:39:53 +00:00
bert-e 569c9f4368 Merge branch 'bugfix/CLDSRV-483/ceph-tests' into tmp/octopus/w/8.7/bugfix/CLDSRV-483/ceph-tests 2023-12-31 10:39:52 +00:00
Nicolas Humbert 92cf03254a CLDSRV-483 Improve Ruby test output readability and Enable backtrace 2023-12-31 11:08:21 +01:00
Nicolas Humbert c57ae9c8ea CLDSRV-483 Bump ruby patch version to fix malformed header response
More info about the malformed header response: https://github.com/excon/excon/issues/845
2023-12-31 11:08:16 +01:00
Mickael Bourgois 5bec42d051
CLDSRV-479: test index with bucket policy 2023-12-29 17:43:34 +01:00
Mickael Bourgois f427fc9b70
CLDSRV-479: bump version 2023-12-28 15:20:59 +01:00
Mickael Bourgois 9aad4ae3ea
CLDSRV-479: fix error on index using bucket policy
The variable holding the new objectKey with index suffix
is not propagated to bucket policy function.
_checkBucketPolicyResources function extract objectKey from request.
2023-12-28 15:20:48 +01:00
bert-e 1a3cb8108c Merge branch 'q/5495/8.7/improvement/CLDSRV-451-specific-7.70-apis-update' into tmp/normal/q/8.8 2023-12-15 06:44:04 +00:00
bert-e 042120b17e Merge branch 'q/5495/8.6/improvement/CLDSRV-451-specific-7.70-apis-update' into tmp/normal/q/8.7 2023-12-15 06:44:04 +00:00
bert-e ba4593592d Merge branch 'w/8.8/improvement/CLDSRV-451-specific-7.70-apis-update' into tmp/normal/q/8.8 2023-12-15 06:44:04 +00:00
bert-e 6efdb627da Merge branch 'w/8.7/improvement/CLDSRV-451-specific-7.70-apis-update' into tmp/normal/q/8.7 2023-12-15 06:44:04 +00:00
bert-e 5306bf0b5c Merge branch 'q/5495/7.70/improvement/CLDSRV-451-specific-7.70-apis-update' into tmp/normal/q/8.6 2023-12-15 06:44:03 +00:00
bert-e 5b22819c3f Merge branch 'w/8.6/improvement/CLDSRV-451-specific-7.70-apis-update' into tmp/normal/q/8.6 2023-12-15 06:44:03 +00:00
bert-e 126ca3560f Merge branch 'improvement/CLDSRV-451-specific-7.70-apis-update' into q/7.70 2023-12-15 06:44:02 +00:00
bert-e e5b692f3db Merge branch 'w/8.7/improvement/CLDSRV-467/add_reindex_opt_only_count_latest' into tmp/octopus/w/8.8/improvement/CLDSRV-467/add_reindex_opt_only_count_latest 2023-12-14 18:30:49 +00:00
bert-e 548ae8cd12 Merge branch 'w/8.6/improvement/CLDSRV-467/add_reindex_opt_only_count_latest' into tmp/octopus/w/8.7/improvement/CLDSRV-467/add_reindex_opt_only_count_latest 2023-12-14 18:30:48 +00:00
Taylor McKinnon 80376405df Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-467/add_reindex_opt_only_count_latest' into w/8.6/improvement/CLDSRV-467/add_reindex_opt_only_count_latest 2023-12-14 10:30:13 -08:00
Taylor McKinnon a612e5c27c Merge remote-tracking branch 'origin/improvement/CLDSRV-467/add_reindex_opt_only_count_latest' into w/7.70/improvement/CLDSRV-467/add_reindex_opt_only_count_latest 2023-12-14 10:27:03 -08:00
Taylor McKinnon c3b7662086 impr(CLDSRV-467): Bump Utapi dependency to 7.10.15 2023-12-14 10:17:18 -08:00
Taylor McKinnon 818b1e60d1 impr(CLDSRV-467): Add new Utapi Reindex option `utapi.reindex.onlyCountLatestWhenObjectLocked` 2023-12-14 10:17:18 -08:00
bert-e 2a919af071 Merge branch 'w/8.7/improvement/CLDSRV-451-specific-7.70-apis-update' into tmp/octopus/w/8.8/improvement/CLDSRV-451-specific-7.70-apis-update 2023-12-14 17:21:47 +00:00
bert-e 5c300b8b6c Merge branch 'w/8.6/improvement/CLDSRV-451-specific-7.70-apis-update' into tmp/octopus/w/8.7/improvement/CLDSRV-451-specific-7.70-apis-update 2023-12-14 17:21:46 +00:00
Maha Benzekri ad3ebd3db2
CLDSRV-451: fix on gettagging 2023-12-14 18:21:24 +01:00
Maha Benzekri 99068e7265
Merge remote-tracking branch 'origin/w/8.7/improvement/CLDSRV-451-specific-7.70-apis-update' into w/8.8/improvement/CLDSRV-451-specific-7.70-apis-update 2023-12-14 17:36:17 +01:00
Maha Benzekri cd039d8133
Merge remote-tracking branch 'origin/w/8.6/improvement/CLDSRV-451-specific-7.70-apis-update' into w/8.7/improvement/CLDSRV-451-specific-7.70-apis-update
In this commit the only api change compared to the 8.6 is the
routeVeeam.
2023-12-14 17:33:03 +01:00
Maha Benzekri dd3ec25d74
Merge remote-tracking branch 'origin/improvement/CLDSRV-451-specific-7.70-apis-update' into w/8.6/improvement/CLDSRV-451-specific-7.70-apis-update
In this merge, we have updated the tagging apis along with the
lifecycle apis and metadata search apis and objectRestore, unit test
for objectRestore has been updated as well.
2023-12-14 17:28:46 +01:00
Maha Benzekri 717228bdfc
CLDSRV-451: bump Cloudserver version 2023-12-14 16:59:10 +01:00
Maha Benzekri 836fc80560
CLDSRV-451: updating buckettagging apis for impDeny 2023-12-14 16:58:14 +01:00
Maha Benzekri 75b293df8d
Merge remote-tracking branch 'origin/w/8.7/improvement/CLDSRV-431-misc-api-implicitDeny' into w/8.8/improvement/CLDSRV-431-misc-api-implicitDeny 2023-12-14 13:37:14 +01:00
Maha Benzekri a855e38998
Merge remote-tracking branch 'origin/w/8.6/improvement/CLDSRV-431-misc-api-implicitDeny' into w/8.7/improvement/CLDSRV-431-misc-api-implicitDeny 2023-12-14 13:35:02 +01:00
Maha Benzekri 51d5666bec
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-431-misc-api-implicitDeny' into w/8.6/improvement/CLDSRV-431-misc-api-implicitDeny 2023-12-14 13:32:36 +01:00
Maha Benzekri ecb74a2db3
Merge remote-tracking branch 'origin/improvement/CLDSRV-431-misc-api-implicitDeny' into w/7.70/improvement/CLDSRV-431-misc-api-implicitDeny 2023-12-14 13:26:57 +01:00
Maha Benzekri cdcdf8eff0
CLDSRV-431: cloudserver version bump 2023-12-14 12:22:42 +01:00
Maha Benzekri dc39b37877
CLDSRV-431: arsenal bump 2023-12-14 12:21:53 +01:00
Maha Benzekri 4897b3c720
CLDSRV-431: changes on misc api for impDeny 2023-12-13 11:14:21 +01:00
Maha Benzekri ffe4ea4afe
Merge remote-tracking branch 'origin/w/8.7/improvement/CLDSRV-474-fix-multiObjectDelete-api-aut' into w/8.8/improvement/CLDSRV-474-fix-multiObjectDelete-api-aut 2023-12-12 14:47:24 +01:00
Maha Benzekri a16cfad0fc
CLDSRV-474: mongodb_image on all jobs 2023-12-12 14:06:02 +01:00
bert-e 556163e3e9 Merge branch 'w/8.6/improvement/CLDSRV-474-fix-multiObjectDelete-api-aut' into tmp/octopus/w/8.7/improvement/CLDSRV-474-fix-multiObjectDelete-api-aut 2023-12-12 12:55:56 +00:00
Maha Benzekri 8fe9f16661
CLDSRV-474: Removing the docker-compose commands from the tests.yaml 2023-12-12 13:53:53 +01:00
Maha Benzekri eb9ff85bd9
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-474-fix-multiObjectDelete-api-aut' into w/8.6/improvement/CLDSRV-474-fix-multiObjectDelete-api-aut 2023-12-12 13:52:50 +01:00
bert-e 52994c0177 Merge branch 'improvement/CLDSRV-474-fix-multiObjectDelete-api-aut' into tmp/octopus/w/7.70/improvement/CLDSRV-474-fix-multiObjectDelete-api-aut 2023-12-12 12:44:56 +00:00
tmacro e109b0fca7
CLDSRV-474: fix CI fail 2023-12-12 10:21:01 +01:00
Maha Benzekri 9940699f9d
CLDSRV-474: fixup on mutiObjectDelete 2023-12-12 10:11:18 +01:00
Maha Benzekri 869d554e43
Merge remote-tracking branch 'origin/w/8.7/improvement/CLDSRV-430-delete-api-implicitDeny' into w/8.8/improvement/CLDSRV-430-delete-api-implicitDeny 2023-12-08 18:42:25 +01:00
Maha Benzekri 2f8b228595
Merge remote-tracking branch 'origin/w/8.6/improvement/CLDSRV-430-delete-api-implicitDeny' into w/8.7/improvement/CLDSRV-430-delete-api-implicitDeny 2023-12-08 18:39:20 +01:00
Maha Benzekri 539b2c1630
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-430-delete-api-implicitDeny' into w/8.6/improvement/CLDSRV-430-delete-api-implicitDeny 2023-12-08 18:35:11 +01:00
Maha Benzekri 320766e7b2
Merge remote-tracking branch 'origin/improvement/CLDSRV-430-delete-api-implicitDeny' into w/7.70/improvement/CLDSRV-430-delete-api-implicitDeny 2023-12-08 18:31:56 +01:00
Maha Benzekri 74425d03f8
CLDSRV-430: version bump 2023-12-08 18:29:19 +01:00
Maha Benzekri 91629a0d18
CLDSRV-430: add delete API implicit deny logic
As for multiObjectDelete,a new function was added to
ensure that all actions are allowed.
2023-12-08 18:29:17 +01:00
Maha Benzekri e44b7ed918
Merge remote-tracking branch 'origin/w/8.7/improvement/CLDSRV-429-get-apis-implicitDeny' into w/8.8/improvement/CLDSRV-429-get-apis-implicitDeny 2023-12-05 12:00:50 +01:00
Maha Benzekri 3cb29f7f8e
CLDSRV-429: version bump for version release 2023-12-05 12:00:09 +01:00
Maha Benzekri 4f08a4dff2
Merge remote-tracking branch 'origin/development/8.8' into w/8.8/improvement/CLDSRV-429-get-apis-implicitDeny 2023-12-05 11:58:27 +01:00
Maha Benzekri 15a1aa7965
Merge remote-tracking branch 'origin/development/8.7' into w/8.7/improvement/CLDSRV-429-get-apis-implicitDeny 2023-12-05 10:58:04 +01:00
Maha Benzekri 4470ee9125
CLDSRV-429: version bump for release 2023-12-05 10:55:31 +01:00
Francois Ferrand d8c12597ea
Release cloudserver 8.8.7
Issue: CLDSRV-471
2023-12-01 19:03:38 +01:00
Francois Ferrand c8eb9025fa
Merge remote-tracking branch 'origin/improvement/CLDSRV-471' into w/8.8/improvement/CLDSRV-471 2023-12-01 19:03:17 +01:00
Francois Ferrand 57e0f71e6a
Release cloudserver 8.7.33
Issue: CLDSRV-471
2023-12-01 19:01:30 +01:00
Francois Ferrand f22f920ee2
Bump arsenal 8.1.115
Issue: CLDSRV-471
2023-12-01 18:42:26 +01:00
Maha Benzekri ed1bb6301d
Merge remote-tracking branch 'origin/w/8.7/improvement/CLDSRV-429-get-apis-implicitDeny' into w/8.8/improvement/CLDSRV-429-get-apis-implicitDeny 2023-12-01 11:31:50 +01:00
Maha Benzekri 70dfa5b11b
Merge remote-tracking branch 'origin/w/8.6/improvement/CLDSRV-429-get-apis-implicitDeny' into w/8.7/improvement/CLDSRV-429-get-apis-implicitDeny 2023-12-01 11:29:14 +01:00
Maha Benzekri f17e7677fa
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-429-get-apis-implicitDeny' into w/8.6/improvement/CLDSRV-429-get-apis-implicitDeny 2023-12-01 11:27:44 +01:00
Maha Benzekri 63b00fef55
Merge remote-tracking branch 'origin/improvement/CLDSRV-429-get-apis-implicitDeny' into w/7.70/improvement/CLDSRV-429-get-apis-implicitDeny 2023-12-01 11:25:04 +01:00
Maha Benzekri b4f0d34abd
CLDSRV-429: version bump 2023-12-01 10:27:58 +01:00
Maha Benzekri e18f83ef0d
CLDSRV-429: update get apis with impDeny logic 2023-11-30 17:17:30 +01:00
Francois Ferrand a4e6f9d034
Add lifecycle restore duration metrics
Issue: CLDSRV-471
2023-11-30 14:55:01 +01:00
Maha Benzekri cf94b9de6a
Merge remote-tracking branch 'origin/w/8.7/improvement/CLDSRV-428-put-apis-impDeny' into w/8.8/improvement/CLDSRV-428-put-apis-impDeny 2023-11-29 16:23:08 +01:00
Maha Benzekri da0492d2bb
Merge remote-tracking branch 'origin/development/8.8' into w/8.8/improvement/CLDSRV-428-put-apis-impDeny 2023-11-29 16:22:32 +01:00
Maha Benzekri 979b9065ed
Merge remote-tracking branch 'origin/w/8.6/improvement/CLDSRV-428-put-apis-impDeny' into w/8.7/improvement/CLDSRV-428-put-apis-impDeny 2023-11-29 16:19:27 +01:00
Maha Benzekri d5a3923f74
Merge remote-tracking branch 'origin/development/8.7' into w/8.7/improvement/CLDSRV-428-put-apis-impDeny 2023-11-29 16:18:06 +01:00
Maha Benzekri 23cbbdaaed
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-428-put-apis-impDeny' into w/8.6/improvement/CLDSRV-428-put-apis-impDeny 2023-11-29 16:17:05 +01:00
Maha Benzekri e506dea140
Merge remote-tracking branch 'origin/development/8.6' into w/8.6/improvement/CLDSRV-428-put-apis-impDeny 2023-11-29 16:15:52 +01:00
Maha Benzekri 78721be7f7
Merge remote-tracking branch 'origin/improvement/CLDSRV-428-put-apis-impDeny' into w/7.70/improvement/CLDSRV-428-put-apis-impDeny 2023-11-29 16:10:26 +01:00
Maha Benzekri 02c5a46d14
Merge remote-tracking branch 'origin/development/7.70' into w/7.70/improvement/CLDSRV-428-put-apis-impDeny 2023-11-29 16:06:41 +01:00
Maha Benzekri b138955ef2
Merge remote-tracking branch 'origin/development/7.10' into HEAD 2023-11-29 16:01:54 +01:00
Maha Benzekri 7d10e5d69e
CLDSRV-428:Bump CLDSRV version 2023-11-29 15:59:28 +01:00
bert-e bc291fe3a7 Merge branches 'w/8.8/bugfix/CLDSRV-463/bump_cloudserver' and 'q/5444/8.7/bugfix/CLDSRV-463/bump_cloudserver' into tmp/octopus/q/8.8 2023-11-27 17:16:14 +00:00
bert-e 8dc7432c51 Merge branches 'w/8.7/bugfix/CLDSRV-463/bump_cloudserver' and 'q/5444/8.6/bugfix/CLDSRV-463/bump_cloudserver' into tmp/octopus/q/8.7 2023-11-27 17:16:14 +00:00
bert-e 040fe53e53 Merge branches 'w/8.6/bugfix/CLDSRV-463/bump_cloudserver' and 'q/5444/7.70/bugfix/CLDSRV-463/bump_cloudserver' into tmp/octopus/q/8.6 2023-11-27 17:16:13 +00:00
bert-e 60e350a5cf Merge branches 'w/7.70/bugfix/CLDSRV-463/bump_cloudserver' and 'q/5444/7.10/bugfix/CLDSRV-463/bump_cloudserver' into tmp/octopus/q/7.70 2023-11-27 17:16:13 +00:00
bert-e 5de00c80f8 Merge branch 'bugfix/CLDSRV-463/bump_cloudserver' into q/7.10 2023-11-27 17:16:13 +00:00
bert-e 6f963bdcd9 Merge branch 'w/8.7/improvement/CLDSRV-428-put-apis-impDeny' into tmp/octopus/w/8.8/improvement/CLDSRV-428-put-apis-impDeny 2023-11-27 16:02:56 +00:00
bert-e cd9024fd32 Merge branch 'w/8.6/improvement/CLDSRV-428-put-apis-impDeny' into tmp/octopus/w/8.7/improvement/CLDSRV-428-put-apis-impDeny 2023-11-27 16:02:55 +00:00
Maha Benzekri 37649bf49b
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-428-put-apis-impDeny' into w/8.6/improvement/CLDSRV-428-put-apis-impDeny 2023-11-27 17:01:43 +01:00
Maha Benzekri abf5ea33a9
Merge remote-tracking branch 'origin/improvement/CLDSRV-428-put-apis-impDeny' into w/7.70/improvement/CLDSRV-428-put-apis-impDeny 2023-11-27 16:59:09 +01:00
Maha Benzekri 2596f3fda8
CLDSRV-428: put apis updated for implicit deny
In this commit put apis have been updated to check for implicit deny
returned by vault and added as a parameter in the request Object.
Tests have also been added for the metadataUtils validateBucket
function.
MetadataUtils functions have been updated to check for implicit
deny.
The goal is to implement the same authorization
logic as AWS, where an implicit deny from IAM and an Allow from
the Bucket Policy should allow the request for example.
For the delete on the objectPutCopyPart and objectPutPart as we need to
deferentiate between the vault request and the external backend once
a delete is applied to the request directly as it's unique per API call
this value is then added to the request object. here's the link to the
design doc for more details:
https://github.com/scality/citadel/blob/development/1.0/docs/design/bucket-policies.md?plain=1#L263
2023-11-27 16:47:43 +01:00
bert-e dff7610060 Merge branch 'w/8.7/improvement/CLDSRV-427-permissions-checks' into tmp/octopus/w/8.8/improvement/CLDSRV-427-permissions-checks 2023-11-17 11:30:07 +00:00
bert-e 757c2537ef Merge branch 'w/8.6/improvement/CLDSRV-427-permissions-checks' into tmp/octopus/w/8.7/improvement/CLDSRV-427-permissions-checks 2023-11-17 11:30:06 +00:00
Maha Benzekri c445322685
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-427-permissions-checks' into w/8.6/improvement/CLDSRV-427-permissions-checks 2023-11-17 12:28:19 +01:00
bert-e 2344204746 Merge branch 'improvement/CLDSRV-427-permissions-checks' into tmp/octopus/w/7.70/improvement/CLDSRV-427-permissions-checks 2023-11-17 10:50:50 +00:00
Maha Benzekri 693ddf8d35
Merge branch 'development/7.10' into improvement/CLDSRV-427-permissions-checks 2023-11-17 11:40:17 +01:00
Maha Benzekri 6caa5cc26a
CLDSRV-427: Improving functions using helper function
- In this commit , I added a helper (processBucketPolicy) function
for the bycket policies checks that are shared between the
isbucketAuthorized, isObjAuthorized and evaluateBucketPolicyWithIAM
for a better code readability and to avoid long functions.

(cherry picked from commit 33d7c99e0c)
2023-11-17 11:36:22 +01:00
bert-e 4515b2adbf Merge branch 'w/8.7/bugfix/CLDSRV-463/bump_cloudserver' into tmp/octopus/w/8.8/bugfix/CLDSRV-463/bump_cloudserver 2023-11-16 23:26:50 +00:00
bert-e 50ffdd260b Merge branch 'w/8.6/bugfix/CLDSRV-463/bump_cloudserver' into tmp/octopus/w/8.7/bugfix/CLDSRV-463/bump_cloudserver 2023-11-16 23:26:50 +00:00
Taylor McKinnon 3836848c05 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-463/bump_cloudserver' into w/8.6/bugfix/CLDSRV-463/bump_cloudserver 2023-11-16 15:26:23 -08:00
Taylor McKinnon 813a1553d2 Merge remote-tracking branch 'origin/bugfix/CLDSRV-463/bump_cloudserver' into w/7.70/bugfix/CLDSRV-463/bump_cloudserver 2023-11-16 15:25:13 -08:00
Taylor McKinnon 1238cd809c bf(CLDSRV-463): Bump cloudserver to 7.10.34/7.70.31 2023-11-16 15:23:14 -08:00
bert-e b5f22d8c68 Merge branches 'w/8.8/bugfix/CLDSRV-463/strictly_check_algo_headers' and 'q/5403/8.7/bugfix/CLDSRV-463/strictly_check_algo_headers' into tmp/octopus/q/8.8 2023-11-16 19:43:14 +00:00
bert-e 68ff54d49a Merge branches 'w/8.7/bugfix/CLDSRV-463/strictly_check_algo_headers' and 'q/5403/8.6/bugfix/CLDSRV-463/strictly_check_algo_headers' into tmp/octopus/q/8.7 2023-11-16 19:43:13 +00:00
bert-e a74b3eacf8 Merge branches 'w/8.6/bugfix/CLDSRV-463/strictly_check_algo_headers' and 'q/5403/7.70/bugfix/CLDSRV-463/strictly_check_algo_headers' into tmp/octopus/q/8.6 2023-11-16 19:43:13 +00:00
bert-e f00a2f2d9e Merge branch 'q/5403/7.10/bugfix/CLDSRV-463/strictly_check_algo_headers' into tmp/normal/q/7.70 2023-11-16 19:43:13 +00:00
bert-e 02bb60253a Merge branch 'bugfix/CLDSRV-463/strictly_check_algo_headers' into q/7.10 2023-11-16 19:43:12 +00:00
bert-e 3fe5579c80 Merge branch 'w/8.7/bugfix/CLDSRV-463/strictly_check_algo_headers' into tmp/octopus/w/8.8/bugfix/CLDSRV-463/strictly_check_algo_headers 2023-11-16 19:25:08 +00:00
bert-e 3fdd2bce21 Merge branch 'w/8.6/bugfix/CLDSRV-463/strictly_check_algo_headers' into tmp/octopus/w/8.7/bugfix/CLDSRV-463/strictly_check_algo_headers 2023-11-16 19:25:07 +00:00
Taylor McKinnon 44e6eb2550 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-463/strictly_check_algo_headers' into w/8.6/bugfix/CLDSRV-463/strictly_check_algo_headers 2023-11-16 11:20:55 -08:00
Taylor McKinnon c148c770ac Merge remote-tracking branch 'origin/bugfix/CLDSRV-463/strictly_check_algo_headers' into w/7.70/bugfix/CLDSRV-463/strictly_check_algo_headers 2023-11-16 11:17:40 -08:00
Maha Benzekri fa2f877825
CLDSRV-427: linting fixups and retrocompatibility changes
(cherry picked from commit a7396a721c)
2023-11-15 11:26:02 +01:00
Will Toozs 0e323fbefe
CLDSRV-427: update bucket/object perm checks to account for implicit …
…denies

(cherry picked from commit c01898f1a0)
(cherry picked from commit 7aa326cba9)
2023-11-15 11:26:01 +01:00
bert-e c9b512174f Merge branches 'w/8.8/bugfix/CLDSRV-460-forward-system-signals' and 'q/5431/8.7/bugfix/CLDSRV-460-forward-system-signals' into tmp/octopus/q/8.8 2023-11-15 10:14:18 +00:00
bert-e 7b48624cf7 Merge branch 'bugfix/CLDSRV-460-forward-system-signals' into q/8.7 2023-11-15 10:14:17 +00:00
bert-e 55b07def2e Merge branch 'bugfix/CLDSRV-460-forward-system-signals' into tmp/octopus/w/8.8/bugfix/CLDSRV-460-forward-system-signals 2023-11-15 09:43:35 +00:00
bert-e 62ae2b2c69 Merge branch 'w/7.70/improvement/CLDSRV-468-version-bump' into tmp/octopus/w/8.6/improvement/CLDSRV-468-version-bump 2023-11-14 11:06:35 +00:00
bert-e fcc9468b63 Merge branch 'w/8.6/improvement/CLDSRV-468-version-bump' into tmp/octopus/w/8.7/improvement/CLDSRV-468-version-bump 2023-11-14 11:06:35 +00:00
bert-e efc44a620d Merge branch 'w/8.7/improvement/CLDSRV-468-version-bump' into tmp/octopus/w/8.8/improvement/CLDSRV-468-version-bump 2023-11-14 11:06:35 +00:00
Maha Benzekri 72342f6654
Merge remote-tracking branch 'origin/improvement/CLDSRV-468-version-bump' into w/7.70/improvement/CLDSRV-468-version-bump 2023-11-14 12:06:02 +01:00
Maha Benzekri fa11e58d57
CLDSRV-468:CLDSRV version bump 2023-11-14 11:57:09 +01:00
bert-e 1bc19b39d7 Merge branches 'w/8.7/improvement/CLDSRV-466/timestamps_in_stderr' and 'q/5406/8.6/improvement/CLDSRV-466/timestamps_in_stderr' into tmp/octopus/q/8.7 2023-11-13 17:20:17 +00:00
bert-e b5fa3a1fd3 Merge branches 'w/8.8/improvement/CLDSRV-466/timestamps_in_stderr' and 'q/5406/8.7/improvement/CLDSRV-466/timestamps_in_stderr' into tmp/octopus/q/8.8 2023-11-13 17:20:17 +00:00
bert-e 68a6fc659c Merge branches 'w/8.6/improvement/CLDSRV-466/timestamps_in_stderr' and 'q/5406/7.70/improvement/CLDSRV-466/timestamps_in_stderr' into tmp/octopus/q/8.6 2023-11-13 17:20:16 +00:00
bert-e 2624a05018 Merge branches 'w/7.70/improvement/CLDSRV-466/timestamps_in_stderr' and 'q/5406/7.10/improvement/CLDSRV-466/timestamps_in_stderr' into tmp/octopus/q/7.70 2023-11-13 17:20:16 +00:00
bert-e 0882bfffb9 Merge branch 'improvement/CLDSRV-466/timestamps_in_stderr' into q/7.10 2023-11-13 17:20:15 +00:00
bert-e c0fc958365 Merge branch 'w/8.7/improvement/CLDSRV-426-acl-impl-deny' into tmp/octopus/w/8.8/improvement/CLDSRV-426-acl-impl-deny 2023-11-13 16:03:26 +00:00
bert-e d3c74d2c16 Merge branch 'w/8.6/improvement/CLDSRV-426-acl-impl-deny' into tmp/octopus/w/8.7/improvement/CLDSRV-426-acl-impl-deny 2023-11-13 16:03:25 +00:00
Maha Benzekri 9001285177
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-426-acl-impl-deny' into w/8.6/improvement/CLDSRV-426-acl-impl-deny 2023-11-13 17:02:22 +01:00
bert-e bae6e8ecb3 Merge branch 'improvement/CLDSRV-426-acl-impl-deny' into tmp/octopus/w/7.70/improvement/CLDSRV-426-acl-impl-deny 2023-11-13 15:56:01 +00:00
Will Toozs e0eab954aa
CLDSRV-426: add tests for ACL permission check updates
CLDSRV-426: additionnal test for ACL permission
2023-11-13 13:10:38 +01:00
Will Toozs 19b4e25373
CLDSRV-426: update ACL permission checks for implicitDeny logic
CLDSRV-426:fixups on ACL permission checks for implicitDeny logic

CLDSRV-426:better readability on ACL permission
2023-11-13 13:10:37 +01:00
Kerkesni 07eda89a3f
forward system signals to the node process using tini
npm run doesn’t handle signal forwarding and crashes
on the SIGTERM signal sent by Kubernetes.

Tini spawns a process at PID 1 that handles forwarding
system signals to all it's child processes.

Issue: CLDSRV-460
2023-11-13 12:07:29 +01:00
bert-e 27b4066ca4 Merge branch 'w/8.6/improvement/CLDSRV-466/timestamps_in_stderr' into tmp/octopus/w/8.7/improvement/CLDSRV-466/timestamps_in_stderr 2023-11-10 16:18:45 +00:00
bert-e 2ee5b356fa Merge branch 'w/8.7/improvement/CLDSRV-466/timestamps_in_stderr' into tmp/octopus/w/8.8/improvement/CLDSRV-466/timestamps_in_stderr 2023-11-10 16:18:45 +00:00
bert-e 233955a0d3 Merge branch 'w/7.70/improvement/CLDSRV-466/timestamps_in_stderr' into tmp/octopus/w/8.6/improvement/CLDSRV-466/timestamps_in_stderr 2023-11-10 16:18:44 +00:00
bert-e ab51522110 Merge branch 'improvement/CLDSRV-466/timestamps_in_stderr' into tmp/octopus/w/7.70/improvement/CLDSRV-466/timestamps_in_stderr 2023-11-10 16:18:44 +00:00
Rahul Padigela b1b2d2ada6 improvement CLDSRV-466 add timestamp for exceptions 2023-11-10 08:17:34 -08:00
bert-e f5d3433413 Merge branches 'w/8.8/improvement/CLDSRV-464/support_mpu_scuba' and 'q/5405/8.7/improvement/CLDSRV-464/support_mpu_scuba' into tmp/octopus/q/8.8 2023-11-09 17:31:36 +00:00
bert-e 62b4b9bc25 Merge branches 'w/8.7/improvement/CLDSRV-464/support_mpu_scuba' and 'q/5405/8.6/improvement/CLDSRV-464/support_mpu_scuba' into tmp/octopus/q/8.7 2023-11-09 17:31:35 +00:00
bert-e ce4b2b5a27 Merge branches 'w/8.6/improvement/CLDSRV-464/support_mpu_scuba' and 'q/5405/7.70/improvement/CLDSRV-464/support_mpu_scuba' into tmp/octopus/q/8.6 2023-11-09 17:31:34 +00:00
bert-e 96bd67ee60 Merge branch 'improvement/CLDSRV-464/support_mpu_scuba' into q/7.70 2023-11-09 17:31:34 +00:00
bert-e ec56c77881 Merge branch 'w/8.7/improvement/CLDSRV-464/support_mpu_scuba' into tmp/octopus/w/8.8/improvement/CLDSRV-464/support_mpu_scuba 2023-11-08 17:19:07 +00:00
bert-e d0abde3962 Merge branch 'w/8.6/improvement/CLDSRV-464/support_mpu_scuba' into tmp/octopus/w/8.7/improvement/CLDSRV-464/support_mpu_scuba 2023-11-08 17:19:07 +00:00
bert-e f08a3f434b Merge branch 'improvement/CLDSRV-464/support_mpu_scuba' into tmp/octopus/w/8.6/improvement/CLDSRV-464/support_mpu_scuba 2023-11-08 17:19:06 +00:00
bert-e fdc682f2db Merge branches 'w/8.8/improvement/CLDSRV-424-apicall-auth-update' and 'q/5322/8.7/improvement/CLDSRV-424-apicall-auth-update' into tmp/octopus/q/8.8 2023-11-07 09:32:43 +00:00
bert-e b184606dc2 Merge branches 'w/8.7/improvement/CLDSRV-424-apicall-auth-update' and 'q/5322/8.6/improvement/CLDSRV-424-apicall-auth-update' into tmp/octopus/q/8.7 2023-11-07 09:32:43 +00:00
bert-e 172ec4a714 Merge branches 'w/8.6/improvement/CLDSRV-424-apicall-auth-update' and 'q/5322/7.70/improvement/CLDSRV-424-apicall-auth-update' into tmp/octopus/q/8.6 2023-11-07 09:32:42 +00:00
bert-e ae770d0d3f Merge branch 'improvement/CLDSRV-424-apicall-auth-update' into q/7.10 2023-11-07 09:32:41 +00:00
bert-e 7d2613e9a3 Merge branches 'w/7.70/improvement/CLDSRV-424-apicall-auth-update' and 'q/5322/7.10/improvement/CLDSRV-424-apicall-auth-update' into tmp/octopus/q/7.70 2023-11-07 09:32:41 +00:00
Maha Benzekri 9ce0f2c2b6
Merge remote-tracking branch 'origin/w/8.7/improvement/CLDSRV-424-apicall-auth-update' into w/8.8/improvement/CLDSRV-424-apicall-auth-update 2023-11-07 09:20:41 +01:00
Maha Benzekri 43b4e0c713
Merge remote-tracking branch 'origin/w/8.6/improvement/CLDSRV-424-apicall-auth-update' into w/8.7/improvement/CLDSRV-424-apicall-auth-update 2023-11-07 09:18:48 +01:00
Maha Benzekri 2bda761518
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-424-apicall-auth-update' into w/8.6/improvement/CLDSRV-424-apicall-auth-update 2023-11-07 09:16:48 +01:00
Maha Benzekri bfc9ca68c9
Merge remote-tracking branch 'origin/improvement/CLDSRV-424-apicall-auth-update' into w/7.70/improvement/CLDSRV-424-apicall-auth-update 2023-11-07 09:13:45 +01:00
Maha Benzekri 6abb0d96a9
CLDSRV-424:CLDSRV version bump
Update lib/api/api.js

Co-authored-by: Jonathan Gramain <jonathan.gramain@scality.com>
2023-11-07 09:06:23 +01:00
Maha Benzekri 733f424a4b
CLDSRV-424:ARSN version bump 2023-11-03 12:39:09 +01:00
Will Toozs 8d4ff7df5f
CLDSRV-424: api call updated with implicit deny logic
change variable names for clarity

edit: update arsenal package
2023-11-03 12:39:01 +01:00
Taylor McKinnon 59b87479df possible => unsupported 2023-11-01 10:17:20 -07:00
Taylor McKinnon 967ab966fa impr(CLDSRV-464): Add owner-id to mpu part metadata 2023-11-01 09:06:20 -07:00
Taylor McKinnon 212c7f506c impr(CLDSRV-464): Pass overhead fields for complete MPU and subsequent part bath delete 2023-11-01 09:04:32 -07:00
Taylor McKinnon 1e9ee0ef0b bf(CLDSRV-463): Strictly validate checksum algorithm headers 2023-10-30 10:54:34 -07:00
bert-e 9185f16554 Merge branch 'w/8.7/bugfix/CLDSRV-462/tags' into tmp/octopus/w/8.8/bugfix/CLDSRV-462/tags 2023-10-25 18:44:17 +00:00
bert-e 2df9a57f9c Merge branch 'w/8.6/bugfix/CLDSRV-462/tags' into tmp/octopus/w/8.7/bugfix/CLDSRV-462/tags 2023-10-25 18:44:17 +00:00
Nicolas Humbert c96706ff28 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-462/tags' into w/8.6/bugfix/CLDSRV-462/tags 2023-10-25 20:42:14 +02:00
Nicolas Humbert daa6f46b14 Merge remote-tracking branch 'origin/bugfix/CLDSRV-462/tags' into w/7.70/bugfix/CLDSRV-462/tags 2023-10-25 20:21:40 +02:00
Nicolas Humbert 44315057df CLDSRV-462 bump project version 2023-10-25 19:59:47 +02:00
Nicolas Humbert 61fe64a3ac CLDSRV-462 Expiration header is not compatible with legacy object md
Before the Object Metadata refactor done around May 31, 2017 (c22e44f63d), if no tags were set, the object tag was stored as undefined.

After the commit, if no tags are set, the object tag is stored as an empty object '{}'.

When the expiration response headers were implemented on 812b09afef around Nov 22, 2021, the empty object was handled, but not the undefined tag logic, which made the expiration response headers not backward compatible.

We need to address both cases: the undefined property and the empty object '{}'.
2023-10-25 19:59:07 +02:00
bert-e 68535f83d6 Merge branches 'w/8.8/bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion' and 'q/5384/8.7/bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion' into tmp/octopus/q/8.8 2023-10-24 18:40:33 +00:00
bert-e 41d63650be Merge branches 'w/8.7/bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion' and 'q/5384/8.6/bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion' into tmp/octopus/q/8.7 2023-10-24 18:40:32 +00:00
bert-e 4ebb5d449a Merge branches 'w/8.6/bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion' and 'q/5384/7.70/bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion' into tmp/octopus/q/8.6 2023-10-24 18:40:32 +00:00
bert-e 48abedc6f7 Merge branch 'bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion' into q/7.70 2023-10-24 18:40:31 +00:00
bert-e 12185f7c3b Merge branches 'w/8.8/improvement/CLDSRV-449/pass_overhead_fields' and 'q/5354/8.7/improvement/CLDSRV-449/pass_overhead_fields' into tmp/octopus/q/8.8 2023-10-19 20:36:18 +00:00
bert-e 5f82ee2d0e Merge branches 'w/8.7/improvement/CLDSRV-449/pass_overhead_fields' and 'q/5354/8.6/improvement/CLDSRV-449/pass_overhead_fields' into tmp/octopus/q/8.7 2023-10-19 20:36:18 +00:00
bert-e 7e0f9c63fe Merge branches 'w/8.6/improvement/CLDSRV-449/pass_overhead_fields' and 'q/5354/7.70/improvement/CLDSRV-449/pass_overhead_fields' into tmp/octopus/q/8.6 2023-10-19 20:36:17 +00:00
bert-e 9f5ac17357 Merge branch 'improvement/CLDSRV-449/pass_overhead_fields' into q/7.70 2023-10-19 20:36:17 +00:00
Taylor McKinnon d72bc5c6b9 Merge remote-tracking branch 'origin/w/8.7/improvement/CLDSRV-449/pass_overhead_fields' into w/8.8/improvement/CLDSRV-449/pass_overhead_fields 2023-10-19 13:16:26 -07:00
Taylor McKinnon 0e47810963 Merge remote-tracking branch 'origin/w/8.6/improvement/CLDSRV-449/pass_overhead_fields' into w/8.7/improvement/CLDSRV-449/pass_overhead_fields 2023-10-19 12:40:23 -07:00
Taylor McKinnon 8d83546ee3 Merge remote-tracking branch 'origin/improvement/CLDSRV-449/pass_overhead_fields' into w/8.6/improvement/CLDSRV-449/pass_overhead_fields 2023-10-16 12:01:03 -07:00
Taylor McKinnon fff4fd5d22 impr(CLDSRV-449): Add unit tests for overheadField param 2023-10-16 11:13:49 -07:00
Taylor McKinnon 1016a6826d impr(CLDSRV-449): Pass overheadField to Metadata in API handlers 2023-10-16 11:13:49 -07:00
bert-e 3b36cef85f Merge branch 'w/8.7/bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion' into tmp/octopus/w/8.8/bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion 2023-10-11 18:57:46 +00:00
Jonathan Gramain 114b885c7f Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion' into w/8.7/bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion 2023-10-11 11:35:23 -07:00
Jonathan Gramain e56d4e3744 Merge remote-tracking branch 'origin/bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion' into w/8.6/bugfix/CLDSRV-458-fixBucketdParamsUpdatingLegacyNullVersion 2023-10-11 11:05:32 -07:00
Jonathan Gramain 15144e4adf CLDSRV-458 bump cloudserver version 2023-10-11 11:03:02 -07:00
Jonathan Gramain 3985e2a712 bf: CLDSRV-458 fix bucketd params on null version update
On in-place updates of "legacy" null versions (those without the
"isNull2" attribute, using the "nullVersionId" chain instead of null
keys), we mustn't pass the "isNull" query parameter when sending the
update request to bucketd. Otherwise, it creates a null key which
causes issues when deleting the null version later.

Use a helper to pass the right set of parameters in all request types
that update versions in-place.
2023-10-11 10:59:56 -07:00
williamlardier 3b95c033d2 Merge remote-tracking branch 'origin/bugfix/CLDSRV-457-fix-memory-leak-in-arsenal' into w/8.8/bugfix/CLDSRV-457-fix-memory-leak-in-arsenal 2023-10-06 17:59:22 +02:00
williamlardier 04091dc316 CLDSRV-457: bump project version 2023-10-06 14:54:35 +02:00
williamlardier 56023a80ed CLDSRV-457: bump arsenal 2023-10-06 14:54:34 +02:00
bert-e 2deaebd89a Merge branch 'w/8.7/bugfix/CLDSRV-455/skip' into tmp/octopus/w/8.8/bugfix/CLDSRV-455/skip 2023-10-05 16:41:46 +00:00
bert-e c706ccf9c6 Merge branch 'w/8.6/bugfix/CLDSRV-455/skip' into tmp/octopus/w/8.7/bugfix/CLDSRV-455/skip 2023-10-05 16:41:45 +00:00
Nicolas Humbert 4afb2476f8 Merge remote-tracking branch 'origin/bugfix/CLDSRV-455/skip' into w/8.6/bugfix/CLDSRV-455/skip 2023-10-05 18:21:54 +02:00
Nicolas Humbert 91a7e7f24f CLDSRV-455 orphan delete marker list interruption skips processed key
The key marker in the orphan delete marker listing response should match the last key in the response's key array.
This ensures that the next listing begins after the key that has already been returned.
2023-10-05 15:55:45 +02:00
Taylor McKinnon 2f344cde70 impr(CLDSRV-449): Pass overheadField through helper functions to MetadataWrapper 2023-10-04 15:04:10 -07:00
Taylor McKinnon ad154085ac impr(CLDSRV-449): Use correct method in log message 2023-10-04 15:04:10 -07:00
Francois Ferrand 583ea8490f
Bump 8.8.3
Issue: CLDSRV-454
2023-10-04 11:18:25 +02:00
bert-e 85a9480793 Merge branch 'w/8.8/improvement/CLDSRV-446/bump' into tmp/octopus/q/8.8 2023-10-03 10:44:50 +00:00
bert-e be2f65b69e Merge branch 'bugfix/CLDSRV-423-test-sproxyd' into q/8.8 2023-10-03 10:12:16 +00:00
bert-e 1ee6d0a87d Merge branch 'w/8.7/improvement/CLDSRV-446/bump' into tmp/octopus/w/8.8/improvement/CLDSRV-446/bump 2023-10-02 15:25:13 +00:00
bert-e 224af9a5d2 Merge branch 'w/8.6/improvement/CLDSRV-446/bump' into tmp/octopus/w/8.7/improvement/CLDSRV-446/bump 2023-10-02 15:25:12 +00:00
Nicolas Humbert 9e2ad48c5c Merge remote-tracking branch 'origin/improvement/CLDSRV-446/bump' into w/8.6/improvement/CLDSRV-446/bump 2023-10-02 17:12:32 +02:00
Nicolas Humbert 780971ce10 CLDSRV-446 bump version 2023-10-02 17:08:41 +02:00
bert-e 74f05377f0 Merge branch 'w/8.7/improvement/CLDSRV-446/listing-scanned-limit' into tmp/octopus/w/8.8/improvement/CLDSRV-446/listing-scanned-limit 2023-10-02 13:38:08 +00:00
bert-e 111e14cc89 Merge branch 'w/8.6/improvement/CLDSRV-446/listing-scanned-limit' into tmp/octopus/w/8.7/improvement/CLDSRV-446/listing-scanned-limit 2023-10-02 13:38:07 +00:00
Nicolas Humbert fd6fb5a26c Merge remote-tracking branch 'origin/improvement/CLDSRV-446/listing-scanned-limit' into w/8.6/improvement/CLDSRV-446/listing-scanned-limit 2023-10-02 15:30:02 +02:00
Nicolas Humbert 8df540dcc1 CLDSRV-446 Limiting entries scanned during the lifecycle listing 2023-10-02 15:08:37 +02:00
Florent Monjalet 00b20f00d1 Merge remote-tracking branch 'origin/development/8.8' into bugfix/CLDSRV-423-test-sproxyd 2023-10-02 13:45:58 +02:00
Florent Monjalet a91d53a12c CLDSRV-423: test distinct and overwriting PUTs 2023-09-27 11:58:20 +02:00
Florent Monjalet 63d2637046 CLDSRV-423: improve async series usage in test 2023-09-27 11:50:44 +02:00
Maha Benzekri 5d416ad190
Merge remote-tracking branch 'origin/w/8.7/bugfix/CLDSRV-444-id-resource-policy' into w/8.8/bugfix/CLDSRV-444-id-resource-policy 2023-09-27 11:49:03 +02:00
Maha Benzekri ff29cda03f
Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-444-id-resource-policy' into w/8.7/bugfix/CLDSRV-444-id-resource-policy 2023-09-27 11:47:33 +02:00
Maha Benzekri 5685b2e972
Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-444-id-resource-policy' into w/8.6/bugfix/CLDSRV-444-id-resource-policy 2023-09-27 11:45:19 +02:00
Maha Benzekri 4e4ea2ab84
Merge remote-tracking branch 'origin/bugfix/CLDSRV-444-id-resource-policy' into w/7.70/bugfix/CLDSRV-444-id-resource-policy 2023-09-27 11:43:20 +02:00
Florent Monjalet cb8baf2dab CLDSRV-423: provide a proper dockerfile for test sproxyd 2023-09-27 11:36:49 +02:00
Maha Benzekri 67e5694d26
CLDSRV-447:CLDSRV version bump 2023-09-27 11:23:26 +02:00
bert-e 22f470c6eb Merge branch 'w/8.7/bugfix/CLDSRV-444-id-resource-policy' into tmp/octopus/w/8.8/bugfix/CLDSRV-444-id-resource-policy 2023-09-27 08:28:17 +00:00
bert-e e510473116 Merge branch 'w/8.6/bugfix/CLDSRV-444-id-resource-policy' into tmp/octopus/w/8.7/bugfix/CLDSRV-444-id-resource-policy 2023-09-27 08:28:16 +00:00
Maha Benzekri d046e8a294
Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-444-id-resource-policy' into w/8.6/bugfix/CLDSRV-444-id-resource-policy 2023-09-27 10:27:47 +02:00
Maha Benzekri 20a730788a
Merge remote-tracking branch 'origin/bugfix/CLDSRV-444-id-resource-policy' into w/7.70/bugfix/CLDSRV-444-id-resource-policy 2023-09-27 10:15:37 +02:00
Maha Benzekri 47958591ec
CLDSRV-447:ARSN version bump
fixes after reviews
2023-09-26 16:13:14 +02:00
Maha Benzekri 4195b6ae6a
CLDSRV-447:Test add for principal && arsn bump 2023-09-25 15:13:58 +02:00
Maha Benzekri feefd13b68
CLDSRV-444: unit test with Id and arsenal version 2023-09-12 22:51:04 +02:00
Florent Monjalet 17a6808fe4 CLDSRV-423: bump arsenal and sproxydclient to fix SPRXCLT-12 2023-08-31 19:07:44 +02:00
Florent Monjalet df646e4802 CLDSRV-423: disable failing tests that have just been reenabled
They had been disabled for a long while and cannot be reenabled yet
because they don't pass, so keep on skipping them for now.

Tickets have been created to take care of them:

- CLDSRV-440
- CLDSRV-441
- CLDSRV-442
- CLDSRV-443
2023-08-31 19:06:34 +02:00
Florent Monjalet 267770d256 CLDSRV-423: reproduce SPRXCLT-12 more often 2023-08-31 19:06:34 +02:00
Florent Monjalet 1b92dc2c05 CLDSRV-423: perform two successive put in multiple backend tests
This tests for SPRXCLT-12 issue
2023-08-31 19:06:34 +02:00
Florent Monjalet f80bb2f34b CLDSRV-423: don't run sproxyd test when testing Ceph 2023-08-31 19:06:34 +02:00
Florent Monjalet 4f89b67bb9 CLDSRV-423: Add missing mock logger method 2023-08-31 19:06:34 +02:00
Florent Monjalet 8b5630923c CLDSRV-423: refactor multiple backend put tests to avoid duplication 2023-08-31 19:06:34 +02:00
Florent Monjalet 9ff5e376e5 CLDSRV-423: reenable a good chunk of multiple backend tests 2023-08-31 19:06:34 +02:00
Florent Monjalet a9b5a2e3a4 CLDSRV-423: add put test for sproxyd 2023-08-31 19:06:34 +02:00
Florent Monjalet 7e9ec22ae3 CLDSRV-423: deploy sproxyd for multiple backend tests 2023-08-31 19:06:34 +02:00
bert-e 9d4664ae06 Merge branch 'w/8.7/bugfix/CLDSRV-439/bump_arsenal_for_bugfix' into tmp/octopus/w/8.8/bugfix/CLDSRV-439/bump_arsenal_for_bugfix 2023-08-30 16:44:30 +00:00
bert-e 662265ba2e Merge branch 'w/8.6/bugfix/CLDSRV-439/bump_arsenal_for_bugfix' into tmp/octopus/w/8.7/bugfix/CLDSRV-439/bump_arsenal_for_bugfix 2023-08-30 16:44:30 +00:00
bert-e c7da82dda7 Merge branch 'w/7.70/bugfix/CLDSRV-439/bump_arsenal_for_bugfix' into tmp/octopus/w/8.6/bugfix/CLDSRV-439/bump_arsenal_for_bugfix 2023-08-30 16:44:29 +00:00
Taylor McKinnon 960b4b2dd4 Merge remote-tracking branch 'origin/bugfix/CLDSRV-439/bump_arsenal_for_bugfix' into w/7.70/bugfix/CLDSRV-439/bump_arsenal_for_bugfix 2023-08-30 09:41:53 -07:00
Taylor McKinnon 1e9af343b9 bf(CLDSRV-439): Bump version to 7.10.30 2023-08-30 09:25:48 -07:00
Taylor McKinnon 8bb7338080 bf(CLDSRV-439): Bump arsenal to 7.10.47 2023-08-30 09:24:17 -07:00
Taylor McKinnon 17e4f14f9c Merge remote-tracking branch 'origin/w/8.7/bugfix/CLDSRV-413/bump_version' into w/8.8/bugfix/CLDSRV-413/bump_version 2023-08-18 10:10:01 -07:00
Taylor McKinnon 014b071536 Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-413/bump_version' into w/8.7/bugfix/CLDSRV-413/bump_version 2023-08-18 10:07:14 -07:00
Taylor McKinnon 9130f323d4 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-413/bump_version' into w/8.6/bugfix/CLDSRV-413/bump_version 2023-08-18 10:05:33 -07:00
Taylor McKinnon c09d3282dc Merge remote-tracking branch 'origin/bugfix/CLDSRV-413/bump_version' into w/7.70/bugfix/CLDSRV-413/bump_version 2023-08-18 09:54:43 -07:00
Taylor McKinnon fb9175579f bf(CLDSRV-413): Bump cloudserver version 2023-08-18 09:44:53 -07:00
bert-e 2d45f92ae1 Merge branches 'w/8.8/feature/CLDSRV-420/backport' and 'q/5268/8.7/feature/CLDSRV-420/backport' into tmp/octopus/q/8.8 2023-08-18 14:53:18 +00:00
bert-e 48452496fa Merge branches 'w/8.7/feature/CLDSRV-420/backport' and 'q/5268/8.6/feature/CLDSRV-420/backport' into tmp/octopus/q/8.7 2023-08-18 14:53:18 +00:00
bert-e b89773eba6 Merge branch 'q/5268/7.70/feature/CLDSRV-420/backport' into tmp/normal/q/8.6 2023-08-18 14:53:18 +00:00
bert-e c738e0924e Merge branch 'feature/CLDSRV-420/backport' into q/7.70 2023-08-18 14:53:16 +00:00
bert-e 18bf6b8d4a Merge branch 'w/8.7/feature/CLDSRV-420/backport' into tmp/octopus/w/8.8/feature/CLDSRV-420/backport 2023-08-18 11:19:15 +00:00
bert-e 858c31a542 Merge branch 'w/8.6/feature/CLDSRV-420/backport' into tmp/octopus/w/8.7/feature/CLDSRV-420/backport 2023-08-18 11:19:15 +00:00
Nicolas Humbert 75a759de27 Merge remote-tracking branch 'origin/feature/CLDSRV-420/backport' into w/8.6/feature/CLDSRV-420/backport 2023-08-18 12:57:48 +02:00
bert-e 19d3e0bc9d Merge branch 'w/8.7/bugfix/CLDSRV-413/crr_existing_null_version' into tmp/octopus/w/8.8/bugfix/CLDSRV-413/crr_existing_null_version 2023-08-17 17:03:00 +00:00
bert-e bac044dc8f Merge branch 'w/8.6/bugfix/CLDSRV-413/crr_existing_null_version' into tmp/octopus/w/8.7/bugfix/CLDSRV-413/crr_existing_null_version 2023-08-17 17:02:59 +00:00
Taylor McKinnon 8f77cd18c8 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-413/crr_existing_null_version' into w/8.6/bugfix/CLDSRV-413/crr_existing_null_version 2023-08-17 10:02:29 -07:00
bert-e cb7609b173 Merge branch 'bugfix/CLDSRV-413/crr_existing_null_version' into tmp/octopus/w/7.70/bugfix/CLDSRV-413/crr_existing_null_version 2023-08-17 17:00:57 +00:00
Taylor McKinnon 2926048735 bf(CLDSRV-413): Set isNull in objectMD fin backbeat putMetadata route for current null versions 2023-08-17 09:59:58 -07:00
Taylor McKinnon 656ef3fcee bf(CLDSRV-413): improve backbeat route testing setup cleanup 2023-08-16 14:25:31 -07:00
bert-e 8c0f709014 Merge branch 'bugfix/CLDSRV-422' into tmp/octopus/w/8.8/bugfix/CLDSRV-422 2023-08-16 11:46:43 +00:00
Francois Ferrand ce92d33a5d
Fix use of http_requests_total metrics
It was missed when metric names were updated. In addition, the dashboard
was not up-to-date with the python source, and needed to be regenerated.

Issue: CLDSRV-422
2023-08-14 14:36:14 +02:00
Kerkesni 0381cce85c
Merge remote-tracking branch 'origin/improvement/CLDSRV-408-Fix-metadata-getting-deleted-when-restoring' into w/8.8/improvement/CLDSRV-408-Fix-metadata-getting-deleted-when-restoring 2023-08-10 16:07:42 +02:00
Kerkesni 20a08a2a4e
bump version to 8.7.26 2023-08-10 16:04:25 +02:00
Kerkesni ff73d8ab12
add tests for keeping object properties after restore
Issue: CLDSRV-408
2023-08-10 16:03:58 +02:00
Kerkesni 1ee44bc6d3
keep same object properties after a restore of a cold object
Object properties such as ACLs and custom user metadata should
not be removed after the restore of a cold object.

Issue: CLDSRV-408
2023-08-10 12:58:18 +02:00
bert-e 614e876536 Merge branches 'w/8.8/improvement/CLDSRV-400' and 'q/5191/8.7/improvement/CLDSRV-400' into tmp/octopus/q/8.8 2023-08-09 16:42:42 +00:00
bert-e b40a77d94b Merge branch 'improvement/CLDSRV-400' into q/8.7 2023-08-09 16:42:42 +00:00
bert-e 3a3a73b756 Merge branch 'improvement/CLDSRV-400' into tmp/octopus/w/8.8/improvement/CLDSRV-400 2023-08-09 16:19:33 +00:00
Nicolas Humbert 6789959109 CLDSRV-420 add us-east-2 location 2023-08-09 10:14:47 -04:00
Nicolas Humbert bf9b53eea9 CLDSRV-420 fix linter 2023-08-09 10:14:47 -04:00
Nicolas Humbert aa04d23e68 CLDSRV-420 test v1 and v0 bucket format 2023-08-09 10:14:47 -04:00
Nicolas Humbert e08aaa7bcc CLDSRV-412 Test null version in Lifecycle list of non-current versions
(cherry picked from commit 98f855f997)
2023-08-09 10:14:47 -04:00
Nicolas Humbert 1d9c44126a CLDSRV-375 Exclude already transitioned keys from lifecycle listings
(cherry picked from commit 4c189b2d9e)
2023-08-09 10:14:47 -04:00
Nicolas Humbert 70a28ab620 CLDSRV-420 fix import metrics 2023-08-09 10:14:47 -04:00
Nicolas Humbert 550451eefa CLDSRV-372 Current lifecycle versions should include version id
(cherry picked from commit f7f77c6cd2)
2023-08-09 10:14:47 -04:00
Nicolas Humbert 96befd3f28 CLDSRV-371 ETag should be surrounded by double quotes
(cherry picked from commit f20bf1becf)
2023-08-09 10:14:47 -04:00
Nicolas Humbert 75288f1b56 CLDSRV-366 Clear list orphan delete markers response
(cherry picked from commit f2292f1ca3)
2023-08-09 10:14:47 -04:00
Nicolas Humbert 6847f2b0c4 CLDSRV-363 ETag instead of Etag for lifecycle listings Contents
(cherry picked from commit 049f52bf95)
2023-08-09 10:14:47 -04:00
Nicolas Humbert 050059548e CLDSRV-317 Implement listLifecycleOrphans
(cherry picked from commit ec9ed94555)
2023-08-09 10:14:47 -04:00
Nicolas Humbert 9b2a557a05 CLDSRV-316 Implement listLifecycleNonCurrents
(cherry picked from commit 41cc399d85)
2023-08-09 10:14:47 -04:00
Nicolas Humbert 7a7e2f4b91 CLDSRV-314 Implement listLifecycleCurrents
(cherry picked from commit 6b8a2581b6)
2023-08-09 10:14:42 -04:00
bert-e 3f6e85590d Merge branches 'w/8.8/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' and 'q/5243/8.7/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' into tmp/octopus/q/8.8 2023-08-07 17:27:19 +00:00
bert-e de589a07e8 Merge branches 'w/8.6/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' and 'q/5243/7.70/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' into tmp/octopus/q/8.6 2023-08-07 17:27:18 +00:00
bert-e bc009945d2 Merge branches 'w/8.7/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' and 'q/5243/8.6/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' into tmp/octopus/q/8.7 2023-08-07 17:27:18 +00:00
bert-e 8db04f4486 Merge branches 'w/7.70/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' and 'q/5243/7.10/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' into tmp/octopus/q/7.70 2023-08-07 17:27:17 +00:00
bert-e 328b7bc335 Merge branch 'bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' into q/7.10 2023-08-07 17:27:17 +00:00
bert-e 3ac30d9bab Merge branch 'w/8.7/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' into tmp/octopus/w/8.8/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710 2023-07-20 16:27:46 +00:00
bert-e 32204fbfbf Merge branch 'w/8.6/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' into tmp/octopus/w/8.7/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710 2023-07-20 16:27:46 +00:00
bert-e b1eda2a73a Merge branch 'w/7.70/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' into tmp/octopus/w/8.6/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710 2023-07-20 16:27:45 +00:00
bert-e 0249ad9bcf Merge branch 'bugfix/CLDSRV-418/CLDSRV_196_backport_to_710' into tmp/octopus/w/7.70/bugfix/CLDSRV-418/CLDSRV_196_backport_to_710 2023-07-20 16:27:45 +00:00
bert-e 5a26e1a80d Merge branch 'w/8.7/improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/8.8/improvement/CLDSRV-411-impose-last-modified 2023-07-20 08:27:00 +00:00
bert-e 507a2d4ff5 Merge branch 'w/8.6/improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/8.7/improvement/CLDSRV-411-impose-last-modified 2023-07-20 08:27:00 +00:00
bert-e 8cdd35950b Merge branch 'w/7.70/improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/8.6/improvement/CLDSRV-411-impose-last-modified 2023-07-20 08:26:59 +00:00
bert-e bfa366cd27 Merge branch 'improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/7.70/improvement/CLDSRV-411-impose-last-modified 2023-07-20 08:26:59 +00:00
Dimitrios Vasilas d132757696 CLDSRV-411: Add tests for imposing last-modified in testing mode 2023-07-20 09:58:09 +02:00
Alexander Chan 2a4be31b8a CLDSRV-196: create new werelogs object over using global werelogs
(cherry picked from commit 7fd547db24)
2023-07-19 12:25:55 -07:00
bert-e 1207a6fb70 Merge branch 'w/8.7/improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/8.8/improvement/CLDSRV-411-impose-last-modified 2023-07-19 08:45:28 +00:00
bert-e 5883286864 Merge branch 'w/8.6/improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/8.7/improvement/CLDSRV-411-impose-last-modified 2023-07-19 08:45:28 +00:00
bert-e b206728342 Merge branch 'w/7.70/improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/8.6/improvement/CLDSRV-411-impose-last-modified 2023-07-19 08:45:27 +00:00
bert-e 347a7391b9 Merge branch 'improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/7.70/improvement/CLDSRV-411-impose-last-modified 2023-07-19 08:45:27 +00:00
Dimitrios Vasilas 6273eebe66
CLDSRV-411: Use method to set last-modified
Co-authored-by: Jonathan Gramain <jonathan.gramain@scality.com>
2023-07-19 10:45:17 +02:00
bert-e 2a37e809d9 Merge branch 'w/8.7/improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/8.8/improvement/CLDSRV-411-impose-last-modified 2023-07-18 12:47:22 +00:00
bert-e 86ce7691cd Merge branch 'w/8.6/improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/8.7/improvement/CLDSRV-411-impose-last-modified 2023-07-18 12:47:21 +00:00
bert-e c04f663480 Merge branch 'w/7.70/improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/8.6/improvement/CLDSRV-411-impose-last-modified 2023-07-18 12:47:21 +00:00
Dimitrios Vasilas f2493e982f CLDSRV-411: Remove double import 2023-07-18 14:15:44 +02:00
bert-e e466b5e92a Merge branch 'w/8.7/improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/8.8/improvement/CLDSRV-411-impose-last-modified 2023-07-17 16:54:16 +00:00
bert-e a4bc10f730 Merge branch 'w/8.6/improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/8.7/improvement/CLDSRV-411-impose-last-modified 2023-07-17 16:54:15 +00:00
bert-e e826033bf0 Merge branch 'w/7.70/improvement/CLDSRV-411-impose-last-modified' into tmp/octopus/w/8.6/improvement/CLDSRV-411-impose-last-modified 2023-07-17 16:54:15 +00:00
Dimitrios Vasilas c23dad6fb8 Merge remote-tracking branch 'origin/improvement/CLDSRV-411-impose-last-modified' into w/7.70/improvement/CLDSRV-411-impose-last-modified 2023-07-17 18:50:30 +02:00
Dimitrios Vasilas 5fcdaa5a97 CLDSRV-411: Add mechanism for imposing the last-modified
When the configuration parameter "testingMode" is set to true,
a put can specify a custom last-modified date using the header
x-amz-meta-x-scal-last-modified.

This is intended to be used in tests only.
2023-07-17 18:40:18 +02:00
Dimitrios Vasilas 9f61ef9a3b CLDSRV-411: Add testing mode 2023-07-17 12:05:16 +02:00
Nicolas Humbert c480301e95 Merge remote-tracking branch 'origin/improvement/CLDSRV-414/bump' into w/8.8/improvement/CLDSRV-414/bump 2023-07-14 15:52:57 -04:00
Nicolas Humbert 276be285cc CLDSRV-414 bump version 2023-07-14 15:47:13 -04:00
bert-e 897d41392a Merge branch 'w/8.7/bugfix/CLDSRV-412/null' into tmp/octopus/w/8.8/bugfix/CLDSRV-412/null 2023-07-14 14:08:38 +00:00
bert-e f4e3a19d61 Merge branch 'bugfix/CLDSRV-412/null' into tmp/octopus/w/8.7/bugfix/CLDSRV-412/null 2023-07-14 14:08:37 +00:00
Nicolas Humbert ee84a03d2c bump arsenal 2023-07-14 09:49:30 -04:00
Nicolas Humbert 98f855f997 CLDSRV-412 Test null version in Lifecycle list of non-current versions 2023-07-14 09:48:47 -04:00
williamlardier 7c52fcbbb0
CLDSRV-402: bump project version 2023-07-13 17:45:06 +02:00
bert-e da52688a39 Merge branch 'w/8.7/improvement/CLDSRV-402-optimize-multiobjectdelete-api' into tmp/octopus/w/8.8/improvement/CLDSRV-402-optimize-multiobjectdelete-api 2023-07-13 15:12:26 +00:00
bert-e 1cb54a66f8 Merge branch 'w/8.6/improvement/CLDSRV-402-optimize-multiobjectdelete-api' into tmp/octopus/w/8.7/improvement/CLDSRV-402-optimize-multiobjectdelete-api 2023-07-13 15:12:25 +00:00
williamlardier 0bb61ddb5b
Merge remote-tracking branch 'origin/improvement/CLDSRV-402-optimize-multiobjectdelete-api' into w/8.6/improvement/CLDSRV-402-optimize-multiobjectdelete-api 2023-07-13 17:12:05 +02:00
williamlardier 68e4b0610a
CLDSRV-402: bump project version 2023-07-13 17:10:06 +02:00
bert-e d9fffdad9e Merge branch 'w/8.7/improvement/CLDSRV-402-optimize-multiobjectdelete-api' into tmp/octopus/w/8.8/improvement/CLDSRV-402-optimize-multiobjectdelete-api 2023-07-13 13:08:26 +00:00
williamlardier 389c32f819
Merge remote-tracking branch 'origin/w/8.6/improvement/CLDSRV-402-optimize-multiobjectdelete-api' into w/8.7/improvement/CLDSRV-402-optimize-multiobjectdelete-api 2023-07-13 15:06:34 +02:00
williamlardier c2df0bd3eb
Merge remote-tracking branch 'origin/improvement/CLDSRV-402-optimize-multiobjectdelete-api' into w/8.6/improvement/CLDSRV-402-optimize-multiobjectdelete-api 2023-07-13 14:22:48 +02:00
williamlardier af0436f1cd
CLDSRV-402: bump project version 2023-07-13 09:54:35 +02:00
williamlardier f7593d385e
CLDSRV-402: bump arsenal dependency 2023-07-13 09:54:35 +02:00
williamlardier 84068b205e
CLDSRV-402: test multi object delete optimization 2023-07-13 09:54:34 +02:00
williamlardier 9774d31b03
CLDSRV-402: optimize multideleteobject API
- Parallelism is increased to reduce the latency
- If the backend supports it, batching is used
- Batch the deletion of objects from storage
- Flag to disable or enable the optimization, as well as
  a way to tune it.
2023-07-13 09:54:34 +02:00
Kerkesni d26b8bcfcc
test keeping same storage class when restoring a cold object
Issue: CLDSRV-400
2023-06-23 11:22:10 +02:00
Kerkesni e4634621ee
keep storage class as cold for restored objects
To be compliant with the AWS S3 standard, the storage class
of restored objects should be left as cold location

Issue: CLDSRV-400
2023-06-23 11:22:10 +02:00
williamlardier 0b58b3ad2a
CLDSRV390: bump mongodb to 4.4 2023-06-22 16:56:53 +02:00
bert-e 652bf92536 Merge branch 'w/8.6/improvement/CLDSRV-409-fix-python-version' into tmp/octopus/w/8.7/improvement/CLDSRV-409-fix-python-version 2023-06-22 13:14:43 +00:00
bert-e c5b1ef63ee Merge branch 'w/7.70/improvement/CLDSRV-409-fix-python-version' into tmp/octopus/w/8.6/improvement/CLDSRV-409-fix-python-version 2023-06-22 13:14:42 +00:00
bert-e 227de16bca Merge branch 'improvement/CLDSRV-409-fix-python-version' into tmp/octopus/w/7.70/improvement/CLDSRV-409-fix-python-version 2023-06-22 13:14:41 +00:00
williamlardier c57a6e3c57
CLDSRV-409: fix s3cmd test 2023-06-22 15:14:24 +02:00
bert-e 344ee8a014 Merge branch 'w/8.6/improvement/CLDSRV-409-fix-python-version' into tmp/octopus/w/8.7/improvement/CLDSRV-409-fix-python-version 2023-06-22 12:44:35 +00:00
bert-e 5d7a434306 Merge branch 'w/7.70/improvement/CLDSRV-409-fix-python-version' into tmp/octopus/w/8.6/improvement/CLDSRV-409-fix-python-version 2023-06-22 12:44:35 +00:00
williamlardier 852ae72a13
Merge remote-tracking branch 'origin/improvement/CLDSRV-409-fix-python-version' into w/7.70/improvement/CLDSRV-409-fix-python-version 2023-06-22 14:43:18 +02:00
williamlardier 507782bd17
CLDSRV-409: remove virtualenv 2023-06-22 14:42:09 +02:00
bert-e b7e7f65d52 Merge branch 'w/8.6/improvement/CLDSRV-409-fix-python-version' into tmp/octopus/w/8.7/improvement/CLDSRV-409-fix-python-version 2023-06-22 12:37:53 +00:00
williamlardier d00320a8ba
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-409-fix-python-version' into w/8.6/improvement/CLDSRV-409-fix-python-version 2023-06-22 14:36:28 +02:00
williamlardier 4cf07193d9
Merge remote-tracking branch 'origin/improvement/CLDSRV-409-fix-python-version' into w/7.70/improvement/CLDSRV-409-fix-python-version 2023-06-22 14:34:03 +02:00
williamlardier aef272ea3c
CLDSRV-409: remove python 2.7 2023-06-22 14:30:43 +02:00
williamlardier 31d1734d5c
CLDSRV-409: use latest s3cmd with python3 2023-06-22 13:52:14 +02:00
bert-e c5b7450a4d Merge branches 'w/8.7/improvement/CLDSRV-388-implement-GHAS' and 'q/5169/8.6/improvement/CLDSRV-388-implement-GHAS' into tmp/octopus/q/8.7 2023-06-12 22:01:22 +00:00
bert-e eb5affdced Merge branches 'w/8.6/improvement/CLDSRV-388-implement-GHAS' and 'q/5169/8.5/improvement/CLDSRV-388-implement-GHAS' into tmp/octopus/q/8.6 2023-06-12 22:01:21 +00:00
bert-e cdaf6db929 Merge branches 'w/8.5/improvement/CLDSRV-388-implement-GHAS' and 'q/5169/7.70/improvement/CLDSRV-388-implement-GHAS' into tmp/octopus/q/8.5 2023-06-12 22:01:21 +00:00
bert-e 91ada795d0 Merge branches 'w/7.70/improvement/CLDSRV-388-implement-GHAS' and 'q/5169/7.10/improvement/CLDSRV-388-implement-GHAS' into tmp/octopus/q/7.70 2023-06-12 22:01:20 +00:00
bert-e 2b420a85e0 Merge branch 'w/7.10/improvement/CLDSRV-388-implement-GHAS' into tmp/octopus/q/7.10 2023-06-12 22:01:19 +00:00
Nicolas Humbert 18c8d4ecac CLDSRV-404 bump version 2023-06-09 11:48:56 -04:00
Nicolas Humbert c8150c6857 CLDSRV-397 Introduce the time-progression-factor flag
The "time-progression-factor" variable serves as a testing-specific feature that accelerates the progression of time within a system.
By reducing the significance of each day, it enables the swift execution of specific actions, such as expiration, transition, and object locking, which are typically associated with longer timeframes.

This capability allows for efficient testing and evaluation of outcomes, optimizing the observation of processes that would normally take days or even years.
It's important to note that this variable is intended exclusively for testing purposes and is not employed in live production environments, where real-time progression is crucial for accurate results.
2023-06-08 12:14:36 -04:00
bert-e 399a2a53ab Merge branch 'improvement/CLDSRV-399/addWorkflowDispatch' into q/8.7 2023-06-05 20:39:18 +00:00
Alexander Chan bbad049b5f CLDSRV-399: add workflow_dispatch 2023-06-05 11:30:35 -07:00
bert-e 2a4e2e1584 Merge branch 'w/8.6/improvement/CLDSRV-398/bump' into tmp/octopus/w/8.7/improvement/CLDSRV-398/bump 2023-06-02 20:19:28 +00:00
bert-e 08e43f5084 Merge branch 'w/8.5/improvement/CLDSRV-398/bump' into tmp/octopus/w/8.6/improvement/CLDSRV-398/bump 2023-06-02 20:19:27 +00:00
Nicolas Humbert cc153c99d6 Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-398/bump' into w/8.5/improvement/CLDSRV-398/bump 2023-06-02 15:58:58 -04:00
Nicolas Humbert d3f9870389 Merge remote-tracking branch 'origin/improvement/CLDSRV-398/bump' into w/7.70/improvement/CLDSRV-398/bump 2023-06-02 15:14:08 -04:00
Nicolas Humbert 0fa264693d CLDSRV-398 bump version 2023-06-02 15:05:47 -04:00
bert-e b304d05614 Merge branch 'w/8.6/bugfix/CLDSRV-396/put-metadata-null' into tmp/octopus/w/8.7/bugfix/CLDSRV-396/put-metadata-null 2023-06-02 18:31:27 +00:00
bert-e 751f6ce559 Merge branch 'w/8.5/bugfix/CLDSRV-396/put-metadata-null' into tmp/octopus/w/8.6/bugfix/CLDSRV-396/put-metadata-null 2023-06-02 18:31:26 +00:00
bert-e 0330597679 Merge branch 'w/7.70/bugfix/CLDSRV-396/put-metadata-null' into tmp/octopus/w/8.5/bugfix/CLDSRV-396/put-metadata-null 2023-06-02 18:31:26 +00:00
Nicolas Humbert 27cacc9552 CLDSRV-396 add nullVersionCompatMode condition 2023-06-02 14:14:05 -04:00
bert-e 004bd63368 Merge branch 'w/8.6/bugfix/CLDSRV-396/put-metadata-null' into tmp/octopus/w/8.7/bugfix/CLDSRV-396/put-metadata-null 2023-06-02 16:12:33 +00:00
bert-e e047ae6fbb Merge branch 'w/8.5/bugfix/CLDSRV-396/put-metadata-null' into tmp/octopus/w/8.6/bugfix/CLDSRV-396/put-metadata-null 2023-06-02 16:12:32 +00:00
Nicolas Humbert ebca8dd05e Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-396/put-metadata-null' into w/8.5/bugfix/CLDSRV-396/put-metadata-null 2023-06-02 12:09:50 -04:00
bert-e 52535fb44b Merge branch 'bugfix/CLDSRV-396/put-metadata-null' into tmp/octopus/w/7.70/bugfix/CLDSRV-396/put-metadata-null 2023-06-02 15:56:03 +00:00
Nicolas Humbert 1ed32b2cae CLDSRV-396 If put metadata for a null version, set options.isNull to true 2023-06-02 11:55:35 -04:00
Nicolas Humbert 960d736962 Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-396/put-metadata-null' into w/8.7/bugfix/CLDSRV-396/put-metadata-null 2023-06-01 11:24:34 -04:00
bert-e 11098dd113 Merge branch 'w/8.5/bugfix/CLDSRV-396/put-metadata-null' into tmp/octopus/w/8.6/bugfix/CLDSRV-396/put-metadata-null 2023-06-01 15:13:36 +00:00
Nicolas Humbert 9cc7362fbd Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-396/put-metadata-null' into w/8.5/bugfix/CLDSRV-396/put-metadata-null 2023-06-01 08:27:34 -04:00
KillianG 32401c9a83
bump 8.7.23 2023-05-30 09:40:36 +00:00
KillianG 5f05b676cc
Merge remote-tracking branch 'origin/development/8.7' into HEAD 2023-05-26 09:46:21 +00:00
KillianG fd662a8c2c
Bump arsenal 8.1.101 and test delete markers are not listed when bucket versionning is suspended
Issue: CLDSRV-347
2023-05-26 08:46:42 +00:00
bert-e a843d53939 Merge branch 'bugfix/CLDSRV-396/put-metadata-null' into tmp/octopus/w/7.70/bugfix/CLDSRV-396/put-metadata-null 2023-05-25 21:48:52 +00:00
Nicolas Humbert f889100798 CLDSRV-396 putMetadata API route is not updating null version properly 2023-05-25 17:42:47 -04:00
bert-e 5d54dd58be Merge branch 'bugfix/CLDSRV-393' into q/8.7 2023-05-25 19:47:24 +00:00
Nicolas Humbert 1bd0deafcf CLDSRV-395 bump to 8.7.21 2023-05-25 14:02:47 -04:00
Francois Ferrand 7c788d3dbf Bump github actions
Issue: CLDSRV-393
2023-05-25 14:02:47 -04:00
Nicolas Humbert 50cb6a2bf1 CLDSRV-374 putMetadata API route is not updating null version properly
Instead of using the provided "null" value, the metadata "null version id" is now used when updating the metadata of a null version.
2023-05-25 09:40:20 -04:00
bert-e 58f7bb2877 Merge branch 'w/8.6/improvement/CLDSRV-388-implement-GHAS' into tmp/octopus/w/8.7/improvement/CLDSRV-388-implement-GHAS 2023-05-24 22:42:13 +00:00
bert-e f899337284 Merge branch 'w/8.5/improvement/CLDSRV-388-implement-GHAS' into tmp/octopus/w/8.6/improvement/CLDSRV-388-implement-GHAS 2023-05-24 22:42:13 +00:00
bert-e b960a913ec Merge branch 'w/7.70/improvement/CLDSRV-388-implement-GHAS' into tmp/octopus/w/8.5/improvement/CLDSRV-388-implement-GHAS 2023-05-24 22:42:12 +00:00
bert-e 5436c0698e Merge branch 'w/7.10/improvement/CLDSRV-388-implement-GHAS' into tmp/octopus/w/7.70/improvement/CLDSRV-388-implement-GHAS 2023-05-24 22:42:12 +00:00
bert-e 3ff7856a94 Merge branch 'improvement/CLDSRV-388-implement-GHAS' into tmp/octopus/w/7.10/improvement/CLDSRV-388-implement-GHAS 2023-05-24 22:42:11 +00:00
gaspardmoindrot 57fb5f1403 [CLDSRV-388] Implement GHAS 2023-05-24 22:39:31 +00:00
Francois Ferrand ea284508d7
Update x-amz-restore when updating the expiry date
Issue: CLDSRV-393
2023-05-24 19:20:52 +02:00
Francois Ferrand 0981fa42f3
Add version name in release runs
Issue: CLDSRV-393
2023-05-24 19:20:52 +02:00
Francois Ferrand 7e63064a52
Bump github actions
Issue: CLDSRV-393
2023-05-24 19:20:52 +02:00
Francois Ferrand 71b21e40ca
Add eslint rule to prevent exclusive tests
Lint will fail if it finds any `describe.only` or `it.only`.

Issue: CLDSRV-393
2023-05-24 17:14:24 +02:00
Francois Ferrand ff894bb545
Remove describe.only
This should never have been commited, as it disables most unit tests from
CI.

This caused some tests to actually fail:
* bad import of refactored `objectDelete` api
* getting an object while transitioning (archiving) is allowed

Issue: CLDSRV-393
2023-05-24 17:09:33 +02:00
Francois Ferrand ae9f24e1bb
Update expiry date on s3:restore on restored object
If the object is already restored, we simply need to update the expiry
date, as per AWS docs:
> After restoring an archived object, you can update the restoration
> period by reissuing the request with a new period. Amazon S3 updates
> the restoration period relative to the current time.

Issue: CLDSRV-393
2023-05-24 16:52:45 +02:00
bert-e 2dc01ce3ed Merge branch 'w/8.7/improvement/CLDSRV-370-build-dev-img-release' into tmp/octopus/q/8.7 2023-05-15 16:39:05 +00:00
Kerkesni 9bd9bef6c7
bump version in package.json to 8.7.20
Issue: CLDSRV-386
2023-05-11 10:34:27 +02:00
bert-e 3a8bbefb6c Merge branch 'w/8.5/improvement/CLDSRV-370-build-dev-img-release' into tmp/octopus/w/8.6/improvement/CLDSRV-370-build-dev-img-release 2023-05-10 14:27:25 +00:00
bert-e a6a5c273d5 Merge branch 'w/8.6/improvement/CLDSRV-370-build-dev-img-release' into tmp/octopus/w/8.7/improvement/CLDSRV-370-build-dev-img-release 2023-05-10 14:27:25 +00:00
Dimitrios Vasilas c329d9684d Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-370-build-dev-img-release' into w/8.5/improvement/CLDSRV-370-build-dev-img-release 2023-05-10 16:23:51 +02:00
bert-e ec5baf1f85 Merge branch 'improvement/CLDSRV-370-build-dev-img-release' into tmp/octopus/w/7.70/improvement/CLDSRV-370-build-dev-img-release 2023-05-10 14:23:30 +00:00
Dimitrios Vasilas d844fb4fa1 CLDSRV-370: Push non-Federation image as cloudserver/cloudserver:<tag> 2023-05-10 16:19:18 +02:00
Kerkesni 6479076fec
bump node version to 16.20 in Dockerfile
Issue: CLDSRV-386
2023-05-10 13:35:54 +02:00
bert-e c436e2657c Merge branch 'w/8.5/improvement/CLDSRV-370-build-dev-img-release' into tmp/octopus/w/8.6/improvement/CLDSRV-370-build-dev-img-release 2023-05-09 17:22:48 +00:00
bert-e df45f481d0 Merge branch 'w/8.6/improvement/CLDSRV-370-build-dev-img-release' into tmp/octopus/w/8.7/improvement/CLDSRV-370-build-dev-img-release 2023-05-09 17:22:48 +00:00
Dimitrios Vasilas 406f3f0093 Revert "CLDSRV-370: docker-entrypoint: make bucketd bootstrap configurable"
This reverts commit 1d76f61d88.
2023-05-09 19:19:47 +02:00
Dimitrios Vasilas 6952b91539 CLDSRV-370: Pin virtualenv version to 20.21.0
Virtualenv setup fails with the latest version (20.23)
2023-05-09 19:18:40 +02:00
Dimitrios Vasilas eea1ebb5ec Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-370-build-dev-img-release' into w/8.5/improvement/CLDSRV-370-build-dev-img-release 2023-05-09 19:17:35 +02:00
bert-e dae5b7dc28 Merge branch 'improvement/CLDSRV-370-build-dev-img-release' into tmp/octopus/w/7.70/improvement/CLDSRV-370-build-dev-img-release 2023-05-09 17:12:54 +00:00
Dimitrios Vasilas 1d76f61d88 CLDSRV-370: docker-entrypoint: make bucketd bootstrap configurable 2023-05-09 19:07:50 +02:00
Dimitrios Vasilas 8abe809141 CLDSRV-370: Build dev docker image on release 2023-05-09 19:07:50 +02:00
Dimitrios Vasilas 94b14a258e CLDSRV-370: Pin virtualenv version to 20.21
Virtualenv setup fails with the latest version (20.23)
2023-05-09 19:07:46 +02:00
bert-e cd8c589eba Merge branch 'improvement/CLDSRV-375/exclude-keys' into tmp/octopus/w/8.7/improvement/CLDSRV-375/exclude-keys 2023-04-28 18:20:48 +00:00
williamlardier daec2661ae
CLDSRV-385: use mongodb v4.2 for the CI 2023-04-21 15:03:01 +02:00
Francois Ferrand 0f266371a0
Bump version 8.7.18
Issue: CLDSRV-383
2023-04-17 23:36:28 +02:00
Francois Ferrand 73e56963bf
Fix originOp when deleting a version
DeleteMarkerCreated was sent instead of the expect Delete, which breaks
bucket notifications.

Issue: CLDSRV-383
2023-04-17 23:14:49 +02:00
Nicolas Humbert 4c189b2d9e CLDSRV-375 Exclude already transitioned keys from lifecycle listings 2023-04-16 21:54:16 -07:00
Alexander Chan fb11d0f42e Merge remote-tracking branch 'origin/feature/CLDSRV-368/addBackbeatRouteForIndexingOps' into w/8.7/feature/CLDSRV-368/addBackbeatRouteForIndexingOps 2023-04-14 18:35:38 -07:00
Alexander Chan fe6690da92 bump arsenal 2023-04-14 18:08:42 -07:00
williamlardier 9cbd9f7be8
CLDSRV-381: bump project version 2023-04-14 22:29:03 +02:00
williamlardier c2fc8873cb
CLDSRV-381: bump arsenal 2023-04-14 22:28:47 +02:00
Francois Ferrand bee1ae04bf
Bump version 8.7.15
Issue: CLDSRV-380
2023-04-14 09:06:04 +02:00
Francois Ferrand eb86552a57
Allow reading transition-in-progress objects
This “transition in progress” state does not exist in AWS S3 (so we have no reference), and we need to access the data for cold storage framework.

When the transition has been performed, the archive id and storage class will be updated first (as well as clearing the ‘transitioning’ flag) before triggering the “GC” to remove the (local) data.

So we are sure that data is available in this state, and that simply checking that the object is in cold storage is enough.

Issue: CLDSRV-380
2023-04-14 09:02:32 +02:00
Alexander Chan 80fbf78d62 CLDSRV-368: add indexing routes 2023-04-13 15:17:03 -07:00
bert-e f5d8f2fac5 Merge branch 'w/8.6/feature/CLDSRV-359-passGetDeleteMarkerFlag' into tmp/octopus/w/8.7/feature/CLDSRV-359-passGetDeleteMarkerFlag 2023-04-13 19:07:07 +00:00
bert-e b1e13d6efa Merge branch 'w/8.5/feature/CLDSRV-359-passGetDeleteMarkerFlag' into tmp/octopus/w/8.6/feature/CLDSRV-359-passGetDeleteMarkerFlag 2023-04-13 19:07:07 +00:00
Jonathan Gramain e7ef437b27 Merge remote-tracking branch 'origin/feature/CLDSRV-359-passGetDeleteMarkerFlag' into w/8.5/feature/CLDSRV-359-passGetDeleteMarkerFlag 2023-04-13 11:42:08 -07:00
Jonathan Gramain 55f652ecc4 CLDSRV-359 bump cloudserver version 2023-04-13 11:40:29 -07:00
Jonathan Gramain 77f56d1fa1 feat: CLDSRV-359 pass getDeleteMarker flag to metadata when needed
Pass the `getDeleteMarker` flag to the Metadata backend when the
Cloudserver operation requires to distinguish if the target is a
delete marker or does not exist, to set response header
"x-amz-delete-marker" or return a specific error code.
2023-04-13 11:39:36 -07:00
bert-e 36e841b542 Merge branches 'w/8.7/feature/CLDSRV-355-activateNullKeys' and 'q/5069/8.6/feature/CLDSRV-355-activateNullKeys' into tmp/octopus/q/8.7 2023-04-13 18:35:42 +00:00
bert-e a2404ed622 Merge branch 'w/8.6/feature/CLDSRV-355-activateNullKeys' into tmp/octopus/q/8.6 2023-04-13 18:35:41 +00:00
williamlardier 1d12a430a0
CLDSRV-376: bump to 8.7.13 2023-04-13 16:59:28 +02:00
williamlardier bea27b4fb4
CLDSRV-376: update mongoClient used in functional tests 2023-04-13 16:59:13 +02:00
williamlardier 76405d9179
CLDSRV-376: bump mongodb driver 2023-04-13 16:59:12 +02:00
Alexander Chan 31b7f1e71c bump version 2023-04-12 15:36:00 -07:00
Alexander Chan 8674cac9f8 CLDSRV-379: bump arsenal 2023-04-12 15:35:27 -07:00
KillianG d5b666a246
Better indentation and use bool isExpiration only in the first function, after, uses directly originOp string
Issue: CLDSRV-367
2023-04-11 14:59:02 +00:00
KillianG 4360772971
Improve the way we pass originOp to make it clearer
Issue: CLDSRV-367
2023-04-11 13:43:37 +00:00
KillianG 6e152e33d5
Use boolean in parameter instead of hardcoded originOP
Issue: CLDSRV-367
2023-04-11 13:43:37 +00:00
KillianG 94f34979a5
add origin op to all delete object calls
Issue: CLDSRV-367
2023-04-11 13:43:36 +00:00
bert-e 4be430c313 Merge branch 'improvement/CLDSRV-372/vid' into q/8.6 2023-04-07 18:35:02 +00:00
bert-e 4b0f165b46 Merge branches 'w/8.7/improvement/CLDSRV-372/vid' and 'q/5109/8.6/improvement/CLDSRV-372/vid' into tmp/octopus/q/8.7 2023-04-07 18:35:02 +00:00
Nicolas Humbert 3590377554 Merge remote-tracking branch 'origin/improvement/CLDSRV-372/vid' into w/8.7/improvement/CLDSRV-372/vid 2023-04-07 07:58:01 -04:00
Nicolas Humbert f7f77c6cd2 CLDSRV-372 Current lifecycle versions should include version id 2023-04-06 19:09:04 -04:00
bert-e 8a08f97492 Merge branch 'w/8.6/feature/CLDSRV-355-activateNullKeys' into tmp/octopus/w/8.7/feature/CLDSRV-355-activateNullKeys 2023-04-05 18:16:48 +00:00
bert-e a908d09cc8 Merge branch 'w/8.5/feature/CLDSRV-355-activateNullKeys' into tmp/octopus/w/8.6/feature/CLDSRV-355-activateNullKeys 2023-04-05 18:16:47 +00:00
Jonathan Gramain 170a68a4f8 CLDSRV-355 [8.5+] fixup problematic automerge
Restore missing `require('../Config')` in lib/api/objectDelete.js
2023-04-05 10:57:42 -07:00
bert-e 448afa50e3 Merge branch 'w/8.6/feature/CLDSRV-355-activateNullKeys' into tmp/octopus/w/8.7/feature/CLDSRV-355-activateNullKeys 2023-04-05 00:48:47 +00:00
bert-e a0fff19611 Merge branch 'w/8.5/feature/CLDSRV-355-activateNullKeys' into tmp/octopus/w/8.6/feature/CLDSRV-355-activateNullKeys 2023-04-05 00:48:46 +00:00
bert-e 6ad1643ba8 Merge branch 'w/8.4/feature/CLDSRV-355-activateNullKeys' into tmp/octopus/w/8.5/feature/CLDSRV-355-activateNullKeys 2023-04-05 00:48:45 +00:00
Jonathan Gramain 5ce253ef62 Merge remote-tracking branch 'origin/feature/CLDSRV-355-activateNullKeys' into w/8.4/feature/CLDSRV-355-activateNullKeys 2023-04-04 17:27:11 -07:00
Jonathan Gramain 72f4c36077 CLDSRV-355 set 'isNull2' attr in copied null key
In order to other logic to detect properly null keys written from
non-compat Cloudservers, we also need to set the 'isNull2' param in
those when we copy them from the master key.
2023-04-04 16:23:25 -07:00
Jonathan Gramain e534af856f feat: CLDSRV-355 activate null keys behavior
Activate the use of null keys in place of null versioned keys by Cloudserver:

- allow processVersioningState() and preprocessingVersioningDelete()
  helpers to return the associated fields for null key handling, which
  tells Cloudserver to set its behavior to create/delete null keys,
  via sending PUT/DELETE requests with `versionId="null"` to the
  Metadata backend

- pass 'isNull' parameter in version-specific requests to hint the
  Metadata backend on what to do (most useful for V1 backend, but also
  to hint V0 backend that it should handle null keys appropriately)

- set "isNull2" metadata attribute when writing a null master, for
  optimization purpose (allows to avoid checking the null versioned
  key on update)
2023-04-04 16:23:25 -07:00
bert-e 5dd8d9057a Merge branch 'w/8.5/feature/CLDSRV-358-preprocessingVersioningDeleteNullKeys' into tmp/octopus/w/8.6/feature/CLDSRV-358-preprocessingVersioningDeleteNullKeys 2023-04-04 22:55:49 +00:00
bert-e 50b738cfff Merge branch 'w/8.6/feature/CLDSRV-358-preprocessingVersioningDeleteNullKeys' into tmp/octopus/w/8.7/feature/CLDSRV-358-preprocessingVersioningDeleteNullKeys 2023-04-04 22:55:49 +00:00
bert-e 2be3ce21c7 Merge branch 'w/8.4/feature/CLDSRV-358-preprocessingVersioningDeleteNullKeys' into tmp/octopus/w/8.5/feature/CLDSRV-358-preprocessingVersioningDeleteNullKeys 2023-04-04 22:55:49 +00:00
bert-e 70ff6fc4ee Merge branch 'feature/CLDSRV-358-preprocessingVersioningDeleteNullKeys' into tmp/octopus/w/8.4/feature/CLDSRV-358-preprocessingVersioningDeleteNullKeys 2023-04-04 22:55:48 +00:00
bert-e c5214d19a6 Merge branch 'w/8.5/feature/CLDSRV-378-forceEnableNullCompatMode' into tmp/octopus/w/8.6/feature/CLDSRV-378-forceEnableNullCompatMode 2023-04-04 22:27:20 +00:00
bert-e 951a98fcaf Merge branch 'w/8.6/feature/CLDSRV-378-forceEnableNullCompatMode' into tmp/octopus/w/8.7/feature/CLDSRV-378-forceEnableNullCompatMode 2023-04-04 22:27:20 +00:00
Jonathan Gramain ebb0fed48a Merge remote-tracking branch 'origin/feature/CLDSRV-378-forceEnableNullCompatMode' into w/8.5/feature/CLDSRV-378-forceEnableNullCompatMode 2023-04-04 15:17:14 -07:00
Jonathan Gramain 5f85c14ab9 CLDSRV-378 [8.x] force null version compat mode
force null version compatibility mode to be enabled, so that
Cloudserver stays compatible with MongoDB backend not supporting null
keys.

Remove the associated aws-sdk functional test suite related to
compatibility mode
2023-04-04 14:39:00 -07:00
bert-e 8ca770dcb7 Merge branch 'w/8.6/feature/CLDSRV-357-versioningPreprocessingNullKeySupport' into tmp/octopus/w/8.7/feature/CLDSRV-357-versioningPreprocessingNullKeySupport 2023-04-04 21:28:15 +00:00
bert-e 7923977300 Merge branch 'w/8.5/feature/CLDSRV-357-versioningPreprocessingNullKeySupport' into tmp/octopus/w/8.6/feature/CLDSRV-357-versioningPreprocessingNullKeySupport 2023-04-04 21:28:14 +00:00
Jonathan Gramain 3597c146d7 CLDSRV-358 fix issue deleting master null version
Fix an issue that occurred when deleting a null version that was the
current version AND that had a null key. This may happen in various
cases, e.g. if the repair process did repair the master by the null
version, in which case it would not delete the null versioned key
(this has been fixed with null keys).

The fix is to not send the `isNull: true` parameter to Metadata only
because Cloudserver is not in compatibility mode, instead, only send
this parameter if the master key has the `isNull2: true` parameter set
(meaning it was put by a non-compat Cloudserver).
2023-04-04 14:14:56 -07:00
Jonathan Gramain c81e49ba9b CLDSRV-358 fix deletion of null key in null compat mode
A version-specific DELETE of the null version did not work if:

- the request comes from a compat-mode Cloudserver

- the null version had been put by a non-compat mode Cloudserver

To handle this case properly, we look at the "isNull2" field of the
null version fetched, which is only set on non-compat Cloudserver, in
which case we send the "isNull" param to Metadata to instruct to
delete the null key instead of a null versioned key.
2023-04-04 14:14:56 -07:00
Jonathan Gramain e93c064b5f feat: CLDSRV-358 preprocessingVersioningDelete() update for null keys
Add support for null keys in the preprocessingVersioningDelete()
helper, essentially, set a 'isNull' boolean param that gets passed to
Metadata, which tells whether the version to delete is a null version.

NOTE: The null version compatibility mode is still enforced for now
until all pieces are updated to make functional tests pass without the
compatibility mode.
2023-04-04 14:14:55 -07:00
Jonathan Gramain 2b3774600d CLDSRV-358 [test] fix error code checking
In functional tests of 'objectDelete', an "afterAll" cleanup can crash
because it checks an error code before checking if there's an error
object.

It does not crash in normal circumstances because there is an actual
error due to the last unit test trying to clean the bucket, but if
anything changes in the unit tests that leaves the bucket existing
would have triggered this issue.
2023-04-04 14:14:55 -07:00
Jonathan Gramain a6951f2ef8 CLDSRV-358 [cleanup] objectDelete: remove unused assignment
Remove unused assignment of 'deleteInfo.isNull'
2023-04-04 14:14:55 -07:00
Jonathan Gramain 9fb232861f Merge remote-tracking branch 'origin/w/8.4/feature/CLDSRV-357-versioningPreprocessingNullKeySupport' into w/8.5/feature/CLDSRV-357-versioningPreprocessingNullKeySupport 2023-04-04 14:03:37 -07:00
Jonathan Gramain 6cf4e291fa Merge remote-tracking branch 'origin/feature/CLDSRV-357-versioningPreprocessingNullKeySupport' into w/8.4/feature/CLDSRV-357-versioningPreprocessingNullKeySupport 2023-04-04 13:21:30 -07:00
Jonathan Gramain 06b4320e7d CLDSRV-357 fix behavior change causing issue with mongo backend
When looking for a null version to delete, the code had changed its
behavior: it triggered a delete on the metadata even when the null
version was deleted. While technically not an issue for S3C as
Metadata returns a 200 when attempting to delete a non-existing
version (albeit creating a useless request), the MongoDB backend
returns NoSuchKey error in that case, leading to an issue.

Revert the behavior so to not trigger a delete of the null version
when it does not exist.
2023-04-04 13:17:24 -07:00
bert-e 3585b8d5eb Merge branch 'w/8.6/feature/CLDSRV-353-modifyPreprocessingVersioningForNullKeys' into tmp/octopus/w/8.7/feature/CLDSRV-353-modifyPreprocessingVersioningForNullKeys 2023-04-04 17:12:28 +00:00
bert-e 9331c0a375 Merge branch 'w/8.5/feature/CLDSRV-353-modifyPreprocessingVersioningForNullKeys' into tmp/octopus/w/8.6/feature/CLDSRV-353-modifyPreprocessingVersioningForNullKeys 2023-04-04 17:12:28 +00:00
bert-e 70f368408d Merge branch 'w/8.4/feature/CLDSRV-353-modifyPreprocessingVersioningForNullKeys' into tmp/octopus/w/8.5/feature/CLDSRV-353-modifyPreprocessingVersioningForNullKeys 2023-04-04 17:12:28 +00:00
Jonathan Gramain a63762ae71 CLDSRV-357 honor 'delOptions.deleteData' even if master is null
Move check of 'delOptions.deleteData' in prepareNullVersionDeletion()
earlier, so that it is honored even if the master key is a null
version.

This goes with the new possibility to return 'delOptions' without
deleteData in order to delete an existing null key for the master key
(done as part of CLDSRV-353).
2023-04-04 09:52:22 -07:00
Jonathan Gramain f0420572c8 feat: CLDSRV-357 pass deleteNullKey param to backend
Pass the 'deleteNullKey' param that processVersioningState() may set
to the Metadata backend, which tells it to delete the null key as the
PUT is executed.
2023-04-04 09:52:22 -07:00
Jonathan Gramain b1fd915ba3 feat: CLDSRV-357 update versioningPreprocessing() helper for null keys
Modify the code flow of versioningPreprocessing() to support null keys
in addition to the legacy "null versioned keys".

NOTE: The null version compatibility mode is still enforced for now
until all pieces are updated to make functional tests pass without the
compatibility mode.
2023-04-04 09:52:22 -07:00
Jonathan Gramain 4285c18e44 Merge remote-tracking branch 'origin/feature/CLDSRV-353-modifyPreprocessingVersioningForNullKeys' into w/8.4/feature/CLDSRV-353-modifyPreprocessingVersioningForNullKeys 2023-04-04 09:49:28 -07:00
Jonathan Gramain 71ffd004df CLDSRV-353 bump arsenal dep 2023-04-04 09:46:07 -07:00
Jonathan Gramain f674104825 CLDSRV-353 + case of delete null versioned key
Add a case in processVersioningState() to delete the null versioned
key where the key is a legacy null version (we know this because
`isNull2` is not set) and we are going to write it as a null key,
because in older Cloudservers there may be an associated null version
as the master null in certain circumstances.
2023-04-03 18:11:15 -07:00
Jonathan Gramain 9c9d4b3e7c CLDSRV-353 remove conversion of null version key to null key
Fix an issue that occurred when converting a null versioned key into a
null key (that would occur when a non-compat mode Cloudserver updates
a compat-mode object having a noncurrent null version). The issue was
that the null key was updated with the master's contents instead of
the original null version contents.

The fix consists of keeping the backward compatibility by setting a
`nullVersionId` instead, which avoids to have to read the null version
metadata first. It is not important to convert those keys as the
migration from V0 to V1 will necessarily have to convert existing
legacy null versions anyway.
2023-04-03 18:11:15 -07:00
Jonathan Gramain 13265a3d6e CLDSRV-353 [optim] no legacy null version key deletion
In case the master has been updated with a null-key-enabled
Cloudserver, there can be no more versioned key associated (as the new
behavior guarantees that a null master cannot have an associated null
versioned key, see S3C-7526).

Thanks to this, we can avoid having to check the versioned key for
deletion when a null master version is updated on a
versioning-suspended bucket, which is arguably a rather common use
case.

To implement that, we will add a "isNull2" attribute to the master key
in a subsequent commit (part of CLDSRV-355) when Cloudserver is not in
null version compatibility mode.

This commit is implementing the optimization by checking the new
"isNull2" metadata attribute, and skipping the null version check in
case the flag is set.
2023-04-03 18:10:07 -07:00
Jonathan Gramain 31c5316a7e feat: CLDSRV-353 processVersioningState() null key support
Add support for null key handling for the helper
processVersioningState(), and maintain the null version compatibility
mode to keep the old behavior - for now, the calling code sets this
flag to "true" without using the config to maintain current behavior,
it will be changed with CLDSRV-355.

One slight behavior change in compatibility mode is that when an old
null versioned key is deleted due to a PUT overwriting the null
version, we do not send the "replayId" to the DELETE request, but
instead, rely on the "oldReplayId" sent by the PUT, because this is
the normal way of letting metadata know how to get rid of the existing
replayId for the existing null version on versioning-suspended buckets.
2023-04-03 18:10:07 -07:00
bert-e 0a1489ee46 Merge branch 'w/8.6/improvement/CLDSRV-373-func-test-reproducing-s3c-5139' into tmp/octopus/w/8.7/improvement/CLDSRV-373-func-test-reproducing-s3c-5139 2023-04-03 16:34:59 +00:00
bert-e 71f80544ac Merge branch 'w/8.5/improvement/CLDSRV-373-func-test-reproducing-s3c-5139' into tmp/octopus/w/8.6/improvement/CLDSRV-373-func-test-reproducing-s3c-5139 2023-04-03 16:34:58 +00:00
bert-e 270080a75b Merge branch 'w/8.4/improvement/CLDSRV-373-func-test-reproducing-s3c-5139' into tmp/octopus/w/8.5/improvement/CLDSRV-373-func-test-reproducing-s3c-5139 2023-04-03 16:34:58 +00:00
bert-e 74717b2acb Merge branch 'w/7.70/improvement/CLDSRV-373-func-test-reproducing-s3c-5139' into tmp/octopus/w/8.4/improvement/CLDSRV-373-func-test-reproducing-s3c-5139 2023-04-03 16:34:57 +00:00
bert-e ef81f3e58f Merge branch 'improvement/CLDSRV-373-func-test-reproducing-s3c-5139' into tmp/octopus/w/7.70/improvement/CLDSRV-373-func-test-reproducing-s3c-5139 2023-04-03 16:34:57 +00:00
Jonathan Gramain aa55a87a65 impr: CLDSRV-373 new func test for S3C-5139
Add a new functional test in aws-node-sdk that reproduces the steps
that formerly triggered S3C-5139, which was silently fixed since it
was opened.

It is useful also for S3C-7352 because it covers a corner case for
both old and new null version handling in v0 and v1 metadata formats.
2023-04-03 09:30:35 -07:00
Xin LI de5b4331e2 Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-365-fix-legal-hold-can-be-deleted' into w/8.7/bugfix/CLDSRV-365-fix-legal-hold-can-be-deleted 2023-03-31 11:00:24 +02:00
Xin LI e1a4f1ef8c bugfix: CLDSRV-365 bump 2023-03-31 10:58:07 +02:00
bert-e 46dff0321d Merge branch 'w/8.6/bugfix/CLDSRV-365-fix-legal-hold-can-be-deleted' into tmp/octopus/w/8.7/bugfix/CLDSRV-365-fix-legal-hold-can-be-deleted 2023-03-31 08:54:07 +00:00
bert-e f3c7580510 Merge branch 'w/8.4/bugfix/CLDSRV-365-fix-legal-hold-can-be-deleted' into tmp/octopus/w/8.5/bugfix/CLDSRV-365-fix-legal-hold-can-be-deleted 2023-03-31 08:54:06 +00:00
bert-e 2145bb3ae3 Merge branch 'w/8.5/bugfix/CLDSRV-365-fix-legal-hold-can-be-deleted' into tmp/octopus/w/8.6/bugfix/CLDSRV-365-fix-legal-hold-can-be-deleted 2023-03-31 08:54:06 +00:00
Xin LI 468162c81c Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-365-fix-legal-hold-can-be-deleted' into w/8.4/bugfix/CLDSRV-365-fix-legal-hold-can-be-deleted 2023-03-31 10:53:21 +02:00
Xin LI 89f9139203 Merge remote-tracking branch 'origin/bugfix/CLDSRV-365-fix-legal-hold-can-be-deleted' into w/7.70/bugfix/CLDSRV-365-fix-legal-hold-can-be-deleted 2023-03-31 10:48:44 +02:00
Xin LI 8153554a4c bugfix: CLDSRV-365 bump version 2023-03-31 10:46:47 +02:00
Xin LI fb9063bccc bugfix: CLDSRV-365 fix legal hold can be deleted issue and add more tests 2023-03-31 10:46:47 +02:00
bert-e ddc6ea72be Merge branch 'improvement/CLDSRV-371/etag' into tmp/octopus/w/8.7/improvement/CLDSRV-371/etag 2023-03-29 20:22:38 +00:00
Nicolas Humbert f20bf1becf CLDSRV-371 ETag should be surrounded by double quotes 2023-03-29 16:16:52 -04:00
bert-e d31c773e77 Merge branch 'w/8.4/improvement/CLDSRV-369-versionSpecificDeleteWithNullFuncTest' into tmp/octopus/w/8.5/improvement/CLDSRV-369-versionSpecificDeleteWithNullFuncTest 2023-03-28 21:14:17 +00:00
bert-e d266ff4e9f Merge branch 'w/8.6/improvement/CLDSRV-369-versionSpecificDeleteWithNullFuncTest' into tmp/octopus/w/8.7/improvement/CLDSRV-369-versionSpecificDeleteWithNullFuncTest 2023-03-28 21:14:17 +00:00
bert-e 6ff21996f5 Merge branch 'w/8.5/improvement/CLDSRV-369-versionSpecificDeleteWithNullFuncTest' into tmp/octopus/w/8.6/improvement/CLDSRV-369-versionSpecificDeleteWithNullFuncTest 2023-03-28 21:14:17 +00:00
bert-e 15d1b3ba86 Merge branch 'w/7.70/improvement/CLDSRV-369-versionSpecificDeleteWithNullFuncTest' into tmp/octopus/w/8.4/improvement/CLDSRV-369-versionSpecificDeleteWithNullFuncTest 2023-03-28 21:14:16 +00:00
bert-e 827c752e9a Merge branch 'improvement/CLDSRV-369-versionSpecificDeleteWithNullFuncTest' into tmp/octopus/w/7.70/improvement/CLDSRV-369-versionSpecificDeleteWithNullFuncTest 2023-03-28 21:14:15 +00:00
Jonathan Gramain 82dc837610 impr: CLDSRV-369 new version-specific DELETE functional test
Add a new functional test in the aws-node-sdk test suite, that

- creates a nonversioned key

- then enables versioning and creates 5 more versioned keys

- then deletes the 5 versioned keys in a batch delete

- it expects the null version to be the only remaining version

Its primary purpose is to check version-specific DELETE with a null
version in V1 metadata format (MD-847), but it is a good addition to
the existing test suite also in V0 format.
2023-03-28 14:09:10 -07:00
bert-e 7dc2f07cb6 Merge branch 'w/8.7/improvement/CLDSRV-366/clear' into tmp/octopus/q/8.7 2023-03-28 13:25:15 +00:00
Kerkesni 6c22d87c55
bump version to 8.7.11
Issue: CLDSRV-362
2023-03-28 12:25:11 +02:00
Kerkesni 310f67d3a7
throw error when getting a transitioning object
Issue: CLDSRV-362
2023-03-28 12:24:50 +02:00
Kerkesni 49841c5e0e
throw error when copying parts from a cold object
A cold object should not be allowed to get copied as the data
is not accessible.

Issue: CLDSRV-362
2023-03-28 12:24:49 +02:00
Kerkesni b5334baca8
throw error when copying a cold or transitioning object
A cold object should not be allowed to get copied as the data
is not accessible.

Same issue happens when copying an object that is transitioning,
the data might get deleted while copying is still in progress.

Issue: CLDSRV-362
2023-03-28 12:24:49 +02:00
Kerkesni e592671b54
add helper to check if object is in cold storage
Issue: CLDSRV-362
2023-03-28 12:24:48 +02:00
bert-e 6e0b66849d Merge branch 'improvement/CLDSRV-366/clear' into tmp/octopus/w/8.7/improvement/CLDSRV-366/clear 2023-03-28 03:45:02 +00:00
Nicolas Humbert f2292f1ca3 CLDSRV-366 Clear list orphan delete markers response 2023-03-27 15:52:49 -04:00
bert-e 18a1bfd325 Merge branch 'w/8.6/improvement/CLDSRV-356-enhanceProcessVersioningState' into tmp/octopus/w/8.7/improvement/CLDSRV-356-enhanceProcessVersioningState 2023-03-24 23:39:13 +00:00
bert-e c2b54702f6 Merge branch 'w/8.5/improvement/CLDSRV-356-enhanceProcessVersioningState' into tmp/octopus/w/8.6/improvement/CLDSRV-356-enhanceProcessVersioningState 2023-03-24 23:39:12 +00:00
Jonathan Gramain 13a5e14da5 impr: CLDSRV-356 [8.5+] adapt overwritingVersioning() for archive
Due to the change in what processVersioningState() returns
(nullVersionId embedded in an "extraMD" field for clarity), modify the
overwritingVersioning() helper that needs to have the same contract
than the former function.
2023-03-24 16:37:24 -07:00
Jonathan Gramain 891913fd16 Merge remote-tracking branch 'origin/w/8.4/improvement/CLDSRV-356-enhanceProcessVersioningState' into w/8.5/improvement/CLDSRV-356-enhanceProcessVersioningState 2023-03-24 15:52:32 -07:00
bert-e 7baa2501e6 Merge branch 'improvement/CLDSRV-356-enhanceProcessVersioningState' into tmp/octopus/w/8.4/improvement/CLDSRV-356-enhanceProcessVersioningState 2023-03-24 22:50:02 +00:00
Jonathan Gramain 8e808afec9 impr: CLDSRV-356 enhance processVersioningState() and fix replayId
- enhance general flow of processVersioningState(), to make it easier
  to read and update for null key handling

- fix an issue related to passing the uploadId for nonversioned
  buckets (linked to S3C-7361): remove a check "master.isNull" to also
  pass the uploadId as replayId when the master is non-versioned, so
  that it can be deleted by passing it to the metadata DELETE request

- make processVersioningState() return a 'nullVersionId' param rather
  than a "storeOptions", as it is always used to copy master to a null
  version, it simplifies a bit the API

- make processVersioningState() return extra metadata params to set in
  the object ("nullVersionId" and "nullUploadId") in their own
  "extraMD" object, for clarity

- remove undefined params returned by the function to have clean unit
  tests
2023-03-24 12:51:48 -07:00
bert-e 2c999f4c10 Merge branch 'w/8.6/feature/CLDSRV-354-modifyMetadataGetObjectForNullKeys' into tmp/octopus/w/8.7/feature/CLDSRV-354-modifyMetadataGetObjectForNullKeys 2023-03-24 19:46:54 +00:00
bert-e b23472a754 Merge branch 'w/8.5/feature/CLDSRV-354-modifyMetadataGetObjectForNullKeys' into tmp/octopus/w/8.6/feature/CLDSRV-354-modifyMetadataGetObjectForNullKeys 2023-03-24 19:46:54 +00:00
bert-e a4999c1bfb Merge branch 'w/8.4/feature/CLDSRV-354-modifyMetadataGetObjectForNullKeys' into tmp/octopus/w/8.5/feature/CLDSRV-354-modifyMetadataGetObjectForNullKeys 2023-03-24 19:46:53 +00:00
bert-e fe0b0f8b2f Merge branch 'feature/CLDSRV-354-modifyMetadataGetObjectForNullKeys' into tmp/octopus/w/8.4/feature/CLDSRV-354-modifyMetadataGetObjectForNullKeys 2023-03-24 19:46:53 +00:00
Jonathan Gramain c2bee23fd1 ft: CLDSRV-354 handle null keys in metadata GET helpers
Update the helpers in metadataUtils to handle null keys, as well as
keeping backward compatibility with null versioned keys.

The main change in logic for null keys is that, instead of fetching
first the master key then the null versioned key, we first attempt to
fetch the null key, and if not found, we fetch the master key (we may
then also have to fetch the null versioned key for backward
compatibility).

Take the chance to reduce tech debt by reorganizing the helpers
responsibilities in a better way, and by using the "validateBucket"
helper.
2023-03-24 12:45:07 -07:00
Jonathan Gramain e87c2a4e5f CLDSRV-354 [cleanup] new helper metadataUtils.validateBucket()
Factorize logic to validate a bucket and return the relevant error
code in a helper function (checks on bucketShield, bucket policies,
then bucket authorization)
2023-03-24 12:42:02 -07:00
Jonathan Gramain db943cd634 CLDSRV-354 [optim] remove unnecessary check in getVersionIdResHeader() 2023-03-24 12:42:02 -07:00
bert-e bf7a643d45 Merge branch 'w/8.6/feature/CLDSRV-349-nullVersionCompatModeConfigAndTests' into tmp/octopus/w/8.7/feature/CLDSRV-349-nullVersionCompatModeConfigAndTests 2023-03-24 19:07:15 +00:00
bert-e 874a53c767 Merge branch 'w/8.5/feature/CLDSRV-349-nullVersionCompatModeConfigAndTests' into tmp/octopus/w/8.6/feature/CLDSRV-349-nullVersionCompatModeConfigAndTests 2023-03-24 19:07:14 +00:00
Jonathan Gramain c7e1c6921b Merge remote-tracking branch 'origin/w/8.4/feature/CLDSRV-349-nullVersionCompatModeConfigAndTests' into w/8.5/feature/CLDSRV-349-nullVersionCompatModeConfigAndTests 2023-03-24 11:43:47 -07:00
Jonathan Gramain 6d2d56bc1e Merge remote-tracking branch 'origin/feature/CLDSRV-349-nullVersionCompatModeConfigAndTests' into w/8.4/feature/CLDSRV-349-nullVersionCompatModeConfigAndTests 2023-03-24 11:04:47 -07:00
bert-e 3f3bf0fdf0 Merge branches 'w/8.7/bugfix/CLDSRV-363/etag' and 'q/5078/8.6/bugfix/CLDSRV-363/etag' into tmp/octopus/q/8.7 2023-03-24 18:01:38 +00:00
bert-e 1922facb7b Merge branch 'bugfix/CLDSRV-363/etag' into q/8.6 2023-03-24 18:01:38 +00:00
Jonathan Gramain fff03d3320 CLDSRV-349 [tests] func test stage for ENABLE_NULL_VERSION_COMPAT_MODE
Turn the "file-ft-tests" job into a matrix, to duplicate the suite
with and without the ENABLE_NULL_VERSION_COMPAT_MODE environment variable.

This will make sure Cloudserver behaves correctly (versioning, null
version handling etc.)  when the compatibility mode is active.
2023-03-24 10:36:39 -07:00
Jonathan Gramain 6e79d3f1a4 ft: CLDSRV-349 support ENABLE_NULL_VERSION_COMPAT_MODE env var
Cloudserver sets a flag in its configuration when the
ENABLE_NULL_VERSION_COMPAT_MODE environment variable is set to "true".
2023-03-24 10:36:39 -07:00
bert-e 2a44949048 Merge branches 'development/8.7' and 'w/8.6/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion' into tmp/octopus/w/8.7/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion 2023-03-24 16:52:06 +00:00
bert-e 1576352613 Merge branch 'w/8.5/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion' into tmp/octopus/w/8.6/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion 2023-03-24 16:52:06 +00:00
bert-e 74978f423e Merge branch 'w/8.4/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion' into tmp/octopus/w/8.5/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion 2023-03-24 16:52:05 +00:00
bert-e 6f4cd75d6f Merge branch 'w/7.70/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion' into tmp/octopus/w/8.4/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion 2023-03-24 16:52:05 +00:00
bert-e 00906d04f5 Merge branch 'bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion' into tmp/octopus/w/7.70/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion 2023-03-24 16:52:04 +00:00
Jonathan Gramain 270339f2bb CLDSRV-361 guard before accessing err.is field 2023-03-24 09:51:22 -07:00
bert-e 6660626190 Merge branch 'bugfix/CLDSRV-363/etag' into tmp/octopus/w/8.7/bugfix/CLDSRV-363/etag 2023-03-24 13:23:06 +00:00
Nicolas Humbert 049f52bf95 CLDSRV-363 ETag instead of Etag for lifecycle listings Contents 2023-03-23 16:51:12 -04:00
williamlardier 58fc0b7146
CLDSRV-350: bump to 8.7.10 2023-03-21 13:52:26 +01:00
williamlardier 11e3d7ecb2
CLDSRV-350: update veeam put and delete routes with new arsenal methods
We must ensure that concurrent updates of the bucket metadata won't conflict
with each other, by separately updating the capabilities fields. This change
ensures that two files can be uploaded at the same without any problem,
regardless of the number of cloudserver instances.
2023-03-21 13:52:25 +01:00
williamlardier 1bab851ce3
CLDSRV-350: bump arsenal version 2023-03-21 13:52:25 +01:00
bert-e 0bc0341f33 Merge branch 'w/8.6/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion' into tmp/octopus/w/8.7/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion 2023-03-20 22:27:37 +00:00
bert-e b5af652dc8 Merge branch 'w/8.5/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion' into tmp/octopus/w/8.6/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion 2023-03-20 22:27:37 +00:00
bert-e 6c29be5137 Merge branch 'w/8.4/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion' into tmp/octopus/w/8.5/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion 2023-03-20 22:27:36 +00:00
Jonathan Gramain 2967f327ed Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion' into w/8.4/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion 2023-03-20 15:08:01 -07:00
Jonathan Gramain 0f8a56e9b5 Merge remote-tracking branch 'origin/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion' into w/7.70/bugfix/CLDSRV-361-fixExceptionBatchDeleteNullVersion 2023-03-20 14:25:03 -07:00
Jonathan Gramain c1d2601237 CLDSRV-361 [8.x fix] backbeat multiple backend route fix
Fix handling of response of metadataGetObject() in multiple backend
PutTagging and DeleteTagging when target object does not exist: return
error code NoSuchKey in this case.

Note: NoSuchKey used to be returned by metadataGetObject() helper but
not always, now it's never returned, instead the "objMD" returned
value is null.
2023-03-20 13:25:09 -07:00
Jonathan Gramain 885f95606c bugfix: CLDSRV-361 fix exception with batch delete of null version
- Fix an exception in Cloudserver when doing a batch delete containing a
  deletion of a null version that does not exist on a versioned object

- The changes also fix the return code when fetching a noncurrent null
  version that was deleted: in such case it was returning NoSuchKey,
  now returns NoSuchVersion per spec.

The change consists of:

- consolidate and fix the API contract of metadataGetObject() and
  metadataValidateBucketAndObj() for "not found" objects or object
  versions: there was a mix of return codes from Metadata (NoSuchKey)
  and an OK response with no metadata returned in others, depending on
  the exact scenario and object state. Fixed by always returning an OK
  response and no metadata if the target version is not found, to let
  the caller set the appropriate error code in the API response

- for multiObjectDelete, fix by handling the OK response with empty
  object metadata response specifically as a "not found" case, like
  for other API calls
2023-03-20 13:12:02 -07:00
bert-e b5b0f6482b Merge branch 'feature/CLDSRV-317/listLifecycleOrphans' into tmp/octopus/w/8.7/feature/CLDSRV-317/listLifecycleOrphans 2023-03-20 13:53:09 +00:00
Nicolas Humbert ec9ed94555 CLDSRV-317 Implement listLifecycleOrphans 2023-03-20 09:52:42 -04:00
bert-e 755f282f8e Merge branch 'feature/CLDSRV-316/listLifecycleNonCurrents' into tmp/octopus/w/8.7/feature/CLDSRV-316/listLifecycleNonCurrents 2023-03-17 18:00:21 +00:00
Nicolas Humbert 41cc399d85 CLDSRV-316 Implement listLifecycleNonCurrents 2023-03-17 13:58:22 -04:00
bert-e c4dc928de2 Merge branch 'feature/CLDSRV-314/listLifecycleCurrents' into tmp/octopus/w/8.7/feature/CLDSRV-314/listLifecycleCurrents 2023-03-17 16:20:16 +00:00
Nicolas Humbert 6b8a2581b6 CLDSRV-314 Implement listLifecycleCurrents 2023-03-17 11:48:02 -04:00
Killian Gardahaut a0087e8d77
Bump 8.7.9
Issue: ZKOP-219
2023-03-17 09:58:21 +01:00
KillianG 8e5bea56b6
Refacto tests for more readability
Issue: CLDSRV-337
2023-03-17 09:58:21 +01:00
KillianG 976e349036
Add tests
Adding test for the function azureArchiveLocationConstraintAssert

Issue: CLDSRV-337
2023-03-17 09:58:16 +01:00
KillianG de1c23ac1b
Add test on location constraints to ensure the location is well configured
Issue: CLDSRV-337
2023-03-17 09:56:35 +01:00
KillianG 0b4d04a2a3
Add location azure archive to cold storage locations
Issue: CLDSRV-337
2023-03-17 09:56:35 +01:00
KillianG 049d396c8d
Add azure_archive location type
ISSUE: CLDSRV-337
2023-03-17 09:56:35 +01:00
Naren 5c04cbe6d1 Merge remote-tracking branch 'origin/w/8.6/improvement/CLDSRV-327-cloudserver-metrics' into w/8.7/improvement/CLDSRV-327-cloudserver-metrics 2023-03-16 22:36:59 -07:00
Naren d3e538087a Merge remote-tracking branch 'origin/w/8.5/improvement/CLDSRV-327-cloudserver-metrics' into w/8.6/improvement/CLDSRV-327-cloudserver-metrics 2023-03-16 22:05:26 -07:00
bert-e 7cc37c7f3d Merge branch 'w/8.4/improvement/CLDSRV-327-cloudserver-metrics' into tmp/octopus/w/8.5/improvement/CLDSRV-327-cloudserver-metrics 2023-03-17 03:50:44 +00:00
Naren 399d081d68 impr: CLDSRV-327 upgrade arsenal, bucketclient, prom-client, utapi, vaultclient 2023-03-16 20:33:03 -07:00
Naren c3fac24366 Merge remote-tracking branch 'origin/improvement/CLDSRV-327-cloudserver-metrics' into w/8.4/improvement/CLDSRV-327-cloudserver-metrics 2023-03-16 20:23:37 -07:00
Naren 82687aa1a7 impr: CLDSRV-327 minor updates
add info log on metric server start.
refactor unit tests.
2023-03-16 17:35:28 -07:00
Naren 820ada48ce impr: CLDSRV-327 bump version to 7.70.18 2023-03-16 15:43:56 -07:00
Naren df73cc7ebc impr: CLDSRV-327 upgrade arsenal, bucketclient, prom-client, utapi, vaultclient 2023-03-16 15:43:07 -07:00
Naren 429c62087d impr: CLDSRV-327 refactor metrics server
prom-client will not aggregate metrics from all workers, so moved
metrics server to master and aggregate metrics using AggregatorRegistry.
Metrics are moved to a separate file to not confuse with metrics
handler.
2023-03-16 15:39:13 -07:00
Naren 13fa26986d impr: CLDSRV-327 linter corrections 2023-03-02 09:52:39 -08:00
bert-e 5cb63991a8 Merge branch 'w/8.6/improvement/CLDSRV-328-adapt-prescribed-metric-names' into tmp/octopus/w/8.7/improvement/CLDSRV-328-adapt-prescribed-metric-names 2023-03-02 16:30:18 +00:00
Naren d5b336d1d9 Merge remote-tracking branch 'origin/w/8.5/improvement/CLDSRV-328-adapt-prescribed-metric-names' into w/8.6/improvement/CLDSRV-328-adapt-prescribed-metric-names 2023-03-02 08:29:39 -08:00
bert-e 750223500d Merge branch 'improvement/CLDSRV-328-adapt-prescribed-metric-names' into tmp/octopus/w/8.5/improvement/CLDSRV-328-adapt-prescribed-metric-names 2023-03-02 15:31:17 +00:00
Naren 23ffbf77d2 impr: CLDSRV-328 fix ceph java deps installation 2023-03-02 07:29:22 -08:00
Naren 6ea18bcef4 impr: CLDSRV-328 adapt metric naming conventions 2023-03-02 06:44:45 -08:00
Naren c45dac7ffc impr: CLDSRV-327 add monitoring tests 2023-02-28 11:33:12 -08:00
Naren 878fc6819f impr: CLDSRV-327 generate cloudserver metrics 2023-02-28 11:32:37 -08:00
Naren 43592f9392 impr: CLDSRV-327 setup cloudserver monitoring 2023-02-28 11:32:13 -08:00
Naren dbd1383c32 impr: CLDSRV-327 add prom-client 2023-02-28 11:32:13 -08:00
Alexander Chan c310cb3dd1 Merge remote-tracking branch 'origin/w/8.6/feature/CLDSRV-336/supportNewerNoncurrentVersions' into w/8.7/feature/CLDSRV-336/supportNewerNoncurrentVersions 2023-02-26 18:47:15 -08:00
bert-e 7fe0e2891b Merge branch 'w/8.5/feature/CLDSRV-336/supportNewerNoncurrentVersions' into tmp/octopus/w/8.6/feature/CLDSRV-336/supportNewerNoncurrentVersions 2023-02-27 02:43:32 +00:00
bert-e 93442fed68 Merge branch 'w/8.4/feature/CLDSRV-336/supportNewerNoncurrentVersions' into tmp/octopus/w/8.5/feature/CLDSRV-336/supportNewerNoncurrentVersions 2023-02-27 02:43:31 +00:00
Alexander Chan 21612cfadd Merge remote-tracking branch 'origin/w/7.70/feature/CLDSRV-336/supportNewerNoncurrentVersions' into w/8.4/feature/CLDSRV-336/supportNewerNoncurrentVersions 2023-02-26 18:20:17 -08:00
bert-e 644062f088 Merge branch 'feature/CLDSRV-336/supportNewerNoncurrentVersions' into tmp/octopus/w/7.70/feature/CLDSRV-336/supportNewerNoncurrentVersions 2023-02-25 01:39:32 +00:00
Alexander Chan d0eb81539e bump arsenal depedency 2023-02-24 16:52:33 -08:00
bert-e 22cda51944 Merge branch 'w/8.7/bugfix/CLDSRV-344-doubleCallbackInFuncTest' into tmp/octopus/q/8.7 2023-02-22 16:39:53 +00:00
williamlardier 408d0de732
CLDSRV-343: bump cloudserver to the next version 2023-02-22 09:59:09 +01:00
williamlardier 83916c91fb
CLDSRV-343: enable back some CEPH backend tests
These tests also cover the ObjectTagging API with multiple backend.
Enabling them back will allow us to better avoid issues like this
in the future.
2023-02-17 14:24:59 +01:00
bert-e 110b2a35ed Merge branch 'w/8.6/bugfix/CLDSRV-344-doubleCallbackInFuncTest' into tmp/octopus/w/8.7/bugfix/CLDSRV-344-doubleCallbackInFuncTest 2023-02-17 00:55:38 +00:00
bert-e 3b5f5875f3 Merge branch 'w/8.5/bugfix/CLDSRV-344-doubleCallbackInFuncTest' into tmp/octopus/w/8.6/bugfix/CLDSRV-344-doubleCallbackInFuncTest 2023-02-17 00:55:37 +00:00
bert-e bdaf92023f Merge branch 'w/8.4/bugfix/CLDSRV-344-doubleCallbackInFuncTest' into tmp/octopus/w/8.5/bugfix/CLDSRV-344-doubleCallbackInFuncTest 2023-02-17 00:55:37 +00:00
bert-e 25d1cd9601 Merge branch 'w/7.70/bugfix/CLDSRV-344-doubleCallbackInFuncTest' into tmp/octopus/w/8.4/bugfix/CLDSRV-344-doubleCallbackInFuncTest 2023-02-17 00:55:36 +00:00
bert-e 91c9eb6faa Merge branch 'bugfix/CLDSRV-344-doubleCallbackInFuncTest' into tmp/octopus/w/7.70/bugfix/CLDSRV-344-doubleCallbackInFuncTest 2023-02-17 00:55:36 +00:00
Jonathan Gramain 6306cf7fc7 [tests] CLDSRV-344 fix func test double callback
In the completeMPU test of the aws-node-sdk test suite, fix a case of
double callback being called, that can fail when launching locally
with mocha.
2023-02-16 16:52:14 -08:00
williamlardier a8117ca037
CLDSRV-343: use bucket name for backend tagging operations 2023-02-16 15:51:49 +01:00
bert-e 9145d1cf79 Merge branches 'w/8.7/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' and 'q/5003/8.6/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' into tmp/octopus/q/8.7 2023-02-15 20:43:46 +00:00
bert-e 0fb54c9d31 Merge branches 'w/8.6/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' and 'q/5003/8.5/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' into tmp/octopus/q/8.6 2023-02-15 20:43:46 +00:00
bert-e 63dc33a339 Merge branches 'w/8.5/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' and 'q/5003/8.4/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' into tmp/octopus/q/8.5 2023-02-15 20:43:45 +00:00
bert-e 49d46dfe04 Merge branches 'w/8.4/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' and 'q/5003/7.70/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' into tmp/octopus/q/8.4 2023-02-15 20:43:44 +00:00
bert-e 4bb331392e Merge branch 'improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' into q/7.70 2023-02-15 20:43:44 +00:00
bert-e ae1b6dc3d1 Merge branch 'w/8.6/feature/CLDSRV-342/bump-7.70.16' into tmp/octopus/w/8.7/feature/CLDSRV-342/bump-7.70.16 2023-02-14 20:05:16 +00:00
bert-e 162157580f Merge branch 'w/8.5/feature/CLDSRV-342/bump-7.70.16' into tmp/octopus/w/8.6/feature/CLDSRV-342/bump-7.70.16 2023-02-14 20:05:16 +00:00
bert-e 4e4435d82e Merge branch 'w/8.4/feature/CLDSRV-342/bump-7.70.16' into tmp/octopus/w/8.5/feature/CLDSRV-342/bump-7.70.16 2023-02-14 20:05:16 +00:00
Alexander Chan b0db1f9a94 Merge remote-tracking branch 'origin/feature/CLDSRV-342/bump-7.70.16' into w/8.4/feature/CLDSRV-342/bump-7.70.16 2023-02-14 12:03:01 -08:00
Alexander Chan 35d269c27c bump version 7.70.16 2023-02-13 10:49:17 -08:00
bert-e b1304b5f7f Merge branches 'w/8.7/bugfix/CLDSRV-338/fixMaxKeysV2Listing' and 'q/5000/8.6/bugfix/CLDSRV-338/fixMaxKeysV2Listing' into tmp/octopus/q/8.7 2023-02-10 12:57:22 +00:00
bert-e c355422a7e Merge branches 'w/8.6/bugfix/CLDSRV-338/fixMaxKeysV2Listing' and 'q/5000/8.5/bugfix/CLDSRV-338/fixMaxKeysV2Listing' into tmp/octopus/q/8.6 2023-02-10 12:57:22 +00:00
bert-e d44334ad22 Merge branches 'w/8.5/bugfix/CLDSRV-338/fixMaxKeysV2Listing' and 'q/5000/8.4/bugfix/CLDSRV-338/fixMaxKeysV2Listing' into tmp/octopus/q/8.5 2023-02-10 12:57:21 +00:00
bert-e 6e9c50eeba Merge branches 'w/8.4/bugfix/CLDSRV-338/fixMaxKeysV2Listing' and 'q/5000/7.70/bugfix/CLDSRV-338/fixMaxKeysV2Listing' into tmp/octopus/q/8.4 2023-02-10 12:57:21 +00:00
bert-e 6c7be8892c Merge branch 'bugfix/CLDSRV-338/fixMaxKeysV2Listing' into q/7.10 2023-02-10 12:57:20 +00:00
bert-e 82df91debb Merge branches 'w/7.70/bugfix/CLDSRV-338/fixMaxKeysV2Listing' and 'q/5000/7.10/bugfix/CLDSRV-338/fixMaxKeysV2Listing' into tmp/octopus/q/7.70 2023-02-10 12:57:20 +00:00
bert-e 6b1f8c61ec Merge branch 'w/8.6/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' into tmp/octopus/w/8.7/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete 2023-02-09 23:05:09 +00:00
bert-e a12d44dc18 Merge branch 'w/8.5/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' into tmp/octopus/w/8.6/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete 2023-02-09 23:05:08 +00:00
bert-e d5ec32fc5c Merge branch 'w/8.4/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' into tmp/octopus/w/8.5/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete 2023-02-09 23:05:08 +00:00
Jonathan Gramain e16da9ab11 Merge remote-tracking branch 'origin/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete' into w/8.4/improvement/CLDSRV-340-simplifyPreprocessingVersioningDelete 2023-02-09 14:43:41 -08:00
Jonathan Gramain d43e8d01bf impr: CLDSRV-340 improve preprocessingVersioningDelete()
- remove dead code handling nullVersionId and when no reference to
  null version is found: the confusion is that the metadata is already
  coming from the null version fetch due to the "reqVersionId" being
  "null", so it can only be the null version itself.

- remove the "isNull" argument returned as it is misused (for metrics)
  and can be incorrect, see S3C-7440

- simplify the function by reorganizing the logic and removing the
  callback argument, just returning an options object

- delete replay keys on multiobject delete, instead of passing the
  replayId option via preprocessingVersioningDelete() (a similar
  change has been done in objectDelete but it resulted in duplication
  of replayId setting)

- remove all MPU-related tests for this helper, as the helper does not
  return replayId anymore, they became irrelevant
2023-02-09 14:00:56 -08:00
bert-e 335bfabed1 Merge branch 'w/8.6/bugfix/CLDSRV-338/fixMaxKeysV2Listing' into tmp/octopus/w/8.7/bugfix/CLDSRV-338/fixMaxKeysV2Listing 2023-02-09 19:30:15 +00:00
bert-e 3b92eaaef2 Merge branch 'w/8.5/bugfix/CLDSRV-338/fixMaxKeysV2Listing' into tmp/octopus/w/8.6/bugfix/CLDSRV-338/fixMaxKeysV2Listing 2023-02-09 19:30:15 +00:00
bert-e a6fd8b2261 Merge branch 'w/8.4/bugfix/CLDSRV-338/fixMaxKeysV2Listing' into tmp/octopus/w/8.5/bugfix/CLDSRV-338/fixMaxKeysV2Listing 2023-02-09 19:30:15 +00:00
bert-e 00ab8d482d Merge branch 'w/7.70/bugfix/CLDSRV-338/fixMaxKeysV2Listing' into tmp/octopus/w/8.4/bugfix/CLDSRV-338/fixMaxKeysV2Listing 2023-02-09 19:30:14 +00:00
bert-e 29551f7edf Merge branch 'bugfix/CLDSRV-338/fixMaxKeysV2Listing' into tmp/octopus/w/7.70/bugfix/CLDSRV-338/fixMaxKeysV2Listing 2023-02-09 19:30:14 +00:00
Alexander Chan 7dd022f6cb CLDSRV-338: fix v2 listing with max-keys 0
addresses issue in which listing XML does not return a value for
`max-keys' when it is 0.
2023-02-08 21:10:19 -08:00
bert-e 3398db3c0f Merge branch 'w/8.6/bugfix/CLDSRV-339-revert-S3C-7054' into tmp/octopus/w/8.7/bugfix/CLDSRV-339-revert-S3C-7054 2023-02-06 23:08:19 +00:00
bert-e 00a793be6e Merge branch 'w/8.5/bugfix/CLDSRV-339-revert-S3C-7054' into tmp/octopus/w/8.6/bugfix/CLDSRV-339-revert-S3C-7054 2023-02-06 23:08:18 +00:00
bert-e 68bb824b57 Merge branch 'w/8.4/bugfix/CLDSRV-339-revert-S3C-7054' into tmp/octopus/w/8.5/bugfix/CLDSRV-339-revert-S3C-7054 2023-02-06 23:08:18 +00:00
Jonathan Gramain 432680841e Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-339-revert-S3C-7054' into w/8.4/bugfix/CLDSRV-339-revert-S3C-7054 2023-02-06 14:52:41 -08:00
bert-e b2641f5c1b Merge branch 'bugfix/CLDSRV-339-revert-S3C-7054' into tmp/octopus/w/7.70/bugfix/CLDSRV-339-revert-S3C-7054 2023-02-06 22:49:55 +00:00
Jonathan Gramain 66c34e0272 CLDSRV-331 Revert "CLDSRV-301 - use isNonversionedBucket flag"
This reverts commit 3da8f88a12.
2023-02-06 14:41:06 -08:00
bert-e 836e9fb22d Merge branch 'w/8.6/bugfix/CLDSRV-335-build-federation-image-tests' into tmp/octopus/w/8.7/bugfix/CLDSRV-335-build-federation-image-tests 2023-02-02 09:21:46 +00:00
bert-e 9bc7fa49ea Merge branch 'w/8.5/bugfix/CLDSRV-335-build-federation-image-tests' into tmp/octopus/w/8.6/bugfix/CLDSRV-335-build-federation-image-tests 2023-02-02 09:21:45 +00:00
bert-e e3087fb940 Merge branch 'w/8.4/bugfix/CLDSRV-335-build-federation-image-tests' into tmp/octopus/w/8.5/bugfix/CLDSRV-335-build-federation-image-tests 2023-02-02 09:21:45 +00:00
Dimitrios Vasilas 67e126320c Revert "CLDSRV-335: build federation image in tests workflow"
This reverts commit fd669664a6.
2023-02-02 04:21:00 -05:00
Dimitrios Vasilas 66520571d3 Revert "CLDSRV-335: build federation image in tests workflow"
This reverts commit fd669664a6.
2023-02-02 04:19:20 -05:00
bert-e ead7f5f7c2 Merge branch 'w/8.6/bugfix/CLDSRV-335-build-federation-image-tests' into tmp/octopus/w/8.7/bugfix/CLDSRV-335-build-federation-image-tests 2023-01-31 10:46:42 +00:00
bert-e fe636d22fc Merge branch 'w/8.5/bugfix/CLDSRV-335-build-federation-image-tests' into tmp/octopus/w/8.6/bugfix/CLDSRV-335-build-federation-image-tests 2023-01-31 10:46:41 +00:00
bert-e 6530e70761 Merge branch 'w/8.4/bugfix/CLDSRV-335-build-federation-image-tests' into tmp/octopus/w/8.5/bugfix/CLDSRV-335-build-federation-image-tests 2023-01-31 10:46:41 +00:00
bert-e 6d14bda3ed Merge branch 'w/7.70/bugfix/CLDSRV-335-build-federation-image-tests' into tmp/octopus/w/8.4/bugfix/CLDSRV-335-build-federation-image-tests 2023-01-31 10:46:40 +00:00
bert-e 416634cf11 Merge branch 'bugfix/CLDSRV-335-build-federation-image-tests' into tmp/octopus/w/7.70/bugfix/CLDSRV-335-build-federation-image-tests 2023-01-31 10:46:39 +00:00
Dimitrios Vasilas fd669664a6 CLDSRV-335: build federation image in tests workflow 2023-01-31 05:20:46 -05:00
bert-e c17059dc77 Merge branch 'w/8.6/improvement/CLDSRV-333-handle-MPU-delete-overwrite' into tmp/octopus/w/8.7/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-27 17:21:51 +00:00
bert-e b4617f1362 Merge branch 'w/8.5/improvement/CLDSRV-333-handle-MPU-delete-overwrite' into tmp/octopus/w/8.6/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-27 17:21:51 +00:00
bert-e 624d4708cf Merge branch 'w/8.4/improvement/CLDSRV-333-handle-MPU-delete-overwrite' into tmp/octopus/w/8.5/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-27 17:21:50 +00:00
bert-e 95c180e9d9 Merge branch 'w/7.70/improvement/CLDSRV-333-handle-MPU-delete-overwrite' into tmp/octopus/w/8.4/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-27 17:21:50 +00:00
bert-e 5a2b465d0f Merge branch 'improvement/CLDSRV-333-handle-MPU-delete-overwrite' into tmp/octopus/w/7.70/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-27 17:21:49 +00:00
Dimitrios Vasilas 2cd10e7195 CLDSRV-333 - Make docker image compatible with scality's registry
Images built with the `provenance` flag set to `true`
are currently not supported by our registry and older versions of docker.
2023-01-27 12:20:36 -05:00
bert-e 8ace5b24a5 Merge branches 'development/8.7' and 'w/8.6/improvement/CLDSRV-333-handle-MPU-delete-overwrite' into tmp/octopus/w/8.7/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-25 15:02:48 +00:00
bert-e 4b1dcd531d Merge branch 'w/8.5/improvement/CLDSRV-333-handle-MPU-delete-overwrite' into tmp/octopus/w/8.6/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-25 15:02:47 +00:00
bert-e 13ef509cbc Merge branch 'w/8.4/improvement/CLDSRV-333-handle-MPU-delete-overwrite' into tmp/octopus/w/8.5/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-25 15:02:47 +00:00
Dimitrios Vasilas d4feda7bbd CLDSRV-333: remove parentheses around single function argument 2023-01-25 09:34:47 -05:00
bert-e 39f7035dbd Merge branch 'w/8.6/improvement/CLDSRV-333-handle-MPU-delete-overwrite' into tmp/octopus/w/8.7/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-24 14:13:42 +00:00
bert-e 7d3ab342f6 Merge branch 'w/8.5/improvement/CLDSRV-333-handle-MPU-delete-overwrite' into tmp/octopus/w/8.6/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-24 14:13:41 +00:00
Dimitrios Vasilas af60df4caf Merge remote-tracking branch 'origin/w/8.4/improvement/CLDSRV-333-handle-MPU-delete-overwrite' into w/8.5/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-24 09:12:54 -05:00
Dimitrios Vasilas 2acd7348d4 Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-333-handle-MPU-delete-overwrite' into w/8.4/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-24 09:08:52 -05:00
williamlardier bb62ed4fa7
CLDSRV-334: bump cloudserver to 8.7.7 2023-01-24 12:33:37 +01:00
williamlardier c95368858d
CLDSRV-334: bump arsenal to 8.1.82 2023-01-24 12:33:17 +01:00
bert-e ffafe6ecfc Merge branch 'improvement/CLDSRV-333-handle-MPU-delete-overwrite' into tmp/octopus/w/7.70/improvement/CLDSRV-333-handle-MPU-delete-overwrite 2023-01-24 08:20:21 +00:00
Dimitrios Vasilas 4301fc57e2 CLDSRV-333: catch unintentional error 2023-01-23 10:50:44 -05:00
Dimitrios Vasilas 072d8324ca CLDSRV-333: test MPU delete/overwrite handling.
Add unit tests that check that the correct
fields are sent to Metadata when a MPU is
deleted or overwritten.
2023-01-23 10:50:44 -05:00
Dimitrios Vasilas 25276dae3f CLDSRV-333: add mpuUtils
Avoid code duplication in tests that create MPUs.
2023-01-23 10:50:44 -05:00
Dimitrios Vasilas bdeeb25d19 CLDSRV-333: add oldReplayId to metadataStoreParams on overwrite
When a MPU is overwritten by a PUT (createAndStoreObject)
or a copy (objectCopy), add the MPU's uploadId to
metadataStoreParams in a field called oldReplayId.
Metadata will use this field to cleanup the replay
key corrensponding to the overwritten MPU.
2023-01-23 10:50:44 -05:00
Dimitrios Vasilas 5dc17db9df CLDSRV-333: objectDelete: add MPU uploadId to delOptions
If the object to be deleted is a MPU,
include its uploadId as a field named replayId
to the options with which services.deleteObject
is called.
Metadata will use this field to cleanup the replay
key corresponding to the overwriten MPU.
2023-01-23 10:50:35 -05:00
bert-e d8ff1377fc Merge branch 'w/8.6/feature/CLDSRV-329/migrateToGithubActions-8.x' into tmp/octopus/w/8.7/feature/CLDSRV-329/migrateToGithubActions-8.x 2023-01-20 02:29:31 +00:00
bert-e 425a9167ca Merge branch 'w/8.5/feature/CLDSRV-329/migrateToGithubActions-8.x' into tmp/octopus/w/8.6/feature/CLDSRV-329/migrateToGithubActions-8.x 2023-01-20 02:29:30 +00:00
bert-e 2f21b9cc52 Merge branch 'feature/CLDSRV-329/migrateToGithubActions-8.x' into tmp/octopus/w/8.5/feature/CLDSRV-329/migrateToGithubActions-8.x 2023-01-20 02:29:30 +00:00
Alexander Chan d6433961a1 CLDSRV-329: adapt release step for 8.x 2023-01-19 18:28:59 -08:00
Alexander Chan 090b276f23 CLDSRV-329 migrate mongodb and ceph functional tests 2023-01-18 17:50:12 -08:00
Jonathan Gramain 28f4c5baee Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-330-setNextMarkerToCommonPrefix' into w/8.7/bugfix/CLDSRV-330-setNextMarkerToCommonPrefix 2023-01-14 11:49:44 -08:00
Jonathan Gramain 89a1c646ad Merge remote-tracking branch 'origin/w/8.5/bugfix/CLDSRV-330-setNextMarkerToCommonPrefix' into w/8.6/bugfix/CLDSRV-330-setNextMarkerToCommonPrefix 2023-01-14 11:49:05 -08:00
Jonathan Gramain 5c249f0c56 Merge remote-tracking branch 'origin/w/8.4/bugfix/CLDSRV-330-setNextMarkerToCommonPrefix' into w/8.5/bugfix/CLDSRV-330-setNextMarkerToCommonPrefix 2023-01-14 11:48:27 -08:00
Jonathan Gramain c971669b9b Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-330-setNextMarkerToCommonPrefix' into w/8.4/bugfix/CLDSRV-330-setNextMarkerToCommonPrefix 2023-01-14 11:47:23 -08:00
Jonathan Gramain 04e553b968 Merge remote-tracking branch 'origin/bugfix/CLDSRV-330-setNextMarkerToCommonPrefix' into w/7.70/bugfix/CLDSRV-330-setNextMarkerToCommonPrefix 2023-01-14 11:44:21 -08:00
Jonathan Gramain 57ef76548e CLDSRV-330 version bump 2023-01-14 00:21:24 -08:00
Jonathan Gramain 717a3274fc CLDSRV-330 Revert "CLDSRV-312 bump arsenal dep"
This reverts commit 3992ac2809.
2023-01-14 00:16:53 -08:00
Jonathan Gramain 1b59d0efb8 CLDSRV-330 bump arsenal version 2023-01-14 00:16:42 -08:00
bert-e 0a8f846f4b Merge branch 'w/8.6/feature/CLDSRV-244/migrateToGithubActions' into tmp/octopus/w/8.7/feature/CLDSRV-244/migrateToGithubActions 2023-01-12 22:54:04 +00:00
bert-e 045602fc00 Merge branch 'w/8.5/feature/CLDSRV-244/migrateToGithubActions' into tmp/octopus/w/8.6/feature/CLDSRV-244/migrateToGithubActions 2023-01-12 22:54:04 +00:00
Alexander Chan 5048c1fef1 Merge remote-tracking branch 'origin/w/8.4/feature/CLDSRV-244/migrateToGithubActions' into w/8.5/feature/CLDSRV-244/migrateToGithubActions 2023-01-12 14:34:55 -08:00
Alexander Chan 1e95d108be Merge remote-tracking branch 'origin/w/7.70/feature/CLDSRV-244/migrateToGithubActions' into w/8.4/feature/CLDSRV-244/migrateToGithubActions 2023-01-12 14:04:54 -08:00
bert-e 04abefd799 Merge branch 'feature/CLDSRV-244/migrateToGithubActions' into tmp/octopus/w/7.70/feature/CLDSRV-244/migrateToGithubActions 2023-01-12 21:14:30 +00:00
Alexander Chan 2772976e86 remove duplications 2023-01-12 13:13:45 -08:00
Alexander Chan 51905f82ba remove eve CI files 2023-01-12 13:13:45 -08:00
Alexander Chan b72adc50a7 CLDSRV-244: migrate eve CI to GitHub Actions
- multibackend functional tests
- file functional tests
- utapi v2 functional tests
- kmip functional tests
2023-01-12 13:13:45 -08:00
Thomas Carmet b6def80347 CLDSRV-115 migration to gh actions 2023-01-11 16:32:35 -08:00
Thomas Carmet b3f7a22a07 CLDSRV-115 migration github actions 2023-01-11 16:32:35 -08:00
Jonathan Gramain ac5de47ca1 Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-312-bumpArsenal' into w/8.7/bugfix/CLDSRV-312-bumpArsenal 2023-01-09 16:03:10 -08:00
Jonathan Gramain 3c0f3e671a Merge remote-tracking branch 'origin/w/8.5/bugfix/CLDSRV-312-bumpArsenal' into w/8.6/bugfix/CLDSRV-312-bumpArsenal 2023-01-09 13:48:58 -08:00
Jonathan Gramain a3dc3f9fb8 Merge remote-tracking branch 'origin/w/8.4/bugfix/CLDSRV-312-bumpArsenal' into w/8.5/bugfix/CLDSRV-312-bumpArsenal 2023-01-09 13:44:46 -08:00
Jonathan Gramain e4bf9500a3 Merge branch 'w/7.70/bugfix/CLDSRV-312-bumpArsenal' into w/8.4/bugfix/CLDSRV-312-bumpArsenal 2023-01-09 12:41:33 -08:00
Jonathan Gramain ac33897f25 Merge remote-tracking branch 'origin/bugfix/CLDSRV-312-bumpArsenal' into w/7.70/bugfix/CLDSRV-312-bumpArsenal 2023-01-09 11:33:15 -08:00
Jonathan Gramain c6a640ca9d CLDSRV-312 bump cloudserver version 2023-01-09 10:39:19 -08:00
Jonathan Gramain 3992ac2809 CLDSRV-312 bump arsenal dep
Bump arsenal dependency and fix NextMarker in listing unit tests

Because of changes in the Delimiter/DelimiterMaster listing
algorithms, the NextMarker is now always an actual listable key
instead of possibly be a common prefix, so existing tests need to
reflect this.
2023-01-09 10:39:00 -08:00
williamlardier c147785464
CLDSRV-322: bump cloudserver version 2023-01-06 09:04:04 +01:00
williamlardier ca8c788757
CLDSRV-322: code improvements 2023-01-06 09:04:03 +01:00
williamlardier cb2af364bb
CLDSRV-322: Implement test for custom routes
Unit and funcitonal tests are implemented to test the custom routes.
The LISTing is not yet tested, as it requires more changes to
generate a valid signature, from Mocha.
2023-01-05 15:31:33 +01:00
williamlardier 1eb27d610b
CLDSRV-322: Support custom files for MultiObjectDelete
MultiObjectDelete is implemented by the product UI to delete the
files in buckets. This method is a POST that relies on the request
body to filter the objects, hence, it is not possible to filter
it as an ingress rule in nginx.

The implementation tries to avoid adding any complexity
by extending existing loops, and implementing a new step if elligible
files are found.

These files are extracted from the Veeam route list of accepted files,
but this implementation might change if more custom APIs are supported
in the future.
2023-01-05 15:31:33 +01:00
williamlardier 73b295c91d
CLDSRV-322: Implement LIST for SOSAPI routes
Listing of objects is needed for consistent user experience in the
product's User Interface.

Listing is implemented as a `GET` request with a specific query parameter
`list-type` and folder `.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c`.

This API:
- Handles both versioned and non-versioned listing
- Relies on predefined templates to fill the response content
- Extracts the system.xml and capacity.xml files from the bucket
  metadata
- Computes the listing response based on the input query parameters
  and files currently in the bucket md capabilities
- Handle errors if any non supported query parameter is used. As any GET
  request is routed to this method, we return InvalidRequest if the requested
  action is not supported (i.e., not a listing v2)
2023-01-05 15:31:32 +01:00
williamlardier 8186c84bf9
CLDSRV-322: Implement DELETE for SOSAPI routes
Deletion of Veam SOSAPI files are required to ensure consistent
user experience. This API is only exposed to API/CLI clients, not
User Interface. The User Interface relies on MultiObjectDelete to
perform the deletions, and is handled in a separate commit.

This API:
- Checks that the requested file exists
- Erase the bucket metadata according to the file
- Update the bucket metadata with the updated values.
- Handle errors if the Veaam capability is not yet enabled for the bucket
2023-01-05 15:31:32 +01:00
williamlardier 93ef2d0545
CLDSRV-322: Implement HEAD for SOSAPI routes
HEAD object is not formally required by Veeam SOSAPI, but Veeam
relies on the last-modified date value of the capacity.xml file.
To suppoort any change in future SOSAPI standard, the HEAD method
is implemented, and is similar to the GET method, where only the metadata
are returned.
2023-01-05 15:31:31 +01:00
williamlardier d7d0a31bb1
CLDSRV-322: Implement PUT for SOSAPI routes
In the SOSAPI context, the user is requested to pre-created two files,
system.xml and capacity.xml under the veeam folder, to enable the feature.

This API:
- Extracts the XML from the provided file, convert it to JSON
- Validate that the JSON is valid against joi schemes, if applicable
- Updates the bucket metadata, including the last-modified date
- Update in the database the bucket metadata
- Return the standard success code response
- Handle invalid XML or XML structure, and return an error accordingly
2023-01-05 15:31:31 +01:00
williamlardier 4c69b82508
CLDSRV-322: Implement GET for SOSAPI
The GET method is used by SOSAPI to determine if SOS API is enabled
or not on a bucket.

Two files are supported: system.xml and capacity.xml.

This API:
- Get the bucket metadata
- Dynamically recomputes a valid XML based on the bucket md content
  using xml2js as headless, to enforce the same XML as the one
  from SOSAPI standard
- Rejects the request with an error if the bucket metadata does not
  exist
- Handle the `?tagging` request, required for versioned bucket, to
  return a static content.

Output stream relies on the utils file.
2023-01-05 15:31:30 +01:00
williamlardier ca13284da3
CLDSRV-322: implement common util functions
Custom SOSAPI routes might either retreive or stream data. The utils file
re-implement, with support for this particular context, some functions
from the standard API paths, from Arsenal.

These changes mostly introduce ways to compute the right HTTP headers as
well as input our output streams to handle GET or PUT request types.
2023-01-05 15:31:30 +01:00
williamlardier c6ed75a1d7
CLDSRV-322: implement SOSAPI scheme validator
SOSAPI relies on standard XML files for both the system and the capacity.
It is used by Veeam12+ to determine what capabilities and/or
configuration should be enforced for a given S3-integrated Bucket used
for backups.

The commit introduces scheme validation for JSON objects, as XML will
be first converted using xml2js.

The system.xml file includes the protocol version of SOSAPI: if the
version is not know, no validation is made, to allow for future changes
without formal need to update the product.

Note: maximum XML file size, in case of unsupported protocol version, will
be enforced to avoid spacing issues with the database.
2023-01-05 15:31:30 +01:00
williamlardier 402d0dea1a
CLDSRV-322: Create a new route for Veeam12 SOS API.
This new route is exposed through special nginx rules
from Zenko-Operator, to redirect any call to the veeam
folder, located under .system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c
at the root of the bucket. The goal is to store files in
the bucket metadata, to ease their update by internal jobs.
To avoid impacting standard API, we rely on custom routes
with dedicated logic to handle these files, in a generic
way.

This commit introduces a new route that will manage, in turn,
the:
- Handling of incoming request.
- Validity checks, including list of suppoorted APIs according
  to the HTTP verb and query parameters.
- Authentication and Authorization with Vault, in the same
  way as usual files.
- Check of the targeted bucket and/or keys, to extract the
  bucket metadata.
- Routing of the request to the right API handler.
2023-01-05 15:31:29 +01:00
williamlardier 95faec1db0
CLDSRV-322: bump arsenal version 2023-01-05 15:31:29 +01:00
Jonathan Gramain ca9d53f430 Merge remote-tracking branch 'origin/w/8.6/bugfix/CLDSRV-321-version-bump' into w/8.7/bugfix/CLDSRV-321-version-bump 2022-12-26 11:19:03 -08:00
Jonathan Gramain ba27ff7980 Merge remote-tracking branch 'origin/w/8.5/bugfix/CLDSRV-321-version-bump' into w/8.6/bugfix/CLDSRV-321-version-bump 2022-12-26 11:18:27 -08:00
Jonathan Gramain 8957997e23 Merge remote-tracking branch 'origin/w/8.4/bugfix/CLDSRV-321-version-bump' into w/8.5/bugfix/CLDSRV-321-version-bump 2022-12-26 11:17:26 -08:00
Jonathan Gramain 3caeda5d39 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-321-version-bump' into w/8.4/bugfix/CLDSRV-321-version-bump 2022-12-26 11:15:32 -08:00
Jonathan Gramain feed423f56 Merge remote-tracking branch 'origin/bugfix/CLDSRV-321-version-bump' into w/7.70/bugfix/CLDSRV-321-version-bump 2022-12-26 11:12:06 -08:00
Jonathan Gramain 4981d8f342 CLDSRV-321 bump version 2022-12-26 10:56:55 -08:00
bert-e b1ee1f8ef7 Merge branch 'w/8.6/bugfix/CLDSRV-321/fix_retention_extension_check' into tmp/octopus/w/8.7/bugfix/CLDSRV-321/fix_retention_extension_check 2022-12-21 23:27:26 +00:00
bert-e 28d778c2d4 Merge branch 'w/8.5/bugfix/CLDSRV-321/fix_retention_extension_check' into tmp/octopus/w/8.6/bugfix/CLDSRV-321/fix_retention_extension_check 2022-12-21 23:27:26 +00:00
bert-e b180aac9ba Merge branch 'w/8.4/bugfix/CLDSRV-321/fix_retention_extension_check' into tmp/octopus/w/8.5/bugfix/CLDSRV-321/fix_retention_extension_check 2022-12-21 23:27:25 +00:00
Taylor McKinnon c353452128 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-321/fix_retention_extension_check' into w/8.4/bugfix/CLDSRV-321/fix_retention_extension_check 2022-12-21 15:26:32 -08:00
bert-e 101b13abce Merge branch 'bugfix/CLDSRV-321/fix_retention_extension_check' into tmp/octopus/w/7.70/bugfix/CLDSRV-321/fix_retention_extension_check 2022-12-21 23:24:18 +00:00
Taylor McKinnon 9f5ae852bf bf(CLDSRV-321): Fix retention extension check to consider same date as extended 2022-12-21 15:11:21 -08:00
williamlardier e882cb6781
Merge remote-tracking branch 'origin/bugfix/CLDSRV-320-bump-arsenal' into w/8.7/bugfix/CLDSRV-320-bump-arsenal 2022-12-20 17:44:48 +01:00
williamlardier 8543f1a934
CLDSRV-320: bump project version 2022-12-20 17:43:02 +01:00
williamlardier fc871fbbfa
CLDSRV-320: bump arsenal to 8.1.77 2022-12-20 17:42:59 +01:00
Francois Ferrand cb7303636c
Release bump 8.7.1
Issue: CLDSRV-306
2022-12-16 19:56:19 +01:00
Francois Ferrand 6d0f889c23
Merge remote-tracking branch 'origin/feature/CLDSRV-306' into w/8.7/feature/CLDSRV-306 2022-12-16 19:54:23 +01:00
Francois Ferrand c13f2ae6a5
Merge remote-tracking branch 'origin/improvement/CLDSRV-305' into w/8.7/improvement/CLDSRV-305 2022-12-16 18:08:52 +01:00
Francois Ferrand 03058371e9
Release bump 8.6.4
Issue: CLDSRV-306
2022-12-16 17:54:09 +01:00
Francois Ferrand 473fed7594
Migrate tests to new azure storage sdk
Issue: CLDSRV-305
2022-12-16 17:54:09 +01:00
Francois Ferrand d86b9144be
Handle isSameAzureAccount() for other auth methods
Issue: CLDSRV-306
2022-12-16 17:54:09 +01:00
Francois Ferrand 2f2d9ced4c
Add unit tests for azure auth config
Issue: CLDSRV-306
2022-12-16 17:54:09 +01:00
Francois Ferrand 57a0ffc746
Support alternate azure auth methods in config
Issue: CLDSRV-306
2022-12-16 17:54:09 +01:00
Francois Ferrand d839cf2394
Bump arsenal
https://github.com/scality/Arsenal/tree/improvement/ARSN-281

Issue: CLDSRV-305
2022-12-16 16:21:43 +01:00
bert-e b6611c4711 Merge branch 'w/8.6/bugfix/CLDSRV-173-CLDSRV-170-CLDSRV-177-S3C-5390-development-7.10' into tmp/octopus/w/8.7/bugfix/CLDSRV-173-CLDSRV-170-CLDSRV-177-S3C-5390-development-7.10 2022-12-15 22:52:48 +00:00
bert-e 461f5ac5f9 Merge branch 'w/8.4/bugfix/CLDSRV-173-CLDSRV-170-CLDSRV-177-S3C-5390-development-7.10' into tmp/octopus/w/8.5/bugfix/CLDSRV-173-CLDSRV-170-CLDSRV-177-S3C-5390-development-7.10 2022-12-15 22:52:47 +00:00
bert-e 413a42adf0 Merge branch 'w/8.5/bugfix/CLDSRV-173-CLDSRV-170-CLDSRV-177-S3C-5390-development-7.10' into tmp/octopus/w/8.6/bugfix/CLDSRV-173-CLDSRV-170-CLDSRV-177-S3C-5390-development-7.10 2022-12-15 22:52:47 +00:00
Jonathan Gramain 7be27e0a83 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-173-CLDSRV-170-CLDSRV-177-S3C-5390-development-7.10' into w/8.4/bugfix/CLDSRV-173-CLDSRV-170-CLDSRV-177-S3C-5390-development-7.10 2022-12-15 14:51:13 -08:00
Jonathan Gramain 3d3252361d Merge remote-tracking branch 'origin/bugfix/CLDSRV-173-CLDSRV-170-CLDSRV-177-S3C-5390-development-7.10' into w/7.70/bugfix/CLDSRV-173-CLDSRV-170-CLDSRV-177-S3C-5390-development-7.10 2022-12-15 12:21:16 -08:00
Jonathan Gramain dad8bc7195 CLDSRV-173 CLDSRV-170 CLDSRV-177 S3C-5390 bump version 2022-12-15 11:47:24 -08:00
Artem Bakalov e6bda3460b remove .only
(cherry picked from commit b4725aa032)
(cherry picked from commit af95fea311)
2022-12-15 11:47:24 -08:00
Artem Bakalov 64334db65a S3C-5390 s3api head-object with part-number 1 on empty file fails: httpCode 416
(cherry picked from commit 4f3195a6ca)
(cherry picked from commit 8f3e737664)
2022-12-15 11:47:24 -08:00
Jonathan Gramain fa562ae85a CLDSRV-177 fix backbeat func tests for CLDSRV-290
Since CLDSRV-290, backbeat routes functional tests need to pass a
versionId in the query string in the `/_/backbeat/metadata` route,
hence the original tests from CLDSRV-177 needed to be updated.
2022-12-15 11:47:24 -08:00
Jonathan Gramain 6f32ebb2ce [7.10] CLDSRV-177 add missing test helper checkObjectData
(cherry picked from commit 413ebe743c)
(cherry picked from commit a1c4420eab)
2022-12-14 23:31:35 -08:00
Jonathan Gramain 2b32ec6163 bugfix: CLDSRV-177 fix crash with empty object replication
Fix a case of crash when a replication occurs with an empty object
over a non-empty object.

It is not clear how this happens in practice but there can be some
corner cases with race conditions between object replication and
versioning getting suspended on the target bucket at the same time,
that could lead to this situation, as the check between replication
configuration and actual replication is not atomic.

(cherry picked from commit a4e8cbebe6)
(cherry picked from commit 108d1c920f)
2022-12-14 23:31:29 -08:00
Jonathan Gramain d4063e157a bugfix: CLDSRV-170 skip orphan cleanup in UploadPart[Copy]
Do not delete orphan data in UploadPart/UploadPartCopy on overwrite
iff a CompleteMPU of the target MPU is already in progress.

This is to prevent a race condition where a CompleteMPU is running
while UploadPart is uploading a part for the same MPU.

It leaves an orphan in storage since only one of the upload data will
be present in the finished MPU, but the window is limited to the
CompleteMPU execution and should only occur when there are retries of
UploadPart due to prior stuck requests, or with broken clients
misusing the MPU API, so it should be acceptable.

Implementation details:

- set a flag in the MPU overview key when starting the CompleteMPU
  process, before listing the parts from metadata to construct the
  manifest

- in UploadPart/UploadPartCopy, after the part metadata is written and
  if the same part already existed, re-fetch the MPU overview key to
  check the flag: if set, skip the deletion of the old data of this
  part, since the CompleteMPU process in progress may choose either
  part data depending on the exact timing of the listing vs. the
  part overwrite.

(cherry picked from commit 8496111518)
(cherry picked from commit c2dbbfa008)
2022-12-14 23:31:07 -08:00
Nicolas Humbert a481384538 CLDSRV-173 DeleteMarkers created by Lifecycle should not be replicated
(cherry picked from commit d30430a81c)
2022-12-14 23:31:07 -08:00
bert-e ae4ece471b Merge branch 'w/8.7/improvement/CLDSRV-301-md-get-object-from-non-versioned-buckets-flag' into tmp/octopus/q/8.7 2022-12-14 21:19:55 +00:00
williamlardier 15b61cd947
CLDSRV-297: bump cloudserver to 8.7.0 2022-12-14 18:16:55 +01:00
williamlardier 91536c575f
CLDSRV-297: bump projects versions 2022-12-14 18:16:52 +01:00
bert-e 864ce1f27d Merge branch 'w/8.4/improvement/CLDSRV-301-md-get-object-from-non-versioned-buckets-flag' into tmp/octopus/w/8.5/improvement/CLDSRV-301-md-get-object-from-non-versioned-buckets-flag 2022-12-14 04:56:43 +00:00
bert-e 9d007a76b1 Merge branch 'w/8.5/improvement/CLDSRV-301-md-get-object-from-non-versioned-buckets-flag' into tmp/octopus/w/8.6/improvement/CLDSRV-301-md-get-object-from-non-versioned-buckets-flag 2022-12-14 04:56:43 +00:00
Artem Bakalov f4e292c6f9 Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-301-md-get-object-from-non-versioned-buckets-flag' into w/8.4/improvement/CLDSRV-301-md-get-object-from-non-versioned-buckets-flag 2022-12-13 20:55:44 -08:00
bert-e 436d1a9eab Merge branch 'improvement/CLDSRV-301-md-get-object-from-non-versioned-buckets-flag' into tmp/octopus/w/7.70/improvement/CLDSRV-301-md-get-object-from-non-versioned-buckets-flag 2022-12-14 04:52:48 +00:00
Artem Bakalov 3da8f88a12 CLDSRV-301 - use isNonversionedBucket flag 2022-12-13 20:51:28 -08:00
Francois Ferrand a2eb347fe3
Join latency lines when no operation
Latency is expected to be NaN when there are no operation: and we should
not mask this as "0 latency", which would be incorrect.

To make the graph easier to read and less confusing, we now 'join' the
lines if there is less than 3 minutes with no data.

As we plot the individual data points on the graph, the missing data can
still be observed by looking for these points.

Issue: CLDSRV-309
2022-11-25 14:03:35 +01:00
Francois Ferrand 0ff1262f97
Display metrics in op/s
This is more practical for ops, when dealing with large amount of data.

Issue: CLDSRV-309
2022-11-25 14:03:35 +01:00
Francois Ferrand 54a23d90c1
Fix http method breakdown
Should compute value based on all values in the current window.

Issue: CLDSRV-309
2022-11-23 21:42:06 +01:00
Kerkesni eb3dc9b79f
feature: CLDSRV-308 bump version in package.json to 8.6.3 2022-11-18 14:48:56 +01:00
bert-e 2c8968ef4a Merge branch 'feature/CLDSRV-304-support-object-restore-completed-notification' into q/8.6 2022-11-16 09:42:12 +00:00
Kerkesni a449aa35f4
feature: CLDSRV-303 support s3:ObjectRestore:Completed event notification 2022-11-15 19:11:49 +01:00
Kerkesni c2c8582585
feature: CLDSRV-303 support s3:ObjectRestore:Post event notification 2022-11-15 17:31:22 +01:00
Kerkesni 82c1bd7211
feature: CLDSRV-277 bump arsenal to 8.1.72 2022-11-14 11:23:02 +01:00
bert-e 776af747f2 Merge branch 'feature/CLDSRV-295/bumpArsenalVersion' into tmp/octopus/w/8.6/feature/CLDSRV-295/bumpArsenalVersion 2022-11-12 10:40:37 +00:00
Alexander Chan 453fec0cb0 CLDSRV-295: bump arsenal 8.1.71 2022-11-11 22:03:21 -08:00
bert-e f9fd3cae16 Merge branch 'w/8.5/bugfix/CLDSRV-293/bump_cloudserver_version' into tmp/octopus/w/8.6/bugfix/CLDSRV-293/bump_cloudserver_version 2022-11-11 22:01:36 +00:00
bert-e 3662c406ec Merge branch 'w/8.4/bugfix/CLDSRV-293/bump_cloudserver_version' into tmp/octopus/w/8.5/bugfix/CLDSRV-293/bump_cloudserver_version 2022-11-11 22:01:36 +00:00
Taylor McKinnon 243876ef81 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-293/bump_cloudserver_version' into w/8.4/bugfix/CLDSRV-293/bump_cloudserver_version 2022-11-11 14:01:05 -08:00
Taylor McKinnon cf4706816f Merge remote-tracking branch 'origin/bugfix/CLDSRV-293/bump_cloudserver_version' into w/7.70/bugfix/CLDSRV-293/bump_cloudserver_version 2022-11-11 13:56:54 -08:00
Taylor McKinnon 368971dacb bf(CLDSRV-293): Bump version 2022-11-11 13:46:00 -08:00
bert-e f6fe11b763 Merge branch 'w/8.5/bugfix/CLDSRV-293/refactor_olock_checks' into tmp/octopus/w/8.6/bugfix/CLDSRV-293/refactor_olock_checks 2022-11-11 19:54:55 +00:00
Taylor McKinnon 5f94fce344 Merge remote-tracking branch 'origin/w/8.4/bugfix/CLDSRV-293/refactor_olock_checks' into w/8.5/bugfix/CLDSRV-293/refactor_olock_checks 2022-11-11 11:54:16 -08:00
Taylor McKinnon af8420fe3c Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-293/refactor_olock_checks' into w/8.4/bugfix/CLDSRV-293/refactor_olock_checks 2022-11-11 11:52:21 -08:00
bert-e c3b209cbb5 Merge branch 'bugfix/CLDSRV-293/refactor_olock_checks' into tmp/octopus/w/7.70/bugfix/CLDSRV-293/refactor_olock_checks 2022-11-11 19:47:53 +00:00
Taylor McKinnon 3d6b7354a5 bf(CLDSRV-293): convert multiObjectDelete to use ObjectLockInfo helper 2022-11-11 11:47:14 -08:00
Taylor McKinnon a5d694a92c bf(CLDSRV-293): convert objectDelete to use ObjectLockInfo helper 2022-11-11 11:47:14 -08:00
Taylor McKinnon 990e821ac8 bf(CLDSRV-293): convert objectPutRetention to use ObjectLockInfo helper 2022-11-11 09:20:31 -08:00
Taylor McKinnon 8170bb9965 bf(CLDSRV-293): Add tests for object lock refactor 2022-11-11 09:20:07 -08:00
Taylor McKinnon 7e559d08c9 bf(CLDSRV-293): Refactor object lock helpers to centralize logic 2022-11-11 09:20:07 -08:00
bert-e e5c58ecc3d Merge branch 'w/8.5/bugfix/CLDSRV-296-removeGetProductVersion' into tmp/octopus/w/8.6/bugfix/CLDSRV-296-removeGetProductVersion 2022-11-04 20:17:34 +00:00
bert-e 6ef88fd60e Merge branch 'w/8.4/bugfix/CLDSRV-296-removeGetProductVersion' into tmp/octopus/w/8.5/bugfix/CLDSRV-296-removeGetProductVersion 2022-11-04 20:17:34 +00:00
bert-e 483e91a8d6 Merge branch 'w/7.70/bugfix/CLDSRV-296-removeGetProductVersion' into tmp/octopus/w/8.4/bugfix/CLDSRV-296-removeGetProductVersion 2022-11-04 20:17:33 +00:00
bert-e 3c99c67a33 Merge branch 'bugfix/CLDSRV-296-removeGetProductVersion' into tmp/octopus/w/7.70/bugfix/CLDSRV-296-removeGetProductVersion 2022-11-04 20:17:33 +00:00
Jonathan Gramain 29f87c7f2f chore: CLDSRV-296 remove get_product_version.sh
Not useful anymore and does not support hotfix versions x.y.z-p
2022-11-04 13:15:58 -07:00
Jonathan Gramain 7692d2c376 Merge remote-tracking branch 'origin/w/8.5/feature/CLDSRV-294-bump-7.10.19' into w/8.6/feature/CLDSRV-294-bump-7.10.19 2022-11-03 21:37:43 -07:00
Jonathan Gramain a0d7b07dc6 Merge remote-tracking branch 'origin/w/8.4/feature/CLDSRV-294-bump-7.10.19' into w/8.5/feature/CLDSRV-294-bump-7.10.19 2022-11-03 21:36:36 -07:00
Jonathan Gramain a9c21b98f9 Merge remote-tracking branch 'origin/w/7.70/feature/CLDSRV-294-bump-7.10.19' into w/8.4/feature/CLDSRV-294-bump-7.10.19 2022-11-03 21:23:12 -07:00
Jonathan Gramain fa9232f137 Merge remote-tracking branch 'origin/feature/CLDSRV-294-bump-7.10.19' into w/7.70/feature/CLDSRV-294-bump-7.10.19 2022-11-03 17:07:18 -07:00
Jonathan Gramain bcf3b4a16a CLDSRV-294 bump version to 7.10.19 2022-11-03 17:04:56 -07:00
bert-e 3257f4e905 Merge branch 'w/8.5/bugfix/CLDSRV-291-testObjectHeadWithBucketPolicy' into tmp/octopus/w/8.6/bugfix/CLDSRV-291-testObjectHeadWithBucketPolicy 2022-11-03 22:35:18 +00:00
bert-e 1d190019f7 Merge branch 'w/8.4/bugfix/CLDSRV-291-testObjectHeadWithBucketPolicy' into tmp/octopus/w/8.5/bugfix/CLDSRV-291-testObjectHeadWithBucketPolicy 2022-11-03 22:35:17 +00:00
Jonathan Gramain 79e7dc3946 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-291-testObjectHeadWithBucketPolicy' into w/8.4/bugfix/CLDSRV-291-testObjectHeadWithBucketPolicy 2022-11-03 15:31:25 -07:00
bert-e 1144e6bb33 Merge branch 'bugfix/CLDSRV-291-testObjectHeadWithBucketPolicy' into tmp/octopus/w/7.70/bugfix/CLDSRV-291-testObjectHeadWithBucketPolicy 2022-11-02 22:05:56 +00:00
Jonathan Gramain 950542237f CLDSRV-291 update arsenal dep 2022-11-02 15:02:29 -07:00
Jonathan Gramain a3c3511ff9 bugfix: CLDSRV-291 test for HEAD object with bucket policy
Add unit tests to show that HEAD object requests are allowed when
bucket policy grants the `s3:GetObject` permission to another account
or user
2022-11-01 18:28:57 -07:00
Francois Ferrand 7db26fae9a
Release 8.6.1 2022-10-28 15:36:35 +02:00
Francois Ferrand 7faf8c2366
Fix chunk upload/download size
The formula is not statistically accurate, but it gives an estimation,
assuming the repartition of object size is somewhat linear.

Issue: CLDSRV-288
2022-10-28 15:36:35 +02:00
Francois Ferrand e803078952
Add per-operation latency and count
Issue: CLDSRV-288
2022-10-28 15:36:35 +02:00
Francois Ferrand cfd72f3a38
Fix last report query
Issue: CLDSRV-288
2022-10-28 15:36:35 +02:00
Francois Ferrand 69a96d3993
Fix rounding of counts
Add round() operator in query instead of limiting to 0 decimals, to
allow grafana to display fractional value when there is a "unit" (like
`1.25K`)

Issue: CLDSRV-288
2022-10-28 15:36:35 +02:00
Taylor McKinnon d5bb8d8ed3 Merge remote-tracking branch 'origin/w/8.5/bugfix/CLDSRV-289/fix_utapiv2_delete_obj_suspended_versioning' into w/8.6/bugfix/CLDSRV-289/fix_utapiv2_delete_obj_suspended_versioning 2022-10-27 13:43:28 -07:00
Taylor McKinnon aeb8de54db Merge remote-tracking branch 'origin/w/8.4/bugfix/CLDSRV-289/fix_utapiv2_delete_obj_suspended_versioning' into w/8.5/bugfix/CLDSRV-289/fix_utapiv2_delete_obj_suspended_versioning 2022-10-27 13:37:03 -07:00
Taylor McKinnon 8f62260d70 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-289/fix_utapiv2_delete_obj_suspended_versioning' into w/8.4/bugfix/CLDSRV-289/fix_utapiv2_delete_obj_suspended_versioning 2022-10-27 13:36:19 -07:00
Taylor McKinnon 293930ff74 Merge remote-tracking branch 'origin/bugfix/CLDSRV-289/fix_utapiv2_delete_obj_suspended_versioning' into w/7.70/bugfix/CLDSRV-289/fix_utapiv2_delete_obj_suspended_versioning 2022-10-27 11:11:39 -07:00
Taylor McKinnon dd6deff075 bf(CLDSRV-289): Bump version to 7.10.18 2022-10-27 09:30:57 -07:00
Taylor McKinnon 4174106c2d bf(CLDSRV-289): Fix putDeleteMarkerObject metric for version suspended buckets 2022-10-27 09:30:57 -07:00
bert-e 29985f8955 Merge branch 'w/8.5/bugfix/CLDSRV-290-putMetadataRouteShouldNotRemoveValidData' into tmp/octopus/w/8.6/bugfix/CLDSRV-290-putMetadataRouteShouldNotRemoveValidData 2022-10-25 22:53:59 +00:00
Jonathan Gramain b081918317 Merge remote-tracking branch 'origin/w/8.4/bugfix/CLDSRV-290-putMetadataRouteShouldNotRemoveValidData' into w/8.5/bugfix/CLDSRV-290-putMetadataRouteShouldNotRemoveValidData 2022-10-25 15:53:06 -07:00
Jonathan Gramain 9049555887 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-290-putMetadataRouteShouldNotRemoveValidData' into w/8.4/bugfix/CLDSRV-290-putMetadataRouteShouldNotRemoveValidData 2022-10-25 10:47:34 -07:00
bert-e 41063705a9 Merge branch 'bugfix/CLDSRV-290-putMetadataRouteShouldNotRemoveValidData' into tmp/octopus/w/7.70/bugfix/CLDSRV-290-putMetadataRouteShouldNotRemoveValidData 2022-10-25 17:35:08 +00:00
Jonathan Gramain 7cdb395ee3 bugfix: CLDSRV-290 fix `PUT /_/backbeat/metadata` versioning logic
Fix the logic by always using the provided `versionId` in the query
string as the version to put, instead of relying on the version stored
in the metadata. Not passing a `versionId` now amounts to creating a
new version.

The previous logic was causing a possible confusion when no
`versionId` was passed in the query string, that allowed valid data
locations to be removed as if it was an overwrite.
2022-10-24 17:46:49 -07:00
Jonathan Gramain 45c6aefc35 bugfix: CLDSRV-290 [test] update `PUT /_/backbeat/metadata` tests
- modify existing `PUT /_/backbeat/metadata` tests to always pass the
  `versionId` in the query string, as it should be with the updated
  API contract

- create a new test that does not pass the `versionId` in the query
  string on an update, and expects a new version to be created (and
  both versions to be readable to ensure no cleanup occurred)

- general tech debt cleanup: update the test `versionId` to be in the
  new base64 format when encoded by removing the extra info, making it
  exactly 27 characters long
2022-10-24 17:45:24 -07:00
bert-e b125bcb0b7 Merge branch 'w/8.5/bugfix/CLDSRV-275/bump-utapi' into tmp/octopus/w/8.6/bugfix/CLDSRV-275/bump-utapi 2022-10-21 18:25:46 +00:00
bert-e dd93e2f0be Merge branch 'w/8.4/bugfix/CLDSRV-275/bump-utapi' into tmp/octopus/w/8.5/bugfix/CLDSRV-275/bump-utapi 2022-10-21 18:25:46 +00:00
Taylor McKinnon d8dc35f1cf Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-275/bump-utapi' into w/8.4/bugfix/CLDSRV-275/bump-utapi 2022-10-21 11:25:09 -07:00
Taylor McKinnon 3df9712648 Merge remote-tracking branch 'origin/bugfix/CLDSRV-275/bump-utapi' into w/7.70/bugfix/CLDSRV-275/bump-utapi 2022-10-21 11:21:38 -07:00
Taylor McKinnon d45b543053 bf(CLDSRV-275): Bump version to 7.10.17 2022-10-18 11:29:53 -07:00
Taylor McKinnon 3910b25f1c bf(CLDSRV-275): Bump utapi to 7.10.10 2022-10-18 11:27:11 -07:00
Francois Ferrand 4e935dff1a
Release 8.6.0
Issue: CLDSRV-287
2022-10-17 10:10:04 +02:00
Francois Ferrand ecd54df821
Use node:16.17.1 bulleye slim base image
* Use more recent base image to get CVE fixes
* Use separate builder image to minimize the prod image

Issue: CLDSRV-287
2022-10-17 10:09:30 +02:00
bert-e d523b6f1b6 Merge branch 'w/8.4/bugfix/CLDSRV-275-utapi-v1-delete-inconsistency-with-versioning-suspended' into tmp/octopus/w/8.5/bugfix/CLDSRV-275-utapi-v1-delete-inconsistency-with-versioning-suspended 2022-10-15 02:10:11 +00:00
Artem Bakalov ab95973786 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-275-utapi-v1-delete-inconsistency-with-versioning-suspended' into w/8.4/bugfix/CLDSRV-275-utapi-v1-delete-inconsistency-with-versioning-suspended 2022-10-14 19:09:19 -07:00
bert-e fa99e2f3b2 Merge branch 'bugfix/CLDSRV-275-utapi-v1-delete-inconsistency-with-versioning-suspended' into tmp/octopus/w/7.70/bugfix/CLDSRV-275-utapi-v1-delete-inconsistency-with-versioning-suspended 2022-10-15 02:07:33 +00:00
Artem Bakalov 49fded7d5f CLDSRV-275 - utapi-v1 delete inconsistency with versioning suspended 2022-10-14 18:49:49 -07:00
bert-e de094c53cd Merge branch 'w/8.4/bugfix/CLDSRV-285-update-release-dockerfile' into tmp/octopus/w/8.5/bugfix/CLDSRV-285-update-release-dockerfile 2022-10-07 20:49:44 +00:00
bert-e 0234ec7461 Merge branch 'w/7.70/bugfix/CLDSRV-285-update-release-dockerfile' into tmp/octopus/w/8.4/bugfix/CLDSRV-285-update-release-dockerfile 2022-10-07 20:49:44 +00:00
bert-e 34ece584a2 Merge branch 'bugfix/CLDSRV-285-update-release-dockerfile' into tmp/octopus/w/7.70/bugfix/CLDSRV-285-update-release-dockerfile 2022-10-07 20:49:44 +00:00
Naren 464930ff16 bf: CLDSRV-285 execute symlinking scality-kms separately 2022-10-07 13:48:01 -07:00
bert-e 4f1bd8e634 Merge branch 'w/8.4/bugfix/CLDSRV-285-update-release-dockerfile' into tmp/octopus/w/8.5/bugfix/CLDSRV-285-update-release-dockerfile 2022-10-06 23:08:24 +00:00
bert-e bcabab454c Merge branch 'w/7.70/bugfix/CLDSRV-285-update-release-dockerfile' into tmp/octopus/w/8.4/bugfix/CLDSRV-285-update-release-dockerfile 2022-10-06 23:08:23 +00:00
Naren fdcecbf5ef Merge remote-tracking branch 'origin/bugfix/CLDSRV-285-update-release-dockerfile' into w/7.70/bugfix/CLDSRV-285-update-release-dockerfile 2022-10-06 15:46:33 -07:00
Naren 9e186f7107 bf: CLDSRV-285 bump version to 7.10.16 2022-10-06 13:33:20 -07:00
Naren 82316c7b10 bf: CLDSRV-285 update release dockerfile 2022-10-06 13:29:34 -07:00
bert-e 47352b1df1 Merge branch 'w/8.4/bugfix/CLDSRV-285-update-release-dockerfile' into tmp/octopus/w/8.5/bugfix/CLDSRV-285-update-release-dockerfile 2022-10-06 18:52:22 +00:00
bert-e a019e89ebb Merge branch 'w/7.70/bugfix/CLDSRV-285-update-release-dockerfile' into tmp/octopus/w/8.4/bugfix/CLDSRV-285-update-release-dockerfile 2022-10-06 18:52:21 +00:00
bert-e d0eef7bf3f Merge branch 'bugfix/CLDSRV-285-update-release-dockerfile' into tmp/octopus/w/7.70/bugfix/CLDSRV-285-update-release-dockerfile 2022-10-06 18:52:21 +00:00
Naren 1db16d1cda bf: CLDSRV-285 update release dockerfile
install git-lfs as root
2022-10-06 11:51:27 -07:00
bert-e 59c6a9fb2a Merge branches 'w/8.5/bugfix/CLDSRV-285-correct-docker-image' and 'q/4824/8.4/bugfix/CLDSRV-285-correct-docker-image' into tmp/octopus/q/8.5 2022-10-05 16:29:08 +00:00
bert-e 0c27fbebea Merge branch 'w/8.4/bugfix/CLDSRV-285-correct-docker-image' into tmp/octopus/q/8.4 2022-10-05 16:29:08 +00:00
bert-e 01afc596e9 Merge branch 'w/8.4/bugfix/CLDSRV-285-correct-docker-image' into tmp/octopus/w/8.5/bugfix/CLDSRV-285-correct-docker-image 2022-10-05 02:51:42 +00:00
bert-e dff4c42971 Merge branch 'w/7.70/bugfix/CLDSRV-285-correct-docker-image' into tmp/octopus/w/8.4/bugfix/CLDSRV-285-correct-docker-image 2022-10-05 02:51:42 +00:00
bert-e 55710d6a64 Merge branch 'bugfix/CLDSRV-285-correct-docker-image' into tmp/octopus/w/7.70/bugfix/CLDSRV-285-correct-docker-image 2022-10-05 02:51:41 +00:00
Naren 6e714cdb84 bf: CLDSRV-285 update dockerfile to match federation 2022-10-04 19:50:17 -07:00
williamlardier 3ce13ddde9
Merge remote-tracking branch 'origin/bugfix/CLDSRV-286-bump-arsenal-fix-authz-regression' into w/8.5/bugfix/CLDSRV-286-bump-arsenal-fix-authz-regression 2022-10-04 19:43:21 +02:00
williamlardier a327aa83c1
CLDSRV-286: use latest Arsenal version
s
2022-10-04 19:39:14 +02:00
williamlardier 667cd471a4
CLDSRV-286: bump dockerfile base image 2022-10-04 14:53:00 +02:00
williamlardier 1b6b2ef4ed
Merge remote-tracking branch 'origin/w/8.4/improvement/CLDSRV-281-complete-missing-permission-checks' into w/8.5/improvement/CLDSRV-281-complete-missing-permission-checks 2022-09-27 14:05:51 +02:00
williamlardier 04b1d6c6a4
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-281-complete-missing-permission-checks' into w/8.4/improvement/CLDSRV-281-complete-missing-permission-checks 2022-09-27 13:55:20 +02:00
williamlardier 887ab2510d
Merge remote-tracking branch 'origin/improvement/CLDSRV-281-complete-missing-permission-checks' into w/7.70/improvement/CLDSRV-281-complete-missing-permission-checks 2022-09-27 13:54:10 +02:00
williamlardier 51f7e390e8
CLDSRV-281: bump project version 2022-09-27 13:52:07 +02:00
williamlardier 16b5ef230a
CLDSRV-281: add permission checks tests 2022-09-27 13:52:04 +02:00
williamlardier f543eb30e8
CLDSRV-281: add missing permission checks 2022-09-27 13:51:59 +02:00
williamlardier 50efadb55b
CLDSRV-281: bump arsenal version 2022-09-27 10:17:08 +02:00
bert-e a149336c1a Merge branch 'w/8.4/bugfix/CLDSRV-282/put-bucket-object-lock-regression' into tmp/octopus/w/8.5/bugfix/CLDSRV-282/put-bucket-object-lock-regression 2022-09-23 18:53:07 +00:00
bert-e d3847224a4 Merge branch 'w/7.70/bugfix/CLDSRV-282/put-bucket-object-lock-regression' into tmp/octopus/w/8.4/bugfix/CLDSRV-282/put-bucket-object-lock-regression 2022-09-23 18:53:07 +00:00
Ronnie Smith 66848a31e6
Merge remote-tracking branch 'origin/bugfix/CLDSRV-282/put-bucket-object-lock-regression' into w/7.70/bugfix/CLDSRV-282/put-bucket-object-lock-regression 2022-09-23 11:13:29 -07:00
Ronnie Smith 9e76f3b769
CLDSRV-282 bump package.json version 2022-09-23 11:06:08 -07:00
bert-e ee090c4f03 Merge branch 'w/8.4/bugfix/CLDSRV-282/put-bucket-object-lock-regression' into tmp/octopus/w/8.5/bugfix/CLDSRV-282/put-bucket-object-lock-regression 2022-09-22 20:58:55 +00:00
bert-e 8f2aa95ec8 Merge branch 'w/7.70/bugfix/CLDSRV-282/put-bucket-object-lock-regression' into tmp/octopus/w/8.4/bugfix/CLDSRV-282/put-bucket-object-lock-regression 2022-09-22 20:58:55 +00:00
bert-e 17595bf0af Merge branch 'bugfix/CLDSRV-282/put-bucket-object-lock-regression' into tmp/octopus/w/7.70/bugfix/CLDSRV-282/put-bucket-object-lock-regression 2022-09-22 20:58:54 +00:00
Ronnie Smith bbb3e2f7ce
bugfix: CLDSRV-282 fix regression from CLDSRV-280
* auth results can be an array of arrays
* Object.assign needs an empty object or it modifies the original
* should be `_isObjectLockEnabled` not `!_isObjectLockEnabled`
* add unit tests
2022-09-22 13:58:17 -07:00
bert-e c8cdd8eacb Merge branch 'w/8.4/feature/CLDSRV-278/fix-github-action-typo' into tmp/octopus/w/8.5/feature/CLDSRV-278/fix-github-action-typo 2022-09-21 21:21:35 +00:00
bert-e 30455b9d6f Merge branch 'w/7.70/feature/CLDSRV-278/fix-github-action-typo' into tmp/octopus/w/8.4/feature/CLDSRV-278/fix-github-action-typo 2022-09-21 21:21:35 +00:00
bert-e 773bfe1f14 Merge branch 'feature/CLDSRV-278/fix-github-action-typo' into tmp/octopus/w/7.70/feature/CLDSRV-278/fix-github-action-typo 2022-09-21 21:21:34 +00:00
Ronnie Smith 2bc80795a9
feature: CLDSRV-278 remove trailing quote 2022-09-21 14:19:28 -07:00
bert-e 73474be2fe Merge branch 'w/8.4/feature/CLDSRV-278/fix-workflow-env-var-usage' into tmp/octopus/w/8.5/feature/CLDSRV-278/fix-workflow-env-var-usage 2022-09-21 04:47:17 +00:00
bert-e 5cdbe049cf Merge branch 'w/7.70/feature/CLDSRV-278/fix-workflow-env-var-usage' into tmp/octopus/w/8.4/feature/CLDSRV-278/fix-workflow-env-var-usage 2022-09-21 04:47:16 +00:00
bert-e 2e40cd1b4c Merge branch 'feature/CLDSRV-278/fix-workflow-env-var-usage' into tmp/octopus/w/7.70/feature/CLDSRV-278/fix-workflow-env-var-usage 2022-09-21 04:47:16 +00:00
Ronnie Smith bace3047ec
feature: CLDSRV-278 no need for needs 2022-09-20 21:46:57 -07:00
bert-e d819e9128a Merge branch 'w/8.4/feature/CLDSRV-278/fix-workflow-env-var-usage' into tmp/octopus/w/8.5/feature/CLDSRV-278/fix-workflow-env-var-usage 2022-09-21 01:25:38 +00:00
bert-e d5dcd1f2c1 Merge branch 'w/7.70/feature/CLDSRV-278/fix-workflow-env-var-usage' into tmp/octopus/w/8.4/feature/CLDSRV-278/fix-workflow-env-var-usage 2022-09-21 01:25:38 +00:00
Ronnie Smith 9079221ba0
Merge remote-tracking branch 'origin/feature/CLDSRV-278/fix-workflow-env-var-usage' into w/7.70/feature/CLDSRV-278/fix-workflow-env-var-usage 2022-09-20 18:02:13 -07:00
Ronnie Smith 0f06277dce
feature: CLDSRV-278 bump cloudserver version 2022-09-20 17:38:47 -07:00
bert-e bc835899d0 Merge branch 'w/8.4/feature/CLDSRV-278/fix-workflow-env-var-usage' into tmp/octopus/w/8.5/feature/CLDSRV-278/fix-workflow-env-var-usage 2022-09-20 23:13:44 +00:00
bert-e 0e741e0b6f Merge branch 'w/7.70/feature/CLDSRV-278/fix-workflow-env-var-usage' into tmp/octopus/w/8.4/feature/CLDSRV-278/fix-workflow-env-var-usage 2022-09-20 23:13:43 +00:00
bert-e 33df88ac2d Merge branch 'feature/CLDSRV-278/fix-workflow-env-var-usage' into tmp/octopus/w/7.70/feature/CLDSRV-278/fix-workflow-env-var-usage 2022-09-20 23:13:43 +00:00
Ronnie Smith 71143fd0cf
feature: CLDSRV-278 use separate job to store env vars 2022-09-20 15:59:35 -07:00
bert-e bfcfb43999 Merge branch 'w/8.4/bugfix/CLDSRV-280/iam-user-can-put-object-with-retention-when-policy-deny' into tmp/octopus/w/8.5/bugfix/CLDSRV-280/iam-user-can-put-object-with-retention-when-policy-deny 2022-09-20 20:57:08 +00:00
bert-e 00fc3496ac Merge branch 'w/7.70/bugfix/CLDSRV-280/iam-user-can-put-object-with-retention-when-policy-deny' into tmp/octopus/w/8.4/bugfix/CLDSRV-280/iam-user-can-put-object-with-retention-when-policy-deny 2022-09-20 20:57:08 +00:00
bert-e 547a8cc2d0 Merge branch 'bugfix/CLDSRV-280/iam-user-can-put-object-with-retention-when-policy-deny' into tmp/octopus/w/7.70/bugfix/CLDSRV-280/iam-user-can-put-object-with-retention-when-policy-deny 2022-09-20 20:57:08 +00:00
Ronnie 2d7990cb59
Merge branch 'development/7.10' into bugfix/CLDSRV-280/iam-user-can-put-object-with-retention-when-policy-deny 2022-09-20 10:20:02 -07:00
Ronnie 3569d816bd
Fix typo of context
Co-authored-by: William <91462779+williamlardier@users.noreply.github.com>
2022-09-20 09:16:50 -07:00
bert-e 1d702112f0 Merge branch 'w/8.4/feature/CLDSRV-278/provide-image-for-federation' into tmp/octopus/w/8.5/feature/CLDSRV-278/provide-image-for-federation 2022-09-19 19:59:20 +00:00
Ronnie Smith 1b35948ce2
Merge remote-tracking branch 'origin/w/7.70/feature/CLDSRV-278/provide-image-for-federation' into w/8.4/feature/CLDSRV-278/provide-image-for-federation 2022-09-19 12:58:31 -07:00
bert-e c25d4661b4 Merge branch 'feature/CLDSRV-278/provide-image-for-federation' into tmp/octopus/w/7.70/feature/CLDSRV-278/provide-image-for-federation 2022-09-16 18:09:37 +00:00
Ronnie b554a7e517
Add spacing 2022-09-16 11:09:31 -07:00
Ronnie Smith 58e7c66eae
bugfix: CLDSRV-280 add check for object put retention for object put 2022-09-16 01:18:12 -07:00
bert-e 664ea4a23a Merge branch 'feature/CLDSRV-278/provide-image-for-federation' into tmp/octopus/w/7.70/feature/CLDSRV-278/provide-image-for-federation 2022-09-15 19:26:22 +00:00
Ronnie Smith 89843bc2ab
feature: CLDSRV-278 add github action to release image for fed 2022-09-15 12:25:55 -07:00
Alexander Chan 113c5c166f Merge remote-tracking branch 'origin/w/8.4/improvement/CLDSRV-276/bump-7.10.12' into w/8.5/improvement/CLDSRV-276/bump-7.10.12 2022-09-12 07:52:21 -07:00
Alexander Chan 53a988b167 Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-276/bump-7.10.12' into w/8.4/improvement/CLDSRV-276/bump-7.10.12 2022-09-12 03:02:22 -07:00
Alexander Chan 6bee199b56 Merge remote-tracking branch 'origin/improvement/CLDSRV-276/bump-7.10.12' into w/7.70/improvement/CLDSRV-276/bump-7.10.12 2022-09-11 23:20:00 -07:00
Alexander Chan 2a935b34ed CLDSRV-276: bump 7.10.12 2022-09-09 16:48:35 -07:00
bert-e d5b31cb669 Merge branch 'w/8.4/bugfix/CLDSRV-269/policy-checks-for-put-bucket-with-object-lock' into tmp/octopus/w/8.5/bugfix/CLDSRV-269/policy-checks-for-put-bucket-with-object-lock 2022-09-09 22:11:58 +00:00
Ronnie Smith 9bf176b7fb
Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-269/policy-checks-for-put-bucket-with-object-lock' into w/8.4/bugfix/CLDSRV-269/policy-checks-for-put-bucket-with-object-lock 2022-09-09 15:11:06 -07:00
bert-e d2d5b1ee0a Merge branch 'bugfix/CLDSRV-269/policy-checks-for-put-bucket-with-object-lock' into tmp/octopus/w/7.70/bugfix/CLDSRV-269/policy-checks-for-put-bucket-with-object-lock 2022-09-09 22:09:33 +00:00
Ronnie Smith a916cac32d
bugfix: CLDSRV-269 add unit test and check all results 2022-09-07 23:09:02 -07:00
Ronnie 58bc54db87
Merge branch 'development/7.10' into bugfix/CLDSRV-269/policy-checks-for-put-bucket-with-object-lock 2022-09-02 00:01:58 -07:00
Ronnie Smith 5d42dad5c3
Merge branch 'bugfix/CLDSRV-269/policy-checks-for-put-bucket-with-object-lock' of github.com:scality/cloudserver into bugfix/CLDSRV-269/policy-checks-for-put-bucket-with-object-lock 2022-09-02 00:01:02 -07:00
Ronnie 09c8cd0fbd
Merge branch 'development/7.10' into bugfix/CLDSRV-269/policy-checks-for-put-bucket-with-object-lock 2022-09-01 22:29:23 -07:00
bert-e 7c92f34ee0 Merge branches 'w/8.5/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' and 'q/4757/8.4/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' into tmp/octopus/q/8.5 2022-08-31 10:03:38 +00:00
bert-e 81d34525c7 Merge branches 'w/8.4/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' and 'q/4757/7.70/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' into tmp/octopus/q/8.4 2022-08-31 10:03:37 +00:00
bert-e a571bfc721 Merge branches 'w/7.70/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' and 'q/4757/7.10/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' into tmp/octopus/q/7.70 2022-08-31 10:03:37 +00:00
bert-e 84a7547628 Merge branch 'bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' into q/7.10 2022-08-31 10:03:37 +00:00
bert-e 9af76eb0ce Merge branches 'development/8.5' and 'w/8.4/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' into tmp/octopus/w/8.5/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass 2022-08-31 02:32:21 +00:00
bert-e d3a622ea27 Merge branches 'development/8.4' and 'w/7.70/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' into tmp/octopus/w/8.4/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass 2022-08-31 02:32:21 +00:00
bert-e a0b1e6b308 Merge branches 'development/7.70' and 'bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' into tmp/octopus/w/7.70/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass 2022-08-31 02:32:21 +00:00
Ronnie fedb0547e1
add a little more detail to invalid object lock state 2022-08-30 19:32:12 -07:00
bert-e 76301c9ec4 Merge branches 'w/8.5/bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi' and 'q/4772/8.4/bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi' into tmp/octopus/q/8.5 2022-08-31 01:32:15 +00:00
bert-e 3b6a2c9a55 Merge branches 'w/8.4/bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi' and 'q/4772/7.70/bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi' into tmp/octopus/q/8.4 2022-08-31 01:32:15 +00:00
bert-e c0b89c650e Merge branch 'bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi' into q/7.10 2022-08-31 01:32:14 +00:00
bert-e 97feb483c0 Merge branches 'w/7.70/bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi' and 'q/4772/7.10/bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi' into tmp/octopus/q/7.70 2022-08-31 01:32:14 +00:00
bert-e 5fb3cf0ede Merge branch 'w/8.4/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' into tmp/octopus/w/8.5/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass 2022-08-30 09:04:07 +00:00
Ronnie Smith 3a7c8f920b
Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' into w/8.4/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass 2022-08-30 02:03:20 -07:00
bert-e 0e2b3b0f53 Merge branch 'bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass' into tmp/octopus/w/7.70/bugfix/CLDSRV-270/retention-time-cannot-be-increased-without-bypass 2022-08-30 09:00:06 +00:00
bert-e 83cf51a3d0 Merge branch 'w/8.4/bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi' into tmp/octopus/w/8.5/bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi 2022-08-30 04:19:27 +00:00
Artem Bakalov 9544b18f2e Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi' into w/8.4/bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi 2022-08-29 21:18:19 -07:00
bert-e b73f28964a Merge branch 'bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi' into tmp/octopus/w/7.70/bugfix/CLDSRV-264/ignore_lifecycle_event_in_utapi 2022-08-29 23:21:03 +00:00
Ronnie Smith 3239e8da75
bugfix: CLDSRV-270 do not check permissions if bypass is not required 2022-08-29 10:22:47 -07:00
bert-e 22905c8967 Merge branch 'w/8.4/improvement/CLDSRV-274-release-7-10-10' into tmp/octopus/w/8.5/improvement/CLDSRV-274-release-7-10-10 2022-08-29 11:40:07 +00:00
KillianG d05f027837
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-274-release-7-10-10' into w/8.4/improvement/CLDSRV-274-release-7-10-10 2022-08-29 13:39:22 +02:00
KillianG 0a3986adb7
Merge remote-tracking branch 'origin/improvement/CLDSRV-274-release-7-10-10' into w/7.70/improvement/CLDSRV-274-release-7-10-10 2022-08-29 13:34:18 +02:00
Killian Gardahaut 5a97f88b14
Merge branch 'development/7.10' into improvement/CLDSRV-274-release-7-10-10 2022-08-29 08:35:18 +02:00
Taylor McKinnon fa43f86ce1 bf(CLDSRV-264): Ignore lifecycle non-data touching events in utapi metrics 2022-08-26 18:36:42 -07:00
bert-e f0b34678f7 Merge branch 'w/8.4/bugfix/CLDSRV-273/doNotSetEmptyStringDelimiter' into tmp/octopus/w/8.5/bugfix/CLDSRV-273/doNotSetEmptyStringDelimiter 2022-08-26 19:08:09 +00:00
bert-e cddaef2bb9 Merge branch 'w/7.70/bugfix/CLDSRV-273/doNotSetEmptyStringDelimiter' into tmp/octopus/w/8.4/bugfix/CLDSRV-273/doNotSetEmptyStringDelimiter 2022-08-26 19:08:08 +00:00
bert-e c9d5c6cc18 Merge branch 'bugfix/CLDSRV-273/doNotSetEmptyStringDelimiter' into tmp/octopus/w/7.70/bugfix/CLDSRV-273/doNotSetEmptyStringDelimiter 2022-08-26 19:08:08 +00:00
Alexander Chan 1217a496ff CLDSRV-273: skip delimiter as listing param if empty 2022-08-26 12:00:38 -07:00
Killian Gardahaut 3911c37d8e
CLDSRV-274: release 7.10.10 2022-08-26 13:07:14 +02:00
bert-e fdbfec2bcc Merge branch 'w/8.4/improvement/CLDSRV-271-bump-arsenal-7-10-34' into tmp/octopus/w/8.5/improvement/CLDSRV-271-bump-arsenal-7-10-34 2022-08-26 08:28:39 +00:00
KillianG 3f11dab32b
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-271-bump-arsenal-7-10-34' into w/8.4/improvement/CLDSRV-271-bump-arsenal-7-10-34 2022-08-26 10:27:54 +02:00
bert-e e1ae7b84f2 Merge branch 'improvement/CLDSRV-271-bump-arsenal-7-10-34' into tmp/octopus/w/7.70/improvement/CLDSRV-271-bump-arsenal-7-10-34 2022-08-26 08:25:58 +00:00
Killian Gardahaut 2b35351649 CLDSRV-271: Update yarnlock 2022-08-26 08:06:58 +00:00
Killian Gardahaut dacbc85e62
Merge branch 'development/7.10' into improvement/CLDSRV-271-bump-arsenal-7-10-34 2022-08-25 15:25:29 +02:00
Killian Gardahaut 1ed1513729 CLDSRV-271: bump arsenal 7.10.34 2022-08-25 13:23:28 +00:00
Ronnie ab84030e8e
Merge branch 'development/7.10' into bugfix/CLDSRV-269/policy-checks-for-put-bucket-with-object-lock 2022-08-22 12:01:27 -07:00
Ronnie Smith e4c17569ce
bugfix: CLDSRV-269 add policy checks for object lock in put bucket 2022-08-22 00:24:33 -07:00
Jonathan Gramain 3be5f2633c Merge remote-tracking branch 'origin/w/8.4/feature/CLDSRV-268-release-7.10.9' into w/8.5/feature/CLDSRV-268-release-7.10.9 2022-08-19 15:21:29 -07:00
Jonathan Gramain d947e4ffb2 Merge remote-tracking branch 'origin/w/7.70/feature/CLDSRV-268-release-7.10.9' into w/8.4/feature/CLDSRV-268-release-7.10.9 2022-08-19 15:18:43 -07:00
Jonathan Gramain 8a982bbc37 Merge remote-tracking branch 'origin/feature/CLDSRV-268-release-7.10.9' into w/7.70/feature/CLDSRV-268-release-7.10.9 2022-08-19 14:23:45 -07:00
Jonathan Gramain 2360e410e3 CLDSRV-268 release 7.10.9 2022-08-19 14:19:02 -07:00
bert-e 4407b46d06 Merge branch 'w/8.5/bugfix/CLDSRV-257-callTagConditionKeyAuthOnce' into tmp/octopus/q/8.5 2022-08-18 00:04:22 +00:00
bert-e b1d42091b3 Merge branch 'bugfix/CLDSRV-267/bump' into q/8.5 2022-08-17 23:35:54 +00:00
Nicolas Humbert dbc99acd0d CLDSRV-267 bump arsenal 8.1.65 2022-08-17 19:04:19 -04:00
bert-e 8ec404dc7a Merge branch 'w/8.4/bugfix/CLDSRV-257-callTagConditionKeyAuthOnce' into tmp/octopus/w/8.5/bugfix/CLDSRV-257-callTagConditionKeyAuthOnce 2022-08-17 23:00:24 +00:00
Jonathan Gramain a1a7e4d888 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-257-callTagConditionKeyAuthOnce' into w/8.4/bugfix/CLDSRV-257-callTagConditionKeyAuthOnce 2022-08-17 15:59:43 -07:00
bert-e 93751da82a Merge branch 'bugfix/CLDSRV-257-callTagConditionKeyAuthOnce' into tmp/octopus/w/7.70/bugfix/CLDSRV-257-callTagConditionKeyAuthOnce 2022-08-17 22:57:35 +00:00
bert-e 1b560fa584 Merge branch 'w/8.4/bugfix/CLDSRV-266-checkCheckTagConditionsBeforeIsAllowed' into tmp/octopus/w/8.5/bugfix/CLDSRV-266-checkCheckTagConditionsBeforeIsAllowed 2022-08-17 22:16:17 +00:00
Jonathan Gramain 8a1828ef4c Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-266-checkCheckTagConditionsBeforeIsAllowed' into w/8.4/bugfix/CLDSRV-266-checkCheckTagConditionsBeforeIsAllowed 2022-08-17 15:15:36 -07:00
bert-e f4243e6408 Merge branch 'bugfix/CLDSRV-266-checkCheckTagConditionsBeforeIsAllowed' into tmp/octopus/w/7.70/bugfix/CLDSRV-266-checkCheckTagConditionsBeforeIsAllowed 2022-08-17 22:11:42 +00:00
bert-e 717d9f844e Merge branch 'w/8.4/bugfix/CLDSRV-256-fixAndTestUpdateRequestContexts' into tmp/octopus/w/8.5/bugfix/CLDSRV-256-fixAndTestUpdateRequestContexts 2022-08-17 21:02:13 +00:00
Jonathan Gramain 04e2396b3b CLDSRV-256 [8.4] disable linter check for ES6 array iteration 2022-08-17 14:01:49 -07:00
Jonathan Gramain ffcf5517e8 bugfix: CLDSRV-257 call tagConditionKeyAuth only once
For requests not of type "objectPut" or "objectPutPart", an extra call
to tagConditionKeyAuth was made, resulting in unnecessary requests.

Refactored callApiMethod() wrapper to first gather the POST data which
may contain request tags used for condition checks, then authorize the
request.
2022-08-16 17:14:34 -07:00
Jonathan Gramain e426faa324 bugfix: CLDSRV-266 prioritize "checkTagConditions" over "isAllowed"
In the generic API handler callApiMethod(), authorization checks are
done via a call to a local helper checkAuthResults(). When tag
conditions need to be checked, a flag "checkTagConditions" is set by
Vault in the authorization response, however, currently the flag
"isAllowed" is also set to true (but the request is not necessarily
allowed in the end, after tag conditions are checked later on).

In order to avoid requests being allowed by default when tag
conditions are present but cannot be checked yet, we need to
prioritize checking the "checkTagConditions" flag before checking the
"isAllowed" flag, and on the other hand, have Vault return "isAllowed:
false" for such requests. This commit is doing the first part: moving
the authorization check after tag condition checks.

Also: removed an extra argument "updatedContexts" to the callback of
tagConditionKeyAuth() since the function does not return such argument
(the code ignored the argument when it was missing, so it is only a
cleanup).
2022-08-15 16:44:24 -07:00
bert-e 443f239b8e Merge branch 'w/8.4/bugfix/CLDSRV-256-fixAndTestUpdateRequestContexts' into tmp/octopus/w/8.5/bugfix/CLDSRV-256-fixAndTestUpdateRequestContexts 2022-08-12 20:28:24 +00:00
Jonathan Gramain c57b6ff0e4 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-256-fixAndTestUpdateRequestContexts' into w/8.4/bugfix/CLDSRV-256-fixAndTestUpdateRequestContexts 2022-08-12 13:27:39 -07:00
bert-e d807379c2d Merge branch 'bugfix/CLDSRV-256-fixAndTestUpdateRequestContexts' into tmp/octopus/w/7.70/bugfix/CLDSRV-256-fixAndTestUpdateRequestContexts 2022-08-12 20:25:56 +00:00
Jonathan Gramain 5265c1b35d CLDSRV-256 fix and test tag conditions updateRequestContexts() helper
Various issues were in this function that needed a full refactor.

Merged functional tests that were essentially unit tests into the
existing unit tests.
2022-08-12 13:23:54 -07:00
Jonathan Gramain 43cc84ac9b Merge remote-tracking branch 'origin/w/8.4/improvement/CLDSRV-261-release-7.10.8' into w/8.5/improvement/CLDSRV-261-release-7.10.8 2022-08-11 12:15:52 -07:00
Jonathan Gramain 806c79be7c Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-261-release-7.10.8' into w/8.4/improvement/CLDSRV-261-release-7.10.8 2022-08-11 11:54:56 -07:00
Jonathan Gramain 0cdaf92b00 Merge remote-tracking branch 'origin/improvement/CLDSRV-261-release-7.10.8' into w/7.70/improvement/CLDSRV-261-release-7.10.8 2022-08-11 10:54:36 -07:00
Jonathan Gramain 46454ac80b improvement: CLDSRV-261 release 7.10.8 2022-08-11 10:50:11 -07:00
bert-e 760ef6e0d8 Merge branch 'w/8.4/improvement/CLDSRV-260-remove-package-lock-json' into tmp/octopus/w/8.5/improvement/CLDSRV-260-remove-package-lock-json 2022-08-11 08:40:21 +00:00
bert-e dcc1b32049 Merge branch 'w/7.70/improvement/CLDSRV-260-remove-package-lock-json' into tmp/octopus/w/8.4/improvement/CLDSRV-260-remove-package-lock-json 2022-08-11 08:40:21 +00:00
bert-e 7d85a7702e Merge branch 'improvement/CLDSRV-260-remove-package-lock-json' into tmp/octopus/w/7.70/improvement/CLDSRV-260-remove-package-lock-json 2022-08-11 08:40:20 +00:00
Killian Gardahaut 54fa51c101 CLDSRV-260: remove package-lock.json 2022-08-11 08:38:37 +00:00
bert-e 7bd3ec9954 Merge branch 'w/8.4/improvement/CLDSRV-259-bump-arsenal-7-10-31' into tmp/octopus/w/8.5/improvement/CLDSRV-259-bump-arsenal-7-10-31 2022-08-10 12:34:55 +00:00
KillianG 0b7c6a76cc
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-259-bump-arsenal-7-10-31' into w/8.4/improvement/CLDSRV-259-bump-arsenal-7-10-31 2022-08-10 14:34:32 +02:00
bert-e 4dbbd31599 Merge branch 'improvement/CLDSRV-259-bump-arsenal-7-10-31' into tmp/octopus/w/7.70/improvement/CLDSRV-259-bump-arsenal-7-10-31 2022-08-10 11:58:18 +00:00
KillianG 4179227367
CLDSRV-259: update lockfiles 2022-08-10 13:58:07 +02:00
bert-e f115aeb7c2 Merge branch 'w/8.4/improvement/CLDSRV-259-bump-arsenal-7-10-31' into tmp/octopus/w/8.5/improvement/CLDSRV-259-bump-arsenal-7-10-31 2022-08-10 10:07:27 +00:00
KillianG add9e37712
Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-259-bump-arsenal-7-10-31' into w/8.4/improvement/CLDSRV-259-bump-arsenal-7-10-31 2022-08-10 12:06:35 +02:00
bert-e ce0f7383aa Merge branch 'improvement/CLDSRV-259-bump-arsenal-7-10-31' into tmp/octopus/w/7.70/improvement/CLDSRV-259-bump-arsenal-7-10-31 2022-08-10 10:05:28 +00:00
Killian Gardahaut cc4eac28ac
CLDSRV-259: bump arsenal 7.10.31 2022-08-10 12:04:22 +02:00
bert-e c0082d495d Merge branch 'w/8.4/bugfix/CLDSRV-258-setOriginOpInTaggingAndACLOps' into tmp/octopus/w/8.5/bugfix/CLDSRV-258-setOriginOpInTaggingAndACLOps 2022-08-08 20:46:14 +00:00
Jonathan Gramain 555b583354 Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-258-setOriginOpInTaggingAndACLOps' into w/8.4/bugfix/CLDSRV-258-setOriginOpInTaggingAndACLOps 2022-08-08 13:45:30 -07:00
bert-e 2b21481eb8 Merge branch 'bugfix/CLDSRV-258-setOriginOpInTaggingAndACLOps' into tmp/octopus/w/7.70/bugfix/CLDSRV-258-setOriginOpInTaggingAndACLOps 2022-08-08 20:44:26 +00:00
Jonathan Gramain 2225a8ebb4 bugfix: CLDSRV-258 set 'originOp' in MD for ACL ops
Set properly the 'originOp' field in object metadata to the
corresponding notification event type when doing ACL operations, so
that bucket notifications do not take this event for an object
creation (as the original field value was left as a create-type
event).
2022-08-08 13:42:51 -07:00
Kerkesni e394cc304c feature: CLDSRV-176 support s3:ObjectTagging:* notifications
Note: cherry-picked as part of CLDSRV-258 to fix an issue with
notifications.

(cherry picked from commit 53d9f8cf99)
2022-08-08 13:41:54 -07:00
bert-e e70753446f Merge branch 'w/8.4/improvement/CLDSRV-253-skipFlakyACLTests' into tmp/octopus/w/8.5/improvement/CLDSRV-253-skipFlakyACLTests 2022-07-29 23:49:15 +00:00
bert-e 29a3a6d845 Merge branch 'w/7.70/improvement/CLDSRV-253-skipFlakyACLTests' into tmp/octopus/w/8.4/improvement/CLDSRV-253-skipFlakyACLTests 2022-07-29 23:49:14 +00:00
bert-e 5110275a7e Merge branch 'improvement/CLDSRV-253-skipFlakyACLTests' into tmp/octopus/w/7.70/improvement/CLDSRV-253-skipFlakyACLTests 2022-07-29 23:49:14 +00:00
Jonathan Gramain 70d4646af6 improvement: CLDSRV-253 skip flaky ACL tests on E2E 2022-07-29 16:46:55 -07:00
Nicolas Humbert f2e7aec6c8 CLDSRV-252 Bump CloudServer 8.5.11 2022-07-28 17:45:32 -04:00
bert-e 9e85e8dd9e Merge branch 'bugfix/CLDSRV-246/objectPutVersion' into q/8.5 2022-07-28 17:39:28 +00:00
Nicolas Humbert f3f4937578 CLDSRV-246 PutObjectVersion always returns 403 2022-07-27 17:11:38 -04:00
bert-e bcb879c2ff Merge branch 'w/8.4/bugfix/CLDSRV-250/fix_acl_replication' into tmp/octopus/w/8.5/bugfix/CLDSRV-250/fix_acl_replication 2022-07-26 21:09:42 +00:00
Taylor McKinnon 615ace071f Merge remote-tracking branch 'origin/w/7.70/bugfix/CLDSRV-250/fix_acl_replication' into w/8.4/bugfix/CLDSRV-250/fix_acl_replication 2022-07-26 14:09:05 -07:00
Taylor McKinnon 3cfcc9aa28 Merge remote-tracking branch 'origin/bugfix/CLDSRV-250/fix_acl_replication' into w/7.70/bugfix/CLDSRV-250/fix_acl_replication 2022-07-26 14:06:33 -07:00
Taylor McKinnon 49a9146550 impr(CLDSRV-250): Release 7.10.7 2022-07-26 13:57:21 -07:00
Taylor McKinnon 885315bb93 bf(CLDSRV-250): Set replication status to `PENDING` when putting an ACL
Rollback objectPutACL changes from - feature: CLDSRV-162 use metadata and data from new arsenal fbd68e6839
2022-07-26 13:55:43 -07:00
Taylor McKinnon d4ae083b5a Merge remote-tracking branch 'origin/w/8.4/improvement/CLDSRV-249/release_7_10_6' into w/8.5/improvement/CLDSRV-249/release_7_10_6 2022-07-22 14:39:36 -07:00
Taylor McKinnon e97e410ee4 Merge remote-tracking branch 'origin/w/7.70/improvement/CLDSRV-249/release_7_10_6' into w/8.4/improvement/CLDSRV-249/release_7_10_6 2022-07-22 14:33:33 -07:00
Taylor McKinnon 91c202edec Merge remote-tracking branch 'origin/improvement/CLDSRV-249/release_7_10_6' into w/7.70/improvement/CLDSRV-249/release_7_10_6 2022-07-22 14:26:33 -07:00
Taylor McKinnon b2ad4dfa96 impr(CLDSRV-249): Bump agressore memory and and git-lfs to builder image 2022-07-22 14:24:44 -07:00
Taylor McKinnon 11029f2d90 impr(CLDSRV-249): Release 7.10.6 2022-07-22 11:49:11 -07:00
Kerkesni 3d86abd70a
bugfix: CLDSRV-248 bump cloudserver to 8.5.9 2022-07-22 15:49:24 +02:00
Kerkesni dc4dd2595e
bugfix: CLDSRV-248 fix azure mpu replication issue 2022-07-22 15:49:24 +02:00
bert-e b712df6a1f Merge branch 'feature/CLDSRV-243-allow-backbeat-to-update-non-versioned-objects' into q/8.5 2022-07-20 08:14:26 +00:00
Kerkesni 356c9f1e9c
feature: CLDSRV-243 bump cloudserver 8.5.8 2022-07-20 09:45:03 +02:00
Nicolas Humbert 51e28def0e CLDSRV-247 Bump Arsenal 8.1.62 2022-07-19 09:23:05 +02:00
bert-e 55b0400b25 Merge branches 'w/8.5/improvement/CLDSRV-239-version-bump' and 'q/4656/8.4/improvement/CLDSRV-239-version-bump' into tmp/octopus/q/8.5 2022-07-18 11:36:08 +00:00
bert-e 4fc5ac5e58 Merge branches 'w/8.4/improvement/CLDSRV-239-version-bump' and 'q/4656/7.70/improvement/CLDSRV-239-version-bump' into tmp/octopus/q/8.4 2022-07-18 11:36:07 +00:00
bert-e 95b607d991 Merge branch 'q/4656/7.10/improvement/CLDSRV-239-version-bump' into tmp/normal/q/7.70 2022-07-18 11:36:07 +00:00
bert-e 09006f9a68 Merge branch 'improvement/CLDSRV-239-version-bump' into q/7.10 2022-07-18 11:36:06 +00:00
bert-e c2f86b63eb Merge branch 'w/8.4/improvement/CLDSRV-239-version-bump' into tmp/octopus/w/8.5/improvement/CLDSRV-239-version-bump 2022-07-18 11:09:11 +00:00
bert-e 35ecf8e556 Merge branch 'w/7.70/improvement/CLDSRV-239-version-bump' into tmp/octopus/w/8.4/improvement/CLDSRV-239-version-bump 2022-07-18 11:09:11 +00:00
Will Toozs aa6297a35b
Merge remote-tracking branch 'origin/improvement/CLDSRV-239-version-bump' into w/7.70/improvement/CLDSRV-239-version-bump 2022-07-18 13:06:15 +02:00
bert-e 4e89f4b025 Merge branches 'w/8.5/improvement/CLDSRV-234-bump-arsenal' and 'q/4630/8.4/improvement/CLDSRV-234-bump-arsenal' into tmp/octopus/q/8.5 2022-07-18 09:28:11 +00:00
bert-e fc5bde533c Merge branch 'improvement/CLDSRV-234-bump-arsenal' into q/8.4 2022-07-18 09:28:11 +00:00
Rached Ben Mustapha 4fa5bf7409 CLDSRV-245: temporarily disable objectPutVersion action match 2022-07-14 15:27:19 +00:00
Will Toozs 5effe07ebd
Merge remote-tracking branch 'origin/improvement/CLDSRV-234-bump-arsenal' into w/8.5/improvement/CLDSRV-234-bump-arsenal 2022-07-13 18:30:19 +02:00
Will Toozs dce1f83322
improvement/CLDSRV-234: change build memory 2022-07-13 17:35:47 +02:00
Will Toozs 89259c82cb
CLDSRV-239: bump version for bucket tagging 2022-07-13 17:03:38 +02:00
Kerkesni 7e405ff963
feature: CLDSRV-243 allow backbeat to update non versioned objects
When replicating a non versioned OOB object, Backbeat’s ReplicationStatusProcessor
needs to only update the master version of that object.
2022-07-13 15:29:29 +02:00
bert-e ce705c8e78 Merge branch 'w/8.4/improvement/CLDSRV-237-revert-tagging' into tmp/octopus/w/8.5/improvement/CLDSRV-237-revert-tagging 2022-07-13 09:37:38 +00:00
Will Toozs 2ca6fb2fe6
CLDSRV-237-revert-bucket-tagging 2022-07-13 11:36:42 +02:00
bert-e 7b423666fe Merge branch 'w/7.70/improvement/CLDSRV-237-revert-tagging' into tmp/octopus/w/8.4/improvement/CLDSRV-237-revert-tagging 2022-07-13 09:30:58 +00:00
Will Toozs 07fc8c35d1
Revert "CLDSRV-237-revert-bucket-tagging"
This reverts commit 0f4a09fecc.
2022-07-13 11:29:38 +02:00
bert-e ec07bedd0b Merge branch 'w/8.4/improvement/CLDSRV-237-revert-tagging' into tmp/octopus/w/8.5/improvement/CLDSRV-237-revert-tagging 2022-07-13 09:18:08 +00:00
bert-e dc76bbb5c4 Merge branch 'w/7.70/improvement/CLDSRV-237-revert-tagging' into tmp/octopus/w/8.4/improvement/CLDSRV-237-revert-tagging 2022-07-13 09:18:08 +00:00
bert-e 75757a541b Merge branch 'improvement/CLDSRV-237-revert-tagging' into tmp/octopus/w/7.70/improvement/CLDSRV-237-revert-tagging 2022-07-13 09:18:07 +00:00
bert-e e130629ff9 Merge branch 'bugfix/CLDSRV-241/put-metadata' into q/8.5 2022-07-12 16:40:11 +00:00
Nicolas Humbert 8e60c2d300 CLDSRV-241 Allow updating object metadata with empty data location 2022-07-12 18:18:54 +02:00
Will Toozs 5c1b237e9e
CLDSRV-234: lockfile 2022-07-12 11:43:46 +02:00
Will Toozs 8d7bd0809c
CLDSRV-234: bump arsenal 2022-07-12 11:43:45 +02:00
Will Toozs 0f4a09fecc
CLDSRV-237-revert-bucket-tagging 2022-07-11 17:57:25 +02:00
bert-e 31b3469e4a Merge branch 'w/8.4/improvement/CLDSRV-235-bump-package.json' into tmp/octopus/w/8.5/improvement/CLDSRV-235-bump-package.json 2022-07-11 12:12:17 +00:00
Anurag Mittal d841a31bf3
Merge remote-tracking branch 'origin/improvement/CLDSRV-235-bump-package.json' into w/8.4/improvement/CLDSRV-235-bump-package.json 2022-07-11 14:11:43 +02:00
Anurag Mittal ca91cb1b9d
improvement: CLDSRV-235-bump-package.json-to-v7.70.0 2022-07-11 14:08:03 +02:00
Taylor McKinnon 47c6c7acf3 Merge remote-tracking branch 'origin/development/8.5' into w/8.5/bugfix/CLDSRV-232/dont_send_empty_NextContinuationToken_on_listing_end 2022-07-06 10:56:04 -07:00
Nicolas Humbert 493ba63384 CLDSRV-233 Bump CloudServer 8.5.5 2022-07-06 15:39:51 +02:00
Nicolas Humbert 6f1c3286a9 fix object Lock tests 2022-07-06 10:01:38 +02:00
bert-e b8058920d7 Merge branch 'bugfix/CLDSRV-232/dont_send_empty_NextContinuationToken_on_listing_end' into tmp/octopus/w/8.4/bugfix/CLDSRV-232/dont_send_empty_NextContinuationToken_on_listing_end 2022-07-05 22:59:46 +00:00
bert-e 1138a5952c Merge branch 'w/8.4/bugfix/CLDSRV-232/dont_send_empty_NextContinuationToken_on_listing_end' into tmp/octopus/w/8.5/bugfix/CLDSRV-232/dont_send_empty_NextContinuationToken_on_listing_end 2022-07-05 22:59:46 +00:00
Taylor McKinnon 4069a94f78 bf(CLDSRV-232): Prevent empty NextContinuationToken from being sent at listing end 2022-07-05 15:57:09 -07:00
Nicolas Humbert 53708a5197 update arsenal 8.1.58 2022-07-05 22:00:12 +02:00
Nicolas Humbert 4e0a497367 CLDSRV-231 location type DMF prevents cloudserver from starting 2022-07-01 15:34:13 +02:00
williamlardier 7e6d5e5629
CLDSRV-224: return only once 2022-06-21 11:32:30 +02:00
williamlardier 266776650e
CLDSRV-224: bump package.json version 2022-06-21 10:01:24 +02:00
williamlardier 36d910fd97
CLDSRV-224: switch back to callback mode 2022-06-21 10:00:08 +02:00
williamlardier f5781c3609
CLDSRV-224: add monitoring for bucket tagging APIs 2022-06-21 10:00:08 +02:00
williamlardier 3378b6a439
CLDSRV-224: tests fixups 2022-06-21 10:00:07 +02:00
Will Toozs f47687de10
CLDSRV-224: bucket tagging tests 2022-06-21 10:00:07 +02:00
Will Toozs 9899e95cab
CLDSRV-224: add getBucketTagging 2022-06-21 10:00:07 +02:00
williamlardier 4c60757086
CLDSRV-225: switch back to callback mode 2022-06-21 09:57:03 +02:00
Will Toozs 09062e3020
CLDSRV-225: Implement delete bucket tagging 2022-06-21 09:57:01 +02:00
bert-e 2389f36f34 Merge branch 'feature/CLDSRV-223-implement-putbuckettagging' into q/8.5 2022-06-21 07:18:20 +00:00
Ronnie Smith 0852be8a2b
feature: CLDSRV-229 bump to 8.5.3 2022-06-17 11:47:02 -07:00
KillianG 3a5236239e
CLDSRV-167: use arsenal parse tag function 2022-06-17 10:43:23 +02:00
williamlardier a77e558d53
CLDSRV-223: Bump Arsenal 2022-06-17 10:43:02 +02:00
williamlardier 888b760834
CLDSRV-223: Fix comparison and add tests 2022-06-17 10:43:01 +02:00
KillianG a1c8c9adc1
CLDSRV-223: Implement bucket owner check and change check order
Checking order has been changed to fit with AWS one, meaning that we
raise the first error we face

- Checking every tag one by one
- Checking that the tag key is unique, the error is raised only if we
  already seen a tag with the same key before
- Checking that the key length is less than 128
- Checking that the tag value length is less than 256
- After seeing more than 50 tags, if there is one more, raise the too
  many tags error
2022-06-17 10:43:01 +02:00
KillianG 7ac2adb23f
CLDSRV-167: implement put bucket tagging 2022-06-17 10:43:01 +02:00
bert-e 3e42758950 Merge branch 'feature/CLDSRV-220/add-s3-object-restore-operation' into q/8.5 2022-06-15 17:41:43 +00:00
bert-e aa70b840b7 Merge branch 'w/8.4/improvement/CLDSRV-228-bump-buckettagging' into tmp/octopus/w/8.5/improvement/CLDSRV-228-bump-buckettagging 2022-06-15 14:08:00 +00:00
Will Toozs 90b7316043
Merge remote-tracking branch 'origin/improvement/CLDSRV-228-bump-buckettagging' into w/8.4/improvement/CLDSRV-228-bump-buckettagging 2022-06-15 16:07:33 +02:00
Will Toozs a2ae5cc5e4
CLDSRV-228: bump version to 7.10.4 2022-06-15 16:01:43 +02:00
bert-e 8daac3c50b Merge branches 'w/8.5/improvement/CLDSRV-169-get-bucket-tagging' and 'q/4481/8.4/improvement/CLDSRV-169-get-bucket-tagging' into tmp/octopus/q/8.5 2022-06-15 08:39:13 +00:00
bert-e 835cd193d4 Merge branches 'w/8.4/improvement/CLDSRV-169-get-bucket-tagging' and 'q/4481/7.10/improvement/CLDSRV-169-get-bucket-tagging' into tmp/octopus/q/8.4 2022-06-15 08:39:12 +00:00
bert-e 75cbd72c52 Merge branch 'improvement/CLDSRV-169-get-bucket-tagging' into q/7.10 2022-06-15 08:39:12 +00:00
bert-e 27777296bf Merge branch 'w/8.4/improvement/CLDSRV-169-get-bucket-tagging' into tmp/octopus/w/8.5/improvement/CLDSRV-169-get-bucket-tagging 2022-06-15 08:11:55 +00:00
Will Toozs d8f73ce56c
Merge remote-tracking branch 'origin/improvement/CLDSRV-169-get-bucket-tagging' into w/8.4/improvement/CLDSRV-169-get-bucket-tagging 2022-06-15 10:11:29 +02:00
bert-e 4dbfe27254 Merge branch 'improvement/CLDSRV-168-delete-bucket-tagging' into q/7.10 2022-06-14 18:53:30 +00:00
bert-e 8fe33dee76 Merge branches 'w/8.5/improvement/CLDSRV-168-delete-bucket-tagging' and 'q/4480/8.4/improvement/CLDSRV-168-delete-bucket-tagging' into tmp/octopus/q/8.5 2022-06-14 18:53:30 +00:00
bert-e cff15fe737 Merge branches 'w/8.4/improvement/CLDSRV-168-delete-bucket-tagging' and 'q/4480/7.10/improvement/CLDSRV-168-delete-bucket-tagging' into tmp/octopus/q/8.4 2022-06-14 18:53:30 +00:00
bert-e dec7a13106 Merge branch 'w/8.4/improvement/CLDSRV-168-delete-bucket-tagging' into tmp/octopus/w/8.5/improvement/CLDSRV-168-delete-bucket-tagging 2022-06-14 18:29:02 +00:00
Will Toozs 6ce675ce01
Merge remote-tracking branch 'origin/improvement/CLDSRV-168-delete-bucket-tagging' into w/8.4/improvement/CLDSRV-168-delete-bucket-tagging 2022-06-14 20:28:28 +02:00
bert-e 32658b7e3b Merge branch 'w/8.4/improvement/CLDSRV-167-put-bucket-tagging' into tmp/octopus/w/8.5/improvement/CLDSRV-167-put-bucket-tagging 2022-06-14 15:31:55 +00:00
KillianG fcf617acf0
Merge remote-tracking branch 'origin/improvement/CLDSRV-167-put-bucket-tagging' into w/8.4/improvement/CLDSRV-167-put-bucket-tagging 2022-06-14 17:31:29 +02:00
KillianG 910e62e0c9
CLDSRV-1167: use arsenal parse tag function 2022-06-14 16:52:58 +02:00
bert-e 67e1611edb Merge branches 'development/8.5' and 'w/8.4/improvement/CLDSRV-167-put-bucket-tagging' into tmp/octopus/w/8.5/improvement/CLDSRV-167-put-bucket-tagging 2022-06-14 12:57:38 +00:00
bert-e 05532878d6 Merge branch 'development/8.4' into tmp/octopus/w/8.4/improvement/CLDSRV-167-put-bucket-tagging 2022-06-14 12:57:37 +00:00
KillianG 68037356f9
Merge remote-tracking branch 'origin/improvement/CLDSRV-167-put-bucket-tagging' into w/8.4/improvement/CLDSRV-167-put-bucket-tagging 2022-06-14 14:57:09 +02:00
Will Toozs 3e83c7d836
CLDSRV-169: bucket tagging tests 2022-06-14 14:10:30 +02:00
Will Toozs 63ae80e76e
CLDSRV-169: add getBucketTagging 2022-06-14 13:35:20 +02:00
Will Toozs 8729f86db0
CLDSRV-168: add deleteBucketTagging 2022-06-14 12:55:30 +02:00
Killian Gardahaut 66110ad2ac
Update bucketPutTagging.js 2022-06-14 10:41:09 +02:00
Ronnie Smith fdfb76a99a
feature: CLDSRV-220 add origin op for object restore 2022-06-13 15:19:14 -07:00
williamlardier ab59e98977
Merge remote-tracking branch 'origin/bugfix/CLDSRV-219-bump-arsenal-version-with-sproxydclient-fix' into w/8.5/bugfix/CLDSRV-219-bump-arsenal-version-with-sproxydclient-fix 2022-06-10 14:48:14 +02:00
williamlardier 505c421014
CLDSRV-219: bump arsenal version to fix ts related errors 2022-06-10 14:45:08 +02:00
williamlardier e046b87eec
CLDSRV-219: skip NonCurrentTransition obsolete test 2022-06-10 11:42:18 +02:00
williamlardier 1b2f2478c8
CLDSRV-219: bump Arsenal version 2022-06-10 11:41:44 +02:00
bert-e f476a11faf Merge branches 'w/8.5/improvement/CLDSRV-216' and 'q/4575/8.4/improvement/CLDSRV-216' into tmp/octopus/q/8.5 2022-06-08 20:26:59 +00:00
bert-e f61c585184 Merge branch 'improvement/CLDSRV-216' into q/8.4 2022-06-08 20:26:59 +00:00
bert-e 0174252f54 Merge branch 'improvement/CLDSRV-216' into tmp/octopus/w/8.5/improvement/CLDSRV-216 2022-06-08 18:46:13 +00:00
Francois Ferrand 5a13eecd05
Use standard release procedure
Bump version **when** doing the release, not afterwards, as it is done
almost everywhere else:

* It makes the git history harder to read, with a tag on any commit AND
a separate commit (anywhere) which bumps the version
* It does not work with “yarn version” : which creates a misleadingly
commit named “v8.x.x” in our repos
* It requires creating an extra ticket after the release ; and this
ticket does not actually represent anything: it is just a placeholder to “please” Bert-e (whereas a ticket fo>
* It creates the risk this extra ticket/bump is done, and someone
creating another bump when doing the next release (i.e. skipping a
release)

Issue: CLDSRV-216
2022-06-08 20:00:50 +02:00
bert-e bbd2dfefd7 Merge branch 'feature/CLDSRV-211' into q/8.4 2022-06-07 22:12:29 +00:00
bert-e b491a3e92c Merge branches 'w/8.5/feature/CLDSRV-211' and 'q/4549/8.4/feature/CLDSRV-211' into tmp/octopus/q/8.5 2022-06-07 22:12:29 +00:00
bert-e 9cb550514e Merge branch 'feature/CLDSRV-214' into q/8.5 2022-06-07 21:39:26 +00:00
Nicolas Humbert 74df4fcd65 CLDSRV-217 Bump CloudServer 8.5.2 2022-06-07 08:35:25 -04:00
bert-e 3a842d3b93 Merge branch 'w/8.4/improvement/CLDSRV-167-put-bucket-tagging' into tmp/octopus/w/8.5/improvement/CLDSRV-167-put-bucket-tagging 2022-06-07 11:55:06 +00:00
KillianG 63a790a3d7
Revert "Merge remote-tracking branch 'origin/improvement/CLDSRV-167-put-bucket-tagging' into w/8.4/improvement/CLDSRV-167-put-bucket-tagging"
This reverts commit 518253dc5e, reversing
changes made to ca91044fde.
2022-06-07 13:54:50 +02:00
KillianG 518253dc5e
Merge remote-tracking branch 'origin/improvement/CLDSRV-167-put-bucket-tagging' into w/8.4/improvement/CLDSRV-167-put-bucket-tagging 2022-06-07 13:54:37 +02:00
Killian Gardahaut c3d3171906
Merge branch 'development/7.10' into improvement/CLDSRV-167-put-bucket-tagging 2022-06-07 11:12:17 +02:00
Francois Ferrand 717c5c6a7d
Remove left-over tiny version id
Tiny version id code has been removed, there is only Short version id,
which is handled fully in Arsenal by setting S3_VERSION_ID_ENCODING_TYPE
variable.

The `VersionId.encode()` function does not even actually support passing
an encoding type anymore, so the code was of no use anymore.

Issue: CLDSRV-214
2022-06-07 10:09:46 +02:00
bert-e 22fdaad636 Merge branches 'w/8.5/bugfix/CLDSRV-215/delete-objects' and 'q/4560/8.4/bugfix/CLDSRV-215/delete-objects' into tmp/octopus/q/8.5 2022-06-02 19:42:44 +00:00
bert-e ca91044fde Merge branch 'q/4560/7.10/bugfix/CLDSRV-215/delete-objects' into tmp/normal/q/8.4 2022-06-02 19:42:44 +00:00
bert-e 266e0f9aa3 Merge branch 'w/8.4/bugfix/CLDSRV-215/delete-objects' into tmp/normal/q/8.4 2022-06-02 19:42:44 +00:00
bert-e e9d41f8db3 Merge branch 'bugfix/CLDSRV-215/delete-objects' into q/7.10 2022-06-02 19:42:42 +00:00
bert-e b63f210b8f Merge branch 'w/8.4/bugfix/CLDSRV-215/delete-objects' into tmp/octopus/w/8.5/bugfix/CLDSRV-215/delete-objects 2022-06-02 12:53:16 +00:00
Nicolas Humbert 61f30e659c CLDSRV-215 adapt tests for CEPH 2022-06-02 08:52:48 -04:00
bert-e 2c375ee10d Merge branch 'w/8.4/bugfix/CLDSRV-215/delete-objects' into tmp/octopus/w/8.5/bugfix/CLDSRV-215/delete-objects 2022-06-02 12:45:05 +00:00
bert-e 8638f00c95 Merge branch 'bugfix/CLDSRV-215/delete-objects' into tmp/octopus/w/8.4/bugfix/CLDSRV-215/delete-objects 2022-06-02 12:45:05 +00:00
Ronnie Smith 9b4b755cf8
Merge remote-tracking branch 'origin/w/8.4/feature/CLDSRV-174-use-armory-work' into w/8.5/feature/CLDSRV-174-use-armory-work 2022-06-01 22:18:29 -07:00
Ronnie Smith 0f5ab42233
Merge remote-tracking branch 'origin/feature/CLDSRV-174-use-armory-work' into w/8.4/feature/CLDSRV-174-use-armory-work 2022-06-01 19:31:28 -07:00
Nicolas Humbert d99397f604 CLDSRV-215 S3.DeleteObjects should clean up data 2022-06-01 19:10:36 -04:00
Nicolas Humbert cb591f06fc CLDSRV-213 Check object has a cold location before putObjectVersion 2022-05-26 14:25:26 -04:00
Jordi Bertran de Balanda f26286667f CLDSRV-174 - fix bucketclient error introspection
Due to circular dependency issues, we can't simply switch to a new
bucketclient. Instead, we rely on the old error format for now.
2022-05-25 18:43:11 +02:00
bert-e abcbb75ad0 Merge branch 'feature/CLDSRV-211' into tmp/octopus/w/8.5/feature/CLDSRV-211 2022-05-25 09:17:32 +00:00
Francois Ferrand 1807abe656
Remove default replication endpoint mandate
The default replication endpoint should not be needed, esp. since the
one we use (in zenko) does not relate to an existing location: it is
used only to let Cloudserver start.

Issue: CLDSRV-211
2022-05-25 10:52:05 +02:00
Jordi Bertran de Balanda 3e74ead98a CLDSRV-174 - update Arsenal 2022-05-24 12:06:24 +02:00
Ronnie Smith b8fae2d659 feature: CLDSRV-162 use data retrieval params over fn 2022-05-24 12:06:24 +02:00
Ronnie Smith ea328177de feature: CLDSRV-162 update bad version ids to be proper 2022-05-24 12:06:24 +02:00
Ronnie Smith fbd68e6839 feature: CLDSRV-162 use metadata and data from new arsenal 2022-05-24 12:06:17 +02:00
williamlardier 8c939b3a05
CLDSRV-188: bump package.json to 8.5.1 2022-05-24 09:08:52 +02:00
williamlardier 5795a44e4c
CLDSRV-184: accept list of valid storage classes
Co-authored-by: Naren <54150791+naren-rajendran@users.noreply.github.com>
2022-05-24 08:26:33 +02:00
williamlardier 8c7862b51d
CLDSRV-184: apply changes to CopyObject API 2022-05-24 08:26:15 +02:00
williamlardier bf9d9fe3c5
CLDSRV-184: apply changes to MPU API 2022-05-24 08:26:15 +02:00
williamlardier 127ce9c619
CLDSRV-184: Update PutObject and add unit tests 2022-05-24 08:26:14 +02:00
bert-e a10c705310 Merge branch 'feature/CLDSRV-183-implement-restore-object-api' into q/8.5 2022-05-23 17:14:07 +00:00
Xin LI cb518a8f08 improvement: CLDSRV-183 implement restoreObject api 2022-05-23 18:52:20 +02:00
bert-e dc408bcc3c Merge branch 'feature/CLDSRV-205/update-restore' into q/8.5 2022-05-23 14:54:36 +00:00
Nicolas Humbert 7afced91ba CLDSRV-205 Update restore properties when putting object version 2022-05-20 16:27:55 -04:00
williamlardier 91c1eca7d6
CLDSRV-209: fix dmf checker and default values 2022-05-20 13:43:25 +02:00
williamlardier 78498030ab
CLDSRV-209: switch to guard clause 2022-05-20 10:50:54 +02:00
williamlardier 60a917cc08
CLDSRV-209: allow to run cloudserver with empty details for dmf 2022-05-20 09:55:01 +02:00
bert-e 687b8565f9 Merge branch 'improvement/CLDSRV-208' into tmp/octopus/w/8.5/improvement/CLDSRV-208 2022-05-19 17:55:52 +00:00
Francois Ferrand da633f6b49
Use pre-built python 3.7 from distribution
Issue: CLDSRV-208
2022-05-19 19:25:30 +02:00
bert-e f6a66881fd Merge branch 'feature/CLDSRV-185-put-bucket-support-restore-object' into q/8.5 2022-05-18 13:53:03 +00:00
bert-e fda4c099b3 Merge branch 'feature/CLDSRV-204/simple-mpu-version' into q/8.5 2022-05-17 16:21:58 +00:00
williamlardier 3d0ee3fc4b
CLDSRV-185: add functional tests 2022-05-17 09:48:42 +02:00
williamlardier 6c2661eea5
CLDSRV-185: add unit tests 2022-05-17 08:46:19 +02:00
williamlardier 413b52f4d0
CLDSRV-185: add dmf location in json config file 2022-05-17 08:45:17 +02:00
williamlardier bf23f09c41
CLDSRV-185: clean dmf check function 2022-05-17 08:44:50 +02:00
Ronnie Smith 89d030bfe9
CLDSRV-185: add dmf location type 2022-05-17 08:43:52 +02:00
williamlardier 2d91180469
CLDSRV-185: reject bucket put if cold storage location 2022-05-17 08:37:24 +02:00
bert-e 64b83e9f2c Merge branch 'feature/CLDSRV-186-get-object-to-support-restore-object' into q/8.5 2022-05-17 06:34:06 +00:00
Nicolas Humbert 2ab4f3139e clean up putVersion tests 2022-05-16 22:31:51 -04:00
Nicolas Humbert 87b199dc6f CLDSRV-204 Complete MPU with x-scal-s3-version-id header 2022-05-16 20:54:04 -04:00
Nicolas Humbert a0c97d2c06 update the microVersionId 2022-05-16 15:16:11 -04:00
Nicolas Humbert 8fd7b3ed30 CLDSRV-202 Put object version with x-scal-s3-version-id 2022-05-16 11:01:52 -04:00
williamlardier eeb6ff8c3b
CLDSRV-186: comply with AWS errors 2022-05-16 11:44:21 +02:00
williamlardier 9922acf8f9
CLDSRV-186: fixups 2022-05-16 11:44:21 +02:00
williamlardier 9d7c7dedbb
CLDSRV-186: add unit tests for new Get Object 2022-05-16 11:44:21 +02:00
williamlardier 3ff27f488f
CLDSRV-186: return cold storage related errors in Get Object 2022-05-16 11:44:20 +02:00
williamlardier a5df203ca2
CLDSRV-187: update yarn lock 2022-05-16 11:43:02 +02:00
williamlardier 58e2625e3e
CLDSRV-187: move coldStorage file 2022-05-16 11:43:02 +02:00
williamlardier c0acd4dc1b
CLDSRV-187: fixups 2022-05-16 11:43:02 +02:00
williamlardier 78f131df56
CLDSRV-187: remove undefined header from response 2022-05-16 11:43:01 +02:00
williamlardier 721f54cb92
CLDSRV-187: add dmf in test config legacy 2022-05-16 11:43:01 +02:00
Yutaka Oishi a39c3f475c
CLDSRV-187: Use util function to compute returned header 2022-05-16 11:43:01 +02:00
williamlardier dd20b82745
CLDSRV-187: Cleanup 2022-05-16 11:43:00 +02:00
williamlardier 62163eb8aa
CLDSRV-187: Linting 2022-05-16 11:43:00 +02:00
williamlardier 7ae162ee0c
CLDSRV-187: Add tests for new HEAD object feature 2022-05-16 11:43:00 +02:00
williamlardier 54d313fffa
CLDSRV-187: Add util file for bucket and object MD mocking 2022-05-16 11:42:59 +02:00
williamlardier d22b6b6aa5
CLDSRV-187: HEAD to handle object in cold storage 2022-05-16 11:42:59 +02:00
williamlardier 5acdfccdea
CLDSRV-187: update arsenal 2022-05-16 11:42:59 +02:00
Nicolas Humbert 30f6c803af CLDSRV-192 Introduce s3:PutObjectVersion permission 2022-05-12 14:46:45 -07:00
Francois Ferrand a235a1a175
Merge remote-tracking branch 'origin/bugfix/CLDSRV-201' into w/8.5/bugfix/CLDSRV-201 2022-05-12 15:20:18 +02:00
Francois Ferrand d5a9960967
Bump version 8.4.10
Issue: CLDSRV-201
2022-05-12 15:13:33 +02:00
Francois Ferrand fffba0bdbb
Increase memory for cloudserver docker image build
Issue: CLDSRV-201
2022-05-12 11:58:33 +02:00
bert-e 2a165dc0da Merge branch 'w/8.4/bugfix/CLDSRV-196/fixLoggerConfiguration' into tmp/octopus/w/8.5/bugfix/CLDSRV-196/fixLoggerConfiguration 2022-05-11 21:45:38 +00:00
bert-e 1003b77abd Merge branch 'bugfix/CLDSRV-196/fixLoggerConfiguration' into tmp/octopus/w/8.4/bugfix/CLDSRV-196/fixLoggerConfiguration 2022-05-11 21:45:38 +00:00
Alexander Chan 7fd547db24 CLDSRV-196: create new werelogs object over using global werelogs 2022-05-11 14:36:27 -07:00
Rahul Padigela 830ddb6e3c Merge remote-tracking branch 'origin/improvement/CLDSRV-193-bump-cloudserver-8-4-9' into w/8.5/improvement/CLDSRV-193-bump-cloudserver-8-4-9 2022-05-11 11:55:04 -07:00
KillianG 8d00ef947d
CLDSRV-198: bump cloudserver 8.5.0 2022-05-11 16:32:34 +02:00
KillianG fe432a0edb
CLDSRV-193: bump cloudserver 8.4.9 2022-05-11 16:01:48 +02:00
Kerkesni 25a70dddff
feature: CLDSRV-178 match aws putBucketNotification error 2022-05-09 17:52:58 +02:00
Xin LI 03fa7d00a2 improvement: CLDSRV-191 migrate arsenal to cloudserver 8.x 2022-05-09 16:44:53 +02:00
KillianG ea107a50e1
CLDSRV-167: Implement bucket owner check and change check order
Checking order has been changed to fit with AWS one, meaning that we
raise the first error we face

- Checking every tag one by one
- Checking that the tag key is unique, the error is raised only if we
  already seen a tag with the same key before
- Checking that the key length is less than 128
- Checking that the tag value length is less than 256
- After seeing more than 50 tags, if there is one more, raise the too
  many tags error
2022-04-29 14:57:49 +02:00
KillianG 4b4f35d36f
CLDSRV-167: implement put bucket tagging 2022-04-29 14:54:53 +02:00
bert-e 90c98b5fae Merge branch 'feature/CLDSRV-175-support-object-acl-put-notification' into q/8.4 2022-04-22 16:56:48 +00:00
bert-e 8d52258ec5 Merge branch 'w/8.2/bugfix/CLDSRV-179-align-with-arsenal-mddb-api-change' into tmp/octopus/w/8.3/bugfix/CLDSRV-179-align-with-arsenal-mddb-api-change 2022-04-21 16:49:33 +00:00
bert-e 44549c79d2 Merge branch 'w/8.3/bugfix/CLDSRV-179-align-with-arsenal-mddb-api-change' into tmp/octopus/w/8.4/bugfix/CLDSRV-179-align-with-arsenal-mddb-api-change 2022-04-21 16:49:33 +00:00
Jordi Bertran de Balanda 9837c0a0a7 Merge remote-tracking branch 'origin/bugfix/CLDSRV-179-align-with-arsenal-mddb-api-change' into w/8.2/bugfix/CLDSRV-179-align-with-arsenal-mddb-api-change 2022-04-21 18:35:28 +02:00
Jordi Bertran de Balanda e109daa2bc CLDSRV-179 - update imports
All file-related metadata/bucket file interfaces in `storage` have moved
under the `file` key.
2022-04-21 18:07:35 +02:00
Jordi Bertran de Balanda be77fe9977 CLDSRV-179 - align with ARSN-128 breaking change
ARSN-128 changed the `open` operation from directly returning the DB
handle to passing the DB handle as callback. This was making s3server
crash.
2022-04-21 16:27:31 +02:00
Kerkesni 89f286eab6
feature: CLDSRV-175 add support for s3:ObjectACL:Put notification
Notification event should only be instanciated when a new ACL is created
or when an old one is changed, ACL requests that change nothing don't generate
a notification event
2022-04-21 16:26:29 +02:00
bert-e e83057c75c Merge branch 'feature/CLDSRV-176-support-object-tagging-notifications' into q/8.4 2022-04-21 08:16:39 +00:00
bert-e 48aaede6db Merge branch 'w/8.3/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck' into tmp/octopus/w/8.4/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck 2022-04-20 17:04:59 +00:00
bert-e 0c704913a3 Merge branch 'w/8.2/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck' into tmp/octopus/w/8.3/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck 2022-04-20 17:04:58 +00:00
Jonathan Gramain ad9e34c5fd Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck' into w/8.2/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck 2022-04-20 10:04:40 -07:00
Jonathan Gramain 413ebe743c [7.10] CLDSRV-177 add missing test helper checkObjectData 2022-04-20 10:00:50 -07:00
bert-e b486489888 Merge branch 'w/8.3/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck' into tmp/octopus/w/8.4/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck 2022-04-19 22:59:28 +00:00
bert-e 8b0093b56b Merge branch 'w/8.2/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck' into tmp/octopus/w/8.3/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck 2022-04-19 22:59:28 +00:00
Jonathan Gramain a667315a4c Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck' into w/8.2/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck 2022-04-19 15:38:47 -07:00
bert-e 7c3ad7b095 Merge branch 'bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck' into tmp/octopus/w/7.10/bugfix/CLDSRV-177-checkEmptyLocationsInLocationsSanityCheck 2022-04-19 22:32:59 +00:00
Jonathan Gramain a4e8cbebe6 bugfix: CLDSRV-177 fix crash with empty object replication
Fix a case of crash when a replication occurs with an empty object
over a non-empty object.

It is not clear how this happens in practice but there can be some
corner cases with race conditions between object replication and
versioning getting suspended on the target bucket at the same time,
that could lead to this situation, as the check between replication
configuration and actual replication is not atomic.
2022-04-19 15:26:35 -07:00
Kerkesni 53d9f8cf99
feature: CLDSRV-176 support s3:ObjectTagging:* notifications 2022-04-19 16:05:48 +02:00
bert-e fe923f679d Merge branches 'w/8.4/bugfix/CLDSRV-173/delete-marker-lifecycle' and 'q/4444/8.3/bugfix/CLDSRV-173/delete-marker-lifecycle' into tmp/octopus/q/8.4 2022-04-14 21:18:11 +00:00
bert-e 84cbef6210 Merge branches 'w/8.3/bugfix/CLDSRV-173/delete-marker-lifecycle' and 'q/4444/8.2/bugfix/CLDSRV-173/delete-marker-lifecycle' into tmp/octopus/q/8.3 2022-04-14 21:18:11 +00:00
bert-e 59e49ee50d Merge branches 'w/8.2/bugfix/CLDSRV-173/delete-marker-lifecycle' and 'q/4444/7.10/bugfix/CLDSRV-173/delete-marker-lifecycle' into tmp/octopus/q/8.2 2022-04-14 21:18:10 +00:00
bert-e d0adc439ab Merge branch 'bugfix/CLDSRV-173/delete-marker-lifecycle' into q/7.10 2022-04-14 21:18:10 +00:00
bert-e 3b63f5ec27 Merge branch 'w/8.3/bugfix/CLDSRV-173/delete-marker-lifecycle' into tmp/octopus/w/8.4/bugfix/CLDSRV-173/delete-marker-lifecycle 2022-04-14 20:55:54 +00:00
bert-e 8bae222521 Merge branch 'w/8.2/bugfix/CLDSRV-173/delete-marker-lifecycle' into tmp/octopus/w/8.3/bugfix/CLDSRV-173/delete-marker-lifecycle 2022-04-14 20:55:54 +00:00
Nicolas Humbert 95d218c824 Merge remote-tracking branch 'origin/bugfix/CLDSRV-173/delete-marker-lifecycle' into w/8.2/bugfix/CLDSRV-173/delete-marker-lifecycle 2022-04-14 16:35:13 -04:00
Nicolas Humbert da238658da CLDSRV-173 DeleteMarkers created by Lifecycle should not be replicated 2022-04-14 14:38:41 -04:00
Francois Ferrand 2a80c583a6
Rename dashboard
Issue: CLDSRV-172
2022-04-13 12:20:09 +02:00
bert-e 74d1a26dc9 Merge branches 'w/8.3/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' and 'q/4417/8.2/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' into tmp/octopus/q/8.3 2022-04-12 02:10:52 +00:00
bert-e 3fd0b95665 Merge branches 'w/8.4/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' and 'q/4417/8.3/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' into tmp/octopus/q/8.4 2022-04-12 02:10:52 +00:00
bert-e c79597e387 Merge branches 'w/8.2/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' and 'q/4417/7.10/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' into tmp/octopus/q/8.2 2022-04-12 02:10:51 +00:00
bert-e 34abca3d9a Merge branch 'bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' into q/7.4 2022-04-12 02:10:50 +00:00
bert-e 919c6effc8 Merge branches 'w/7.10/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' and 'q/4417/7.4/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' into tmp/octopus/q/7.10 2022-04-12 02:10:50 +00:00
bert-e 4c3378d1b9 Merge branch 'w/8.3/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' into tmp/octopus/w/8.4/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart 2022-04-12 01:39:21 +00:00
bert-e 44744ffbe3 Merge branch 'w/8.2/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' into tmp/octopus/w/8.3/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart 2022-04-12 01:39:20 +00:00
Jonathan Gramain 0e4d685957 Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' into w/8.2/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart 2022-04-11 18:23:39 -07:00
bert-e e963a72f51 Merge branch 'bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart' into tmp/octopus/w/7.10/bugfix/CLDSRV-170-raceCompleteMPUAndUploadPart 2022-04-12 01:22:51 +00:00
Jonathan Gramain 8496111518 bugfix: CLDSRV-170 skip orphan cleanup in UploadPart[Copy]
Do not delete orphan data in UploadPart/UploadPartCopy on overwrite
iff a CompleteMPU of the target MPU is already in progress.

This is to prevent a race condition where a CompleteMPU is running
while UploadPart is uploading a part for the same MPU.

It leaves an orphan in storage since only one of the upload data will
be present in the finished MPU, but the window is limited to the
CompleteMPU execution and should only occur when there are retries of
UploadPart due to prior stuck requests, or with broken clients
misusing the MPU API, so it should be acceptable.

Implementation details:

- set a flag in the MPU overview key when starting the CompleteMPU
  process, before listing the parts from metadata to construct the
  manifest

- in UploadPart/UploadPartCopy, after the part metadata is written and
  if the same part already existed, re-fetch the MPU overview key to
  check the flag: if set, skip the deletion of the old data of this
  part, since the CompleteMPU process in progress may choose either
  part data depending on the exact timing of the listing vs. the
  part overwrite.
2022-04-11 18:21:59 -07:00
bert-e 683c7c96af Merge branch 'w/8.3/bugfix/S3C-5390-s3api_head-object_with_part-number_1_on_empty_file_fails' into tmp/octopus/w/8.4/bugfix/S3C-5390-s3api_head-object_with_part-number_1_on_empty_file_fails 2022-04-11 23:00:07 +00:00
bert-e b21ba96907 Merge branch 'w/8.2/bugfix/S3C-5390-s3api_head-object_with_part-number_1_on_empty_file_fails' into tmp/octopus/w/8.3/bugfix/S3C-5390-s3api_head-object_with_part-number_1_on_empty_file_fails 2022-04-11 23:00:07 +00:00
bert-e 2cdd882b42 Merge branch 'w/7.10/bugfix/S3C-5390-s3api_head-object_with_part-number_1_on_empty_file_fails' into tmp/octopus/w/8.2/bugfix/S3C-5390-s3api_head-object_with_part-number_1_on_empty_file_fails 2022-04-11 23:00:07 +00:00
bert-e b2b95fb986 Merge branch 'bugfix/S3C-5390-s3api_head-object_with_part-number_1_on_empty_file_fails' into tmp/octopus/w/7.10/bugfix/S3C-5390-s3api_head-object_with_part-number_1_on_empty_file_fails 2022-04-11 23:00:06 +00:00
Artem Bakalov b4725aa032 remove .only 2022-04-08 13:46:25 -07:00
Artem Bakalov 4f3195a6ca S3C-5390 s3api head-object with part-number 1 on empty file fails: httpCode 416 2022-04-06 16:42:56 -07:00
Francois Ferrand 46eda1cc14
Use 'mean' computatino for dl/ul chunk size panels
Issue: CLDSRV-166
2022-04-06 18:57:16 +02:00
Francois Ferrand 3366f2339c
Display internal cloudserver metrics in dashboard
Use a templating variable to let the user select which information to
show.

Issue: CLDSRV-166
2022-04-06 18:57:16 +02:00
Francois Ferrand 10e3d4da63
Bump version to 8.4.8 2022-03-31 21:46:07 +02:00
Francois Ferrand 94ba153d7f
Fix computation for http requests panels
Need to use `sum` aggregation to get a meaningful Stat value.

Issue: CLDSRV-164
2022-03-30 10:16:54 +02:00
Francois Ferrand 12408bffa8
Fix success rate gauge
- Filtering out period with no http requests (NaN)
- Display greyed-out "-" when no data
- Compute mean value over current range

Issue: CLDSRV-164
2022-03-30 10:15:01 +02:00
bert-e 229c3eb251 Merge branch 'w/8.3/bugfix/CLDSRV-163-skip-failing-test' into tmp/octopus/w/8.4/bugfix/CLDSRV-163-skip-failing-test 2022-03-29 19:39:43 +00:00
bert-e 6e7fdfd8a9 Merge branch 'w/8.2/bugfix/CLDSRV-163-skip-failing-test' into tmp/octopus/w/8.3/bugfix/CLDSRV-163-skip-failing-test 2022-03-29 19:39:42 +00:00
bert-e 625bc7f5e5 Merge branch 'w/7.10/bugfix/CLDSRV-163-skip-failing-test' into tmp/octopus/w/8.2/bugfix/CLDSRV-163-skip-failing-test 2022-03-29 19:39:42 +00:00
bert-e 62f432a4c9 Merge branch 'bugfix/CLDSRV-163-skip-failing-test' into tmp/octopus/w/7.10/bugfix/CLDSRV-163-skip-failing-test 2022-03-29 19:39:41 +00:00
Rahul Padigela 1a21fbd023 bugfix: S3C-5657 skip test in Integration 2022-03-29 11:58:23 -07:00
Kerkesni 4b073de34f
improvement: CLDSRV-152 bump version to 8.4.7 2022-03-25 19:22:17 +01:00
bert-e ad257db42d Merge branches 'w/8.4/bugfix/CLDSRV-159-include-listing-phd-bugfix' and 'q/4376/8.3/bugfix/CLDSRV-159-include-listing-phd-bugfix' into tmp/octopus/q/8.4 2022-03-25 15:43:06 +00:00
bert-e 7cc1926fd2 Merge branch 'bugfix/CLDSRV-159-include-listing-phd-bugfix' into q/8.3 2022-03-25 15:43:05 +00:00
Kerkesni 05f99c74e3
Merge remote-tracking branch 'origin/bugfix/CLDSRV-159-include-listing-phd-bugfix' into w/8.4/bugfix/CLDSRV-159-include-listing-phd-bugfix 2022-03-25 15:35:53 +01:00
Kerkesni 84aefc46dd
bugfix: CLDSRV-159 bump arsenal to 8.1.38 2022-03-25 15:33:52 +01:00
bert-e 3bc1150a2a Merge branch 'q/4373/8.3/bugfix/CLDSRV-157' into tmp/normal/q/8.4 2022-03-25 10:00:30 +00:00
bert-e 4617917714 Merge branch 'bugfix/CLDSRV-157' into q/8.3 2022-03-25 10:00:29 +00:00
bert-e a371727022 Merge branch 'w/8.4/bugfix/CLDSRV-157' into tmp/normal/q/8.4 2022-03-25 10:00:29 +00:00
Xin LI 4c400bdc3f improvement: CLDSRV-158 bump version to 8.4.6 2022-03-25 09:38:48 +01:00
Francois Ferrand c6af1e5cea
Set noValue on download/upload chunk size
Issue: CLDSRV-157
2022-03-24 16:50:14 +01:00
Francois Ferrand 806c0ad2dd
Fix upload/download chunk size
Issue: CLDSRV-157
2022-03-24 16:47:02 +01:00
Francois Ferrand cde78ba363
Merge remote-tracking branch 'origin/bugfix/CLDSRV-157' into w/8.4/bugfix/CLDSRV-157 2022-03-24 16:45:38 +01:00
Francois Ferrand ae841997f6
Fix upload/download chunk size
Issue: CLDSRV-157
2022-03-24 16:36:23 +01:00
bert-e 192fd70d53 Merge branch 'improvement/CLDSRV-155-search_bucket-support-list-versions' into q/8.4 2022-03-24 09:19:24 +00:00
bert-e 1499f5e67d Merge branch 'w/8.3/improvement/CLDSRV-156-upgrade-vaultclient' into tmp/octopus/w/8.4/improvement/CLDSRV-156-upgrade-vaultclient 2022-03-24 07:11:57 +00:00
Naren b7059b8329 Merge remote-tracking branch 'origin/w/8.2/improvement/CLDSRV-156-upgrade-vaultclient' into w/8.3/improvement/CLDSRV-156-upgrade-vaultclient 2022-03-23 23:40:54 -07:00
Naren b925fa7c8c Merge remote-tracking branch 'origin/w/7.10/improvement/CLDSRV-156-upgrade-vaultclient' into w/8.2/improvement/CLDSRV-156-upgrade-vaultclient 2022-03-23 23:01:14 -07:00
Naren 5a9b9a569b Merge remote-tracking branch 'origin/improvement/CLDSRV-156-upgrade-vaultclient' into w/7.10/improvement/CLDSRV-156-upgrade-vaultclient 2022-03-23 22:29:25 -07:00
Naren dcd935027b improvement: CLDSRV-156 upgrade vaultclient 2022-03-23 12:46:41 -07:00
Francois Ferrand 886dc44ab1
Fix wrong label on reporter's Up panel
Issue: CLDSRV-149
2022-03-23 15:07:43 +01:00
Francois Ferrand 9d4723834d
Use last value for Up panels
Up and Reporter panels must show the instantaneous value, even if null:
i.e. if there are no pods.

Issue: CLDSRV-149
2022-03-23 15:06:32 +01:00
Francois Ferrand e0c1f529a6
Fix last report counter
- Formula must account for the date of last succesful `count-items`
cronjob execution.
- Unit is actually second.
- Display last value, even if null.

Issue: CLDSRV-149
2022-03-23 15:06:08 +01:00
Francois Ferrand 8b2523e78c
Merge remote-tracking branch 'origin/bugfix/CLDSRV-149-use-last-count-time' into w/8.4/bugfix/CLDSRV-149-use-last-count-time 2022-03-23 15:04:26 +01:00
Francois Ferrand f528fc9bf9
Use last value for Up panels
Up and Reporter panels must show the instantaneous value, even if null:
i.e. if there are no pods.

Issue: CLDSRV-149
2022-03-23 14:20:34 +01:00
Francois Ferrand 68b27be192
Fix last report counter
- Formula must account for the date of last succesful `count-items`
cronjob execution.
- Unit is actually second.
- Display last value, even if null.

Issue: CLDSRV-149
2022-03-23 14:20:34 +01:00
bert-e 0abac6cb61 Merge branch 'bugfix/CLDSRV-149' into q/8.3 2022-03-23 10:36:09 +00:00
bert-e a3bf81e8b7 Merge branches 'w/8.4/bugfix/CLDSRV-149' and 'q/4350/8.3/bugfix/CLDSRV-149' into tmp/octopus/q/8.4 2022-03-23 10:36:09 +00:00
Xin LI 40d67e69a5 improvement: CLDSRV-155-search_bucket support list versions 2022-03-23 09:18:23 +01:00
Francois Ferrand cbf1ee3733
Merge remote-tracking branch 'origin/bugfix/CLDSRV-149' into w/8.4/bugfix/CLDSRV-149 2022-03-21 18:09:00 +01:00
Francois Ferrand fe8fe2c13b
Merge remote-tracking branch 'origin/development/8.4' into w/8.4/bugfix/CLDSRV-149 2022-03-21 18:07:02 +01:00
Xin LI 24a49c2fb8 improvement: CLDSRV-154-bump-version 2022-03-21 16:25:52 +01:00
Xin LI 21871de66e improvement: CLDSRV-153-bump-arsenal 2022-03-21 15:17:35 +01:00
Francois Ferrand 265a212f0e
Show status of report-handler
Issue: CLDSRV-149
2022-03-21 09:22:03 +01:00
Francois Ferrand 3c37a63b75
Fix wrong computation for disk available
Issue: CLDSRV-149
2022-03-21 09:16:18 +01:00
Francois Ferrand f8fc95b778
Remove local data storage graph
Display bigger panel form buckets and objects count, and add ingestion
rate panels (both obj/s and byte/s).

Issue: CLDSRV-149
2022-03-21 09:13:31 +01:00
Francois Ferrand 170d0b2034
Fix computation for injection rate panel
Use "mean" value, which is more relevant for a "rate", and limit to 1 decimal.

Issue: CLDSRV-149
2022-03-21 09:13:31 +01:00
Francois Ferrand 6a32c8a0cb
Use number of objects/buckets from last report
The value from report is now always exposed. The logic of adding report
and dynamic operations would work in a single-instance scenario; but
things cannot "just" be added when it gets distributed over multiple
pods: there must be a single report handler, all other instances will
report only their own operations, and there is no safe way to combine
all the metrics together.

For now, just use the value from report handler, so the value is roughly
correct, although with some delay. Also expose the timestamp of the
report, so this delay may be displayed (or eventually used to improve
computations).

Issue: CLDSRV-149
2022-03-21 09:13:31 +01:00
Francois Ferrand 0700006207
Use piechart for top10 panels
Issue: CLDSRV-149
2022-03-21 09:13:31 +01:00
Francois Ferrand 403e1484cd
Fix labels & description for overview panels
Issue: CLDSRV-149
2022-03-21 09:13:31 +01:00
Francois Ferrand 5ceba2dd87
Merge remote-tracking branch 'origin/bugfix/CLDSRV-149' into w/8.4/bugfix/CLDSRV-149 2022-03-18 13:36:56 +01:00
Francois Ferrand 79fa3254ae
Show status of report-handler
Issue: CLDSRV-149
2022-03-17 15:54:36 +01:00
Francois Ferrand b69ccf28f7
Remove local data storage graph
Display bigger panel form buckets and objects count, and add ingestion
rate panels (both obj/s and byte/s).

Issue: CLDSRV-149
2022-03-17 15:43:31 +01:00
Francois Ferrand e854f6ccac
Fix computation for injection rate panel
Use "mean" value, which is more relevant for a "rate", and limit to 1 decimal.

Issue: CLDSRV-149
2022-03-17 14:54:45 +01:00
Francois Ferrand 3274009341
Use number of objects/buckets from last report
The value from report is now always exposed. The logic of adding report
and dynamic operations would work in a single-instance scenario; but
things cannot "just" be added when it gets distributed over multiple
pods: there must be a single report handler, all other instances will
report only their own operations, and there is no safe way to combine
all the metrics together.

For now, just use the value from report handler, so the value is roughly
correct, although with some delay. Also expose the timestamp of the
report, so this delay may be displayed (or eventually used to improve
computations).

Issue: CLDSRV-149
2022-03-17 13:30:11 +01:00
Francois Ferrand bde2df02f6
Use piechart for top10 panels
Issue: CLDSRV-149
2022-03-17 13:29:53 +01:00
Francois Ferrand fc44bc9e97
Fix labels & description for overview panels
Issue: CLDSRV-149
2022-03-17 13:29:53 +01:00
Alexandre Lavigne 6012e5c9b2
CLDSRV-148 - set only required inputs 2022-03-17 13:16:58 +01:00
Alexandre Lavigne 0dafa7c665
CLDSRV-148 - test alerts
Run action `action-prom-render-test` to test alerts
2022-03-17 13:15:42 +01:00
bert-e 2de0086801 Merge branch 'feature/CLDSRV-134-integrate-new-bucket-format' into tmp/octopus/w/8.4/feature/CLDSRV-134-integrate-new-bucket-format 2022-03-16 11:42:17 +00:00
Kerkesni 0d8fe7ba5e
feature: CLDSRV-134 add tests for mixed bucket formats
Added functional tests for testing if s3 operations respect
the bucket vFormat when having buckets in different formats
2022-03-16 12:08:05 +01:00
Kerkesni f3db9b8ed3
feature: CLDSRV-134 Added CI steps to run tests in both formats
Added CI steps to run functional tests in both bucket metadata formats
2022-03-16 12:08:05 +01:00
Kerkesni 6ad4f03da0
feature: CLDSRV-134 Bump Arsenal to 8.1.34 2022-03-16 12:08:01 +01:00
KillianG b18b5ba14c
CLDSRV-146: bump cloudserver version 2022-03-11 11:52:30 +01:00
Francois Ferrand ffb8ed7a5b Remove type annotations
The version of flake8 in Eve's worker does not seem to support
type annotations, and fails with: `E999 SyntaxError: invalid syntax`.

Issue: CLDSRV-147
2022-03-10 11:43:22 +01:00
Francois Ferrand c90d36de02 Make flake8 compliant
Issue: CLDSRV-147
2022-03-10 11:43:20 +01:00
Francois Ferrand a76709474d Migrate dashboard to grafanalib
Issue: CLDSRV-147
2022-03-10 11:42:33 +01:00
bert-e 12bcb1088c Merge branch 'improvement/CLDSRV-143-bump-package-to-8.3.12' into q/8.3 2022-03-01 08:03:56 +00:00
bert-e 0c7f4fd832 Merge branches 'w/8.4/improvement/CLDSRV-143-bump-package-to-8.3.12' and 'q/4311/8.3/improvement/CLDSRV-143-bump-package-to-8.3.12' into tmp/octopus/q/8.4 2022-03-01 08:03:56 +00:00
bert-e f945a77e50 Merge branches 'w/8.3/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' and 'q/4314/8.2/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/q/8.3 2022-02-28 18:20:36 +00:00
bert-e 952b98e9c2 Merge branches 'w/8.4/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' and 'q/4314/8.3/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/q/8.4 2022-02-28 18:20:36 +00:00
bert-e 17425df1aa Merge branches 'w/8.2/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' and 'q/4314/7.10/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/q/8.2 2022-02-28 18:20:35 +00:00
bert-e 1a5696dde4 Merge branches 'w/7.10/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' and 'q/4314/7.4/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/q/7.10 2022-02-28 18:20:34 +00:00
bert-e 8a003f985c Merge branch 'bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into q/7.4 2022-02-28 18:20:34 +00:00
bert-e 326b825ba9 Merge branch 'w/8.2/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/8.3/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-28 17:54:06 +00:00
bert-e 9e8459bd90 Merge branch 'w/8.3/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/8.4/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-28 17:54:06 +00:00
Ronnie Smith 921e68f03a
feature: CLDSRV-144 use array iterable over for loop 2022-02-28 09:53:33 -08:00
bert-e 74e2bb251f Merge branch 'w/8.2/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/8.3/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-26 03:05:38 +00:00
bert-e d1c5b75aa1 Merge branch 'w/8.3/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/8.4/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-26 03:05:38 +00:00
bert-e fc46fd1675 Merge branch 'bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/7.10/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-26 03:05:37 +00:00
bert-e c205437587 Merge branch 'w/7.10/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/8.2/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-26 03:05:37 +00:00
Ronnie 2d843ae408
update other cannot
Co-authored-by: Jonathan Gramain <jonathan.gramain@scality.com>
2022-02-25 19:05:29 -08:00
bert-e 32b1369772 Merge branch 'w/8.2/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/8.3/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-26 01:42:52 +00:00
bert-e 7f64198565 Merge branch 'w/8.3/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/8.4/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-26 01:42:52 +00:00
bert-e 27052c5c2b Merge branch 'bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/7.10/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-26 01:42:51 +00:00
bert-e de6d6cf8d2 Merge branch 'w/7.10/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/8.2/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-26 01:42:51 +00:00
Ronnie e01d62a5ad
bugfix: CLDSRV-144: Also check 127 and grammar suggestion
Co-authored-by: Jonathan Gramain <jonathan.gramain@scality.com>
2022-02-25 17:42:40 -08:00
bert-e e54edce1fe Merge branch 'w/8.3/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/8.4/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-26 00:31:26 +00:00
bert-e 59027d1790 Merge branch 'w/8.2/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/8.3/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-26 00:31:25 +00:00
bert-e d540133ab7 Merge branch 'w/7.10/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into tmp/octopus/w/8.2/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-26 00:31:25 +00:00
Ronnie Smith 9149cb4abb
Merge remote-tracking branch 'origin/bugfix/CLDSRV-144/do-not-allow-non-printable-chars' into w/7.10/bugfix/CLDSRV-144/do-not-allow-non-printable-chars 2022-02-25 16:29:25 -08:00
Ronnie Smith d9d3ec682d
bugfix: CLDSRV-144 Do not allow non-printable characters in object names 2022-02-25 15:39:14 -08:00
williamlardier 0e13475285
Merge remote-tracking branch 'origin/improvement/CLDSRV-143-bump-package-to-8.3.12' into w/8.4/improvement/CLDSRV-143-bump-package-to-8.3.12 2022-02-25 14:02:56 +01:00
williamlardier 8760f955c4
CLDSRV-143: bump package.json to v8.3.12 2022-02-25 14:01:28 +01:00
williamlardier a7b436f05c
Merge remote-tracking branch 'origin/bugfix/CLDSRV-139-allow-extending-re-period-in-governance-without-bypass-header' into w/8.4/bugfix/CLDSRV-139-allow-extending-re-period-in-governance-without-bypass-header 2022-02-25 13:36:53 +01:00
williamlardier d6dac34024
CLDSRV-193: bump version in package.json 2022-02-25 12:35:53 +01:00
williamlardier 1b02a4a8d2
CLDSRV-193: allow extension of ret period without header 2022-02-25 12:35:53 +01:00
bert-e 0778f1b342 Merge branch 'w/8.3/feature/CLDSRV-102/aborted_mpu_ghost_put' into tmp/octopus/w/8.4/feature/CLDSRV-102/aborted_mpu_ghost_put 2022-02-24 19:47:57 +00:00
bert-e e3faf94954 Merge branch 'w/8.2/feature/CLDSRV-102/aborted_mpu_ghost_put' into tmp/octopus/w/8.3/feature/CLDSRV-102/aborted_mpu_ghost_put 2022-02-24 19:47:57 +00:00
Taylor McKinnon 80b686eff3 Merge remote-tracking branch 'origin/w/7.10/feature/CLDSRV-102/aborted_mpu_ghost_put' into w/8.2/feature/CLDSRV-102/aborted_mpu_ghost_put 2022-02-24 11:45:00 -08:00
Taylor McKinnon 44f6a037ea Merge remote-tracking branch 'origin/feature/CLDSRV-102/aborted_mpu_ghost_put' into w/7.10/feature/CLDSRV-102/aborted_mpu_ghost_put 2022-02-24 10:10:11 -08:00
Taylor McKinnon 9eba583eb0 ft(CLDSRV-102): Add Aborted MPU PUT 2022-02-24 10:06:25 -08:00
bert-e 793c46936b Merge branch 'feature/CLDSRV-128' into tmp/octopus/w/8.4/feature/CLDSRV-128 2022-02-21 13:41:34 +00:00
Francois Ferrand 6e66deffa0 Introduce Cloudserver alerts tests
Issue: CLDSRV-128
2022-02-17 12:25:31 +01:00
Xin LI 274838a953 improvement: CLDSRV-125-typo-fix 2022-02-09 16:45:10 +01:00
Xin LI 033e3cb725 improvement: CLDSRV-125-modify-search_bucket.js-to-support-sessionUser 2022-02-09 15:34:58 +01:00
bert-e bf1357b3cc Merge branch 'improvement/CLDSRV-123-bump-version-8.4.1' into q/8.4 2022-02-09 10:57:54 +00:00
Xin LI ea864cd432 improvement: CLDSRV-123-bump-version-8.4.2 2022-02-09 11:13:51 +01:00
bert-e 2bba7f22bb Merge branches 'w/8.4/improvement/CLDSRV-49/fix-test' and 'q/4257/8.3/improvement/CLDSRV-49/fix-test' into tmp/octopus/q/8.4 2022-02-09 10:00:42 +00:00
bert-e b0e0d3b43e Merge branch 'improvement/CLDSRV-49/fix-test' into q/8.2 2022-02-09 10:00:41 +00:00
bert-e 8948de1945 Merge branches 'w/8.3/improvement/CLDSRV-49/fix-test' and 'q/4257/8.2/improvement/CLDSRV-49/fix-test' into tmp/octopus/q/8.3 2022-02-09 10:00:41 +00:00
Nicolas Humbert 117ef04e27 metadataSearch implemented 2022-02-09 10:36:43 +01:00
bert-e fb04099921 Merge branch 'w/8.3/improvement/CLDSRV-49/fix-test' into tmp/octopus/w/8.4/improvement/CLDSRV-49/fix-test 2022-02-09 09:34:03 +00:00
Nicolas Humbert 7d8ddc08f1 Merge remote-tracking branch 'origin/improvement/CLDSRV-49/fix-test' into w/8.3/improvement/CLDSRV-49/fix-test 2022-02-09 10:04:09 +01:00
Nicolas Humbert 7bb77208a8 CLDSRV-49 handle metadataSearch before implementation 2022-02-09 09:59:32 +01:00
Xin LI 547562956b improvement: CLDSRV-123-bump-version-8.4.1 2022-02-08 19:13:33 +01:00
Nicolas Humbert a622dc9a07 Merge remote-tracking branch 'origin/w/8.3/feature/CLDSRV-49/UpgradeToNode16-clean' into w/8.4/feature/CLDSRV-49/UpgradeToNode16-clean 2022-02-08 18:32:16 +01:00
Xin LI 3795552534 improvement: CLDSRV-122-bump-version-8.4.0 2022-02-08 17:40:33 +01:00
Nicolas Humbert 964fb0175a fix: event close is always emitted 2022-02-08 17:25:51 +01:00
Nicolas Humbert 0ecfa9a2e2 Merge remote-tracking branch 'origin/w/8.2/feature/CLDSRV-49/UpgradeToNode16-clean' into w/8.3/feature/CLDSRV-49/UpgradeToNode16-clean 2022-02-08 17:20:52 +01:00
bert-e 2f503ac62e Merge branch 'feature/CLDSRV-112-add-mdsearch-api' into q/8.4 2022-02-08 16:10:11 +00:00
Nicolas Humbert 686994075c fix: NodeJS 16 upgrade 2022-02-08 16:10:58 +01:00
Xin LI 7c511c7f76 feature: CLDSRV-112-add-new-mdsearch-api(copy from bucketGet) 2022-02-08 14:38:40 +01:00
Nicolas Humbert 0a4542b990 Merge remote-tracking branch 'origin/w/7.10/feature/CLDSRV-49/UpgradeToNode16-clean' into w/8.2/feature/CLDSRV-49/UpgradeToNode16-clean 2022-02-08 14:03:39 +01:00
Nicolas Humbert a8772d312d fix: upgrade to Node 16 2022-02-08 13:19:26 +01:00
Nicolas Humbert 682f35dc7e Merge remote-tracking branch 'origin/feature/CLDSRV-49/UpgradeToNode16-clean' into w/7.10/feature/CLDSRV-49/UpgradeToNode16-clean 2022-02-08 13:14:30 +01:00
Nicolas Humbert 845807148f CLDSRV-49 upgrade to Node 16 2022-02-08 12:35:06 +01:00
bert-e c4093d5c9a Merge branch 'bugfix/CLDSRV-120' into q/8.3 2022-02-04 09:05:02 +00:00
bert-e 457d447dd6 Merge branch 'w/8.2/bugfix/CLDSRV-111-log-client-ip-correctly-stabilization' into tmp/octopus/w/8.3/bugfix/CLDSRV-111-log-client-ip-correctly-stabilization 2022-02-03 19:40:53 +00:00
Naren 95fe7d9a8e Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-111-log-client-ip-correctly-stabilization' into w/8.2/bugfix/CLDSRV-111-log-client-ip-correctly-stabilization 2022-02-03 11:21:34 -08:00
Naren 32d06cb1b4 Merge remote-tracking branch 'origin/bugfix/CLDSRV-111-log-client-ip-correctly-stabilization' into w/7.10/bugfix/CLDSRV-111-log-client-ip-correctly-stabilization 2022-02-03 10:24:59 -08:00
Francois Ferrand d2aa29d03a Display separate in/out scales for bandwidth
Display 'in' bandwidth on the left, and 'out' bandwidth on the right.

Issue: CLDSRV-120
2022-02-03 15:46:12 +01:00
Francois Ferrand 23fa11de6c Fix datasource in success rate panel
Issue: CLDSRV-120
2022-02-03 15:17:05 +01:00
Naren ddb4cf4855 bf: CLDSRV-111 log correct client ip
(cherry picked from commit 8b8f84d648)
2022-02-03 00:12:42 -08:00
Naren 8fbba4e3c4 Merge remote-tracking branch 'origin/w/8.2/bugfix/CLDSRV-111-log-client-ip-correctly' into w/8.3/bugfix/CLDSRV-111-log-client-ip-correctly 2022-02-02 23:11:35 -08:00
Naren 3fc455ec18 bf: CLDSRV-111 skip buggy gcp test 2022-02-02 22:35:32 -08:00
Naren a6224e87ce bf: CLDSRV-111 use arsenal v8.1.25 2022-02-02 14:40:37 -08:00
Naren 114b4c9560 bf: CLDSRV-111 override dependency resolutions
override dependencies resolutions for ioctl, npmlog and npmcli/fs, this
is done to avoid ioctl, fnctl compilation issues with node v10.
2022-02-02 12:52:52 -08:00
Naren b05d723640 Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-111-log-client-ip-correctly' into w/8.2/bugfix/CLDSRV-111-log-client-ip-correctly 2022-02-02 12:51:47 -08:00
Naren a8e4e34231 Merge remote-tracking branch 'origin/bugfix/CLDSRV-111-log-client-ip-correctly' into w/7.10/bugfix/CLDSRV-111-log-client-ip-correctly 2022-01-31 19:01:33 -08:00
Naren 8b8f84d648 bf: CLDSRV-111 log correct client ip 2022-01-31 10:45:32 -08:00
bert-e 72617f4ab1 Merge branch 'feature/CLDSRV-110' into tmp/octopus/w/8.3/feature/CLDSRV-110 2022-01-27 15:32:52 +00:00
Francois Ferrand f567903ce3 Relax client health check
Fail the check (both on startup and /ready route) only if all backends
are failing: so that the working backends can still be used.

Issue: CLDSRV-110
2022-01-27 15:06:36 +01:00
Alexander Chan 122b4ed89f fix merge conflict 2022-01-24 21:07:40 -08:00
Alexander Chan dd55f09be8 Merge remote-tracking branch 'origin/w/8.2/bugfix/CLDSRV-105/request-times-out-with-big-header' into w/8.3/bugfix/CLDSRV-105/request-times-out-with-big-header 2022-01-24 18:25:28 -08:00
Alexander Chan 8428404fb3 Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-105/request-times-out-with-big-header' into w/8.2/bugfix/CLDSRV-105/request-times-out-with-big-header 2022-01-24 16:59:14 -08:00
Alexander Chan 71b32660d5 Merge remote-tracking branch 'origin/w/7.4/bugfix/CLDSRV-105/request-times-out-with-big-header' into w/7.10/bugfix/CLDSRV-105/request-times-out-with-big-header 2022-01-24 15:18:41 -08:00
bert-e 0915b88f2b Merge branch 'bugfix/CLDSRV-105/request-times-out-with-big-header' into tmp/octopus/w/7.4/bugfix/CLDSRV-105/request-times-out-with-big-header 2022-01-24 21:44:02 +00:00
Naren 107fc53b68 improvement: CLDSRV-109 bump version to 8.3.10 2022-01-21 16:29:36 -08:00
Naren ffd1fcd2c1 Merge remote-tracking branch 'origin/improvement/CLDSRV-108-bump-to-8-2-19' into w/8.3/improvement/CLDSRV-108-bump-to-8-2-19 2022-01-21 15:30:01 -08:00
Naren 6ebe0d3211 improvement: CLDSRV-108 bump version to 8.2.19 2022-01-21 15:07:21 -08:00
bert-e c3c408688e Merge branch 'w/8.2/bugfix/CLDSRV-106-put-object-retention-with-sub-seconds-fail' into tmp/octopus/w/8.3/bugfix/CLDSRV-106-put-object-retention-with-sub-seconds-fail 2022-01-21 17:11:56 +00:00
Naren b1f013ad5e Merge remote-tracking branch 'origin/bugfix/CLDSRV-106-put-object-retention-with-sub-seconds-fail' into w/8.2/bugfix/CLDSRV-106-put-object-retention-with-sub-seconds-fail 2022-01-21 08:49:16 -08:00
Naren 2ff0af0135 bf: CLDSRV-106 retention timestamp with milliseconds
put object retention timestamp with milliseconds is accepted now.
2022-01-20 19:19:10 -08:00
Alexander Chan 532e5a8327 yarn ignore engines 2022-01-19 17:47:14 -08:00
Alexander Chan b66495fc85 upgrade arsenal to 7.4.14 2022-01-19 17:47:14 -08:00
Ilke a52635dfd3 bugfix: CLDSRV-47 check http header size 2022-01-19 17:47:14 -08:00
bert-e 3dab88a658 Merge branch 'w/8.2/bugfix/CLDSRV-96-forward-408-errors-to-client-on-put' into tmp/octopus/w/8.3/bugfix/CLDSRV-96-forward-408-errors-to-client-on-put 2022-01-17 17:31:58 +00:00
Artem Bakalov baf4b87b84 Updated Yarn lockfile 2022-01-17 09:30:35 -08:00
bert-e dc6bd4bfab Merge branch 'w/8.2/bugfix/CLDSRV-96-forward-408-errors-to-client-on-put' into tmp/octopus/w/8.3/bugfix/CLDSRV-96-forward-408-errors-to-client-on-put 2022-01-17 17:13:30 +00:00
Artem Bakalov 19b31f16fc Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-96-forward-408-errors-to-client-on-put' into w/8.2/bugfix/CLDSRV-96-forward-408-errors-to-client-on-put 2022-01-17 09:12:42 -08:00
Artem Bakalov cb8dadd093 Merge remote-tracking branch 'origin/bugfix/CLDSRV-96-forward-408-errors-to-client-on-put' into w/7.10/bugfix/CLDSRV-96-forward-408-errors-to-client-on-put 2022-01-17 09:06:30 -08:00
Francois Ferrand cde2a224ef Bump version to 8.3.9
Issue: CLDSRV-99
2022-01-17 13:43:27 +01:00
Francois Ferrand 2bf2df789f Fix type of alerts' config params
Issue: CLDSRV-99
2022-01-17 13:43:27 +01:00
Artem Bakalov dd875bacd4 bugfix: CLDSRV-96 S3C should send httpCode 400 back to client on 408 metadata response
This allows TS to quickly diagnose when an error is caused by network issues on the client.
2022-01-14 09:11:45 -08:00
Francois Ferrand 485f5159a5 Fix typo in "aggregated status over time" panel
Issue: CLDSRV-98
2022-01-13 18:00:00 +01:00
Alexandre Lavigne ed8ebb5c7f
CLDSRV-77 - bump version to 8.3.8 2022-01-12 16:58:04 +01:00
Alexandre Lavigne fcb5ae28ed
CLDSRV-89 - revert push server health check route
Revert the route used to check the health of the push server.
This route is not being updated in zenko-operator, so it should
not change to make sure it matches what the operator deploys
2022-01-12 14:29:28 +01:00
Alexandre Lavigne 7b67328f95
CLDSRV-89 - specify health check port in documentation 2022-01-12 14:29:28 +01:00
Alexandre Lavigne 38228044c8
CLDSRV-89 - rename route `healthcheck` to `live`
Rename the route `/healthcheck` to `/live` to match
kubernetes probe name *liveness*.

Update the route `/healthcheck/deep` to `/ready` to match
kubernetes probe name *readiness*
2022-01-12 14:29:28 +01:00
Alexandre Lavigne af872f5452
CLDSRV-89 - update doc, tests to new health check route
Add branch `user/*` to trigger a CI run, in order for everyone
to be able to run CI on personal branches.
2022-01-12 14:29:28 +01:00
Alexandre Lavigne 776367dcf4
CLDSRV-89 - update liveness/readiness path
Use the shorter path `/healthcheck`, use the dedicated function `healthckeckhandler`
with it.
2022-01-12 14:29:28 +01:00
Alexandre Lavigne af0c42863d
ZENKO-3725 - update kubernetes readiness/liveness probe route
Update the readiness and liveness route for kubernetes.
The operator will configure the pod to check for readiness
and liveness on the same port as for the metrics.

In this commit allow the new routes on the metric code path.
2022-01-12 14:29:28 +01:00
Killian Gardahaut 421c2117cf
Merge remote-tracking branch 'origin/improvement/CLDSRV-91-bump-arsenal-version' into w/8.3/improvement/CLDSRV-91-bump-arsenal-version 2022-01-11 16:42:32 +01:00
Killian Gardahaut c3385a4225
CLDSRV-93 Disable multiple backend tests 2022-01-11 16:40:58 +01:00
Killian Gardahaut 18594575bc
CLDSRV-91 update cloudserver version 2022-01-11 11:43:18 +01:00
Killian Gardahaut b5c94ffce9
CLDSRV-91 bump arsenal version 2022-01-11 11:38:59 +01:00
Xin LI a74a9e6361 improvement: CLDSRV-90-bump-version-8.3.7 2022-01-06 18:54:22 +01:00
bert-e a0f4e31839 Merge branch 'bugfix/CLDSRV-86-fix-versionID' into q/8.3 2022-01-06 14:02:35 +00:00
williamlardier 937778b64c
bugfix: CLDSRV-86 bump arsenal and vaultclient 2022-01-06 11:45:27 +01:00
bert-e 8e252ac5cb Merge branch 'feature/CLDSRV-74' into q/8.3 2022-01-06 09:24:27 +00:00
Francois Ferrand 1997c5831d Display the "current" http requests count stat
Used to display the "total" count, show only the increase over current
time range.

Issue: CLDSRV-74
2022-01-04 12:43:30 +01:00
bert-e ca3ac649a6 Merge branch 'feature/CLDSRV-87' into q/8.3 2022-01-04 11:40:16 +00:00
Xin LI 4fa7efefce bugfix: CLDSRV-88-bump version 8.3.6 2022-01-03 15:28:15 +01:00
Francois Ferrand 8bd3ff5533 Make alerts configurable
Issue: CLDSRV-87
2021-12-31 15:54:56 +01:00
Francois Ferrand e13911a096 Display top 10 buckets with errors
Separate panels for 404, 500 and 50X errors.

Issue: CLDSRV-74
2021-12-29 17:47:02 +01:00
Francois Ferrand 44f0855057 Improve data rate display
- Use non-SI byte unit (kB/s, MB/s...)
- Fix capitalization of label
- Display upload packet size distribution
- Display download packet size distribution

Issue: CLDSRV-74
2021-12-29 17:47:02 +01:00
Francois Ferrand 39b649b3ef Improve latency display
- Display overall and per-action latencies on the same graph
- Add heatmap showing the request duration "repartition" over time

Issue: CLDSRV-74
2021-12-29 17:47:02 +01:00
Francois Ferrand 5844beca12 Improve response code graphs
- Compute the "increase" instead of "rate", to display the actual number
of operations
- Use "multi" tooltips, to allow viewing the value under cursor
- Use separate colors and stack series for aggregated statuses
- Fill graphs & use smooth lines

Issue: CLDSRV-74
2021-12-29 16:01:24 +01:00
Francois Ferrand 9101bb94ae Add overview panels on top
- "up" replicas counter
- "Total requests" counter
- Number of buckets & objects
- Data disk storage
- Data & object injection rate
- Responses counter by category (200, 4xx, 5xx)

Issue: CLDSRV-74
2021-12-29 16:01:21 +01:00
bert-e e426635c1f Merge branch 'bugfix/CLDSRV-83' into q/8.3 2021-12-28 19:15:07 +00:00
Francois Ferrand 96115259c7 Add description to all alerts
Issue: CLDSRV-83
2021-12-28 10:48:49 +01:00
Francois Ferrand df271f7aa9 Fix typo in variable name
Request time is measured in nanosecond, fix name to reflect the correct
unit.

Issue: CLDSRV-83
2021-12-28 09:01:57 +01:00
Francois Ferrand f548ea8810 Generate average latency alerts
Issue: CLDSRV-83
2021-12-27 17:54:08 +01:00
Francois Ferrand 653f73e257 Use only deleteObject in DeleteLatencyWarning
Ignore deleteBucket and multiObjectDelete, to better match requirements and
avoid unsignificant alerts (as these multi-object delete operations can
take longer, esp. when many objects are deleted in a single command).

Issue: CLDSRV-83
2021-12-27 17:43:16 +01:00
Francois Ferrand 37cbb69ab5 Use only listBucket in ListingLatencyWarning
Ignore getService (e.g. listing buckets), to better match requirements.

Issue: CLDSRV-83
2021-12-27 17:41:06 +01:00
Xin LI 2a859a8c7c bugfix: CLDSRV-85-revert some parts 2021-12-27 16:47:57 +01:00
Xin LI 187ed092fa improvement: CLDSRV-84-bump-to-8.3.5 2021-12-27 14:21:04 +01:00
Xin LI b97ce84204 improvement: CLDSRV-84-bump-up-8.3.4 2021-12-27 12:45:08 +01:00
bert-e 9667339bee Merge branch 'improvement/CLDSRV-79-support-checkPolicy-for-session-user' into q/8.3 2021-12-23 12:57:18 +00:00
bert-e 91b1078870 Merge branch 'w/8.3/bugfix/CLDSRV-43/addMissingLifecycleHeaders' into tmp/octopus/q/8.3 2021-12-23 11:26:13 +00:00
Xin LI 7ae4fba220 improvement: CLDSRV-79-replace all iamuser check to nonaccount user check 2021-12-22 15:15:35 +01:00
Alexandre Lavigne d4d8537935
CLDSRV-80 - use git tag to tag dashboard during release 2021-12-20 11:18:57 +01:00
Kerkesni 7c80be1136
improvement: CLDSRV-76 replace vault with cloudserver in release docs 2021-12-17 14:18:46 +01:00
Kerkesni bb89dd5919
improvement: CLDSRV-76 improved release docs 2021-12-17 13:51:10 +01:00
Kerkesni 9d8a2cbe81
improvement: CLDSRV-76 bump cloudserver version 2021-12-17 13:49:59 +01:00
Walid El Ansari d96b098bc1
CLDSRV-78-Add workdir 2021-12-17 10:42:29 +01:00
Walid El Ansari 9c619364fc CLDSRV-78-fix ci release 2021-12-16 18:38:30 +01:00
Francois Ferrand f6dbe23504 De-duplicate list of dashboard image layers
Issue: CLDSRV-72
2021-12-16 15:59:54 +01:00
Alexandre Lavigne e954e03dbb CLDSRV-72 - upload alerts to registry 2021-12-16 15:51:58 +01:00
Francois Ferrand 6e3b2962c8 Add overall cloudserver status alert
Issue: CLDSRV-75
2021-12-16 11:30:59 +01:00
bert-e a64a7a31e9 Merge branch 'feature/CLDSRV-65_create_dashboard' into q/8.3 2021-12-15 13:38:51 +00:00
Alexandre Lavigne 43ed45e264
CLDSRV-65 - push dashboard to registry
Add a script that pushes the dashboard to the registry.
One can override, using env variables:
- the registry URL
- the registry project
2021-12-15 14:00:33 +01:00
Alexandre Lavigne 72817d5f78
CLDSRV-65 - add cloudserver dashboard
Created the dashboard using grafana UI.
then go to setting, JSON model, copy/paste the content here.
2021-12-15 14:00:33 +01:00
bert-e cd5231a397 Merge branch 'bugfix/CLDSRV-73_request_time' into q/8.3 2021-12-15 12:09:47 +00:00
Alexandre Lavigne 903c6d9be9
CLDSRV-73 - convert nanoseconds to seconds
during metric measurement we get the request time in nanoseconds.
Convert it to seconds, as per the standard from prometheus, by
dividing it by 1000000000.
2021-12-15 11:53:44 +01:00
bert-e 844345483a Merge branch 'feature/CLDSRV-70-alerts' into q/8.3 2021-12-10 14:58:30 +00:00
Francois Ferrand 60c149d4b3 Add tests for http request & response size 2021-12-10 14:52:32 +01:00
Francois Ferrand d16e6d9553 Do not use 'or' in unit test assertions
We know the behavior, simply check what we expect.

Issue: CLDSRV-71
2021-12-10 14:07:17 +01:00
Francois Ferrand fd9f8b8507 Remove 'it' from testcase names
Issue: CLDSRV-71
2021-12-10 14:06:33 +01:00
Francois Ferrand 0c5823407e Fix typo in action name
Issue: CLDSRV-71
2021-12-10 12:08:51 +01:00
Francois Ferrand cf6b160744 Add cloudserver alerting rules
The alerts will be deployed by Zenko-Operator as PrometheusRule
objects, after "resolving" the input variables.

Issue: CLDSRV-70
2021-12-10 09:29:21 +01:00
bert-e 96ee301762 Merge branch 'w/8.2/bugfix/CLDSRV-43/addMissingLifecycleHeaders' into tmp/octopus/w/8.3/bugfix/CLDSRV-43/addMissingLifecycleHeaders 2021-12-09 01:08:46 +00:00
saitharunthodupunuru 21be0940fe Merge branch bugfix/CLDSRV-43/addMissingLifecycleHeaders into w/8.2/bugfix/CLDSRV-43/addMissingLifecycleHeaders 2021-12-08 16:29:04 -08:00
Sai Tharun Thodupunuru 1d6665c087
Merge branch 'development/7.10' into bugfix/CLDSRV-43/addMissingLifecycleHeaders 2021-12-08 13:10:36 -08:00
williamlardier 271ab5d4bd
improvement: CLDSRV-69 release 8.3.2 2021-12-08 15:36:12 +01:00
bert-e 20a75ab111 Merge branch 'w/8.2/improvement/CLDSRV-60-cleanupReplayIdOfPreviousVersions' into tmp/octopus/w/8.3/improvement/CLDSRV-60-cleanupReplayIdOfPreviousVersions 2021-12-02 02:02:23 +00:00
Jonathan Gramain 57fffbabb7 Merge remote-tracking branch 'origin/w/7.10/improvement/CLDSRV-60-cleanupReplayIdOfPreviousVersions' into w/8.2/improvement/CLDSRV-60-cleanupReplayIdOfPreviousVersions 2021-12-01 17:27:19 -08:00
Jonathan Gramain 60586d662d Merge remote-tracking branch 'origin/improvement/CLDSRV-60-cleanupReplayIdOfPreviousVersions' into w/7.10/improvement/CLDSRV-60-cleanupReplayIdOfPreviousVersions 2021-12-01 16:37:45 -08:00
Jonathan Gramain 6d741d1312 CLDSRV-60 address review: improve test self-doc
In versioning tests, add a 'description' field in test cases instead
of comments to make the description part of the test name
2021-12-01 16:32:52 -08:00
Jonathan Gramain 2d44334a1f improvement: CLDSRV-60 cleanup replay IDs of null versions
Replace the previous way of passing replayId option to metadata DELETE
by the result of the preprocessingVersioningDelete function, since it
handles more cases like when a bucket with versioning suspended has to
delete the previous null version, in which case, the null version's
replay IDs have to be cleaned up.
2021-12-01 16:21:51 -08:00
Jonathan Gramain a6e1c2ec83 improvement: CLDSRV-60 process upload IDs in versioning helpers
In versioning helpers, make sure we pass on the nullUploadId/replayId
to the metadataStoreObject() or deleteObject() function, so that they
can update ObjectMD and pass the appropriate replay options to metadata
2021-12-01 16:21:51 -08:00
Jonathan Gramain 9d18cf5d03 CLDSRV-60 bump arsenal dep 2021-12-01 16:21:51 -08:00
williamlardier 41635b82c3
improvement: CLDSRV bump arsenal version 2021-11-30 15:32:04 +01:00
bert-e 744d1e5d43 Merge branches 'w/8.2/improvement/CLDSRV-61-refactorAndTestVersioningHelpers' and 'q/4071/7.10/improvement/CLDSRV-61-refactorAndTestVersioningHelpers' into tmp/octopus/q/8.2 2021-11-29 23:15:44 +00:00
bert-e a563245dc8 Merge branches 'w/8.3/improvement/CLDSRV-61-refactorAndTestVersioningHelpers' and 'q/4071/8.2/improvement/CLDSRV-61-refactorAndTestVersioningHelpers' into tmp/octopus/q/8.3 2021-11-29 23:15:44 +00:00
bert-e 7f37ab9198 Merge branches 'w/7.10/improvement/CLDSRV-61-refactorAndTestVersioningHelpers' and 'q/4071/7.4/improvement/CLDSRV-61-refactorAndTestVersioningHelpers' into tmp/octopus/q/7.10 2021-11-29 23:15:43 +00:00
bert-e 743ef59fc0 Merge branch 'improvement/CLDSRV-61-refactorAndTestVersioningHelpers' into q/7.4 2021-11-29 23:15:43 +00:00
bert-e 63d71560ba Merge branch 'feature/CLDSRV-63-metrics' into q/8.3 2021-11-26 16:00:58 +00:00
Francois Ferrand ba5ded8b30 Use high resolution clock to fix flacky tests
Issue: CLDSRV-63
2021-11-26 15:45:50 +01:00
Francois Ferrand 8d0d1370a5 Rename label "route" to "action"
It avoids the confusion with the actual route, and will be easier to
understand when looking at the metrics.

Issue: CLDSRV-63
2021-11-26 15:05:30 +01:00
Francois Ferrand 8ce7a63ea1 fixup! Add http active requests metrics 2021-11-26 13:56:08 +01:00
Francois Ferrand da3cc74f1a fixup! Add functionnal test 2021-11-25 15:52:56 +01:00
Francois Ferrand 7718d11706 fixup! Add functionnal test 2021-11-25 14:51:35 +01:00
Francois Ferrand 1285ab7497 fixup! Update prom-client to 14.0.1 2021-11-25 14:05:03 +01:00
Alexander Chan 28d435b5f1 CLDSRV-43: add expiration header tests 2021-11-24 14:53:19 -08:00
Francois Ferrand 706d688985 Fix missing response on malformed requests
The server was not replying to a request (missing "resourceType") made
on backbeat or workflow-engine-operator internal routes.
2021-11-24 17:45:28 +01:00
Francois Ferrand 91fbc0fd61 Add functionnal test
Issue: CLDSRV-63
2021-11-24 17:18:14 +01:00
Francois Ferrand f3a3e442d1 Fix counting of http requests
All requests must be counted, includes rejected.

Issue: CLDSRV-63
2021-11-23 23:19:19 +01:00
Francois Ferrand 7f6fa22044 Update prom-client to 14.0.1
This is needed to support 'optional' labels.

Issue: CLDSRV-63
2021-11-23 23:19:16 +01:00
Francois Ferrand d3966151c6 Add unit tests
Issue: CLDSRV-63
2021-11-23 23:18:40 +01:00
Francois Ferrand 5c2076552b Add http active requests metrics
Issue: CLDSRV-63
2021-11-23 23:17:25 +01:00
Francois Ferrand 78f76476bf Fix error handling on prometheus endpoint
Issue: CLDSRV-63
2021-11-23 23:16:19 +01:00
Francois Ferrand 04b1642a04 Add http requests duration metrics
Issue: CLDSRV-63
2021-11-23 23:16:16 +01:00
bert-e 5986e207bb Merge branch 'w/7.10/bugfix/CLDSRV-66/fix_utapiv1_mpu_versioned_bucket' into tmp/octopus/w/8.2/bugfix/CLDSRV-66/fix_utapiv1_mpu_versioned_bucket 2021-11-23 22:02:45 +00:00
bert-e 6dcd86f719 Merge branch 'w/8.2/bugfix/CLDSRV-66/fix_utapiv1_mpu_versioned_bucket' into tmp/octopus/w/8.3/bugfix/CLDSRV-66/fix_utapiv1_mpu_versioned_bucket 2021-11-23 22:02:45 +00:00
bert-e 0f1b2c2c71 Merge branch 'bugfix/CLDSRV-66/fix_utapiv1_mpu_versioned_bucket' into tmp/octopus/w/7.10/bugfix/CLDSRV-66/fix_utapiv1_mpu_versioned_bucket 2021-11-23 22:02:44 +00:00
Taylor McKinnon 1364dc895d use version config instead 2021-11-23 10:53:28 -08:00
bert-e e0801a7b75 Merge branches 'development/8.2' and 'w/7.10/improvement/CLDSRV-61-refactorAndTestVersioningHelpers' into tmp/octopus/w/8.2/improvement/CLDSRV-61-refactorAndTestVersioningHelpers 2021-11-23 17:05:40 +00:00
bert-e 647d8135b5 Merge branches 'development/8.3' and 'w/8.2/improvement/CLDSRV-61-refactorAndTestVersioningHelpers' into tmp/octopus/w/8.3/improvement/CLDSRV-61-refactorAndTestVersioningHelpers 2021-11-23 17:05:40 +00:00
bert-e dae08d5024 Merge branches 'development/7.10' and 'improvement/CLDSRV-61-refactorAndTestVersioningHelpers' into tmp/octopus/w/7.10/improvement/CLDSRV-61-refactorAndTestVersioningHelpers 2021-11-23 17:05:39 +00:00
Francois Ferrand 9c3440c954 Tweak http metrics
* Remove `cloud_server` prefix from standard metrics
* Remove `cloud_server` label
* Add a route label, to store the operation/api method

Issue: CLDSRV-63
2021-11-23 10:21:48 +01:00
Taylor McKinnon 190a068828 bf(CLDSRV-66): Don't pass `oldByteLength` for completeMPU if versioned bucket 2021-11-22 15:25:59 -08:00
Jonathan Gramain 8f913101a6 CLDSRV-61 add JSDoc
Add JSDoc for processVersioningState() and getMasterState() versioning
helper functions
2021-11-22 14:17:09 -08:00
Alexander Chan 812b09afef CLDSRV-43: add expiration response headers to APIs
add expiration response headers to APIs:
* PutObject
* GetObject
* HeadObject
* CopyObject
* ListParts
* InitiateMultipartUploads
* CompleteMultipartUploads
2021-11-22 12:07:30 -08:00
bert-e 7d5af27c1e Merge branch 'q/4060/7.10/feature/CLDSRV-55/utapi_event_filtering' into tmp/normal/q/8.2 2021-11-19 01:48:23 +00:00
bert-e 2082cadf82 Merge branches 'w/8.3/feature/CLDSRV-55/utapi_event_filtering' and 'q/4060/8.2/feature/CLDSRV-55/utapi_event_filtering' into tmp/octopus/q/8.3 2021-11-19 01:48:23 +00:00
bert-e 4d4e81cf34 Merge branch 'feature/CLDSRV-55/utapi_event_filtering' into q/7.10 2021-11-19 01:48:22 +00:00
bert-e dea2204260 Merge branch 'w/8.2/feature/CLDSRV-55/utapi_event_filtering' into tmp/octopus/w/8.3/feature/CLDSRV-55/utapi_event_filtering 2021-11-18 23:33:06 +00:00
Taylor McKinnon 427061deb7 Merge remote-tracking branch 'origin/feature/CLDSRV-55/utapi_event_filtering' into w/8.2/feature/CLDSRV-55/utapi_event_filtering 2021-11-18 15:32:47 -08:00
Taylor McKinnon ce6b211697 ft(CLDSRV-55): Add config loading for utapi v2 event filter 2021-11-18 15:30:15 -08:00
bert-e d1b6ebf0dd Merge branch 'w/8.2/improvement/CLDSRV-61-refactorAndTestVersioningHelpers' into tmp/octopus/w/8.3/improvement/CLDSRV-61-refactorAndTestVersioningHelpers 2021-11-18 05:13:19 +00:00
bert-e ca4a788dcd Merge branch 'w/7.10/improvement/CLDSRV-61-refactorAndTestVersioningHelpers' into tmp/octopus/w/8.2/improvement/CLDSRV-61-refactorAndTestVersioningHelpers 2021-11-18 05:13:18 +00:00
bert-e 1d09dbd4fb Merge branch 'improvement/CLDSRV-61-refactorAndTestVersioningHelpers' into tmp/octopus/w/7.10/improvement/CLDSRV-61-refactorAndTestVersioningHelpers 2021-11-18 05:13:18 +00:00
Jonathan Gramain 1ccd36f9bf improvement: CLDSRV-61 test preprocessingVersioningDelete
Add unit tests for preprocessingVersioningDelete helper
2021-11-17 20:48:49 -08:00
Jonathan Gramain 22f7f253a1 improvement: CLDSRV-61 refactor and test processVersioningState
- add unit tests for processVersioningState() helper

- remove the callback argument, instead, return the list of parameters
  as an object, it simplifies and enhances testability
2021-11-17 20:48:49 -08:00
bert-e 6da6a9acd4 Merge branches 'w/8.3/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete' and 'q/4044/8.2/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete' into tmp/octopus/q/8.3 2021-11-17 19:02:30 +00:00
bert-e 64dd6b443d Merge branches 'w/7.10/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete' and 'q/4044/7.4/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete' into tmp/octopus/q/7.10 2021-11-17 19:02:29 +00:00
bert-e 906731ff6d Merge branches 'w/8.2/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete' and 'q/4044/7.10/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete' into tmp/octopus/q/8.2 2021-11-17 19:02:29 +00:00
bert-e 221c4d3c14 Merge branch 'improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete' into q/7.4 2021-11-17 19:02:28 +00:00
Rached Ben Mustapha b30265da43
chore: release 8.3.1 2021-11-15 09:37:34 -08:00
Rached Ben Mustapha f8235facdb
chore: bump arsenal version 2021-11-15 09:36:16 -08:00
bert-e 3f07fc3397 Merge branch 'improvement/CLDSRV-59-release-8.3.0' into q/8.3 2021-11-12 16:13:28 +00:00
Francois Ferrand 08fc544481 Bump version to 8.3.0
Issue: CLDSRV-59
2021-11-12 16:00:28 +01:00
bert-e dc2e6af001 Merge branch 'feature/CLDSRV-51-new-http-server-for-metrics' into q/8.3 2021-11-12 14:17:04 +00:00
Francois Ferrand 3e3352b798 Use port 8002 for cloudserver metrics
Issue: CLDSRV-51
2021-11-12 14:23:13 +01:00
Francois Ferrand fc9277589a Use /metrics path
As defined in https://scality.atlassian.net/wiki/spaces/OS/pages/1819148292/Strategy+for+exposing+metrics

Issue: CLDSRV-51
2021-11-12 14:23:00 +01:00
Jonathan Gramain aafc46dc66 Merge remote-tracking branch 'origin/w/8.2/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete' into w/8.3/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete 2021-11-11 17:33:28 -08:00
bert-e 8c0cf823b8 Merge branches 'development/8.2' and 'w/7.10/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete' into tmp/octopus/w/8.2/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete 2021-11-12 00:14:31 +00:00
bert-e 11cd2a30ec Merge branch 'development/7.10' into tmp/octopus/w/7.10/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete 2021-11-12 00:14:30 +00:00
Jonathan Gramain f50b81be78 CLDSRV-53 fixup path on 8.2 branch 2021-11-11 15:30:37 -08:00
bert-e c5a3d75622 Merge branch 'w/8.3/bugfix/CLDSRV-56_DoNotEncodeTokensAgain' into tmp/octopus/q/8.3 2021-11-11 22:55:07 +00:00
Jonathan Gramain f9d041b1fd Merge remote-tracking branch 'origin/w/7.10/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete' into w/8.2/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete 2021-11-11 14:16:42 -08:00
Jonathan Gramain 1d3a1821f7 Merge remote-tracking branch 'origin/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete' into w/7.10/improvement/CLDSRV-53-passReplayIdOnMPUCompleteAndDelete 2021-11-11 13:28:46 -08:00
bert-e 4a37347c9d Merge branch 'w/8.2/bugfix/CLDSRV-56_DoNotEncodeTokensAgain' into tmp/octopus/w/8.3/bugfix/CLDSRV-56_DoNotEncodeTokensAgain 2021-11-10 23:46:51 +00:00
bert-e f7bdcb9335 Merge branch 'bugfix/CLDSRV-56_DoNotEncodeTokensAgain' into tmp/octopus/w/7.10/bugfix/CLDSRV-56_DoNotEncodeTokensAgain 2021-11-10 23:46:50 +00:00
bert-e 0083b7b719 Merge branch 'w/7.10/bugfix/CLDSRV-56_DoNotEncodeTokensAgain' into tmp/octopus/w/8.2/bugfix/CLDSRV-56_DoNotEncodeTokensAgain 2021-11-10 23:46:50 +00:00
Ronnie Smith 166bddd6dc
bugfix: CLDSRV-56 Do not encode tokens again 2021-11-10 15:45:54 -08:00
Nicolas Humbert 2bec6c2143 CLDSRV-57 Artesca 1.3 will only support ingestion from S3C 2021-11-09 12:17:09 -05:00
Jonathan Gramain 5fc53b3b95 CLDSRV-53 pass replayId when completing/deleting an MPU
Pass the 'replayId' option to metadata when:

- completing an MPU, to allow metadata to check or write a replay key

- deleting an MPU, to allow metadata to delete the replay key
2021-11-08 17:42:19 -08:00
Jonathan Gramain 54a3553c20 CLDSRV-53 arsenal version bump 2021-11-08 17:42:08 -08:00
bert-e 12354f20a4 Merge branch 'w/8.2/bugfix/CLDSRV-48-configure-utapi-redis-retry-main' into tmp/octopus/w/8.3/bugfix/CLDSRV-48-configure-utapi-redis-retry-main 2021-11-09 00:52:35 +00:00
Rached Ben Mustapha 92e78e6452 bugfix: utapi unit test 2021-11-09 00:52:24 +00:00
bert-e 65fb1cd9e5 Merge branch 'w/8.2/bugfix/CLDSRV-48-configure-utapi-redis-retry-main' into tmp/octopus/w/8.3/bugfix/CLDSRV-48-configure-utapi-redis-retry-main 2021-11-08 23:15:52 +00:00
Rached Ben Mustapha adc9b7bedd bugfix: utapi unit test 2021-11-08 23:15:33 +00:00
bert-e 998723fcdd Merge branch 'w/8.2/bugfix/CLDSRV-48-configure-utapi-redis-retry-main' into tmp/octopus/w/8.3/bugfix/CLDSRV-48-configure-utapi-redis-retry-main 2021-11-08 20:39:07 +00:00
Rached Ben Mustapha 1eda439539 Merge remote-tracking branch 'origin/bugfix/CLDSRV-48-configure-utapi-redis-retry-main' into w/8.2/bugfix/CLDSRV-48-configure-utapi-redis-retry-main 2021-11-08 20:25:27 +00:00
Rached Ben Mustapha c53ebe1b9a feature: configure utapi retry for cloudserver 2021-11-08 19:42:47 +00:00
Rached Ben Mustapha 2b89922b7f chore: bump utapi version 2021-11-08 19:42:47 +00:00
Francois Ferrand 102fed0403 Merge remote-tracking branch 'origin/development/8.3' into feature/CLDSRV-51-new-http-server-for-metrics 2021-11-08 13:55:50 +01:00
bert-e 9276d722db Merge branch 'w/8.3/bugfix/CLDSRV-54' into tmp/octopus/q/8.3 2021-11-06 08:23:03 +00:00
Ronnie Smith 506913b78d
feature: CLDSRV-46 Use patch locations from Arsena 2021-11-05 14:05:16 -07:00
Francois Ferrand 203c7ec4e7 Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-54' into w/8.2/bugfix/CLDSRV-54 2021-11-05 10:33:23 +01:00
bert-e 65f980c144 Merge branch 'bugfix/CLDSRV-54' into tmp/octopus/w/7.10/bugfix/CLDSRV-54 2021-11-05 09:29:52 +00:00
Francois Ferrand 7bec09e761 Ensure to close all servers on shutdown
Issue: CLDSRV-54
2021-11-05 10:14:08 +01:00
Francois Ferrand 2a1389666c Ensure to close all servers on shutdown
Issue: CLDSRV-51
2021-11-04 16:56:06 +01:00
Francois Ferrand 6ed87b5a67 Make metrics endpoint configurable
Issue: CLDSRV-51
2021-11-04 16:56:06 +01:00
Francois Ferrand 0db92e0a49 Expose metrics port in Docker image
Issue: CLDSRV-51
2021-11-04 16:56:06 +01:00
Francois Ferrand 47f109a5d6 Proper routing of metrics requests
Issue: CLDSRV-51
2021-11-04 16:56:06 +01:00
Jordi Bertran de Balanda 9ae98ce550 feature: CLDSRV-51 - spawn a new HTTP server to serve metrics 2021-11-04 10:39:44 +01:00
Jordi Bertran de Balanda e046884c3c feature: CLDSRV-51 - switch case for prometheus object updates 2021-11-04 10:39:06 +01:00
bert-e 24f9fa6a65 Merge branches 'w/8.2/bugfix/CLDSRV-45/add_crr_metrics_utapiv2' and 'q/4019/7.10/bugfix/CLDSRV-45/add_crr_metrics_utapiv2' into tmp/octopus/q/8.2 2021-11-02 21:51:38 +00:00
bert-e 6c12d83cdc Merge branch 'bugfix/CLDSRV-45/add_crr_metrics_utapiv2' into q/7.10 2021-11-02 21:51:38 +00:00
Taylor McKinnon c4d49607ae Merge remote-tracking branch 'origin/bugfix/CLDSRV-45/add_crr_metrics_utapiv2' into w/8.2/bugfix/CLDSRV-45/add_crr_metrics_utapiv2 2021-11-02 10:09:20 -07:00
Taylor McKinnon ef166efc62 bf(CLDSRV-45): Add crr metrics for utapiv2 2021-11-02 09:56:18 -07:00
bert-e 103f077176 Merge branch 'w/7.10/bugfix/CLDSRV-37-pushReplicationMetrics' into tmp/octopus/w/8.2/bugfix/CLDSRV-37-pushReplicationMetrics 2021-11-01 18:18:47 +00:00
Taylor McKinnon ceab89c6da Merge remote-tracking branch 'origin/bugfix/CLDSRV-37-pushReplicationMetrics' into w/7.10/bugfix/CLDSRV-37-pushReplicationMetrics 2021-11-01 11:18:10 -07:00
Jonathan Gramain f345db91c0 bf(CLDSRV-37): Push metrics for backbeat routes 2021-11-01 11:13:04 -07:00
bert-e e9731164a2 Merge branch 'feature/CLDSRV-41-bumpversion' into q/8.2 2021-10-29 16:07:52 +00:00
bert-e 487e2a1d1d Merge branch 'w/8.2/improvement/CLDSRV-44-upgrade-arsenal' into tmp/octopus/q/8.2 2021-10-29 15:29:03 +00:00
Francois Ferrand e0bfe70b21 Bump clouserver version to 8.2.17
Issue: CLDSRV-41
2021-10-29 15:24:03 +02:00
Rahul Padigela 88df896458 Merge remote-tracking branch 'origin/w/7.10/improvement/CLDSRV-44-upgrade-arsenal' into w/8.2/improvement/CLDSRV-44-upgrade-arsenal 2021-10-27 16:52:10 -07:00
Rahul Padigela 7957f96d16 Merge remote-tracking branch 'origin/improvement/CLDSRV-44-upgrade-arsenal' into w/7.10/improvement/CLDSRV-44-upgrade-arsenal 2021-10-27 16:41:32 -07:00
Rahul Padigela b1b336cd0e chore: upgrade arsenal version
Release notes - Arsenal - Version 7.4.11

Bug

ARSN-31 Return empty string for invalid encoding requests

Feature

ARSN-20 Migrate CI to GitHub Actions
2021-10-27 16:22:09 -07:00
Francois Ferrand 497188dac5 Do not use JSON.parse() to parse variable
Issue: CLDSRV-41
Issue: ARTESCA-2190
2021-10-27 13:42:57 +02:00
Francois Ferrand 501fbda5c2 Allow configuring CLDSRV-41 capability
Issue: CLDSRV-41
Issue: ARTESCA-2190
2021-10-25 09:23:25 +02:00
Naren 05ff4d91e2 improvement: CLDSRV-42 bump cloudserver version
bump cloudserver version to 8.2.16
2021-10-22 15:21:41 -07:00
Naren 643c201325 improvement: CLDSRV-26 bump arsenal version
bump arsenal version to 8.1.7
2021-10-22 14:05:59 -07:00
Nicolas Humbert c3c82e086b CLDSRV-40 bump version to 8.2.15 2021-10-20 15:28:04 -04:00
bert-e c0b64fa490 Merge branch 'bugfix/CLDSRV-39/arsenal' into q/8.2 2021-10-19 21:39:57 +00:00
Nicolas Humbert 9f84c8c6df CLDSRV-39 interrogate the default region for getBucketLocation 2021-10-19 16:31:12 -04:00
bert-e 2e9d691511 Merge branches 'w/8.2/bugfix/CLDSRV-36-backport-S3C-2775-to-7.4' and 'q/3948/7.10/bugfix/CLDSRV-36-backport-S3C-2775-to-7.4' into tmp/octopus/q/8.2 2021-10-15 20:51:05 +00:00
bert-e 504887a79a Merge branches 'w/7.10/bugfix/CLDSRV-36-backport-S3C-2775-to-7.4' and 'q/3948/7.4/bugfix/CLDSRV-36-backport-S3C-2775-to-7.4' into tmp/octopus/q/7.10 2021-10-15 20:51:04 +00:00
bert-e fe03437eab Merge branch 'bugfix/CLDSRV-36-backport-S3C-2775-to-7.4' into q/7.4 2021-10-15 20:51:04 +00:00
bert-e 6f84440bdd Merge branch 'w/7.10/bugfix/CLDSRV-36-backport-S3C-2775-to-7.4' into tmp/octopus/w/8.2/bugfix/CLDSRV-36-backport-S3C-2775-to-7.4 2021-10-14 21:06:22 +00:00
bert-e 6daad06eba Merge branch 'bugfix/CLDSRV-36-backport-S3C-2775-to-7.4' into tmp/octopus/w/7.10/bugfix/CLDSRV-36-backport-S3C-2775-to-7.4 2021-10-14 21:06:22 +00:00
Jonathan Gramain 7d5b5e3ca2 bugfix: S3C-2775 update logic for counting object replicas
Before, we were relying on the presence of ACL change to know if the
operation is metadata-only. This was not correct, as a new object can
be put with specific ACL too.

Instead, consider that if there is already an object present with the
same version, we are necessarily updating this object hence the object
counter should not be updated.

NOTE: this change will come along with another change in backbeat: the
PUT /_/backbeat/metadata route will be provided the version ID of the
object to PUT, so that the validation of the existing object (through
the metadataValidateBucketAndObj helper) checks the specific version
instead of checking the master key (which for backbeat replication
purpose is also incorrect). Without this backbeat change, the
replicated object count metric becomes wrong when an object is
rewritten.

(cherry picked from commit 69e6d678a0)
2021-10-14 14:01:01 -07:00
Alexandre Lavigne 580b9611df
CLDSRV-35 - update key server to a working key server 2021-10-14 17:11:29 +02:00
bert-e 93b2beed54 Merge branch 'bugfix/CLDSRV-35_update_aggressor_ci_image' into tmp/octopus/w/7.10/bugfix/CLDSRV-35_update_aggressor_ci_image 2021-10-14 15:08:08 +00:00
bert-e f423854d6b Merge branch 'w/7.10/bugfix/CLDSRV-35_update_aggressor_ci_image' into tmp/octopus/w/8.2/bugfix/CLDSRV-35_update_aggressor_ci_image 2021-10-14 15:08:08 +00:00
Alexandre Lavigne d04abd5339
CLDSRV-35 - Update aggressor image to bionic
Xenial is now out of date and has reached end-of-support.
SSL certificates are not updated anymore, so we can't download
anything from HTTPS, we can't retrieve gpg keys.
2021-10-14 17:03:00 +02:00
bert-e 2b19e2703c Merge branch 'w/8.2/bugfix/CLDSRV-30-deleteOldDataLocationsOnBackbeatPutMetadata' into tmp/octopus/q/8.2 2021-10-13 23:15:46 +00:00
Alexandre Lavigne 829a22fe94
CLDSRV-34 - fix time computation for test comparison
need a new date that is 30 minutes in the past.
Computation is based on a date already 30 minutes in the future,
take that date and rewind it by 60 minutes
- 30 to go back to the original data
- 30 to go back 30 minutes in the past
2021-10-13 14:37:53 +02:00
Jonathan Gramain f22f9bc0a1 CLDSRV-30 fix routeBackbeat logic on development/8.2 branch 2021-10-12 16:22:56 -07:00
Jonathan Gramain e27ecd89f5 Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-30-deleteOldDataLocationsOnBackbeatPutMetadata' into w/8.2/bugfix/CLDSRV-30-deleteOldDataLocationsOnBackbeatPutMetadata 2021-10-12 16:22:30 -07:00
Jonathan Gramain 3bfe4bcd21 CLDSRV-30 fix routeBackbeat tests on development/7.10 branch 2021-10-12 16:19:33 -07:00
bert-e 82d83898a6 Merge branch 'bugfix/CLDSRV-30-deleteOldDataLocationsOnBackbeatPutMetadata' into tmp/octopus/w/7.10/bugfix/CLDSRV-30-deleteOldDataLocationsOnBackbeatPutMetadata 2021-10-12 21:34:31 +00:00
Naren 0bb7a918c0 improvement: ARSN-25 update version to 8.1.5 2021-10-12 13:41:02 -07:00
Jonathan Gramain e7b7d28015 bugfix: CLDSRV-30 remove old data keys on CRR target version overwrite
Remove data keys belonging to the old object when overwriting a CRR
target object version with a new object, to avoid creating orphaned
data keys.

The code checks whether the data keys have changed before removing the
old ones, in order to cope with replays of the PUT metadata operation
only. Similarly, metadata-only operations do not trigger any data key
removal.
2021-10-11 16:27:27 -07:00
Jonathan Gramain 07ae7e492a bugfix: CLDSRV-30 failing test
Add a failing test to show that old data locations are not properly
cleaned up on backbeat PUT metadata overwriting an existing version.
2021-10-08 16:44:43 -07:00
Jonathan Gramain ba6765bea4 refactor: CLDSRV-30 rename locationKeysSanityCheck to locationKeysHaveChanged
The function name now reflects what it is returning (a boolean) which
is clearer in the new context where it will be used. The jsdoc was
also improved and fixed.
2021-10-08 16:44:43 -07:00
bert-e 6df2b1698d Merge branch 'q/3841/7.10/feature/CLDSRV-22-pin-arsenal' into tmp/normal/q/8.2 2021-10-08 17:45:17 +00:00
bert-e 9233b658f8 Merge branch 'q/3841/7.4/feature/CLDSRV-22-pin-arsenal' into tmp/normal/q/7.10 2021-10-08 17:45:16 +00:00
bert-e 779ed8a1b7 Merge branch 'w/7.10/feature/CLDSRV-22-pin-arsenal' into tmp/normal/q/7.10 2021-10-08 17:45:16 +00:00
bert-e 5e08207d47 Merge branch 'feature/CLDSRV-22-pin-arsenal' into q/7.4 2021-10-08 17:45:15 +00:00
Thomas Carmet af00189942 Merge remote-tracking branch 'origin/w/7.10/feature/CLDSRV-22-pin-arsenal' into w/8.2/feature/CLDSRV-22-pin-arsenal 2021-10-08 10:11:19 -07:00
bert-e c5be7e3741 Merge branches 'w/8.2/feature/CLDSRV-33-package-version-set' and 'q/3918/7.10/feature/CLDSRV-33-package-version-set' into tmp/octopus/q/8.2 2021-10-08 16:17:47 +00:00
bert-e 51b608cace Merge branch 'w/7.10/feature/CLDSRV-33-package-version-set' into tmp/octopus/q/7.10 2021-10-08 16:17:46 +00:00
bert-e bd39a80952 Merge branch 'w/8.2/bugfix/CLDSRV-31-disableKMSHealthcheck' into tmp/octopus/q/8.2 2021-10-07 20:22:35 +00:00
Thomas Carmet e1dd682495 Merge branch 'w/7.10/feature/CLDSRV-33-package-version-set' into w/8.2/feature/CLDSRV-33-package-version-set 2021-10-07 11:01:40 -07:00
Thomas Carmet 36293d5c64 Merge branch 'feature/CLDSRV-33-package-version-set' into w/7.10/feature/CLDSRV-33-package-version-set 2021-10-07 11:01:03 -07:00
Thomas Carmet a7864fe803 CLDSRV-33 set package.json version accordingly 2021-10-07 10:59:21 -07:00
Thomas Carmet 05eed1cf18 CLDSRV-22 stop counting on commit hash size for release 2021-10-06 17:20:14 -07:00
Thomas Carmet ccc425249b Merge branch 'feature/CLDSRV-22-pin-arsenal' into w/7.10/feature/CLDSRV-22-pin-arsenal 2021-10-06 14:26:48 -07:00
Thomas Carmet 38200e83b1 CLDSRV-22 pin arsenal version 2021-10-06 14:20:40 -07:00
bert-e f7ce2931b4 Merge branch 'bugfix/CLDSRV-31-disableKMSHealthcheck' into tmp/octopus/w/8.2/bugfix/CLDSRV-31-disableKMSHealthcheck 2021-10-05 18:33:35 +00:00
Jonathan Gramain d7fd815d6d bugfix: CLDSRV-31 disable KMS healthchecks
Disable KMS healthchecks introduced with S3C-4562, because when the
KMS is unavailable, Cloudserver would not start, or if already
started, would reject all requests (including unencrypted ones).

The KMS healthchecks will be reworked later, this is a quick fix for now.
2021-10-05 11:32:40 -07:00
bert-e 804f59d5f9 Merge branch 'bugfix/CLDSRV-28-put-bucket-notification-crashes-with-cross-origin-requests' into tmp/octopus/w/8.2/bugfix/CLDSRV-28-put-bucket-notification-crashes-with-cross-origin-requests 2021-09-30 19:28:50 +00:00
Naren ced0ac31d2 bf: CLDSRV-28 changed error response
Cloudserver response message to an unknown or unsupported destination in
the request notification configuration is modified to match AWS's
message.
2021-09-30 12:27:45 -07:00
bert-e 0bbb40a7da Merge branch 'bugfix/CLDSRV-28-put-bucket-notification-crashes-with-cross-origin-requests' into tmp/octopus/w/8.2/bugfix/CLDSRV-28-put-bucket-notification-crashes-with-cross-origin-requests 2021-09-30 15:25:53 +00:00
Naren f78a05c965 bf: CLDSRV-28 fixed server crash
Cloudserver crashes when a cross origin request is made to put bucket
notification configuration with invalid arn or a configuration with unknown configuration.
2021-09-29 21:24:36 -07:00
Naren 41533ca80f improvement: bump version to 8.2.13 2021-09-28 16:06:08 -07:00
bert-e 8fae0bfa76 Merge branch 'w/7.10/feature/CLDSRV-20-pin-bucketclient' into tmp/normal/w/8.2/feature/CLDSRV-20-pin-bucketclient 2021-09-01 20:26:26 +00:00
bert-e bc2fad37c9 Merge branch 'development/8.2' into tmp/normal/w/8.2/feature/CLDSRV-20-pin-bucketclient 2021-09-01 20:26:26 +00:00
bert-e a7b98abb7a Merge branch 'feature/CLDSRV-20-pin-bucketclient' into tmp/normal/w/7.10/feature/CLDSRV-20-pin-bucketclient 2021-09-01 20:26:25 +00:00
bert-e 31cdf91b27 Merge branch 'development/7.10' into tmp/normal/w/7.10/feature/CLDSRV-20-pin-bucketclient 2021-09-01 20:26:25 +00:00
Thomas Carmet 1922cadb9c Update yarn.lock 2021-09-01 13:26:08 -07:00
Thomas Carmet 1fc86f78b7 Merge remote-tracking branch 'origin/w/7.10/feature/CLDSRV-20-pin-bucketclient' into w/8.2/feature/CLDSRV-20-pin-bucketclient 2021-08-31 12:59:16 -07:00
Thomas Carmet 74deabdcef Merge remote-tracking branch 'origin/feature/CLDSRV-20-pin-bucketclient' into w/7.10/feature/CLDSRV-20-pin-bucketclient 2021-08-30 14:17:17 -07:00
Thomas Carmet a5b7e90408 CLDSRV-20 pin bucketclient 2021-08-30 14:11:18 -07:00
Thomas Carmet 8b9b1b0978 Merge remote-tracking branch 'origin/w/7.10/feature/CLDSRV-19-bump-sproxydclient' into w/8.2/feature/CLDSRV-19-bump-sproxydclient 2021-08-30 09:42:01 -07:00
Thomas Carmet 993246d278 Merge remote-tracking branch 'origin/feature/CLDSRV-19-bump-sproxydclient' into w/7.10/feature/CLDSRV-19-bump-sproxydclient 2021-08-30 09:38:05 -07:00
Thomas Carmet 7686261d5a CLDSRV-19 pin sproxydclient version 2021-08-30 09:34:30 -07:00
bert-e c03c33162f Merge branches 'w/8.2/bugfix/CLDSRV-18-set-default-region' and 'q/3825/7.10/bugfix/CLDSRV-18-set-default-region' into tmp/octopus/q/8.2 2021-08-27 19:51:26 +00:00
bert-e e0c5a85356 Merge branches 'w/7.10/bugfix/CLDSRV-18-set-default-region' and 'q/3825/7.10.0/bugfix/CLDSRV-18-set-default-region' into tmp/octopus/q/7.10 2021-08-27 19:51:26 +00:00
bert-e 72c5994eb1 Merge branch 'bugfix/CLDSRV-18-set-default-region' into q/7.10.0 2021-08-27 19:51:25 +00:00
bert-e d10ef350ba Merge branches 'w/8.2/bugfix/CLDSRV-16/fixObjectLockComparisonCheck' and 'q/3818/7.10/bugfix/CLDSRV-16/fixObjectLockComparisonCheck' into tmp/octopus/q/8.2 2021-08-27 19:30:56 +00:00
bert-e 44030670dd Merge branch 'w/7.10/bugfix/CLDSRV-16/fixObjectLockComparisonCheck' into tmp/octopus/q/7.10 2021-08-27 19:30:55 +00:00
Naren fa175e413e Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-18-set-default-region' into w/8.2/bugfix/CLDSRV-18-set-default-region 2021-08-27 11:37:16 -07:00
bert-e be1036f873 Merge branch 'bugfix/CLDSRV-18-set-default-region' into tmp/octopus/w/7.10/bugfix/CLDSRV-18-set-default-region 2021-08-27 18:32:37 +00:00
Naren a55fecfa8e bf: CLDSRV-18 correct package.json 2021-08-27 11:32:22 -07:00
bert-e 0fdcbed429 Merge branch 'w/7.10/bugfix/CLDSRV-18-set-default-region' into tmp/octopus/w/8.2/bugfix/CLDSRV-18-set-default-region 2021-08-27 18:25:03 +00:00
bert-e 566fba320c Merge branch 'bugfix/CLDSRV-18-set-default-region' into tmp/octopus/w/7.10/bugfix/CLDSRV-18-set-default-region 2021-08-27 18:25:03 +00:00
Naren 9e87138379 bf: CLDSRV-15 use updated vaultclient
use correct uuid package.

(cherry picked from commit 54b8de8b62)
2021-08-27 11:23:00 -07:00
naren-scality e2493c66f6 bf: CLDSRV-15 use updated vaultclient
use updated vault client with the fix setting us-east-1 as default region when region is not provided.

(cherry picked from commit 027798a525)
2021-08-27 11:22:37 -07:00
Naren 901cfb18f0 bf: CLDSRV-15 use updated vaultclient
use correct uuid package.
2021-08-26 04:04:58 -07:00
bert-e 16ea051c70 Merge branch 'w/7.10/bugfix/CLDSRV-15-use-updated-vault-client' into tmp/octopus/w/8.2/bugfix/CLDSRV-15-use-updated-vault-client 2021-08-26 10:57:33 +00:00
Naren 54b8de8b62 bf: CLDSRV-15 use updated vaultclient
use correct uuid package.
2021-08-26 03:31:23 -07:00
Naren faf421de17 Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-15-use-updated-vault-client' into w/8.2/bugfix/CLDSRV-15-use-updated-vault-client 2021-08-26 02:40:41 -07:00
Naren ee879e719e Merge remote-tracking branch 'origin/bugfix/CLDSRV-15-use-updated-vault-client' into w/7.10/bugfix/CLDSRV-15-use-updated-vault-client 2021-08-26 02:37:11 -07:00
bert-e 8e5ee89ceb Merge branch 'w/7.10/bugfix/CLDSRV-16/fixObjectLockComparisonCheck' into tmp/octopus/w/8.2/bugfix/CLDSRV-16/fixObjectLockComparisonCheck 2021-08-26 01:12:17 +00:00
bert-e 14e45a5e4e Merge branch 'bugfix/CLDSRV-16/fixObjectLockComparisonCheck' into tmp/octopus/w/7.10/bugfix/CLDSRV-16/fixObjectLockComparisonCheck 2021-08-26 01:12:16 +00:00
Alexander Chan c170e2f29f CLDSRV-16: fix object-lock comparison checks 2021-08-25 17:54:36 -07:00
naren-scality 027798a525 bf: CLDSRV-15 use updated vaultclient
use updated vault client with the fix setting us-east-1 as default region when region is not provided.
2021-08-25 11:26:12 -07:00
bert-e 977dbab35d Merge branch 'w/7.10/bugfix/CLDSRV-11_StabCherryPickWriteChecks' into tmp/octopus/w/8.2/bugfix/CLDSRV-11_StabCherryPickWriteChecks 2021-08-19 22:58:57 +00:00
bert-e 279520cda7 Merge branch 'bugfix/CLDSRV-11_StabCherryPickWriteChecks' into tmp/octopus/w/7.10/bugfix/CLDSRV-11_StabCherryPickWriteChecks 2021-08-19 22:58:56 +00:00
bert-e e8a3b9fee2
Merge branch 'bugfix/CLDSRV-11_RemoveExtraChecksForOwnership' into tmp/octopus/w/7.10/bugfix/CLDSRV-11_RemoveExtraChecksForOwnership
(cherry picked from commit 0246067b6f)
2021-08-19 15:56:47 -07:00
Ronnie Smith 8de6f0b89a
Merge remote-tracking branch 'origin/bugfix/CLDSRV-11_PredefinedACLForWrite' into w/7.10/bugfix/CLDSRV-11_PredefinedACLForWrite
(cherry picked from commit 64ed86cfd0)
2021-08-19 15:56:35 -07:00
Ronnie Smith cf95928e02
Merge branch 'w/8.2/bugfix/CLDSRV-11_RemoveExtraChecksForOwnership' of github.com:scality/cloudserver into w/8.2/bugfix/CLDSRV-11_RemoveExtraChecksForOwnership 2021-08-19 15:11:48 -07:00
Ronnie Smith 6a5aea842b
Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-11_RemoveExtraChecksForOwnership' into w/8.2/bugfix/CLDSRV-11_RemoveExtraChecksForOwnership 2021-08-19 15:09:58 -07:00
Ronnie Smith 7e41e5d85a
Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-11_RemoveExtraChecksForOwnership' into w/8.2/bugfix/CLDSRV-11_RemoveExtraChecksForOwnership 2021-08-19 14:35:35 -07:00
bert-e 0246067b6f Merge branch 'bugfix/CLDSRV-11_RemoveExtraChecksForOwnership' into tmp/octopus/w/7.10/bugfix/CLDSRV-11_RemoveExtraChecksForOwnership 2021-08-19 21:33:53 +00:00
Ronnie Smith cf742df7ae
bugfix: CLDSRV-11 Revert checks and adjust tests 2021-08-19 14:15:15 -07:00
bert-e 772993b2b8 Merge branches 'w/8.2/bugfix/CLDSRV-12-bucket-policy-resource-security-fix' and 'q/3780/7.10/bugfix/CLDSRV-12-bucket-policy-resource-security-fix' into tmp/octopus/q/8.2 2021-08-19 17:49:47 +00:00
bert-e 3f0e52ae7d Merge branch 'bugfix/CLDSRV-12-bucket-policy-resource-security-fix' into q/7.10.0 2021-08-19 17:49:46 +00:00
bert-e 4c68bf2c4a Merge branches 'w/7.10/bugfix/CLDSRV-12-bucket-policy-resource-security-fix' and 'q/3780/7.10.0/bugfix/CLDSRV-12-bucket-policy-resource-security-fix' into tmp/octopus/q/7.10 2021-08-19 17:49:46 +00:00
naren-scality 384dfd854c Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-12-bucket-policy-resource-security-fix' into w/8.2/bugfix/CLDSRV-12-bucket-policy-resource-security-fix 2021-08-18 23:46:39 -07:00
bert-e 4f96c15475 Merge branch 'bugfix/CLDSRV-12-bucket-policy-resource-security-fix' into tmp/octopus/w/7.10/bugfix/CLDSRV-12-bucket-policy-resource-security-fix 2021-08-19 06:32:19 +00:00
naren-scality 53da779d6c bf: CLDSRV-12 bucket policy resource security fix
Adds resource check for bucket policies. S3 actions are modified to pass
request object for resource validation.
2021-08-18 22:30:26 -07:00
bert-e 24894f6ed3 Merge branches 'w/8.2/bugfix/CLDSRV-11_PredefinedACLForWrite' and 'q/3782/7.10/bugfix/CLDSRV-11_PredefinedACLForWrite' into tmp/octopus/q/8.2 2021-08-17 23:58:26 +00:00
bert-e 24e13b83d4 Merge branch 'bugfix/CLDSRV-11_PredefinedACLForWrite' into q/7.4 2021-08-17 23:58:25 +00:00
bert-e eaf8119780 Merge branches 'w/7.10/bugfix/CLDSRV-11_PredefinedACLForWrite' and 'q/3782/7.4/bugfix/CLDSRV-11_PredefinedACLForWrite' into tmp/octopus/q/7.10 2021-08-17 23:58:25 +00:00
Ronnie Smith 0a441f8726
bugfix: CLDSRV-11 Check for service account on object delete 2021-08-16 15:51:00 -07:00
bert-e 240151e14d Merge branch 'w/7.10/bugfix/CLDSRV-11_PredefinedACLForWrite' into tmp/octopus/w/8.2/bugfix/CLDSRV-11_PredefinedACLForWrite 2021-08-16 22:35:01 +00:00
Ronnie Smith 64ed86cfd0
Merge remote-tracking branch 'origin/bugfix/CLDSRV-11_PredefinedACLForWrite' into w/7.10/bugfix/CLDSRV-11_PredefinedACLForWrite 2021-08-16 15:20:49 -07:00
bert-e 7bcd88dc49 Merge branch 'w/7.10/bugfix/CLDSRV-10-get-acl-with-predefined-groups-stabilization' into tmp/octopus/w/8.2/bugfix/CLDSRV-10-get-acl-with-predefined-groups-stabilization 2021-08-14 00:45:31 +00:00
bert-e 434c5bf1c5 Merge branch 'bugfix/CLDSRV-10-get-acl-with-predefined-groups-stabilization' into tmp/octopus/w/7.10/bugfix/CLDSRV-10-get-acl-with-predefined-groups-stabilization 2021-08-14 00:45:31 +00:00
naren-scality 221033455f bf CLDSRV-10 get acl with predefined groups
Refactor acl util

(cherry picked from commit 7d89a6f250)
2021-08-13 17:40:10 -07:00
naren-scality 15d18ecdd6 bf CLDSRV-10 get acl with predefined groups
Corrected listing ACLs for an object or a bucket when predefined groups
are used.

(cherry picked from commit bca7a57fbf)
2021-08-13 17:39:56 -07:00
bert-e 654edb134a Merge branches 'w/8.2/feature/S3C-4726/log' and 'q/3765/7.10/feature/S3C-4726/log' into tmp/octopus/q/8.2 2021-08-13 20:15:18 +00:00
bert-e b821c83338 Merge branch 'w/7.10/feature/S3C-4726/log' into tmp/octopus/q/7.10 2021-08-13 20:15:18 +00:00
bert-e e6c83347f5 Merge branch 'w/7.10/feature/S3C-4726/log' into tmp/octopus/w/8.2/feature/S3C-4726/log 2021-08-13 19:36:06 +00:00
bert-e f637704d73 Merge branch 'feature/S3C-4726/log' into tmp/octopus/w/7.10/feature/S3C-4726/log 2021-08-13 19:36:05 +00:00
bert-e c066be2dab Merge branches 'w/8.2/bugfix/CLDSRV-10-get-acl-with-predefined-groups' and 'q/3769/7.10/bugfix/CLDSRV-10-get-acl-with-predefined-groups' into tmp/octopus/q/8.2 2021-08-13 16:30:31 +00:00
bert-e 6dcea89a99 Merge branches 'w/7.10/bugfix/CLDSRV-10-get-acl-with-predefined-groups' and 'q/3769/7.4/bugfix/CLDSRV-10-get-acl-with-predefined-groups' into tmp/octopus/q/7.10 2021-08-13 16:30:31 +00:00
bert-e 92defdb05d Merge branch 'bugfix/CLDSRV-10-get-acl-with-predefined-groups' into q/7.4 2021-08-13 16:30:30 +00:00
Ronnie ccb076a0b5
Merge branch 'development/7.4' into bugfix/CLDSRV-11_PredefinedACLForWrite 2021-08-13 09:10:49 -07:00
bert-e 6651519557 Merge branch 'w/7.10/bugfix/CLDSRV-10-get-acl-with-predefined-groups' into tmp/octopus/w/8.2/bugfix/CLDSRV-10-get-acl-with-predefined-groups 2021-08-12 09:24:38 +00:00
bert-e aa15f6c760 Merge branch 'bugfix/CLDSRV-10-get-acl-with-predefined-groups' into tmp/octopus/w/7.10/bugfix/CLDSRV-10-get-acl-with-predefined-groups 2021-08-12 09:24:37 +00:00
naren-scality 7d89a6f250 bf CLDSRV-10 get acl with predefined groups
Refactor acl util
2021-08-12 02:23:52 -07:00
Ronnie Smith 330beed953
bugfix: CLDSRV-11 Write ACL via predefined groups 2021-08-12 01:15:25 -07:00
naren-scality bca7a57fbf bf CLDSRV-10 get acl with predefined groups
Corrected listing ACLs for an object or a bucket when predefined groups
are used.
2021-08-11 21:19:20 -07:00
Nicolas Humbert ed31504ec5 S3C-4726 Log bytes deleted by AbortMultipartUpload calls 2021-08-10 16:02:25 -04:00
Thomas Carmet e8b54a707a Merge remote-tracking branch 'origin/w/7.10/feature/CLDSRV-9-upgrade-werelogs' into w/8.2/feature/CLDSRV-9-upgrade-werelogs 2021-08-06 11:16:53 -07:00
Thomas Carmet 82a9e998d4 Merge remote-tracking branch 'origin/feature/CLDSRV-9-upgrade-werelogs' into w/7.10/feature/CLDSRV-9-upgrade-werelogs 2021-08-05 15:18:51 -07:00
Thomas Carmet d13392183e CLDSRV-9 update werelogs to a properly tagged version 2021-08-05 15:13:01 -07:00
Jonathan Gramain 9372af75a3 Merge remote-tracking branch 'origin/improvement/CLDSRV-8-kmipDeepHealthcheck' into w/8.2/improvement/CLDSRV-8-kmipDeepHealthcheck 2021-08-04 18:19:13 -07:00
Jonathan Gramain 0efd4b8873 improvement: CLDSRV-8 server-side encryption deep healthcheck
Implement a deep healthcheck for server-side encryption in the
Cloudserver deep healthcheck route.

For now, only the KMIP backend has an implementation for the deep
healthcheck, hence all other backends (NAE KMS, file etc.) will just
return success without checking anything.
2021-08-04 11:57:10 -07:00
Ronnie Smith 859a68b158
Merge remote-tracking branch 'origin/bugfix/CLDSRV-7_RemoveHTTPCodeAndMessageFromLogs' into w/8.2/bugfix/CLDSRV-7_RemoveHTTPCodeAndMessageFromLogs 2021-08-03 14:05:48 -07:00
Ronnie Smith 7f75e942b8
bugfix: CLDSRV-7 Remove HTTP code and message from logs
* Update arsenal for fix
2021-08-03 14:00:09 -07:00
bert-e 19ddf81950 Merge branches 'w/8.2/bugfix/CLDSRV-6/403-errors' and 'q/3750/7.10/bugfix/CLDSRV-6/403-errors' into tmp/octopus/q/8.2 2021-08-03 19:56:37 +00:00
bert-e c5ac082304 Merge branch 'bugfix/CLDSRV-6/403-errors' into q/7.10 2021-08-03 19:56:37 +00:00
bert-e 2eeeb22138 Merge branch 'bugfix/CLDSRV-6/403-errors' into tmp/octopus/w/8.2/bugfix/CLDSRV-6/403-errors 2021-08-03 17:08:09 +00:00
Nicolas Humbert 69a24439ba CLDSRV-6 unexpected 403 errors 2021-08-03 13:03:10 -04:00
Ronnie Smith 0739e5c04e
Merge remote-tracking branch 'origin/w/7.10/bugfix/CLDSRV-5_RemoveExtraHeadersFrom304Responses' into w/8.2/bugfix/CLDSRV-5_RemoveExtraHeadersFrom304Responses 2021-07-30 20:59:09 -07:00
Ronnie Smith c63304f62c
Merge remote-tracking branch 'origin/bugfix/CLDSRV-5_RemoveExtraHeadersFrom304Responses' into w/7.10/bugfix/CLDSRV-5_RemoveExtraHeadersFrom304Responses 2021-07-30 20:43:53 -07:00
Ronnie Smith 1a544a02fe
bugfix: CLDSRV-5 Update arsenal for hotfix 2021-07-30 20:40:13 -07:00
Ronnie Smith 6fa84810f5
bugfix: CLDSRV-5 Remove extra headers from 304 responses 2021-07-30 14:52:28 -07:00
bert-e 8b6f5a1ee3 Merge branches 'w/8.2/feature/CLDSRV-2/addLifecycleUID' and 'q/3734/7.10/feature/CLDSRV-2/addLifecycleUID' into tmp/octopus/q/8.2 2021-07-28 22:07:17 +00:00
bert-e 0ee38fd787 Merge branch 'feature/CLDSRV-2/addLifecycleUID' into q/7.10 2021-07-28 22:07:16 +00:00
Alexander Chan 11419e2937 Merge remote-tracking branch 'origin/feature/CLDSRV-2/addLifecycleUID' into w/8.2/feature/CLDSRV-2/addLifecycleUID 2021-07-28 14:16:48 -07:00
Nicolas Humbert 8e9441b923 CLDSRV-4 fix functional tests 2021-07-28 12:38:17 -04:00
Alexander Chan 599dd4138b CLDSRV-2: add bucket UID for lifecycle 2021-07-27 18:20:08 -07:00
bert-e 9cd7ce9278 Merge branch 'bugfix/CLDSRV-4/lifecycle' into tmp/octopus/w/8.2/bugfix/CLDSRV-4/lifecycle 2021-07-27 17:31:49 +00:00
Nicolas Humbert fde9337a98 CLDSRV-4 S3 putBucketLifecycle crashes Cloudserver worker 2021-07-27 13:01:37 -04:00
bert-e f597b7c44b Merge branch 'improvement/S3C-4312-backbeatEncryptionSupport' into q/7.10 2021-07-26 22:25:40 +00:00
bert-e 60c62d8332 Merge branches 'w/8.2/improvement/S3C-4312-backbeatEncryptionSupport' and 'q/3717/7.10/improvement/S3C-4312-backbeatEncryptionSupport' into tmp/octopus/q/8.2 2021-07-26 22:25:40 +00:00
Jonathan Gramain e52a69e6ef Merge remote-tracking branch 'origin/improvement/S3C-4312-backbeatEncryptionSupport' into w/8.2/improvement/S3C-4312-backbeatEncryptionSupport 2021-07-26 14:51:26 -07:00
bert-e 7054da645c Merge branch 'bugfix/CLDSRV-3-pykmipImageBuildFix' into tmp/octopus/w/8.2/bugfix/CLDSRV-3-pykmipImageBuildFix 2021-07-23 20:05:13 +00:00
Jonathan Gramain 744c03d52f bugfix: CLDSRV-3 fix pykmip CI image build
Install the latest version of rust instead of the base package
available in the alpine distribution, in order to successfully build
wheel and cryptography python modules.
2021-07-23 11:59:10 -07:00
Jonathan Gramain a1c70a729d improvement: S3C-4312 backbeat data route encryption support
Add encryption support to backbeat PUT data route
2021-07-21 11:11:46 -07:00
bert-e 82efa99a8f Merge branches 'w/8.2/bugfix/S3C-4533-prevent-suspending-versioning-locked-bucket' and 'q/3715/7.10/bugfix/S3C-4533-prevent-suspending-versioning-locked-bucket' into tmp/octopus/q/8.2 2021-07-21 17:00:56 +00:00
bert-e 07d9d4753c Merge branch 'bugfix/S3C-4533-prevent-suspending-versioning-locked-bucket' into q/7.10 2021-07-21 17:00:56 +00:00
anurag4DSB d19d6b4c18
Merge remote-tracking branch 'origin/bugfix/S3C-4533-prevent-suspending-versioning-locked-bucket' into w/8.2/bugfix/S3C-4533-prevent-suspending-versioning-locked-bucket 2021-07-21 18:25:20 +02:00
Jonathan Gramain 0f9bd86b37 ft(S3C-4508) 8.2: use S3KMS=file
set S3KMS=file in multiple backend and ceph tests to persist
encryption parameters in bucket and avoid exceptions
2021-07-20 18:35:42 -07:00
Jonathan Gramain df9fa85942 ft(S3C-4508) 8.2: update arsenal dep 2021-07-20 17:40:48 -07:00
Ilke ff1e223e4f bf: S3C-4533 prevent suspending version on locked buckets 2021-07-15 12:49:58 -07:00
Ilke ddd29f0cf0 bf: S3C-4533 locked bucket version should not suspended 2021-07-15 12:49:58 -07:00
Taylor McKinnon d8707561fc Merge remote-tracking branch 'origin/feature/S3C-4508_per_object_encryption_headers' into w/8.2/feature/S3C-4508_per_object_encryption_headers 2021-07-07 10:14:38 -07:00
Taylor McKinnon f336541f6a ft(S3C-4508): per object encryption headers support 2021-07-07 10:07:56 -07:00
bert-e a4ff425488 Merge branch 'feature/S3C-4551-tiny-version-ids' into q/7.10 2021-07-06 18:35:18 +00:00
bert-e 1c45364dc6 Merge branches 'w/8.2/feature/S3C-4551-tiny-version-ids' and 'q/3660/7.10/feature/S3C-4551-tiny-version-ids' into tmp/octopus/q/8.2 2021-07-06 18:35:18 +00:00
bert-e 7a6a1ca8d6 Merge branch 'bugfix/S3C-4420/correctErrorResponse' into q/7.10 2021-07-03 00:09:42 +00:00
bert-e 8cf441cc50 Merge branches 'w/8.2/bugfix/S3C-4420/correctErrorResponse' and 'q/3636/7.10/bugfix/S3C-4420/correctErrorResponse' into tmp/octopus/q/8.2 2021-07-03 00:09:42 +00:00
bert-e c3245a4245 Merge branch 'bugfix/S3C-4610_add_missing_utapi_header' into q/7.10 2021-07-02 17:00:35 +00:00
bert-e f7387d729e Merge branches 'w/8.2/bugfix/S3C-4610_add_missing_utapi_header' and 'q/3676/7.10/bugfix/S3C-4610_add_missing_utapi_header' into tmp/octopus/q/8.2 2021-07-02 17:00:35 +00:00
bert-e 86ad4e4c6a Merge branch 'bugfix/S3C-4420/correctErrorResponse' into tmp/octopus/w/8.2/bugfix/S3C-4420/correctErrorResponse 2021-07-01 17:17:34 +00:00
vrancurel cc081768f6 fix aws test 2021-06-30 19:03:26 -07:00
vrancurel c7725254ce add config line 2021-06-30 17:19:44 -07:00
Alexander Chan 6205fa69b5 S3C-4420: fix missing object access response with bucket policy 2021-06-30 15:44:45 -07:00
vrancurel 0ed73e0752 Merge remote-tracking branch 'origin/feature/S3C-4551-tiny-version-ids' into w/8.2/feature/S3C-4551-tiny-version-ids 2021-06-30 09:56:53 -07:00
vrancurel 44a40b807f ft: S3C-4551 tiny version IDs
The tiny mode is selected by default when enabling versioning.
2021-06-30 09:51:42 -07:00
bert-e 974a54bbe7 Merge branches 'w/8.2/feature/S3C-4569_x-amz-bucket-region_header' and 'q/3682/7.10/feature/S3C-4569_x-amz-bucket-region_header' into tmp/octopus/q/8.2 2021-06-30 00:57:52 +00:00
bert-e afbbc8c587 Merge branch 'feature/S3C-4569_x-amz-bucket-region_header' into q/7.10 2021-06-30 00:57:51 +00:00
= ae72ae2c69 Merge remote-tracking branch 'origin/feature/S3C-4569_x-amz-bucket-region_header' into w/8.2/feature/S3C-4569_x-amz-bucket-region_header
merge
2021-06-29 16:43:30 -07:00
artem bakalov 6b19636a1b feature: S3C-4569 Implement x-amz-bucket-region header 2021-06-29 15:22:03 -07:00
bert-e b6d81e2bff Merge branches 'w/8.2/improvement/S3C-4110/backport' and 'q/3659/7.10/improvement/S3C-4110/backport' into tmp/octopus/q/8.2 2021-06-29 17:25:24 +00:00
bert-e 59c65c565e Merge branch 'improvement/S3C-4110/backport' into q/7.10 2021-06-29 17:25:24 +00:00
Nicolas Humbert ad073b2549 Merge remote-tracking branch 'origin/improvement/S3C-4110/backport' into w/8.2/improvement/S3C-4110/backport 2021-06-29 09:26:14 -04:00
Nicolas Humbert aec7068dc2 Merge remote-tracking branch 'origin/development/8.2' into w/8.2/improvement/S3C-4110/backport 2021-06-29 09:21:52 -04:00
Nicolas Humbert f4b6c005ca update arsenal dependency 2021-06-29 09:20:04 -04:00
bert-e 159d7b7fa4 Merge branch 'bugfix/S3C-4610_add_missing_utapi_header' into tmp/octopus/w/8.2/bugfix/S3C-4610_add_missing_utapi_header 2021-06-23 16:56:46 +00:00
Taylor McKinnon 852b9d8d54 bf(S3C-4610): Add missing content-length header to utapi util 2021-06-23 09:56:05 -07:00
bert-e e89e2ff705 Merge branch 'bugfix/S3C-4324-bucketPolicyAllowsNotificationActions' into tmp/octopus/w/8.2/bugfix/S3C-4324-bucketPolicyAllowsNotificationActions 2021-06-11 01:11:12 +00:00
Jonathan Gramain c40736a9d3 bugfix: S3C-4324 shared action map to check bucket policy actions
When checking bucket policy actions, instead of using a hard-coded
local action map which was missing a few items, check against the
shared bucket policy action map in arsenal.
2021-06-10 15:44:32 -07:00
Jonathan Gramain 030c121e20 bugfix: S3C-4324 deny any unsupported action
As a security feature rather than a functional change, all action
types that are not explicitly known are denied instead of allowed (or
subject to further non-action policy controls).
2021-06-10 15:42:12 -07:00
Jonathan Gramain f1fa5c3bce S3C-4324: unit tests for denying unknown action types 2021-06-10 15:42:12 -07:00
Nicolas Humbert 39b412dd64 clean yarn.lock 2021-06-10 14:45:02 -05:00
Nicolas Humbert 77c9f485a1 Merge remote-tracking branch 'origin/improvement/S3C-4110/backport' into w/8.2/improvement/S3C-4110/backport 2021-06-10 14:19:27 -05:00
Nicolas Humbert 1d465e1da3 S3C-4110 test NoncurrentVersionTransitions/Transitons rules NotImplemented 2021-06-09 16:56:25 -05:00
Dora Korpar d94b64574c bf: ZENKO 773 failing lifecycle unit tests 2021-06-09 16:53:45 -05:00
Bennett Buchanan 9d8de712a4 feature: ZENKO-1317 AWS lifecycle compat 2021-06-09 16:53:37 -05:00
bert-e 9f9291b008 Merge branch 'feature/S3C-4382_add_expected_bucket_owner_support' into tmp/octopus/w/8.2/feature/S3C-4382_add_expected_bucket_owner_support 2021-05-19 22:24:26 +00:00
Taylor McKinnon bc81861964 ft(S3C-4382): Add support for x-amz-expected-bucket-owner to put/getBucketEncryption 2021-05-19 15:23:52 -07:00
bert-e e7b27ea528 Merge branch 'w/7.10/bugfix/S3C-2663/correctErrorResponse' into tmp/octopus/w/8.2/bugfix/S3C-2663/correctErrorResponse 2021-05-18 02:03:59 +00:00
Alexander Chan adf195227a Merge remote-tracking branch 'origin/bugfix/S3C-2663/correctErrorResponse' into w/7.10/bugfix/S3C-2663/correctErrorResponse 2021-05-17 19:03:00 -07:00
Alexander Chan 81ba70b515 S3C-2663: fix missing object access response 2021-05-17 18:37:37 -07:00
Taylor McKinnon 0c15f95932 Merge remote-tracking branch 'origin/feature/S3C-3755_add_bucketDeleteEncryption_handler' into w/8.2/feature/S3C-3755_add_bucketDeleteEncryption_handler 2021-05-17 11:24:17 -07:00
Taylor McKinnon 92b4c14da9 ft(S3C-3755): add bucketDeleteEncryption handler 2021-05-17 11:20:52 -07:00
bert-e 1c0ccb51e4 Merge branches 'w/8.2/bugfix/S3C-3061-bucket-acl-using-predefined-groups' and 'q/3584/7.10/bugfix/S3C-3061-bucket-acl-using-predefined-groups' into tmp/octopus/q/8.2 2021-05-15 07:09:47 +00:00
bert-e 0943453ec2 Merge branches 'w/7.10/bugfix/S3C-3061-bucket-acl-using-predefined-groups' and 'q/3584/7.4/bugfix/S3C-3061-bucket-acl-using-predefined-groups' into tmp/octopus/q/7.10 2021-05-15 07:09:47 +00:00
bert-e 3f00a29334 Merge branch 'bugfix/S3C-3061-bucket-acl-using-predefined-groups' into q/7.4 2021-05-15 07:09:46 +00:00
bert-e e14a6e74fb Merge branch 'w/7.10/bugfix/S3C-3061-bucket-acl-using-predefined-groups' into tmp/octopus/w/8.2/bugfix/S3C-3061-bucket-acl-using-predefined-groups 2021-05-15 05:14:23 +00:00
naren-scality d10ddad5e7 bf: S3C-3061 allow bucket acl via predefined groups
corrections are applied to permissionChecks, forwarded from aclChecks
which are used in older branches.
2021-05-14 22:12:00 -07:00
bert-e d0cc88c232 Merge branch 'w/7.10/bugfix/S3C-3061-bucket-acl-using-predefined-groups' into tmp/octopus/w/8.2/bugfix/S3C-3061-bucket-acl-using-predefined-groups 2021-05-15 05:02:19 +00:00
naren-scality 8b2d85fd85 Merge remote-tracking branch 'origin/bugfix/S3C-3061-bucket-acl-using-predefined-groups' into w/7.10/bugfix/S3C-3061-bucket-acl-using-predefined-groups 2021-05-14 21:14:18 -07:00
naren-scality fe99c01012 bf: S3C-3061 revert writes acl fix
reverting writes acl fix as this conflicts with service accounts
behavior, which will be handled separately.
2021-05-14 19:24:28 -07:00
naren-scality 6c35af3e2c bf: S3C-3061 allow bucket acl via predefined groups 2021-05-13 23:49:40 -07:00
Ilke cb500f6e3d Merge remote-tracking branch 'origin/bugfix/S3C-4358-support-versioned-obj-lock-requests' into w/8.2/bugfix/S3C-4358-support-versioned-obj-lock-requests 2021-05-13 17:08:54 -07:00
Ilke dac1d8ea89 bf: S3C-4358 support versioned obj lock requests 2021-05-13 16:46:17 -07:00
bert-e 22417386c1 Merge branches 'w/8.2/improvement/S3C-3727-deprecated-buffer-usage' and 'q/3218/8.1/improvement/S3C-3727-deprecated-buffer-usage' into tmp/octopus/q/8.2 2021-05-12 20:07:45 +00:00
bert-e edffa80691 Merge branches 'w/8.1/improvement/S3C-3727-deprecated-buffer-usage' and 'q/3218/7.10/improvement/S3C-3727-deprecated-buffer-usage' into tmp/octopus/q/8.1 2021-05-12 20:07:45 +00:00
bert-e f1f11b869e Merge branch 'improvement/S3C-3727-deprecated-buffer-usage' into q/7.4 2021-05-12 20:07:44 +00:00
bert-e 6116c56eaf Merge branches 'w/7.10/improvement/S3C-3727-deprecated-buffer-usage' and 'q/3218/7.4/improvement/S3C-3727-deprecated-buffer-usage' into tmp/octopus/q/7.10 2021-05-12 20:07:44 +00:00
bert-e fc089213b6 Merge branches 'w/8.2/dependabot/npm_and_yarn/development/7.4/aws-sdk-2.905.0' and 'q/3603/8.1/dependabot/npm_and_yarn/development/7.4/aws-sdk-2.905.0' into tmp/octopus/q/8.2 2021-05-12 19:56:46 +00:00
bert-e 01e84295c0 Merge branches 'w/8.1/dependabot/npm_and_yarn/development/7.4/aws-sdk-2.905.0' and 'q/3603/7.10/dependabot/npm_and_yarn/development/7.4/aws-sdk-2.905.0' into tmp/octopus/q/8.1 2021-05-12 19:56:45 +00:00
bert-e 775c04d865 Merge branch 'q/3603/7.4/dependabot/npm_and_yarn/development/7.4/aws-sdk-2.905.0' into tmp/normal/q/7.10 2021-05-12 19:56:45 +00:00
bert-e 853706b089 Merge branch 'dependabot/npm_and_yarn/development/7.4/aws-sdk-2.905.0' into q/7.4 2021-05-12 19:56:44 +00:00
bert-e 40e329138e Merge branch 'w/8.1/dependabot/npm_and_yarn/development/7.4/aws-sdk-2.905.0' into tmp/octopus/w/8.2/dependabot/npm_and_yarn/development/7.4/aws-sdk-2.905.0 2021-05-12 18:50:37 +00:00
bert-e 208a1ec456 Merge branch 'w/7.10/dependabot/npm_and_yarn/development/7.4/aws-sdk-2.905.0' into tmp/octopus/w/8.1/dependabot/npm_and_yarn/development/7.4/aws-sdk-2.905.0 2021-05-12 18:50:36 +00:00
bert-e ec19cafe48 Merge branch 'w/8.1/improvement/S3C-3727-deprecated-buffer-usage' into tmp/octopus/w/8.2/improvement/S3C-3727-deprecated-buffer-usage 2021-05-12 18:34:18 +00:00
bert-e 67413869a4 Merge branch 'w/7.10/improvement/S3C-3727-deprecated-buffer-usage' into tmp/octopus/w/8.1/improvement/S3C-3727-deprecated-buffer-usage 2021-05-12 18:34:18 +00:00
bert-e 601149b232 Merge branch 'improvement/S3C-3727-deprecated-buffer-usage' into tmp/octopus/w/7.10/improvement/S3C-3727-deprecated-buffer-usage 2021-05-12 18:34:17 +00:00
Rahul Padigela 7223d33699 Merge remote-tracking branch 'origin/dependabot/npm_and_yarn/development/7.4/aws-sdk-2.905.0' into w/7.10/dependabot/npm_and_yarn/development/7.4/aws-sdk-2.905.0 2021-05-12 11:32:57 -07:00
dependabot[bot] 47ff0b9c84
build(deps): bump aws-sdk from 2.831.0 to 2.905.0
Bumps [aws-sdk](https://github.com/aws/aws-sdk-js) from 2.831.0 to 2.905.0.
- [Release notes](https://github.com/aws/aws-sdk-js/releases)
- [Changelog](https://github.com/aws/aws-sdk-js/blob/master/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-js/compare/v2.831.0...v2.905.0)

Signed-off-by: dependabot[bot] <support@github.com>
2021-05-12 13:02:04 +00:00
bert-e 3ea63d364c Merge branches 'w/8.2/improvement/S3C-4363-remove-node-uuid' and 'q/3585/8.1/improvement/S3C-4363-remove-node-uuid' into tmp/octopus/q/8.2 2021-05-11 17:49:52 +00:00
bert-e e611bc5c27 Merge branch 'improvement/S3C-4363-remove-node-uuid' into q/7.4 2021-05-11 17:49:51 +00:00
bert-e d963b38f0e Merge branches 'w/8.1/improvement/S3C-4363-remove-node-uuid' and 'q/3585/7.10/improvement/S3C-4363-remove-node-uuid' into tmp/octopus/q/8.1 2021-05-11 17:49:51 +00:00
bert-e 76fda51fdb Merge branches 'w/7.10/improvement/S3C-4363-remove-node-uuid' and 'q/3585/7.4/improvement/S3C-4363-remove-node-uuid' into tmp/octopus/q/7.10 2021-05-11 17:49:51 +00:00
Rahul Padigela 37f0b90655 improvement: S3C-3727 update deprecated Buffer usage 2021-05-10 19:12:09 -07:00
bert-e bc60e8b08b Merge branches 'w/8.2/bugfix/S3C-3778-deduplicateCompleteMPU' and 'q/3577/8.1/bugfix/S3C-3778-deduplicateCompleteMPU' into tmp/octopus/q/8.2 2021-05-11 01:58:08 +00:00
bert-e 142fe7cc72 Merge branch 'bugfix/S3C-3778-deduplicateCompleteMPU' into q/7.4 2021-05-11 01:58:07 +00:00
bert-e a2fb0215fb Merge branches 'w/8.1/bugfix/S3C-3778-deduplicateCompleteMPU' and 'q/3577/7.10/bugfix/S3C-3778-deduplicateCompleteMPU' into tmp/octopus/q/8.1 2021-05-11 01:58:07 +00:00
bert-e 3ec7f93fb0 Merge branches 'w/7.10/bugfix/S3C-3778-deduplicateCompleteMPU' and 'q/3577/7.4/bugfix/S3C-3778-deduplicateCompleteMPU' into tmp/octopus/q/7.10 2021-05-11 01:58:07 +00:00
bert-e 1b332a273c Merge branch 'w/8.1/improvement/S3C-4363-remove-node-uuid' into tmp/octopus/w/8.2/improvement/S3C-4363-remove-node-uuid 2021-05-11 00:33:31 +00:00
Rahul Padigela 6f56a6d215 improvement: S3C-4363 update uuid references 2021-05-10 17:32:57 -07:00
bert-e ec40c8a520 Merge branch 'w/8.1/improvement/S3C-4363-remove-node-uuid' into tmp/octopus/w/8.2/improvement/S3C-4363-remove-node-uuid 2021-05-11 00:07:25 +00:00
Rahul Padigela c9444fdbc4 improvement: S3C-4363 update uuid references 2021-05-10 17:06:12 -07:00
bert-e e534dc15c3 Merge branch 'w/8.1/improvement/dependabot-builds' into tmp/octopus/w/8.2/improvement/dependabot-builds 2021-05-10 23:34:27 +00:00
bert-e fd7fe45046 Merge branch 'w/7.10/improvement/dependabot-builds' into tmp/octopus/w/8.1/improvement/dependabot-builds 2021-05-10 23:34:27 +00:00
bert-e 447dafcc8b Merge branch 'improvement/dependabot-builds' into tmp/octopus/w/7.10/improvement/dependabot-builds 2021-05-10 23:34:27 +00:00
Rahul Padigela c90f93bbd1 improvement: increase yaml len default to 120 chars 2021-05-10 16:33:52 -07:00
bert-e d008ccc3ec Merge branch 'w/8.1/improvement/dependabot-builds' into tmp/octopus/w/8.2/improvement/dependabot-builds 2021-05-10 22:21:05 +00:00
Rahul Padigela 9a69b5c00d Merge remote-tracking branch 'origin/w/7.10/improvement/dependabot-builds' into w/8.1/improvement/dependabot-builds 2021-05-10 15:03:11 -07:00
bert-e 3edfbcbe30 Merge branch 'improvement/dependabot-builds' into tmp/octopus/w/7.10/improvement/dependabot-builds 2021-05-10 22:00:28 +00:00
bert-e c313d201d4 Merge branch 'w/8.1/bugfix/S3C-3778-deduplicateCompleteMPU' into tmp/octopus/w/8.2/bugfix/S3C-3778-deduplicateCompleteMPU 2021-05-10 20:56:15 +00:00
Jonathan Gramain f03f6bf2c9 S3C-3778 8.1+ fix: disable sanity check for external backend
For external backends like AWS, the sanity check fails because it does
not support external locations: do not change the existing behavior
for external backends.
2021-05-10 13:55:51 -07:00
bert-e a7ab9ef522 Merge branch 'w/8.1/bugfix/S3C-3778-deduplicateCompleteMPU' into tmp/octopus/w/8.2/bugfix/S3C-3778-deduplicateCompleteMPU 2021-05-10 18:32:44 +00:00
Jonathan Gramain b98898dfe1 Merge remote-tracking branch 'origin/w/7.10/bugfix/S3C-3778-deduplicateCompleteMPU' into w/8.1/bugfix/S3C-3778-deduplicateCompleteMPU 2021-05-10 11:24:20 -07:00
bert-e 9b8bb2c49e Merge branch 'bugfix/S3C-3778-deduplicateCompleteMPU' into tmp/octopus/w/7.10/bugfix/S3C-3778-deduplicateCompleteMPU 2021-05-10 18:22:29 +00:00
Jonathan Gramain 60d55c0a6c S3C-3778: new test for non-versioned complete MPU replay
Add a unit test to make sure we skip the data deletion on
non-versioned buckets when a complete MPU is replayed. It was already
the case but it should help to catch any regression.
2021-05-10 11:16:27 -07:00
Jonathan Gramain 3464487c2b bugfix: S3C-3778 prevent MPU with duplicate data keys
Sometimes, complete MPU request may be replayed due to a previous
failure leaving the original MPU metadata untouched but with a valid
completed object as well. This could lead to a new version of the
object being created, pointing to the same data keys than the previous
version, created from the same MPU that failed from the client
perspective.

The fix consists of checking whether we are trying to update the
metadata with the same data location keys than the existing ones, and
skip the metadata update and data location deletions if this is the
case.

It's a generalization of the previous fix S3C-1959 that consisted in
skipping data deletions if they were already in the object
metadata. It worked on non-versioned buckets because there, the keys
to delete are the original master key locations because the master key
gets overwritten, but it did not work on versioned buckets since there
is no key to delete as we are creating a new version.
2021-05-10 11:16:27 -07:00
Jonathan Gramain 2b1f87511b S3C-3778 optim: do not call getObjectMD(mpuOverviewKey) twice
Since metadataValidateMultipart already retrieves the metadata of the
overview key, we can piggy-back it and avoid retrieving it twice
during the completeMultipartUpload processing.
2021-05-10 11:15:11 -07:00
Jonathan Gramain 826d2fe2e8 bugfix: S3C-3778 unit test showing a duplicate version
Create a unit test showing that a duplicate version is created after a
complete-multipart-upload fails and is retried. Having more than one
version in this case creates shared data keys that cause a risk of
data loss once one of the versions is deleted.
2021-05-10 11:15:11 -07:00
Rahul Padigela 983350ee61 improvement: build dependabot pull requests 2021-05-10 08:13:11 -07:00
bert-e 1624a1488b Merge branch 'w/8.1/improvement/S3C-4363-remove-node-uuid' into tmp/octopus/w/8.2/improvement/S3C-4363-remove-node-uuid 2021-05-10 15:12:43 +00:00
Rahul Padigela 6a94c7bf73 Merge remote-tracking branch 'origin/w/7.10/improvement/S3C-4363-remove-node-uuid' into w/8.1/improvement/S3C-4363-remove-node-uuid 2021-05-10 08:07:09 -07:00
Rahul Padigela 8c7f4941f7 Merge remote-tracking branch 'origin/improvement/S3C-4363-remove-node-uuid' into w/7.10/improvement/S3C-4363-remove-node-uuid 2021-05-10 07:57:54 -07:00
Rahul Padigela 68cbe00b14 improvement: S3C-4363 remove node-uuid module 2021-05-10 07:44:48 -07:00
Rahul Padigela 40af650ad9 improvement: S3C-4363 upgrade uuid module 2021-05-10 07:44:44 -07:00
Rahul Padigela 9109cd1610 Set theme jekyll-theme-modernist 2021-05-07 17:44:39 -07:00
bert-e bc3dfbd315 Merge branches 'w/8.1/improvement/S3C-4336_move_ModelVersion' and 'q/3573/7.10/improvement/S3C-4336_move_ModelVersion' into tmp/octopus/q/8.1 2021-05-06 00:05:55 +00:00
bert-e 34fa2d6448 Merge branches 'w/8.2/improvement/S3C-4336_move_ModelVersion' and 'q/3573/8.1/improvement/S3C-4336_move_ModelVersion' into tmp/octopus/q/8.2 2021-05-06 00:05:55 +00:00
bert-e 74938748df Merge branch 'w/7.10/improvement/S3C-4336_move_ModelVersion' into tmp/octopus/q/7.10 2021-05-06 00:05:54 +00:00
bert-e 054c92bc8a Merge branch 'w/8.1/improvement/S3C-4336_move_ModelVersion' into tmp/octopus/w/8.2/improvement/S3C-4336_move_ModelVersion 2021-05-05 18:31:54 +00:00
Taylor McKinnon ad1a299b9f Merge remote-tracking branch 'origin/w/7.10/improvement/S3C-4336_move_ModelVersion' into w/8.1/improvement/S3C-4336_move_ModelVersion 2021-05-05 11:31:39 -07:00
Taylor McKinnon 811750ede7 Merge remote-tracking branch 'origin/improvement/S3C-4336_move_ModelVersion' into w/7.10/improvement/S3C-4336_move_ModelVersion 2021-05-05 11:30:47 -07:00
Taylor McKinnon 9bc8b0e5dc impr(S3C-4336): Remove ModelVersion.md 2021-05-05 11:28:59 -07:00
bert-e e9adc508b3 Merge branch 'w/8.1/feature/S3C-3749_add_PutBucketEncyption_handler' into tmp/octopus/w/8.2/feature/S3C-3749_add_PutBucketEncyption_handler 2021-05-04 23:47:28 +00:00
Taylor McKinnon 33a79b88a7 Merge remote-tracking branch 'origin/feature/S3C-3749_add_PutBucketEncyption_handler' into w/8.1/feature/S3C-3749_add_PutBucketEncyption_handler 2021-05-04 16:46:54 -07:00
Taylor McKinnon ccb04ced4f ft(S3C-3749): Add PutBucketEncryptionHandler 2021-05-04 16:34:42 -07:00
Rached Ben Mustapha 1359292388 feature: bump version to 8.2.12 2021-04-30 16:36:23 -07:00
Rached Ben Mustapha c27c4ef3e7 bugfix: report currently running overlay version 2021-04-29 16:59:11 -07:00
bert-e b5b4743822 Merge branch 'w/8.1/feature/S3C-3752_add_GetBucketEncryption_api' into tmp/octopus/w/8.2/feature/S3C-3752_add_GetBucketEncryption_api 2021-04-27 21:55:04 +00:00
Taylor McKinnon ba3a6fe19f Merge remote-tracking branch 'origin/feature/S3C-3752_add_GetBucketEncryption_api' into w/8.1/feature/S3C-3752_add_GetBucketEncryption_api 2021-04-27 14:52:58 -07:00
Alexander Chan 8ec1cf01c7 bump cloudserver version 2021-04-24 15:40:45 -07:00
Alexander Chan a05b89f707 ZENKO-3368: add auth chain backend 2021-04-24 15:40:21 -07:00
Taylor McKinnon 1c0bc8ab52 ft(S3C-3752): Add GetBucketEncryption handler 2021-04-23 10:11:06 -07:00
bert-e 3f0d46f706 Merge branch 'documentation/add-go-sample' into q/8.2 2021-04-15 03:32:15 +00:00
bert-e f32f353244 Merge branch 'bugfix/S3C-4245_enforce_bypassgovernancemode_policy' into tmp/octopus/w/8.1/bugfix/S3C-4245_enforce_bypassgovernancemode_policy 2021-04-14 00:37:08 +00:00
bert-e 58b070dec4 Merge branch 'w/8.1/bugfix/S3C-4245_enforce_bypassgovernancemode_policy' into tmp/octopus/w/8.2/bugfix/S3C-4245_enforce_bypassgovernancemode_policy 2021-04-14 00:37:08 +00:00
Taylor McKinnon fcd2aa8218 bf(S3C-4245): Enforce x-amz-bypass-governance-retention header and s:BypassGovernanceRetention user policy 2021-04-13 17:36:36 -07:00
bert-e 6adf4eb265 Merge branch 'w/8.2/bugfix/S3C-4245_compliance_mode_fixes' into tmp/octopus/q/8.2 2021-04-12 19:14:24 +00:00
Ronnie Smith d934090322
Merge branch 'development/8.2' into documentation/add-go-sample 2021-04-12 10:30:48 -07:00
Ronnie Smith 45fa71e6e5
feature S3C-4213: Add docs for using vault 2021-04-09 16:23:59 -07:00
bert-e 18a68a1778 Merge branch 'bugfix/S3C-4245_compliance_mode_fixes' into tmp/octopus/w/8.1/bugfix/S3C-4245_compliance_mode_fixes 2021-04-09 22:06:22 +00:00
bert-e 6ce0829832 Merge branch 'w/8.1/bugfix/S3C-4245_compliance_mode_fixes' into tmp/octopus/w/8.2/bugfix/S3C-4245_compliance_mode_fixes 2021-04-09 22:06:22 +00:00
Taylor McKinnon 11ddeffe40 bf(S3C-4245): Don't allow COMPLIANCE mode to be shortened or disabled 2021-04-09 15:05:50 -07:00
bert-e 6e95095943 Merge branch 'w/8.1/bugfix/S3C-4276_update_kmip_certs' into tmp/octopus/w/8.2/bugfix/S3C-4276_update_kmip_certs 2021-04-09 15:56:50 +00:00
bert-e e7b12e2d1b Merge branch 'bugfix/S3C-4276_update_kmip_certs' into tmp/octopus/w/8.1/bugfix/S3C-4276_update_kmip_certs 2021-04-09 15:56:50 +00:00
Taylor McKinnon 10f8e5a125 bf(S3C-4276): Update KMIP test certificates 2021-04-09 08:56:04 -07:00
Ronnie Smith 75b22cbe35
docs: Add go sample to clients 2021-04-06 16:06:18 -07:00
bert-e 9dfd60b629 Merge branch 'feature/ZENKO-3356-release-8.2.10' into q/8.2 2021-04-02 22:10:17 +00:00
Rached Ben Mustapha 029703d5d7 feature: bump version to 8.2.10 2021-04-02 15:05:04 -07:00
bert-e e83c8ec255 Merge branch 'w/8.2/bugfix/S3C-4263-limitConcurrentOpsInTest' into tmp/octopus/q/8.2 2021-04-02 21:50:56 +00:00
bert-e 7b44b994d3 Merge branch 'w/7.10/bugfix/S3C-4263-limitConcurrentOpsInTest' into tmp/octopus/w/8.1/bugfix/S3C-4263-limitConcurrentOpsInTest 2021-04-02 00:46:14 +00:00
bert-e cdc1d8879b Merge branch 'w/8.1/bugfix/S3C-4263-limitConcurrentOpsInTest' into tmp/octopus/w/8.2/bugfix/S3C-4263-limitConcurrentOpsInTest 2021-04-02 00:46:14 +00:00
bert-e a0835a8a06 Merge branch 'bugfix/S3C-4263-limitConcurrentOpsInTest' into tmp/octopus/w/7.9/bugfix/S3C-4263-limitConcurrentOpsInTest 2021-04-02 00:46:13 +00:00
Jonathan Gramain b7c17cd19b bugfix: S3C-4263 limit concurrent operations during versioning tests
Add a limit of 10x10=100 concurrent operations during versioning tests
ingestion, this should help with integration tests having timeouts due
to too many operations happening at the same time.
2021-04-01 17:21:28 -07:00
Rached Ben Mustapha 2c902b188a bf: upgrade arsenal for hdclient fix 2021-03-26 23:58:34 +00:00
Alexander Chan 916cb62e60 update utapi dependency 2021-03-18 15:44:30 -07:00
bert-e 985c811f74 Merge branch 'w/8.1/bugfix/S3C-4166_putDeleteMarkerObject_increments_numberOfObjects' into tmp/octopus/w/8.2/bugfix/S3C-4166_putDeleteMarkerObject_increments_numberOfObjects 2021-03-17 20:50:58 +00:00
bert-e f8591690cb Merge branch 'w/7.10/bugfix/S3C-4166_putDeleteMarkerObject_increments_numberOfObjects' into tmp/octopus/w/8.1/bugfix/S3C-4166_putDeleteMarkerObject_increments_numberOfObjects 2021-03-17 20:50:57 +00:00
bert-e 417c91db03 Merge branch 'bugfix/S3C-4166_putDeleteMarkerObject_increments_numberOfObjects' into tmp/octopus/w/7.9/bugfix/S3C-4166_putDeleteMarkerObject_increments_numberOfObjects 2021-03-17 20:50:56 +00:00
Taylor McKinnon 90c32b4b6d bf(S3C-4166): putDeleteMarkerObject should increment numberOfObjects 2021-03-17 13:50:18 -07:00
Alexander Chan 83ed02ad08 bugfix: ZENKO-3294 use correctly formatted utapi local cache sentinel param 2021-03-17 08:20:29 -07:00
naren-scality c811b7d533 Merge remote-tracking branch 'origin/bugfix/zenko-3284-mongo-tests-failure-corrections' into w/8.2/bugfix/zenko-3284-mongo-tests-failure-corrections 2021-03-17 07:40:24 -07:00
naren-scality 112a08ee92 bf zenko-3284 mongo ft tests failure corrections 2021-03-16 14:15:24 -07:00
bert-e c023939414 Merge branch 'w/8.1/bugfix/S3C-4138_fix_aborted_mpu_metrics' into tmp/octopus/w/8.2/bugfix/S3C-4138_fix_aborted_mpu_metrics 2021-03-16 03:42:52 +00:00
bert-e 337714fe21 Merge branch 'w/7.10/bugfix/S3C-4138_fix_aborted_mpu_metrics' into tmp/octopus/w/8.1/bugfix/S3C-4138_fix_aborted_mpu_metrics 2021-03-16 03:42:51 +00:00
bert-e d06194d316 Merge branch 'bugfix/S3C-4138_fix_aborted_mpu_metrics' into tmp/octopus/w/7.9/bugfix/S3C-4138_fix_aborted_mpu_metrics 2021-03-16 03:42:50 +00:00
Taylor McKinnon 16916ca09b fix expected metrics 2021-03-15 15:21:42 -07:00
Taylor McKinnon 569b924485 bf(S3C-4138): Decrement storage counter when abortMultipartUpload is pushed 2021-03-15 11:58:56 -07:00
bert-e f2527c2816 Merge branch 'w/8.1/bugfix/S3C-4134_prevent_nan_sizedelta' into tmp/octopus/w/8.2/bugfix/S3C-4134_prevent_nan_sizedelta 2021-03-15 16:30:13 +00:00
bert-e 8ccb3013e7 Merge branch 'w/7.10/bugfix/S3C-4134_prevent_nan_sizedelta' into tmp/octopus/w/8.1/bugfix/S3C-4134_prevent_nan_sizedelta 2021-03-15 16:30:12 +00:00
bert-e a0c584cbd6 Merge branch 'bugfix/S3C-4134_prevent_nan_sizedelta' into tmp/octopus/w/7.9/bugfix/S3C-4134_prevent_nan_sizedelta 2021-03-15 16:30:11 +00:00
Taylor McKinnon 72f038022e bf(S3C-4134): Prevent NaN sizeDelta from completeMultipartUpload 2021-03-15 09:29:36 -07:00
Rahul Padigela 9ff6364b2e Merge remote-tracking branch 'origin/w/8.1/bugfix/S3C-4055-cloudserver-metric-corrections' into w/8.2/bugfix/S3C-4055-cloudserver-metric-corrections 2021-03-09 21:34:32 -08:00
Rahul Padigela d812970cc8 bugfix: add utapi v2 test cmd 2021-03-09 21:27:06 -08:00
Rahul Padigela a64ad91e61 Merge remote-tracking branch 'origin/w/8.1/bugfix/S3C-4055-cloudserver-metric-corrections' into w/8.2/bugfix/S3C-4055-cloudserver-metric-corrections 2021-03-09 21:20:24 -08:00
Rahul Padigela 651abc372b Merge remote-tracking branch 'origin/w/7.10/bugfix/S3C-4055-cloudserver-metric-corrections' into w/8.1/bugfix/S3C-4055-cloudserver-metric-corrections 2021-03-09 21:14:05 -08:00
Rahul Padigela 1af9c05329 Merge remote-tracking branch 'origin/bugfix/S3C-4055-cloudserver-metric-corrections' into w/7.9/bugfix/S3C-4055-cloudserver-metric-corrections 2021-03-09 20:43:10 -08:00
naren-scality 150ee49d00 bf: S3C-4055 cloudserver metrics computation correction
metrics like incomingBytes, sizeDelta were not calculated correctly for actions like getObject and it is corrected now.
2021-03-08 11:50:20 -08:00
Alexander Chan 4a687feaaf ZENKO-3230: fix Orbit instance registration request
fixes issue where the registration response is stored as a string; add
`json` option to registration request in order parse response body as an
object.
2021-03-08 08:54:46 -08:00
Taylor McKinnon 7cd6de8178 Merge remote-tracking branch 'origin/w/8.1/improvement/S3C-3971_reduce_event_footprint' into w/8.2/improvement/S3C-3971_reduce_event_footprint 2021-02-11 13:15:39 -08:00
Taylor McKinnon 09d21ecfdf Merge remote-tracking branch 'origin/w/7.10/improvement/S3C-3971_reduce_event_footprint' into w/8.1/improvement/S3C-3971_reduce_event_footprint 2021-02-11 13:12:18 -08:00
Taylor McKinnon a1f59dbc8e Merge remote-tracking branch 'origin/improvement/S3C-3971_reduce_event_footprint' into w/7.9/improvement/S3C-3971_reduce_event_footprint 2021-02-11 13:06:59 -08:00
Taylor McKinnon 38a52ed62c impr(S3C-3971): Filter object and versionId from pushed metrics 2021-02-11 13:04:32 -08:00
bert-e 2f26577680 Merge branch 'bugfix/ZENKO-2153-fix-tests' into tmp/octopus/w/8.2/bugfix/ZENKO-2153-fix-tests 2021-02-11 10:04:46 +00:00
alexandre merle 1845d9e06d tests: ZENKO-2153: skip retention / object lock / legal hold for ceph 2021-02-11 10:59:10 +01:00
alexandre merle 5d0dc99b80 tests: ZENKO-2153: fix versioning state change if replication 2021-02-11 10:59:10 +01:00
alexandre merle 3619c4c296 bugfix: ZENKO-2153: init metadata wrapper for tests 2021-02-11 10:59:10 +01:00
alexandre merle 237913e604 bugfix: ZENKO-2153: use MPU testing for multiple backend 2021-02-11 10:59:10 +01:00
alexandre merle 9ef5a12af5 bugfix: ZENKO-3243: sanity check on complete upload 2021-02-11 10:59:10 +01:00
alexandre merle 230a4cc2e7 bugfix: ZENKO-3241: considering range start 0 as valid 2021-02-11 10:59:10 +01:00
alexandre merle 798191ac8d tests: ZENKO-2153: activate aws debug logging 2021-02-11 10:59:10 +01:00
alexandre merle b094948064 bugfix: ZENKO-2352: fix aws tests 2021-02-11 10:59:10 +01:00
alexandre merle 7fca140ab9 improv: ZENKO-2153: use agentkeepalive for global agent 2021-02-11 10:59:06 +01:00
alexandre merle a2fb48c9da bugfix: ZENKO-2153: aws sdk upgrade 2021-02-11 10:57:57 +01:00
alexandre merle 01bd8783e9 bugfix: ZENKO-2153: Remove .only() 2021-02-11 10:57:57 +01:00
alexandre merle 3ebb55a59c Merge remote-tracking branch 'origin/w/8.1/bugfix/ZENKO-2153-update-arsenal' into w/8.2/bugfix/ZENKO-2153-update-arsenal 2021-02-11 09:32:52 +01:00
alexandre merle 8d7f242998 Merge remote-tracking branch 'origin/w/7.10/bugfix/ZENKO-2153-update-arsenal' into w/8.1/bugfix/ZENKO-2153-update-arsenal 2021-02-11 09:31:27 +01:00
alexandre merle f6f17ecc44 Merge remote-tracking branch 'origin/bugfix/ZENKO-2153-update-arsenal' into w/7.9/bugfix/ZENKO-2153-update-arsenal 2021-02-11 09:28:22 +01:00
alexandre merle 3dffdce776 ZENKO-2153: update arsenal 2021-02-11 09:19:08 +01:00
bert-e 89dde58a58 Merge branch 'w/8.1/improvement/S3C-3974-remove-s3blaster' into tmp/octopus/w/8.2/improvement/S3C-3974-remove-s3blaster 2021-02-11 03:10:48 +00:00
Rahul Padigela 61fa056e70 Merge remote-tracking branch 'origin/w/7.10/improvement/S3C-3974-remove-s3blaster' into w/8.1/improvement/S3C-3974-remove-s3blaster 2021-02-10 18:04:37 -08:00
Rahul Padigela 0d234f1b3b Merge remote-tracking branch 'origin/improvement/S3C-3974-remove-s3blaster' into w/7.9/improvement/S3C-3974-remove-s3blaster 2021-02-10 17:05:07 -08:00
Rahul Padigela 1503750a3f improvement: S3C-3974 remove s3blaster 2021-02-10 13:26:17 -08:00
bert-e 69aabd61b7 Merge branch 'w/8.1/bugfix/S3C-2642-multi-object-delete-xml-decl' into tmp/octopus/w/8.2/bugfix/S3C-2642-multi-object-delete-xml-decl 2021-02-06 08:56:58 +00:00
bert-e 20c7c2c564 Merge branch 'bugfix/S3C-2642-multi-object-delete-xml-decl' into tmp/octopus/w/7.9/bugfix/S3C-2642-multi-object-delete-xml-decl 2021-02-06 08:56:57 +00:00
bert-e 96edc0ff9b Merge branch 'w/7.9/bugfix/S3C-2642-multi-object-delete-xml-decl' into tmp/octopus/w/8.1/bugfix/S3C-2642-multi-object-delete-xml-decl 2021-02-06 08:56:57 +00:00
alexandre merle 4a0fcb4619 S3C-2642: multi object delete xml declaration
Include xml declaration in multi object delete
2021-02-06 09:56:12 +01:00
alexandre merle 9b571ffd4a Merge remote-tracking branch 'origin/w/8.1/bugfix/S3C-3904-better-s3-action-logs' into w/8.2/bugfix/S3C-3904-better-s3-action-logs 2021-02-05 21:58:13 +01:00
alexandre merle c5cc941609 Merge remote-tracking branch 'origin/w/7.9/bugfix/S3C-3904-better-s3-action-logs' into w/8.1/bugfix/S3C-3904-better-s3-action-logs 2021-02-05 21:57:22 +01:00
alexandre merle d0eb21193c Merge remote-tracking branch 'origin/bugfix/S3C-3904-better-s3-action-logs' into w/7.9/bugfix/S3C-3904-better-s3-action-logs 2021-02-05 21:55:16 +01:00
alexandre merle 73356e1819 update arsenal version 2021-02-05 21:53:02 +01:00
alexandre merle a3b84882aa check for action without log
add a safety check to ensure no api calls is not mapped
to a monitoring action
2021-02-05 21:53:02 +01:00
alexandre merle 347f0e7b8d bugfix: S3C-3904: better s3 action logs
Improve s3 actions logs by using an override
from arsenal
2021-02-05 18:54:36 +01:00
Thomas Carmet e0e3cbc2a5 ZENKO-3106 updating package.json version to 8.2.8 2021-02-04 17:42:32 -08:00
Thomas Carmet 92cf756f78 Merge remote-tracking branch 'origin/feature/ZENKO-3106-add-release-stage' into w/8.2/feature/ZENKO-3106-add-release-stage 2021-02-04 17:42:25 -08:00
Thomas Carmet 9bc11309eb ZENKO-3106 updating package.json to the next version 2021-02-04 17:41:46 -08:00
Thomas Carmet d8dd11101b ZENKO-3106 setting up release stage for cloudserver
Co-authored-by: William Abernathy <william.abernathy@scality.com>
2021-02-04 17:41:46 -08:00
bert-e 890ec70e01 Merge branch 'w/8.1/bugfix/S3C-3860-fix-corner-case-multiple-backend' into tmp/octopus/w/8.2/bugfix/S3C-3860-fix-corner-case-multiple-backend 2021-02-02 02:08:09 +00:00
alexandre merle 5df7ecf776
Merge remote-tracking branch 'origin/w/7.9/bugfix/S3C-3860-fix-corner-case-multiple-backend' into w/8.1/bugfix/S3C-3860-fix-corner-case-multiple-backend 2021-02-02 03:07:19 +01:00
bert-e 9757343b69 Merge branch 'bugfix/S3C-3860-fix-corner-case-multiple-backend' into tmp/octopus/w/7.9/bugfix/S3C-3860-fix-corner-case-multiple-backend 2021-02-02 01:48:56 +00:00
alexandre merle b466ba9490 bgufix: S3C-3960: fix corner case multiple backend
in case of multiple backend, we were writing twice
'continue' in the socket, leading to econnreset from the client
side
2021-02-02 02:43:02 +01:00
bert-e e6fc52f0a7 Merge branch 'w/8.1/feature/S3C-2767-bp-error-code' into tmp/octopus/w/8.2/feature/S3C-2767-bp-error-code 2021-01-28 18:20:46 +00:00
bert-e 75f9ae2ab2 Merge branch 'feature/S3C-2767-bp-error-code' into tmp/octopus/w/8.1/feature/S3C-2767-bp-error-code 2021-01-28 18:20:45 +00:00
Dora Korpar 63cfebd818 ft: S3C-2767 bucket policy not implemented error 2021-01-28 09:59:16 -08:00
alexandre merle 2300ef77ed Merge remote-tracking branch 'origin/w/8.1/improvement/S3C-3897-upgrade-aws-sdk' into w/8.2/improvement/S3C-3897-upgrade-aws-sdk 2021-01-27 20:57:50 +01:00
alexandre merle 37cedc6ade
Merge remote-tracking branch 'origin/w/7.9/improvement/S3C-3897-upgrade-aws-sdk' into w/8.1/improvement/S3C-3897-upgrade-aws-sdk 2021-01-27 20:53:51 +01:00
alexandre merle 3dd6485309
Merge remote-tracking branch 'origin/improvement/S3C-3897-upgrade-aws-sdk' into w/7.9/improvement/S3C-3897-upgrade-aws-sdk 2021-01-27 19:17:17 +01:00
alexandre merle d95524b480
improvement: S3C-3897: upgrade aws sdk to 2.831.0
Upgrade aws-sdk to 2.831.0

Fixes https://scality.atlassian.net/browse/S3C-3897
2021-01-27 15:37:49 +01:00
bert-e 862911f745 Merge branch 'w/8.1/bugfix/S3C-2201-econnreset-rest-client-keep-alive' into tmp/octopus/w/8.2/bugfix/S3C-2201-econnreset-rest-client-keep-alive 2021-01-26 10:50:28 +00:00
alexandre merle b553a2a84b
Merge remote-tracking branch 'origin/w/7.9/bugfix/S3C-2201-econnreset-rest-client-keep-alive' into w/8.1/bugfix/S3C-2201-econnreset-rest-client-keep-alive 2021-01-26 11:49:38 +01:00
alexandre merle 6d3a704a37
Merge remote-tracking branch 'origin/bugfix/S3C-2201-econnreset-rest-client-keep-alive' into w/7.9/bugfix/S3C-2201-econnreset-rest-client-keep-alive 2021-01-26 11:42:30 +01:00
alexandre merle 2e52ad2072
update arsenal version 2021-01-26 11:35:35 +01:00
bert-e 36feeac832 Merge branches 'w/8.2/bugfix/S3C-3860-put-object-continue-too-early' and 'q/3282/8.1/bugfix/S3C-3860-put-object-continue-too-early' into tmp/octopus/q/8.2 2021-01-26 07:17:54 +00:00
bert-e 223739e2c9 Merge branches 'w/8.1/bugfix/S3C-3860-put-object-continue-too-early' and 'q/3282/7.9/bugfix/S3C-3860-put-object-continue-too-early' into tmp/octopus/q/8.1 2021-01-26 07:17:54 +00:00
bert-e c4a16060ce Merge branches 'w/7.9/bugfix/S3C-3860-put-object-continue-too-early' and 'q/3282/7.4/bugfix/S3C-3860-put-object-continue-too-early' into tmp/octopus/q/7.9 2021-01-26 07:17:53 +00:00
bert-e f33a1559c7 Merge branch 'bugfix/S3C-3860-put-object-continue-too-early' into q/7.4 2021-01-26 07:17:53 +00:00
bert-e c79ffc5216 Merge branch 'w/8.1/bugfix/S3C-3860-put-object-continue-too-early' into tmp/octopus/w/8.2/bugfix/S3C-3860-put-object-continue-too-early 2021-01-25 20:07:44 +00:00
alexandre merle 23e1235fb0
Merge remote-tracking branch 'origin/w/7.9/bugfix/S3C-3860-put-object-continue-too-early' into w/8.1/bugfix/S3C-3860-put-object-continue-too-early 2021-01-25 21:07:16 +01:00
alexandre merle f1a6e2b1d3
Merge remote-tracking branch 'origin/bugfix/S3C-3860-put-object-continue-too-early' into w/7.9/bugfix/S3C-3860-put-object-continue-too-early 2021-01-25 21:01:47 +01:00
alexandre merle 6d37b5d345
bugfix: S3C-3860: object put write continue too early
Continue message was send before checking auths, which could lead
to api calls to send the object data, while cloudserver will
check auth and send back an error, closing the connection.
All this was leading to an EPIPE error from the client
perspective.

Fixes https://scality.atlassian.net/browse/S3C-3860
Fixes https://scality.atlassian.net/browse/S3C-3063
2021-01-25 20:50:54 +01:00
bert-e 2e20d665ae Merge branches 'w/8.2/bugfix/S3C-3425-bump-arsenal-version' and 'q/3223/8.1/bugfix/S3C-3425-bump-arsenal-version' into tmp/octopus/q/8.2 2021-01-22 22:30:06 +00:00
bert-e 3449c5aae2 Merge branches 'w/8.1/bugfix/S3C-3425-bump-arsenal-version' and 'q/3223/7.9/bugfix/S3C-3425-bump-arsenal-version' into tmp/octopus/q/8.1 2021-01-22 22:30:05 +00:00
bert-e 7ce221bc81 Merge branch 'q/3223/7.4/bugfix/S3C-3425-bump-arsenal-version' into tmp/normal/q/7.9 2021-01-22 22:30:05 +00:00
bert-e c55e1f6aab Merge branch 'bugfix/S3C-3425-bump-arsenal-version' into q/7.4 2021-01-22 22:30:04 +00:00
bert-e 638afaced4 Merge branch 'w/8.1/bugfix/S3C-3425-bump-arsenal-version' into tmp/octopus/w/8.2/bugfix/S3C-3425-bump-arsenal-version 2021-01-22 22:15:12 +00:00
Ilke 14bc466af7 Merge remote-tracking branch 'origin/w/7.9/bugfix/S3C-3425-bump-arsenal-version' into w/8.1/bugfix/S3C-3425-bump-arsenal-version 2021-01-22 14:02:57 -08:00
Ilke 8fe26d46ed Merge remote-tracking branch 'origin/bugfix/S3C-3425-bump-arsenal-version' into w/7.9/bugfix/S3C-3425-bump-arsenal-version 2021-01-22 12:32:35 -08:00
bert-e f29eb7c837 Merge branch 'w/8.1/bugfix/S3C-3416-data-backend-uuid-for-delete-request' into tmp/octopus/w/8.2/bugfix/S3C-3416-data-backend-uuid-for-delete-request 2021-01-21 15:28:53 +00:00
bert-e ce4d1eecf7 Merge branch 'w/7.9/bugfix/S3C-3416-data-backend-uuid-for-delete-request' into tmp/octopus/w/8.1/bugfix/S3C-3416-data-backend-uuid-for-delete-request 2021-01-21 15:28:52 +00:00
bert-e f2983726fa Merge branch 'bugfix/S3C-3416-data-backend-uuid-for-delete-request' into tmp/octopus/w/7.9/bugfix/S3C-3416-data-backend-uuid-for-delete-request 2021-01-21 15:28:52 +00:00
alexandre merle fa3491d4a3
bf: S3C-3416: data backend uuid when delete
A new logger was created for service delete
operations without propagating logger uuid,
leading to some issue with tracking the
further requests.
2021-01-21 16:14:56 +01:00
bert-e 58d9bf4453 Merge branch 'w/8.1/improvement/S3C-3779-add-parts-count-header' into tmp/octopus/w/8.2/improvement/S3C-3779-add-parts-count-header 2021-01-13 02:18:39 +00:00
Dora Korpar f9fbe2ca15 Merge remote-tracking branch 'origin/w/7.9/improvement/S3C-3779-add-parts-count-header' into w/8.1/improvement/S3C-3779-add-parts-count-header 2021-01-12 18:06:55 -08:00
Dora Korpar 84375b3f73 Merge remote-tracking branch 'origin/improvement/S3C-3779-add-parts-count-header' into w/7.9/improvement/S3C-3779-add-parts-count-header 2021-01-12 16:59:15 -08:00
Dora Korpar 0f4154636a imprv: S3C-3779 add mp-parts-count header 2021-01-12 16:54:08 -08:00
bert-e 07adb04d6b Merge branch 'w/8.1/feature/S3C-3772_add_utapiv2_client_tls_config_support' into tmp/octopus/w/8.2/feature/S3C-3772_add_utapiv2_client_tls_config_support 2021-01-06 23:04:35 +00:00
Taylor McKinnon c0a2ee190e Merge remote-tracking branch 'origin/feature/S3C-3772_add_utapiv2_client_tls_config_support' into w/8.1/feature/S3C-3772_add_utapiv2_client_tls_config_support 2021-01-06 15:04:01 -08:00
Taylor McKinnon 5513b5d62b ft(S3C-3772): Add support for Utapiv2 client to communicate over TLS 2021-01-06 14:55:43 -08:00
bert-e ad7b370211 Merge branches 'w/8.2/bugfix/S3C-3554-bucket-notif-iam-policy-eval' and 'q/3179/8.1/bugfix/S3C-3554-bucket-notif-iam-policy-eval' into tmp/octopus/q/8.2 2020-12-28 05:43:45 +00:00
bert-e 0821069a91 Merge branch 'q/3179/7.9/bugfix/S3C-3554-bucket-notif-iam-policy-eval' into tmp/normal/q/8.1 2020-12-28 05:43:44 +00:00
bert-e 6c66be0689 Merge branch 'bugfix/S3C-3554-bucket-notif-iam-policy-eval' into q/7.9 2020-12-28 05:43:43 +00:00
bert-e 637e01d7ee Merge branch 'w/8.1/bugfix/S3C-3554-bucket-notif-iam-policy-eval' into tmp/octopus/w/8.2/bugfix/S3C-3554-bucket-notif-iam-policy-eval 2020-12-28 05:16:31 +00:00
Dora Korpar 020e65b548 Merge remote-tracking branch 'origin/bugfix/S3C-3554-bucket-notif-iam-policy-eval' into w/8.1/bugfix/S3C-3554-bucket-notif-iam-policy-eval 2020-12-27 21:14:59 -08:00
bert-e 646238f9ee Merge branch 'w/8.1/bugfix/S3C-2665-backportBatchDeleteBackbeatRoute' into tmp/octopus/w/8.2/bugfix/S3C-2665-backportBatchDeleteBackbeatRoute 2020-12-22 21:36:30 +00:00
Jonathan Gramain d210c45dd4 Merge remote-tracking branch 'origin/w/7.9/bugfix/S3C-2665-backportBatchDeleteBackbeatRoute' into w/8.1/bugfix/S3C-2665-backportBatchDeleteBackbeatRoute 2020-12-22 13:18:33 -08:00
Jonathan Gramain 01b4fc4d0c Merge remote-tracking branch 'origin/bugfix/S3C-2665-backportBatchDeleteBackbeatRoute' into w/7.9/bugfix/S3C-2665-backportBatchDeleteBackbeatRoute 2020-12-22 12:35:26 -08:00
Jonathan Gramain 5bc06d7c37 ft: ZENKO-143 batchDelete backbeat route
- Implement 'POST /_/backbeat/batchdelete' backbeat route to get rid
  of an array of data locations. The route will be used by the garbage
  collector service.

  * This includes some reorganization of backbeat routes sanity checks

- Handle gracefully 404 errors from datastore backend:

  * no need to retry 404 errors as they are permanent

  * batch delete can also gracefully handle missing data locations and
    still delete other existing locations (may happen if a retry
    occurs, when replaying a kafka message among other cases).

- Support service-gc service account with account owner access rights.

(cherry picked from commit 8874f97045)

Ticket associated with the cherry-pick: S3C-2665
2020-12-22 12:31:50 -08:00
Dora Korpar 4803e9ead3 bf: S3C-3581 add bucket notif apis to iam policy eval 2020-12-22 10:55:18 -08:00
Ilke fcab730720 bf: S3C-3425 bump arsenal version 2020-12-17 10:05:54 -08:00
bert-e 4db246df92 Merge branch 'w/8.1/bugfix/S3C-3313-fix-bucketdel-abortmpu' into tmp/octopus/w/8.2/bugfix/S3C-3313-fix-bucketdel-abortmpu 2020-12-10 00:33:14 +00:00
bert-e 1b51ac7f94 Merge branch 'w/7.9/bugfix/S3C-3313-fix-bucketdel-abortmpu' into tmp/octopus/w/8.1/bugfix/S3C-3313-fix-bucketdel-abortmpu 2020-12-10 00:33:13 +00:00
bert-e 0de72801cc Merge branch 'w/7.4/bugfix/S3C-3313-fix-bucketdel-abortmpu' into tmp/octopus/w/7.9/bugfix/S3C-3313-fix-bucketdel-abortmpu 2020-12-10 00:33:13 +00:00
Dora Korpar d449543eff bf: S3C-3313 add canonicalid for abortmpu metric 2020-12-09 16:31:00 -08:00
bert-e 198f449da6 Merge branch 'w/8.1/bugfix/S3C-3313-fix-storageused' into tmp/octopus/w/8.2/bugfix/S3C-3313-fix-storageused 2020-12-08 08:03:04 +00:00
bert-e 1f5f22cf85 Merge branch 'w/7.9/bugfix/S3C-3313-fix-storageused' into tmp/octopus/w/8.1/bugfix/S3C-3313-fix-storageused 2020-12-08 08:03:03 +00:00
bert-e 93fb24adf1 Merge branch 'w/7.4/bugfix/S3C-3313-fix-storageused' into tmp/octopus/w/7.9/bugfix/S3C-3313-fix-storageused 2020-12-08 08:03:03 +00:00
Dora Korpar ead94d9193 bf: S3C-3313 storageUtilized for deleteBucket abortMPUs 2020-12-07 17:36:37 -08:00
bert-e cd3aac2d8d Merge branch 'w/8.1/improvement/S3C-3653-add-fields' into tmp/octopus/w/8.2/improvement/S3C-3653-add-fields 2020-12-02 01:20:39 +00:00
bert-e 130639ab09 Merge branch 'w/7.9/improvement/S3C-3653-add-fields' into tmp/octopus/w/8.1/improvement/S3C-3653-add-fields 2020-12-02 01:20:39 +00:00
bert-e 131ebabfbe Merge branch 'w/7.4/improvement/S3C-3653-add-fields' into tmp/octopus/w/7.8/improvement/S3C-3653-add-fields 2020-12-02 01:20:38 +00:00
Rahul Padigela ec35668eb2 improvement: S3C-3653 add server ip, port to log
These fields are added to support transformation to LEEF log format
for S3C. It's redundant to the fields address and port but they are not
removed to maintain backwards compatibility.
2020-12-01 17:17:55 -08:00
bert-e 22f88f40af Merge branch 'w/8.1/improvement/S3C-3512-bucket-obj-names-logs' into tmp/octopus/w/8.2/improvement/S3C-3512-bucket-obj-names-logs 2020-11-19 04:50:09 +00:00
bert-e 658cbc1c58 Merge branch 'w/7.9/improvement/S3C-3512-bucket-obj-names-logs' into tmp/octopus/w/8.1/improvement/S3C-3512-bucket-obj-names-logs 2020-11-19 04:50:08 +00:00
bert-e 78626c2864 Merge branch 'improvement/S3C-3512-bucket-obj-names-logs' into tmp/octopus/w/7.8/improvement/S3C-3512-bucket-obj-names-logs 2020-11-19 04:50:07 +00:00
bert-e f75d044951 Merge branch 'w/8.1/improvement/S3C-3475-add-actions-in-logs' into tmp/octopus/w/8.2/improvement/S3C-3475-add-actions-in-logs 2020-11-19 02:28:14 +00:00
Dora Korpar 3f21abc555 revert mongodb for reportHandler tests 2020-11-18 18:27:55 -08:00
bert-e d35898b7c4 Merge branch 'w/8.1/improvement/S3C-3475-add-actions-in-logs' into tmp/octopus/w/8.2/improvement/S3C-3475-add-actions-in-logs 2020-11-19 00:12:00 +00:00
Dora Korpar 2f55560ea7 Merge remote-tracking branch 'origin/w/7.9/improvement/S3C-3475-add-actions-in-logs' into w/8.1/improvement/S3C-3475-add-actions-in-logs 2020-11-18 16:11:13 -08:00
Dora Korpar 9d7f1cf5d8 Merge remote-tracking branch 'origin/improvement/S3C-3475-add-actions-in-logs' into w/7.8/improvement/S3C-3475-add-actions-in-logs 2020-11-18 16:05:43 -08:00
Dora Korpar cf7c506c93 imprv: S3C-3512 add bucket/obj names in logs 2020-11-18 15:51:58 -08:00
Dora Korpar c81fb3781a imprv: S3C-3475 add s3 actions in logs 2020-11-17 12:55:47 -08:00
bert-e 501067f8c2 Merge branch 'bugfix/ZENKO-2866-updateArsenalDep' into tmp/octopus/w/8.2/bugfix/ZENKO-2866-updateArsenalDep 2020-11-11 19:50:19 +00:00
Jonathan Gramain 73b6c3805f bugfix: ZENKO-2866 update arsenal dependency 2020-11-11 11:46:06 -08:00
Jonathan Gramain 0895df0c66 bugfix: ZENKO-2905 update arsenal dependency 2020-11-04 16:58:54 -08:00
bert-e a5c3d2cfe1 Merge branch 'w/8.1/bugfix/S3C-2687-acl-invalid-id' into tmp/octopus/w/8.2/bugfix/S3C-2687-acl-invalid-id 2020-11-02 21:43:15 +00:00
bert-e 1df9d24171 Merge branch 'w/7.9/bugfix/S3C-2687-acl-invalid-id' into tmp/octopus/w/8.1/bugfix/S3C-2687-acl-invalid-id 2020-11-02 21:43:14 +00:00
bert-e 785d58d2c5 Merge branch 'bugfix/S3C-2687-acl-invalid-id' into tmp/octopus/w/7.8/bugfix/S3C-2687-acl-invalid-id 2020-11-02 21:43:13 +00:00
Dora Korpar 5320e7ce69 bf: S3C-2687 update acl id regex 2020-11-02 11:12:34 -08:00
Dora Korpar 9b5ef5a540 bf: S3C-2687 acl invalid id tests 2020-10-30 15:54:08 -07:00
bert-e deed88b51d Merge branches 'w/8.2/bugfix/S3C-3350-partnumber-exception-port' and 'q/3115/8.1/bugfix/S3C-3350-partnumber-exception-port' into tmp/octopus/q/8.2 2020-10-30 17:19:13 +00:00
bert-e 63a3aa886d Merge branches 'w/8.1/bugfix/S3C-3350-partnumber-exception-port' and 'q/3115/7.9/bugfix/S3C-3350-partnumber-exception-port' into tmp/octopus/q/8.1 2020-10-30 17:19:12 +00:00
bert-e cf04e79c33 Merge branches 'w/7.8/bugfix/S3C-3350-partnumber-exception-port' and 'q/3115/7.4/bugfix/S3C-3350-partnumber-exception-port' into tmp/octopus/q/7.8 2020-10-30 17:19:11 +00:00
bert-e 52e37446a1 Merge branch 'bugfix/S3C-3350-partnumber-exception-port' into q/7.4 2020-10-30 17:19:11 +00:00
bert-e 8ad69c5e35 Merge branch 'w/8.1/bugfix/S3C-3428-skip-obj-copy-tests' into tmp/octopus/w/8.2/bugfix/S3C-3428-skip-obj-copy-tests 2020-10-29 17:46:09 +00:00
bert-e fd395a8672 Merge branch 'w/7.9/bugfix/S3C-3428-skip-obj-copy-tests' into tmp/octopus/w/8.1/bugfix/S3C-3428-skip-obj-copy-tests 2020-10-29 17:46:08 +00:00
Dora Korpar 9b6c165f42 Merge remote-tracking branch 'origin/bugfix/S3C-3428-skip-obj-copy-tests' into w/7.8/bugfix/S3C-3428-skip-obj-copy-tests 2020-10-29 10:31:59 -07:00
bert-e 18f32b56b4 Merge branch 'w/8.1/bugfix/S3C-3350-partnumber-exception-port' into tmp/octopus/w/8.2/bugfix/S3C-3350-partnumber-exception-port 2020-10-29 04:40:42 +00:00
bert-e a6b854d258 Merge branch 'w/7.9/bugfix/S3C-3350-partnumber-exception-port' into tmp/octopus/w/8.1/bugfix/S3C-3350-partnumber-exception-port 2020-10-29 04:40:42 +00:00
bert-e 31b6ebd82d Merge branch 'bugfix/S3C-3350-partnumber-exception-port' into tmp/octopus/w/7.8/bugfix/S3C-3350-partnumber-exception-port 2020-10-29 04:40:41 +00:00
Rahul Padigela 48cbb06b76 bugfix: S3C-3350 avoid throwing exception for legacy objects
(cherry picked from commit fbfda37ae6)
2020-10-28 21:40:03 -07:00
Dora Korpar c0de398709 bf: S3C-3482 skip object copy tests 2020-10-28 16:14:03 -07:00
bert-e 5534af953e Merge branch 'w/8.1/bugfix/S3C-3460_increment_objDelta_only_on_mpu_completion' into tmp/octopus/w/8.2/bugfix/S3C-3460_increment_objDelta_only_on_mpu_completion 2020-10-27 23:59:40 +00:00
bert-e a45a129e86 Merge branch 'w/7.9/bugfix/S3C-3460_increment_objDelta_only_on_mpu_completion' into tmp/octopus/w/8.1/bugfix/S3C-3460_increment_objDelta_only_on_mpu_completion 2020-10-27 23:59:40 +00:00
bert-e 2562982715 Merge branch 'bugfix/S3C-3460_increment_objDelta_only_on_mpu_completion' into tmp/octopus/w/7.8/bugfix/S3C-3460_increment_objDelta_only_on_mpu_completion 2020-10-27 23:59:39 +00:00
Taylor McKinnon 66534f0a40 bf(S3C-3460): Increment numberOfObjects only on MPU completion 2020-10-27 16:50:31 -07:00
bert-e f622e75711 Merge branch 'w/8.1/bugfix/S3C-3428-skip-obj-copy-tests-e2e' into tmp/octopus/w/8.2/bugfix/S3C-3428-skip-obj-copy-tests-e2e 2020-10-27 19:03:49 +00:00
bert-e 0ee82114a9 Merge branch 'w/7.9/bugfix/S3C-3428-skip-obj-copy-tests-e2e' into tmp/octopus/w/8.1/bugfix/S3C-3428-skip-obj-copy-tests-e2e 2020-10-27 19:03:48 +00:00
bert-e 828dfd47e0 Merge branch 'bugfix/S3C-3428-skip-obj-copy-tests-e2e' into tmp/octopus/w/7.8/bugfix/S3C-3428-skip-obj-copy-tests-e2e 2020-10-27 19:03:48 +00:00
Dora Korpar 50199b814b bf: S3C-3428 skip obj copy tests in e2e 2020-10-27 11:59:24 -07:00
bert-e ebdec29c88 Merge branch 'w/8.1/bugfix/S3C-3465-remove-extra-ms-digits' into tmp/octopus/w/8.2/bugfix/S3C-3465-remove-extra-ms-digits 2020-10-23 23:18:13 +00:00
bert-e c8067d3206 Merge branch 'w/7.9/bugfix/S3C-3465-remove-extra-ms-digits' into tmp/octopus/w/8.1/bugfix/S3C-3465-remove-extra-ms-digits 2020-10-23 23:18:13 +00:00
Ilke 7da19aa077 bugfix: S3C-3465 fix ms difference is tests 2020-10-23 16:13:34 -07:00
bert-e 227873f2e5 Merge branches 'w/8.2/bugfix/S3C-3330-bucket-policy-action-mapping' and 'q/3058/8.1/bugfix/S3C-3330-bucket-policy-action-mapping' into tmp/octopus/q/8.2 2020-10-10 02:53:34 +00:00
bert-e 4dfd3c04ad Merge branches 'w/8.1/bugfix/S3C-3330-bucket-policy-action-mapping' and 'q/3058/7.9/bugfix/S3C-3330-bucket-policy-action-mapping' into tmp/octopus/q/8.1 2020-10-10 02:53:34 +00:00
bert-e 81e6d09be1 Merge branch 'bugfix/S3C-3330-bucket-policy-action-mapping' into q/7.8.0 2020-10-10 02:53:33 +00:00
bert-e c94eadf24b Merge branch 'w/8.1/bugfix/S3C-3350-partnumber-exception' into tmp/octopus/w/8.2/bugfix/S3C-3350-partnumber-exception 2020-10-09 06:15:17 +00:00
Rahul Padigela e40203378f Merge remote-tracking branch 'origin/w/7.9/bugfix/S3C-3350-partnumber-exception' into w/8.1/bugfix/S3C-3350-partnumber-exception 2020-10-08 23:06:56 -07:00
Rahul Padigela fbfda37ae6 bugfix: S3C-3350 avoid throwing exception for legacy objects 2020-10-08 23:00:53 -07:00
bert-e 834d4c1721 Merge branch 'w/8.1/bugfix/S3C-3330-bucket-policy-action-mapping' into tmp/octopus/w/8.2/bugfix/S3C-3330-bucket-policy-action-mapping 2020-10-08 05:01:55 +00:00
bert-e 06f56dd2f0 Merge branch 'w/7.9/bugfix/S3C-3330-bucket-policy-action-mapping' into tmp/octopus/w/8.1/bugfix/S3C-3330-bucket-policy-action-mapping 2020-10-08 05:01:55 +00:00
Dora Korpar c8e7c16397 bf: S3C-3330 bucket policy action mapping 2020-10-07 22:00:58 -07:00
bert-e 9519087fe7 Merge branches 'w/8.2/feature/ZENKO-2810-upload-short-version-docker-registry' and 'q/3031/8.1/feature/ZENKO-2810-upload-short-version-docker-registry' into tmp/octopus/q/8.2 2020-10-05 15:44:00 +00:00
bert-e 6cd445849e Merge branch 'feature/ZENKO-2810-upload-short-version-docker-registry' into q/8.1 2020-10-05 15:43:59 +00:00
Dora Korpar 9bea862bdf Merge remote-tracking branch 'origin/w/8.1/feature/S3C-1801-policy-tag-condition-keys' into w/8.2/feature/S3C-1801-policy-tag-condition-keys 2020-09-30 17:02:17 -07:00
Dora Korpar 878102280a Merge remote-tracking branch 'origin/feature/S3C-1801-policy-tag-condition-keys' into w/8.1/feature/S3C-1801-policy-tag-condition-keys 2020-09-30 16:05:14 -07:00
Dora Korpar 33c445aba6 S3C-1801 update arsenal 2020-09-30 15:48:04 -07:00
Dora Korpar a9ca26696f ft: S3C-1801 policy tag condition keys 2020-09-30 15:48:04 -07:00
bert-e 4e6f7b1309 Merge branches 'w/8.2/bugfix/S3C-2775-utapiCRRFix' and 'q/3014/8.1/bugfix/S3C-2775-utapiCRRFix' into tmp/octopus/q/8.2 2020-09-16 23:09:25 +00:00
bert-e 1715bc3dce Merge branches 'w/8.1/bugfix/S3C-2775-utapiCRRFix' and 'q/3014/7.8/bugfix/S3C-2775-utapiCRRFix' into tmp/octopus/q/8.1 2020-09-16 23:09:24 +00:00
bert-e 9df4912a03 Merge branch 'bugfix/S3C-2775-utapiCRRFix' into q/7.8 2020-09-16 23:09:24 +00:00
bert-e 200bfe594e Merge branch 'feature/ZENKO-2810-upload-short-version-docker-registry' into tmp/octopus/w/8.2/feature/ZENKO-2810-upload-short-version-docker-registry 2020-09-16 21:46:53 +00:00
Thomas Carmet e83ec2e1ca ZENKO-2810 upload docker image with short version tag
The goal of this code is to allow us to obtain docker images tag
in our registry that look like the following: `latest-8.1` or `latest-8.2`

Those tags will match the content of the tip of a given development/x.y branch
2020-09-16 14:44:11 -07:00
bert-e 67a1516221 Merge branch 'w/8.1/bugfix/S3C-3042-repl-versioning' into tmp/octopus/w/8.2/bugfix/S3C-3042-repl-versioning 2020-09-15 06:25:57 +00:00
Rahul Padigela 86e8d3412f Merge remote-tracking branch 'origin/w/7.8/bugfix/S3C-3042-repl-versioning' into w/8.1/bugfix/S3C-3042-repl-versioning 2020-09-14 23:25:09 -07:00
bert-e 5ae26dc7b3 Merge branch 'bugfix/S3C-3042-repl-versioning' into tmp/octopus/w/7.8/bugfix/S3C-3042-repl-versioning 2020-09-15 06:17:46 +00:00
Rahul Padigela 6a0636667b bugfix: S3C-3042 prevent suspending version on replication buckets
This fix which is compatibilty with AWS S3, ensures that versioning
cannot be suspended on buckets that have replication enabled.
2020-09-14 23:14:52 -07:00
bert-e e6efb5816e Merge branch 'w/8.1/bugfix/S3C-2775-utapiCRRFix' into tmp/octopus/w/8.2/bugfix/S3C-2775-utapiCRRFix 2020-09-14 22:44:01 +00:00
Jonathan Gramain 4127cbef21 Merge remote-tracking branch 'origin/bugfix/S3C-2775-utapiCRRFix' into w/8.1/bugfix/S3C-2775-utapiCRRFix 2020-09-14 15:38:59 -07:00
Jonathan Gramain 69e6d678a0 bugfix: S3C-2775 update logic for counting object replicas
Before, we were relying on the presence of ACL change to know if the
operation is metadata-only. This was not correct, as a new object can
be put with specific ACL too.

Instead, consider that if there is already an object present with the
same version, we are necessarily updating this object hence the object
counter should not be updated.

NOTE: this change will come along with another change in backbeat: the
PUT /_/backbeat/metadata route will be provided the version ID of the
object to PUT, so that the validation of the existing object (through
the metadataValidateBucketAndObj helper) checks the specific version
instead of checking the master key (which for backbeat replication
purpose is also incorrect). Without this backbeat change, the
replicated object count metric becomes wrong when an object is
rewritten.
2020-09-14 14:38:05 -07:00
bert-e 3139fa0f9f Merge branch 'w/8.1/bugfix/S3C-1232_add_account_id_validation_for_list_metrics' into tmp/octopus/w/8.2/bugfix/S3C-1232_add_account_id_validation_for_list_metrics 2020-09-04 22:23:42 +00:00
bert-e 211cfa9a4e Merge branch 'w/7.8/bugfix/S3C-1232_add_account_id_validation_for_list_metrics' into tmp/octopus/w/8.1/bugfix/S3C-1232_add_account_id_validation_for_list_metrics 2020-09-04 22:23:41 +00:00
bert-e 02f9113a0f Merge branch 'bugfix/S3C-1232_add_account_id_validation_for_list_metrics' into tmp/octopus/w/7.8/bugfix/S3C-1232_add_account_id_validation_for_list_metrics 2020-09-04 22:23:40 +00:00
Taylor McKinnon a6fc4a5003 bf(S3C-1232): Validate passed account ids for list_metrics.js 2020-09-04 15:22:51 -07:00
bert-e ac0558fb1f Merge branch 'w/8.1/bugfix/S3C-3303-put-empty-notif-config' into tmp/octopus/w/8.2/bugfix/S3C-3303-put-empty-notif-config 2020-09-03 02:08:43 +00:00
Dora Korpar 3068b6ddf3 Merge remote-tracking branch 'origin/bugfix/S3C-3303-put-empty-notif-config' into w/8.1/bugfix/S3C-3303-put-empty-notif-config 2020-09-02 18:56:39 -07:00
Dora Korpar 704f8d9290 bf: S3C-3303 allow empty notif config 2020-09-02 17:18:09 -07:00
bert-e ae94b94b4c Merge branch 'w/8.1/bugfix/S3C-3301-bucket-notif-config-check-fix' into tmp/octopus/w/8.2/bugfix/S3C-3301-bucket-notif-config-check-fix 2020-09-01 15:32:13 +00:00
bert-e 5510c329ec Merge branch 'bugfix/S3C-3301-bucket-notif-config-check-fix' into tmp/octopus/w/8.1/bugfix/S3C-3301-bucket-notif-config-check-fix 2020-09-01 15:32:13 +00:00
naren-scality f3d7758d38 bf: S3C-3301 bucket notif config fix
Bucket notification configuratio check is modified to allow the
properties 'port', 'auth' be optional. 'port' can be a string.
2020-08-31 14:03:12 -07:00
bert-e e7fe656fcf Merge branch 'w/8.1/feature/S3C-2798-get-bucket-notification' into tmp/octopus/w/8.2/feature/S3C-2798-get-bucket-notification 2020-08-27 03:39:37 +00:00
Dora Korpar 11f88abe35 Merge remote-tracking branch 'origin/feature/S3C-2798-get-bucket-notification' into w/8.1/feature/S3C-2798-get-bucket-notification 2020-08-26 20:28:54 -07:00
Dora Korpar d2d008dad4 update arsenal 2020-08-26 19:05:06 -07:00
Dora Korpar 00235bf7a9 ft: S3C-2798 get bucket notification 2020-08-26 19:05:06 -07:00
bert-e 0d322df197 Merge branch 'w/8.1/feature/S3C-2797-put-bucket-notification' into tmp/octopus/w/8.2/feature/S3C-2797-put-bucket-notification 2020-08-26 06:30:22 +00:00
Dora Korpar 885d973766 revert mongo yarn lock upgrade 2020-08-25 23:30:03 -07:00
bert-e a9cc81ad3a Merge branch 'w/8.1/feature/S3C-2797-put-bucket-notification' into tmp/octopus/w/8.2/feature/S3C-2797-put-bucket-notification 2020-08-25 21:32:57 +00:00
Dora Korpar 4ea7b2e496 Merge remote-tracking branch 'origin/feature/S3C-2797-put-bucket-notification' into w/8.1/feature/S3C-2797-put-bucket-notification 2020-08-25 14:19:53 -07:00
Dora Korpar 25f8e25eea ft: S3C-2797 update BucketInfo ModelVersion doc 2020-08-25 14:14:21 -07:00
Dora Korpar 44aad38da3 ft:S3C 2797 update Arsenal 2020-08-25 14:14:21 -07:00
Dora Korpar f836c12af7 ft: S3C-2797 put bucket notification 2020-08-25 14:14:21 -07:00
bert-e 118926a4a0 Merge branches 'w/8.2/feature/S3C-3238-bucket-notification-config' and 'q/2934/8.1/feature/S3C-3238-bucket-notification-config' into tmp/octopus/q/8.2 2020-08-17 18:37:47 +00:00
bert-e 431cc40a94 Merge branches 'w/8.1/feature/S3C-3238-bucket-notification-config' and 'q/2934/7.8/feature/S3C-3238-bucket-notification-config' into tmp/octopus/q/8.1 2020-08-17 18:37:47 +00:00
bert-e 83f9af7449 Merge branch 'feature/S3C-3238-bucket-notification-config' into q/7.8 2020-08-17 18:37:47 +00:00
bert-e 3172fe2621 Merge branch 'w/8.1/bugfix/S3C-3263_incorrect_objectDelta' into tmp/octopus/w/8.2/bugfix/S3C-3263_incorrect_objectDelta 2020-08-14 21:42:19 +00:00
bert-e 95f2782fa4 Merge branch 'bugfix/S3C-3263_incorrect_objectDelta' into tmp/octopus/w/8.1/bugfix/S3C-3263_incorrect_objectDelta 2020-08-14 21:42:18 +00:00
Taylor McKinnon 6371721667 fix typo 2020-08-14 14:42:06 -07:00
bert-e 8fb1160deb Merge branch 'w/8.1/bugfix/S3C-3263_incorrect_objectDelta' into tmp/octopus/w/8.2/bugfix/S3C-3263_incorrect_objectDelta 2020-08-14 21:06:21 +00:00
bert-e 1b59974eba Merge branch 'bugfix/S3C-3263_incorrect_objectDelta' into tmp/octopus/w/8.1/bugfix/S3C-3263_incorrect_objectDelta 2020-08-14 21:06:20 +00:00
Taylor McKinnon 9b3a998d7c fix objectCopy 2020-08-14 14:06:07 -07:00
bert-e a16ceb6586 Merge branch 'w/8.1/bugfix/S3C-3263_incorrect_objectDelta' into tmp/octopus/w/8.2/bugfix/S3C-3263_incorrect_objectDelta 2020-08-14 19:47:23 +00:00
bert-e 6c86fec043 Merge branch 'bugfix/S3C-3263_incorrect_objectDelta' into tmp/octopus/w/8.1/bugfix/S3C-3263_incorrect_objectDelta 2020-08-14 19:47:22 +00:00
Taylor McKinnon 319bbbc911 bf(S3C-3263): Fix objectDelta field values 2020-08-14 12:44:23 -07:00
bert-e a903862aab Merge branch 'w/8.1/bugfix/ZENKO-2729-raw-node-tests-remove-only' into tmp/octopus/w/8.2/bugfix/ZENKO-2729-raw-node-tests-remove-only 2020-08-12 20:54:08 +00:00
Jonathan Gramain ceb8d3b6f9 Merge remote-tracking branch 'origin/w/7.8/bugfix/ZENKO-2729-raw-node-tests-remove-only' into w/8.1/bugfix/ZENKO-2729-raw-node-tests-remove-only 2020-08-12 13:53:50 -07:00
bert-e 66a3631f65 Merge branch 'bugfix/ZENKO-2729-raw-node-tests-remove-only' into tmp/octopus/w/7.8/bugfix/ZENKO-2729-raw-node-tests-remove-only 2020-08-12 20:18:06 +00:00
Jonathan Gramain 98d89b47b2 bugfix: ZENKO-2729 fix routeBackbeat mixed use of Promise and callback
In recent versions of mocha, the current test code before() hook in
routeBackbeat.js triggers an error "Error: Resolution method is
overspecified. Specify a callback *or* return a Promise; not both".

Fix this by not returning the promise object, so that more recent
versions of mocha are happy.
2020-08-12 13:15:21 -07:00
bert-e 1b7a0182c2 Merge branches 'development/8.2' and 'w/8.1/bugfix/ZENKO-2729-raw-node-tests-remove-only' into tmp/octopus/w/8.2/bugfix/ZENKO-2729-raw-node-tests-remove-only 2020-08-12 19:56:48 +00:00
bert-e c35cac674b Merge branches 'development/8.1' and 'w/7.8/bugfix/ZENKO-2729-raw-node-tests-remove-only' into tmp/octopus/w/8.1/bugfix/ZENKO-2729-raw-node-tests-remove-only 2020-08-12 19:56:47 +00:00
bert-e fc3ab96f05 Merge branches 'development/7.8' and 'bugfix/ZENKO-2729-raw-node-tests-remove-only' into tmp/octopus/w/7.8/bugfix/ZENKO-2729-raw-node-tests-remove-only 2020-08-12 19:56:47 +00:00
Jonathan Gramain a7b13b50fb bugfix: ZENKO-2729 async raw-node headObject test before/after
try to reduce flakiness by waiting until the before/after hooks are
done creating/deleting the test bucket
2020-08-12 12:53:42 -07:00
bert-e aecbec4e6c Merge branch 'w/8.1/feature/S3C-3238-bucket-notification-config' into tmp/octopus/w/8.2/feature/S3C-3238-bucket-notification-config 2020-08-12 07:21:37 +00:00
bert-e 9f0023157b Merge branch 'feature/S3C-3238-bucket-notification-config' into tmp/octopus/w/8.1/feature/S3C-3238-bucket-notification-config 2020-08-12 07:21:37 +00:00
Dora Korpar 1f511a000a [squash] address comments 2020-08-12 00:21:20 -07:00
bert-e d414c4a5bb Merge branches 'w/8.2/feature/S3C-3229-bucketnotif-objmd-update' and 'q/2929/8.1/feature/S3C-3229-bucketnotif-objmd-update' into tmp/octopus/q/8.2 2020-08-12 07:16:11 +00:00
bert-e a9b2b891f8 Merge branches 'w/8.1/feature/S3C-3229-bucketnotif-objmd-update' and 'q/2929/7.8/feature/S3C-3229-bucketnotif-objmd-update' into tmp/octopus/q/8.1 2020-08-12 07:16:10 +00:00
bert-e 4f5344990d Merge branch 'feature/S3C-3229-bucketnotif-objmd-update' into q/7.8 2020-08-12 07:16:10 +00:00
bert-e 8e247df0c4 Merge branch 'w/8.1/feature/S3C-3229-bucketnotif-objmd-update' into tmp/octopus/w/8.2/feature/S3C-3229-bucketnotif-objmd-update 2020-08-12 04:10:23 +00:00
Dora Korpar 430ddf1cad revert yarn mongo update 2020-08-11 21:09:49 -07:00
bert-e af69c7edc5 Merge branch 'w/8.1/feature/S3C-3238-bucket-notification-config' into tmp/octopus/w/8.2/feature/S3C-3238-bucket-notification-config 2020-08-12 02:05:18 +00:00
bert-e 73636b9af6 Merge branch 'feature/S3C-3238-bucket-notification-config' into tmp/octopus/w/8.1/feature/S3C-3238-bucket-notification-config 2020-08-12 02:05:18 +00:00
Dora Korpar ec63f332dc ft: S3C-3238 add bucket notification config 2020-08-11 17:42:27 -07:00
bert-e 3d8dc4c83e Merge branch 'w/8.1/feature/S3C-3229-bucketnotif-objmd-update' into tmp/octopus/w/8.2/feature/S3C-3229-bucketnotif-objmd-update 2020-08-11 23:25:47 +00:00
Dora Korpar fe87fbebe6 Merge remote-tracking branch 'origin/feature/S3C-3229-bucketnotif-objmd-update' into w/8.1/feature/S3C-3229-bucketnotif-objmd-update 2020-08-11 16:12:18 -07:00
bert-e 748f2276dc Merge branch 'w/8.1/bugfix/ZENKO-2729-raw-node-tests-remove-only' into tmp/octopus/w/8.2/bugfix/ZENKO-2729-raw-node-tests-remove-only 2020-08-11 19:36:27 +00:00
bert-e 5ea3bbd496 Merge branch 'w/7.8/bugfix/ZENKO-2729-raw-node-tests-remove-only' into tmp/octopus/w/8.1/bugfix/ZENKO-2729-raw-node-tests-remove-only 2020-08-11 19:36:26 +00:00
bert-e abbe0e3c75 Merge branch 'bugfix/ZENKO-2729-raw-node-tests-remove-only' into tmp/octopus/w/7.8/bugfix/ZENKO-2729-raw-node-tests-remove-only 2020-08-11 19:36:26 +00:00
Dora Korpar fd1ca26024 bf: ZENKO-2729 remove .only from raw-node tests 2020-08-11 12:35:50 -07:00
bert-e b24cb5f367 Merge branch 'w/8.1/bugfix/S3C-3246-pushmetric-getobject' into tmp/octopus/w/8.2/bugfix/S3C-3246-pushmetric-getobject 2020-08-11 17:41:15 +00:00
bert-e 20e58d85f9 Merge branch 'bugfix/S3C-3246-pushmetric-getobject' into tmp/octopus/w/8.1/bugfix/S3C-3246-pushmetric-getobject 2020-08-11 17:41:14 +00:00
Dora Korpar a37eab03ce bf: S3C-3246 getObject pushMetric missing key 2020-08-11 10:38:52 -07:00
Dora Korpar be669545e6 ft: S3C-3229 update arsenal 2020-08-10 15:27:51 -07:00
Dora Korpar 4f206993b3 ft: S3C-3229 add originOp objMD 2020-08-10 15:27:51 -07:00
bert-e 5d930f52a6 Merge branch 'w/8.1/bugfix/S3C-2801-tagging-functional-tests' into tmp/octopus/w/8.2/bugfix/S3C-2801-tagging-functional-tests 2020-08-08 00:56:40 +00:00
bert-e 7ebd0c9d58 Merge branch 'w/7.8/bugfix/S3C-2801-tagging-functional-tests' into tmp/octopus/w/8.1/bugfix/S3C-2801-tagging-functional-tests 2020-08-08 00:56:40 +00:00
bert-e 0cdbe7425c Merge branch 'bugfix/S3C-2801-tagging-functional-tests' into tmp/octopus/w/7.8/bugfix/S3C-2801-tagging-functional-tests 2020-08-08 00:56:39 +00:00
Dora Korpar 2b24fbfe75 bf: S3C-2801 mpu tagging tests 2020-08-07 17:34:48 -07:00
bert-e ab9b0b8de6 Merge branches 'w/8.2/feature/S3C-3132-utapi-v2-push-metric' and 'q/2912/8.1/feature/S3C-3132-utapi-v2-push-metric' into tmp/octopus/q/8.2 2020-08-07 21:45:21 +00:00
bert-e 16a7b44090 Merge branches 'w/8.1/feature/S3C-3132-utapi-v2-push-metric' and 'q/2912/7.8/feature/S3C-3132-utapi-v2-push-metric' into tmp/octopus/q/8.1 2020-08-07 21:45:20 +00:00
bert-e 9455bb269b Merge branch 'feature/S3C-3132-utapi-v2-push-metric' into q/7.8 2020-08-07 21:45:20 +00:00
bert-e dbf44586a6 Merge branch 'w/8.1/feature/S3C-3132-utapi-v2-push-metric' into tmp/octopus/w/8.2/feature/S3C-3132-utapi-v2-push-metric 2020-08-07 21:26:31 +00:00
Dora Korpar 1c42602831 remove mongo package updates 2020-08-07 13:50:27 -07:00
Dora Korpar 2c968a5cf9 Merge remote-tracking branch 'origin/w/8.1/feature/S3C-3132-utapi-v2-push-metric' into w/8.2/feature/S3C-3132-utapi-v2-push-metric 2020-08-05 01:11:20 -07:00
Dora Korpar 1bd0da04b9 Merge remote-tracking branch 'origin/feature/S3C-3132-utapi-v2-push-metric' into w/8.1/feature/S3C-3132-utapi-v2-push-metric 2020-08-04 22:19:40 -07:00
Dora Korpar 651f62ef61 ft: S3C-3132 update apis with new metric info 2020-08-04 15:37:13 -07:00
Dora Korpar 1aefa45548 ft: S3C-3132 Support utapi pushmetric v2 2020-08-04 10:47:59 -07:00
bert-e 7d15a4460f Merge branches 'w/7.8/dependabot/add-v2-config-file' and 'q/2743/7.4/dependabot/add-v2-config-file' into tmp/octopus/q/7.8 2020-07-31 21:25:03 +00:00
bert-e 031ffe0344 Merge branches 'w/8.2/dependabot/add-v2-config-file' and 'q/2743/8.1/dependabot/add-v2-config-file' into tmp/octopus/q/8.2 2020-07-31 21:25:03 +00:00
bert-e f77069fd37 Merge branches 'w/8.1/dependabot/add-v2-config-file' and 'q/2743/7.8/dependabot/add-v2-config-file' into tmp/octopus/q/8.1 2020-07-31 21:25:03 +00:00
bert-e ff594f908c Merge branch 'dependabot/add-v2-config-file' into q/7.4 2020-07-31 21:25:02 +00:00
bert-e b16d0ab46f Merge branch 'w/8.1/dependabot/add-v2-config-file' into tmp/octopus/w/8.2/dependabot/add-v2-config-file 2020-07-29 18:46:11 +00:00
bert-e ec507e9ca0 Merge branch 'dependabot/add-v2-config-file' into tmp/octopus/w/7.8/dependabot/add-v2-config-file 2020-07-29 18:46:10 +00:00
bert-e 8db4db3e96 Merge branch 'w/7.8/dependabot/add-v2-config-file' into tmp/octopus/w/8.1/dependabot/add-v2-config-file 2020-07-29 18:46:10 +00:00
Dora Korpar ca1c3a4287 Merge remote-tracking branch 'origin/w/8.1/feature/S3C-3196-update-node' into w/8.2/feature/S3C-3196-update-node 2020-07-28 14:32:57 -07:00
Dora Korpar 0d23133315 Merge remote-tracking branch 'origin/w/7.8/feature/S3C-3196-update-node' into w/8.1/feature/S3C-3196-update-node 2020-07-28 12:38:22 -07:00
Dora Korpar 91be6d419f Merge remote-tracking branch 'origin/feature/S3C-3196-update-node' into w/7.7/feature/S3C-3196-update-node 2020-07-28 10:36:16 -07:00
Dora Korpar 3c93e79397 ft: S3C-3196 update node 2020-07-27 00:52:52 -07:00
Rahul Padigela 7c28dfa857 chore: add dependabot config 2020-07-21 12:18:19 -07:00
Jonathan Gramain 827cfd2745 Merge remote-tracking branch 'origin/w/8.1/bugfix/S3C-2513-fixSproxydLocationConfigParsing' into w/8.2/bugfix/S3C-2513-fixSproxydLocationConfigParsing 2020-07-14 17:18:49 -07:00
Jonathan Gramain 8d9c6924d5 Merge remote-tracking branch 'origin/w/7.8/bugfix/S3C-2513-fixSproxydLocationConfigParsing' into w/8.1/bugfix/S3C-2513-fixSproxydLocationConfigParsing 2020-07-14 17:01:29 -07:00
Jonathan Gramain 603df09d5b Merge remote-tracking branch 'origin/bugfix/S3C-2513-fixSproxydLocationConfigParsing' into w/7.7/bugfix/S3C-2513-fixSproxydLocationConfigParsing 2020-07-14 16:49:43 -07:00
Jonathan Gramain 87fc368c6a bugfix: S3C-2513 use joi to parse sproxyd config
Use the @hapi/joi module to parse the sproxyd location config, for
better maintainability.
2020-07-14 16:43:48 -07:00
Jonathan Gramain f5e30c7cb5 bugfix: S3C-2513 fix a typo parsing locationConfig.json sproxyd chordCos/path
The parsing logic was overriding chordCos with the path info,
resulting in an invalid chordCos parameter.
2020-07-14 14:05:27 -07:00
bert-e abca264313 Merge branch 'w/8.1/improvement/S3C-3119_AddGetHeadObjFuncTests' into tmp/octopus/w/8.2/improvement/S3C-3119_AddGetHeadObjFuncTests 2020-07-10 01:05:25 +00:00
bert-e 4d24149ffd Merge branch 'w/7.8/improvement/S3C-3119_AddGetHeadObjFuncTests' into tmp/octopus/w/8.1/improvement/S3C-3119_AddGetHeadObjFuncTests 2020-07-10 01:05:24 +00:00
bert-e 32002627e0 Merge branch 'improvement/S3C-3119_AddGetHeadObjFuncTests' into tmp/octopus/w/7.7/improvement/S3C-3119_AddGetHeadObjFuncTests 2020-07-10 01:05:24 +00:00
Ilke 772db47c23 improvement: S3C-3119 add func test to get head obj 2020-07-09 17:14:40 -07:00
bert-e e7f55cbdd3 Merge branches 'w/8.2/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket' and 'q/2843/8.1/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket' into tmp/octopus/q/8.2 2020-07-08 01:09:40 +00:00
bert-e fe36fe4e79 Merge branches 'w/8.1/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket' and 'q/2843/7.8/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket' into tmp/octopus/q/8.1 2020-07-08 01:09:40 +00:00
bert-e 76c128ffd1 Merge branches 'w/7.7/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket' and 'q/2843/7.7.0/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket' into tmp/octopus/q/7.7 2020-07-08 01:09:39 +00:00
bert-e 025f65e900 Merge branch 'bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket' into q/7.7.0 2020-07-08 01:09:39 +00:00
bert-e bad637485c Merge branches 'development/8.2' and 'w/8.1/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket' into tmp/octopus/w/8.2/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket 2020-07-08 00:34:46 +00:00
bert-e 540df54c1a Merge branches 'development/8.1' and 'w/7.8/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket' into tmp/octopus/w/8.1/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket 2020-07-08 00:34:45 +00:00
bert-e e14aca31dc Merge branch 'development/7.7' into tmp/octopus/w/7.7/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket 2020-07-08 00:34:44 +00:00
Ilke 79dcc2a3cf update arsenal 2020-07-07 17:34:23 -07:00
bert-e a73843ffa1 Merge branch 'w/8.1/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket' into tmp/octopus/w/8.2/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket 2020-07-07 23:51:53 +00:00
Ilke d2ef79a7f3 Merge remote-tracking branch 'origin/w/7.8/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket' into w/8.1/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket 2020-07-07 16:46:28 -07:00
bert-e 484711de04 Merge branch 'bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket' into tmp/octopus/w/7.7/bugfix/S3C-3130_handleObjectLockDisabledCaseForBucket 2020-07-07 23:44:14 +00:00
Ilke e47b3fc997 bf: S3C-3130 update arsenal to pass put config without rule 2020-07-07 16:43:14 -07:00
Ilke 03ec776e21 bf: S3C-3130 put config without rule throws exception 2020-07-07 16:43:14 -07:00
bert-e 79c6d2c714 Merge branch 'w/8.1/bugfix/S3C-3145-error-log-to-debug' into tmp/octopus/w/8.2/bugfix/S3C-3145-error-log-to-debug 2020-07-07 23:36:36 +00:00
bert-e 2a54e6ac00 Merge branch 'w/7.8/bugfix/S3C-3145-error-log-to-debug' into tmp/octopus/w/8.1/bugfix/S3C-3145-error-log-to-debug 2020-07-07 23:36:36 +00:00
bert-e 3e038bdc69 Merge branch 'bugfix/S3C-3145-error-log-to-debug' into tmp/octopus/w/7.7/bugfix/S3C-3145-error-log-to-debug 2020-07-07 23:36:35 +00:00
Dora Korpar 972a1d2aee bf: S3C-3145 change log to trace 2020-07-07 16:29:26 -07:00
bert-e cd710415e1 Merge branch 'w/8.1/bugfix/S3C-3145-obj-lock-delete-log-fix' into tmp/octopus/w/8.2/bugfix/S3C-3145-obj-lock-delete-log-fix 2020-07-07 22:19:41 +00:00
bert-e 34146b41b6 Merge branch 'w/7.8/bugfix/S3C-3145-obj-lock-delete-log-fix' into tmp/octopus/w/8.1/bugfix/S3C-3145-obj-lock-delete-log-fix 2020-07-07 22:19:40 +00:00
bert-e 6d0c744ff4 Merge branch 'bugfix/S3C-3145-obj-lock-delete-log-fix' into tmp/octopus/w/7.7/bugfix/S3C-3145-obj-lock-delete-log-fix 2020-07-07 22:19:40 +00:00
Dora Korpar f506344860 bf: S3C-3145 - add objectLocked field to AccessDenied errors 2020-07-07 15:11:53 -07:00
bert-e f7619ffa62 Merge branch 'w/8.1/bugfix/S3C-3121_fixPutBucketApiObjLock' into tmp/octopus/w/8.2/bugfix/S3C-3121_fixPutBucketApiObjLock 2020-07-03 01:16:50 +00:00
bert-e c8290ccb0c Merge branch 'w/7.8/bugfix/S3C-3121_fixPutBucketApiObjLock' into tmp/octopus/w/8.1/bugfix/S3C-3121_fixPutBucketApiObjLock 2020-07-03 01:16:49 +00:00
bert-e b073968fd1 Merge branch 'bugfix/S3C-3121_fixPutBucketApiObjLock' into tmp/octopus/w/7.7/bugfix/S3C-3121_fixPutBucketApiObjLock 2020-07-03 01:16:49 +00:00
Ilke 71b693254a bluebird error 2020-07-02 18:16:27 -07:00
bert-e 2cfc03cfd1 Merge branch 'w/8.1/bugfix/S3C-3121_fixPutBucketApiObjLock' into tmp/octopus/w/8.2/bugfix/S3C-3121_fixPutBucketApiObjLock 2020-07-03 01:13:42 +00:00
Ilke 499706b300 Merge remote-tracking branch 'origin/w/7.8/bugfix/S3C-3121_fixPutBucketApiObjLock' into w/8.1/bugfix/S3C-3121_fixPutBucketApiObjLock 2020-07-02 18:04:36 -07:00
bert-e 7a6ae2ad48 Merge branch 'bugfix/S3C-3121_fixPutBucketApiObjLock' into tmp/octopus/w/7.7/bugfix/S3C-3121_fixPutBucketApiObjLock 2020-07-03 01:01:52 +00:00
Ilke e4852df3bb bf: S3C-3121 should create bucket without object lock 2020-07-02 18:00:37 -07:00
bert-e 86cee89dae Merge branch 'w/8.1/bugfix/S3C-3106-multiobj-delete-obj-lock-error' into tmp/octopus/w/8.2/bugfix/S3C-3106-multiobj-delete-obj-lock-error 2020-07-02 15:37:29 +00:00
bert-e b7fac2f4e3 Merge branch 'w/7.8/bugfix/S3C-3106-multiobj-delete-obj-lock-error' into tmp/octopus/w/8.1/bugfix/S3C-3106-multiobj-delete-obj-lock-error 2020-07-02 15:37:28 +00:00
bert-e a320cb3167 Merge branch 'bugfix/S3C-3106-multiobj-delete-obj-lock-error' into tmp/octopus/w/7.7/bugfix/S3C-3106-multiobj-delete-obj-lock-error 2020-07-02 15:37:28 +00:00
Dora Korpar 58a75b6c86 bf: S3C-3106 multiobj delete obj lock error 2020-07-02 07:43:07 -07:00
Dora Korpar 6f33877795 removeObjLock func to changeObjLock 2020-07-02 07:43:07 -07:00
bert-e 9758da31a4 Merge branches 'w/8.2/improvement/S3C-3120_updateBucketGetObjectLockApi' and 'q/2804/8.1/improvement/S3C-3120_updateBucketGetObjectLockApi' into tmp/octopus/q/8.2 2020-07-02 02:42:00 +00:00
bert-e 1cfb8bf663 Merge branches 'w/8.1/improvement/S3C-3120_updateBucketGetObjectLockApi' and 'q/2804/7.8/improvement/S3C-3120_updateBucketGetObjectLockApi' into tmp/octopus/q/8.1 2020-07-02 02:42:00 +00:00
bert-e 56506689f0 Merge branches 'w/7.7/improvement/S3C-3120_updateBucketGetObjectLockApi' and 'q/2804/7.7.0/improvement/S3C-3120_updateBucketGetObjectLockApi' into tmp/octopus/q/7.7 2020-07-02 02:41:59 +00:00
bert-e db74afbaa2 Merge branch 'improvement/S3C-3120_updateBucketGetObjectLockApi' into q/7.7.0 2020-07-02 02:41:59 +00:00
bert-e 555cd1a8ff Merge branches 'w/8.2/feature/S3C-3118-flatten-objmd-retention' and 'q/2794/8.1/feature/S3C-3118-flatten-objmd-retention' into tmp/octopus/q/8.2 2020-07-02 00:22:09 +00:00
bert-e 3e9f96fe39 Merge branches 'w/8.1/feature/S3C-3118-flatten-objmd-retention' and 'q/2794/7.8/feature/S3C-3118-flatten-objmd-retention' into tmp/octopus/q/8.1 2020-07-02 00:22:08 +00:00
bert-e 831da81ffe Merge branches 'w/7.7/feature/S3C-3118-flatten-objmd-retention' and 'q/2794/7.7.0/feature/S3C-3118-flatten-objmd-retention' into tmp/octopus/q/7.7 2020-07-02 00:22:08 +00:00
bert-e 0482eb83c7 Merge branch 'feature/S3C-3118-flatten-objmd-retention' into q/7.7.0 2020-07-02 00:22:07 +00:00
bert-e 29ff4f308b Merge branch 'w/8.1/improvement/S3C-3120_updateBucketGetObjectLockApi' into tmp/octopus/w/8.2/improvement/S3C-3120_updateBucketGetObjectLockApi 2020-07-01 23:58:30 +00:00
bert-e ece78c560e Merge branch 'w/7.8/improvement/S3C-3120_updateBucketGetObjectLockApi' into tmp/octopus/w/8.1/improvement/S3C-3120_updateBucketGetObjectLockApi 2020-07-01 23:58:30 +00:00
bert-e 1699917644 Merge branch 'improvement/S3C-3120_updateBucketGetObjectLockApi' into tmp/octopus/w/7.7/improvement/S3C-3120_updateBucketGetObjectLockApi 2020-07-01 23:58:29 +00:00
Ilke d725783014 improvement: S3C-3120 bucket get object lock use setter 2020-07-01 16:57:23 -07:00
bert-e 63209c1fd0 Merge branch 'w/8.1/feature/S3C-3118-flatten-objmd-retention' into tmp/octopus/w/8.2/feature/S3C-3118-flatten-objmd-retention 2020-07-01 23:13:10 +00:00
Dora Korpar 5a8ea20a00 Merge remote-tracking branch 'origin/w/7.8/feature/S3C-3118-flatten-objmd-retention' into w/8.1/feature/S3C-3118-flatten-objmd-retention 2020-07-01 16:07:17 -07:00
bert-e 11e85aeba8 Merge branch 'feature/S3C-3118-flatten-objmd-retention' into tmp/octopus/w/7.7/feature/S3C-3118-flatten-objmd-retention 2020-07-01 22:57:28 +00:00
Dora Korpar 3b7c0319c2 ft: S3C-3118 flatten objmd retention 2020-07-01 15:49:30 -07:00
bert-e 1c40688450 Merge branches 'w/8.2/improvement/S3C-3113_PutGetObjectLockConfigFuncTests' and 'q/2761/8.1/improvement/S3C-3113_PutGetObjectLockConfigFuncTests' into tmp/octopus/q/8.2 2020-07-01 19:25:51 +00:00
bert-e 184d4f195c Merge branches 'w/8.1/improvement/S3C-3113_PutGetObjectLockConfigFuncTests' and 'q/2761/7.8/improvement/S3C-3113_PutGetObjectLockConfigFuncTests' into tmp/octopus/q/8.1 2020-07-01 19:25:51 +00:00
bert-e c589b0cadd Merge branches 'w/7.7/improvement/S3C-3113_PutGetObjectLockConfigFuncTests' and 'q/2761/7.7.0/improvement/S3C-3113_PutGetObjectLockConfigFuncTests' into tmp/octopus/q/7.7 2020-07-01 19:25:50 +00:00
bert-e 1394f0aa52 Merge branch 'improvement/S3C-3113_PutGetObjectLockConfigFuncTests' into q/7.7.0 2020-07-01 19:25:50 +00:00
bert-e 7e778b0a73 Merge branches 'w/8.2/bugfix/S3C-3115-skip-external-tests' and 'q/2774/8.1/bugfix/S3C-3115-skip-external-tests' into tmp/octopus/q/8.2 2020-07-01 05:53:50 +00:00
bert-e 453631bf99 Merge branches 'w/8.1/bugfix/S3C-3115-skip-external-tests' and 'q/2774/7.8/bugfix/S3C-3115-skip-external-tests' into tmp/octopus/q/8.1 2020-07-01 05:53:49 +00:00
bert-e 83bb202c1b Merge branches 'w/7.7/bugfix/S3C-3115-skip-external-tests' and 'q/2774/7.7.0/bugfix/S3C-3115-skip-external-tests' into tmp/octopus/q/7.7 2020-07-01 05:53:48 +00:00
bert-e 5b766bc682 Merge branch 'bugfix/S3C-3115-skip-external-tests' into q/7.7.0 2020-07-01 05:53:48 +00:00
bert-e 469774c107 Merge branch 'w/8.1/bugfix/S3C-3115-skip-external-tests' into tmp/octopus/w/8.2/bugfix/S3C-3115-skip-external-tests 2020-06-30 23:33:53 +00:00
bert-e 8008b452c7 Merge branch 'w/7.8/bugfix/S3C-3115-skip-external-tests' into tmp/octopus/w/8.1/bugfix/S3C-3115-skip-external-tests 2020-06-30 23:33:52 +00:00
bert-e c7a8ffaee9 Merge branch 'bugfix/S3C-3115-skip-external-tests' into tmp/octopus/w/7.7/bugfix/S3C-3115-skip-external-tests 2020-06-30 23:33:51 +00:00
Rahul Padigela 7bcc5fbb30 bugfix: S3C-3115 skip ext. backend tests for Integration 2020-06-30 16:33:00 -07:00
bert-e 8a35787e01 Merge branch 'w/8.1/feature/S3C-3112_updateArsenalForObjLockSetter' into tmp/octopus/w/8.2/feature/S3C-3112_updateArsenalForObjLockSetter 2020-06-30 22:02:52 +00:00
Ilke 5e6a451fa3 Merge remote-tracking branch 'origin/w/7.8/feature/S3C-3112_updateArsenalForObjLockSetter' into w/8.1/feature/S3C-3112_updateArsenalForObjLockSetter 2020-06-30 14:49:53 -07:00
bert-e 840c0a5743 Merge branch 'feature/S3C-3112_updateArsenalForObjLockSetter' into tmp/octopus/w/7.7/feature/S3C-3112_updateArsenalForObjLockSetter 2020-06-30 21:48:19 +00:00
Ilke 45a2d32169 ft: S3C-3112 update arsenal 2020-06-30 14:47:27 -07:00
bert-e e179468afd Merge branch 'w/8.1/improvement/S3C-3113_PutGetObjectLockConfigFuncTests' into tmp/octopus/w/8.2/improvement/S3C-3113_PutGetObjectLockConfigFuncTests 2020-06-30 17:05:02 +00:00
bert-e 798fb01eb0 Merge branch 'w/7.8/improvement/S3C-3113_PutGetObjectLockConfigFuncTests' into tmp/octopus/w/8.1/improvement/S3C-3113_PutGetObjectLockConfigFuncTests 2020-06-30 17:05:01 +00:00
bert-e 79fe04ecd8 Merge branch 'improvement/S3C-3113_PutGetObjectLockConfigFuncTests' into tmp/octopus/w/7.7/improvement/S3C-3113_PutGetObjectLockConfigFuncTests 2020-06-30 17:05:00 +00:00
Ilke f7a9da1dc0 improvement: S3C-3113 put object lock config func tests 2020-06-30 10:03:47 -07:00
bert-e e15d1e9b10 Merge branch 'w/8.1/feature/S3C-2944_putObjectLegalHoldFunctionalTests' into tmp/octopus/w/8.2/feature/S3C-2944_putObjectLegalHoldFunctionalTests 2020-06-30 13:19:08 +00:00
bert-e cb26b99281 Merge branch 'w/7.8/feature/S3C-2944_putObjectLegalHoldFunctionalTests' into tmp/octopus/w/8.1/feature/S3C-2944_putObjectLegalHoldFunctionalTests 2020-06-30 13:19:08 +00:00
bert-e c5fe7b6314 Merge branch 'feature/S3C-2944_putObjectLegalHoldFunctionalTests' into tmp/octopus/w/7.7/feature/S3C-2944_putObjectLegalHoldFunctionalTests 2020-06-30 13:19:07 +00:00
Ilke 167ddc8d4d ft: S3C-2945 get object legal hold functional tests 2020-06-30 06:17:54 -07:00
Ilke 98d816dbbd ft: S3C-2944 put object legal hold functional tests 2020-06-30 05:31:27 -07:00
bert-e c674193a09 Merge branch 'w/8.1/feature/S3C-2788-obj-retention-func-tests' into tmp/octopus/w/8.2/feature/S3C-2788-obj-retention-func-tests 2020-06-30 00:48:07 +00:00
Dora Korpar 75e1dd1847 Merge remote-tracking branch 'origin/w/7.8/feature/S3C-2788-obj-retention-func-tests' into w/8.1/feature/S3C-2788-obj-retention-func-tests 2020-06-29 17:42:08 -07:00
bert-e b496783c37 Merge branch 'feature/S3C-2788-obj-retention-func-tests' into tmp/octopus/w/7.7/feature/S3C-2788-obj-retention-func-tests 2020-06-29 23:50:20 +00:00
Dora Korpar 39a3b945b3 ft: S3C-2788 put/get obj retention func tests 2020-06-29 16:46:55 -07:00
bert-e 8ce48bdc61 Merge branch 'w/8.1/feature/S3C-2802_addFuncTestToHeadObj' into tmp/octopus/w/8.2/feature/S3C-2802_addFuncTestToHeadObj 2020-06-24 16:41:17 +00:00
bert-e d9999981c5 Merge branch 'w/7.8/feature/S3C-2802_addFuncTestToHeadObj' into tmp/octopus/w/8.1/feature/S3C-2802_addFuncTestToHeadObj 2020-06-24 16:41:16 +00:00
Ilke 3133fe58bb [squash] linter 2020-06-24 09:40:57 -07:00
bert-e 50bbac994c Merge branch 'w/8.1/feature/S3C-2802_addFuncTestToHeadObj' into tmp/octopus/w/8.2/feature/S3C-2802_addFuncTestToHeadObj 2020-06-24 16:18:47 +00:00
bert-e 3bb1c5f4a3 Merge branch 'w/7.8/feature/S3C-2802_addFuncTestToHeadObj' into tmp/octopus/w/8.1/feature/S3C-2802_addFuncTestToHeadObj 2020-06-24 16:18:46 +00:00
Ilke 939bbadf75 ft: S3C-2802 add func test for head obj header 2020-06-24 09:15:03 -07:00
bert-e fd38774b90 Merge branch 'w/8.1/feature/S3C-2960-update-utapi' into tmp/octopus/w/8.2/feature/S3C-2960-update-utapi 2020-06-18 01:41:32 +00:00
Dora Korpar c80282bde7 Merge remote-tracking branch 'origin/w/7.8/feature/S3C-2960-update-utapi' into w/8.1/feature/S3C-2960-update-utapi 2020-06-17 18:39:43 -07:00
Dora Korpar 9d6d48c050 ft: S3C-2960 update utapi for object lock metrics 2020-06-17 18:28:09 -07:00
bert-e ada86583a6 Merge branches 'w/8.2/feature/S3C-2974-object-lock-delete-apis' and 'q/2634/8.1/feature/S3C-2974-object-lock-delete-apis' into tmp/octopus/q/8.2 2020-06-18 00:53:37 +00:00
bert-e 4f18682a47 Merge branches 'w/8.1/feature/S3C-2974-object-lock-delete-apis' and 'q/2634/7.8/feature/S3C-2974-object-lock-delete-apis' into tmp/octopus/q/8.1 2020-06-18 00:53:36 +00:00
bert-e f8ae24515a Merge branch 'feature/S3C-2974-object-lock-delete-apis' into q/7.7 2020-06-18 00:53:36 +00:00
Dora Korpar e0d0d39e1f Merge remote-tracking branch 'origin/w/8.1/feature/S3C-2974-object-lock-delete-apis' into w/8.2/feature/S3C-2974-object-lock-delete-apis 2020-06-17 16:34:58 -07:00
Dora Korpar d54caa1f1f Merge remote-tracking branch 'origin/w/7.8/feature/S3C-2974-object-lock-delete-apis' into w/8.1/feature/S3C-2974-object-lock-delete-apis 2020-06-17 16:31:40 -07:00
Dora Korpar f1354b500f ft:S3C-2974 update existing tests for locked delete 2020-06-17 16:09:37 -07:00
Dora Korpar d862655aec ft: S3C-2974 object lock delete apis 2020-06-17 16:09:37 -07:00
bert-e be6799bb5f Merge branches 'w/8.2/feature/S3C-3040_addObjLockActionsToBucketPolicy' and 'q/2683/8.1/feature/S3C-3040_addObjLockActionsToBucketPolicy' into tmp/octopus/q/8.2 2020-06-17 20:10:06 +00:00
bert-e fdd1331f08 Merge branches 'w/8.1/feature/S3C-3040_addObjLockActionsToBucketPolicy' and 'q/2683/7.8/feature/S3C-3040_addObjLockActionsToBucketPolicy' into tmp/octopus/q/8.1 2020-06-17 20:10:05 +00:00
bert-e e4a4283157 Merge branch 'feature/S3C-3040_addObjLockActionsToBucketPolicy' into q/7.7 2020-06-17 20:10:05 +00:00
bert-e 6db95c459b Merge branches 'w/8.2/feature/S3C-2792-mpu-object-retention' and 'q/2635/8.1/feature/S3C-2792-mpu-object-retention' into tmp/octopus/q/8.2 2020-06-17 19:23:09 +00:00
bert-e b925c4cca2 Merge branches 'w/8.1/feature/S3C-2792-mpu-object-retention' and 'q/2635/7.8/feature/S3C-2792-mpu-object-retention' into tmp/octopus/q/8.1 2020-06-17 19:23:09 +00:00
bert-e 7b3f8b22f5 Merge branch 'feature/S3C-2792-mpu-object-retention' into q/7.7 2020-06-17 19:23:08 +00:00
bert-e fb8ec2adc1 Merge branch 'w/8.1/feature/S3C-3040_addObjLockActionsToBucketPolicy' into tmp/octopus/w/8.2/feature/S3C-3040_addObjLockActionsToBucketPolicy 2020-06-17 17:58:25 +00:00
bert-e 063cc79104 Merge branch 'w/7.8/feature/S3C-3040_addObjLockActionsToBucketPolicy' into tmp/octopus/w/8.1/feature/S3C-3040_addObjLockActionsToBucketPolicy 2020-06-17 17:58:25 +00:00
Ilke cc18e083d2 ft: S3C-3040 add obj lock actions to bucket policy 2020-06-17 02:34:17 -07:00
bert-e 403a638356 Merge branch 'w/8.1/feature/S3C-2792-mpu-object-retention' into tmp/octopus/w/8.2/feature/S3C-2792-mpu-object-retention 2020-06-17 08:16:50 +00:00
Dora Korpar 872ac69d72 Merge remote-tracking branch 'origin/w/7.8/feature/S3C-2792-mpu-object-retention' into w/8.1/feature/S3C-2792-mpu-object-retention 2020-06-16 17:41:53 -07:00
Dora Korpar d4dafb9fcf ft:S3C-2792-mpu-object-retention 2020-06-16 16:45:46 -07:00
bert-e 631c0e243a Merge branches 'w/8.2/improvement/S3C-3044-audit-logs-arsenal-hash-update' and 'q/2643/8.1/improvement/S3C-3044-audit-logs-arsenal-hash-update' into tmp/octopus/q/8.2 2020-06-16 19:00:29 +00:00
bert-e 9ec36eb24d Merge branches 'w/8.1/improvement/S3C-3044-audit-logs-arsenal-hash-update' and 'q/2643/7.8/improvement/S3C-3044-audit-logs-arsenal-hash-update' into tmp/octopus/q/8.1 2020-06-16 19:00:28 +00:00
bert-e 3d93bc184d Merge branches 'w/7.7/improvement/S3C-3044-audit-logs-arsenal-hash-update' and 'q/2643/7.4/improvement/S3C-3044-audit-logs-arsenal-hash-update' into tmp/octopus/q/7.7 2020-06-16 19:00:28 +00:00
bert-e 3f72112a7f Merge branch 'improvement/S3C-3044-audit-logs-arsenal-hash-update' into q/7.4 2020-06-16 19:00:27 +00:00
bert-e 259331b579 Merge branch 'w/8.1/improvement/S3C-3044-audit-logs-arsenal-hash-update' into tmp/octopus/w/8.2/improvement/S3C-3044-audit-logs-arsenal-hash-update 2020-06-16 17:49:22 +00:00
bert-e 21e087def0 Merge branch 'w/7.8/improvement/S3C-3044-audit-logs-arsenal-hash-update' into tmp/octopus/w/8.1/improvement/S3C-3044-audit-logs-arsenal-hash-update 2020-06-16 17:49:21 +00:00
Anurag Mittal e964226bb4
Merge remote-tracking branch 'origin/improvement/S3C-3044-audit-logs-arsenal-hash-update' into w/7.7/improvement/S3C-3044-audit-logs-arsenal-hash-update 2020-06-16 19:48:23 +02:00
bert-e 653445f4cd Merge branch 'w/8.1/feature/S3C-2791_CopyObjectApiSupportsObjectLock' into tmp/octopus/w/8.2/feature/S3C-2791_CopyObjectApiSupportsObjectLock 2020-06-16 05:39:02 +00:00
Ilke 8f19bcd71f Merge remote-tracking branch 'origin/w/7.8/feature/S3C-2791_CopyObjectApiSupportsObjectLock' into w/8.1/feature/S3C-2791_CopyObjectApiSupportsObjectLock 2020-06-15 22:27:21 -07:00
Ilke 7d1b0dd2bd ft: S3C-2791 extend copy object for object lock 2020-06-15 22:09:23 -07:00
bert-e 3d313728e1 Merge branch 'w/8.1/feature/S3C-2946_GetObjectApiSupportsObjectLock' into tmp/octopus/w/8.2/feature/S3C-2946_GetObjectApiSupportsObjectLock 2020-06-16 03:25:08 +00:00
Ilke 78f060c19d Merge remote-tracking branch 'origin/w/7.8/feature/S3C-2946_GetObjectApiSupportsObjectLock' into w/8.1/feature/S3C-2946_GetObjectApiSupportsObjectLock 2020-06-15 20:17:54 -07:00
Ilke 3130c2f7ea ft: S3C-2946 get and head object support object lock 2020-06-15 20:11:17 -07:00
bert-e 6196024246 Merge branch 'w/8.1/bugfix/S3C-3062-skip-tests' into tmp/octopus/w/8.2/bugfix/S3C-3062-skip-tests 2020-06-15 21:13:30 +00:00
bert-e cdaafb9b20 Merge branch 'w/7.8/bugfix/S3C-3062-skip-tests' into tmp/octopus/w/8.1/bugfix/S3C-3062-skip-tests 2020-06-15 21:13:29 +00:00
bert-e 046184bb64 Merge branch 'w/7.4/bugfix/S3C-3062-skip-tests' into tmp/octopus/w/7.7/bugfix/S3C-3062-skip-tests 2020-06-15 21:13:29 +00:00
bert-e 42072e70be Merge branch 'bugfix/S3C-3062-skip-tests' into tmp/octopus/w/7.4/bugfix/S3C-3062-skip-tests 2020-06-15 21:13:28 +00:00
Anurag Mittal cb5bdf7e9f
improvement: S3C-3044-update-arsenal-hash 2020-06-15 20:44:16 +02:00
bert-e bc6490fc14 Merge branch 'w/8.2/feature/S3C-2791_PutObjectApiSupportsObjectLock' into tmp/octopus/q/8.2 2020-06-15 16:55:06 +00:00
bert-e f21bca0397 Merge branch 'w/8.1/feature/S3C-2791_PutObjectApiSupportsObjectLock' into tmp/octopus/w/8.2/feature/S3C-2791_PutObjectApiSupportsObjectLock 2020-06-15 16:19:15 +00:00
Ilke 82a7ffd5c1 fix unit test 2020-06-15 09:17:03 -07:00
bert-e e11928a0c7 Merge branch 'w/8.1/feature/S3C-2791_PutObjectApiSupportsObjectLock' into tmp/octopus/w/8.2/feature/S3C-2791_PutObjectApiSupportsObjectLock 2020-06-15 16:13:28 +00:00
Ilke 428632b943 Merge remote-tracking branch 'origin/w/7.8/feature/S3C-2791_PutObjectApiSupportsObjectLock' into w/8.1/feature/S3C-2791_PutObjectApiSupportsObjectLock 2020-06-15 09:07:40 -07:00
Ilke c3fd2ce5bb ft: S3C-2791 extend put object api for object lock 2020-06-15 08:06:42 -07:00
Rahul Padigela 6c453a8b03 bugfix: S3C-3062 fix tests in Integration 2020-06-14 22:55:40 -07:00
bert-e 8d9660b9ef Merge branch 'bugfix/ZENKO-2564/noProxyRequests' into q/8.2 2020-06-11 21:48:24 +00:00
bert-e 9ef8a05582 Merge branch 'w/8.1/feature/S3C-2788-get-obj-retention' into tmp/octopus/w/8.2/feature/S3C-2788-get-obj-retention 2020-06-10 07:27:17 +00:00
bert-e 754155c088 Merge branch 'w/7.8/feature/S3C-2788-get-obj-retention' into tmp/octopus/w/8.1/feature/S3C-2788-get-obj-retention 2020-06-10 07:27:16 +00:00
Dora Korpar 8c6d2685d4 ft: S3C-2788-get-object-retention 2020-06-10 00:12:47 -07:00
bert-e a8c731334a Merge branch 'w/8.1/bugfix/S3C-3018-multiobj-delete-error' into tmp/octopus/w/8.2/bugfix/S3C-3018-multiobj-delete-error 2020-06-10 01:03:01 +00:00
bert-e 6db9164b1a Merge branch 'w/7.8/bugfix/S3C-3018-multiobj-delete-error' into tmp/octopus/w/8.1/bugfix/S3C-3018-multiobj-delete-error 2020-06-10 01:03:01 +00:00
bert-e febea5b697 Merge branch 'w/7.4/bugfix/S3C-3018-multiobj-delete-error' into tmp/octopus/w/7.7/bugfix/S3C-3018-multiobj-delete-error 2020-06-10 01:03:00 +00:00
bert-e 605f1d3048 Merge branch 'bugfix/S3C-3018-multiobj-delete-error' into tmp/octopus/w/7.4/bugfix/S3C-3018-multiobj-delete-error 2020-06-10 01:02:59 +00:00
Dora Korpar ef8845609c bf: S3C-3018 multiobjdel enable utapi 2020-06-09 17:44:48 -07:00
bert-e 7e9e54dd6b Merge branch 'w/8.1/feature/S3C-2787-put-obj-retention' into tmp/octopus/w/8.2/feature/S3C-2787-put-obj-retention 2020-06-06 05:24:35 +00:00
Dora Korpar 5204fa7964 Merge remote-tracking branch 'origin/w/7.8/feature/S3C-2787-put-obj-retention' into w/8.1/feature/S3C-2787-put-obj-retention 2020-06-05 22:16:19 -07:00
Dora Korpar 1807dec24d ft:S3C-2787 update arsenal 2020-06-05 21:45:14 -07:00
Dora Korpar cc16517c31 ft: S3C-2787 put object retention 2020-06-05 21:43:09 -07:00
bert-e 8a96f2d971 Merge branch 'w/8.1/feature/S3C-2945_getObjectLegalHoldApi' into tmp/octopus/w/8.2/feature/S3C-2945_getObjectLegalHoldApi 2020-06-05 21:19:30 +00:00
bert-e 24b09ba587 Merge branch 'w/7.8/feature/S3C-2945_getObjectLegalHoldApi' into tmp/octopus/w/8.1/feature/S3C-2945_getObjectLegalHoldApi 2020-06-05 21:19:30 +00:00
Ilke 1e9271d9cc feature: S3C-2945 get object legal hold api 2020-06-05 14:04:51 -07:00
bert-e 77ae3346a7 Merge branch 'w/8.1/feature/S3C-2944_putObjectLegalHoldApi' into tmp/octopus/w/8.2/feature/S3C-2944_putObjectLegalHoldApi 2020-06-05 20:11:19 +00:00
Ilke 503698c96d Merge remote-tracking branch 'origin/w/7.8/feature/S3C-2944_putObjectLegalHoldApi' into w/8.1/feature/S3C-2944_putObjectLegalHoldApi 2020-06-05 13:03:28 -07:00
Ilke 9018d45557 feature: S3C-2944 put object legal hold api 2020-06-05 12:40:39 -07:00
bert-e b9839e00b9 Merge branch 'w/8.1/feature/S3C-2777-object-lock-test-plan' into tmp/octopus/w/8.2/feature/S3C-2777-object-lock-test-plan 2020-06-03 17:17:13 +00:00
bert-e 910342e592 Merge branch 'w/7.8/feature/S3C-2777-object-lock-test-plan' into tmp/octopus/w/8.1/feature/S3C-2777-object-lock-test-plan 2020-06-03 17:17:12 +00:00
Alexander Chan 6b9606a740 bugfix: ZENKO-2564 non-proxied request for metrics
remove the use of `request` module for backbeat metric requests
2020-06-01 20:23:07 +00:00
Dora Korpar 8bba56d717 ft: S3-2777 object lock test plan 2020-06-01 10:16:31 -07:00
bert-e b849b74beb Merge branch 'w/8.1/feature/S3C-2790_GetObjectLockConfigurationApi' into tmp/octopus/w/8.2/feature/S3C-2790_GetObjectLockConfigurationApi 2020-05-21 23:58:00 +00:00
bert-e 7f2731f8f2 Merge branch 'feature/S3C-2790_GetObjectLockConfigurationApi' into tmp/octopus/w/8.1/feature/S3C-2790_GetObjectLockConfigurationApi 2020-05-21 23:58:00 +00:00
Ilke 356d0bc2d2 [squash] linter 2020-05-21 16:57:43 -07:00
bert-e 8587a12bd4 Merge branch 'w/8.1/feature/S3C-2790_GetObjectLockConfigurationApi' into tmp/octopus/w/8.2/feature/S3C-2790_GetObjectLockConfigurationApi 2020-05-21 23:45:03 +00:00
bert-e 010bb8c063 Merge branch 'feature/S3C-2790_GetObjectLockConfigurationApi' into tmp/octopus/w/8.1/feature/S3C-2790_GetObjectLockConfigurationApi 2020-05-21 23:45:02 +00:00
Ilke 5dc44b10a0 [squash] linter 2020-05-21 16:44:47 -07:00
bert-e b9feecae7d Merge branch 'w/8.1/feature/S3C-2790_GetObjectLockConfigurationApi' into tmp/octopus/w/8.2/feature/S3C-2790_GetObjectLockConfigurationApi 2020-05-21 23:28:56 +00:00
Ilke b22b836638 Merge remote-tracking branch 'origin/feature/S3C-2790_GetObjectLockConfigurationApi' into w/8.1/feature/S3C-2790_GetObjectLockConfigurationApi 2020-05-21 16:26:34 -07:00
Ilke 0f822d7e6b ft: S3C-2790 get object lock configuration 2020-05-21 16:19:21 -07:00
bert-e edfc9d8ea1 Merge branch 'w/8.1/feature/S3C-2789-put-object-lock-configuration' into tmp/octopus/w/8.2/feature/S3C-2789-put-object-lock-configuration 2020-05-20 04:37:59 +00:00
Dora Korpar 192521d597 Merge remote-tracking branch 'origin/feature/S3C-2789-put-object-lock-configuration' into w/8.1/feature/S3C-2789-put-object-lock-configuration 2020-05-19 21:20:09 -07:00
bert-e 96af26c790 Merge branch 'bugfix/S3C-2959_ObjectLockAndNFSEnabledLocations' into tmp/octopus/w/8.2/bugfix/S3C-2959_ObjectLockAndNFSEnabledLocations 2020-05-19 16:49:44 +00:00
Ilke e1cb3c6b9c bf:S3C-2959 fix objectLock and NFSEnabled locations 2020-05-18 19:42:42 -07:00
Dora Korpar df049ff1f9 update ModelVersion readme 2020-05-18 14:31:18 -07:00
Dora Korpar 821cde9495 update Arsenal 2020-05-18 14:31:18 -07:00
Dora Korpar 0b51bb0297 ft: S3C 2789 put obj lock config api 2020-05-18 14:31:18 -07:00
bert-e 2d8a6bee93 Merge branch 'w/8.1/feature/S3C-2785_AddObjectLockForPutBucket' into tmp/octopus/w/8.2/feature/S3C-2785_AddObjectLockForPutBucket 2020-05-15 05:05:34 +00:00
Ilke e3b98648b8 Merge remote-tracking branch 'origin/feature/S3C-2785_AddObjectLockForPutBucket' into w/8.1/feature/S3C-2785_AddObjectLockForPutBucket 2020-05-14 21:53:10 -07:00
Ilke 5a071cc181 ft: S3C-2785 extend putBucket api for object lock 2020-05-14 18:22:32 -07:00
bert-e d33522db6b Merge branch 'w/8.1/bugfix/S3C-2880-update-aws-sdk' into tmp/octopus/w/8.2/bugfix/S3C-2880-update-aws-sdk 2020-05-14 19:24:53 +00:00
Dora Korpar 7287ef5243 Add suffix param in Promisify call 2020-05-14 12:24:23 -07:00
bert-e 0e1bee1d98 Merge branch 'w/8.1/bugfix/S3C-2880-update-aws-sdk' into tmp/octopus/w/8.2/bugfix/S3C-2880-update-aws-sdk 2020-05-14 19:05:41 +00:00
Dora Korpar f3136bfd0c -Async to -Promise 2020-05-14 12:04:48 -07:00
Dora Korpar de08fe3263 -Async to -Promise 2020-05-14 11:54:58 -07:00
bert-e da57d2ca42 Merge branch 'w/8.1/bugfix/S3C-2880-update-aws-sdk' into tmp/octopus/w/8.2/bugfix/S3C-2880-update-aws-sdk 2020-05-14 17:32:23 +00:00
Dora Korpar f23ffa4b01 Merge remote-tracking branch 'origin/w/7.7/bugfix/S3C-2880-update-aws-sdk' into w/8.1/bugfix/S3C-2880-update-aws-sdk 2020-05-14 10:24:19 -07:00
Dora Korpar a9d45d0545 Merge remote-tracking branch 'origin/bugfix/S3C-2880-update-aws-sdk' into w/7.7/bugfix/S3C-2880-update-aws-sdk 2020-05-13 15:04:04 -07:00
Dora Korpar aa34bfaf7a Move failing tests to different suites 2020-05-13 12:00:59 -07:00
Dora Korpar 6baacf4f55 bf: S3C-2880 update aws-sdk 2020-05-07 16:19:57 -07:00
bert-e b3324d9c9c Merge branch 'w/8.1/bugfix/S3C-2726-updateArsenalDependency' into tmp/octopus/w/8.2/bugfix/S3C-2726-updateArsenalDependency 2020-04-24 21:19:02 +00:00
Jonathan Gramain 9e44017d2f Merge remote-tracking branch 'origin/w/7.7/bugfix/S3C-2726-updateArsenalDependency' into w/8.1/bugfix/S3C-2726-updateArsenalDependency 2020-04-24 14:06:21 -07:00
Jonathan Gramain c2653b6cff Merge remote-tracking branch 'origin/w/7.6/bugfix/S3C-2726-updateArsenalDependency' into w/7.7/bugfix/S3C-2726-updateArsenalDependency 2020-04-24 12:23:03 -07:00
Jonathan Gramain 02f1be22b7 Merge remote-tracking branch 'origin/bugfix/S3C-2726-updateArsenalDependency' into w/7.6/bugfix/S3C-2726-updateArsenalDependency 2020-04-24 10:46:35 -07:00
Jonathan Gramain ad63251496 bugfix: S3C-2726 update arsenal dependency 2020-04-24 10:24:23 -07:00
Rahul Padigela fb0417e383 Merge remote-tracking branch 'origin/w/8.1/improvement/S3C-2804-remove-mongodb-tests' into w/8.2/improvement/S3C-2804-remove-mongodb-tests 2020-04-22 23:30:14 -07:00
Rahul Padigela 5c6c94e80d Merge remote-tracking branch 'origin/w/7.7/improvement/S3C-2804-remove-mongodb-tests' into w/8.1/improvement/S3C-2804-remove-mongodb-tests 2020-04-22 23:24:57 -07:00
bert-e f474b19fa7 Merge branch 'w/7.6/improvement/S3C-2804-remove-mongodb-tests' into tmp/octopus/w/7.7/improvement/S3C-2804-remove-mongodb-tests 2020-04-23 06:06:31 +00:00
Rahul Padigela 7d8dd9b8c0 Merge remote-tracking branch 'origin/improvement/S3C-2804-remove-mongodb-tests' into w/7.6/improvement/S3C-2804-remove-mongodb-tests 2020-04-22 22:50:28 -07:00
Rahul Padigela f55abf44d1 improvement: S3C-2508 remove mongodb backend 2020-04-22 21:54:31 -07:00
bert-e 9a0e1f53d6 Merge branch 'w/8.1/bugfix/S3C-2408/mpu-overwrite' into tmp/octopus/w/8.2/bugfix/S3C-2408/mpu-overwrite 2020-04-22 03:43:36 +00:00
Rahul Padigela f0423bbe11 Merge remote-tracking branch 'origin/w/7.7/bugfix/S3C-2408/mpu-overwrite' into w/8.1/bugfix/S3C-2408/mpu-overwrite 2020-04-21 20:30:47 -07:00
bert-e 3fe522447a Merge branch 'w/7.6/bugfix/S3C-2408/mpu-overwrite' into tmp/octopus/w/7.7/bugfix/S3C-2408/mpu-overwrite 2020-04-22 03:04:11 +00:00
Rahul Padigela 3a18fb7d9a Merge remote-tracking branch 'origin/bugfix/S3C-2408/mpu-overwrite' into w/7.6/bugfix/S3C-2408/mpu-overwrite 2020-04-21 19:47:56 -07:00
Rahul Padigela 0faba6cb7a bugfix: S3C-2408 update Utapi package 2020-04-21 19:32:59 -07:00
bennettbuchanan 371fa79454 bugfix: S3C-2408 MPU overwrite 2020-04-21 19:28:49 -07:00
bert-e 848262b6d7 Merge branches 'w/8.2/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' and 'q/2449/8.1/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' into tmp/octopus/q/8.2 2020-04-21 21:38:46 +00:00
bert-e 00772ea319 Merge branches 'w/8.1/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' and 'q/2449/7.7/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' into tmp/octopus/q/8.1 2020-04-21 21:38:45 +00:00
bert-e 13e9fa8491 Merge branches 'w/7.7/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' and 'q/2449/7.6/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' into tmp/octopus/q/7.7 2020-04-21 21:38:44 +00:00
bert-e eafedafb25 Merge branches 'w/7.6/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' and 'q/2449/7.4/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' into tmp/octopus/q/7.6 2020-04-21 21:38:44 +00:00
bert-e e1d28ad999 Merge branch 'bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' into q/7.4 2020-04-21 21:38:44 +00:00
Rahul Padigela 9dc41db70a Merge remote-tracking branch 'origin/w/8.1/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' into w/8.2/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix 2020-04-21 13:52:08 -07:00
Rahul Padigela 54daba70e1 Merge remote-tracking branch 'origin/w/7.7/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' into w/8.1/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix 2020-04-21 13:45:49 -07:00
Rahul Padigela ba6533ff5f Merge remote-tracking branch 'origin/w/7.6/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' into w/7.7/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix 2020-04-21 13:40:12 -07:00
Rahul Padigela 878dde3c09 Merge remote-tracking branch 'origin/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix' into w/7.6/bugfix/S3C-2668_update_arsenal_for_tagging_utf8_fix 2020-04-21 11:55:53 -07:00
bert-e 8af0c52695 Merge branch 'w/8.1/bugfix/S3C-2571_SupportAcceptRangesHeader' into tmp/octopus/w/8.2/bugfix/S3C-2571_SupportAcceptRangesHeader 2020-04-18 06:29:48 +00:00
bert-e 882c91c817 Merge branch 'w/7.7/bugfix/S3C-2571_SupportAcceptRangesHeader' into tmp/octopus/w/8.1/bugfix/S3C-2571_SupportAcceptRangesHeader 2020-04-18 06:29:47 +00:00
bert-e 450783ef92 Merge branch 'w/7.6/bugfix/S3C-2571_SupportAcceptRangesHeader' into tmp/octopus/w/7.7/bugfix/S3C-2571_SupportAcceptRangesHeader 2020-04-18 06:29:47 +00:00
bert-e 473796e0bb Merge branch 'bugfix/S3C-2571_SupportAcceptRangesHeader' into tmp/octopus/w/7.6/bugfix/S3C-2571_SupportAcceptRangesHeader 2020-04-18 06:29:46 +00:00
Ilke 606c9b2c35 bugfix: S3C-2571 enable accept ranges header 2020-04-17 23:12:18 -07:00
Ronnie Smith 039fbe4bdd bugfix: S3C 2668 Fix broken tests from regex changes 2020-04-16 12:51:13 -07:00
Ronnie Smith b043486617 bugfix: S3C-2668 Update arsenal for downstream fix 2020-04-15 16:14:51 -07:00
bert-e d18b9eb058 Merge branch 'w/8.1/improvement/S3C-2749_TestUrlDuration' into tmp/octopus/w/8.2/improvement/S3C-2749_TestUrlDuration 2020-04-13 22:55:10 +00:00
Ilke 27c064fb46 update arsenal 2020-04-13 15:54:37 -07:00
bert-e 9e7a26cb07 Merge branch 'w/8.1/improvement/S3C-2749_TestUrlDuration' into tmp/octopus/w/8.2/improvement/S3C-2749_TestUrlDuration 2020-04-13 22:27:09 +00:00
Ilke 002416038e update 2020-04-13 15:26:53 -07:00
bert-e a8c37200f6 Merge branch 'w/8.1/improvement/S3C-2749_TestUrlDuration' into tmp/octopus/w/8.2/improvement/S3C-2749_TestUrlDuration 2020-04-13 22:19:45 +00:00
Ilke 7c91b8242c Merge remote-tracking branch 'origin/improvement/S3C-2749_TestUrlDuration' into w/8.1/improvement/S3C-2749_TestUrlDuration 2020-04-13 15:13:39 -07:00
Ilke 478b978e23 update arsenal 2020-04-13 14:29:49 -07:00
bert-e 28cbcb755c Merge branch 'w/8.1/feature/S3C-2729-arsenal-update-for-7.7' into tmp/octopus/w/8.2/feature/S3C-2729-arsenal-update-for-7.7 2020-04-09 17:43:38 +00:00
bert-e 303f9e04d0 Merge branch 'feature/S3C-2729-arsenal-update-for-7.7' into tmp/octopus/w/8.1/feature/S3C-2729-arsenal-update-for-7.7 2020-04-09 17:43:37 +00:00
ilkescality 312b1c6f44
Merge branch 'development/7.7' into feature/S3C-2729-arsenal-update-for-7.7 2020-04-09 10:43:29 -07:00
bert-e c01d24a7c9 Merge branches 'development/8.2' and 'w/8.1/feature/S3C-2729-arsenal-update-for-7.7' into tmp/octopus/w/8.2/feature/S3C-2729-arsenal-update-for-7.7 2020-04-08 23:35:09 +00:00
ilkescality 663b8fb889
Merge branch 'development/8.1' into w/8.1/feature/S3C-2729-arsenal-update-for-7.7 2020-04-08 16:34:59 -07:00
bert-e 5741387682 Merge branch 'w/8.1/bugfix/S3C-2542_add_default_logger_fields_for_backbeat_routes' into tmp/octopus/w/8.2/bugfix/S3C-2542_add_default_logger_fields_for_backbeat_routes 2020-04-07 20:16:02 +00:00
Taylor McKinnon 84c1ac06da Merge remote-tracking branch 'origin/w/7.7/bugfix/S3C-2542_add_default_logger_fields_for_backbeat_routes' into w/8.1/bugfix/S3C-2542_add_default_logger_fields_for_backbeat_routes 2020-04-07 13:15:33 -07:00
bert-e b3e1a94be1 Merge branch 'bugfix/S3C-2542_add_default_logger_fields_for_backbeat_routes' into tmp/octopus/w/7.6/bugfix/S3C-2542_add_default_logger_fields_for_backbeat_routes 2020-04-07 20:10:37 +00:00
bert-e 111ccddab9 Merge branch 'w/8.1/bugfix/S3C-2756_update_pykmip_ci_certificates' into tmp/octopus/w/8.2/bugfix/S3C-2756_update_pykmip_ci_certificates 2020-04-07 18:07:08 +00:00
bert-e cc1b341147 Merge branch 'w/7.7/bugfix/S3C-2756_update_pykmip_ci_certificates' into tmp/octopus/w/8.1/bugfix/S3C-2756_update_pykmip_ci_certificates 2020-04-07 18:07:08 +00:00
Taylor McKinnon e6c942e3ea bf(S3C-2756): Update pykmip certificates used in CI 2020-04-07 11:06:09 -07:00
bert-e 85c02af52c Merge branch 'w/8.1/feature/S3C-2729-arsenal-update-for-7.7' into tmp/octopus/w/8.2/feature/S3C-2729-arsenal-update-for-7.7 2020-04-07 00:36:50 +00:00
Ilke 38ff53c2a0 Fix arsenal version for 8.1 2020-04-06 17:36:36 -07:00
bert-e 985e6567f2 Merge branch 'w/8.1/feature/S3C-2729-arsenal-update-for-7.7' into tmp/octopus/w/8.2/feature/S3C-2729-arsenal-update-for-7.7 2020-04-06 19:42:42 +00:00
Ilke d7a4f9299a Merge remote-tracking branch 'origin/feature/S3C-2729-arsenal-update-for-7.7' into w/8.1/feature/S3C-2729-arsenal-update-for-7.7 2020-04-06 12:36:05 -07:00
Taylor McKinnon eee0b4982a bf(S3C-2542): add default logger fields for backbeat routes 2020-04-06 12:06:16 -07:00
Ilke 05a10aac32 feature/S3C-2729: update arsenal dependency version
Update Arsenal dependency version to make use of S3 pre-sign
URL duration configuration
2020-04-03 18:13:45 -07:00
bert-e 22d4ae8c04 Merge branch 'w/8.1/bugfix/S3C-2714-fix-permissions' into tmp/octopus/w/8.2/bugfix/S3C-2714-fix-permissions 2020-03-19 22:14:01 +00:00
bert-e 9ec7252d29 Merge branch 'w/7.7/bugfix/S3C-2714-fix-permissions' into tmp/octopus/w/8.1/bugfix/S3C-2714-fix-permissions 2020-03-19 22:14:00 +00:00
bert-e d546aecab8 Merge branch 'bugfix/S3C-2714-fix-permissions' into tmp/octopus/w/7.6/bugfix/S3C-2714-fix-permissions 2020-03-19 22:13:59 +00:00
Dora Korpar c536f5f3a4 bf: S3C 2714 update permissions tests for no bucketOwnerActions 2020-03-19 12:45:26 -07:00
Dora Korpar c506d77985 bf: S3C 2714 remove 'bucketOwnerAction' from bucket perms and fix obj logic 2020-03-19 12:45:26 -07:00
bert-e 345d126488 Merge branch 'bugfix/ZENKO-1670-propagate-updates-if-master-version-is-gte' into tmp/octopus/w/8.2/bugfix/ZENKO-1670-propagate-updates-if-master-version-is-gte 2020-03-19 01:30:16 +00:00
Rahul Padigela 7bcf78634d bugfix: ZENKO-1670-propagate-updates-if-master-version 2020-03-18 18:28:34 -07:00
bert-e b20785150c Merge branch 'w/8.1/bugfix/S3C-2697-acl-perms-regression' into tmp/octopus/w/8.2/bugfix/S3C-2697-acl-perms-regression 2020-03-17 17:33:42 +00:00
Dora Korpar 7f50dfee79 Merge remote-tracking branch 'origin/w/7.7/bugfix/S3C-2697-acl-perms-regression' into w/8.1/bugfix/S3C-2697-acl-perms-regression 2020-03-17 10:22:11 -07:00
bert-e 656c0baca6 Merge branch 'bugfix/S3C-2697-acl-perms-regression' into tmp/octopus/w/7.6/bugfix/S3C-2697-acl-perms-regression 2020-03-16 17:02:02 +00:00
Dora Korpar 77f8c672ad bf: S3C-2697 acl permissions regression 2020-03-11 12:03:27 -07:00
bert-e bfc2648257 Merge branch 'bugfix/ZENKO-2352-httpError424IfLocationDoesNotExist' into tmp/octopus/w/8.2/bugfix/ZENKO-2352-httpError424IfLocationDoesNotExist 2020-02-27 23:14:02 +00:00
Jonathan Gramain a694f210e6 bugfix: ZENKO-2352 arsenal dependency update
Also remove an error log when LocationNotFound error is returned by
the cloud backend.
2020-02-27 15:13:42 -08:00
bert-e 6d9f3638b3 Merge branches 'w/8.2/bugfix/S3C-2502-vault-req-ip-header-port' and 'q/2378/8.1/bugfix/S3C-2502-vault-req-ip-header-port' into tmp/octopus/q/8.2 2020-02-26 19:43:24 +00:00
bert-e 7f5caa7da6 Merge branches 'w/8.1/bugfix/S3C-2502-vault-req-ip-header-port' and 'q/2378/7.7/bugfix/S3C-2502-vault-req-ip-header-port' into tmp/octopus/q/8.1 2020-02-26 19:43:24 +00:00
bert-e 38a27756e6 Merge branches 'w/7.6/bugfix/S3C-2502-vault-req-ip-header-port' and 'q/2378/7.6.0/bugfix/S3C-2502-vault-req-ip-header-port' into tmp/octopus/q/7.6 2020-02-26 19:43:23 +00:00
bert-e 9907b68abd Merge branch 'bugfix/S3C-2502-vault-req-ip-header-port' into q/7.6.0 2020-02-26 19:43:22 +00:00
bert-e dff6e95e44 Merge branch 'w/8.1/bugfix/S3C-2502-vault-req-ip-header-port' into tmp/octopus/w/8.2/bugfix/S3C-2502-vault-req-ip-header-port 2020-02-26 19:06:21 +00:00
Dora Korpar ff128a1713 Merge remote-tracking branch 'origin/w/7.7/bugfix/S3C-2502-vault-req-ip-header-port' into w/8.1/bugfix/S3C-2502-vault-req-ip-header-port 2020-02-26 11:05:06 -08:00
Dora Korpar 7dbf894785 Merge remote-tracking branch 'origin/bugfix/S3C-2502-vault-req-ip-header-port' into w/7.6/bugfix/S3C-2502-vault-req-ip-header-port 2020-02-26 10:46:48 -08:00
bert-e 2c7ed368cf Merge branches 'w/8.2/bugfix/S3C-2544-diff-acct-metrics-port-2' and 'q/2363/8.1/bugfix/S3C-2544-diff-acct-metrics-port-2' into tmp/octopus/q/8.2 2020-02-26 00:47:04 +00:00
bert-e b9c96fd052 Merge branches 'w/8.1/bugfix/S3C-2544-diff-acct-metrics-port-2' and 'q/2363/7.7/bugfix/S3C-2544-diff-acct-metrics-port-2' into tmp/octopus/q/8.1 2020-02-26 00:47:03 +00:00
bert-e b15b1b645d Merge branch 'bugfix/S3C-2544-diff-acct-metrics-port-2' into q/7.6.0 2020-02-26 00:47:02 +00:00
bert-e 001f4d95eb Merge branches 'w/7.6/bugfix/S3C-2544-diff-acct-metrics-port-2' and 'q/2363/7.6.0/bugfix/S3C-2544-diff-acct-metrics-port-2' into tmp/octopus/q/7.6 2020-02-26 00:47:02 +00:00
Dora Korpar ff90d7f2fa bf: S3C 2502 move ip util to arsenal
(cherry picked from commit 1fd8d0e601)
2020-02-25 16:02:54 -08:00
bert-e 513fa24031 Merge branches 'w/8.2/bugfix/S3C-2597-bucket-policy-can-id-port' and 'q/2368/8.1/bugfix/S3C-2597-bucket-policy-can-id-port' into tmp/octopus/q/8.2 2020-02-25 23:37:40 +00:00
bert-e 4644fdc30a Merge branches 'w/8.1/bugfix/S3C-2597-bucket-policy-can-id-port' and 'q/2368/7.7/bugfix/S3C-2597-bucket-policy-can-id-port' into tmp/octopus/q/8.1 2020-02-25 23:37:39 +00:00
bert-e d345466e1b Merge branch 'w/7.6/bugfix/S3C-2597-bucket-policy-can-id-port' into tmp/octopus/q/7.6 2020-02-25 23:37:39 +00:00
bert-e eef183edf2 Merge branch 'w/8.1/bugfix/S3C-2597-bucket-policy-can-id-port' into tmp/octopus/w/8.2/bugfix/S3C-2597-bucket-policy-can-id-port 2020-02-25 06:08:13 +00:00
bert-e e6350ea3e8 Merge branch 'w/7.7/bugfix/S3C-2597-bucket-policy-can-id-port' into tmp/octopus/w/8.1/bugfix/S3C-2597-bucket-policy-can-id-port 2020-02-25 06:08:12 +00:00
bert-e 930edd8718 Merge branch 'bugfix/S3C-2597-bucket-policy-can-id-port' into tmp/octopus/w/7.6/bugfix/S3C-2597-bucket-policy-can-id-port 2020-02-25 06:08:11 +00:00
Dora Korpar 7b340f621b S3C-2597-bucket-policy-can-id
(cherry picked from commit 5cf12f2cdc)
2020-02-24 22:07:21 -08:00
bert-e 1137e3a6b6 Merge branch 'w/8.1/bugfix/S3C-2544-diff-acct-metrics-port-2' into tmp/octopus/w/8.2/bugfix/S3C-2544-diff-acct-metrics-port-2 2020-02-25 06:01:11 +00:00
bert-e 5096cbc3e3 Merge branch 'w/7.7/bugfix/S3C-2544-diff-acct-metrics-port-2' into tmp/octopus/w/8.1/bugfix/S3C-2544-diff-acct-metrics-port-2 2020-02-25 06:01:11 +00:00
bert-e 86876c1489 Merge branch 'bugfix/S3C-2544-diff-acct-metrics-port-2' into tmp/octopus/w/7.6/bugfix/S3C-2544-diff-acct-metrics-port-2 2020-02-25 06:01:10 +00:00
Dora Korpar 23f2709c93 bf: S3C-2544 fix metric handling on put/del obj
(cherry picked from commit e6449e4a1a)
2020-02-24 22:00:08 -08:00
bert-e e7835064dc Merge branches 'w/8.2/bugfix/S3C-2502-vault-req-ip-header' and 'q/2333/8.1/bugfix/S3C-2502-vault-req-ip-header' into tmp/octopus/q/8.2 2020-02-24 20:43:46 +00:00
bert-e 7720a82fd9 Merge branches 'w/8.1/bugfix/S3C-2502-vault-req-ip-header' and 'q/2333/7.7/bugfix/S3C-2502-vault-req-ip-header' into tmp/octopus/q/8.1 2020-02-24 20:43:46 +00:00
bert-e 2d34f67a5a Merge branch 'bugfix/S3C-2502-vault-req-ip-header' into q/7.4 2020-02-24 20:43:44 +00:00
bert-e 1e2656d34a Merge branches 'w/7.6/bugfix/S3C-2502-vault-req-ip-header' and 'q/2333/7.4/bugfix/S3C-2502-vault-req-ip-header' into tmp/octopus/q/7.6 2020-02-24 20:43:44 +00:00
bert-e d72319c732 Merge branches 'development/8.2' and 'w/8.1/bugfix/S3C-2502-vault-req-ip-header' into tmp/octopus/w/8.2/bugfix/S3C-2502-vault-req-ip-header 2020-02-24 20:05:40 +00:00
bert-e 74935b7bb1 Merge branch 'w/7.7/bugfix/S3C-2502-vault-req-ip-header' into tmp/octopus/w/8.1/bugfix/S3C-2502-vault-req-ip-header 2020-02-24 20:05:39 +00:00
bert-e 18e656c00a Merge branch 'development/7.6' into tmp/octopus/w/7.6/bugfix/S3C-2502-vault-req-ip-header 2020-02-24 20:05:38 +00:00
Dora Korpar 5b9f414bbb Merge remote-tracking branch 'origin/w/7.7/bugfix/S3C-2502-vault-req-ip-header' into w/8.1/bugfix/S3C-2502-vault-req-ip-header 2020-02-24 12:05:19 -08:00
Dora Korpar f51b2d373c Merge remote-tracking branch 'origin/development/8.1' into w/8.1/bugfix/S3C-2502-vault-req-ip-header 2020-02-24 12:01:44 -08:00
bert-e c6b1f214c6 Merge branch 'improvement/ZENKO-2340-rename-zenko-registry-namespace' into tmp/octopus/w/8.2/improvement/ZENKO-2340-rename-zenko-registry-namespace 2020-02-24 14:26:10 +00:00
Thomas Carmet 7f65d5f417 ci: ZENKO-2340 rename the docker registry namespace from zenko-dev to zenko 2020-02-24 15:25:16 +01:00
bert-e 3c975e181a Merge branch 'w/8.1/bugfix/S3C-2597-bucket-policy-can-id' into tmp/octopus/w/8.2/bugfix/S3C-2597-bucket-policy-can-id 2020-02-21 07:04:20 +00:00
Dora Korpar 7b74202223 fix test 2020-02-20 23:04:00 -08:00
bert-e 8ecdd07fd6 Merge branch 'w/8.1/bugfix/S3C-2597-bucket-policy-can-id' into tmp/octopus/w/8.2/bugfix/S3C-2597-bucket-policy-can-id 2020-02-20 19:29:43 +00:00
Dora Korpar 21beb7fcdf Merge remote-tracking branch 'origin/w/7.7/bugfix/S3C-2597-bucket-policy-can-id' into w/8.1/bugfix/S3C-2597-bucket-policy-can-id 2020-02-20 11:26:34 -08:00
Dora Korpar 5cf12f2cdc S3C-2597-bucket-policy-can-id 2020-02-20 10:55:55 -08:00
bert-e 7af23a1526 Merge branch 'w/8.1/bugfix/S3C-2544-diff-acct-metrics' into tmp/octopus/w/8.2/bugfix/S3C-2544-diff-acct-metrics 2020-02-14 06:23:11 +00:00
bert-e 27ed0058a6 Merge branch 'w/7.7/bugfix/S3C-2544-diff-acct-metrics' into tmp/octopus/w/8.1/bugfix/S3C-2544-diff-acct-metrics 2020-02-14 06:23:11 +00:00
bert-e e2b8666ec6 Merge branch 'bugfix/S3C-2544-diff-acct-metrics' into tmp/octopus/w/7.6/bugfix/S3C-2544-diff-acct-metrics 2020-02-14 06:23:10 +00:00
Dora Korpar e6449e4a1a bf: S3C-2544 fix metric handling on put/del obj 2020-02-13 22:02:20 -08:00
Dora Korpar 86c9f54541 fix arsenal version 2020-02-06 16:42:54 -08:00
Dora Korpar 6091cf1b36 fix arsenal version 2020-02-06 16:41:39 -08:00
Dora Korpar 6c5c3f9ba5 fix arsenal version 2020-02-06 16:17:12 -08:00
bert-e 2f7eb3b9be Merge branch 'w/8.1/bugfix/S3C-2502-vault-req-ip-header' into tmp/octopus/w/8.2/bugfix/S3C-2502-vault-req-ip-header 2020-02-06 23:32:29 +00:00
Dora Korpar 059299f981 Merge remote-tracking branch 'origin/w/7.7/bugfix/S3C-2502-vault-req-ip-header' into w/8.1/bugfix/S3C-2502-vault-req-ip-header 2020-02-06 15:18:47 -08:00
Dora Korpar 11c709dd6d Merge remote-tracking branch 'origin/bugfix/S3C-2502-vault-req-ip-header' into w/7.6/bugfix/S3C-2502-vault-req-ip-header 2020-02-06 10:13:28 -08:00
Dora Korpar 1fd8d0e601 bf: S3C 2502 move ip util to arsenal 2020-01-23 14:59:29 -08:00
bert-e fdcbd315d0 Merge branch 'w/8.1/bugfix/S3C-2564-kms-remove-ssl-option-quotes' into tmp/octopus/w/8.2/bugfix/S3C-2564-kms-remove-ssl-option-quotes 2020-01-17 08:44:46 +00:00
bert-e 6b4671b291 Merge branch 'w/7.6/bugfix/S3C-2564-kms-remove-ssl-option-quotes' into tmp/octopus/w/8.1/bugfix/S3C-2564-kms-remove-ssl-option-quotes 2020-01-17 08:44:46 +00:00
bert-e 8117dd02e3 Merge branch 'bugfix/S3C-2564-kms-remove-ssl-option-quotes' into tmp/octopus/w/7.6/bugfix/S3C-2564-kms-remove-ssl-option-quotes 2020-01-17 08:44:45 +00:00
Remi Debay 396f100aca S3C-2564: KMS - fix typo in ssl option
Remove unneeded quotes.
2020-01-17 09:43:40 +01:00
bert-e 605fdd9254 Merge branch 'w/8.1/bugfix/S3C-2503-allow-same-email-acl' into tmp/octopus/w/8.2/bugfix/S3C-2503-allow-same-email-acl 2020-01-16 21:18:16 +00:00
bert-e e77aeeb3a5 Merge branch 'w/7.6/bugfix/S3C-2503-allow-same-email-acl' into tmp/octopus/w/8.1/bugfix/S3C-2503-allow-same-email-acl 2020-01-16 21:18:15 +00:00
bert-e b9df3094e2 Merge branch 'bugfix/S3C-2503-allow-same-email-acl' into tmp/octopus/w/7.6/bugfix/S3C-2503-allow-same-email-acl 2020-01-16 21:18:15 +00:00
Jianqin Wang 8d26509382 bugfix: ZENKO-2503 allow ACLs with same email in single request 2020-01-16 13:15:52 -08:00
bert-e 4bf24747c8 Merge branch 'w/8.1/bugfix/S3C-2582-update-maven-repo' into tmp/octopus/w/8.2/bugfix/S3C-2582-update-maven-repo 2020-01-16 18:17:09 +00:00
bert-e 85e178d0fa Merge branch 'w/7.6/bugfix/S3C-2582-update-maven-repo' into tmp/octopus/w/8.1/bugfix/S3C-2582-update-maven-repo 2020-01-16 18:17:09 +00:00
bert-e 93101d733e Merge branch 'bugfix/S3C-2582-update-maven-repo' into tmp/octopus/w/7.6/bugfix/S3C-2582-update-maven-repo 2020-01-16 18:17:08 +00:00
Thomas Carmet 46a304d35d S3C-2582 Setting up maven repo on https instead of http 2020-01-16 19:15:25 +01:00
bert-e d4a7c1d7c8 Merge branch 'bugfix/ZENKO-2277-fix-cloudserver-image-build' into tmp/octopus/w/8.2/bugfix/ZENKO-2277-fix-cloudserver-image-build 2020-01-13 09:19:53 +00:00
Thomas Carmet 1e73d90126 ZENKO-2277 moving away from a floating tag on Docker image 2020-01-13 10:19:05 +01:00
bert-e 8e5353929c Merge branch 'bugfix/ZENKO-2277-fix-cloudserver-image-build' into tmp/octopus/w/8.2/bugfix/ZENKO-2277-fix-cloudserver-image-build 2020-01-10 16:26:27 +00:00
Thomas Carmet bea2bf9d60 ZENKO-2277 installing curl and gpg due to parent image upgrade
as the image node:10-slim received an update and removed the commands
curl and gpg, we lost the ability to build cloudserver.
We're now installing them specifically.
2020-01-10 17:24:21 +01:00
bert-e a240db4e03 Merge branches 'w/8.2/bugfix/ZENKO-2200/report' and 'q/2231/8.1/bugfix/ZENKO-2200/report' into tmp/octopus/q/8.2 2020-01-03 20:21:28 +00:00
bert-e a4de12e331 Merge branch 'bugfix/ZENKO-2200/report' into q/8.1 2020-01-03 20:21:28 +00:00
bert-e 0b7e2e6ed8 Merge branch 'w/8.1/bugfix/ZENKOIO-20/update-aws-sdk' into tmp/octopus/w/8.2/bugfix/ZENKOIO-20/update-aws-sdk 2020-01-02 22:39:34 +00:00
Dora Korpar 1f2174e224 Merge remote-tracking branch 'origin/w/7.6/bugfix/ZENKOIO-20/update-aws-sdk' into w/8.1/bugfix/ZENKOIO-20/update-aws-sdk 2020-01-02 14:25:13 -08:00
Dora Korpar 3c87fd29c4 Merge remote-tracking branch 'origin/bugfix/ZENKOIO-20/update-aws-sdk' into w/7.6/bugfix/ZENKOIO-20/update-aws-sdk 2020-01-02 13:43:48 -08:00
Dora Korpar c888960b0f bf: ZENKOIO 20 update aws sdk 2019-12-31 10:29:25 -08:00
bert-e ed4cd8cad0 Merge branches 'w/8.2/bugfix/S3C-2527-sproxydclientDependencyUpdate' and 'q/2293/8.1/bugfix/S3C-2527-sproxydclientDependencyUpdate' into tmp/octopus/q/8.2 2019-12-30 21:48:11 +00:00
bert-e 3239927d1a Merge branch 'w/8.1/bugfix/S3C-2527-sproxydclientDependencyUpdate' into tmp/octopus/q/8.1 2019-12-30 21:48:10 +00:00
bert-e 5e9b57b883 Merge branch 'bugfix/ZENKO-2261-arsenalDependencyUpdate' into tmp/octopus/w/8.2/bugfix/ZENKO-2261-arsenalDependencyUpdate 2019-12-30 20:19:39 +00:00
Jonathan Gramain b6cc788e97 bugfix: ZENKO-2261 effectively reuse sproxyd connections
Dependency update on Arsenal, which transitively depends on updated
sproxydclient.
2019-12-30 12:08:00 -08:00
bert-e 97c95f3bd9 Merge branch 'w/8.1/bugfix/S3C-2527-sproxydclientDependencyUpdate' into tmp/octopus/w/8.2/bugfix/S3C-2527-sproxydclientDependencyUpdate 2019-12-30 19:29:18 +00:00
Jonathan Gramain 9529fda244 Merge remote-tracking branch 'origin/w/7.6/bugfix/S3C-2527-sproxydclientDependencyUpdate' into w/8.1/bugfix/S3C-2527-sproxydclientDependencyUpdate 2019-12-30 11:22:22 -08:00
Jonathan Gramain e4fbefb2b3 Merge remote-tracking branch 'origin/bugfix/S3C-2527-sproxydclientDependencyUpdate' into w/7.6/bugfix/S3C-2527-sproxydclientDependencyUpdate 2019-12-30 10:53:06 -08:00
Jonathan Gramain 5ec0875ad3 bugfix: S3C-2527 effectively reuse sproxyd connections
Dependency update on sproxydclient repo
2019-12-30 10:49:25 -08:00
bert-e 4332d0a10f Merge branch 'bugfix/ZENKO-2250-reworkChunkedUploadStreamHandling' into tmp/octopus/w/8.2/bugfix/ZENKO-2250-reworkChunkedUploadStreamHandling 2019-12-11 18:49:40 +00:00
Jonathan Gramain 16b65ed208 bugfix: ZENKO-2250 rework chunked upload stream handling
Update Arsenal dependency and re-enable streaming V4 functional test
2019-12-11 10:45:52 -08:00
bert-e a78da8c24c Merge branch 'w/8.1/bugfix/S3C-2504-cleanupBucketAfterTest' into tmp/octopus/w/8.2/bugfix/S3C-2504-cleanupBucketAfterTest 2019-12-11 01:21:09 +00:00
Jonathan Gramain d3d8229664 Merge remote-tracking branch 'origin/w/7.6/bugfix/S3C-2504-cleanupBucketAfterTest' into w/8.1/bugfix/S3C-2504-cleanupBucketAfterTest 2019-12-10 17:18:28 -08:00
bert-e 6f0b50a96c Merge branch 'bugfix/S3C-2504-cleanupBucketAfterTest' into tmp/octopus/w/7.6/bugfix/S3C-2504-cleanupBucketAfterTest 2019-12-11 01:16:09 +00:00
Jonathan Gramain 9c524ca9e5 bugfix: S3C-2504 cleanup test bucket
Delete bucket after streaming V4 test to avoid side-effects with other
tests (observed on 8.1+ branches with mongodb
reportHandler::countItems test).
2019-12-10 17:14:05 -08:00
bert-e 4b8b60c7c1 Merge branch 'w/8.1/bugfix/S3C-2504-revertYarnChangeOnUtapiDependencies' into tmp/octopus/w/8.2/bugfix/S3C-2504-revertYarnChangeOnUtapiDependencies 2019-12-10 22:50:24 +00:00
bert-e 985f49b5cf Merge branch 'w/7.6/bugfix/S3C-2504-revertYarnChangeOnUtapiDependencies' into tmp/octopus/w/8.1/bugfix/S3C-2504-revertYarnChangeOnUtapiDependencies 2019-12-10 22:50:23 +00:00
bert-e b16fc06d90 Merge branch 'bugfix/S3C-2504-revertYarnChangeOnUtapiDependencies' into tmp/octopus/w/7.6/bugfix/S3C-2504-revertYarnChangeOnUtapiDependencies 2019-12-10 14:44:58 -08:00
Jonathan Gramain 71186d618a bugfix: S3C-2504 revert changes in UTAPI dependencies
Revert changes introduced to UTAPI dependencies and to "joi"
dependency in yarn.lock in PR #2247.

Those changes are suspicious and not needed, so safer to revert them.
2019-12-10 14:27:08 -08:00
bert-e 7351a778cb Merge branch 'w/8.1/bugfix/S3C-2504-reworkChunkedUploadStreamHandling' into tmp/octopus/w/8.2/bugfix/S3C-2504-reworkChunkedUploadStreamHandling 2019-12-10 19:38:31 +00:00
Jonathan Gramain 3e2d1f7ab3 Merge remote-tracking branch 'origin/w/7.6/bugfix/S3C-2504-reworkChunkedUploadStreamHandling' into w/8.1/bugfix/S3C-2504-reworkChunkedUploadStreamHandling 2019-12-10 11:26:46 -08:00
Jonathan Gramain 84ccffea72 Merge remote-tracking branch 'origin/bugfix/S3C-2504-reworkChunkedUploadStreamHandling' into w/7.6/bugfix/S3C-2504-reworkChunkedUploadStreamHandling 2019-12-09 16:55:10 -08:00
Jonathan Gramain 53d382f5c3 bugfix: S3C-2504 rework chunked upload stream handling
- make sure the V4Transform._transform() callback is always called
  even in case of error. When there is a client error (like a chunked
  stream signature mismatch), continue consuming the data but do not
  send it to sproxyd. Do not emit an 'error' event as this confuses
  the HTTP stack and retains bad client connections,

- instead of an 'error' event, emit a custom 'clientError' event on
  V4Transform in case of client error to allow a cleanup of sproxyd
  connections,

- depend on a fix of sproxydclient that aborts sproxyd requests when
  the input stream is closed.
2019-12-09 16:38:27 -08:00
bert-e 0f999147e5 Merge branches 'w/8.2/feature/ZENKO-2238-post-merge-deploy-to-new-registry' and 'q/2265/8.1/feature/ZENKO-2238-post-merge-deploy-to-new-registry' into tmp/octopus/q/8.2 2019-12-03 00:54:51 +00:00
bert-e cdfda7c8fb Merge branch 'feature/ZENKO-2238-post-merge-deploy-to-new-registry' into q/8.1 2019-12-03 00:54:50 +00:00
bert-e dbdbf3c2c2 Merge branch 'feature/ZENKO-2238-post-merge-deploy-to-new-registry' into tmp/octopus/w/8.2/feature/ZENKO-2238-post-merge-deploy-to-new-registry 2019-12-02 22:50:30 +00:00
Thomas Carmet 319988c310 ZENKO-2238 changing worker type to kube pod
An opinionated change regarding the build environment, we may
require more tooling in the future when building cloudserver
so it is better that we stop executing docker commands on eve's
backend.
2019-11-27 16:04:25 -08:00
Thomas Carmet 3b111ce5d1 ZENKO-2238 now pushing docker image to registry.scality.com
Creating a zenko-dev workspace for all zenko related images.

We're also removing push that were made to docker hub because
the tagging was badly made and we're also letting the docker hub
build system handling that part for now.
2019-11-27 16:04:25 -08:00
bert-e 3a21f7fd81 Merge branch 'w/7.6/bugfix/ZENKO-2240-fixup-get-product-version' into tmp/octopus/w/8.1/bugfix/ZENKO-2240-fixup-get-product-version 2019-11-27 19:09:28 +00:00
bert-e 796da8b5ac Merge branch 'w/8.1/bugfix/ZENKO-2240-fixup-get-product-version' into tmp/octopus/w/8.2/bugfix/ZENKO-2240-fixup-get-product-version 2019-11-27 19:09:28 +00:00
bert-e cd3a8ec628 Merge branch 'w/7.4/bugfix/ZENKO-2240-fixup-get-product-version' into tmp/octopus/w/7.6/bugfix/ZENKO-2240-fixup-get-product-version 2019-11-27 19:09:27 +00:00
Thomas Carmet 876846f891 Merge remote-tracking branch 'origin/bugfix/ZENKO-2240-fixup-get-product-version' into w/7.4/bugfix/ZENKO-2240-fixup-get-product-version 2019-11-27 11:08:37 -08:00
Thomas Carmet 7bb975de33 ZENKO-2240 ensure version is correct for 6.4 2019-11-27 11:02:56 -08:00
Thomas Carmet b630c01fa1 ZENKO-2240 now retrieving the version from package.json file 2019-11-27 11:02:32 -08:00
Thomas Carmet bc03944601 bf: ZENKO-2240 use bash on get_product_version
the script itself contains some bashism and it does not work when
executed with ./
2019-11-27 10:13:55 -08:00
bert-e 011c977acb Merge branch 'w/8.1/feature/S3C-2235-injectAccount' into tmp/octopus/w/8.2/feature/S3C-2235-injectAccount 2019-11-25 13:46:50 +00:00
Anurag Mittal 27df131a31
ft: S3C-2235-updated vaultclient 2019-11-25 19:16:24 +05:30
bert-e c2c6e231c6 Merge branch 'w/8.1/feature/S3C-2235-injectAccount' into tmp/octopus/w/8.2/feature/S3C-2235-injectAccount 2019-11-25 13:42:39 +00:00
Anurag Mittal b09655759d
Merge remote-tracking branch 'origin/feature/S3C-2235-injectAccount' into w/8.1/feature/S3C-2235-injectAccount 2019-11-25 19:11:11 +05:30
Anurag Mittal fba489f2be
ft: S3C-2235 inject accountid, accesskey and secretkey 2019-11-25 13:29:48 +05:30
bert-e f5942e3ee5 Merge branch 'w/8.1/improvement/S3C-2193-sse-s3-2' into tmp/octopus/w/8.2/improvement/S3C-2193-sse-s3-2 2019-11-21 20:04:31 +00:00
naren-scality f3331b3d3b merge changes 2019-11-21 12:03:53 -08:00
bert-e 28797860c6 Merge branch 'w/8.1/improvement/S3C-2193-sse-s3-2' into tmp/octopus/w/8.2/improvement/S3C-2193-sse-s3-2 2019-11-21 19:58:34 +00:00
naren-scality c65312411c Merge remote-tracking branch 'origin/improvement/S3C-2193-sse-s3-2' into w/8.1/improvement/S3C-2193-sse-s3-2 2019-11-21 11:54:13 -08:00
naren-scality 14e2403369 improvement: allow sse header for encrypted buckets 2019-11-21 10:40:49 -08:00
Rached Ben Mustapha adf5ffb1e2 feature: add metrics and secure channel servers
This allows using specialized connections for each usage:
- configuration push
- metrics collection (to be later moved to prom scraping)
- secure channel

This way each connection handler can be restricted in what it can
do for reliability and security purposes.
2019-11-20 12:21:42 -08:00
vrancurel 00a2cc43c9 bf: special treatment for MPU MD5 if mdonly
Rclone originally does not handle correctly the MD5 for MPU objects,
because their Etag is structure as "md5chksum-numparts".
Therefore a patch was made to not only transmit an x-amz-meta-md5chksum like
before but also to include an additional x-amz-meta-md5numparts.
This fix will check for this new header and reconstitute the MD5 correcly.
For non-MPU objects the behavior is unchanged and therefore backward
compatible.
2019-11-15 10:44:39 -08:00
Dasha Gurova fa9ca65eb9
Merge branch 'development/8.2' into documentation/ZENKOIO-116-smallUpdatesToDocs 2019-11-08 11:54:48 -08:00
Dasha Gurova 3c015f6de8 Rached's comment 2019-11-08 11:53:55 -08:00
bert-e 3e4d249495 Merge branch 'w/7.6/bugfix/S3C-2517-fixCrashWithInvalidChunkedUpload' into tmp/octopus/w/8.1/bugfix/S3C-2517-fixCrashWithInvalidChunkedUpload 2019-11-07 00:38:33 +00:00
bert-e b65b5d4fb7 Merge branch 'w/8.1/bugfix/S3C-2517-fixCrashWithInvalidChunkedUpload' into tmp/octopus/w/8.2/bugfix/S3C-2517-fixCrashWithInvalidChunkedUpload 2019-11-07 00:38:33 +00:00
bert-e b94fcf23bc Merge branch 'bugfix/S3C-2517-fixCrashWithInvalidChunkedUpload' into tmp/octopus/w/7.6/bugfix/S3C-2517-fixCrashWithInvalidChunkedUpload 2019-11-07 00:38:32 +00:00
Jonathan Gramain cc66f17d7c bugfix: S3C-2517 fix crash with invalid chunked-upload
A Cloudserver worker can crash if a client provides the
"x-amz-content-sha256" header with a value of
"STREAMING-AWS4-HMAC-SHA256-PAYLOAD" to use chunked upload, but
provides a valid AWS V2 signature (chunked-upload requires a V4
signature).

This fixes to instead respond with a 400 Bad Request.
2019-11-06 16:37:40 -08:00
Dasha Gurova 3873d1e0d5 suggestions from review 2019-11-06 12:51:56 -08:00
Dasha Gurova 0dd8e5cc14 docs: ZENKOIO-116 small updates 2019-11-06 11:38:50 -08:00
bert-e 90a26d3432 Merge branch 'bugfix/ZENKO-2200/report' into tmp/octopus/w/8.2/bugfix/ZENKO-2200/report 2019-10-22 20:19:56 +00:00
Nicolas Humbert 0c797cf84e ZENKO-2200 Keep the .git/HEAD file to properly report version 2019-10-22 13:18:18 -07:00
bert-e 3dd86aa30a Merge branches 'w/8.2/bugfix/S3C-2392-mpu-tagging' and 'q/2219/8.1/bugfix/S3C-2392-mpu-tagging' into tmp/octopus/q/8.2 2019-10-17 18:29:52 +00:00
bert-e 8c8902cc74 Merge branches 'w/8.1/bugfix/S3C-2392-mpu-tagging' and 'q/2219/7.6/bugfix/S3C-2392-mpu-tagging' into tmp/octopus/q/8.1 2019-10-17 18:29:51 +00:00
bert-e a772ffcdbb Merge branches 'w/7.6/bugfix/S3C-2392-mpu-tagging' and 'q/2219/7.4/bugfix/S3C-2392-mpu-tagging' into tmp/octopus/q/7.6 2019-10-17 18:29:51 +00:00
bert-e dfff880a59 Merge branch 'bugfix/S3C-2392-mpu-tagging' into q/7.4 2019-10-17 18:29:50 +00:00
bert-e 1ce0a4db92 Merge branches 'w/8.2/bugfix/S3C-2206/x_forwarded_for_ip_extraction' and 'q/2201/8.1/bugfix/S3C-2206/x_forwarded_for_ip_extraction' into tmp/octopus/q/8.2 2019-10-17 16:44:35 +00:00
bert-e 29a174a0bc Merge branches 'w/8.1/bugfix/S3C-2206/x_forwarded_for_ip_extraction' and 'q/2201/7.6/bugfix/S3C-2206/x_forwarded_for_ip_extraction' into tmp/octopus/q/8.1 2019-10-17 16:44:34 +00:00
bert-e 7d6d45c582 Merge branches 'w/7.6/bugfix/S3C-2206/x_forwarded_for_ip_extraction' and 'q/2201/7.4/bugfix/S3C-2206/x_forwarded_for_ip_extraction' into tmp/octopus/q/7.6 2019-10-17 16:44:33 +00:00
bert-e 6ef96d0011 Merge branch 'bugfix/S3C-2206/x_forwarded_for_ip_extraction' into q/7.4 2019-10-17 16:44:33 +00:00
bert-e 360cf0287a Merge branch 'w/8.1/bugfix/S3C-2206/x_forwarded_for_ip_extraction' into tmp/octopus/w/8.2/bugfix/S3C-2206/x_forwarded_for_ip_extraction 2019-10-16 22:39:25 +00:00
naren-scality c7b6b752fd Merge remote-tracking branch 'origin/w/7.6/bugfix/S3C-2206/x_forwarded_for_ip_extraction' into w/8.1/bugfix/S3C-2206/x_forwarded_for_ip_extraction 2019-10-16 15:30:20 -07:00
bert-e 644a66d01d Merge branch 'bugfix/S3C-2206/x_forwarded_for_ip_extraction' into tmp/octopus/w/7.6/bugfix/S3C-2206/x_forwarded_for_ip_extraction 2019-10-16 22:25:40 +00:00
naren-scality 855404dd01 bugfix: 2206 x-forwarded-for ip extraction 2019-10-16 15:24:29 -07:00
bert-e fc6875d640 Merge branch 'w/8.1/bugfix/S3C-2392-mpu-tagging' into tmp/octopus/w/8.2/bugfix/S3C-2392-mpu-tagging 2019-10-16 21:38:17 +00:00
Dora Korpar 39fcbdcb49 Merge remote-tracking branch 'origin/w/7.6/bugfix/S3C-2392-mpu-tagging' into w/8.1/bugfix/S3C-2392-mpu-tagging 2019-10-16 14:27:17 -07:00
bert-e c423a00b5a Merge branch 'bugfix/S3C-2392-mpu-tagging' into tmp/octopus/w/7.6/bugfix/S3C-2392-mpu-tagging 2019-10-16 21:07:37 +00:00
bert-e 25e28357d1 Merge branches 'w/8.2/bugfix/S3C-2392-update-aws-sdk' and 'q/2212/8.1/bugfix/S3C-2392-update-aws-sdk' into tmp/octopus/q/8.2 2019-10-16 20:11:28 +00:00
bert-e a178a16087 Merge branches 'w/8.1/bugfix/S3C-2392-update-aws-sdk' and 'q/2212/7.6/bugfix/S3C-2392-update-aws-sdk' into tmp/octopus/q/8.1 2019-10-16 20:11:28 +00:00
bert-e c0bebd2f95 Merge branch 'bugfix/S3C-2392-update-aws-sdk' into q/7.4 2019-10-16 20:11:27 +00:00
bert-e 08b3b17b70 Merge branches 'w/7.6/bugfix/S3C-2392-update-aws-sdk' and 'q/2212/7.4/bugfix/S3C-2392-update-aws-sdk' into tmp/octopus/q/7.6 2019-10-16 20:11:27 +00:00
Dora Korpar fc18dfb9d7 bf: S3C 2392 mpu tagging 2019-10-16 12:55:26 -07:00
bert-e 7ca6288ba6 Merge branch 'w/8.1/bugfix/S3C-2392-update-aws-sdk' into tmp/octopus/w/8.2/bugfix/S3C-2392-update-aws-sdk 2019-10-16 17:15:43 +00:00
Dora Korpar 77c0476dd4 Merge remote-tracking branch 'origin/w/7.6/bugfix/S3C-2392-update-aws-sdk' into w/8.1/bugfix/S3C-2392-update-aws-sdk 2019-10-16 10:02:04 -07:00
Dora Korpar d638236aaa Merge remote-tracking branch 'origin/bugfix/S3C-2392-update-aws-sdk' into w/7.6/bugfix/S3C-2392-update-aws-sdk 2019-10-16 09:30:41 -07:00
Dora Korpar 056316f4cf bf: S3C 2392 update aws sdk 2019-10-16 09:18:12 -07:00
bert-e bc2897bf13 Merge branches 'w/8.2/bugfix/S3C-2397/invalid-range' and 'q/2194/8.1/bugfix/S3C-2397/invalid-range' into tmp/octopus/q/8.2 2019-10-16 07:04:33 +00:00
bert-e 99f240b2be Merge branches 'w/8.1/bugfix/S3C-2397/invalid-range' and 'q/2194/7.6/bugfix/S3C-2397/invalid-range' into tmp/octopus/q/8.1 2019-10-16 07:04:33 +00:00
bert-e 7b3eee5480 Merge branch 'bugfix/S3C-2397/invalid-range' into q/7.4 2019-10-16 07:04:32 +00:00
bert-e e128f921e6 Merge branches 'w/7.6/bugfix/S3C-2397/invalid-range' and 'q/2194/7.4/bugfix/S3C-2397/invalid-range' into tmp/octopus/q/7.6 2019-10-16 07:04:32 +00:00
bert-e 8f17fe87e8 Merge branch 'w/8.1/bugfix/S3C-2397/invalid-range' into tmp/octopus/w/8.2/bugfix/S3C-2397/invalid-range 2019-10-09 23:06:21 +00:00
bert-e bf9b8f138e Merge branch 'w/7.6/bugfix/S3C-2397/invalid-range' into tmp/octopus/w/8.1/bugfix/S3C-2397/invalid-range 2019-10-09 23:06:20 +00:00
bert-e c9f3f64d47 Merge branch 'bugfix/S3C-2397/invalid-range' into tmp/octopus/w/7.6/bugfix/S3C-2397/invalid-range 2019-10-09 23:06:20 +00:00
Bennett Buchanan 284273d4ef bugfix: S3C-2397 Check invalid range 2019-10-09 15:59:05 -07:00
bert-e eb42e2128a Merge branches 'w/8.2/bugfix/ZENKO-2131' and 'q/2190/8.1/bugfix/ZENKO-2131' into tmp/octopus/q/8.2 2019-10-08 00:00:37 +00:00
bert-e d9fd24e16e Merge branch 'bugfix/ZENKO-2131' into q/8.1 2019-10-08 00:00:37 +00:00
bert-e 206551847e Merge branch 'bugfix/ZENKO-2131' into tmp/octopus/w/8.2/bugfix/ZENKO-2131 2019-10-07 23:21:42 +00:00
Salim a58331bc34 bf: fix mdOnly version ID
ZENKOIO-107
This commit fixes a bug where "metadata only" puts used for ingestion do not
add the server version ID. Without this, getting older versions of an ingested
object would fail.
2019-10-07 15:48:09 -07:00
Salim 163563bc80 bf: fix ingest MD5 inconsistencies
ZENKO-2131
This commit fixes a bug where cloudserver can crash if there is an ingestion
request without the x-amz-meta-md5chksum header. By assigning null to the value
in the event that it is undefined, this behavior is prevented.
2019-10-07 15:23:48 -07:00
bert-e dc5efbafdc Merge branch 'w/8.1/bugfix/S3C-1805/bucket_name_with_consecutive_hyphens' into tmp/octopus/w/8.2/bugfix/S3C-1805/bucket_name_with_consecutive_hyphens 2019-10-07 21:06:09 +00:00
naren-scality de3ff7507e Merge remote-tracking branch 'origin/w/7.6/bugfix/S3C-1805/bucket_name_with_consecutive_hyphens' into w/8.1/bugfix/S3C-1805/bucket_name_with_consecutive_hyphens 2019-10-07 13:51:40 -07:00
naren-scality 42fba9e2bb bugfix: S3C-1805 Consecutive hyphen in bucketnames 2019-10-07 11:41:52 -07:00
naren-scality 14d3bc8b2d Merge remote-tracking branch 'origin/bugfix/S3C-1805/bucket_name_with_consecutive_hyphens' into w/7.6/bugfix/S3C-1805/bucket_name_with_consecutive_hyphens 2019-10-04 16:55:05 -07:00
naren-scality f2b5eb3ec7 bugfix: S3C-1805 Consecutive hyphen in bucketnames 2019-10-04 11:36:35 -07:00
bert-e 1f201b8b3a Merge branches 'development/8.2' and 'w/8.1/bugfix/S3C-2410/handle_unsupported_operations' into tmp/octopus/w/8.2/bugfix/S3C-2410/handle_unsupported_operations 2019-10-02 17:28:15 +00:00
bert-e 8bbb93b931 Merge branches 'development/8.1' and 'w/7.6/bugfix/S3C-2410/handle_unsupported_operations' into tmp/octopus/w/8.1/bugfix/S3C-2410/handle_unsupported_operations 2019-10-02 17:28:14 +00:00
naren-scality eecfc962b4 bugfix: S3C-2410 Unsupported operations are responded with Not Implemented 2019-10-02 10:27:23 -07:00
bert-e bd8f92ee6f Merge branches 'w/8.2/improvement/ZENKOIO-100' and 'q/2157/8.1/improvement/ZENKOIO-100' into tmp/octopus/q/8.2 2019-10-01 22:03:41 +00:00
bert-e 14023dee2d Merge branch 'improvement/ZENKOIO-100' into q/8.1 2019-10-01 22:03:41 +00:00
Salim 6322e527af Merge branch 'improvement/ZENKOIO-100' into w/8.2/improvement/ZENKOIO-100 2019-10-01 13:26:44 -07:00
bert-e 9c0ef12b10 Merge branch 'w/8.1/bugfix/S3C-2410/handle_unsupported_operations' into tmp/octopus/w/8.2/bugfix/S3C-2410/handle_unsupported_operations 2019-10-01 18:01:37 +00:00
bert-e 82fa88c40e Merge branch 'w/7.6/bugfix/S3C-2410/handle_unsupported_operations' into tmp/octopus/w/8.1/bugfix/S3C-2410/handle_unsupported_operations 2019-10-01 18:01:36 +00:00
naren-scality e963e9beba Merge remote-tracking branch 'origin/bugfix/S3C-2410/handle_unsupported_operations' into w/7.6/bugfix/S3C-2410/handle_unsupported_operations 2019-10-01 10:47:11 -07:00
naren-scality 599d86313d bugfix: S3C-2410 Unsupported operations are responded with Not Implemented 2019-10-01 10:27:30 -07:00
vrancurel 2aab4be5d4 feature: workflow engine operator proxy 2019-09-27 09:59:03 -07:00
Salim 0bbe82b5b9 bf: fix putObjectPart typo 2019-09-26 16:18:04 -07:00
Salim ec720c3040 bf: fix metrics reporting of request size
ZENKOIO-101
This fixes a bug where Cloudserver will reports the byte size for a PUT request as NewSizeLength - OldSizeLength causing inaccuracies when putting Objects into a non versioned bucket. (e.g. I put a 100MB file into a bucket, the counter will reflect accurately, but if I put the same object again, the counter won't change even though 100MB just passed through).
2019-09-26 16:18:04 -07:00
Salim d1c59e5c06 improvement: add mdonly ingestion metrics 2019-09-26 16:18:03 -07:00
bert-e f04608a0ce Merge branches 'w/8.2/bugfix/S3C-2440-get-policy-xml-error' and 'q/2154/8.1/bugfix/S3C-2440-get-policy-xml-error' into tmp/octopus/q/8.2 2019-09-25 20:41:21 +00:00
bert-e b1082add04 Merge branches 'w/8.1/bugfix/S3C-2440-get-policy-xml-error' and 'q/2154/7.6/bugfix/S3C-2440-get-policy-xml-error' into tmp/octopus/q/8.1 2019-09-25 20:41:20 +00:00
bert-e ee15b73dcf Merge branch 'bugfix/S3C-2440-get-policy-xml-error' into q/7.6 2019-09-25 20:41:20 +00:00
bert-e 19415e36d5 Merge branch 'w/8.1/bugfix/S3C-2440-get-policy-xml-error' into tmp/octopus/w/8.2/bugfix/S3C-2440-get-policy-xml-error 2019-09-24 20:07:04 +00:00
Dora Korpar 8d5db7da81 Merge remote-tracking branch 'origin/bugfix/S3C-2440-get-policy-xml-error' into w/8.1/bugfix/S3C-2440-get-policy-xml-error 2019-09-24 12:55:06 -07:00
bert-e 589a465c57 Merge branches 'w/8.2/bugfix/S3C-2439-precise-request-types-mpu' and 'q/2151/8.1/bugfix/S3C-2439-precise-request-types-mpu' into tmp/octopus/q/8.2 2019-09-24 01:10:11 +00:00
bert-e fb67c4b6e9 Merge branches 'w/8.1/bugfix/S3C-2439-precise-request-types-mpu' and 'q/2151/7.6/bugfix/S3C-2439-precise-request-types-mpu' into tmp/octopus/q/8.1 2019-09-24 01:10:11 +00:00
bert-e c35bf054a5 Merge branch 'bugfix/S3C-2439-precise-request-types-mpu' into q/7.6 2019-09-24 01:10:11 +00:00
Dora Korpar cc30a2eb42 bf: S3C-2440 get policy xml error and tests 2019-09-23 16:56:25 -07:00
bert-e 957631cea0 Merge branch 'w/8.1/bugfix/S3C-2439-precise-request-types-mpu' into tmp/octopus/w/8.2/bugfix/S3C-2439-precise-request-types-mpu 2019-09-23 22:52:39 +00:00
bert-e 0a48822254 Merge branch 'bugfix/S3C-2439-precise-request-types-mpu' into tmp/octopus/w/8.1/bugfix/S3C-2439-precise-request-types-mpu 2019-09-23 22:52:39 +00:00
Dora Korpar e09dbd7804 bf: S3C-2439 mpu apis precise request type 2019-09-20 12:36:10 -07:00
bert-e 0faa470247 Merge branch 'w/8.1/bugfix/S3C-2399/incorrect_part_number_size' into tmp/octopus/w/8.2/bugfix/S3C-2399/incorrect_part_number_size 2019-09-20 07:15:46 +00:00
naren-scality a3c4dd865c Merge remote-tracking branch 'origin/w/7.6/bugfix/S3C-2399/incorrect_part_number_size' into w/8.1/bugfix/S3C-2399/incorrect_part_number_size 2019-09-20 00:04:48 -07:00
bert-e c7d9992777 Merge branch 'bugfix/S3C-2399/incorrect_part_number_size' into tmp/octopus/w/7.6/bugfix/S3C-2399/incorrect_part_number_size 2019-09-20 07:00:59 +00:00
naren-scality 8bb00cb17b bugfix: S3C-2399 head-object part number size corrections 2019-09-19 23:04:54 -07:00
bert-e aa874cf6fd Merge branch 'w/8.1/bugfix/S3C-2435-fix-obj-action-parse' into tmp/octopus/w/8.2/bugfix/S3C-2435-fix-obj-action-parse 2019-09-17 22:55:41 +00:00
Dora Korpar d24aa96014 Merge remote-tracking branch 'origin/bugfix/S3C-2435-fix-obj-action-parse' into w/8.1/bugfix/S3C-2435-fix-obj-action-parse 2019-09-17 15:46:19 -07:00
Dora Korpar c55aa32e61 bf: S3C 2435 fix object action parse 2019-09-17 15:39:47 -07:00
bert-e 3ab5d23dab Merge branch 'w/8.2/bugfix/S3C-2391/list_objects_v2_no_fetch_owner_corrections' into tmp/octopus/q/8.2 2019-09-17 16:40:48 +00:00
bert-e fa4d3a934e Merge branch 'w/7.6/bugfix/S3C-2391/list_objects_v2_no_fetch_owner_corrections' into tmp/octopus/w/8.1/bugfix/S3C-2391/list_objects_v2_no_fetch_owner_corrections 2019-09-13 22:58:53 +00:00
bert-e 6e77f53f09 Merge branch 'bugfix/S3C-2391/list_objects_v2_no_fetch_owner_corrections' into tmp/octopus/w/7.6/bugfix/S3C-2391/list_objects_v2_no_fetch_owner_corrections 2019-09-13 22:58:52 +00:00
naren-scality 300023b5b2 bugfix: S3C-2391 list_objects_v2 no-fetch-owner corrections 2019-09-13 14:54:43 -07:00
Salim 6ea92cf9ce ft: enable AWS and Ceph OOB on reportHandler 2019-09-12 15:23:47 -07:00
bert-e 9615d51c9e Merge branch 'w/7.6/bugfix/S3C-2169/invalid_user_metadata_header' into tmp/octopus/w/8.1/bugfix/S3C-2169/invalid_user_metadata_header 2019-09-12 19:03:18 +00:00
bert-e 5f53d55923 Merge branch 'bugfix/S3C-2169/invalid_user_metadata_header' into tmp/octopus/w/7.6/bugfix/S3C-2169/invalid_user_metadata_header 2019-09-12 19:03:18 +00:00
naren-scality 5a63967c31 bugfix: S3C-2169 filter invalid user metadata headers 2019-09-12 11:41:06 -07:00
bert-e 74d7c53ffb Merge branch 'bugfix/ZENKO-2101/update-utapi-dependency' into q/8.1 2019-09-11 23:59:07 +00:00
bennettbuchanan 0e3dc452c0 bugfix: ZENKO-2101 Update UTAPI dependency 2019-09-11 16:07:56 -07:00
bert-e 2985c610f5 Merge branch 'w/7.6/bugfix/S3C-2169/invalid_user_metadata_header' into tmp/octopus/q/7.6 2019-09-11 22:53:37 +00:00
bert-e 9bd78d311c Merge branches 'w/8.1/bugfix/S3C-2169/invalid_user_metadata_header' and 'q/2123/7.6/bugfix/S3C-2169/invalid_user_metadata_header' into tmp/octopus/q/8.1 2019-09-11 22:53:37 +00:00
bert-e dbbadac2ee Merge branch 'feature/ZENKO-1616_Add_fixes_for_update_to_ObjectMD_model' into q/8.1 2019-09-11 22:53:04 +00:00
bert-e 86459b0b5a Merge branch 'bugfix/S3C-2424-bucket-policy-api-perms' into tmp/octopus/w/8.1/bugfix/S3C-2424-bucket-policy-api-perms 2019-09-11 18:21:56 +00:00
bert-e eaa9495b4d Merge branch 'w/7.6/bugfix/S3C-2169/invalid_user_metadata_header' into tmp/octopus/w/8.1/bugfix/S3C-2169/invalid_user_metadata_header 2019-09-11 17:57:16 +00:00
naren-scality fce1927cd7 Merge remote-tracking branch 'origin/bugfix/S3C-2169/invalid_user_metadata_header' into w/7.6/bugfix/S3C-2169/invalid_user_metadata_header 2019-09-11 10:44:21 -07:00
Dora Korpar 92e57803de bf: S3C 2424 bucket policy api perms 2019-09-10 16:04:51 -07:00
naren-scality 83e3f96f40 bugfix: S3C-2169 filter invalid user metadata headers 2019-09-10 13:28:04 -07:00
Salim b30263bcc6 improvement: add OOB reporting options for supported locations 2019-09-03 12:34:42 -07:00
bert-e 323885394d Merge branch 'w/7.5/bugfix/S3C-2052/revert-changes-stabilization' into tmp/octopus/w/8.1/bugfix/S3C-2052/revert-changes-stabilization 2019-08-29 20:52:59 +00:00
bbuchanan9 60fd43ab00 Merge remote-tracking branch 'origin/bugfix/S3C-2052/revert-changes-stabilization' into w/7.5/bugfix/S3C-2052/revert-changes-stabilization 2019-08-29 13:45:29 -07:00
bbuchanan9 dbdc9fd86a Revert "bugfix: S3C-2052 Delete orphaned data"
This reverts commit dab7c63414.
2019-08-29 13:32:25 -07:00
bert-e 2b72415a83 Merge branches 'w/8.1/bugfix/S3C-2052/revert-changes' and 'q/2107/7.5/bugfix/S3C-2052/revert-changes' into tmp/octopus/q/8.1 2019-08-29 19:29:45 +00:00
bert-e aac7c8cb5a Merge branch 'bugfix/S3C-2052/revert-changes' into q/7.4 2019-08-29 19:29:44 +00:00
bert-e 765d572a31 Merge branches 'w/7.5/bugfix/S3C-2052/revert-changes' and 'q/2107/7.4/bugfix/S3C-2052/revert-changes' into tmp/octopus/q/7.5 2019-08-29 19:29:44 +00:00
bbuchanan9 989530a41b Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2052/revert-changes' into w/8.1/bugfix/S3C-2052/revert-changes 2019-08-29 12:08:18 -07:00
bbuchanan9 d892434c07 Merge remote-tracking branch 'origin/bugfix/S3C-2052/revert-changes' into w/7.5/bugfix/S3C-2052/revert-changes 2019-08-29 10:49:07 -07:00
bbuchanan9 edd425ecf7 Revert "bugfix: S3C-2052 Delete orphaned data"
This reverts commit d45fbdbf25.
2019-08-28 17:02:26 -07:00
bert-e 6e54403319 Merge branch 'w/7.5/bugfix/S3C-2052/revert-api-changes-stabilization' into tmp/octopus/w/8.1/bugfix/S3C-2052/revert-api-changes-stabilization 2019-08-28 23:53:19 +00:00
bert-e 027372f5da Merge branch 'bugfix/S3C-2052/revert-api-changes-stabilization' into tmp/octopus/w/7.5/bugfix/S3C-2052/revert-api-changes-stabilization 2019-08-28 23:53:18 +00:00
bbuchanan9 2ea3c036df Revert "bugfix: S3C-2052 Delete orphaned data in APIs"
This reverts commit 6ca0c6fbcc.
2019-08-28 16:51:23 -07:00
bert-e bd38e24ed3 Merge branches 'w/8.1/bugfix/S3C-2052/revert-api-changes' and 'q/2101/7.5/bugfix/S3C-2052/revert-api-changes' into tmp/octopus/q/8.1 2019-08-28 23:01:48 +00:00
bert-e 3d840895d2 Merge branch 'bugfix/S3C-2052/revert-api-changes' into q/7.4 2019-08-28 23:01:47 +00:00
bert-e c4f52154aa Merge branches 'w/7.5/bugfix/S3C-2052/revert-api-changes' and 'q/2101/7.4/bugfix/S3C-2052/revert-api-changes' into tmp/octopus/q/7.5 2019-08-28 23:01:47 +00:00
bbuchanan9 85285f9356 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2052/revert-api-changes' into w/8.1/bugfix/S3C-2052/revert-api-changes 2019-08-28 15:21:50 -07:00
bert-e 5641be0d9c Merge branch 'bugfix/S3C-2052/revert-api-changes' into tmp/octopus/w/7.5/bugfix/S3C-2052/revert-api-changes 2019-08-28 21:43:55 +00:00
bbuchanan9 5dd11b7b44 Revert "bugfix: S3C-2052 Delete orphaned data in APIs"
This reverts commit 5bf5fc861c.
2019-08-28 14:34:40 -07:00
Taylor McKinnon b3d6cd1253 ft(ZENKO-1616): Add fixes for new ObjectMD properties 2019-08-27 10:21:07 -07:00
bert-e 7f36eff8c3 Merge branch 'w/7.5/bugfix/S3C-2274/exit-process' into tmp/octopus/w/8.1/bugfix/S3C-2274/exit-process 2019-08-27 16:09:24 +00:00
bert-e 0c6181bb38 Merge branch 'bugfix/S3C-2274/exit-process' into tmp/octopus/w/7.5/bugfix/S3C-2274/exit-process 2019-08-27 16:09:23 +00:00
Nicolas Humbert d61a34d31b bugfix: S3C-2274 Fail to exit process before startup 2019-08-26 13:20:10 -07:00
bert-e f85c3ca489 Merge branch 'bugfix/S3C-2412-bucket-pol-principal-eval' into tmp/octopus/w/8.1/bugfix/S3C-2412-bucket-pol-principal-eval 2019-08-22 21:45:52 +00:00
Dora Korpar 16e05ad6f6 bf: S3C 2412 fix bucket pol principal eval 2019-08-22 14:21:21 -07:00
bert-e 2c132289e8 Merge branch 'w/7.5/bugfix/S3C-2052/delete-orphaned-data-remaining-APIs-forwardport' into tmp/octopus/w/8.1/bugfix/S3C-2052/delete-orphaned-data-remaining-APIs-forwardport 2019-08-20 20:46:48 +00:00
bert-e 5bab0c3642 Merge branch 'bugfix/S3C-2052/delete-orphaned-data-remaining-APIs-forwardport' into tmp/octopus/w/7.5/bugfix/S3C-2052/delete-orphaned-data-remaining-APIs-forwardport 2019-08-20 20:46:47 +00:00
bert-e d5e5719843 Merge branches 'w/8.1/bugfix/S3C-2052/delete-orphaned-data-remaining-APIs' and 'q/2074/7.5/bugfix/S3C-2052/delete-orphaned-data-remaining-APIs' into tmp/octopus/q/8.1 2019-08-20 19:56:20 +00:00
bert-e 21284f242f Merge branch 'bugfix/S3C-2052/delete-orphaned-data-remaining-APIs' into q/7.4 2019-08-20 19:56:19 +00:00
bert-e 999fc0335e Merge branches 'w/7.5/bugfix/S3C-2052/delete-orphaned-data-remaining-APIs' and 'q/2074/7.4/bugfix/S3C-2052/delete-orphaned-data-remaining-APIs' into tmp/octopus/q/7.5 2019-08-20 19:56:19 +00:00
bbuchanan9 6ca0c6fbcc bugfix: S3C-2052 Delete orphaned data in APIs
Cleanup orphaned data in error cases for
the following APIs:

* objectPut
* objectCopy
* objectCopyPart
2019-08-20 11:54:11 -07:00
bbuchanan9 daf841cf53 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2052/delete-orphaned-data-remaining-APIs' into w/8.1/bugfix/S3C-2052/delete-orphaned-data-remaining-APIs 2019-08-20 11:15:10 -07:00
bert-e 2790f3da72 Merge branch 'bugfix/S3C-2052/delete-orphaned-data-remaining-APIs' into tmp/octopus/w/7.5/bugfix/S3C-2052/delete-orphaned-data-remaining-APIs 2019-08-20 18:05:33 +00:00
Dora Korpar 2ba697f540 Merge remote-tracking branch 'origin/bugfix/S3C-2396-fix-bucket-policy-parsing' into w/8.1/bugfix/S3C-2396-fix-bucket-policy-parsing 2019-08-20 10:42:46 -07:00
Dora Korpar beebb84576 update arsenal 2019-08-20 10:31:13 -07:00
Dora Korpar 66c84c1803 bf: S3C 2396 fix action parsing 2019-08-20 10:06:03 -07:00
bert-e b545fc47d8 Merge branches 'w/8.1/bugfix/S3C-2052/delete-orphaned-data-fwdpt' and 'q/2077/7.5/bugfix/S3C-2052/delete-orphaned-data-fwdpt' into tmp/octopus/q/8.1 2019-08-19 23:26:52 +00:00
bert-e 0456a35c36 Merge branch 'w/7.5/bugfix/S3C-2052/delete-orphaned-data-fwdpt' into tmp/octopus/q/7.5 2019-08-19 23:26:51 +00:00
bert-e dd76b681c9 Merge branches 'w/8.1/improvement/S3C-2365-install-yarn' and 'q/2067/7.5/improvement/S3C-2365-install-yarn' into tmp/octopus/q/8.1 2019-08-19 23:04:28 +00:00
bert-e c2d9fd0f18 Merge branch 'improvement/S3C-2365-install-yarn' into q/7.4 2019-08-19 23:04:27 +00:00
bert-e bf014bf0f6 Merge branches 'w/7.5/improvement/S3C-2365-install-yarn' and 'q/2067/7.4/improvement/S3C-2365-install-yarn' into tmp/octopus/q/7.5 2019-08-19 23:04:27 +00:00
bert-e 726a4b2aa8 Merge branch 'w/7.5/bugfix/S3C-2052/delete-orphaned-data-fwdpt' into tmp/octopus/w/8.1/bugfix/S3C-2052/delete-orphaned-data-fwdpt 2019-08-19 22:48:19 +00:00
Rahul Padigela beddc485af Merge remote-tracking branch 'origin/bugfix/S3C-2052/delete-orphaned-data-fwdpt' into w/7.5/bugfix/S3C-2052/delete-orphaned-data-fwdpt 2019-08-19 15:31:40 -07:00
anurag4dsb bac9a10c49
updated package.json and yarn.lock 2019-08-16 10:52:23 -07:00
bbuchanan9 5bf5fc861c bugfix: S3C-2052 Delete orphaned data in APIs
Cleanup orphaned data in error cases for
the following APIs:

* objectPut
* objectCopy
* objectCopyPart
2019-08-16 10:48:28 -07:00
bbuchanan9 dab7c63414 bugfix: S3C-2052 Delete orphaned data 2019-08-16 10:32:38 -07:00
anurag4dsb e4416a9687
Merge remote-tracking branch 'origin/w/7.5/improvement/S3C-2365-install-yarn' into w/8.1/improvement/S3C-2365-install-yarn 2019-08-14 16:04:42 -07:00
anurag4dsb 62d6172e33
updated yarn.lock file 2019-08-14 15:06:00 -07:00
anurag4dsb 03d2b3b80b
Merge remote-tracking branch 'origin/improvement/S3C-2365-install-yarn' into w/7.5/improvement/S3C-2365-install-yarn 2019-08-14 14:39:38 -07:00
bbuchanan9 d275a76fea bugfix: S3C-2052 Updates for Arsenal backend 2019-08-13 18:32:01 -07:00
bbuchanan9 6f1619b1cc Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2052/delete-orphaned-data' into w/8.1/bugfix/S3C-2052/delete-orphaned-data 2019-08-13 18:29:56 -07:00
bbuchanan9 7826ad2f98 Merge remote-tracking branch 'origin/bugfix/S3C-2052/delete-orphaned-data' into w/7.5/bugfix/S3C-2052/delete-orphaned-data 2019-08-13 18:05:27 -07:00
bbuchanan9 d45fbdbf25 bugfix: S3C-2052 Delete orphaned data 2019-08-13 17:09:47 -07:00
Katherine Laue b67904ddb3 add yarn.lock 2019-08-13 15:33:42 -07:00
Katherine Laue c9af7ad6d2 Merge remote-tracking branch 'origin/development/7.4' into HEAD 2019-08-13 15:28:16 -07:00
bert-e e21594fb3b Merge branches 'w/8.1/feature/S3C-2371-apis-check-bucket-policy' and 'q/2044/7.5/feature/S3C-2371-apis-check-bucket-policy' into tmp/octopus/q/8.1 2019-08-13 22:27:59 +00:00
bert-e 3724e3a667 Merge branch 'feature/S3C-2371-apis-check-bucket-policy' into q/7.5 2019-08-13 22:27:58 +00:00
Dora Korpar eda89806ba Merge remote-tracking branch 'origin/feature/S3C-2371-apis-check-bucket-policy' into w/8.1/feature/S3C-2371-apis-check-bucket-policy 2019-08-13 14:18:39 -07:00
bert-e 03c3a40f70 Merge branch 'feature/S3C-2027-bucket-policy-design' into tmp/octopus/w/8.1/feature/S3C-2027-bucket-policy-design 2019-08-13 20:51:34 +00:00
Katherine Laue 81bce441bb improvement/S3C-2365 migrate package manager to yarn 2019-08-13 13:01:16 -07:00
Dora Korpar 805f0dfad5 ft: S3C 1976 bucket policy design doc 2019-08-13 12:07:33 -07:00
Rahul Padigela b93066f142 chore: update utapi to fix package issues 2019-08-12 14:47:36 -07:00
Dora Korpar 73e4523738 ft: S3C 2371 update apis 2019-08-12 14:47:36 -07:00
bert-e 9820307c97 Merge branch 'w/8.0/bugfix/S3C-2369-limit-batch-delete-backport' into tmp/octopus/w/8.1/bugfix/S3C-2369-limit-batch-delete-backport 2019-08-12 21:18:26 +00:00
bert-e e7f6f3d060 Merge branch 'w/7.5/bugfix/S3C-2369-limit-batch-delete-backport' into tmp/octopus/w/8.0/bugfix/S3C-2369-limit-batch-delete-backport 2019-08-12 21:18:25 +00:00
Rahul Padigela ea60714215 Merge remote-tracking branch 'origin/bugfix/S3C-2369-limit-batch-delete-backport' into w/7.5/bugfix/S3C-2369-limit-batch-delete-backport 2019-08-12 14:05:08 -07:00
Rahul Padigela fd10cf2bfb bugfix: S3C-2369 pin sproxydclient version 2019-08-12 14:01:23 -07:00
Rahul Padigela 8177129be3 bugfix: S3C-2369 bump sproxydclient with batch delete fix
(cherry picked from commit d9e20eac9b)
2019-08-12 13:58:37 -07:00
Dora Korpar 4b04048cc7 ft: S3C 2371 check bucket policies function 2019-08-12 12:09:25 -07:00
Rahul Padigela 136cf9c5da bugfix: S3C-2369 bump arsenal 2019-08-09 15:28:39 -07:00
Rahul Padigela 0770fc48fe Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-2369-limit-batch-delete' into w/8.1/bugfix/S3C-2369-limit-batch-delete 2019-08-09 14:46:54 -07:00
Rahul Padigela 5588f9fb65 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2369-limit-batch-delete' into w/8.0/bugfix/S3C-2369-limit-batch-delete 2019-08-09 14:12:39 -07:00
Rahul Padigela 24e0b2fe2e Merge remote-tracking branch 'origin/bugfix/S3C-2369-limit-batch-delete' into w/7.5/bugfix/S3C-2369-limit-batch-delete 2019-08-09 12:35:58 -07:00
Rahul Padigela d9e20eac9b bugfix: S3C-2369 bump sproxydclient with batch delete fix 2019-08-09 12:28:40 -07:00
bbuchanan9 2fbdc6608b Merge remote-tracking branch 'origin/w/8.0/improvement/S3C-2345/allow-config-to-disable-counters' into w/8.1/improvement/S3C-2345/allow-config-to-disable-counters 2019-08-07 11:48:45 -07:00
bbuchanan9 b1cb7c1cc7 Merge remote-tracking branch 'origin/w/7.5/improvement/S3C-2345/allow-config-to-disable-counters' into w/8.0/improvement/S3C-2345/allow-config-to-disable-counters 2019-08-07 11:22:53 -07:00
bbuchanan9 2ef491257b Merge remote-tracking branch 'origin/improvement/S3C-2345/allow-config-to-disable-counters' into w/7.5/improvement/S3C-2345/allow-config-to-disable-counters 2019-08-07 11:03:21 -07:00
bbuchanan9 a36fb20cce improvement: S3C-234 Operation counters config 2019-08-06 15:54:28 -07:00
bert-e 1284f449b1 Merge branch 'w/8.0/feature/S3C-2284-bucket-policy-delete' into tmp/octopus/w/8.1/feature/S3C-2284-bucket-policy-delete 2019-08-01 18:00:27 +00:00
bert-e 5e6dba28a6 Merge branch 'feature/S3C-2284-bucket-policy-delete' into tmp/octopus/w/8.0/feature/S3C-2284-bucket-policy-delete 2019-08-01 18:00:27 +00:00
Dora Korpar 82233b115e ft: S3C-2284 delete bucket policy api 2019-08-01 10:59:42 -07:00
bert-e 9590838e82 Merge branch 'w/8.0/feature/S3C-2283-bucket-policy-get' into tmp/octopus/w/8.1/feature/S3C-2283-bucket-policy-get 2019-08-01 01:06:38 +00:00
bert-e d896145d60 Merge branch 'feature/S3C-2283-bucket-policy-get' into tmp/octopus/w/8.0/feature/S3C-2283-bucket-policy-get 2019-08-01 01:06:37 +00:00
Dora Korpar 4411adaef0 ft: S3C-2283 get bucket policy api 2019-07-31 18:05:40 -07:00
bert-e 9a8adc9ad3 Merge branch 'w/8.1/improvement/ZENKO-2031-update-cdmiclient' into tmp/octopus/q/8.1 2019-08-01 00:35:55 +00:00
bert-e 93d50448a1 Merge branch 'w/8.0/improvement/ZENKO-2031-update-cdmiclient' into tmp/octopus/w/8.1/improvement/ZENKO-2031-update-cdmiclient 2019-07-31 23:56:11 +00:00
Rahul Padigela b5517b50f8 Merge remote-tracking branch 'origin/w/7.5/improvement/ZENKO-2031-update-cdmiclient' into w/8.0/improvement/ZENKO-2031-update-cdmiclient 2019-07-31 16:40:36 -07:00
bert-e 6d2c7eba8d Merge branch 'improvement/ZENKO-2031-update-cdmiclient' into tmp/octopus/w/7.5/improvement/ZENKO-2031-update-cdmiclient 2019-07-31 23:38:56 +00:00
Rahul Padigela 49dff02ff9 improvement: ZENKO-2031 update cdmiclient 2019-07-31 16:38:10 -07:00
Salim ccefcecc78 ci: enable ft_awssdk_external_backends 2019-07-30 19:01:25 -07:00
Salim aa02034926 tests: md only header put 2019-07-30 19:01:25 -07:00
Salim c6aff6f6d3 ft: higher level meta-mdonly put
This adds the meta-mdonly header check from the client libary to a higher order
function. This allows metadata only puts to all backends types.
2019-07-30 19:01:22 -07:00
bert-e 20281be5ef Merge branch 'w/8.0/feature/S3C-2277-bucket-policy-put' into tmp/octopus/w/8.1/feature/S3C-2277-bucket-policy-put 2019-07-30 00:19:09 +00:00
bert-e 538da6ec17 Merge branch 'feature/S3C-2277-bucket-policy-put' into tmp/octopus/w/8.0/feature/S3C-2277-bucket-policy-put 2019-07-30 00:19:08 +00:00
Dora Korpar 12061dc5a9 ft: S3C-2277 put bucket policy api 2019-07-29 17:18:25 -07:00
Rahul Padigela 48550bbe89 Merge remote-tracking branch 'origin/w/8.0/improvement/S3C-2216-bump-tags-limit' into w/8.1/improvement/S3C-2216-bump-tags-limit 2019-07-29 10:54:50 -07:00
Rahul Padigela 4ce2fc43ed Merge remote-tracking branch 'origin/w/7.5/improvement/S3C-2216-bump-tags-limit' into w/8.0/improvement/S3C-2216-bump-tags-limit 2019-07-29 10:32:04 -07:00
Rahul Padigela 602791af75 Merge remote-tracking branch 'origin/improvement/S3C-2216-bump-tags-limit' into w/7.5/improvement/S3C-2216-bump-tags-limit 2019-07-29 10:12:07 -07:00
Rahul Padigela 16cab686b9 improvement: increase the limit of num. of allowed tags
This increases the limit of number of allowed tags on an object
from 10 to 50. This is to be inline and retain compatibility with
AWS S3.
2019-07-29 09:27:49 -07:00
Alexander Chan 306dea4616 feature: ZENKO-2004 search for configs 2019-07-25 14:55:32 -07:00
bert-e 7072b365c3 Merge branches 'w/8.1/bugfix/ZENKO-1829_Add_init_container_for_certs' and 'q/1992/8.0/bugfix/ZENKO-1829_Add_init_container_for_certs' into tmp/octopus/q/8.1 2019-07-24 23:57:44 +00:00
bert-e 2069df7ca8 Merge branches 'w/8.0/bugfix/ZENKO-1829_Add_init_container_for_certs' and 'q/1992/7.5/bugfix/ZENKO-1829_Add_init_container_for_certs' into tmp/octopus/q/8.0 2019-07-24 23:57:43 +00:00
bert-e 33788dc839 Merge branch 'bugfix/ZENKO-1829_Add_init_container_for_certs' into q/7.5 2019-07-24 23:57:43 +00:00
bert-e 2c514cb08b Merge branches 'w/8.1/feature/RELENG-2986-the-no-burst-policy' and 'q/1995/8.0/feature/RELENG-2986-the-no-burst-policy' into tmp/octopus/q/8.1 2019-07-24 23:21:44 +00:00
bert-e 41eed2de1f Merge branches 'w/8.0/feature/RELENG-2986-the-no-burst-policy' and 'q/1995/7.5/feature/RELENG-2986-the-no-burst-policy' into tmp/octopus/q/8.0 2019-07-24 23:21:43 +00:00
bert-e a2f6f9a9c6 Merge branch 'q/1995/7.4/feature/RELENG-2986-the-no-burst-policy' into tmp/normal/q/7.5 2019-07-24 23:21:43 +00:00
bert-e a5bfd6bdb2 Merge branch 'feature/RELENG-2986-the-no-burst-policy' into q/7.4 2019-07-24 23:21:42 +00:00
Thomas Carmet 4c680e1b53 Merge remote-tracking branch 'origin/w/8.0/feature/RELENG-2986-the-no-burst-policy' into w/8.1/feature/RELENG-2986-the-no-burst-policy 2019-07-24 13:23:42 -07:00
Thomas Carmet b7fed1898a Merge remote-tracking branch 'origin/w/7.5/feature/RELENG-2986-the-no-burst-policy' into w/8.0/feature/RELENG-2986-the-no-burst-policy 2019-07-24 13:18:48 -07:00
Thomas Carmet aaaf09f90c ci(RELENG-2986): applying 0 burst policy to kmip stage 2019-07-24 13:18:02 -07:00
Thomas Carmet 9ba72525ed Merge remote-tracking branch 'origin/feature/RELENG-2986-the-no-burst-policy' into w/7.5/feature/RELENG-2986-the-no-burst-policy 2019-07-24 13:17:01 -07:00
Thomas Carmet 43e09750dc ci(RELENG-2986): applying a 0 burst policy
On CI workloads, you cannot guess the state of the node you're going
to schedule in, therefore, for functional tests it is advised to not do
any burst on resources.
2019-07-24 13:12:32 -07:00
bert-e febd9803da Merge branch 'w/8.0/bugfix/ZENKO-1829_Add_init_container_for_certs' into tmp/octopus/w/8.1/bugfix/ZENKO-1829_Add_init_container_for_certs 2019-07-24 17:24:26 +00:00
bert-e bb4ecec2a3 Merge branch 'bugfix/ZENKO-1829_Add_init_container_for_certs' into tmp/octopus/w/8.0/bugfix/ZENKO-1829_Add_init_container_for_certs 2019-07-24 17:24:25 +00:00
Taylor McKinnon e58f11751a bf(ZENKO-1829): Add init container for installing pykmip certs 2019-07-24 10:22:56 -07:00
bert-e c44e000d1f Merge branch 'w/8.0/feature/ZENKOIO-58-upload-junit-reports' into tmp/octopus/w/8.1/feature/ZENKOIO-58-upload-junit-reports 2019-07-24 16:40:06 +00:00
bert-e 88e01f0d8d Merge branch 'w/7.5/feature/ZENKOIO-58-upload-junit-reports' into tmp/octopus/w/8.0/feature/ZENKOIO-58-upload-junit-reports 2019-07-24 16:40:05 +00:00
Thomas Carmet d137d9b225 ZENKOIO-58 upload junit on kmip stage 2019-07-24 09:39:32 -07:00
Thomas Carmet 338d944aeb ZENKOIO-58 add junit upload on ceph stage 2019-07-23 17:56:04 -07:00
bert-e deecb70edf Merge branch 'w/8.0/feature/ZENKOIO-58-upload-junit-reports' into tmp/octopus/w/8.1/feature/ZENKOIO-58-upload-junit-reports 2019-07-24 00:52:25 +00:00
Thomas Carmet 2fa0e3967a ZENKOIO-58 upload junit on kmip stage 2019-07-23 17:51:56 -07:00
Thomas Carmet 7ea47e74fe Merge remote-tracking branch 'origin/w/8.0/feature/ZENKOIO-58-upload-junit-reports' into w/8.1/feature/ZENKOIO-58-upload-junit-reports 2019-07-23 17:34:10 -07:00
Thomas Carmet 78b2f33943 Merge remote-tracking branch 'origin/w/7.5/feature/ZENKOIO-58-upload-junit-reports' into w/8.0/feature/ZENKOIO-58-upload-junit-reports 2019-07-23 17:31:12 -07:00
Thomas Carmet c829336848 Merge remote-tracking branch 'origin/feature/ZENKOIO-58-upload-junit-reports' into w/7.5/feature/ZENKOIO-58-upload-junit-reports 2019-07-23 17:25:58 -07:00
Thomas Carmet 7bc841130f ZENKOIO-58 git ignoring junit reports folder 2019-07-23 16:56:10 -07:00
Thomas Carmet 6662fce481 ZENKOIO-58 making tests junit reports 2019-07-23 16:56:10 -07:00
anurag4dsb e9b87afd11
arrow style 2019-07-19 11:44:44 -07:00
anurag4dsb aafc55396f
trying arrow style linter 2019-07-19 11:12:27 -07:00
anurag4dsb 6ac5122a80
added retuen in cb 2019-07-19 10:51:31 -07:00
anurag4dsb 84dcd8ff34
hopefully this passes 2019-07-19 10:35:35 -07:00
anurag4dsb 8a7a0128fa
cbonce 2019-07-19 10:26:56 -07:00
anurag4dsb c80147cac2
added console.log 2019-07-19 10:16:41 -07:00
anurag4dsb 550604aa9b
only added 2019-07-19 10:14:53 -07:00
anurag4dsb bb6201d6cf
reverted mocha 2019-07-19 09:49:07 -07:00
anurag4dsb 75c23df174
changed mocha script 2019-07-18 12:45:57 -07:00
anurag4dsb 722cc304c6
updated mocha 2019-07-18 11:25:11 -07:00
anurag4dsb 478a264845
changed mocha minor version 2019-07-18 11:01:03 -07:00
anurag4dsb ec0c7f6eff
reverted everything 2019-07-18 10:35:56 -07:00
anurag4dsb b03bf4e192
[s/d] 100continue test 2019-07-18 10:28:17 -07:00
anurag4dsb 7eb39abc70
[s/d] running specific test 2019-07-18 10:20:53 -07:00
Katherine Laue 57294c8d5c updated internal deps for 8.1 2019-07-17 16:58:03 -07:00
Katherine Laue 6243922a20 Merge remote-tracking branch 'origin/w/8.0/improvement/S3C-2293-upgradeNodeJS' into w/8.1/improvement/S3C-2293-upgradeNodeJS 2019-07-17 16:47:57 -07:00
Katherine Laue 5b68f94c48 updated internal deps for 8.0 2019-07-17 16:44:24 -07:00
Katherine Laue a8006a4991 Merge remote-tracking branch 'origin/w/7.5/improvement/S3C-2293-upgradeNodeJS' into w/8.0/improvement/S3C-2293-upgradeNodeJS 2019-07-17 16:42:27 -07:00
Katherine Laue 66480e16e1 updated internal deps for 7.5 2019-07-17 16:39:04 -07:00
Katherine Laue a73b4633ed Merge remote-tracking branch 'origin/improvement/S3C-2293-upgradeNodeJS' into w/7.5/improvement/S3C-2293-upgradeNodeJS 2019-07-17 16:37:19 -07:00
anurag4dsb 59233b9b9e
improvement: S3C-2293 upgrade NodeJS to v10 2019-07-17 16:34:09 -07:00
bert-e 453b4e5999 Merge branch 'w/8.0/bugfix/S3C-2157-sanity-check' into tmp/octopus/w/8.1/bugfix/S3C-2157-sanity-check 2019-07-12 06:25:57 +00:00
bert-e 3b16d16ccd Merge branch 'w/7.5/bugfix/S3C-2157-sanity-check' into tmp/octopus/w/8.0/bugfix/S3C-2157-sanity-check 2019-07-12 06:25:57 +00:00
bert-e 1b7068f1a5 Merge branch 'w/7.4/bugfix/S3C-2157-sanity-check' into tmp/octopus/w/7.5/bugfix/S3C-2157-sanity-check 2019-07-12 06:25:56 +00:00
Rahul Padigela 8cc924a327 Merge remote-tracking branch 'origin/bugfix/S3C-2157-sanity-check' into w/7.4/bugfix/S3C-2157-sanity-check 2019-07-11 22:47:50 -07:00
Rahul Padigela df2c4b15da bugfix: S3C-1959 add sanity check for data deletion
When metadata service is unstable it's possible that an object has successfully
completed composing, final object written to the destination bucket but metadata
service returned HTTP code 500. In this case, the server returns HTTP code 500 to the client and
if the client retries the complete mpu request, it tries to compose the object again but this
time accidentally deleting data keys from the previous request as it assumes that it is overwriting
an existing object where in reality it is deleting the very own keys of the object it is composing.
This check ensures that the current object being composed does not
accidentally delete current keys.

(cherry picked from commit 111f90e500)
2019-07-11 22:26:08 -07:00
bert-e eb2776b1cd Merge branch 'w/8.0/feature/S3C-2160-eve-backport' into tmp/octopus/w/8.1/feature/S3C-2160-eve-backport 2019-07-12 00:48:57 +00:00
bert-e b59e06a5db Merge branch 'w/7.5/feature/S3C-2160-eve-backport' into tmp/octopus/w/8.0/feature/S3C-2160-eve-backport 2019-07-12 00:48:57 +00:00
bert-e 320f5cddfe Merge branch 'w/7.4/feature/S3C-2160-eve-backport' into tmp/octopus/w/7.5/feature/S3C-2160-eve-backport 2019-07-12 00:48:56 +00:00
Rahul Padigela 5f044a3e14 Merge remote-tracking branch 'origin/feature/S3C-2160-eve-backport' into w/7.4/feature/S3C-2160-eve-backport 2019-07-11 17:31:38 -07:00
Rahul Padigela db9b77835d tests: skip tests and follow up with a fix 2019-07-11 16:50:52 -07:00
Rahul Padigela a850371652 improvement: use nodejs 6 for dockerfiles 2019-07-11 16:50:52 -07:00
Rahul Padigela 91ac1aaac2 feature: S3C-2160 add eve support 2019-07-11 16:50:52 -07:00
bert-e 2a8d541786 Merge branch 'w/8.1/bugfix/S3C-2172-bucket-error' into tmp/octopus/q/8.1 2019-07-11 21:16:38 +00:00
bert-e cf23c3ccf0 Merge branch 'feature/ZENKO-1840-mongo-auth-2' into q/8.1 2019-07-11 18:26:59 +00:00
Rahul Padigela 1861156ff9 Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-2172-bucket-error' into w/8.1/bugfix/S3C-2172-bucket-error 2019-07-11 10:42:13 -07:00
Rahul Padigela 5a66c65296 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2172-bucket-error' into w/8.0/bugfix/S3C-2172-bucket-error 2019-07-11 09:25:57 -07:00
Rahul Padigela e11f443330 Merge remote-tracking branch 'origin/bugfix/S3C-2172-bucket-error' into w/7.5/bugfix/S3C-2172-bucket-error 2019-07-09 14:01:13 -07:00
Salim a7d93b0014 bugfix: allow delete markers on NFS backed objects 2019-07-08 12:03:47 -07:00
Rahul Padigela 20ccab2233 bugfix: ZENKO-1930 update arsenal to fix batchdelete 2019-07-03 17:56:34 -07:00
Rahul Padigela e76cd83772 bugfix: update arsenal 2019-06-28 16:31:04 -07:00
anurag4dsb ca74e04ac7
bf: ZENKO-1948 fix env variable for vault host check 2019-06-26 16:11:33 -07:00
anurag4dsb 7170b5722e
improvement: ZENKO-1944 Make Vaultd host modular 2019-06-26 11:54:30 -07:00
Nicolas Humbert 3806fe7030 ZENKO-1840 fix secure MongoDB access 2019-06-21 17:46:56 -07:00
Rahul Padigela 014c05187c test: add test for deleting invalid bucket name 2019-06-20 21:39:42 -07:00
Nicolas Humbert 30b0c8ceca update package.json 2019-06-19 15:57:14 -07:00
Nicolas Humbert 6ffaca6953 ZENKO-1840 Secure MongoDb access 2019-06-19 15:57:08 -07:00
Rahul Padigela c1c7fa2a1d bugfix: ZENKO-1908 update arsenal for batchdelete 2019-06-18 16:59:04 -07:00
Rahul Padigela cd665b9715 bugfix: remove unused sproxydclient 2019-06-18 16:48:51 -07:00
philipyoo 68cae12e07 bf: ZENKO-1736 update arsenal countItems
Update arsenal for countItems changes
2019-06-14 16:43:10 -07:00
bert-e 5f137d47b1 Merge branch 'feature/ZENKO-1360/doc-update' into q/8.1 2019-06-13 18:44:27 +00:00
bbuchanan9 320336a9e2 bugfix: ZENKO-1872 Check array length 2019-06-12 14:31:55 -07:00
bert-e 51b76c9803 Merge branch 'bugfix/ZENKO-1883-utapi-redis-config' into q/8.0 2019-06-12 18:48:04 +00:00
bert-e 1aba527523 Merge branches 'w/8.1/bugfix/ZENKO-1883-utapi-redis-config' and 'q/1934/8.0/bugfix/ZENKO-1883-utapi-redis-config' into tmp/octopus/q/8.1 2019-06-12 18:48:04 +00:00
Rahul Padigela 38b9f72607 bugfix: S3C-2243 add arsenal dependency 2019-06-11 19:34:25 -07:00
Rahul Padigela 43bf45df23 Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-2243-batchdelete-keys' into w/8.1/bugfix/S3C-2243-batchdelete-keys 2019-06-11 18:47:18 -07:00
Rahul Padigela e40be11058 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2243-batchdelete-keys' into w/8.0/bugfix/S3C-2243-batchdelete-keys 2019-06-11 18:14:18 -07:00
bert-e 1393d57dc5 Merge branch 'w/7.4/bugfix/S3C-2243-batchdelete-keys' into tmp/octopus/w/7.5/bugfix/S3C-2243-batchdelete-keys 2019-06-12 01:12:14 +00:00
bert-e 6cc5027949 Merge branch 'bugfix/S3C-2243-batchdelete-keys' into tmp/octopus/w/7.4/bugfix/S3C-2243-batchdelete-keys 2019-06-12 01:12:13 +00:00
Rahul Padigela d269fa9384 bugfix: S3C-2243 fix check for location type
This fixes the check where the logic should be looking at the type
of location instead of the name to leverage batch delete. It also fixes
the format sent to the sproxydclient which expects and object with keys
as an attribute whose value is an array of sproxyd keys.
2019-06-11 18:11:14 -07:00
bert-e 2325f726b0 Merge branch 'bugfix/ZENKO-1883-utapi-redis-config' into tmp/octopus/w/8.1/bugfix/ZENKO-1883-utapi-redis-config 2019-06-11 18:04:25 +00:00
Jianqin Wang 207b743266 bugfix: fix utapi redis configuration parsing 2019-06-11 11:01:30 -07:00
Salim 3f0ef9a68a Merge branch 'w/8.0/improvement/ZENKO-1867-fix-product-version' into w/8.1/improvement/ZENKO-1867-fix-product-version 2019-06-10 14:22:44 -07:00
Salim e92ba8900f Merge branch 'w/7.5/improvement/ZENKO-1867-fix-product-version' into w/8.0/improvement/ZENKO-1867-fix-product-version 2019-06-10 14:21:57 -07:00
Salim 32c8c4d1eb Merge branch 'improvement/ZENKO-1867-fix-product-version' into w/7.5/improvement/ZENKO-1867-fix-product-version 2019-06-10 14:19:49 -07:00
Salim 1781db1759 ci: only generate version on specific branches 2019-06-10 14:18:56 -07:00
bert-e 9465ecffc1 Merge branch 'w/8.0/bugfix/S3C-2076-utapi-reindex-config' into tmp/octopus/w/8.1/bugfix/S3C-2076-utapi-reindex-config 2019-06-07 05:12:57 +00:00
bert-e c704293b04 Merge branch 'w/7.5/bugfix/S3C-2076-utapi-reindex-config' into tmp/octopus/w/8.0/bugfix/S3C-2076-utapi-reindex-config 2019-06-07 05:12:57 +00:00
bert-e bd3a34b5c3 Merge branch 'w/7.4/bugfix/S3C-2076-utapi-reindex-config' into tmp/octopus/w/7.5/bugfix/S3C-2076-utapi-reindex-config 2019-06-07 05:12:56 +00:00
bert-e b4e4006891 Merge branch 'bugfix/S3C-2076-utapi-reindex-config' into tmp/octopus/w/7.4/bugfix/S3C-2076-utapi-reindex-config 2019-06-07 05:12:56 +00:00
Jianqin Wang df119d48fd bugfix: utapi reindex needs configuration to overwrite defaults 2019-06-06 22:02:06 -07:00
Jianqin Wang cb23a7e81f Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-2019-redis-authentication' into w/8.1/bugfix/S3C-2019-redis-authentication 2019-06-05 16:58:15 -07:00
Jianqin Wang ca7ea1465c Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2019-redis-authentication' into w/8.0/bugfix/S3C-2019-redis-authentication 2019-06-05 16:32:09 -07:00
Jianqin Wang 69f11143a5 Merge remote-tracking branch 'origin/w/7.4/bugfix/S3C-2019-redis-authentication' into w/7.5/bugfix/S3C-2019-redis-authentication 2019-06-05 15:51:36 -07:00
bert-e b720e333e2 Merge branch 'bugfix/S3C-2019-redis-authentication' into tmp/octopus/w/7.4/bugfix/S3C-2019-redis-authentication 2019-06-05 22:49:35 +00:00
Jianqin Wang 332c57f082 bugfix: redis sentinel authentication 2019-06-05 15:47:39 -07:00
bert-e 271c0e3075 Merge branch 'w/8.0/bugfix/S3C-2076/update-utapi-dep' into tmp/octopus/w/8.1/bugfix/S3C-2076/update-utapi-dep 2019-06-05 21:03:14 +00:00
bbuchanan9 0f7a8a59aa Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2076/update-utapi-dep' into w/8.0/bugfix/S3C-2076/update-utapi-dep 2019-06-05 13:58:47 -07:00
bert-e 0adb754d6f Merge branch 'w/7.4/bugfix/S3C-2076/update-utapi-dep' into tmp/octopus/w/7.5/bugfix/S3C-2076/update-utapi-dep 2019-06-05 20:45:36 +00:00
bert-e 039820ee74 Merge branch 'bugfix/S3C-2076/update-utapi-dep' into tmp/octopus/w/7.4/bugfix/S3C-2076/update-utapi-dep 2019-06-05 20:45:36 +00:00
bbuchanan9 e20f024451 bugfix: S3C-2076 Update Utapi dependency 2019-06-05 13:43:28 -07:00
bert-e a9bdc8b1ba Merge branch 'w/8.1/bugfix/S3C-2076/add-utapi-reindex' into tmp/octopus/q/8.1 2019-06-05 07:41:27 +00:00
bbuchanan9 7ee1ec28f6 Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-2076/add-utapi-reindex' into w/8.1/bugfix/S3C-2076/add-utapi-reindex 2019-06-05 00:03:16 -07:00
bbuchanan9 eb67f17397 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2076/add-utapi-reindex' into w/8.0/bugfix/S3C-2076/add-utapi-reindex 2019-06-04 23:29:20 -07:00
bbuchanan9 07d449b533 Merge remote-tracking branch 'origin/w/7.4/bugfix/S3C-2076/add-utapi-reindex' into w/7.5/bugfix/S3C-2076/add-utapi-reindex 2019-06-04 22:38:38 -07:00
bert-e 8e9b488c5a Merge branch 'bugfix/S3C-2076/add-utapi-reindex' into tmp/octopus/w/7.4/bugfix/S3C-2076/add-utapi-reindex 2019-06-05 05:03:48 +00:00
bbuchanan9 983b9ad602 bugfix: S3C-2076 Add Utapi reindex 2019-06-04 21:56:32 -07:00
bert-e 470f230212 Merge branch 'feature/ZENKO-1718-testsMongoClientInterface' into q/8.1 2019-06-04 22:31:43 +00:00
bert-e a6de663ad4 Merge branch 'feature/ZENKO-1842/azure-info-models' into q/8.1 2019-06-04 21:39:10 +00:00
Dora Korpar 8351555c3d bf: ZENKO 1857 utapi package version 2019-06-04 10:34:37 -07:00
Rahul Padigela 01ad582647 bugfix: S3C-1139 sproxyd batchdelete 2019-06-04 00:17:18 -07:00
Rahul Padigela 1dd0aa3fc3 Merge remote-tracking branch 'origin/w/8.0/feature/S3C-1139-sproxyd-batchdelete' into w/8.1/feature/S3C-1139-sproxyd-batchdelete 2019-06-04 00:12:26 -07:00
Rahul Padigela 7aaf561249 Merge remote-tracking branch 'origin/w/7.5/feature/S3C-1139-sproxyd-batchdelete' into w/8.0/feature/S3C-1139-sproxyd-batchdelete 2019-06-03 23:55:02 -07:00
bert-e 55cd73a718 Merge branch 'w/7.4/feature/S3C-1139-sproxyd-batchdelete' into tmp/octopus/w/7.5/feature/S3C-1139-sproxyd-batchdelete 2019-06-04 06:52:27 +00:00
bert-e 950bced0a4 Merge branch 'feature/S3C-1139-sproxyd-batchdelete' into tmp/octopus/w/7.4/feature/S3C-1139-sproxyd-batchdelete 2019-06-04 06:52:27 +00:00
Rahul Padigela eef28436ca bugfix: S3C-1139 use batch delete in api methods
This commits makes the batch delete invocation to use callbacks
so that orphans are not created when sproxyd is unstable.
2019-06-03 23:08:32 -07:00
Rahul Padigela 2fe55782a0 improvement: implement batch delete for sproxydclient
This commit implements the sproxyd's batch delete method in the
data abstractions so that APIs can leverage it for deleting large
number of keys. It also changes the batchDelete function's signature
by adding a callback. It is expected for the api methods to use callback
when invoking the method.
2019-06-03 23:08:23 -07:00
bert-e 2aa5be05ed Merge branch 'w/8.1/bugfix/S3C-2222-artifact-conflicts' into tmp/octopus/q/8.1 2019-05-31 17:51:16 +00:00
dashagurova 738ddcb56a documentation: ZENKOIO-22 Update Old Links
adding documentation

remove break line
2019-05-30 15:42:48 -07:00
bert-e dbb9a5dc6f Merge branch 'w/8.0/bugfix/S3C-2222-artifact-conflicts' into tmp/octopus/w/8.1/bugfix/S3C-2222-artifact-conflicts 2019-05-30 21:58:30 +00:00
bert-e 30216d7d51 Merge branch 'w/7.5/bugfix/S3C-2222-artifact-conflicts' into tmp/octopus/w/8.0/bugfix/S3C-2222-artifact-conflicts 2019-05-30 21:58:29 +00:00
bert-e 60478060b4 Merge branch 'bugfix/S3C-2222-artifact-conflicts' into tmp/octopus/w/7.5/bugfix/S3C-2222-artifact-conflicts 2019-05-30 21:58:29 +00:00
Salim cf60e513a3 ci: remove artifacts property
S3C-2222 This removes the conflicting artifact name which causes issues with
the CI pipeline.
2019-05-30 14:54:10 -07:00
Guillaume Gimenez dcea59418b feature: ZENKO-1842: azure-info-models
Updated ModelVersion.md to refect models change in arsenal
2019-05-30 10:14:07 -07:00
philipyoo 4faed870d3 ft: ZENKO-1718 MongoClientInterface putObj tests 2019-05-28 09:39:10 -07:00
Dora Korpar db85022738 bf: ZENKO 1728 update arsenal 2019-05-24 15:58:02 -07:00
bert-e efacf39cfe Merge branches 'w/8.0/improvement/S3C-2034-bump-ioredis' and 'q/1836/7.5/improvement/S3C-2034-bump-ioredis' into tmp/octopus/q/8.0 2019-05-23 19:15:17 +00:00
bert-e 5e49e63f17 Merge branches 'w/8.1/improvement/S3C-2034-bump-ioredis' and 'q/1836/8.0/improvement/S3C-2034-bump-ioredis' into tmp/octopus/q/8.1 2019-05-23 19:15:17 +00:00
bert-e 297c34b41d Merge branch 'improvement/S3C-2034-bump-ioredis' into q/7.4 2019-05-23 19:15:16 +00:00
bert-e 703698b0f7 Merge branches 'w/7.5/improvement/S3C-2034-bump-ioredis' and 'q/1836/7.4/improvement/S3C-2034-bump-ioredis' into tmp/octopus/q/7.5 2019-05-23 19:15:16 +00:00
bert-e 6e6eb92513 Merge branches 'development/8.1' and 'w/8.0/improvement/S3C-2034-bump-ioredis' into tmp/octopus/w/8.1/improvement/S3C-2034-bump-ioredis 2019-05-23 17:19:03 +00:00
Jianqin Wang ef469216ac Merge remote-tracking branch 'origin/w/7.5/improvement/S3C-2034-bump-ioredis' into w/8.0/improvement/S3C-2034-bump-ioredis 2019-05-23 10:18:44 -07:00
Jianqin Wang 6736acb18f Merge remote-tracking branch 'origin/development/8.0' into w/8.0/improvement/S3C-2034-bump-ioredis 2019-05-23 10:16:38 -07:00
bert-e bd9d843715 Merge branch 'development/7.5' into tmp/octopus/w/7.5/improvement/S3C-2034-bump-ioredis 2019-05-23 17:08:30 +00:00
Jianqin Wang a2024c9c25 update: fix arsenal 7.5 version 2019-05-23 10:07:50 -07:00
bert-e f52c0056cc Merge branch 'w/8.0/bugfix/S3C-2118-abort-mpu-crash-2' into tmp/octopus/w/8.1/bugfix/S3C-2118-abort-mpu-crash-2 2019-05-21 23:14:47 +00:00
Rahul Padigela 9fd8468e9c Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2118-abort-mpu-crash-2' into w/8.0/bugfix/S3C-2118-abort-mpu-crash-2 2019-05-21 16:00:08 -07:00
bert-e d04bed1edc Merge branch 'bugfix/S3C-2118-abort-mpu-crash-2' into tmp/octopus/w/7.5/bugfix/S3C-2118-abort-mpu-crash-2 2019-05-21 22:57:59 +00:00
Jianqin Wang d07d85c944 Merge remote-tracking branch 'origin/w/8.0/improvement/S3C-2034-bump-ioredis' into w/8.1/improvement/S3C-2034-bump-ioredis 2019-05-21 15:30:17 -07:00
Jianqin Wang 91f59f876a update: package-lock.json file 2019-05-21 15:26:05 -07:00
Jianqin Wang 48703f48cf Merge remote-tracking branch 'origin/w/8.0/improvement/S3C-2034-bump-ioredis' into w/8.1/improvement/S3C-2034-bump-ioredis 2019-05-21 15:12:42 -07:00
Jianqin Wang 5a09b0226b Merge remote-tracking branch 'origin/w/7.5/improvement/S3C-2034-bump-ioredis' into w/8.0/improvement/S3C-2034-bump-ioredis 2019-05-21 14:52:24 -07:00
Jianqin Wang 4a8d79544a Merge remote-tracking branch 'origin/improvement/S3C-2034-bump-ioredis' into w/7.5/improvement/S3C-2034-bump-ioredis 2019-05-21 14:41:02 -07:00
Rahul Padigela afeb56b66a bugfix: S3C-2118 fix crash when aborting non-existent mpu 2019-05-21 14:34:29 -07:00
Rahul Padigela 58c8af076d test: add test for aborting non-existent mpu 2019-05-21 14:34:29 -07:00
Rahul Padigela 7a1118e33c improvement: add listener for uncaughtException
This makes sure that uncaught exceptions are logged before the process
quits.
2019-05-21 14:34:29 -07:00
Jianqin Wang dcb89e3ad7 S3C-2034: bump ioredis for authentication 2019-05-21 14:28:04 -07:00
bbuchanan9 d0c6d93921 Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-2195/upload-copy-part-metrics' into w/8.1/bugfix/S3C-2195/upload-copy-part-metrics 2019-05-21 11:22:30 -07:00
bbuchanan9 f3ea4f1103 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2195/upload-copy-part-metrics' into w/8.0/bugfix/S3C-2195/upload-copy-part-metrics 2019-05-21 10:31:23 -07:00
bert-e ce1c19df72 Merge branch 'bugfix/S3C-2195/upload-copy-part-metrics' into tmp/octopus/w/7.5/bugfix/S3C-2195/upload-copy-part-metrics 2019-05-20 23:19:51 +00:00
bbuchanan9 ddfa4fa716 bugfix: S3C-2195 Add uploadPartCopy UTAPI metric 2019-05-20 16:14:16 -07:00
Rahul Padigela 691b75aa18 improvement: ZENKO-1827 move to debian Dockerfile 2019-05-20 15:35:26 -07:00
bert-e b9ec6432cb Merge branch 'bugfix/ZENKO-1506_flaky_s3cmd_test' into q/8.1 2019-05-17 17:16:11 +00:00
bbuchanan9 70d4e010f2 Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-2105/add-utapi-crr-conditions' into w/8.1/bugfix/S3C-2105/add-utapi-crr-conditions 2019-05-17 09:59:03 +02:00
bert-e 62591df809 Merge branch 'w/7.5/bugfix/S3C-2105/add-utapi-crr-conditions' into tmp/octopus/w/8.0/bugfix/S3C-2105/add-utapi-crr-conditions 2019-05-17 07:55:53 +00:00
bert-e 52a7218ac8 Merge branch 'bugfix/S3C-2105/add-utapi-crr-conditions' into tmp/octopus/w/7.5/bugfix/S3C-2105/add-utapi-crr-conditions 2019-05-17 07:55:52 +00:00
bbuchanan9 31de552ac6 bugfix: S3C-2105 Add utapi CRR conditions 2019-05-16 11:00:35 +02:00
Taylor McKinnon e69619f784 bf(ZENKO-1506: Increase pod memory limits 2019-05-15 16:43:47 -07:00
Jonathan Gramain 78efc3f8b8 bugfix: ZENKO-1809 allow replication from md-ingestion account
Allow md-ingestion service account to trigger replication on new
objects, so that NFS ingestion buckets are allowed to replicate to
cloud locations.
2019-05-15 16:21:13 -07:00
Jonathan Gramain c9cc5849b2 bugfix: ZENKO-1809 flag nfs location type as supporting versioning
Change the supportsVersioning flag to true for NFS locations, as
although versioning is set initially by cloudserver, they should also
support new calls to putBucketVersioning() with Status=Enabled. This
partially fixes replication support by making sure the orbit workflow
can be applied to NFS locations.
2019-05-15 16:20:46 -07:00
bert-e e7308d4487 Merge branch 'feature/ZENKO-1755-add-support-for-hd' into q/8.1 2019-05-15 16:59:05 +00:00
Taylor McKinnon c9199c0cc3 bf(ZENKO-1506): Fix flaky s3cmd test 2019-05-13 11:34:18 -07:00
bert-e 3148a71091 Merge branches 'w/8.0/bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes' and 'q/1816/7.5/bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes' into tmp/octopus/q/8.0 2019-05-10 23:36:24 +00:00
bert-e c0782322a3 Merge branches 'w/8.1/bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes' and 'q/1816/8.0/bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes' into tmp/octopus/q/8.1 2019-05-10 23:36:24 +00:00
bert-e 810c5480e1 Merge branches 'w/7.5/bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes' and 'q/1816/7.4/bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes' into tmp/octopus/q/7.5 2019-05-10 23:36:23 +00:00
bert-e 6357d14c13 Merge branch 'bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes' into q/7.4 2019-05-10 23:36:23 +00:00
bbuchanan9 69d28bf0bc Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-1506/upgrade-utapi-dependency' into w/8.1/bugfix/S3C-1506/upgrade-utapi-dependency 2019-05-10 13:47:51 -07:00
bbuchanan9 2cf54d0579 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-1506/upgrade-utapi-dependency' into w/8.0/bugfix/S3C-1506/upgrade-utapi-dependency 2019-05-10 11:52:50 -07:00
bbuchanan9 21f9c7105e Merge remote-tracking branch 'origin/w/7.4/bugfix/S3C-1506/upgrade-utapi-dependency' into w/7.5/bugfix/S3C-1506/upgrade-utapi-dependency 2019-05-10 11:19:39 -07:00
bbuchanan9 a2ece0ba5c Merge remote-tracking branch 'origin/bugfix/S3C-1506/upgrade-utapi-dependency' into w/7.4/bugfix/S3C-1506/upgrade-utapi-dependency 2019-05-10 10:37:51 -07:00
Benoit A 00e5b49b55 ZENKO-1755 Add hdclient dependency to package.json
Signed-off-by: Benoit A <benoit@scality.com>
2019-05-10 10:09:59 +02:00
bbuchanan9 25d68f2510 bugfix: S3C-1506 Upgrade UTAPI dependency 2019-05-09 17:16:51 -07:00
bbuchanan9 a389e9e6d4 Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes' into w/8.1/bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes 2019-05-09 13:21:18 -07:00
bert-e 7e42b43c86 Merge branch 'w/7.5/bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes' into tmp/octopus/w/8.0/bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes 2019-05-08 23:10:25 +00:00
bert-e e6a7cf1834 Merge branch 'bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes' into tmp/octopus/w/7.5/bugfix/S3C-2105/push-utapi-metrics-for-backbeat-routes 2019-05-08 23:10:25 +00:00
bbuchanan9 2e8a5d298d bugfix: S3C-2105 Push UTAPI metrics for CRR 2019-05-08 16:06:47 -07:00
philipyoo 0cc3124999 bf: ZENKO-1585 userMD save hashed instance id
user metadata field for `constants.zenkoIDHeader` uses a
hashed instance id (of the current zenko deployment)
2019-05-02 13:57:45 -07:00
philipyoo fb795d557d bf: ZENKO-1585 apply zenko user metadata field
For objects/versions created in a Zenko deployment, we
apply a user metadata field indicating as so.
Changes apply for following ops:
- objectPut
- multiObjectDelete
- objectCopy
- initiateMPU
2019-05-02 13:57:45 -07:00
bert-e 88abf21f37 Merge branch 'q/1805/8.0/bugfix/S3C-1959-sanity-check' into tmp/normal/q/8.1 2019-05-02 06:39:29 +00:00
bert-e 9ae8e432d5 Merge branch 'w/8.1/bugfix/S3C-1959-sanity-check' into tmp/normal/q/8.1 2019-05-02 06:39:29 +00:00
bert-e b587ff29e2 Merge branches 'w/8.0/bugfix/S3C-1959-sanity-check' and 'q/1805/7.5/bugfix/S3C-1959-sanity-check' into tmp/octopus/q/8.0 2019-05-02 06:39:28 +00:00
bert-e 7efe13bbdf Merge branch 'w/7.5/bugfix/S3C-1959-sanity-check' into tmp/octopus/q/7.5 2019-05-02 06:39:28 +00:00
Rahul Padigela 1df12f1b0e refactor: move tests to arsenal 2019-05-01 22:25:32 -07:00
Rahul Padigela ecd3d803a4 Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-1959-sanity-check' into w/8.1/bugfix/S3C-1959-sanity-check 2019-05-01 21:55:11 -07:00
Rahul Padigela 07339703f5 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-1959-sanity-check' into w/8.0/bugfix/S3C-1959-sanity-check 2019-05-01 17:33:47 -07:00
bert-e 1ab202c9dc Merge branch 'bugfix/S3C-1959-sanity-check' into tmp/octopus/w/7.4/bugfix/S3C-1959-sanity-check 2019-05-02 00:31:48 +00:00
bert-e 2100190607 Merge branch 'w/7.4/bugfix/S3C-1959-sanity-check' into tmp/octopus/w/7.5/bugfix/S3C-1959-sanity-check 2019-05-02 00:31:48 +00:00
Rahul Padigela 0cc2ff62e1 bugfix: fix typo in test setup 2019-05-01 17:17:54 -07:00
Rahul Padigela 9fc9a6d5d6 improvement: avoid coverage use npm test instead 2019-05-01 17:17:54 -07:00
Rahul Padigela 111f90e500 bugfix: S3C-1959 add sanity check for data deletion
When metadata service is unstable it's possible that an object has successfully
completed composing, final object written to the destination bucket but metadata
service returned HTTP code 500. In this case, the server returns HTTP code 500 to the client and
if the client retries the complete mpu request, it tries to compose the object again but this
time accidentally deleting data keys from the previous request as it assumes that it is overwriting
an existing object where in reality it is deleting the very own keys of the object it is composing.
This check ensures that the current object being composed does not
accidentally delete current keys.
2019-05-01 17:17:48 -07:00
Rahul Padigela 32aee44640 test: process mpu parts 2019-05-01 17:16:20 -07:00
bert-e 80784c15cf Merge branches 'w/8.1/feature/S3C-1974_PyKMIP_in_CI' and 'q/1775/8.0/feature/S3C-1974_PyKMIP_in_CI' into tmp/octopus/q/8.1 2019-04-30 14:08:25 +00:00
bert-e 38ddb50a1c Merge branches 'w/8.0/feature/S3C-1974_PyKMIP_in_CI' and 'q/1775/7.5/feature/S3C-1974_PyKMIP_in_CI' into tmp/octopus/q/8.0 2019-04-30 14:08:25 +00:00
bert-e d602a25b18 Merge branch 'feature/S3C-1974_PyKMIP_in_CI' into q/7.5 2019-04-30 14:08:25 +00:00
Dora Korpar 807130100f Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-2120-abort-mpu-timeout' into w/8.1/bugfix/S3C-2120-abort-mpu-timeout 2019-04-24 17:09:31 -07:00
Dora Korpar a779e25ca6 fix linter 2019-04-24 16:37:05 -07:00
Dora Korpar 2426e47a16 S3C 2120 update arsenal 2019-04-24 15:38:03 -07:00
Dora Korpar f897998cb9 Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-2120-abort-mpu-timeout' into w/8.1/bugfix/S3C-2120-abort-mpu-timeout 2019-04-24 15:37:40 -07:00
Dora Korpar 1b958cfa23 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2120-abort-mpu-timeout' into w/8.0/bugfix/S3C-2120-abort-mpu-timeout 2019-04-24 15:20:40 -07:00
bert-e 492e4655ed Merge branch 'bugfix/S3C-2120-abort-mpu-timeout' into tmp/octopus/w/7.4/bugfix/S3C-2120-abort-mpu-timeout 2019-04-24 22:09:26 +00:00
bert-e ab877326b1 Merge branch 'w/7.4/bugfix/S3C-2120-abort-mpu-timeout' into tmp/octopus/w/7.5/bugfix/S3C-2120-abort-mpu-timeout 2019-04-24 22:09:26 +00:00
Dora Korpar 0d85022d1e bf: S3C 2120 mpu legacy check 2019-04-24 15:01:14 -07:00
Dora Korpar c56daa83ec bf: S3C 2120 tests 2019-04-24 14:35:02 -07:00
bert-e 6e431a7a32 Merge branches 'w/8.1/bugfix/ZENKO-1745-Fix-HTTP-Agent_config_backport' and 'q/1793/8.0/bugfix/ZENKO-1745-Fix-HTTP-Agent_config_backport' into tmp/octopus/q/8.1 2019-04-22 17:32:22 +00:00
bert-e 02cb1a8c57 Merge branch 'bugfix/ZENKO-1745-Fix-HTTP-Agent_config_backport' into q/8.0 2019-04-22 17:32:22 +00:00
bert-e 3b31ad1fb1 Merge branch 'bugfix/ZENKO-1745-Fix-HTTP-Agent_config_backport' into tmp/octopus/w/8.1/bugfix/ZENKO-1745-Fix-HTTP-Agent_config_backport 2019-04-19 23:54:08 +00:00
Taylor McKinnon 559a20c702 bf(ZENKO-1745): Fix HTTP Agent configuration in docker-entrypoint.sh 2019-04-19 16:53:09 -07:00
bert-e fb47185f5a Merge branch 'bugfix/ZENKO-1745-Fix-HTTP-Agent_config' into q/8.1 2019-04-19 20:37:12 +00:00
Taylor McKinnon a6b5965ac0 bf(ZENKO-1745): Fix HTTP Agent configuration in docker-entrypoint.sh 2019-04-19 10:49:16 -07:00
bert-e 0f91054f94 Merge branch 'w/8.1/bugfix/S3C-2099-zenkoUserMDFieldDeleteMarkers' into tmp/octopus/q/8.1 2019-04-19 05:25:02 +00:00
bbuchanan9 61a6a159c8 bugfix: ZENKO-1606 MPU tagging during replication 2019-04-18 15:47:15 -07:00
philipyoo 4c5efef677 Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-2099-zenkoUserMDFieldDeleteMarkers' into w/8.1/bugfix/S3C-2099-zenkoUserMDFieldDeleteMarkers 2019-04-18 12:00:54 -07:00
philipyoo 4060341963 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2099-zenkoUserMDFieldDeleteMarkers' into w/8.0/bugfix/S3C-2099-zenkoUserMDFieldDeleteMarkers 2019-04-18 10:37:28 -07:00
bert-e e5b8765e1d Merge branch 'bugfix/S3C-2099-zenkoUserMDFieldDeleteMarkers' into tmp/octopus/w/7.5/bugfix/S3C-2099-zenkoUserMDFieldDeleteMarkers 2019-04-18 17:31:25 +00:00
philipyoo fbed7584be bf: S3C-2099 zenko userMD for delete markers
To solve retro-propagation for S3C ingestion, we need to
mark objects with a user-metadata field on all objects
created in a Zenko deployment.

For delete markers, we cannot send a request specifying
user-metadata as it does not comply with aws s3 standards.
Instead, we must somehow detect these delete markers in
S3C and apply the user-md ourselves. We rely on the
'user-agent' header.
2019-04-18 10:29:07 -07:00
Taylor McKinnon 8e2c9ff6da Merge remote-tracking branch 'origin/w/8.0/feature/S3C-1974_PyKMIP_in_CI' into w/8.1/feature/S3C-1974_PyKMIP_in_CI 2019-04-16 10:15:57 -07:00
Taylor McKinnon 1d5199f226 Merge branch 'feature/S3C-1974_PyKMIP_in_CI' into w/8.0/feature/S3C-1974_PyKMIP_in_CI 2019-04-16 10:09:47 -07:00
Taylor McKinnon 0821d4a223 ft(S3C-1974): Add PyKMIP to CI 2019-04-16 10:02:22 -07:00
bbuchanan9 34400be01a bugfix: ZENKO-1583 Add functional tests 2019-04-12 13:22:42 -07:00
bert-e c898505c01 Merge branch 'bugfix/ZENKO-1606/tagging-during-transition' into q/8.1 2019-04-10 18:32:32 +00:00
bbuchanan9 07f7ed1f1e bugfix: ZENKO-1606 Update Arsenal dependency 2019-04-10 09:43:24 -07:00
bbuchanan9 dc96e2ecfe bugfix: ZENKO-1606 Replicate tags from object PUT 2019-04-05 13:21:19 -07:00
bbuchanan9 ad395c568a bugfix: ZENKO-1583 Azure err code, add log detail 2019-04-03 11:15:49 -07:00
bbuchanan9 f68b216eb5 bugfix: ZENKO-1583 Add if-unmodified-since check 2019-04-02 18:02:16 -07:00
philipyoo 59c86b9144 ft: ZENKO-1661 rename ingestion status report var
Rename ingestion status report variable from
`ingestionStatus` to `ingestStatus`
2019-03-27 17:03:53 -07:00
philipyoo 00c60a986f ft: ZENKO-1661 add ingestion data to reporthandler
Add ingestion info (metrics, pause state) to report handler.

Attempt to rename "stats" -> "metrics" since
"stats" and "state" or "status" are very similar.
2019-03-27 12:11:30 -07:00
bert-e e0faa73dce Merge branch 'w/8.0/feature/S3C-2035/tls-files-loading' into tmp/octopus/w/8.1/feature/S3C-2035/tls-files-loading 2019-03-25 17:08:01 +00:00
bert-e 68dca6cddf Merge branch 'feature/S3C-2035/tls-files-loading' into tmp/octopus/w/8.0/feature/S3C-2035/tls-files-loading 2019-03-25 17:08:00 +00:00
bert-e 56fbbe9c34 Merge branch 'bugfix/ZENKO-1610/disable-non-current-version-in-api' into q/8.1 2019-03-21 22:12:29 +00:00
bbuchanan9 51728c2019 bugfix: ZENKO-1610 Non-current version transition 2019-03-21 13:13:22 -07:00
bert-e afb64940de Merge branch 'feature/ZENKO-1585-replication-group-id' into q/8.1 2019-03-21 15:50:15 +00:00
bbuchanan9 fa9f7a768c feature: ZENKO-1529 Add managed transition report 2019-03-19 13:21:01 -07:00
Dora Korpar 655a74f195 bf: ZENKO 1598 bb routes regression 2019-03-18 15:31:12 -07:00
Guillaume Gimenez 5ea7db73e5 feature: S3C-2035: KMIP, TLS files from env
Load in memory the content of the keys and certificats
when the TLS configuration comes from the environment.
2019-03-18 15:14:29 -07:00
Rahul Padigela c41bf73a5c feature: ZENKO-1585 configure replication group id
Replication group id is used in the calculation of version ids for objects
on buckets that have versioning enabled. It has many benefits including avoiding
collisions in an active-active scenario, identifying distinct entries from an
instance etc.
This allows commit allows configuring a custom replication group id for an instance,
ensuring that the version ids generated by the instance are unique.
2019-03-18 12:04:59 -07:00
bert-e ed8838de54 Merge branch 'bugfix/ZENKO-1568-lc-update-crash' into q/8.1 2019-03-15 23:51:45 +00:00
bert-e 11e3b19475 Merge branch 'feature/ZENKO-1420-returnLocationFromMultipleBackendPutObject' into q/8.1 2019-03-15 23:51:16 +00:00
Dora Korpar 5fa4754ba4 bf: ZENKO 1402 location constraint update event 2019-03-15 16:39:49 -07:00
Jonathan Gramain 086250dc82 feature: ZENKO-1420 return locations from backbeat PUT routes
Return the locations array in the API of putObject and completeMPU
multiple backend routes, so that transition policies can use it to
update metadata with the new location as-is.
2019-03-15 16:24:45 -07:00
Guillaume Gimenez 78a5cd8b6e Merge remote-tracking branch 'origin/w/8.0/feature/S3C-2032/update-arsenal-deps' into w/8.1/feature/S3C-2032/update-arsenal-deps 2019-03-15 12:01:42 -07:00
Guillaume Gimenez 6bf16a6d24 Merge remote-tracking branch 'origin/feature/S3C-2032/update-arsenal-deps' into w/8.0/feature/S3C-2032/update-arsenal-deps 2019-03-14 17:40:28 -07:00
Guillaume Gimenez 8cc8724abc feature: S3C-2032: update arsenal dep
for the latest KMIP fixes
2019-03-14 17:29:19 -07:00
Rahul Padigela 842ada6960 feature: ZENKO-1570 enable versioning for ingestion
This commit ensures that versioning is automatically enabled for ingestion
buckets upon creation. It satisfies the requirement that all ingestion buckets
must be versioning enabled.
2019-03-14 10:11:04 -07:00
bert-e 5b6c67a67d Merge branch 'w/8.0/bugfix/S3C-1396-website-redirect-header-response' into tmp/octopus/w/8.1/bugfix/S3C-1396-website-redirect-header-response 2019-03-12 19:27:50 +00:00
Giacomo Guiulfo c83e623f44 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-1396-website-redirect-header-response' into w/8.0/bugfix/S3C-1396-website-redirect-header-response 2019-03-12 12:26:28 -07:00
bert-e 9836e2112e Merge branch 'bugfix/S3C-1396-website-redirect-header-response' into tmp/octopus/w/7.5/bugfix/S3C-1396-website-redirect-header-response 2019-03-12 19:25:01 +00:00
Giacomo Guiulfo d132e16e13 bugfix: WebsiteRedirectLocation header response 2019-03-12 12:24:22 -07:00
Rahul Padigela 7a538b012c Merge remote-tracking branch 'origin/w/8.0/bugfix/S3C-2002-restricted-admin-access' into w/8.1/bugfix/S3C-2002-restricted-admin-access 2019-03-11 22:08:37 -07:00
Rahul Padigela 68ab0821b7 Merge remote-tracking branch 'origin/w/7.5/bugfix/S3C-2002-restricted-admin-access' into w/8.0/bugfix/S3C-2002-restricted-admin-access 2019-03-11 17:30:50 -07:00
Rahul Padigela 7294d4704d Merge remote-tracking branch 'origin/w/7.4/bugfix/S3C-2002-restricted-admin-access' into w/7.5/bugfix/S3C-2002-restricted-admin-access 2019-03-11 17:05:07 -07:00
Rahul Padigela b363d8be02 bugfix: restrict metadata proxy routes to policy
These changes ensure that the routes are available only to account credentials
or credentials of users who are assigned metadata:admin policy
2019-03-10 11:28:09 -07:00
bert-e 66cd32f08c Merge branch 'w/8.0/feature/S3C-1969/kmip-cloudserver-integration' into tmp/octopus/w/8.1/feature/S3C-1969/kmip-cloudserver-integration 2019-03-07 00:22:16 +00:00
bert-e 460a9cec7e Merge branch 'feature/S3C-1969/kmip-cloudserver-integration' into tmp/octopus/w/8.0/feature/S3C-1969/kmip-cloudserver-integration 2019-03-07 00:22:16 +00:00
Guillaume Gimenez 3c6b706532 feature: S3C-1969: KMIP Cloudserver Integration 2019-03-06 16:21:05 -08:00
bert-e 0ffb1fea7f Merge branch 'feature/ZENKO-1402-move-data-wrapper' into q/8.1 2019-03-04 19:29:15 +00:00
Taylor McKinnon 3b47d0a636 bf(ZENKO-1511): Update Ceph CI Image 2019-03-01 15:11:27 -08:00
anurag4DSB 1823d56afd Merge remote-tracking branch 'origin/w/8.0/improvement/S3C-2005-updateVaultClient' into w/8.1/improvement/S3C-2005-updateVaultClient
Conflicts:
	package-lock.json
2019-02-27 22:06:02 +00:00
anurag4DSB 00023f9274 Merge remote-tracking branch 'origin/improvement/S3C-2005-updateVaultClient' into w/8.0/improvement/S3C-2005-updateVaultClient
Conflicts:
	package-lock.json
	package.json
2019-02-27 21:56:09 +00:00
anurag4DSB 5638af7ae9 improvement:S3C-1561- update vaultclient version 2019-02-27 21:32:27 +00:00
Dora Korpar 26004e59fa [chore] update arsenal 2019-02-26 14:54:37 -08:00
Dora Korpar 5aa25752a8 ft: ZENKO 1402 move data wrapper to arsenal 2019-02-26 14:54:11 -08:00
Taylor McKinnon 279c71382f Spawn file operations syncronously 2019-02-19 16:23:13 -08:00
Taylor McKinnon f1d2cd5d00 Bump test pod memory limits 2019-02-19 11:00:03 -08:00
Taylor McKinnon ceec8edae3 merge 2019-02-19 10:50:23 -08:00
Taylor McKinnon ae04387989 feat(ZENKO-1508): Add Ceph Orbit support 2019-02-15 17:41:47 -08:00
Taylor McKinnon 74b9369a93 feat(ZENKO-1508): Add Ceph Orbit support 2019-02-15 15:23:23 -08:00
Salim b95f6da45b ci: update mongo to 3.6.8 2019-02-13 17:34:38 -08:00
bert-e 20393b891e Merge branch 'w/8.1/bugfix/ZENKO-1460-maven-fix' into tmp/octopus/q/8.1 2019-02-12 01:45:23 +00:00
bert-e 0ce57a3d34 Merge branch 'w/7.5/bugfix/ZENKO-1460-maven-fix' into tmp/octopus/w/8.0/bugfix/ZENKO-1460-maven-fix 2019-02-11 19:42:46 +00:00
bert-e aa2459fdaf Merge branch 'w/8.0/bugfix/ZENKO-1460-maven-fix' into tmp/octopus/w/8.1/bugfix/ZENKO-1460-maven-fix 2019-02-11 19:42:46 +00:00
Rahul Padigela b03eea7976 bugfix: ZENKO-1460 maven fix for ci upgrade 2019-02-11 11:38:45 -08:00
Dora Korpar 5c7667f90c [chore] increase test aggressor mem 2019-02-11 10:21:53 -08:00
Dora Korpar ac8490c2cd update Arsenal version 2019-02-11 10:21:53 -08:00
Dora Korpar fb19b61d69 ft: ZENKO 1402 move data backends to arsenal 2019-02-11 10:21:53 -08:00
bert-e 7d1f0f8884 Merge branch 'bugfix/ZENKO-1453-ci-proxyvars' into tmp/octopus/w/8.1/bugfix/ZENKO-1453-ci-proxyvars 2019-02-08 23:20:47 +00:00
Rahul Padigela 766eb10980 bugfix: ZENKO-1452 avoid proxy conflicts in ci 2019-02-08 15:05:01 -08:00
William Abernathy 8970e5ef23 bugfix: S3C-1922_Remove_scality/s3server_refs
S3Server references were out of date (changed to CloudServer).
Fixing them revealed most of the links were broken, a lot of the
language was inelegant, and the organization made the document
hard to follow. Hence the voluminous corrections.

changed all instances of "Cloudserver" to "CloudServer"
2019-02-06 10:14:15 -08:00
Guillaume Gimenez c6362f22f3 feature: ZENKO-1360 fix typo in doc
(this is an external contribution)
2019-02-05 22:22:32 -08:00
William Abernathy b16e4574b6 feature: ZENKO-1466 convert md search doc to rst 2019-02-05 21:54:32 -08:00
Rahul Padigela 44c45374cb improvement: ZENKO-1465 avoid overlogging 2019-02-05 17:47:04 -08:00
Taylor McKinnon 961bddd9c1 Disable failing (or possibly flaky) tests ZENKO-1445 2019-01-28 10:55:28 -08:00
Taylor McKinnon 0cd34a7739 Buffer.byteLength doesn't accept null or undefined 2019-01-28 10:54:23 -08:00
Taylor McKinnon be04c145a5 Move to new docker image based on node:8-alpine 2019-01-28 10:54:19 -08:00
bert-e 91c19e522f Merge branch 'bugfix/ZENKO-1435-pathstyle-https' into q/8.1 2019-01-10 01:07:07 +00:00
Rahul Padigela 16d953d251 bugfix: ZENKO-1435 use CI_CEPH has test flag
The usage of CI env as a test flag for configuring https, pathStyle
options breaks other tests, so the env is being changed to be CI_CEPH
confining the scope to only CEPH backend tests.
2019-01-09 16:28:58 -08:00
Rahul Padigela 39a271d9b3 bugfix: ZENKO-1435 remove coverage commands
This switches up the coverage cmds of istanbul in favor of vanilla
mocha output for test results accuracy.
2019-01-09 15:32:07 -08:00
bert-e d8183f95bf Merge branch 'w/8.0/bugfix/S3C-1933-remove-pod-ttl' into tmp/octopus/w/8.1/bugfix/S3C-1933-remove-pod-ttl 2019-01-09 15:24:37 +00:00
bert-e e5f208b8d2 Merge branch 'bugfix/S3C-1933-remove-pod-ttl' into tmp/octopus/w/8.0/bugfix/S3C-1933-remove-pod-ttl 2019-01-09 15:24:36 +00:00
Thomas Carmet d9d8e4132f ci: S3C-1933 remove active deadline on ci workers 2019-01-09 16:22:26 +01:00
Giacomo Guiulfo 61c99fb4bb bugfix(pfsd): add noCache and noSync options 2019-01-07 17:27:24 -08:00
Giacomo Guiulfo 7d1ea8698f Merge remote-tracking branch 'origin/bugfix/ZENKO-1369-no-cache-option' into w/8.1/bugfix/ZENKO-1369-no-cache-option 2019-01-07 16:58:58 -08:00
Giacomo Guiulfo dd56a3c25c bugfix(s3-data): add noCache option 2019-01-07 16:52:28 -08:00
Rahul Padigela 7e9068a293 improvement: ZENKO-1421 ignore new rules for now 2019-01-06 23:22:25 -08:00
Rahul Padigela 9ef5bb9a41 improvement: ZENKO-1421 update eslint modules 2019-01-06 23:21:59 -08:00
Giacomo Guiulfo f3e403f31c feat: add nfs-mount-v1 support 2018-12-28 10:32:05 -08:00
Bennett Buchanan 8913851975 feature: ZENKO-1399 Update Arsenal dependency 2018-12-20 13:21:45 -08:00
bert-e 624b1ff73d Merge branch 'w/8.1/feature/S3C-1903-public-read' into tmp/octopus/q/8.1 2018-12-20 01:53:44 +00:00
bert-e 480e8f21f7 Merge branch 'improvement/ZENKO-1362-bump-arsenal' into q/8.1 2018-12-20 01:46:19 +00:00
bert-e 4122b85bc6 Merge branch 'w/8.0/feature/S3C-1903-public-read' into tmp/octopus/w/8.1/feature/S3C-1903-public-read 2018-12-20 01:31:36 +00:00
Rahul Padigela d4e7091c09 Merge remote-tracking branch 'origin/feature/S3C-1903-public-read' into w/8.0/feature/S3C-1903-public-read 2018-12-19 17:09:50 -08:00
JianqinWang 20b487ecf0 ZENKO-1343: include md-ingestion as valid bb acc
- move list of service accounts to constants.js
2018-12-19 16:29:29 -08:00
Rahul Padigela 5c31a09f11 feature: S3C-1903 allow public reads on buckets
This change supports proving a whitelist of buckets that have public-read
canned acl set, to allow reads from anonymous users on objects without
having to explicitly set public-read acl on them.
2018-12-19 14:52:27 -08:00
Giacomo Guiulfo 11b20c7302 chore: bump arsenal 2018-12-19 14:58:43 -05:00
Bennett Buchanan df1afd5252 feature: ZENKO-1388 Versioning for MD updates 2018-12-18 09:37:04 -08:00
bert-e 74fa5138c0 Merge branch 'feature/ZENKO-1317/aws-lifecycle-compat' into q/8.1 2018-12-17 21:56:34 +00:00
Salim 33a8e1c479 Merge remote-tracking branch 'origin/w/8.0/bugfix/ZENKO-1337-escape-char' into w/8.1/bugfix/ZENKO-1337-escape-char 2018-12-17 13:33:46 -08:00
bert-e 32e505e7b6 Merge branch 'bugfix/ZENKO-1337-escape-char' into tmp/octopus/w/8.0/bugfix/ZENKO-1337-escape-char 2018-12-17 21:31:46 +00:00
Salim 2eee7b90f3 ci: missing escape characters 2018-12-17 13:10:07 -08:00
Bennett Buchanan c59fc739a5 feature: ZENKO-1317 AWS lifecycle compat 2018-12-17 12:49:08 -08:00
bert-e a365c63b1c Merge branch 'bugfix/ZENKO-1365-orbit-ring-s3-https' into tmp/octopus/w/8.1/bugfix/ZENKO-1365-orbit-ring-s3-https 2018-12-14 20:06:59 +00:00
Dora Korpar f20b32fbe8 bugfix: ZENKO-1365 Orbit set https for RING+S3 2018-12-14 12:04:38 -08:00
Bennett Buchanan d47b71d612 feature: ZENKO-733 Lifecycle non-versioned buckets 2018-12-12 15:47:57 -08:00
bert-e e4fc624855 Merge branch 'feature/ZENKO-557-gcCloudLocations' into q/8.1 2018-12-12 19:28:00 +00:00
bert-e 0c0bfd75f6 Merge branch 'bugfix/ZENKO-1290-Add-CEPH-copy-object-support' into q/8.1 2018-12-12 19:21:07 +00:00
Bennett Buchanan de0115553c feature: ZENKO-732 Lifecycle transition policies 2018-12-11 16:58:18 -08:00
Jonathan Gramain 9bf709839b ZENKO-557 pass 'deleteVersion' param to backend
Cloud backends require the 'deleteVersion' param to be set to true in
order to delete a specific version.

Add a functional test to GC a versioned AWS location (all cloud
locations use the same interface from multipleBackend, hence should
"just work", assuming backends delete() is tested individually).

Note that backbeat route tests have been moved to
tests/multipleBackends because they require a real AWS location.

Fixed existing backbeat DELETE route test: this test has been moved to
multipleBackend along with other backbeat route tests, which seems to
have enabled it while it used to be disabled, it now fails because of
extra params passed to the listMultipartUploads AWS sdk call. Removing
the extra params fixes the test.

Also removed unused left-over file "tests.bash".
2018-12-11 15:54:45 -08:00
Taylor McKinnon 65b0514d5e bugfix(ZENKO-1290): Add CEPH copy object support 2018-12-11 14:47:59 -08:00
bert-e 7f25974c08 Merge branch 'feature/ZENKO-1351-pfs-client-delete' into q/8.1 2018-12-11 01:44:35 +00:00
Giacomo Guiulfo 8606524b0e feat(PfsClient): add delete functionality 2018-12-11 00:28:37 +00:00
Rahul Padigela b5e17918b2 feature: ZENKO-1359 add ingestion capability
This flag allows Orbit to identify Zenko instances which have Ingestion
feature capabilities.
2018-12-10 15:12:02 -08:00
Taylor McKinnon c6c08508c6 Make CI env var force path style buckets 2018-12-03 13:21:13 -08:00
bert-e 476acd822e Merge branch 'w/8.0/bugfix/S3C-1843-fix-npm-dependency-vuln' into tmp/octopus/w/8.1/bugfix/S3C-1843-fix-npm-dependency-vuln 2018-11-28 01:46:33 +00:00
JianqinWang 5839daf56c Merge remote-tracking branch 'origin/bugfix/S3C-1843-fix-npm-dependency-vuln' into w/8.0/bugfix/S3C-1843-fix-npm-dependency-vuln 2018-11-27 17:25:22 -08:00
JianqinWang 1574715f42 bugfix: S3C-1843 update ver. for npm-run-all 2018-11-27 16:50:41 -08:00
bert-e 8fa9ef3f19 Merge branch 'feature/ZENKO-717' into q/8.1 2018-11-16 00:23:25 +00:00
jeremyds f2e28db1e4 ft: ZENKO-717: replicationBackends constant moved to Arsenal
With the management code moved from cloud server to its own repository,
this constant should be shared in Arsenal constants.
2018-11-15 15:51:06 -08:00
bert-e 324a0013d1 Merge branch 'improvement/update-python-dependencies' into tmp/octopus/w/8.1/improvement/update-python-dependencies 2018-11-15 23:00:32 +00:00
Rahul Padigela 3a1b34a7b0 improvement: ignore .tox dir 2018-11-15 14:57:46 -08:00
Rahul Padigela d100db1950 improvement: update python dependencies for docs
fixes CVE-2018-18074 alert
2018-11-15 14:55:59 -08:00
JianqinWang f2d59bc4f9 ZENKO-833: include ingestion option for bucket creation 2018-11-15 12:11:15 -08:00
Rahul Padigela 360928b60b Merge remote-tracking branch 'origin/w/8.0/feature/S3C-1807-MD-admin-routes' into w/8.1/feature/S3C-1807-MD-admin-routes 2018-11-15 11:42:51 -08:00
Rahul Padigela 8ffb5d2e71 Merge remote-tracking branch 'origin/feature/S3C-1807-MD-admin-routes' into w/8.0/feature/S3C-1807-MD-admin-routes 2018-11-15 11:11:17 -08:00
Rahul Padigela d09b6da7be feature: S3C-1807 proxy route for metadata
This allows routing of requests to metadata transparently with
Cloudserver acting as a proxy enabling HTTPS, authenticated routes
for bucketd.
2018-11-15 11:00:04 -08:00
Giacomo Guiulfo d587e3d63a feat(ZENKO-1270): pfs external backend 2018-11-14 16:57:51 -08:00
Giacomo Guiulfo 144ade3d25 feat(ZENKO-1270): pfs daemon 2018-11-14 16:48:49 -08:00
bert-e c1d72f47ab Merge branch 'w/8.1/bugfix/ZENKO-1311-ci-fixes' into tmp/octopus/q/8.1 2018-11-14 22:03:28 +00:00
Salim 75510a2512 improvement: push to latest tag 2018-11-14 13:38:55 -08:00
Salim c2d44466b7 Merge remote-tracking branch 'origin/w/8.0/bugfix/ZENKO-1311-ci-fixes' into w/8.1/bugfix/ZENKO-1311-ci-fixes 2018-11-14 13:38:32 -08:00
Salim fca1bee0bd Merge remote-tracking branch 'origin/bugfix/ZENKO-1311-ci-fixes' into w/8.0/bugfix/ZENKO-1311-ci-fixes 2018-11-14 13:36:32 -08:00
bert-e de3bd1f4d5 Merge branch 'feature/ZENKO-1044-Ceph-backend-tests' into q/8.1 2018-11-14 21:35:44 +00:00
Salim 2802f7836f improvement: push to docker hub 2018-11-14 13:33:29 -08:00
Salim 36e4eac37e bf: unit coverage 2018-11-14 13:33:11 -08:00
Taylor McKinnon 7ccfc141bf feat(ZENKO-1044): Add ceph backend tests 2018-11-14 13:14:29 -08:00
Salim a9c9367564 bf: trailing spaces 2018-11-14 10:41:10 -08:00
Salim 82993b241c ci: exit correctly on failure 2018-11-14 10:41:10 -08:00
vrancurel f65a384ced improvement: use Arsenal BucketClientInterface 2018-11-13 12:09:25 -08:00
vrancurel d810b27eb4 cleanup: Remove duplicated code
Bucketclient and bucketfile seem to be identical to Arsenal.
  We need to keep a dependency on bucket client lib for avoiding
  circular dependency.
2018-11-13 11:35:51 -08:00
bert-e 2512b29ff8 Merge branches 'w/8.1/bugfix/S3C-1752-legacy-mpu' and 'q/1618/8.0/bugfix/S3C-1752-legacy-mpu' into tmp/octopus/q/8.1 2018-11-10 01:28:23 +00:00
bert-e 8497eac258 Merge branches 'w/8.0/bugfix/S3C-1752-legacy-mpu' and 'q/1618/7.4/bugfix/S3C-1752-legacy-mpu' into tmp/octopus/q/8.0 2018-11-10 01:28:22 +00:00
bert-e a5395b6125 Merge branch 'bugfix/S3C-1752-legacy-mpu' into q/7.4 2018-11-10 01:28:22 +00:00
bert-e 6d3abd1b06 Merge branch 'w/8.0/bugfix/S3C-1752-legacy-mpu' into tmp/octopus/w/8.1/bugfix/S3C-1752-legacy-mpu 2018-11-10 01:05:04 +00:00
Dora Korpar 697d3e0ab3 Merge remote-tracking branch 'origin/bugfix/S3C-1752-legacy-mpu' into w/8.0/bugfix/S3C-1752-legacy-mpu 2018-11-09 16:55:22 -08:00
Dora Korpar a1bde14b45 bf: S3C-1752 add legacy support to mpu apis 2018-11-09 16:46:18 -08:00
bert-e 129239e095 Merge branch 'w/8.0/improvement/simplify-ci-unit-test' into tmp/octopus/w/8.1/improvement/simplify-ci-unit-test 2018-11-09 18:19:41 +00:00
bert-e 1618a840a4 Merge branch 'improvement/simplify-ci-unit-test' into tmp/octopus/w/8.0/improvement/simplify-ci-unit-test 2018-11-09 18:19:40 +00:00
JianqinWang 8526114120 improvement: use simpler unit test script 2018-11-09 10:10:31 -08:00
Rahul Padigela 5b39afff13 Merge remote-tracking branch 'origin/improvement/ZENKO-1291-extBackends-keepAlive' into w/8.1/improvement/ZENKO-1291-extBackends-keepAlive 2018-11-02 14:22:35 -07:00
Rahul Padigela 7a32b82b2d improvement: ZENKO-1291 disable keepAlive for AWS
This commit disables keep-alive on connections to AWS as connection
reuse showed intermittent socket hang ups.
2018-11-02 14:16:06 -07:00
Rahul Padigela 9db17e5578 improvement: ZENKO-1291 configurable httpAgent options
This commit adds the ability to configure httpAgent options
for external backends. Currently only AWS and GCP are
suppored. Azure is not supported as there is no straight
forward way to set a custom httpAgent in the Azure SDK.
The defaults are expressed to be sensible and explicit.
The default maxSockets is inifinity which is expressed as
`null` and other values are inspired by node.js defaults.
This configuration is applied globally for all locations of
the same type of external backend.
2018-11-02 14:16:01 -07:00
bert-e 04c987d097 Merge branch 'improvement/ZENKO-1281-logGCPRequestIds' into tmp/octopus/w/8.1/improvement/ZENKO-1281-logGCPRequestIds 2018-10-30 18:40:08 +00:00
Rahul Padigela 654d6105aa improvement: parse and include gcp request id on errors 2018-10-30 11:39:27 -07:00
bert-e bb0fbcd0cb Merge branch 'feature/ua-tagging' into tmp/octopus/w/8.1/feature/ua-tagging 2018-10-30 18:00:45 +00:00
Dora Korpar 75646b55ed ft: ZENKO 1042 azure user-agent tagging" 2018-10-30 10:58:52 -07:00
Dora Korpar 73aa224e30 ft: ZENKO 390 aws user-agent tagging 2018-10-30 10:58:04 -07:00
bert-e a994d54e6e Merge branches 'w/8.1/improvement/ZENKO-1281-logRequestIds' and 'q/1599/8.0/improvement/ZENKO-1281-logRequestIds' into tmp/octopus/q/8.1 2018-10-29 20:18:29 +00:00
bert-e e370bd6aa4 Merge branch 'improvement/ZENKO-1281-logRequestIds' into q/8.0 2018-10-29 20:18:29 +00:00
bert-e aabc2e0ba2 Merge branch 'improvement/ZENKO-1281-logRequestIds' into tmp/octopus/w/8.1/improvement/ZENKO-1281-logRequestIds 2018-10-29 04:35:18 +00:00
Rahul Padigela 763c58f36c improvement: log external request id 2018-10-28 21:32:44 -07:00
bert-e 7974726ebf Merge branch 'bugfix/ZENKO-1274-fixLogProperty' into q/8.1 2018-10-26 17:25:08 +00:00
bert-e 82f1d088f7 Merge branch 'w/8.0/improvement/S3C-1745-bucket-deletion' into tmp/octopus/w/8.1/improvement/S3C-1745-bucket-deletion 2018-10-25 01:56:50 +00:00
Rahul Padigela 1b8c4cc15f Merge remote-tracking branch 'origin/improvement/S3C-1745-bucket-deletion' into w/8.0/improvement/S3C-1745-bucket-deletion 2018-10-24 18:54:25 -07:00
Rahul Padigela 6e9eb606e9 refactor: remove circular dependency
This introduces invisiblyDelete and deleteUserBucketEntry as modules
that can by used without having to require bucketDeletion as a whole.
This is required to avoid circular dependency when abortMultipartUpload
module is required within bucketDeletion.js
2018-10-24 17:11:19 -07:00
Rahul Padigela fc0707954b improvement: abort mpus on bucket deletion
This aborts any ongoing mpus in a bucket including deleting all parts before
deleting the bucket. It enables Cloudserver to be inline with AWS S3's
behavior of deletion of buckets.
2018-10-24 17:11:19 -07:00
Rahul Padigela b967f9e96d refactor: obfuscate abortMPU logic into a utility
This enables code re-use for aborting mpus when deleting a bucket
2018-10-24 17:11:15 -07:00
bert-e 2a5fc70daa Merge branch 'bugfix/wait-for-local-port-fix' into tmp/octopus/w/8.1/bugfix/wait-for-local-port-fix 2018-10-24 22:22:49 +00:00
jeremyds dd500ed601 Revert "bugfix: reduce flakiness of server startup wait"
This reverts commit a65eb80873.
2018-10-24 11:07:38 -07:00
Dora Korpar 9f2580a397 ft: ZENKO 390 aws user-agent tagging 2018-10-24 11:01:15 -07:00
Rahul Padigela 16cb2934ec improvement: add aws request ids to logs 2018-10-24 10:45:43 -07:00
Dora Korpar b40706bd18 ft: ZENKO 1042 azure user-agent tagging" 2018-10-23 14:22:48 -07:00
philipyoo 01c2ef617e bf: ZENKO-1274 fix log property name 2018-10-19 16:49:08 -07:00
Rahul Padigela 417d16988f bugfix: ZENKO-1269 fix undefined callback 2018-10-18 16:09:41 -07:00
bert-e 3b04f89e3e Merge branch 'w/8.0/feature/S3C-1499-apiv2' into tmp/octopus/w/8.1/feature/S3C-1499-apiv2 2018-10-17 19:11:16 +00:00
Dora Korpar 2404d52a78 Merge remote-tracking branch 'origin/feature/S3C-1499-apiv2' into w/8.0/feature/S3C-1499-apiv2 2018-10-17 11:50:14 -07:00
Dora Korpar b8d78e6ed5 [chore] update arsenal 2018-10-15 16:35:21 -07:00
Dora Korpar 56c635c731 doc: design doc 2018-10-08 15:42:13 -07:00
Dora Korpar e629dad579 ft: S3C-1171 list objects v2 2018-10-08 15:42:13 -07:00
Dora Korpar b60c592aeb [bf] cherry pick #337b049 and #67eb20c for maxkeys=0 2018-09-25 15:15:55 -07:00
605 changed files with 59504 additions and 23148 deletions

View File

@ -1,3 +1,9 @@
node_modules
localData/*
localMetadata/*
# Keep the .git/HEAD file in order to properly report version
.git/objects
.github
.tox
coverage
.DS_Store

View File

@ -1 +1,54 @@
{ "extends": "scality" }
{
"extends": "scality",
"plugins": [
"mocha"
],
"rules": {
"import/extensions": "off",
"lines-around-directive": "off",
"no-underscore-dangle": "off",
"indent": "off",
"object-curly-newline": "off",
"operator-linebreak": "off",
"function-paren-newline": "off",
"import/newline-after-import": "off",
"prefer-destructuring": "off",
"implicit-arrow-linebreak": "off",
"no-bitwise": "off",
"dot-location": "off",
"comma-dangle": "off",
"no-undef-init": "off",
"global-require": "off",
"import/no-dynamic-require": "off",
"class-methods-use-this": "off",
"no-plusplus": "off",
"no-else-return": "off",
"object-property-newline": "off",
"import/order": "off",
"no-continue": "off",
"no-tabs": "off",
"lines-between-class-members": "off",
"prefer-spread": "off",
"no-lonely-if": "off",
"no-useless-escape": "off",
"no-restricted-globals": "off",
"no-buffer-constructor": "off",
"import/no-extraneous-dependencies": "off",
"space-unary-ops": "off",
"no-useless-return": "off",
"no-unexpected-multiline": "off",
"no-mixed-operators": "off",
"newline-per-chained-call": "off",
"operator-assignment": "off",
"spaced-comment": "off",
"comma-style": "off",
"no-restricted-properties": "off",
"new-parens": "off",
"no-multi-spaces": "off",
"quote-props": "off",
"mocha/no-exclusive-tests": "error",
},
"parserOptions": {
"ecmaVersion": 2020
}
}

View File

@ -48,7 +48,7 @@ Describe the results you expected
- Node.js version,
- Docker version,
- npm version,
- yarn version,
- distribution/OS,
- optional: anything else you deem helpful to us.

43
.github/actions/setup-ci/action.yaml vendored Normal file
View File

@ -0,0 +1,43 @@
---
name: "Setup CI environment"
description: "Setup Cloudserver CI environment"
runs:
using: composite
steps:
- name: Setup etc/hosts
shell: bash
run: sudo echo "127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" | sudo tee -a /etc/hosts
- name: Setup Credentials
shell: bash
run: bash .github/scripts/credentials.bash
- name: Setup job artifacts directory
shell: bash
run: |-
set -exu;
mkdir -p /tmp/artifacts/${JOB_NAME}/;
- uses: actions/setup-node@v4
with:
node-version: '16'
cache: 'yarn'
- name: install dependencies
shell: bash
run: yarn install --ignore-engines --frozen-lockfile --network-concurrency 1
- uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip
- uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Setup python2 test environment
shell: bash
run: |
sudo apt-get install -y libdigest-hmac-perl
pip install 's3cmd==2.3.0'
- name: fix sproxyd.conf permissions
shell: bash
run: sudo chown root:root .github/docker/sproxyd/conf/sproxyd0.conf
- name: ensure fuse kernel module is loaded (for sproxyd)
shell: bash
run: sudo modprobe fuse

25
.github/ceph/Dockerfile vendored Normal file
View File

@ -0,0 +1,25 @@
FROM ceph/daemon:v3.2.1-stable-3.2-mimic-centos-7
ENV CEPH_DAEMON demo
ENV CEPH_DEMO_DAEMONS mon,mgr,osd,rgw
ENV CEPH_DEMO_UID zenko
ENV CEPH_DEMO_ACCESS_KEY accessKey1
ENV CEPH_DEMO_SECRET_KEY verySecretKey1
ENV CEPH_DEMO_BUCKET zenkobucket
ENV CEPH_PUBLIC_NETWORK 0.0.0.0/0
ENV MON_IP 0.0.0.0
ENV NETWORK_AUTO_DETECT 4
ENV RGW_CIVETWEB_PORT 8001
RUN rm /etc/yum.repos.d/tcmu-runner.repo
ADD ./entrypoint-wrapper.sh /
RUN chmod +x /entrypoint-wrapper.sh && \
yum install -y python-pip && \
yum clean all && \
pip install awscli && \
rm -rf /root/.cache/pip
ENTRYPOINT [ "/entrypoint-wrapper.sh" ]

37
.github/ceph/entrypoint-wrapper.sh vendored Normal file
View File

@ -0,0 +1,37 @@
#!/bin/sh
touch /artifacts/ceph.log
mkfifo /tmp/entrypoint_output
# We run this in the background so that we can tail the RGW log after init,
# because entrypoint.sh never returns
# The next line will be needed when ceph builds 3.2.2 so I'll leave it here
# bash /opt/ceph-container/bin/entrypoint.sh > /tmp/entrypoint_output &
bash /entrypoint.sh > /tmp/entrypoint_output &
entrypoint_pid="$!"
while read -r line; do
echo $line
# When we find this line server has started
if [ -n "$(echo $line | grep 'Creating bucket')" ]; then
break
fi
done < /tmp/entrypoint_output
# Make our buckets - CEPH_DEMO_BUCKET is set to force the "Creating bucket" message, but unused
s3cmd mb s3://cephbucket s3://cephbucket2
mkdir /root/.aws
cat > /root/.aws/credentials <<EOF
[default]
aws_access_key_id = accessKey1
aws_secret_access_key = verySecretKey1
EOF
# Enable versioning on them
for bucket in cephbucket cephbucket2; do
echo "Enabling versiong for $bucket"
aws --endpoint http://127.0.0.1:8001 s3api put-bucket-versioning --bucket $bucket --versioning Status=Enabled
done
tail -f /var/log/ceph/client.rgw.*.log | tee -a /artifacts/ceph.log
wait $entrypoint_pid

11
.github/ceph/wait_for_ceph.sh vendored Normal file
View File

@ -0,0 +1,11 @@
#!/bin/sh
# This script is needed because RADOS Gateway
# will open the port before beginning to serve traffic
# causing wait_for_local_port.bash to exit immediately
echo 'Waiting for ceph'
while [ -z "$(curl 127.0.0.1:8001 2>/dev/null)" ]; do
sleep 1
echo -n "."
done

10
.github/dependabot.yml vendored Normal file
View File

@ -0,0 +1,10 @@
---
version: 2
updates:
- package-ecosystem: npm
directory: "/"
schedule:
interval: daily
time: "13:00"
open-pull-requests-limit: 10
target-branch: "development/7.4"

36
.github/docker/creds.env vendored Normal file
View File

@ -0,0 +1,36 @@
azurebackend_AZURE_STORAGE_ACCESS_KEY
azurebackend_AZURE_STORAGE_ACCOUNT_NAME
azurebackend_AZURE_STORAGE_ENDPOINT
azurebackend2_AZURE_STORAGE_ACCESS_KEY
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME
azurebackend2_AZURE_STORAGE_ENDPOINT
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME
azurebackendmismatch_AZURE_STORAGE_ENDPOINT
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT
azuretest_AZURE_BLOB_ENDPOINT
b2backend_B2_ACCOUNT_ID
b2backend_B2_STORAGE_ACCESS_KEY
GOOGLE_SERVICE_EMAIL
GOOGLE_SERVICE_KEY
AWS_S3_BACKEND_ACCESS_KEY
AWS_S3_BACKEND_SECRET_KEY
AWS_S3_BACKEND_ACCESS_KEY_2
AWS_S3_BACKEND_SECRET_KEY_2
AWS_GCP_BACKEND_ACCESS_KEY
AWS_GCP_BACKEND_SECRET_KEY
AWS_GCP_BACKEND_ACCESS_KEY_2
AWS_GCP_BACKEND_SECRET_KEY_2
b2backend_B2_STORAGE_ENDPOINT
gcpbackend2_GCP_SERVICE_EMAIL
gcpbackend2_GCP_SERVICE_KEY
gcpbackend2_GCP_SERVICE_KEYFILE
gcpbackend_GCP_SERVICE_EMAIL
gcpbackend_GCP_SERVICE_KEY
gcpbackendmismatch_GCP_SERVICE_EMAIL
gcpbackendmismatch_GCP_SERVICE_KEY
gcpbackend_GCP_SERVICE_KEYFILE
gcpbackendmismatch_GCP_SERVICE_KEYFILE
gcpbackendnoproxy_GCP_SERVICE_KEYFILE

92
.github/docker/docker-compose.yaml vendored Normal file
View File

@ -0,0 +1,92 @@
services:
cloudserver:
image: ${CLOUDSERVER_IMAGE}
command: sh -c "yarn start > /artifacts/s3.log"
network_mode: "host"
volumes:
- /tmp/ssl:/ssl
- /tmp/ssl-kmip:/ssl-kmip
- ${HOME}/.aws/credentials:/root/.aws/credentials
- /tmp/artifacts/${JOB_NAME}:/artifacts
environment:
- CI=true
- ENABLE_LOCAL_CACHE=true
- REDIS_HOST=0.0.0.0
- REDIS_PORT=6379
- REPORT_TOKEN=report-token-1
- REMOTE_MANAGEMENT_DISABLE=1
- HEALTHCHECKS_ALLOWFROM=0.0.0.0/0
- DATA_HOST=0.0.0.0
- METADATA_HOST=0.0.0.0
- S3BACKEND
- S3DATA
- S3METADATA
- MPU_TESTING
- S3VAULT
- S3_LOCATION_FILE
- ENABLE_UTAPI_V2
- BUCKET_DENY_FILTER
- S3KMS
- S3KMIP_PORT
- S3KMIP_HOSTS
- S3KMIP-COMPOUND_CREATE
- S3KMIP_BUCKET_ATTRIBUTE_NAME
- S3KMIP_PIPELINE_DEPTH
- S3KMIP_KEY
- S3KMIP_CERT
- S3KMIP_CA
- MONGODB_HOSTS=0.0.0.0:27018
- MONGODB_RS=rs0
- DEFAULT_BUCKET_KEY_FORMAT
- METADATA_MAX_CACHED_BUCKETS
- ENABLE_NULL_VERSION_COMPAT_MODE
- SCUBA_HOST
- SCUBA_PORT
- SCUBA_HEALTHCHECK_FREQUENCY
- S3QUOTA
- QUOTA_ENABLE_INFLIGHTS
env_file:
- creds.env
depends_on:
- redis
extra_hosts:
- "bucketwebsitetester.s3-website-us-east-1.amazonaws.com:127.0.0.1"
- "pykmip.local:127.0.0.1"
redis:
image: redis:alpine
network_mode: "host"
squid:
network_mode: "host"
profiles: ['ci-proxy']
image: scality/ci-squid
command: >-
sh -c 'mkdir -p /ssl &&
openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
-subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
-keyout /ssl/myca.pem -out /ssl/myca.pem &&
cp /ssl/myca.pem /ssl/CA.pem &&
squid -f /etc/squid/squid.conf -N -z &&
squid -f /etc/squid/squid.conf -NYCd 1'
volumes:
- /tmp/ssl:/ssl
pykmip:
network_mode: "host"
profiles: ['pykmip']
image: ${PYKMIP_IMAGE:-ghcr.io/scality/cloudserver/pykmip}
volumes:
- /tmp/artifacts/${JOB_NAME}:/artifacts
mongo:
network_mode: "host"
profiles: ['mongo', 'ceph']
image: ${MONGODB_IMAGE}
ceph:
network_mode: "host"
profiles: ['ceph']
image: ghcr.io/scality/cloudserver/ci-ceph
sproxyd:
network_mode: "host"
profiles: ['sproxyd']
image: sproxyd-standalone
build: ./sproxyd
user: 0:0
privileged: yes

28
.github/docker/mongodb/Dockerfile vendored Normal file
View File

@ -0,0 +1,28 @@
FROM mongo:5.0.21
ENV USER=scality \
HOME_DIR=/home/scality \
CONF_DIR=/conf \
DATA_DIR=/data
# Set up directories and permissions
RUN mkdir -p /data/db /data/configdb && chown -R mongodb:mongodb /data/db /data/configdb; \
mkdir /logs; \
adduser --uid 1000 --disabled-password --gecos --quiet --shell /bin/bash scality
# Set up environment variables and directories for scality user
RUN mkdir ${CONF_DIR} && \
chown -R ${USER} ${CONF_DIR} && \
chown -R ${USER} ${DATA_DIR}
# copy the mongo config file
COPY /conf/mongod.conf /conf/mongod.conf
COPY /conf/mongo-run.sh /conf/mongo-run.sh
COPY /conf/initReplicaSet /conf/initReplicaSet.js
EXPOSE 27017/tcp
EXPOSE 27018
# Set up CMD
ENTRYPOINT ["bash", "/conf/mongo-run.sh"]
CMD ["bash", "/conf/mongo-run.sh"]

View File

@ -0,0 +1,4 @@
rs.initiate({
_id: "rs0",
members: [{ _id: 0, host: "127.0.0.1:27018" }]
});

View File

@ -0,0 +1,10 @@
#!/bin/bash
set -exo pipefail
init_RS() {
sleep 5
mongo --port 27018 /conf/initReplicaSet.js
}
init_RS &
mongod --bind_ip_all --config=/conf/mongod.conf

15
.github/docker/mongodb/conf/mongod.conf vendored Normal file
View File

@ -0,0 +1,15 @@
storage:
journal:
enabled: true
engine: wiredTiger
dbPath: "/data/db"
processManagement:
fork: false
net:
port: 27018
bindIp: 0.0.0.0
replication:
replSetName: "rs0"
enableMajorityReadConcern: true
security:
authorization: disabled

3
.github/docker/sproxyd/Dockerfile vendored Normal file
View File

@ -0,0 +1,3 @@
FROM ghcr.io/scality/federation/sproxyd:7.10.6.8
ADD ./conf/supervisord.conf ./conf/nginx.conf ./conf/fastcgi_params ./conf/sproxyd0.conf /conf/
RUN chown root:root /conf/sproxyd0.conf

View File

@ -0,0 +1,26 @@
fastcgi_param QUERY_STRING $query_string;
fastcgi_param REQUEST_METHOD $request_method;
fastcgi_param CONTENT_TYPE $content_type;
fastcgi_param CONTENT_LENGTH $content_length;
#fastcgi_param SCRIPT_NAME $fastcgi_script_name;
fastcgi_param SCRIPT_NAME /var/www;
fastcgi_param PATH_INFO $document_uri;
fastcgi_param REQUEST_URI $request_uri;
fastcgi_param DOCUMENT_URI $document_uri;
fastcgi_param DOCUMENT_ROOT $document_root;
fastcgi_param SERVER_PROTOCOL $server_protocol;
fastcgi_param HTTPS $https if_not_empty;
fastcgi_param GATEWAY_INTERFACE CGI/1.1;
fastcgi_param SERVER_SOFTWARE nginx/$nginx_version;
fastcgi_param REMOTE_ADDR $remote_addr;
fastcgi_param REMOTE_PORT $remote_port;
fastcgi_param SERVER_ADDR $server_addr;
fastcgi_param SERVER_PORT $server_port;
fastcgi_param SERVER_NAME $server_name;
# PHP only, required if PHP was built with --enable-force-cgi-redirect
fastcgi_param REDIRECT_STATUS 200;

88
.github/docker/sproxyd/conf/nginx.conf vendored Normal file
View File

@ -0,0 +1,88 @@
worker_processes 1;
error_log /logs/error.log;
user root root;
events {
worker_connections 1000;
reuse_port on;
multi_accept on;
}
worker_rlimit_nofile 20000;
http {
root /var/www/;
upstream sproxyds {
least_conn;
keepalive 40;
server 127.0.0.1:20000;
}
server {
client_max_body_size 0;
client_body_timeout 150;
client_header_timeout 150;
postpone_output 0;
client_body_postpone_size 0;
keepalive_requests 1100;
keepalive_timeout 300s;
server_tokens off;
default_type application/octet-stream;
gzip off;
tcp_nodelay on;
tcp_nopush on;
sendfile on;
listen 81;
server_name localhost;
rewrite ^/arc/(.*)$ /dc1/$1 permanent;
location ~* ^/proxy/(.*)$ {
rewrite ^/proxy/(.*)$ /$1 last;
}
allow 127.0.0.1;
deny all;
set $usermd '-';
set $sentusermd '-';
set $elapsed_ms '-';
set $now '-';
log_by_lua '
if not(ngx.var.http_x_scal_usermd == nil) and string.len(ngx.var.http_x_scal_usermd) > 2 then
ngx.var.usermd = string.sub(ngx.decode_base64(ngx.var.http_x_scal_usermd),1,-3)
end
if not(ngx.var.sent_http_x_scal_usermd == nil) and string.len(ngx.var.sent_http_x_scal_usermd) > 2 then
ngx.var.sentusermd = string.sub(ngx.decode_base64(ngx.var.sent_http_x_scal_usermd),1,-3)
end
local elapsed_ms = tonumber(ngx.var.request_time)
if not ( elapsed_ms == nil) then
elapsed_ms = elapsed_ms * 1000
ngx.var.elapsed_ms = tostring(elapsed_ms)
end
local time = tonumber(ngx.var.msec) * 1000
ngx.var.now = time
';
log_format irm '{ "time":"$now","connection":"$connection","request":"$connection_requests","hrtime":"$msec",'
'"httpMethod":"$request_method","httpURL":"$uri","elapsed_ms":$elapsed_ms,'
'"httpCode":$status,"requestLength":$request_length,"bytesSent":$bytes_sent,'
'"contentLength":"$content_length","sentContentLength":"$sent_http_content_length",'
'"contentType":"$content_type","s3Address":"$remote_addr",'
'"requestUserMd":"$usermd","responseUserMd":"$sentusermd",'
'"ringKeyVersion":"$sent_http_x_scal_version","ringStatus":"$sent_http_x_scal_ring_status",'
'"s3Port":"$remote_port","sproxydStatus":"$upstream_status","req_id":"$http_x_scal_request_uids",'
'"ifMatch":"$http_if_match","ifNoneMatch":"$http_if_none_match",'
'"range":"$http_range","contentRange":"$sent_http_content_range","nginxPID":$PID,'
'"sproxydAddress":"$upstream_addr","sproxydResponseTime_s":"$upstream_response_time" }';
access_log /dev/stdout irm;
error_log /dev/stdout error;
location / {
proxy_request_buffering off;
fastcgi_request_buffering off;
fastcgi_no_cache 1;
fastcgi_cache_bypass 1;
fastcgi_buffering off;
fastcgi_ignore_client_abort on;
fastcgi_keep_conn on;
include fastcgi_params;
fastcgi_pass sproxyds;
fastcgi_next_upstream error timeout;
fastcgi_send_timeout 285s;
fastcgi_read_timeout 285s;
}
}
}

View File

@ -0,0 +1,12 @@
{
"general": {
"ring": "DATA",
"port": 20000,
"syslog_facility": "local0"
},
"ring_driver:0": {
"alias": "dc1",
"type": "local",
"queue_path": "/tmp/ring-objs"
},
}

View File

@ -0,0 +1,43 @@
[supervisord]
nodaemon = true
loglevel = info
logfile = %(ENV_LOG_DIR)s/supervisord.log
pidfile = %(ENV_SUP_RUN_DIR)s/supervisord.pid
logfile_maxbytes = 20MB
logfile_backups = 2
[unix_http_server]
file = %(ENV_SUP_RUN_DIR)s/supervisor.sock
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
[program:nginx]
directory=%(ENV_SUP_RUN_DIR)s
command=bash -c "/usr/sbin/nginx -c %(ENV_CONF_DIR)s/nginx.conf -g 'daemon off;'"
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
stdout_logfile_maxbytes=100MB
stdout_logfile_backups=7
stderr_logfile_maxbytes=100MB
stderr_logfile_backups=7
autorestart=true
autostart=true
user=root
[program:sproxyd]
directory=%(ENV_SUP_RUN_DIR)s
process_name=%(program_name)s-%(process_num)s
numprocs=1
numprocs_start=0
command=/usr/bin/sproxyd -dlw -V127 -c %(ENV_CONF_DIR)s/sproxyd%(process_num)s.conf -P /run%(process_num)s
stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
stdout_logfile_maxbytes=100MB
stdout_logfile_backups=7
redirect_stderr=true
autorestart=true
autostart=true
user=root

29
.github/pykmip/Dockerfile vendored Normal file
View File

@ -0,0 +1,29 @@
FROM python:3.10-alpine
RUN apk add --no-cache \
libressl && \
apk add --no-cache --virtual .build-deps \
python3-dev \
libffi-dev \
libressl-dev \
sqlite-dev \
build-base \
curl
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
RUN pip3 install -U pip && \
pip3 install pykmip requests && \
apk del .build-deps && \
mkdir /pykmip
ADD ./bin /usr/local/bin
ADD ./certs /ssl
ADD policy.json /etc/pykmip/policies/policy.json
ADD server.conf /etc/pykmip/server.conf
ADD docker-entrypoint.sh /
RUN chmod +x /docker-entrypoint.sh
ENTRYPOINT ["/docker-entrypoint.sh"]

View File

@ -0,0 +1,156 @@
#!/usr/bin/env python
from cryptography import x509
from cryptography.hazmat import backends
from cryptography.hazmat.primitives import hashes
from cryptography.hazmat.primitives import serialization
from cryptography.hazmat.primitives.asymmetric import rsa
import datetime
import argparse
import sys
def get_args():
parser = argparse.ArgumentParser(
prog=sys.argv[0],
description='Tool to generate a x509 CA root, server and client certs')
parser.add_argument('-c', '--common-name', action='store',
default='localhost',
help='Set the common name for the server-side cert')
return parser.parse_args()
def create_rsa_private_key(key_size=2048, public_exponent=65537):
private_key = rsa.generate_private_key(
public_exponent=public_exponent,
key_size=key_size,
backend=backends.default_backend()
)
return private_key
def create_self_signed_certificate(subject_name,
private_key,
days_valid=36500):
subject = x509.Name([
x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, u"Scality"),
x509.NameAttribute(x509.NameOID.COMMON_NAME, subject_name)
])
certificate = x509.CertificateBuilder().subject_name(
subject
).issuer_name(
subject
).public_key(
private_key.public_key()
).serial_number(
x509.random_serial_number()
).not_valid_before(
datetime.datetime.utcnow()
).not_valid_after(
datetime.datetime.utcnow() + datetime.timedelta(days=days_valid)
).add_extension(
x509.BasicConstraints(True, None),
critical=True
).sign(private_key, hashes.SHA256(), backends.default_backend())
return certificate
def create_certificate(subject_name,
private_key,
signing_certificate,
signing_key,
days_valid=36500,
client_auth=False):
subject = x509.Name([
x509.NameAttribute(x509.NameOID.ORGANIZATION_NAME, u"Scality"),
x509.NameAttribute(x509.NameOID.COMMON_NAME, subject_name)
])
builder = x509.CertificateBuilder().subject_name(
subject
).issuer_name(
signing_certificate.subject
).public_key(
private_key.public_key()
).serial_number(
x509.random_serial_number()
).not_valid_before(
datetime.datetime.utcnow()
).not_valid_after(
datetime.datetime.utcnow() + datetime.timedelta(days=days_valid)
)
if client_auth:
builder = builder.add_extension(
x509.ExtendedKeyUsage([x509.ExtendedKeyUsageOID.CLIENT_AUTH]),
critical=True
)
certificate = builder.sign(
signing_key,
hashes.SHA256(),
backends.default_backend()
)
return certificate
def main(common_name):
root_key = create_rsa_private_key()
root_certificate = create_self_signed_certificate(
u"Root CA",
root_key
)
server_key = create_rsa_private_key()
server_certificate = create_certificate(
common_name,
server_key,
root_certificate,
root_key
)
john_doe_client_key = create_rsa_private_key()
john_doe_client_certificate = create_certificate(
u"John Doe",
john_doe_client_key,
root_certificate,
root_key,
client_auth=True
)
with open("certs/kmip-ca.pem", "wb") as f:
f.write(
root_certificate.public_bytes(
serialization.Encoding.PEM
)
)
with open("certs/kmip-key.pem", "wb") as f:
f.write(server_key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption()
))
with open("certs/kmip-cert.pem", "wb") as f:
f.write(
server_certificate.public_bytes(
serialization.Encoding.PEM
)
)
with open("certs/kmip-client-key.pem", "wb") as f:
f.write(john_doe_client_key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.PKCS8,
encryption_algorithm=serialization.NoEncryption()
))
with open("certs/kmip-client-cert.pem", "wb") as f:
f.write(
john_doe_client_certificate.public_bytes(
serialization.Encoding.PEM
)
)
if __name__ == '__main__':
args = get_args()
main(args.common_name)

26
.github/pykmip/bin/run_server.py vendored Normal file
View File

@ -0,0 +1,26 @@
#!/usr/bin/env python
# Copyright (c) 2016 The Johns Hopkins University/Applied Physics Laboratory
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging # noqa: E402
logging.basicConfig(level=logging.DEBUG)
from kmip.services.server import server # noqa: E402
if __name__ == '__main__':
print('Starting PyKMIP server on 0.0.0.0:5696')
server.main()

18
.github/pykmip/certs/kmip-ca.pem vendored Normal file
View File

@ -0,0 +1,18 @@
-----BEGIN CERTIFICATE-----
MIIC6zCCAdOgAwIBAgIUPIpMY95b4HjKAk+FyydZApAEFskwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowJDEQMA4GA1UECgwHU2NhbGl0eTEQ
MA4GA1UEAwwHUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
AKqLFEsWtfRTxnoZrQe63tq+rQnVgninHMahRmXkzyjK/uNhoKnIh8bXdTC/eCZ6
FBROqBYNL0TJb0HDv1FzcZS1UCUldRqTlvr6wZb0pfrp40fvztsqQgAh1t/Blg5i
Zv5+ESSlNs5rWbFTxtq+FbMW/ERYTrVfnMkBiLg4Gq0HwID9a5jvJatzrrno2s1m
OfZCT3HaE3tMZ6vvYuoamvLNdvdH+9KeTmBCursfNejt0rSGjIqfi6DvFJSayydQ
is5DMSTbCLGdKQmA85VfEQmlQ8v0232WDSd6gVfp2tthDEDHnCbgWkEd1vsTyS85
ubdt5v4CWGOWV+mu3bf8xM0CAwEAAaMTMBEwDwYDVR0TAQH/BAUwAwEB/zANBgkq
hkiG9w0BAQsFAAOCAQEARTjc2zV/ol1/LsSzZy6l1R0uFBmR2KumH+Se1Yq2vKpY
Dv6xmrvmjOUr5RBO77nRhIgdcQA+LyAg8ii2Dfzc8r1RTD+j1bYOxESXctBOBcXM
Chy6FEBydR6m7S8qQyL+caJWO1WZWp2tapcm6sUG1oRVznWtK1/SHKIzOBwsmJ07
79KsCJ6wf9tzD05EDTI2QhAObE9/thy+zc8l8cmv9A6p3jKkx9rwXUttSUqTn0CW
w45bgKg6+DDcrhZ+MATbzuTfhuA4NFUTzK7KeX9sMuOV03Zs8SA3VhAOXmu063M3
0f9X7P/0RmGTTp7GGCqEINcZdbLh3k7CpFb2Ox998Q==
-----END CERTIFICATE-----

18
.github/pykmip/certs/kmip-cert.pem vendored Normal file
View File

@ -0,0 +1,18 @@
-----BEGIN CERTIFICATE-----
MIIC2zCCAcOgAwIBAgIUIlE8UAkqQ+6mbJDtrt9kkmi8aJYwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowKTEQMA4GA1UECgwHU2NhbGl0eTEV
MBMGA1UEAwwMcHlrbWlwLmxvY2FsMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
CgKCAQEAtxr7pq/lnzVeZz4z52Yc3DeaPqjNfRSyW5cPUlT7ABXFb7+tja7K2C7u
DYVK+Q+2yJCQwYJY47aKJB++ewam9t2V8Xy0Z8S+0I2ImCwuyeihaD/f6uJZRzms
ycdECH22BA6tCPlQLnlboRiZzI6rcIvXAbUMvLvFm3nyYIs9qidExRnfyMjISknM
V+83LT5QW4IcHgKYqzdz2ZmOnk+f4wmMmitcivTdIZCL8Z0cxr7BJlOh5JZ/V5uj
WUXeNa+ttW0RKKBlg9T+wj0JvwoJBPZTmsMAy3tI9tjLg3DwGYKsflbFeU2tebXI
gncGFZ/dFxj331GGtq3kz1PzAUYf2wIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQB1
8HgJ0fu6/pCrDxAm90eESFjmaTFyTN8q00zhq4Cb3zAT9KMWzAygkZ9n4ZFgELPo
7kBE2H6RcDdoBmjVYd8HnBloDdYzYbncKgt5YBvxRaMSF4/l65BM8wjatyXErqnH
QLLTRe5AuF0/F0KtPeDQ2JFVu8dZ35W3fyKGPRsEdVOSCTHROmqpGhZCpscyUP4W
Hb0dBTESQ9mQHw14OCaaahARd0X5WdcA/E+m0fpGqj1rQCXS+PrRcSLe1E1hqPlK
q/hXSXD5nybwipktELvJCbB7l4HmJr2pIpldeR5+ef68Cs8hqs6DRlsJX9sK2ng+
TFe5v6SCarqZ9kFvr6Yp
-----END CERTIFICATE-----

View File

@ -0,0 +1,18 @@
-----BEGIN CERTIFICATE-----
MIIC8zCCAdugAwIBAgIUBs6nVXQXhrFbClub3aSLg72/DiYwDQYJKoZIhvcNAQEL
BQAwJDEQMA4GA1UECgwHU2NhbGl0eTEQMA4GA1UEAwwHUm9vdCBDQTAgFw0yMTA0
MDkwMDI4MTFaGA8yMTIxMDMxNjAwMjgxMVowJTEQMA4GA1UECgwHU2NhbGl0eTER
MA8GA1UEAwwISm9obiBEb2UwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
AQC6neSYoBoWh/i2mBpduJnTlXacpJ0iQqLezvcGy8qR0s/48mtfV2IRGTNVsq4L
jLLRsPGt9KkJlUhHGWhG00cBGEsIiJiBUr+WrEsO04ME/Sk76kX8wk/t9Oljl7jt
UDnQUwshj+hRFe0iKAyE65JIutu5EiiNtOqMzbVgPNfNniAaGlrgwByJaS9arzsH
PVju9yZBYzYhwAMyYFcXUGrgvHRCHKmxBi4QmV7DX4TeN4l9TrCyEmqDev4PRFip
yR2Fh3WGSwWh45HgMT+Jp6Uv6yI4wMXWJAcNkHdx1OhjBoUQrkavvdeVEnCwjQ+p
SMLm0T4iNxedQWBtDM7ts4EjAgMBAAGjGjAYMBYGA1UdJQEB/wQMMAoGCCsGAQUF
BwMCMA0GCSqGSIb3DQEBCwUAA4IBAQCMi9HEhZc5jHJMj18Wq00fZy4O9XtjCe0J
nntW9tzi3rTQcQWKA7i9uVdDoCg+gMFVxWMvV7luFEUc/VYV1v8hFfbIFygzFsZY
xwv4GQaIwbsgzD+oziia53w0FSuNL0uE0MeKvrt3yzHxCxylHyl+TQd/UdAtAo+k
RL1sI0mBZx5qo6d1J7ZMCxzAGaT7KjnJvziFr/UbfSNnwDsxsUwGaI1ZeAxJN8DI
zTrg3f3lrrmHcauEgKnuQwIqaMZR6veG6RkjtcYSlJYID1irkE6njs7+wivOAkzt
fBt/0PD76FmAI0VArgU/zDB8dGyYzrq39W749LuEfm1TPmlnUtDr
-----END CERTIFICATE-----

View File

@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6neSYoBoWh/i2
mBpduJnTlXacpJ0iQqLezvcGy8qR0s/48mtfV2IRGTNVsq4LjLLRsPGt9KkJlUhH
GWhG00cBGEsIiJiBUr+WrEsO04ME/Sk76kX8wk/t9Oljl7jtUDnQUwshj+hRFe0i
KAyE65JIutu5EiiNtOqMzbVgPNfNniAaGlrgwByJaS9arzsHPVju9yZBYzYhwAMy
YFcXUGrgvHRCHKmxBi4QmV7DX4TeN4l9TrCyEmqDev4PRFipyR2Fh3WGSwWh45Hg
MT+Jp6Uv6yI4wMXWJAcNkHdx1OhjBoUQrkavvdeVEnCwjQ+pSMLm0T4iNxedQWBt
DM7ts4EjAgMBAAECggEANNXdUeUKXdSzcycPV/ea/c+0XFcy8e9B46lfQTpTqQOx
xD8GbWD1L/gdk6baJgT43+ukEWdSsJbmdtLXti29Ta8OF2VtIDhIbCVtvs3dq3zt
vrvugsiVDr8nkP306qOrKrNIVIFE+igmEmSaXsu/h/33ladxeeV9/s2DC7NOOjWN
Mu4KYr5BBbu3qAavdzbrcz7Sch+GzsYqK/pBounCTQu3o9E4TSUcmcsasWmtHN3u
e6G2UjObdzEW7J0wWvvtJ0wHQUVRueHfqwqKf0dymcZ3xOlx3ZPhKPz5n4F1UGUt
RQaNazqs5SzZpUgDuPw4k8h/aCHK21Yexw/l4+O9KQKBgQD1WZSRK54zFoExBQgt
OZSBNZW3Ibti5lSiF0M0g+66yNZSWfPuABEH0tu5CXopdPDXo4kW8NLGEqQStWTX
RGK0DE9buEL3eebOfjIdS2IZ3t3dX3lMypplVCj4HzAgITlweSH1LLTyAtaaOpwa
jksqfcn5Zw+XGkyc6GBBVaZetQKBgQDCt6Xf/g26+zjvHscjdzsfBhnYvTOrr6+F
xqFFxOEOocGr+mL7UTAs+a9m/6lOWhlagk+m+TIZNL8o3IN7KFTYxPYPxTiewgVE
rIm3JBmPxRiPn01P3HrtjaqfzsXF30j3ele7ix5OxieZq4vsW7ZXP3GZE34a08Ov
12sE1DlvdwKBgQDzpYQOLhyqazzcqzyVfMrnDYmiFVN7QXTmiudobWRUBUIhAcdl
oJdJB7K/rJOuO704x+RJ7dnCbZyWH6EGzZifaGIemXuXO21jvpqR0NyZCGOXhUp2
YfS1j8AntwEZxyS9du2sBjui4gKvomiHTquChOxgSmKHEcznPTTpbN8MyQKBgF5F
LVCZniolkLXsL7tS8VOez4qoZ0i6wP7CYLf3joJX+/z4N023S9yqcaorItvlMRsp
tciAIyoi6F2vDRTmPNXJ3dtav4PVKVnLMs1w89MwOCjoljSQ6Q7zpGTEZenbpWbz
W2BYBS9cLjXu4MpoyInLFINo9YeleLs8TvrCiKAXAoGBANsduqLnlUW/f5zDb5Fe
SB51+KhBjsVIeYmU+8xtur9Z7IxZXK28wpoEsm7LmX7Va5dERjI+tItBiJ5+Unu1
Xs2ljDg35ARKHs0dWBJGpbnZg4dbT6xpIL4YMPXm1Zu++PgRpxPIMn646xqd8GlH
bavm6Km/fXNG58xus+EeLpV5
-----END PRIVATE KEY-----

28
.github/pykmip/certs/kmip-key.pem vendored Normal file
View File

@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC3Gvumr+WfNV5n
PjPnZhzcN5o+qM19FLJblw9SVPsAFcVvv62NrsrYLu4NhUr5D7bIkJDBgljjtook
H757Bqb23ZXxfLRnxL7QjYiYLC7J6KFoP9/q4llHOazJx0QIfbYEDq0I+VAueVuh
GJnMjqtwi9cBtQy8u8WbefJgiz2qJ0TFGd/IyMhKScxX7zctPlBbghweApirN3PZ
mY6eT5/jCYyaK1yK9N0hkIvxnRzGvsEmU6Hkln9Xm6NZRd41r621bREooGWD1P7C
PQm/CgkE9lOawwDLe0j22MuDcPAZgqx+VsV5Ta15tciCdwYVn90XGPffUYa2reTP
U/MBRh/bAgMBAAECggEABCvcMcbuDztzBB0Zp5re63Fk1SqZS9Et4wJE+hYvhaf5
UHtoY8LoohYnnC0+MQBXpKgOdCoZBk8BRKNofnr/UL5pjQ/POFH2GuAujXDsO/NN
wgc6fapcaE/7DLm6ZgsfG2aOMJclaXmgScI6trtFUpIM+t/6A06vyMP1bpeddwPW
Fqu7NvpDiEcTRUGd+z1JooYgUhGgC7peYUx5+9zqFrwoDBKxnUOnz3BkDsXBy3qm
65Vu0BSjuJzf6vVMpNGUHY6JXjopVNWku+JAX0wD+iikOd5sziNVdIj1fnZ+IHIf
7G5h5owHpvSGzJFQ18/g5VHtJdCm+4WQSnbSJRsCAQKBgQDu4IH8yspyeH44fhoS
PAp/OtILqSP+Da0zAp2LbhrOgyzyuSTdEAYyptqjqHS6QkB1Bu1H44FS0BYUxRXc
iu2e9AndiLVCGngsE7TpA/ZVLN1B0LEZEHjM6p4d6zZM6iveKVnPAOkTWTBAgzCt
b31nj4jL8PdlPKQil1AMrOlRAQKBgQDEOwshzIdr2Iy6B/n4CuBViEtwnbAd5f/c
atA9bcfF8kCahokJsI4eCCLgBwDZpYKD+v0AwOBlacF6t6TX+vdlJsi5EP7uxZ22
ILsuWqVm/0H77PACuckc5/qLZoGGC81l0DhnpoeMEb6r/TKOo5xAK1gxdlwNNrq+
nP1zdZnU2wKBgBAS92xFUR4m0YeHpMV5WNN658t1FEDyNqdqE6PgQtmGpi2nG73s
aB5cb/X3TfOCpce6MZlWy8sAyZuYL4Jprte1YDySCHBsS43bvZ64b4kHvdPB8UjY
fOh9GSq2Oy8tysnmSm7NhuGQbNjKeyoQiIXBeNkQW/VqATl6qR5RPFoBAoGACNqV
JQBCd/Y8W0Ry3eM3vgQ5SyqCQMcY5UwYez0Rz3efvJknY72InAhH8o2+VxOlsOjJ
M5iAR3MfHLdeg7Q6J2E5m0gOCJ34ALi3WV8TqXMI+iH1rlnNnjVFU7bbTz4HFXnw
oZSc9w/x53a0KkVtjmOmRg0OGDaI9ILG2MfMmhMCgYB8ZqJtX8qZ2TqKU3XdLZ4z
T2N7xMFuKohWP420r5jKm3Xw85IC+y1SUTB9XGcL79r2eJzmzmdKQ3A3sf3oyUH3
RdYWxtKcZ5PAE8hVRtn1ETZqUgxASGOUn/6w0npkYSOXPU5bc0W6RSLkjES0i+c3
fv3OMNI8qpmQhEjpHHQS1g==
-----END PRIVATE KEY-----

3
.github/pykmip/docker-entrypoint.sh vendored Executable file
View File

@ -0,0 +1,3 @@
#!/bin/sh
python3 /usr/local/bin/run_server.py 2>&1 | tee -a /artifacts/pykmip.log

168
.github/pykmip/policy.json vendored Normal file
View File

@ -0,0 +1,168 @@
{
"example": {
"preset": {
"CERTIFICATE": {
"LOCATE": "ALLOW_ALL",
"CHECK": "ALLOW_ALL",
"GET": "ALLOW_ALL",
"GET_ATTRIBUTES": "ALLOW_ALL",
"GET_ATTRIBUTE_LIST": "ALLOW_ALL",
"ADD_ATTRIBUTE": "ALLOW_OWNER",
"MODIFY_ATTRIBUTE": "ALLOW_OWNER",
"DELETE_ATTRIBUTE": "ALLOW_OWNER",
"OBTAIN_LEASE": "ALLOW_ALL",
"ACTIVATE": "ALLOW_OWNER",
"REVOKE": "ALLOW_OWNER",
"DESTROY": "ALLOW_OWNER",
"ARCHIVE": "ALLOW_OWNER",
"RECOVER": "ALLOW_OWNER"
},
"SYMMETRIC_KEY": {
"REKEY": "ALLOW_OWNER",
"REKEY_KEY_PAIR": "ALLOW_OWNER",
"DERIVE_KEY": "ALLOW_OWNER",
"LOCATE": "ALLOW_OWNER",
"CHECK": "ALLOW_OWNER",
"GET": "ALLOW_OWNER",
"GET_ATTRIBUTES": "ALLOW_OWNER",
"GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
"ADD_ATTRIBUTE": "ALLOW_OWNER",
"MODIFY_ATTRIBUTE": "ALLOW_OWNER",
"DELETE_ATTRIBUTE": "ALLOW_OWNER",
"OBTAIN_LEASE": "ALLOW_OWNER",
"GET_USAGE_ALLOCATION": "ALLOW_OWNER",
"ACTIVATE": "ALLOW_OWNER",
"REVOKE": "ALLOW_OWNER",
"DESTROY": "ALLOW_OWNER",
"ARCHIVE": "ALLOW_OWNER",
"RECOVER": "ALLOW_OWNER"
},
"PUBLIC_KEY": {
"LOCATE": "ALLOW_ALL",
"CHECK": "ALLOW_ALL",
"GET": "ALLOW_ALL",
"GET_ATTRIBUTES": "ALLOW_ALL",
"GET_ATTRIBUTE_LIST": "ALLOW_ALL",
"ADD_ATTRIBUTE": "ALLOW_OWNER",
"MODIFY_ATTRIBUTE": "ALLOW_OWNER",
"DELETE_ATTRIBUTE": "ALLOW_OWNER",
"OBTAIN_LEASE": "ALLOW_ALL",
"ACTIVATE": "ALLOW_OWNER",
"REVOKE": "ALLOW_OWNER",
"DESTROY": "ALLOW_OWNER",
"ARCHIVE": "ALLOW_OWNER",
"RECOVER": "ALLOW_OWNER"
},
"PRIVATE_KEY": {
"REKEY": "ALLOW_OWNER",
"REKEY_KEY_PAIR": "ALLOW_OWNER",
"DERIVE_KEY": "ALLOW_OWNER",
"LOCATE": "ALLOW_OWNER",
"CHECK": "ALLOW_OWNER",
"GET": "ALLOW_OWNER",
"GET_ATTRIBUTES": "ALLOW_OWNER",
"GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
"ADD_ATTRIBUTE": "ALLOW_OWNER",
"MODIFY_ATTRIBUTE": "ALLOW_OWNER",
"DELETE_ATTRIBUTE": "ALLOW_OWNER",
"OBTAIN_LEASE": "ALLOW_OWNER",
"GET_USAGE_ALLOCATION": "ALLOW_OWNER",
"ACTIVATE": "ALLOW_OWNER",
"REVOKE": "ALLOW_OWNER",
"DESTROY": "ALLOW_OWNER",
"ARCHIVE": "ALLOW_OWNER",
"RECOVER": "ALLOW_OWNER"
},
"SPLIT_KEY": {
"REKEY": "ALLOW_OWNER",
"REKEY_KEY_PAIR": "ALLOW_OWNER",
"DERIVE_KEY": "ALLOW_OWNER",
"LOCATE": "ALLOW_OWNER",
"CHECK": "ALLOW_OWNER",
"GET": "ALLOW_OWNER",
"GET_ATTRIBUTES": "ALLOW_OWNER",
"GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
"ADD_ATTRIBUTE": "ALLOW_OWNER",
"MODIFY_ATTRIBUTE": "ALLOW_OWNER",
"DELETE_ATTRIBUTE": "ALLOW_OWNER",
"OBTAIN_LEASE": "ALLOW_OWNER",
"GET_USAGE_ALLOCATION": "ALLOW_OWNER",
"ACTIVATE": "ALLOW_OWNER",
"REVOKE": "ALLOW_OWNER",
"DESTROY": "ALLOW_OWNER",
"ARCHIVE": "ALLOW_OWNER",
"RECOVER": "ALLOW_OWNER"
},
"TEMPLATE": {
"LOCATE": "ALLOW_OWNER",
"GET": "ALLOW_OWNER",
"GET_ATTRIBUTES": "ALLOW_OWNER",
"GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
"ADD_ATTRIBUTE": "ALLOW_OWNER",
"MODIFY_ATTRIBUTE": "ALLOW_OWNER",
"DELETE_ATTRIBUTE": "ALLOW_OWNER",
"DESTROY": "ALLOW_OWNER"
},
"SECRET_DATA": {
"REKEY": "ALLOW_OWNER",
"REKEY_KEY_PAIR": "ALLOW_OWNER",
"DERIVE_KEY": "ALLOW_OWNER",
"LOCATE": "ALLOW_OWNER",
"CHECK": "ALLOW_OWNER",
"GET": "ALLOW_OWNER",
"GET_ATTRIBUTES": "ALLOW_OWNER",
"GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
"ADD_ATTRIBUTE": "ALLOW_OWNER",
"MODIFY_ATTRIBUTE": "ALLOW_OWNER",
"DELETE_ATTRIBUTE": "ALLOW_OWNER",
"OBTAIN_LEASE": "ALLOW_OWNER",
"GET_USAGE_ALLOCATION": "ALLOW_OWNER",
"ACTIVATE": "ALLOW_OWNER",
"REVOKE": "ALLOW_OWNER",
"DESTROY": "ALLOW_OWNER",
"ARCHIVE": "ALLOW_OWNER",
"RECOVER": "ALLOW_OWNER"
},
"OPAQUE_DATA": {
"REKEY": "ALLOW_OWNER",
"REKEY_KEY_PAIR": "ALLOW_OWNER",
"DERIVE_KEY": "ALLOW_OWNER",
"LOCATE": "ALLOW_OWNER",
"CHECK": "ALLOW_OWNER",
"GET": "ALLOW_OWNER",
"GET_ATTRIBUTES": "ALLOW_OWNER",
"GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
"ADD_ATTRIBUTE": "ALLOW_OWNER",
"MODIFY_ATTRIBUTE": "ALLOW_OWNER",
"DELETE_ATTRIBUTE": "ALLOW_OWNER",
"OBTAIN_LEASE": "ALLOW_OWNER",
"GET_USAGE_ALLOCATION": "ALLOW_OWNER",
"ACTIVATE": "ALLOW_OWNER",
"REVOKE": "ALLOW_OWNER",
"DESTROY": "ALLOW_OWNER",
"ARCHIVE": "ALLOW_OWNER",
"RECOVER": "ALLOW_OWNER"
},
"PGP_KEY": {
"REKEY": "ALLOW_OWNER",
"REKEY_KEY_PAIR": "ALLOW_OWNER",
"DERIVE_KEY": "ALLOW_OWNER",
"LOCATE": "ALLOW_OWNER",
"CHECK": "ALLOW_OWNER",
"GET": "ALLOW_OWNER",
"GET_ATTRIBUTES": "ALLOW_OWNER",
"GET_ATTRIBUTE_LIST": "ALLOW_OWNER",
"ADD_ATTRIBUTE": "ALLOW_OWNER",
"MODIFY_ATTRIBUTE": "ALLOW_OWNER",
"DELETE_ATTRIBUTE": "ALLOW_OWNER",
"OBTAIN_LEASE": "ALLOW_OWNER",
"GET_USAGE_ALLOCATION": "ALLOW_OWNER",
"ACTIVATE": "ALLOW_OWNER",
"REVOKE": "ALLOW_OWNER",
"DESTROY": "ALLOW_OWNER",
"ARCHIVE": "ALLOW_OWNER",
"RECOVER": "ALLOW_OWNER"
}
}
}
}

15
.github/pykmip/server.conf vendored Normal file
View File

@ -0,0 +1,15 @@
[server]
hostname=0.0.0.0
port=5696
certificate_path=/ssl/kmip-cert.pem
key_path=/ssl/kmip-key.pem
ca_path=/ssl/kmip-ca.pem
auth_suite=TLS1.2
policy_path=/etc/pykmip/policies
enable_tls_client_auth=True
database_path=/pykmip/pykmip.db
tls_cipher_suites=
TLS_RSA_WITH_AES_128_CBC_SHA256
TLS_RSA_WITH_AES_256_CBC_SHA256
TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384
logging_level=DEBUG

View File

@ -2,9 +2,9 @@
set -x #echo on
set -e #exit at the first error
mkdir -p ~/.aws
mkdir -p $HOME/.aws
cat >>/root/.aws/credentials <<EOF
cat >>$HOME/.aws/credentials <<EOF
[default]
aws_access_key_id = $AWS_S3_BACKEND_ACCESS_KEY
aws_secret_access_key = $AWS_S3_BACKEND_SECRET_KEY

35
.github/workflows/alerts.yaml vendored Normal file
View File

@ -0,0 +1,35 @@
name: Test alerts
on:
push:
branches-ignore:
- 'development/**'
- 'q/*/**'
jobs:
run-alert-tests:
runs-on: ubuntu-latest
strategy:
matrix:
tests:
- name: 1 minute interval tests
file: monitoring/alerts.test.yaml
- name: 10 seconds interval tests
file: monitoring/alerts.10s.test.yaml
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Render and test ${{ matrix.tests.name }}
uses: scality/action-prom-render-test@1.0.3
with:
alert_file_path: monitoring/alerts.yaml
test_file_path: ${{ matrix.tests.file }}
alert_inputs: |
namespace=zenko
service=artesca-data-connector-s3api-metrics
reportJob=artesca-data-ops-report-handler
replicas=3
github_token: ${{ secrets.GITHUB_TOKEN }}

25
.github/workflows/codeql.yaml vendored Normal file
View File

@ -0,0 +1,25 @@
---
name: codeQL
on:
push:
branches: [w/**, q/*]
pull_request:
branches: [development/*, stabilization/*, hotfix/*]
workflow_dispatch:
jobs:
analyze:
name: Static analysis with CodeQL
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: javascript, python, ruby
- name: Build and analyze
uses: github/codeql-action/analyze@v3

View File

@ -0,0 +1,16 @@
---
name: dependency review
on:
pull_request:
branches: [development/*, stabilization/*, hotfix/*]
jobs:
dependency-review:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@v4
- name: 'Dependency Review'
uses: actions/dependency-review-action@v4

80
.github/workflows/release.yaml vendored Normal file
View File

@ -0,0 +1,80 @@
---
name: release
run-name: release ${{ inputs.tag }}
on:
workflow_dispatch:
inputs:
tag:
description: 'Tag to be released'
required: true
env:
PROJECT_NAME: ${{ github.event.repository.name }}
jobs:
build-federation-image:
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Build and push image for federation
uses: docker/build-push-action@v5
with:
push: true
context: .
file: images/svc-base/Dockerfile
tags: |
ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-svc-base
cache-from: type=gha,scope=federation
cache-to: type=gha,mode=max,scope=federation
release:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildk
uses: docker/setup-buildx-action@v3
- name: Login to Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Push dashboards into the production namespace
run: |
oras push ghcr.io/${{ github.repository }}/${{ env.PROJECT_NAME }}-dashboards:${{ github.event.inputs.tag }} \
dashboard.json:application/grafana-dashboard+json \
alerts.yaml:application/prometheus-alerts+yaml
working-directory: monitoring
- name: Build and push
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Create Release
uses: softprops/action-gh-release@v2
env:
GITHUB_TOKEN: ${{ github.token }}
with:
name: Release ${{ github.event.inputs.tag }}
tag_name: ${{ github.event.inputs.tag }}
generate_release_notes: true
target_commitish: ${{ github.sha }}

533
.github/workflows/tests.yaml vendored Normal file
View File

@ -0,0 +1,533 @@
---
name: tests
on:
workflow_dispatch:
push:
branches-ignore:
- 'development/**'
- 'q/*/**'
env:
# Secrets
azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
azurebackend_AZURE_STORAGE_ENDPOINT: >-
${{ secrets.AZURE_STORAGE_ENDPOINT }}
azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
${{ secrets.AZURE_STORAGE_ACCESS_KEY_2 }}
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME_2 }}
azurebackend2_AZURE_STORAGE_ENDPOINT: >-
${{ secrets.AZURE_STORAGE_ENDPOINT_2 }}
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
${{ secrets.AZURE_STORAGE_ENDPOINT }}
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
${{ secrets.AZURE_STORAGE_ACCESS_KEY }}
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
${{ secrets.AZURE_STORAGE_ACCOUNT_NAME }}
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
${{ secrets.AZURE_STORAGE_ENDPOINT }}
azuretest_AZURE_BLOB_ENDPOINT: "${{ secrets.AZURE_STORAGE_ENDPOINT }}"
b2backend_B2_ACCOUNT_ID: "${{ secrets.B2BACKEND_B2_ACCOUNT_ID }}"
b2backend_B2_STORAGE_ACCESS_KEY: >-
${{ secrets.B2BACKEND_B2_STORAGE_ACCESS_KEY }}
GOOGLE_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
GOOGLE_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
AWS_S3_BACKEND_ACCESS_KEY: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY }}"
AWS_S3_BACKEND_SECRET_KEY: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY }}"
AWS_S3_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_S3_BACKEND_ACCESS_KEY_2 }}"
AWS_S3_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_S3_BACKEND_SECRET_KEY_2 }}"
AWS_GCP_BACKEND_ACCESS_KEY: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY }}"
AWS_GCP_BACKEND_SECRET_KEY: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY }}"
AWS_GCP_BACKEND_ACCESS_KEY_2: "${{ secrets.AWS_GCP_BACKEND_ACCESS_KEY_2 }}"
AWS_GCP_BACKEND_SECRET_KEY_2: "${{ secrets.AWS_GCP_BACKEND_SECRET_KEY_2 }}"
b2backend_B2_STORAGE_ENDPOINT: "${{ secrets.B2BACKEND_B2_STORAGE_ENDPOINT }}"
gcpbackend2_GCP_SERVICE_EMAIL: "${{ secrets.GCP2_SERVICE_EMAIL }}"
gcpbackend2_GCP_SERVICE_KEY: "${{ secrets.GCP2_SERVICE_KEY }}"
gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackend_GCP_SERVICE_EMAIL: "${{ secrets.GCP_SERVICE_EMAIL }}"
gcpbackend_GCP_SERVICE_KEY: "${{ secrets.GCP_SERVICE_KEY }}"
gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_EMAIL }}
gcpbackendmismatch_GCP_SERVICE_KEY: >-
${{ secrets.GCPBACKENDMISMATCH_GCP_SERVICE_KEY }}
gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
# Configs
ENABLE_LOCAL_CACHE: "true"
REPORT_TOKEN: "report-token-1"
REMOTE_MANAGEMENT_DISABLE: "1"
# https://github.com/git-lfs/git-lfs/issues/5749
GIT_CLONE_PROTECTION_ACTIVE: 'false'
jobs:
linting-coverage:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: '16'
cache: yarn
- name: install dependencies
run: yarn install --frozen-lockfile --network-concurrency 1
- uses: actions/setup-python@v5
with:
python-version: '3.9'
- uses: actions/cache@v4
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip
- name: Install python deps
run: pip install flake8
- name: Lint Javascript
run: yarn run --silent lint -- --max-warnings 0
- name: Lint Markdown
run: yarn run --silent lint_md
- name: Lint python
run: flake8 $(git ls-files "*.py")
- name: Lint Yaml
run: yamllint -c yamllint.yml $(git ls-files "*.yml")
- name: Unit Coverage
run: |
set -ex
mkdir -p $CIRCLE_TEST_REPORTS/unit
yarn test
yarn run test_legacy_location
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
CIRCLE_TEST_REPORTS: /tmp
CIRCLE_ARTIFACTS: /tmp
CI_REPORTS: /tmp
- name: Unit Coverage logs
run: find /tmp/unit -exec cat {} \;
- name: preparing junit files for upload
run: |
mkdir -p artifacts/junit
find . -name "*junit*.xml" -exec cp {} artifacts/junit/ ";"
if: always()
- name: Upload files to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: artifacts
if: always()
build:
runs-on: ubuntu-20.04
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Login to GitHub Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Build and push cloudserver image
uses: docker/build-push-action@v5
with:
push: true
context: .
provenance: false
tags: |
ghcr.io/${{ github.repository }}:${{ github.sha }}
labels: |
git.repository=${{ github.repository }}
git.commit-sha=${{ github.sha }}
cache-from: type=gha,scope=cloudserver
cache-to: type=gha,mode=max,scope=cloudserver
- name: Build and push pykmip image
uses: docker/build-push-action@v5
with:
push: true
context: .github/pykmip
tags: |
ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
labels: |
git.repository=${{ github.repository }}
git.commit-sha=${{ github.sha }}
cache-from: type=gha,scope=pykmip
cache-to: type=gha,mode=max,scope=pykmip
- name: Build and push MongoDB
uses: docker/build-push-action@v5
with:
push: true
context: .github/docker/mongodb
tags: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
cache-from: type=gha,scope=mongodb
cache-to: type=gha,mode=max,scope=mongodb
multiple-backend:
runs-on: ubuntu-latest
needs: build
env:
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
S3BACKEND: mem
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
S3DATA: multiple
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Login to Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile sproxyd up -d
working-directory: .github/docker
- name: Run multiple backend test
run: |-
set -o pipefail;
bash wait_for_local_port.bash 8000 40
bash wait_for_local_port.bash 81 40
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/tests.log
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
mongo-v0-ft-tests:
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: mem
MPU_TESTING: "yes"
S3METADATA: mongodb
S3KMS: file
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
DEFAULT_BUCKET_KEY_FORMAT: v0
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile mongo up -d
working-directory: .github/docker
- name: Run functional tests
run: |-
set -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
mongo-v1-ft-tests:
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: mem
MPU_TESTING: "yes"
S3METADATA: mongodb
S3KMS: file
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigTests.json
DEFAULT_BUCKET_KEY_FORMAT: v1
METADATA_MAX_CACHED_BUCKETS: 1
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile mongo up -d
working-directory: .github/docker
- name: Run functional tests
run: |-
set -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log
yarn run ft_mixed_bucket_format_version | tee /tmp/artifacts/${{ github.job }}/mixed-tests.log
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
file-ft-tests:
strategy:
matrix:
include:
- job-name: file-ft-tests
name: ${{ matrix.job-name }}
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: file
S3VAULT: mem
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
MPU_TESTING: "yes"
JOB_NAME: ${{ matrix.job-name }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup matrix job artifacts directory
shell: bash
run: |
set -exu
mkdir -p /tmp/artifacts/${{ matrix.job-name }}/
- name: Setup CI services
run: docker compose up -d
working-directory: .github/docker
- name: Run file ft tests
run: |-
set -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run ft_test | tee /tmp/artifacts/${{ matrix.job-name }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
utapi-v2-tests:
runs-on: ubuntu-latest
needs: build
env:
ENABLE_UTAPI_V2: t
S3BACKEND: mem
BUCKET_DENY_FILTER: utapi-event-filter-deny-bucket
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose up -d
working-directory: .github/docker
- name: Run file utapi v2 tests
run: |-
set -ex -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run test_utapi_v2 | tee /tmp/artifacts/${{ github.job }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
quota-tests:
runs-on: ubuntu-latest
needs: build
strategy:
matrix:
inflights:
- name: "With Inflights"
value: "true"
- name: "Without Inflights"
value: "false"
env:
S3METADATA: mongodb
S3BACKEND: mem
S3QUOTA: scuba
QUOTA_ENABLE_INFLIGHTS: ${{ matrix.inflights.value }}
SCUBA_HOST: localhost
SCUBA_PORT: 8100
SCUBA_HEALTHCHECK_FREQUENCY: 100
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Setup CI services
run: docker compose --profile mongo up -d
working-directory: .github/docker
- name: Run quota tests
run: |-
set -ex -o pipefail;
bash wait_for_local_port.bash 8000 40
yarn run test_quota | tee /tmp/artifacts/${{ github.job }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
kmip-ft-tests:
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: file
S3VAULT: mem
MPU_TESTING: "yes"
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
PYKMIP_IMAGE: ghcr.io/${{ github.repository }}/pykmip:${{ github.sha }}
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- name: Copy KMIP certs
run: cp -r ./certs /tmp/ssl-kmip
working-directory: .github/pykmip
- name: Setup CI services
run: docker compose --profile pykmip up -d
working-directory: .github/docker
- name: Run file KMIP tests
run: |-
set -ex -o pipefail;
bash wait_for_local_port.bash 8000 40
bash wait_for_local_port.bash 5696 40
yarn run ft_kmip | tee /tmp/artifacts/${{ github.job }}/tests.log
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()
ceph-backend-test:
runs-on: ubuntu-latest
needs: build
env:
S3BACKEND: mem
S3DATA: multiple
S3KMS: file
CI_CEPH: 'true'
MPU_TESTING: "yes"
S3_LOCATION_FILE: /usr/src/app/tests/locationConfig/locationConfigCeph.json
MONGODB_IMAGE: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
CLOUDSERVER_IMAGE: ghcr.io/${{ github.repository }}:${{ github.sha }}
JOB_NAME: ${{ github.job }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Login to GitHub Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ github.token }}
- name: Setup CI environment
uses: ./.github/actions/setup-ci
- uses: ruby/setup-ruby@v1
with:
ruby-version: '2.5.9'
- name: Install Ruby dependencies
run: |
gem install nokogiri:1.12.5 excon:0.109.0 fog-aws:1.3.0 json mime-types:3.1 rspec:3.5
- name: Install Java dependencies
run: |
sudo apt-get update && sudo apt-get install -y --fix-missing default-jdk maven
- name: Setup CI services
run: docker compose --profile ceph up -d
working-directory: .github/docker
env:
S3METADATA: mongodb
- name: Run Ceph multiple backend tests
run: |-
set -ex -o pipefail;
bash .github/ceph/wait_for_ceph.sh
bash wait_for_local_port.bash 27018 40
bash wait_for_local_port.bash 8000 40
yarn run multiple_backend_test | tee /tmp/artifacts/${{ github.job }}/multibackend-tests.log
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
S3METADATA: mem
- name: Run Java tests
run: |-
set -ex -o pipefail;
mvn test | tee /tmp/artifacts/${{ github.job }}/java-tests.log
working-directory: tests/functional/jaws
- name: Run Ruby tests
run: |-
set -ex -o pipefail;
rspec -fd --backtrace tests.rb | tee /tmp/artifacts/${{ github.job }}/ruby-tests.log
working-directory: tests/functional/fog
- name: Run Javascript AWS SDK tests
run: |-
set -ex -o pipefail;
yarn run ft_awssdk | tee /tmp/artifacts/${{ github.job }}/js-awssdk-tests.log;
yarn run ft_s3cmd | tee /tmp/artifacts/${{ github.job }}/js-s3cmd-tests.log;
env:
S3_LOCATION_FILE: tests/locationConfig/locationConfigCeph.json
S3BACKEND: file
S3VAULT: mem
S3METADATA: mongodb
- name: Upload logs to artifacts
uses: scality/action-artifacts@v4
with:
method: upload
url: https://artifacts.scality.net
user: ${{ secrets.ARTIFACTS_USER }}
password: ${{ secrets.ARTIFACTS_PASSWORD }}
source: /tmp/artifacts
if: always()

5
.gitignore vendored
View File

@ -28,3 +28,8 @@ _build
# Dependency directory
# https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git
node_modules
yarn.lock
.tox
# Junit directory
junit

View File

@ -1,28 +1,60 @@
FROM node:6-slim
MAINTAINER Giorgio Regni <gr@scality.com>
ARG NODE_VERSION=16.20-bullseye-slim
FROM node:${NODE_VERSION} as builder
WORKDIR /usr/src/app
# Keep the .git directory in order to properly report version
COPY ./package.json .
RUN apt-get update \
&& apt-get install -y jq python git build-essential --no-install-recommends \
&& npm install --production \
&& apt-get autoremove --purge -y python git build-essential \
&& rm -rf /var/lib/apt/lists/* \
&& npm cache clear \
&& rm -rf ~/.node-gyp \
&& rm -rf /tmp/npm-*
&& apt-get install -y --no-install-recommends \
build-essential \
ca-certificates \
curl \
git \
gnupg2 \
jq \
python3 \
ssh \
wget \
libffi-dev \
zlib1g-dev \
&& apt-get clean \
&& mkdir -p /root/ssh \
&& ssh-keyscan -H github.com > /root/ssh/known_hosts
COPY ./ ./
ENV PYTHON=python3
COPY package.json yarn.lock /usr/src/app/
RUN npm install typescript -g
RUN yarn install --production --ignore-optional --frozen-lockfile --ignore-engines --network-concurrency 1
VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
################################################################################
FROM node:${NODE_VERSION}
RUN apt-get update && \
apt-get install -y --no-install-recommends \
jq \
&& rm -rf /var/lib/apt/lists/*
ENV NO_PROXY localhost,127.0.0.1
ENV no_proxy localhost,127.0.0.1
ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
CMD [ "npm", "start" ]
EXPOSE 8000
EXPOSE 8002
RUN apt-get update && \
apt-get install -y --no-install-recommends \
jq \
tini \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /usr/src/app
# Keep the .git directory in order to properly report version
COPY . /usr/src/app
COPY --from=builder /usr/src/app/node_modules ./node_modules/
VOLUME ["/usr/src/app/localData","/usr/src/app/localMetadata"]
ENTRYPOINT ["tini", "--", "/usr/src/app/docker-entrypoint.sh"]
CMD [ "yarn", "start" ]

View File

@ -7,16 +7,16 @@ COPY . /usr/src/app
RUN apt-get update \
&& apt-get install -y jq python git build-essential --no-install-recommends \
&& npm install --production \
&& yarn install --production \
&& apt-get autoremove --purge -y python git build-essential \
&& rm -rf /var/lib/apt/lists/* \
&& npm cache clear \
&& yarn cache clean \
&& rm -rf ~/.node-gyp \
&& rm -rf /tmp/npm-*
&& rm -rf /tmp/yarn-*
ENV S3BACKEND mem
ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
CMD [ "npm", "start" ]
CMD [ "yarn", "start" ]
EXPOSE 8000

View File

@ -1,6 +1,7 @@
# S3 Healthcheck
Scality S3 exposes a healthcheck route `/_/healthcheck` which returns a
Scality S3 exposes a healthcheck route `/live` on the port used
for the metrics (defaults to port 8002) which returns a
response with HTTP code
- 200 OK

166
README.md
View File

@ -1,12 +1,7 @@
# Zenko CloudServer
# Zenko CloudServer with Vitastor Backend
![Zenko CloudServer logo](res/scality-cloudserver-logo.png)
[![CircleCI][badgepub]](https://circleci.com/gh/scality/S3)
[![Scality CI][badgepriv]](http://ci.ironmann.io/gh/scality/S3)
[![Docker Pulls][badgedocker]](https://hub.docker.com/r/scality/s3server/)
[![Docker Pulls][badgetwitter]](https://twitter.com/zenko)
## Overview
CloudServer (formerly S3 Server) is an open-source Amazon S3-compatible
@ -16,126 +11,71 @@ Scalitys Open Source Multi-Cloud Data Controller.
CloudServer provides a single AWS S3 API interface to access multiple
backend data storage both on-premise or public in the cloud.
CloudServer is useful for Developers, either to run as part of a
continous integration test environment to emulate the AWS S3 service locally
or as an abstraction layer to develop object storage enabled
application on the go.
This repository contains a fork of CloudServer with [Vitastor](https://git.yourcmc.ru/vitalif/vitastor)
backend support.
## Learn more at [www.zenko.io/cloudserver](https://www.zenko.io/cloudserver/)
## Quick Start with Vitastor
## [May I offer you some lovely documentation?](http://s3-server.readthedocs.io/en/latest/)
Vitastor Backend is in experimental status, however you can already try to
run it and write or read something, or even mount it with [GeeseFS](https://github.com/yandex-cloud/geesefs),
it works too 😊.
## Docker
Installation instructions:
[Run your Zenko CloudServer with Docker](https://hub.docker.com/r/scality/s3server/)
### Install Vitastor
## Contributing
Refer to [Vitastor Quick Start Manual](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/intro/quickstart.en.md).
In order to contribute, please follow the
[Contributing Guidelines](
https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md).
### Install Zenko with Vitastor Backend
## Installation
- Clone this repository: `git clone https://git.yourcmc.ru/vitalif/zenko-cloudserver-vitastor`
- Install dependencies: `npm install --omit dev` or just `npm install`
- Clone Vitastor repository: `git clone https://git.yourcmc.ru/vitalif/vitastor`
- Build Vitastor node.js binding by running `npm install` in `node-binding` subdirectory of Vitastor repository.
You need `node-gyp` and `vitastor-client-dev` (Vitastor client library) for it to succeed.
- Symlink Vitastor module to Zenko: `ln -s /path/to/vitastor/node-binding /path/to/zenko/node_modules/vitastor`
### Dependencies
### Install and Configure MongoDB
Building and running the Zenko CloudServer requires node.js 6.9.5 and npm v3
. Up-to-date versions can be found at
[Nodesource](https://github.com/nodesource/distributions).
Refer to [MongoDB Manual](https://www.mongodb.com/docs/manual/installation/).
### Clone source code
### Setup Zenko
```shell
git clone https://github.com/scality/S3.git
- Create a separate pool for S3 object data in your Vitastor cluster: `vitastor-cli create-pool s3-data`
- Retrieve ID of the new pool from `vitastor-cli ls-pools --detail s3-data`
- In another pool, create an image for storing Vitastor volume metadata: `vitastor-cli create -s 10G s3-volume-meta`
- Copy `config.json.vitastor` to `config.json`, adjust it to match your domain
- Copy `authdata.json.example` to `authdata.json` - this is where you set S3 access & secret keys,
and also adjust them if you want to. Scality seems to use a separate auth service "Scality Vault" for
access keys, but it's not published, so let's use a file for now.
- Copy `locationConfig.json.vitastor` to `locationConfig.json` - this is where you set Vitastor cluster access data.
You should put correct values for `pool_id` (pool ID from the second step) and `metadata_image` (from the third step)
in this file.
Note: `locationConfig.json` in this version corresponds to storage classes (like STANDARD, COLD, etc)
instead of "locations" (zones like us-east-1) as it was in original Zenko CloudServer.
### Start Zenko
Start the S3 server with: `node index.js`
If you use default settings, Zenko CloudServer starts on port 8000.
The default access key is `accessKey1` with a secret key of `verySecretKey1`.
Now you can access your S3 with `s3cmd` or `geesefs`:
```
s3cmd --access_key=accessKey1 --secret_key=verySecretKey1 --host=http://localhost:8000 mb s3://testbucket
```
### Install js dependencies
Go to the ./S3 folder,
```shell
npm install
```
AWS_ACCESS_KEY_ID=accessKey1 \
AWS_SECRET_ACCESS_KEY=verySecretKey1 \
geesefs --endpoint http://localhost:8000 testbucket mountdir
```
If you get an error regarding installation of the diskUsage module,
please install g++.
# Author & License
If you get an error regarding level-down bindings, try clearing your npm cache:
```shell
npm cache clear
```
## Run it with a file backend
```shell
npm start
```
This starts a Zenko CloudServer on port 8000. Two additional ports 9990 and
9991 are also open locally for internal transfer of metadata and data,
respectively.
The default access key is accessKey1 with
a secret key of verySecretKey1.
By default the metadata files will be saved in the
localMetadata directory and the data files will be saved
in the localData directory within the ./S3 directory on your
machine. These directories have been pre-created within the
repository. If you would like to save the data or metadata in
different locations of your choice, you must specify them with absolute paths.
So, when starting the server:
```shell
mkdir -m 700 $(pwd)/myFavoriteDataPath
mkdir -m 700 $(pwd)/myFavoriteMetadataPath
export S3DATAPATH="$(pwd)/myFavoriteDataPath"
export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
npm start
```
## Run it with multiple data backends
```shell
export S3DATA='multiple'
npm start
```
This starts a Zenko CloudServer on port 8000.
The default access key is accessKey1 with
a secret key of verySecretKey1.
With multiple backends, you have the ability to
choose where each object will be saved by setting
the following header with a locationConstraint on
a PUT request:
```shell
'x-amz-meta-scal-location-constraint':'myLocationConstraint'
```
If no header is sent with a PUT object request, the
location constraint of the bucket will determine
where the data is saved. If the bucket has no location
constraint, the endpoint of the PUT request will be
used to determine location.
See the Configuration section in our documentation
[here](http://s3-server.readthedocs.io/en/latest/GETTING_STARTED/#configuration)
to learn how to set location constraints.
## Run it with an in-memory backend
```shell
npm run mem_backend
```
This starts a Zenko CloudServer on port 8000.
The default access key is accessKey1 with
a secret key of verySecretKey1.
[badgetwitter]: https://img.shields.io/twitter/follow/zenko.svg?style=social&label=Follow
[badgedocker]: https://img.shields.io/docker/pulls/scality/s3server.svg
[badgepub]: https://circleci.com/gh/scality/S3.svg?style=svg
[badgepriv]: http://ci.ironmann.io/gh/scality/S3.svg?style=svg&circle-token=1f105b7518b53853b5b7cf72302a3f75d8c598ae
- [Zenko CloudServer](https://s3-server.readthedocs.io/en/latest/) author is Scality, licensed under [Apache License, version 2.0](https://www.apache.org/licenses/LICENSE-2.0)
- [Vitastor](https://git.yourcmc.ru/vitalif/vitastor/) and Zenko Vitastor backend author is Vitaliy Filippov, licensed under [VNPL-1.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/VNPL-1.1.txt)
(a "network copyleft" license based on AGPL/SSPL, but worded in a better way)

View File

@ -1,2 +1,2 @@
---
theme: jekyll-theme-minimal
theme: jekyll-theme-modernist

View File

@ -13,20 +13,26 @@ function _performSearch(host,
port,
bucketName,
query,
listVersions,
accessKey,
secretKey,
sessionToken,
verbose, ssl) {
const escapedSearch = encodeURIComponent(query);
const options = {
host,
port,
method: 'GET',
path: `/${bucketName}/?search=${escapedSearch}`,
path: `/${bucketName}/?search=${escapedSearch}${listVersions ? '&&versions' : ''}`,
headers: {
'Content-Length': 0,
},
rejectUnauthorized: false,
versions: '',
};
if (sessionToken) {
options.headers['x-amz-security-token'] = sessionToken;
}
const transport = ssl ? https : http;
const request = transport.request(options, response => {
if (verbose) {
@ -55,9 +61,9 @@ function _performSearch(host,
// generateV4Headers exepects request object with path that does not
// include query
request.path = `/${bucketName}`;
auth.client.generateV4Headers(request, { search: query },
accessKey, secretKey, 's3');
request.path = `/${bucketName}?search=${escapedSearch}`;
const requestData = listVersions ? { search: query, versions: '' } : { search: query };
auth.client.generateV4Headers(request, requestData, accessKey, secretKey, 's3');
request.path = `/${bucketName}?search=${escapedSearch}${listVersions ? '&&versions' : ''}`;
if (verbose) {
logger.info('request headers', { headers: request._headers });
}
@ -76,15 +82,17 @@ function searchBucket() {
.version('0.0.1')
.option('-a, --access-key <accessKey>', 'Access key id')
.option('-k, --secret-key <secretKey>', 'Secret access key')
.option('-t, --session-token <sessionToken>', 'Session token')
.option('-b, --bucket <bucket>', 'Name of the bucket')
.option('-q, --query <query>', 'Search query')
.option('-h, --host <host>', 'Host of the server')
.option('-p, --port <port>', 'Port of the server')
.option('-s', '--ssl', 'Enable ssl')
.option('-l, --list-versions', 'List all versions of the objects that meet the search query, ' +
'otherwise only list the latest version')
.option('-v, --verbose')
.parse(process.argv);
const { host, port, accessKey, secretKey, bucket, query, verbose, ssl } =
const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } =
commander;
if (!host || !port || !accessKey || !secretKey || !bucket || !query) {
@ -93,7 +101,7 @@ function searchBucket() {
process.exit(1);
}
_performSearch(host, port, bucket, query, accessKey, secretKey, verbose,
_performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose,
ssl);
}

View File

@ -1,7 +1,10 @@
{
"port": 8000,
"listenOn": [],
"metricsPort": 8002,
"metricsListenOn": [],
"replicationGroupId": "RG001",
"workers": 4,
"restEndpoints": {
"localhost": "us-east-1",
"127.0.0.1": "us-east-1",
@ -39,6 +42,10 @@
"host": "localhost",
"port": 8900
},
"workflowEngineOperator": {
"host": "localhost",
"port": 3001
},
"cdmi": {
"host": "localhost",
"port": 81,
@ -46,7 +53,7 @@
"readonly": true
},
"bucketd": {
"bootstrap": ["localhost"]
"bootstrap": ["localhost:9000"]
},
"vaultd": {
"host": "localhost",
@ -68,6 +75,10 @@
"host": "localhost",
"port": 9991
},
"pfsClient": {
"host": "localhost",
"port": 9992
},
"metadataDaemon": {
"bindAddress": "localhost",
"port": 9990
@ -76,15 +87,57 @@
"bindAddress": "localhost",
"port": 9991
},
"pfsDaemon": {
"bindAddress": "localhost",
"port": 9992
},
"recordLog": {
"enabled": true,
"recordLogName": "s3-recordlog"
},
"mongodb": {
"replicaSetHosts": "localhost:27018,localhost:27019,localhost:27020",
"writeConcern": "majority",
"replicaSet": "rs0",
"readPreference": "primary",
"database": "metadata"
}
"replicaSetHosts": "localhost:27018,localhost:27019,localhost:27020",
"writeConcern": "majority",
"replicaSet": "rs0",
"readPreference": "primary",
"database": "metadata"
},
"authdata": "authdata.json",
"backends": {
"auth": "file",
"data": "file",
"metadata": "mongodb",
"kms": "file",
"quota": "none"
},
"externalBackends": {
"aws_s3": {
"httpAgent": {
"keepAlive": false,
"keepAliveMsecs": 1000,
"maxFreeSockets": 256,
"maxSockets": null
}
},
"gcp": {
"httpAgent": {
"keepAlive": true,
"keepAliveMsecs": 1000,
"maxFreeSockets": 256,
"maxSockets": null
}
}
},
"requests": {
"viaProxy": false,
"trustedProxyCIDRs": [],
"extractClientIPFromHeader": ""
},
"bucketNotificationDestinations": [
{
"resource": "target1",
"type": "dummy",
"host": "localhost:6000"
}
]
}

71
config.json.vitastor Normal file
View File

@ -0,0 +1,71 @@
{
"port": 8000,
"listenOn": [],
"metricsPort": 8002,
"metricsListenOn": [],
"replicationGroupId": "RG001",
"restEndpoints": {
"localhost": "STANDARD",
"127.0.0.1": "STANDARD",
"yourhostname.ru": "STANDARD"
},
"websiteEndpoints": [
"static.yourhostname.ru"
],
"replicationEndpoints": [ {
"site": "zenko",
"servers": ["127.0.0.1:8000"],
"default": true
} ],
"log": {
"logLevel": "info",
"dumpLevel": "error"
},
"healthChecks": {
"allowFrom": ["127.0.0.1/8", "::1"]
},
"backends": {
"metadata": "mongodb"
},
"mongodb": {
"replicaSetHosts": "127.0.0.1:27017",
"writeConcern": "majority",
"replicaSet": "rs0",
"readPreference": "primary",
"database": "s3",
"authCredentials": {
"username": "s3",
"password": ""
}
},
"externalBackends": {
"aws_s3": {
"httpAgent": {
"keepAlive": false,
"keepAliveMsecs": 1000,
"maxFreeSockets": 256,
"maxSockets": null
}
},
"gcp": {
"httpAgent": {
"keepAlive": true,
"keepAliveMsecs": 1000,
"maxFreeSockets": 256,
"maxSockets": null
}
}
},
"requests": {
"viaProxy": false,
"trustedProxyCIDRs": [],
"extractClientIPFromHeader": ""
},
"bucketNotificationDestinations": [
{
"resource": "target1",
"type": "dummy",
"host": "localhost:6000"
}
]
}

View File

@ -86,40 +86,51 @@ const constants = {
// In testing, AWS seems to allow up to 88 more bytes, so we do the same.
maximumMetaHeadersSize: 2136,
// Maximum HTTP headers size allowed
maxHttpHeadersSize: 14122,
// hex digest of sha256 hash of empty string:
emptyStringHash: crypto.createHash('sha256')
.update('', 'binary').digest('hex'),
// Queries supported by AWS that we do not currently support.
// Non-bucket queries
unsupportedQueries: [
'accelerate',
'analytics',
'inventory',
'list-type',
'logging',
'metrics',
'notification',
'policy',
'policyStatus',
'publicAccessBlock',
'requestPayment',
'restore',
'torrent',
],
// Headers supported by AWS that we do not currently support.
unsupportedHeaders: [
'x-amz-server-side-encryption',
'x-amz-server-side-encryption-customer-algorithm',
'x-amz-server-side-encryption-aws-kms-key-id',
'x-amz-server-side-encryption-context',
'x-amz-server-side-encryption-customer-key',
'x-amz-server-side-encryption-customer-key-md5',
],
// user metadata header to set object locationConstraint
objectLocationConstraintHeader: 'x-amz-meta-scal-location-constraint',
objectLocationConstraintHeader: 'x-amz-storage-class',
lastModifiedHeader: 'x-amz-meta-x-scal-last-modified',
legacyLocations: ['sproxyd', 'legacy'],
// declare here all existing service accounts and their properties
// (if any, otherwise an empty object)
serviceAccountProperties: {
replication: {},
lifecycle: {},
gc: {},
'md-ingestion': {
canReplicate: true,
},
},
/* eslint-disable camelcase */
externalBackends: { aws_s3: true, azure: true, gcp: true },
replicationBackends: { aws_s3: true, azure: true, gcp: true },
externalBackends: { aws_s3: true, azure: true, gcp: true, pfs: true, dmf: true, azure_archive: true },
// some of the available data backends (if called directly rather
// than through the multiple backend gateway) need a key provided
// as a string as first parameter of the get/delete methods.
@ -138,6 +149,100 @@ const constants = {
azureAccountNameRegex: /^[a-z0-9]{3,24}$/,
base64Regex: new RegExp('^(?:[A-Za-z0-9+/]{4})*' +
'(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$'),
productName: 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko',
// location constraint delimiter
zenkoSeparator: ':',
// user metadata applied on zenko objects
zenkoIDHeader: 'x-amz-meta-zenko-instance-id',
bucketOwnerActions: [
'bucketDeleteCors',
'bucketDeleteLifecycle',
'bucketDeletePolicy',
'bucketDeleteReplication',
'bucketDeleteWebsite',
'bucketGetCors',
'bucketGetLifecycle',
'bucketGetLocation',
'bucketGetPolicy',
'bucketGetReplication',
'bucketGetVersioning',
'bucketGetWebsite',
'bucketPutCors',
'bucketPutLifecycle',
'bucketPutPolicy',
'bucketPutReplication',
'bucketPutVersioning',
'bucketPutWebsite',
'objectDeleteTagging',
'objectGetTagging',
'objectPutTagging',
'objectPutLegalHold',
'objectPutRetention',
],
// response header to be sent when there are invalid
// user metadata in the object's metadata
invalidObjectUserMetadataHeader: 'x-amz-missing-meta',
// Bucket specific queries supported by AWS that we do not currently support
// these queries may or may not be supported at object level
unsupportedBucketQueries: [
],
suppressedUtapiEventFields: [
'object',
'location',
'versionId',
],
allowedUtapiEventFilterFields: [
'operationId',
'location',
'account',
'user',
'bucket',
],
arrayOfAllowed: [
'objectPutTagging',
'objectPutLegalHold',
'objectPutRetention',
],
allowedUtapiEventFilterStates: ['allow', 'deny'],
allowedRestoreObjectRequestTierValues: ['Standard'],
lifecycleListing: {
CURRENT_TYPE: 'current',
NON_CURRENT_TYPE: 'noncurrent',
ORPHAN_DM_TYPE: 'orphan',
},
multiObjectDeleteConcurrency: 50,
maxScannedLifecycleListingEntries: 10000,
overheadField: [
'content-length',
'owner-id',
'versionId',
'isNull',
'isDeleteMarker',
],
unsupportedSignatureChecksums: new Set([
'STREAMING-UNSIGNED-PAYLOAD-TRAILER',
'STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER',
'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD',
'STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER',
]),
supportedSignatureChecksums: new Set([
'UNSIGNED-PAYLOAD',
'STREAMING-AWS4-HMAC-SHA256-PAYLOAD',
]),
ipv4Regex: /^(\d{1,3}\.){3}\d{1,3}(\/(3[0-2]|[12]?\d))?$/,
ipv6Regex: /^([\da-f]{1,4}:){7}[\da-f]{1,4}$/i,
// The AWS assumed Role resource type
assumedRoleArnResourceType: 'assumed-role',
// Session name of the backbeat lifecycle assumed role session.
backbeatLifecycleSessionName: 'backbeat-lifecycle',
actionsToConsiderAsObjectPut: [
'initiateMultipartUpload',
'objectPutPart',
'completeMultipartUpload',
],
// if requester is not bucket owner, bucket policy actions should be denied with
// MethodNotAllowed error
onlyOwnerAllowed: ['bucketDeletePolicy', 'bucketGetPolicy', 'bucketPutPolicy'],
};
module.exports = constants;

View File

@ -4,16 +4,30 @@ const arsenal = require('arsenal');
const { config } = require('./lib/Config.js');
const logger = require('./lib/utilities/logger');
process.on('uncaughtException', err => {
logger.fatal('caught error', {
error: err.message,
stack: err.stack,
workerId: this.worker ? this.worker.id : undefined,
workerPid: this.worker ? this.worker.process.pid : undefined,
});
process.exit(1);
});
if (config.backends.data === 'file' ||
(config.backends.data === 'multiple' &&
config.backends.metadata !== 'scality')) {
const dataServer = new arsenal.network.rest.RESTServer(
{ bindAddress: config.dataDaemon.bindAddress,
port: config.dataDaemon.port,
dataStore: new arsenal.storage.data.file.DataFileStore(
{ dataPath: config.dataDaemon.dataPath,
log: config.log }),
log: config.log });
const dataServer = new arsenal.network.rest.RESTServer({
bindAddress: config.dataDaemon.bindAddress,
port: config.dataDaemon.port,
dataStore: new arsenal.storage.data.file.DataFileStore({
dataPath: config.dataDaemon.dataPath,
log: config.log,
noSync: config.dataDaemon.noSync,
noCache: config.dataDaemon.noCache,
}),
log: config.log,
});
dataServer.setup(err => {
if (err) {
logger.error('Error initializing REST data server',

View File

@ -71,9 +71,14 @@ fi
if [[ "$LISTEN_ADDR" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataDaemon.bindAddress=\"$LISTEN_ADDR\""
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataDaemon.bindAddress=\"$LISTEN_ADDR\""
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsDaemon.bindAddress=\"$LISTEN_ADDR\""
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .listenOn=[\"$LISTEN_ADDR:8000\"]"
fi
if [[ "$REPLICATION_GROUP_ID" ]] ; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .replicationGroupId=\"$REPLICATION_GROUP_ID\""
fi
if [[ "$DATA_HOST" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .dataClient.host=\"$DATA_HOST\""
fi
@ -82,6 +87,10 @@ if [[ "$METADATA_HOST" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .metadataClient.host=\"$METADATA_HOST\""
fi
if [[ "$PFSD_HOST" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .pfsClient.host=\"$PFSD_HOST\""
fi
if [[ "$MONGODB_HOSTS" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .mongodb.replicaSetHosts=\"$MONGODB_HOSTS\""
fi
@ -138,10 +147,62 @@ if [[ "$CRR_METRICS_PORT" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .backbeat.port=$CRR_METRICS_PORT"
fi
if [[ "$WE_OPERATOR_HOST" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.host=\"$WE_OPERATOR_HOST\""
fi
if [[ "$WE_OPERATOR_PORT" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .workflowEngineOperator.port=$WE_OPERATOR_PORT"
fi
if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]"
fi
# external backends http(s) agent config
# AWS
if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAlive=$AWS_S3_HTTPAGENT_KEEPALIVE"
fi
if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MS" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.keepAliveMsecs=$AWS_S3_HTTPAGENT_KEEPALIVE_MS"
fi
if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
fi
if [[ "$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.aws_s3.httpAgent.maxFreeSockets=$AWS_S3_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
fi
#GCP
if [[ "$GCP_HTTPAGENT_KEEPALIVE" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAlive=$GCP_HTTPAGENT_KEEPALIVE"
fi
if [[ "$GCP_HTTPAGENT_KEEPALIVE_MS" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.keepAliveMsecs=$GCP_HTTPAGENT_KEEPALIVE_MS"
fi
if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_SOCKETS"
fi
if [[ "$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .externalBackends.gcp.httpAgent.maxFreeSockets=$GCP_HTTPAGENT_KEEPALIVE_MAX_FREE_SOCKETS"
fi
if [[ -n "$BUCKET_DENY_FILTER" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .utapi.filter.deny.bucket=[\"$BUCKET_DENY_FILTER\"]"
fi
if [[ "$TESTING_MODE" ]]; then
JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .testingMode=true"
fi
if [[ $JQ_FILTERS_CONFIG != "." ]]; then
jq "$JQ_FILTERS_CONFIG" config.json > config.json.tmp
mv config.json.tmp config.json

View File

@ -66,7 +66,7 @@ The second section, `"Implementation of Bucket Versioning in
API" <#implementation-of-bucket-versioning-in-api>`__, describes the way
the metadata options are used in the API within S3 actions to create new
versions, update their metadata, and delete them. The management of null
versions and creation of delete markers are also described in this
versions and creation of delete markers is also described in this
section.
Implementation of Bucket Versioning in Metadata
@ -746,7 +746,7 @@ Operation
Startup
~~~~~~~
The simplest deployment is still to launch with npm start, this will
The simplest deployment is still to launch with yarn start, this will
start one instance of the Zenko CloudServer connector and will listen on the
locally bound dmd ports 9990 and 9991 (by default, see below).
@ -755,7 +755,7 @@ command in the Zenko CloudServer directory:
::
npm run start_dmd
yarn run start_dmd
This will open two ports:
@ -770,7 +770,7 @@ elsewhere with:
.. code:: sh
npm run start_s3server
yarn run start_s3server
Configuration
~~~~~~~~~~~~~

146
docs/BUCKET_POLICIES.md Normal file
View File

@ -0,0 +1,146 @@
# Bucket Policy Documentation
## Description
Bucket policy is a method of controlling access to a user's account at the
resource level.
There are three associated APIs:
- PUT Bucket policy (see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketPUTpolicy.html)
- GET Bucket policy (see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETpolicy.html)
- DELETE Bucket policy (see https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketDELETEpolicy.html)
More information on bucket policies in general can be found at
https://docs.aws.amazon.com/AmazonS3/latest/dev/using-iam-policies.html.
## Requirements
To prevent loss of access to a bucket, the root owner of a bucket will always
be able to perform any of the three bucket policy-related operations, even
if permission is explicitly denied.
All other users must have permission to perform the desired operation.
## Design
On a PUTBucketPolicy request, the user provides a policy in JSON format.
The policy is evaluated against our policy schema in Arsenal and, once
validated, is stored as part of the bucket's metadata.
On a GETBucketPolicy request, the policy is retrieved from the bucket's
metadata.
On a DELETEBucketPolicy request, the policy is deleted from the bucket's
metadata.
All other APIs are updated to check if a bucket policy is attached to the bucket
the request is made on. If there is a policy, user authorization to perform
the requested action is checked.
### Differences Between Bucket and IAM Policies
IAM policies are attached to an IAM identity and define what actions that
identity is allowed to or denied from doing on what resource.
Bucket policies attach only to buckets and define what actions are allowed or
denied for which principles on that bucket. Permissions specified in a bucket
policy apply to all objects in that bucket unless otherwise specified.
Besides their attachment origins, the main structural difference between
IAM policy and bucket policy is the requirement of a "Principal" element in
bucket policies. This field is redundant in IAM policies.
### Policy Validation
For general guidelines for bucket policy structure, see examples here:
https://docs.aws.amazon.com/AmazonS3/latest/dev//example-bucket-policies.html.
Each bucket policy statement object requires at least four keys:
"Effect", "Principle", "Resource", and "Action".
"Effect" defines the effect of the policy and can have a string value of either
"Allow" or "Deny".
"Resource" defines to which bucket or list of buckets a policy is attached.
An object within the bucket is also a valid resource. The element value can be
either a single bucket or object ARN string or an array of ARNs.
"Action" lists which action(s) the policy controls. Its value can also be either
a string or array of S3 APIs. Each action is the API name prepended by "s3:".
"Principle" specifies which user(s) are granted or denied access to the bucket
resource. Its value can be a string or an object containing an array of users.
Valid users can be identified with an account ARN, account id, or user ARN.
There are also two optional bucket policy statement keys: Sid and Condition.
"Sid" stands for "statement id". If this key is not included, one will be
generated for the statement.
"Condition" lists the condition under which a statement will take affect.
The possibilities are as follows:
- ArnEquals
- ArnEqualsIfExists
- ArnLike
- ArnLikeIfExists
- ArnNotEquals
- ArnNotEqualsIfExists
- ArnNotLike
- ArnNotLikeIfExists
- BinaryEquals
- BinaryEqualsIfExists
- BinaryNotEquals
- BinaryNotEqualsIfExists
- Bool
- BoolIfExists
- DateEquals
- DateEqualsIfExists
- DateGreaterThan
- DateGreaterThanEquals
- DateGreaterThanEqualsIfExists
- DateGreaterThanIfExists
- DateLessThan
- DateLessThanEquals
- DateLessThanEqualsIfExists
- DateLessThanIfExists
- DateNotEquals
- DateNotEqualsIfExists
- IpAddress
- IpAddressIfExists
- NotIpAddress
- NotIpAddressIfExists
- Null
- NumericEquals
- NumericEqualsIfExists
- NumericGreaterThan
- NumericGreaterThanEquals
- NumericGreaterThanEqualsIfExists
- NumericGreaterThanIfExists
- NumericLessThan
- NumericLessThanEquals
- NumericLessThanEqualsIfExists
- NumericLessThanIfExists
- NumericNotEquals
- NumericNotEqualsIfExists
- StringEquals
- StringEqualsIfExists
- StringEqualsIgnoreCase
- StringEqualsIgnoreCaseIfExists
- StringLike
- StringLikeIfExists
- StringNotEquals
- StringNotEqualsIfExists
- StringNotEqualsIgnoreCase
- StringNotEqualsIgnoreCaseIfExists
- StringNotLike
- StringNotLikeIfExists
The value of the Condition key will be an object containing the desired
condition name as that key. The value of inner object can be a string, boolean,
number, or object, depending on the condition.
## Authorization with Multiple Access Control Mechanisms
In the case where multiple access control mechanisms (such as IAM policies,
bucket policies, and ACLs) refer to the same resource, the principle of
least-privilege is applied. Unless an action is explicitly allowed, access will
by default be denied. An explicit DENY in any policy will trump another
policy's ALLOW for an action. The request will only be allowed if at least one
policy specifies an ALLOW, and there is no overriding DENY.
The following diagram illustrates this logic:
![Access_Control_Authorization_Chart](./images/access_control_authorization.png)

View File

@ -295,3 +295,51 @@ Should force path-style requests even though v3 advertises it does by default.
$client->createBucket(array(
'Bucket' => 'bucketphp',
));
Go
~~
`AWS Go SDK <https://github.com/aws/aws-sdk-go>`__
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
.. code:: go
package main
import (
"context"
"fmt"
"log"
"os"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
func main() {
os.Setenv("AWS_ACCESS_KEY_ID", "accessKey1")
os.Setenv("AWS_SECRET_ACCESS_KEY", "verySecretKey1")
endpoint := "http://localhost:8000"
timeout := time.Duration(10) * time.Second
sess := session.Must(session.NewSession())
// Create a context with a timeout that will abort the upload if it takes
// more than the passed in timeout.
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
svc := s3.New(sess, &aws.Config{
Region: aws.String(endpoints.UsEast1RegionID),
Endpoint: &endpoint,
})
out, err := svc.ListBucketsWithContext(ctx, &s3.ListBucketsInput{})
if err != nil {
log.Fatal(err)
} else {
fmt.Println(out)
}
}

View File

@ -14,7 +14,7 @@ Got an idea? Get started!
In order to contribute, please follow the `Contributing
Guidelines <https://github.com/scality/Guidelines/blob/master/CONTRIBUTING.md>`__.
If anything is unclear to you, reach out to us on
`slack <https://zenko-io.slack.com/>`__ or via a GitHub issue.
`forum <https://forum.zenko.io/>`__ or via a GitHub issue.
Don't write code? There are other ways to help!
-----------------------------------------------

View File

@ -1,11 +1,7 @@
Docker
======
- `Environment Variables <#environment-variables>`__
- `Tunables and setup tips <#tunables-and-setup-tips>`__
- `Examples for continuous integration with
Docker <#continuous-integration-with-docker-hosted CloudServer>`__
- `Examples for going in production with Docker <#in-production-with-docker-hosted CloudServer>`__
.. _environment-variables:
Environment Variables
---------------------
@ -15,21 +11,23 @@ S3DATA
S3DATA=multiple
^^^^^^^^^^^^^^^
Allows you to run Scality Zenko CloudServer with multiple data backends, defined
This variable enables running CloudServer with multiple data backends, defined
as regions.
When using multiple data backends, a custom ``locationConfig.json`` file is
mandatory. It will allow you to set custom regions. You will then need to
provide associated rest_endpoints for each custom region in your
``config.json`` file.
`Learn more about multiple backends configuration <../GETTING_STARTED/#location-configuration>`__
If you are using Scality RING endpoints, please refer to your customer
documentation.
For multiple data backends, a custom locationConfig.json file is required.
This file enables you to set custom regions. You must provide associated
rest_endpoints for each custom region in config.json.
Running it with an AWS S3 hosted backend
""""""""""""""""""""""""""""""""""""""""
To run CloudServer with an S3 AWS backend, you will have to add a new section
to your ``locationConfig.json`` file with the ``aws_s3`` location type:
`Learn more about multiple-backend configurations <GETTING_STARTED.html#location-configuration>`__
If you are using Scality RING endpoints, refer to your customer documentation.
Running CloudServer with an AWS S3-Hosted Backend
"""""""""""""""""""""""""""""""""""""""""""""""""
To run CloudServer with an S3 AWS backend, add a new section to the
``locationConfig.json`` file with the ``aws_s3`` location type:
.. code:: json
@ -45,10 +43,9 @@ to your ``locationConfig.json`` file with the ``aws_s3`` location type:
}
(...)
You will also have to edit your AWS credentials file to be able to use your
command line tool of choice. This file should mention credentials for all the
backends you're using. You can use several profiles when using multiple
profiles.
Edit your AWS credentials file to enable your preferred command-line tool.
This file must mention credentials for all backends in use. You can use
several profiles if multiple profiles are configured.
.. code:: json
@ -59,110 +56,124 @@ profiles.
aws_access_key_id={{YOUR_ACCESS_KEY}}
aws_secret_access_key={{YOUR_SECRET_KEY}}
Just as you need to mount your locationConfig.json, you will need to mount your
AWS credentials file at run time:
``-v ~/.aws/credentials:/root/.aws/credentials`` on Linux, OS X, or Unix or
As with locationConfig.json, the AWS credentials file must be mounted at
run time: ``-v ~/.aws/credentials:/root/.aws/credentials`` on Unix-like
systems (Linux, OS X, etc.), or
``-v C:\Users\USERNAME\.aws\credential:/root/.aws/credentials`` on Windows
NOTE: One account can't copy to another account with a source and
destination on real AWS unless the account associated with the
access Key/secret Key pairs used for the destination bucket has rights
to get in the source bucket. ACL's would have to be updated
on AWS directly to enable this.
.. note:: One account cannot copy to another account with a source and
destination on real AWS unless the account associated with the
accessKey/secretKey pairs used for the destination bucket has source
bucket access privileges. To enable this, update ACLs directly on AWS.
S3BACKEND
~~~~~~~~~
S3BACKEND=file
^^^^^^^^^^^^^^
When storing file data, for it to be persistent you must mount docker volumes
for both data and metadata. See `this section <#using-docker-volumes-in-production>`__
For stored file data to persist, you must mount Docker volumes
for both data and metadata. See :ref:`In Production with a Docker-Hosted CloudServer <in-production-w-a-Docker-hosted-cloudserver>`
S3BACKEND=mem
^^^^^^^^^^^^^
This is ideal for testing - no data will remain after container is shutdown.
This is ideal for testing: no data remains after the container is shut down.
ENDPOINT
~~~~~~~~
This variable specifies your endpoint. If you have a domain such as
new.host.com, by specifying that here, you and your users can direct s3
server requests to new.host.com.
This variable specifies the endpoint. To direct CloudServer requests to
new.host.com, for example, specify the endpoint with:
.. code-block:: shell
$ docker run -d --name s3server -p 8000:8000 -e ENDPOINT=new.host.com scality/s3server
$ docker run -d --name cloudserver -p 8000:8000 -e ENDPOINT=new.host.com zenko/cloudserver
Note: In your ``/etc/hosts`` file on Linux, OS X, or Unix with root
permissions, make sure to associate 127.0.0.1 with ``new.host.com``
.. note:: On Unix-like systems (Linux, OS X, etc.) edit /etc/hosts
to associate 127.0.0.1 with new.host.com.
REMOTE_MANAGEMENT_DISABLE
~~~~~~~~~~~~~~~~~~~~~~~~~
CloudServer is a part of `Zenko <https://www.zenko.io/>`__. When you run CloudServer standalone it will still try to connect to Orbit by default (browser-based graphical user interface for Zenko).
Setting this variable to true(1) will default to accessKey1 and verySecretKey1 for credentials and disable the automatic Orbit management:
.. code-block:: shell
$ docker run -d --name cloudserver -p 8000:8000 -e REMOTE_MANAGEMENT_DISABLE=1 zenko/cloudserver
SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
These variables specify authentication credentials for an account named
"CustomAccount".
“CustomAccount”.
You can set credentials for many accounts by editing
``conf/authdata.json`` (see below for further info), but if you just
want to specify one set of your own, you can use these environment
variables.
Set account credentials for multiple accounts by editing conf/authdata.json
(see below for further details). To specify one set for personal use, set these
environment variables:
.. code-block:: shell
docker run -d --name s3server -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey
-e SCALITY_SECRET_ACCESS_KEY=newSecretKey scality/s3server
$ docker run -d --name cloudserver -p 8000:8000 -e SCALITY_ACCESS_KEY_ID=newAccessKey \
-e SCALITY_SECRET_ACCESS_KEY=newSecretKey zenko/cloudserver
Note: Anything in the ``authdata.json`` file will be ignored. Note: The
old ``ACCESS_KEY`` and ``SECRET_KEY`` environment variables are now
deprecated
.. note:: This takes precedence over the contents of the authdata.json
file. The authdata.json file is ignored.
.. note:: The ACCESS_KEY and SECRET_KEY environment variables are
deprecated.
LOG\_LEVEL
~~~~~~~~~~
This variable allows you to change the log level: info, debug or trace.
The default is info. Debug will give you more detailed logs and trace
will give you the most detailed.
This variable changes the log level. There are three levels: info, debug,
and trace. The default is info. Debug provides more detailed logs, and trace
provides the most detailed logs.
.. code-block:: shell
$ docker run -d --name s3server -p 8000:8000 -e LOG_LEVEL=trace scality/s3server
$ docker run -d --name cloudserver -p 8000:8000 -e LOG_LEVEL=trace zenko/cloudserver
SSL
~~~
This variable set to true allows you to run S3 with SSL:
Set true, this variable runs CloudServer with SSL.
**Note1**: You also need to specify the ENDPOINT environment variable.
**Note2**: In your ``/etc/hosts`` file on Linux, OS X, or Unix with root
permissions, make sure to associate 127.0.0.1 with ``<YOUR_ENDPOINT>``
If SSL is set true:
**Warning**: These certs, being self-signed (and the CA being generated
inside the container) will be untrusted by any clients, and could
disappear on a container upgrade. That's ok as long as it's for quick
testing. Also, best security practice for non-testing would be to use an
extra container to do SSL/TLS termination such as haproxy/nginx/stunnel
to limit what an exploit on either component could expose, as well as
certificates in a mounted volume
* The ENDPOINT environment variable must also be specified.
.. code-block:: shell
* On Unix-like systems (Linux, OS X, etc.), 127.0.0.1 must be associated with
<YOUR_ENDPOINT> in /etc/hosts.
$ docker run -d --name s3server -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT>
scality/s3server
.. Warning:: Self-signed certs with a CA generated within the container are
suitable for testing purposes only. Clients cannot trust them, and they may
disappear altogether on a container upgrade. The best security practice for
production environments is to use an extra container, such as
haproxy/nginx/stunnel, for SSL/TLS termination and to pull certificates
from a mounted volume, limiting what an exploit on either component
can expose.
More information about how to use S3 server with SSL
`here <https://s3.scality.com/v1.0/page/scality-with-ssl>`__
.. code:: shell
$ docker run -d --name cloudserver -p 8000:8000 -e SSL=TRUE -e ENDPOINT=<YOUR_ENDPOINT> \
zenko/cloudserver
For more information about using ClousdServer with SSL, see `Using SSL <GETTING_STARTED.html#Using SSL>`__
LISTEN\_ADDR
~~~~~~~~~~~~
This variable instructs the Zenko CloudServer, and its data and metadata
components to listen on the specified address. This allows starting the data
or metadata servers as standalone services, for example.
This variable causes CloudServer and its data and metadata components to
listen on the specified address. This allows starting the data or metadata
servers as standalone services, for example.
.. code-block:: shell
.. code:: shell
$ docker run -d --name s3server-data -p 9991:9991 -e LISTEN_ADDR=0.0.0.0
scality/s3server npm run start_dataserver
docker run -d --name s3server-data -p 9991:9991 -e LISTEN_ADDR=0.0.0.0
scality/s3server yarn run start_dataserver
DATA\_HOST and METADATA\_HOST
@ -172,10 +183,10 @@ These variables configure the data and metadata servers to use,
usually when they are running on another host and only starting the stateless
Zenko CloudServer.
.. code-block:: shell
.. code:: shell
$ docker run -d --name s3server -e DATA_HOST=s3server-data
-e METADATA_HOST=s3server-metadata scality/s3server npm run start_s3server
$ docker run -d --name cloudserver -e DATA_HOST=cloudserver-data \
-e METADATA_HOST=cloudserver-metadata zenko/cloudserver yarn run start_s3server
REDIS\_HOST
~~~~~~~~~~~
@ -183,21 +194,23 @@ REDIS\_HOST
Use this variable to connect to the redis cache server on another host than
localhost.
.. code-block:: shell
.. code:: shell
$ docker run -d --name s3server -p 8000:8000
-e REDIS_HOST=my-redis-server.example.com scality/s3server
$ docker run -d --name cloudserver -p 8000:8000 \
-e REDIS_HOST=my-redis-server.example.com zenko/cloudserver
REDIS\_PORT
~~~~~~~~~~~
Use this variable to connect to the redis cache server on another port than
the default 6379.
Use this variable to connect to the Redis cache server on a port other
than the default 6379.
.. code-block:: shell
.. code:: shell
$ docker run -d --name s3server -p 8000:8000
-e REDIS_PORT=6379 scality/s3server
$ docker run -d --name cloudserver -p 8000:8000 \
-e REDIS_PORT=6379 zenko/cloudserver
.. _tunables-and-setup-tips:
Tunables and Setup Tips
-----------------------
@ -205,61 +218,57 @@ Tunables and Setup Tips
Using Docker Volumes
~~~~~~~~~~~~~~~~~~~~
Zenko CloudServer runs with a file backend by default.
CloudServer runs with a file backend by default, meaning that data is
stored inside the CloudServers Docker container.
So, by default, the data is stored inside your Zenko CloudServer Docker
container.
However, if you want your data and metadata to persist, you **MUST** use
Docker volumes to host your data and metadata outside your Zenko CloudServer
Docker container. Otherwise, the data and metadata will be destroyed
when you erase the container.
For data and metadata to persist, data and metadata must be hosted in Docker
volumes outside the CloudServers Docker container. Otherwise, the data
and metadata are destroyed when the container is erased.
.. code-block:: shell
$ docker run -­v $(pwd)/data:/usr/src/app/localData -­v $(pwd)/metadata:/usr/src/app/localMetadata
-p 8000:8000 ­-d scality/s3server
$ docker run -­v $(pwd)/data:/usr/src/app/localData -­v $(pwd)/metadata:/usr/src/app/localMetadata \
-p 8000:8000 ­-d zenko/cloudserver
This command mounts the host directory, ``./data``, into the container
at ``/usr/src/app/localData`` and the host directory, ``./metadata``, into
the container at ``/usr/src/app/localMetaData``. It can also be any host
mount point, like ``/mnt/data`` and ``/mnt/metadata``.
This command mounts the ./data host directory to the container
at /usr/src/app/localData and the ./metadata host directory to
the container at /usr/src/app/localMetaData.
Adding modifying or deleting accounts or users credentials
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. tip:: These host directories can be mounted to any accessible mount
point, such as /mnt/data and /mnt/metadata, for example.
1. Create locally a customized ``authdata.json`` based on our ``/conf/authdata.json``.
Adding, Modifying, or Deleting Accounts or Credentials
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
2. Use `Docker
Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__
to override the default ``authdata.json`` through a docker file mapping.
1. Create a customized authdata.json file locally based on /conf/authdata.json.
2. Use `Docker volumes <https://docs.docker.com/storage/volumes/>`__
to override the default ``authdata.json`` through a Docker file mapping.
For example:
.. code-block:: shell
$ docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d
scality/s3server
$ docker run -v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json -p 8000:8000 -d \
zenko/cloudserver
Specifying your own host name
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Specifying a Host Name
~~~~~~~~~~~~~~~~~~~~~~
To specify a host name (e.g. s3.domain.name), you can provide your own
`config.json <https://github.com/scality/S3/blob/master/config.json>`__
using `Docker
Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__.
To specify a host name (for example, s3.domain.name), provide your own
`config.json <https://github.com/scality/cloudserver/blob/master/config.json>`__
file using `Docker volumes <https://docs.docker.com/storage/volumes/>`__.
First add a new key-value pair in the restEndpoints section of your
config.json. The key in the key-value pair should be the host name you
would like to add and the value is the default location\_constraint for
this endpoint.
First, add a new key-value pair to the restEndpoints section of your
config.json. Make the key the host name you want, and the value the default
location\_constraint for this endpoint.
For example, ``s3.example.com`` is mapped to ``us-east-1`` which is one
of the ``location_constraints`` listed in your locationConfig.json file
`here <https://github.com/scality/S3/blob/master/locationConfig.json>`__.
More information about location configuration
`here <https://github.com/scality/S3/blob/master/README.md#location-configuration>`__
For more information about location configuration, see:
`GETTING STARTED <GETTING_STARTED.html#location-configuration>`__
.. code:: json
@ -267,31 +276,31 @@ More information about location configuration
"localhost": "file",
"127.0.0.1": "file",
...
"s3.example.com": "us-east-1"
"cloudserver.example.com": "us-east-1"
},
Then, run your Scality S3 Server using `Docker
Volume <https://docs.docker.com/engine/tutorials/dockervolumes/>`__:
Next, run CloudServer using a `Docker volume
<https://docs.docker.com/engine/tutorials/dockervolumes/>`__:
.. code-block:: shell
$ docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d scality/s3server
$ docker run -v $(pwd)/config.json:/usr/src/app/config.json -p 8000:8000 -d zenko/cloudserver
Your local ``config.json`` file will override the default one through a
docker file mapping.
The local ``config.json`` file overrides the default one through a Docker
file mapping.
Running as an unprivileged user
Running as an Unprivileged User
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Zenko CloudServer runs as root by default.
CloudServer runs as root by default.
You can change that by modifing the dockerfile and specifying a user
before the entrypoint.
To change this, modify the dockerfile and specify a user before the
entry point.
The user needs to exist within the container, and own the folder
**/usr/src/app** for Scality Zenko CloudServer to run properly.
The user must exist within the container, and must own the
/usr/src/app directory for CloudServer to run.
For instance, you can modify these lines in the dockerfile:
For example, the following dockerfile lines can be modified:
.. code-block:: shell
@ -305,54 +314,58 @@ For instance, you can modify these lines in the dockerfile:
USER scality
ENTRYPOINT ["/usr/src/app/docker-entrypoint.sh"]
Continuous integration with Docker hosted CloudServer
-----------------------------------------------------
.. _continuous-integration-with-docker-hosted-cloudserver:
When you start the Docker Scality Zenko CloudServer image, you can adjust the
configuration of the Scality Zenko CloudServer instance by passing one or more
environment variables on the docker run command line.
Continuous Integration with a Docker-Hosted CloudServer
-------------------------------------------------------
Sample ways to run it for CI are:
When you start the Docker CloudServer image, you can adjust the
configuration of the CloudServer instance by passing one or more
environment variables on the ``docker run`` command line.
- With custom locations (one in-memory, one hosted on AWS), and custom
credentials mounted:
To run CloudServer for CI with custom locations (one in-memory,
one hosted on AWS), and custom credentials mounted:
.. code-block:: shell
docker run --name CloudServer -p 8000:8000
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
-v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json
-v ~/.aws/credentials:/root/.aws/credentials
-e S3DATA=multiple -e S3BACKEND=mem scality/s3server
$ docker run --name CloudServer -p 8000:8000 \
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
-v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \
-v ~/.aws/credentials:/root/.aws/credentials \
-e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver
- With custom locations, (one in-memory, one hosted on AWS, one file),
and custom credentials set as environment variables
(see `this section <#scality-access-key-id-and-scality-secret-access-key>`__):
To run CloudServer for CI with custom locations, (one in-memory, one
hosted on AWS, and one file), and custom credentials `set as environment
variables <GETTING_STARTED.html#scality-access-key-id-and-scality-secret-access-key>`__):
.. code-block:: shell
docker run --name CloudServer -p 8000:8000
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
-v ~/.aws/credentials:/root/.aws/credentials
-v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata
-e SCALITY_ACCESS_KEY_ID=accessKey1
-e SCALITY_SECRET_ACCESS_KEY=verySecretKey1
-e S3DATA=multiple -e S3BACKEND=mem scality/s3server
$ docker run --name CloudServer -p 8000:8000 \
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
-v ~/.aws/credentials:/root/.aws/credentials \
-v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
-e SCALITY_ACCESS_KEY_ID=accessKey1 \
-e SCALITY_SECRET_ACCESS_KEY=verySecretKey1 \
-e S3DATA=multiple -e S3BACKEND=mem zenko/cloudserver
In production with Docker hosted CloudServer
--------------------------------------------
.. _in-production-w-a-Docker-hosted-cloudserver:
In production, we expect that data will be persistent, that you will use the
multiple backends capabilities of Zenko CloudServer, and that you will have a
custom endpoint for your local storage, and custom credentials for your local
storage:
In Production with a Docker-Hosted CloudServer
----------------------------------------------
Because data must persist in production settings, CloudServer offers
multiple-backend capabilities. This requires a custom endpoint
and custom credentials for local storage.
Customize these with:
.. code-block:: shell
docker run -d --name CloudServer
-v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json
-v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json
-v ~/.aws/credentials:/root/.aws/credentials -e S3DATA=multiple
-e ENDPOINT=custom.endpoint.com
-p 8000:8000 ­-d scality/s3server
$ docker run -d --name CloudServer \
-v $(pwd)/data:/usr/src/app/localData -v $(pwd)/metadata:/usr/src/app/localMetadata \
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
-v $(pwd)/authdata.json:/usr/src/app/conf/authdata.json \
-v ~/.aws/credentials:/root/.aws/credentials -e S3DATA=multiple \
-e ENDPOINT=custom.endpoint.com \
-p 8000:8000 ­-d zenko/cloudserver \

View File

@ -4,212 +4,218 @@ Getting Started
.. figure:: ../res/scality-cloudserver-logo.png
:alt: Zenko CloudServer logo
|CircleCI| |Scality CI|
Dependencies
------------
Building and running the Scality Zenko CloudServer requires node.js 10.x and
yarn v1.17.x. Up-to-date versions can be found at
`Nodesource <https://github.com/nodesource/distributions>`__.
Installation
------------
Dependencies
~~~~~~~~~~~~
1. Clone the source code
Building and running the Scality Zenko CloudServer requires node.js 6.9.5 and
npm v3 . Up-to-date versions can be found at
`Nodesource <https://github.com/nodesource/distributions>`__.
.. code-block:: shell
Clone source code
~~~~~~~~~~~~~~~~~
$ git clone https://github.com/scality/cloudserver.git
2. Go to the cloudserver directory and use yarn to install the js dependencies.
.. code-block:: shell
$ cd cloudserver
$ yarn install
Running CloudServer with a File Backend
---------------------------------------
.. code-block:: shell
git clone https://github.com/scality/S3.git
$ yarn start
Install js dependencies
~~~~~~~~~~~~~~~~~~~~~~~
This starts a Zenko CloudServer on port 8000. Two additional ports, 9990
and 9991, are also open locally for internal transfer of metadata and
data, respectively.
Go to the ./S3 folder,
The default access key is accessKey1. The secret key is verySecretKey1.
By default, metadata files are saved in the localMetadata directory and
data files are saved in the localData directory in the local ./cloudserver
directory. These directories are pre-created within the repository. To
save data or metadata in different locations, you must specify them using
absolute paths. Thus, when starting the server:
.. code-block:: shell
npm install
$ mkdir -m 700 $(pwd)/myFavoriteDataPath
$ mkdir -m 700 $(pwd)/myFavoriteMetadataPath
$ export S3DATAPATH="$(pwd)/myFavoriteDataPath"
$ export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
$ yarn start
Run it with a file backend
--------------------------
Running CloudServer with Multiple Data Backends
-----------------------------------------------
.. code-block:: shell
npm start
$ export S3DATA='multiple'
$ yarn start
This starts an Zenko CloudServer on port 8000. Two additional ports 9990 and
9991 are also open locally for internal transfer of metadata and data,
respectively.
This starts a Zenko CloudServer on port 8000.
The default access key is accessKey1 with a secret key of
verySecretKey1.
The default access key is accessKey1. The secret key is verySecretKey1.
By default the metadata files will be saved in the localMetadata
directory and the data files will be saved in the localData directory
within the ./S3 directory on your machine. These directories have been
pre-created within the repository. If you would like to save the data or
metadata in different locations of your choice, you must specify them
with absolute paths. So, when starting the server:
.. code-block:: shell
mkdir -m 700 $(pwd)/myFavoriteDataPath
mkdir -m 700 $(pwd)/myFavoriteMetadataPath
export S3DATAPATH="$(pwd)/myFavoriteDataPath"
export S3METADATAPATH="$(pwd)/myFavoriteMetadataPath"
npm start
Run it with multiple data backends
----------------------------------
.. code-block:: shell
export S3DATA='multiple'
npm start
This starts an Zenko CloudServer on port 8000. The default access key is
accessKey1 with a secret key of verySecretKey1.
With multiple backends, you have the ability to choose where each object
will be saved by setting the following header with a locationConstraint
on a PUT request:
With multiple backends, you can choose where each object is saved by setting
the following header with a location constraint in a PUT request:
.. code-block:: shell
'x-amz-meta-scal-location-constraint':'myLocationConstraint'
If no header is sent with a PUT object request, the location constraint
of the bucket will determine where the data is saved. If the bucket has
no location constraint, the endpoint of the PUT request will be used to
determine location.
If no header is sent with a PUT object request, the buckets location
constraint determines where the data is saved. If the bucket has no
location constraint, the endpoint of the PUT request determines location.
See the Configuration section below to learn how to set location
constraints.
See the Configuration_ section to set location constraints.
Run it with an in-memory backend
--------------------------------
Run CloudServer with an In-Memory Backend
-----------------------------------------
.. code-block:: shell
npm run mem_backend
$ yarn run mem_backend
This starts an Zenko CloudServer on port 8000. The default access key is
accessKey1 with a secret key of verySecretKey1.
This starts a Zenko CloudServer on port 8000.
Run it for continuous integration testing or in production with Docker
----------------------------------------------------------------------
The default access key is accessKey1. The secret key is verySecretKey1.
`DOCKER <../DOCKER/>`__
Run CloudServer with Vault User Management
------------------------------------------
.. code:: shell
export S3VAULT=vault
yarn start
Note: Vault is proprietary and must be accessed separately.
This starts a Zenko CloudServer using Vault for user management.
Run CloudServer for Continuous Integration Testing or in Production with Docker
-------------------------------------------------------------------------------
Run Cloudserver with `DOCKER <DOCKER.html>`__
Testing
-------
~~~~~~~
You can run the unit tests with the following command:
Run unit tests with the command:
.. code-block:: shell
npm test
$ yarn test
You can run the multiple backend unit tests with:
Run multiple-backend unit tests with:
.. code-block:: shell
CI=true S3DATA=multiple npm start
npm run multiple_backend_test
$ CI=true S3DATA=multiple yarn start
$ yarn run multiple_backend_test
You can run the linter with:
Run the linter with:
.. code-block:: shell
npm run lint
$ yarn run lint
Running functional tests locally:
Running Functional Tests Locally
--------------------------------
For the AWS backend and Azure backend tests to pass locally,
you must modify tests/locationConfigTests.json so that awsbackend
specifies a bucketname of a bucket you have access to based on
your credentials profile and modify "azurebackend" with details
for your Azure account.
To pass AWS and Azure backend tests locally, modify
tests/locationConfig/locationConfigTests.json so that ``awsbackend``
specifies the bucketname of a bucket you have access to based on your
credentials, and modify ``azurebackend`` with details for your Azure account.
The test suite requires additional tools, **s3cmd** and **Redis**
installed in the environment the tests are running in.
- Install `s3cmd <http://s3tools.org/download>`__
- Install `redis <https://redis.io/download>`__ and start Redis.
- Add localCache section to your ``config.json``:
1. Install `s3cmd <http://s3tools.org/download>`__
::
2. Install `redis <https://redis.io/download>`__ and start Redis.
"localCache": {
3. Add localCache section to ``config.json``:
.. code:: json
"localCache": {
"host": REDIS_HOST,
"port": REDIS_PORT
}
}
where ``REDIS_HOST`` is your Redis instance IP address (``"127.0.0.1"``
if your Redis is running locally) and ``REDIS_PORT`` is your Redis
instance port (``6379`` by default)
where ``REDIS_HOST`` is the Redis instance IP address (``"127.0.0.1"``
if Redis is running locally) and ``REDIS_PORT`` is the Redis instance
port (``6379`` by default)
- Add the following to the etc/hosts file on your machine:
4. Add the following to the local etc/hosts file:
.. code-block:: shell
.. code-block:: shell
127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com
127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com
- Start the Zenko CloudServer in memory and run the functional tests:
5. Start Zenko CloudServer in memory and run the functional tests:
.. code-block:: shell
.. code-block:: shell
CI=true npm run mem_backend
CI=true npm run ft_test
$ CI=true yarn run mem_backend
$ CI=true yarn run ft_test
.. _Configuration:
Configuration
-------------
There are three configuration files for your Scality Zenko CloudServer:
There are three configuration files for Zenko CloudServer:
1. ``conf/authdata.json``, described above for authentication
* ``conf/authdata.json``, for authentication.
2. ``locationConfig.json``, to set up configuration options for
* ``locationConfig.json``, to configure where data is saved.
where data will be saved
* ``config.json``, for general configuration options.
3. ``config.json``, for general configuration options
.. _location-configuration:
Location Configuration
~~~~~~~~~~~~~~~~~~~~~~
You must specify at least one locationConstraint in your
locationConfig.json (or leave as pre-configured).
You must specify at least one locationConstraint in locationConfig.json
(or leave it as pre-configured).
You must also specify 'us-east-1' as a locationConstraint so if you only
define one locationConstraint, that would be it. If you put a bucket to
an unknown endpoint and do not specify a locationConstraint in the put
bucket call, us-east-1 will be used.
You must also specify 'us-east-1' as a locationConstraint. If you put a
bucket to an unknown endpoint and do not specify a locationConstraint in
the PUT bucket call, us-east-1 is used.
For instance, the following locationConstraint will save data sent to
For instance, the following locationConstraint saves data sent to
``myLocationConstraint`` to the file backend:
.. code:: json
"myLocationConstraint": {
"type": "file",
"legacyAwsBehavior": false,
"details": {}
},
"myLocationConstraint": {
"type": "file",
"legacyAwsBehavior": false,
"details": {}
},
Each locationConstraint must include the ``type``,
``legacyAwsBehavior``, and ``details`` keys. ``type`` indicates which
backend will be used for that region. Currently, mem, file, and scality
are the supported backends. ``legacyAwsBehavior`` indicates whether the
region will have the same behavior as the AWS S3 'us-east-1' region. If
the locationConstraint type is scality, ``details`` should contain
connector information for sproxyd. If the locationConstraint type is mem
or file, ``details`` should be empty.
Each locationConstraint must include the ``type``, ``legacyAwsBehavior``,
and ``details`` keys. ``type`` indicates which backend is used for that
region. Supported backends are mem, file, and scality.``legacyAwsBehavior``
indicates whether the region behaves the same as the AWS S3 'us-east-1'
region. If the locationConstraint type is ``scality``, ``details`` must
contain connector information for sproxyd. If the locationConstraint type
is ``mem`` or ``file``, ``details`` must be empty.
Once you have your locationConstraints in your locationConfig.json, you
can specify a default locationConstraint for each of your endpoints.
Once locationConstraints is set in locationConfig.json, specify a default
locationConstraint for each endpoint.
For instance, the following sets the ``localhost`` endpoint to the
``myLocationConstraint`` data backend defined above:
@ -220,26 +226,24 @@ For instance, the following sets the ``localhost`` endpoint to the
"localhost": "myLocationConstraint"
},
If you would like to use an endpoint other than localhost for your
Scality Zenko CloudServer, that endpoint MUST be listed in your
``restEndpoints``. Otherwise if your server is running with a:
To use an endpoint other than localhost for Zenko CloudServer, the endpoint
must be listed in ``restEndpoints``. Otherwise, if the server is running
with a:
- **file backend**: your default location constraint will be ``file``
- **memory backend**: your default location constraint will be ``mem``
* **file backend**: The default location constraint is ``file``
* **memory backend**: The default location constraint is ``mem``
Endpoints
~~~~~~~~~
Note that our Zenko CloudServer supports both:
The Zenko CloudServer supports endpoints that are rendered in either:
- path-style: http://myhostname.com/mybucket
- hosted-style: http://mybucket.myhostname.com
* path style: http://myhostname.com/mybucket or
* hosted style: http://mybucket.myhostname.com
However, hosted-style requests will not hit the server if you are using
an ip address for your host. So, make sure you are using path-style
requests in that case. For instance, if you are using the AWS SDK for
JavaScript, you would instantiate your client like this:
However, if an IP address is specified for the host, hosted-style requests
cannot reach the server. Use path-style requests in that case. For example,
if you are using the AWS SDK for JavaScript, instantiate your client like this:
.. code:: js
@ -248,87 +252,99 @@ JavaScript, you would instantiate your client like this:
s3ForcePathStyle: true,
});
Setting your own access key and secret key pairs
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Setting Your Own Access and Secret Key Pairs
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
You can set credentials for many accounts by editing
``conf/authdata.json`` but if you want to specify one set of your own
credentials, you can use ``SCALITY_ACCESS_KEY_ID`` and
``SCALITY_SECRET_ACCESS_KEY`` environment variables.
Credentials can be set for many accounts by editing ``conf/authdata.json``,
but use the ``SCALITY_ACCESS_KEY_ID`` and ``SCALITY_SECRET_ACCESS_KEY``
environment variables to specify your own credentials.
_`scality-access-key-id-and-scality-secret-access-key`
SCALITY\_ACCESS\_KEY\_ID and SCALITY\_SECRET\_ACCESS\_KEY
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
These variables specify authentication credentials for an account named
"CustomAccount".
“CustomAccount”.
Note: Anything in the ``authdata.json`` file will be ignored.
.. note:: Anything in the ``authdata.json`` file is ignored.
.. code-block:: shell
SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey npm start
$ SCALITY_ACCESS_KEY_ID=newAccessKey SCALITY_SECRET_ACCESS_KEY=newSecretKey yarn start
.. _Using_SSL:
Scality with SSL
~~~~~~~~~~~~~~~~~~~~~~
Using SSL
~~~~~~~~~
If you wish to use https with your local Zenko CloudServer, you need to set up
SSL certificates. Here is a simple guide of how to do it.
To use https with your local CloudServer, you must set up
SSL certificates.
Deploying Zenko CloudServer
^^^^^^^^^^^^^^^^^^^^^^^^^^^
1. Deploy CloudServer using `our DockerHub page
<https://hub.docker.com/r/zenko/cloudserver/>`__ (run it with a file
backend).
First, you need to deploy **Zenko CloudServer**. This can be done very easily
via `our **DockerHub**
page <https://hub.docker.com/r/scality/s3server/>`__ (you want to run it
with a file backend).
.. Note:: If Docker is not installed locally, follow the
`instructions to install it for your distribution
<https://docs.docker.com/engine/installation/>`__
*Note:* *- If you don't have docker installed on your machine, here
are the `instructions to install it for your
distribution <https://docs.docker.com/engine/installation/>`__*
2. Update the CloudServer containers config
Updating your Zenko CloudServer container's config
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Add your certificates to your container. To do this,
#. exec inside the CloudServer container.
You're going to add your certificates to your container. In order to do
so, you need to exec inside your Zenko CloudServer container. Run a
``$> docker ps`` and find your container's id (the corresponding image
name should be ``scality/s3server``. Copy the corresponding container id
(here we'll use ``894aee038c5e``, and run:
#. Run ``$> docker ps`` to find the containers ID (the corresponding
image name is ``scality/cloudserver``.
#. Copy the corresponding container ID (``894aee038c5e`` in the present
example), and run:
.. code-block:: shell
.. code-block:: shell
$> docker exec -it 894aee038c5e bash
$> docker exec -it 894aee038c5e bash
You're now inside your container, using an interactive terminal :)
This puts you inside your container, using an interactive terminal.
Generate SSL key and certificates
**********************************
3. Generate the SSL key and certificates. The paths where the different
files are stored are defined after the ``-out`` option in each of the
following commands.
There are 5 steps to this generation. The paths where the different
files are stored are defined after the ``-out`` option in each command
#. Generate a private key for your certificate signing request (CSR):
.. code-block:: shell
.. code-block:: shell
# Generate a private key for your CSR
$> openssl genrsa -out ca.key 2048
# Generate a self signed certificate for your local Certificate Authority
$> openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 -subj "/C=US/ST=Country/L=City/O=Organization/CN=scality.test"
$> openssl genrsa -out ca.key 2048
# Generate a key for Zenko CloudServer
$> openssl genrsa -out test.key 2048
# Generate a Certificate Signing Request for S3 Server
$> openssl req -new -key test.key -out test.csr -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.scality.test"
# Generate a local-CA-signed certificate for S3 Server
$> openssl x509 -req -in test.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out test.crt -days 99999 -sha256
#. Generate a self-signed certificate for your local certificate
authority (CA):
Update Zenko CloudServer ``config.json``
****************************************
.. code:: shell
Add a ``certFilePaths`` section to ``./config.json`` with the
appropriate paths:
$> openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 -subj "/C=US/ST=Country/L=City/O=Organization/CN=scality.test"
.. code:: json
#. Generate a key for the CloudServer:
.. code:: shell
$> openssl genrsa -out test.key 2048
#. Generate a CSR for CloudServer:
.. code:: shell
$> openssl req -new -key test.key -out test.csr -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.scality.test"
#. Generate a certificate for CloudServer signed by the local CA:
.. code:: shell
$> openssl x509 -req -in test.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out test.crt -days 99999 -sha256
4. Update Zenko CloudServer ``config.json``. Add a ``certFilePaths``
section to ``./config.json`` with appropriate paths:
.. code:: json
"certFilePaths": {
"key": "./test.key",
@ -336,42 +352,36 @@ appropriate paths:
"ca": "./ca.crt"
}
Run your container with the new config
**************************************
5. Run your container with the new config.
First, you need to exit your container. Simply run ``$> exit``. Then,
you need to restart your container. Normally, a simple
``$> docker restart s3server`` should do the trick.
#. Exit the container by running ``$> exit``.
Update your host config
^^^^^^^^^^^^^^^^^^^^^^^
#. Restart the container with ``$> docker restart cloudserver``.
Associates local IP addresses with hostname
*******************************************
6. Update the host configuration by adding s3.scality.test
to /etc/hosts:
In your ``/etc/hosts`` file on Linux, OS X, or Unix (with root
permissions), edit the line of localhost so it looks like this:
.. code:: bash
::
127.0.0.1 localhost s3.scality.test
127.0.0.1 localhost s3.scality.test
7. Copy the local certificate authority (ca.crt in step 4) from your
container. Choose the path to save this file to (in the present
example, ``/root/ca.crt``), and run:
Copy the local certificate authority from your container
********************************************************
.. code:: shell
In the above commands, it's the file named ``ca.crt``. Choose the path
you want to save this file at (here we chose ``/root/ca.crt``), and run
something like:
$> docker cp 894aee038c5e:/usr/src/app/ca.crt /root/ca.crt
.. code-block:: shell
.. note:: Your container ID will be different, and your path to
ca.crt may be different.
$> docker cp 894aee038c5e:/usr/src/app/ca.crt /root/ca.crt
Test the Config
^^^^^^^^^^^^^^^
Test your config
^^^^^^^^^^^^^^^^^
If aws-sdk is not installed, run ``$> yarn install aws-sdk``.
If you do not have aws-sdk installed, run ``$> npm install aws-sdk``. In
a ``test.js`` file, paste the following script:
Paste the following script into a file named "test.js":
.. code:: js
@ -411,8 +421,13 @@ a ``test.js`` file, paste the following script:
});
});
Now run that script with ``$> nodejs test.js``. If all goes well, it
should output ``SSL is cool!``. Enjoy that added security!
Now run this script with:
.. code::
$> nodejs test.js
On success, the script outputs ``SSL is cool!``.
.. |CircleCI| image:: https://circleci.com/gh/scality/S3.svg?style=svg

69
docs/GET_BUCKET_V2.md Normal file
View File

@ -0,0 +1,69 @@
# Get Bucket Version 2 Documentation
## Description
This feature implements version 2 of the GET Bucket (List Objects)
operation, following AWS specifications
(see https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html).
## Requirements
The user must have READ access to the bucket.
## Design
### Request
The `delimiter`, `encoding-type`, `max-keys`, and `prefix` request parameters
from GET Bucket v1 remain unchanged.
In order to specify v2, the parameter `list-type` must be included and
set to `2`.
The `marker` v1 parameter's functionality has been split in two and replaced by
`start-after` and `continuation-token` in v2. The `start-after` parameter is
a specific object key after which the API will return key names. It is only
valid in the first GET request. If both the `start-after` and
`continuation-token` parameters are included in a request, the API will
ignore the `start-after` parameter in favor of the `continuation-token`.
If the GET Bucket v2 response is truncated, a `NextContinuationToken` will
also be included. To list the next set of objects, the `NextContinuationToken`
can be used as the `continuation-token` in the next request. The continuation
token is an obfuscated string of 57 characters that CloudServer understands and
interprets.
By default, the v2 response does not include object owner information. To
include owner information like the default v1 response, use the `fetch-owner`
request parameter set to `true`.
### Response
The GET Bucket v1 and v2 responses are largely the same, with only a few changes.
The `NextMarker` v1 parameter has been replaced by the
`NextContinuationToken`. The `NextContinuationToken` is included with any
truncated response, even if no delimiter is sent in the request. Its value is an
obfuscated string that can be passed at the `continuation-token` in the next
request, which will be interpreted by CloudServer.
The `KeyCounter` parameter is returned in every response. Its value is the
number of keys included in the response. It is always less than or equal to
the `MaxKeys` value.
If the `start-after` or `continuation-token` parameter is used in the
request, it is also included in the response.
By default, the v2 response does not include object owner information, unlike
the v1 response. See the `Request` section for including it.
### Continuation Token
An example continuation token:
```
NextContinuationToken: '1bunC4s+crlZNAAbKUGBLyajJUQKp22TOdUR6/01snxD2cZtjJD0ugA=='
```
In order to generate a comparable token, CloudServer uses base64 encoding to
obfuscate the key name of the next object to be listed.
Encoded continuation tokens are similarly decoded in order for listing to
continue from the correct object.
## Performing Get Bucket V2 Operation
When performing the GET Bucket V2 operation, if the request is built manually,
the parameter `list-type` must be included and set to `2`.
Using the AWS cli client, the command becomes `list-objects-v2`.

View File

@ -4,479 +4,415 @@ Integrations
High Availability
=================
`Docker swarm <https://docs.docker.com/engine/swarm/>`__ is a
clustering tool developped by Docker and ready to use with its
containers. It allows to start a service, which we define and use as a
means to ensure Zenko CloudServer's continuous availability to the end user.
Indeed, a swarm defines a manager and n workers among n+1 servers. We
will do a basic setup in this tutorial, with just 3 servers, which
already provides a strong service resiliency, whilst remaining easy to
do as an individual. We will use NFS through docker to share data and
`Docker Swarm <https://docs.docker.com/engine/swarm/>`__ is a clustering tool
developed by Docker for use with its containers. It can be used to start
services, which we define to ensure CloudServer's continuous availability to
end users. A swarm defines a manager and *n* workers among *n* + 1 servers.
This tutorial shows how to perform a basic setup with three servers, which
provides strong service resiliency, while remaining easy to use and
maintain. We will use NFS through Docker to share data and
metadata between the different servers.
You will see that the steps of this tutorial are defined as **On
Server**, **On Clients**, **On All Machines**. This refers respectively
to NFS Server, NFS Clients, or NFS Server and Clients. In our example,
the IP of the Server will be **10.200.15.113**, while the IPs of the
Clients will be **10.200.15.96 and 10.200.15.97**
Sections are labeled **On Server**, **On Clients**, or
**On All Machines**, referring respectively to NFS server, NFS clients, or
NFS server and clients. In the present example, the servers IP address is
**10.200.15.113** and the client IP addresses are **10.200.15.96** and
**10.200.15.97**
Installing docker
-----------------
1. Install Docker (on All Machines)
Any version from docker 1.12.6 onwards should work; we used Docker
17.03.0-ce for this tutorial.
Docker 17.03.0-ce is used for this tutorial. Docker 1.12.6 and later will
likely work, but is not tested.
On All Machines
~~~~~~~~~~~~~~~
* On Ubuntu 14.04
Install Docker CE for Ubuntu as `documented at Docker
<https://docs.docker.com/install/linux/docker-ce/ubuntu/>`__.
Install the aufs dependency as recommended by Docker. The required
commands are:
On Ubuntu 14.04
^^^^^^^^^^^^^^^
.. code:: sh
The docker website has `solid
documentation <https://docs.docker.com/engine/installation/linux/ubuntu/>`__.
We have chosen to install the aufs dependency, as recommended by Docker.
Here are the required commands:
$> sudo apt-get update
$> sudo apt-get install linux-image-extra-$(uname -r) linux-image-extra-virtual
$> sudo apt-get install apt-transport-https ca-certificates curl software-properties-common
$> curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
$> sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
$> sudo apt-get update
$> sudo apt-get install docker-ce
* On CentOS 7
Install Docker CE as `documented at Docker
<https://docs.docker.com/install/linux/docker-ce/centos/>`__.
The required commands are:
.. code:: sh
$> sudo yum install -y yum-utils
$> sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
$> sudo yum makecache fast
$> sudo yum install docker-ce
$> sudo systemctl start docker
2. Install NFS on Client(s)
NFS clients mount Docker volumes over the NFS servers shared folders.
If the NFS commons are installed, manual mounts are no longer needed.
* On Ubuntu 14.04
Install the NFS commons with apt-get:
.. code:: sh
$> sudo apt-get install nfs-common
* On CentOS 7
Install the NFS utils; then start required services:
.. code:: sh
$> yum install nfs-utils
$> sudo systemctl enable rpcbind
$> sudo systemctl enable nfs-server
$> sudo systemctl enable nfs-lock
$> sudo systemctl enable nfs-idmap
$> sudo systemctl start rpcbind
$> sudo systemctl start nfs-server
$> sudo systemctl start nfs-lock
$> sudo systemctl start nfs-idmap
3. Install NFS (on Server)
The NFS server hosts the data and metadata. The package(s) to install on it
differs from the package installed on the clients.
* On Ubuntu 14.04
Install the NFS server-specific package and the NFS commons:
.. code:: sh
$> sudo apt-get install nfs-kernel-server nfs-common
* On CentOS 7
Install the NFS utils and start the required services:
.. code:: sh
$> yum install nfs-utils
$> sudo systemctl enable rpcbind
$> sudo systemctl enable nfs-server
$> sudo systemctl enable nfs-lock
$> sudo systemctl enable nfs-idmap
$> sudo systemctl start rpcbind
$> sudo systemctl start nfs-server
$> sudo systemctl start nfs-lock
$> sudo systemctl start nfs-idmap
For both distributions:
#. Choose where shared data and metadata from the local
`CloudServer <http://www.zenko.io/cloudserver/>`__ shall be stored (The
present example uses /var/nfs/data and /var/nfs/metadata). Set permissions
for these folders for
sharing over NFS:
.. code:: sh
$> mkdir -p /var/nfs/data /var/nfs/metadata
$> chmod -R 777 /var/nfs/
#. The /etc/exports file configures network permissions and r-w-x permissions
for NFS access. Edit /etc/exports, adding the following lines:
.. code:: sh
/var/nfs/data 10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
/var/nfs/metadata 10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
Ubuntu applies the no\_subtree\_check option by default, so both
folders are declared with the same permissions, even though theyre in
the same tree.
#. Export this new NFS table:
.. code:: sh
$> sudo exportfs -a
#. Edit the ``MountFlags`` option in the Docker config in
/lib/systemd/system/docker.service to enable NFS mount from Docker volumes
on other machines:
.. code:: sh
MountFlags=shared
#. Restart the NFS server and Docker daemons to apply these changes.
* On Ubuntu 14.04
.. code:: sh
$> sudo service nfs-kernel-server restart
$> sudo service docker restart
* On CentOS 7
.. code:: sh
$> sudo systemctl restart nfs-server
$> sudo systemctl daemon-reload
$> sudo systemctl restart docker
4. Set Up a Docker Swarm
* On all machines and distributions:
Set up the Docker volumes to be mounted to the NFS server for CloudServers
data and metadata storage. The following commands must be replicated on all
machines:
.. code:: sh
$> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/data --name data
$> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/metadata --name metadata
There is no need to ``docker exec`` these volumes to mount them: the
Docker Swarm manager does this when the Docker service is started.
* On a server:
To start a Docker service on a Docker Swarm cluster, initialize the cluster
(that is, define a manager), prompt workers/nodes to join in, and then start
the service.
Initialize the swarm cluster, and review its response:
.. code:: sh
$> docker swarm init --advertise-addr 10.200.15.113
Swarm initialized: current node (db2aqfu3bzfzzs9b1kfeaglmq) is now a manager.
To add a worker to this swarm, run the following command:
docker swarm join \
--token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka \
10.200.15.113:2377
To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
* On clients:
Copy and paste the command provided by your Docker Swarm init. A successful
request/response will resemble:
.. code:: sh
$> docker swarm join --token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka 10.200.15.113:2377
This node joined a swarm as a worker.
Set Up Docker Swarm on Clients on a Server
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Start the service on the Swarm cluster.
.. code:: sh
$> sudo apt-get update
$> sudo apt-get install linux-image-extra-$(uname -r) linux-image-extra-virtual
$> sudo apt-get install apt-transport-https ca-certificates curl software-properties-common
$> curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
$> sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
$> sudo apt-get update
$> sudo apt-get install docker-ce
$> docker service create --name s3 --replicas 1 --mount type=volume,source=data,target=/usr/src/app/localData --mount type=volume,source=metadata,target=/usr/src/app/localMetadata -p 8000:8000 scality/cloudserver
On CentOS 7
^^^^^^^^^^^
The docker website has `solid
documentation <https://docs.docker.com/engine/installation/linux/centos/>`__.
Here are the required commands:
.. code:: sh
$> sudo yum install -y yum-utils
$> sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
$> sudo yum makecache fast
$> sudo yum install docker-ce
$> sudo systemctl start docker
Configure NFS
-------------
On Clients
~~~~~~~~~~
Your NFS Clients will mount Docker volumes over your NFS Server's shared
folders. Hence, you don't have to mount anything manually, you just have
to install the NFS commons:
On Ubuntu 14.04
^^^^^^^^^^^^^^^
Simply install the NFS commons:
.. code:: sh
$> sudo apt-get install nfs-common
On CentOS 7
^^^^^^^^^^^
Install the NFS utils, and then start the required services:
.. code:: sh
$> yum install nfs-utils
$> sudo systemctl enable rpcbind
$> sudo systemctl enable nfs-server
$> sudo systemctl enable nfs-lock
$> sudo systemctl enable nfs-idmap
$> sudo systemctl start rpcbind
$> sudo systemctl start nfs-server
$> sudo systemctl start nfs-lock
$> sudo systemctl start nfs-idmap
On Server
~~~~~~~~~
Your NFS Server will be the machine to physically host the data and
metadata. The package(s) we will install on it is slightly different
from the one we installed on the clients.
On Ubuntu 14.04
^^^^^^^^^^^^^^^
Install the NFS server specific package and the NFS commons:
.. code:: sh
$> sudo apt-get install nfs-kernel-server nfs-common
On CentOS 7
^^^^^^^^^^^
Same steps as with the client: install the NFS utils and start the
required services:
.. code:: sh
$> yum install nfs-utils
$> sudo systemctl enable rpcbind
$> sudo systemctl enable nfs-server
$> sudo systemctl enable nfs-lock
$> sudo systemctl enable nfs-idmap
$> sudo systemctl start rpcbind
$> sudo systemctl start nfs-server
$> sudo systemctl start nfs-lock
$> sudo systemctl start nfs-idmap
On Ubuntu 14.04 and CentOS 7
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Choose where your shared data and metadata from your local `Zenko CloudServer
<http://www.zenko.io/cloudserver/>`__ will be stored.
We chose to go with /var/nfs/data and /var/nfs/metadata. You also need
to set proper sharing permissions for these folders as they'll be shared
over NFS:
.. code:: sh
$> mkdir -p /var/nfs/data /var/nfs/metadata
$> chmod -R 777 /var/nfs/
Now you need to update your **/etc/exports** file. This is the file that
configures network permissions and rwx permissions for NFS access. By
default, Ubuntu applies the no\_subtree\_check option, so we declared
both folders with the same permissions, even though they're in the same
tree:
.. code:: sh
$> sudo vim /etc/exports
In this file, add the following lines:
.. code:: sh
/var/nfs/data 10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
/var/nfs/metadata 10.200.15.96(rw,sync,no_root_squash) 10.200.15.97(rw,sync,no_root_squash)
Export this new NFS table:
.. code:: sh
$> sudo exportfs -a
Eventually, you need to allow for NFS mount from Docker volumes on other
machines. You need to change the Docker config in
**/lib/systemd/system/docker.service**:
.. code:: sh
$> sudo vim /lib/systemd/system/docker.service
In this file, change the **MountFlags** option:
.. code:: sh
MountFlags=shared
Now you just need to restart the NFS server and docker daemons so your
changes apply.
On Ubuntu 14.04
^^^^^^^^^^^^^^^
Restart your NFS Server and docker services:
.. code:: sh
$> sudo service nfs-kernel-server restart
$> sudo service docker restart
On CentOS 7
^^^^^^^^^^^
Restart your NFS Server and docker daemons:
.. code:: sh
$> sudo systemctl restart nfs-server
$> sudo systemctl daemon-reload
$> sudo systemctl restart docker
Set up your Docker Swarm service
--------------------------------
On All Machines
~~~~~~~~~~~~~~~
On Ubuntu 14.04 and CentOS 7
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
We will now set up the Docker volumes that will be mounted to the NFS
Server and serve as data and metadata storage for Zenko CloudServer. These two
commands have to be replicated on all machines:
.. code:: sh
$> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/data --name data
$> docker volume create --driver local --opt type=nfs --opt o=addr=10.200.15.113,rw --opt device=:/var/nfs/metadata --name metadata
There is no need to ""docker exec" these volumes to mount them: the
Docker Swarm manager will do it when the Docker service will be started.
On Server
^^^^^^^^^
To start a Docker service on a Docker Swarm cluster, you first have to
initialize that cluster (i.e.: define a manager), then have the
workers/nodes join in, and then start the service. Initialize the swarm
cluster, and look at the response:
.. code:: sh
$> docker swarm init --advertise-addr 10.200.15.113
Swarm initialized: current node (db2aqfu3bzfzzs9b1kfeaglmq) is now a manager.
To add a worker to this swarm, run the following command:
docker swarm join \
--token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka \
10.200.15.113:2377
To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
On Clients
^^^^^^^^^^
Simply copy/paste the command provided by your docker swarm init. When
all goes well, you'll get something like this:
.. code:: sh
$> docker swarm join --token SWMTKN-1-5yxxencrdoelr7mpltljn325uz4v6fe1gojl14lzceij3nujzu-2vfs9u6ipgcq35r90xws3stka 10.200.15.113:2377
This node joined a swarm as a worker.
On Server
^^^^^^^^^
Start the service on your swarm cluster!
.. code:: sh
$> docker service create --name s3 --replicas 1 --mount type=volume,source=data,target=/usr/src/app/localData --mount type=volume,source=metadata,target=/usr/src/app/localMetadata -p 8000:8000 scality/s3server
If you run a docker service ls, you should have the following output:
On a successful installation, ``docker service ls`` returns the following
output:
.. code:: sh
$> docker service ls
ID NAME MODE REPLICAS IMAGE
ocmggza412ft s3 replicated 1/1 scality/s3server:latest
ocmggza412ft s3 replicated 1/1 scality/cloudserver:latest
If your service won't start, consider disabling apparmor/SELinux.
If the service does not start, consider disabling apparmor/SELinux.
Testing your High Availability S3Server
---------------------------------------
Testing the High-Availability CloudServer
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
On All Machines
~~~~~~~~~~~~~~~
On Ubuntu 14.04 and CentOS 7
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Try to find out where your Scality Zenko CloudServer is actually running using
the **docker ps** command. It can be on any node of the swarm cluster,
manager or worker. When you find it, you can kill it, with **docker stop
<container id>** and you'll see it respawn on a different node of the
swarm cluster. Now you see, if one of your servers falls, or if docker
stops unexpectedly, your end user will still be able to access your
local Zenko CloudServer.
On all machines (client/server) and distributions (Ubuntu and CentOS),
determine where CloudServer is running using ``docker ps``. CloudServer can
operate on any node of the Swarm cluster, manager or worker. When you find
it, you can kill it with ``docker stop <container id>``. It will respawn
on a different node. Now, if one server falls, or if Docker stops
unexpectedly, the end user will still be able to access your the local CloudServer.
Troubleshooting
---------------
~~~~~~~~~~~~~~~
To troubleshoot the service you can run:
To troubleshoot the service, run:
.. code:: sh
$> docker service ps s3docker service ps s3
ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR
0ar81cw4lvv8chafm8pw48wbc s3.1 scality/s3server localhost.localdomain.localdomain Running Running 7 days ago
cvmf3j3bz8w6r4h0lf3pxo6eu \_ s3.1 scality/s3server localhost.localdomain.localdomain Shutdown Failed 7 days ago "task: non-zero exit (137)"
0ar81cw4lvv8chafm8pw48wbc s3.1 scality/cloudserver localhost.localdomain.localdomain Running Running 7 days ago
cvmf3j3bz8w6r4h0lf3pxo6eu \_ s3.1 scality/cloudserver localhost.localdomain.localdomain Shutdown Failed 7 days ago "task: non-zero exit (137)"
If the error is truncated it is possible to have a more detailed view of
the error by inspecting the docker task ID:
If the error is truncated, view the error in detail by inspecting the
Docker task ID:
.. code:: sh
$> docker inspect cvmf3j3bz8w6r4h0lf3pxo6eu
Off you go!
-----------
Let us know what you use this functionality for, and if you'd like any
specific developments around it. Or, even better: come and contribute to
our `Github repository <https://github.com/scality/s3/>`__! We look
forward to meeting you!
~~~~~~~~~~~
Let us know how you use this and if you'd like any specific developments
around it. Even better: come and contribute to our `Github repository
<https://github.com/scality/s3/>`__! We look forward to meeting you!
S3FS
====
Export your buckets as a filesystem with s3fs on top of Zenko CloudServer
You can export buckets as a filesystem with s3fs on CloudServer.
`s3fs <https://github.com/s3fs-fuse/s3fs-fuse>`__ is an open source
tool that allows you to mount an S3 bucket on a filesystem-like backend.
It is available both on Debian and RedHat distributions. For this
tutorial, we used an Ubuntu 14.04 host to deploy and use s3fs over
Scality's Zenko CloudServer.
tool, available both on Debian and RedHat distributions, that enables
you to mount an S3 bucket on a filesystem-like backend. This tutorial uses
an Ubuntu 14.04 host to deploy and use s3fs over CloudServer.
Deploying Zenko CloudServer with SSL
------------------------------------
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
First, you need to deploy **Zenko CloudServer**. This can be done very easily
via `our DockerHub
page <https://hub.docker.com/r/scality/s3server/>`__ (you want to run it
with a file backend).
First, deploy CloudServer with a file backend using `our DockerHub page
<https://hub.docker.com/r/zenko/cloudserver>`__.
*Note:* *- If you don't have docker installed on your machine, here
are the `instructions to install it for your
distribution <https://docs.docker.com/engine/installation/>`__*
.. note::
You also necessarily have to set up SSL with Zenko CloudServer to use s3fs. We
have a nice
`tutorial <https://s3.scality.com/v1.0/page/scality-with-ssl>`__ to help
you do it.
If Docker is not installed on your machine, follow
`these instructions <https://docs.docker.com/engine/installation/>`__
to install it for your distribution.
s3fs setup
----------
You must also set up SSL with CloudServer to use s3fs. See `Using SSL
<./GETTING_STARTED#Using_SSL>`__ for instructions.
s3fs Setup
~~~~~~~~~~
Installing s3fs
~~~~~~~~~~~~~~~
---------------
s3fs has quite a few dependencies. As explained in their
`README <https://github.com/s3fs-fuse/s3fs-fuse/blob/master/README.md#installation>`__,
the following commands should install everything for Ubuntu 14.04:
Follow the instructions in the s3fs `README
<https://github.com/s3fs-fuse/s3fs-fuse/blob/master/README.md#installation-from-pre-built-packages>`__,
Check that s3fs is properly installed. A version check should return
a response resembling:
.. code:: sh
$> sudo apt-get install automake autotools-dev g++ git libcurl4-gnutls-dev
$> sudo apt-get install libfuse-dev libssl-dev libxml2-dev make pkg-config
Now you want to install s3fs per se:
.. code:: sh
$> git clone https://github.com/s3fs-fuse/s3fs-fuse.git
$> cd s3fs-fuse
$> ./autogen.sh
$> ./configure
$> make
$> sudo make install
Check that s3fs is properly installed by checking its version. it should
answer as below:
.. code:: sh
$> s3fs --version
$> s3fs --version
Amazon Simple Storage Service File System V1.80(commit:d40da2c) with OpenSSL
Copyright (C) 2010 Randy Rizun <rrizun@gmail.com>
License GPL2: GNU GPL version 2 <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Configuring s3fs
~~~~~~~~~~~~~~~~
----------------
s3fs expects you to provide it with a password file. Our file is
``/etc/passwd-s3fs``. The structure for this file is
``ACCESSKEYID:SECRETKEYID``, so, for S3Server, you can run:
``ACCESSKEYID:SECRETKEYID``, so, for CloudServer, you can run:
.. code:: sh
$> echo 'accessKey1:verySecretKey1' > /etc/passwd-s3fs
$> chmod 600 /etc/passwd-s3fs
Using Zenko CloudServer with s3fs
---------------------------------
Using CloudServer with s3fs
---------------------------
First, you're going to need a mountpoint; we chose ``/mnt/tests3fs``:
1. Use /mnt/tests3fs as a mount point.
.. code:: sh
.. code:: sh
$> mkdir /mnt/tests3fs
Then, you want to create a bucket on your local Zenko CloudServer; we named it
``tests3fs``:
2. Create a bucket on your local CloudServer. In the present example it is
named “tests3fs”.
.. code:: sh
.. code:: sh
$> s3cmd mb s3://tests3fs
*Note:* *- If you've never used s3cmd with our Zenko CloudServer, our README
provides you with a `recommended
config <https://github.com/scality/S3/blob/master/README.md#s3cmd>`__*
3. Mount the bucket to your mount point with s3fs:
Now you can mount your bucket to your mountpoint with s3fs:
.. code:: sh
.. code:: sh
$> s3fs tests3fs /mnt/tests3fs -o passwd_file=/etc/passwd-s3fs -o url="https://s3.scality.test:8000/" -o use_path_request_style
*If you're curious, the structure of this command is*
``s3fs BUCKET_NAME PATH/TO/MOUNTPOINT -o OPTIONS``\ *, and the
options are mandatory and serve the following purposes:
* ``passwd_file``\ *: specifiy path to password file;
* ``url``\ *: specify the hostname used by your SSL provider;
* ``use_path_request_style``\ *: force path style (by default, s3fs
uses subdomains (DNS style)).*
The structure of this command is:
``s3fs BUCKET_NAME PATH/TO/MOUNTPOINT -o OPTIONS``. Of these mandatory
options:
| From now on, you can either add files to your mountpoint, or add
objects to your bucket, and they'll show in the other.
| For example, let's' create two files, and then a directory with a file
in our mountpoint:
* ``passwd_file`` specifies the path to the password file.
* ``url`` specifies the host name used by your SSL provider.
* ``use_path_request_style`` forces the path style (by default,
s3fs uses DNS-style subdomains).
.. code:: sh
Once the bucket is mounted, files added to the mount point or
objects added to the bucket will appear in both locations.
$> touch /mnt/tests3fs/file1 /mnt/tests3fs/file2
$> mkdir /mnt/tests3fs/dir1
$> touch /mnt/tests3fs/dir1/file3
Example
-------
Now, I can use s3cmd to show me what is actually in S3Server:
Create two files, and then a directory with a file in our mount point:
.. code:: sh
.. code:: sh
$> s3cmd ls -r s3://tests3fs
$> touch /mnt/tests3fs/file1 /mnt/tests3fs/file2
$> mkdir /mnt/tests3fs/dir1
$> touch /mnt/tests3fs/dir1/file3
2017-02-28 17:28 0 s3://tests3fs/dir1/
2017-02-28 17:29 0 s3://tests3fs/dir1/file3
2017-02-28 17:28 0 s3://tests3fs/file1
2017-02-28 17:28 0 s3://tests3fs/file2
Now, use s3cmd to show what is in CloudServer:
Now you can enjoy a filesystem view on your local Zenko CloudServer!
.. code:: sh
$> s3cmd ls -r s3://tests3fs
2017-02-28 17:28 0 s3://tests3fs/dir1/
2017-02-28 17:29 0 s3://tests3fs/dir1/file3
2017-02-28 17:28 0 s3://tests3fs/file1
2017-02-28 17:28 0 s3://tests3fs/file2
Now you can enjoy a filesystem view on your local CloudServer.
Duplicity
=========
How to backup your files with Zenko CloudServer.
How to back up your files with CloudServer.
Installing
-----------
Installing Duplicity and its dependencies
Installing Duplicity and its Dependencies
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Second, you want to install
`Duplicity <http://duplicity.nongnu.org/index.html>`__. You have to
download `this
tarball <https://code.launchpad.net/duplicity/0.7-series/0.7.11/+download/duplicity-0.7.11.tar.gz>`__,
decompress it, and then checkout the README inside, which will give you
a list of dependencies to install. If you're using Ubuntu 14.04, this is
your lucky day: here is a lazy step by step install.
To install `Duplicity <http://duplicity.nongnu.org/>`__,
go to `this site <https://code.launchpad.net/duplicity/0.7-series>`__.
Download the latest tarball. Decompress it and follow the instructions
in the README.
.. code:: sh
$> tar zxvf duplicity-0.7.11.tar.gz
$> cd duplicity-0.7.11
$> python setup.py install
You may receive error messages indicating the need to install some or all
of the following dependencies:
.. code:: sh
@ -484,30 +420,20 @@ your lucky day: here is a lazy step by step install.
$> apt-get install python-dev python-pip python-lockfile
$> pip install -U boto
Then you want to actually install Duplicity:
Testing the Installation
------------------------
.. code:: sh
1. Check that CloudServer is running. Run ``$> docker ps``. You should
see one container named ``scality/cloudserver``. If you do not, run
``$> docker start cloudserver`` and check again.
$> tar zxvf duplicity-0.7.11.tar.gz
$> cd duplicity-0.7.11
$> python setup.py install
Using
------
Testing your installation
~~~~~~~~~~~~~~~~~~~~~~~~~~~
First, we're just going to quickly check that Zenko CloudServer is actually
running. To do so, simply run ``$> docker ps`` . You should see one
container named ``scality/s3server``. If that is not the case, try
``$> docker start s3server``, and check again.
Secondly, as you probably know, Duplicity uses a module called **Boto**
to send requests to S3. Boto requires a configuration file located in
**``/etc/boto.cfg``** to have your credentials and preferences. Here is
a minimalistic config `that you can finetune following these
instructions <http://boto.cloudhackers.com/en/latest/getting_started.html>`__.
2. Duplicity uses a module called “Boto” to send requests to S3. Boto
requires a configuration file located in ``/etc/boto.cfg`` to store
your credentials and preferences. A minimal configuration
you can fine tune `following these instructions
<http://boto.cloudhackers.com/en/latest/getting_started.html>`__ is
shown here:
::
@ -521,54 +447,51 @@ instructions <http://boto.cloudhackers.com/en/latest/getting_started.html>`__.
# If using SSL, unmute and provide absolute path to local CA certificate
# ca_certificates_file = /absolute/path/to/ca.crt
*Note:* *If you want to set up SSL with Zenko CloudServer, check out our
`tutorial <http://link/to/SSL/tutorial>`__*
.. note:: To set up SSL with CloudServer, check out our `Using SSL
<./GETTING_STARTED#Using_SSL>`__ in GETTING STARTED.
At this point, we've met all the requirements to start running Zenko CloudServer
as a backend to Duplicity. So we should be able to back up a local
folder/file to local S3. Let's try with the duplicity decompressed
folder:
3. At this point all requirements to run CloudServer as a backend to Duplicity
have been met. A local folder/file should back up to the local S3.
Try it with the decompressed Duplicity folder:
.. code:: sh
$> duplicity duplicity-0.7.11 "s3://127.0.0.1:8000/testbucket/"
*Note:* *Duplicity will prompt you for a symmetric encryption
passphrase. Save it somewhere as you will need it to recover your
data. Alternatively, you can also add the ``--no-encryption`` flag
and the data will be stored plain.*
.. note:: Duplicity will prompt for a symmetric encryption passphrase.
Save it carefully, as you will need it to recover your data.
Alternatively, you can add the ``--no-encryption`` flag
and the data will be stored plain.
If this command is succesful, you will get an output looking like this:
If this command is successful, you will receive an output resembling:
::
.. code:: sh
--------------[ Backup Statistics ]--------------
StartTime 1486486547.13 (Tue Feb 7 16:55:47 2017)
EndTime 1486486547.40 (Tue Feb 7 16:55:47 2017)
ElapsedTime 0.27 (0.27 seconds)
SourceFiles 388
SourceFileSize 6634529 (6.33 MB)
NewFiles 388
NewFileSize 6634529 (6.33 MB)
DeletedFiles 0
ChangedFiles 0
ChangedFileSize 0 (0 bytes)
ChangedDeltaSize 0 (0 bytes)
DeltaEntries 388
RawDeltaSize 6392865 (6.10 MB)
TotalDestinationSizeChange 2003677 (1.91 MB)
Errors 0
-------------------------------------------------
--------------[ Backup Statistics ]--------------
StartTime 1486486547.13 (Tue Feb 7 16:55:47 2017)
EndTime 1486486547.40 (Tue Feb 7 16:55:47 2017)
ElapsedTime 0.27 (0.27 seconds)
SourceFiles 388
SourceFileSize 6634529 (6.33 MB)
NewFiles 388
NewFileSize 6634529 (6.33 MB)
DeletedFiles 0
ChangedFiles 0
ChangedFileSize 0 (0 bytes)
ChangedDeltaSize 0 (0 bytes)
DeltaEntries 388
RawDeltaSize 6392865 (6.10 MB)
TotalDestinationSizeChange 2003677 (1.91 MB)
Errors 0
-------------------------------------------------
Congratulations! You can now backup to your local S3 through duplicity
:)
Congratulations! You can now back up to your local S3 through Duplicity.
Automating backups
~~~~~~~~~~~~~~~~~~~
Automating Backups
------------------
Now you probably want to back up your files periodically. The easiest
way to do this is to write a bash script and add it to your crontab.
Here is my suggestion for such a file:
The easiest way to back up files periodically is to write a bash script
and add it to your crontab. A suggested script follows.
.. code:: sh
@ -577,33 +500,33 @@ Here is my suggestion for such a file:
# Export your passphrase so you don't have to type anything
export PASSPHRASE="mypassphrase"
# If you want to use a GPG Key, put it here and unmute the line below
# To use a GPG key, put it here and uncomment the line below
#GPG_KEY=
# Define your backup bucket, with localhost specified
DEST="s3://127.0.0.1:8000/testbuckets3server/"
DEST="s3://127.0.0.1:8000/testbucketcloudserver/"
# Define the absolute path to the folder you want to backup
# Define the absolute path to the folder to back up
SOURCE=/root/testfolder
# Set to "full" for full backups, and "incremental" for incremental backups
# Warning: you have to perform one full backup befor you can perform
# Warning: you must perform one full backup befor you can perform
# incremental ones on top of it
FULL=incremental
# How long to keep backups for; if you don't want to delete old
# backups, keep empty; otherwise, syntax is "1Y" for one year, "1M"
# for one month, "1D" for one day
# How long to keep backups. If you don't want to delete old backups, keep
# this value empty; otherwise, the syntax is "1Y" for one year, "1M" for
# one month, "1D" for one day.
OLDER_THAN="1Y"
# is_running checks whether duplicity is currently completing a task
# is_running checks whether Duplicity is currently completing a task
is_running=$(ps -ef | grep duplicity | grep python | wc -l)
# If duplicity is already completing a task, this will simply not run
# If Duplicity is already completing a task, this will not run
if [ $is_running -eq 0 ]; then
echo "Backup for ${SOURCE} started"
# If you want to delete backups older than a certain time, we do it here
# To delete backups older than a certain time, do it here
if [ "$OLDER_THAN" != "" ]; then
echo "Removing backups older than ${OLDER_THAN}"
duplicity remove-older-than ${OLDER_THAN} ${DEST}
@ -626,17 +549,17 @@ Here is my suggestion for such a file:
# Forget the passphrase...
unset PASSPHRASE
So let's say you put this file in ``/usr/local/sbin/backup.sh.`` Next
you want to run ``crontab -e`` and paste your configuration in the file
that opens. If you're unfamiliar with Cron, here is a good `How
To <https://help.ubuntu.com/community/CronHowto>`__. The folder I'm
backing up is a folder I modify permanently during my workday, so I want
incremental backups every 5mn from 8AM to 9PM monday to friday. Here is
the line I will paste in my crontab:
Put this file in ``/usr/local/sbin/backup.sh``. Run ``crontab -e`` and
paste your configuration into the file that opens. If you're unfamiliar
with Cron, here is a good `HowTo
<https://help.ubuntu.com/community/CronHowto>`__. If the folder being
backed up is a folder to be modified permanently during the work day,
we can set incremental backups every 5 minutes from 8 AM to 9 PM Monday
through Friday by pasting the following line into crontab:
.. code:: sh
*/5 8-20 * * 1-5 /usr/local/sbin/backup.sh
Now I can try and add / remove files from the folder I'm backing up, and
I will see incremental backups in my bucket.
Adding or removing files from the folder being backed up will result in
incremental backups in the bucket.

View File

@ -1,229 +0,0 @@
# Metadata Search Documenation
## Description
This feature enables metadata search to be performed on the metadata of objects
stored in Zenko.
## Requirements
+ MongoDB
## Design
The MD Search feature expands on the existing `GET Bucket` S3 API. It allows
users to conduct metadata searches by adding the custom Zenko querystring
parameter, `search`. The `search` parameter is of a pseudo
SQL WHERE clause structure and supports basic SQL operators:
ex. `"A=1 AND B=2 OR C=3"` (more complex queries can also be achieved with the
use of nesting operators, `(` and `)`).
The search process is as follows:
+ Zenko receives a `GET` request.
```
# regular getBucket request
GET /bucketname HTTP/1.1
Host: 127.0.0.1:8000
Date: Wed, 18 Oct 2018 17:50:00 GMT
Authorization: authorization string
# getBucket versions request
GET /bucketname?versions HTTP/1.1
Host: 127.0.0.1:8000
Date: Wed, 18 Oct 2018 17:50:00 GMT
Authorization: authorization string
# search getBucket request
GET /bucketname?search=key%3Dsearch-item HTTP/1.1
Host: 127.0.0.1:8000
Date: Wed, 18 Oct 2018 17:50:00 GMT
Authorization: authorization string
```
+ If the request does not contain the query param `search`, a normal bucket
listing is performed and a XML result containing the list of objects will be
returned as the response.
+ If the request does contain the query parameter `search`, the search string is
parsed and validated.
+ If the search string is invalid, an `InvalidArgument` error will be
returned as response.
+ If the search string is valid, it will be parsed and an abstract syntax
tree (AST) is generated.
+ The AST is then passed to the MongoDB backend to be used as the query filter
for retrieving objects in a bucket that satisfies the requested search
conditions.
+ The filtered results are then parsed and returned as the response.
The results from MD search is of the same structure as the `GET Bucket`
results:
```xml
<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>bucketname</Name>
<Prefix/>
<Marker/>
<MaxKeys>1000</MaxKeys>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>objectKey</Key>
<LastModified>2018-04-19T18:31:49.426Z</LastModified>
<ETag>&quot;d41d8cd98f00b204e9800998ecf8427e&quot;</ETag>
<Size>0</Size>
<Owner>
<ID>79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be</ID>
<DisplayName>Bart</DisplayName>
</Owner>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
...
</Contents>
</ListBucketResult>
```
## Performing MD Search with Zenko
To make a successful request to Zenko, you would need
+ Zenko Credentials
+ Sign request with Auth V4
With requirements, you can peform metadata searches by:
+ using the `search_bucket` tool in the
[Scality/S3](https://github.com/scality/S3) GitHub repository.
+ creating an AuthV4 signed HTTP request to Zenko in the programming language of
choice
### Using the S3 Tool
After cloning the [Scality/S3](https://github.com/scality/S3) GitHub repository
and installing the necessary dependencies, you can run the following command
in the S3 project root directory to access the search tool.
```
node bin/search_bucket
```
This will generate the following output
```
Usage: search_bucket [options]
Options:
-V, --version output the version number
-a, --access-key <accessKey> Access key id
-k, --secret-key <secretKey> Secret access key
-b, --bucket <bucket> Name of the bucket
-q, --query <query> Search query
-h, --host <host> Host of the server
-p, --port <port> Port of the server
-s --ssl
-v, --verbose
-h, --help output usage information
```
In the following examples, our Zenko Server is accessible on endpoint
`http://127.0.0.1:8000` and contains the bucket `zenkobucket`.
```
# search for objects with metadata "blue"
node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
-q "x-amz-meta-color=blue" -h 127.0.0.1 -p 8000
# search for objects tagged with "type=color"
node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
-q "tags.type=color" -h 127.0.0.1 -p 8000
```
### Coding Examples
Search requests can be also performed by making HTTP requests authenticated
with the `AWS Signature version 4` scheme.\
See the following urls for more information about the V4 authentication scheme.
+ http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
+ http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
You can also view examples for making requests with Auth V4 in various
languages [here](../exmaples).
### Specifying Metadata Fields
To search common metadata headers:
```
{metadata-key}{supported SQL op}{search value}
# example
key = blueObject
size > 0
key LIKE "blue.*"
```
To search custom user metadata:
```
# metadata must be prefixed with "x-amz-meta-"
x-amz-meta-{usermetadata-key}{supported SQL op}{search value}
# example
x-amz-meta-color = blue
x-amz-meta-color != red
x-amz-meta-color LIKE "b.*"
```
To search tags:
```
# tag searches must be prefixed with "tags."
tags.{tag-key}{supported SQL op}{search value}
# example
tags.type = color
```
### Differences from SQL
The MD search queries are similar to the `WHERE` clauses of SQL queries, but
they differ in that:
+ MD search queries follow the `PCRE` format
+ Search queries do not require values with hyphens to be enclosed in
backticks, ``(`)``
```
# SQL query
`x-amz-meta-search-item` = `ice-cream-cone`
# MD Search query
x-amz-meta-search-item = ice-cream-cone
```
+ The search queries do not support all of the SQL operators.
+ Supported SQL Operators: `=`, `<`, `>`, `<=`, `>=`, `!=`,
`AND`, `OR`, `LIKE`, `<>`
+ Unsupported SQL Operators: `NOT`, `BETWEEN`, `IN`, `IS`, `+`,
`-`, `%`, `^`, `/`, `*`, `!`
#### Using Regular Expressions in MD Search
+ Regular expressions used in MD search differs from SQL in that wildcards are
represented with `.*` instead of `%`.
+ Regex patterns must be wrapped in quotes as not doing so can lead to
misinterpretation of patterns.
+ Regex patterns can be written in form of the `/pattern/` syntax or
just the pattern if one does not require regex options, similar to `PCRE`.
Example regular expressions:
```
# search for strings containing word substring "helloworld"
".*helloworld.*"
"/.*helloworld.*/"
"/.*helloworld.*/i"
```

263
docs/MD_SEARCH.rst Normal file
View File

@ -0,0 +1,263 @@
Metadata Search Documentation
=============================
Description
-----------
This feature enables metadata search to be performed on the metadata of objects
stored in Zenko.
Requirements
------------
* MongoDB
Design
------
The Metadata Search feature expands on the existing :code:`GET Bucket` S3 API by
enabling users to conduct metadata searches by adding the custom Zenko query
string parameter, :code:`search`. The :code:`search` parameter is structured as a pseudo
SQL WHERE clause, and supports basic SQL operators. For example:
:code:`"A=1 AND B=2 OR C=3"` (complex queries can be built using nesting
operators, :code:`(` and :code:`)`).
The search process is as follows:
* Zenko receives a :code:`GET` request.
.. code::
# regular getBucket request
GET /bucketname HTTP/1.1
Host: 127.0.0.1:8000
Date: Wed, 18 Oct 2018 17:50:00 GMT
Authorization: authorization string
# getBucket versions request
GET /bucketname?versions HTTP/1.1
Host: 127.0.0.1:8000
Date: Wed, 18 Oct 2018 17:50:00 GMT
Authorization: authorization string
# search getBucket request
GET /bucketname?search=key%3Dsearch-item HTTP/1.1
Host: 127.0.0.1:8000
Date: Wed, 18 Oct 2018 17:50:00 GMT
Authorization: authorization string
* If the request does *not* contain the :code:`search` query parameter, Zenko performs
a normal bucket listing and returns an XML result containing the list of
objects.
* If the request *does* contain the :code:`search` query parameter, Zenko parses and
validates the search string.
- If the search string is invalid, Zenko returns an :code:`InvalidArgument` error.
.. code::
<?xml version=\"1.0\" encoding=\"UTF-8\"?>
<Error>
<Code>InvalidArgument</Code>
<Message>Invalid sql where clause sent as search query</Message>
<Resource></Resource>
<RequestId>d1d6afc64345a8e1198e</RequestId>
</Error>
- If the search string is valid, Zenko parses it and generates an abstract
syntax tree (AST). The AST is then passed to the MongoDB backend to be
used as the query filter for retrieving objects from a bucket that
satisfies the requested search conditions. Zenko parses the filtered
results and returns them as the response.
Metadata search results have the same structure as a :code:`GET Bucket` response:
.. code:: xml
<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>bucketname</Name>
<Prefix/>
<Marker/>
<MaxKeys>1000</MaxKeys>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>objectKey</Key>
<LastModified>2018-04-19T18:31:49.426Z</LastModified>
<ETag>&quot;d41d8cd98f00b204e9800998ecf8427e&quot;</ETag>
<Size>0</Size>
<Owner>
<ID>79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be</ID>
<DisplayName>Bart</DisplayName>
</Owner>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
...
</Contents>
</ListBucketResult>
Performing Metadata Searches with Zenko
---------------------------------------
You can perform metadata searches by:
+ Using the :code:`search_bucket` tool in the
`Scality/S3 <https://github.com/scality/S3>`_ GitHub repository.
+ Creating a signed HTTP request to Zenko in your preferred programming
language.
Using the S3 Tool
+++++++++++++++++
After cloning the `Scality/S3 <https://github.com/scality/S3>`_ GitHub repository
and installing the necessary dependencies, run the following command in the S3
projects root directory to access the search tool:
.. code::
node bin/search_bucket
This generates the following output:
.. code::
Usage: search_bucket [options]
Options:
-V, --version output the version number
-a, --access-key <accessKey> Access key id
-k, --secret-key <secretKey> Secret access key
-b, --bucket <bucket> Name of the bucket
-q, --query <query> Search query
-h, --host <host> Host of the server
-p, --port <port> Port of the server
-s --ssl
-v, --verbose
-h, --help output usage information
In the following examples, Zenko Server is accessible on endpoint
:code:`http://127.0.0.1:8000` and contains the bucket :code:`zenkobucket`.
.. code::
# search for objects with metadata "blue"
node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
-q "x-amz-meta-color=blue" -h 127.0.0.1 -p 8000
# search for objects tagged with "type=color"
node bin/search_bucket -a accessKey1 -k verySecretKey1 -b zenkobucket \
-q "tags.type=color" -h 127.0.0.1 -p 8000
Coding Examples
+++++++++++++++
Search requests can be also performed by making HTTP requests authenticated
with one of the AWS Signature schemes: version 2 or version 4. \
For more about authentication scheme, see:
* https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
* http://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html
* http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
You can also view examples for making requests with Auth V4 in various
languages `here <../../../examples>`__.
Specifying Metadata Fields
~~~~~~~~~~~~~~~~~~~~~~~~~~
To search system metadata headers:
.. code::
{system-metadata-key}{supported SQL op}{search value}
# example
key = blueObject
size > 0
key LIKE "blue.*"
To search custom user metadata:
.. code::
# metadata must be prefixed with "x-amz-meta-"
x-amz-meta-{user-metadata-key}{supported SQL op}{search value}
# example
x-amz-meta-color = blue
x-amz-meta-color != red
x-amz-meta-color LIKE "b.*"
To search tags:
.. code::
# tag searches must be prefixed with "tags."
tags.{tag-key}{supported SQL op}{search value}
# example
tags.type = color
Examples queries:
.. code::
# searching for objects with custom metadata "color"=blue" and are tagged
# "type"="color"
tags.type="color" AND x-amz-meta-color="blue"
# searching for objects with the object key containing the substring "blue"
# or (custom metadata "color"=blue" and are tagged "type"="color")
key LIKE '.*blue.*' OR (x-amz-meta-color="blue" AND tags.type="color")
Differences from SQL
++++++++++++++++++++
Zenko metadata search queries are similar to SQL-query :code:`WHERE` clauses, but
differ in that:
* They follow the :code:`PCRE` format
* They do not require values with hyphens to be enclosed in
backticks, :code:``(`)``
.. code::
# SQL query
`x-amz-meta-search-item` = `ice-cream-cone`
# MD Search query
x-amz-meta-search-item = ice-cream-cone
* Search queries do not support all SQL operators.
.. code::
# Supported SQL operators:
=, <, >, <=, >=, !=, AND, OR, LIKE, <>
# Unsupported SQL operators:
NOT, BETWEEN, IN, IS, +, -, %, ^, /, *, !
Using Regular Expressions in Metadata Search
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Regular expressions in Zenko metadata search differ from SQL in the following
ways:
+ Wildcards are represented with :code:`.*` instead of :code:`%`.
+ Regex patterns must be wrapped in quotes. Failure to do this can lead to
misinterpretation of patterns.
+ As with :code:`PCRE`, regular expressions can be entered in either the
:code:`/pattern/` syntax or as the pattern itself if regex options are
not required.
Example regular expressions:
.. code::
# search for strings containing word substring "helloworld"
".*helloworld.*"
"/.*helloworld.*/"
"/.*helloworld.*/i"

View File

@ -0,0 +1,161 @@
# Object Lock Feature Test Plan
## Feature Component Description
Implementing Object Lock will introduce six new APIs:
- putObjectLockConfiguration
- getObjectLockConfiguration
- putObjectRetention
- getObjectRetention
- putObjectLegalHold
- getObjectLegalHold
Along with these APIs, putBucket, putObject, deleteObject, and multiObjectDelete
be affected. In Arsenal, both the BucketInfo and ObjectMD models will be
updated. Bucket policy and IAM policy permissions will be updated to include
the new API actions.
## Functional Tests
### putBucket tests
- passing option to enable object lock updates bucket metadata and enables
bucket versioning
### putBucketVersioning tests
- suspending versioning on bucket with object lock enabled returns error
### putObject tests
- putting retention configuration on object should be allowed
- putting invalid retention configuration returns error
### getObject tests
- getting object with retention information should include retention information
### copyObject tests
- copying object with retention information should include retention information
### initiateMultipartUpload tests
- mpu object initiated with retention information should include retention
information
### putObjectLockConfiguration tests
- putting configuration as non-bucket-owner user returns AccessDenied error
- disabling object lock on bucket created with object lock returns error
- enabling object lock on bucket created without object lock returns
InvalidBucketState error
- enabling object lock with token on bucket created without object lock succeeds
- putting valid object lock configuration when bucket does not have object
lock enabled returns error (InvalidRequest?)
- putting valid object lock configuration updates bucket metadata
- putting invalid object lock configuration returns error
- ObjectLockEnabled !== "Enabled"
- Rule object doesn't contain DefaultRetention key
- Mode !== "GOVERNANCE" or "COMPLIANCE"
- Days are not an integer
- Years are not an integer
### getObjectLockConfiguration tests
- getting configuration as non-bucket-owner user returns AccessDenied error
- getting configuration when none is set returns
ObjectLockConfigurationNotFoundError error
- getting configuration returns correct object lock configuration for bucket
### putObjectRetention
- putting retention as non-bucket-owner user returns AccessDenied error
- putting retention on object in bucket without object lock enabled returns
InvalidRequest error
- putting valid retention period updates object metadata
### getObjectRetention
- getting retention as non-bucket-owner user returns AccessDenied error
- getting retention when none is set returns NoSuchObjectLockConfiguration
error
- getting retention returns correct object retention period
### putObjectLegalHold
- putting legal hold as non-bucket-owner user returns AccessDenied error
- putting legal hold on object in bucket without object lock enabled returns
InvalidRequest error
- putting valid legal hold updates object metadata
### getObjectLegalHold
- getting legal hold as non-bucket-owner user returns AccessDenied error
- getting legal hold when none is set returns NoSuchObjectLockConfiguration
error
- getting legal hold returns correct object legal hold
## End to End Tests
### Scenarios
- Create bucket with object lock enabled. Put object. Put object lock
configuration. Put another object.
- Ensure object put before configuration does not have retention period set
- Ensure object put after configuration does have retention period set
- Create bucket without object lock. Put object. Enable object lock with token
and put object lock configuration. Put another object.
- Ensure object put before configuration does not have retention period set
- Ensure object put after configuration does have retention period set
- Create bucket with object lock enabled and put configuration with COMPLIANCE
mode. Put object.
- Ensure object cannot be deleted (returns AccessDenied error).
- Ensure object cannot be overwritten.
- Create bucket with object lock enabled and put configuration with GOVERNANCE
mode. Put object.
- Ensure user without permission cannot delete object
- Ensure user without permission cannot overwrite object
- Ensure user with permission can delete object
- Ensure user with permission can overwrite object
- Ensure user with permission can lengthen retention period
- Ensure user with permission cannot shorten retention period
- Create bucket with object lock enabled and put configuration. Edit bucket
metadata so retention period is expired. Put object.
- Ensure object can be deleted.
- Ensure object can be overwritten.
- Create bucket with object lock enabled and put configuration. Edit bucket
metadata so retention period is expired. Put object. Put new retention
period on object.
- Ensure object cannot be deleted.
- Ensure object cannot be overwritten.
- Create bucket with object locked enabled and put configuration. Put object.
Edit object metadata so retention period is past expiration.
- Ensure object can be deleted.
- Ensure object can be overwritten.
- Create bucket with object lock enabled and put configuration. Edit bucket
metadata so retention period is expired. Put object. Put legal hold
on object.
- Ensure object cannot be deleted.
- Ensure object cannot be overwritten.
- Create bucket with object lock enabled and put configuration. Put object.
Check object retention. Change bucket object lock configuration.
- Ensure object retention period has not changed with bucket configuration.
- Create bucket with object lock enabled. Put object with legal hold.
- Ensure object cannot be deleted.
- Ensure object cannot be overwritten.
- Create bucket with object lock enabled. Put object with legal hold. Remove
legal hold.
- Ensure object can be deleted.
- Ensure object can be overwritten.

73
docs/RELEASE.md Normal file
View File

@ -0,0 +1,73 @@
# Cloudserver Release Plan
## Docker Image Generation
Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages).
CloudServer has a few images there:
* Cloudserver container image: ghcr.io/scality/cloudserver
* Dashboard oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
* Policies oras image: ghcr.io/scality/cloudserver/cloudser-dashboard
With every CI build, the CI will push images, tagging the
content with the developer branch's short SHA-1 commit hash.
This allows those images to be used by developers, CI builds,
build chain and so on.
Tagged versions of cloudserver will be stored in the production namespace.
## How to Pull Docker Images
```sh
docker pull ghcr.io/scality/cloudserver:<commit hash>
docker pull ghcr.io/scality/cloudserver:<tag>
```
## Release Process
To release a production image:
* Create a PR to bump the package version
Update Cloudserver's `package.json` by bumping it to the relevant next
version in a new PR. Per example if the last released version was
`8.4.7`, the next version would be `8.4.8`.
```js
{
"name": "cloudserver",
"version": "8.4.8", <--- Here
[...]
}
```
* Review & merge the PR
* Create the release on GitHub
* Go the Release tab (https://github.com/scality/cloudserver/releases);
* Click on the `Draft new release button`;
* In the `tag` field, type the name of the release (`8.4.8`), and confirm
to create the tag on publish;
* Click on `Generate release notes` button to fill the fields;
* Rename the release to `Release x.y.z` (e.g. `Release 8.4.8` in this case);
* Click to `Publish the release` to create the GitHub release and git tag
Notes:
* the Git tag will be created automatically.
* this should be done as soon as the PR is merged, so that the tag
is put on the "version bump" commit.
* With the following parameters, [force a build here](https://eve.devsca.com/github/scality/cloudserver/#/builders/3/force/force)
* Branch Name: The one used for the tag earlier. In this example `development/8.4`
* Override Stage: 'release'
* Extra properties:
* name: `'tag'`, value: `[release version]`, in this example`'8.4.8'`
* Release the release version on Jira
* Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page)
* Create a next version
* Name: `[next version]`, in this example `8.4.9`
* Click `...` and select `Release` on the recently released version (`8.4.8`)
* Fill in the field to move incomplete version to the next one

View File

@ -6,7 +6,7 @@ Using Public Clouds as data backends
Introduction
------------
As stated in our `GETTING STARTED guide <../GETTING_STARTED/#location-configuration>`__,
As stated in our `GETTING STARTED guide <GETTING_STARTED.html#location-configuration>`__,
new data backends can be added by creating a region (also called location
constraint) with the right endpoint and credentials.
This section of the documentation shows you how to set up our currently
@ -139,7 +139,7 @@ to start the server and start writing data to AWS S3 through CloudServer.
.. code:: shell
# Start the server locally
$> S3DATA=multiple npm start
$> S3DATA=multiple yarn start
Run the server as a docker container with the ability to write to AWS S3
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -162,8 +162,8 @@ CloudServer.
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
-v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
-v ~/.aws/credentials:/root/.aws/credentials \
-e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000
-d scality/s3server
-e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000 \
-d scality/cloudserver
Testing: put an object to AWS S3 using CloudServer
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -306,7 +306,7 @@ to start the server and start writing data to MS Azure through CloudServer.
.. code:: shell
# Start the server locally
$> S3DATA=multiple npm start
$> S3DATA=multiple yarn start
Run the server as a docker container with the ability to write to MS Azure
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -324,7 +324,7 @@ CloudServer.
-v $(pwd)/locationConfig.json:/usr/src/app/locationConfig.json \
-v $(pwd)/conf/authdata.json:/usr/src/app/conf/authdata.json \
-e S3DATA=multiple -e ENDPOINT=http://localhost -p 8000:8000
-d scality/s3server
-d scality/cloudserver
Testing: put an object to MS Azure using CloudServer
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

View File

@ -1,8 +1,8 @@
============================================
Add New Backend Storage To Zenko Cloudserver
Add New Backend Storage To Zenko CloudServer
============================================
This set of documents aims at bootstrapping developers with Zenko's Cloudserver
This set of documents aims at bootstrapping developers with Zenko's CloudServer
module, so they can then go on and contribute features.
.. toctree::
@ -43,7 +43,7 @@ Openstack Swift x
and filling out the "Feature Request" section of our
template.
To add support for a new backend support to Cloudserver official
To add support for a new backend support to CloudServer official
repository, please follow these steps:
- familiarize yourself with our `Contributing Guidelines`_

View File

@ -2,28 +2,28 @@
Add A New Backend
=================
Supporting all possible public cloud storage APIs is Cloudserver's
Supporting all possible public cloud storage APIs is CloudServer's
ultimate goal. As an open source project, contributions are welcome.
The first step is to get familiar with building a custom Docker image
for Cloudserver.
for CloudServer.
Build a Custom Docker Image
~~~~~~~~~~~~~~~~~~~~~~~~~~~
Clone Zenko's Cloudserver, install all dependencies and start the
Clone Zenko's CloudServer, install all dependencies and start the
service:
.. code-block:: shell
$ git clone https://github.com/scality/cloudserver
$ cd cloudserver
$ npm install
$ npm start
$ yarn install
$ yarn start
.. tip::
Some optional dependencies may fail, resulting in you seeing `NPM
Some optional dependencies may fail, resulting in you seeing `yarn
WARN` messages; these can safely be ignored. Refer to the User
documentation for all available options.

View File

@ -6,7 +6,7 @@ These backends abstract the complexity of multiple APIs to let users
work on a single common namespace across multiple clouds.
This documents aims at introducing you to the right files in
Cloudserver (the Zenko stack's subcomponent in charge of API
CloudServer (the Zenko stack's subcomponent in charge of API
translation, among other things) to add support to your own backend of
choice.

View File

@ -1,10 +1,10 @@
======================
S3 compatible backends
S3-Compatible Backends
======================
Adding support in Zenkos Cloudserver
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Adding Support in CloudServer
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
This is the easiest case for backend support integration: there is nothing to do
but configuration! Follow the steps described in our
@ -33,7 +33,7 @@ definition for that backend will look something like:
}
},
Adding support in Zenko Orbit
Adding Support in Zenko Orbit
#############################
This can only be done by our core developpers' team. If thats what youre

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

View File

@ -4,17 +4,17 @@
#
# tox -e pip-compile
#
alabaster==0.7.11 \
--hash=sha256:674bb3bab080f598371f4443c5008cbfeb1a5e622dd312395d2d82af2c54c456 \
--hash=sha256:b63b1f4dc77c074d386752ec4a8a7517600f6c0db8cd42980cae17ab7b3275d7 \
alabaster==0.7.12 \
--hash=sha256:446438bdcca0e05bd45ea2de1668c1d9b032e1a9154c2c259092d77031ddd359 \
--hash=sha256:a661d72d58e6ea8a57f7a86e37d86716863ee5e92788398526d58b26a4e4dc02 \
# via sphinx
babel==2.6.0 \
--hash=sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669 \
--hash=sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23 \
# via sphinx
certifi==2018.4.16 \
--hash=sha256:13e698f54293db9f89122b0581843a782ad0934a4fe0172d2a980ba77fc61bb7 \
--hash=sha256:9fa520c1bacfb634fa7af20a76bcbd3d5fb390481724c597da32c719a7dca4b0 \
certifi==2018.10.15 \
--hash=sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c \
--hash=sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a \
# via requests
chardet==3.0.4 \
--hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae \
@ -32,39 +32,66 @@ idna==2.7 \
--hash=sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e \
--hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16 \
# via requests
imagesize==1.0.0 \
--hash=sha256:3620cc0cadba3f7475f9940d22431fc4d407269f1be59ec9b8edcca26440cf18 \
--hash=sha256:5b326e4678b6925158ccc66a9fa3122b6106d7c876ee32d7de6ce59385b96315 \
imagesize==1.1.0 \
--hash=sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8 \
--hash=sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5 \
# via sphinx
jinja2==2.10 \
--hash=sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd \
--hash=sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4 \
# via sphinx
markupsafe==1.0 \
--hash=sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665 \
markupsafe==1.1.0 \
--hash=sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432 \
--hash=sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b \
--hash=sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9 \
--hash=sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af \
--hash=sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834 \
--hash=sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd \
--hash=sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d \
--hash=sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7 \
--hash=sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b \
--hash=sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3 \
--hash=sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c \
--hash=sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2 \
--hash=sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7 \
--hash=sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36 \
--hash=sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1 \
--hash=sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e \
--hash=sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1 \
--hash=sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c \
--hash=sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856 \
--hash=sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550 \
--hash=sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492 \
--hash=sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672 \
--hash=sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401 \
--hash=sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6 \
--hash=sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6 \
--hash=sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c \
--hash=sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd \
--hash=sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1 \
# via jinja2
packaging==17.1 \
--hash=sha256:e9215d2d2535d3ae866c3d6efc77d5b24a0192cce0ff20e42896cc0664f889c0 \
--hash=sha256:f019b770dd64e585a99714f1fd5e01c7a8f11b45635aa953fd41c689a657375b \
packaging==18.0 \
--hash=sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807 \
--hash=sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9 \
# via sphinx
pygments==2.2.0 \
--hash=sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d \
--hash=sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc \
# via sphinx
pyparsing==2.2.0 \
--hash=sha256:0832bcf47acd283788593e7a0f542407bd9550a55a8a8435214a1960e04bcb04 \
--hash=sha256:fee43f17a9c4087e7ed1605bd6df994c6173c1e977d7ade7b651292fab2bd010 \
pyparsing==2.3.0 \
--hash=sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b \
--hash=sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592 \
# via packaging
pytz==2018.5 \
--hash=sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053 \
--hash=sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277 \
pytz==2018.7 \
--hash=sha256:31cb35c89bd7d333cd32c5f278fca91b523b0834369e757f4c5641ea252236ca \
--hash=sha256:8e0f8568c118d3077b46be7d654cc8167fa916092e28320cde048e54bfc9f1e6 \
# via babel
recommonmark==0.4.0 \
--hash=sha256:6e29c723abcf5533842376d87c4589e62923ecb6002a8e059eb608345ddaff9d \
--hash=sha256:cd8bf902e469dae94d00367a8197fb7b81fcabc9cfb79d520e0d22d0fbeaa8b7
requests==2.19.1 \
--hash=sha256:63b52e3c866428a224f97cab011de738c36aec0185aa91cfacd418b5d58911d1 \
--hash=sha256:ec22d826a36ed72a7358ff3fe56cbd4ba69dd7a6718ffd450ff0e9df7a47ce6a \
requests==2.20.1 \
--hash=sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54 \
--hash=sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263 \
# via sphinx
six==1.11.0 \
--hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
@ -74,19 +101,19 @@ snowballstemmer==1.2.1 \
--hash=sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128 \
--hash=sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89 \
# via sphinx
sphinx==1.7.5 \
--hash=sha256:85f7e32c8ef07f4ba5aeca728e0f7717bef0789fba8458b8d9c5c294cad134f3 \
--hash=sha256:d45480a229edf70d84ca9fae3784162b1bc75ee47e480ffe04a4b7f21a95d76d
sphinx==1.8.2 \
--hash=sha256:120732cbddb1b2364471c3d9f8bfd4b0c5b550862f99a65736c77f970b142aea \
--hash=sha256:b348790776490894e0424101af9c8413f2a86831524bd55c5f379d3e3e12ca64
sphinxcontrib-websupport==1.1.0 \
--hash=sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd \
--hash=sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9 \
# via sphinx
typing==3.6.4 \
--hash=sha256:3a887b021a77b292e151afb75323dea88a7bc1b3dfa92176cff8e44c8b68bddf \
--hash=sha256:b2c689d54e1144bbcfd191b0832980a21c2dbcf7b5ff7a66248a60c90e951eb8 \
--hash=sha256:d400a9344254803a2368533e4533a4200d21eb7b6b729c173bc38201a74db3f2 \
typing==3.6.6 \
--hash=sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d \
--hash=sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4 \
--hash=sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a \
# via sphinx
urllib3==1.23 \
--hash=sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf \
--hash=sha256:b5725a0bd4ba422ab0e66e89e030c806576753ea3ee08554382c14e685d117b5 \
urllib3==1.24.1 \
--hash=sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39 \
--hash=sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22 \
# via requests

View File

@ -1,283 +0,0 @@
---
version: 0.2
branches:
feature/*, improvement/*, bugfix/*, w/*, q/*, hotfix/*:
stage: pre-merge
development/*:
stage: post-merge
models:
- env: &global-env
azurebackend_AZURE_STORAGE_ACCESS_KEY: >-
%(secret:azure_storage_access_key)s
azurebackend_AZURE_STORAGE_ACCOUNT_NAME: >-
%(secret:azure_storage_account_name)s
azurebackend_AZURE_STORAGE_ENDPOINT: >-
%(secret:azure_storage_endpoint)s
azurebackend2_AZURE_STORAGE_ACCESS_KEY: >-
%(secret:azure_storage_access_key_2)s
azurebackend2_AZURE_STORAGE_ACCOUNT_NAME: >-
%(secret:azure_storage_account_name_2)s
azurebackend2_AZURE_STORAGE_ENDPOINT: >-
%(secret:azure_storage_endpoint_2)s
azurebackendmismatch_AZURE_STORAGE_ACCESS_KEY: >-
%(secret:azure_storage_access_key)s
azurebackendmismatch_AZURE_STORAGE_ACCOUNT_NAME: >-
%(secret:azure_storage_account_name)s
azurebackendmismatch_AZURE_STORAGE_ENDPOINT: >-
%(secret:azure_storage_endpoint)s
azurenonexistcontainer_AZURE_STORAGE_ACCESS_KEY: >-
%(secret:azure_storage_access_key)s
azurenonexistcontainer_AZURE_STORAGE_ACCOUNT_NAME: >-
%(secret:azure_storage_account_name)s
azurenonexistcontainer_AZURE_STORAGE_ENDPOINT: >-
%(secret:azure_storage_endpoint)s
azuretest_AZURE_BLOB_ENDPOINT: "%(secret:azure_storage_endpoint)s"
b2backend_B2_ACCOUNT_ID: "%(secret:b2backend_b2_account_id)s"
b2backend_B2_STORAGE_ACCESS_KEY: >-
%(secret:b2backend_b2_storage_access_key)s
GOOGLE_SERVICE_EMAIL: "%(secret:gcp_service_email)s"
GOOGLE_SERVICE_KEY: "%(secret:gcp_service_key)s"
AWS_S3_BACKEND_ACCESS_KEY: "%(secret:aws_s3_backend_access_key)s"
AWS_S3_BACKEND_SECRET_KEY: "%(secret:aws_s3_backend_secret_key)s"
AWS_S3_BACKEND_ACCESS_KEY_2: "%(secret:aws_s3_backend_access_key_2)s"
AWS_S3_BACKEND_SECRET_KEY_2: "%(secret:aws_s3_backend_secret_key_2)s"
AWS_GCP_BACKEND_ACCESS_KEY: "%(secret:aws_gcp_backend_access_key)s"
AWS_GCP_BACKEND_SECRET_KEY: "%(secret:aws_gcp_backend_secret_key)s"
AWS_GCP_BACKEND_ACCESS_KEY_2: "%(secret:aws_gcp_backend_access_key_2)s"
AWS_GCP_BACKEND_SECRET_KEY_2: "%(secret:aws_gcp_backend_secret_key_2)s"
b2backend_B2_STORAGE_ENDPOINT: "%(secret:b2backend_b2_storage_endpoint)s"
gcpbackend2_GCP_SERVICE_EMAIL: "%(secret:gcp2_service_email)s"
gcpbackend2_GCP_SERVICE_KEY: "%(secret:gcp2_service_key)s"
gcpbackend2_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackend_GCP_SERVICE_EMAIL: "%(secret:gcp_service_email)s"
gcpbackend_GCP_SERVICE_KEY: "%(secret:gcp_service_key)s"
gcpbackendmismatch_GCP_SERVICE_EMAIL: >-
%(secret:gcpbackendmismatch_gcp_service_email)s
gcpbackendmismatch_GCP_SERVICE_KEY: >-
%(secret:gcpbackendmismatch_gcp_service_key)s
gcpbackend_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendmismatch_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendnoproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
gcpbackendproxy_GCP_SERVICE_KEYFILE: /root/.gcp/servicekey
- env: &mongo-vars
S3BACKEND: "mem"
MPU_TESTING: "yes"
S3METADATA: mongodb
- env: &multiple-backend-vars
S3BACKEND: "mem"
S3DATA: "multiple"
- env: &file-mem-mpu
S3BACKEND: "file"
S3VAULT: "mem"
MPU_TESTING: "yes"
- Git: &clone
name: Pull repo
repourl: '%(prop:git_reference)s'
shallow: True
retryFetch: True
haltOnFailure: True
- ShellCommand: &credentials
name: Setup Credentials
command: bash eve/workers/build/credentials.bash
haltOnFailure: True
env: *global-env
- ShellCommand: &npm-install
name: install modules
command: npm install
haltOnFailure: True
- Upload: &upload-artifacts
source: /artifacts
urls:
- "*"
- ShellCommand: &follow-s3-log
logfiles:
s3:
filename: /artifacts/s3.log
follow: true
- ShellCommand: &add-hostname
name: add hostname
command: |
echo "127.0.0.1 testrequestbucket.localhost" >> /etc/hosts
echo \
"127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com" \
>> /etc/hosts
haltOnFailure: True
stages:
pre-merge:
worker:
type: local
steps:
- SetProperty:
property: artifacts_name
value: "scality-s3-%(prop:buildnumber)s"
haltOnFailure: True
- TriggerStages:
name: Launch all workers
stage_names:
- linting-coverage
- file-ft-tests
- multiple-backend-test
- mongo-ft-tests
waitForFinish: True
haltOnFailure: True
linting-coverage:
worker:
type: docker
path: eve/workers/build
volumes: &default_volumes
- '/home/eve/workspace'
steps:
- Git: *clone
- ShellCommand: *npm-install
- ShellCommand: *add-hostname
- ShellCommand:
name: Linting
command: |
npm run --silent lint -- --max-warnings 0
npm run --silent lint_md
flake8 $(git ls-files "*.py")
yamllint $(git ls-files "*.yml")
- ShellCommand:
name: Unit Coverage
command: |
mkdir -p $CIRCLE_TEST_REPORTS/unit
npm run unit_coverage
npm run unit_coverage_legacy_location
env: &shared-vars
CIRCLE_TEST_REPORTS: /tmp
CIRCLE_ARTIFACTS: /tmp
CI_REPORTS: /tmp
- ShellCommand:
name: Unit Coverage logs
command: find /tmp/unit -exec cat {} \;
multiple-backend-test:
worker:
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
vars:
aggressorMemLimit: "2Gi"
s3MemLimit: "2Gi"
env:
<<: *multiple-backend-vars
<<: *global-env
steps:
- Git: *clone
- ShellCommand: *credentials
- ShellCommand: *npm-install
- ShellCommand:
command: |
bash -c "
source /root/.aws/exports &> /dev/null
set -ex
bash wait_for_local_port.bash 8000 40
npm run multiple_backend_test"
<<: *follow-s3-log
env:
<<: *multiple-backend-vars
<<: *global-env
S3_LOCATION_FILE: tests/locationConfig/locationConfigTests.json
- ShellCommand:
command: mvn test
workdir: build/tests/functional/jaws
<<: *follow-s3-log
env:
<<: *multiple-backend-vars
- ShellCommand:
command: rspec tests.rb
workdir: build/tests/functional/fog
<<: *follow-s3-log
env:
<<: *multiple-backend-vars
- Upload: *upload-artifacts
mongo-ft-tests:
worker: &s3-pod
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
vars:
aggressorMemLimit: "2Gi"
s3MemLimit: "1664Mi"
redis: enabled
env:
<<: *mongo-vars
<<: *global-env
steps:
- Git: *clone
- ShellCommand: *credentials
- ShellCommand: *npm-install
- ShellCommand:
command: |
set -ex
bash wait_for_local_port.bash 8000 40
npm run ft_test
<<: *follow-s3-log
env:
<<: *mongo-vars
<<: *global-env
- Upload: *upload-artifacts
file-ft-tests:
worker:
type: kube_pod
path: eve/workers/pod.yaml
images:
aggressor: eve/workers/build
s3: "."
vars:
aggressorMemLimit: "1920Mi"
s3MemLimit: "2Gi"
redis: enabled
env:
<<: *file-mem-mpu
<<: *global-env
steps:
- Git: *clone
- ShellCommand: *credentials
- ShellCommand: *npm-install
- ShellCommand:
command: |
set -ex
bash wait_for_local_port.bash 8000 40
npm run ft_test
<<: *follow-s3-log
env:
<<: *file-mem-mpu
<<: *global-env
- Upload: *upload-artifacts
post-merge:
worker:
type: local
steps:
- Git: *clone
- ShellCommand: &docker_login
name: Private Registry Login
command: >
docker login
-u '%(secret:private_registry_username)s'
-p '%(secret:private_registry_password)s'
'%(secret:private_registry_url)s'
- SetProperty: &docker_image_name
name: Set docker image name property
property: docker_image_name
value:
"%(secret:private_registry_url)s/zenko/cloudserver:\
%(prop:commit_short_revision)s"
- ShellCommand:
name: Build docker image
command: >-
docker build --no-cache -t %(prop:docker_image_name)s .
- ShellCommand:
name: Push image
command: docker push %(prop:docker_image_name)s

View File

@ -1,57 +0,0 @@
FROM buildpack-deps:stretch-curl
#
# Install packages needed by the buildchain
#
ENV LANG C.UTF-8
COPY ./s3_packages.list ./buildbot_worker_packages.list /tmp/
RUN curl -sS http://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - \
&& echo "deb http://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list \
&& apt-get update \
&& cat /tmp/*packages.list | xargs apt-get install -y \
&& git clone https://github.com/tj/n.git \
&& make -C ./n \
&& n 6 latest \
&& pip install pip==9.0.1 \
&& rm -rf ./n \
&& rm -rf /var/lib/apt/lists/* \
&& rm -f /tmp/packages.list
#
# Add user eve
#
RUN adduser -u 1042 --home /home/eve --disabled-password --gecos "" eve \
&& adduser eve sudo \
&& sed -ri 's/(%sudo.*)ALL$/\1NOPASSWD:ALL/' /etc/sudoers
#
# Install Dependencies
#
# Install RVM and gems
ENV RUBY_VERSION="2.4.1"
COPY ./gems.list /tmp/
RUN cat /tmp/gems.list | xargs gem install
#RUN gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 \
# && curl -sSL https://get.rvm.io | bash -s stable --ruby=$RUBY_VERSION \
# && usermod -a -G rvm eve
#RUN /bin/bash -l -c "\
# source /usr/local/rvm/scripts/rvm \
# && cat /tmp/gems.list | xargs gem install \
# && rm /tmp/gems.list"
# Install Pip packages
COPY ./pip_packages.list /tmp/
RUN cat /tmp/pip_packages.list | xargs pip install \
&& rm -f /tmp/pip_packages.list \
&& mkdir /home/eve/.aws \
&& chown eve /home/eve/.aws
#
# Run buildbot-worker on startup
#
ARG BUILDBOT_VERSION
RUN pip install buildbot-worker==$BUILDBOT_VERSION
CMD ["/bin/bash", "-l", "-c", "buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS && buildbot-worker start --nodaemon"]

View File

@ -1,13 +0,0 @@
ca-certificates
git
gnupg
libffi-dev
libssl-dev
python-pip
python2.7
python2.7-dev
software-properties-common
sudo
tcl
wget
procps

View File

@ -1,4 +0,0 @@
fog-aws:1.3.0
json
mime-types:3.1
rspec:3.5

View File

@ -1,3 +0,0 @@
flake8
s3cmd==1.6.1
yamllint

View File

@ -1,11 +0,0 @@
build-essential
curl
default-jdk
libdigest-hmac-perl
lsof
maven
netcat
redis-server
ruby-full
yarn=1.7.0-1
zlib1g-dev

View File

@ -1,162 +0,0 @@
---
apiVersion: v1
kind: Pod
metadata:
name: "proxy-ci-test-pod"
spec:
activeDeadlineSeconds: 3600
restartPolicy: Never
terminationGracePeriodSeconds: 10
hostAliases:
- ip: "127.0.0.1"
hostnames:
- "bucketwebsitetester.s3-website-us-east-1.amazonaws.com"
- "testrequestbucket.localhost"
containers:
{% if vars.env.S3METADATA is defined and vars.env.S3METADATA == "mongodb" -%}
- name: mongo
image: scality/ci-mongo:3.4
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 1Gi
limits:
cpu: 500m
memory: 1Gi
{%- endif %}
- name: aggressor
image: {{ images.aggressor }}
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 1Gi
limits:
cpu: "1"
memory: {{ vars.aggressorMemLimit }}
volumeMounts:
- name: creds
readOnly: false
mountPath: /root/.aws
- name: artifacts
readOnly: true
mountPath: /artifacts
command:
- bash
- -lc
- |
buildbot-worker create-worker . $BUILDMASTER:$BUILDMASTER_PORT $WORKERNAME $WORKERPASS
buildbot-worker start --nodaemon
env:
- name: CI
value: "true"
- name: ENABLE_LOCAL_CACHE
value: "true"
- name: REPORT_TOKEN
value: "report-token-1"
- name: REMOTE_MANAGEMENT_DISABLE
value: "1"
{% for key, value in vars.env.items() %}
- name: {{ key }}
value: "{{ value }}"
{% endfor %}
- name: s3
image: {{ images.s3 }}
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 200m
memory: 1Gi
limits:
cpu: "2"
memory: {{ vars.s3MemLimit }}
volumeMounts:
- name: creds
readOnly: false
mountPath: /root/.aws
- name: certs
readOnly: true
mountPath: /tmp
- name: artifacts
readOnly: false
mountPath: /artifacts
command:
- bash
- -ec
- |
sleep 10 # wait for mongo
/usr/src/app/docker-entrypoint.sh npm start | tee -a /artifacts/s3.log
env:
{% if vars.env.S3DATA is defined and vars.env.S3DATA == "multiple" -%}
- name: S3_LOCATION_FILE
value: "/usr/src/app/tests/locationConfig/locationConfigTests.json"
{%- endif %}
- name: CI
value: "true"
- name: ENABLE_LOCAL_CACHE
value: "true"
- name: MONGODB_HOSTS
value: "localhost:27018"
- name: MONGODB_RS
value: "rs0"
- name: REDIS_HOST
value: "localhost"
- name: REDIS_PORT
value: "6379"
- name: REPORT_TOKEN
value: "report-token-1"
- name: REMOTE_MANAGEMENT_DISABLE
value: "1"
- name: HEALTHCHECKS_ALLOWFROM
value: "0.0.0.0/0"
{% for key, value in vars.env.items() %}
- name: {{ key }}
value: "{{ value }}"
{% endfor %}
{% if vars.redis is defined and vars.redis == "enabled" -%}
- name: redis
image: redis:alpine
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 200m
memory: 128Mi
{%- endif %}
{% if vars.env.CI_PROXY is defined and vars.env.CI_PROXY == "true" -%}
- name: squid
image: scality/ci-squid
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 250m
memory: 128Mi
volumeMounts:
- name: certs
readOnly: false
mountPath: /ssl
command:
- sh
- -exc
- |
mkdir -p /ssl
openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 \
-subj "/C=US/ST=Country/L=City/O=Organization/CN=CN=scality-proxy" \
-keyout /ssl/myca.pem -out /ssl/myca.pem
cp /ssl/myca.pem /ssl/CA.pem
squid -f /etc/squid/squid.conf -N -z
squid -f /etc/squid/squid.conf -NYCd 1
{%- endif %}
volumes:
- name: creds
emptyDir: {}
- name: certs
emptyDir: {}
- name: artifacts
emptyDir: {}

View File

@ -48,7 +48,7 @@ signed_headers = 'host;x-amz-content-sha256;x-amz-date'
canonical_request = '{0}\n{1}\n{2}\n{3}\n{4}\n{5}' \
.format(method, canonical_uri, canonical_querystring, canonical_headers,
signed_headers, payload_hash)
print canonical_request
print(canonical_request)
credential_scope = '{0}/{1}/{2}/aws4_request' \
.format(date_stamp, region, service)
@ -76,4 +76,4 @@ headers = {
endpoint = 'http://' + host + canonical_uri + '?' + canonical_querystring
r = requests.get(endpoint, headers=headers)
print (r.text)
print(r.text)

View File

@ -0,0 +1,28 @@
FROM ghcr.io/scality/federation/nodesvc-base:7.10.6.0
ENV S3_CONFIG_FILE=${CONF_DIR}/config.json
ENV S3_LOCATION_FILE=${CONF_DIR}/locationConfig.json
COPY . ${HOME_DIR}/s3
RUN chown -R ${USER} ${HOME_DIR}
RUN pip3 install redis===3.5.3 requests==2.27.1 && \
apt-get install -y git-lfs
USER ${USER}
WORKDIR ${HOME_DIR}/s3
RUN rm -f ~/.gitconfig && \
git config --global --add safe.directory . && \
git lfs install && \
GIT_LFS_SKIP_SMUDGE=1 && \
yarn global add typescript && \
yarn install --frozen-lockfile --production --network-concurrency 1 && \
yarn cache clean --all && \
yarn global remove typescript
# run symlinking separately to avoid yarn installation errors
# we might have to check if the symlinking is really needed!
RUN ln -sf /scality-kms node_modules
EXPOSE 8000
CMD bash -c "source ${CONF_DIR}/env && export && supervisord -c ${CONF_DIR}/supervisord.conf"

View File

@ -1,3 +1,10 @@
'use strict'; // eslint-disable-line strict
require('werelogs').stderrUtils.catchAndTimestampStderr(
undefined,
// Do not exit as workers have their own listener that will exit
// But primary don't have another listener
require('cluster').isPrimary ? 1 : null,
);
require('./lib/server.js')();

File diff suppressed because it is too large Load Diff

View File

@ -1,24 +1,41 @@
const { auth, errors } = require('arsenal');
const { auth, errors, policies } = require('arsenal');
const async = require('async');
const bucketDelete = require('./bucketDelete');
const bucketDeleteCors = require('./bucketDeleteCors');
const bucketDeleteEncryption = require('./bucketDeleteEncryption');
const bucketDeleteWebsite = require('./bucketDeleteWebsite');
const bucketDeleteLifecycle = require('./bucketDeleteLifecycle');
const bucketGet = require('./bucketGet');
const bucketDeletePolicy = require('./bucketDeletePolicy');
const bucketDeleteQuota = require('./bucketDeleteQuota');
const { bucketGet } = require('./bucketGet');
const bucketGetACL = require('./bucketGetACL');
const bucketGetCors = require('./bucketGetCors');
const bucketGetVersioning = require('./bucketGetVersioning');
const bucketGetWebsite = require('./bucketGetWebsite');
const bucketGetLocation = require('./bucketGetLocation');
const bucketGetLifecycle = require('./bucketGetLifecycle');
const bucketGetNotification = require('./bucketGetNotification');
const bucketGetObjectLock = require('./bucketGetObjectLock');
const bucketGetPolicy = require('./bucketGetPolicy');
const bucketGetQuota = require('./bucketGetQuota');
const bucketGetEncryption = require('./bucketGetEncryption');
const bucketHead = require('./bucketHead');
const { bucketPut } = require('./bucketPut');
const bucketPutACL = require('./bucketPutACL');
const bucketPutCors = require('./bucketPutCors');
const bucketPutVersioning = require('./bucketPutVersioning');
const bucketPutTagging = require('./bucketPutTagging');
const bucketDeleteTagging = require('./bucketDeleteTagging');
const bucketGetTagging = require('./bucketGetTagging');
const bucketPutWebsite = require('./bucketPutWebsite');
const bucketPutReplication = require('./bucketPutReplication');
const bucketPutLifecycle = require('./bucketPutLifecycle');
const bucketPutNotification = require('./bucketPutNotification');
const bucketPutEncryption = require('./bucketPutEncryption');
const bucketPutPolicy = require('./bucketPutPolicy');
const bucketPutObjectLock = require('./bucketPutObjectLock');
const bucketUpdateQuota = require('./bucketUpdateQuota');
const bucketGetReplication = require('./bucketGetReplication');
const bucketDeleteReplication = require('./bucketDeleteReplication');
const corsPreflight = require('./corsPreflight');
@ -26,40 +43,75 @@ const completeMultipartUpload = require('./completeMultipartUpload');
const initiateMultipartUpload = require('./initiateMultipartUpload');
const listMultipartUploads = require('./listMultipartUploads');
const listParts = require('./listParts');
const metadataSearch = require('./metadataSearch');
const { multiObjectDelete } = require('./multiObjectDelete');
const multipartDelete = require('./multipartDelete');
const objectCopy = require('./objectCopy');
const objectDelete = require('./objectDelete');
const { objectDelete } = require('./objectDelete');
const objectDeleteTagging = require('./objectDeleteTagging');
const objectGet = require('./objectGet');
const objectGetACL = require('./objectGetACL');
const objectGetLegalHold = require('./objectGetLegalHold');
const objectGetRetention = require('./objectGetRetention');
const objectGetTagging = require('./objectGetTagging');
const objectHead = require('./objectHead');
const objectPut = require('./objectPut');
const objectPutACL = require('./objectPutACL');
const objectPutLegalHold = require('./objectPutLegalHold');
const objectPutTagging = require('./objectPutTagging');
const objectPutPart = require('./objectPutPart');
const objectPutCopyPart = require('./objectPutCopyPart');
const objectPutRetention = require('./objectPutRetention');
const objectRestore = require('./objectRestore');
const prepareRequestContexts
= require('./apiUtils/authorization/prepareRequestContexts');
const serviceGet = require('./serviceGet');
const vault = require('../auth/vault');
const websiteGet = require('./websiteGet');
const websiteHead = require('./websiteHead');
const website = require('./website');
const writeContinue = require('../utilities/writeContinue');
const validateQueryAndHeaders = require('../utilities/validateQueryAndHeaders');
const parseCopySource = require('./apiUtils/object/parseCopySource');
const { tagConditionKeyAuth } = require('./apiUtils/authorization/tagConditionKeys');
const { isRequesterASessionUser } = require('./apiUtils/authorization/permissionChecks');
const checkHttpHeadersSize = require('./apiUtils/object/checkHttpHeadersSize');
const monitoringMap = policies.actionMaps.actionMonitoringMapS3;
auth.setHandler(vault);
/* eslint-disable no-param-reassign */
const api = {
callApiMethod(apiMethod, request, response, log, callback) {
// Attach the apiMethod method to the request, so it can used by monitoring in the server
// eslint-disable-next-line no-param-reassign
request.apiMethod = apiMethod;
// Array of end of API callbacks, used to perform some logic
// at the end of an API.
// eslint-disable-next-line no-param-reassign
request.finalizerHooks = [];
const actionLog = monitoringMap[apiMethod];
if (!actionLog &&
apiMethod !== 'websiteGet' &&
apiMethod !== 'websiteHead' &&
apiMethod !== 'corsPreflight') {
log.error('callApiMethod(): No actionLog for this api method', {
apiMethod,
});
}
log.addDefaultFields({
service: 's3',
action: actionLog,
bucketName: request.bucketName,
});
if (request.objectKey) {
log.addDefaultFields({
objectKey: request.objectKey,
});
}
let returnTagCount = true;
const validationRes =
validateQueryAndHeaders(request.method, request.query,
request.headers, log);
const validationRes = validateQueryAndHeaders(request, log);
if (validationRes.error) {
log.debug('request query / header validation failed', {
error: validationRes.error,
@ -71,6 +123,7 @@ const api = {
// no need to check auth on website or cors preflight requests
if (apiMethod === 'websiteGet' || apiMethod === 'websiteHead' ||
apiMethod === 'corsPreflight') {
request.actionImplicitDenies = false;
return this[apiMethod](request, log, callback);
}
@ -83,128 +136,242 @@ const api = {
return process.nextTick(callback, parsingError);
}
const { httpHeadersSizeError } = checkHttpHeadersSize(request.headers);
if (httpHeadersSizeError) {
log.debug('http header size limit exceeded', {
error: httpHeadersSizeError,
});
return process.nextTick(callback, httpHeadersSizeError);
}
const requestContexts = prepareRequestContexts(apiMethod, request,
sourceBucket, sourceObject, sourceVersionId);
return auth.server.doAuth(request, log, (err, userInfo,
authorizationResults, streamingV4Params) => {
// Extract all the _apiMethods and store them in an array
const apiMethods = requestContexts ? requestContexts.map(context => context._apiMethod) : [];
// Attach the names to the current request
// eslint-disable-next-line no-param-reassign
request.apiMethods = apiMethods;
function checkAuthResults(authResults) {
let returnTagCount = true;
const isImplicitDeny = {};
let isOnlyImplicitDeny = true;
if (apiMethod === 'objectGet') {
// first item checks s3:GetObject(Version) action
if (!authResults[0].isAllowed && !authResults[0].isImplicit) {
log.trace('get object authorization denial from Vault');
return errors.AccessDenied;
}
// TODO add support for returnTagCount in the bucket policy
// checks
isImplicitDeny[authResults[0].action] = authResults[0].isImplicit;
// second item checks s3:GetObject(Version)Tagging action
if (!authResults[1].isAllowed) {
log.trace('get tagging authorization denial ' +
'from Vault');
returnTagCount = false;
}
} else {
for (let i = 0; i < authResults.length; i++) {
isImplicitDeny[authResults[i].action] = true;
if (!authResults[i].isAllowed && !authResults[i].isImplicit) {
// Any explicit deny rejects the current API call
log.trace('authorization denial from Vault');
return errors.AccessDenied;
}
if (authResults[i].isAllowed) {
// If the action is allowed, the result is not implicit
// Deny.
isImplicitDeny[authResults[i].action] = false;
isOnlyImplicitDeny = false;
}
}
}
// These two APIs cannot use ACLs or Bucket Policies, hence, any
// implicit deny from vault must be treated as an explicit deny.
if ((apiMethod === 'bucketPut' || apiMethod === 'serviceGet') && isOnlyImplicitDeny) {
return errors.AccessDenied;
}
return { returnTagCount, isImplicitDeny };
}
return async.waterfall([
next => auth.server.doAuth(
request, log, (err, userInfo, authorizationResults, streamingV4Params, infos) => {
if (err) {
// VaultClient returns standard errors, but the route requires
// Arsenal errors
const arsenalError = err.metadata ? err : errors[err.code] || errors.InternalError;
log.trace('authentication error', { error: err });
return next(arsenalError);
}
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
}, 's3', requestContexts),
(userInfo, authorizationResults, streamingV4Params, infos, next) => {
const authNames = { accountName: userInfo.getAccountDisplayName() };
if (userInfo.isRequesterAnIAMUser()) {
authNames.userName = userInfo.getIAMdisplayName();
}
if (isRequesterASessionUser(userInfo)) {
authNames.sessionName = userInfo.getShortid().split(':')[1];
}
log.addDefaultFields(authNames);
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
}
// issue 100 Continue to the client
writeContinue(request, response);
const MAX_POST_LENGTH = request.method === 'POST' ?
1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
const post = [];
let postLength = 0;
request.on('data', chunk => {
postLength += chunk.length;
// Sanity check on post length
if (postLength <= MAX_POST_LENGTH) {
post.push(chunk);
}
});
request.on('error', err => {
log.trace('error receiving request', {
error: err,
});
return next(errors.InternalError);
});
request.on('end', () => {
if (postLength > MAX_POST_LENGTH) {
log.error('body length is too long for request type',
{ postLength });
return next(errors.InvalidRequest);
}
// Convert array of post buffers into one string
request.post = Buffer.concat(post, postLength).toString();
return next(null, userInfo, authorizationResults, streamingV4Params, infos);
});
return undefined;
},
// Tag condition keys require information from CloudServer for evaluation
(userInfo, authorizationResults, streamingV4Params, infos, next) => tagConditionKeyAuth(
authorizationResults,
request,
requestContexts,
apiMethod,
log,
(err, authResultsWithTags) => {
if (err) {
log.trace('tag authentication error', { error: err });
return next(err);
}
return next(null, userInfo, authResultsWithTags, streamingV4Params, infos);
},
),
], (err, userInfo, authorizationResults, streamingV4Params, infos) => {
if (err) {
log.trace('authentication error', { error: err });
return callback(err);
}
request.accountQuotas = infos?.accountQuota;
if (authorizationResults) {
if (apiMethod === 'objectGet') {
// first item checks s3:GetObject(Version) action
if (!authorizationResults[0].isAllowed) {
log.trace('get object authorization denial from Vault');
return callback(errors.AccessDenied);
}
// second item checks s3:GetObject(Version)Tagging action
if (!authorizationResults[1].isAllowed) {
log.trace('get tagging authorization denial ' +
'from Vault');
returnTagCount = false;
}
} else {
for (let i = 0; i < authorizationResults.length; i++) {
if (!authorizationResults[i].isAllowed) {
log.trace('authorization denial from Vault');
return callback(errors.AccessDenied);
}
}
const checkedResults = checkAuthResults(authorizationResults);
if (checkedResults instanceof Error) {
return callback(checkedResults);
}
returnTagCount = checkedResults.returnTagCount;
request.actionImplicitDenies = checkedResults.isImplicitDeny;
} else {
// create an object of keys apiMethods with all values to false:
// for backward compatibility, all apiMethods are allowed by default
// thus it is explicitly allowed, so implicit deny is false
request.actionImplicitDenies = apiMethods.reduce((acc, curr) => {
acc[curr] = false;
return acc;
}, {});
}
// issue 100 Continue to the client
writeContinue(request, response);
const methodCallback = (err, ...results) => async.forEachLimit(request.finalizerHooks, 5,
(hook, done) => hook(err, done),
() => callback(err, ...results));
if (apiMethod === 'objectPut' || apiMethod === 'objectPutPart') {
request._response = response;
return this[apiMethod](userInfo, request, streamingV4Params,
log, callback);
log, methodCallback, authorizationResults);
}
const MAX_POST_LENGTH = request.method.toUpperCase() === 'POST' ?
1024 * 1024 : 1024 * 1024 / 2; // 1 MB or 512 KB
const post = [];
let postLength = 0;
request.on('data', chunk => {
postLength += chunk.length;
// Sanity check on post length
if (postLength <= MAX_POST_LENGTH) {
post.push(chunk);
}
return undefined;
});
request.on('error', err => {
log.trace('error receiving request', {
error: err,
});
return callback(errors.InternalError);
});
request.on('end', () => {
if (postLength > MAX_POST_LENGTH) {
log.error('body length is too long for request type',
{ postLength });
return callback(errors.InvalidRequest);
}
// Convert array of post buffers into one string
request.post = Buffer.concat(post, postLength).toString();
if (apiMethod === 'objectCopy' ||
apiMethod === 'objectPutCopyPart') {
return this[apiMethod](userInfo, request, sourceBucket,
sourceObject, sourceVersionId, log, callback);
}
if (apiMethod === 'objectGet') {
return this[apiMethod](userInfo, request,
returnTagCount, log, callback);
}
return this[apiMethod](userInfo, request, log, callback);
});
return undefined;
}, 's3', requestContexts);
if (apiMethod === 'objectCopy' || apiMethod === 'objectPutCopyPart') {
return this[apiMethod](userInfo, request, sourceBucket,
sourceObject, sourceVersionId, log, methodCallback);
}
if (apiMethod === 'objectGet') {
return this[apiMethod](userInfo, request, returnTagCount, log, callback);
}
return this[apiMethod](userInfo, request, log, methodCallback);
});
},
bucketDelete,
bucketDeleteCors,
bucketDeleteEncryption,
bucketDeleteWebsite,
bucketGet,
bucketGetACL,
bucketGetCors,
bucketGetObjectLock,
bucketGetVersioning,
bucketGetWebsite,
bucketGetLocation,
bucketGetEncryption,
bucketHead,
bucketPut,
bucketPutACL,
bucketPutCors,
bucketPutVersioning,
bucketPutTagging,
bucketDeleteTagging,
bucketGetTagging,
bucketPutWebsite,
bucketPutReplication,
bucketGetReplication,
bucketDeleteReplication,
bucketDeleteQuota,
bucketPutLifecycle,
bucketUpdateQuota,
bucketGetLifecycle,
bucketDeleteLifecycle,
bucketPutPolicy,
bucketGetPolicy,
bucketGetQuota,
bucketDeletePolicy,
bucketPutObjectLock,
bucketPutNotification,
bucketGetNotification,
bucketPutEncryption,
corsPreflight,
completeMultipartUpload,
initiateMultipartUpload,
listMultipartUploads,
listParts,
metadataSearch,
multiObjectDelete,
multipartDelete,
objectDelete,
objectDeleteTagging,
objectGet,
objectGetACL,
objectGetLegalHold,
objectGetRetention,
objectGetTagging,
objectCopy,
objectHead,
objectPut,
objectPutACL,
objectPutLegalHold,
objectPutTagging,
objectPutPart,
objectPutCopyPart,
objectPutRetention,
objectRestore,
serviceGet,
websiteGet,
websiteHead,
websiteGet: website,
websiteHead: website,
};
module.exports = api;

View File

@ -1,135 +0,0 @@
const constants = require('../../../../constants');
function isBackbeatUser(canonicalID) {
const canonicalIDArray = canonicalID.split('/');
const serviceName = canonicalIDArray[canonicalIDArray.length - 1];
return ['replication', 'lifecycle', 'gc'].includes(serviceName);
}
function isBucketAuthorized(bucket, requestType, canonicalID) {
// Check to see if user is authorized to perform a
// particular action on bucket based on ACLs.
// TODO: Add IAM checks and bucket policy checks.
if (bucket.getOwner() === canonicalID || isBackbeatUser(canonicalID)) {
return true;
} else if (requestType === 'bucketOwnerAction') {
// only bucket owner can modify or retrieve this property of a bucket
return false;
}
const bucketAcl = bucket.getAcl();
if (requestType === 'bucketGet' || requestType === 'bucketHead') {
if (bucketAcl.Canned === 'public-read'
|| bucketAcl.Canned === 'public-read-write'
|| (bucketAcl.Canned === 'authenticated-read'
&& canonicalID !== constants.publicId)) {
return true;
} else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.READ.indexOf(canonicalID) > -1) {
return true;
}
}
if (requestType === 'bucketGetACL') {
if ((bucketAcl.Canned === 'log-delivery-write'
&& canonicalID === constants.logId)
|| bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.READ_ACP.indexOf(canonicalID) > -1) {
return true;
}
}
if (requestType === 'bucketPutACL') {
if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) {
return true;
}
}
if (requestType === 'bucketDelete' && bucket.getOwner() === canonicalID) {
return true;
}
if (requestType === 'objectDelete' || requestType === 'objectPut') {
if (bucketAcl.Canned === 'public-read-write'
|| bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.WRITE.indexOf(canonicalID) > -1) {
return true;
}
}
// Note that an account can have the ability to do objectPutACL,
// objectGetACL, objectHead or objectGet even if the account has no rights
// to the bucket holding the object. So, if the request type is
// objectPutACL, objectGetACL, objectHead or objectGet, the bucket
// authorization check should just return true so can move on to check
// rights at the object level.
return (requestType === 'objectPutACL' || requestType === 'objectGetACL' ||
requestType === 'objectGet' || requestType === 'objectHead');
}
function isObjAuthorized(bucket, objectMD, requestType, canonicalID) {
const bucketOwner = bucket.getOwner();
if (!objectMD) {
return false;
}
if (objectMD['owner-id'] === canonicalID) {
return true;
}
if (isBackbeatUser(canonicalID)) {
return true;
}
// account is authorized if:
// - requesttype is "bucketOwnerAction" (example: for objectTagging) and
// - account is the bucket owner
if (requestType === 'bucketOwnerAction' && bucketOwner === canonicalID) {
return true;
}
if (requestType === 'objectGet' || requestType === 'objectHead') {
if (objectMD.acl.Canned === 'public-read'
|| objectMD.acl.Canned === 'public-read-write'
|| (objectMD.acl.Canned === 'authenticated-read'
&& canonicalID !== constants.publicId)) {
return true;
} else if (objectMD.acl.Canned === 'bucket-owner-read'
&& bucketOwner === canonicalID) {
return true;
} else if ((objectMD.acl.Canned === 'bucket-owner-full-control'
&& bucketOwner === canonicalID)
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|| objectMD.acl.READ.indexOf(canonicalID) > -1) {
return true;
}
}
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
// bucket has canned ACL public-read-write
if (requestType === 'objectPut' || requestType === 'objectDelete') {
return true;
}
if (requestType === 'objectPutACL') {
if ((objectMD.acl.Canned === 'bucket-owner-full-control'
&& bucketOwner === canonicalID)
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|| objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1) {
return true;
}
}
if (requestType === 'objectGetACL') {
if ((objectMD.acl.Canned === 'bucket-owner-full-control'
&& bucketOwner === canonicalID)
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|| objectMD.acl.READ_ACP.indexOf(canonicalID) > -1) {
return true;
}
}
return false;
}
module.exports = {
isBucketAuthorized,
isObjAuthorized,
isBackbeatUser,
};

View File

@ -0,0 +1,29 @@
const { errors } = require('arsenal');
const vault = require('../../../auth/vault');
function checkExpectedBucketOwner(headers, bucket, log, cb) {
const expectedOwner = headers['x-amz-expected-bucket-owner'];
if (expectedOwner === undefined) {
return cb();
}
const bucketOwner = bucket.getOwner();
return vault.getAccountIds([bucketOwner], log, (error, res) => {
if (error) {
log.error('error fetch accountId from vault', {
method: 'checkExpectedBucketOwner',
error,
});
}
if (error || res[bucketOwner] !== expectedOwner) {
return cb(errors.AccessDenied);
}
return cb();
});
}
module.exports = {
checkExpectedBucketOwner,
};

View File

@ -0,0 +1,641 @@
const { evaluators, actionMaps, RequestContext, requestUtils } = require('arsenal').policies;
const { errors } = require('arsenal');
const { parseCIDR, isValid } = require('ipaddr.js');
const constants = require('../../../../constants');
const { config } = require('../../../Config');
const {
allAuthedUsersId,
bucketOwnerActions,
logId,
publicId,
arrayOfAllowed,
assumedRoleArnResourceType,
backbeatLifecycleSessionName,
actionsToConsiderAsObjectPut,
} = constants;
// whitelist buckets to allow public read on objects
const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS
? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : [];
function getServiceAccountProperties(canonicalID) {
const canonicalIDArray = canonicalID.split('/');
const serviceName = canonicalIDArray[canonicalIDArray.length - 1];
return constants.serviceAccountProperties[serviceName];
}
function isServiceAccount(canonicalID) {
return getServiceAccountProperties(canonicalID) !== undefined;
}
function isRequesterASessionUser(authInfo) {
const regexpAssumedRoleArn = /^arn:aws:sts::[0-9]{12}:assumed-role\/.*$/;
return regexpAssumedRoleArn.test(authInfo.getArn());
}
function isRequesterNonAccountUser(authInfo) {
return authInfo.isRequesterAnIAMUser() || isRequesterASessionUser(authInfo);
}
/**
* Checks the access control for a given bucket based on the request type and user's canonical ID.
*
* @param {Bucket} bucket - The bucket to check access control for.
* @param {string} requestType - The list of s3 actions to check within the API call.
* @param {string} canonicalID - The canonical ID of the user making the request.
* @param {string} mainApiCall - The main API call (first item of the requestType).
*
* @returns {boolean} - Returns true if the user has the necessary access rights, otherwise false.
*/
function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) {
// Same logic applies on the Versioned APIs, so let's simplify it.
let requestTypeParsed = requestType.endsWith('Version') ?
requestType.slice(0, 'Version'.length * -1) : requestType;
requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ?
'objectPut' : requestTypeParsed;
const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
'objectPut' : mainApiCall;
if (bucket.getOwner() === canonicalID) {
return true;
}
if (parsedMainApiCall === 'objectGet') {
if (requestTypeParsed === 'objectGetTagging') {
return true;
}
}
if (parsedMainApiCall === 'objectPut') {
if (arrayOfAllowed.includes(requestTypeParsed)) {
return true;
}
}
const bucketAcl = bucket.getAcl();
if (requestTypeParsed === 'bucketGet' || requestTypeParsed === 'bucketHead') {
if (bucketAcl.Canned === 'public-read'
|| bucketAcl.Canned === 'public-read-write'
|| (bucketAcl.Canned === 'authenticated-read'
&& canonicalID !== publicId)) {
return true;
} else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.READ.indexOf(canonicalID) > -1) {
return true;
} else if (bucketAcl.READ.indexOf(publicId) > -1
|| (bucketAcl.READ.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
}
}
if (requestTypeParsed === 'bucketGetACL') {
if ((bucketAcl.Canned === 'log-delivery-write'
&& canonicalID === logId)
|| bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.READ_ACP.indexOf(canonicalID) > -1) {
return true;
} else if (bucketAcl.READ_ACP.indexOf(publicId) > -1
|| (bucketAcl.READ_ACP.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
}
}
if (requestTypeParsed === 'bucketPutACL') {
if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) {
return true;
} else if (bucketAcl.WRITE_ACP.indexOf(publicId) > -1
|| (bucketAcl.WRITE_ACP.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
}
}
if (requestTypeParsed === 'objectDelete' || requestTypeParsed === 'objectPut') {
if (bucketAcl.Canned === 'public-read-write'
|| bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1
|| bucketAcl.WRITE.indexOf(canonicalID) > -1) {
return true;
} else if (bucketAcl.WRITE.indexOf(publicId) > -1
|| (bucketAcl.WRITE.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
}
}
// Note that an account can have the ability to do objectPutACL,
// objectGetACL, objectHead or objectGet even if the account has no rights
// to the bucket holding the object. So, if the request type is
// objectPutACL, objectGetACL, objectHead or objectGet, the bucket
// authorization check should just return true so can move on to check
// rights at the object level.
return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL'
|| requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
}
function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser,
isUserUnauthenticated, mainApiCall) {
const bucketOwner = bucket.getOwner();
const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ?
'objectPut' : requestType;
const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ?
'objectPut' : mainApiCall;
// acls don't distinguish between users and accounts, so both should be allowed
if (bucketOwnerActions.includes(requestTypeParsed)
&& (bucketOwner === canonicalID)) {
return true;
}
if (objectMD['owner-id'] === canonicalID) {
return true;
}
// Backward compatibility
if (parsedMainApiCall === 'objectGet') {
if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id']))
&& requestTypeParsed === 'objectGetTagging') {
return true;
}
}
if (!objectMD.acl) {
return false;
}
if (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead') {
if (objectMD.acl.Canned === 'public-read'
|| objectMD.acl.Canned === 'public-read-write'
|| (objectMD.acl.Canned === 'authenticated-read'
&& canonicalID !== publicId)) {
return true;
} else if (objectMD.acl.Canned === 'bucket-owner-read'
&& bucketOwner === canonicalID) {
return true;
} else if ((objectMD.acl.Canned === 'bucket-owner-full-control'
&& bucketOwner === canonicalID)
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|| objectMD.acl.READ.indexOf(canonicalID) > -1) {
return true;
} else if (objectMD.acl.READ.indexOf(publicId) > -1
|| (objectMD.acl.READ.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
}
}
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
// bucket has canned ACL public-read-write
if (requestTypeParsed === 'objectPut' || requestTypeParsed === 'objectDelete') {
return true;
}
if (requestTypeParsed === 'objectPutACL') {
if ((objectMD.acl.Canned === 'bucket-owner-full-control'
&& bucketOwner === canonicalID)
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|| objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1) {
return true;
} else if (objectMD.acl.WRITE_ACP.indexOf(publicId) > -1
|| (objectMD.acl.WRITE_ACP.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
}
}
if (requestTypeParsed === 'objectGetACL') {
if ((objectMD.acl.Canned === 'bucket-owner-full-control'
&& bucketOwner === canonicalID)
|| objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1
|| objectMD.acl.READ_ACP.indexOf(canonicalID) > -1) {
return true;
} else if (objectMD.acl.READ_ACP.indexOf(publicId) > -1
|| (objectMD.acl.READ_ACP.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1
&& canonicalID !== publicId)
|| objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) {
return true;
}
}
// allow public reads on buckets that are whitelisted for anonymous reads
// TODO: remove this after bucket policies are implemented
const bucketAcl = bucket.getAcl();
const allowPublicReads = publicReadBuckets.includes(bucket.getName())
&& bucketAcl.Canned === 'public-read'
&& (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead');
if (allowPublicReads) {
return true;
}
return false;
}
function _checkBucketPolicyActions(requestType, actions, log) {
const mappedAction = actionMaps.actionMapBP[requestType];
// Deny any action that isn't in list of controlled actions
if (!mappedAction) {
return false;
}
return evaluators.isActionApplicable(mappedAction, actions, log);
}
function _checkBucketPolicyResources(request, resource, log) {
if (!request || (Array.isArray(resource) && resource.length === 0)) {
return true;
}
// build request context from the request!
const requestContext = new RequestContext(request.headers, request.query,
request.bucketName, request.objectKey, null,
request.connection.encrypted, request.resourceType, 's3');
return evaluators.isResourceApplicable(requestContext, resource, log);
}
function _checkBucketPolicyConditions(request, conditions, log) {
const ip = request ? requestUtils.getClientIp(request, config) : undefined;
if (!conditions) {
return true;
}
// build request context from the request!
const requestContext = new RequestContext(request.headers, request.query,
request.bucketName, request.objectKey, ip,
request.connection.encrypted, request.resourceType, 's3', null, null,
null, null, null, null, null, null, null, null, null,
request.objectLockRetentionDays);
return evaluators.meetConditions(requestContext, conditions, log);
}
function _getAccountId(arn) {
// account or user arn is of format 'arn:aws:iam::<12-digit-acct-id>:etc...
return arn.substr(13, 12);
}
function _isAccountId(principal) {
return (principal.length === 12 && /^\d+$/.test(principal));
}
function _checkPrincipal(requester, principal) {
if (principal === '*') {
return true;
}
if (principal === requester) {
return true;
}
if (_isAccountId(principal)) {
return _getAccountId(requester) === principal;
}
if (principal.endsWith('root')) {
return _getAccountId(requester) === _getAccountId(principal);
}
return false;
}
function _checkPrincipals(canonicalID, arn, principal) {
if (principal === '*') {
return true;
}
if (principal.CanonicalUser) {
if (Array.isArray(principal.CanonicalUser)) {
return principal.CanonicalUser.some(p => _checkPrincipal(canonicalID, p));
}
return _checkPrincipal(canonicalID, principal.CanonicalUser);
}
if (principal.AWS) {
if (Array.isArray(principal.AWS)) {
return principal.AWS.some(p => _checkPrincipal(arn, p));
}
return _checkPrincipal(arn, principal.AWS);
}
return false;
}
function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, log, request, actionImplicitDenies) {
let permission = 'defaultDeny';
// if requester is user within bucket owner account, actions should be
// allowed unless explicitly denied (assumes allowed by IAM policy)
if (bucketOwner === canonicalID && actionImplicitDenies[requestType] === false) {
permission = 'allow';
}
let copiedStatement = JSON.parse(JSON.stringify(policy.Statement));
while (copiedStatement.length > 0) {
const s = copiedStatement[0];
const principalMatch = _checkPrincipals(canonicalID, arn, s.Principal);
const actionMatch = _checkBucketPolicyActions(requestType, s.Action, log);
const resourceMatch = _checkBucketPolicyResources(request, s.Resource, log);
const conditionsMatch = _checkBucketPolicyConditions(request, s.Condition, log);
if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Deny') {
// explicit deny trumps any allows, so return immediately
return 'explicitDeny';
}
if (principalMatch && actionMatch && resourceMatch && conditionsMatch && s.Effect === 'Allow') {
permission = 'allow';
}
copiedStatement = copiedStatement.splice(1);
}
return permission;
}
function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, log,
request, aclPermission, results, actionImplicitDenies) {
const bucketPolicy = bucket.getBucketPolicy();
let processedResult = results[requestType];
if (!bucketPolicy) {
processedResult = actionImplicitDenies[requestType] === false && aclPermission;
} else {
const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType, canonicalID, arn,
bucketOwner, log, request, actionImplicitDenies);
if (bucketPolicyPermission === 'explicitDeny') {
processedResult = false;
} else if (bucketPolicyPermission === 'allow') {
processedResult = true;
} else {
processedResult = actionImplicitDenies[requestType] === false && aclPermission;
}
}
return processedResult;
}
function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, log, request,
actionImplicitDeniesInput = {}, isWebsite = false) {
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
const mainApiCall = requestTypes[0];
const results = {};
return requestTypes.every(_requestType => {
// By default, all missing actions are defined as allowed from IAM, to be
// backward compatible
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
// Check to see if user is authorized to perform a
// particular action on bucket based on ACLs.
// TODO: Add IAM checks
let requesterIsNotUser = true;
let arn = null;
if (authInfo) {
requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
arn = authInfo.getArn();
}
// if the bucket owner is an account, users should not have default access
if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) {
results[_requestType] = actionImplicitDenies[_requestType] === false;
return results[_requestType];
}
const aclPermission = checkBucketAcls(bucket, _requestType, canonicalID, mainApiCall);
// In case of error bucket access is checked with bucketGet
// For website, bucket policy only uses objectGet and ignores bucketGet
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/WebsiteAccessPermissionsReqd.html
// bucketGet should be used to check acl but switched to objectGet for bucket policy
if (isWebsite && _requestType === 'bucketGet') {
// eslint-disable-next-line no-param-reassign
_requestType = 'objectGet';
actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false;
}
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
request, aclPermission, results, actionImplicitDenies);
});
}
function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, authInfo, actionImplicitDeniesInput = {},
log, request) {
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
const results = {};
return requestTypes.every(_requestType => {
// By default, all missing actions are defined as allowed from IAM, to be
// backward compatible
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
let arn = null;
if (authInfo) {
arn = authInfo.getArn();
}
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log,
request, true, results, actionImplicitDenies);
});
}
function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authInfo, log, request,
actionImplicitDeniesInput = {}, isWebsite = false) {
const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput];
const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput;
const results = {};
const mainApiCall = requestTypes[0];
return requestTypes.every(_requestType => {
// By default, all missing actions are defined as allowed from IAM, to be
// backward compatible
actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false;
const parsedMethodName = _requestType.endsWith('Version')
? _requestType.slice(0, -7) : _requestType;
const bucketOwner = bucket.getOwner();
if (!objectMD) {
// check bucket has read access
// 'bucketGet' covers listObjects and listMultipartUploads, bucket read actions
let permission = 'bucketGet';
if (actionsToConsiderAsObjectPut.includes(_requestType)) {
permission = 'objectPut';
}
results[_requestType] = isBucketAuthorized(bucket, permission, canonicalID, authInfo, log, request,
actionImplicitDenies, isWebsite);
// User is already authorized on the bucket for FULL_CONTROL or WRITE or
// bucket has canned ACL public-read-write
if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete')
&& results[_requestType] === false) {
results[_requestType] = actionImplicitDenies[_requestType] === false;
}
return results[_requestType];
}
let requesterIsNotUser = true;
let arn = null;
let isUserUnauthenticated = false;
if (authInfo) {
requesterIsNotUser = !isRequesterNonAccountUser(authInfo);
arn = authInfo.getArn();
isUserUnauthenticated = arn === undefined;
}
if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) {
results[_requestType] = actionImplicitDenies[_requestType] === false;
return results[_requestType];
}
// account is authorized if:
// - requesttype is included in bucketOwnerActions and
// - account is the bucket owner
// - requester is account, not user
if (bucketOwnerActions.includes(parsedMethodName)
&& (bucketOwner === canonicalID)
&& requesterIsNotUser) {
results[_requestType] = actionImplicitDenies[_requestType] === false;
return results[_requestType];
}
const aclPermission = checkObjectAcls(bucket, objectMD, parsedMethodName,
canonicalID, requesterIsNotUser, isUserUnauthenticated, mainApiCall);
return processBucketPolicy(_requestType, bucket, canonicalID, arn, bucketOwner,
log, request, aclPermission, results, actionImplicitDenies);
});
}
function _checkResource(resource, bucketArn) {
if (resource === bucketArn) {
return true;
}
if (resource.includes('/')) {
const rSubs = resource.split('/');
return rSubs[0] === bucketArn;
}
return false;
}
// the resources specified in the bucket policy should contain the bucket name
function validatePolicyResource(bucketName, policy) {
const bucketArn = `arn:aws:s3:::${bucketName}`;
return policy.Statement.every(s => {
if (Array.isArray(s.Resource)) {
return s.Resource.every(r => _checkResource(r, bucketArn));
}
if (typeof s.Resource === 'string') {
return _checkResource(s.Resource, bucketArn);
}
return false;
});
}
function checkIp(value) {
const errString = 'Invalid IP address in Conditions';
const values = Array.isArray(value) ? value : [value];
for (let i = 0; i < values.length; i++) {
// these preliminary checks are validating the provided
// ip address against ipaddr.js, the library we use when
// evaluating IP condition keys. It ensures compatibility,
// but additional checks are required to enforce the right
// notation (e.g., xxx.xxx.xxx.xxx/xx for IPv4). Otherwise,
// we would accept different ip formats, which is not
// standard in an AWS use case.
try {
try {
parseCIDR(values[i]);
} catch (err) {
isValid(values[i]);
}
} catch (err) {
return errString;
}
// Apply the existing IP validation logic to each element
const validateIpRegex = ip => {
if (constants.ipv4Regex.test(ip)) {
return ip.split('.').every(part => parseInt(part, 10) <= 255);
}
if (constants.ipv6Regex.test(ip)) {
return ip.split(':').every(part => part.length <= 4);
}
return false;
};
if (validateIpRegex(values[i]) !== true) {
return errString;
}
}
// If the function hasn't returned by now, all elements are valid
return null;
}
// This function checks all bucket policy conditions if the values provided
// are valid for the condition type. If not it returns a relevant Malformed policy error string
function validatePolicyConditions(policy) {
const validConditions = [
{ conditionKey: 'aws:SourceIp', conditionValueTypeChecker: checkIp },
{ conditionKey: 's3:object-lock-remaining-retention-days' },
];
// keys where value type does not seem to be checked by AWS:
// - s3:object-lock-remaining-retention-days
if (!policy.Statement || !Array.isArray(policy.Statement) || policy.Statement.length === 0) {
return null;
}
// there can be multiple statements in the policy, each with a Condition enclosure
for (let i = 0; i < policy.Statement.length; i++) {
const s = policy.Statement[i];
if (s.Condition) {
const conditionOperators = Object.keys(s.Condition);
// there can be multiple condition operations in the Condition enclosure
// eslint-disable-next-line no-restricted-syntax
for (const conditionOperator of conditionOperators) {
const conditionKey = Object.keys(s.Condition[conditionOperator])[0];
const conditionValue = s.Condition[conditionOperator][conditionKey];
const validCondition = validConditions.find(validCondition =>
validCondition.conditionKey === conditionKey
);
// AWS returns does not return an error if the condition starts with 'aws:'
// so we reproduce this behaviour
if (!validCondition && !conditionKey.startsWith('aws:')) {
return errors.MalformedPolicy.customizeDescription('Policy has an invalid condition key');
}
if (validCondition && validCondition.conditionValueTypeChecker) {
const conditionValueTypeError = validCondition.conditionValueTypeChecker(conditionValue);
if (conditionValueTypeError) {
return errors.MalformedPolicy.customizeDescription(conditionValueTypeError);
}
}
}
}
}
return null;
}
/** isLifecycleSession - check if it is the Lifecycle assumed role session arn.
* @param {string} arn - Amazon resource name - example:
* arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle
* @return {boolean} true if Lifecycle assumed role session arn, false if not.
*/
function isLifecycleSession(arn) {
if (!arn) {
return false;
}
const arnSplits = arn.split(':');
const service = arnSplits[2];
const resourceNames = arnSplits[arnSplits.length - 1].split('/');
const resourceType = resourceNames[0];
const sessionName = resourceNames[resourceNames.length - 1];
return (service === 'sts'
&& resourceType === assumedRoleArnResourceType
&& sessionName === backbeatLifecycleSessionName);
}
module.exports = {
isBucketAuthorized,
isObjAuthorized,
getServiceAccountProperties,
isServiceAccount,
isRequesterASessionUser,
isRequesterNonAccountUser,
checkBucketAcls,
checkObjectAcls,
validatePolicyResource,
validatePolicyConditions,
isLifecycleSession,
evaluateBucketPolicyWithIAM,
};

View File

@ -1,10 +1,20 @@
const { policies } = require('arsenal');
const { config } = require('../../../Config');
const RequestContext = policies.RequestContext;
const { RequestContext, requestUtils } = policies;
let apiMethodAfterVersionCheck;
const apiMethodWithVersion = { objectGetACL: true, objectPutACL: true,
objectGet: true, objectDelete: true, objectPutTagging: true,
objectGetTagging: true, objectDeleteTagging: true };
const apiMethodWithVersion = {
objectGetACL: true,
objectPutACL: true,
objectGet: true,
objectDelete: true,
objectPutTagging: true,
objectGetTagging: true,
objectDeleteTagging: true,
objectGetLegalHold: true,
objectPutLegalHold: true,
objectPutRetention: true,
};
function isHeaderAcl(headers) {
return headers['x-amz-grant-read'] || headers['x-amz-grant-read-acp'] ||
@ -33,8 +43,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
// null as the requestContext to Vault so it will only do an authentication
// check.
const ip = request.headers['x-forwarded-for'] ||
request.socket.remoteAddress;
const ip = requestUtils.getClientIp(request, config);
function generateRequestContext(apiMethod) {
return new RequestContext(request.headers,
@ -43,7 +52,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
apiMethod, 's3');
}
if (apiMethod === 'multiObjectDelete' || apiMethod === 'bucketPut') {
if (apiMethod === 'bucketPut') {
return null;
}
@ -56,7 +65,17 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
const requestContexts = [];
if (apiMethodAfterVersionCheck === 'objectCopy'
if (apiMethod === 'multiObjectDelete') {
// MultiObjectDelete does not require any authorization when evaluating
// the API. Instead, we authorize each object passed.
// But in order to get any relevant information from the authorization service
// for example, the account quota, we must send a request context object
// with no `specificResource`. We expect the result to be an implicit deny.
// In the API, we then ignore these authorization results, and we can use
// any information returned, e.g., the quota.
const requestContextMultiObjectDelete = generateRequestContext('objectDelete');
requestContexts.push(requestContextMultiObjectDelete);
} else if (apiMethodAfterVersionCheck === 'objectCopy'
|| apiMethodAfterVersionCheck === 'objectPutCopyPart') {
const objectGetAction = sourceVersionId ? 'objectGetVersion' :
'objectGet';
@ -88,27 +107,126 @@ function prepareRequestContexts(apiMethod, request, sourceBucket,
const objectGetTaggingAction = (request.query &&
request.query.versionId) ? 'objectGetTaggingVersion' :
'objectGetTagging';
if (request.headers['x-amz-version-id']) {
const objectGetVersionAction = 'objectGetVersion';
const getVersionResourceVersion =
generateRequestContext(objectGetVersionAction);
requestContexts.push(getVersionResourceVersion);
}
const getRequestContext =
generateRequestContext(apiMethodAfterVersionCheck);
const getTaggingRequestContext =
generateRequestContext(objectGetTaggingAction);
requestContexts.push(getRequestContext, getTaggingRequestContext);
} else if (apiMethodAfterVersionCheck === 'objectPut') {
const putRequestContext =
generateRequestContext(apiMethodAfterVersionCheck);
requestContexts.push(putRequestContext);
// if put object (versioning) with tag set
if (request.headers['x-amz-tagging']) {
const putTaggingRequestContext =
generateRequestContext('objectPutTagging');
requestContexts.push(putTaggingRequestContext);
} else if (apiMethodAfterVersionCheck === 'objectGetTagging') {
const objectGetTaggingAction = 'objectGetTagging';
const getTaggingResourceVersion =
generateRequestContext(objectGetTaggingAction);
requestContexts.push(getTaggingResourceVersion);
if (request.headers['x-amz-version-id']) {
const objectGetTaggingVersionAction = 'objectGetTaggingVersion';
const getTaggingVersionResourceVersion =
generateRequestContext(objectGetTaggingVersionAction);
requestContexts.push(getTaggingVersionResourceVersion);
}
} else if (apiMethodAfterVersionCheck === 'objectHead') {
const objectHeadAction = 'objectHead';
const headObjectAction =
generateRequestContext(objectHeadAction);
requestContexts.push(headObjectAction);
if (request.headers['x-amz-version-id']) {
const objectHeadVersionAction = 'objectGetVersion';
const headObjectVersion =
generateRequestContext(objectHeadVersionAction);
requestContexts.push(headObjectVersion);
}
} else if (apiMethodAfterVersionCheck === 'objectPutTagging') {
const putObjectTaggingRequestContext =
generateRequestContext('objectPutTagging');
requestContexts.push(putObjectTaggingRequestContext);
if (request.headers['x-amz-version-id']) {
const putObjectVersionRequestContext =
generateRequestContext('objectPutTaggingVersion');
requestContexts.push(putObjectVersionRequestContext);
}
} else if (apiMethodAfterVersionCheck === 'objectPutCopyPart') {
const putObjectRequestContext =
generateRequestContext('objectPut');
requestContexts.push(putObjectRequestContext);
const getObjectRequestContext =
generateRequestContext('objectGet');
requestContexts.push(getObjectRequestContext);
} else if (apiMethodAfterVersionCheck === 'objectPut') {
// if put object with version
if (request.headers['x-scal-s3-version-id'] ||
request.headers['x-scal-s3-version-id'] === '') {
const putVersionRequestContext =
generateRequestContext('objectPutVersion');
requestContexts.push(putVersionRequestContext);
} else {
const putRequestContext =
generateRequestContext(apiMethodAfterVersionCheck);
requestContexts.push(putRequestContext);
// if put object (versioning) with tag set
if (request.headers['x-amz-tagging']) {
const putTaggingRequestContext =
generateRequestContext('objectPutTagging');
requestContexts.push(putTaggingRequestContext);
}
if (['ON', 'OFF'].includes(request.headers['x-amz-object-lock-legal-hold-status'])) {
const putLegalHoldStatusAction =
generateRequestContext('objectPutLegalHold');
requestContexts.push(putLegalHoldStatusAction);
}
// if put object (versioning) with ACL
if (isHeaderAcl(request.headers)) {
const putAclRequestContext =
generateRequestContext('objectPutACL');
requestContexts.push(putAclRequestContext);
}
if (request.headers['x-amz-object-lock-mode']) {
const putObjectLockRequestContext =
generateRequestContext('objectPutRetention');
requestContexts.push(putObjectLockRequestContext);
}
if (request.headers['x-amz-version-id']) {
const putObjectVersionRequestContext =
generateRequestContext('objectPutTaggingVersion');
requestContexts.push(putObjectVersionRequestContext);
}
}
} else if (apiMethodAfterVersionCheck === 'initiateMultipartUpload' ||
apiMethodAfterVersionCheck === 'objectPutPart' ||
apiMethodAfterVersionCheck === 'completeMultipartUpload'
) {
if (request.headers['x-scal-s3-version-id'] ||
request.headers['x-scal-s3-version-id'] === '') {
const putVersionRequestContext =
generateRequestContext('objectPutVersion');
requestContexts.push(putVersionRequestContext);
} else {
const putRequestContext =
generateRequestContext(apiMethodAfterVersionCheck);
requestContexts.push(putRequestContext);
}
// if put object (versioning) with ACL
if (isHeaderAcl(request.headers)) {
const putAclRequestContext =
generateRequestContext('objectPutACL');
requestContexts.push(putAclRequestContext);
}
if (request.headers['x-amz-object-lock-mode']) {
const putObjectLockRequestContext =
generateRequestContext('objectPutRetention');
requestContexts.push(putObjectLockRequestContext);
}
if (request.headers['x-amz-version-id']) {
const putObjectVersionRequestContext =
generateRequestContext('objectPutTaggingVersion');
requestContexts.push(putObjectVersionRequestContext);
}
} else {
const requestContext =
generateRequestContext(apiMethodAfterVersionCheck);

View File

@ -0,0 +1,99 @@
const async = require('async');
const { auth, s3middleware } = require('arsenal');
const metadata = require('../../../metadata/wrapper');
const { decodeVersionId } = require('../object/versioning');
const { parseTagXml } = s3middleware.tagging;
function makeTagQuery(tags) {
return Object.entries(tags)
.map(i => i.join('='))
.join('&');
}
function updateRequestContextsWithTags(request, requestContexts, apiMethod, log, cb) {
async.waterfall([
next => {
if (request.headers['x-amz-tagging']) {
return next(null, request.headers['x-amz-tagging']);
}
if (request.post && apiMethod === 'objectPutTagging') {
return parseTagXml(request.post, log, (err, tags) => {
if (err) {
log.trace('error parsing request tags');
return next(err);
}
return next(null, makeTagQuery(tags));
});
}
return next(null, null);
},
(requestTagsQuery, next) => {
const objectKey = request.objectKey;
const bucketName = request.bucketName;
const decodedVidResult = decodeVersionId(request.query);
if (decodedVidResult instanceof Error) {
log.trace('invalid versionId query', {
versionId: request.query.versionId,
error: decodedVidResult,
});
return next(decodedVidResult);
}
const reqVersionId = decodedVidResult;
return metadata.getObjectMD(
bucketName, objectKey, { versionId: reqVersionId }, log, (err, objMD) => {
if (err) {
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (err.NoSuchKey) {
return next(null, requestTagsQuery, null);
}
log.trace('error getting request object tags');
return next(err);
}
const existingTagsQuery = objMD.tags && makeTagQuery(objMD.tags);
return next(null, requestTagsQuery, existingTagsQuery);
});
},
], (err, requestTagsQuery, existingTagsQuery) => {
if (err) {
log.trace('error processing tag condition key evaluation');
return cb(err);
}
// FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter
// eslint-disable-next-line no-restricted-syntax
for (const rc of requestContexts) {
rc.setNeedTagEval(true);
if (requestTagsQuery) {
rc.setRequestObjTags(requestTagsQuery);
}
if (existingTagsQuery) {
rc.setExistingObjTag(existingTagsQuery);
}
}
return cb();
});
}
function tagConditionKeyAuth(authorizationResults, request, requestContexts, apiMethod, log, cb) {
if (!authorizationResults) {
return cb();
}
if (!authorizationResults.some(authRes => authRes.checkTagConditions)) {
return cb(null, authorizationResults);
}
return updateRequestContextsWithTags(request, requestContexts, apiMethod, log, err => {
if (err) {
return cb(err);
}
return auth.server.doAuth(request, log,
(err, userInfo, authResults) => cb(err, authResults), 's3', requestContexts);
});
}
module.exports = {
tagConditionKeyAuth,
updateRequestContextsWithTags,
makeTagQuery,
};

View File

@ -6,29 +6,34 @@ const acl = require('../../../metadata/acl');
const BucketInfo = require('arsenal').models.BucketInfo;
const constants = require('../../../../constants');
const createKeyForUserBucket = require('./createKeyForUserBucket');
const { parseBucketEncryptionHeaders } = require('./bucketEncryption');
const metadata = require('../../../metadata/wrapper');
const kms = require('../../../kms/wrapper');
const isLegacyAWSBehavior = require('../../../utilities/legacyAWSBehavior');
const { isBackbeatUser } = require('../authorization/aclChecks');
const { isServiceAccount } = require('../authorization/permissionChecks');
const usersBucket = constants.usersBucket;
const oldUsersBucket = constants.oldUsersBucket;
const zenkoSeparator = constants.zenkoSeparator;
const userBucketOwner = 'admin';
function addToUsersBucket(canonicalID, bucketName, log, cb) {
function addToUsersBucket(canonicalID, bucketName, bucketMD, log, cb) {
// BACKWARD: Simplify once do not have to deal with old
// usersbucket name and old splitter
// Get new format usersBucket to see if it exists
return metadata.getBucket(usersBucket, log, (err, usersBucketAttrs) => {
if (err && !err.NoSuchBucket && !err.BucketAlreadyExists) {
if (err && !err.is.NoSuchBucket && !err.is.BucketAlreadyExists) {
return cb(err);
}
const splitter = usersBucketAttrs ?
constants.splitter : constants.oldSplitter;
let key = createKeyForUserBucket(canonicalID, splitter, bucketName);
const omVal = { creationDate: new Date().toJSON() };
const omVal = {
creationDate: new Date().toJSON(),
ingestion: bucketMD.getIngestion(),
};
// If the new format usersbucket does not exist, try to put the
// key in the old usersBucket using the old splitter.
// Otherwise put the key in the new format usersBucket
@ -36,7 +41,7 @@ function addToUsersBucket(canonicalID, bucketName, log, cb) {
usersBucket : oldUsersBucket;
return metadata.putObjectMD(usersBucketBeingCalled, key,
omVal, {}, log, err => {
if (err && err.NoSuchBucket) {
if (err?.is?.NoSuchBucket) {
// There must be no usersBucket so createBucket
// one using the new format
log.trace('users bucket does not exist, ' +
@ -56,9 +61,8 @@ function addToUsersBucket(canonicalID, bucketName, log, cb) {
// from getting a BucketAlreadyExists
// error with respect
// to the usersBucket.
if (err &&
err !==
errors.BucketAlreadyExists) {
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (err && !err.BucketAlreadyExists) {
log.error('error from metadata', {
error: err,
});
@ -92,7 +96,7 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) {
return callback(err);
}
log.trace('created bucket in metadata');
return addToUsersBucket(canonicalID, bucketName, log, err => {
return addToUsersBucket(canonicalID, bucketName, bucket, log, err => {
if (err) {
return callback(err);
}
@ -115,7 +119,7 @@ function freshStartCreateBucket(bucket, canonicalID, log, callback) {
*/
function cleanUpBucket(bucketMD, canonicalID, log, callback) {
const bucketName = bucketMD.getName();
return addToUsersBucket(canonicalID, bucketName, log, err => {
return addToUsersBucket(canonicalID, bucketName, bucketMD, log, err => {
if (err) {
return callback(err);
}
@ -167,14 +171,36 @@ function createBucket(authInfo, bucketName, headers,
authInfo.getAccountDisplayName();
const creationDate = new Date().toJSON();
const isNFSEnabled = headers['x-scal-nfs-enabled'] === 'true';
const bucket = new BucketInfo(bucketName,
canonicalID, ownerDisplayName, creationDate,
BucketInfo.currentModelVersion(), null, null, null,
null, null, null, null, null, null, null, null,
null, isNFSEnabled);
const headerObjectLock = headers['x-amz-bucket-object-lock-enabled'];
const objectLockEnabled
= headerObjectLock && headerObjectLock.toLowerCase() === 'true';
const bucket = new BucketInfo(bucketName, canonicalID, ownerDisplayName,
creationDate, BucketInfo.currentModelVersion(), null, null, null, null,
null, null, null, null, null, null, null, null, null, isNFSEnabled,
null, null, objectLockEnabled);
let locationConstraintVal = null;
if (locationConstraint !== undefined) {
bucket.setLocationConstraint(locationConstraint);
if (locationConstraint) {
const [locationConstraintStr, ingestion] =
locationConstraint.split(zenkoSeparator);
if (locationConstraintStr) {
locationConstraintVal = locationConstraintStr;
bucket.setLocationConstraint(locationConstraintStr);
}
if (ingestion === 'ingest') {
bucket.enableIngestion();
//automatically enable versioning for ingestion buckets
bucket.setVersioningConfiguration({ Status: 'Enabled' });
}
}
if (objectLockEnabled) {
// default versioning configuration AWS sets
// when a bucket is created with object lock
const versioningConfiguration = {
Status: 'Enabled',
MfaDelete: 'Disabled',
};
bucket.setVersioningConfiguration(versioningConfiguration);
}
const parseAclParams = {
headers,
@ -197,6 +223,7 @@ function createBucket(authInfo, bucketName, headers,
},
getAnyExistingBucketInfo: function getAnyExistingBucketInfo(callback) {
metadata.getBucket(bucketName, log, (err, data) => {
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (err && err.NoSuchBucket) {
return callback(null, 'NoBucketYet');
}
@ -215,14 +242,15 @@ function createBucket(authInfo, bucketName, headers,
const existingBucketMD = results.getAnyExistingBucketInfo;
if (existingBucketMD instanceof BucketInfo &&
existingBucketMD.getOwner() !== canonicalID &&
!isBackbeatUser(canonicalID)) {
!isServiceAccount(canonicalID)) {
// return existingBucketMD to collect cors headers
return cb(errors.BucketAlreadyExists, existingBucketMD);
}
const newBucketMD = results.prepareNewBucketMD;
if (existingBucketMD === 'NoBucketYet') {
const sseConfig = parseBucketEncryptionHeaders(headers);
return bucketLevelServerSideEncryption(
bucketName, headers, log,
bucketName, sseConfig, log,
(err, sseInfo) => {
if (err) {
return cb(err);
@ -245,7 +273,7 @@ function createBucket(authInfo, bucketName, headers,
// error unless old AWS behavior (us-east-1)
// Existing locationConstraint must have legacyAwsBehavior === true
// New locationConstraint should have legacyAwsBehavior === true
if (isLegacyAWSBehavior(locationConstraint) &&
if (isLegacyAWSBehavior(locationConstraintVal) &&
isLegacyAWSBehavior(existingBucketMD.getLocationConstraint())) {
log.trace('returning 200 instead of 409 to mirror us-east-1');
return cb(null, existingBucketMD);

View File

@ -2,60 +2,21 @@ const assert = require('assert');
const async = require('async');
const { errors } = require('arsenal');
const logger = require('../../../utilities/logger');
const abortMultipartUpload = require('../object/abortMultipartUpload');
const { pushMetric } = require('../../../utapi/utilities');
const constants = require('../../../../constants');
const createKeyForUserBucket = require('./createKeyForUserBucket');
const { splitter, oldSplitter, mpuBucketPrefix } =
require('../../../../constants');
const metadata = require('../../../metadata/wrapper');
const kms = require('../../../kms/wrapper');
const usersBucket = constants.usersBucket;
const oldUsersBucket = constants.oldUsersBucket;
function _deleteUserBucketEntry(bucketName, canonicalID, log, cb) {
log.trace('deleting bucket name from users bucket', { method:
'_deleteUserBucketEntry' });
const keyForUserBucket = createKeyForUserBucket(canonicalID,
constants.splitter, bucketName);
metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => {
// If the object representing the bucket is not in the
// users bucket just continue
if (error && error.NoSuchKey) {
return cb(null);
// BACKWARDS COMPATIBILITY: Remove this once no longer
// have old user bucket format
} else if (error && error.NoSuchBucket) {
const keyForUserBucket2 = createKeyForUserBucket(canonicalID,
constants.oldSplitter, bucketName);
return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2,
{}, log, error => {
if (error && !error.NoSuchKey) {
log.error('from metadata while deleting user bucket',
{ error });
return cb(error);
}
log.trace('deleted bucket from user bucket',
{ method: '_deleteUserBucketEntry' });
return cb(null);
});
} else if (error) {
log.error('from metadata while deleting user bucket', { error,
method: '_deleteUserBucketEntry' });
return cb(error);
}
log.trace('deleted bucket from user bucket', {
method: '_deleteUserBucketEntry' });
return cb(null);
});
}
const deleteUserBucketEntry = require('./deleteUserBucketEntry');
function _deleteMPUbucket(destinationBucketName, log, cb) {
const mpuBucketName =
`${constants.mpuBucketPrefix}${destinationBucketName}`;
`${mpuBucketPrefix}${destinationBucketName}`;
return metadata.deleteBucket(mpuBucketName, log, err => {
// If the mpu bucket does not exist, just move on
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (err && err.NoSuchBucket) {
return cb();
}
@ -63,48 +24,38 @@ function _deleteMPUbucket(destinationBucketName, log, cb) {
});
}
/**
* Invisibly finishes deleting a bucket that already has a deleted flag
* by deleting the object in the users bucket representing the created bucket
* and then deleting the bucket in metadata
* @param {string} bucketName - name of bucket
* @param {string} canonicalID - bucket owner's canonicalID
* @return {undefined}
*/
function invisiblyDelete(bucketName, canonicalID) {
const log = logger.newRequestLogger();
log.trace('deleting bucket with deleted flag invisibly', { bucketName });
return _deleteUserBucketEntry(bucketName, canonicalID, log, err => {
if (err) {
log.error('error invisibly deleting bucket name from user bucket',
{ error: err });
return log.end();
}
log.trace('deleted bucket name from user bucket');
return metadata.deleteBucket(bucketName, log, error => {
log.trace('deleting bucket from metadata',
{ method: 'invisiblyDelete' });
if (error) {
log.error('error deleting bucket from metadata', { error });
return log.end();
}
log.trace('invisible deletion of bucket succeeded',
{ method: 'invisiblyDelete' });
return log.end();
});
});
function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log, cb) {
async.mapLimit(mpus, 1, (mpu, next) => {
const splitterChar = mpu.key.includes(oldSplitter) ?
oldSplitter : splitter;
// `overview${splitter}${objectKey}${splitter}${uploadId}
const [, objectKey, uploadId] = mpu.key.split(splitterChar);
abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
(err, destBucket, partSizeSum) => {
pushMetric('abortMultipartUpload', log, {
authInfo,
canonicalID: bucketMD.getOwner(),
bucket: bucketName,
keys: [objectKey],
byteLength: partSizeSum,
});
next(err);
}, request);
}, cb);
}
/**
* deleteBucket - Delete bucket from namespace
* @param {object} authInfo - authentication info
* @param {object} bucketMD - bucket attributes/metadata
* @param {string} bucketName - bucket in which objectMetadata is stored
* @param {string} canonicalID - account canonicalID of requester
* @param {object} request - request object given by router
* including normalized headers
* @param {object} log - Werelogs logger
* @param {function} cb - callback from async.waterfall in bucketDelete
* @return {undefined}
*/
function deleteBucket(bucketMD, bucketName, canonicalID, log, cb) {
function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log, cb) {
log.trace('deleting bucket from metadata');
assert.strictEqual(typeof bucketName, 'string');
assert.strictEqual(typeof canonicalID, 'string');
@ -133,18 +84,16 @@ function deleteBucket(bucketMD, bucketName, canonicalID, log, cb) {
return next();
});
},
// Note: This does not mirror AWS behavior. AWS will allow a user to
// delete a bucket even if there are ongoing multipart uploads.
function deleteMPUbucketStep(next) {
const MPUBucketName = `${constants.mpuBucketPrefix}${bucketName}`;
const MPUBucketName = `${mpuBucketPrefix}${bucketName}`;
// check to see if there are any mpu overview objects (so ignore
// any orphaned part objects)
return metadata.listObject(MPUBucketName,
{ maxKeys: 1, prefix: 'overview' }, log,
(err, objectsListRes) => {
return metadata.listObject(MPUBucketName, { prefix: 'overview' },
log, (err, objectsListRes) => {
// If no shadow bucket ever created, no ongoing MPU's, so
// continue with deletion
if (err && err.NoSuchBucket) {
if (err?.is.NoSuchBucket) {
return next();
}
if (err) {
@ -152,12 +101,14 @@ function deleteBucket(bucketMD, bucketName, canonicalID, log, cb) {
return next(err);
}
if (objectsListRes.Contents.length) {
log.debug('bucket delete failed',
{ error: errors.MPUinProgress });
// Return non-AWS standard error
// regarding ongoing MPUs so user
// understands what is occurring
return next(errors.MPUinProgress);
return _deleteOngoingMPUs(authInfo, bucketName,
bucketMD, objectsListRes.Contents, request, log, err => {
if (err) {
return next(err);
}
log.trace('deleting shadow MPU bucket');
return _deleteMPUbucket(bucketName, log, next);
});
}
log.trace('deleting shadow MPU bucket');
return _deleteMPUbucket(bucketName, log, next);
@ -173,7 +124,7 @@ function deleteBucket(bucketMD, bucketName, canonicalID, log, cb) {
},
function deleteUserBucketEntryStep(next) {
log.trace('deleting bucket name from user bucket');
return _deleteUserBucketEntry(bucketName, canonicalID, log, next);
return deleteUserBucketEntry(bucketName, canonicalID, log, next);
},
],
// eslint-disable-next-line prefer-arrow-callback
@ -197,7 +148,4 @@ function deleteBucket(bucketMD, bucketName, canonicalID, log, cb) {
});
}
module.exports = {
invisiblyDelete,
deleteBucket,
};
module.exports = deleteBucket;

View File

@ -0,0 +1,255 @@
const { errors } = require('arsenal');
const metadata = require('../../../metadata/wrapper');
const kms = require('../../../kms/wrapper');
const { parseString } = require('xml2js');
/**
* ServerSideEncryptionInfo - user configuration for server side encryption
* @typedef {Object} ServerSideEncryptionInfo
* @property {string} algorithm - Algorithm to use for encryption. Either AES256 or aws:kms.
* @property {string} masterKeyId - Key id for the kms key used to encrypt data keys.
* @property {string} configuredMasterKeyId - User configured master key id.
* @property {boolean} mandatory - Whether a default encryption policy has been enabled.
*/
/**
* @callback ServerSideEncryptionInfo~callback
* @param {Object} error - Instance of Arsenal error
* @param {ServerSideEncryptionInfo} - SSE configuration
*/
/**
* parseEncryptionXml - Parses and validates a ServerSideEncryptionConfiguration xml document
* @param {object} xml - ServerSideEncryptionConfiguration doc
* @param {object} log - logger
* @param {ServerSideEncryptionInfo~callback} cb - callback
* @returns {undefined}
*/
function parseEncryptionXml(xml, log, cb) {
return parseString(xml, (err, parsed) => {
if (err) {
log.trace('xml parsing failed', {
error: err,
method: 'parseEncryptionXml',
});
log.debug('invalid xml', { xml });
return cb(errors.MalformedXML);
}
if (!parsed
|| !parsed.ServerSideEncryptionConfiguration
|| !parsed.ServerSideEncryptionConfiguration.Rule) {
log.trace('error in sse config, invalid ServerSideEncryptionConfiguration section', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
const { Rule } = parsed.ServerSideEncryptionConfiguration;
if (!Array.isArray(Rule)
|| Rule.length > 1
|| !Rule[0]
|| !Rule[0].ApplyServerSideEncryptionByDefault
|| !Rule[0].ApplyServerSideEncryptionByDefault[0]) {
log.trace('error in sse config, invalid ApplyServerSideEncryptionByDefault section', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
const [encConfig] = Rule[0].ApplyServerSideEncryptionByDefault;
if (!encConfig.SSEAlgorithm || !encConfig.SSEAlgorithm[0]) {
log.trace('error in sse config, no SSEAlgorithm provided', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
const [algorithm] = encConfig.SSEAlgorithm;
if (algorithm !== 'AES256' && algorithm !== 'aws:kms') {
log.trace('error in sse config, unknown SSEAlgorithm', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
const result = { algorithm, mandatory: true };
if (encConfig.KMSMasterKeyID) {
if (algorithm === 'AES256') {
log.trace('error in sse config, can not specify KMSMasterKeyID when using AES256', {
method: 'parseEncryptionXml',
});
return cb(errors.InvalidArgument.customizeDescription(
'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms'));
}
if (!encConfig.KMSMasterKeyID[0] || typeof encConfig.KMSMasterKeyID[0] !== 'string') {
log.trace('error in sse config, invalid KMSMasterKeyID', {
method: 'parseEncryptionXml',
});
return cb(errors.MalformedXML);
}
result.configuredMasterKeyId = encConfig.KMSMasterKeyID[0];
}
return cb(null, result);
});
}
/**
* hydrateEncryptionConfig - Constructs a ServerSideEncryptionInfo object from arguments
* ensuring no invalid or undefined keys are added
*
* @param {string} algorithm - Algorithm to use for encryption. Either AES256 or aws:kms.
* @param {string} configuredMasterKeyId - User configured master key id.
* @param {boolean} [mandatory] - Whether a default encryption policy has been enabled.
* @returns {ServerSideEncryptionInfo} - SSE configuration
*/
function hydrateEncryptionConfig(algorithm, configuredMasterKeyId, mandatory = null) {
if (algorithm !== 'AES256' && algorithm !== 'aws:kms') {
return {
algorithm: null,
};
}
const sseConfig = { algorithm, mandatory };
if (algorithm === 'aws:kms' && configuredMasterKeyId) {
sseConfig.configuredMasterKeyId = configuredMasterKeyId;
}
if (mandatory !== null) {
sseConfig.mandatory = mandatory;
}
return sseConfig;
}
/**
* parseBucketEncryptionHeaders - retrieves bucket level sse configuration from request headers
* @param {object} headers - Request headers
* @returns {ServerSideEncryptionInfo} - SSE configuration
*/
function parseBucketEncryptionHeaders(headers) {
const sseAlgorithm = headers['x-amz-scal-server-side-encryption'];
const configuredMasterKeyId = headers['x-amz-scal-server-side-encryption-aws-kms-key-id'] || null;
return hydrateEncryptionConfig(sseAlgorithm, configuredMasterKeyId, true);
}
/**
* parseObjectEncryptionHeaders - retrieves bucket level sse configuration from request headers
* @param {object} headers - Request headers
* @returns {ServerSideEncryptionInfo} - SSE configuration
*/
function parseObjectEncryptionHeaders(headers) {
const sseAlgorithm = headers['x-amz-server-side-encryption'];
const configuredMasterKeyId = headers['x-amz-server-side-encryption-aws-kms-key-id'] || null;
if (sseAlgorithm && sseAlgorithm !== 'AES256' && sseAlgorithm !== 'aws:kms') {
return {
error: errors.InvalidArgument.customizeDescription('The encryption method specified is not supported'),
};
}
if (sseAlgorithm !== 'aws:kms' && configuredMasterKeyId) {
return {
error: errors.InvalidArgument.customizeDescription(
'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms'),
};
}
return { objectSSE: hydrateEncryptionConfig(sseAlgorithm, configuredMasterKeyId) };
}
/**
* createDefaultBucketEncryptionMetadata - Creates master key and sets up default server side encryption configuration
* @param {BucketInfo} bucket - bucket metadata
* @param {object} log - werelogs logger
* @param {ServerSideEncryptionInfo~callback} cb - callback
* @returns {undefined}
*/
function createDefaultBucketEncryptionMetadata(bucket, log, cb) {
return kms.bucketLevelEncryption(
bucket.getName(),
{ algorithm: 'AES256', mandatory: false },
log,
(error, sseConfig) => {
if (error) {
return cb(error);
}
bucket.setServerSideEncryption(sseConfig);
return metadata.updateBucket(bucket.getName(), bucket, log, err => cb(err, sseConfig));
});
}
/**
*
* @param {object} headers - request headers
* @param {BucketInfo} bucket - BucketInfo model
* @param {*} log - werelogs logger
* @param {ServerSideEncryptionInfo~callback} cb - callback
* @returns {undefined}
*/
function getObjectSSEConfiguration(headers, bucket, log, cb) {
const bucketSSE = bucket.getServerSideEncryption();
const { error, objectSSE } = parseObjectEncryptionHeaders(headers);
if (error) {
return cb(error);
}
// If a per object sse algo has been passed through
// x-amz-server-side-encryption
if (objectSSE.algorithm) {
// If aws:kms and a custom key id
// pass it through without updating the bucket md
if (objectSSE.algorithm === 'aws:kms' && objectSSE.configuredMasterKeyId) {
return cb(null, objectSSE);
}
// If the client has not specified a key id,
// and we have a default config, then we reuse
// it and pass it through
if (!objectSSE.configuredMasterKeyId && bucketSSE) {
// The default configs algo is overridden with the one passed in the
// request headers. Our implementations of AES256 and aws:kms are the
// same underneath so this is only cosmetic change.
const sseConfig = Object.assign({}, bucketSSE, { algorithm: objectSSE.algorithm });
return cb(null, sseConfig);
}
// If the client has not specified a key id, and we
// don't have a default config, generate it
if (!objectSSE.configuredMasterKeyId && !bucketSSE) {
return createDefaultBucketEncryptionMetadata(bucket, log, (error, sseConfig) => {
if (error) {
return cb(error);
}
// Override the algorithm, for the same reasons as above.
Object.assign(sseConfig, { algorithm: objectSSE.algorithm });
return cb(null, sseConfig);
});
}
}
// If the bucket has a default encryption config, and it is mandatory
// (created with putBucketEncryption or legacy headers)
// pass it through
if (bucketSSE && bucketSSE.mandatory) {
return cb(null, bucketSSE);
}
// No encryption config
return cb(null, null);
}
module.exports = {
createDefaultBucketEncryptionMetadata,
getObjectSSEConfiguration,
hydrateEncryptionConfig,
parseEncryptionXml,
parseBucketEncryptionHeaders,
parseObjectEncryptionHeaders,
};

View File

@ -1,4 +1,5 @@
const { invisiblyDelete } = require('./bucketDeletion');
const invisiblyDelete = require('./invisiblyDelete');
const constants = require('../../../../constants');
/**
* Checks whether to proceed with a request based on the bucket flags
@ -8,9 +9,17 @@ const { invisiblyDelete } = require('./bucketDeletion');
* @return {boolean} true if the bucket should be shielded, false otherwise
*/
function bucketShield(bucket, requestType) {
const invisiblyDeleteRequests = ['bucketGet', 'bucketHead',
'bucketGetACL', 'bucketOwnerAction', 'objectGet', 'objectGetACL',
'objectHead', 'objectPutACL', 'objectDelete'];
const invisiblyDeleteRequests = constants.bucketOwnerActions.concat(
[
'bucketGet',
'bucketHead',
'bucketGetACL',
'objectGet',
'objectGetACL',
'objectHead',
'objectPutACL',
'objectDelete',
]);
if (invisiblyDeleteRequests.indexOf(requestType) > -1 &&
bucket.hasDeletedFlag()) {
invisiblyDelete(bucket.getName(), bucket.getOwner());
@ -21,6 +30,9 @@ function bucketShield(bucket, requestType) {
// Otherwise return an error to the client
if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) &&
(requestType !== 'objectPut' &&
requestType !== 'initiateMultipartUpload' &&
requestType !== 'objectPutPart' &&
requestType !== 'completeMultipartUpload' &&
requestType !== 'bucketPutACL' &&
requestType !== 'bucketDelete')) {
return true;

View File

@ -0,0 +1,44 @@
const createKeyForUserBucket = require('./createKeyForUserBucket');
const { usersBucket, oldUsersBucket, splitter, oldSplitter } =
require('../../../../constants');
const metadata = require('../../../metadata/wrapper');
function deleteUserBucketEntry(bucketName, canonicalID, log, cb) {
log.trace('deleting bucket name from users bucket', { method:
'_deleteUserBucketEntry' });
const keyForUserBucket = createKeyForUserBucket(canonicalID, splitter,
bucketName);
metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => {
// If the object representing the bucket is not in the
// users bucket just continue
if (error?.is.NoSuchKey) {
return cb(null);
// BACKWARDS COMPATIBILITY: Remove this once no longer
// have old user bucket format
} else if (error?.is.NoSuchBucket) {
const keyForUserBucket2 = createKeyForUserBucket(canonicalID,
oldSplitter, bucketName);
return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2,
{}, log, error => {
// TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver
if (error && !error.NoSuchKey) {
log.error('from metadata while deleting user bucket',
{ error });
return cb(error);
}
log.trace('deleted bucket from user bucket',
{ method: '_deleteUserBucketEntry' });
return cb(null);
});
} else if (error) {
log.error('from metadata while deleting user bucket', { error,
method: '_deleteUserBucketEntry' });
return cb(error);
}
log.trace('deleted bucket from user bucket', {
method: '_deleteUserBucketEntry' });
return cb(null);
});
}
module.exports = deleteUserBucketEntry;

View File

@ -0,0 +1,37 @@
const { errors, models } = require('arsenal');
const { NotificationConfiguration } = models;
const { config } = require('../../../Config');
function getNotificationConfiguration(parsedXml) {
const notifConfig = new NotificationConfiguration(parsedXml).getValidatedNotificationConfiguration();
// if notifConfig is empty object, effectively delete notification configuration
if (notifConfig.error || Object.keys(notifConfig).length === 0) {
return notifConfig;
}
if (!config.bucketNotificationDestinations) {
return { error: errors.InvalidArgument.customizeDescription(
'Unable to validate the following destination configurations') };
}
const targets = new Set(config.bucketNotificationDestinations.map(t => t.resource));
const notifConfigTargets = notifConfig.queueConfig.map(t => t.queueArn.split(':')[5]);
// getting invalid targets
const invalidTargets = [];
notifConfigTargets.forEach((t, i) => {
if (!targets.has(t)) {
invalidTargets.push({
ArgumentName: notifConfig.queueConfig[i].queueArn,
ArgumentValue: 'The destination queue does not exist',
});
}
});
if (invalidTargets.length > 0) {
const errDesc = 'Unable to validate the following destination configurations';
let error = errors.InvalidArgument.customizeDescription(errDesc);
error = error.addMetadataEntry('invalidArguments', invalidTargets);
return { error };
}
return notifConfig;
}
module.exports = getNotificationConfiguration;

View File

@ -0,0 +1,37 @@
const logger = require('../../../utilities/logger');
const deleteUserBucketEntry = require('./deleteUserBucketEntry');
const metadata = require('../../../metadata/wrapper');
/**
* Invisibly finishes deleting a bucket that already has a deleted flag
* by deleting the object in the users bucket representing the created bucket
* and then deleting the bucket in metadata
* @param {string} bucketName - name of bucket
* @param {string} canonicalID - bucket owner's canonicalID
* @return {undefined}
*/
function invisiblyDelete(bucketName, canonicalID) {
const log = logger.newRequestLogger();
log.trace('deleting bucket with deleted flag invisibly', { bucketName });
return deleteUserBucketEntry(bucketName, canonicalID, log, err => {
if (err) {
log.error('error invisibly deleting bucket name from user bucket',
{ error: err });
return log.end();
}
log.trace('deleted bucket name from user bucket');
return metadata.deleteBucket(bucketName, log, error => {
log.trace('deleting bucket from metadata',
{ method: 'invisiblyDelete' });
if (error) {
log.error('error deleting bucket from metadata', { error });
return log.end();
}
log.trace('invisible deletion of bucket succeeded',
{ method: 'invisiblyDelete' });
return log.end();
});
});
}
module.exports = invisiblyDelete;

View File

@ -1,233 +0,0 @@
const { config } = require('../../../Config');
const { legacyLocations } = require('../../../../constants.js');
const { locationConstraints } = config;
const escapeForXml = require('arsenal').s3middleware.escapeForXml;
class BackendInfo {
/**
* Represents the info necessary to evaluate which data backend to use
* on a data put call.
* @constructor
* @param {string | undefined} objectLocationConstraint - location constraint
* for object based on user meta header
* @param {string | undefined } bucketLocationConstraint - location
* constraint for bucket based on bucket metadata
* @param {string} requestEndpoint - endpoint to which request was made
* @param {string | undefined } legacyLocationConstraint - legacy location
* constraint
*/
constructor(objectLocationConstraint, bucketLocationConstraint,
requestEndpoint, legacyLocationConstraint) {
this._objectLocationConstraint = objectLocationConstraint;
this._bucketLocationConstraint = bucketLocationConstraint;
this._requestEndpoint = requestEndpoint;
this._legacyLocationConstraint = legacyLocationConstraint;
return this;
}
/**
* validate proposed location constraint against config
* @param {string | undefined} locationConstraint - value of user
* metadata location constraint header or bucket location constraint
* @param {object} log - werelogs logger
* @return {boolean} - true if valid, false if not
*/
static isValidLocationConstraint(locationConstraint, log) {
if (Object.keys(config.locationConstraints).
indexOf(locationConstraint) < 0) {
log.trace('proposed locationConstraint is invalid',
{ locationConstraint });
return false;
}
return true;
}
/**
* validate that request endpoint is listed in the restEndpoint config
* @param {string} requestEndpoint - request endpoint
* @param {object} log - werelogs logger
* @return {boolean} - true if present, false if not
*/
static isRequestEndpointPresent(requestEndpoint, log) {
if (Object.keys(config.restEndpoints).indexOf(requestEndpoint) < 0) {
log.trace('requestEndpoint does not match config restEndpoints',
{ requestEndpoint });
return false;
}
return true;
}
/**
* validate that locationConstraint for request Endpoint matches
* one config locationConstraint
* @param {string} requestEndpoint - request endpoint
* @param {object} log - werelogs logger
* @return {boolean} - true if matches, false if not
*/
static isRequestEndpointValueValid(requestEndpoint, log) {
if (Object.keys(config.locationConstraints).indexOf(config
.restEndpoints[requestEndpoint]) < 0) {
log.trace('the default locationConstraint for request' +
'Endpoint does not match any config locationConstraint',
{ requestEndpoint });
return false;
}
return true;
}
/**
* validate that s3 server is running with a file or memory backend
* @param {string} requestEndpoint - request endpoint
* @param {object} log - werelogs logger
* @return {boolean} - true if running with file/mem backend, false if not
*/
static isMemOrFileBackend(requestEndpoint, log) {
if (config.backends.data === 'mem' ||
config.backends.data === 'file') {
log.trace('use data backend for the location', {
dataBackend: config.backends.data,
method: 'isMemOrFileBackend',
});
return true;
}
return false;
}
/**
* validate requestEndpoint against config or mem/file data backend
* - if there is no match for the request endpoint in the config
* restEndpoints and data backend is set to mem or file we will use this
* data backend for the location.
* - if locationConstraint for request Endpoint does not match
* any config locationConstraint, we will return an error
* @param {string} requestEndpoint - request endpoint
* @param {object} log - werelogs logger
* @return {boolean} - true if valid, false if not
*/
static isValidRequestEndpointOrBackend(requestEndpoint, log) {
if (!BackendInfo.isRequestEndpointPresent(requestEndpoint, log)) {
return BackendInfo.isMemOrFileBackend(requestEndpoint, log);
}
return BackendInfo.isRequestEndpointValueValid(requestEndpoint, log);
}
/**
* validate controlling BackendInfo Parameter
* @param {string | undefined} objectLocationConstraint - value of user
* metadata location constraint header
* @param {string | null} bucketLocationConstraint - location
* constraint from bucket metadata
* @param {string} requestEndpoint - endpoint of request
* @param {object} log - werelogs logger
* @return {object} - location constraint validity
*/
static controllingBackendParam(objectLocationConstraint,
bucketLocationConstraint, requestEndpoint, log) {
if (objectLocationConstraint) {
if (BackendInfo.isValidLocationConstraint(objectLocationConstraint,
log)) {
log.trace('objectLocationConstraint is valid');
return { isValid: true };
}
log.trace('objectLocationConstraint is invalid');
return { isValid: false, description: 'Object Location Error - ' +
`Your object location "${escapeForXml(objectLocationConstraint)}"` +
'is not in your location config - Please update.' };
}
if (bucketLocationConstraint) {
if (BackendInfo.isValidLocationConstraint(bucketLocationConstraint,
log)) {
log.trace('bucketLocationConstraint is valid');
return { isValid: true };
}
log.trace('bucketLocationConstraint is invalid');
return { isValid: false, description: 'Bucket Location Error - ' +
`Your bucket location "${escapeForXml(bucketLocationConstraint)}"` +
' is not in your location config - Please update.' };
}
const legacyLocationConstraint =
BackendInfo.getLegacyLocationConstraint();
if (legacyLocationConstraint) {
log.trace('legacy location is valid');
return { isValid: true, legacyLocationConstraint };
}
if (!BackendInfo.isValidRequestEndpointOrBackend(requestEndpoint,
log)) {
return { isValid: false, description: 'Endpoint Location Error - ' +
`Your endpoint "${requestEndpoint}" is not in restEndpoints ` +
'in your config OR the default location constraint for request ' +
`endpoint "${escapeForXml(requestEndpoint)}" does not ` +
'match any config locationConstraint - Please update.' };
}
if (BackendInfo.isRequestEndpointPresent(requestEndpoint, log)) {
return { isValid: true };
}
return { isValid: true, defaultedToDataBackend: true };
}
/**
* Return legacyLocationConstraint
* @return {string | undefined} legacyLocationConstraint;
*/
static getLegacyLocationConstraint() {
return legacyLocations.find(ll => locationConstraints[ll]);
}
/**
* Return objectLocationConstraint
* @return {string | undefined} objectLocationConstraint;
*/
getObjectLocationConstraint() {
return this._objectLocationConstraint;
}
/**
* Return bucketLocationConstraint
* @return {string | undefined} bucketLocationConstraint;
*/
getBucketLocationConstraint() {
return this._bucketLocationConstraint;
}
/**
* Return requestEndpoint
* @return {string} requestEndpoint;
*/
getRequestEndpoint() {
return this._requestEndpoint;
}
/**
* Return locationConstraint that should be used with put request
* Order of priority is:
* (1) objectLocationConstraint,
* (2) bucketLocationConstraint,
* (3) legacyLocationConstraint,
* (4) default locationConstraint for requestEndpoint if requestEndpoint
* is listed in restEndpoints in config.json
* (5) default data backend
* @return {string} locationConstraint;
*/
getControllingLocationConstraint() {
const objectLC = this.getObjectLocationConstraint();
const bucketLC = this.getBucketLocationConstraint();
const reqEndpoint = this.getRequestEndpoint();
if (objectLC) {
return objectLC;
}
if (bucketLC) {
return bucketLC;
}
if (this._legacyLocationConstraint) {
return this._legacyLocationConstraint;
}
if (config.restEndpoints[reqEndpoint]) {
return config.restEndpoints[reqEndpoint];
}
return config.backends.data;
}
}
module.exports = {
BackendInfo,
};

View File

@ -0,0 +1,131 @@
const async = require('async');
const constants = require('../../../../constants');
const { data } = require('../../../data/wrapper');
const locationConstraintCheck = require('../object/locationConstraintCheck');
const { standardMetadataValidateBucketAndObj } =
require('../../../metadata/metadataUtils');
const services = require('../../../services');
function abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log,
callback, request) {
const metadataValMPUparams = {
authInfo,
bucketName,
objectKey,
uploadId,
preciseRequestType: request.apiMethods || 'multipartDelete',
request,
};
// For validating the request at the destinationBucket level
// params are the same as validating at the MPU level
// but the requestType is the more general 'objectDelete'
const metadataValParams = Object.assign({}, metadataValMPUparams);
metadataValParams.requestType = 'objectPut';
const authzIdentityResult = request ? request.actionImplicitDenies : false;
async.waterfall([
function checkDestBucketVal(next) {
standardMetadataValidateBucketAndObj(metadataValParams, authzIdentityResult, log,
(err, destinationBucket) => {
if (err) {
return next(err, destinationBucket);
}
if (destinationBucket.policies) {
// TODO: Check bucket policies to see if user is granted
// permission or forbidden permission to take
// given action.
// If permitted, add 'bucketPolicyGoAhead'
// attribute to params for validating at MPU level.
// This is GH Issue#76
metadataValMPUparams.requestType =
'bucketPolicyGoAhead';
}
return next(null, destinationBucket);
});
},
function checkMPUval(destBucket, next) {
metadataValParams.log = log;
services.metadataValidateMultipart(metadataValParams,
(err, mpuBucket, mpuOverviewObj) => {
if (err) {
return next(err, destBucket);
}
return next(err, mpuBucket, mpuOverviewObj, destBucket);
});
},
function abortExternalMpu(mpuBucket, mpuOverviewObj, destBucket,
next) {
const location = mpuOverviewObj.controllingLocationConstraint;
const originalIdentityAuthzResults = request.actionImplicitDenies;
// eslint-disable-next-line no-param-reassign
delete request.actionImplicitDenies;
return data.abortMPU(objectKey, uploadId, location, bucketName,
request, destBucket, locationConstraintCheck, log,
(err, skipDataDelete) => {
// eslint-disable-next-line no-param-reassign
request.actionImplicitDenies = originalIdentityAuthzResults;
if (err) {
return next(err, destBucket);
}
// for Azure and GCP we do not need to delete data
// for all other backends, skipDataDelete will be set to false
return next(null, mpuBucket, destBucket, skipDataDelete);
});
},
function getPartLocations(mpuBucket, destBucket, skipDataDelete,
next) {
services.getMPUparts(mpuBucket.getName(), uploadId, log,
(err, result) => {
if (err) {
return next(err, destBucket);
}
const storedParts = result.Contents;
return next(null, mpuBucket, storedParts, destBucket,
skipDataDelete);
});
},
function deleteData(mpuBucket, storedParts, destBucket,
skipDataDelete, next) {
if (skipDataDelete) {
return next(null, mpuBucket, storedParts, destBucket);
}
// The locations were sent to metadata as an array
// under partLocations. Pull the partLocations.
let locations = storedParts.map(item => item.value.partLocations);
if (locations.length === 0) {
return next(null, mpuBucket, storedParts, destBucket);
}
// flatten the array
locations = [].concat(...locations);
return async.eachLimit(locations, 5, (loc, cb) => {
data.delete(loc, log, err => {
if (err) {
log.fatal('delete ObjectPart failed', { err });
}
cb();
});
}, () => next(null, mpuBucket, storedParts, destBucket));
},
function deleteMetadata(mpuBucket, storedParts, destBucket, next) {
let splitter = constants.splitter;
// BACKWARD: Remove to remove the old splitter
if (mpuBucket.getMdBucketModelVersion() < 2) {
splitter = constants.oldSplitter;
}
// Reconstruct mpuOverviewKey
const mpuOverviewKey =
`overview${splitter}${objectKey}${splitter}${uploadId}`;
// Get the sum of all part sizes to include in pushMetric object
const partSizeSum = storedParts.map(item => item.value.Size)
.reduce((currPart, nextPart) => currPart + nextPart, 0);
const keysToDelete = storedParts.map(item => item.key);
keysToDelete.push(mpuOverviewKey);
services.batchDeleteObjectMetadata(mpuBucket.getName(),
keysToDelete, log, err => next(err, destBucket, partSizeSum));
},
], callback);
}
module.exports = abortMultipartUpload;

View File

@ -0,0 +1,19 @@
const { zenkoIDHeader } = require('arsenal').constants;
const _config = require('../../../Config').config;
/**
* applyZenkoUserMD - if request is within a Zenko deployment, apply user
* metadata called "zenko-source" to the object
* @param {Object} metaHeaders - user metadata object
* @return {undefined}
*/
function applyZenkoUserMD(metaHeaders) {
if (process.env.REMOTE_MANAGEMENT_DISABLE === '0' &&
!metaHeaders[zenkoIDHeader]) {
// eslint-disable-next-line no-param-reassign
metaHeaders[zenkoIDHeader] = _config.getPublicInstanceId();
}
}
module.exports = applyZenkoUserMD;

View File

@ -0,0 +1,25 @@
const { errors } = require('arsenal');
const { maxHttpHeadersSize } = require('../../../../constants');
/**
* Checks the size of the HTTP headers
* @param {object} requestHeaders - HTTP request headers
* @return {object} object with error or null
*/
function checkHttpHeadersSize(requestHeaders) {
let httpHeadersSize = 0;
Object.keys(requestHeaders).forEach(header => {
httpHeadersSize += Buffer.byteLength(header, 'utf8') +
Buffer.byteLength(requestHeaders[header], 'utf8');
});
if (httpHeadersSize > maxHttpHeadersSize) {
return {
httpHeadersSizeError: errors.HttpHeadersTooLarge,
};
}
return {};
}
module.exports = checkHttpHeadersSize;

View File

@ -0,0 +1,38 @@
const { maximumMetaHeadersSize,
invalidObjectUserMetadataHeader } = require('../../../../constants');
/**
* Checks the size of the user metadata in the object metadata and removes
* them from the response if the size of the user metadata is larger than
* the maximum size allowed. A custome metadata key is added to the response
* with the number of user metadata keys not returned as its value
* @param {object} responseMetadata - response metadata
* @return {object} responseMetaHeaders headers with object metadata to include
* in response to client
*/
function checkUserMetadataSize(responseMetadata) {
let userMetadataSize = 0;
// collect the user metadata keys from the object metadata
const userMetadataHeaders = Object.keys(responseMetadata)
.filter(key => key.startsWith('x-amz-meta-'));
// compute the size of all user metadata key and its value
userMetadataHeaders.forEach(header => {
userMetadataSize += header.length + responseMetadata[header].length;
});
// check the size computed against the maximum allowed
// if the computed size is greater, then remove all the
// user metadata from the response object
if (userMetadataSize > maximumMetaHeadersSize) {
const md = Object.assign({}, responseMetadata);
userMetadataHeaders.forEach(header => {
delete md[header];
});
// add the prescribed/custom metadata with number of user metadata
// as its value
md[invalidObjectUserMetadataHeader] = userMetadataHeaders.length;
return md;
}
return responseMetadata;
}
module.exports = checkUserMetadataSize;

View File

@ -0,0 +1,247 @@
/*
* Code based on Yutaka Oishi (Fujifilm) contributions
* Date: 11 Sep 2020
*/
const { ObjectMDArchive } = require('arsenal').models;
const errors = require('arsenal').errors;
const { config } = require('../../../Config');
const { locationConstraints } = config;
const { scaledMsPerDay } = config.getTimeOptions();
/**
* Get response header "x-amz-restore"
* Be called by objectHead.js
* @param {object} objMD - object's metadata
* @returns {string|undefined} x-amz-restore
*/
function getAmzRestoreResHeader(objMD) {
if (objMD.archive &&
objMD.archive.restoreRequestedAt &&
!objMD.archive.restoreCompletedAt) {
// Avoid race condition by relying on the `archive` MD of the object
// and return the right header after a RESTORE request.
// eslint-disable-next-line
return `ongoing-request="true"`;
}
if (objMD['x-amz-restore']) {
if (objMD['x-amz-restore']['expiry-date']) {
const utcDateTime = new Date(objMD['x-amz-restore']['expiry-date']).toUTCString();
// eslint-disable-next-line
return `ongoing-request="${objMD['x-amz-restore']['ongoing-request']}", expiry-date="${utcDateTime}"`;
}
}
return undefined;
}
/**
* Check if restore can be done.
*
* @param {ObjectMD} objectMD - object metadata
* @param {object} log - werelogs logger
* @return {ArsenalError|undefined} - undefined if the conditions for RestoreObject are fulfilled
*/
function _validateStartRestore(objectMD, log) {
if (objectMD.archive?.restoreCompletedAt) {
if (new Date(objectMD.archive?.restoreWillExpireAt) < new Date(Date.now())) {
// return InvalidObjectState error if the restored object is expired
// but restore info md of this object has not yet been cleared
log.debug('The restored object already expired.',
{
archive: objectMD.archive,
method: '_validateStartRestore',
});
return errors.InvalidObjectState;
}
// If object is already restored, no further check is needed
// Furthermore, we cannot check if the location is cold, as the `dataStoreName` would have
// been reset.
return undefined;
}
const isLocationCold = locationConstraints[objectMD.dataStoreName]?.isCold;
if (!isLocationCold) {
// return InvalidObjectState error if the object is not in cold storage,
// not in cold storage means either location cold flag not exists or cold flag is explicit false
log.debug('The bucket of the object is not in a cold storage location.',
{
isLocationCold,
method: '_validateStartRestore',
});
return errors.InvalidObjectState;
}
if (objectMD.archive?.restoreRequestedAt) {
// return RestoreAlreadyInProgress error if the object is currently being restored
// check if archive.restoreRequestAt exists and archive.restoreCompletedAt not yet exists
log.debug('The object is currently being restored.',
{
archive: objectMD.archive,
method: '_validateStartRestore',
});
return errors.RestoreAlreadyInProgress;
}
return undefined;
}
/**
* Check if "put version id" is allowed
*
* @param {ObjectMD} objMD - object metadata
* @param {string} versionId - object's version id
* @param {object} log - werelogs logger
* @return {ArsenalError|undefined} - undefined if "put version id" is allowed
*/
function validatePutVersionId(objMD, versionId, log) {
if (!objMD) {
const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey;
log.error('error no object metadata found', { method: 'validatePutVersionId', versionId });
return err;
}
if (objMD.isDeleteMarker) {
log.error('version is a delete marker', { method: 'validatePutVersionId', versionId });
return errors.MethodNotAllowed;
}
const isLocationCold = locationConstraints[objMD.dataStoreName]?.isCold;
if (!isLocationCold) {
log.error('The object data is not stored in a cold storage location.',
{
isLocationCold,
dataStoreName: objMD.dataStoreName,
method: 'validatePutVersionId',
});
return errors.InvalidObjectState;
}
// make sure object archive restoration is in progress
// NOTE: we do not use putObjectVersion to update the restoration period.
if (!objMD.archive || !objMD.archive.restoreRequestedAt || !objMD.archive.restoreRequestedDays
|| objMD.archive.restoreCompletedAt || objMD.archive.restoreWillExpireAt) {
log.error('object archive restoration is not in progress',
{ method: 'validatePutVersionId', versionId });
return errors.InvalidObjectState;
}
return undefined;
}
/**
* Check if the object is already restored, and update the expiration date accordingly:
* > After restoring an archived object, you can update the restoration period by reissuing the
* > request with a new period. Amazon S3 updates the restoration period relative to the current
* > time.
*
* @param {ObjectMD} objectMD - object metadata
* @param {object} log - werelogs logger
* @return {boolean} - true if the object is already restored
*/
function _updateObjectExpirationDate(objectMD, log) {
// Check if restoreCompletedAt field exists
// Normally, we should check `archive.restoreWillExpireAt > current time`; however this is
// checked earlier in the process, so checking again here would create weird states
const isObjectAlreadyRestored = !!objectMD.archive.restoreCompletedAt;
log.debug('The restore status of the object.', {
isObjectAlreadyRestored,
method: 'isObjectAlreadyRestored'
});
if (isObjectAlreadyRestored) {
const expiryDate = new Date(objectMD.archive.restoreRequestedAt);
expiryDate.setTime(expiryDate.getTime() + (objectMD.archive.restoreRequestedDays * scaledMsPerDay));
/* eslint-disable no-param-reassign */
objectMD.archive.restoreWillExpireAt = expiryDate;
objectMD['x-amz-restore'] = {
'ongoing-request': false,
'expiry-date': expiryDate,
};
/* eslint-enable no-param-reassign */
}
return isObjectAlreadyRestored;
}
/**
* update restore expiration date.
*
* @param {ObjectMD} objectMD - objectMD instance
* @param {object} restoreParam - restore param
* @param {object} log - werelogs logger
* @return {ArsenalError|undefined} internal error if object MD is not valid
*
*/
function _updateRestoreInfo(objectMD, restoreParam, log) {
if (!objectMD.archive) {
log.debug('objectMD.archive doesn\'t exits', {
objectMD,
method: '_updateRestoreInfo'
});
return errors.InternalError.customizeDescription('Archive metadata is missing.');
}
/* eslint-disable no-param-reassign */
objectMD.archive.restoreRequestedAt = new Date();
objectMD.archive.restoreRequestedDays = restoreParam.days;
objectMD.originOp = 's3:ObjectRestore:Post';
/* eslint-enable no-param-reassign */
if (!ObjectMDArchive.isValid(objectMD.archive)) {
log.debug('archive is not valid', {
archive: objectMD.archive,
method: '_updateRestoreInfo'
});
return errors.InternalError.customizeDescription('Invalid archive metadata.');
}
return undefined;
}
/**
* start to restore object.
* If not exist x-amz-restore, add it to objectMD.(x-amz-restore = false)
* calculate restore expiry-date and add it to objectMD.
* Be called by objectRestore.js
*
* @param {ObjectMD} objectMD - objectMd instance
* @param {object} restoreParam - bucket name
* @param {object} log - werelogs logger
* @param {function} cb - bucket name
* @return {undefined}
*
*/
function startRestore(objectMD, restoreParam, log, cb) {
log.info('Validating if restore can be done or not.');
const checkResultError = _validateStartRestore(objectMD, log);
if (checkResultError) {
return cb(checkResultError);
}
log.info('Updating restore information.');
const updateResultError = _updateRestoreInfo(objectMD, restoreParam, log);
if (updateResultError) {
return cb(updateResultError);
}
const isObjectAlreadyRestored = _updateObjectExpirationDate(objectMD, log);
return cb(null, isObjectAlreadyRestored);
}
/**
* checks if object data is available or if it's in cold storage
* @param {ObjectMD} objMD Object metadata
* @returns {ArsenalError|null} error if object data is not available
*/
function verifyColdObjectAvailable(objMD) {
// return error when object is cold
if (objMD.archive &&
// Object is in cold backend
(!objMD.archive.restoreRequestedAt ||
// Object is being restored
(objMD.archive.restoreRequestedAt && !objMD.archive.restoreCompletedAt))) {
const err = errors.InvalidObjectState
.customizeDescription('The operation is not valid for the object\'s storage class');
return err;
}
return null;
}
module.exports = {
startRestore,
getAmzRestoreResHeader,
validatePutVersionId,
verifyColdObjectAvailable,
};

View File

@ -0,0 +1,28 @@
/**
* generateToken - generates obfuscated continue token from object keyName
* @param {String} keyName - name of key to obfuscate
* @return {String} - obfuscated continue token
*/
function generateToken(keyName) {
if (keyName === '' || keyName === undefined) {
return undefined;
}
return Buffer.from(keyName).toString('base64');
}
/**
* decryptToken - decrypts object keyName from obfuscated continue token
* @param {String} token - obfuscated continue token
* @return {String} - object keyName
*/
function decryptToken(token) {
if (token === '' || token === undefined) {
return undefined;
}
return Buffer.from(token, 'base64').toString('utf8');
}
module.exports = {
generateToken,
decryptToken,
};

View File

@ -3,25 +3,24 @@ const { errors, s3middleware } = require('arsenal');
const getMetaHeaders = s3middleware.userMetadata.getMetaHeaders;
const constants = require('../../../../constants');
const data = require('../../../data/wrapper');
const { data } = require('../../../data/wrapper');
const services = require('../../../services');
const logger = require('../../../utilities/logger');
const { dataStore } = require('./storeObject');
const locationConstraintCheck = require('./locationConstraintCheck');
const { versioningPreprocessing } = require('./versioning');
const { versioningPreprocessing, overwritingVersioning } = require('./versioning');
const removeAWSChunked = require('./removeAWSChunked');
const getReplicationInfo = require('./getReplicationInfo');
const { config } = require('../../../Config');
const validateWebsiteHeader = require('./websiteServing')
.validateWebsiteHeader;
const applyZenkoUserMD = require('./applyZenkoUserMD');
const { externalBackends, versioningNotImplBackends } = constants;
const externalVersioningErrorMessage = 'We do not currently support putting ' +
'a versioned object to a location-constraint of type Azure or GCP.';
function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
metadataStoreParams, dataToDelete, deleteLog, requestMethod, callback) {
metadataStoreParams, dataToDelete, log, requestMethod, callback) {
services.metadataStoreObject(bucketName, dataGetInfo,
cipherBundle, metadataStoreParams, (err, result) => {
if (err) {
@ -30,8 +29,8 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
if (dataToDelete) {
const newDataStoreName = Array.isArray(dataGetInfo) ?
dataGetInfo[0].dataStoreName : null;
data.batchDelete(dataToDelete, requestMethod,
newDataStoreName, deleteLog);
return data.batchDelete(dataToDelete, requestMethod,
newDataStoreName, log, err => callback(err, result));
}
return callback(null, result);
});
@ -51,7 +50,9 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
* @param {(object|null)} streamingV4Params - if v4 auth, object containing
* accessKey, signatureFromRequest, region, scopeDate, timestamp, and
* credentialScope (to be used for streaming v4 auth if applicable)
* @param {(object|null)} overheadField - fields to be included in metadata overhead
* @param {RequestLogger} log - logger instance
* @param {string} originOp - Origin operation
* @param {function} callback - callback function
* @return {undefined} and call callback with (err, result) -
* result.contentMD5 - content md5 of new object or version
@ -59,7 +60,10 @@ function _storeInMDandDeleteData(bucketName, dataGetInfo, cipherBundle,
*/
function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
canonicalID, cipherBundle, request, isDeleteMarker, streamingV4Params,
log, callback) {
overheadField, log, originOp, callback) {
const putVersionId = request.headers['x-scal-s3-version-id'];
const isPutVersion = putVersionId || putVersionId === '';
const size = isDeleteMarker ? 0 : request.parsedContentLength;
// although the request method may actually be 'DELETE' if creating a
// delete marker, for our purposes we consider this to be a 'PUT'
@ -82,6 +86,9 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
});
return process.nextTick(() => callback(metaHeaders));
}
// if the request occurs within a Zenko deployment, we place a user-metadata
// field on the object
applyZenkoUserMD(metaHeaders);
log.trace('meta headers', { metaHeaders, method: 'objectPut' });
const objectKeyContext = {
@ -109,8 +116,24 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
isDeleteMarker,
replicationInfo: getReplicationInfo(
objectKey, bucketMD, false, size, null, null, authInfo),
overheadField,
log,
};
// For Azure BlobStorage API compatability
// If an object already exists copy/repair creation-time
// creation-time must remain static after an object is created
// --> EVEN FOR VERSIONS <--
if (objMD) {
if (objMD['creation-time']) {
metadataStoreParams.creationTime = objMD['creation-time'];
} else {
// If creation-time is not set (for old objects)
// fall back to the last modified and store it back to the db
metadataStoreParams.creationTime = objMD['last-modified'];
}
}
if (!isDeleteMarker) {
metadataStoreParams.contentType = request.headers['content-type'];
metadataStoreParams.cacheControl = request.headers['cache-control'];
@ -120,6 +143,13 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
removeAWSChunked(request.headers['content-encoding']);
metadataStoreParams.expires = request.headers.expires;
metadataStoreParams.tagging = request.headers['x-amz-tagging'];
metadataStoreParams.originOp = originOp;
const defaultObjectLockConfiguration
= bucketMD.getObjectLockConfiguration();
if (defaultObjectLockConfiguration) {
metadataStoreParams.defaultRetention
= defaultObjectLockConfiguration;
}
}
// if creating new delete marker and there is an existing object, copy
@ -128,6 +158,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
// eslint-disable-next-line no-param-reassign
request.headers[constants.objectLocationConstraintHeader] =
objMD[constants.objectLocationConstraintHeader];
metadataStoreParams.originOp = originOp;
}
const backendInfoObj =
@ -158,17 +189,49 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
}
}
if (objMD && objMD.uploadId) {
metadataStoreParams.oldReplayId = objMD.uploadId;
}
/* eslint-disable camelcase */
const dontSkipBackend = externalBackends;
/* eslint-enable camelcase */
const requestLogger =
logger.newRequestLoggerFromSerializedUids(log.getSerializedUids());
const mdOnlyHeader = request.headers['x-amz-meta-mdonly'];
const mdOnlySize = request.headers['x-amz-meta-size'];
return async.waterfall([
function storeData(next) {
if (size === 0 && !dontSkipBackend[locationType]) {
metadataStoreParams.contentMD5 = constants.emptyFileMd5;
return next(null, null, null);
if (size === 0) {
if (!dontSkipBackend[locationType]) {
metadataStoreParams.contentMD5 = constants.emptyFileMd5;
return next(null, null, null);
}
// Handle mdOnlyHeader as a metadata only operation. If
// the object in question is actually 0 byte or has a body size
// then handle normally.
if (mdOnlyHeader === 'true' && mdOnlySize > 0) {
log.debug('metadata only operation x-amz-meta-mdonly');
const md5 = request.headers['x-amz-meta-md5chksum']
? new Buffer(request.headers['x-amz-meta-md5chksum'],
'base64').toString('hex') : null;
const numParts = request.headers['x-amz-meta-md5numparts'];
let _md5;
if (numParts === undefined) {
_md5 = md5;
} else {
_md5 = `${md5}-${numParts}`;
}
const versionId = request.headers['x-amz-meta-version-id'];
const dataGetInfo = {
key: objectKey,
dataStoreName: location,
dataStoreType: locationType,
dataStoreVersionId: versionId,
dataStoreMD5: _md5,
};
return next(null, dataGetInfo, _md5);
}
}
return dataStore(objectKeyContext, cipherBundle, request, size,
streamingV4Params, backendInfo, log, next);
@ -193,16 +256,25 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
dataGetInfoArr[0].cipheredDataKey =
cipherBundle.cipheredDataKey;
}
if (mdOnlyHeader === 'true') {
metadataStoreParams.size = mdOnlySize;
dataGetInfoArr[0].size = mdOnlySize;
}
metadataStoreParams.contentMD5 = calculatedHash;
return next(null, dataGetInfoArr);
},
function getVersioningInfo(infoArr, next) {
// if x-scal-s3-version-id header is specified, we overwrite the object/version metadata.
if (isPutVersion) {
const options = overwritingVersioning(objMD, metadataStoreParams);
return process.nextTick(() => next(null, options, infoArr));
}
return versioningPreprocessing(bucketName, bucketMD,
metadataStoreParams.objectKey, objMD, log, (err, options) => {
if (err) {
// TODO: check AWS error when user requested a specific
// version before any versions have been put
const logLvl = err === errors.BadRequest ?
const logLvl = err.is.BadRequest ?
'debug' : 'error';
log[logLvl]('error getting versioning info', {
error: err,
@ -216,10 +288,13 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
metadataStoreParams.versionId = options.versionId;
metadataStoreParams.versioning = options.versioning;
metadataStoreParams.isNull = options.isNull;
metadataStoreParams.nullVersionId = options.nullVersionId;
metadataStoreParams.deleteNullKey = options.deleteNullKey;
if (options.extraMD) {
Object.assign(metadataStoreParams, options.extraMD);
}
return _storeInMDandDeleteData(bucketName, infoArr,
cipherBundle, metadataStoreParams,
options.dataToDelete, requestLogger, requestMethod, next);
options.dataToDelete, log, requestMethod, next);
},
], callback);
}

Some files were not shown because too many files have changed in this diff Show More