Compare commits
1257 Commits
developmen
...
w/8.1/bugf
Author | SHA1 | Date |
---|---|---|
bert-e | 8463fc4d40 | |
Nicolas Humbert | 9cc66d397f | |
bert-e | 7e372b7bd5 | |
bert-e | a121810552 | |
Nicolas Humbert | 06402c6c94 | |
Nicolas Humbert | a6f3c82827 | |
bert-e | a1eed4fefb | |
Nicolas Humbert | 68204448a1 | |
Nicolas Humbert | 40e271f7e2 | |
bert-e | d8f7f18f5a | |
bert-e | 2482fdfafc | |
bert-e | e151b3fff1 | |
Nicolas Humbert | b8bbdbbd81 | |
williamlardier | b6bc11881a | |
williamlardier | 648257612b | |
williamlardier | 7423fac674 | |
williamlardier | 9647043a02 | |
williamlardier | f9e1f91791 | |
Jonathan Gramain | 1a0a981271 | |
bert-e | a45b2eb6a4 | |
bert-e | b00378d46d | |
Jonathan Gramain | c72d8be223 | |
Jonathan Gramain | f63cb3c762 | |
bert-e | 15fd621c5c | |
bert-e | effbf63dd4 | |
bert-e | 285fe2f63b | |
bert-e | 1d8ebe6a9c | |
bert-e | 00555597e0 | |
bert-e | bddc2ccd01 | |
Jonathan Gramain | 7908654b51 | |
Jonathan Gramain | 0d7cf8d40a | |
Jonathan Gramain | c4c75e976c | |
Jonathan Gramain | 1266a14253 | |
williamlardier | 851c72bd0f | |
bert-e | 722b6ae699 | |
bert-e | 29925a15ad | |
Jonathan Gramain | 8dc3ba7ca6 | |
bert-e | 3c2283b062 | |
Jonathan Gramain | a6a76acede | |
Jonathan Gramain | 6a116734a9 | |
Jonathan Gramain | 9325ea4996 | |
Jonathan Gramain | 33ba89f0cf | |
Jonathan Gramain | c67331d350 | |
Jonathan Gramain | 6d6f1860ef | |
Mickael Bourgois | be1557d972 | |
Mickael Bourgois | a03463061c | |
Mickael Bourgois | 8ad0ea73a7 | |
Mickael Bourgois | a94040d13b | |
Frédéric Meinnel | 5a5ef7c572 | |
Frédéric Meinnel | 918c2c5473 | |
Frédéric Meinnel | f8ce90f9c3 | |
Frédéric Meinnel | 5734d11cf1 | |
Jonathan Gramain | 6f58f9dd68 | |
Jonathan Gramain | 3b9c93be68 | |
Jonathan Gramain | 081af3e795 | |
bert-e | 042f541a45 | |
bert-e | 39f42d9cb4 | |
Mickael Bourgois | 02f126f040 | |
bert-e | 1477a70e47 | |
Mickael Bourgois | 7233ec2635 | |
Frédéric Meinnel | 59d47a3e21 | |
Frédéric Meinnel | 6b61347c29 | |
Frédéric Meinnel | aea4663ff2 | |
bert-e | c2f6b45116 | |
bert-e | a0322b131c | |
bert-e | 993b9e6093 | |
bert-e | ddd6c87831 | |
bert-e | 7440794d93 | |
Mickael Bourgois | 1efab676bc | |
bert-e | 087369b37d | |
bert-e | 2d2030dfe4 | |
Will Toozs | da80e12dab | |
Will Toozs | a7cf94d0fe | |
Jonathan Gramain | 2a82095d03 | |
Jonathan Gramain | 44b3d25459 | |
Jonathan Gramain | f1d6e30fb6 | |
Jonathan Gramain | 9186643caa | |
Jonathan Gramain | 485a76ceb9 | |
Jonathan Gramain | 00109a2c44 | |
Jonathan Gramain | aed1247825 | |
bert-e | 37234efd14 | |
Jonathan Gramain | 2799381ef2 | |
Jonathan Gramain | f4e83086d6 | |
Jonathan Gramain | d08a267965 | |
Jonathan Gramain | 063a2fb8fb | |
Jonathan Gramain | 1bc3360daf | |
Jonathan Gramain | 206f14bdf5 | |
Maha Benzekri | 74ff1691a0 | |
Maha Benzekri | 5ffae72693 | |
bert-e | 2a4ea38301 | |
bert-e | df4c22154e | |
Francois Ferrand | d800179f86 | |
Francois Ferrand | c1c45a4af9 | |
Francois Ferrand | da536ed037 | |
Nicolas Humbert | 06901104e8 | |
Nicolas Humbert | a99a6d9d97 | |
Benoit A. | 863f45d256 | |
KillianG | 4b642cf8b4 | |
KillianG | 2537f8aa9a | |
Maha Benzekri | 7866a1d06f | |
Maha Benzekri | 29ef2ef265 | |
Maha Benzekri | 1509f1bdfe | |
Maha Benzekri | 13d349d211 | |
Maha Benzekri | 34a32c967d | |
Maha Benzekri | 90ab985271 | |
bert-e | d79ed1b9c8 | |
bert-e | c34ad0dc31 | |
williamlardier | 17b5bbc233 | |
williamlardier | 4aa8b5cc6e | |
williamlardier | 5deed6c2e1 | |
Nicolas Humbert | af34571771 | |
Nicolas Humbert | 79b83a9067 | |
Nicolas Humbert | 5fd675a316 | |
Nicolas Humbert | d84cc974d3 | |
Maha Benzekri | dcf0f902ff | |
Maha Benzekri | 0177fbe98f | |
bert-e | 5b66f8d089 | |
bert-e | b61d178b18 | |
Florent Monjalet | e51b06cfea | |
Florent Monjalet | f2bc701f8c | |
Nicolas Humbert | 4d6b03ba47 | |
Nicolas Humbert | f03f049683 | |
Nicolas Humbert | d7b51de024 | |
Nicolas Humbert | cf51adf1c7 | |
Nicolas Humbert | 8a7c1be2d1 | |
Nicolas Humbert | c049df0a97 | |
Nicolas Humbert | 2b2667e29a | |
Nicolas Humbert | 8eb4a29c36 | |
bert-e | 862317703e | |
Nicolas Humbert | e69a97f240 | |
Nicolas Humbert | 81e838000f | |
bert-e | 547ce816e0 | |
Nicolas Humbert | 8256d6debf | |
bert-e | 15d5e93a2d | |
Nicolas Humbert | 69c1698eb7 | |
bert-e | d11bcb56e9 | |
Nicolas Humbert | c2cd90925f | |
bert-e | 0ed35c3d86 | |
bert-e | b1723594eb | |
Nicolas Humbert | c0218821ff | |
Nicolas Humbert | 49e32758fb | |
Nicolas Humbert | e13d0f5ed8 | |
Nicolas Humbert | 0d5907956f | |
Nicolas Humbert | f0c5d60ce9 | |
Nicolas Humbert | 8c2f4cf357 | |
Nicolas Humbert | f3f1da9bb3 | |
Nicolas Humbert | 036b75842e | |
Nicolas Humbert | 7ac5774635 | |
Nicolas Humbert | f3b928fce0 | |
Nicolas Humbert | 7173a357d9 | |
Nicolas Humbert | 7c4f461196 | |
Nicolas Humbert | 0a4d6f862f | |
bert-e | 8716fee67d | |
bert-e | 2938bb0c88 | |
williamlardier | 05c93446ab | |
williamlardier | 8d758327dd | |
williamlardier | be63c09624 | |
Nicolas Humbert | 4615875462 | |
Rahul Padigela | bdb59a0e63 | |
bert-e | a89d1d8d75 | |
williamlardier | 57e84980c8 | |
williamlardier | 51bfd41bea | |
Nicolas Humbert | 96cbaeb821 | |
Nicolas Humbert | cb01346d07 | |
bert-e | 15b68fa9fa | |
Nicolas Humbert | 51703a65f5 | |
bert-e | 09aaa2d5ee | |
Nicolas Humbert | ad39d90b6f | |
Jonathan Gramain | 20e9fe4adb | |
bert-e | e9c67f7f67 | |
bert-e | 536d474f57 | |
bert-e | 55e68cfa17 | |
KillianG | 25be9014c9 | |
KillianG | ed42f24580 | |
KillianG | ce076cb3df | |
KillianG | 4bc3de52ff | |
bert-e | beb5f69be3 | |
bert-e | 5f3540a0d5 | |
Alexander Chan | 4093bf2b04 | |
Alexander Chan | d0bb6d5b0c | |
bert-e | 3f7229eebe | |
bert-e | 7eb9d52da5 | |
Nicolas Humbert | e216c9dd20 | |
williamlardier | 0c1afe535b | |
williamlardier | 73335ae6ec | |
Alexander Chan | 99c514e8f2 | |
Alexander Chan | cfd9fdcfc4 | |
Alexander Chan | d809dac5e3 | |
williamlardier | 53dac8d233 | |
williamlardier | 6d5ef07eee | |
williamlardier | 272166e406 | |
williamlardier | 3af05e672b | |
williamlardier | 8b0c90cb2f | |
Alexander Chan | dfc9b761e2 | |
Alexander Chan | 04f1eb7f04 | |
bert-e | c204b90847 | |
bert-e | 78d6e7fd72 | |
Alexander Chan | 7768fa8d35 | |
KillianG | 4d9a9adc48 | |
KillianG | c4804e52ee | |
KillianG | 671cf3a679 | |
Jonathan Gramain | 9a5e27f97b | |
Jonathan Gramain | d744a709d2 | |
Jonathan Gramain | a9d003c6f8 | |
Jonathan Gramain | 99e04bd6fa | |
Jonathan Gramain | d3bdddeba3 | |
bert-e | 3252f7de03 | |
Jonathan Gramain | c4cc5a2c3d | |
Jonathan Gramain | fedd0190cc | |
Jonathan Gramain | 56fd4ad734 | |
Jonathan Gramain | ebe6b65fcf | |
Nicolas Humbert | 7994bf7b96 | |
Nicolas Humbert | 4be0a06c4a | |
bert-e | da7dbdc51f | |
Will Toozs | 2103ef1237 | |
Will Toozs | dbc1c54246 | |
bert-e | 6c22f8404d | |
KillianG | 00e03f0592 | |
KillianG | d453758b7d | |
KillianG | a964dc99c3 | |
Jonathan Gramain | 3a4da1d7c0 | |
williamlardier | 5074e6c0a4 | |
williamlardier | bd05dd6918 | |
williamlardier | fbda12ce3c | |
Nicolas Humbert | b02934bb39 | |
Nicolas Humbert | c9a444969b | |
Nicolas Humbert | 5d018860ec | |
bert-e | 5838e02096 | |
Nicolas Humbert | ecd600ac4b | |
Naren | ab0324da05 | |
Naren | 2b353b33af | |
Naren | 5377b20ceb | |
Naren | 21b329b301 | |
bert-e | fd57f47be1 | |
bert-e | 94edf8be70 | |
Jonathan Gramain | 58e47e5015 | |
Jonathan Gramain | 4d782ecec6 | |
Jonathan Gramain | 655a10ce52 | |
Jonathan Gramain | 0c7f0e607d | |
Jonathan Gramain | caa5d53e9b | |
Jonathan Gramain | 21da975187 | |
bert-e | e0df67a115 | |
Naren | 7e18ae77e0 | |
Naren | 4750118f85 | |
Naren | c273c8b823 | |
Jonathan Gramain | d3b50fafa8 | |
Naren | 47e68a9b60 | |
Alexander Chan | bf4072151f | |
Alexander Chan | f33cd69e45 | |
Jonathan Gramain | 22fa04b7e7 | |
Jonathan Gramain | 10a94a0a96 | |
bert-e | 4d71a834d5 | |
Alexander Chan | fa26a487f5 | |
Jonathan Gramain | 66740f5aba | |
williamlardier | 9c46703b89 | |
williamlardier | 47672d60ce | |
Jonathan Gramain | 6d41d103e8 | |
Jonathan Gramain | 890ac08dcd | |
Jonathan Gramain | 4949b7cc35 | |
Jonathan Gramain | 2b6fee4e84 | |
williamlardier | c460338163 | |
williamlardier | f17d52b602 | |
williamlardier | a6b234b7a8 | |
williamlardier | ff353bb4d6 | |
williamlardier | 0f9c9c2f18 | |
williamlardier | f6b2cf2c1a | |
Kerkesni | ecafbae36a | |
Kerkesni | d1cd7e8dba | |
Francois Ferrand | 3da6719200 | |
Francois Ferrand | c0dd54ef51 | |
Francois Ferrand | 7910792390 | |
Francois Ferrand | a4f4c51290 | |
Francois Ferrand | 66c4bc52b5 | |
Francois Ferrand | 81cd6652d6 | |
Francois Ferrand | 2a07f67244 | |
Francois Ferrand | 1a634015ee | |
williamlardier | 7a88a54918 | |
williamlardier | b25e620750 | |
williamlardier | 38ef89cc83 | |
williamlardier | 1a6c828bfc | |
williamlardier | 3d769c6960 | |
williamlardier | 8a27920a85 | |
williamlardier | 7642a22176 | |
bert-e | 8f63687ef3 | |
Kerkesni | 26f45fa81a | |
Kerkesni | 76b59057f7 | |
Kerkesni | ae0da3d605 | |
bert-e | 162d9ec46b | |
Kerkesni | ccd6462015 | |
Kerkesni | 665c77570c | |
Kerkesni | 27307b397c | |
Kerkesni | 414eada32b | |
Kerkesni | fdf0c6fe99 | |
Kerkesni | 8cc0be7da2 | |
bert-e | 65231633a7 | |
Alexander Chan | 92c567414a | |
Alexander Chan | ec55e39175 | |
Jonathan Gramain | c343820cae | |
williamlardier | 8307a1513e | |
williamlardier | 706c2425fe | |
williamlardier | 8618d77de9 | |
Artem Bakalov | 8abe746222 | |
bert-e | e74cca6795 | |
bert-e | 1427abecb7 | |
bert-e | 4771ce3067 | |
williamlardier | 4e8a907d99 | |
Killian Gardahaut | 6f42b3e64c | |
Jonathan Gramain | 237872a5a3 | |
bert-e | 390fd97edf | |
Nicolas Humbert | 1c9e4eb93d | |
bert-e | a4f163f466 | |
Nicolas Humbert | 4d0cc9bc12 | |
bert-e | 657f969d05 | |
bert-e | b43cf22b2c | |
Killian Gardahaut | 46c44ccaa6 | |
bert-e | 90c63168c1 | |
Jonathan Gramain | fe5f868f43 | |
Killian Gardahaut | c0ee81eb7a | |
bert-e | 604a0170f1 | |
Killian Gardahaut | 9d8f4793c9 | |
Killian Gardahaut | 69d33a3341 | |
Jonathan Gramain | 981c9c1a23 | |
KillianG | 806f988334 | |
KillianG | 976a05c3e5 | |
Killian Gardahaut | c5004cb521 | |
KillianG | bc9cfb0b6d | |
KillianG | 4b6e342ff8 | |
Kerkesni | 480f5a4427 | |
bert-e | 852ae9bd0f | |
Taylor McKinnon | 3d77540c47 | |
Taylor McKinnon | 4f0506cf31 | |
Nicolas Humbert | d92a91f076 | |
Nicolas Humbert | 28779db602 | |
Alexander Chan | 8db16c5532 | |
Jordi Bertran de Balanda | 33439ec215 | |
bert-e | 9873c0f112 | |
Nicolas Humbert | 725a492c2c | |
Nicolas Humbert | e446e3e132 | |
bert-e | 25c6b34a1e | |
Jordi Bertran de Balanda | 721d7ede93 | |
Nicolas Humbert | fbbba32d69 | |
Jordi Bertran de Balanda | 56c1ba5c21 | |
Will Toozs | 73431094a3 | |
bert-e | 5919d20fa4 | |
Nicolas Humbert | 56665069c1 | |
Nicolas Humbert | 61fe54bd73 | |
Francois Ferrand | e227d9d5ca | |
Francois Ferrand | cdcc44d272 | |
Xin LI | 5acef6895f | |
williamlardier | f7d360fe0b | |
williamlardier | 0a61b43252 | |
williamlardier | c014e630be | |
williamlardier | a747d5feda | |
KillianG | 765857071a | |
KillianG | 91b39da7e5 | |
williamlardier | 2cc6ebe9b4 | |
williamlardier | 7887d22d0d | |
williamlardier | 2f142aea7f | |
williamlardier | 26a046c9b2 | |
bert-e | ab23d59daf | |
bert-e | 6950df200a | |
williamlardier | 3265d162a7 | |
bert-e | 67200d80ad | |
bert-e | aa2992cd9f | |
williamlardier | 0e2071ed3b | |
williamlardier | ad579b2bd2 | |
Guillaume Hivert | 139da904a7 | |
Guillaume Hivert | e8851b40c0 | |
Naren | cd9456b510 | |
Alexander Chan | 15f07538d8 | |
Guillaume Hivert | e95d07af12 | |
Guillaume Hivert | b21f7f3440 | |
Guillaume Hivert | ca2d23710f | |
Guillaume Hivert | 310fd30266 | |
Guillaume Hivert | 8743e9c3ac | |
bert-e | b2af7c0aea | |
bert-e | 58c24376aa | |
Guillaume Hivert | 62c13c1eed | |
Guillaume Hivert | ee81fa5829 | |
bert-e | d7df1df2b6 | |
bert-e | f59b1b5e07 | |
Guillaume Hivert | a3418603d0 | |
Guillaume Hivert | 947ccd90d9 | |
Guillaume Hivert | f460ffdb21 | |
Guillaume Hivert | dfa49c79c5 | |
Guillaume Hivert | e582882883 | |
Guillaume Hivert | dd61c1abbe | |
Guillaume Hivert | a15f8a56e3 | |
Guillaume Hivert | 43e82f7f33 | |
bert-e | d7625ced17 | |
Guillaume Hivert | a2c1989a5d | |
bert-e | 24755c8472 | |
bert-e | fb39a4095e | |
bert-e | 32dfba2f89 | |
Guillaume Hivert | a2ca197bd8 | |
Xin LI | 3ed46f2d16 | |
williamlardier | 5c936c94ee | |
Xin LI | f87101eef6 | |
Xin LI | 14f86282b6 | |
Xin LI | f9dba52d38 | |
Yutaka Oishi | 6714aed351 | |
williamlardier | 99f96dd377 | |
williamlardier | ae08d89d7d | |
williamlardier | c48e2948f0 | |
williamlardier | fc942febca | |
williamlardier | a4fe998c34 | |
williamlardier | 1460e94488 | |
williamlardier | dcc7117d88 | |
williamlardier | 99cee367aa | |
williamlardier | ad5a4c152d | |
bert-e | b608c043f5 | |
bert-e | 079c09e1ec | |
bert-e | 75f07440ef | |
bert-e | 3a6bac1158 | |
bert-e | f2d119326a | |
Guillaume Hivert | 2a019f3788 | |
bert-e | 5e22900c0f | |
Guillaume Hivert | e62ed598e8 | |
bert-e | a217ad58e8 | |
bert-e | 10cf10daa4 | |
Guillaume Hivert | 6ec2f99a91 | |
bert-e | dfd8f20bf2 | |
Guillaume Hivert | fc17ab4299 | |
Guillaume Hivert | 44f398b01f | |
Guillaume Hivert | dc32d78b0f | |
Guillaume Hivert | 073d752ad8 | |
bert-e | 3454e934f5 | |
Jordi Bertran de Balanda | 399fdaaed0 | |
Jordi Bertran de Balanda | 5084c8f971 | |
williamlardier | 3388de6fb6 | |
Alexander Chan | 86e9d4a356 | |
williamlardier | a0010efbdd | |
Nicolas Humbert | 8eb7efd58a | |
williamlardier | 25ae7e443b | |
williamlardier | 4afa1ed78d | |
williamlardier | 706dfddf5f | |
williamlardier | 4cce306a12 | |
williamlardier | f3bf6f2615 | |
williamlardier | bbe51b2e5e | |
williamlardier | 3cd06256d6 | |
Yutaka Oishi | 6e42216549 | |
williamlardier | e37712e94f | |
williamlardier | ac30d29509 | |
Xin LI | 1f235d569d | |
williamlardier | 320713a764 | |
Artem Bakalov | fbf686feab | |
Guillaume Hivert | 4b795a245c | |
Guillaume Hivert | 983d59d565 | |
Guillaume Hivert | fd7f0a1a91 | |
bert-e | 459fd99316 | |
Guillaume Hivert | 235b2ac6d4 | |
bert-e | 8025ce08fe | |
bert-e | bffb00266f | |
bert-e | a6cd3a67e0 | |
dependabot[bot] | 18605a9546 | |
dependabot[bot] | 74d7fe5e68 | |
dependabot[bot] | e707cf4398 | |
bert-e | 47c34a4f5c | |
bert-e | 59f7e32037 | |
Jordi Bertran de Balanda | fb286c6403 | |
williamlardier | 7f93695300 | |
bert-e | cecb5fc1b1 | |
bert-e | 75ba3733aa | |
dependabot[bot] | 7c6f5d34b8 | |
bert-e | 7e3190a600 | |
Jordi Bertran de Balanda | e9c4a5ce99 | |
Guillaume Hivert | f378a85799 | |
bert-e | 23ea19bcb3 | |
KillianG | d2c1400cb6 | |
bert-e | 6da31dfd18 | |
Yutaka Oishi | ee1e65d778 | |
williamlardier | 3534927ccf | |
Jordi Bertran de Balanda | 0e3edb847e | |
bert-e | a9f9fe99a5 | |
Jordi Bertran de Balanda | a587f78242 | |
Guillaume Hivert | 40e5100cd8 | |
Guillaume Hivert | 0851aa1406 | |
bert-e | 3ce4effafb | |
bert-e | b1897708e5 | |
bert-e | 019907e2ab | |
bert-e | 73729c7bdb | |
Kerkesni | 3f5e553d8a | |
bert-e | efea69ff70 | |
Guillaume Hivert | 8a2b62815b | |
bert-e | 0dbbb80bea | |
Guillaume Hivert | 2eecda3079 | |
bert-e | 011606e146 | |
Guillaume Hivert | 8271b3ba21 | |
Guillaume Hivert | a1b980b95b | |
bert-e | 4c47264a78 | |
bert-e | f69087814e | |
Ronnie Smith | cd432fa920 | |
Ronnie Smith | af0ab673d7 | |
Ronnie Smith | 334edbc17b | |
bert-e | 271b28e59b | |
bert-e | 7f641d2755 | |
bert-e | df91750c5a | |
bert-e | 1f2caf6a01 | |
Ronnie Smith | 1333195dcd | |
bert-e | f822c7bad9 | |
bert-e | b3ce76d7d8 | |
Artem Bakalov | 18887d10b3 | |
Ronnie Smith | 223897bbff | |
bert-e | e4d888c07b | |
bert-e | dece118ba9 | |
Will Toozs | a077cc199f | |
bert-e | b0cb6d9c0f | |
Alexander Chan | e0da963226 | |
bert-e | 209f3bae44 | |
Guillaume Hivert | e311f0d83d | |
Guillaume Hivert | dab763884a | |
Guillaume Hivert | 4f22e526ee | |
Guillaume Hivert | 3951bb289c | |
Guillaume Hivert | b97de6505c | |
Guillaume Hivert | a5ad298c3b | |
bert-e | 6919af95f2 | |
Guillaume Hivert | b94c13a115 | |
Guillaume Hivert | 666da6b1aa | |
Guillaume Hivert | 7192d4bc93 | |
bert-e | 1523f6baa6 | |
bert-e | c517e4531a | |
Kerkesni | 7bcb81985a | |
bert-e | 68ac02ad54 | |
Guillaume Hivert | 0d479c82c5 | |
Guillaume Hivert | f958ed3204 | |
bert-e | 7d80db5d7f | |
bert-e | 34ef6d0434 | |
bert-e | 0ce6a79961 | |
Kerkesni | 7477b881ed | |
Guillaume Hivert | 3874d16f42 | |
Guillaume Hivert | fac5605a18 | |
bert-e | 72057b1efc | |
bert-e | 529840fa37 | |
Guillaume Hivert | 0a5f7c4ea9 | |
bert-e | 0e4ac99d9d | |
KillianG | 218d21b819 | |
bert-e | 9333323301 | |
bert-e | e5929b9f91 | |
bert-e | 8998544c06 | |
KillianG | df33583aea | |
KillianG | 050d649db5 | |
bert-e | de81f65306 | |
bert-e | 5eaf67ac93 | |
bert-e | 193a399ae2 | |
bert-e | 4de18e5b26 | |
Jordi Bertran de Balanda | c7e2743bf9 | |
Jordi Bertran de Balanda | a8029d8779 | |
bert-e | d639f4cffe | |
Guillaume Hivert | b2ec34c8f2 | |
KillianG | fb31f93829 | |
bert-e | 6c6ee31f34 | |
Kerkesni | 64351cf20d | |
Ronnie Smith | b58b4d0773 | |
Ronnie Smith | 9a0915d40e | |
Ronnie Smith | 36d3a67a68 | |
Xin LI | 3d156a58dd | |
Xin LI | 7737ec4904 | |
Kerkesni | d18f4d10bd | |
Kerkesni | e0bc4383cd | |
bert-e | de17f221bf | |
Kerkesni | d46301b498 | |
Kerkesni | 0bb2a44912 | |
Guillaume Hivert | 2c1fb773fd | |
Xin.LI | 3528c24276 | |
Xin LI | 6d8294d0c0 | |
Xin LI | 23bfc17a26 | |
bert-e | 0f6a1f2982 | |
Nicolas Humbert | bff13f1190 | |
bert-e | c857e743c8 | |
Kerkesni | 5f8edd35e9 | |
Kerkesni | 3c4359b696 | |
Kerkesni | 8ecf1d9808 | |
Kerkesni | 74e4934654 | |
Kerkesni | eac87fc9de | |
Kerkesni | e2be4d895d | |
bert-e | c0f7ebbaa9 | |
Kerkesni | 60fcedc251 | |
Kerkesni | 10ef395501 | |
Kerkesni | d1c8e67901 | |
Kerkesni | 266aabef37 | |
Kerkesni | b63c909808 | |
Kerkesni | 02ee339214 | |
Kerkesni | 5ca7f86350 | |
Kerkesni | 50a4fd8dc1 | |
bert-e | 5de0c2a7da | |
Kerkesni | b942516dca | |
Kerkesni | 54181af522 | |
Kerkesni | 21af204956 | |
Kerkesni | 68a27be345 | |
Kerkesni | 06350ffe15 | |
Taylor McKinnon | 5da4cd88ff | |
bert-e | 6bb68ee0e3 | |
Taylor McKinnon | 9a4bae40e6 | |
bert-e | 54e9635cab | |
Vianney Rancurel | b8f803338b | |
Guillaume Hivert | 4a1215adb5 | |
Guillaume Hivert | fc8d7532c6 | |
Guillaume Hivert | 1818bfe6c8 | |
Guillaume Hivert | 5cd929ea8a | |
Guillaume Hivert | 1138ce43af | |
Guillaume Hivert | 8b4e9cc0aa | |
Guillaume Hivert | ff6ea2a6d5 | |
Guillaume Hivert | 3b3600db92 | |
bert-e | 51c5247d01 | |
Vianney Rancurel | 7813a312b5 | |
Thomas Carmet | 35a4552c0f | |
Vianney Rancurel | 0dbdff3a00 | |
bert-e | 80b91d724d | |
bert-e | 40843d4bed | |
bert-e | b3fd77d08f | |
Taylor McKinnon | ed6bc63e75 | |
Rached Ben Mustapha | c95f84e887 | |
Nicolas Humbert | 3c9ab1bb99 | |
Nicolas Humbert | 3c30adaf85 | |
bert-e | 98edeae3f2 | |
bert-e | 4f15e4f267 | |
Xin LI | 68c5b42e6f | |
Xin LI | 6933bb8422 | |
Xin LI | 7e180fcad8 | |
Naren | 41d482cf7d | |
Nicolas Humbert | 1e334924f9 | |
Naren | 49239cc76e | |
williamlardier | 8d17fcac0f | |
williamlardier | 1c3fcc5a65 | |
Ronnie Smith | f5b0f1e082 | |
williamlardier | 708aab707d | |
williamlardier | 3a1cbdeedb | |
bert-e | faf5701248 | |
Ronnie Smith | 4cbb5a5dd6 | |
bert-e | 22eca9b61c | |
Naren | 59a679831b | |
bert-e | 26da124e27 | |
bert-e | 47b121c17b | |
Ronnie Smith | c605c1e1a2 | |
bert-e | 994bd0a6be | |
Ronnie Smith | 1e2a6c387e | |
Ronnie Smith | 1348fc820f | |
Ronnie Smith | 79a363786f | |
bert-e | 86e3c02126 | |
bert-e | 8f6731aa6a | |
Artem Bakalov | ea2f8ebd01 | |
Artem Bakalov | b640bbb45e | |
Taylor McKinnon | d9fcf275ce | |
Ronnie Smith | 66b03695c3 | |
Rahul Padigela | 3575e651e3 | |
Rahul Padigela | fa19a34306 | |
Xin LI | 3ab7ef4e8d | |
Xin LI | e531d3eae1 | |
Nicolas Humbert | 9ebcc9690e | |
Nicolas Humbert | 95759509cb | |
williamlardier | 6cdae52d57 | |
williamlardier | 995cb59db4 | |
Alexander Chan | 385e34b472 | |
Jonathan Gramain | f102c5ec8c | |
bert-e | e912617f02 | |
williamlardier | 3abde0bc74 | |
bert-e | cf49c7d8bf | |
Alexander Chan | e6e49a70c9 | |
Rached Ben Mustapha | 77f971957b | |
Ronnie Smith | ed1d6c12c2 | |
williamlardier | 27f17f9535 | |
williamlardier | 4658651593 | |
Jonathan Gramain | 7af6a73b3b | |
bert-e | 8728ff5c80 | |
Ronnie Smith | 7c16652e57 | |
bert-e | 5a9d667936 | |
Rahul Padigela | 29dd069a5f | |
Rahul Padigela | f1793bfe51 | |
Rahul Padigela | b42f1d3943 | |
Naren | c27b359fba | |
Alexandre Lavigne | bb8bdbc6ea | |
Nicolas Humbert | 413f0c9433 | |
Nicolas Humbert | ab3fa2f13d | |
Naren | bfbda5d38b | |
Naren | 2e6b1791bb | |
Naren | 1f8cfecf43 | |
Alexandre Lavigne | 6a250feea9 | |
Thomas Carmet | 0a33d4b74e | |
Thomas Carmet | 9a544b9890 | |
Ronnie Smith | a2b6846e2e | |
Ronnie Smith | 3fdfc7196b | |
Ronnie Smith | f602fb9601 | |
Thomas Carmet | c237a25448 | |
Thomas Carmet | 5aaec6a4e6 | |
Thomas Carmet | 11278e7334 | |
bert-e | c0fe2efbc2 | |
Jonathan Gramain | b0633d8a13 | |
bert-e | b27caf5814 | |
bert-e | f5f6cb5692 | |
bert-e | 87ba4a7b4a | |
bert-e | 9ff605f875 | |
Thomas Carmet | 4e160db87d | |
bert-e | dc698f4d5c | |
bert-e | 8c7907f753 | |
bert-e | 395a881d92 | |
bert-e | 3d6306d2a3 | |
bert-e | 681740fbe7 | |
Alexander Chan | d381ec14d8 | |
bert-e | 0bdcd866bc | |
Jonathan Gramain | 856a1634d4 | |
Jonathan Gramain | 2921864aac | |
bert-e | 4665f3da5c | |
Jonathan Gramain | 0df0d952d2 | |
bert-e | 54eb3ede5f | |
bert-e | be4dea481d | |
Rached Ben Mustapha | d15e2d5df6 | |
Taylor McKinnon | 93503cf505 | |
bert-e | 0f63de2f05 | |
bert-e | 16a5e6a550 | |
Rached Ben Mustapha | 864d2e8a28 | |
vrancurel | 15703aafca | |
bert-e | db000bc5e1 | |
vrancurel | 06c35c15a5 | |
bert-e | 68c8189f53 | |
bert-e | 041731e6eb | |
Nicolas Humbert | d51361ce06 | |
Nicolas Humbert | 453fd8b722 | |
bert-e | 2621aa7e53 | |
bert-e | b4aeab77b9 | |
bert-e | e1a3b05330 | |
bert-e | 0151504158 | |
bert-e | 048e8b02bc | |
bert-e | 1d899efec8 | |
Taylor McKinnon | 4cb8f715e9 | |
bert-e | 580dda4d48 | |
bert-e | a17054e3a4 | |
bert-e | a8df2b7b96 | |
Taylor McKinnon | d572fc953b | |
Alexander Chan | 2a78d4f413 | |
Alexander Chan | d2c7165214 | |
bert-e | 1999a586fd | |
bert-e | a1c0dd2472 | |
bert-e | a22032f9a5 | |
bert-e | dd38e32797 | |
bert-e | 274bf80720 | |
Ronnie Smith | 25bd1f6111 | |
Jonathan Gramain | 2d41b034aa | |
Rached Ben Mustapha | bb8ec629bf | |
Rached Ben Mustapha | 4bbaa83b87 | |
bert-e | 58697f7915 | |
Ronnie Smith | bf4a6fe01b | |
alexandre merle | c703ba66e7 | |
alexandre merle | 20c77f9f85 | |
alexandre merle | edb27cc9a8 | |
alexandre merle | 79e0dfa38f | |
alexandre merle | e1118803e6 | |
bert-e | 1230e72c49 | |
bert-e | 372df634c4 | |
bert-e | 2b96888eb7 | |
bert-e | a0909885f1 | |
alexandre merle | 5d100645aa | |
bert-e | 356edf8478 | |
bert-e | 1cfb869631 | |
bert-e | 0403ca65fc | |
Rahul Padigela | 269e005198 | |
bert-e | 10627f51d1 | |
bert-e | aa5f714081 | |
Jonathan Gramain | d27c0577ee | |
Jonathan Gramain | ff539645ea | |
Jonathan Gramain | e5c3bb188a | |
Jonathan Gramain | 2461b5c2f7 | |
Jonathan Gramain | 747307cac2 | |
Jonathan Gramain | 5942d9d70c | |
bert-e | 8ed84786fc | |
bert-e | 1e40e76bb2 | |
bert-e | f4058dd6ef | |
bert-e | 04f7692bad | |
bert-e | 32752ac504 | |
vrancurel | 549f187893 | |
bert-e | 93cd582e3a | |
vrancurel | 2582108f97 | |
bert-e | b25867f9c2 | |
bert-e | 7b60166d08 | |
bert-e | 8887a67261 | |
Ronnie Smith | 437ecc57f9 | |
bert-e | 759f0ef949 | |
bert-e | 0014aa3467 | |
Dora Korpar | 1727f4bd3f | |
Dora Korpar | d71c8eac86 | |
bert-e | 7eb6304956 | |
bert-e | ce98e9d104 | |
bert-e | 36d932bbce | |
bert-e | 7f2c40cf6d | |
bert-e | 6a78af0f39 | |
bert-e | f73dc3dd68 | |
Jonathan Gramain | 8ec0611d08 | |
Jonathan Gramain | 6baca6f1e2 | |
bert-e | 78d62636c3 | |
Dora Korpar | 9b8f813d02 | |
Dora Korpar | 0f70366774 | |
bert-e | fb8cf65091 | |
Jonathan Gramain | 7792f7c603 | |
bert-e | 668d90b7d0 | |
bert-e | c1cfc59a0e | |
bert-e | f956b02387 | |
Jonathan Gramain | 86bca2502e | |
bert-e | 3aa49eed1d | |
Jonathan Gramain | a9c3b2218f | |
Jonathan Gramain | f459498e18 | |
bert-e | 55323aa7a2 | |
bert-e | a20e875908 | |
bert-e | a3a83f5ec8 | |
bert-e | 51d3312de8 | |
Ilke | 6383d14d49 | |
Jonathan Gramain | 0e4035d45b | |
Jonathan Gramain | a18285ced8 | |
Rahul Padigela | dc4e1829fc | |
bert-e | 3b438e03cd | |
bert-e | f2787ec013 | |
bert-e | 560ccef3ec | |
Dora Korpar | 3f4ed31153 | |
Jonathan Gramain | fc23f68d0f | |
bert-e | 2a4da20c0a | |
bert-e | 14c4696482 | |
bert-e | 275226278f | |
bert-e | b4b5712df7 | |
bert-e | 750c021c37 | |
bert-e | ee4d94c0fb | |
bert-e | 98f1d219a9 | |
Dora Korpar | fb363030c0 | |
Dora Korpar | 7aeb32e223 | |
bert-e | 5bdee7eb8a | |
bert-e | b8fd646097 | |
bert-e | a9d6e05c6e | |
Ilke | dc412e8953 | |
bert-e | 36b68be051 | |
bert-e | 3f19a00b32 | |
bert-e | ea8166cf7a | |
bert-e | c06f735e82 | |
bert-e | b8c4ae4203 | |
Dora Korpar | 0cf9a9cdd5 | |
bert-e | d201e572fd | |
bert-e | 400dc24281 | |
bert-e | f59cea6b34 | |
bert-e | f19feb949d | |
Jonathan Gramain | bbef1964d7 | |
bert-e | 43cd5f59b0 | |
bert-e | dd7390ade6 | |
Dora Korpar | a3739cc836 | |
bert-e | 97682f56bf | |
bert-e | ce4ca533e2 | |
bert-e | 26bff09887 | |
Pepijn Van Eeckhoudt | f6165146ec | |
Ilke | 9f580444f3 | |
Ilke | 93fe6fa94d | |
Jonathan Gramain | d9ff2c2060 | |
bert-e | e553342616 | |
Ilke | 8a9dbc4de7 | |
Jonathan Gramain | 81d05b6ea8 | |
bert-e | 44b8de565f | |
vrancurel | 3ed66c50f6 | |
bert-e | 90e1cff9f9 | |
Jonathan Gramain | 9f323b32ea | |
bert-e | dee53c8ad8 | |
bert-e | 9680071e1a | |
bert-e | 6dd3aa92a4 | |
bert-e | a9618bc0bb | |
bert-e | b6042035c0 | |
bert-e | d2fafe8ef3 | |
bert-e | fb18cba367 | |
bert-e | bab9d5dc24 | |
Alexander Chan | e531e5e711 | |
bert-e | f54d356669 | |
Jonathan Gramain | c1bb2ac058 | |
Jonathan Gramain | d76eeeea89 | |
Alexander Chan | ad58f66981 | |
bert-e | 85b5599ce2 | |
Dora Korpar | 3121d29140 | |
Jonathan Gramain | a75db3122f | |
bert-e | d994e2ae60 | |
Rached Ben Mustapha | c443793968 | |
Rached Ben Mustapha | 517a034291 | |
Rached Ben Mustapha | cc6671f37c | |
Rached Ben Mustapha | 87bb3126a3 | |
bert-e | cedd08686a | |
bert-e | 635d2fe6d9 | |
Jianqin Wang | 9557e36438 | |
bert-e | 2bb0e171d8 | |
bert-e | 68f5d3c9f2 | |
vrancurel | 71caf08c19 | |
Guillaume Gimenez | 38403b84aa | |
Jianqin Wang | 21610dd88d | |
bbuchanan9 | 7566d1f0a9 | |
bbuchanan9 | 28415a5c9b | |
Taylor McKinnon | 506a9ad37d | |
bert-e | 1c6e56e8ef | |
bbuchanan9 | 9d02f86cf5 | |
bert-e | 5c4547a3a9 | |
bbuchanan9 | 5de85713ef | |
Rahul Padigela | 68defde532 | |
Dora Korpar | 9e5d4ae95b | |
Dora Korpar | 633ce2c069 | |
Dora Korpar | 08ddc07d1c | |
Katherine Laue | bc6c9c8c36 | |
bert-e | 3dc9b958f7 | |
vrancurel | 4b5c0ff923 | |
vrancurel | 62536f66df | |
bert-e | 9032b89e6f | |
vrancurel | 9014761c70 | |
bert-e | 8d9864264d | |
Rahul Padigela | 839182292c | |
Rahul Padigela | a197b2b6a4 | |
bert-e | adf6cfc8e4 | |
bert-e | 40aa7d836f | |
bert-e | 4fa15fce2a | |
bert-e | 279f08c870 | |
anurag4dsb | 05a8475f1c | |
anurag4dsb | 8c664d9076 | |
Jianqin Wang | 77172f33f8 | |
Guillaume Gimenez | 0a0fe7f1da | |
Salim | 6d7437a776 | |
bert-e | 1a6174dadf | |
vrancurel | c57cde88bb | |
Rahul Padigela | 6e97c01edd | |
Rahul Padigela | dd6fde61bb | |
Benoit A | 3e8c43e05b | |
Nicolas Humbert | 633efcbc50 | |
Alexander Chan | d99b430ac4 | |
philipyoo | 8f71d4ff03 | |
Rahul Padigela | d0f77cee75 | |
bert-e | 4419db7b23 | |
Rahul Padigela | 3672df0fc4 | |
Dora Korpar | 9b223bea87 | |
Guillaume Gimenez | b7dfc3a9c0 | |
Dora Korpar | 787f66458f | |
Dora Korpar | 618b179d5c | |
bert-e | e6ddad1193 | |
bert-e | 6575be0050 | |
Jianqin Wang | 1f7263c320 | |
Jianqin Wang | 9da1a8e1f7 | |
Jianqin Wang | 14f8690a9a | |
Jianqin Wang | 700cb4eb48 | |
philipyoo | 7dd4dca7e5 | |
bert-e | a5d248000e | |
Taylor McKinnon | dae12b245b | |
bert-e | c0129eb0d7 | |
philipyoo | bd0d6c1942 | |
Jonathan Gramain | ed2d393e98 | |
bert-e | 886110138a | |
Jonathan Gramain | 397eecb370 | |
bert-e | 3623b992da | |
Jonathan Gramain | 78b64bebed | |
Dora Korpar | e857bb5f5a | |
Benoit A | 9c1dab1055 | |
bert-e | e18850911e | |
Jonathan Gramain | 2ff9cf866d | |
bbuchanan9 | cc6ed165dd | |
Dora Korpar | a6b5c21e5d | |
bbuchanan9 | 64426b1450 | |
bert-e | 160fe96b18 | |
Taylor McKinnon | 59290513e3 | |
Rahul Padigela | 6b9be35d8e | |
bbuchanan9 | dffcbefe9b | |
bbuchanan9 | c470cfb5b1 | |
philipyoo | abcff1b04e | |
bbuchanan9 | 6791d1b561 | |
bert-e | a8e0a30918 | |
philipyoo | 487fe8bf35 | |
bert-e | b7c84ef7d3 | |
bert-e | b55295818f | |
philipyoo | 0213bcfd25 | |
bert-e | 32b0946679 | |
JianqinWang | bef886d8ad | |
philipyoo | d44c2f123e | |
bert-e | f199d52c54 | |
bert-e | b9c419dde7 | |
bert-e | 5cf3948ba2 | |
bert-e | 226088c8fb | |
Rahul Padigela | bca10414bc | |
bert-e | 8f0cab8d91 | |
Jonathan Gramain | 40c234bb5f | |
bert-e | 26e2b5e425 | |
bert-e | df5a61cb8d | |
bert-e | b01a390c46 | |
Guillaume Gimenez | 87103f83e1 | |
bert-e | 9ba5d64cd2 | |
bert-e | f4d4c9b76e | |
bert-e | 2c149ea9b1 | |
philipyoo | 735ad74bda | |
bert-e | 1636c87556 | |
bert-e | 8e2d6d42a8 | |
bert-e | f11d6e223d | |
philipyoo | ebe2d1f24d | |
bert-e | 6a1bc69336 | |
bert-e | 0144158a37 | |
bert-e | aea19c9cc2 | |
bert-e | daaeb5637a | |
Dora Korpar | c479933448 | |
JianqinWang | f804aa9657 | |
Jonathan Gramain | ad35b9ec78 | |
Jonathan Gramain | 9fe0ba5c8c | |
bert-e | 2fe1e4da3c | |
bert-e | 6a4784417f | |
bert-e | 0ed8c750c9 | |
bert-e | 0d33e5a69f | |
bert-e | ac470f4233 | |
bert-e | 23d406dc81 | |
JianqinWang | f11ccbfefa | |
bert-e | c8c0527f65 | |
JianqinWang | d81d309420 | |
Dora Korpar | c657b4b469 | |
Dora Korpar | 65c99ff86d | |
Jonathan Gramain | 645433ed0c | |
JianqinWang | f9bb82ce43 | |
bert-e | ab4500b842 | |
bert-e | 40a802b715 | |
Giacomo Guiulfo | 84bf7bd511 | |
Giacomo Guiulfo | b5fa54ec11 | |
Bennett Buchanan | 58e9f26ae0 | |
Giacomo Guiulfo | d6fdd153aa | |
Giacomo Guiulfo | 1e05f0f54e | |
Giacomo Guiulfo | 9c66b7ceba | |
bert-e | 0555d0b41a | |
Guillaume Gimenez | 39f2a53beb | |
Bennett Buchanan | 0a75792ca6 | |
bert-e | 5225fc231d | |
Guillaume Gimenez | 30c3ce1e2b | |
Taylor McKinnon | aa157c6d13 | |
Bennett Buchanan | 699890d2d7 | |
Jonathan Gramain | ea1a7d4d87 | |
bert-e | a9297e707a | |
Bennett Buchanan | 75dccc528d | |
bert-e | 5d7cf78eda | |
Giacomo Guiulfo | 0a364fe379 | |
Rahul Padigela | 345031f2bd | |
greenkeeper[bot] | 0bc1fe1a71 | |
greenkeeper[bot] | f23e457b83 | |
greenkeeper[bot] | 09aca2dcf4 | |
greenkeeper[bot] | d304334e92 | |
greenkeeper[bot] | 7955b97810 | |
Rahul Padigela | d14cef843b | |
Dora Korpar | f2b39fb3d7 | |
Dora Korpar | 9a009746be | |
Jeremy Desanlis | 3e08bad2da | |
philipyoo | 13b156b226 | |
JianqinWang | 07f655c2f8 | |
JianqinWang | f496cec8bf | |
bert-e | 7f5413699d | |
Jonathan Gramain | d620fef517 | |
Jonathan Gramain | 8ac3cf5548 | |
Giacomo Guiulfo | ebd9a74666 | |
bert-e | a1f9bef60e | |
philipyoo | 899107913c | |
Jonathan Gramain | 18dfc6b4fa | |
Rahul Padigela | 9fe16c64fa | |
vrancurel | 3dee6e2d0b | |
vrancurel | 3545eb4d62 | |
Dora Korpar | 0a85eeb8b7 | |
Dora Korpar | 83759870f2 | |
Alexander Chan | 0d4bf3c17f | |
Alexander Chan | 0117b39dcf | |
Bennett Buchanan | 549ca1f683 | |
bert-e | e4a66343fb | |
philipyoo | a89fdde6fd | |
philipyoo | 872a2d88e5 | |
philipyoo | 0c9c462634 | |
philipyoo | a3973ac7d3 | |
bert-e | d1a8693fe5 | |
Jeremy Desanlis | 5687a48599 | |
Nicolas Humbert | 9dca871e1b | |
philipyoo | 7088812c80 | |
philipyoo | 9f742d4921 | |
bert-e | 2c31728905 | |
Bennett Buchanan | 125ccbbfa9 | |
bert-e | 40c8b37b30 | |
bert-e | 879075e4ec | |
philipyoo | 79ed68ce9f | |
bert-e | cbfacb5ec0 | |
philipyoo | 06dfdd9612 | |
philipyoo | bf95506495 | |
Alexander Chan | db743f8269 | |
Alexander Chan | a2311bb69c | |
Alexander Chan | c8f323237f | |
Rahul Padigela | 5cf55fcb68 | |
Rahul Padigela | de94a0e62e | |
Rahul Padigela | 2b13994795 | |
Rahul Padigela | 769a461178 | |
Rahul Padigela | c11fc1d9d8 | |
bert-e | b8ad86a1f1 | |
Giacomo Guiulfo | 12c4df722b | |
bert-e | f566e32322 | |
philipyoo | 6413c92fbc | |
bert-e | 29182cce05 | |
Jonathan Gramain | 9fb5b8b10d | |
vrancurel | 5631a892c6 | |
Rahul Padigela | dfcdea46fc | |
Rahul Padigela | be02e59bfe | |
Rahul Padigela | fdbeed1c4e | |
bert-e | 91fbc3fd23 | |
philipyoo | 241338bcfa | |
Rached Ben Mustapha | 6db80e9411 | |
bert-e | d701352635 | |
Alexander Chan | b291ccc03f | |
Bennett Buchanan | 0426f44dee | |
Rahul Padigela | 1b9242788a | |
Bennett Buchanan | 1a2ea2f353 | |
Bennett Buchanan | c36280a6e8 | |
bert-e | c749725410 | |
Alexander Chan | 3d06ec6230 | |
Jonathan Gramain | 159ebb4283 | |
Alexander Chan | e17333b19e | |
philipyoo | b3b22292c4 | |
bert-e | 68d27ed5bf | |
bert-e | 1e79964253 | |
philipyoo | 5f76343c2e | |
Alexander Chan | d907c9942d | |
Alexander Chan | c63b0713c0 | |
Alexander Chan | 6a9a88800a | |
Dora Korpar | 5834f15397 | |
bert-e | b50f6c4678 | |
bert-e | edeab02107 | |
David Pineau | c64cccdf55 | |
vrancurel | af2b3a4bc3 | |
philipyoo | 1e9ad08830 | |
David Pineau | 9e66fda610 | |
Rahul Padigela | 888e154f0e | |
Nicolas Humbert | 8448f909e4 | |
bert-e | 2b16e84733 | |
philipyoo | a1a6f65364 | |
bert-e | 7cf0c97d8e | |
Taylor McKinnon | 10e7b976d5 | |
vrancurel | e80ea95ad8 | |
Jeremy Desanlis | 7075318dd2 | |
bert-e | 38f68fba1a | |
vrancurel | 16f9a6f5f6 | |
bert-e | c48e4b89bd | |
Bennett Buchanan | 2a8169e936 | |
Alexander Chan | 1af67fffc7 | |
Guillaume Gimenez | e9ac11b1fe | |
bert-e | 30dcd6ef86 | |
Alexander Chan | 2ce9db4e01 | |
philipyoo | 9e234e2b41 | |
philipyoo | 83a831f512 | |
Guillaume Gimenez | 32c2a6fe99 | |
Rahul Padigela | 063361377c | |
Rahul Padigela | ea7f28c82d | |
Rahul Padigela | a9e760b32e | |
Rahul Padigela | 3b16a307b8 | |
Rahul Padigela | f8dfa378a1 | |
Jonathan Gramain | e16eadb474 | |
Rahul Padigela | 5bf7fef53c | |
philipyoo | 659aee2fc2 | |
Rahul Padigela | bde52ab89b | |
Jonathan Gramain | 0ddb4da8a9 | |
Rached Ben Mustapha | 56e280236b | |
Rached Ben Mustapha | f904f04401 | |
Rahul Padigela | db45fee9e8 | |
JianqinWang | ecc431c715 | |
JianqinWang | 6f694ae7f4 | |
Rahul Padigela | e7862d3922 | |
Jonathan Gramain | de7ebf70d7 | |
Rahul Padigela | 1425f03c1e | |
Alexander Chan | ad527911a2 | |
Rahul Padigela | 6c528688ee | |
Nicolas Humbert | e53aa2efd2 | |
Rahul Padigela | 873bc9b647 | |
Nicolas Humbert | 160b960607 | |
Rahul Padigela | 843bd1fe13 | |
Alexander Chan | 93a2a79699 | |
Rahul Padigela | ef32d5e94d | |
Alexander Chan | 45d9c3d999 | |
Rahul Padigela | a2ce46d8d0 | |
anurag4DSB | 0c0bffa2c3 | |
ironman-machine | d966c0bda9 | |
Rahul Padigela | cb86a857cc | |
Alexander Chan | 55c9441bd7 | |
David Pineau | cae55a65c8 | |
philipyoo | 114cbf5571 | |
Alexander Chan | f2bab3b3d6 | |
philipyoo | 3276d235bb | |
philipyoo | ee2aed10f3 | |
Rahul Padigela | 19bee770ea | |
Rahul Padigela | e0c5d03436 | |
Rahul Padigela | c8a7148645 | |
Rahul Padigela | 8ca5dce4fe | |
Bennett Buchanan | 599fb5709b | |
Rahul Padigela | 1161d5f75d | |
Rahul Padigela | 26b6c5d1d9 | |
Bennett Buchanan | 8fd50cd20e | |
Rahul Padigela | 1f6b5bf2bd | |
Rached Ben Mustapha | a7813daea9 | |
Rahul Padigela | 5d4eb84425 | |
Alexander Chan | 9511fff479 | |
Rahul Padigela | d70f64a6d0 | |
Alexander Chan | ee66dc811c | |
Rahul Padigela | 2710471726 | |
Dora Korpar | 9aee9f6cf0 | |
Rahul Padigela | a168fab266 | |
Dora Korpar | 92da4c90e5 | |
Rahul Padigela | a95d5ea15d | |
Salim | aad05faa12 | |
Rahul Padigela | ab230ebfe7 | |
Salim | b3103e1307 | |
Salim | f3b0091210 | |
Rahul Padigela | f633b91072 | |
Alexander Chan | 87807462dc | |
Rahul Padigela | d7f114d504 | |
Rached Ben Mustapha | 5ef168e654 | |
Rahul Padigela | 82b4055c6c | |
Rached Ben Mustapha | 91ccccfe85 | |
Rached Ben Mustapha | 696999874b | |
Rached Ben Mustapha | d2bed3bf9a | |
Rahul Padigela | ad42baa5ff | |
Rached Ben Mustapha | 6ac92b2ad2 | |
Rahul Padigela | 13dbf48867 | |
Rached Ben Mustapha | e79ad68e96 | |
Rahul Padigela | a4a5fe0db0 | |
Bennett Buchanan | f838fcc31f | |
VR | eb9dd23b14 | |
JianqinWang | edbf7ab650 | |
Rahul Padigela | e068950903 | |
Rahul Padigela | 1ceb7b264c | |
vrancurel | 5a29aaa10c | |
Rahul Padigela | 7587f7ba25 | |
Rahul Padigela | 795b145594 | |
Jeremy Desanlis | 58f027a693 | |
Rahul Padigela | e09348d658 | |
Alexander Chan | bddb90c6a1 | |
Rahul Padigela | 94efaaccc2 | |
Rahul Padigela | 463a8ebe15 | |
philipyoo | f17ce17857 | |
Rahul Padigela | 3a5250e2e9 | |
ironman-machine | 48cb7b3b05 | |
Nicolas Humbert | 84c4c147a2 | |
Rahul Padigela | 958e818655 | |
philipyoo | 91dd219c47 | |
Alexander Chan | 5f3d478edb | |
Rahul Padigela | 04d56cfdff | |
Rahul Padigela | 73dd529c29 | |
philipyoo | a9aa40c168 | |
ironman-machine | 189194a4e7 | |
JianqinWang | a9a6b2433d | |
JianqinWang | fa19fc8859 | |
JianqinWang | a269619698 | |
Rahul Padigela | da1da43597 | |
Rahul Padigela | caac4e4e7e | |
Rahul Padigela | 67250133dc | |
JianqinWang | d3f3be03ae | |
ironman-machine | 1a9f1afd2c | |
JianqinWang | 9a5afdbc5c | |
JianqinWang | 83cf54512b | |
ironman-machine | 7e3ad64456 | |
Nicolas Humbert | eba0cb6116 | |
Lauren Spiegel | fd23e82ab9 | |
Lauren Spiegel | d7cf5e8ccf | |
flavien-scality | d0f4f95f0d | |
Alexandre Merle | 0e606b1061 | |
ironman-machine | 44ead88d83 | |
vrancurel | d8e1497940 | |
ThibaultRiviere | 4193394340 | |
Thibault Riviere | 0f1b0dad01 | |
ironman-machine | 393d6edc07 | |
vrancurel | 70638eaf7a | |
Lauren Spiegel | 9d0156dfdf | |
Lauren Spiegel | 8d8028b83f | |
Lauren Spiegel | b99fe2cd8d | |
Lauren Spiegel | cc26f288be |
|
@ -1 +1,6 @@
|
|||
{ "extends": "scality" }
|
||||
{
|
||||
"extends": "scality",
|
||||
"parserOptions": {
|
||||
"ecmaVersion": 2020
|
||||
}
|
||||
}
|
||||
|
|
|
@ -14,7 +14,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v2
|
||||
|
|
|
@ -10,7 +10,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@v3
|
||||
|
|
|
@ -25,24 +25,28 @@ jobs:
|
|||
- 6379:6379
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
- uses: actions/setup-node@v2
|
||||
uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '16'
|
||||
cache: 'yarn'
|
||||
- name: install dependencies
|
||||
run: yarn install --frozen-lockfile --prefer-offline
|
||||
run: yarn install --frozen-lockfile --prefer-offline --network-concurrency 1
|
||||
continue-on-error: true # TODO ARSN-97 Remove it when no errors in TS
|
||||
- name: lint yaml
|
||||
run: yarn --silent lint_yml
|
||||
- name: lint javascript
|
||||
run: yarn --silent lint -- --max-warnings 0
|
||||
run: yarn --silent lint --max-warnings 0
|
||||
- name: lint markdown
|
||||
run: yarn --silent lint_md
|
||||
- name: run unit tests
|
||||
run: yarn test
|
||||
- name: add hostname
|
||||
run: |
|
||||
sudo sh -c "echo '127.0.0.1 testrequestbucket.localhost' >> /etc/hosts"
|
||||
- name: test and coverage
|
||||
run: yarn --silent coverage
|
||||
- name: run functional tests
|
||||
run: yarn ft_test
|
||||
- uses: codecov/codecov-action@v3
|
||||
- name: run executables tests
|
||||
run: yarn install && yarn test
|
||||
working-directory: 'lib/executables/pensieveCreds/'
|
||||
|
@ -53,9 +57,9 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
uses: actions/checkout@v4
|
||||
- name: Install NodeJS
|
||||
uses: actions/setup-node@v2
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '16'
|
||||
cache: yarn
|
||||
|
@ -66,7 +70,7 @@ jobs:
|
|||
run: yarn build
|
||||
continue-on-error: true # TODO ARSN-97 Remove it when no errors in TS
|
||||
- name: Upload artifacts
|
||||
uses: scality/action-artifacts@v2
|
||||
uses: scality/action-artifacts@v3
|
||||
with:
|
||||
url: https://artifacts.scality.net
|
||||
user: ${{ secrets.ARTIFACTS_USER }}
|
||||
|
|
|
@ -1,5 +1,7 @@
|
|||
# Arsenal
|
||||
|
||||
[![codecov](https://codecov.io/gh/scality/Arsenal/branch/development/8.1/graph/badge.svg?token=X0esXhJSwb)](https://codecov.io/gh/scality/Arsenal)
|
||||
|
||||
Common utilities for the S3 project components
|
||||
|
||||
Within this repository, you will be able to find the shared libraries for the
|
||||
|
|
|
@ -85,6 +85,66 @@ Used to store the bucket lifecycle configuration info
|
|||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._uid = uid || uuid();
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to set a unique identifier on a bucket
|
||||
|
||||
## Model version 8
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._readLocationConstraint = readLocationConstraint || null;
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to store default read location of the bucket
|
||||
|
||||
## Model version 9
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._isNFS = isNFS || null;
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to determine whether the bucket may be accessed through NFS
|
||||
|
||||
## Model version 10
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._ingestion = ingestionConfig || null;
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to store the ingestion status of a bucket
|
||||
|
||||
## Model version 11
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._azureInfo = azureInfo || null;
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to store Azure storage account specific information
|
||||
|
||||
## Model version 12
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._objectLockEnabled = objectLockEnabled || false;
|
||||
this._objectLockConfiguration = objectLockConfiguration || null;
|
||||
|
@ -95,7 +155,7 @@ this._objectLockConfiguration = objectLockConfiguration || null;
|
|||
Used to determine whether object lock capabilities are enabled on a bucket and
|
||||
to store the object lock configuration of the bucket
|
||||
|
||||
## Model version 8
|
||||
## Model version 13
|
||||
|
||||
### Properties Added
|
||||
|
||||
|
@ -107,7 +167,7 @@ this._notificationConfiguration = notificationConfiguration || null;
|
|||
|
||||
Used to store the bucket notification configuration info
|
||||
|
||||
## Model version 9
|
||||
## Model version 14
|
||||
|
||||
### Properties Added
|
||||
|
||||
|
@ -119,19 +179,7 @@ this._serverSideEncryption.configuredMasterKeyId = configuredMasterKeyId || unde
|
|||
|
||||
Used to store the users configured KMS key id
|
||||
|
||||
## Model version 10
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._uid = uid || uuid();
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to set a unique identifier on a bucket
|
||||
|
||||
## Model version 11
|
||||
## Model version 15
|
||||
|
||||
### Properties Added
|
||||
|
||||
|
@ -139,6 +187,62 @@ Used to set a unique identifier on a bucket
|
|||
this._tags = tags || null;
|
||||
```
|
||||
|
||||
The Tag Set of a bucket is an array of objects with Key and Value:
|
||||
|
||||
```javascript
|
||||
[
|
||||
{
|
||||
Key: 'something',
|
||||
Value: 'some_data'
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
## Model version 16
|
||||
|
||||
### Properties Added
|
||||
|
||||
```javascript
|
||||
this._capabilities = capabilities || undefined;
|
||||
```
|
||||
|
||||
For capacity-enabled buckets, contains the following data:
|
||||
|
||||
```javascript
|
||||
{
|
||||
_capabilities: {
|
||||
VeeamSOSApi?: {
|
||||
SystemInfo?: {
|
||||
ProtocolVersion: String,
|
||||
ModelName: String,
|
||||
ProtocolCapabilities: {
|
||||
CapacityInfo: Boolean,
|
||||
UploadSessions: Boolean,
|
||||
IAMSTS: Boolean,
|
||||
},
|
||||
APIEndpoints: {
|
||||
IAMEndpoint: String,
|
||||
STSEndpoint: String,
|
||||
},
|
||||
SystemRecommendations?: {
|
||||
S3ConcurrentTaskLimit: Number,
|
||||
S3MultiObjectDelete: Number,
|
||||
StorageCurrentTasksLimit: Number,
|
||||
KbBlockSize: Number,
|
||||
}
|
||||
LastModified?: String,
|
||||
},
|
||||
CapacityInfo?: {
|
||||
Capacity: Number,
|
||||
Available: Number,
|
||||
Used: Number,
|
||||
LastModified?: String,
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
Used to store bucket tagging
|
||||
Used to store bucket tagging
|
|
@ -0,0 +1,33 @@
|
|||
# DelimiterVersions
|
||||
|
||||
The DelimiterVersions class handles raw listings from the database of a
|
||||
versioned or non-versioned bucket with an optional delimiter, and
|
||||
fills in a curated listing with "Versions" and "CommonPrefixes" as a
|
||||
result.
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
- lists individual distinct versions of versioned buckets
|
||||
|
||||
- only lists keys belonging to the given **prefix** (if provided)
|
||||
|
||||
- groups listed keys that have a common prefix ending with a delimiter
|
||||
inside CommonPrefixes
|
||||
|
||||
- can take a **keyMarker** and optionally a **versionIdMarker** to
|
||||
list from a specific key or version
|
||||
|
||||
- can take a **maxKeys** parameter to limit how many keys can be returned
|
||||
|
||||
- skips internal keys like replay keys
|
||||
|
||||
## State Chart
|
||||
|
||||
- States with grey background are *Idle* states, which are waiting for
|
||||
a new listing key
|
||||
|
||||
- States with blue background are *Processing* states, which are
|
||||
actively processing a new listing key passed by the filter()
|
||||
function
|
||||
|
||||
![DelimiterVersions State Chart](./pics/delimiterVersionsStateChart.svg)
|
|
@ -0,0 +1,50 @@
|
|||
digraph {
|
||||
node [shape="box",style="filled,rounded",fontsize=16,fixedsize=true,width=3];
|
||||
edge [fontsize=14];
|
||||
rankdir=TB;
|
||||
|
||||
START [shape="circle",width=0.2,label="",style="filled",fillcolor="black"]
|
||||
END [shape="circle",width=0.2,label="",style="filled",fillcolor="black",peripheries=2]
|
||||
|
||||
node [fillcolor="lightgrey"];
|
||||
"NotSkipping.Idle" [label="NotSkipping",group="NotSkipping",width=4];
|
||||
"SkippingPrefix.Idle" [label="SkippingPrefix",group="SkippingPrefix"];
|
||||
"WaitForNullKey.Idle" [label="WaitForNullKey",group="WaitForNullKey"];
|
||||
"SkippingVersions.Idle" [label="SkippingVersions",group="SkippingVersions"];
|
||||
|
||||
node [fillcolor="lightblue"];
|
||||
"NotSkipping.Processing" [label="NotSkipping",group="NotSkipping",width=4];
|
||||
"NotSkippingV0.Processing" [label="NotSkippingV0",group="NotSkipping",width=4];
|
||||
"NotSkippingV1.Processing" [label="NotSkippingV1",group="NotSkipping",width=4];
|
||||
"NotSkippingCommon.Processing" [label="NotSkippingCommon",group="NotSkipping",width=4];
|
||||
"SkippingPrefix.Processing" [label="SkippingPrefix",group="SkippingPrefix"];
|
||||
"WaitForNullKey.Processing" [label="WaitForNullKey",group="WaitForNullKey"];
|
||||
"SkippingVersions.Processing" [label="SkippingVersions",group="SkippingVersions"];
|
||||
|
||||
START -> "WaitForNullKey.Idle" [label="[versionIdMarker != undefined]"]
|
||||
START -> "NotSkipping.Idle" [label="[versionIdMarker == undefined]"]
|
||||
|
||||
"NotSkipping.Idle" -> "NotSkipping.Processing" [label="filter(key, value)"]
|
||||
"SkippingPrefix.Idle" -> "SkippingPrefix.Processing" [label="filter(key, value)"]
|
||||
"WaitForNullKey.Idle" -> "WaitForNullKey.Processing" [label="filter(key, value)"]
|
||||
"SkippingVersions.Idle" -> "SkippingVersions.Processing" [label="filter(key, value)"]
|
||||
|
||||
"NotSkipping.Processing" -> "NotSkippingV0.Processing" [label="vFormat='v0'"]
|
||||
"NotSkipping.Processing" -> "NotSkippingV1.Processing" [label="vFormat='v1'"]
|
||||
|
||||
"WaitForNullKey.Processing" -> "NotSkipping.Processing" [label="master(key) != keyMarker"]
|
||||
"WaitForNullKey.Processing" -> "SkippingVersions.Processing" [label="master(key) == keyMarker"]
|
||||
"NotSkippingV0.Processing" -> "SkippingPrefix.Idle" [label="[key.startsWith(<ReplayPrefix>)]\n/ prefix <- <ReplayPrefix>\n-> FILTER_SKIP"]
|
||||
"NotSkippingV0.Processing" -> "NotSkipping.Idle" [label="[Version.isPHD(value)]\n-> FILTER_ACCEPT"]
|
||||
"NotSkippingV0.Processing" -> "NotSkippingCommon.Processing" [label="[not key.startsWith(<ReplayPrefix>)\nand not Version.isPHD(value)]"]
|
||||
"NotSkippingV1.Processing" -> "NotSkippingCommon.Processing" [label="[always]"]
|
||||
"NotSkippingCommon.Processing" -> END [label="[isListableKey(key, value) and\nKeys == maxKeys]\n-> FILTER_END"]
|
||||
"NotSkippingCommon.Processing" -> "SkippingPrefix.Idle" [label="[isListableKey(key, value) and\nnKeys < maxKeys and\nhasDelimiter(key)]\n/ prefix <- prefixOf(key)\n/ CommonPrefixes.append(prefixOf(key))\n-> FILTER_ACCEPT"]
|
||||
"NotSkippingCommon.Processing" -> "NotSkipping.Idle" [label="[isListableKey(key, value) and\nnKeys < maxKeys and\nnot hasDelimiter(key)]\n/ Contents.append(key, versionId, value)\n-> FILTER_ACCEPT"]
|
||||
|
||||
"SkippingPrefix.Processing" -> "SkippingPrefix.Idle" [label="[key.startsWith(prefix)]\n-> FILTER_SKIP"]
|
||||
"SkippingPrefix.Processing" -> "NotSkipping.Processing" [label="[not key.startsWith(prefix)]"]
|
||||
"SkippingVersions.Processing" -> "NotSkipping.Processing" [label="master(key) !== keyMarker or \nversionId > versionIdMarker"]
|
||||
"SkippingVersions.Processing" -> "SkippingVersions.Idle" [label="master(key) === keyMarker and \nversionId < versionIdMarker\n-> FILTER_SKIP"]
|
||||
"SkippingVersions.Processing" -> "SkippingVersions.Idle" [label="master(key) === keyMarker and \nversionId == versionIdMarker\n-> FILTER_ACCEPT"]
|
||||
}
|
|
@ -0,0 +1,265 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.43.0 (0)
|
||||
-->
|
||||
<!-- Title: %3 Pages: 1 -->
|
||||
<svg width="1522pt" height="922pt"
|
||||
viewBox="0.00 0.00 1522.26 922.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 918)">
|
||||
<title>%3</title>
|
||||
<polygon fill="white" stroke="transparent" points="-4,4 -4,-918 1518.26,-918 1518.26,4 -4,4"/>
|
||||
<!-- START -->
|
||||
<g id="node1" class="node">
|
||||
<title>START</title>
|
||||
<ellipse fill="black" stroke="black" cx="393.26" cy="-907" rx="7" ry="7"/>
|
||||
</g>
|
||||
<!-- NotSkipping.Idle -->
|
||||
<g id="node3" class="node">
|
||||
<title>NotSkipping.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M436.26,-675C436.26,-675 172.26,-675 172.26,-675 166.26,-675 160.26,-669 160.26,-663 160.26,-663 160.26,-651 160.26,-651 160.26,-645 166.26,-639 172.26,-639 172.26,-639 436.26,-639 436.26,-639 442.26,-639 448.26,-645 448.26,-651 448.26,-651 448.26,-663 448.26,-663 448.26,-669 442.26,-675 436.26,-675"/>
|
||||
<text text-anchor="middle" x="304.26" y="-653.2" font-family="Times,serif" font-size="16.00">NotSkipping</text>
|
||||
</g>
|
||||
<!-- START->NotSkipping.Idle -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>START->NotSkipping.Idle</title>
|
||||
<path fill="none" stroke="black" d="M391.06,-899.87C380.45,-870.31 334.26,-741.58 313.93,-684.93"/>
|
||||
<polygon fill="black" stroke="black" points="317.12,-683.46 310.45,-675.23 310.53,-685.82 317.12,-683.46"/>
|
||||
<text text-anchor="middle" x="470.76" y="-783.8" font-family="Times,serif" font-size="14.00">[versionIdMarker == undefined]</text>
|
||||
</g>
|
||||
<!-- WaitForNullKey.Idle -->
|
||||
<g id="node5" class="node">
|
||||
<title>WaitForNullKey.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M692.26,-849C692.26,-849 500.26,-849 500.26,-849 494.26,-849 488.26,-843 488.26,-837 488.26,-837 488.26,-825 488.26,-825 488.26,-819 494.26,-813 500.26,-813 500.26,-813 692.26,-813 692.26,-813 698.26,-813 704.26,-819 704.26,-825 704.26,-825 704.26,-837 704.26,-837 704.26,-843 698.26,-849 692.26,-849"/>
|
||||
<text text-anchor="middle" x="596.26" y="-827.2" font-family="Times,serif" font-size="16.00">WaitForNullKey</text>
|
||||
</g>
|
||||
<!-- START->WaitForNullKey.Idle -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>START->WaitForNullKey.Idle</title>
|
||||
<path fill="none" stroke="black" d="M399.56,-903.7C420.56,-896.05 489.7,-870.85 540.08,-852.48"/>
|
||||
<polygon fill="black" stroke="black" points="541.38,-855.73 549.57,-849.02 538.98,-849.16 541.38,-855.73"/>
|
||||
<text text-anchor="middle" x="608.76" y="-870.8" font-family="Times,serif" font-size="14.00">[versionIdMarker != undefined]</text>
|
||||
</g>
|
||||
<!-- END -->
|
||||
<g id="node2" class="node">
|
||||
<title>END</title>
|
||||
<ellipse fill="black" stroke="black" cx="45.26" cy="-120" rx="7" ry="7"/>
|
||||
<ellipse fill="none" stroke="black" cx="45.26" cy="-120" rx="11" ry="11"/>
|
||||
</g>
|
||||
<!-- NotSkipping.Processing -->
|
||||
<g id="node7" class="node">
|
||||
<title>NotSkipping.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M761.26,-558C761.26,-558 497.26,-558 497.26,-558 491.26,-558 485.26,-552 485.26,-546 485.26,-546 485.26,-534 485.26,-534 485.26,-528 491.26,-522 497.26,-522 497.26,-522 761.26,-522 761.26,-522 767.26,-522 773.26,-528 773.26,-534 773.26,-534 773.26,-546 773.26,-546 773.26,-552 767.26,-558 761.26,-558"/>
|
||||
<text text-anchor="middle" x="629.26" y="-536.2" font-family="Times,serif" font-size="16.00">NotSkipping</text>
|
||||
</g>
|
||||
<!-- NotSkipping.Idle->NotSkipping.Processing -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>NotSkipping.Idle->NotSkipping.Processing</title>
|
||||
<path fill="none" stroke="black" d="M333.17,-638.98C364.86,-620.99 417.68,-592.92 466.26,-576 483.64,-569.95 502.44,-564.74 520.88,-560.34"/>
|
||||
<polygon fill="black" stroke="black" points="521.83,-563.71 530.78,-558.04 520.25,-556.89 521.83,-563.71"/>
|
||||
<text text-anchor="middle" x="524.26" y="-594.8" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Idle -->
|
||||
<g id="node4" class="node">
|
||||
<title>SkippingPrefix.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M662.26,-138C662.26,-138 470.26,-138 470.26,-138 464.26,-138 458.26,-132 458.26,-126 458.26,-126 458.26,-114 458.26,-114 458.26,-108 464.26,-102 470.26,-102 470.26,-102 662.26,-102 662.26,-102 668.26,-102 674.26,-108 674.26,-114 674.26,-114 674.26,-126 674.26,-126 674.26,-132 668.26,-138 662.26,-138"/>
|
||||
<text text-anchor="middle" x="566.26" y="-116.2" font-family="Times,serif" font-size="16.00">SkippingPrefix</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Processing -->
|
||||
<g id="node11" class="node">
|
||||
<title>SkippingPrefix.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M779.26,-36C779.26,-36 587.26,-36 587.26,-36 581.26,-36 575.26,-30 575.26,-24 575.26,-24 575.26,-12 575.26,-12 575.26,-6 581.26,0 587.26,0 587.26,0 779.26,0 779.26,0 785.26,0 791.26,-6 791.26,-12 791.26,-12 791.26,-24 791.26,-24 791.26,-30 785.26,-36 779.26,-36"/>
|
||||
<text text-anchor="middle" x="683.26" y="-14.2" font-family="Times,serif" font-size="16.00">SkippingPrefix</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Idle->SkippingPrefix.Processing -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>SkippingPrefix.Idle->SkippingPrefix.Processing</title>
|
||||
<path fill="none" stroke="black" d="M552.64,-101.74C543.31,-87.68 534.41,-67.95 545.26,-54 549.71,-48.29 559.34,-43.36 571.56,-39.15"/>
|
||||
<polygon fill="black" stroke="black" points="572.87,-42.41 581.36,-36.07 570.77,-35.73 572.87,-42.41"/>
|
||||
<text text-anchor="middle" x="603.26" y="-65.3" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- WaitForNullKey.Processing -->
|
||||
<g id="node12" class="node">
|
||||
<title>WaitForNullKey.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M692.26,-762C692.26,-762 500.26,-762 500.26,-762 494.26,-762 488.26,-756 488.26,-750 488.26,-750 488.26,-738 488.26,-738 488.26,-732 494.26,-726 500.26,-726 500.26,-726 692.26,-726 692.26,-726 698.26,-726 704.26,-732 704.26,-738 704.26,-738 704.26,-750 704.26,-750 704.26,-756 698.26,-762 692.26,-762"/>
|
||||
<text text-anchor="middle" x="596.26" y="-740.2" font-family="Times,serif" font-size="16.00">WaitForNullKey</text>
|
||||
</g>
|
||||
<!-- WaitForNullKey.Idle->WaitForNullKey.Processing -->
|
||||
<g id="edge5" class="edge">
|
||||
<title>WaitForNullKey.Idle->WaitForNullKey.Processing</title>
|
||||
<path fill="none" stroke="black" d="M596.26,-812.8C596.26,-801.16 596.26,-785.55 596.26,-772.24"/>
|
||||
<polygon fill="black" stroke="black" points="599.76,-772.18 596.26,-762.18 592.76,-772.18 599.76,-772.18"/>
|
||||
<text text-anchor="middle" x="654.26" y="-783.8" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Idle -->
|
||||
<g id="node6" class="node">
|
||||
<title>SkippingVersions.Idle</title>
|
||||
<path fill="lightgrey" stroke="black" d="M1241.26,-558C1241.26,-558 1049.26,-558 1049.26,-558 1043.26,-558 1037.26,-552 1037.26,-546 1037.26,-546 1037.26,-534 1037.26,-534 1037.26,-528 1043.26,-522 1049.26,-522 1049.26,-522 1241.26,-522 1241.26,-522 1247.26,-522 1253.26,-528 1253.26,-534 1253.26,-534 1253.26,-546 1253.26,-546 1253.26,-552 1247.26,-558 1241.26,-558"/>
|
||||
<text text-anchor="middle" x="1145.26" y="-536.2" font-family="Times,serif" font-size="16.00">SkippingVersions</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Processing -->
|
||||
<g id="node13" class="node">
|
||||
<title>SkippingVersions.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M1241.26,-675C1241.26,-675 1049.26,-675 1049.26,-675 1043.26,-675 1037.26,-669 1037.26,-663 1037.26,-663 1037.26,-651 1037.26,-651 1037.26,-645 1043.26,-639 1049.26,-639 1049.26,-639 1241.26,-639 1241.26,-639 1247.26,-639 1253.26,-645 1253.26,-651 1253.26,-651 1253.26,-663 1253.26,-663 1253.26,-669 1247.26,-675 1241.26,-675"/>
|
||||
<text text-anchor="middle" x="1145.26" y="-653.2" font-family="Times,serif" font-size="16.00">SkippingVersions</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Idle->SkippingVersions.Processing -->
|
||||
<g id="edge6" class="edge">
|
||||
<title>SkippingVersions.Idle->SkippingVersions.Processing</title>
|
||||
<path fill="none" stroke="black" d="M1145.26,-558.25C1145.26,-576.77 1145.26,-606.45 1145.26,-628.25"/>
|
||||
<polygon fill="black" stroke="black" points="1141.76,-628.53 1145.26,-638.53 1148.76,-628.53 1141.76,-628.53"/>
|
||||
<text text-anchor="middle" x="1203.26" y="-594.8" font-family="Times,serif" font-size="14.00">filter(key, value)</text>
|
||||
</g>
|
||||
<!-- NotSkippingV0.Processing -->
|
||||
<g id="node8" class="node">
|
||||
<title>NotSkippingV0.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M436.26,-411C436.26,-411 172.26,-411 172.26,-411 166.26,-411 160.26,-405 160.26,-399 160.26,-399 160.26,-387 160.26,-387 160.26,-381 166.26,-375 172.26,-375 172.26,-375 436.26,-375 436.26,-375 442.26,-375 448.26,-381 448.26,-387 448.26,-387 448.26,-399 448.26,-399 448.26,-405 442.26,-411 436.26,-411"/>
|
||||
<text text-anchor="middle" x="304.26" y="-389.2" font-family="Times,serif" font-size="16.00">NotSkippingV0</text>
|
||||
</g>
|
||||
<!-- NotSkipping.Processing->NotSkippingV0.Processing -->
|
||||
<g id="edge7" class="edge">
|
||||
<title>NotSkipping.Processing->NotSkippingV0.Processing</title>
|
||||
<path fill="none" stroke="black" d="M573.96,-521.95C558.07,-516.64 540.84,-510.46 525.26,-504 460.22,-477.02 387.62,-439.36 343.97,-415.84"/>
|
||||
<polygon fill="black" stroke="black" points="345.57,-412.72 335.11,-411.04 342.24,-418.88 345.57,-412.72"/>
|
||||
<text text-anchor="middle" x="573.76" y="-462.8" font-family="Times,serif" font-size="14.00">vFormat='v0'</text>
|
||||
</g>
|
||||
<!-- NotSkippingV1.Processing -->
|
||||
<g id="node9" class="node">
|
||||
<title>NotSkippingV1.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M758.26,-411C758.26,-411 494.26,-411 494.26,-411 488.26,-411 482.26,-405 482.26,-399 482.26,-399 482.26,-387 482.26,-387 482.26,-381 488.26,-375 494.26,-375 494.26,-375 758.26,-375 758.26,-375 764.26,-375 770.26,-381 770.26,-387 770.26,-387 770.26,-399 770.26,-399 770.26,-405 764.26,-411 758.26,-411"/>
|
||||
<text text-anchor="middle" x="626.26" y="-389.2" font-family="Times,serif" font-size="16.00">NotSkippingV1</text>
|
||||
</g>
|
||||
<!-- NotSkipping.Processing->NotSkippingV1.Processing -->
|
||||
<g id="edge8" class="edge">
|
||||
<title>NotSkipping.Processing->NotSkippingV1.Processing</title>
|
||||
<path fill="none" stroke="black" d="M628.91,-521.8C628.39,-496.94 627.44,-450.74 626.83,-421.23"/>
|
||||
<polygon fill="black" stroke="black" points="630.32,-421.11 626.62,-411.18 623.33,-421.25 630.32,-421.11"/>
|
||||
<text text-anchor="middle" x="676.76" y="-462.8" font-family="Times,serif" font-size="14.00">vFormat='v1'</text>
|
||||
</g>
|
||||
<!-- NotSkippingV0.Processing->NotSkipping.Idle -->
|
||||
<g id="edge12" class="edge">
|
||||
<title>NotSkippingV0.Processing->NotSkipping.Idle</title>
|
||||
<path fill="none" stroke="black" d="M304.26,-411.25C304.26,-455.74 304.26,-574.61 304.26,-628.62"/>
|
||||
<polygon fill="black" stroke="black" points="300.76,-628.81 304.26,-638.81 307.76,-628.81 300.76,-628.81"/>
|
||||
<text text-anchor="middle" x="385.76" y="-543.8" font-family="Times,serif" font-size="14.00">[Version.isPHD(value)]</text>
|
||||
<text text-anchor="middle" x="385.76" y="-528.8" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- NotSkippingV0.Processing->SkippingPrefix.Idle -->
|
||||
<g id="edge11" class="edge">
|
||||
<title>NotSkippingV0.Processing->SkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M448.41,-376.93C508.52,-369.95 565.63,-362.09 570.26,-357 622.9,-299.12 594.8,-196.31 577.11,-147.78"/>
|
||||
<polygon fill="black" stroke="black" points="580.33,-146.4 573.53,-138.28 573.78,-148.87 580.33,-146.4"/>
|
||||
<text text-anchor="middle" x="720.26" y="-297.8" font-family="Times,serif" font-size="14.00">[key.startsWith(<ReplayPrefix>)]</text>
|
||||
<text text-anchor="middle" x="720.26" y="-282.8" font-family="Times,serif" font-size="14.00">/ prefix <- <ReplayPrefix></text>
|
||||
<text text-anchor="middle" x="720.26" y="-267.8" font-family="Times,serif" font-size="14.00">-> FILTER_SKIP</text>
|
||||
</g>
|
||||
<!-- NotSkippingCommon.Processing -->
|
||||
<g id="node10" class="node">
|
||||
<title>NotSkippingCommon.Processing</title>
|
||||
<path fill="lightblue" stroke="black" d="M436.26,-304.5C436.26,-304.5 172.26,-304.5 172.26,-304.5 166.26,-304.5 160.26,-298.5 160.26,-292.5 160.26,-292.5 160.26,-280.5 160.26,-280.5 160.26,-274.5 166.26,-268.5 172.26,-268.5 172.26,-268.5 436.26,-268.5 436.26,-268.5 442.26,-268.5 448.26,-274.5 448.26,-280.5 448.26,-280.5 448.26,-292.5 448.26,-292.5 448.26,-298.5 442.26,-304.5 436.26,-304.5"/>
|
||||
<text text-anchor="middle" x="304.26" y="-282.7" font-family="Times,serif" font-size="16.00">NotSkippingCommon</text>
|
||||
</g>
|
||||
<!-- NotSkippingV0.Processing->NotSkippingCommon.Processing -->
|
||||
<g id="edge13" class="edge">
|
||||
<title>NotSkippingV0.Processing->NotSkippingCommon.Processing</title>
|
||||
<path fill="none" stroke="black" d="M304.26,-374.74C304.26,-358.48 304.26,-333.85 304.26,-314.9"/>
|
||||
<polygon fill="black" stroke="black" points="307.76,-314.78 304.26,-304.78 300.76,-314.78 307.76,-314.78"/>
|
||||
<text text-anchor="middle" x="435.26" y="-345.8" font-family="Times,serif" font-size="14.00">[not key.startsWith(<ReplayPrefix>)</text>
|
||||
<text text-anchor="middle" x="435.26" y="-330.8" font-family="Times,serif" font-size="14.00">and not Version.isPHD(value)]</text>
|
||||
</g>
|
||||
<!-- NotSkippingV1.Processing->NotSkippingCommon.Processing -->
|
||||
<g id="edge14" class="edge">
|
||||
<title>NotSkippingV1.Processing->NotSkippingCommon.Processing</title>
|
||||
<path fill="none" stroke="black" d="M616.43,-374.83C606.75,-359.62 590.48,-338.14 570.26,-327 549.98,-315.83 505.48,-307.38 458.57,-301.23"/>
|
||||
<polygon fill="black" stroke="black" points="458.9,-297.74 448.53,-299.95 458.01,-304.69 458.9,-297.74"/>
|
||||
<text text-anchor="middle" x="632.26" y="-338.3" font-family="Times,serif" font-size="14.00">[always]</text>
|
||||
</g>
|
||||
<!-- NotSkippingCommon.Processing->END -->
|
||||
<g id="edge15" class="edge">
|
||||
<title>NotSkippingCommon.Processing->END</title>
|
||||
<path fill="none" stroke="black" d="M159.92,-279.56C109.8,-274.24 62.13,-264.33 46.26,-246 20.92,-216.72 30.42,-167.54 38.5,-140.42"/>
|
||||
<polygon fill="black" stroke="black" points="41.94,-141.16 41.67,-130.57 35.27,-139.02 41.94,-141.16"/>
|
||||
<text text-anchor="middle" x="152.76" y="-212.3" font-family="Times,serif" font-size="14.00">[isListableKey(key, value) and</text>
|
||||
<text text-anchor="middle" x="152.76" y="-197.3" font-family="Times,serif" font-size="14.00">Keys == maxKeys]</text>
|
||||
<text text-anchor="middle" x="152.76" y="-182.3" font-family="Times,serif" font-size="14.00">-> FILTER_END</text>
|
||||
</g>
|
||||
<!-- NotSkippingCommon.Processing->NotSkipping.Idle -->
|
||||
<g id="edge17" class="edge">
|
||||
<title>NotSkippingCommon.Processing->NotSkipping.Idle</title>
|
||||
<path fill="none" stroke="black" d="M214.74,-304.54C146.51,-322.73 57.06,-358.99 13.26,-429 -49.27,-528.95 128.43,-602.49 233.32,-635.95"/>
|
||||
<polygon fill="black" stroke="black" points="232.34,-639.31 242.93,-638.97 234.43,-632.63 232.34,-639.31"/>
|
||||
<text text-anchor="middle" x="156.76" y="-492.8" font-family="Times,serif" font-size="14.00">[isListableKey(key, value) and</text>
|
||||
<text text-anchor="middle" x="156.76" y="-477.8" font-family="Times,serif" font-size="14.00">nKeys < maxKeys and</text>
|
||||
<text text-anchor="middle" x="156.76" y="-462.8" font-family="Times,serif" font-size="14.00">not hasDelimiter(key)]</text>
|
||||
<text text-anchor="middle" x="156.76" y="-447.8" font-family="Times,serif" font-size="14.00">/ Contents.append(key, versionId, value)</text>
|
||||
<text text-anchor="middle" x="156.76" y="-432.8" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- NotSkippingCommon.Processing->SkippingPrefix.Idle -->
|
||||
<g id="edge16" class="edge">
|
||||
<title>NotSkippingCommon.Processing->SkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M292.14,-268.23C288.18,-261.59 284.27,-253.75 282.26,-246 272.21,-207.28 255.76,-185.96 282.26,-156 293.6,-143.18 374.98,-134.02 447.74,-128.3"/>
|
||||
<polygon fill="black" stroke="black" points="448.24,-131.77 457.94,-127.51 447.7,-124.79 448.24,-131.77"/>
|
||||
<text text-anchor="middle" x="428.26" y="-234.8" font-family="Times,serif" font-size="14.00">[isListableKey(key, value) and</text>
|
||||
<text text-anchor="middle" x="428.26" y="-219.8" font-family="Times,serif" font-size="14.00">nKeys < maxKeys and</text>
|
||||
<text text-anchor="middle" x="428.26" y="-204.8" font-family="Times,serif" font-size="14.00">hasDelimiter(key)]</text>
|
||||
<text text-anchor="middle" x="428.26" y="-189.8" font-family="Times,serif" font-size="14.00">/ prefix <- prefixOf(key)</text>
|
||||
<text text-anchor="middle" x="428.26" y="-174.8" font-family="Times,serif" font-size="14.00">/ CommonPrefixes.append(prefixOf(key))</text>
|
||||
<text text-anchor="middle" x="428.26" y="-159.8" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Processing->SkippingPrefix.Idle -->
|
||||
<g id="edge18" class="edge">
|
||||
<title>SkippingPrefix.Processing->SkippingPrefix.Idle</title>
|
||||
<path fill="none" stroke="black" d="M681.57,-36.04C679.28,-50.54 673.9,-71.03 661.26,-84 656.4,-88.99 650.77,-93.28 644.72,-96.95"/>
|
||||
<polygon fill="black" stroke="black" points="642.71,-94.06 635.6,-101.92 646.05,-100.21 642.71,-94.06"/>
|
||||
<text text-anchor="middle" x="759.26" y="-72.8" font-family="Times,serif" font-size="14.00">[key.startsWith(prefix)]</text>
|
||||
<text text-anchor="middle" x="759.26" y="-57.8" font-family="Times,serif" font-size="14.00">-> FILTER_SKIP</text>
|
||||
</g>
|
||||
<!-- SkippingPrefix.Processing->NotSkipping.Processing -->
|
||||
<g id="edge19" class="edge">
|
||||
<title>SkippingPrefix.Processing->NotSkipping.Processing</title>
|
||||
<path fill="none" stroke="black" d="M791.46,-33.51C815.84,-38.71 837.21,-45.46 846.26,-54 868.07,-74.57 864.26,-89.02 864.26,-119 864.26,-394 864.26,-394 864.26,-394 864.26,-462.4 791.27,-499.6 726.64,-519.12"/>
|
||||
<polygon fill="black" stroke="black" points="725.39,-515.84 716.77,-521.99 727.35,-522.56 725.39,-515.84"/>
|
||||
<text text-anchor="middle" x="961.26" y="-282.8" font-family="Times,serif" font-size="14.00">[not key.startsWith(prefix)]</text>
|
||||
</g>
|
||||
<!-- WaitForNullKey.Processing->NotSkipping.Processing -->
|
||||
<g id="edge9" class="edge">
|
||||
<title>WaitForNullKey.Processing->NotSkipping.Processing</title>
|
||||
<path fill="none" stroke="black" d="M599.08,-725.78C604.81,-690.67 617.89,-610.59 624.8,-568.31"/>
|
||||
<polygon fill="black" stroke="black" points="628.3,-568.61 626.46,-558.18 621.39,-567.48 628.3,-568.61"/>
|
||||
<text text-anchor="middle" x="707.26" y="-653.3" font-family="Times,serif" font-size="14.00">master(key) != keyMarker</text>
|
||||
</g>
|
||||
<!-- WaitForNullKey.Processing->SkippingVersions.Processing -->
|
||||
<g id="edge10" class="edge">
|
||||
<title>WaitForNullKey.Processing->SkippingVersions.Processing</title>
|
||||
<path fill="none" stroke="black" d="M704.4,-726.26C797.32,-711.87 931.09,-691.16 1026.87,-676.33"/>
|
||||
<polygon fill="black" stroke="black" points="1027.55,-679.77 1036.89,-674.78 1026.47,-672.85 1027.55,-679.77"/>
|
||||
<text text-anchor="middle" x="1001.26" y="-696.8" font-family="Times,serif" font-size="14.00">master(key) == keyMarker</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Processing->SkippingVersions.Idle -->
|
||||
<g id="edge21" class="edge">
|
||||
<title>SkippingVersions.Processing->SkippingVersions.Idle</title>
|
||||
<path fill="none" stroke="black" d="M1241.89,-638.98C1249.74,-634.29 1256.75,-628.4 1262.26,-621 1274.21,-604.96 1274.21,-592.04 1262.26,-576 1258.82,-571.38 1254.79,-567.34 1250.33,-563.82"/>
|
||||
<polygon fill="black" stroke="black" points="1252.11,-560.8 1241.89,-558.02 1248.15,-566.57 1252.11,-560.8"/>
|
||||
<text text-anchor="middle" x="1392.26" y="-609.8" font-family="Times,serif" font-size="14.00">master(key) === keyMarker and </text>
|
||||
<text text-anchor="middle" x="1392.26" y="-594.8" font-family="Times,serif" font-size="14.00">versionId < versionIdMarker</text>
|
||||
<text text-anchor="middle" x="1392.26" y="-579.8" font-family="Times,serif" font-size="14.00">-> FILTER_SKIP</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Processing->SkippingVersions.Idle -->
|
||||
<g id="edge22" class="edge">
|
||||
<title>SkippingVersions.Processing->SkippingVersions.Idle</title>
|
||||
<path fill="none" stroke="black" d="M1036.97,-654.38C978.97,-650.96 915.73,-642.25 897.26,-621 884.15,-605.9 884.15,-591.1 897.26,-576 914.65,-555.99 971.71,-547.1 1026.73,-543.28"/>
|
||||
<polygon fill="black" stroke="black" points="1027.21,-546.76 1036.97,-542.62 1026.76,-539.77 1027.21,-546.76"/>
|
||||
<text text-anchor="middle" x="1019.26" y="-609.8" font-family="Times,serif" font-size="14.00">master(key) === keyMarker and </text>
|
||||
<text text-anchor="middle" x="1019.26" y="-594.8" font-family="Times,serif" font-size="14.00">versionId == versionIdMarker</text>
|
||||
<text text-anchor="middle" x="1019.26" y="-579.8" font-family="Times,serif" font-size="14.00">-> FILTER_ACCEPT</text>
|
||||
</g>
|
||||
<!-- SkippingVersions.Processing->NotSkipping.Processing -->
|
||||
<g id="edge20" class="edge">
|
||||
<title>SkippingVersions.Processing->NotSkipping.Processing</title>
|
||||
<path fill="none" stroke="black" d="M1037.02,-651.24C897.84,-644.67 672.13,-632.37 657.26,-621 641.04,-608.6 634.18,-586.13 631.3,-568.16"/>
|
||||
<polygon fill="black" stroke="black" points="634.76,-567.68 630.02,-558.21 627.82,-568.57 634.76,-567.68"/>
|
||||
<text text-anchor="middle" x="770.26" y="-602.3" font-family="Times,serif" font-size="14.00">master(key) !== keyMarker or </text>
|
||||
<text text-anchor="middle" x="770.26" y="-587.3" font-family="Times,serif" font-size="14.00">versionId > versionIdMarker</text>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
After Width: | Height: | Size: 21 KiB |
|
@ -0,0 +1,28 @@
|
|||
{
|
||||
"groups": {
|
||||
"default": {
|
||||
"packages": [
|
||||
"lib/executables/pensieveCreds/package.json",
|
||||
"package.json"
|
||||
]
|
||||
}
|
||||
},
|
||||
"branchPrefix": "improvement/greenkeeper.io/",
|
||||
"commitMessages": {
|
||||
"initialBadge": "docs(readme): add Greenkeeper badge",
|
||||
"initialDependencies": "chore(package): update dependencies",
|
||||
"initialBranches": "chore(bert-e): whitelist greenkeeper branches",
|
||||
"dependencyUpdate": "fix(package): update ${dependency} to version ${version}",
|
||||
"devDependencyUpdate": "chore(package): update ${dependency} to version ${version}",
|
||||
"dependencyPin": "fix: pin ${dependency} to ${oldVersionResolved}",
|
||||
"devDependencyPin": "chore: pin ${dependency} to ${oldVersionResolved}",
|
||||
"closes": "\n\nCloses #${number}"
|
||||
},
|
||||
"ignore": [
|
||||
"ajv",
|
||||
"eslint",
|
||||
"eslint-plugin-react",
|
||||
"eslint-config-airbnb",
|
||||
"eslint-config-scality"
|
||||
]
|
||||
}
|
24
index.ts
24
index.ts
|
@ -4,11 +4,13 @@ import RequestContext from './lib/policyEvaluator/RequestContext';
|
|||
import * as requestUtils from './lib/policyEvaluator/requestUtils';
|
||||
import * as actionMaps from './lib/policyEvaluator/utils/actionMaps';
|
||||
import { validateUserPolicy } from './lib/policy/policyValidator'
|
||||
import * as locationConstraints from './lib/patches/locationConstraints';
|
||||
import * as userMetadata from './lib/s3middleware/userMetadata';
|
||||
import convertToXml from './lib/s3middleware/convertToXml';
|
||||
import escapeForXml from './lib/s3middleware/escapeForXml';
|
||||
import * as objectLegalHold from './lib/s3middleware/objectLegalHold';
|
||||
import * as tagging from './lib/s3middleware/tagging';
|
||||
import { checkDateModifiedHeaders } from './lib/s3middleware/validateConditionalHeaders';
|
||||
import { validateConditionalHeaders } from './lib/s3middleware/validateConditionalHeaders';
|
||||
import MD5Sum from './lib/s3middleware/MD5Sum';
|
||||
import NullStream from './lib/s3middleware/nullStream';
|
||||
|
@ -16,11 +18,14 @@ import * as objectUtils from './lib/s3middleware/objectUtils';
|
|||
import * as mpuUtils from './lib/s3middleware/azureHelpers/mpuUtils';
|
||||
import ResultsCollector from './lib/s3middleware/azureHelpers/ResultsCollector';
|
||||
import SubStreamInterface from './lib/s3middleware/azureHelpers/SubStreamInterface';
|
||||
import { prepareStream } from './lib/s3middleware/prepareStream';
|
||||
import * as processMpuParts from './lib/s3middleware/processMpuParts';
|
||||
import * as retention from './lib/s3middleware/objectRetention';
|
||||
import * as objectRestore from './lib/s3middleware/objectRestore';
|
||||
import * as lifecycleHelpers from './lib/s3middleware/lifecycleHelpers';
|
||||
export { default as errors } from './lib/errors';
|
||||
export { default as Clustering } from './lib/Clustering';
|
||||
export * as ClusterRPC from './lib/clustering/ClusterRPC';
|
||||
export * as ipCheck from './lib/ipCheck';
|
||||
export * as auth from './lib/auth/auth';
|
||||
export * as constants from './lib/constants';
|
||||
|
@ -33,21 +38,19 @@ export * as stream from './lib/stream';
|
|||
export * as jsutil from './lib/jsutil';
|
||||
export { default as stringHash } from './lib/stringHash';
|
||||
export * as db from './lib/db';
|
||||
export * as errorUtils from './lib/errorUtils';
|
||||
export { default as shuffle } from './lib/shuffle';
|
||||
export * as models from './lib/models';
|
||||
|
||||
export const algorithms = {
|
||||
list: {
|
||||
Basic: require('./lib/algos/list/basic').List,
|
||||
Delimiter: require('./lib/algos/list/delimiter').Delimiter,
|
||||
DelimiterVersions: require('./lib/algos/list/delimiterVersions').DelimiterVersions,
|
||||
DelimiterMaster: require('./lib/algos/list/delimiterMaster').DelimiterMaster,
|
||||
MPU: require('./lib/algos/list/MPU').MultipartUploads,
|
||||
},
|
||||
list: require('./lib/algos/list/exportAlgos'),
|
||||
listTools: {
|
||||
DelimiterTools: require('./lib/algos/list/tools'),
|
||||
Skip: require('./lib/algos/list/skip'),
|
||||
},
|
||||
cache: {
|
||||
GapSet: require('./lib/algos/cache/GapSet'),
|
||||
GapCache: require('./lib/algos/cache/GapCache'),
|
||||
LRUCache: require('./lib/algos/cache/LRUCache'),
|
||||
},
|
||||
stream: {
|
||||
|
@ -76,6 +79,7 @@ export const s3middleware = {
|
|||
escapeForXml,
|
||||
objectLegalHold,
|
||||
tagging,
|
||||
checkDateModifiedHeaders,
|
||||
validateConditionalHeaders,
|
||||
MD5Sum,
|
||||
NullStream,
|
||||
|
@ -85,8 +89,10 @@ export const s3middleware = {
|
|||
ResultsCollector,
|
||||
SubStreamInterface,
|
||||
},
|
||||
prepareStream,
|
||||
processMpuParts,
|
||||
retention,
|
||||
objectRestore,
|
||||
lifecycleHelpers,
|
||||
};
|
||||
|
||||
|
@ -157,3 +163,7 @@ export const storage = {
|
|||
export const pensieve = {
|
||||
credentialUtils: require('./lib/executables/pensieveCreds/utils'),
|
||||
};
|
||||
|
||||
export const patches = {
|
||||
locationConstraints,
|
||||
};
|
||||
|
|
|
@ -0,0 +1,363 @@
|
|||
import { OrderedSet } from '@js-sdsl/ordered-set';
|
||||
import {
|
||||
default as GapSet,
|
||||
GapSetEntry,
|
||||
} from './GapSet';
|
||||
|
||||
// the API is similar but is not strictly a superset of GapSetInterface
|
||||
// so we don't extend from it
|
||||
export interface GapCacheInterface {
|
||||
exposureDelayMs: number;
|
||||
maxGapWeight: number;
|
||||
size: number;
|
||||
|
||||
setGap: (firstKey: string, lastKey: string, weight: number) => void;
|
||||
removeOverlappingGaps: (overlappingKeys: string[]) => number;
|
||||
lookupGap: (minKey: string, maxKey?: string) => Promise<GapSetEntry | null>;
|
||||
[Symbol.iterator]: () => Iterator<GapSetEntry>;
|
||||
toArray: () => GapSetEntry[];
|
||||
};
|
||||
|
||||
class GapCacheUpdateSet {
|
||||
newGaps: GapSet;
|
||||
updatedKeys: OrderedSet<string>;
|
||||
|
||||
constructor(maxGapWeight: number) {
|
||||
this.newGaps = new GapSet(maxGapWeight);
|
||||
this.updatedKeys = new OrderedSet();
|
||||
}
|
||||
|
||||
addUpdateBatch(updatedKeys: OrderedSet<string>): void {
|
||||
this.updatedKeys.union(updatedKeys);
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Cache of listing "gaps" i.e. ranges of keys that can be skipped
|
||||
* over during listing (because they only contain delete markers as
|
||||
* latest versions).
|
||||
*
|
||||
* Typically, a single GapCache instance would be attached to a raft session.
|
||||
*
|
||||
* The API usage is as follows:
|
||||
*
|
||||
* - Initialize a GapCache instance by calling start() (this starts an internal timer)
|
||||
*
|
||||
* - Insert a gap or update an existing one via setGap()
|
||||
*
|
||||
* - Lookup existing gaps via lookupGap()
|
||||
*
|
||||
* - Invalidate gaps that overlap a specific set of keys via removeOverlappingGaps()
|
||||
*
|
||||
* - Shut down a GapCache instance by calling stop() (this stops the internal timer)
|
||||
*
|
||||
* Gaps inserted via setGap() are not exposed immediately to lookupGap(), but only:
|
||||
*
|
||||
* - after a certain delay always larger than 'exposureDelayMs' and usually shorter
|
||||
* than twice this value (but might be slightly longer in rare cases)
|
||||
*
|
||||
* - and only if they haven't been invalidated by a recent call to removeOverlappingGaps()
|
||||
*
|
||||
* This ensures atomicity between gap creation and invalidation from updates under
|
||||
* the condition that a gap is created from first key to last key within the time defined
|
||||
* by 'exposureDelayMs'.
|
||||
*
|
||||
* The implementation is based on two extra temporary "update sets" on top of the main
|
||||
* exposed gap set, one called "staging" and the other "frozen", each containing a
|
||||
* temporary updated gap set and a list of updated keys to invalidate gaps with (coming
|
||||
* from calls to removeOverlappingGaps()). Every "exposureDelayMs" milliseconds, the frozen
|
||||
* gaps are invalidated by all key updates coming from either of the "staging" or "frozen"
|
||||
* update set, then merged into the exposed gaps set, after which the staging updates become
|
||||
* the frozen updates and won't receive any new gap until the next cycle.
|
||||
*/
|
||||
export default class GapCache implements GapCacheInterface {
|
||||
_exposureDelayMs: number;
|
||||
maxGaps: number;
|
||||
|
||||
_stagingUpdates: GapCacheUpdateSet;
|
||||
_frozenUpdates: GapCacheUpdateSet;
|
||||
_exposedGaps: GapSet;
|
||||
_exposeFrozenInterval: NodeJS.Timeout | null;
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
*
|
||||
* @param {number} exposureDelayMs - minimum delay between
|
||||
* insertion of a gap via setGap() and its exposure via
|
||||
* lookupGap()
|
||||
* @param {number} maxGaps - maximum number of cached gaps, after
|
||||
* which no new gap can be added by setGap(). (Note: a future
|
||||
* improvement could replace this by an eviction strategy)
|
||||
* @param {number} maxGapWeight - maximum "weight" of individual
|
||||
* cached gaps, which is also the granularity for
|
||||
* invalidation. Individual gaps can be chained together,
|
||||
* which lookupGap() transparently consolidates in the response
|
||||
* into a single large gap.
|
||||
*/
|
||||
constructor(exposureDelayMs: number, maxGaps: number, maxGapWeight: number) {
|
||||
this._exposureDelayMs = exposureDelayMs;
|
||||
this.maxGaps = maxGaps;
|
||||
|
||||
this._stagingUpdates = new GapCacheUpdateSet(maxGapWeight);
|
||||
this._frozenUpdates = new GapCacheUpdateSet(maxGapWeight);
|
||||
this._exposedGaps = new GapSet(maxGapWeight);
|
||||
this._exposeFrozenInterval = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a GapCache from an array of exposed gap entries (used in tests)
|
||||
*
|
||||
* @return {GapCache} - a new GapCache instance
|
||||
*/
|
||||
static createFromArray(
|
||||
gaps: GapSetEntry[],
|
||||
exposureDelayMs: number,
|
||||
maxGaps: number,
|
||||
maxGapWeight: number
|
||||
): GapCache {
|
||||
const gapCache = new GapCache(exposureDelayMs, maxGaps, maxGapWeight);
|
||||
gapCache._exposedGaps = GapSet.createFromArray(gaps, maxGapWeight)
|
||||
return gapCache;
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal helper to remove gaps in the staging and frozen sets
|
||||
* overlapping with previously updated keys, right before the
|
||||
* frozen gaps get exposed.
|
||||
*
|
||||
* @return {undefined}
|
||||
*/
|
||||
_removeOverlappingGapsBeforeExpose(): void {
|
||||
for (const { updatedKeys } of [this._stagingUpdates, this._frozenUpdates]) {
|
||||
if (updatedKeys.size() === 0) {
|
||||
continue;
|
||||
}
|
||||
for (const { newGaps } of [this._stagingUpdates, this._frozenUpdates]) {
|
||||
if (newGaps.size === 0) {
|
||||
continue;
|
||||
}
|
||||
newGaps.removeOverlappingGaps(updatedKeys);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This function is the core mechanism that updates the exposed gaps in the
|
||||
* cache. It is called on a regular interval defined by 'exposureDelayMs'.
|
||||
*
|
||||
* It does the following in order:
|
||||
*
|
||||
* - remove gaps from the frozen set that overlap with any key present in a
|
||||
* batch passed to removeOverlappingGaps() since the last two triggers of
|
||||
* _exposeFrozen()
|
||||
*
|
||||
* - merge the remaining gaps from the frozen set to the exposed set, which
|
||||
* makes them visible from calls to lookupGap()
|
||||
*
|
||||
* - rotate by freezing the currently staging updates and initiating a new
|
||||
* staging updates set
|
||||
*
|
||||
* @return {undefined}
|
||||
*/
|
||||
_exposeFrozen(): void {
|
||||
this._removeOverlappingGapsBeforeExpose();
|
||||
for (const gap of this._frozenUpdates.newGaps) {
|
||||
// Use a trivial strategy to keep the cache size within
|
||||
// limits: refuse to add new gaps when the size is above
|
||||
// the 'maxGaps' threshold. We solely rely on
|
||||
// removeOverlappingGaps() to make space for new gaps.
|
||||
if (this._exposedGaps.size < this.maxGaps) {
|
||||
this._exposedGaps.setGap(gap.firstKey, gap.lastKey, gap.weight);
|
||||
}
|
||||
}
|
||||
this._frozenUpdates = this._stagingUpdates;
|
||||
this._stagingUpdates = new GapCacheUpdateSet(this.maxGapWeight);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the internal GapCache timer
|
||||
*
|
||||
* @return {undefined}
|
||||
*/
|
||||
start(): void {
|
||||
if (this._exposeFrozenInterval) {
|
||||
return;
|
||||
}
|
||||
this._exposeFrozenInterval = setInterval(
|
||||
() => this._exposeFrozen(),
|
||||
this._exposureDelayMs);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the internal GapCache timer
|
||||
*
|
||||
* @return {undefined}
|
||||
*/
|
||||
stop(): void {
|
||||
if (this._exposeFrozenInterval) {
|
||||
clearInterval(this._exposeFrozenInterval);
|
||||
this._exposeFrozenInterval = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a gap between two keys, associated with a weight to
|
||||
* limit individual gap's spanning ranges in the cache, for a more
|
||||
* granular invalidation.
|
||||
*
|
||||
* The function handles splitting and merging existing gaps to
|
||||
* maintain an optimal weight of cache entries.
|
||||
*
|
||||
* NOTE 1: the caller must ensure that the full length of the gap
|
||||
* between 'firstKey' and 'lastKey' has been built from a listing
|
||||
* snapshot that is more recent than 'exposureDelayMs' milliseconds,
|
||||
* in order to guarantee that the exposed gap will be fully
|
||||
* covered (and potentially invalidated) from recent calls to
|
||||
* removeOverlappingGaps().
|
||||
*
|
||||
* NOTE 2: a usual pattern when building a large gap from multiple
|
||||
* calls to setGap() is to start the next gap from 'lastKey',
|
||||
* which will be passed as 'firstKey' in the next call, so that
|
||||
* gaps can be chained together and consolidated by lookupGap().
|
||||
*
|
||||
* @param {string} firstKey - first key of the gap
|
||||
* @param {string} lastKey - last key of the gap, must be greater
|
||||
* or equal than 'firstKey'
|
||||
* @param {number} weight - total weight between 'firstKey' and 'lastKey'
|
||||
* @return {undefined}
|
||||
*/
|
||||
setGap(firstKey: string, lastKey: string, weight: number): void {
|
||||
this._stagingUpdates.newGaps.setGap(firstKey, lastKey, weight);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove gaps that overlap with a given set of keys. Used to
|
||||
* invalidate gaps when keys are inserted or deleted.
|
||||
*
|
||||
* @param {OrderedSet<string> | string[]} overlappingKeys - remove gaps that
|
||||
* overlap with any of this set of keys
|
||||
* @return {number} - how many gaps were removed from the exposed
|
||||
* gaps only (overlapping gaps not yet exposed are also invalidated
|
||||
* but are not accounted for in the returned value)
|
||||
*/
|
||||
removeOverlappingGaps(overlappingKeys: OrderedSet<string> | string[]): number {
|
||||
let overlappingKeysSet;
|
||||
if (Array.isArray(overlappingKeys)) {
|
||||
overlappingKeysSet = new OrderedSet(overlappingKeys);
|
||||
} else {
|
||||
overlappingKeysSet = overlappingKeys;
|
||||
}
|
||||
this._stagingUpdates.addUpdateBatch(overlappingKeysSet);
|
||||
return this._exposedGaps.removeOverlappingGaps(overlappingKeysSet);
|
||||
}
|
||||
|
||||
/**
|
||||
* Lookup the next exposed gap that overlaps with [minKey, maxKey]. Internally
|
||||
* chained gaps are coalesced in the response into a single contiguous large gap.
|
||||
*
|
||||
* @param {string} minKey - minimum key overlapping with the returned gap
|
||||
* @param {string} [maxKey] - maximum key overlapping with the returned gap
|
||||
* @return {Promise<GapSetEntry | null>} - result of the lookup if a gap
|
||||
* was found, null otherwise, as a Promise
|
||||
*/
|
||||
lookupGap(minKey: string, maxKey?: string): Promise<GapSetEntry | null> {
|
||||
return this._exposedGaps.lookupGap(minKey, maxKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the maximum weight setting for individual gaps.
|
||||
*
|
||||
* @return {number} - maximum weight of individual gaps
|
||||
*/
|
||||
get maxGapWeight(): number {
|
||||
return this._exposedGaps.maxWeight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the maximum weight setting for individual gaps.
|
||||
*
|
||||
* @param {number} gapWeight - maximum weight of individual gaps
|
||||
*/
|
||||
set maxGapWeight(gapWeight: number) {
|
||||
this._exposedGaps.maxWeight = gapWeight;
|
||||
// also update transient gap sets
|
||||
this._stagingUpdates.newGaps.maxWeight = gapWeight;
|
||||
this._frozenUpdates.newGaps.maxWeight = gapWeight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the exposure delay in milliseconds, which is the minimum
|
||||
* time after which newly cached gaps will be exposed by
|
||||
* lookupGap().
|
||||
*
|
||||
* @return {number} - exposure delay in milliseconds
|
||||
*/
|
||||
get exposureDelayMs(): number {
|
||||
return this._exposureDelayMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the exposure delay in milliseconds, which is the minimum
|
||||
* time after which newly cached gaps will be exposed by
|
||||
* lookupGap(). Setting this attribute automatically updates the
|
||||
* internal state to honor the new value.
|
||||
*
|
||||
* @param {number} - exposure delay in milliseconds
|
||||
*/
|
||||
set exposureDelayMs(exposureDelayMs: number) {
|
||||
if (exposureDelayMs !== this._exposureDelayMs) {
|
||||
this._exposureDelayMs = exposureDelayMs;
|
||||
if (this._exposeFrozenInterval) {
|
||||
// invalidate all pending gap updates, as the new interval may not be
|
||||
// safe for them
|
||||
this._stagingUpdates = new GapCacheUpdateSet(this.maxGapWeight);
|
||||
this._frozenUpdates = new GapCacheUpdateSet(this.maxGapWeight);
|
||||
|
||||
// reinitialize the _exposeFrozenInterval timer with the updated delay
|
||||
this.stop();
|
||||
this.start();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of exposed gaps
|
||||
*
|
||||
* @return {number} number of exposed gaps
|
||||
*/
|
||||
get size(): number {
|
||||
return this._exposedGaps.size;
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate over exposed gaps
|
||||
*
|
||||
* @return {Iterator<GapSetEntry>} an iterator over exposed gaps
|
||||
*/
|
||||
[Symbol.iterator](): Iterator<GapSetEntry> {
|
||||
return this._exposedGaps[Symbol.iterator]();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an array of all exposed gaps
|
||||
*
|
||||
* @return {GapSetEntry[]} array of exposed gaps
|
||||
*/
|
||||
toArray(): GapSetEntry[] {
|
||||
return this._exposedGaps.toArray();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all exposed and staging gaps from the cache.
|
||||
*
|
||||
* Note: retains invalidating updates from removeOverlappingGaps()
|
||||
* for correctness of gaps inserted afterwards.
|
||||
*
|
||||
* @return {undefined}
|
||||
*/
|
||||
clear(): void {
|
||||
this._stagingUpdates.newGaps = new GapSet(this.maxGapWeight);
|
||||
this._frozenUpdates.newGaps = new GapSet(this.maxGapWeight);
|
||||
this._exposedGaps = new GapSet(this.maxGapWeight);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,366 @@
|
|||
import assert from 'assert';
|
||||
import { OrderedSet } from '@js-sdsl/ordered-set';
|
||||
|
||||
import errors from '../../errors';
|
||||
|
||||
export type GapSetEntry = {
|
||||
firstKey: string,
|
||||
lastKey: string,
|
||||
weight: number,
|
||||
};
|
||||
|
||||
export interface GapSetInterface {
|
||||
maxWeight: number;
|
||||
size: number;
|
||||
|
||||
setGap: (firstKey: string, lastKey: string, weight: number) => GapSetEntry;
|
||||
removeOverlappingGaps: (overlappingKeys: string[]) => number;
|
||||
lookupGap: (minKey: string, maxKey?: string) => Promise<GapSetEntry | null>;
|
||||
[Symbol.iterator]: () => Iterator<GapSetEntry>;
|
||||
toArray: () => GapSetEntry[];
|
||||
};
|
||||
|
||||
/**
|
||||
* Specialized data structure to support caching of listing "gaps",
|
||||
* i.e. ranges of keys that can be skipped over during listing
|
||||
* (because they only contain delete markers as latest versions)
|
||||
*/
|
||||
export default class GapSet implements GapSetInterface, Iterable<GapSetEntry> {
|
||||
_gaps: OrderedSet<GapSetEntry>;
|
||||
_maxWeight: number;
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
|
||||
* @param {number} maxWeight - weight threshold for each cached
|
||||
* gap (unitless). Triggers splitting gaps when reached
|
||||
*/
|
||||
constructor(maxWeight: number) {
|
||||
this._gaps = new OrderedSet(
|
||||
[],
|
||||
(left: GapSetEntry, right: GapSetEntry) => (
|
||||
left.firstKey < right.firstKey ? -1 :
|
||||
left.firstKey > right.firstKey ? 1 : 0
|
||||
)
|
||||
);
|
||||
this._maxWeight = maxWeight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a GapSet from an array of gap entries (used in tests)
|
||||
*/
|
||||
static createFromArray(gaps: GapSetEntry[], maxWeight: number): GapSet {
|
||||
const gapSet = new GapSet(maxWeight);
|
||||
for (const gap of gaps) {
|
||||
gapSet._gaps.insert(gap);
|
||||
}
|
||||
return gapSet;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a gap between two keys, associated with a weight to limit
|
||||
* individual gap sizes in the cache.
|
||||
*
|
||||
* The function handles splitting and merging existing gaps to
|
||||
* maintain an optimal weight of cache entries.
|
||||
*
|
||||
* @param {string} firstKey - first key of the gap
|
||||
* @param {string} lastKey - last key of the gap, must be greater
|
||||
* or equal than 'firstKey'
|
||||
* @param {number} weight - total weight between 'firstKey' and 'lastKey'
|
||||
* @return {GapSetEntry} - existing or new gap entry
|
||||
*/
|
||||
setGap(firstKey: string, lastKey: string, weight: number): GapSetEntry {
|
||||
assert(lastKey >= firstKey);
|
||||
|
||||
// Step 1/4: Find the closest left-overlapping gap, and either re-use it
|
||||
// or chain it with a new gap depending on the weights if it exists (otherwise
|
||||
// just creates a new gap).
|
||||
const curGapIt = this._gaps.reverseLowerBound(<GapSetEntry>{ firstKey });
|
||||
let curGap;
|
||||
if (curGapIt.isAccessible()) {
|
||||
curGap = curGapIt.pointer;
|
||||
if (curGap.lastKey >= lastKey) {
|
||||
// return fully overlapping gap already cached
|
||||
return curGap;
|
||||
}
|
||||
}
|
||||
let remainingWeight = weight;
|
||||
if (!curGap // no previous gap
|
||||
|| curGap.lastKey < firstKey // previous gap not overlapping
|
||||
|| (curGap.lastKey === firstKey // previous gap overlapping by one key...
|
||||
&& curGap.weight + weight > this._maxWeight) // ...but we can't extend it
|
||||
) {
|
||||
// create a new gap indexed by 'firstKey'
|
||||
curGap = { firstKey, lastKey: firstKey, weight: 0 };
|
||||
this._gaps.insert(curGap);
|
||||
} else if (curGap.lastKey > firstKey && weight > this._maxWeight) {
|
||||
// previous gap is either fully or partially contained in the new gap
|
||||
// and cannot be extended: substract its weight from the total (heuristic
|
||||
// in case the previous gap doesn't start at 'firstKey', which is the
|
||||
// uncommon case)
|
||||
remainingWeight -= curGap.weight;
|
||||
|
||||
// there may be an existing chained gap starting with the previous gap's
|
||||
// 'lastKey': use it if it exists
|
||||
const chainedGapIt = this._gaps.find(<GapSetEntry>{ firstKey: curGap.lastKey });
|
||||
if (chainedGapIt.isAccessible()) {
|
||||
curGap = chainedGapIt.pointer;
|
||||
} else {
|
||||
// no existing chained gap: chain a new gap to the previous gap
|
||||
curGap = {
|
||||
firstKey: curGap.lastKey,
|
||||
lastKey: curGap.lastKey,
|
||||
weight: 0,
|
||||
};
|
||||
this._gaps.insert(curGap);
|
||||
}
|
||||
}
|
||||
// Step 2/4: Cleanup existing gaps fully included in firstKey -> lastKey, and
|
||||
// aggregate their weights in curGap to define the minimum weight up to the
|
||||
// last merged gap.
|
||||
let nextGap;
|
||||
while (true) {
|
||||
const nextGapIt = this._gaps.upperBound(<GapSetEntry>{ firstKey: curGap.firstKey });
|
||||
nextGap = nextGapIt.isAccessible() && nextGapIt.pointer;
|
||||
// stop the cleanup when no more gap or if the next gap is not fully
|
||||
// included in curGap
|
||||
if (!nextGap || nextGap.lastKey > lastKey) {
|
||||
break;
|
||||
}
|
||||
this._gaps.eraseElementByIterator(nextGapIt);
|
||||
curGap.lastKey = nextGap.lastKey;
|
||||
curGap.weight += nextGap.weight;
|
||||
}
|
||||
|
||||
// Step 3/4: Extend curGap to lastKey, adjusting the weight.
|
||||
// At this point, curGap weight is the minimum weight of the finished gap, save it
|
||||
// for step 4.
|
||||
let minMergedWeight = curGap.weight;
|
||||
if (curGap.lastKey === firstKey && firstKey !== lastKey) {
|
||||
// extend the existing gap by the full amount 'firstKey -> lastKey'
|
||||
curGap.lastKey = lastKey;
|
||||
curGap.weight += remainingWeight;
|
||||
} else if (curGap.lastKey <= lastKey) {
|
||||
curGap.lastKey = lastKey;
|
||||
curGap.weight = remainingWeight;
|
||||
}
|
||||
|
||||
// Step 4/4: Find the closest right-overlapping gap, and if it exists, either merge
|
||||
// it or chain it with curGap depending on the weights.
|
||||
if (nextGap && nextGap.firstKey <= lastKey) {
|
||||
// nextGap overlaps with the new gap: check if we can merge it
|
||||
minMergedWeight += nextGap.weight;
|
||||
let mergedWeight;
|
||||
if (lastKey === nextGap.firstKey) {
|
||||
// nextGap is chained with curGap: add the full weight of nextGap
|
||||
mergedWeight = curGap.weight + nextGap.weight;
|
||||
} else {
|
||||
// strict overlap: don't add nextGap's weight unless
|
||||
// it's larger than the sum of merged ranges (as it is
|
||||
// then included in `minMergedWeight`)
|
||||
mergedWeight = Math.max(curGap.weight, minMergedWeight);
|
||||
}
|
||||
if (mergedWeight <= this._maxWeight) {
|
||||
// merge nextGap into curGap
|
||||
curGap.lastKey = nextGap.lastKey;
|
||||
curGap.weight = mergedWeight;
|
||||
this._gaps.eraseElementByKey(nextGap);
|
||||
} else {
|
||||
// adjust the last key to chain with nextGap and substract the next
|
||||
// gap's weight from curGap (heuristic)
|
||||
curGap.lastKey = nextGap.firstKey;
|
||||
curGap.weight = Math.max(mergedWeight - nextGap.weight, 0);
|
||||
curGap = nextGap;
|
||||
}
|
||||
}
|
||||
// return a copy of curGap
|
||||
return Object.assign({}, curGap);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove gaps that overlap with one or more keys in a given array or
|
||||
* OrderedSet. Used to invalidate gaps when keys are inserted or deleted.
|
||||
*
|
||||
* @param {OrderedSet<string> | string[]} overlappingKeys - remove gaps that overlap
|
||||
* with any of this set of keys
|
||||
* @return {number} - how many gaps were removed
|
||||
*/
|
||||
removeOverlappingGaps(overlappingKeys: OrderedSet<string> | string[]): number {
|
||||
// To optimize processing with a large number of keys and/or gaps, this function:
|
||||
//
|
||||
// 1. converts the overlappingKeys array to a OrderedSet (if not already a OrderedSet)
|
||||
// 2. queries both the gaps set and the overlapping keys set in a loop, which allows:
|
||||
// - skipping ranges of overlapping keys at once when there is no new overlapping gap
|
||||
// - skipping ranges of gaps at once when there is no overlapping key
|
||||
//
|
||||
// This way, it is efficient when the number of non-overlapping gaps is large
|
||||
// (which is the most common case in practice).
|
||||
|
||||
let overlappingKeysSet;
|
||||
if (Array.isArray(overlappingKeys)) {
|
||||
overlappingKeysSet = new OrderedSet(overlappingKeys);
|
||||
} else {
|
||||
overlappingKeysSet = overlappingKeys;
|
||||
}
|
||||
const firstKeyIt = overlappingKeysSet.begin();
|
||||
let currentKey = firstKeyIt.isAccessible() && firstKeyIt.pointer;
|
||||
let nRemoved = 0;
|
||||
while (currentKey) {
|
||||
const closestGapIt = this._gaps.reverseUpperBound(<GapSetEntry>{ firstKey: currentKey });
|
||||
if (closestGapIt.isAccessible()) {
|
||||
const closestGap = closestGapIt.pointer;
|
||||
if (currentKey <= closestGap.lastKey) {
|
||||
// currentKey overlaps closestGap: remove the gap
|
||||
this._gaps.eraseElementByIterator(closestGapIt);
|
||||
nRemoved += 1;
|
||||
}
|
||||
}
|
||||
const nextGapIt = this._gaps.lowerBound(<GapSetEntry>{ firstKey: currentKey });
|
||||
if (!nextGapIt.isAccessible()) {
|
||||
// no more gap: we're done
|
||||
return nRemoved;
|
||||
}
|
||||
const nextGap = nextGapIt.pointer;
|
||||
// advance to the last key potentially overlapping with nextGap
|
||||
let currentKeyIt = overlappingKeysSet.reverseLowerBound(nextGap.lastKey);
|
||||
if (currentKeyIt.isAccessible()) {
|
||||
currentKey = currentKeyIt.pointer;
|
||||
if (currentKey >= nextGap.firstKey) {
|
||||
// currentKey overlaps nextGap: remove the gap
|
||||
this._gaps.eraseElementByIterator(nextGapIt);
|
||||
nRemoved += 1;
|
||||
}
|
||||
}
|
||||
// advance to the first key potentially overlapping with another gap
|
||||
currentKeyIt = overlappingKeysSet.lowerBound(nextGap.lastKey);
|
||||
currentKey = currentKeyIt.isAccessible() && currentKeyIt.pointer;
|
||||
}
|
||||
return nRemoved;
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal helper to coalesce multiple chained gaps into a single gap.
|
||||
*
|
||||
* It is only used to construct lookupGap() return values and
|
||||
* doesn't modify the GapSet.
|
||||
*
|
||||
* NOTE: The function may take a noticeable amount of time and CPU
|
||||
* to execute if a large number of chained gaps have to be
|
||||
* coalesced, but it should never take more than a few seconds. In
|
||||
* most cases it should take less than a millisecond. It regularly
|
||||
* yields to the nodejs event loop to avoid blocking it during a
|
||||
* long execution.
|
||||
*
|
||||
* @param {GapSetEntry} firstGap - first gap of the chain to coalesce with
|
||||
* the next ones in the chain
|
||||
* @return {Promise<GapSetEntry>} - a new coalesced entry, as a Promise
|
||||
*/
|
||||
_coalesceGapChain(firstGap: GapSetEntry): Promise<GapSetEntry> {
|
||||
return new Promise(resolve => {
|
||||
const coalescedGap: GapSetEntry = Object.assign({}, firstGap);
|
||||
const coalesceGapChainIteration = () => {
|
||||
// efficiency trade-off: 100 iterations of log(N) complexity lookups should
|
||||
// not block the event loop for too long
|
||||
for (let opCounter = 0; opCounter < 100; ++opCounter) {
|
||||
const chainedGapIt = this._gaps.find(
|
||||
<GapSetEntry>{ firstKey: coalescedGap.lastKey });
|
||||
if (!chainedGapIt.isAccessible()) {
|
||||
// chain is complete
|
||||
return resolve(coalescedGap);
|
||||
}
|
||||
const chainedGap = chainedGapIt.pointer;
|
||||
if (chainedGap.firstKey === chainedGap.lastKey) {
|
||||
// found a single-key gap: chain is complete
|
||||
return resolve(coalescedGap);
|
||||
}
|
||||
coalescedGap.lastKey = chainedGap.lastKey;
|
||||
coalescedGap.weight += chainedGap.weight;
|
||||
}
|
||||
// yield to the event loop before continuing the process
|
||||
// of coalescing the gap chain
|
||||
return process.nextTick(coalesceGapChainIteration);
|
||||
};
|
||||
coalesceGapChainIteration();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Lookup the next gap that overlaps with [minKey, maxKey]. Internally chained
|
||||
* gaps are coalesced in the response into a single contiguous large gap.
|
||||
*
|
||||
* @param {string} minKey - minimum key overlapping with the returned gap
|
||||
* @param {string} [maxKey] - maximum key overlapping with the returned gap
|
||||
* @return {Promise<GapSetEntry | null>} - result of the lookup if a gap
|
||||
* was found, null otherwise, as a Promise
|
||||
*/
|
||||
async lookupGap(minKey: string, maxKey?: string): Promise<GapSetEntry | null> {
|
||||
let firstGap: GapSetEntry | null = null;
|
||||
const minGapIt = this._gaps.reverseLowerBound(<GapSetEntry>{ firstKey: minKey });
|
||||
const minGap = minGapIt.isAccessible() && minGapIt.pointer;
|
||||
if (minGap && minGap.lastKey >= minKey) {
|
||||
firstGap = minGap;
|
||||
} else {
|
||||
const maxGapIt = this._gaps.upperBound(<GapSetEntry>{ firstKey: minKey });
|
||||
const maxGap = maxGapIt.isAccessible() && maxGapIt.pointer;
|
||||
if (maxGap && (maxKey === undefined || maxGap.firstKey <= maxKey)) {
|
||||
firstGap = maxGap;
|
||||
}
|
||||
}
|
||||
if (!firstGap) {
|
||||
return null;
|
||||
}
|
||||
return this._coalesceGapChain(firstGap);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the maximum weight setting for individual gaps.
|
||||
*
|
||||
* @return {number} - maximum weight of individual gaps
|
||||
*/
|
||||
get maxWeight(): number {
|
||||
return this._maxWeight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the maximum weight setting for individual gaps.
|
||||
*
|
||||
* @param {number} gapWeight - maximum weight of individual gaps
|
||||
*/
|
||||
set maxWeight(gapWeight: number) {
|
||||
this._maxWeight = gapWeight;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of gaps stored in this set.
|
||||
*
|
||||
* @return {number} - number of gaps stored in this set
|
||||
*/
|
||||
get size(): number {
|
||||
return this._gaps.size();
|
||||
}
|
||||
|
||||
/**
|
||||
* Iterate over each gap of the set, ordered by first key
|
||||
*
|
||||
* @return {Iterator<GapSetEntry>} - an iterator over all gaps
|
||||
* Example:
|
||||
* for (const gap of myGapSet) { ... }
|
||||
*/
|
||||
[Symbol.iterator](): Iterator<GapSetEntry> {
|
||||
return this._gaps[Symbol.iterator]();
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an array containing all gaps, ordered by first key
|
||||
*
|
||||
* NOTE: there is a toArray() method in the OrderedSet implementation
|
||||
* but it does not scale well and overflows the stack quickly. This is
|
||||
* why we provide an implementation based on an iterator.
|
||||
*
|
||||
* @return {GapSetEntry[]} - an array containing all gaps
|
||||
*/
|
||||
toArray(): GapSetEntry[] {
|
||||
return [...this];
|
||||
}
|
||||
}
|
|
@ -1,6 +1,6 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const { FILTER_SKIP, SKIP_NONE } = require('./tools');
|
||||
const { FILTER_ACCEPT, SKIP_NONE } = require('./tools');
|
||||
|
||||
// Use a heuristic to amortize the cost of JSON
|
||||
// serialization/deserialization only on largest metadata where the
|
||||
|
@ -92,21 +92,26 @@ class Extension {
|
|||
* @param {object} entry - a listing entry from metadata
|
||||
* expected format: { key, value }
|
||||
* @return {number} - result of filtering the entry:
|
||||
* > 0: entry is accepted and included in the result
|
||||
* = 0: entry is accepted but not included (skipping)
|
||||
* < 0: entry is not accepted, listing should finish
|
||||
* FILTER_ACCEPT: entry is accepted and may or not be included
|
||||
* in the result
|
||||
* FILTER_SKIP: listing may skip directly (with "gte" param) to
|
||||
* the key returned by the skipping() method
|
||||
* FILTER_END: the results are complete, listing can be stopped
|
||||
*/
|
||||
filter(entry) {
|
||||
return entry ? FILTER_SKIP : FILTER_SKIP;
|
||||
filter(/* entry: { key, value } */) {
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides the insight into why filter is skipping an entry. This could be
|
||||
* because it is skipping a range of delimited keys or a range of specific
|
||||
* version when doing master version listing.
|
||||
* Provides the next key at which the listing task is allowed to skip to.
|
||||
* This could allow to skip over:
|
||||
* - a key prefix ending with the delimiter
|
||||
* - all remaining versions of an object when doing a current
|
||||
* versions listing in v0 format
|
||||
* - a cached "gap" of deleted objects when doing a current
|
||||
* versions listing in v0 format
|
||||
*
|
||||
* @return {string} - the insight: a common prefix or a master key,
|
||||
* or SKIP_NONE if there is no insight
|
||||
* @return {string} - the next key at which the listing task is allowed to skip to
|
||||
*/
|
||||
skipping() {
|
||||
return SKIP_NONE;
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const { inc, checkLimit, listingParamsMasterKeysV0ToV1,
|
||||
FILTER_END, FILTER_ACCEPT } = require('./tools');
|
||||
FILTER_END, FILTER_ACCEPT, SKIP_NONE } = require('./tools');
|
||||
const DEFAULT_MAX_KEYS = 1000;
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
|
||||
|
@ -163,7 +163,7 @@ class MultipartUploads {
|
|||
}
|
||||
|
||||
skipping() {
|
||||
return '';
|
||||
return SKIP_NONE;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
const Extension = require('./Extension').default;
|
||||
|
||||
const { checkLimit, FILTER_END, FILTER_ACCEPT, FILTER_SKIP } = require('./tools');
|
||||
const { checkLimit, FILTER_END, FILTER_ACCEPT } = require('./tools');
|
||||
const DEFAULT_MAX_KEYS = 10000;
|
||||
|
||||
/**
|
||||
|
@ -91,7 +91,7 @@ class List extends Extension {
|
|||
* < 0 : listing done
|
||||
*/
|
||||
filter(elem) {
|
||||
// Check first in case of maxkeys <= 0
|
||||
// Check if the result array is full
|
||||
if (this.keys >= this.maxKeys) {
|
||||
return FILTER_END;
|
||||
}
|
||||
|
@ -99,7 +99,7 @@ class List extends Extension {
|
|||
this.filterKeyStartsWith !== undefined) &&
|
||||
typeof elem === 'object' &&
|
||||
!this.customFilter(elem.value)) {
|
||||
return FILTER_SKIP;
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
if (typeof elem === 'object') {
|
||||
this.res.push({
|
||||
|
|
|
@ -32,7 +32,7 @@ export interface DelimiterFilterState_SkippingPrefix extends FilterState {
|
|||
|
||||
type KeyHandler = (key: string, value: string) => FilterReturnValue;
|
||||
|
||||
type ResultObject = {
|
||||
export type ResultObject = {
|
||||
CommonPrefixes: string[];
|
||||
Contents: {
|
||||
key: string;
|
||||
|
@ -305,7 +305,7 @@ export class Delimiter extends Extension {
|
|||
switch (this.state.id) {
|
||||
case DelimiterFilterStateId.SkippingPrefix:
|
||||
const { prefix } = <DelimiterFilterState_SkippingPrefix> this.state;
|
||||
return prefix;
|
||||
return inc(prefix);
|
||||
|
||||
default:
|
||||
return SKIP_NONE;
|
||||
|
|
|
@ -0,0 +1,127 @@
|
|||
const { DelimiterMaster } = require('./delimiterMaster');
|
||||
const { FILTER_ACCEPT, FILTER_END } = require('./tools');
|
||||
|
||||
type ResultObject = {
|
||||
Contents: {
|
||||
key: string;
|
||||
value: string;
|
||||
}[];
|
||||
IsTruncated: boolean;
|
||||
NextMarker ?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters. This extends the base class DelimiterMaster
|
||||
* to return the master/current versions.
|
||||
*/
|
||||
class DelimiterCurrent extends DelimiterMaster {
|
||||
/**
|
||||
* Delimiter listing of current versions.
|
||||
* @param {Object} parameters - listing parameters
|
||||
* @param {String} parameters.beforeDate - limit the response to keys older than beforeDate
|
||||
* @param {String} parameters.excludedDataStoreName - excluded datatore name
|
||||
* @param {Number} parameters.maxScannedLifecycleListingEntries - max number of entries to be scanned
|
||||
* @param {RequestLogger} logger - The logger of the request
|
||||
* @param {String} [vFormat] - versioning key format
|
||||
*/
|
||||
constructor(parameters, logger, vFormat) {
|
||||
super(parameters, logger, vFormat);
|
||||
|
||||
this.beforeDate = parameters.beforeDate;
|
||||
this.excludedDataStoreName = parameters.excludedDataStoreName;
|
||||
this.maxScannedLifecycleListingEntries = parameters.maxScannedLifecycleListingEntries;
|
||||
this.scannedKeys = 0;
|
||||
}
|
||||
|
||||
genMDParamsV0() {
|
||||
const params = super.genMDParamsV0();
|
||||
// lastModified and dataStoreName parameters are used by metadata that enables built-in filtering,
|
||||
// a feature currently exclusive to MongoDB
|
||||
if (this.beforeDate) {
|
||||
params.lastModified = {
|
||||
lt: this.beforeDate,
|
||||
};
|
||||
}
|
||||
|
||||
if (this.excludedDataStoreName) {
|
||||
params.dataStoreName = {
|
||||
ne: this.excludedDataStoreName,
|
||||
}
|
||||
}
|
||||
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the stringified entry's value.
|
||||
* @param s - sringified value
|
||||
* @return - undefined if parsing fails, otherwise it contains the parsed value.
|
||||
*/
|
||||
_parse(s) {
|
||||
let p;
|
||||
try {
|
||||
p = JSON.parse(s);
|
||||
} catch (e: any) {
|
||||
this.logger.warn(
|
||||
'Could not parse Object Metadata while listing',
|
||||
{ err: e.toString() });
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
/**
|
||||
* check if the max keys count has been reached and set the
|
||||
* final state of the result if it is the case
|
||||
*
|
||||
* specialized implementation on DelimiterCurrent to also check
|
||||
* the number of scanned keys
|
||||
*
|
||||
* @return {Boolean} - indicates if the iteration has to stop
|
||||
*/
|
||||
_reachedMaxKeys(): boolean {
|
||||
if (this.maxScannedLifecycleListingEntries && this.scannedKeys >= this.maxScannedLifecycleListingEntries) {
|
||||
this.IsTruncated = true;
|
||||
this.logger.info('listing stopped due to reaching the maximum scanned entries limit',
|
||||
{
|
||||
maxScannedLifecycleListingEntries: this.maxScannedLifecycleListingEntries,
|
||||
scannedKeys: this.scannedKeys,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
return super._reachedMaxKeys();
|
||||
}
|
||||
|
||||
addContents(key, value) {
|
||||
++this.scannedKeys;
|
||||
const parsedValue = this._parse(value);
|
||||
// if parsing fails, skip the key.
|
||||
if (parsedValue) {
|
||||
const lastModified = parsedValue['last-modified'];
|
||||
const dataStoreName = parsedValue.dataStoreName;
|
||||
// We then check if the current version is older than the "beforeDate" and
|
||||
// "excludedDataStoreName" is not specified or if specified and the data store name is different.
|
||||
if ((!this.beforeDate || (lastModified && lastModified < this.beforeDate)) &&
|
||||
(!this.excludedDataStoreName || dataStoreName !== this.excludedDataStoreName)) {
|
||||
super.addContents(key, value);
|
||||
}
|
||||
// In the event of a timeout occurring before any content is added,
|
||||
// NextMarker is updated even if the object is not eligible.
|
||||
// It minimizes the amount of data that the client needs to re-process if the request times out.
|
||||
this.nextMarker = key;
|
||||
}
|
||||
}
|
||||
|
||||
result(): object {
|
||||
const result: ResultObject = {
|
||||
Contents: this.Contents,
|
||||
IsTruncated: this.IsTruncated,
|
||||
};
|
||||
|
||||
if (this.IsTruncated) {
|
||||
result.NextMarker = this.nextMarker;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
module.exports = { DelimiterCurrent };
|
|
@ -5,18 +5,23 @@ import {
|
|||
DelimiterFilterStateId,
|
||||
DelimiterFilterState_NotSkipping,
|
||||
DelimiterFilterState_SkippingPrefix,
|
||||
ResultObject,
|
||||
} from './delimiter';
|
||||
const Version = require('../../versioning/Version').Version;
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { BucketVersioningKeyFormat } = VSConst;
|
||||
const { FILTER_ACCEPT, FILTER_SKIP, FILTER_END } = require('./tools');
|
||||
const { FILTER_ACCEPT, FILTER_SKIP, FILTER_END, SKIP_NONE, inc } = require('./tools');
|
||||
|
||||
import { GapSetEntry } from '../cache/GapSet';
|
||||
import { GapCacheInterface } from '../cache/GapCache';
|
||||
|
||||
const VID_SEP = VSConst.VersionId.Separator;
|
||||
const { DbPrefixes } = VSConst;
|
||||
|
||||
const enum DelimiterMasterFilterStateId {
|
||||
export const enum DelimiterMasterFilterStateId {
|
||||
SkippingVersionsV0 = 101,
|
||||
WaitVersionAfterPHDV0 = 102,
|
||||
SkippingGapV0 = 103,
|
||||
};
|
||||
|
||||
interface DelimiterMasterFilterState_SkippingVersionsV0 extends FilterState {
|
||||
|
@ -29,37 +34,121 @@ interface DelimiterMasterFilterState_WaitVersionAfterPHDV0 extends FilterState {
|
|||
masterKey: string,
|
||||
};
|
||||
|
||||
interface DelimiterMasterFilterState_SkippingGapV0 extends FilterState {
|
||||
id: DelimiterMasterFilterStateId.SkippingGapV0,
|
||||
};
|
||||
|
||||
export const enum GapCachingState {
|
||||
NoGapCache = 0, // there is no gap cache
|
||||
UnknownGap = 1, // waiting for a cache lookup
|
||||
GapLookupInProgress = 2, // asynchronous gap lookup in progress
|
||||
GapCached = 3, // an upcoming or already skippable gap is cached
|
||||
NoMoreGap = 4, // the cache doesn't have any more gaps inside the listed range
|
||||
};
|
||||
|
||||
type GapCachingInfo_NoGapCache = {
|
||||
state: GapCachingState.NoGapCache;
|
||||
};
|
||||
|
||||
type GapCachingInfo_NoCachedGap = {
|
||||
state: GapCachingState.UnknownGap
|
||||
| GapCachingState.GapLookupInProgress
|
||||
gapCache: GapCacheInterface;
|
||||
};
|
||||
|
||||
type GapCachingInfo_GapCached = {
|
||||
state: GapCachingState.GapCached;
|
||||
gapCache: GapCacheInterface;
|
||||
gapCached: GapSetEntry;
|
||||
};
|
||||
|
||||
type GapCachingInfo_NoMoreGap = {
|
||||
state: GapCachingState.NoMoreGap;
|
||||
};
|
||||
|
||||
type GapCachingInfo = GapCachingInfo_NoGapCache
|
||||
| GapCachingInfo_NoCachedGap
|
||||
| GapCachingInfo_GapCached
|
||||
| GapCachingInfo_NoMoreGap;
|
||||
|
||||
|
||||
export const enum GapBuildingState {
|
||||
Disabled = 0, // no gap cache or no gap building needed (e.g. in V1 versioning format)
|
||||
NotBuilding = 1, // not currently building a gap (i.e. not listing within a gap)
|
||||
Building = 2, // currently building a gap (i.e. listing within a gap)
|
||||
Expired = 3, // not allowed to build due to exposure delay timeout
|
||||
};
|
||||
|
||||
type GapBuildingInfo_NothingToBuild = {
|
||||
state: GapBuildingState.Disabled | GapBuildingState.Expired;
|
||||
};
|
||||
|
||||
type GapBuildingParams = {
|
||||
/**
|
||||
* minimum weight for a gap to be created in the cache
|
||||
*/
|
||||
minGapWeight: number;
|
||||
/**
|
||||
* trigger a cache setGap() call every N skippable keys
|
||||
*/
|
||||
triggerSaveGapWeight: number;
|
||||
/**
|
||||
* timestamp to assess whether we're still inside the validity period to
|
||||
* be allowed to build gaps
|
||||
*/
|
||||
initTimestamp: number;
|
||||
};
|
||||
|
||||
type GapBuildingInfo_NotBuilding = {
|
||||
state: GapBuildingState.NotBuilding;
|
||||
gapCache: GapCacheInterface;
|
||||
params: GapBuildingParams;
|
||||
};
|
||||
|
||||
type GapBuildingInfo_Building = {
|
||||
state: GapBuildingState.Building;
|
||||
gapCache: GapCacheInterface;
|
||||
params: GapBuildingParams;
|
||||
/**
|
||||
* Gap currently being created
|
||||
*/
|
||||
gap: GapSetEntry;
|
||||
/**
|
||||
* total current weight of the gap being created
|
||||
*/
|
||||
gapWeight: number;
|
||||
};
|
||||
|
||||
type GapBuildingInfo = GapBuildingInfo_NothingToBuild
|
||||
| GapBuildingInfo_NotBuilding
|
||||
| GapBuildingInfo_Building;
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters. This extends the base class Delimiter
|
||||
* to return the raw master versions of existing objects.
|
||||
*/
|
||||
export class DelimiterMaster extends Delimiter {
|
||||
|
||||
_gapCaching: GapCachingInfo;
|
||||
_gapBuilding: GapBuildingInfo;
|
||||
_refreshedBuildingParams: GapBuildingParams | null;
|
||||
|
||||
/**
|
||||
* Delimiter listing of master versions.
|
||||
* @param {Object} parameters - listing parameters
|
||||
* @param {String} parameters.delimiter - delimiter per amazon format
|
||||
* @param {String} parameters.prefix - prefix per amazon format
|
||||
* @param {String} parameters.marker - marker per amazon format
|
||||
* @param {Number} parameters.maxKeys - number of keys to list
|
||||
* @param {Boolean} parameters.v2 - indicates whether v2 format
|
||||
* @param {String} parameters.startAfter - marker per amazon v2 format
|
||||
* @param {String} parameters.continuationToken - obfuscated amazon token
|
||||
* @param {RequestLogger} logger - The logger of the request
|
||||
* @param {String} [vFormat] - versioning key format
|
||||
* @param {String} [parameters.delimiter] - delimiter per amazon format
|
||||
* @param {String} [parameters.prefix] - prefix per amazon format
|
||||
* @param {String} [parameters.marker] - marker per amazon format
|
||||
* @param {Number} [parameters.maxKeys] - number of keys to list
|
||||
* @param {Boolean} [parameters.v2] - indicates whether v2 format
|
||||
* @param {String} [parameters.startAfter] - marker per amazon v2 format
|
||||
* @param {String} [parameters.continuationToken] - obfuscated amazon token
|
||||
* @param {RequestLogger} logger - The logger of the request
|
||||
* @param {String} [vFormat="v0"] - versioning key format
|
||||
*/
|
||||
constructor(parameters, logger, vFormat) {
|
||||
constructor(parameters, logger, vFormat?: string) {
|
||||
super(parameters, logger, vFormat);
|
||||
|
||||
Object.assign(this, {
|
||||
[BucketVersioningKeyFormat.v0]: {
|
||||
skipping: this.skippingV0,
|
||||
},
|
||||
[BucketVersioningKeyFormat.v1]: {
|
||||
skipping: this.skippingV1,
|
||||
},
|
||||
}[this.vFormat]);
|
||||
|
||||
if (this.vFormat === BucketVersioningKeyFormat.v0) {
|
||||
// override Delimiter's implementation of NotSkipping for
|
||||
// DelimiterMaster logic (skipping versions and special
|
||||
|
@ -77,6 +166,10 @@ export class DelimiterMaster extends Delimiter {
|
|||
DelimiterMasterFilterStateId.WaitVersionAfterPHDV0,
|
||||
this.keyHandler_WaitVersionAfterPHDV0.bind(this));
|
||||
|
||||
this.setKeyHandler(
|
||||
DelimiterMasterFilterStateId.SkippingGapV0,
|
||||
this.keyHandler_SkippingGapV0.bind(this));
|
||||
|
||||
if (this.marker) {
|
||||
// distinct initial state to include some special logic
|
||||
// before the first master key is found that does not have
|
||||
|
@ -90,9 +183,186 @@ export class DelimiterMaster extends Delimiter {
|
|||
id: DelimiterFilterStateId.NotSkipping,
|
||||
};
|
||||
}
|
||||
} else {
|
||||
// save base implementation of the `NotSkipping` state in
|
||||
// Delimiter before overriding it with ours, to be able to call it from there
|
||||
this.keyHandler_NotSkipping_Delimiter = this.keyHandlers[DelimiterFilterStateId.NotSkipping];
|
||||
this.setKeyHandler(
|
||||
DelimiterFilterStateId.NotSkipping,
|
||||
this.keyHandler_NotSkippingPrefixNorVersionsV1.bind(this));
|
||||
}
|
||||
// in v1, we can directly use Delimiter's implementation,
|
||||
// which is already set to the proper state
|
||||
|
||||
// default initialization of the gap cache and building states, can be
|
||||
// set by refreshGapCache()
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.NoGapCache,
|
||||
};
|
||||
this._gapBuilding = {
|
||||
state: GapBuildingState.Disabled,
|
||||
};
|
||||
this._refreshedBuildingParams = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the validity period left before a refresh of the gap cache is needed
|
||||
* to continue building new gaps.
|
||||
*
|
||||
* @return {number|null} one of:
|
||||
* - the remaining time in milliseconds in which gaps can be added to the
|
||||
* cache before a call to refreshGapCache() is required
|
||||
* - or 0 if there is no time left and a call to refreshGapCache() is required
|
||||
* to resume caching gaps
|
||||
* - or null if refreshing the cache is never needed (because the gap cache
|
||||
* is either not available or not used)
|
||||
*/
|
||||
getGapBuildingValidityPeriodMs(): number | null {
|
||||
let gapBuilding;
|
||||
switch (this._gapBuilding.state) {
|
||||
case GapBuildingState.Disabled:
|
||||
return null;
|
||||
case GapBuildingState.Expired:
|
||||
return 0;
|
||||
case GapBuildingState.NotBuilding:
|
||||
gapBuilding = <GapBuildingInfo_NotBuilding> this._gapBuilding;
|
||||
break;
|
||||
case GapBuildingState.Building:
|
||||
gapBuilding = <GapBuildingInfo_Building> this._gapBuilding;
|
||||
break;
|
||||
}
|
||||
const { gapCache, params } = gapBuilding;
|
||||
const elapsedTime = Date.now() - params.initTimestamp;
|
||||
return Math.max(gapCache.exposureDelayMs - elapsedTime, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Refresh the gaps caching logic (gaps are series of current delete markers
|
||||
* in V0 bucket metadata format). It has two effects:
|
||||
*
|
||||
* - starts exposing existing and future gaps from the cache to efficiently
|
||||
* skip over series of current delete markers that have been seen and cached
|
||||
* earlier
|
||||
*
|
||||
* - enables building and caching new gaps (or extend existing ones), for a
|
||||
* limited time period defined by the `gapCacheProxy.exposureDelayMs` value
|
||||
* in milliseconds. To refresh the validity period and resume building and
|
||||
* caching new gaps, one must restart a new listing from the database (starting
|
||||
* at the current listing key, included), then call refreshGapCache() again.
|
||||
*
|
||||
* @param {GapCacheInterface} gapCacheProxy - API proxy to the gaps cache
|
||||
* (the proxy should handle prefixing object keys with the bucket name)
|
||||
* @param {number} [minGapWeight=100] - minimum weight of a gap for it to be
|
||||
* added in the cache
|
||||
* @param {number} [triggerSaveGapWeight] - cumulative weight to wait for
|
||||
* before saving the current building gap. Cannot be greater than
|
||||
* `gapCacheProxy.maxGapWeight` (the value is thresholded to `maxGapWeight`
|
||||
* otherwise). Defaults to `gapCacheProxy.maxGapWeight / 2`.
|
||||
* @return {undefined}
|
||||
*/
|
||||
refreshGapCache(
|
||||
gapCacheProxy: GapCacheInterface,
|
||||
minGapWeight?: number,
|
||||
triggerSaveGapWeight?: number
|
||||
): void {
|
||||
if (this.vFormat !== BucketVersioningKeyFormat.v0) {
|
||||
return;
|
||||
}
|
||||
if (this._gapCaching.state === GapCachingState.NoGapCache) {
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.UnknownGap,
|
||||
gapCache: gapCacheProxy,
|
||||
};
|
||||
}
|
||||
const refreshedBuildingParams: GapBuildingParams = {
|
||||
minGapWeight: minGapWeight || 100,
|
||||
triggerSaveGapWeight: triggerSaveGapWeight
|
||||
|| Math.trunc(gapCacheProxy.maxGapWeight / 2),
|
||||
initTimestamp: Date.now(),
|
||||
};
|
||||
if (this._gapBuilding.state === GapBuildingState.Building) {
|
||||
// refreshed params will be applied as soon as the current building gap is saved
|
||||
this._refreshedBuildingParams = refreshedBuildingParams;
|
||||
} else {
|
||||
this._gapBuilding = {
|
||||
state: GapBuildingState.NotBuilding,
|
||||
gapCache: gapCacheProxy,
|
||||
params: refreshedBuildingParams,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Trigger a lookup of the closest upcoming or already skippable gap.
|
||||
*
|
||||
* @param {string} fromKey - lookup a gap not before 'fromKey'
|
||||
* @return {undefined} - the lookup is asynchronous and its
|
||||
* response is handled inside this function
|
||||
*/
|
||||
_triggerGapLookup(gapCaching: GapCachingInfo_NoCachedGap, fromKey: string): void {
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.GapLookupInProgress,
|
||||
gapCache: gapCaching.gapCache,
|
||||
};
|
||||
const maxKey = this.prefix ? inc(this.prefix) : undefined;
|
||||
gapCaching.gapCache.lookupGap(fromKey, maxKey).then(_gap => {
|
||||
const gap = <GapSetEntry | null> _gap;
|
||||
if (gap) {
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.GapCached,
|
||||
gapCache: gapCaching.gapCache,
|
||||
gapCached: gap,
|
||||
};
|
||||
} else {
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.NoMoreGap,
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
_checkGapOnMasterDeleteMarker(key: string): FilterReturnValue {
|
||||
switch (this._gapBuilding.state) {
|
||||
case GapBuildingState.Disabled:
|
||||
case GapBuildingState.Expired:
|
||||
break;
|
||||
case GapBuildingState.NotBuilding:
|
||||
this._createBuildingGap(key, 1);
|
||||
break;
|
||||
case GapBuildingState.Building:
|
||||
this._updateBuildingGap(key);
|
||||
break;
|
||||
}
|
||||
if (this._gapCaching.state === GapCachingState.GapCached) {
|
||||
const { gapCached } = this._gapCaching;
|
||||
if (key >= gapCached.firstKey) {
|
||||
if (key <= gapCached.lastKey) {
|
||||
// we are inside the last looked up cached gap: transition to
|
||||
// 'SkippingGapV0' state
|
||||
this.setState(<DelimiterMasterFilterState_SkippingGapV0> {
|
||||
id: DelimiterMasterFilterStateId.SkippingGapV0,
|
||||
});
|
||||
// cut the current gap before skipping, it will be merged or
|
||||
// chained with the existing one (depending on its weight)
|
||||
if (this._gapBuilding.state === GapBuildingState.Building) {
|
||||
// substract 1 from the weight because we are going to chain this gap,
|
||||
// which has an overlap of one key.
|
||||
this._gapBuilding.gap.weight -= 1;
|
||||
this._cutBuildingGap();
|
||||
}
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
// as we are past the cached gap, we will need another lookup
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.UnknownGap,
|
||||
gapCache: this._gapCaching.gapCache,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (this._gapCaching.state === GapCachingState.UnknownGap) {
|
||||
this._triggerGapLookup(this._gapCaching, key);
|
||||
}
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
|
||||
filter_onNewMasterKeyV0(key: string, value: string): FilterReturnValue {
|
||||
|
@ -104,7 +374,7 @@ export class DelimiterMaster extends Delimiter {
|
|||
id: DelimiterMasterFilterStateId.SkippingVersionsV0,
|
||||
masterKey: key,
|
||||
});
|
||||
return FILTER_ACCEPT;
|
||||
return this._checkGapOnMasterDeleteMarker(key);
|
||||
}
|
||||
if (Version.isPHD(value)) {
|
||||
// master version is a PHD version: wait for the first
|
||||
|
@ -116,6 +386,9 @@ export class DelimiterMaster extends Delimiter {
|
|||
});
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
// cut the current gap as soon as a non-deleted entry is seen
|
||||
this._cutBuildingGap();
|
||||
|
||||
if (key.startsWith(DbPrefixes.Replay)) {
|
||||
// skip internal replay prefix entirely
|
||||
this.setState(<DelimiterFilterState_SkippingPrefix> {
|
||||
|
@ -127,6 +400,7 @@ export class DelimiterMaster extends Delimiter {
|
|||
if (this._reachedMaxKeys()) {
|
||||
return FILTER_END;
|
||||
}
|
||||
|
||||
const commonPrefix = this.addCommonPrefixOrContents(key, value);
|
||||
if (commonPrefix) {
|
||||
// transition into SkippingPrefix state to skip all following keys
|
||||
|
@ -149,11 +423,30 @@ export class DelimiterMaster extends Delimiter {
|
|||
return this.filter_onNewMasterKeyV0(key, value);
|
||||
}
|
||||
|
||||
filter_onNewMasterKeyV1(key: string, value: string): FilterReturnValue {
|
||||
// if this master key is a delete marker, accept it without
|
||||
// adding the version to the contents
|
||||
if (Version.isDeleteMarker(value)) {
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
// use base Delimiter's implementation
|
||||
return this.keyHandler_NotSkipping_Delimiter(key, value);
|
||||
}
|
||||
|
||||
keyHandler_NotSkippingPrefixNorVersionsV1(key: string, value: string): FilterReturnValue {
|
||||
return this.filter_onNewMasterKeyV1(key, value);
|
||||
}
|
||||
|
||||
keyHandler_SkippingVersionsV0(key: string, value: string): FilterReturnValue {
|
||||
/* In the SkippingVersionsV0 state, skip all version keys
|
||||
* (<key><versionIdSeparator><version>) */
|
||||
const versionIdIndex = key.indexOf(VID_SEP);
|
||||
if (versionIdIndex !== -1) {
|
||||
// version keys count in the building gap weight because they must
|
||||
// also be listed until skipped
|
||||
if (this._gapBuilding.state === GapBuildingState.Building) {
|
||||
this._updateBuildingGap(key);
|
||||
}
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
return this.filter_onNewMasterKeyV0(key, value);
|
||||
|
@ -177,14 +470,151 @@ export class DelimiterMaster extends Delimiter {
|
|||
return this.filter_onNewMasterKeyV0(key, value);
|
||||
}
|
||||
|
||||
keyHandler_SkippingGapV0(key: string, value: string): FilterReturnValue {
|
||||
const { gapCache, gapCached } = <GapCachingInfo_GapCached> this._gapCaching;
|
||||
if (key <= gapCached.lastKey) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
this._gapCaching = {
|
||||
state: GapCachingState.UnknownGap,
|
||||
gapCache,
|
||||
};
|
||||
this.setState(<DelimiterMasterFilterState_SkippingVersionsV0> {
|
||||
id: DelimiterMasterFilterStateId.SkippingVersionsV0,
|
||||
});
|
||||
// Start a gap with weight=0 from the latest skippable key. This will
|
||||
// allow to extend the gap just skipped with a chained gap in case
|
||||
// other delete markers are seen after the existing gap is skipped.
|
||||
this._createBuildingGap(gapCached.lastKey, 0, gapCached.weight);
|
||||
|
||||
return this.handleKey(key, value);
|
||||
}
|
||||
|
||||
skippingBase(): string | undefined {
|
||||
switch (this.state.id) {
|
||||
case DelimiterMasterFilterStateId.SkippingVersionsV0:
|
||||
const { masterKey } = <DelimiterMasterFilterState_SkippingVersionsV0> this.state;
|
||||
return masterKey + VID_SEP;
|
||||
return masterKey + inc(VID_SEP);
|
||||
|
||||
case DelimiterMasterFilterStateId.SkippingGapV0:
|
||||
const { gapCached } = <GapCachingInfo_GapCached> this._gapCaching;
|
||||
return gapCached.lastKey;
|
||||
|
||||
default:
|
||||
return super.skippingBase();
|
||||
}
|
||||
}
|
||||
|
||||
result(): ResultObject {
|
||||
this._cutBuildingGap();
|
||||
return super.result();
|
||||
}
|
||||
|
||||
_checkRefreshedBuildingParams(params: GapBuildingParams): GapBuildingParams {
|
||||
if (this._refreshedBuildingParams) {
|
||||
const newParams = this._refreshedBuildingParams;
|
||||
this._refreshedBuildingParams = null;
|
||||
return newParams;
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save the gap being built if allowed (i.e. still within the
|
||||
* allocated exposure time window).
|
||||
*
|
||||
* @return {boolean} - true if the gap was saved, false if we are
|
||||
* outside the allocated exposure time window.
|
||||
*/
|
||||
_saveBuildingGap(): boolean {
|
||||
const { gapCache, params, gap, gapWeight } =
|
||||
<GapBuildingInfo_Building> this._gapBuilding;
|
||||
const totalElapsed = Date.now() - params.initTimestamp;
|
||||
if (totalElapsed >= gapCache.exposureDelayMs) {
|
||||
this._gapBuilding = {
|
||||
state: GapBuildingState.Expired,
|
||||
};
|
||||
this._refreshedBuildingParams = null;
|
||||
return false;
|
||||
}
|
||||
const { firstKey, lastKey, weight } = gap;
|
||||
gapCache.setGap(firstKey, lastKey, weight);
|
||||
this._gapBuilding = {
|
||||
state: GapBuildingState.Building,
|
||||
gapCache,
|
||||
params: this._checkRefreshedBuildingParams(params),
|
||||
gap: {
|
||||
firstKey: gap.lastKey,
|
||||
lastKey: gap.lastKey,
|
||||
weight: 0,
|
||||
},
|
||||
gapWeight,
|
||||
};
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new gap to be extended afterwards
|
||||
*
|
||||
* @param {string} newKey - gap's first key
|
||||
* @param {number} startWeight - initial weight of the building gap (usually 0 or 1)
|
||||
* @param {number} [cachedWeight] - if continuing a cached gap, weight of the existing
|
||||
* cached portion
|
||||
* @return {undefined}
|
||||
*/
|
||||
_createBuildingGap(newKey: string, startWeight: number, cachedWeight?: number): void {
|
||||
if (this._gapBuilding.state === GapBuildingState.NotBuilding) {
|
||||
const { gapCache, params } = <GapBuildingInfo_NotBuilding> this._gapBuilding;
|
||||
this._gapBuilding = {
|
||||
state: GapBuildingState.Building,
|
||||
gapCache,
|
||||
params: this._checkRefreshedBuildingParams(params),
|
||||
gap: {
|
||||
firstKey: newKey,
|
||||
lastKey: newKey,
|
||||
weight: startWeight,
|
||||
},
|
||||
gapWeight: (cachedWeight || 0) + startWeight,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
_updateBuildingGap(newKey: string): void {
|
||||
const gapBuilding = <GapBuildingInfo_Building> this._gapBuilding;
|
||||
const { params, gap } = gapBuilding;
|
||||
gap.lastKey = newKey;
|
||||
gap.weight += 1;
|
||||
gapBuilding.gapWeight += 1;
|
||||
// the GapCache API requires updating a gap regularly because it can only split
|
||||
// it once per update, by the known last key. In practice the default behavior
|
||||
// is to trigger an update after a number of keys that is half the maximum weight.
|
||||
// It is also useful for other listings to benefit from the cache sooner.
|
||||
if (gapBuilding.gapWeight >= params.minGapWeight &&
|
||||
gap.weight >= params.triggerSaveGapWeight) {
|
||||
this._saveBuildingGap();
|
||||
}
|
||||
}
|
||||
|
||||
_cutBuildingGap(): void {
|
||||
if (this._gapBuilding.state === GapBuildingState.Building) {
|
||||
let gapBuilding = <GapBuildingInfo_Building> this._gapBuilding;
|
||||
let { gapCache, params, gap, gapWeight } = gapBuilding;
|
||||
// only set gaps that are significant enough in weight and
|
||||
// with a non-empty extension
|
||||
if (gapWeight >= params.minGapWeight && gap.weight > 0) {
|
||||
// we're done if we were not allowed to save the gap
|
||||
if (!this._saveBuildingGap()) {
|
||||
return;
|
||||
}
|
||||
// params may have been refreshed, reload them
|
||||
gapBuilding = <GapBuildingInfo_Building> this._gapBuilding;
|
||||
params = gapBuilding.params;
|
||||
}
|
||||
this._gapBuilding = {
|
||||
state: GapBuildingState.NotBuilding,
|
||||
gapCache,
|
||||
params,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,202 @@
|
|||
const { DelimiterVersions } = require('./delimiterVersions');
|
||||
const { FILTER_END, FILTER_SKIP } = require('./tools');
|
||||
|
||||
const TRIM_METADATA_MIN_BLOB_SIZE = 10000;
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters. This extends the base class DelimiterVersions
|
||||
* to return the raw non-current versions objects.
|
||||
*/
|
||||
class DelimiterNonCurrent extends DelimiterVersions {
|
||||
/**
|
||||
* Delimiter listing of non-current versions.
|
||||
* @param {Object} parameters - listing parameters
|
||||
* @param {String} parameters.keyMarker - key marker
|
||||
* @param {String} parameters.versionIdMarker - version id marker
|
||||
* @param {String} parameters.beforeDate - limit the response to keys with stale date older than beforeDate.
|
||||
* “stale date” is the date on when a version becomes non-current.
|
||||
* @param {Number} parameters.maxScannedLifecycleListingEntries - max number of entries to be scanned
|
||||
* @param {String} parameters.excludedDataStoreName - exclude dataStoreName matches from the versions
|
||||
* @param {RequestLogger} logger - The logger of the request
|
||||
* @param {String} [vFormat] - versioning key format
|
||||
*/
|
||||
constructor(parameters, logger, vFormat) {
|
||||
super(parameters, logger, vFormat);
|
||||
|
||||
this.beforeDate = parameters.beforeDate;
|
||||
this.excludedDataStoreName = parameters.excludedDataStoreName;
|
||||
this.maxScannedLifecycleListingEntries = parameters.maxScannedLifecycleListingEntries;
|
||||
|
||||
// internal state
|
||||
this.prevKey = null;
|
||||
this.staleDate = null;
|
||||
|
||||
this.scannedKeys = 0;
|
||||
}
|
||||
|
||||
getLastModified(value) {
|
||||
let lastModified;
|
||||
try {
|
||||
const v = JSON.parse(value);
|
||||
lastModified = v['last-modified'];
|
||||
} catch (e) {
|
||||
this.logger.warn('could not parse Object Metadata while listing',
|
||||
{
|
||||
method: 'getLastModified',
|
||||
err: e.toString(),
|
||||
});
|
||||
}
|
||||
return lastModified;
|
||||
}
|
||||
|
||||
// Overwrite keyHandler_SkippingVersions to include the last version from the previous listing.
|
||||
// The creation (last-modified) date of this version will be the stale date for the following version.
|
||||
// eslint-disable-next-line camelcase
|
||||
keyHandler_SkippingVersions(key, versionId, value) {
|
||||
if (key === this.keyMarker) {
|
||||
// since the nonversioned key equals the marker, there is
|
||||
// necessarily a versionId in this key
|
||||
const _versionId = versionId;
|
||||
if (_versionId < this.versionIdMarker) {
|
||||
// skip all versions until marker
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
}
|
||||
this.setState({
|
||||
id: 1 /* NotSkipping */,
|
||||
});
|
||||
return this.handleKey(key, versionId, value);
|
||||
}
|
||||
|
||||
filter(obj) {
|
||||
if (this.maxScannedLifecycleListingEntries && this.scannedKeys >= this.maxScannedLifecycleListingEntries) {
|
||||
this.IsTruncated = true;
|
||||
this.logger.info('listing stopped due to reaching the maximum scanned entries limit',
|
||||
{
|
||||
maxScannedLifecycleListingEntries: this.maxScannedLifecycleListingEntries,
|
||||
scannedKeys: this.scannedKeys,
|
||||
});
|
||||
return FILTER_END;
|
||||
}
|
||||
++this.scannedKeys;
|
||||
return super.filter(obj);
|
||||
}
|
||||
|
||||
/**
|
||||
* NOTE: Each version of a specific key is sorted from the latest to the oldest
|
||||
* thanks to the way version ids are generated.
|
||||
* DESCRIPTION: Skip the version if it represents the master key, but keep its last-modified date in memory,
|
||||
* which will be the stale date of the following version.
|
||||
* The following version is pushed only:
|
||||
* - if the "stale date" (picked up from the previous version) is available (JSON.parse has not failed),
|
||||
* - if "beforeDate" is not specified or if specified and the "stale date" is older.
|
||||
* - if "excludedDataStoreName" is not specified or if specified and the data store name is different
|
||||
* The in-memory "stale date" is then updated with the version's last-modified date to be used for
|
||||
* the following version.
|
||||
* The process stops and returns the available results if either:
|
||||
* - no more metadata key is left to be processed
|
||||
* - the listing reaches the maximum number of key to be returned
|
||||
* - the internal timeout is reached
|
||||
* @param {String} key - The key to add
|
||||
* @param {String} versionId - The version id
|
||||
* @param {String} value - The value of the key
|
||||
* @return {undefined}
|
||||
*/
|
||||
addVersion(key, versionId, value) {
|
||||
this.nextKeyMarker = key;
|
||||
this.nextVersionIdMarker = versionId;
|
||||
|
||||
// Skip the version if it represents the non-current version, but keep its last-modified date,
|
||||
// which will be the stale date of the following version.
|
||||
const isCurrentVersion = key !== this.prevKey;
|
||||
if (isCurrentVersion) {
|
||||
this.staleDate = this.getLastModified(value);
|
||||
this.prevKey = key;
|
||||
return;
|
||||
}
|
||||
|
||||
// The following version is pushed only:
|
||||
// - if the "stale date" (picked up from the previous version) is available (JSON.parse has not failed),
|
||||
// - if "beforeDate" is not specified or if specified and the "stale date" is older.
|
||||
// - if "excludedDataStoreName" is not specified or if specified and the data store name is different
|
||||
let lastModified;
|
||||
if (this.staleDate && (!this.beforeDate || this.staleDate < this.beforeDate)) {
|
||||
const parsedValue = this._parse(value);
|
||||
// if parsing fails, skip the key.
|
||||
if (parsedValue) {
|
||||
const dataStoreName = parsedValue.dataStoreName;
|
||||
lastModified = parsedValue['last-modified'];
|
||||
if (!this.excludedDataStoreName || dataStoreName !== this.excludedDataStoreName) {
|
||||
const s = this._stringify(parsedValue, this.staleDate);
|
||||
// check that _stringify succeeds to only push objects with a defined staleDate.
|
||||
if (s) {
|
||||
this.Versions.push({ key, value: s });
|
||||
++this.keys;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The in-memory "stale date" is then updated with the version's last-modified date to be used for
|
||||
// the following version.
|
||||
this.staleDate = lastModified || this.getLastModified(value);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the stringified entry's value and remove the location property if too large.
|
||||
* @param {string} s - sringified value
|
||||
* @return {object} p - undefined if parsing fails, otherwise it contains the parsed value.
|
||||
*/
|
||||
_parse(s) {
|
||||
let p;
|
||||
try {
|
||||
p = JSON.parse(s);
|
||||
if (s.length >= TRIM_METADATA_MIN_BLOB_SIZE) {
|
||||
delete p.location;
|
||||
}
|
||||
} catch (e) {
|
||||
this.logger.warn('Could not parse Object Metadata while listing', {
|
||||
method: 'DelimiterNonCurrent._parse',
|
||||
err: e.toString(),
|
||||
});
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
_stringify(parsedMD, staleDate) {
|
||||
const p = parsedMD;
|
||||
let s = undefined;
|
||||
p.staleDate = staleDate;
|
||||
try {
|
||||
s = JSON.stringify(p);
|
||||
} catch (e) {
|
||||
this.logger.warn('could not stringify Object Metadata while listing', {
|
||||
method: 'DelimiterNonCurrent._stringify',
|
||||
err: e.toString(),
|
||||
});
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
result() {
|
||||
const { Versions, IsTruncated, NextKeyMarker, NextVersionIdMarker } = super.result();
|
||||
|
||||
const result = {
|
||||
Contents: Versions,
|
||||
IsTruncated,
|
||||
};
|
||||
|
||||
if (NextKeyMarker) {
|
||||
result.NextKeyMarker = NextKeyMarker;
|
||||
}
|
||||
|
||||
if (NextVersionIdMarker) {
|
||||
result.NextVersionIdMarker = NextVersionIdMarker;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
module.exports = { DelimiterNonCurrent };
|
|
@ -0,0 +1,204 @@
|
|||
const DelimiterVersions = require('./delimiterVersions').DelimiterVersions;
|
||||
const { FILTER_END } = require('./tools');
|
||||
const TRIM_METADATA_MIN_BLOB_SIZE = 10000;
|
||||
/**
|
||||
* Handle object listing with parameters. This extends the base class DelimiterVersions
|
||||
* to return the orphan delete markers. Orphan delete markers are also
|
||||
* refered as expired object delete marker.
|
||||
* They are delete marker with zero noncurrent versions.
|
||||
*/
|
||||
class DelimiterOrphanDeleteMarker extends DelimiterVersions {
|
||||
/**
|
||||
* Delimiter listing of orphan delete markers.
|
||||
* @param {Object} parameters - listing parameters
|
||||
* @param {String} parameters.beforeDate - limit the response to keys older than beforeDate
|
||||
* @param {Number} parameters.maxScannedLifecycleListingEntries - max number of entries to be scanned
|
||||
* @param {RequestLogger} logger - The logger of the request
|
||||
* @param {String} [vFormat] - versioning key format
|
||||
*/
|
||||
constructor(parameters, logger, vFormat) {
|
||||
const {
|
||||
marker,
|
||||
maxKeys,
|
||||
prefix,
|
||||
beforeDate,
|
||||
maxScannedLifecycleListingEntries,
|
||||
} = parameters;
|
||||
|
||||
const versionParams = {
|
||||
// The orphan delete marker logic uses the term 'marker' instead of 'keyMarker',
|
||||
// as the latter could suggest the presence of a 'versionIdMarker'.
|
||||
keyMarker: marker,
|
||||
maxKeys,
|
||||
prefix,
|
||||
};
|
||||
super(versionParams, logger, vFormat);
|
||||
|
||||
this.maxScannedLifecycleListingEntries = maxScannedLifecycleListingEntries;
|
||||
this.beforeDate = beforeDate;
|
||||
// this.prevKeyName is used as a marker for the next listing when the current one reaches its entry limit.
|
||||
// We cannot rely on this.keyName, as it contains the name of the current key.
|
||||
// In the event of a listing interruption due to reaching the maximum scanned entries,
|
||||
// relying on this.keyName would cause the next listing to skip the current key because S3 starts
|
||||
// listing after the marker.
|
||||
this.prevKeyName = null;
|
||||
this.keyName = null;
|
||||
this.value = null;
|
||||
this.scannedKeys = 0;
|
||||
}
|
||||
|
||||
_reachedMaxKeys() {
|
||||
if (this.keys >= this.maxKeys) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
_addOrphan() {
|
||||
const parsedValue = this._parse(this.value);
|
||||
// if parsing fails, skip the key.
|
||||
if (parsedValue) {
|
||||
const lastModified = parsedValue['last-modified'];
|
||||
const isDeleteMarker = parsedValue.isDeleteMarker;
|
||||
// We then check if the orphan version is a delete marker and if it is older than the "beforeDate"
|
||||
if ((!this.beforeDate || (lastModified && lastModified < this.beforeDate)) && isDeleteMarker) {
|
||||
// Prefer returning an untrimmed data rather than stopping the service in case of parsing failure.
|
||||
const s = this._stringify(parsedValue) || this.value;
|
||||
this.Versions.push({ key: this.keyName, value: s });
|
||||
this.nextKeyMarker = this.keyName;
|
||||
++this.keys;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the stringified entry's value and remove the location property if too large.
|
||||
* @param {string} s - sringified value
|
||||
* @return {object} p - undefined if parsing fails, otherwise it contains the parsed value.
|
||||
*/
|
||||
_parse(s) {
|
||||
let p;
|
||||
try {
|
||||
p = JSON.parse(s);
|
||||
if (s.length >= TRIM_METADATA_MIN_BLOB_SIZE) {
|
||||
delete p.location;
|
||||
}
|
||||
} catch (e) {
|
||||
this.logger.warn('Could not parse Object Metadata while listing', {
|
||||
method: 'DelimiterOrphanDeleteMarker._parse',
|
||||
err: e.toString(),
|
||||
});
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
_stringify(value) {
|
||||
const p = value;
|
||||
let s = undefined;
|
||||
try {
|
||||
s = JSON.stringify(p);
|
||||
} catch (e) {
|
||||
this.logger.warn('could not stringify Object Metadata while listing',
|
||||
{
|
||||
method: 'DelimiterOrphanDeleteMarker._stringify',
|
||||
err: e.toString(),
|
||||
});
|
||||
}
|
||||
return s;
|
||||
}
|
||||
/**
|
||||
* The purpose of _isMaxScannedEntriesReached is to restrict the number of scanned entries,
|
||||
* thus controlling resource overhead (CPU...).
|
||||
* @return {boolean} isMaxScannedEntriesReached - true if the maximum limit on the number
|
||||
* of entries scanned has been reached, false otherwise.
|
||||
*/
|
||||
_isMaxScannedEntriesReached() {
|
||||
return this.maxScannedLifecycleListingEntries && this.scannedKeys >= this.maxScannedLifecycleListingEntries;
|
||||
}
|
||||
|
||||
filter(obj) {
|
||||
if (this._isMaxScannedEntriesReached()) {
|
||||
this.nextKeyMarker = this.prevKeyName;
|
||||
this.IsTruncated = true;
|
||||
this.logger.info('listing stopped due to reaching the maximum scanned entries limit',
|
||||
{
|
||||
maxScannedLifecycleListingEntries: this.maxScannedLifecycleListingEntries,
|
||||
scannedKeys: this.scannedKeys,
|
||||
});
|
||||
return FILTER_END;
|
||||
}
|
||||
++this.scannedKeys;
|
||||
return super.filter(obj);
|
||||
}
|
||||
|
||||
/**
|
||||
* NOTE: Each version of a specific key is sorted from the latest to the oldest
|
||||
* thanks to the way version ids are generated.
|
||||
* DESCRIPTION: For a given key, the latest version is kept in memory since it is the current version.
|
||||
* If the following version reference a new key, it means that the previous one was an orphan version.
|
||||
* We then check if the orphan version is a delete marker and if it is older than the "beforeDate"
|
||||
* The process stops and returns the available results if either:
|
||||
* - no more metadata key is left to be processed
|
||||
* - the listing reaches the maximum number of key to be returned
|
||||
* - the internal timeout is reached
|
||||
* NOTE: we cannot leverage MongoDB to list keys older than "beforeDate"
|
||||
* because then we will not be able to assess its orphanage.
|
||||
* @param {String} key - The object key.
|
||||
* @param {String} versionId - The object version id.
|
||||
* @param {String} value - The value of the key
|
||||
* @return {undefined}
|
||||
*/
|
||||
addVersion(key, versionId, value) {
|
||||
// For a given key, the youngest version is kept in memory since it represents the current version.
|
||||
if (key !== this.keyName) {
|
||||
// If this.value is defined, it means that <this.keyName, this.value> pair is "allowed" to be an orphan.
|
||||
if (this.value) {
|
||||
this._addOrphan();
|
||||
}
|
||||
this.prevKeyName = this.keyName;
|
||||
this.keyName = key;
|
||||
this.value = value;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// If the key is not the current version, we can skip it in the next listing
|
||||
// in the case where the current listing is interrupted due to reaching the maximum scanned entries.
|
||||
this.prevKeyName = key;
|
||||
this.keyName = key;
|
||||
this.value = null;
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
result() {
|
||||
// Only check for remaining last orphan delete marker if the listing is not interrupted.
|
||||
// This will help avoid false positives.
|
||||
if (!this._isMaxScannedEntriesReached()) {
|
||||
// The following check makes sure the last orphan delete marker is not forgotten.
|
||||
if (this.keys < this.maxKeys) {
|
||||
if (this.value) {
|
||||
this._addOrphan();
|
||||
}
|
||||
// The following make sure that if makeKeys is reached, isTruncated is set to true.
|
||||
// We moved the "isTruncated" from _reachedMaxKeys to make sure we take into account the last entity
|
||||
// if listing is truncated right before the last entity and the last entity is a orphan delete marker.
|
||||
} else {
|
||||
this.IsTruncated = this.maxKeys > 0;
|
||||
}
|
||||
}
|
||||
|
||||
const result = {
|
||||
Contents: this.Versions,
|
||||
IsTruncated: this.IsTruncated,
|
||||
};
|
||||
|
||||
if (this.IsTruncated) {
|
||||
result.NextMarker = this.nextKeyMarker;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { DelimiterOrphanDeleteMarker };
|
|
@ -1,304 +0,0 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const Delimiter = require('./delimiter').Delimiter;
|
||||
const Version = require('../../versioning/Version').Version;
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { inc, FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } =
|
||||
require('./tools');
|
||||
|
||||
const VID_SEP = VSConst.VersionId.Separator;
|
||||
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters
|
||||
*
|
||||
* @prop {String[]} CommonPrefixes - 'folders' defined by the delimiter
|
||||
* @prop {String[]} Contents - 'files' to list
|
||||
* @prop {Boolean} IsTruncated - truncated listing flag
|
||||
* @prop {String|undefined} NextMarker - marker per amazon format
|
||||
* @prop {Number} keys - count of listed keys
|
||||
* @prop {String|undefined} delimiter - separator per amazon format
|
||||
* @prop {String|undefined} prefix - prefix per amazon format
|
||||
* @prop {Number} maxKeys - number of keys to list
|
||||
*/
|
||||
class DelimiterVersions extends Delimiter {
|
||||
constructor(parameters, logger, vFormat) {
|
||||
super(parameters, logger, vFormat);
|
||||
// specific to version listing
|
||||
this.keyMarker = parameters.keyMarker;
|
||||
this.versionIdMarker = parameters.versionIdMarker;
|
||||
// internal state
|
||||
this.masterKey = undefined;
|
||||
this.masterVersionId = undefined;
|
||||
// listing results
|
||||
this.NextMarker = parameters.keyMarker;
|
||||
this.NextVersionIdMarker = undefined;
|
||||
this.inReplayPrefix = false;
|
||||
|
||||
Object.assign(this, {
|
||||
[BucketVersioningKeyFormat.v0]: {
|
||||
genMDParams: this.genMDParamsV0,
|
||||
filter: this.filterV0,
|
||||
skipping: this.skippingV0,
|
||||
},
|
||||
[BucketVersioningKeyFormat.v1]: {
|
||||
genMDParams: this.genMDParamsV1,
|
||||
filter: this.filterV1,
|
||||
skipping: this.skippingV1,
|
||||
},
|
||||
}[this.vFormat]);
|
||||
}
|
||||
|
||||
genMDParamsV0() {
|
||||
const params = {};
|
||||
if (this.parameters.prefix) {
|
||||
params.gte = this.parameters.prefix;
|
||||
params.lt = inc(this.parameters.prefix);
|
||||
}
|
||||
if (this.parameters.keyMarker) {
|
||||
if (params.gte && params.gte > this.parameters.keyMarker) {
|
||||
return params;
|
||||
}
|
||||
delete params.gte;
|
||||
if (this.parameters.versionIdMarker) {
|
||||
// versionIdMarker should always come with keyMarker
|
||||
// but may not be the other way around
|
||||
params.gt = this.parameters.keyMarker
|
||||
+ VID_SEP
|
||||
+ this.parameters.versionIdMarker;
|
||||
} else {
|
||||
params.gt = inc(this.parameters.keyMarker + VID_SEP);
|
||||
}
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
genMDParamsV1() {
|
||||
// return an array of two listing params sets to ask for
|
||||
// synchronized listing of M and V ranges
|
||||
const params = [{}, {}];
|
||||
if (this.parameters.prefix) {
|
||||
params[0].gte = DbPrefixes.Master + this.parameters.prefix;
|
||||
params[0].lt = DbPrefixes.Master + inc(this.parameters.prefix);
|
||||
params[1].gte = DbPrefixes.Version + this.parameters.prefix;
|
||||
params[1].lt = DbPrefixes.Version + inc(this.parameters.prefix);
|
||||
} else {
|
||||
params[0].gte = DbPrefixes.Master;
|
||||
params[0].lt = inc(DbPrefixes.Master); // stop after the last master key
|
||||
params[1].gte = DbPrefixes.Version;
|
||||
params[1].lt = inc(DbPrefixes.Version); // stop after the last version key
|
||||
}
|
||||
if (this.parameters.keyMarker) {
|
||||
if (params[1].gte <= DbPrefixes.Version + this.parameters.keyMarker) {
|
||||
delete params[0].gte;
|
||||
delete params[1].gte;
|
||||
params[0].gt = DbPrefixes.Master + inc(this.parameters.keyMarker + VID_SEP);
|
||||
if (this.parameters.versionIdMarker) {
|
||||
// versionIdMarker should always come with keyMarker
|
||||
// but may not be the other way around
|
||||
params[1].gt = DbPrefixes.Version
|
||||
+ this.parameters.keyMarker
|
||||
+ VID_SEP
|
||||
+ this.parameters.versionIdMarker;
|
||||
} else {
|
||||
params[1].gt = DbPrefixes.Version
|
||||
+ inc(this.parameters.keyMarker + VID_SEP);
|
||||
}
|
||||
}
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
/**
|
||||
* Used to synchronize listing of M and V prefixes by object key
|
||||
*
|
||||
* @param {object} masterObj object listed from first range
|
||||
* returned by genMDParamsV1() (the master keys range)
|
||||
* @param {object} versionObj object listed from second range
|
||||
* returned by genMDParamsV1() (the version keys range)
|
||||
* @return {number} comparison result:
|
||||
* * -1 if master key < version key
|
||||
* * 1 if master key > version key
|
||||
*/
|
||||
compareObjects(masterObj, versionObj) {
|
||||
const masterKey = masterObj.key.slice(DbPrefixes.Master.length);
|
||||
const versionKey = versionObj.key.slice(DbPrefixes.Version.length);
|
||||
return masterKey < versionKey ? -1 : 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a (key, versionId, value) tuple to the listing.
|
||||
* Set the NextMarker to the current key
|
||||
* Increment the keys counter
|
||||
* @param {object} obj - the entry to add to the listing result
|
||||
* @param {String} obj.key - The key to add
|
||||
* @param {String} obj.versionId - versionId
|
||||
* @param {String} obj.value - The value of the key
|
||||
* @return {Boolean} - indicates if iteration should continue
|
||||
*/
|
||||
addContents(obj) {
|
||||
if (this._reachedMaxKeys()) {
|
||||
return FILTER_END;
|
||||
}
|
||||
this.Contents.push({
|
||||
key: obj.key,
|
||||
value: this.trimMetadata(obj.value),
|
||||
versionId: obj.versionId,
|
||||
});
|
||||
this.NextMarker = obj.key;
|
||||
this.NextVersionIdMarker = obj.versionId;
|
||||
++this.keys;
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a Common Prefix in the list
|
||||
* @param {String} key - object name
|
||||
* @param {Number} index - after prefix starting point
|
||||
* @return {Boolean} - indicates if iteration should continue
|
||||
*/
|
||||
addCommonPrefix(key, index) {
|
||||
const commonPrefix = key.substring(0, index + this.delimiter.length);
|
||||
if (this.CommonPrefixes.indexOf(commonPrefix) === -1
|
||||
&& this.NextMarker !== commonPrefix) {
|
||||
if (this._reachedMaxKeys()) {
|
||||
return FILTER_END;
|
||||
}
|
||||
this.CommonPrefixes.push(commonPrefix);
|
||||
this.NextMarker = commonPrefix;
|
||||
++this.keys;
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter to apply on each iteration if bucket is in v0
|
||||
* versioning key format, based on:
|
||||
* - prefix
|
||||
* - delimiter
|
||||
* - maxKeys
|
||||
* The marker is being handled directly by levelDB
|
||||
* @param {Object} obj - The key and value of the element
|
||||
* @param {String} obj.key - The key of the element
|
||||
* @param {String} obj.value - The value of the element
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
filterV0(obj) {
|
||||
if (obj.key.startsWith(DbPrefixes.Replay)) {
|
||||
this.inReplayPrefix = true;
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
this.inReplayPrefix = false;
|
||||
|
||||
if (Version.isPHD(obj.value)) {
|
||||
// return accept to avoid skipping the next values in range
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
return this.filterCommon(obj.key, obj.value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter to apply on each iteration if bucket is in v1
|
||||
* versioning key format, based on:
|
||||
* - prefix
|
||||
* - delimiter
|
||||
* - maxKeys
|
||||
* The marker is being handled directly by levelDB
|
||||
* @param {Object} obj - The key and value of the element
|
||||
* @param {String} obj.key - The key of the element
|
||||
* @param {String} obj.value - The value of the element
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
filterV1(obj) {
|
||||
if (Version.isPHD(obj.value)) {
|
||||
// return accept to avoid skipping the next values in range
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
// this function receives both M and V keys, but their prefix
|
||||
// length is the same so we can remove their prefix without
|
||||
// looking at the type of key
|
||||
return this.filterCommon(obj.key.slice(DbPrefixes.Master.length),
|
||||
obj.value);
|
||||
}
|
||||
|
||||
filterCommon(key, value) {
|
||||
if (this.prefix && !key.startsWith(this.prefix)) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
let nonversionedKey;
|
||||
let versionId = undefined;
|
||||
const versionIdIndex = key.indexOf(VID_SEP);
|
||||
if (versionIdIndex < 0) {
|
||||
nonversionedKey = key;
|
||||
this.masterKey = key;
|
||||
this.masterVersionId =
|
||||
Version.from(value).getVersionId() || 'null';
|
||||
versionId = this.masterVersionId;
|
||||
} else {
|
||||
nonversionedKey = key.slice(0, versionIdIndex);
|
||||
versionId = key.slice(versionIdIndex + 1);
|
||||
// skip a version key if it is the master version
|
||||
if (this.masterKey === nonversionedKey && this.masterVersionId === versionId) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
this.masterKey = undefined;
|
||||
this.masterVersionId = undefined;
|
||||
}
|
||||
if (this.delimiter) {
|
||||
const baseIndex = this.prefix ? this.prefix.length : 0;
|
||||
const delimiterIndex = nonversionedKey.indexOf(this.delimiter, baseIndex);
|
||||
if (delimiterIndex >= 0) {
|
||||
return this.addCommonPrefix(nonversionedKey, delimiterIndex);
|
||||
}
|
||||
}
|
||||
return this.addContents({ key: nonversionedKey, value, versionId });
|
||||
}
|
||||
|
||||
skippingV0() {
|
||||
if (this.inReplayPrefix) {
|
||||
return DbPrefixes.Replay;
|
||||
}
|
||||
if (this.NextMarker) {
|
||||
const index = this.NextMarker.lastIndexOf(this.delimiter);
|
||||
if (index === this.NextMarker.length - 1) {
|
||||
return this.NextMarker;
|
||||
}
|
||||
}
|
||||
return SKIP_NONE;
|
||||
}
|
||||
|
||||
skippingV1() {
|
||||
const skipV0 = this.skippingV0();
|
||||
if (skipV0 === SKIP_NONE) {
|
||||
return SKIP_NONE;
|
||||
}
|
||||
// skip to the same object key in both M and V range listings
|
||||
return [DbPrefixes.Master + skipV0,
|
||||
DbPrefixes.Version + skipV0];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an object containing all mandatory fields to use once the
|
||||
* iteration is done, doesn't show a NextMarker field if the output
|
||||
* isn't truncated
|
||||
* @return {Object} - following amazon format
|
||||
*/
|
||||
result() {
|
||||
/* NextMarker is only provided when delimiter is used.
|
||||
* specified in v1 listing documentation
|
||||
* http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html
|
||||
*/
|
||||
return {
|
||||
CommonPrefixes: this.CommonPrefixes,
|
||||
Versions: this.Contents,
|
||||
IsTruncated: this.IsTruncated,
|
||||
NextKeyMarker: this.IsTruncated ? this.NextMarker : undefined,
|
||||
NextVersionIdMarker: this.IsTruncated ?
|
||||
this.NextVersionIdMarker : undefined,
|
||||
Delimiter: this.delimiter,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { DelimiterVersions };
|
|
@ -0,0 +1,535 @@
|
|||
'use strict'; // eslint-disable-line strict
|
||||
|
||||
const Extension = require('./Extension').default;
|
||||
|
||||
import {
|
||||
FilterState,
|
||||
FilterReturnValue,
|
||||
} from './delimiter';
|
||||
|
||||
const Version = require('../../versioning/Version').Version;
|
||||
const VSConst = require('../../versioning/constants').VersioningConstants;
|
||||
const { inc, FILTER_END, FILTER_ACCEPT, FILTER_SKIP, SKIP_NONE } =
|
||||
require('./tools');
|
||||
|
||||
const VID_SEP = VSConst.VersionId.Separator;
|
||||
const { DbPrefixes, BucketVersioningKeyFormat } = VSConst;
|
||||
|
||||
export const enum DelimiterVersionsFilterStateId {
|
||||
NotSkipping = 1,
|
||||
SkippingPrefix = 2,
|
||||
SkippingVersions = 3,
|
||||
};
|
||||
|
||||
export interface DelimiterVersionsFilterState_NotSkipping extends FilterState {
|
||||
id: DelimiterVersionsFilterStateId.NotSkipping,
|
||||
};
|
||||
|
||||
export interface DelimiterVersionsFilterState_SkippingPrefix extends FilterState {
|
||||
id: DelimiterVersionsFilterStateId.SkippingPrefix,
|
||||
prefix: string;
|
||||
};
|
||||
|
||||
export interface DelimiterVersionsFilterState_SkippingVersions extends FilterState {
|
||||
id: DelimiterVersionsFilterStateId.SkippingVersions,
|
||||
gt: string;
|
||||
};
|
||||
|
||||
type KeyHandler = (key: string, versionId: string | undefined, value: string) => FilterReturnValue;
|
||||
|
||||
type ResultObject = {
|
||||
CommonPrefixes: string[],
|
||||
Versions: {
|
||||
key: string;
|
||||
value: string;
|
||||
versionId: string;
|
||||
}[];
|
||||
IsTruncated: boolean;
|
||||
Delimiter ?: string;
|
||||
NextKeyMarker ?: string;
|
||||
NextVersionIdMarker ?: string;
|
||||
};
|
||||
|
||||
type GenMDParamsItem = {
|
||||
gt ?: string,
|
||||
gte ?: string,
|
||||
lt ?: string,
|
||||
};
|
||||
|
||||
/**
|
||||
* Handle object listing with parameters
|
||||
*
|
||||
* @prop {String[]} CommonPrefixes - 'folders' defined by the delimiter
|
||||
* @prop {String[]} Contents - 'files' to list
|
||||
* @prop {Boolean} IsTruncated - truncated listing flag
|
||||
* @prop {String|undefined} NextMarker - marker per amazon format
|
||||
* @prop {Number} keys - count of listed keys
|
||||
* @prop {String|undefined} delimiter - separator per amazon format
|
||||
* @prop {String|undefined} prefix - prefix per amazon format
|
||||
* @prop {Number} maxKeys - number of keys to list
|
||||
*/
|
||||
export class DelimiterVersions extends Extension {
|
||||
|
||||
state: FilterState;
|
||||
keyHandlers: { [id: number]: KeyHandler };
|
||||
|
||||
constructor(parameters, logger, vFormat) {
|
||||
super(parameters, logger);
|
||||
// original listing parameters
|
||||
this.delimiter = parameters.delimiter;
|
||||
this.prefix = parameters.prefix;
|
||||
this.maxKeys = parameters.maxKeys || 1000;
|
||||
// specific to version listing
|
||||
this.keyMarker = parameters.keyMarker;
|
||||
this.versionIdMarker = parameters.versionIdMarker;
|
||||
// internal state
|
||||
this.masterKey = undefined;
|
||||
this.masterVersionId = undefined;
|
||||
this.nullKey = null;
|
||||
this.vFormat = vFormat || BucketVersioningKeyFormat.v0;
|
||||
// listing results
|
||||
this.CommonPrefixes = [];
|
||||
this.Versions = [];
|
||||
this.IsTruncated = false;
|
||||
this.nextKeyMarker = parameters.keyMarker;
|
||||
this.nextVersionIdMarker = undefined;
|
||||
|
||||
this.keyHandlers = {};
|
||||
|
||||
Object.assign(this, {
|
||||
[BucketVersioningKeyFormat.v0]: {
|
||||
genMDParams: this.genMDParamsV0,
|
||||
getObjectKey: this.getObjectKeyV0,
|
||||
skipping: this.skippingV0,
|
||||
},
|
||||
[BucketVersioningKeyFormat.v1]: {
|
||||
genMDParams: this.genMDParamsV1,
|
||||
getObjectKey: this.getObjectKeyV1,
|
||||
skipping: this.skippingV1,
|
||||
},
|
||||
}[this.vFormat]);
|
||||
|
||||
if (this.vFormat === BucketVersioningKeyFormat.v0) {
|
||||
this.setKeyHandler(
|
||||
DelimiterVersionsFilterStateId.NotSkipping,
|
||||
this.keyHandler_NotSkippingV0.bind(this));
|
||||
} else {
|
||||
this.setKeyHandler(
|
||||
DelimiterVersionsFilterStateId.NotSkipping,
|
||||
this.keyHandler_NotSkippingV1.bind(this));
|
||||
}
|
||||
this.setKeyHandler(
|
||||
DelimiterVersionsFilterStateId.SkippingPrefix,
|
||||
this.keyHandler_SkippingPrefix.bind(this));
|
||||
|
||||
this.setKeyHandler(
|
||||
DelimiterVersionsFilterStateId.SkippingVersions,
|
||||
this.keyHandler_SkippingVersions.bind(this));
|
||||
|
||||
if (this.versionIdMarker) {
|
||||
this.state = <DelimiterVersionsFilterState_SkippingVersions> {
|
||||
id: DelimiterVersionsFilterStateId.SkippingVersions,
|
||||
gt: `${this.keyMarker}${VID_SEP}${this.versionIdMarker}`,
|
||||
};
|
||||
} else {
|
||||
this.state = <DelimiterVersionsFilterState_NotSkipping> {
|
||||
id: DelimiterVersionsFilterStateId.NotSkipping,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
genMDParamsV0() {
|
||||
const params: GenMDParamsItem = {};
|
||||
if (this.prefix) {
|
||||
params.gte = this.prefix;
|
||||
params.lt = inc(this.prefix);
|
||||
}
|
||||
if (this.keyMarker && this.delimiter) {
|
||||
const commonPrefix = this.getCommonPrefix(this.keyMarker);
|
||||
if (commonPrefix) {
|
||||
const afterPrefix = inc(commonPrefix);
|
||||
if (!params.gte || afterPrefix > params.gte) {
|
||||
params.gte = afterPrefix;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (this.keyMarker && (!params.gte || this.keyMarker >= params.gte)) {
|
||||
delete params.gte;
|
||||
if (this.versionIdMarker) {
|
||||
// start from the beginning of versions so we can
|
||||
// check if there's a null key and fetch it
|
||||
// (afterwards, we can skip the rest of versions until
|
||||
// we reach versionIdMarker)
|
||||
params.gte = `${this.keyMarker}${VID_SEP}`;
|
||||
} else {
|
||||
params.gt = `${this.keyMarker}${inc(VID_SEP)}`;
|
||||
}
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
genMDParamsV1() {
|
||||
// return an array of two listing params sets to ask for
|
||||
// synchronized listing of M and V ranges
|
||||
const v0Params: GenMDParamsItem = this.genMDParamsV0();
|
||||
const mParams: GenMDParamsItem = {};
|
||||
const vParams: GenMDParamsItem = {};
|
||||
if (v0Params.gt) {
|
||||
mParams.gt = `${DbPrefixes.Master}${v0Params.gt}`;
|
||||
vParams.gt = `${DbPrefixes.Version}${v0Params.gt}`;
|
||||
} else if (v0Params.gte) {
|
||||
mParams.gte = `${DbPrefixes.Master}${v0Params.gte}`;
|
||||
vParams.gte = `${DbPrefixes.Version}${v0Params.gte}`;
|
||||
} else {
|
||||
mParams.gte = DbPrefixes.Master;
|
||||
vParams.gte = DbPrefixes.Version;
|
||||
}
|
||||
if (v0Params.lt) {
|
||||
mParams.lt = `${DbPrefixes.Master}${v0Params.lt}`;
|
||||
vParams.lt = `${DbPrefixes.Version}${v0Params.lt}`;
|
||||
} else {
|
||||
mParams.lt = inc(DbPrefixes.Master);
|
||||
vParams.lt = inc(DbPrefixes.Version);
|
||||
}
|
||||
return [mParams, vParams];
|
||||
}
|
||||
|
||||
/**
|
||||
* check if the max keys count has been reached and set the
|
||||
* final state of the result if it is the case
|
||||
* @return {Boolean} - indicates if the iteration has to stop
|
||||
*/
|
||||
_reachedMaxKeys(): boolean {
|
||||
if (this.keys >= this.maxKeys) {
|
||||
// In cases of maxKeys <= 0 -> IsTruncated = false
|
||||
this.IsTruncated = this.maxKeys > 0;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Used to synchronize listing of M and V prefixes by object key
|
||||
*
|
||||
* @param {object} masterObj object listed from first range
|
||||
* returned by genMDParamsV1() (the master keys range)
|
||||
* @param {object} versionObj object listed from second range
|
||||
* returned by genMDParamsV1() (the version keys range)
|
||||
* @return {number} comparison result:
|
||||
* * -1 if master key < version key
|
||||
* * 1 if master key > version key
|
||||
*/
|
||||
compareObjects(masterObj, versionObj) {
|
||||
const masterKey = masterObj.key.slice(DbPrefixes.Master.length);
|
||||
const versionKey = versionObj.key.slice(DbPrefixes.Version.length);
|
||||
return masterKey < versionKey ? -1 : 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a listing key into its nonversioned key and version ID components
|
||||
*
|
||||
* @param {string} key - full listing key
|
||||
* @return {object} obj
|
||||
* @return {string} obj.key - nonversioned part of key
|
||||
* @return {string} [obj.versionId] - version ID in the key
|
||||
*/
|
||||
parseKey(fullKey: string): { key: string, versionId ?: string } {
|
||||
const versionIdIndex = fullKey.indexOf(VID_SEP);
|
||||
if (versionIdIndex === -1) {
|
||||
return { key: fullKey };
|
||||
}
|
||||
const nonversionedKey: string = fullKey.slice(0, versionIdIndex);
|
||||
let versionId: string = fullKey.slice(versionIdIndex + 1);
|
||||
return { key: nonversionedKey, versionId };
|
||||
}
|
||||
|
||||
/**
|
||||
* Include a key in the listing output, in the Versions or CommonPrefix result
|
||||
*
|
||||
* @param {string} key - key (without version ID)
|
||||
* @param {string} versionId - version ID
|
||||
* @param {string} value - metadata value
|
||||
* @return {undefined}
|
||||
*/
|
||||
addKey(key: string, versionId: string, value: string) {
|
||||
// add the subprefix to the common prefixes if the key has the delimiter
|
||||
const commonPrefix = this.getCommonPrefix(key);
|
||||
if (commonPrefix) {
|
||||
this.addCommonPrefix(commonPrefix);
|
||||
// transition into SkippingPrefix state to skip all following keys
|
||||
// while they start with the same prefix
|
||||
this.setState(<DelimiterVersionsFilterState_SkippingPrefix> {
|
||||
id: DelimiterVersionsFilterStateId.SkippingPrefix,
|
||||
prefix: commonPrefix,
|
||||
});
|
||||
} else {
|
||||
this.addVersion(key, versionId, value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a (key, versionId, value) tuple to the listing.
|
||||
* Set the NextMarker to the current key
|
||||
* Increment the keys counter
|
||||
* @param {String} key - The key to add
|
||||
* @param {String} versionId - versionId
|
||||
* @param {String} value - The value of the key
|
||||
* @return {undefined}
|
||||
*/
|
||||
addVersion(key: string, versionId: string, value: string) {
|
||||
this.Versions.push({
|
||||
key,
|
||||
versionId,
|
||||
value: this.trimMetadata(value),
|
||||
});
|
||||
this.nextKeyMarker = key;
|
||||
this.nextVersionIdMarker = versionId;
|
||||
++this.keys;
|
||||
}
|
||||
|
||||
getCommonPrefix(key: string): string | undefined {
|
||||
if (!this.delimiter) {
|
||||
return undefined;
|
||||
}
|
||||
const baseIndex = this.prefix ? this.prefix.length : 0;
|
||||
const delimiterIndex = key.indexOf(this.delimiter, baseIndex);
|
||||
if (delimiterIndex === -1) {
|
||||
return undefined;
|
||||
}
|
||||
return key.substring(0, delimiterIndex + this.delimiter.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a Common Prefix in the list
|
||||
* @param {String} commonPrefix - common prefix to add
|
||||
* @return {undefined}
|
||||
*/
|
||||
addCommonPrefix(commonPrefix: string): void {
|
||||
// add the new prefix to the list
|
||||
this.CommonPrefixes.push(commonPrefix);
|
||||
++this.keys;
|
||||
this.nextKeyMarker = commonPrefix;
|
||||
this.nextVersionIdMarker = undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cache the current null key, to save it for outputting it later at
|
||||
* the correct position
|
||||
*
|
||||
* @param {String} key - nonversioned key of the null key
|
||||
* @param {String} versionId - real version ID of the null key
|
||||
* @param {String} value - value of the null key
|
||||
* @return {undefined}
|
||||
*/
|
||||
cacheNullKey(key: string, versionId: string, value: string): void {
|
||||
this.nullKey = { key, versionId, value };
|
||||
}
|
||||
|
||||
getObjectKeyV0(obj: { key: string }): string {
|
||||
return obj.key;
|
||||
}
|
||||
|
||||
getObjectKeyV1(obj: { key: string }): string {
|
||||
return obj.key.slice(DbPrefixes.Master.length);
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter to apply on each iteration, based on:
|
||||
* - prefix
|
||||
* - delimiter
|
||||
* - maxKeys
|
||||
* The marker is being handled directly by levelDB
|
||||
* @param {Object} obj - The key and value of the element
|
||||
* @param {String} obj.key - The key of the element
|
||||
* @param {String} obj.value - The value of the element
|
||||
* @return {number} - indicates if iteration should continue
|
||||
*/
|
||||
filter(obj: { key: string, value: string }): FilterReturnValue {
|
||||
const key = this.getObjectKey(obj);
|
||||
const value = obj.value;
|
||||
|
||||
const { key: nonversionedKey, versionId: keyVersionId } = this.parseKey(key);
|
||||
if (this.nullKey) {
|
||||
if (this.nullKey.key !== nonversionedKey
|
||||
|| this.nullKey.versionId < <string> keyVersionId) {
|
||||
this.handleKey(
|
||||
this.nullKey.key, this.nullKey.versionId, this.nullKey.value);
|
||||
this.nullKey = null;
|
||||
}
|
||||
}
|
||||
if (keyVersionId === '') {
|
||||
// null key
|
||||
this.cacheNullKey(nonversionedKey, Version.from(value).getVersionId(), value);
|
||||
if (this.state.id === DelimiterVersionsFilterStateId.SkippingVersions) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
return this.handleKey(nonversionedKey, keyVersionId, value);
|
||||
}
|
||||
|
||||
setState(state: FilterState): void {
|
||||
this.state = state;
|
||||
}
|
||||
|
||||
setKeyHandler(stateId: number, keyHandler: KeyHandler): void {
|
||||
this.keyHandlers[stateId] = keyHandler;
|
||||
}
|
||||
|
||||
handleKey(key: string, versionId: string | undefined, value: string): FilterReturnValue {
|
||||
return this.keyHandlers[this.state.id](key, versionId, value);
|
||||
}
|
||||
|
||||
keyHandler_NotSkippingV0(key: string, versionId: string | undefined, value: string): FilterReturnValue {
|
||||
if (key.startsWith(DbPrefixes.Replay)) {
|
||||
// skip internal replay prefix entirely
|
||||
this.setState(<DelimiterVersionsFilterState_SkippingPrefix> {
|
||||
id: DelimiterVersionsFilterStateId.SkippingPrefix,
|
||||
prefix: DbPrefixes.Replay,
|
||||
});
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
if (Version.isPHD(value)) {
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
return this.filter_onNewKey(key, versionId, value);
|
||||
}
|
||||
|
||||
keyHandler_NotSkippingV1(key: string, versionId: string | undefined, value: string): FilterReturnValue {
|
||||
// NOTE: this check on PHD is only useful for Artesca, S3C
|
||||
// does not use PHDs in V1 format
|
||||
if (Version.isPHD(value)) {
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
return this.filter_onNewKey(key, versionId, value);
|
||||
}
|
||||
|
||||
filter_onNewKey(key: string, versionId: string | undefined, value: string): FilterReturnValue {
|
||||
if (this._reachedMaxKeys()) {
|
||||
return FILTER_END;
|
||||
}
|
||||
if (versionId === undefined) {
|
||||
this.masterKey = key;
|
||||
this.masterVersionId = Version.from(value).getVersionId() || 'null';
|
||||
this.addKey(this.masterKey, this.masterVersionId, value);
|
||||
} else {
|
||||
if (this.masterKey === key && this.masterVersionId === versionId) {
|
||||
// do not add a version key if it is the master version
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
this.addKey(key, versionId, value);
|
||||
}
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
|
||||
keyHandler_SkippingPrefix(key: string, versionId: string | undefined, value: string): FilterReturnValue {
|
||||
const { prefix } = <DelimiterVersionsFilterState_SkippingPrefix> this.state;
|
||||
if (key.startsWith(prefix)) {
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
this.setState(<DelimiterVersionsFilterState_NotSkipping> {
|
||||
id: DelimiterVersionsFilterStateId.NotSkipping,
|
||||
});
|
||||
return this.handleKey(key, versionId, value);
|
||||
}
|
||||
|
||||
keyHandler_SkippingVersions(key: string, versionId: string | undefined, value: string): FilterReturnValue {
|
||||
if (key === this.keyMarker) {
|
||||
// since the nonversioned key equals the marker, there is
|
||||
// necessarily a versionId in this key
|
||||
const _versionId = <string> versionId;
|
||||
if (_versionId < this.versionIdMarker) {
|
||||
// skip all versions until marker
|
||||
return FILTER_SKIP;
|
||||
}
|
||||
if (_versionId === this.versionIdMarker) {
|
||||
// nothing left to skip, so return ACCEPT, but don't add this version
|
||||
return FILTER_ACCEPT;
|
||||
}
|
||||
}
|
||||
this.setState(<DelimiterVersionsFilterState_NotSkipping> {
|
||||
id: DelimiterVersionsFilterStateId.NotSkipping,
|
||||
});
|
||||
return this.handleKey(key, versionId, value);
|
||||
}
|
||||
|
||||
skippingBase(): string | undefined {
|
||||
switch (this.state.id) {
|
||||
case DelimiterVersionsFilterStateId.SkippingPrefix:
|
||||
const { prefix } = <DelimiterVersionsFilterState_SkippingPrefix> this.state;
|
||||
return inc(prefix);
|
||||
|
||||
case DelimiterVersionsFilterStateId.SkippingVersions:
|
||||
const { gt } = <DelimiterVersionsFilterState_SkippingVersions> this.state;
|
||||
// the contract of skipping() is to return the first key
|
||||
// that can be skipped to, so adding a null byte to skip
|
||||
// over the existing versioned key set in 'gt'
|
||||
return `${gt}\0`;
|
||||
|
||||
default:
|
||||
return SKIP_NONE;
|
||||
}
|
||||
}
|
||||
|
||||
skippingV0() {
|
||||
return this.skippingBase();
|
||||
}
|
||||
|
||||
skippingV1() {
|
||||
const skipTo = this.skippingBase();
|
||||
if (skipTo === SKIP_NONE) {
|
||||
return SKIP_NONE;
|
||||
}
|
||||
// skip to the same object key in both M and V range listings
|
||||
return [
|
||||
`${DbPrefixes.Master}${skipTo}`,
|
||||
`${DbPrefixes.Version}${skipTo}`,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Return an object containing all mandatory fields to use once the
|
||||
* iteration is done, doesn't show a NextMarker field if the output
|
||||
* isn't truncated
|
||||
* @return {Object} - following amazon format
|
||||
*/
|
||||
result() {
|
||||
// Add the last null key if still in cache (when it is the
|
||||
// last version of the last key)
|
||||
//
|
||||
// NOTE: _reachedMaxKeys sets IsTruncated to true when it
|
||||
// returns true. Here we want this because either:
|
||||
//
|
||||
// - we did not reach the max keys yet so the result is not
|
||||
// - truncated, and there is still room for the null key in
|
||||
// - the results
|
||||
//
|
||||
// - OR we reached it already while having to process a new
|
||||
// key (so the result is truncated even without the null key)
|
||||
//
|
||||
// - OR we are *just* below the limit but the null key to add
|
||||
// does not fit, so we know the result is now truncated
|
||||
// because there remains the null key to be output.
|
||||
//
|
||||
if (this.nullKey) {
|
||||
this.handleKey(this.nullKey.key, this.nullKey.versionId, this.nullKey.value);
|
||||
}
|
||||
const result: ResultObject = {
|
||||
CommonPrefixes: this.CommonPrefixes,
|
||||
Versions: this.Versions,
|
||||
IsTruncated: this.IsTruncated,
|
||||
};
|
||||
if (this.delimiter) {
|
||||
result.Delimiter = this.delimiter;
|
||||
}
|
||||
if (this.IsTruncated) {
|
||||
result.NextKeyMarker = this.nextKeyMarker;
|
||||
if (this.nextVersionIdMarker) {
|
||||
result.NextVersionIdMarker = this.nextVersionIdMarker;
|
||||
}
|
||||
};
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { DelimiterVersions };
|
|
@ -6,4 +6,7 @@ module.exports = {
|
|||
DelimiterMaster: require('./delimiterMaster')
|
||||
.DelimiterMaster,
|
||||
MPU: require('./MPU').MultipartUploads,
|
||||
DelimiterCurrent: require('./delimiterCurrent').DelimiterCurrent,
|
||||
DelimiterNonCurrent: require('./delimiterNonCurrent').DelimiterNonCurrent,
|
||||
DelimiterOrphanDeleteMarker: require('./delimiterOrphanDeleteMarker').DelimiterOrphanDeleteMarker,
|
||||
};
|
||||
|
|
|
@ -52,21 +52,21 @@ class Skip {
|
|||
assert(this.skipRangeCb);
|
||||
|
||||
const filteringResult = this.extension.filter(entry);
|
||||
const skippingRange = this.extension.skipping();
|
||||
const skipTo = this.extension.skipping();
|
||||
|
||||
if (filteringResult === FILTER_END) {
|
||||
this.listingEndCb();
|
||||
} else if (filteringResult === FILTER_SKIP
|
||||
&& skippingRange !== SKIP_NONE) {
|
||||
&& skipTo !== SKIP_NONE) {
|
||||
if (++this.streakLength >= MAX_STREAK_LENGTH) {
|
||||
let newRange;
|
||||
if (Array.isArray(skippingRange)) {
|
||||
if (Array.isArray(skipTo)) {
|
||||
newRange = [];
|
||||
for (let i = 0; i < skippingRange.length; ++i) {
|
||||
newRange.push(this._inc(skippingRange[i]));
|
||||
for (let i = 0; i < skipTo.length; ++i) {
|
||||
newRange.push(skipTo[i]);
|
||||
}
|
||||
} else {
|
||||
newRange = this._inc(skippingRange);
|
||||
newRange = skipTo;
|
||||
}
|
||||
/* Avoid to loop on the same range again and again. */
|
||||
if (newRange === this.gteParams) {
|
||||
|
@ -79,16 +79,6 @@ class Skip {
|
|||
this.streakLength = 0;
|
||||
}
|
||||
}
|
||||
|
||||
_inc(str) {
|
||||
if (!str) {
|
||||
return str;
|
||||
}
|
||||
const lastCharValue = str.charCodeAt(str.length - 1);
|
||||
const lastCharNewValue = String.fromCharCode(lastCharValue + 1);
|
||||
|
||||
return `${str.slice(0, str.length - 1)}${lastCharNewValue}`;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -298,7 +298,7 @@ export default class Vault {
|
|||
callback: (err: Error | null, data?: { [key: string]: string }) => void
|
||||
) {
|
||||
log.trace('getting accountIds from Vault based on canonicalIDs',
|
||||
{ canonicalIDs });
|
||||
{ canonicalIDs });
|
||||
this.client.getAccountIds(canonicalIDs,
|
||||
// @ts-expect-error
|
||||
{ reqUid: log.getSerializedUids() },
|
||||
|
@ -313,14 +313,14 @@ export default class Vault {
|
|||
const result = {};
|
||||
/* If the accountId was not found in Vault, do not
|
||||
send the canonicalID back to the API */
|
||||
Object.keys(infoFromVault).forEach(key => {
|
||||
if (infoFromVault[key] !== 'NotFound' &&
|
||||
Object.keys(infoFromVault).forEach(key => {
|
||||
if (infoFromVault[key] !== 'NotFound' &&
|
||||
infoFromVault[key] !== 'WrongFormat') {
|
||||
result[key] = infoFromVault[key];
|
||||
}
|
||||
result[key] = infoFromVault[key];
|
||||
}
|
||||
});
|
||||
return callback(null, result);
|
||||
});
|
||||
return callback(null, result);
|
||||
});
|
||||
}
|
||||
|
||||
/** checkPolicies -- call Vault to evaluate policies
|
||||
|
|
|
@ -9,10 +9,12 @@ import * as constants from '../constants';
|
|||
import constructStringToSignV2 from './v2/constructStringToSign';
|
||||
import constructStringToSignV4 from './v4/constructStringToSign';
|
||||
import { convertUTCtoISO8601 } from './v4/timeUtils';
|
||||
import * as vaultUtilities from './in_memory/vaultUtilities';
|
||||
import * as backend from './in_memory/Backend';
|
||||
import validateAuthConfig from './in_memory/validateAuthConfig';
|
||||
import AuthLoader from './in_memory/AuthLoader';
|
||||
import * as vaultUtilities from './backends/in_memory/vaultUtilities';
|
||||
import * as inMemoryBackend from './backends/in_memory/Backend';
|
||||
import baseBackend from './backends/base';
|
||||
import chainBackend from './backends/ChainBackend';
|
||||
import validateAuthConfig from './backends/in_memory/validateAuthConfig';
|
||||
import AuthLoader from './backends/in_memory/AuthLoader';
|
||||
import Vault from './Vault';
|
||||
|
||||
let vault: Vault | null = null;
|
||||
|
@ -76,7 +78,7 @@ function extractParams(
|
|||
version = 'v4';
|
||||
} else {
|
||||
log.trace('invalid authorization security header',
|
||||
{ header: authHeader });
|
||||
{ header: authHeader });
|
||||
return { err: errors.AccessDenied };
|
||||
}
|
||||
} else if (data.Signature) {
|
||||
|
@ -91,7 +93,7 @@ function extractParams(
|
|||
if (version !== null && method !== null) {
|
||||
if (!checkFunctions[version] || !checkFunctions[version][method]) {
|
||||
log.trace('invalid auth version or method',
|
||||
{ version, authMethod: method });
|
||||
{ version, authMethod: method });
|
||||
return { err: errors.NotImplemented };
|
||||
}
|
||||
log.trace('identified auth method', { version, authMethod: method });
|
||||
|
@ -233,16 +235,16 @@ function generateV4Headers(
|
|||
headerName.startsWith('x-amz-')
|
||||
|| headerName.startsWith('x-scal-')
|
||||
|| headerName === 'content-md5'
|
||||
|| headerName === 'host'
|
||||
|| headerName === 'host',
|
||||
).sort().join(';');
|
||||
const params = { request, signedHeaders, payloadChecksum,
|
||||
credentialScope, timestamp, query: data,
|
||||
awsService: service, proxyPath };
|
||||
const stringToSign = constructStringToSignV4(params);
|
||||
const signingKey = vaultUtilities.calculateSigningKey(secretKeyValue,
|
||||
region,
|
||||
scopeDate,
|
||||
service);
|
||||
region,
|
||||
scopeDate,
|
||||
service);
|
||||
const signature = crypto.createHmac('sha256', signingKey)
|
||||
.update(stringToSign as string, 'binary').digest('hex');
|
||||
const authorizationHeader = `${algorithm} Credential=${accessKey}` +
|
||||
|
@ -254,7 +256,8 @@ function generateV4Headers(
|
|||
|
||||
export const server = { extractParams, doAuth }
|
||||
export const client = { generateV4Headers, constructStringToSignV2 }
|
||||
export const inMemory = { backend, validateAuthConfig, AuthLoader }
|
||||
export const inMemory = { backend: inMemoryBackend, validateAuthConfig, AuthLoader }
|
||||
export const backends = { baseBackend, chainBackend }
|
||||
export {
|
||||
setAuthHandler as setHandler,
|
||||
AuthInfo,
|
||||
|
|
|
@ -0,0 +1,215 @@
|
|||
import assert from 'assert';
|
||||
import async from 'async';
|
||||
import errors from '../../errors';
|
||||
import BaseBackend from './base';
|
||||
|
||||
/**
|
||||
* Class that provides an authentication backend that will verify signatures
|
||||
* and retrieve emails and canonical ids associated with an account using a
|
||||
* given list of authentication backends and vault clients.
|
||||
*
|
||||
* @class ChainBackend
|
||||
*/
|
||||
export default class ChainBackend extends BaseBackend {
|
||||
_clients: any[];
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
* @param {string} service - service id
|
||||
* @param {object[]} clients - list of authentication backends or vault clients
|
||||
*/
|
||||
constructor(service: string, clients: any[]) {
|
||||
super(service);
|
||||
|
||||
assert(Array.isArray(clients) && clients.length > 0, 'invalid client list');
|
||||
assert(clients.every(client =>
|
||||
typeof client.verifySignatureV4 === 'function' &&
|
||||
typeof client.verifySignatureV2 === 'function' &&
|
||||
typeof client.getCanonicalIds === 'function' &&
|
||||
typeof client.getEmailAddresses === 'function' &&
|
||||
typeof client.checkPolicies === 'function' &&
|
||||
typeof client.healthcheck === 'function',
|
||||
), 'invalid client: missing required auth backend methods');
|
||||
this._clients = clients;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* try task against each client for one to be successful
|
||||
*/
|
||||
_tryEachClient(task: any, cb: any) {
|
||||
// @ts-ignore
|
||||
async.tryEach(this._clients.map(client => done => task(client, done)), cb);
|
||||
}
|
||||
|
||||
/*
|
||||
* apply task to all clients
|
||||
*/
|
||||
_forEachClient(task: any, cb: any) {
|
||||
async.map(this._clients, task, cb);
|
||||
}
|
||||
|
||||
verifySignatureV2(
|
||||
stringToSign: string,
|
||||
signatureFromRequest: string,
|
||||
accessKey: string,
|
||||
options: any,
|
||||
callback: any,
|
||||
) {
|
||||
this._tryEachClient((client, done) => client.verifySignatureV2(
|
||||
stringToSign,
|
||||
signatureFromRequest,
|
||||
accessKey,
|
||||
options,
|
||||
done,
|
||||
), callback);
|
||||
}
|
||||
|
||||
verifySignatureV4(
|
||||
stringToSign: string,
|
||||
signatureFromRequest: string,
|
||||
accessKey: string,
|
||||
region: string,
|
||||
scopeDate: string,
|
||||
options: any,
|
||||
callback: any,
|
||||
) {
|
||||
this._tryEachClient((client, done) => client.verifySignatureV4(
|
||||
stringToSign,
|
||||
signatureFromRequest,
|
||||
accessKey,
|
||||
region,
|
||||
scopeDate,
|
||||
options,
|
||||
done,
|
||||
), callback);
|
||||
}
|
||||
|
||||
static _mergeObjects(objectResponses: any) {
|
||||
return objectResponses.reduce(
|
||||
(retObj, resObj) => Object.assign(retObj, resObj.message.body),
|
||||
{});
|
||||
}
|
||||
|
||||
getCanonicalIds(emailAddresses: string[], options: any, callback: any) {
|
||||
this._forEachClient(
|
||||
(client, done) => client.getCanonicalIds(emailAddresses, options, done),
|
||||
(err, res) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
// TODO: atm naive merge, better handling of conflicting email results
|
||||
return callback(null, {
|
||||
message: {
|
||||
body: ChainBackend._mergeObjects(res),
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
getEmailAddresses(canonicalIDs: string[], options: any, callback: any) {
|
||||
this._forEachClient(
|
||||
(client, done) => client.getEmailAddresses(canonicalIDs, options, done),
|
||||
(err, res) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
return callback(null, {
|
||||
message: {
|
||||
body: ChainBackend._mergeObjects(res),
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/*
|
||||
* merge policy responses into a single message
|
||||
*/
|
||||
static _mergePolicies(policyResponses: any) {
|
||||
const policyMap: any = {};
|
||||
|
||||
policyResponses.forEach(resp => {
|
||||
if (!resp.message || !Array.isArray(resp.message.body)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const check = (policy) => {
|
||||
const key = (policy.arn || '') + (policy.versionId || '') + (policy.action || '');
|
||||
if (!policyMap[key] || !policyMap[key].isAllowed) {
|
||||
policyMap[key] = policy;
|
||||
}
|
||||
// else is duplicate policy
|
||||
};
|
||||
|
||||
resp.message.body.forEach(policy => {
|
||||
if (Array.isArray(policy)) {
|
||||
policy.forEach(authResult => check(authResult));
|
||||
} else {
|
||||
check(policy);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
return Object.keys(policyMap).map(key => {
|
||||
const policyRes: any = { isAllowed: policyMap[key].isAllowed };
|
||||
if (policyMap[key].arn !== '') {
|
||||
policyRes.arn = policyMap[key].arn;
|
||||
}
|
||||
if (policyMap[key].versionId) {
|
||||
policyRes.versionId = policyMap[key].versionId;
|
||||
}
|
||||
if (policyMap[key].isImplicit !== undefined) {
|
||||
policyRes.isImplicit = policyMap[key].isImplicit;
|
||||
}
|
||||
if (policyMap[key].action) {
|
||||
policyRes.action = policyMap[key].action;
|
||||
}
|
||||
return policyRes;
|
||||
});
|
||||
}
|
||||
|
||||
/*
|
||||
response format:
|
||||
{ message: {
|
||||
body: [{}],
|
||||
code: number,
|
||||
message: string,
|
||||
} }
|
||||
*/
|
||||
checkPolicies(requestContextParams: any, userArn: string, options: any, callback: any) {
|
||||
this._forEachClient((client, done) => client.checkPolicies(
|
||||
requestContextParams,
|
||||
userArn,
|
||||
options,
|
||||
done,
|
||||
), (err, res) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
return callback(null, {
|
||||
message: {
|
||||
body: ChainBackend._mergePolicies(res),
|
||||
},
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
healthcheck(reqUid: string, callback: any) {
|
||||
this._forEachClient((client, done) =>
|
||||
client.healthcheck(reqUid, (err, res) => done(null, {
|
||||
error: !!err ? err : null,
|
||||
status: res,
|
||||
}),
|
||||
), (err, res) => {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
const isError = res.some(results => !!results.error);
|
||||
if (isError) {
|
||||
return callback(errors.InternalError, res);
|
||||
}
|
||||
return callback(null, res);
|
||||
});
|
||||
}
|
||||
}
|
|
@ -0,0 +1,96 @@
|
|||
import errors from '../../errors';
|
||||
|
||||
/**
|
||||
* Base backend class
|
||||
*
|
||||
* @class BaseBackend
|
||||
*/
|
||||
export default class BaseBackend {
|
||||
service: string;
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
* @param {string} service - service identifer for construction arn
|
||||
*/
|
||||
constructor(service: string) {
|
||||
this.service = service;
|
||||
}
|
||||
|
||||
/** verifySignatureV2
|
||||
* @param stringToSign - string to sign built per AWS rules
|
||||
* @param signatureFromRequest - signature sent with request
|
||||
* @param accessKey - account accessKey
|
||||
* @param options - contains algorithm (SHA1 or SHA256)
|
||||
* @param callback - callback with either error or user info
|
||||
* @return calls callback
|
||||
*/
|
||||
verifySignatureV2(
|
||||
stringToSign: string,
|
||||
signatureFromRequest: string,
|
||||
accessKey: string,
|
||||
options: any,
|
||||
callback: any
|
||||
) {
|
||||
return callback(errors.AuthMethodNotImplemented);
|
||||
}
|
||||
|
||||
|
||||
/** verifySignatureV4
|
||||
* @param stringToSign - string to sign built per AWS rules
|
||||
* @param signatureFromRequest - signature sent with request
|
||||
* @param accessKey - account accessKey
|
||||
* @param region - region specified in request credential
|
||||
* @param scopeDate - date specified in request credential
|
||||
* @param options - options to send to Vault
|
||||
* (just contains reqUid for logging in Vault)
|
||||
* @param callback - callback with either error or user info
|
||||
* @return calls callback
|
||||
*/
|
||||
verifySignatureV4(
|
||||
stringToSign: string,
|
||||
signatureFromRequest: string,
|
||||
accessKey: string,
|
||||
region: string,
|
||||
scopeDate: string,
|
||||
options: any,
|
||||
callback: any
|
||||
) {
|
||||
return callback(errors.AuthMethodNotImplemented);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets canonical ID's for a list of accounts
|
||||
* based on email associated with account
|
||||
* @param emails - list of email addresses
|
||||
* @param options - to send log id to vault
|
||||
* @param callback - callback to calling function
|
||||
* @returns callback with either error or
|
||||
* object with email addresses as keys and canonical IDs
|
||||
* as values
|
||||
*/
|
||||
getCanonicalIds(emails: string[], options: any, callback: any) {
|
||||
return callback(errors.AuthMethodNotImplemented);
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets email addresses (referred to as diplay names for getACL's)
|
||||
* for a list of accounts based on canonical IDs associated with account
|
||||
* @param canonicalIDs - list of canonicalIDs
|
||||
* @param options - to send log id to vault
|
||||
* @param callback - callback to calling function
|
||||
* @returns callback with either error or
|
||||
* an object from Vault containing account canonicalID
|
||||
* as each object key and an email address as the value (or "NotFound")
|
||||
*/
|
||||
getEmailAddresses(canonicalIDs: string[], options: any, callback: any) {
|
||||
return callback(errors.AuthMethodNotImplemented);
|
||||
}
|
||||
|
||||
checkPolicies(requestContextParams: any, userArn: string, options: any, callback: any) {
|
||||
return callback(null, { message: { body: [] } });
|
||||
}
|
||||
|
||||
healthcheck(reqUid: string, callback: any) {
|
||||
return callback(null, { code: 200, message: 'OK' });
|
||||
}
|
||||
}
|
|
@ -4,7 +4,7 @@ import joi from 'joi';
|
|||
import werelogs from 'werelogs';
|
||||
import * as types from './types';
|
||||
import { Account, Accounts } from './types';
|
||||
import ARN from '../../models/ARN';
|
||||
import ARN from '../../../models/ARN';
|
||||
|
||||
/** Load authentication information from files or pre-loaded account objects */
|
||||
export default class AuthLoader {
|
|
@ -1,7 +1,9 @@
|
|||
import * as crypto from 'crypto';
|
||||
import errors from '../../errors';
|
||||
import crypto from 'crypto';
|
||||
import { Logger } from 'werelogs';
|
||||
import errors from '../../../errors';
|
||||
import { calculateSigningKey, hashSignature } from './vaultUtilities';
|
||||
import Indexer from './Indexer';
|
||||
import BaseBackend from '../base';
|
||||
import { Accounts } from './types';
|
||||
|
||||
function _formatResponse(userInfoToSend: any) {
|
||||
|
@ -15,26 +17,32 @@ function _formatResponse(userInfoToSend: any) {
|
|||
/**
|
||||
* Class that provides a memory backend for verifying signatures and getting
|
||||
* emails and canonical ids associated with an account.
|
||||
*
|
||||
* @class InMemoryBackend
|
||||
*/
|
||||
class Backend {
|
||||
class InMemoryBackend extends BaseBackend {
|
||||
indexer: Indexer;
|
||||
service: string;
|
||||
formatResponse: any;
|
||||
|
||||
constructor(service: string, indexer: Indexer) {
|
||||
this.service = service;
|
||||
/**
|
||||
* @constructor
|
||||
* @param service - service identifer for construction arn
|
||||
* @param indexer - indexer instance for retrieving account info
|
||||
* @param formatter - function which accepts user info to send
|
||||
* back and returns it in an object
|
||||
*/
|
||||
constructor(service: string, indexer: Indexer, formatter: typeof _formatResponse) {
|
||||
super(service);
|
||||
this.indexer = indexer;
|
||||
this.formatResponse = formatter;
|
||||
}
|
||||
|
||||
// CODEQUALITY-TODO-SYNC Should be synchronous
|
||||
verifySignatureV2(
|
||||
stringToSign: string,
|
||||
signatureFromRequest: string,
|
||||
accessKey: string,
|
||||
options: { algo: 'SHA256' | 'SHA1' },
|
||||
callback: (
|
||||
error: Error | null,
|
||||
data?: ReturnType<typeof _formatResponse>
|
||||
) => void
|
||||
options: any,
|
||||
callback: any,
|
||||
) {
|
||||
const entity = this.indexer.getEntityByKey(accessKey);
|
||||
if (!entity) {
|
||||
|
@ -50,27 +58,21 @@ class Backend {
|
|||
accountDisplayName: this.indexer.getAcctDisplayName(entity),
|
||||
canonicalID: entity.canonicalID,
|
||||
arn: entity.arn,
|
||||
// TODO Why?
|
||||
// @ts-ignore
|
||||
IAMdisplayName: entity.IAMdisplayName,
|
||||
};
|
||||
const vaultReturnObject = _formatResponse(userInfoToSend);
|
||||
const vaultReturnObject = this.formatResponse(userInfoToSend);
|
||||
return callback(null, vaultReturnObject);
|
||||
}
|
||||
|
||||
// TODO Options not used. Why ?
|
||||
// CODEQUALITY-TODO-SYNC Should be synchronous
|
||||
verifySignatureV4(
|
||||
stringToSign: string,
|
||||
signatureFromRequest: string,
|
||||
accessKey: string,
|
||||
region: string,
|
||||
scopeDate: string,
|
||||
_options: { algo: 'SHA256' | 'SHA1' },
|
||||
callback: (
|
||||
err: Error | null,
|
||||
data?: ReturnType<typeof _formatResponse>
|
||||
) => void
|
||||
options: any,
|
||||
callback: any,
|
||||
) {
|
||||
const entity = this.indexer.getEntityByKey(accessKey);
|
||||
if (!entity) {
|
||||
|
@ -87,21 +89,14 @@ class Backend {
|
|||
accountDisplayName: this.indexer.getAcctDisplayName(entity),
|
||||
canonicalID: entity.canonicalID,
|
||||
arn: entity.arn,
|
||||
// TODO Why?
|
||||
// @ts-ignore
|
||||
IAMdisplayName: entity.IAMdisplayName,
|
||||
};
|
||||
const vaultReturnObject = _formatResponse(userInfoToSend);
|
||||
const vaultReturnObject = this.formatResponse(userInfoToSend);
|
||||
return callback(null, vaultReturnObject);
|
||||
}
|
||||
|
||||
// TODO log not used. Why ?
|
||||
// CODEQUALITY-TODO-SYNC Should be synchronous
|
||||
getCanonicalIds(
|
||||
emails: string[],
|
||||
_log: any,
|
||||
cb: (err: null, data: { message: { body: any } }) => void
|
||||
) {
|
||||
getCanonicalIds(emails: string[], log: Logger, cb: any) {
|
||||
const results = {};
|
||||
emails.forEach(email => {
|
||||
const lowercasedEmail = email.toLowerCase();
|
||||
|
@ -121,13 +116,7 @@ class Backend {
|
|||
return cb(null, vaultReturnObject);
|
||||
}
|
||||
|
||||
// TODO options not used. Why ?
|
||||
// CODEQUALITY-TODO-SYNC Should be synchronous
|
||||
getEmailAddresses(
|
||||
canonicalIDs: string[],
|
||||
_options: any,
|
||||
cb: (err: null, data: { message: { body: any } }) => void
|
||||
) {
|
||||
getEmailAddresses(canonicalIDs: string[], options: any, cb: any) {
|
||||
const results = {};
|
||||
canonicalIDs.forEach(canonicalId => {
|
||||
const foundEntity = this.indexer.getEntityByCanId(canonicalId);
|
||||
|
@ -145,24 +134,17 @@ class Backend {
|
|||
return cb(null, vaultReturnObject);
|
||||
}
|
||||
|
||||
// TODO options not used. Why ?
|
||||
// CODEQUALITY-TODO-SYNC Should be synchronous
|
||||
/**
|
||||
* Gets accountIds for a list of accounts based on
|
||||
* the canonical IDs associated with the account
|
||||
* @param canonicalIDs - list of canonicalIDs
|
||||
* @param _options - to send log id to vault
|
||||
* @param options - to send log id to vault
|
||||
* @param cb - callback to calling function
|
||||
* @returns The next is wrong. Here to keep archives.
|
||||
* callback with either error or
|
||||
* @returns callback with either error or
|
||||
* an object from Vault containing account canonicalID
|
||||
* as each object key and an accountId as the value (or "NotFound")
|
||||
*/
|
||||
getAccountIds(
|
||||
canonicalIDs: string[],
|
||||
_options: any,
|
||||
cb: (err: null, data: { message: { body: any } }) => void
|
||||
) {
|
||||
getAccountIds(canonicalIDs: string[], options: any, cb: any) {
|
||||
const results = {};
|
||||
canonicalIDs.forEach(canonicalID => {
|
||||
const foundEntity = this.indexer.getEntityByCanId(canonicalID);
|
||||
|
@ -181,14 +163,28 @@ class Backend {
|
|||
}
|
||||
}
|
||||
|
||||
class S3AuthBackend extends Backend {
|
||||
constructor(authdata: Accounts) {
|
||||
super('s3', new Indexer(authdata));
|
||||
|
||||
class S3AuthBackend extends InMemoryBackend {
|
||||
/**
|
||||
* @constructor
|
||||
* @param authdata - the authentication config file's data
|
||||
* @param authdata.accounts - array of account objects
|
||||
* @param authdata.accounts[].name - account name
|
||||
* @param authdata.accounts[].email - account email
|
||||
* @param authdata.accounts[].arn - IAM resource name
|
||||
* @param authdata.accounts[].canonicalID - account canonical ID
|
||||
* @param authdata.accounts[].shortid - short account ID
|
||||
* @param authdata.accounts[].keys - array of key objects
|
||||
* @param authdata.accounts[].keys[].access - access key
|
||||
* @param authdata.accounts[].keys[].secret - secret key
|
||||
*/
|
||||
constructor(authdata?: Accounts) {
|
||||
super('s3', new Indexer(authdata), _formatResponse);
|
||||
}
|
||||
|
||||
refreshAuthData(authData: Accounts) {
|
||||
refreshAuthData(authData?: Accounts) {
|
||||
this.indexer = new Indexer(authData);
|
||||
}
|
||||
}
|
||||
|
||||
export { S3AuthBackend as s3 };
|
||||
export { S3AuthBackend as s3 }
|
|
@ -39,5 +39,5 @@ export default function getCanonicalizedAmzHeaders(headers: Headers, clientType:
|
|||
// Build headerString
|
||||
return amzHeaders.reduce((headerStr, current) =>
|
||||
`${headerStr}${current[0]}:${current[1]}\n`,
|
||||
'');
|
||||
'');
|
||||
}
|
||||
|
|
|
@ -21,9 +21,9 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
timestamp = Date.parse(timestamp);
|
||||
if (!timestamp) {
|
||||
log.debug('missing or invalid date header',
|
||||
{ method: 'auth/v2/headerAuthCheck.check' });
|
||||
{ method: 'auth/v2/headerAuthCheck.check' });
|
||||
return { err: errors.AccessDenied.
|
||||
customizeDescription('Authentication requires a valid Date or ' +
|
||||
customizeDescription('Authentication requires a valid Date or ' +
|
||||
'x-amz-date header') };
|
||||
}
|
||||
|
||||
|
|
|
@ -41,12 +41,12 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
|
||||
if (expirationTime > currentTime + preSignedURLExpiry) {
|
||||
log.debug('expires parameter too far in future',
|
||||
{ expires: request.query.Expires });
|
||||
{ expires: request.query.Expires });
|
||||
return { err: errors.AccessDenied };
|
||||
}
|
||||
if (currentTime > expirationTime) {
|
||||
log.debug('current time exceeds expires time',
|
||||
{ expires: request.query.Expires });
|
||||
{ expires: request.query.Expires });
|
||||
return { err: errors.RequestTimeTooSkewed };
|
||||
}
|
||||
const accessKey = data.AWSAccessKeyId;
|
||||
|
|
|
@ -42,37 +42,40 @@ export default function awsURIencode(
|
|||
if (typeof input !== 'string') {
|
||||
return '';
|
||||
}
|
||||
|
||||
// precalc slash and star based on configs
|
||||
let encoded = "";
|
||||
const slash = encodeSlash === undefined || encodeSlash ? '%2F' : '/';
|
||||
const star = noEncodeStar !== undefined && noEncodeStar ? '*' : '%2A';
|
||||
const encoded: string[] = [];
|
||||
|
||||
const charArray = Array.from(input);
|
||||
for (const ch of charArray) {
|
||||
switch (true) {
|
||||
case ch >= 'A' && ch <= 'Z':
|
||||
case ch >= 'a' && ch <= 'z':
|
||||
case ch >= '0' && ch <= '9':
|
||||
case ch === '-':
|
||||
case ch === '_':
|
||||
case ch === '~':
|
||||
case ch === '.':
|
||||
encoded.push(ch);
|
||||
break;
|
||||
case ch === '/':
|
||||
encoded.push(slash);
|
||||
break;
|
||||
case ch === '*':
|
||||
encoded.push(star);
|
||||
break;
|
||||
case ch === ' ':
|
||||
encoded.push('%20');
|
||||
break;
|
||||
default:
|
||||
encoded.push(_toHexUTF8(ch));
|
||||
break;
|
||||
for (let i = 0; i < input.length; i++) {
|
||||
let ch = input.charAt(i);
|
||||
if ((ch >= 'A' && ch <= 'Z') ||
|
||||
(ch >= 'a' && ch <= 'z') ||
|
||||
(ch >= '0' && ch <= '9') ||
|
||||
ch === '_' || ch === '-' ||
|
||||
ch === '~' || ch === '.') {
|
||||
encoded = encoded.concat(ch);
|
||||
} else if (ch === ' ') {
|
||||
encoded = encoded.concat('%20');
|
||||
} else if (ch === '/') {
|
||||
encoded = encoded.concat(slash);
|
||||
} else if (ch === '*') {
|
||||
encoded = encoded.concat(star);
|
||||
} else {
|
||||
if (ch >= '\uD800' && ch <= '\uDBFF') {
|
||||
// If this character is a high surrogate peek the next character
|
||||
// and join it with this one if the next character is a low
|
||||
// surrogate.
|
||||
// Otherwise the encoded URI will contain the two surrogates as
|
||||
// two distinct UTF-8 sequences which is not valid UTF-8.
|
||||
if (i + 1 < input.length) {
|
||||
const ch2 = input.charAt(i + 1);
|
||||
if (ch2 >= '\uDC00' && ch2 <= '\uDFFF') {
|
||||
i++;
|
||||
ch += ch2;
|
||||
}
|
||||
}
|
||||
}
|
||||
encoded = encoded.concat(_toHexUTF8(ch));
|
||||
}
|
||||
}
|
||||
return encoded.join('');
|
||||
return encoded;
|
||||
}
|
||||
|
|
|
@ -93,14 +93,14 @@ export function check(
|
|||
}
|
||||
if (!timestamp) {
|
||||
log.debug('missing or invalid date header',
|
||||
{ method: 'auth/v4/headerAuthCheck.check' });
|
||||
{ method: 'auth/v4/headerAuthCheck.check' });
|
||||
return { err: errors.AccessDenied.
|
||||
customizeDescription('Authentication requires a valid Date or ' +
|
||||
customizeDescription('Authentication requires a valid Date or ' +
|
||||
'x-amz-date header') };
|
||||
}
|
||||
|
||||
const validationResult = validateCredentials(credentialsArr, timestamp,
|
||||
log);
|
||||
log);
|
||||
if (validationResult instanceof Error) {
|
||||
log.debug('credentials in improper format', { credentialsArr,
|
||||
timestamp, validationResult });
|
||||
|
@ -132,6 +132,17 @@ export function check(
|
|||
return { err: errors.RequestTimeTooSkewed };
|
||||
}
|
||||
|
||||
let proxyPath: string | undefined;
|
||||
if (request.headers.proxy_path) {
|
||||
try {
|
||||
proxyPath = decodeURIComponent(request.headers.proxy_path);
|
||||
} catch (err) {
|
||||
log.debug('invalid proxy_path header', { proxyPath, err });
|
||||
return { err: errors.InvalidArgument.customizeDescription(
|
||||
'invalid proxy_path header') };
|
||||
}
|
||||
}
|
||||
|
||||
const stringToSign = constructStringToSign({
|
||||
log,
|
||||
request,
|
||||
|
@ -141,6 +152,7 @@ export function check(
|
|||
timestamp,
|
||||
payloadChecksum,
|
||||
awsService: service,
|
||||
proxyPath,
|
||||
});
|
||||
log.trace('constructed stringToSign', { stringToSign });
|
||||
if (stringToSign instanceof Error) {
|
||||
|
|
|
@ -39,7 +39,7 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
}
|
||||
|
||||
const validationResult = validateCredentials(credential, timestamp,
|
||||
log);
|
||||
log);
|
||||
if (validationResult instanceof Error) {
|
||||
log.debug('credentials in improper format', { credential,
|
||||
timestamp, validationResult });
|
||||
|
@ -56,6 +56,17 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
return { err: errors.RequestTimeTooSkewed };
|
||||
}
|
||||
|
||||
let proxyPath: string | undefined;
|
||||
if (request.headers.proxy_path) {
|
||||
try {
|
||||
proxyPath = decodeURIComponent(request.headers.proxy_path);
|
||||
} catch (err) {
|
||||
log.debug('invalid proxy_path header', { proxyPath });
|
||||
return { err: errors.InvalidArgument.customizeDescription(
|
||||
'invalid proxy_path header') };
|
||||
}
|
||||
}
|
||||
|
||||
// In query v4 auth, the canonical request needs
|
||||
// to include the query params OTHER THAN
|
||||
// the signature so create a
|
||||
|
@ -81,6 +92,7 @@ export function check(request: any, log: Logger, data: { [key: string]: string }
|
|||
credentialScope:
|
||||
`${scopeDate}/${region}/${service}/${requestType}`,
|
||||
awsService: service,
|
||||
proxyPath,
|
||||
});
|
||||
if (stringToSign instanceof Error) {
|
||||
return { err: stringToSign };
|
||||
|
|
|
@ -3,7 +3,7 @@ import async from 'async';
|
|||
import errors from '../../../errors';
|
||||
import { Logger } from 'werelogs';
|
||||
import Vault, { AuthV4RequestParams } from '../../Vault';
|
||||
import { Callback } from '../../in_memory/types';
|
||||
import { Callback } from '../../backends/in_memory/types';
|
||||
|
||||
import constructChunkStringToSign from './constructChunkStringToSign';
|
||||
|
||||
|
|
|
@ -27,20 +27,20 @@ export function validateCredentials(
|
|||
log.warn('accessKey provided is wrong format', { accessKey });
|
||||
return errors.InvalidArgument;
|
||||
}
|
||||
// The scope date (format YYYYMMDD) must be same date as the timestamp
|
||||
// on the request from the x-amz-date param (if queryAuthCheck)
|
||||
// or from the x-amz-date header or date header (if headerAuthCheck)
|
||||
// Format of timestamp is ISO 8601: YYYYMMDDTHHMMSSZ.
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/
|
||||
// sigv4-query-string-auth.html
|
||||
// http://docs.aws.amazon.com/general/latest/gr/
|
||||
// sigv4-date-handling.html
|
||||
// The scope date (format YYYYMMDD) must be same date as the timestamp
|
||||
// on the request from the x-amz-date param (if queryAuthCheck)
|
||||
// or from the x-amz-date header or date header (if headerAuthCheck)
|
||||
// Format of timestamp is ISO 8601: YYYYMMDDTHHMMSSZ.
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/
|
||||
// sigv4-query-string-auth.html
|
||||
// http://docs.aws.amazon.com/general/latest/gr/
|
||||
// sigv4-date-handling.html
|
||||
|
||||
// convert timestamp to format of scopeDate YYYYMMDD
|
||||
// convert timestamp to format of scopeDate YYYYMMDD
|
||||
const timestampDate = timestamp.split('T')[0];
|
||||
if (scopeDate.length !== 8 || scopeDate !== timestampDate) {
|
||||
log.warn('scope date must be the same date as the timestamp date',
|
||||
{ scopeDate, timestampDate });
|
||||
{ scopeDate, timestampDate });
|
||||
return errors.RequestTimeTooSkewed;
|
||||
}
|
||||
if (service !== 's3' && service !== 'iam' && service !== 'ring' &&
|
||||
|
@ -52,7 +52,7 @@ export function validateCredentials(
|
|||
}
|
||||
if (requestType !== 'aws4_request') {
|
||||
log.warn('requestType contained in params is not aws4_request',
|
||||
{ requestType });
|
||||
{ requestType });
|
||||
return errors.InvalidArgument;
|
||||
}
|
||||
return {};
|
||||
|
@ -79,7 +79,7 @@ export function extractQueryParams(
|
|||
// Do not need the algorithm sent back
|
||||
if (queryObj['X-Amz-Algorithm'] !== 'AWS4-HMAC-SHA256') {
|
||||
log.warn('algorithm param incorrect',
|
||||
{ algo: queryObj['X-Amz-Algorithm'] });
|
||||
{ algo: queryObj['X-Amz-Algorithm'] });
|
||||
return authParams;
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,514 @@
|
|||
import cluster, { Worker } from 'cluster';
|
||||
import * as werelogs from 'werelogs';
|
||||
|
||||
import { default as errors } from '../../lib/errors';
|
||||
|
||||
const rpcLogger = new werelogs.Logger('ClusterRPC');
|
||||
|
||||
/**
|
||||
* Remote procedure calls support between cluster workers.
|
||||
*
|
||||
* When using the cluster module, new processes are forked and are
|
||||
* dispatched workloads, usually HTTP requests. The ClusterRPC module
|
||||
* implements a RPC system to send commands to all cluster worker
|
||||
* processes at once from any particular worker, and retrieve their
|
||||
* individual command results, like a distributed map operation.
|
||||
*
|
||||
* The existing nodejs cluster IPC channel is setup from the primary
|
||||
* to each worker, but not between workers, so there has to be a hop
|
||||
* by the primary.
|
||||
*
|
||||
* How a command is treated:
|
||||
*
|
||||
* - a worker sends a command message to the primary
|
||||
*
|
||||
* - the primary then forwards that command to each existing worker
|
||||
* (including the requestor)
|
||||
*
|
||||
* - each worker then executes the command and returns a result or an
|
||||
* error
|
||||
*
|
||||
* - the primary gathers all workers results into an array
|
||||
*
|
||||
* - finally, the primary dispatches the results array to the original
|
||||
* requesting worker
|
||||
*
|
||||
*
|
||||
* Limitations:
|
||||
*
|
||||
* - The command payload must be serializable, which means that:
|
||||
* - it should not contain circular references
|
||||
* - it should be of a reasonable size to be sent in a single RPC message
|
||||
*
|
||||
* - The "toWorkers" parameter of value "*" targets the set of workers
|
||||
* that are available at the time the command is dispatched. Any new
|
||||
* worker spawned after the command has been dispatched for
|
||||
* processing, but before the command completes, don't execute
|
||||
* the command and hence are not part of the results array.
|
||||
*
|
||||
*
|
||||
* To set it up:
|
||||
*
|
||||
* - On the primary:
|
||||
* if (cluster.isPrimary) {
|
||||
* setupRPCPrimary();
|
||||
* }
|
||||
*
|
||||
* - On the workers:
|
||||
* if (!cluster.isPrimary) {
|
||||
* setupRPCWorker({
|
||||
* handler1: (payload: object, uids: string, callback: HandlerCallback) => void,
|
||||
* handler2: ...
|
||||
* });
|
||||
* }
|
||||
* Handler functions will be passed the command payload, request
|
||||
* serialized uids, and must call the callback when the worker is done
|
||||
* processing the command:
|
||||
* callback(error: Error | null | undefined, result?: any)
|
||||
*
|
||||
* When this setup is done, any worker can start sending commands by calling
|
||||
* the async function sendWorkerCommand().
|
||||
*/
|
||||
|
||||
// exported types
|
||||
|
||||
export type ResultObject = {
|
||||
error: Error | null;
|
||||
result: any;
|
||||
};
|
||||
|
||||
/**
|
||||
* saved Promise for sendWorkerCommand
|
||||
*/
|
||||
export type CommandPromise = {
|
||||
resolve: (results?: ResultObject[]) => void;
|
||||
reject: (error: Error) => void;
|
||||
timeout: NodeJS.Timer | null;
|
||||
};
|
||||
export type HandlerCallback = (error: Error | null | undefined, result?: any) => void;
|
||||
export type HandlerFunction = (payload: object, uids: string, callback: HandlerCallback) => void;
|
||||
export type HandlersMap = {
|
||||
[index: string]: HandlerFunction;
|
||||
};
|
||||
|
||||
// private types
|
||||
|
||||
type RPCMessage<T extends string, P> = {
|
||||
type: T;
|
||||
uids: string;
|
||||
payload: P;
|
||||
};
|
||||
|
||||
type RPCCommandMessage = RPCMessage<'cluster-rpc:command', any> & {
|
||||
toWorkers: string;
|
||||
toHandler: string;
|
||||
};
|
||||
|
||||
type MarshalledResultObject = {
|
||||
error: string | null;
|
||||
result: any;
|
||||
};
|
||||
|
||||
type RPCCommandResultMessage = RPCMessage<'cluster-rpc:commandResult', MarshalledResultObject>;
|
||||
|
||||
type RPCCommandResultsMessage = RPCMessage<'cluster-rpc:commandResults', {
|
||||
results: MarshalledResultObject[];
|
||||
}>;
|
||||
|
||||
type RPCCommandErrorMessage = RPCMessage<'cluster-rpc:commandError', {
|
||||
error: string;
|
||||
}>;
|
||||
|
||||
|
||||
/**
|
||||
* In primary: store worker IDs that are waiting to be dispatched
|
||||
* their command's results, as a mapping.
|
||||
*/
|
||||
const uidsToWorkerId: {
|
||||
[index: string]: number;
|
||||
} = {};
|
||||
|
||||
|
||||
/**
|
||||
* In primary: store worker responses for commands in progress as a
|
||||
* mapping.
|
||||
*
|
||||
* Result objects are 'null' while the worker is still processing the
|
||||
* command. When a worker finishes processing it stores the result as:
|
||||
* {
|
||||
* error: string | null,
|
||||
* result: any
|
||||
* }
|
||||
*/
|
||||
const uidsToCommandResults: {
|
||||
[index: string]: {
|
||||
[index: number]: MarshalledResultObject | null;
|
||||
};
|
||||
} = {};
|
||||
|
||||
/**
|
||||
* In workers: store promise callbacks for commands waiting to be
|
||||
* dispatched, as a mapping.
|
||||
*/
|
||||
const uidsToCommandPromise: {
|
||||
[index: string]: CommandPromise;
|
||||
} = {};
|
||||
|
||||
|
||||
function _isRpcMessage(message) {
|
||||
return (message !== null &&
|
||||
typeof message === 'object' &&
|
||||
typeof message.type === 'string' &&
|
||||
message.type.startsWith('cluster-rpc:'));
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup cluster RPC system on the primary
|
||||
*
|
||||
* @return {undefined}
|
||||
*/
|
||||
export function setupRPCPrimary() {
|
||||
cluster.on('message', (worker, message) => {
|
||||
if (_isRpcMessage(message)) {
|
||||
_handlePrimaryMessage(worker?.id, message);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup RPCs on a cluster worker process
|
||||
*
|
||||
* @param {object} handlers - mapping of handler names to handler functions
|
||||
* handler function:
|
||||
* handler({object} payload, {string} uids, {function} callback)
|
||||
* handler callback must be called when worker is done with the command:
|
||||
* callback({Error|null} error, {any} [result])
|
||||
* @return {undefined}
|
||||
* }
|
||||
*/
|
||||
export function setupRPCWorker(handlers: HandlersMap) {
|
||||
if (!process.send) {
|
||||
throw new Error('fatal: cannot setup cluster RPC: "process.send" is not available');
|
||||
}
|
||||
process.on('message', (message: RPCCommandMessage | RPCCommandResultsMessage) => {
|
||||
if (_isRpcMessage(message)) {
|
||||
_handleWorkerMessage(message, handlers);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a command for workers to execute in parallel, and wait for results
|
||||
*
|
||||
* @param {string} toWorkers - which workers should execute the command
|
||||
* Currently the only supported value is "*", meaning all workers will
|
||||
* execute the command
|
||||
* @param {string} toHandler - name of handler that will execute the
|
||||
* command in workers, as declared in setupRPCWorker() parameter object
|
||||
* @param {string} uids - unique identifier of the command, must be
|
||||
* unique across all commands in progress
|
||||
* @param {object} payload - message payload, sent as-is to the handler
|
||||
* @param {number} [timeoutMs=60000] - timeout the command with a
|
||||
* "RequestTimeout" error after this number of milliseconds - set to 0
|
||||
* to disable timeouts (the command may then hang forever)
|
||||
* @returns {Promise}
|
||||
*/
|
||||
export async function sendWorkerCommand(
|
||||
toWorkers: string,
|
||||
toHandler: string,
|
||||
uids: string,
|
||||
payload: object,
|
||||
timeoutMs: number = 60000
|
||||
) {
|
||||
if (typeof uids !== 'string') {
|
||||
rpcLogger.error('missing or invalid "uids" field', { uids });
|
||||
throw errors.MissingParameter;
|
||||
}
|
||||
if (uidsToCommandPromise[uids] !== undefined) {
|
||||
rpcLogger.error('a command is already in progress with same uids', { uids });
|
||||
throw errors.OperationAborted;
|
||||
}
|
||||
rpcLogger.info('sending command', { toWorkers, toHandler, uids, payload });
|
||||
return new Promise((resolve, reject) => {
|
||||
let timeout: NodeJS.Timer | null = null;
|
||||
if (timeoutMs) {
|
||||
timeout = setTimeout(() => {
|
||||
delete uidsToCommandPromise[uids];
|
||||
reject(errors.RequestTimeout);
|
||||
}, timeoutMs);
|
||||
}
|
||||
uidsToCommandPromise[uids] = { resolve, reject, timeout };
|
||||
const message: RPCCommandMessage = {
|
||||
type: 'cluster-rpc:command',
|
||||
toWorkers,
|
||||
toHandler,
|
||||
uids,
|
||||
payload,
|
||||
};
|
||||
return process.send?.(message);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the number of commands in flight
|
||||
* @returns {number}
|
||||
*/
|
||||
export function getPendingCommandsCount() {
|
||||
return Object.keys(uidsToCommandPromise).length;
|
||||
}
|
||||
|
||||
|
||||
function _dispatchCommandResultsToWorker(
|
||||
worker: Worker,
|
||||
uids: string,
|
||||
resultsArray: MarshalledResultObject[]
|
||||
): void {
|
||||
const message: RPCCommandResultsMessage = {
|
||||
type: 'cluster-rpc:commandResults',
|
||||
uids,
|
||||
payload: {
|
||||
results: resultsArray,
|
||||
},
|
||||
};
|
||||
worker.send(message);
|
||||
}
|
||||
|
||||
function _dispatchCommandErrorToWorker(
|
||||
worker: Worker,
|
||||
uids: string,
|
||||
error: Error,
|
||||
): void {
|
||||
const message: RPCCommandErrorMessage = {
|
||||
type: 'cluster-rpc:commandError',
|
||||
uids,
|
||||
payload: {
|
||||
error: error.message,
|
||||
},
|
||||
};
|
||||
worker.send(message);
|
||||
}
|
||||
|
||||
function _handlePrimaryCommandMessage(
|
||||
fromWorkerId: number,
|
||||
logger: any,
|
||||
message: RPCCommandMessage
|
||||
): void {
|
||||
const { toWorkers, toHandler, uids, payload } = message;
|
||||
if (toWorkers === '*') {
|
||||
if (uidsToWorkerId[uids] !== undefined) {
|
||||
logger.warn('new command already has a waiting worker with same uids', {
|
||||
uids, workerId: uidsToWorkerId[uids],
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
const commandResults = {};
|
||||
for (const workerId of Object.keys(cluster.workers || {})) {
|
||||
commandResults[workerId] = null;
|
||||
}
|
||||
uidsToWorkerId[uids] = fromWorkerId;
|
||||
uidsToCommandResults[uids] = commandResults;
|
||||
|
||||
for (const [workerId, worker] of Object.entries(cluster.workers || {})) {
|
||||
logger.debug('sending command message to worker', {
|
||||
workerId, toHandler, payload,
|
||||
});
|
||||
if (worker) {
|
||||
worker.send(message);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.error('unsupported "toWorkers" field from worker command message', {
|
||||
toWorkers,
|
||||
});
|
||||
const fromWorker = cluster.workers?.[fromWorkerId];
|
||||
if (fromWorker) {
|
||||
_dispatchCommandErrorToWorker(fromWorker, uids, errors.NotImplemented);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function _handlePrimaryCommandResultMessage(
|
||||
fromWorkerId: number,
|
||||
logger: any,
|
||||
message: RPCCommandResultMessage
|
||||
): void {
|
||||
const { uids, payload } = message;
|
||||
const commandResults = uidsToCommandResults[uids];
|
||||
if (!commandResults) {
|
||||
logger.warn('received command response message from worker for command not in flight', {
|
||||
workerId: fromWorkerId,
|
||||
uids,
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
if (commandResults[fromWorkerId] === undefined) {
|
||||
logger.warn('received command response message with unexpected worker ID', {
|
||||
workerId: fromWorkerId,
|
||||
uids,
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
if (commandResults[fromWorkerId] !== null) {
|
||||
logger.warn('ignoring duplicate command response from worker', {
|
||||
workerId: fromWorkerId,
|
||||
uids,
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
commandResults[fromWorkerId] = payload;
|
||||
const commandResultsArray = Object.values(commandResults);
|
||||
if (commandResultsArray.every(response => response !== null)) {
|
||||
logger.debug('all workers responded to command', { uids });
|
||||
const completeCommandResultsArray = <MarshalledResultObject[]> commandResultsArray;
|
||||
const toWorkerId = uidsToWorkerId[uids];
|
||||
const toWorker = cluster.workers?.[toWorkerId];
|
||||
|
||||
delete uidsToCommandResults[uids];
|
||||
delete uidsToWorkerId[uids];
|
||||
|
||||
if (!toWorker) {
|
||||
logger.warn('worker shut down while its command was executing', {
|
||||
workerId: toWorkerId, uids,
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
// send back response to original worker
|
||||
_dispatchCommandResultsToWorker(toWorker, uids, completeCommandResultsArray);
|
||||
}
|
||||
}
|
||||
|
||||
function _handlePrimaryMessage(
|
||||
fromWorkerId: number,
|
||||
message: RPCCommandMessage | RPCCommandResultMessage
|
||||
): void {
|
||||
const { type: messageType, uids } = message;
|
||||
const logger = rpcLogger.newRequestLoggerFromSerializedUids(uids);
|
||||
logger.debug('primary received message from worker', {
|
||||
workerId: fromWorkerId, rpcMessage: message,
|
||||
});
|
||||
if (messageType === 'cluster-rpc:command') {
|
||||
return _handlePrimaryCommandMessage(fromWorkerId, logger, message);
|
||||
}
|
||||
if (messageType === 'cluster-rpc:commandResult') {
|
||||
return _handlePrimaryCommandResultMessage(fromWorkerId, logger, message);
|
||||
}
|
||||
logger.error('unsupported message type', {
|
||||
workerId: fromWorkerId, messageType, uids,
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function _sendWorkerCommandResult(
|
||||
uids: string,
|
||||
error: Error | null | undefined,
|
||||
result?: any
|
||||
): void {
|
||||
const message: RPCCommandResultMessage = {
|
||||
type: 'cluster-rpc:commandResult',
|
||||
uids,
|
||||
payload: {
|
||||
error: error ? error.message : null,
|
||||
result,
|
||||
},
|
||||
};
|
||||
process.send?.(message);
|
||||
}
|
||||
|
||||
function _handleWorkerCommandMessage(
|
||||
logger: any,
|
||||
message: RPCCommandMessage,
|
||||
handlers: HandlersMap
|
||||
): void {
|
||||
const { toHandler, uids, payload } = message;
|
||||
const cb: HandlerCallback = (err, result) => _sendWorkerCommandResult(uids, err, result);
|
||||
|
||||
if (toHandler in handlers) {
|
||||
return handlers[toHandler](payload, uids, cb);
|
||||
}
|
||||
logger.error('no such handler in "toHandler" field from worker command message', {
|
||||
toHandler,
|
||||
});
|
||||
return cb(errors.NotImplemented);
|
||||
}
|
||||
|
||||
function _handleWorkerCommandResultsMessage(
|
||||
logger: any,
|
||||
message: RPCCommandResultsMessage,
|
||||
): void {
|
||||
const { uids, payload } = message;
|
||||
const { results } = payload;
|
||||
const commandPromise: CommandPromise = uidsToCommandPromise[uids];
|
||||
if (commandPromise === undefined) {
|
||||
logger.error('missing promise for command results', { uids, payload });
|
||||
return undefined;
|
||||
}
|
||||
if (commandPromise.timeout) {
|
||||
clearTimeout(commandPromise.timeout);
|
||||
}
|
||||
delete uidsToCommandPromise[uids];
|
||||
const unmarshalledResults = results.map(workerResult => {
|
||||
let workerError: Error | null = null;
|
||||
if (workerResult.error) {
|
||||
if (workerResult.error in errors) {
|
||||
workerError = errors[workerResult.error];
|
||||
} else {
|
||||
workerError = new Error(workerResult.error);
|
||||
}
|
||||
}
|
||||
const unmarshalledResult: ResultObject = {
|
||||
error: workerError,
|
||||
result: workerResult.result,
|
||||
};
|
||||
return unmarshalledResult;
|
||||
});
|
||||
return commandPromise.resolve(unmarshalledResults);
|
||||
}
|
||||
|
||||
function _handleWorkerCommandErrorMessage(
|
||||
logger: any,
|
||||
message: RPCCommandErrorMessage,
|
||||
): void {
|
||||
const { uids, payload } = message;
|
||||
const { error } = payload;
|
||||
const commandPromise: CommandPromise = uidsToCommandPromise[uids];
|
||||
if (commandPromise === undefined) {
|
||||
logger.error('missing promise for command results', { uids, payload });
|
||||
return undefined;
|
||||
}
|
||||
if (commandPromise.timeout) {
|
||||
clearTimeout(commandPromise.timeout);
|
||||
}
|
||||
delete uidsToCommandPromise[uids];
|
||||
let commandError: Error | null = null;
|
||||
if (error in errors) {
|
||||
commandError = errors[error];
|
||||
} else {
|
||||
commandError = new Error(error);
|
||||
}
|
||||
return commandPromise.reject(<Error> commandError);
|
||||
}
|
||||
|
||||
function _handleWorkerMessage(
|
||||
message: RPCCommandMessage | RPCCommandResultsMessage | RPCCommandErrorMessage,
|
||||
handlers: HandlersMap
|
||||
): void {
|
||||
const { type: messageType, uids } = message;
|
||||
const workerId = cluster.worker?.id;
|
||||
const logger = rpcLogger.newRequestLoggerFromSerializedUids(uids);
|
||||
logger.debug('worker received message from primary', {
|
||||
workerId, rpcMessage: message,
|
||||
});
|
||||
if (messageType === 'cluster-rpc:command') {
|
||||
return _handleWorkerCommandMessage(logger, message, handlers);
|
||||
}
|
||||
if (messageType === 'cluster-rpc:commandResults') {
|
||||
return _handleWorkerCommandResultsMessage(logger, message);
|
||||
}
|
||||
if (messageType === 'cluster-rpc:commandError') {
|
||||
return _handleWorkerCommandErrorMessage(logger, message);
|
||||
}
|
||||
logger.error('unsupported message type', {
|
||||
workerId, messageType,
|
||||
});
|
||||
return undefined;
|
||||
}
|
|
@ -2,18 +2,18 @@ import * as crypto from 'crypto';
|
|||
|
||||
// The min value here is to manage further backward compat if we
|
||||
// need it
|
||||
const iamSecurityTokenSizeMin = 128;
|
||||
const iamSecurityTokenSizeMax = 128;
|
||||
// Security token is an hex string (no real format from amazon)
|
||||
const iamSecurityTokenPattern = new RegExp(
|
||||
`^[a-f0-9]{${iamSecurityTokenSizeMin},${iamSecurityTokenSizeMax}}$`,
|
||||
);
|
||||
// Default value
|
||||
export const vaultGeneratedIamSecurityTokenSizeMin = 128;
|
||||
// Safe to assume that a typical token size is less than 8192 bytes
|
||||
export const vaultGeneratedIamSecurityTokenSizeMax = 8192;
|
||||
// Base-64
|
||||
export const vaultGeneratedIamSecurityTokenPattern = /^[A-Za-z0-9/+=]*$/;
|
||||
|
||||
// info about the iam security token
|
||||
export const iamSecurityToken = {
|
||||
min: iamSecurityTokenSizeMin,
|
||||
max: iamSecurityTokenSizeMax,
|
||||
pattern: iamSecurityTokenPattern,
|
||||
min: vaultGeneratedIamSecurityTokenSizeMin,
|
||||
max: vaultGeneratedIamSecurityTokenSizeMax,
|
||||
pattern: vaultGeneratedIamSecurityTokenPattern,
|
||||
};
|
||||
// PublicId is used as the canonicalID for a request that contains
|
||||
// no authentication information. Requestor can access
|
||||
|
@ -22,6 +22,7 @@ export const publicId = 'http://acs.amazonaws.com/groups/global/AllUsers';
|
|||
export const zenkoServiceAccount = 'http://acs.zenko.io/accounts/service';
|
||||
export const metadataFileNamespace = '/MDFile';
|
||||
export const dataFileURL = '/DataFile';
|
||||
export const passthroughFileURL = '/PassthroughFile';
|
||||
// AWS states max size for user-defined metadata
|
||||
// (x-amz-meta- headers) is 2 KB:
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html
|
||||
|
@ -31,7 +32,16 @@ export const maximumMetaHeadersSize = 2136;
|
|||
export const emptyFileMd5 = 'd41d8cd98f00b204e9800998ecf8427e';
|
||||
// Version 2 changes the format of the data location property
|
||||
// Version 3 adds the dataStoreName attribute
|
||||
export const mdModelVersion = 3;
|
||||
// Version 4 add the Creation-Time and Content-Language attributes,
|
||||
// and add support for x-ms-meta-* headers in UserMetadata
|
||||
// Version 5 adds the azureInfo structure
|
||||
// Version 6 adds a "deleted" flag that is updated to true before
|
||||
// the object gets deleted. This is done to keep object metadata in the
|
||||
// oplog when deleting the object, as oplog deletion events don't contain
|
||||
// any metadata of the object.
|
||||
// version 6 also adds the "isPHD" flag that is used to indicate that the master
|
||||
// object is a placeholder and is not up to date.
|
||||
export const mdModelVersion = 6;
|
||||
/*
|
||||
* Splitter is used to build the object name for the overview of a
|
||||
* multipart upload and to build the object names for each part of a
|
||||
|
@ -71,19 +81,45 @@ export const mpuBucketPrefix = 'mpuShadowBucket';
|
|||
export const permittedCapitalizedBuckets = {
|
||||
METADATA: true,
|
||||
};
|
||||
// Setting a lower object key limit to account for:
|
||||
// - Mongo key limit of 1012 bytes
|
||||
// - Version ID in Mongo Key if versioned of 33
|
||||
// - Max bucket name length if bucket match false of 63
|
||||
// - Extra prefix slash for bucket prefix if bucket match of 1
|
||||
export const objectKeyByteLimit = 915;
|
||||
/* delimiter for location-constraint. The location constraint will be able
|
||||
* to include the ingestion flag
|
||||
*/
|
||||
export const zenkoSeparator = ':';
|
||||
/* eslint-disable camelcase */
|
||||
export const externalBackends = { aws_s3: true, azure: true, gcp: true, pfs: true }
|
||||
export const hasCopyPartBackends = { aws_s3: true, gcp: true }
|
||||
export const versioningNotImplBackends = { azure: true, gcp: true }
|
||||
export const mpuMDStoredExternallyBackend = { aws_s3: true, gcp: true }
|
||||
export const externalBackends = { aws_s3: true, azure: true, gcp: true, pfs: true };
|
||||
export const replicationBackends = { aws_s3: true, azure: true, gcp: true };
|
||||
// hex digest of sha256 hash of empty string:
|
||||
export const emptyStringHash = crypto.createHash('sha256')
|
||||
.update('', 'binary').digest('hex');
|
||||
export const mpuMDStoredExternallyBackend = { aws_s3: true, gcp: true };
|
||||
// AWS sets a minimum size limit for parts except for the last part.
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadComplete.html
|
||||
export const minimumAllowedPartSize = 5242880;
|
||||
// hex digest of sha256 hash of empty string:
|
||||
export const emptyStringHash = crypto.createHash('sha256').update('', 'binary').digest('hex');
|
||||
export const gcpMaximumAllowedPartCount = 1024;
|
||||
// GCP Object Tagging Prefix
|
||||
export const gcpTaggingPrefix = 'aws-tag-';
|
||||
export const productName = 'APN/1.0 Scality/1.0 Scality CloudServer for Zenko';
|
||||
export const legacyLocations = ['sproxyd', 'legacy'];
|
||||
// healthcheck default call from nginx is every 2 seconds
|
||||
// for external backends, don't call unless at least 1 minute
|
||||
// (60,000 milliseconds) since last call
|
||||
export const externalBackendHealthCheckInterval = 60000;
|
||||
// some of the available data backends (if called directly rather
|
||||
// than through the multiple backend gateway) need a key provided
|
||||
// as a string as first parameter of the get/delete methods.
|
||||
export const clientsRequireStringKey = { sproxyd: true, cdmi: true };
|
||||
export const hasCopyPartBackends = { aws_s3: true, gcp: true };
|
||||
export const versioningNotImplBackends = { azure: true, gcp: true };
|
||||
// user metadata applied on zenko-created objects
|
||||
export const zenkoIDHeader = 'x-amz-meta-zenko-instance-id';
|
||||
// Default expiration value of the S3 pre-signed URL duration
|
||||
// 604800 seconds (seven days).
|
||||
export const legacyLocations = ['sproxyd', 'legacy'];
|
||||
export const defaultPreSignedURLExpiry = 7 * 24 * 60 * 60;
|
||||
// Regex for ISO-8601 formatted date
|
||||
export const shortIso8601Regex = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/;
|
||||
|
@ -96,16 +132,21 @@ export const supportedNotificationEvents = new Set([
|
|||
's3:ObjectRemoved:*',
|
||||
's3:ObjectRemoved:Delete',
|
||||
's3:ObjectRemoved:DeleteMarkerCreated',
|
||||
's3:Replication:OperationFailedReplication',
|
||||
's3:ObjectTagging:*',
|
||||
's3:ObjectTagging:Put',
|
||||
's3:ObjectTagging:Delete',
|
||||
's3:ObjectAcl:Put',
|
||||
's3:ObjectRestore:*',
|
||||
's3:ObjectRestore:Post',
|
||||
's3:ObjectRestore:Completed',
|
||||
's3:ObjectRestore:Delete',
|
||||
's3:LifecycleTransition',
|
||||
's3:LifecycleExpiration:*',
|
||||
's3:LifecycleExpiration:DeleteMarkerCreated',
|
||||
's3:LifecycleExpiration:Delete',
|
||||
]);
|
||||
export const notificationArnPrefix = 'arn:scality:bucketnotif';
|
||||
// some of the available data backends (if called directly rather
|
||||
// than through the multiple backend gateway) need a key provided
|
||||
// as a string as first parameter of the get/delete methods.
|
||||
export const clientsRequireStringKey = { sproxyd: true, cdmi: true };
|
||||
// HTTP server keep-alive timeout is set to a higher value than
|
||||
// client's free sockets timeout to avoid the risk of triggering
|
||||
// ECONNRESET errors if the server closes the connection at the
|
||||
|
@ -122,10 +163,15 @@ export const supportedLifecycleRules = [
|
|||
'expiration',
|
||||
'noncurrentVersionExpiration',
|
||||
'abortIncompleteMultipartUpload',
|
||||
'transitions',
|
||||
'noncurrentVersionTransition',
|
||||
];
|
||||
// Maximum number of buckets to cache (bucket metadata)
|
||||
export const maxCachedBuckets = process.env.METADATA_MAX_CACHED_BUCKETS ?
|
||||
Number(process.env.METADATA_MAX_CACHED_BUCKETS) : 1000;
|
||||
|
||||
export const validRestoreObjectTiers = new Set(['Expedited', 'Standard', 'Bulk']);
|
||||
export const maxBatchingConcurrentOperations = 5;
|
||||
|
||||
/** For policy resource arn check we allow empty account ID to not break compatibility */
|
||||
export const policyArnAllowedEmptyAccountId = ['utapi', 'scuba'];
|
||||
|
|
|
@ -2,7 +2,7 @@ import type { ServerResponse } from 'http';
|
|||
import * as rawErrors from './arsenalErrors';
|
||||
|
||||
/** All possible errors names. */
|
||||
export type Name = keyof typeof rawErrors
|
||||
export type Name = keyof typeof rawErrors;
|
||||
/** Object containing all errors names. It has the format { [Name]: "Name" } */
|
||||
export type Names = { [Name_ in Name]: Name_ };
|
||||
/** Mapping used to determine an error type. It has the format { [Name]: boolean } */
|
||||
|
@ -13,7 +13,7 @@ export type Errors = { [_ in Name]: ArsenalError };
|
|||
// This object is reused constantly through createIs, we store it there
|
||||
// to avoid recomputation.
|
||||
const isBase = Object.fromEntries(
|
||||
Object.keys(rawErrors).map(key => [key, false])
|
||||
Object.keys(rawErrors).map((key) => [key, false])
|
||||
) as Is;
|
||||
|
||||
// This allows to conditionally add the old behavior of errors to properly
|
||||
|
@ -32,7 +32,7 @@ export const allowUnsafeErrComp = (
|
|||
// the Proxy will return false.
|
||||
const createIs = (type: Name): Is => {
|
||||
const get = (is: Is, value: string | symbol) => is[value] ?? false;
|
||||
const final = Object.freeze({ ...isBase, [type]: true })
|
||||
const final = Object.freeze({ ...isBase, [type]: true });
|
||||
return new Proxy(final, { get });
|
||||
};
|
||||
|
||||
|
@ -46,13 +46,18 @@ export class ArsenalError extends Error {
|
|||
/** Object used to determine the error type.
|
||||
* Example: error.is.InternalError */
|
||||
#is: Is;
|
||||
/** A map of error metadata (can be extra fields
|
||||
* that only show in debug mode) */
|
||||
#metadata: Map<string, Object[]>;
|
||||
|
||||
private constructor(type: Name, code: number, description: string) {
|
||||
private constructor(type: Name, code: number, description: string,
|
||||
metadata?: Map<string, Object[]>) {
|
||||
super(type);
|
||||
this.#code = code;
|
||||
this.#description = description;
|
||||
this.#type = type;
|
||||
this.#is = createIs(type);
|
||||
this.#metadata = metadata ?? new Map<string, Object[]>();
|
||||
|
||||
// This restores the old behavior of errors, to make sure they're now
|
||||
// backward-compatible. Fortunately it's handled by TS, but it cannot
|
||||
|
@ -106,7 +111,22 @@ export class ArsenalError extends Error {
|
|||
customizeDescription(description: string): ArsenalError {
|
||||
const type = this.#type;
|
||||
const code = this.#code;
|
||||
return new ArsenalError(type, code, description);
|
||||
const metadata = new Map(this.#metadata);
|
||||
const err = new ArsenalError(type, code, description, metadata);
|
||||
err.stack = this.stack;
|
||||
return err;
|
||||
}
|
||||
|
||||
/** Clone the error with a new metadata field */
|
||||
addMetadataEntry(key: string, value: Object[]): ArsenalError {
|
||||
const type = this.#type;
|
||||
const code = this.#code;
|
||||
const description = this.#description;
|
||||
const metadata = new Map(this.#metadata);
|
||||
metadata.set(key, value);
|
||||
const err = new ArsenalError(type, code, description, metadata);
|
||||
err.stack = this.stack;
|
||||
return err;
|
||||
}
|
||||
|
||||
/** Used to determine the error type. Example: error.is.InternalError */
|
||||
|
@ -131,9 +151,14 @@ export class ArsenalError extends Error {
|
|||
return this.#type;
|
||||
}
|
||||
|
||||
/** A map of error metadata */
|
||||
get metadata() {
|
||||
return this.#metadata;
|
||||
}
|
||||
|
||||
/** Generate all possible errors. An instance is created by default. */
|
||||
static errors() {
|
||||
const errors = {}
|
||||
const errors = {};
|
||||
Object.entries(rawErrors).forEach((value) => {
|
||||
const name = value[0] as Name;
|
||||
const error = value[1];
|
||||
|
@ -141,7 +166,7 @@ export class ArsenalError extends Error {
|
|||
const get = () => new ArsenalError(name, code, description);
|
||||
Object.defineProperty(errors, name, { get });
|
||||
});
|
||||
return errors as Errors
|
||||
return errors as Errors;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -7,8 +7,8 @@
|
|||
"test": "mocha --recursive --timeout 5500 tests/unit"
|
||||
},
|
||||
"dependencies": {
|
||||
"mocha": "2.5.3",
|
||||
"async": "^2.6.0",
|
||||
"mocha": "5.2.0",
|
||||
"async": "~2.6.1",
|
||||
"node-forge": "^0.7.1"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -20,7 +20,32 @@ export default class RedisClient {
|
|||
return this;
|
||||
}
|
||||
|
||||
/** increment value of a key by 1 and set a ttl */
|
||||
/**
|
||||
* scan a pattern and return matching keys
|
||||
* @param pattern - string pattern to match with all existing keys
|
||||
* @param [count=10] - scan count
|
||||
* @param cb - callback (error, result)
|
||||
*/
|
||||
scan(pattern: string, count = 10, cb: Callback) {
|
||||
const params = { match: pattern, count };
|
||||
const keys: any[] = [];
|
||||
|
||||
const stream = this._client.scanStream(params);
|
||||
stream.on('data', resultKeys => {
|
||||
for (let i = 0; i < resultKeys.length; i++) {
|
||||
keys.push(resultKeys[i]);
|
||||
}
|
||||
});
|
||||
stream.on('end', () => {
|
||||
cb(null, keys);
|
||||
});
|
||||
}
|
||||
|
||||
/** increment value of a key by 1 and set a ttl
|
||||
* @param key - key holding the value
|
||||
* @param expiry - expiry in seconds
|
||||
* @param cb - callback
|
||||
*/
|
||||
incrEx(key: string, expiry: number, cb: Callback) {
|
||||
const exp = expiry.toString();
|
||||
return this._client
|
||||
|
@ -28,7 +53,22 @@ export default class RedisClient {
|
|||
.exec(cb);
|
||||
}
|
||||
|
||||
/** increment value of a key by a given amount and set a ttl */
|
||||
/**
|
||||
* increment value of a key by a given amount
|
||||
* @param key - key holding the value
|
||||
* @param amount - amount to increase by
|
||||
* @param cb - callback
|
||||
*/
|
||||
incrby(key: string, amount: number, cb: Callback) {
|
||||
return this._client.incrby(key, amount, cb);
|
||||
}
|
||||
|
||||
/** increment value of a key by a given amount and set a ttl
|
||||
* @param key - key holding the value
|
||||
* @param amount - amount to increase by
|
||||
* @param expiry - expiry in seconds
|
||||
* @param cb - callback
|
||||
*/
|
||||
incrbyEx(key: string, amount: number, expiry: number, cb: Callback) {
|
||||
const am = amount.toString();
|
||||
const exp = expiry.toString();
|
||||
|
@ -37,13 +77,29 @@ export default class RedisClient {
|
|||
.exec(cb);
|
||||
}
|
||||
|
||||
/** execute a batch of commands */
|
||||
/**
|
||||
* decrement value of a key by a given amount
|
||||
* @param key - key holding the value
|
||||
* @param amount - amount to increase by
|
||||
* @param cb - callback
|
||||
*/
|
||||
decrby(key: string, amount: number, cb: Callback) {
|
||||
return this._client.decrby(key, amount, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* execute a batch of commands
|
||||
* @param cmds - list of commands
|
||||
* @param cb - callback
|
||||
* @return
|
||||
*/
|
||||
batch(cmds: string[][], cb: Callback) {
|
||||
return this._client.pipeline(cmds).exec(cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a key exists
|
||||
* @param key - name of key
|
||||
* @param cb - callback
|
||||
* If cb response returns 0, key does not exist.
|
||||
* If cb response returns 1, key exists.
|
||||
|
@ -52,10 +108,22 @@ export default class RedisClient {
|
|||
return this._client.exists(key, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* get value stored at key
|
||||
* @param key - key holding the value
|
||||
* @param cb - callback
|
||||
*/
|
||||
get(key: string, cb: Callback) {
|
||||
return this._client.get(key, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a value and its score to a sorted set. If no sorted set exists, this
|
||||
* will create a new one for the given key.
|
||||
* @param key - name of key
|
||||
* @param score - score used to order set
|
||||
* @param value - value to store
|
||||
* @param cb - callback
|
||||
*/
|
||||
zadd(key: string, score: number, value: string, cb: Callback) {
|
||||
return this._client.zadd(key, score, value, cb);
|
||||
|
@ -66,6 +134,8 @@ export default class RedisClient {
|
|||
* Note: using this on a key that does not exist will return 0.
|
||||
* Note: using this on an existing key that isn't a sorted set will
|
||||
* return an error WRONGTYPE.
|
||||
* @param key - name of key
|
||||
* @param cb - callback
|
||||
*/
|
||||
zcard(key: string, cb: Callback) {
|
||||
return this._client.zcard(key, cb);
|
||||
|
@ -76,6 +146,9 @@ export default class RedisClient {
|
|||
* Note: using this on a key that does not exist will return nil.
|
||||
* Note: using this on a value that does not exist in a valid sorted set key
|
||||
* will return nil.
|
||||
* @param key - name of key
|
||||
* @param value - value within sorted set
|
||||
* @param cb - callback
|
||||
*/
|
||||
zscore(key: string, value: string, cb: Callback) {
|
||||
return this._client.zscore(key, value, cb);
|
||||
|
@ -83,8 +156,10 @@ export default class RedisClient {
|
|||
|
||||
/**
|
||||
* Remove a value from a sorted set
|
||||
* @param value - value within sorted set. Can specify multiple values within an array
|
||||
* @param {function} cb - callback
|
||||
* @param key - name of key
|
||||
* @param value - value within sorted set. Can specify
|
||||
* multiple values within an array
|
||||
* @param cb - callback
|
||||
* The cb response returns number of values removed
|
||||
*/
|
||||
zrem(key: string, value: string | string[], cb: Callback) {
|
||||
|
@ -93,8 +168,10 @@ export default class RedisClient {
|
|||
|
||||
/**
|
||||
* Get specified range of elements in a sorted set
|
||||
* @param key - name of key
|
||||
* @param start - start index (inclusive)
|
||||
* @param end - end index (inclusive) (can use -1)
|
||||
* @param cb - callback
|
||||
*/
|
||||
zrange(key: string, start: number, end: number, cb: Callback) {
|
||||
return this._client.zrange(key, start, end, cb);
|
||||
|
@ -102,10 +179,12 @@ export default class RedisClient {
|
|||
|
||||
/**
|
||||
* Get range of elements in a sorted set based off score
|
||||
* @param key - name of key
|
||||
* @param min - min score value (inclusive)
|
||||
* (can use "-inf")
|
||||
* @param max - max score value (inclusive)
|
||||
* (can use "+inf")
|
||||
* @param cb - callback
|
||||
*/
|
||||
zrangebyscore(
|
||||
key: string,
|
||||
|
@ -116,6 +195,15 @@ export default class RedisClient {
|
|||
return this._client.zrangebyscore(key, min, max, cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* get TTL or expiration in seconds
|
||||
* @param key - name of key
|
||||
* @param cb - callback
|
||||
*/
|
||||
ttl(key: string, cb: Callback) {
|
||||
return this._client.ttl(key, cb);
|
||||
}
|
||||
|
||||
clear(cb: Callback) {
|
||||
return this._client.flushdb(cb);
|
||||
}
|
||||
|
@ -123,4 +211,8 @@ export default class RedisClient {
|
|||
disconnect() {
|
||||
this._client.disconnect();
|
||||
}
|
||||
|
||||
listClients(cb: Callback) {
|
||||
return this._client.client('list', cb);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,6 +2,8 @@ import async from 'async';
|
|||
import RedisClient from './RedisClient';
|
||||
import { Logger } from 'werelogs';
|
||||
|
||||
export type Callback = (error: Error | null, value?: any) => void;
|
||||
|
||||
export default class StatsClient {
|
||||
_redis: RedisClient;
|
||||
_interval: number;
|
||||
|
@ -48,7 +50,7 @@ export default class StatsClient {
|
|||
* @param d - Date instance
|
||||
* @return key - key for redis
|
||||
*/
|
||||
_buildKey(name: string, d: Date): string {
|
||||
buildKey(name: string, d: Date): string {
|
||||
return `${name}:${this._normalizeTimestamp(d)}`;
|
||||
}
|
||||
|
||||
|
@ -91,11 +93,33 @@ export default class StatsClient {
|
|||
amount = (typeof incr === 'number') ? incr : 1;
|
||||
}
|
||||
|
||||
const key = this._buildKey(`${id}:requests`, new Date());
|
||||
const key = this.buildKey(`${id}:requests`, new Date());
|
||||
|
||||
return this._redis.incrbyEx(key, amount, this._expiry, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Increment the given key by the given value.
|
||||
* @param key - The Redis key to increment
|
||||
* @param incr - The value to increment by
|
||||
* @param [cb] - callback
|
||||
*/
|
||||
incrementKey(key: string, incr: number, cb: Callback) {
|
||||
const callback = cb || this._noop;
|
||||
return this._redis.incrby(key, incr, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* Decrement the given key by the given value.
|
||||
* @param key - The Redis key to decrement
|
||||
* @param decr - The value to decrement by
|
||||
* @param [cb] - callback
|
||||
*/
|
||||
decrementKey(key: string, decr: number, cb: Callback) {
|
||||
const callback = cb || this._noop;
|
||||
return this._redis.decrby(key, decr, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* report/record a request that ended up being a 500 on the server
|
||||
* @param id - service identifier
|
||||
|
@ -105,10 +129,53 @@ export default class StatsClient {
|
|||
return undefined;
|
||||
}
|
||||
const callback = cb || this._noop;
|
||||
const key = this._buildKey(`${id}:500s`, new Date());
|
||||
const key = this.buildKey(`${id}:500s`, new Date());
|
||||
return this._redis.incrEx(key, this._expiry, callback);
|
||||
}
|
||||
|
||||
/**
|
||||
* wrapper on `getStats` that handles a list of keys
|
||||
* @param log - Werelogs request logger
|
||||
* @param ids - service identifiers
|
||||
* @param cb - callback to call with the err/result
|
||||
*/
|
||||
getAllStats(log: Logger, ids: string[], cb: Callback) {
|
||||
if (!this._redis) {
|
||||
return cb(null, {});
|
||||
}
|
||||
|
||||
const statsRes = {
|
||||
'requests': 0,
|
||||
'500s': 0,
|
||||
'sampleDuration': this._expiry,
|
||||
};
|
||||
let requests = 0;
|
||||
let errors = 0;
|
||||
|
||||
// for now set concurrency to default of 10
|
||||
return async.eachLimit(ids, 10, (id: string, done) => {
|
||||
this.getStats(log, id, (err, res) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
requests += res.requests;
|
||||
errors += res['500s'];
|
||||
return done();
|
||||
});
|
||||
}, error => {
|
||||
if (error) {
|
||||
log.error('error getting stats', {
|
||||
error,
|
||||
method: 'StatsClient.getAllStats',
|
||||
});
|
||||
return cb(null, statsRes);
|
||||
}
|
||||
statsRes.requests = requests;
|
||||
statsRes['500s'] = errors;
|
||||
return cb(null, statsRes);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* get stats for the last x seconds, x being the sampling duration
|
||||
* @param log - Werelogs request logger
|
||||
|
@ -123,8 +190,8 @@ export default class StatsClient {
|
|||
const reqsKeys: ['get', string][] = [];
|
||||
const req500sKeys: ['get', string][] = [];
|
||||
for (let i = 0; i < totalKeys; i++) {
|
||||
reqsKeys.push(['get', this._buildKey(`${id}:requests`, d)]);
|
||||
req500sKeys.push(['get', this._buildKey(`${id}:500s`, d)]);
|
||||
reqsKeys.push(['get', this.buildKey(`${id}:requests`, d)]);
|
||||
req500sKeys.push(['get', this.buildKey(`${id}:500s`, d)]);
|
||||
this._setPrevInterval(d);
|
||||
}
|
||||
return async.parallel([
|
||||
|
|
|
@ -1,4 +1,8 @@
|
|||
import StatsClient from './StatsClient';
|
||||
import { Logger } from 'werelogs';
|
||||
import async from 'async';
|
||||
|
||||
export type Callback = (error: Error | null, value?: any) => void;
|
||||
|
||||
/**
|
||||
* @class StatsModel
|
||||
|
@ -7,12 +11,145 @@ import StatsClient from './StatsClient';
|
|||
* rather than by seconds
|
||||
*/
|
||||
export default class StatsModel extends StatsClient {
|
||||
/**
|
||||
* Utility method to convert 2d array rows to columns, and vice versa
|
||||
* See also: https://docs.ruby-lang.org/en/2.0.0/Array.html#method-i-zip
|
||||
* @param arrays - 2d array of integers
|
||||
* @return converted array
|
||||
*/
|
||||
_zip(arrays: number[][]) {
|
||||
if (arrays.length > 0 && arrays.every(a => Array.isArray(a))) {
|
||||
return arrays[0].map((_, i) => arrays.map(a => a[i]));
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* normalize to the nearest interval
|
||||
* @param d - Date instance
|
||||
* @return timestamp - normalized to the nearest interval
|
||||
*/
|
||||
_normalizeTimestamp(d: Date) {
|
||||
const m = d.getMinutes();
|
||||
return d.setMinutes(m - m % (Math.floor(this._interval / 60)), 0, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* override the method to get the count as an array of integers separated
|
||||
* by each interval
|
||||
* typical input looks like [[null, '1'], [null, '2'], [null, null]...]
|
||||
* @param arr - each index contains the result of each batch command
|
||||
* where index 0 signifies the error and index 1 contains the result
|
||||
* @return array of integers, ordered from most recent interval to
|
||||
* oldest interval with length of (expiry / interval)
|
||||
*/
|
||||
// @ts-expect-errors
|
||||
_getCount(arr: [any, string | null][]): number[] {
|
||||
const size = Math.floor(this._expiry / this._interval);
|
||||
const array = arr.reduce((store, i) => {
|
||||
let num = parseInt(i[1] ?? '', 10);
|
||||
num = Number.isNaN(num) ? 0 : num;
|
||||
store.push(num);
|
||||
return store;
|
||||
}, [] as number[]);
|
||||
|
||||
if (array.length < size) {
|
||||
array.push(...Array(size - array.length).fill(0));
|
||||
}
|
||||
return array;
|
||||
}
|
||||
|
||||
/**
|
||||
* wrapper on `getStats` that handles a list of keys
|
||||
* override the method to reduce the returned 2d array from `_getCount`
|
||||
* @param log - Werelogs request logger
|
||||
* @param ids - service identifiers
|
||||
* @param cb - callback to call with the err/result
|
||||
*/
|
||||
getAllStats(log: Logger, ids: string[], cb: Callback) {
|
||||
if (!this._redis) {
|
||||
return cb(null, {});
|
||||
}
|
||||
|
||||
const size = Math.floor(this._expiry / this._interval);
|
||||
const statsRes = {
|
||||
'requests': Array(size).fill(0),
|
||||
'500s': Array(size).fill(0),
|
||||
'sampleDuration': this._expiry,
|
||||
};
|
||||
const requests: any[] = [];
|
||||
const errors: any[] = [];
|
||||
|
||||
if (ids.length === 0) {
|
||||
return cb(null, statsRes);
|
||||
}
|
||||
|
||||
// for now set concurrency to default of 10
|
||||
return async.eachLimit(ids, 10, (id, done) => {
|
||||
this.getStats(log, id, (err, res) => {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
requests.push(res.requests);
|
||||
errors.push(res['500s']);
|
||||
return done();
|
||||
});
|
||||
}, error => {
|
||||
if (error) {
|
||||
log.error('error getting stats', {
|
||||
error,
|
||||
method: 'StatsModel.getAllStats',
|
||||
});
|
||||
return cb(null, statsRes);
|
||||
}
|
||||
|
||||
statsRes.requests = this._zip(requests).map(arr =>
|
||||
arr.reduce((acc, i) => acc + i), 0);
|
||||
statsRes['500s'] = this._zip(errors).map(arr =>
|
||||
arr.reduce((acc, i) => acc + i), 0);
|
||||
|
||||
return cb(null, statsRes);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles getting a list of global keys.
|
||||
* @param ids - Service identifiers
|
||||
* @param log - Werelogs request logger
|
||||
* @param cb - Callback
|
||||
*/
|
||||
getAllGlobalStats(ids: string[], log: Logger, cb: Callback) {
|
||||
const reqsKeys = ids.map(key => (['get', key]));
|
||||
return this._redis.batch(reqsKeys, (err, res) => {
|
||||
const statsRes = { requests: 0 };
|
||||
if (err) {
|
||||
log.error('error getting metrics', {
|
||||
error: err,
|
||||
method: 'StatsClient.getAllGlobalStats',
|
||||
});
|
||||
return cb(null, statsRes);
|
||||
}
|
||||
statsRes.requests = res.reduce((sum, curr) => {
|
||||
const [cmdErr, val] = curr;
|
||||
if (cmdErr) {
|
||||
// Log any individual request errors from the batch request.
|
||||
log.error('error getting metrics', {
|
||||
error: cmdErr,
|
||||
method: 'StatsClient.getAllGlobalStats',
|
||||
});
|
||||
}
|
||||
return sum + (Number.parseInt(val, 10) || 0);
|
||||
}, 0);
|
||||
return cb(null, statsRes);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* normalize date timestamp to the nearest hour
|
||||
* @param d - Date instance
|
||||
* @return timestamp - normalized to the nearest hour
|
||||
*/
|
||||
normalizeTimestampByHour(d: Date): number {
|
||||
normalizeTimestampByHour(d: Date) {
|
||||
return d.setMinutes(0, 0, 0);
|
||||
}
|
||||
|
||||
|
@ -21,40 +158,10 @@ export default class StatsModel extends StatsClient {
|
|||
* @param d - Date instance
|
||||
* @return timestamp - one hour prior to date passed
|
||||
*/
|
||||
_getDatePreviousHour(d: Date): number {
|
||||
_getDatePreviousHour(d: Date) {
|
||||
return d.setHours(d.getHours() - 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* normalize to the nearest interval
|
||||
* @param d - Date instance
|
||||
* @return timestamp - normalized to the nearest interval
|
||||
*/
|
||||
_normalizeTimestamp(d: Date): number {
|
||||
const m = d.getMinutes();
|
||||
return d.setMinutes(m - m % (Math.floor(this._interval / 60)), 0, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* override the method to get the result as an array of integers separated
|
||||
* by each interval
|
||||
* typical input looks like [[null, '1'], [null, '2'], [null, null]...]
|
||||
* @param arr - each index contains the result of each batch command
|
||||
* where index 0 signifies the error and index 1 contains the result
|
||||
* @return array of integers, ordered from most recent interval to
|
||||
* oldest interval
|
||||
*/
|
||||
// @ts-ignore
|
||||
// TODO change name or conform to parent class method
|
||||
_getCount(arr: [any, string | null][]) {
|
||||
return arr.reduce<number[]>((store, i) => {
|
||||
let num = parseInt(i[1] ?? '', 10);
|
||||
num = Number.isNaN(num) ? 0 : num;
|
||||
store.push(num);
|
||||
return store;
|
||||
}, []);
|
||||
}
|
||||
|
||||
/**
|
||||
* get list of sorted set key timestamps
|
||||
* @param epoch - epoch time
|
||||
|
|
|
@ -0,0 +1,281 @@
|
|||
export type DeleteRetentionPolicy = {
|
||||
enabled: boolean;
|
||||
days: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Helper class to ease access to the Azure specific information for
|
||||
* storage accounts mapped to buckets.
|
||||
*/
|
||||
export default class BucketAzureInfo {
|
||||
_data: {
|
||||
sku: string;
|
||||
accessTier: string;
|
||||
kind: string;
|
||||
systemKeys: string[];
|
||||
tenantKeys: string[];
|
||||
subscriptionId: string;
|
||||
resourceGroup: string;
|
||||
deleteRetentionPolicy: DeleteRetentionPolicy;
|
||||
managementPolicies: any[];
|
||||
httpsOnly: boolean;
|
||||
tags: any;
|
||||
networkACL: any[];
|
||||
cname: string;
|
||||
azureFilesAADIntegration: boolean;
|
||||
hnsEnabled: boolean;
|
||||
logging: any;
|
||||
hourMetrics: any;
|
||||
minuteMetrics: any;
|
||||
serviceVersion: string;
|
||||
}
|
||||
/**
|
||||
* @constructor
|
||||
* @param obj - Raw structure for the Azure info on storage account
|
||||
* @param obj.sku - SKU name of this storage account
|
||||
* @param obj.accessTier - Access Tier name of this storage account
|
||||
* @param obj.kind - Kind name of this storage account
|
||||
* @param obj.systemKeys - pair of shared keys for the system
|
||||
* @param obj.tenantKeys - pair of shared keys for the tenant
|
||||
* @param obj.subscriptionId - subscription ID the storage account
|
||||
* belongs to
|
||||
* @param obj.resourceGroup - Resource group name the storage
|
||||
* account belongs to
|
||||
* @param obj.deleteRetentionPolicy - Delete retention policy
|
||||
* @param obj.deleteRetentionPolicy.enabled -
|
||||
* @param obj.deleteRetentionPolicy.days -
|
||||
* @param obj.managementPolicies - Management policies for this
|
||||
* storage account
|
||||
* @param obj.httpsOnly - Server the content of this storage
|
||||
* account through HTTPS only
|
||||
* @param obj.tags - Set of tags applied on this storage account
|
||||
* @param obj.networkACL - Network ACL of this storage account
|
||||
* @param obj.cname - CNAME of this storage account
|
||||
* @param obj.azureFilesAADIntegration - whether or not Azure
|
||||
* Files AAD Integration is enabled for this storage account
|
||||
* @param obj.hnsEnabled - whether or not a hierarchical namespace
|
||||
* is enabled for this storage account
|
||||
* @param obj.logging - service properties: logging
|
||||
* @param obj.hourMetrics - service properties: hourMetrics
|
||||
* @param obj.minuteMetrics - service properties: minuteMetrics
|
||||
* @param obj.serviceVersion - service properties: serviceVersion
|
||||
*/
|
||||
constructor(obj: {
|
||||
sku: string;
|
||||
accessTier: string;
|
||||
kind: string;
|
||||
systemKeys: string[];
|
||||
tenantKeys: string[];
|
||||
subscriptionId: string;
|
||||
resourceGroup: string;
|
||||
deleteRetentionPolicy: DeleteRetentionPolicy;
|
||||
managementPolicies: any[];
|
||||
httpsOnly: boolean;
|
||||
tags: any;
|
||||
networkACL: any[];
|
||||
cname: string;
|
||||
azureFilesAADIntegration: boolean;
|
||||
hnsEnabled: boolean;
|
||||
logging: any;
|
||||
hourMetrics: any;
|
||||
minuteMetrics: any;
|
||||
serviceVersion: string;
|
||||
}) {
|
||||
this._data = {
|
||||
sku: obj.sku,
|
||||
accessTier: obj.accessTier,
|
||||
kind: obj.kind,
|
||||
systemKeys: obj.systemKeys,
|
||||
tenantKeys: obj.tenantKeys,
|
||||
subscriptionId: obj.subscriptionId,
|
||||
resourceGroup: obj.resourceGroup,
|
||||
deleteRetentionPolicy: obj.deleteRetentionPolicy,
|
||||
managementPolicies: obj.managementPolicies,
|
||||
httpsOnly: obj.httpsOnly,
|
||||
tags: obj.tags,
|
||||
networkACL: obj.networkACL,
|
||||
cname: obj.cname,
|
||||
azureFilesAADIntegration: obj.azureFilesAADIntegration,
|
||||
hnsEnabled: obj.hnsEnabled,
|
||||
logging: obj.logging,
|
||||
hourMetrics: obj.hourMetrics,
|
||||
minuteMetrics: obj.minuteMetrics,
|
||||
serviceVersion: obj.serviceVersion,
|
||||
};
|
||||
}
|
||||
|
||||
getSku() {
|
||||
return this._data.sku;
|
||||
}
|
||||
|
||||
setSku(sku: string) {
|
||||
this._data.sku = sku;
|
||||
return this;
|
||||
}
|
||||
|
||||
getAccessTier() {
|
||||
return this._data.accessTier;
|
||||
}
|
||||
|
||||
setAccessTier(accessTier: string) {
|
||||
this._data.accessTier = accessTier;
|
||||
return this;
|
||||
}
|
||||
|
||||
getKind() {
|
||||
return this._data.kind;
|
||||
}
|
||||
|
||||
setKind(kind: string) {
|
||||
this._data.kind = kind;
|
||||
return this;
|
||||
}
|
||||
|
||||
getSystemKeys() {
|
||||
return this._data.systemKeys;
|
||||
}
|
||||
|
||||
setSystemKeys(systemKeys: string[]) {
|
||||
this._data.systemKeys = systemKeys;
|
||||
return this;
|
||||
}
|
||||
|
||||
getTenantKeys() {
|
||||
return this._data.tenantKeys;
|
||||
}
|
||||
|
||||
setTenantKeys(tenantKeys: string[]) {
|
||||
this._data.tenantKeys = tenantKeys;
|
||||
return this;
|
||||
}
|
||||
|
||||
getSubscriptionId() {
|
||||
return this._data.subscriptionId;
|
||||
}
|
||||
|
||||
setSubscriptionId(subscriptionId: string) {
|
||||
this._data.subscriptionId = subscriptionId;
|
||||
return this;
|
||||
}
|
||||
|
||||
getResourceGroup() {
|
||||
return this._data.resourceGroup;
|
||||
}
|
||||
|
||||
setResourceGroup(resourceGroup: string) {
|
||||
this._data.resourceGroup = resourceGroup;
|
||||
return this;
|
||||
}
|
||||
|
||||
getDeleteRetentionPolicy() {
|
||||
return this._data.deleteRetentionPolicy;
|
||||
}
|
||||
|
||||
setDeleteRetentionPolicy(deleteRetentionPolicy: DeleteRetentionPolicy) {
|
||||
this._data.deleteRetentionPolicy = deleteRetentionPolicy;
|
||||
return this;
|
||||
}
|
||||
|
||||
getManagementPolicies() {
|
||||
return this._data.managementPolicies;
|
||||
}
|
||||
|
||||
setManagementPolicies(managementPolicies: any[]) {
|
||||
this._data.managementPolicies = managementPolicies;
|
||||
return this;
|
||||
}
|
||||
|
||||
getHttpsOnly() {
|
||||
return this._data.httpsOnly;
|
||||
}
|
||||
|
||||
setHttpsOnly(httpsOnly: boolean) {
|
||||
this._data.httpsOnly = httpsOnly;
|
||||
return this;
|
||||
}
|
||||
|
||||
getTags() {
|
||||
return this._data.tags;
|
||||
}
|
||||
|
||||
setTags(tags: any) {
|
||||
this._data.tags = tags;
|
||||
return this;
|
||||
}
|
||||
|
||||
getNetworkACL() {
|
||||
return this._data.networkACL;
|
||||
}
|
||||
|
||||
setNetworkACL(networkACL: any[]) {
|
||||
this._data.networkACL = networkACL;
|
||||
return this;
|
||||
}
|
||||
|
||||
getCname() {
|
||||
return this._data.cname;
|
||||
}
|
||||
|
||||
setCname(cname: string) {
|
||||
this._data.cname = cname;
|
||||
return this;
|
||||
}
|
||||
|
||||
getAzureFilesAADIntegration() {
|
||||
return this._data.azureFilesAADIntegration;
|
||||
}
|
||||
|
||||
setAzureFilesAADIntegration(azureFilesAADIntegration: boolean) {
|
||||
this._data.azureFilesAADIntegration = azureFilesAADIntegration;
|
||||
return this;
|
||||
}
|
||||
|
||||
getHnsEnabled() {
|
||||
return this._data.hnsEnabled;
|
||||
}
|
||||
|
||||
setHnsEnabled(hnsEnabled: boolean) {
|
||||
this._data.hnsEnabled = hnsEnabled;
|
||||
return this;
|
||||
}
|
||||
|
||||
getLogging() {
|
||||
return this._data.logging;
|
||||
}
|
||||
|
||||
setLogging(logging: any) {
|
||||
this._data.logging = logging;
|
||||
return this;
|
||||
}
|
||||
|
||||
getHourMetrics() {
|
||||
return this._data.hourMetrics;
|
||||
}
|
||||
|
||||
setHourMetrics(hourMetrics: any) {
|
||||
this._data.hourMetrics = hourMetrics;
|
||||
return this;
|
||||
}
|
||||
|
||||
getMinuteMetrics() {
|
||||
return this._data.minuteMetrics;
|
||||
}
|
||||
|
||||
setMinuteMetrics(minuteMetrics: any) {
|
||||
this._data.minuteMetrics = minuteMetrics;
|
||||
return this;
|
||||
}
|
||||
|
||||
getServiceVersion() {
|
||||
return this._data.serviceVersion;
|
||||
}
|
||||
|
||||
setServiceVersion(serviceVersion: any) {
|
||||
this._data.serviceVersion = serviceVersion;
|
||||
return this;
|
||||
}
|
||||
|
||||
getValue() {
|
||||
return this._data;
|
||||
}
|
||||
}
|
|
@ -8,10 +8,12 @@ import ObjectLockConfiguration from './ObjectLockConfiguration';
|
|||
import BucketPolicy from './BucketPolicy';
|
||||
import NotificationConfiguration from './NotificationConfiguration';
|
||||
import { ACL as OACL } from './ObjectMD';
|
||||
import { areTagsValid, BucketTag } from '../s3middleware/tagging';
|
||||
|
||||
// WHEN UPDATING THIS NUMBER, UPDATE BucketInfoModelVersion.md CHANGELOG
|
||||
// BucketInfoModelVersion.md can be found in the root of this repository
|
||||
const modelVersion = 10;
|
||||
// BucketInfoModelVersion.md can be found in documentation/ at the root
|
||||
// of this repository
|
||||
const modelVersion = 16;
|
||||
|
||||
export type CORS = {
|
||||
id: string;
|
||||
|
@ -35,6 +37,41 @@ export type VersioningConfiguration = {
|
|||
MfaDelete: any;
|
||||
};
|
||||
|
||||
export type VeeamSOSApi = {
|
||||
SystemInfo?: {
|
||||
ProtocolVersion: string,
|
||||
ModelName: string,
|
||||
ProtocolCapabilities: {
|
||||
CapacityInfo: boolean,
|
||||
UploadSessions: boolean,
|
||||
IAMSTS?: boolean,
|
||||
},
|
||||
APIEndpoints?: {
|
||||
IAMEndpoint: string,
|
||||
STSEndpoint: string,
|
||||
},
|
||||
SystemRecommendations?: {
|
||||
S3ConcurrentTaskLimit: number,
|
||||
S3MultiObjectDelete: number,
|
||||
StorageCurrentTasksLimit: number,
|
||||
KbBlockSize: number,
|
||||
}
|
||||
LastModified?: string,
|
||||
},
|
||||
CapacityInfo?: {
|
||||
Capacity: number,
|
||||
Available: number,
|
||||
Used: number,
|
||||
LastModified?: string,
|
||||
},
|
||||
};
|
||||
|
||||
// Capabilities contains all specifics from external products supported by
|
||||
// our S3 implementation, at bucket level
|
||||
export type Capabilities = {
|
||||
VeeamSOSApi?: VeeamSOSApi,
|
||||
};
|
||||
|
||||
export type ACL = OACL & { WRITE: string[] }
|
||||
|
||||
export default class BucketInfo {
|
||||
|
@ -58,56 +95,68 @@ export default class BucketInfo {
|
|||
_objectLockEnabled?: boolean;
|
||||
_objectLockConfiguration?: any;
|
||||
_notificationConfiguration?: any;
|
||||
_tags?: { key: string; value: string }[] | null;
|
||||
_tags?: Array<BucketTag>;
|
||||
_readLocationConstraint: string | null;
|
||||
_isNFS: boolean | null;
|
||||
_azureInfo: any | null;
|
||||
_ingestion: { status: 'enabled' | 'disabled' } | null;
|
||||
_capabilities?: Capabilities;
|
||||
|
||||
/**
|
||||
* Represents all bucket information.
|
||||
* @constructor
|
||||
* @param {string} name - bucket name
|
||||
* @param {string} owner - bucket owner's name
|
||||
* @param {string} ownerDisplayName - owner's display name
|
||||
* @param {object} creationDate - creation date of bucket
|
||||
* @param {number} mdBucketModelVersion - bucket model version
|
||||
* @param {object} [acl] - bucket ACLs (no need to copy
|
||||
* @param name - bucket name
|
||||
* @param owner - bucket owner's name
|
||||
* @param ownerDisplayName - owner's display name
|
||||
* @param creationDate - creation date of bucket
|
||||
* @param mdBucketModelVersion - bucket model version
|
||||
* @param [acl] - bucket ACLs (no need to copy
|
||||
* ACL object since referenced object will not be used outside of
|
||||
* BucketInfo instance)
|
||||
* @param {boolean} transient - flag indicating whether bucket is transient
|
||||
* @param {boolean} deleted - flag indicating whether attempt to delete
|
||||
* @param {object} serverSideEncryption - sse information for this bucket
|
||||
* @param {number} serverSideEncryption.cryptoScheme -
|
||||
* @param transient - flag indicating whether bucket is transient
|
||||
* @param deleted - flag indicating whether attempt to delete
|
||||
* @param serverSideEncryption - sse information for this bucket
|
||||
* @param serverSideEncryption.cryptoScheme -
|
||||
* cryptoScheme used
|
||||
* @param {string} serverSideEncryption.algorithm -
|
||||
* @param serverSideEncryption.algorithm -
|
||||
* algorithm to use
|
||||
* @param {string} serverSideEncryption.masterKeyId -
|
||||
* @param serverSideEncryption.masterKeyId -
|
||||
* key to get master key
|
||||
* @param {string} serverSideEncryption.configuredMasterKeyId -
|
||||
* @param serverSideEncryption.configuredMasterKeyId -
|
||||
* custom KMS key id specified by user
|
||||
* @param {boolean} serverSideEncryption.mandatory -
|
||||
* @param serverSideEncryption.mandatory -
|
||||
* true for mandatory encryption
|
||||
* bucket has been made
|
||||
* @param {object} versioningConfiguration - versioning configuration
|
||||
* @param {string} versioningConfiguration.Status - versioning status
|
||||
* @param {object} versioningConfiguration.MfaDelete - versioning mfa delete
|
||||
* @param {string} locationConstraint - locationConstraint for bucket
|
||||
* @param {WebsiteConfiguration} [websiteConfiguration] - website
|
||||
* @param versioningConfiguration - versioning configuration
|
||||
* @param versioningConfiguration.Status - versioning status
|
||||
* @param versioningConfiguration.MfaDelete - versioning mfa delete
|
||||
* @param locationConstraint - locationConstraint for bucket that
|
||||
* also includes the ingestion flag
|
||||
* @param [websiteConfiguration] - website
|
||||
* configuration
|
||||
* @param {object[]} [cors] - collection of CORS rules to apply
|
||||
* @param {string} [cors[].id] - optional ID to identify rule
|
||||
* @param {string[]} cors[].allowedMethods - methods allowed for CORS request
|
||||
* @param {string[]} cors[].allowedOrigins - origins allowed for CORS request
|
||||
* @param {string[]} [cors[].allowedHeaders] - headers allowed in an OPTIONS
|
||||
* @param [cors] - collection of CORS rules to apply
|
||||
* @param [cors[].id] - optional ID to identify rule
|
||||
* @param cors[].allowedMethods - methods allowed for CORS request
|
||||
* @param cors[].allowedOrigins - origins allowed for CORS request
|
||||
* @param [cors[].allowedHeaders] - headers allowed in an OPTIONS
|
||||
* request via the Access-Control-Request-Headers header
|
||||
* @param {number} [cors[].maxAgeSeconds] - seconds browsers should cache
|
||||
* @param [cors[].maxAgeSeconds] - seconds browsers should cache
|
||||
* OPTIONS response
|
||||
* @param {string[]} [cors[].exposeHeaders] - headers expose to applications
|
||||
* @param {object} [replicationConfiguration] - replication configuration
|
||||
* @param {object} [lifecycleConfiguration] - lifecycle configuration
|
||||
* @param {object} [bucketPolicy] - bucket policy
|
||||
* @param {string} [uid] - unique identifier for the bucket, necessary
|
||||
* @param {boolean} [objectLockEnabled] - true when object lock enabled
|
||||
* @param {object} [objectLockConfiguration] - object lock configuration
|
||||
* @param {object} [notificationConfiguration] - bucket notification configuration
|
||||
* @param {object[]} [tags] - bucket tags
|
||||
* @param [cors[].exposeHeaders] - headers expose to applications
|
||||
* @param [replicationConfiguration] - replication configuration
|
||||
* @param [lifecycleConfiguration] - lifecycle configuration
|
||||
* @param [bucketPolicy] - bucket policy
|
||||
* @param [uid] - unique identifier for the bucket, necessary
|
||||
* @param readLocationConstraint - readLocationConstraint for bucket
|
||||
* addition for use with lifecycle operations
|
||||
* @param [isNFS] - whether the bucket is on NFS
|
||||
* @param [ingestionConfig] - object for ingestion status: en/dis
|
||||
* @param [azureInfo] - Azure storage account specific info
|
||||
* @param [objectLockEnabled] - true when object lock enabled
|
||||
* @param [objectLockConfiguration] - object lock configuration
|
||||
* @param [notificationConfiguration] - bucket notification configuration
|
||||
* @param [tags] - bucket tag set
|
||||
* @param [capabilities] - capabilities for the bucket
|
||||
*/
|
||||
constructor(
|
||||
name: string,
|
||||
|
@ -127,10 +176,15 @@ export default class BucketInfo {
|
|||
lifecycleConfiguration?: any,
|
||||
bucketPolicy?: any,
|
||||
uid?: string,
|
||||
readLocationConstraint?: string,
|
||||
isNFS?: boolean,
|
||||
ingestionConfig?: { status: 'enabled' | 'disabled' },
|
||||
azureInfo?: any,
|
||||
objectLockEnabled?: boolean,
|
||||
objectLockConfiguration?: any,
|
||||
notificationConfiguration?: any,
|
||||
tags?: { key: string; value: string }[],
|
||||
tags?: Array<BucketTag> | [],
|
||||
capabilities?: Capabilities,
|
||||
) {
|
||||
assert.strictEqual(typeof name, 'string');
|
||||
assert.strictEqual(typeof owner, 'string');
|
||||
|
@ -172,6 +226,15 @@ export default class BucketInfo {
|
|||
if (locationConstraint) {
|
||||
assert.strictEqual(typeof locationConstraint, 'string');
|
||||
}
|
||||
if (ingestionConfig) {
|
||||
assert.strictEqual(typeof ingestionConfig, 'object');
|
||||
}
|
||||
if (azureInfo) {
|
||||
assert.strictEqual(typeof azureInfo, 'object');
|
||||
}
|
||||
if (readLocationConstraint) {
|
||||
assert.strictEqual(typeof readLocationConstraint, 'string');
|
||||
}
|
||||
if (websiteConfiguration) {
|
||||
assert(websiteConfiguration instanceof WebsiteConfiguration);
|
||||
const indexDocument = websiteConfiguration.getIndexDocument();
|
||||
|
@ -217,9 +280,11 @@ export default class BucketInfo {
|
|||
READ: [],
|
||||
READ_ACP: [],
|
||||
};
|
||||
if (tags) {
|
||||
assert(Array.isArray(tags));
|
||||
|
||||
if (tags === undefined) {
|
||||
tags = [] as BucketTag[];
|
||||
}
|
||||
assert.strictEqual(areTagsValid(tags), true);
|
||||
|
||||
// IF UPDATING PROPERTIES, INCREMENT MODELVERSION NUMBER ABOVE
|
||||
this._acl = aclInstance;
|
||||
|
@ -233,16 +298,21 @@ export default class BucketInfo {
|
|||
this._serverSideEncryption = serverSideEncryption || null;
|
||||
this._versioningConfiguration = versioningConfiguration || null;
|
||||
this._locationConstraint = locationConstraint || null;
|
||||
this._readLocationConstraint = readLocationConstraint || null;
|
||||
this._websiteConfiguration = websiteConfiguration || null;
|
||||
this._replicationConfiguration = replicationConfiguration || null;
|
||||
this._cors = cors || null;
|
||||
this._lifecycleConfiguration = lifecycleConfiguration || null;
|
||||
this._bucketPolicy = bucketPolicy || null;
|
||||
this._uid = uid || uuid();
|
||||
this._isNFS = isNFS || null;
|
||||
this._ingestion = ingestionConfig || null;
|
||||
this._azureInfo = azureInfo || null;
|
||||
this._objectLockEnabled = objectLockEnabled || false;
|
||||
this._objectLockConfiguration = objectLockConfiguration || null;
|
||||
this._notificationConfiguration = notificationConfiguration || null;
|
||||
this._tags = tags || null;
|
||||
this._tags = tags;
|
||||
this._capabilities = capabilities || undefined;
|
||||
return this;
|
||||
}
|
||||
|
||||
|
@ -263,16 +333,21 @@ export default class BucketInfo {
|
|||
serverSideEncryption: this._serverSideEncryption,
|
||||
versioningConfiguration: this._versioningConfiguration,
|
||||
locationConstraint: this._locationConstraint,
|
||||
readLocationConstraint: this._readLocationConstraint,
|
||||
websiteConfiguration: undefined,
|
||||
cors: this._cors,
|
||||
replicationConfiguration: this._replicationConfiguration,
|
||||
lifecycleConfiguration: this._lifecycleConfiguration,
|
||||
bucketPolicy: this._bucketPolicy,
|
||||
uid: this._uid,
|
||||
isNFS: this._isNFS,
|
||||
ingestion: this._ingestion,
|
||||
azureInfo: this._azureInfo,
|
||||
objectLockEnabled: this._objectLockEnabled,
|
||||
objectLockConfiguration: this._objectLockConfiguration,
|
||||
notificationConfiguration: this._notificationConfiguration,
|
||||
tags: this._tags,
|
||||
capabilities: this._capabilities,
|
||||
};
|
||||
const final = this._websiteConfiguration
|
||||
? {
|
||||
|
@ -296,8 +371,10 @@ export default class BucketInfo {
|
|||
obj.transient, obj.deleted, obj.serverSideEncryption,
|
||||
obj.versioningConfiguration, obj.locationConstraint, websiteConfig,
|
||||
obj.cors, obj.replicationConfiguration, obj.lifecycleConfiguration,
|
||||
obj.bucketPolicy, obj.uid, obj.objectLockEnabled,
|
||||
obj.objectLockConfiguration, obj.notificationConfiguration, obj.tags);
|
||||
obj.bucketPolicy, obj.uid, obj.readLocationConstraint, obj.isNFS,
|
||||
obj.ingestion, obj.azureInfo, obj.objectLockEnabled,
|
||||
obj.objectLockConfiguration, obj.notificationConfiguration, obj.tags,
|
||||
obj.capabilities);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -321,8 +398,10 @@ export default class BucketInfo {
|
|||
data._versioningConfiguration, data._locationConstraint,
|
||||
data._websiteConfiguration, data._cors,
|
||||
data._replicationConfiguration, data._lifecycleConfiguration,
|
||||
data._bucketPolicy, data._uid, data._objectLockEnabled,
|
||||
data._objectLockConfiguration, data._notificationConfiguration, data._tags);
|
||||
data._bucketPolicy, data._uid, data._readLocationConstraint,
|
||||
data._isNFS, data._ingestion, data._azureInfo,
|
||||
data._objectLockEnabled, data._objectLockConfiguration,
|
||||
data._notificationConfiguration, data._tags, data._capabilities);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -619,6 +698,17 @@ export default class BucketInfo {
|
|||
return this._locationConstraint;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get read location constraint.
|
||||
* @return - bucket read location constraint
|
||||
*/
|
||||
getReadLocationConstraint() {
|
||||
if (this._readLocationConstraint) {
|
||||
return this._readLocationConstraint;
|
||||
}
|
||||
return this._locationConstraint;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set Bucket model version
|
||||
*
|
||||
|
@ -707,6 +797,85 @@ export default class BucketInfo {
|
|||
this._uid = uid;
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Check if the bucket is an NFS bucket.
|
||||
* @return - Wether the bucket is NFS or not
|
||||
*/
|
||||
isNFS() {
|
||||
return this._isNFS;
|
||||
}
|
||||
/**
|
||||
* Set whether the bucket is an NFS bucket.
|
||||
* @param isNFS - Wether the bucket is NFS or not
|
||||
* @return - bucket info instance
|
||||
*/
|
||||
setIsNFS(isNFS: boolean) {
|
||||
this._isNFS = isNFS;
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* enable ingestion, set 'this._ingestion' to { status: 'enabled' }
|
||||
* @return - bucket info instance
|
||||
*/
|
||||
enableIngestion() {
|
||||
this._ingestion = { status: 'enabled' };
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* disable ingestion, set 'this._ingestion' to { status: 'disabled' }
|
||||
* @return - bucket info instance
|
||||
*/
|
||||
disableIngestion() {
|
||||
this._ingestion = { status: 'disabled' };
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Get ingestion configuration
|
||||
* @return - bucket ingestion configuration: Enabled or Disabled
|
||||
*/
|
||||
getIngestion() {
|
||||
return this._ingestion;
|
||||
}
|
||||
|
||||
/**
|
||||
** Check if bucket is an ingestion bucket
|
||||
* @return - 'true' if bucket is ingestion bucket, 'false' if
|
||||
* otherwise
|
||||
*/
|
||||
isIngestionBucket() {
|
||||
const ingestionConfig = this.getIngestion();
|
||||
if (ingestionConfig) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
/**
|
||||
* Check if ingestion is enabled
|
||||
* @return - 'true' if ingestion is enabled, otherwise 'false'
|
||||
*/
|
||||
isIngestionEnabled() {
|
||||
const ingestionConfig = this.getIngestion();
|
||||
return ingestionConfig ? ingestionConfig.status === 'enabled' : false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the Azure specific storage account information for this bucket
|
||||
* @return - a structure suitable for {@link BucketAzureIno}
|
||||
* constructor
|
||||
*/
|
||||
getAzureInfo() {
|
||||
return this._azureInfo;
|
||||
}
|
||||
/**
|
||||
* Set the Azure specific storage account information for this bucket
|
||||
* @param azureInfo - a structure suitable for
|
||||
* {@link BucketAzureInfo} construction
|
||||
* @return - bucket info instance
|
||||
*/
|
||||
setAzureInfo(azureInfo: any) {
|
||||
this._azureInfo = azureInfo;
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* Check if object lock is enabled.
|
||||
* @return - depending on whether object lock is enabled
|
||||
|
@ -726,21 +895,48 @@ export default class BucketInfo {
|
|||
|
||||
/**
|
||||
* Get the value of bucket tags
|
||||
* @return - Array of bucket tags as {"key" : "key", "value": "value"}
|
||||
* @return - Array of bucket tags
|
||||
*/
|
||||
getTags() {
|
||||
return this._tags;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Set bucket tags
|
||||
* @param tags - collection of tags
|
||||
* @param tags[].key - key of the tag
|
||||
* @param tags[].value - value of the tag
|
||||
* @return - bucket info instance
|
||||
*/
|
||||
setTags(tags: { key: string; value: string }[]) {
|
||||
setTags(tags: Array<BucketTag>) {
|
||||
this._tags = tags;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the value of bucket capabilities
|
||||
* @return - capabilities of the bucket
|
||||
*/
|
||||
getCapabilities() {
|
||||
return this._capabilities;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a specific bucket capability
|
||||
*
|
||||
* @param capability? - if provided, will return a specific capacity
|
||||
* @return - capability of the bucket
|
||||
*/
|
||||
getCapability(capability: string) : VeeamSOSApi | undefined {
|
||||
if (capability && this._capabilities && this._capabilities[capability]) {
|
||||
return this._capabilities[capability];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set bucket capabilities
|
||||
* @return - bucket info instance
|
||||
*/
|
||||
setCapabilities(capabilities: Capabilities) {
|
||||
this._capabilities = capabilities;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -7,6 +7,8 @@ import escapeForXml from '../s3middleware/escapeForXml';
|
|||
import type { XMLRule } from './ReplicationConfiguration';
|
||||
import { Status } from './LifecycleRule';
|
||||
|
||||
const MAX_DAYS = 2147483647; // Max 32-bit signed binary integer.
|
||||
|
||||
/**
|
||||
* Format of xml request:
|
||||
|
||||
|
@ -87,6 +89,7 @@ export default class LifecycleConfiguration {
|
|||
_parsedXML: any;
|
||||
_ruleIDs: string[];
|
||||
_tagKeys: string[];
|
||||
_storageClasses: string[];
|
||||
_config: {
|
||||
error?: ArsenalError;
|
||||
rules?: any[];
|
||||
|
@ -95,10 +98,13 @@ export default class LifecycleConfiguration {
|
|||
/**
|
||||
* Create a Lifecycle Configuration instance
|
||||
* @param xml - the parsed xml
|
||||
* @param config - the CloudServer config
|
||||
* @return - LifecycleConfiguration instance
|
||||
*/
|
||||
constructor(xml: any) {
|
||||
constructor(xml: any, config: { replicationEndpoints: { site: string }[] }) {
|
||||
this._parsedXML = xml;
|
||||
this._storageClasses =
|
||||
config.replicationEndpoints.map(endpoint => endpoint.site);
|
||||
this._ruleIDs = [];
|
||||
this._tagKeys = [];
|
||||
this._config = {};
|
||||
|
@ -219,11 +225,6 @@ export default class LifecycleConfiguration {
|
|||
* }
|
||||
*/
|
||||
_parseRule(rule: XMLRule) {
|
||||
if (rule.Transition || rule.NoncurrentVersionTransition) {
|
||||
const msg = 'Transition lifecycle action not yet implemented';
|
||||
const error = errors.NotImplemented.customizeDescription(msg);
|
||||
return { error };
|
||||
}
|
||||
// Either Prefix or Filter must be included, but can be empty string
|
||||
if ((!rule.Filter && rule.Filter !== '') &&
|
||||
(!rule.Prefix && rule.Prefix !== '')) {
|
||||
|
@ -492,6 +493,172 @@ export default class LifecycleConfiguration {
|
|||
return { ...base, ruleStatus: status }
|
||||
}
|
||||
|
||||
/**
|
||||
* Finds the prefix and/or tags of the given rule and gets the error message
|
||||
* @param rule - The rule to find the prefix in
|
||||
* @return - The prefix of filter information
|
||||
*/
|
||||
_getRuleFilterDesc(rule: { Prefix?: string[]; Filter?: any[] }) {
|
||||
if (rule.Prefix) {
|
||||
return `prefix '${rule.Prefix[0]}'`;
|
||||
}
|
||||
// There must be a filter if no top-level prefix is provided. First
|
||||
// check if there are multiple filters (i.e. `Filter.And`).
|
||||
if (rule.Filter?.[0] === undefined || rule.Filter[0].And === undefined) {
|
||||
const { Prefix, Tag } = rule.Filter?.[0] || {};
|
||||
if (Prefix) {
|
||||
return `filter '(prefix=${Prefix[0]})'`;
|
||||
}
|
||||
if (Tag) {
|
||||
const { Key, Value } = Tag[0];
|
||||
return `filter '(tag: key=${Key[0]}, value=${Value[0]})'`;
|
||||
}
|
||||
return 'filter (all)';
|
||||
}
|
||||
const filters: string[] = [];
|
||||
const { Prefix, Tag } = rule.Filter[0].And[0];
|
||||
if (Prefix) {
|
||||
filters.push(`prefix=${Prefix[0]}`);
|
||||
}
|
||||
Tag.forEach((tag: { Key: string[]; Value: string[] }) => {
|
||||
const { Key, Value } = tag;
|
||||
filters.push(`tag: key=${Key[0]}, value=${Value[0]}`);
|
||||
});
|
||||
const joinedFilters = filters.join(' and ');
|
||||
return `filter '(${joinedFilters})'`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the validity of the given field
|
||||
* @param params - Given function parameters
|
||||
* @param params.days - The value of the field to check
|
||||
* @param params.field - The field name with the value
|
||||
* @param params.ancestor - The immediate ancestor field
|
||||
* @return Returns an error object or `null`
|
||||
*/
|
||||
_checkDays(params: { days: number; field: string; ancestor: string }) {
|
||||
const { days, field, ancestor } = params;
|
||||
if (days < 0) {
|
||||
const msg = `'${field}' in ${ancestor} action must be nonnegative`;
|
||||
return errors.InvalidArgument.customizeDescription(msg);
|
||||
}
|
||||
if (days > MAX_DAYS) {
|
||||
return errors.MalformedXML.customizeDescription(
|
||||
`'${field}' in ${ancestor} action must not exceed ${MAX_DAYS}`);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the validity of the given storage class
|
||||
* @param params - Given function parameters
|
||||
* @param params.usedStorageClasses - Storage classes used in other
|
||||
* rules
|
||||
* @param params.storageClass - The storage class of the current
|
||||
* rule
|
||||
* @param params.ancestor - The immediate ancestor field
|
||||
* @param params.prefix - The prefix of the rule
|
||||
* @return Returns an error object or `null`
|
||||
*/
|
||||
_checkStorageClasses(params: {
|
||||
usedStorageClasses: string[];
|
||||
storageClass: string;
|
||||
ancestor: string;
|
||||
rule: { Prefix?: string[]; Filter?: any };
|
||||
}) {
|
||||
const { usedStorageClasses, storageClass, ancestor, rule } = params;
|
||||
if (!this._storageClasses.includes(storageClass)) {
|
||||
// This differs from the AWS message. This will help the user since
|
||||
// the StorageClass does not conform to AWS specs.
|
||||
const list = `'${this._storageClasses.join("', '")}'`;
|
||||
const msg = `'StorageClass' must be one of ${list}`;
|
||||
return errors.MalformedXML.customizeDescription(msg);
|
||||
}
|
||||
if (usedStorageClasses.includes(storageClass)) {
|
||||
const msg = `'StorageClass' must be different for '${ancestor}' ` +
|
||||
`actions in same 'Rule' with ${this._getRuleFilterDesc(rule)}`;
|
||||
return errors.InvalidRequest.customizeDescription(msg);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure that transition rules are at least a day apart from each other.
|
||||
* @param params - Given function parameters
|
||||
* @param [params.days] - The days of the current transition
|
||||
* @param [params.date] - The date of the current transition
|
||||
* @param params.storageClass - The storage class of the current
|
||||
* rule
|
||||
* @param params.rule - The current rule
|
||||
*/
|
||||
_checkTimeGap(params: {
|
||||
days?: number;
|
||||
date?: string;
|
||||
storageClass: string;
|
||||
rule: { Transition: any[]; Prefix?: string[]; Filter?: any };
|
||||
}) {
|
||||
const { days, date, storageClass, rule } = params;
|
||||
const invalidTransition = rule.Transition.find(transition => {
|
||||
if (storageClass === transition.StorageClass[0]) {
|
||||
return false;
|
||||
}
|
||||
if (days !== undefined) {
|
||||
return Number.parseInt(transition.Days[0], 10) === days;
|
||||
}
|
||||
if (date !== undefined) {
|
||||
const timestamp = new Date(date).getTime();
|
||||
const compareTimestamp = new Date(transition.Date[0]).getTime();
|
||||
const oneDay = 24 * 60 * 60 * 1000; // Milliseconds in a day.
|
||||
return Math.abs(timestamp - compareTimestamp) < oneDay;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
if (invalidTransition) {
|
||||
const timeType = days !== undefined ? 'Days' : 'Date';
|
||||
const filterMsg = this._getRuleFilterDesc(rule);
|
||||
const compareStorageClass = invalidTransition.StorageClass[0];
|
||||
const msg = `'${timeType}' in the 'Transition' action for ` +
|
||||
`StorageClass '${storageClass}' for ${filterMsg} must be at ` +
|
||||
`least one day apart from ${filterMsg} in the 'Transition' ` +
|
||||
`action for StorageClass '${compareStorageClass}'`;
|
||||
return errors.InvalidArgument.customizeDescription(msg);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks transition time type (i.e. 'Date' or 'Days') only occurs once
|
||||
* across transitions and across transitions and expiration policies
|
||||
* @param params - Given function parameters
|
||||
* @param params.usedTimeType - The time type that has been used by
|
||||
* another rule
|
||||
* @param params.currentTimeType - the time type used by the
|
||||
* current rule
|
||||
* @param params.rule - The current rule
|
||||
* @return Returns an error object or `null`
|
||||
*/
|
||||
_checkTimeType(params: {
|
||||
usedTimeType: string | null;
|
||||
currentTimeType: string;
|
||||
rule: { Prefix?: string[]; Filter?: any; Expiration?: any[] };
|
||||
}) {
|
||||
const { usedTimeType, currentTimeType, rule } = params;
|
||||
if (usedTimeType && usedTimeType !== currentTimeType) {
|
||||
const msg = "Found mixed 'Date' and 'Days' based Transition " +
|
||||
'actions in lifecycle rule for ' +
|
||||
`${this._getRuleFilterDesc(rule)}`;
|
||||
return errors.InvalidRequest.customizeDescription(msg);
|
||||
}
|
||||
// Transition time type cannot differ from the expiration, if provided.
|
||||
if (rule.Expiration &&
|
||||
rule.Expiration[0][currentTimeType] === undefined) {
|
||||
const msg = "Found mixed 'Date' and 'Days' based Expiration and " +
|
||||
'Transition actions in lifecycle rule for ' +
|
||||
`${this._getRuleFilterDesc(rule)}`;
|
||||
return errors.InvalidRequest.customizeDescription(msg);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks the validity of the given date
|
||||
|
@ -533,6 +700,159 @@ export default class LifecycleConfiguration {
|
|||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the NonCurrentVersionTransition value
|
||||
* @param rule - Rule object from Rule array from this._parsedXml
|
||||
* @return - Contains error if parsing failed, otherwise contains
|
||||
* the parsed nonCurrentVersionTransition array
|
||||
*
|
||||
* Format of result:
|
||||
* result = {
|
||||
* error: <error>,
|
||||
* nonCurrentVersionTransition: [
|
||||
* {
|
||||
* noncurrentDays: <non-current-days>,
|
||||
* storageClass: <storage-class>,
|
||||
* },
|
||||
* ...
|
||||
* ]
|
||||
* }
|
||||
*/
|
||||
_parseNoncurrentVersionTransition(rule: {
|
||||
NoncurrentVersionTransition: any[];
|
||||
Prefix?: string[];
|
||||
Filter?: any;
|
||||
}) {
|
||||
const nonCurrentVersionTransition: {
|
||||
noncurrentDays: number;
|
||||
storageClass: string;
|
||||
}[] = [];
|
||||
const usedStorageClasses: string[] = [];
|
||||
for (let i = 0; i < rule.NoncurrentVersionTransition.length; i++) {
|
||||
const t = rule.NoncurrentVersionTransition[i]; // Transition object
|
||||
const noncurrentDays: number | undefined =
|
||||
t.NoncurrentDays && Number.parseInt(t.NoncurrentDays[0], 10);
|
||||
const storageClass: string | undefined = t.StorageClass && t.StorageClass[0];
|
||||
if (noncurrentDays === undefined || storageClass === undefined) {
|
||||
return { error: errors.MalformedXML };
|
||||
}
|
||||
let error = this._checkDays({
|
||||
days: noncurrentDays,
|
||||
field: 'NoncurrentDays',
|
||||
ancestor: 'NoncurrentVersionTransition',
|
||||
});
|
||||
if (error) {
|
||||
return { error };
|
||||
}
|
||||
error = this._checkStorageClasses({
|
||||
storageClass,
|
||||
usedStorageClasses,
|
||||
ancestor: 'NoncurrentVersionTransition',
|
||||
rule,
|
||||
});
|
||||
if (error) {
|
||||
return { error };
|
||||
}
|
||||
nonCurrentVersionTransition.push({ noncurrentDays, storageClass });
|
||||
usedStorageClasses.push(storageClass);
|
||||
}
|
||||
return { nonCurrentVersionTransition };
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses the Transition value
|
||||
* @param rule - Rule object from Rule array from this._parsedXml
|
||||
* @return - Contains error if parsing failed, otherwise contains
|
||||
* the parsed transition array
|
||||
*
|
||||
* Format of result:
|
||||
* result = {
|
||||
* error: <error>,
|
||||
* transition: [
|
||||
* {
|
||||
* days: <days>,
|
||||
* date: <date>,
|
||||
* storageClass: <storage-class>,
|
||||
* },
|
||||
* ...
|
||||
* ]
|
||||
* }
|
||||
*/
|
||||
_parseTransition(rule: {
|
||||
Transition: any[];
|
||||
Prefix?: string[];
|
||||
Filter?: any;
|
||||
}) {
|
||||
const transition:
|
||||
({ days: number; storageClass: string }
|
||||
| { date: string; storageClass: string })[] = [];
|
||||
const usedStorageClasses: string[] = [];
|
||||
let usedTimeType: string | null = null;
|
||||
for (let i = 0; i < rule.Transition.length; i++) {
|
||||
const t = rule.Transition[i]; // Transition object
|
||||
const days = t.Days && Number.parseInt(t.Days[0], 10);
|
||||
const date = t.Date && t.Date[0];
|
||||
const storageClass = t.StorageClass && t.StorageClass[0];
|
||||
if ((days === undefined && date === undefined) ||
|
||||
(days !== undefined && date !== undefined) ||
|
||||
(storageClass === undefined)) {
|
||||
return { error: errors.MalformedXML };
|
||||
}
|
||||
let error = this._checkStorageClasses({
|
||||
storageClass,
|
||||
usedStorageClasses,
|
||||
ancestor: 'Transition',
|
||||
rule,
|
||||
});
|
||||
if (error) {
|
||||
return { error };
|
||||
}
|
||||
usedStorageClasses.push(storageClass);
|
||||
if (days !== undefined) {
|
||||
error = this._checkTimeType({
|
||||
usedTimeType,
|
||||
currentTimeType: 'Days',
|
||||
rule,
|
||||
});
|
||||
if (error) {
|
||||
return { error };
|
||||
}
|
||||
usedTimeType = 'Days';
|
||||
error = this._checkDays({
|
||||
days,
|
||||
field: 'Days',
|
||||
ancestor: 'Transition',
|
||||
});
|
||||
if (error) {
|
||||
return { error };
|
||||
}
|
||||
transition.push({ days, storageClass });
|
||||
}
|
||||
if (date !== undefined) {
|
||||
error = this._checkTimeType({
|
||||
usedTimeType,
|
||||
currentTimeType: 'Date',
|
||||
rule,
|
||||
});
|
||||
if (error) {
|
||||
return { error };
|
||||
}
|
||||
usedTimeType = 'Date';
|
||||
error = this._checkDate(date);
|
||||
if (error) {
|
||||
return { error };
|
||||
}
|
||||
transition.push({ date, storageClass });
|
||||
}
|
||||
error = this._checkTimeGap({ days, date, storageClass, rule });
|
||||
if (error) {
|
||||
return { error };
|
||||
}
|
||||
}
|
||||
return { transition };
|
||||
}
|
||||
|
||||
/**
|
||||
* Check that action component of rule is valid
|
||||
* @param rule - a rule object from Rule array from this._parsedXml
|
||||
|
@ -569,8 +889,13 @@ export default class LifecycleConfiguration {
|
|||
propName: 'actions',
|
||||
actions: [],
|
||||
};
|
||||
const validActions = ['AbortIncompleteMultipartUpload',
|
||||
'Expiration', 'NoncurrentVersionExpiration'];
|
||||
const validActions = [
|
||||
'AbortIncompleteMultipartUpload',
|
||||
'Expiration',
|
||||
'NoncurrentVersionExpiration',
|
||||
'NoncurrentVersionTransition',
|
||||
'Transition',
|
||||
];
|
||||
validActions.forEach(a => {
|
||||
if (rule[a]) {
|
||||
actionsObj.actions.push({ actionName: `${a}` });
|
||||
|
@ -587,7 +912,14 @@ export default class LifecycleConfiguration {
|
|||
if (action.error) {
|
||||
actionsObj.error = action.error;
|
||||
} else {
|
||||
const actionTimes = ['days', 'date', 'deleteMarker', 'newerNoncurrentVersions'];
|
||||
const actionTimes = [
|
||||
'days',
|
||||
'date',
|
||||
'deleteMarker',
|
||||
'transition',
|
||||
'nonCurrentVersionTransition',
|
||||
'newerNoncurrentVersions'
|
||||
];
|
||||
actionTimes.forEach(t => {
|
||||
if (action[t]) {
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
|
@ -821,6 +1153,26 @@ export default class LifecycleConfiguration {
|
|||
if (a.deleteMarker) {
|
||||
assert.strictEqual(typeof a.deleteMarker, 'string');
|
||||
}
|
||||
if (a.nonCurrentVersionTransition) {
|
||||
assert.strictEqual(
|
||||
typeof a.nonCurrentVersionTransition, 'object');
|
||||
a.nonCurrentVersionTransition.forEach(t => {
|
||||
assert.strictEqual(typeof t.noncurrentDays, 'number');
|
||||
assert.strictEqual(typeof t.storageClass, 'string');
|
||||
});
|
||||
}
|
||||
if (a.transition) {
|
||||
assert.strictEqual(typeof a.transition, 'object');
|
||||
a.transition.forEach(t => {
|
||||
if (t.days || t.days === 0) {
|
||||
assert.strictEqual(typeof t.days, 'number');
|
||||
}
|
||||
if (t.date !== undefined) {
|
||||
assert.strictEqual(typeof t.date, 'string');
|
||||
}
|
||||
assert.strictEqual(typeof t.storageClass, 'string');
|
||||
});
|
||||
}
|
||||
|
||||
if (a.newerNoncurrentVersions) {
|
||||
assert.strictEqual(typeof a.newerNoncurrentVersions, 'number');
|
||||
|
@ -874,7 +1226,15 @@ export default class LifecycleConfiguration {
|
|||
}
|
||||
|
||||
const Actions = actions.map(action => {
|
||||
const { actionName, days, date, deleteMarker, newerNoncurrentVersions } = action;
|
||||
const {
|
||||
actionName,
|
||||
days,
|
||||
date,
|
||||
deleteMarker,
|
||||
nonCurrentVersionTransition,
|
||||
transition,
|
||||
newerNoncurrentVersions,
|
||||
} = action;
|
||||
let Action: any;
|
||||
if (actionName === 'AbortIncompleteMultipartUpload') {
|
||||
Action = `<${actionName}><DaysAfterInitiation>${days}` +
|
||||
|
@ -893,6 +1253,40 @@ export default class LifecycleConfiguration {
|
|||
Action = `<${actionName}>${Days}${Date}${DelMarker}` +
|
||||
`</${actionName}>`;
|
||||
}
|
||||
if (actionName === 'NoncurrentVersionTransition') {
|
||||
const xml: string[] = [];
|
||||
nonCurrentVersionTransition!.forEach(transition => {
|
||||
const { noncurrentDays, storageClass } = transition;
|
||||
xml.push(
|
||||
`<${actionName}>`,
|
||||
`<NoncurrentDays>${noncurrentDays}` +
|
||||
'</NoncurrentDays>',
|
||||
`<StorageClass>${storageClass}</StorageClass>`,
|
||||
`</${actionName}>`,
|
||||
);
|
||||
});
|
||||
Action = xml.join('');
|
||||
}
|
||||
if (actionName === 'Transition') {
|
||||
const xml: string[] = [];
|
||||
transition!.forEach(transition => {
|
||||
const { days, date, storageClass } = transition;
|
||||
let element: string = '';
|
||||
if (days !== undefined) {
|
||||
element = `<Days>${days}</Days>`;
|
||||
}
|
||||
if (date !== undefined) {
|
||||
element = `<Date>${date}</Date>`;
|
||||
}
|
||||
xml.push(
|
||||
`<${actionName}>`,
|
||||
element,
|
||||
`<StorageClass>${storageClass}</StorageClass>`,
|
||||
`</${actionName}>`,
|
||||
);
|
||||
});
|
||||
Action = xml.join('');
|
||||
}
|
||||
return Action;
|
||||
}).join('');
|
||||
return `<Rule>${ID}${Status}${Filter}${Actions}</Rule>`;
|
||||
|
@ -975,6 +1369,15 @@ export type Rule = {
|
|||
date?: number;
|
||||
deleteMarker?: boolean;
|
||||
newerNoncurrentVersions?: number;
|
||||
nonCurrentVersionTransition?: {
|
||||
noncurrentDays: number;
|
||||
storageClass: string;
|
||||
}[];
|
||||
transition?: {
|
||||
days?: number;
|
||||
date?: string;
|
||||
storageClass: string;
|
||||
}[];
|
||||
}[];
|
||||
filter?: {
|
||||
rulePrefix?: string;
|
||||
|
|
|
@ -28,6 +28,7 @@ export default class LifecycleRule {
|
|||
ncvExpiration?: NoncurrentExpiration;
|
||||
abortMPU?: { DaysAfterInitiation: number };
|
||||
transitions?: any[];
|
||||
ncvTransitions?: any[];
|
||||
prefix?: string;
|
||||
|
||||
constructor(id: string, status: Status) {
|
||||
|
@ -45,6 +46,7 @@ export default class LifecycleRule {
|
|||
NoncurrentVersionExpiration?: NoncurrentExpiration;
|
||||
AbortIncompleteMultipartUpload?: { DaysAfterInitiation: number };
|
||||
Transitions?: any[];
|
||||
NoncurrentVersionTransitions?: any[];
|
||||
Filter?: Filter;
|
||||
Prefix?: '';
|
||||
} = { ID: this.id, Status: this.status };
|
||||
|
@ -61,6 +63,9 @@ export default class LifecycleRule {
|
|||
if (this.transitions) {
|
||||
rule.Transitions = this.transitions;
|
||||
}
|
||||
if (this.ncvTransitions) {
|
||||
rule.NoncurrentVersionTransitions = this.ncvTransitions;
|
||||
}
|
||||
|
||||
const filter = this.buildFilter();
|
||||
|
||||
|
@ -173,4 +178,13 @@ export default class LifecycleRule {
|
|||
this.transitions = transitions;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* NonCurrentVersionTransitions
|
||||
* @param nvcTransitions - NonCurrentVersionTransitions
|
||||
*/
|
||||
addNCVTransitions(nvcTransitions) {
|
||||
this.ncvTransitions = nvcTransitions;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,9 +1,13 @@
|
|||
import * as crypto from 'crypto';
|
||||
import * as constants from '../constants';
|
||||
import * as VersionIDUtils from '../versioning/VersionID';
|
||||
import { VersioningConstants } from '../versioning/constants';
|
||||
import ObjectMDLocation, {
|
||||
ObjectMDLocationData,
|
||||
Location,
|
||||
} from './ObjectMDLocation';
|
||||
import ObjectMDAmzRestore from './ObjectMDAmzRestore';
|
||||
import ObjectMDArchive from './ObjectMDArchive';
|
||||
|
||||
export type ACL = {
|
||||
Canned: string;
|
||||
|
@ -28,6 +32,7 @@ export type ReplicationInfo = {
|
|||
role: string;
|
||||
storageType: string;
|
||||
dataStoreVersionId: string;
|
||||
isNFS: boolean | null;
|
||||
};
|
||||
|
||||
export type ObjectMDData = {
|
||||
|
@ -35,30 +40,33 @@ export type ObjectMDData = {
|
|||
'owner-id': string;
|
||||
'cache-control': string;
|
||||
'content-disposition': string;
|
||||
'content-language': string;
|
||||
'content-encoding': string;
|
||||
'creation-time'?: string;
|
||||
'last-modified'?: string;
|
||||
expires: string;
|
||||
'content-length': number;
|
||||
'content-type': string;
|
||||
'content-md5': string;
|
||||
// simple/no version. will expand once object versioning is
|
||||
// introduced
|
||||
'x-amz-version-id': 'null' | string;
|
||||
'x-amz-server-version-id': string;
|
||||
// TODO: Handle this as a utility function for all object puts
|
||||
// similar to normalizing request but after checkAuth so
|
||||
// string to sign is not impacted. This is GH Issue#89.
|
||||
'x-amz-restore'?: ObjectMDAmzRestore;
|
||||
archive?: ObjectMDArchive;
|
||||
'x-amz-storage-class': string;
|
||||
'x-amz-server-side-encryption': string;
|
||||
'x-amz-server-side-encryption-aws-kms-key-id': string;
|
||||
'x-amz-server-side-encryption-customer-algorithm': string;
|
||||
'x-amz-website-redirect-location': string;
|
||||
'x-amz-scal-transition-in-progress'?: boolean;
|
||||
'x-amz-scal-transition-time'?: string;
|
||||
azureInfo?: any;
|
||||
acl: ACL;
|
||||
key: string;
|
||||
location: null | Location[];
|
||||
// versionId, isNull, nullVersionId and isDeleteMarker
|
||||
// versionId, isNull, isNull2, nullVersionId and isDeleteMarker
|
||||
// should be undefined when not set explicitly
|
||||
isNull?: boolean;
|
||||
isNull2?: boolean;
|
||||
nullVersionId?: string;
|
||||
nullUploadId?: string;
|
||||
isDeleteMarker?: boolean;
|
||||
|
@ -71,6 +79,17 @@ export type ObjectMDData = {
|
|||
replicationInfo: ReplicationInfo;
|
||||
dataStoreName: string;
|
||||
originOp: string;
|
||||
microVersionId?: string;
|
||||
// Deletion flag
|
||||
// Used for keeping object metadata in the oplog event
|
||||
// In case of a deletion the flag is first updated before
|
||||
// deleting the object
|
||||
deleted: boolean;
|
||||
// PHD flag indicates whether the object is a temporary placeholder.
|
||||
// This is the case when the latest version of an object gets deleted
|
||||
// the master is set as a placeholder and gets updated with the new latest
|
||||
// version data after a certain amount of time.
|
||||
isPHD: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
|
@ -99,9 +118,17 @@ export default class ObjectMD {
|
|||
} else {
|
||||
this._updateFromParsedJSON(objMd);
|
||||
}
|
||||
if (!this._data['creation-time']) {
|
||||
const lastModified = this.getLastModified();
|
||||
if (lastModified) {
|
||||
this.setCreationTime(lastModified);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// set newly-created object md modified time to current time
|
||||
this._data['last-modified'] = new Date().toJSON();
|
||||
const dt = new Date().toJSON();
|
||||
this.setLastModified(dt);
|
||||
this.setCreationTime(dt);
|
||||
}
|
||||
// set latest md model version now that we ensured
|
||||
// backward-compat conversion
|
||||
|
@ -156,6 +183,8 @@ export default class ObjectMD {
|
|||
'content-length': 0,
|
||||
'content-type': '',
|
||||
'content-md5': '',
|
||||
'content-language': '',
|
||||
'creation-time': undefined,
|
||||
// simple/no version. will expand once object versioning is
|
||||
// introduced
|
||||
'x-amz-version-id': 'null',
|
||||
|
@ -168,6 +197,7 @@ export default class ObjectMD {
|
|||
'x-amz-server-side-encryption-aws-kms-key-id': '',
|
||||
'x-amz-server-side-encryption-customer-algorithm': '',
|
||||
'x-amz-website-redirect-location': '',
|
||||
'x-amz-scal-transition-in-progress': false,
|
||||
acl: {
|
||||
Canned: 'private',
|
||||
FULL_CONTROL: [],
|
||||
|
@ -177,9 +207,11 @@ export default class ObjectMD {
|
|||
},
|
||||
key: '',
|
||||
location: null,
|
||||
azureInfo: undefined,
|
||||
// versionId, isNull, nullVersionId and isDeleteMarker
|
||||
// should be undefined when not set explicitly
|
||||
isNull: undefined,
|
||||
isNull2: undefined,
|
||||
nullVersionId: undefined,
|
||||
nullUploadId: undefined,
|
||||
isDeleteMarker: undefined,
|
||||
|
@ -195,9 +227,12 @@ export default class ObjectMD {
|
|||
role: '',
|
||||
storageType: '',
|
||||
dataStoreVersionId: '',
|
||||
isNFS: null,
|
||||
},
|
||||
dataStoreName: '',
|
||||
originOp: '',
|
||||
deleted: false,
|
||||
isPHD: false,
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -427,6 +462,50 @@ export default class ObjectMD {
|
|||
return this._data['content-md5'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Set content-language
|
||||
*
|
||||
* @param contentLanguage - content-language
|
||||
* @return itself
|
||||
*/
|
||||
setContentLanguage(contentLanguage: string) {
|
||||
this._data['content-language'] = contentLanguage;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns content-language
|
||||
*
|
||||
* @return content-language
|
||||
*/
|
||||
getContentLanguage() {
|
||||
return this._data['content-language'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Set Creation Date
|
||||
*
|
||||
* @param creationTime - Creation Date
|
||||
* @return itself
|
||||
*/
|
||||
setCreationTime(creationTime: string) {
|
||||
this._data['creation-time'] = creationTime;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns Creation Date
|
||||
*
|
||||
* @return Creation Date
|
||||
*/
|
||||
getCreationTime() {
|
||||
// If creation-time is not set fallback to LastModified
|
||||
if (!this._data['creation-time']) {
|
||||
return this.getLastModified();
|
||||
}
|
||||
return this._data['creation-time'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Set version id
|
||||
*
|
||||
|
@ -567,6 +646,48 @@ export default class ObjectMD {
|
|||
return this._data['x-amz-website-redirect-location'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Set metadata transition in progress value
|
||||
*
|
||||
* @param inProgress - True if transition is in progress, false otherwise
|
||||
* @param transitionTime - Date when the transition started
|
||||
* @return itself
|
||||
*/
|
||||
setTransitionInProgress(inProgress: false): this
|
||||
setTransitionInProgress(inProgress: true, transitionTime: Date|string|number): this
|
||||
setTransitionInProgress(inProgress: boolean, transitionTime?: Date|string|number) {
|
||||
this._data['x-amz-scal-transition-in-progress'] = inProgress;
|
||||
if (!inProgress || !transitionTime) {
|
||||
delete this._data['x-amz-scal-transition-time'];
|
||||
} else {
|
||||
if (typeof transitionTime === 'number') {
|
||||
transitionTime = new Date(transitionTime);
|
||||
}
|
||||
if (transitionTime instanceof Date) {
|
||||
transitionTime = transitionTime.toISOString();
|
||||
}
|
||||
this._data['x-amz-scal-transition-time'] = transitionTime;
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata transition in progress value
|
||||
*
|
||||
* @return True if transition is in progress, false otherwise
|
||||
*/
|
||||
getTransitionInProgress() {
|
||||
return this._data['x-amz-scal-transition-in-progress'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the transition time of the object.
|
||||
* @returns The transition time of the object.
|
||||
*/
|
||||
getTransitionTime() {
|
||||
return this._data['x-amz-scal-transition-time'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Set access control list
|
||||
*
|
||||
|
@ -672,6 +793,29 @@ export default class ObjectMD {
|
|||
return reducedLocations;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the Azure specific information
|
||||
* @param azureInfo - a plain JS structure representing the
|
||||
* Azure specific information for a Blob or a Container (see constructor
|
||||
* of {@link ObjectMDAzureInfo} for a description of the fields of this
|
||||
* structure
|
||||
* @return itself
|
||||
*/
|
||||
setAzureInfo(azureInfo: any) {
|
||||
this._data.azureInfo = azureInfo;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the Azure specific information
|
||||
* @return a plain JS structure representing the Azure specific
|
||||
* information for a Blob or a Container an suitable for the constructor
|
||||
* of {@link ObjectMDAzureInfo}.
|
||||
*/
|
||||
getAzureInfo() {
|
||||
return this._data.azureInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set metadata isNull value
|
||||
*
|
||||
|
@ -692,6 +836,31 @@ export default class ObjectMD {
|
|||
return this._data.isNull || false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set metadata isNull2 value
|
||||
*
|
||||
* @param isNull2 - Whether new version is null or not AND has
|
||||
* been put with a Cloudserver handling null keys (i.e. supporting
|
||||
* S3C-7352)
|
||||
|
||||
* @return itself
|
||||
*/
|
||||
setIsNull2(isNull2: boolean) {
|
||||
this._data.isNull2 = isNull2;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get metadata isNull2 value
|
||||
*
|
||||
* @return isNull2 - Whether new version is null or not AND has
|
||||
* been put with a Cloudserver handling null keys (i.e. supporting
|
||||
* S3C-7352)
|
||||
*/
|
||||
getIsNull2() {
|
||||
return this._data.isNull2 || false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set metadata nullVersionId value
|
||||
*
|
||||
|
@ -753,6 +922,19 @@ export default class ObjectMD {
|
|||
return this._data.isDeleteMarker || false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get if the object is a multipart upload (MPU)
|
||||
*
|
||||
* The function checks the "content-md5" field: if it contains a
|
||||
* dash ('-') it is a MPU, as the content-md5 string ends with
|
||||
* "-[nbparts]" for MPUs.
|
||||
*
|
||||
* @return Whether object is a multipart upload
|
||||
*/
|
||||
isMultipartUpload() {
|
||||
return this.getContentMd5().includes('-');
|
||||
}
|
||||
|
||||
/**
|
||||
* Set metadata versionId value
|
||||
*
|
||||
|
@ -770,6 +952,9 @@ export default class ObjectMD {
|
|||
* @return The object versionId
|
||||
*/
|
||||
getVersionId() {
|
||||
if (this.getIsNull()) {
|
||||
return VersioningConstants.ExternalNullVersionId;
|
||||
}
|
||||
return this._data.versionId;
|
||||
}
|
||||
|
||||
|
@ -777,13 +962,16 @@ export default class ObjectMD {
|
|||
* Get metadata versionId value in encoded form (the one visible
|
||||
* to the S3 API user)
|
||||
*
|
||||
* @return The encoded object versionId
|
||||
* @return {undefined|string} The encoded object versionId
|
||||
*/
|
||||
getEncodedVersionId() {
|
||||
const versionId = this.getVersionId();
|
||||
if (versionId) {
|
||||
if (versionId === VersioningConstants.ExternalNullVersionId) {
|
||||
return versionId;
|
||||
} else if (versionId) {
|
||||
return VersionIDUtils.encode(versionId);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -826,6 +1014,20 @@ export default class ObjectMD {
|
|||
return this._data.tags;
|
||||
}
|
||||
|
||||
getUserMetadata() {
|
||||
const metaHeaders = {};
|
||||
const data = this.getValue();
|
||||
Object.keys(data).forEach(key => {
|
||||
if (key.startsWith('x-amz-meta-')) {
|
||||
metaHeaders[key] = data[key];
|
||||
}
|
||||
});
|
||||
if (Object.keys(metaHeaders).length > 0) {
|
||||
return JSON.stringify(metaHeaders);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set replication information
|
||||
*
|
||||
|
@ -841,6 +1043,7 @@ export default class ObjectMD {
|
|||
role: string;
|
||||
storageType?: string;
|
||||
dataStoreVersionId?: string;
|
||||
isNFS?: boolean;
|
||||
}) {
|
||||
const {
|
||||
status,
|
||||
|
@ -851,6 +1054,7 @@ export default class ObjectMD {
|
|||
role,
|
||||
storageType,
|
||||
dataStoreVersionId,
|
||||
isNFS,
|
||||
} = replicationInfo;
|
||||
this._data.replicationInfo = {
|
||||
status,
|
||||
|
@ -861,6 +1065,7 @@ export default class ObjectMD {
|
|||
role,
|
||||
storageType: storageType || '',
|
||||
dataStoreVersionId: dataStoreVersionId || '',
|
||||
isNFS: isNFS || null,
|
||||
};
|
||||
return this;
|
||||
}
|
||||
|
@ -879,6 +1084,24 @@ export default class ObjectMD {
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set whether the replication is occurring from an NFS bucket.
|
||||
* @param isNFS - Whether replication from an NFS bucket
|
||||
* @return itself
|
||||
*/
|
||||
setReplicationIsNFS(isNFS: boolean) {
|
||||
this._data.replicationInfo.isNFS = isNFS;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get whether the replication is occurring from an NFS bucket.
|
||||
* @return Whether replication from an NFS bucket
|
||||
*/
|
||||
getReplicationIsNFS() {
|
||||
return this._data.replicationInfo.isNFS;
|
||||
}
|
||||
|
||||
setReplicationSiteStatus(site: string, status: string) {
|
||||
const backend = this._data.replicationInfo.backends.find(
|
||||
(o) => o.site === site
|
||||
|
@ -929,6 +1152,11 @@ export default class ObjectMD {
|
|||
return this;
|
||||
}
|
||||
|
||||
setReplicationStorageType(storageType: string) {
|
||||
this._data.replicationInfo.storageType = storageType;
|
||||
return this;
|
||||
}
|
||||
|
||||
setReplicationStorageClass(storageClass: string) {
|
||||
this._data.replicationInfo.storageClass = storageClass;
|
||||
return this;
|
||||
|
@ -1010,6 +1238,9 @@ export default class ObjectMD {
|
|||
Object.keys(metaHeaders).forEach((key) => {
|
||||
if (key.startsWith('x-amz-meta-')) {
|
||||
this._data[key] = metaHeaders[key];
|
||||
} else if (key.startsWith('x-ms-meta-')) {
|
||||
const _key = key.replace('x-ms-meta-', 'x-amz-meta-');
|
||||
this._data[_key] = metaHeaders[key];
|
||||
}
|
||||
});
|
||||
// If a multipart object and the acl is already parsed, we update it
|
||||
|
@ -1019,6 +1250,20 @@ export default class ObjectMD {
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all existing meta headers (used for Azure)
|
||||
*
|
||||
* @return itself
|
||||
*/
|
||||
clearMetadataValues() {
|
||||
Object.keys(this._data).forEach(key => {
|
||||
if (key.startsWith('x-amz-meta')) {
|
||||
delete this._data[key];
|
||||
}
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* overrideMetadataValues (used for complete MPU and object copy)
|
||||
*
|
||||
|
@ -1030,6 +1275,38 @@ export default class ObjectMD {
|
|||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create or update the microVersionId field
|
||||
*
|
||||
* This field can be used to force an update in MongoDB. This can
|
||||
* be needed in the following cases:
|
||||
*
|
||||
* - in case no other metadata field changes
|
||||
*
|
||||
* - to detect a change when fields change but object version does
|
||||
* not change e.g. when ingesting a putObjectTagging coming from
|
||||
* S3C to Zenko
|
||||
*
|
||||
* - to manage conflicts during concurrent updates, using
|
||||
* conditions on the microVersionId field.
|
||||
*
|
||||
* It's a field of 16 hexadecimal characters randomly generated
|
||||
*
|
||||
* @return itself
|
||||
*/
|
||||
updateMicroVersionId() {
|
||||
this._data.microVersionId = crypto.randomBytes(8).toString('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the microVersionId field, or null if not set
|
||||
*
|
||||
* @return the microVersionId field if exists, or {null} if it does not exist
|
||||
*/
|
||||
getMicroVersionId() {
|
||||
return this._data.microVersionId || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set object legal hold status
|
||||
* @param legalHold - true if legal hold is 'ON' false if 'OFF'
|
||||
|
@ -1110,4 +1387,98 @@ export default class ObjectMD {
|
|||
getValue() {
|
||||
return this._data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get x-amz-restore
|
||||
*
|
||||
* @returns x-amz-restore
|
||||
*/
|
||||
getAmzRestore() {
|
||||
return this._data['x-amz-restore'];
|
||||
}
|
||||
|
||||
/**
|
||||
* Set x-amz-restore
|
||||
*
|
||||
* @param value x-amz-restore object
|
||||
* @returns itself
|
||||
* @throws case of invalid parameter
|
||||
*/
|
||||
setAmzRestore(value?: ObjectMDAmzRestore) {
|
||||
if (value) {
|
||||
// Accept object instance of ObjectMDAmzRestore and Object
|
||||
if (!(value instanceof ObjectMDAmzRestore) && !ObjectMDAmzRestore.isValid(value)) {
|
||||
throw new Error('x-amz-restore must be type of ObjectMDAmzRestore.');
|
||||
}
|
||||
this._data['x-amz-restore'] = value;
|
||||
} else {
|
||||
delete this._data['x-amz-restore'];
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get archive
|
||||
*
|
||||
* @returns archive
|
||||
*/
|
||||
getArchive() {
|
||||
return this._data.archive;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set archive
|
||||
*
|
||||
* @param value archive object
|
||||
* @returns itself
|
||||
* @throws case of invalid parameter
|
||||
*/
|
||||
setArchive(value: ObjectMDArchive) {
|
||||
if (value) {
|
||||
// Accept object instance of ObjectMDArchive and Object
|
||||
if (!(value instanceof ObjectMDArchive) && !ObjectMDArchive.isValid(value)) {
|
||||
throw new Error('archive is must be type of ObjectMDArchive.');
|
||||
}
|
||||
this._data.archive = value;
|
||||
} else {
|
||||
delete this._data.archive;
|
||||
}
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set deleted flag
|
||||
* @param {Boolean} value deleted object
|
||||
* @return {ObjectMD}
|
||||
*/
|
||||
setDeleted(value) {
|
||||
this._data.deleted = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get deleted flag
|
||||
* @return {Boolean}
|
||||
*/
|
||||
getDeleted() {
|
||||
return this._data.deleted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set isPHD flag
|
||||
* @param {Boolean} value isPHD value
|
||||
* @return {ObjectMD}
|
||||
*/
|
||||
setIsPHD(value) {
|
||||
this._data.isPHD = value;
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get isPHD flag
|
||||
* @return {Boolean}
|
||||
*/
|
||||
getIsPHD() {
|
||||
return this._data.isPHD;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,94 @@
|
|||
/*
|
||||
* Code based on Yutaka Oishi (Fujifilm) contributions
|
||||
* Date: 11 Sep 2020
|
||||
*/
|
||||
|
||||
/**
|
||||
* class representing the x-amz-restore of object metadata.
|
||||
*
|
||||
* @class
|
||||
*/
|
||||
export default class ObjectMDAmzRestore {
|
||||
'expiry-date': Date | string;
|
||||
'ongoing-request': boolean;
|
||||
|
||||
/**
|
||||
*
|
||||
* @constructor
|
||||
* @param ongoingRequest ongoing-request
|
||||
* @param [expiryDate] expiry-date
|
||||
* @throws case of invalid parameter
|
||||
*/
|
||||
constructor(ongoingRequest: boolean, expiryDate?: Date | string) {
|
||||
this.setOngoingRequest(ongoingRequest);
|
||||
this.setExpiryDate(expiryDate);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param data archiveInfo
|
||||
* @returns true if the provided object is valid
|
||||
*/
|
||||
static isValid(data: { 'ongoing-request': boolean; 'expiry-date': Date | string }) {
|
||||
try {
|
||||
// eslint-disable-next-line no-new
|
||||
new ObjectMDAmzRestore(data['ongoing-request'], data['expiry-date']);
|
||||
return true;
|
||||
} catch (err) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns ongoing-request
|
||||
*/
|
||||
getOngoingRequest() {
|
||||
return this['ongoing-request'];
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param value ongoing-request
|
||||
* @throws case of invalid parameter
|
||||
*/
|
||||
setOngoingRequest(value?: boolean) {
|
||||
if (value === undefined) {
|
||||
throw new Error('ongoing-request is required.');
|
||||
} else if (typeof value !== 'boolean') {
|
||||
throw new Error('ongoing-request must be type of boolean.');
|
||||
}
|
||||
this['ongoing-request'] = value;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns expiry-date
|
||||
*/
|
||||
getExpiryDate() {
|
||||
return this['expiry-date'];
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param value expiry-date
|
||||
* @throws case of invalid parameter
|
||||
*/
|
||||
setExpiryDate(value?: Date | string) {
|
||||
if (value) {
|
||||
const checkWith = (new Date(value)).getTime();
|
||||
if (Number.isNaN(Number(checkWith))) {
|
||||
throw new Error('expiry-date is must be a valid Date.');
|
||||
}
|
||||
this['expiry-date'] = value;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns itself
|
||||
*/
|
||||
getValue() {
|
||||
return this;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,184 @@
|
|||
/**
|
||||
* class representing the archive of object metadata.
|
||||
*
|
||||
* @class
|
||||
*/
|
||||
export default class ObjectMDArchive {
|
||||
archiveInfo: any;
|
||||
// @ts-ignore
|
||||
restoreRequestedAt: Date | string;
|
||||
// @ts-ignore
|
||||
restoreRequestedDays: number;
|
||||
// @ts-ignore
|
||||
restoreCompletedAt: Date | string;
|
||||
// @ts-ignore
|
||||
restoreWillExpireAt: Date | string;
|
||||
|
||||
/**
|
||||
*
|
||||
* @constructor
|
||||
* @param archiveInfo contains the archive info set by the TLP and returned by the TLP jobs
|
||||
* @param [restoreRequestedAt] set at the time restore request is made by the client
|
||||
* @param [restoreRequestedDays] set at the time restore request is made by the client
|
||||
* @param [restoreCompletedAt] set at the time of successful restore
|
||||
* @param [restoreWillExpireAt] computed and stored at the time of restore
|
||||
* @throws case of invalid parameter
|
||||
*/
|
||||
constructor(
|
||||
archiveInfo: any,
|
||||
restoreRequestedAt?: Date | string,
|
||||
restoreRequestedDays?: number,
|
||||
restoreCompletedAt?: Date | string,
|
||||
restoreWillExpireAt?: Date | string,
|
||||
) {
|
||||
this.setArchiveInfo(archiveInfo);
|
||||
this.setRestoreRequestedAt(restoreRequestedAt!);
|
||||
this.setRestoreRequestedDays(restoreRequestedDays!);
|
||||
this.setRestoreCompletedAt(restoreCompletedAt!);
|
||||
this.setRestoreWillExpireAt(restoreWillExpireAt!);
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param data archiveInfo
|
||||
* @returns true if the provided object is valid
|
||||
*/
|
||||
static isValid(data: {
|
||||
archiveInfo: any;
|
||||
restoreRequestedAt?: Date;
|
||||
restoreRequestedDays?: number;
|
||||
restoreCompletedAt?: Date;
|
||||
restoreWillExpireAt?: Date;
|
||||
}) {
|
||||
try {
|
||||
// eslint-disable-next-line no-new
|
||||
new ObjectMDArchive(
|
||||
data.archiveInfo,
|
||||
data.restoreRequestedAt,
|
||||
data.restoreRequestedDays,
|
||||
data.restoreCompletedAt,
|
||||
data.restoreWillExpireAt,
|
||||
);
|
||||
return true;
|
||||
} catch (err) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns archiveInfo
|
||||
*/
|
||||
getArchiveInfo() {
|
||||
return this.archiveInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param value archiveInfo
|
||||
* @throws case of invalid parameter
|
||||
*/
|
||||
setArchiveInfo(value: any) {
|
||||
if (!value) {
|
||||
throw new Error('archiveInfo is required.');
|
||||
} else if (typeof value !== 'object') {
|
||||
throw new Error('archiveInfo must be type of object.');
|
||||
}
|
||||
this.archiveInfo = value;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns restoreRequestedAt
|
||||
*/
|
||||
getRestoreRequestedAt() {
|
||||
return this.restoreRequestedAt;
|
||||
}
|
||||
/**
|
||||
* @param value restoreRequestedAt
|
||||
* @throws case of invalid parameter
|
||||
*/
|
||||
setRestoreRequestedAt(value: Date | string) {
|
||||
if (value) {
|
||||
const checkWith = (new Date(value)).getTime();
|
||||
if (Number.isNaN(Number(checkWith))) {
|
||||
throw new Error('restoreRequestedAt must be a valid Date.');
|
||||
}
|
||||
this.restoreRequestedAt = value;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns restoreRequestedDays
|
||||
*/
|
||||
getRestoreRequestedDays() {
|
||||
return this.restoreRequestedDays;
|
||||
}
|
||||
/**
|
||||
* @param value restoreRequestedDays
|
||||
* @throws case of invalid parameter
|
||||
*/
|
||||
setRestoreRequestedDays(value: number) {
|
||||
if (value) {
|
||||
if (isNaN(value)) {
|
||||
throw new Error('restoreRequestedDays must be type of Number.');
|
||||
}
|
||||
this.restoreRequestedDays = value;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns restoreCompletedAt
|
||||
*/
|
||||
getRestoreCompletedAt() {
|
||||
return this.restoreCompletedAt;
|
||||
}
|
||||
/**
|
||||
* @param value restoreCompletedAt
|
||||
* @throws case of invalid parameter
|
||||
*/
|
||||
setRestoreCompletedAt(value: Date | string) {
|
||||
if (value) {
|
||||
if (!this.restoreRequestedAt || !this.restoreRequestedDays) {
|
||||
throw new Error('restoreCompletedAt must be set after restoreRequestedAt and restoreRequestedDays.');
|
||||
}
|
||||
const checkWith = (new Date(value)).getTime();
|
||||
if (Number.isNaN(Number(checkWith))) {
|
||||
throw new Error('restoreCompletedAt must be a valid Date.');
|
||||
}
|
||||
this.restoreCompletedAt = value;
|
||||
}
|
||||
}
|
||||
/**
|
||||
*
|
||||
* @returns restoreWillExpireAt
|
||||
*/
|
||||
getRestoreWillExpireAt() {
|
||||
return this.restoreWillExpireAt;
|
||||
}
|
||||
/**
|
||||
* @param value restoreWillExpireAt
|
||||
* @throws case of invalid parameter
|
||||
*/
|
||||
setRestoreWillExpireAt(value: Date | string) {
|
||||
if (value) {
|
||||
if (!this.restoreRequestedAt || !this.restoreRequestedDays) {
|
||||
throw new Error('restoreWillExpireAt must be set after restoreRequestedAt and restoreRequestedDays.');
|
||||
}
|
||||
const checkWith = (new Date(value)).getTime();
|
||||
if (Number.isNaN(Number(checkWith))) {
|
||||
throw new Error('restoreWillExpireAt must be a valid Date.');
|
||||
}
|
||||
this.restoreWillExpireAt = value;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @returns itself
|
||||
*/
|
||||
getValue() {
|
||||
return this;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,188 @@
|
|||
/**
|
||||
* Helper class to ease access to the Azure specific information for
|
||||
* Blob and Container objects.
|
||||
*/
|
||||
export default class ObjectMDAzureInfo {
|
||||
_data: {
|
||||
containerPublicAccess: string;
|
||||
containerStoredAccessPolicies: any[];
|
||||
containerImmutabilityPolicy: any;
|
||||
containerLegalHoldStatus: boolean;
|
||||
containerDeletionInProgress: boolean;
|
||||
blobType: string;
|
||||
blobContentMD5: string;
|
||||
blobIssuedETag: string;
|
||||
blobCopyInfo: any;
|
||||
blobSequenceNumber: number;
|
||||
blobAccessTierChangeTime: Date;
|
||||
blobUncommitted: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
* @param obj - Raw structure for the Azure info on Blob/Container
|
||||
* @param obj.containerPublicAccess - Public access authorization
|
||||
* type
|
||||
* @param obj.containerStoredAccessPolicies - Access policies
|
||||
* for Shared Access Signature bearer
|
||||
* @param obj.containerImmutabilityPolicy - data immutability
|
||||
* policy for this container
|
||||
* @param obj.containerLegalHoldStatus - legal hold status for
|
||||
* this container
|
||||
* @param obj.containerDeletionInProgress - deletion in progress
|
||||
* indicator for this container
|
||||
* @param obj.blobType - defines the type of blob for this object
|
||||
* @param obj.blobContentMD5 - whole object MD5 sum set by the
|
||||
* client through the Azure API
|
||||
* @param obj.blobIssuedETag - backup of the issued ETag on MD only
|
||||
* operations like Set Blob Properties and Set Blob Metadata
|
||||
* @param obj.blobCopyInfo - information pertaining to past and
|
||||
* pending copy operation targeting this object
|
||||
* @param obj.blobSequenceNumber - sequence number for a PageBlob
|
||||
* @param obj.blobAccessTierChangeTime - date of change of tier
|
||||
* @param obj.blobUncommitted - A block has been put for a
|
||||
* nonexistent blob which is about to be created
|
||||
*/
|
||||
constructor(obj: {
|
||||
containerPublicAccess: string;
|
||||
containerStoredAccessPolicies: any[];
|
||||
containerImmutabilityPolicy: any;
|
||||
containerLegalHoldStatus: boolean;
|
||||
containerDeletionInProgress: boolean;
|
||||
blobType: string;
|
||||
blobContentMD5: string;
|
||||
blobIssuedETag: string;
|
||||
blobCopyInfo: any;
|
||||
blobSequenceNumber: number;
|
||||
blobAccessTierChangeTime: Date;
|
||||
blobUncommitted: boolean;
|
||||
}) {
|
||||
this._data = {
|
||||
containerPublicAccess: obj.containerPublicAccess,
|
||||
containerStoredAccessPolicies: obj.containerStoredAccessPolicies,
|
||||
containerImmutabilityPolicy: obj.containerImmutabilityPolicy,
|
||||
containerLegalHoldStatus: obj.containerLegalHoldStatus,
|
||||
containerDeletionInProgress: obj.containerDeletionInProgress,
|
||||
blobType: obj.blobType,
|
||||
blobContentMD5: obj.blobContentMD5,
|
||||
blobIssuedETag: obj.blobIssuedETag,
|
||||
blobCopyInfo: obj.blobCopyInfo,
|
||||
blobSequenceNumber: obj.blobSequenceNumber,
|
||||
blobAccessTierChangeTime: obj.blobAccessTierChangeTime,
|
||||
blobUncommitted: obj.blobUncommitted,
|
||||
};
|
||||
}
|
||||
|
||||
getContainerPublicAccess() {
|
||||
return this._data.containerPublicAccess;
|
||||
}
|
||||
|
||||
setContainerPublicAccess(containerPublicAccess: string) {
|
||||
this._data.containerPublicAccess = containerPublicAccess;
|
||||
return this;
|
||||
}
|
||||
|
||||
getContainerStoredAccessPolicies() {
|
||||
return this._data.containerStoredAccessPolicies;
|
||||
}
|
||||
|
||||
setContainerStoredAccessPolicies(containerStoredAccessPolicies: any[]) {
|
||||
this._data.containerStoredAccessPolicies =
|
||||
containerStoredAccessPolicies;
|
||||
return this;
|
||||
}
|
||||
|
||||
getContainerImmutabilityPolicy() {
|
||||
return this._data.containerImmutabilityPolicy;
|
||||
}
|
||||
|
||||
setContainerImmutabilityPolicy(containerImmutabilityPolicy: any) {
|
||||
this._data.containerImmutabilityPolicy = containerImmutabilityPolicy;
|
||||
return this;
|
||||
}
|
||||
|
||||
getContainerLegalHoldStatus() {
|
||||
return this._data.containerLegalHoldStatus;
|
||||
}
|
||||
|
||||
setContainerLegalHoldStatus(containerLegalHoldStatus: boolean) {
|
||||
this._data.containerLegalHoldStatus = containerLegalHoldStatus;
|
||||
return this;
|
||||
}
|
||||
|
||||
getContainerDeletionInProgress() {
|
||||
return this._data.containerDeletionInProgress;
|
||||
}
|
||||
|
||||
setContainerDeletionInProgress(containerDeletionInProgress: boolean) {
|
||||
this._data.containerDeletionInProgress = containerDeletionInProgress;
|
||||
return this;
|
||||
}
|
||||
|
||||
getBlobType() {
|
||||
return this._data.blobType;
|
||||
}
|
||||
|
||||
setBlobType(blobType: string) {
|
||||
this._data.blobType = blobType;
|
||||
return this;
|
||||
}
|
||||
|
||||
getBlobContentMD5() {
|
||||
return this._data.blobContentMD5;
|
||||
}
|
||||
|
||||
setBlobContentMD5(blobContentMD5: string) {
|
||||
this._data.blobContentMD5 = blobContentMD5;
|
||||
return this;
|
||||
}
|
||||
|
||||
getBlobIssuedETag() {
|
||||
return this._data.blobIssuedETag;
|
||||
}
|
||||
|
||||
setBlobIssuedETag(blobIssuedETag: string) {
|
||||
this._data.blobIssuedETag = blobIssuedETag;
|
||||
return this;
|
||||
}
|
||||
|
||||
getBlobCopyInfo() {
|
||||
return this._data.blobCopyInfo;
|
||||
}
|
||||
|
||||
setBlobCopyInfo(blobCopyInfo: any) {
|
||||
this._data.blobCopyInfo = blobCopyInfo;
|
||||
return this;
|
||||
}
|
||||
|
||||
getBlobSequenceNumber() {
|
||||
return this._data.blobSequenceNumber;
|
||||
}
|
||||
|
||||
setBlobSequenceNumber(blobSequenceNumber: number) {
|
||||
this._data.blobSequenceNumber = blobSequenceNumber;
|
||||
return this;
|
||||
}
|
||||
|
||||
getBlobAccessTierChangeTime() {
|
||||
return this._data.blobAccessTierChangeTime;
|
||||
}
|
||||
|
||||
setBlobAccessTierChangeTime(blobAccessTierChangeTime: Date) {
|
||||
this._data.blobAccessTierChangeTime = blobAccessTierChangeTime;
|
||||
return this;
|
||||
}
|
||||
|
||||
getBlobUncommitted() {
|
||||
return this._data.blobUncommitted;
|
||||
}
|
||||
|
||||
setBlobUncommitted(blobUncommitted: boolean) {
|
||||
this._data.blobUncommitted = blobUncommitted;
|
||||
return this;
|
||||
}
|
||||
|
||||
getValue() {
|
||||
return this._data;
|
||||
}
|
||||
}
|
|
@ -5,6 +5,7 @@ export type Location = BaseLocation & {
|
|||
size: number;
|
||||
dataStoreETag: string;
|
||||
dataStoreVersionId: string;
|
||||
blockId?: string;
|
||||
};
|
||||
export type ObjectMDLocationData = {
|
||||
key: string;
|
||||
|
@ -12,6 +13,8 @@ export type ObjectMDLocationData = {
|
|||
size: number;
|
||||
dataStoreName: string;
|
||||
dataStoreETag: string;
|
||||
dataStoreVersionId: string;
|
||||
blockId?: string;
|
||||
cryptoScheme?: number;
|
||||
cipheredDataKey?: string;
|
||||
};
|
||||
|
@ -31,10 +34,14 @@ export default class ObjectMDLocation {
|
|||
* @param locationObj.dataStoreName - type of data store
|
||||
* @param locationObj.dataStoreETag - internal ETag of
|
||||
* data part
|
||||
* @param [locationObj.dataStoreVersionId] - versionId,
|
||||
* needed for cloud backends
|
||||
* @param [location.cryptoScheme] - if location data is
|
||||
* encrypted: the encryption scheme version
|
||||
* @param [location.cipheredDataKey] - if location data
|
||||
* is encrypted: the base64-encoded ciphered data key
|
||||
* @param [locationObj.blockId] - blockId of the part,
|
||||
* set by the Azure Blob Service REST API frontend
|
||||
*/
|
||||
constructor(locationObj: Location | (Location & Ciphered)) {
|
||||
this._data = {
|
||||
|
@ -43,6 +50,8 @@ export default class ObjectMDLocation {
|
|||
size: locationObj.size,
|
||||
dataStoreName: locationObj.dataStoreName,
|
||||
dataStoreETag: locationObj.dataStoreETag,
|
||||
dataStoreVersionId: locationObj.dataStoreVersionId,
|
||||
blockId: locationObj.blockId,
|
||||
};
|
||||
if ('cryptoScheme' in locationObj) {
|
||||
this._data.cryptoScheme = locationObj.cryptoScheme;
|
||||
|
@ -64,6 +73,7 @@ export default class ObjectMDLocation {
|
|||
* @param location - single data location info
|
||||
* @param location.key - data backend key
|
||||
* @param location.dataStoreName - type of data store
|
||||
* @param [location.dataStoreVersionId] - data backend version ID
|
||||
* @param [location.cryptoScheme] - if location data is
|
||||
* encrypted: the encryption scheme version
|
||||
* @param [location.cipheredDataKey] - if location data
|
||||
|
@ -71,15 +81,19 @@ export default class ObjectMDLocation {
|
|||
* @return return this
|
||||
*/
|
||||
setDataLocation(location: BaseLocation | (BaseLocation & Ciphered)) {
|
||||
['key', 'dataStoreName', 'cryptoScheme', 'cipheredDataKey'].forEach(
|
||||
(attrName) => {
|
||||
if (location[attrName] !== undefined) {
|
||||
this._data[attrName] = location[attrName];
|
||||
} else {
|
||||
delete this._data[attrName];
|
||||
}
|
||||
[
|
||||
'key',
|
||||
'dataStoreName',
|
||||
'dataStoreVersionId',
|
||||
'cryptoScheme',
|
||||
'cipheredDataKey',
|
||||
].forEach(attrName => {
|
||||
if (location[attrName] !== undefined) {
|
||||
this._data[attrName] = location[attrName];
|
||||
} else {
|
||||
delete this._data[attrName];
|
||||
}
|
||||
);
|
||||
});
|
||||
return this;
|
||||
}
|
||||
|
||||
|
@ -87,6 +101,10 @@ export default class ObjectMDLocation {
|
|||
return this._data.dataStoreETag;
|
||||
}
|
||||
|
||||
getDataStoreVersionId() {
|
||||
return this._data.dataStoreVersionId;
|
||||
}
|
||||
|
||||
getPartNumber() {
|
||||
return Number.parseInt(this._data.dataStoreETag.split(':')[0], 10);
|
||||
}
|
||||
|
@ -121,6 +139,15 @@ export default class ObjectMDLocation {
|
|||
return this._data.cipheredDataKey;
|
||||
}
|
||||
|
||||
getBlockId() {
|
||||
return this._data.blockId;
|
||||
}
|
||||
|
||||
setBlockId(blockId: string) {
|
||||
this._data.blockId = blockId;
|
||||
return this;
|
||||
}
|
||||
|
||||
getValue() {
|
||||
return this._data;
|
||||
}
|
||||
|
|
|
@ -62,7 +62,8 @@ export default class ReplicationConfiguration {
|
|||
_destination: string | null;
|
||||
_rules: Rule[] | null;
|
||||
_prevStorageClass: null;
|
||||
_hasScalityDestination: boolean;
|
||||
_hasScalityDestination: boolean | null;
|
||||
_preferredReadLocation: string | null;
|
||||
|
||||
/**
|
||||
* Create a ReplicationConfiguration instance
|
||||
|
@ -84,7 +85,8 @@ export default class ReplicationConfiguration {
|
|||
this._destination = null;
|
||||
this._rules = null;
|
||||
this._prevStorageClass = null;
|
||||
this._hasScalityDestination = false;
|
||||
this._hasScalityDestination = null;
|
||||
this._preferredReadLocation = null;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -111,6 +113,18 @@ export default class ReplicationConfiguration {
|
|||
return this._rules;
|
||||
}
|
||||
|
||||
/**
|
||||
* The preferred read location
|
||||
* @return {string|null} - The preferred read location if defined,
|
||||
* otherwise null
|
||||
*
|
||||
* FIXME ideally we should be able to specify one preferred read
|
||||
* location for each rule
|
||||
*/
|
||||
getPreferredReadLocation() {
|
||||
return this._preferredReadLocation;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the replication configuration
|
||||
* @return - The replication configuration
|
||||
|
@ -120,6 +134,7 @@ export default class ReplicationConfiguration {
|
|||
role: this.getRole(),
|
||||
destination: this.getDestination(),
|
||||
rules: this.getRules(),
|
||||
preferredReadLocation: this.getPreferredReadLocation(),
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -326,7 +341,15 @@ export default class ReplicationConfiguration {
|
|||
return undefined;
|
||||
}
|
||||
const storageClasses = destination.StorageClass[0].split(',');
|
||||
const isValidStorageClass = storageClasses.every((storageClass) => {
|
||||
const prefReadIndex = storageClasses.findIndex(storageClass =>
|
||||
storageClass.endsWith(':preferred_read'));
|
||||
if (prefReadIndex !== -1) {
|
||||
const prefRead = storageClasses[prefReadIndex].split(':')[0];
|
||||
// remove :preferred_read tag from storage class name
|
||||
storageClasses[prefReadIndex] = prefRead;
|
||||
this._preferredReadLocation = prefRead;
|
||||
}
|
||||
const isValidStorageClass = storageClasses.every(storageClass => {
|
||||
if (validStorageClasses.includes(storageClass)) {
|
||||
this._hasScalityDestination =
|
||||
defaultEndpoint.type === undefined;
|
||||
|
@ -336,6 +359,11 @@ export default class ReplicationConfiguration {
|
|||
(endpoint: any) => endpoint.site === storageClass
|
||||
);
|
||||
if (endpoint) {
|
||||
// We do not support replication to cold location.
|
||||
// Only transition to cold location is supported.
|
||||
if (endpoint.site && this._config.locationConstraints[endpoint.site]?.isCold) {
|
||||
return false;
|
||||
}
|
||||
// If this._hasScalityDestination was not set to true in any
|
||||
// previous iteration or by a prior rule's storage class, then
|
||||
// check if the current endpoint is a Scality destination.
|
||||
|
|
|
@ -1,11 +1,16 @@
|
|||
export { default as ARN } from './ARN';
|
||||
export { default as BackendInfo } from './BackendInfo';
|
||||
export { default as BucketAzureInfo } from './BucketAzureInfo';
|
||||
export { default as BucketInfo } from './BucketInfo';
|
||||
export { default as ObjectMD } from './ObjectMD';
|
||||
export { default as ObjectMDLocation } from './ObjectMDLocation';
|
||||
export * as WebsiteConfiguration from './WebsiteConfiguration';
|
||||
export { default as ReplicationConfiguration } from './ReplicationConfiguration';
|
||||
export { default as BucketPolicy } from './BucketPolicy';
|
||||
export { default as LifecycleConfiguration } from './LifecycleConfiguration';
|
||||
export { default as LifecycleRule } from './LifecycleRule';
|
||||
export { default as BucketPolicy } from './BucketPolicy';
|
||||
export { default as ObjectLockConfiguration } from './ObjectLockConfiguration';
|
||||
export { default as NotificationConfiguration } from './NotificationConfiguration';
|
||||
export { default as ObjectLockConfiguration } from './ObjectLockConfiguration';
|
||||
export { default as ObjectMD } from './ObjectMD';
|
||||
export { default as ObjectMDAmzRestore } from './ObjectMDAmzRestore';
|
||||
export { default as ObjectMDArchive } from './ObjectMDArchive';
|
||||
export { default as ObjectMDAzureInfo } from './ObjectMDAzureInfo';
|
||||
export { default as ObjectMDLocation } from './ObjectMDLocation';
|
||||
export { default as ReplicationConfiguration } from './ReplicationConfiguration';
|
||||
export * as WebsiteConfiguration from './WebsiteConfiguration';
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
import * as http from 'http';
|
||||
import * as https from 'https';
|
||||
import { https as HttpsAgent } from 'httpagent';
|
||||
import * as tls from 'tls';
|
||||
import * as net from 'net';
|
||||
import assert from 'assert';
|
||||
|
@ -371,6 +372,8 @@ export default class Server {
|
|||
error: err.stack || err,
|
||||
address: sock.address(),
|
||||
});
|
||||
// socket is not systematically destroyed
|
||||
sock.destroy();
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -407,7 +410,11 @@ export default class Server {
|
|||
method: 'arsenal.network.Server.start',
|
||||
port: this._port,
|
||||
});
|
||||
this._https.agent = new https.Agent(this._https);
|
||||
this._https.agent = new HttpsAgent.Agent(this._https, {
|
||||
// Do not enforce the maximum number of sockets for the
|
||||
// main server, as it might be able to serve more clients.
|
||||
maxSockets: false,
|
||||
});
|
||||
this._server = https.createServer(this._https,
|
||||
(req, res) => this._onRequest(req, res));
|
||||
} else {
|
||||
|
|
|
@ -3,10 +3,12 @@ import * as utils from './http/utils';
|
|||
import RESTServer from './rest/RESTServer';
|
||||
import RESTClient from './rest/RESTClient';
|
||||
import * as ProbeServer from './probe/ProbeServer';
|
||||
import HealthProbeServer from './probe/HealthProbeServer';
|
||||
import * as Utils from './probe/Utils';
|
||||
|
||||
export const http = { server, utils };
|
||||
export const rest = { RESTServer, RESTClient };
|
||||
export const probe = { ProbeServer };
|
||||
export const probe = { ProbeServer, HealthProbeServer, Utils };
|
||||
|
||||
export { default as RoundRobin } from './RoundRobin';
|
||||
export { default as kmip } from './kmip';
|
||||
|
|
|
@ -20,7 +20,7 @@ function _ttlvPadVector(vec: any[]) {
|
|||
return vec;
|
||||
}
|
||||
|
||||
function _throwError(logger: werelogs.Logger, msg: string, data?: LogDictionnary) {
|
||||
function _throwError(logger: werelogs.Logger, msg: string, data?: LogDictionary) {
|
||||
logger.error(msg, data);
|
||||
throw Error(msg);
|
||||
}
|
||||
|
|
|
@ -0,0 +1,94 @@
|
|||
import * as http from 'http';
|
||||
import httpServer from '../http/server';
|
||||
import * as werelogs from 'werelogs';
|
||||
import errors from '../../errors';
|
||||
import ZenkoMetrics from '../../metrics/ZenkoMetrics';
|
||||
import { sendSuccess, sendError } from './Utils';
|
||||
|
||||
function checkStub(_log: any) {
|
||||
// eslint-disable-line
|
||||
return true;
|
||||
}
|
||||
|
||||
export default class HealthProbeServer extends httpServer {
|
||||
logging: werelogs.Logger;
|
||||
_reqHandlers: { [key: string]: any };
|
||||
_livenessCheck: (log: any) => boolean;
|
||||
_readinessCheck: (log: any) => boolean;
|
||||
|
||||
constructor(params: {
|
||||
port: number;
|
||||
bindAddress: string;
|
||||
livenessCheck?: (log: any) => boolean;
|
||||
readinessCheck?: (log: any) => boolean;
|
||||
}) {
|
||||
const logging = new werelogs.Logger('HealthProbeServer');
|
||||
super(params.port, logging);
|
||||
this.logging = logging;
|
||||
this.setBindAddress(params.bindAddress || 'localhost');
|
||||
// hooking our request processing function by calling the
|
||||
// parent's method for that
|
||||
this.onRequest(this._onRequest);
|
||||
this._reqHandlers = {
|
||||
'/_/health/liveness': this._onLiveness.bind(this),
|
||||
'/_/health/readiness': this._onReadiness.bind(this),
|
||||
'/_/monitoring/metrics': this._onMetrics.bind(this),
|
||||
};
|
||||
this._livenessCheck = params.livenessCheck || checkStub;
|
||||
this._readinessCheck = params.readinessCheck || checkStub;
|
||||
}
|
||||
|
||||
onLiveCheck(f: (log: any) => boolean) {
|
||||
this._livenessCheck = f;
|
||||
}
|
||||
|
||||
onReadyCheck(f: (log: any) => boolean) {
|
||||
this._readinessCheck = f;
|
||||
}
|
||||
|
||||
_onRequest(req: http.IncomingMessage, res: http.ServerResponse) {
|
||||
const log = this.logging.newRequestLogger();
|
||||
log.debug('request received', { method: req.method, url: req.url });
|
||||
|
||||
if (req.method !== 'GET') {
|
||||
sendError(res, log, errors.MethodNotAllowed);
|
||||
} else if (req.url && req.url in this._reqHandlers) {
|
||||
this._reqHandlers[req.url](req, res, log);
|
||||
} else {
|
||||
sendError(res, log, errors.InvalidURI);
|
||||
}
|
||||
}
|
||||
|
||||
_onLiveness(
|
||||
_req: http.IncomingMessage,
|
||||
res: http.ServerResponse,
|
||||
log: RequestLogger,
|
||||
) {
|
||||
if (this._livenessCheck(log)) {
|
||||
sendSuccess(res, log);
|
||||
} else {
|
||||
sendError(res, log, errors.ServiceUnavailable);
|
||||
}
|
||||
}
|
||||
|
||||
_onReadiness(
|
||||
_req: http.IncomingMessage,
|
||||
res: http.ServerResponse,
|
||||
log: RequestLogger,
|
||||
) {
|
||||
if (this._readinessCheck(log)) {
|
||||
sendSuccess(res, log);
|
||||
} else {
|
||||
sendError(res, log, errors.ServiceUnavailable);
|
||||
}
|
||||
}
|
||||
|
||||
// expose metrics to Prometheus
|
||||
async _onMetrics(_req: http.IncomingMessage, res: http.ServerResponse) {
|
||||
const metrics = await ZenkoMetrics.asPrometheus();
|
||||
res.writeHead(200, {
|
||||
'Content-Type': ZenkoMetrics.asPrometheusContentType(),
|
||||
});
|
||||
res.end(metrics);
|
||||
}
|
||||
}
|
|
@ -4,19 +4,16 @@ import * as werelogs from 'werelogs';
|
|||
import errors from '../../errors';
|
||||
|
||||
export const DEFAULT_LIVE_ROUTE = '/_/live';
|
||||
export const DEFAULT_READY_ROUTE = '/_/live';
|
||||
export const DEFAULT_METRICS_ROUTE = '/_/metrics';
|
||||
export const DEFAULT_READY_ROUTE = '/_/ready';
|
||||
export const DEFAULT_METRICS_ROUTE = '/metrics';
|
||||
|
||||
/**
|
||||
* ProbeDelegate is used to determine if a probe is successful or
|
||||
* if any errors are present.
|
||||
* If everything is working as intended, it is a no-op.
|
||||
* Otherwise, return a string representing what is failing.
|
||||
* ProbeDelegate is used to handle probe checks.
|
||||
* You can sendSuccess and sendError from Utils to handle success
|
||||
* and failure conditions.
|
||||
* @callback ProbeDelegate
|
||||
* @param res - HTTP response for writing
|
||||
* @param log - Werelogs instance for logging if you choose to
|
||||
* @return String representing issues to report. An empty
|
||||
* string or undefined is used to represent no issues.
|
||||
*/
|
||||
|
||||
export type ProbeDelegate = (res: http.ServerResponse, log: RequestLogger) => string | void
|
||||
|
@ -90,6 +87,6 @@ export class ProbeServer extends httpServer {
|
|||
return;
|
||||
}
|
||||
|
||||
this._handlers.get(req.url!)!(res, log);
|
||||
this._handlers.get(req.url ?? '')?.(res, log);
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,46 @@
|
|||
import * as http from 'http';
|
||||
import { ArsenalError } from '../../errors';
|
||||
|
||||
/**
|
||||
* Send a successful HTTP response of 200 OK
|
||||
* @param res - HTTP response for writing
|
||||
* @param log - Werelogs instance for logging if you choose to
|
||||
* @param [message] - Message to send as response, defaults to OK
|
||||
*/
|
||||
export function sendSuccess(
|
||||
res: http.ServerResponse,
|
||||
log: RequestLogger,
|
||||
message = 'OK'
|
||||
) {
|
||||
log.debug('replying with success');
|
||||
res.writeHead(200);
|
||||
res.end(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an Arsenal Error response
|
||||
* @param res - HTTP response for writing
|
||||
* @param log - Werelogs instance for logging if you choose to
|
||||
* @param error - Error to send back to the user
|
||||
* @param [optMessage] - Message to use instead of the errors message
|
||||
*/
|
||||
export function sendError(
|
||||
res: http.ServerResponse,
|
||||
log: RequestLogger,
|
||||
error: ArsenalError,
|
||||
optMessage?: string
|
||||
) {
|
||||
const message = optMessage || error.description || '';
|
||||
log.debug('sending back error response', {
|
||||
httpCode: error.code,
|
||||
errorType: error.message,
|
||||
error: message,
|
||||
});
|
||||
res.writeHead(error.code);
|
||||
res.end(
|
||||
JSON.stringify({
|
||||
errorType: error.message,
|
||||
errorMessage: message,
|
||||
})
|
||||
);
|
||||
}
|
|
@ -4,7 +4,7 @@ import * as werelogs from 'werelogs';
|
|||
import * as constants from '../../constants';
|
||||
import * as utils from './utils';
|
||||
import errors, { ArsenalError } from '../../errors';
|
||||
import HttpAgent from 'agentkeepalive';
|
||||
import { http as HttpAgent } from 'httpagent';
|
||||
import * as stream from 'stream';
|
||||
|
||||
function setRequestUids(reqHeaders: http.IncomingHttpHeaders, reqUids: string) {
|
||||
|
@ -71,8 +71,9 @@ function makeErrorFromHTTPResponse(response: http.IncomingMessage) {
|
|||
export default class RESTClient {
|
||||
host: string;
|
||||
port: number;
|
||||
httpAgent: HttpAgent;
|
||||
httpAgent: http.Agent;
|
||||
logging: werelogs.Logger;
|
||||
isPassthrough: boolean;
|
||||
|
||||
/**
|
||||
* Interface to the data file server
|
||||
|
@ -88,17 +89,19 @@ export default class RESTClient {
|
|||
host: string;
|
||||
port: number;
|
||||
logApi: { Logger: typeof werelogs.Logger };
|
||||
isPassthrough?: boolean;
|
||||
}) {
|
||||
assert(params.host);
|
||||
assert(params.port);
|
||||
|
||||
this.host = params.host;
|
||||
this.port = params.port;
|
||||
this.isPassthrough = params.isPassthrough || false;
|
||||
this.logging = new (params.logApi || werelogs).Logger('DataFileRESTClient');
|
||||
this.httpAgent = new HttpAgent({
|
||||
this.httpAgent = new HttpAgent.Agent({
|
||||
keepAlive: true,
|
||||
freeSocketTimeout: constants.httpClientFreeSocketTimeout,
|
||||
});
|
||||
}) as http.Agent;
|
||||
}
|
||||
|
||||
/** Destroy the HTTP agent, forcing a close of the remaining open connections */
|
||||
|
@ -121,11 +124,13 @@ export default class RESTClient {
|
|||
) {
|
||||
const reqHeaders = headers || {};
|
||||
const urlKey = key || '';
|
||||
const prefix = this.isPassthrough ?
|
||||
constants.passthroughFileURL : constants.dataFileURL;
|
||||
const reqParams = {
|
||||
hostname: this.host,
|
||||
port: this.port,
|
||||
method,
|
||||
path: `${constants.dataFileURL}/${urlKey}`,
|
||||
path: encodeURI(`${prefix}/${urlKey}`),
|
||||
headers: reqHeaders,
|
||||
agent: this.httpAgent,
|
||||
};
|
||||
|
|
|
@ -4,7 +4,7 @@ import * as werelogs from 'werelogs';
|
|||
import * as http from 'http';
|
||||
import httpServer from '../http/server';
|
||||
import * as constants from '../../constants';
|
||||
import * as utils from './utils';
|
||||
import { parseURL } from './utils';
|
||||
import * as httpUtils from '../http/utils';
|
||||
import errors, { ArsenalError } from '../../errors';
|
||||
|
||||
|
@ -38,42 +38,6 @@ function sendError(
|
|||
errorMessage: message })}\n`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the given url and return a pathInfo object. Sanity checks are
|
||||
* performed.
|
||||
*
|
||||
* @param urlStr - URL to parse
|
||||
* @param expectKey - whether the command expects to see a
|
||||
* key in the URL
|
||||
* @return a pathInfo object with URL items containing the
|
||||
* following attributes:
|
||||
* - pathInfo.service {String} - The name of REST service ("DataFile")
|
||||
* - pathInfo.key {String} - The requested key
|
||||
*/
|
||||
function parseURL(urlStr: string, expectKey: boolean) {
|
||||
const urlObj = url.parse(urlStr);
|
||||
const pathInfo = utils.explodePath(urlObj.path!);
|
||||
if (pathInfo.service !== constants.dataFileURL) {
|
||||
throw errors.InvalidAction.customizeDescription(
|
||||
`unsupported service '${pathInfo.service}'`);
|
||||
}
|
||||
if (expectKey && pathInfo.key === undefined) {
|
||||
throw errors.MissingParameter.customizeDescription(
|
||||
'URL is missing key');
|
||||
}
|
||||
if (!expectKey && pathInfo.key !== undefined) {
|
||||
// note: we may implement rewrite functionality by allowing a
|
||||
// key in the URL, though we may still provide the new key in
|
||||
// the Location header to keep immutability property and
|
||||
// atomicity of the update (we would just remove the old
|
||||
// object when the new one has been written entirely in this
|
||||
// case, saving a request over an equivalent PUT + DELETE).
|
||||
throw errors.InvalidURI.customizeDescription(
|
||||
'PUT url cannot contain a key');
|
||||
}
|
||||
return pathInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* @class
|
||||
* @classdesc REST Server interface
|
||||
|
@ -104,7 +68,6 @@ export default class RESTServer extends httpServer {
|
|||
}) {
|
||||
assert(params.port);
|
||||
|
||||
// @ts-expect-error
|
||||
werelogs.configure({
|
||||
level: params.log.logLevel,
|
||||
dump: params.log.dumpLevel,
|
||||
|
|
|
@ -1,6 +1,16 @@
|
|||
import errors from '../../errors';
|
||||
import * as constants from '../../constants';
|
||||
import * as url from 'url';
|
||||
const passthroughPrefixLength = constants.passthroughFileURL.length;
|
||||
|
||||
export function explodePath(path: string) {
|
||||
if (path.startsWith(constants.passthroughFileURL)) {
|
||||
const key = path.slice(passthroughPrefixLength + 1);
|
||||
return {
|
||||
service: constants.passthroughFileURL,
|
||||
key: key.length > 0 ? key : undefined,
|
||||
};
|
||||
}
|
||||
const pathMatch = /^(\/[a-zA-Z0-9]+)(\/([0-9a-f]*))?$/.exec(path);
|
||||
if (pathMatch) {
|
||||
return {
|
||||
|
@ -10,4 +20,41 @@ export function explodePath(path: string) {
|
|||
};
|
||||
}
|
||||
throw errors.InvalidURI.customizeDescription('malformed URI');
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the given url and return a pathInfo object. Sanity checks are
|
||||
* performed.
|
||||
*
|
||||
* @param urlStr - URL to parse
|
||||
* @param expectKey - whether the command expects to see a
|
||||
* key in the URL
|
||||
* @return a pathInfo object with URL items containing the
|
||||
* following attributes:
|
||||
* - pathInfo.service {String} - The name of REST service ("DataFile")
|
||||
* - pathInfo.key {String} - The requested key
|
||||
*/
|
||||
export function parseURL(urlStr: string, expectKey: boolean) {
|
||||
const urlObj = url.parse(urlStr);
|
||||
const pathInfo = explodePath(decodeURI(urlObj.path!));
|
||||
if ((pathInfo.service !== constants.dataFileURL)
|
||||
&& (pathInfo.service !== constants.passthroughFileURL)) {
|
||||
throw errors.InvalidAction.customizeDescription(
|
||||
`unsupported service '${pathInfo.service}'`);
|
||||
}
|
||||
if (expectKey && pathInfo.key === undefined) {
|
||||
throw errors.MissingParameter.customizeDescription(
|
||||
'URL is missing key');
|
||||
}
|
||||
if (!expectKey && pathInfo.key !== undefined) {
|
||||
// note: we may implement rewrite functionality by allowing a
|
||||
// key in the URL, though we may still provide the new key in
|
||||
// the Location header to keep immutability property and
|
||||
// atomicity of the update (we would just remove the old
|
||||
// object when the new one has been written entirely in this
|
||||
// case, saving a request over an equivalent PUT + DELETE).
|
||||
throw errors.InvalidURI.customizeDescription(
|
||||
'PUT url cannot contain a key');
|
||||
}
|
||||
return pathInfo;
|
||||
}
|
||||
|
|
|
@ -0,0 +1,209 @@
|
|||
import { URL } from 'url';
|
||||
import { decryptSecret } from '../executables/pensieveCreds/utils';
|
||||
import { Logger } from 'werelogs';
|
||||
|
||||
export type LocationType =
|
||||
| 'location-mem-v1'
|
||||
| 'location-file-v1'
|
||||
| 'location-azure-v1'
|
||||
| 'location-ceph-radosgw-s3-v1'
|
||||
| 'location-scality-ring-s3-v1'
|
||||
| 'location-aws-s3-v1'
|
||||
| 'location-wasabi-v1'
|
||||
| 'location-do-spaces-v1'
|
||||
| 'location-gcp-v1'
|
||||
| 'location-scality-sproxyd-v1'
|
||||
| 'location-nfs-mount-v1'
|
||||
| 'location-scality-hdclient-v2';
|
||||
|
||||
export interface OverlayLocations {
|
||||
[key: string]: {
|
||||
name: string;
|
||||
objectId: string;
|
||||
details?: any;
|
||||
locationType: string;
|
||||
sizeLimitGB?: number;
|
||||
isTransient?: boolean;
|
||||
legacyAwsBehavior?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export type Location = {
|
||||
type:
|
||||
| 'mem'
|
||||
| 'file'
|
||||
| 'azure'
|
||||
| 'aws_s3'
|
||||
| 'gcp'
|
||||
| 'scality'
|
||||
| 'pfs'
|
||||
| 'scality';
|
||||
name: string;
|
||||
objectId: string;
|
||||
details: { [key: string]: any };
|
||||
locationType: string;
|
||||
sizeLimitGB: number | null;
|
||||
isTransient: boolean;
|
||||
legacyAwsBehavior: boolean;
|
||||
};
|
||||
|
||||
export function patchLocations(
|
||||
overlayLocations: OverlayLocations | undefined | null,
|
||||
creds: any,
|
||||
log: Logger
|
||||
) {
|
||||
const locs = overlayLocations ?? {};
|
||||
return Object.entries(locs).reduce<{ [key: string]: Location }>(
|
||||
(acc, [k, l]) => {
|
||||
const location: Location = {
|
||||
type: 'mem',
|
||||
name: k,
|
||||
objectId: l.objectId,
|
||||
details: l.details || {},
|
||||
locationType: l.locationType,
|
||||
sizeLimitGB: l.sizeLimitGB || null,
|
||||
isTransient: Boolean(l.isTransient),
|
||||
legacyAwsBehavior: Boolean(l.legacyAwsBehavior),
|
||||
};
|
||||
let supportsVersioning = false;
|
||||
let pathStyle = process.env.CI_CEPH !== undefined;
|
||||
|
||||
switch (l.locationType) {
|
||||
case 'location-mem-v1':
|
||||
location.type = 'mem';
|
||||
location.details = { supportsVersioning: true };
|
||||
break;
|
||||
case 'location-file-v1':
|
||||
location.type = 'file';
|
||||
location.details = { supportsVersioning: true };
|
||||
break;
|
||||
case 'location-azure-v1':
|
||||
location.type = 'azure';
|
||||
if (l.details.secretKey && l.details.secretKey.length > 0) {
|
||||
location.details = {
|
||||
bucketMatch: l.details.bucketMatch,
|
||||
azureStorageEndpoint: l.details.endpoint,
|
||||
azureStorageAccountName: l.details.accessKey,
|
||||
azureStorageAccessKey: decryptSecret(
|
||||
creds,
|
||||
l.details.secretKey
|
||||
),
|
||||
azureContainerName: l.details.bucketName,
|
||||
};
|
||||
}
|
||||
break;
|
||||
case 'location-ceph-radosgw-s3-v1':
|
||||
case 'location-scality-ring-s3-v1':
|
||||
pathStyle = true; // fallthrough
|
||||
case 'location-aws-s3-v1':
|
||||
case 'location-wasabi-v1':
|
||||
supportsVersioning = true; // fallthrough
|
||||
case 'location-do-spaces-v1':
|
||||
location.type = 'aws_s3';
|
||||
if (l.details.secretKey && l.details.secretKey.length > 0) {
|
||||
let https = true;
|
||||
let awsEndpoint =
|
||||
l.details.endpoint || 's3.amazonaws.com';
|
||||
if (awsEndpoint.includes('://')) {
|
||||
const url = new URL(awsEndpoint);
|
||||
awsEndpoint = url.host;
|
||||
https = url.protocol.includes('https');
|
||||
}
|
||||
|
||||
location.details = {
|
||||
credentials: {
|
||||
accessKey: l.details.accessKey,
|
||||
secretKey: decryptSecret(
|
||||
creds,
|
||||
l.details.secretKey
|
||||
),
|
||||
},
|
||||
bucketName: l.details.bucketName,
|
||||
bucketMatch: l.details.bucketMatch,
|
||||
serverSideEncryption: Boolean(
|
||||
l.details.serverSideEncryption
|
||||
),
|
||||
region: l.details.region,
|
||||
awsEndpoint,
|
||||
supportsVersioning,
|
||||
pathStyle,
|
||||
https,
|
||||
};
|
||||
}
|
||||
break;
|
||||
case 'location-gcp-v1':
|
||||
location.type = 'gcp';
|
||||
if (l.details.secretKey && l.details.secretKey.length > 0) {
|
||||
location.details = {
|
||||
credentials: {
|
||||
accessKey: l.details.accessKey,
|
||||
secretKey: decryptSecret(
|
||||
creds,
|
||||
l.details.secretKey
|
||||
),
|
||||
},
|
||||
bucketName: l.details.bucketName,
|
||||
mpuBucketName: l.details.mpuBucketName,
|
||||
bucketMatch: l.details.bucketMatch,
|
||||
gcpEndpoint:
|
||||
l.details.endpoint || 'storage.googleapis.com',
|
||||
https: true,
|
||||
};
|
||||
}
|
||||
break;
|
||||
case 'location-scality-sproxyd-v1':
|
||||
location.type = 'scality';
|
||||
if (
|
||||
l.details &&
|
||||
l.details.bootstrapList &&
|
||||
l.details.proxyPath
|
||||
) {
|
||||
location.details = {
|
||||
supportsVersioning: true,
|
||||
connector: {
|
||||
sproxyd: {
|
||||
chordCos: l.details.chordCos || null,
|
||||
bootstrap: l.details.bootstrapList,
|
||||
path: l.details.proxyPath,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
break;
|
||||
case 'location-nfs-mount-v1':
|
||||
location.type = 'pfs';
|
||||
if (l.details) {
|
||||
location.details = {
|
||||
supportsVersioning: true,
|
||||
bucketMatch: true,
|
||||
pfsDaemonEndpoint: {
|
||||
host: `${l.name}-cosmos-pfsd`,
|
||||
port: 80,
|
||||
},
|
||||
};
|
||||
}
|
||||
break;
|
||||
case 'location-scality-hdclient-v2':
|
||||
location.type = 'scality';
|
||||
if (l.details && l.details.bootstrapList) {
|
||||
location.details = {
|
||||
supportsVersioning: true,
|
||||
connector: {
|
||||
hdclient: {
|
||||
bootstrap: l.details.bootstrapList,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
break;
|
||||
default:
|
||||
log.info('unknown location type', {
|
||||
locationType: l.locationType,
|
||||
});
|
||||
return acc;
|
||||
}
|
||||
return { ...acc, [location.name]: location };
|
||||
},
|
||||
{}
|
||||
);
|
||||
}
|
|
@ -38,6 +38,10 @@
|
|||
"type": "string",
|
||||
"pattern": "^arn:aws:iam::[0-9]{12}:saml-provider/[\\w._-]{1,128}$"
|
||||
},
|
||||
"principalFederatedOidcIdp": {
|
||||
"type": "string",
|
||||
"pattern": "^(?:http(s)?:\/\/)?[\\w.-]+(?:\\.[\\w\\.-]+)+[\\w\\-\\._~:/?#[\\]@!\\$&'\\(\\)\\*\\+,;=.]+$"
|
||||
},
|
||||
"principalAWSItem": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
|
@ -98,6 +102,9 @@
|
|||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/definitions/principalFederatedSamlIdp"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/principalFederatedOidcIdp"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
@ -126,7 +126,9 @@ export type RequesterInfo = {
|
|||
principalType: string;
|
||||
principaltype: string;
|
||||
userid: string;
|
||||
username: string,
|
||||
username: string;
|
||||
keycloakGroup: string;
|
||||
keycloakRole: string;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -33,6 +33,7 @@ const sharedActionMap = {
|
|||
bypassGovernanceRetention: 's3:BypassGovernanceRetention',
|
||||
listMultipartUploads: 's3:ListBucketMultipartUploads',
|
||||
listParts: 's3:ListMultipartUploadParts',
|
||||
metadataSearch: 's3:MetadataSearch',
|
||||
multipartDelete: 's3:AbortMultipartUpload',
|
||||
objectDelete: 's3:DeleteObject',
|
||||
objectDeleteTagging: 's3:DeleteObjectTagging',
|
||||
|
@ -47,6 +48,8 @@ const sharedActionMap = {
|
|||
objectPutLegalHold: 's3:PutObjectLegalHold',
|
||||
objectPutRetention: 's3:PutObjectRetention',
|
||||
objectPutTagging: 's3:PutObjectTagging',
|
||||
objectRestore: 's3:RestoreObject',
|
||||
objectPutVersion: 's3:PutObjectVersion',
|
||||
};
|
||||
|
||||
// action map used for request context
|
||||
|
@ -70,8 +73,10 @@ const actionMapRQ = {
|
|||
objectPutTaggingVersion: 's3:PutObjectVersionTagging',
|
||||
serviceGet: 's3:ListAllMyBuckets',
|
||||
objectReplicate: 's3:ReplicateObject',
|
||||
objectPutRetentionVersion: 's3:PutObjectVersionRetention',
|
||||
objectPutLegalHoldVersion: 's3:PutObjectVersionLegalHold',
|
||||
objectGetRetentionVersion: 's3:GetObjectRetention',
|
||||
objectPutRetentionVersion: 's3:PutObjectRetention',
|
||||
objectGetLegalHoldVersion: 's3:GetObjectLegalHold',
|
||||
objectPutLegalHoldVersion: 's3:PutObjectLegalHold',
|
||||
listObjectVersions: 's3:ListBucketVersions',
|
||||
...sharedActionMap,
|
||||
};
|
||||
|
@ -125,6 +130,7 @@ const actionMonitoringMapS3 = {
|
|||
initiateMultipartUpload: 'CreateMultipartUpload',
|
||||
listMultipartUploads: 'ListMultipartUploads',
|
||||
listParts: 'ListParts',
|
||||
metadataSearch: 'MetadataSearch',
|
||||
multiObjectDelete: 'DeleteObjects',
|
||||
multipartDelete: 'AbortMultipartUpload',
|
||||
objectCopy: 'CopyObject',
|
||||
|
@ -143,6 +149,7 @@ const actionMonitoringMapS3 = {
|
|||
objectPutPart: 'UploadPart',
|
||||
objectPutRetention: 'PutObjectRetention',
|
||||
objectPutTagging: 'PutObjectTagging',
|
||||
objectRestore: 'RestoreObject',
|
||||
serviceGet: 'ListBuckets',
|
||||
};
|
||||
|
||||
|
@ -168,6 +175,7 @@ const actionMapIAM = {
|
|||
getPolicyVersion: 'iam:GetPolicyVersion',
|
||||
getUser: 'iam:GetUser',
|
||||
listAccessKeys: 'iam:ListAccessKeys',
|
||||
listEntitiesForPolicy: 'iam:ListEntitiesForPolicy',
|
||||
listGroupPolicies: 'iam:ListGroupPolicies',
|
||||
listGroups: 'iam:ListGroups',
|
||||
listGroupsForUser: 'iam:ListGroupsForUser',
|
||||
|
|
|
@ -142,6 +142,8 @@ export function findConditionKey(
|
|||
// header
|
||||
case 's3:ObjLocationConstraint': return headers['x-amz-meta-scal-location-constraint'];
|
||||
case 'sts:ExternalId': return requestContext.getRequesterExternalId();
|
||||
case 'keycloak:groups': return requesterInfo.keycloakGroup;
|
||||
case 'keycloak:roles': return requesterInfo.keycloakRole;
|
||||
case 'iam:PolicyArn': return requestContext.getPolicyArn();
|
||||
// s3:ExistingObjectTag - Used to check that existing object tag has
|
||||
// specific tag key and value. Extraction of correct tag key is done in CloudServer.
|
||||
|
|
|
@ -30,7 +30,7 @@ export default class ResultsCollector extends EventEmitter {
|
|||
* @emits ResultCollector#done
|
||||
* @emits ResultCollector#error
|
||||
*/
|
||||
pushResult(err: Error | undefined, subPartIndex: number) {
|
||||
pushResult(err: Error | null | undefined, subPartIndex: number) {
|
||||
this._results.push({
|
||||
error: err,
|
||||
subPartIndex,
|
||||
|
|
|
@ -1,11 +1,12 @@
|
|||
import assert from 'assert';
|
||||
import * as crypto from 'crypto';
|
||||
import * as stream from 'stream';
|
||||
import azure from '@azure/storage-blob';
|
||||
import ResultsCollector from './ResultsCollector';
|
||||
import SubStreamInterface from './SubStreamInterface';
|
||||
import * as objectUtils from '../objectUtils';
|
||||
import MD5Sum from '../MD5Sum';
|
||||
import errors from '../../errors';
|
||||
import errors, { ArsenalError } from '../../errors';
|
||||
|
||||
export const splitter = '|';
|
||||
export const overviewMpuKey = 'azure_mpu';
|
||||
|
@ -61,7 +62,7 @@ export const getBlockId = (
|
|||
const paddedSubPart = padString(subPartIndex, 'subPart');
|
||||
const blockId = `${uploadId}${splitter}partNumber${paddedPartNumber}` +
|
||||
`${splitter}subPart${paddedSubPart}${splitter}`;
|
||||
return padString(blockId, 'part');
|
||||
return Buffer.from(padString(blockId, 'part')).toString('base64');
|
||||
};
|
||||
|
||||
export const getSummaryPartId = (partNumber: number, eTag: string, size: number) => {
|
||||
|
@ -100,10 +101,17 @@ export const getSubPartIds = (
|
|||
) => [...Array(part.numberSubParts).keys()].map(subPartIndex =>
|
||||
getBlockId(uploadId, part.partNumber, subPartIndex));
|
||||
|
||||
// TODO Better type this
|
||||
type ErrorWrapperFn = (
|
||||
s3Method: string,
|
||||
azureMethod: string,
|
||||
command: (client: azure.ContainerClient) => Promise<any>,
|
||||
log: RequestLogger,
|
||||
cb: (err: ArsenalError | null | undefined) => void,
|
||||
) => void
|
||||
|
||||
export const putSinglePart = (
|
||||
errorWrapperFn: (first: string, second: string, third: any, log: any, cb: any) => void,
|
||||
request: any,
|
||||
errorWrapperFn: ErrorWrapperFn,
|
||||
request: stream.Readable,
|
||||
params: {
|
||||
bucketName: string;
|
||||
partNumber: number;
|
||||
|
@ -114,44 +122,44 @@ export const putSinglePart = (
|
|||
},
|
||||
dataStoreName: string,
|
||||
log: RequestLogger,
|
||||
cb: any,
|
||||
cb: (err: ArsenalError | null | undefined, dataStoreETag?: string, size?: number) => void,
|
||||
) => {
|
||||
const { bucketName, partNumber, size, objectKey, contentMD5, uploadId }
|
||||
= params;
|
||||
const blockId = getBlockId(uploadId, partNumber, 0);
|
||||
const passThrough = new stream.PassThrough();
|
||||
const options = contentMD5
|
||||
? { useTransactionalMD5: true, transactionalContentMD5: contentMD5 }
|
||||
? { transactionalContentMD5: objectUtils.getMD5Buffer(contentMD5) }
|
||||
: {};
|
||||
request.pipe(passThrough);
|
||||
return errorWrapperFn('uploadPart', 'createBlockFromStream',
|
||||
[blockId, bucketName, objectKey, passThrough, size, options,
|
||||
(err: any | null, result: any) => {
|
||||
if (err) {
|
||||
log.error('Error from Azure data backend uploadPart',
|
||||
{ error: err.message, dataStoreName });
|
||||
if (err.code === 'ContainerNotFound') {
|
||||
return cb(errors.NoSuchBucket);
|
||||
}
|
||||
if (err.code === 'InvalidMd5') {
|
||||
return cb(errors.InvalidDigest);
|
||||
}
|
||||
if (err.code === 'Md5Mismatch') {
|
||||
return cb(errors.BadDigest);
|
||||
}
|
||||
return cb(errors.InternalError.customizeDescription(
|
||||
`Error returned from Azure: ${err.message}`),
|
||||
);
|
||||
}
|
||||
const md5 = result.headers['content-md5'] || '';
|
||||
const eTag = objectUtils.getHexMD5(md5);
|
||||
return cb(null, eTag, size);
|
||||
}], log, cb);
|
||||
return errorWrapperFn('uploadPart', 'createBlockFromStream', async client => {
|
||||
try {
|
||||
const result = await client.getBlockBlobClient(objectKey)
|
||||
.stageBlock(blockId, () => passThrough, size, options);
|
||||
const md5 = result.contentMD5 || '';
|
||||
const eTag = objectUtils.getHexMD5(md5);
|
||||
return eTag
|
||||
} catch (err: any) {
|
||||
log.error('Error from Azure data backend uploadPart',
|
||||
{ error: err.message, dataStoreName });
|
||||
if (err.code === 'ContainerNotFound') {
|
||||
throw errors.NoSuchBucket;
|
||||
}
|
||||
if (err.code === 'InvalidMd5') {
|
||||
throw errors.InvalidDigest;
|
||||
}
|
||||
if (err.code === 'Md5Mismatch') {
|
||||
throw errors.BadDigest;
|
||||
}
|
||||
throw errors.InternalError.customizeDescription(
|
||||
`Error returned from Azure: ${err.message}`
|
||||
);
|
||||
}
|
||||
}, log, cb);
|
||||
};
|
||||
|
||||
// TODO type this
|
||||
export const putNextSubPart = (
|
||||
errorWrapperFn: any,
|
||||
const putNextSubPart = (
|
||||
errorWrapperFn: ErrorWrapperFn,
|
||||
partParams: {
|
||||
uploadId: string;
|
||||
partNumber: number;
|
||||
|
@ -159,11 +167,10 @@ export const putNextSubPart = (
|
|||
objectKey: string;
|
||||
},
|
||||
subPartInfo: { lastPartIndex: number; lastPartSize: number },
|
||||
subPartStream: any,
|
||||
subPartStream: stream.Readable,
|
||||
subPartIndex: number,
|
||||
resultsCollector: ResultsCollector,
|
||||
log: RequestLogger,
|
||||
cb: any,
|
||||
) => {
|
||||
const { uploadId, partNumber, bucketName, objectKey } = partParams;
|
||||
const subPartSize = getSubPartSize(
|
||||
|
@ -171,14 +178,20 @@ export const putNextSubPart = (
|
|||
const subPartId = getBlockId(uploadId, partNumber,
|
||||
subPartIndex);
|
||||
resultsCollector.pushOp();
|
||||
errorWrapperFn('uploadPart', 'createBlockFromStream',
|
||||
[subPartId, bucketName, objectKey, subPartStream, subPartSize,
|
||||
{}, err => resultsCollector.pushResult(err, subPartIndex)], log, cb);
|
||||
errorWrapperFn('uploadPart', 'createBlockFromStream', async client => {
|
||||
try {
|
||||
const result = await client.getBlockBlobClient(objectKey)
|
||||
.stageBlock(subPartId, () => subPartStream, subPartSize, {});
|
||||
resultsCollector.pushResult(null, subPartIndex);
|
||||
} catch (err: any) {
|
||||
resultsCollector.pushResult(err, subPartIndex);
|
||||
}
|
||||
}, log, () => {});
|
||||
};
|
||||
|
||||
export const putSubParts = (
|
||||
errorWrapperFn: any,
|
||||
request: any,
|
||||
errorWrapperFn: ErrorWrapperFn,
|
||||
request: stream.Readable,
|
||||
params: {
|
||||
uploadId: string;
|
||||
partNumber: number;
|
||||
|
@ -188,7 +201,7 @@ export const putSubParts = (
|
|||
},
|
||||
dataStoreName: string,
|
||||
log: RequestLogger,
|
||||
cb: any,
|
||||
cb: (err: ArsenalError | null | undefined, dataStoreETag?: string) => void,
|
||||
) => {
|
||||
const subPartInfo = getSubPartInfo(params.size);
|
||||
const resultsCollector = new ResultsCollector();
|
||||
|
@ -227,14 +240,13 @@ export const putSubParts = (
|
|||
const totalLength = streamInterface.getTotalBytesStreamed();
|
||||
log.trace('successfully put subparts to Azure',
|
||||
{ numberSubParts, totalLength });
|
||||
hashedStream.on('hashed', () => cb(null, hashedStream.completedHash,
|
||||
totalLength));
|
||||
hashedStream.on('hashed', () => cb(null, hashedStream.completedHash));
|
||||
|
||||
// in case the hashed event was already emitted before the
|
||||
// event handler was registered:
|
||||
if (hashedStream.completedHash) {
|
||||
hashedStream.removeAllListeners('hashed');
|
||||
return cb(null, hashedStream.completedHash, totalLength);
|
||||
return cb(null, hashedStream.completedHash);
|
||||
}
|
||||
return undefined;
|
||||
});
|
||||
|
@ -242,7 +254,7 @@ export const putSubParts = (
|
|||
const currentStream = streamInterface.getCurrentStream();
|
||||
// start first put to Azure before we start streaming the data
|
||||
putNextSubPart(errorWrapperFn, params, subPartInfo,
|
||||
currentStream, 0, resultsCollector, log, cb);
|
||||
currentStream, 0, resultsCollector, log);
|
||||
|
||||
request.pipe(hashedStream);
|
||||
hashedStream.on('end', () => {
|
||||
|
@ -262,8 +274,8 @@ export const putSubParts = (
|
|||
}
|
||||
const { nextStream, subPartIndex } =
|
||||
streamInterface.transitionToNextStream();
|
||||
putNextSubPart(errorWrapperFn, params, subPartInfo,
|
||||
nextStream, subPartIndex, resultsCollector, log, cb);
|
||||
putNextSubPart(errorWrapperFn, params, subPartInfo, nextStream,
|
||||
subPartIndex, resultsCollector, log);
|
||||
streamInterface.write(firstChunk);
|
||||
} else {
|
||||
streamInterface.write(data);
|
||||
|
|
|
@ -1,19 +1,25 @@
|
|||
const oneDay = 24 * 60 * 60 * 1000; // Milliseconds in a day.
|
||||
import { scaleMsPerDay } from '../objectUtils';
|
||||
const msInOneDay = 24 * 60 * 60 * 1000; // Milliseconds in a day.
|
||||
|
||||
export default class LifecycleDateTime {
|
||||
_transitionOneDayEarlier?: boolean;
|
||||
_expireOneDayEarlier?: boolean;
|
||||
_timeProgressionFactor?: number;
|
||||
_scaledMsPerDay: number;
|
||||
|
||||
constructor(params?: {
|
||||
transitionOneDayEarlier: boolean;
|
||||
expireOneDayEarlier: boolean;
|
||||
timeProgressionFactor: number;
|
||||
}) {
|
||||
this._transitionOneDayEarlier = params?.transitionOneDayEarlier;
|
||||
this._expireOneDayEarlier = params?.expireOneDayEarlier;
|
||||
this._timeProgressionFactor = params?.timeProgressionFactor || 1;
|
||||
this._scaledMsPerDay = scaleMsPerDay(this._timeProgressionFactor);
|
||||
}
|
||||
|
||||
getCurrentDate() {
|
||||
const timeTravel = this._expireOneDayEarlier ? oneDay : 0;
|
||||
const timeTravel = this._expireOneDayEarlier ? msInOneDay : 0;
|
||||
return Date.now() + timeTravel;
|
||||
}
|
||||
|
||||
|
@ -25,7 +31,7 @@ export default class LifecycleDateTime {
|
|||
findDaysSince(date: Date) {
|
||||
const now = this.getCurrentDate();
|
||||
const diff = now - date.getTime();
|
||||
return Math.floor(diff / (1000 * 60 * 60 * 24));
|
||||
return Math.floor(diff / this._scaledMsPerDay);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -52,8 +58,25 @@ export default class LifecycleDateTime {
|
|||
}
|
||||
if (transition.Days !== undefined) {
|
||||
const lastModifiedTime = this.getTimestamp(lastModified);
|
||||
const timeTravel = this._transitionOneDayEarlier ? -oneDay : 0;
|
||||
return lastModifiedTime + (transition.Days * oneDay) + timeTravel;
|
||||
const timeTravel = this._transitionOneDayEarlier ? -msInOneDay : 0;
|
||||
return lastModifiedTime + (transition.Days * this._scaledMsPerDay) + timeTravel;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the Unix time at which the non-current version transition should occur.
|
||||
* @param transition - A non-current version transition from the lifecycle non-current version transitions
|
||||
* @param lastModified - The object's last modified date
|
||||
* @return - The normalized transition timestamp
|
||||
*/
|
||||
getNCVTransitionTimestamp(
|
||||
transition: { NoncurrentDays?: number },
|
||||
lastModified: string,
|
||||
) {
|
||||
if (transition.NoncurrentDays !== undefined) {
|
||||
const lastModifiedTime = this.getTimestamp(lastModified);
|
||||
const timeTravel = this._transitionOneDayEarlier ? -msInOneDay : 0;
|
||||
return lastModifiedTime + (transition.NoncurrentDays * this._scaledMsPerDay) + timeTravel;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -61,6 +61,47 @@ export default class LifecycleUtils {
|
|||
return trans1 > trans2 ? transition1 : transition2;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two non-current version transition rules and return the one that is most recent.
|
||||
* @param params - The function parameters
|
||||
* @param params.transition1 - A non-current version transition from the current rule
|
||||
* @param params.transition2 - A non-current version transition from the previous rule
|
||||
* @param params.lastModified - The object's last modified
|
||||
* date
|
||||
* @return The most applicable transition rule
|
||||
*/
|
||||
compareNCVTransitions(params: {
|
||||
lastModified: string;
|
||||
transition1: any;
|
||||
transition2?: any;
|
||||
}): number | undefined;
|
||||
compareNCVTransitions(params: {
|
||||
lastModified: string;
|
||||
transition1?: any;
|
||||
transition2: any;
|
||||
}): number | undefined;
|
||||
compareNCVTransitions(params: {
|
||||
lastModified: string;
|
||||
transition1: any;
|
||||
transition2: any;
|
||||
}): number | undefined;
|
||||
compareNCVTransitions(params: {
|
||||
lastModified: string;
|
||||
transition1?: any;
|
||||
transition2?: any;
|
||||
}) {
|
||||
const { transition1, transition2, lastModified } = params;
|
||||
if (transition1 === undefined) {
|
||||
return transition2;
|
||||
}
|
||||
if (transition2 === undefined) {
|
||||
return transition1;
|
||||
}
|
||||
const trans1 = this._datetime.getNCVTransitionTimestamp(transition1!, lastModified)!;
|
||||
const trans2 = this._datetime.getNCVTransitionTimestamp(transition2!, lastModified)!;
|
||||
return trans1 > trans2 ? transition1 : transition2;
|
||||
}
|
||||
|
||||
// TODO Fix This
|
||||
/**
|
||||
* Find the most relevant trantition rule for the given transitions array
|
||||
|
@ -98,6 +139,42 @@ export default class LifecycleUtils {
|
|||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the most relevant non-current version transition rule for the given transitions array
|
||||
* and any previously stored non-current version transition from another rule.
|
||||
* @param params - The function parameters
|
||||
* @param params.transitions - Array of lifecycle non-current version transitions
|
||||
* @param params.lastModified - The object's last modified
|
||||
* date
|
||||
* @return The most applicable non-current version transition rule
|
||||
*/
|
||||
getApplicableNCVTransition(params: {
|
||||
store: any;
|
||||
currentDate: Date;
|
||||
transitions: any[];
|
||||
lastModified: string;
|
||||
}) {
|
||||
const { transitions, store, lastModified, currentDate } = params;
|
||||
const transition = transitions.reduce((result, transition) => {
|
||||
const isApplicable = // Is the transition time in the past?
|
||||
this._datetime.getTimestamp(currentDate) >=
|
||||
this._datetime.getNCVTransitionTimestamp(transition, lastModified)!;
|
||||
if (!isApplicable) {
|
||||
return result;
|
||||
}
|
||||
return this.compareNCVTransitions({
|
||||
transition1: transition,
|
||||
transition2: result,
|
||||
lastModified,
|
||||
});
|
||||
}, undefined);
|
||||
return this.compareNCVTransitions({
|
||||
transition1: transition,
|
||||
transition2: store.NoncurrentVersionTransition,
|
||||
lastModified,
|
||||
});
|
||||
}
|
||||
|
||||
// TODO
|
||||
/**
|
||||
* Filter out all rules based on `Status` and `Filter` (Prefix and Tags)
|
||||
|
@ -241,7 +318,17 @@ export default class LifecycleUtils {
|
|||
currentDate,
|
||||
});
|
||||
}
|
||||
// TODO: Add support for NoncurrentVersionTransitions.
|
||||
|
||||
const ncvt = 'NoncurrentVersionTransitions';
|
||||
const hasNoncurrentVersionTransitions = Array.isArray(rule[ncvt]) && rule[ncvt].length > 0;
|
||||
if (hasNoncurrentVersionTransitions && this._supportedRules.includes('noncurrentVersionTransition')) {
|
||||
store.NoncurrentVersionTransition = this.getApplicableNCVTransition({
|
||||
transitions: rule.NoncurrentVersionTransitions,
|
||||
lastModified: metadata.LastModified,
|
||||
store,
|
||||
currentDate,
|
||||
});
|
||||
}
|
||||
return store;
|
||||
}, {});
|
||||
// Do not transition to a location where the object is already stored.
|
||||
|
@ -249,6 +336,12 @@ export default class LifecycleUtils {
|
|||
&& applicableRules.Transition.StorageClass === metadata.StorageClass) {
|
||||
applicableRules.Transition = undefined;
|
||||
}
|
||||
|
||||
if (applicableRules.NoncurrentVersionTransition
|
||||
&& applicableRules.NoncurrentVersionTransition.StorageClass === metadata.StorageClass) {
|
||||
applicableRules.NoncurrentVersionTransition = undefined;
|
||||
}
|
||||
|
||||
return applicableRules;
|
||||
/* eslint-enable no-param-reassign */
|
||||
}
|
||||
|
|
|
@ -0,0 +1,110 @@
|
|||
import {parseStringPromise} from 'xml2js';
|
||||
import errors, {ArsenalError} from '../errors';
|
||||
import * as werelogs from 'werelogs';
|
||||
import {validRestoreObjectTiers} from "../constants";
|
||||
|
||||
/*
|
||||
Format of xml request:
|
||||
<RestoreRequest>
|
||||
<Days>integer</Days>
|
||||
<Tier>Standard|Bulk|Expedited</Tier>
|
||||
</RestoreRequest>
|
||||
*/
|
||||
|
||||
/**
|
||||
* validate restore request xml
|
||||
* @param restoreRequest - parsed restore request object
|
||||
* @return{ArsenalError|undefined} - error on failure, undefined on success
|
||||
*/
|
||||
function validateRestoreRequest(restoreRequest?: any) {
|
||||
if (!restoreRequest) {
|
||||
const desc = 'request xml does not contain RestoreRequest';
|
||||
return errors.MalformedXML.customizeDescription(desc);
|
||||
}
|
||||
if (!restoreRequest.Days || !restoreRequest.Days[0]) {
|
||||
const desc = 'request xml does not contain RestoreRequest.Days';
|
||||
return errors.MalformedXML.customizeDescription(desc);
|
||||
}
|
||||
// RestoreRequest.Days must be greater than or equal to 1
|
||||
const daysValue = Number.parseInt(restoreRequest.Days[0], 10);
|
||||
if (Number.isNaN(daysValue)) {
|
||||
const desc = `RestoreRequest.Days is invalid type. [${restoreRequest.Days[0]}]`;
|
||||
return errors.MalformedXML.customizeDescription(desc);
|
||||
}
|
||||
if (daysValue < 1) {
|
||||
const desc = `RestoreRequest.Days must be greater than 0. [${restoreRequest.Days[0]}]`;
|
||||
return errors.MalformedXML.customizeDescription(desc);
|
||||
}
|
||||
if (daysValue > 2147483647) {
|
||||
const desc = `RestoreRequest.Days must be less than 2147483648. [${restoreRequest.Days[0]}]`;
|
||||
return errors.MalformedXML.customizeDescription(desc);
|
||||
}
|
||||
if (restoreRequest.Tier && restoreRequest.Tier[0] && !validRestoreObjectTiers.has(restoreRequest.Tier[0])) {
|
||||
const desc = `RestoreRequest.Tier is invalid value. [${restoreRequest.Tier[0]}]`;
|
||||
return errors.MalformedXML.customizeDescription(desc);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* parseRestoreRequestXml - Parse and validate xml body, returning callback with
|
||||
* object restoreReqObj: { days: <value>, tier: <value> }
|
||||
* @param xml - xml body to parse and validate
|
||||
* @param log - Werelogs logger
|
||||
* @param cb - callback to server
|
||||
* @return - calls callback with object restore request or error
|
||||
*/
|
||||
export async function parseRestoreRequestXml(
|
||||
xml: string,
|
||||
log: werelogs.Logger,
|
||||
cb: (err: ArsenalError | null, data?: any) => void,
|
||||
) {
|
||||
let result;
|
||||
try {
|
||||
result = await parseStringPromise(xml);
|
||||
} catch (err) {
|
||||
log.debug('xml parsing failed', {
|
||||
error: err,
|
||||
method: 'parseRestoreXml',
|
||||
xml,
|
||||
});
|
||||
return cb(errors.MalformedXML);
|
||||
}
|
||||
if (!result) {
|
||||
const desc = 'request xml is undefined or empty';
|
||||
return cb(errors.MalformedXML.customizeDescription(desc));
|
||||
}
|
||||
const restoreRequest = result.RestoreRequest;
|
||||
const restoreReqError = validateRestoreRequest(restoreRequest);
|
||||
if (restoreReqError) {
|
||||
log.debug('restore request validation failed', {
|
||||
error: restoreReqError,
|
||||
method: 'validateRestoreRequest',
|
||||
xml,
|
||||
});
|
||||
return cb(restoreReqError);
|
||||
}
|
||||
// If do not specify Tier, set "Standard"
|
||||
return cb(null, {
|
||||
days: Number.parseInt(restoreRequest.Days, 10),
|
||||
tier: restoreRequest.Tier && restoreRequest.Tier[0] ? restoreRequest.Tier[0] : 'Standard',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* convertToXml - Convert restore request info object to xml
|
||||
* @param days - restore days
|
||||
* @param tier - restore tier
|
||||
* @return - returns restore request information xml string
|
||||
*/
|
||||
export function convertToXml(days: string, tier: string) {
|
||||
if (!(days && tier)) {
|
||||
return '';
|
||||
}
|
||||
return [
|
||||
'<RestoreRequest xmlns="http://s3.amazonaws.com/doc/2006-03-01/">',
|
||||
`<Days>${days}</Days>`,
|
||||
`<Tier>${tier}</Tier>`,
|
||||
'</RestoreRequest>',
|
||||
].join('');
|
||||
}
|
|
@ -1,5 +1,21 @@
|
|||
export const getHexMD5 = (base64MD5: WithImplicitCoercion<string>) =>
|
||||
Buffer.from(base64MD5, 'base64').toString('hex');
|
||||
const msInOneDay = 24 * 60 * 60 * 1000; // Milliseconds in a day.
|
||||
|
||||
export const getMD5Buffer = (base64MD5: WithImplicitCoercion<string> | Uint8Array) =>
|
||||
base64MD5 instanceof Uint8Array ? base64MD5 : Buffer.from(base64MD5, 'base64')
|
||||
|
||||
export const getHexMD5 = (base64MD5: WithImplicitCoercion<string> | Uint8Array) =>
|
||||
getMD5Buffer(base64MD5).toString('hex');
|
||||
|
||||
export const getBase64MD5 = (hexMD5: WithImplicitCoercion<string>) =>
|
||||
Buffer.from(hexMD5, 'hex').toString('base64');
|
||||
|
||||
|
||||
/**
|
||||
* Calculates the number of scaled milliseconds per day based on the given time progression factor.
|
||||
* This function is intended for testing and simulation purposes only.
|
||||
* @param {number} timeProgressionFactor - The desired time progression factor for scaling.
|
||||
* @returns {number} The number of scaled milliseconds per day.
|
||||
* If the result is 0, the minimum value of 1 millisecond is returned.
|
||||
*/
|
||||
export const scaleMsPerDay = (timeProgressionFactor: number): number =>
|
||||
Math.round(msInOneDay / (timeProgressionFactor || 1)) || 1;
|
||||
|
|
|
@ -3,6 +3,11 @@ import * as werelogs from 'werelogs';
|
|||
import errors, { ArsenalError } from '../errors';
|
||||
import escapeForXml from './escapeForXml';
|
||||
|
||||
export interface BucketTag {
|
||||
Key: string;
|
||||
Value: string;
|
||||
};
|
||||
|
||||
const errorInvalidArgument = () => errors.InvalidArgument
|
||||
.customizeDescription('The header \'x-amz-tagging\' shall be ' +
|
||||
'encoded as UTF-8 then URLEncoded URL query parameters without ' +
|
||||
|
@ -32,6 +37,15 @@ export const _validator = {
|
|||
&& tag.Key[0] !== undefined && tag.Value[0] !== undefined
|
||||
&& typeof tag.Key[0] === 'string' && typeof tag.Value[0] === 'string',
|
||||
|
||||
// Allowed characters are letters, whitespace, and numbers, plus
|
||||
// the following special characters: + - = . _ : /
|
||||
// Maximum key length: 128 Unicode characters
|
||||
// Maximum value length: 256 Unicode characters
|
||||
validateTagObjectStructure: (tag: BucketTag) => tag
|
||||
&& Object.keys(tag).length === 2
|
||||
&& typeof tag.Key === 'string' && typeof tag.Value === 'string'
|
||||
&& tag.Key.length >= 1 && tag.Value.length >= 1,
|
||||
|
||||
validateXMLStructure: (result: any) =>
|
||||
result && Object.keys(result).length === 1 &&
|
||||
result.Tagging &&
|
||||
|
@ -100,12 +114,47 @@ function _validateTags(tags: Array<{ Key: string[], Value: string[] }>) {
|
|||
}
|
||||
// not repeating keys
|
||||
if (tags.length > Object.keys(tagsResult).length) {
|
||||
return errors.InvalidTag.customizeDescription('Cannot provide ' +
|
||||
'multiple Tags with the same key');
|
||||
return errors.InvalidTag.customizeDescription(
|
||||
'Cannot provide multiple Tags with the same key'
|
||||
);
|
||||
}
|
||||
return tagsResult;
|
||||
}
|
||||
|
||||
/** areTagsValid - Validate bucket tags
|
||||
* @param tags - tags parsed from xml to be validated
|
||||
* @return result - true if the tags are valide, false otherwise
|
||||
*/
|
||||
export function areTagsValid(tags: Array<BucketTag>) {
|
||||
if (tags.length === 0) {
|
||||
return true;
|
||||
}
|
||||
// Maximum number of tags per resource: 50
|
||||
if (tags.length > 50) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const tagsResult = {};
|
||||
for (const tag of tags) {
|
||||
if (!_validator.validateTagObjectStructure(tag)) {
|
||||
return false;
|
||||
}
|
||||
const { Key: key, Value: value } = tag;
|
||||
|
||||
const result = _validator.validateKeyValue(key, value);
|
||||
if (result instanceof Error) {
|
||||
return false;
|
||||
}
|
||||
|
||||
tagsResult[key] = value;
|
||||
}
|
||||
// not repeating keys
|
||||
if (tags.length > Object.keys(tagsResult).length) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** parseTagXml - Parse and validate xml body, returning callback with object
|
||||
* tags : { key: value}
|
||||
* @param xml - xml body to parse and validate
|
||||
|
|
|
@ -77,6 +77,34 @@ export function _checkUnmodifiedSince(
|
|||
return { present: false, error: null };
|
||||
}
|
||||
|
||||
/**
|
||||
* checks 'if-modified-since' and 'if-unmodified-since' headers if included in
|
||||
* request against last-modified date of object
|
||||
* @param headers - headers from request object
|
||||
* @param lastModified - last modified date of object
|
||||
* @return contains modifiedSince and unmodifiedSince res objects
|
||||
*/
|
||||
export function checkDateModifiedHeaders(
|
||||
headers: http.IncomingHttpHeaders,
|
||||
lastModified: string,
|
||||
) {
|
||||
const lastModifiedDate = new Date(lastModified);
|
||||
lastModifiedDate.setMilliseconds(0);
|
||||
const millis = lastModifiedDate.getTime();
|
||||
|
||||
const ifModifiedSinceHeader = headers['if-modified-since'] ||
|
||||
headers['x-amz-copy-source-if-modified-since'];
|
||||
const ifUnmodifiedSinceHeader = headers['if-unmodified-since'] ||
|
||||
headers['x-amz-copy-source-if-unmodified-since'];
|
||||
|
||||
const modifiedSinceRes = _checkModifiedSince(ifModifiedSinceHeader?.toString(),
|
||||
millis);
|
||||
const unmodifiedSinceRes = _checkUnmodifiedSince(ifUnmodifiedSinceHeader?.toString(),
|
||||
millis);
|
||||
|
||||
return { modifiedSinceRes, unmodifiedSinceRes };
|
||||
}
|
||||
|
||||
/**
|
||||
* validateConditionalHeaders - validates 'if-modified-since',
|
||||
* 'if-unmodified-since', 'if-match' or 'if-none-match' headers if included in
|
||||
|
@ -92,21 +120,14 @@ export function validateConditionalHeaders(
|
|||
lastModified: string,
|
||||
contentMD5: string,
|
||||
): {} | { present: boolean; error: ArsenalError } {
|
||||
const lastModifiedDate = new Date(lastModified);
|
||||
lastModifiedDate.setMilliseconds(0);
|
||||
const millis = lastModifiedDate.getTime();
|
||||
const ifMatchHeader = headers['if-match'] ||
|
||||
headers['x-amz-copy-source-if-match'];
|
||||
const ifNoneMatchHeader = headers['if-none-match'] ||
|
||||
headers['x-amz-copy-source-if-none-match'];
|
||||
const ifModifiedSinceHeader = headers['if-modified-since'] ||
|
||||
headers['x-amz-copy-source-if-modified-since'];
|
||||
const ifUnmodifiedSinceHeader = headers['if-unmodified-since'] ||
|
||||
headers['x-amz-copy-source-if-unmodified-since'];
|
||||
const etagMatchRes = _checkEtagMatch(ifMatchHeader?.toString(), contentMD5);
|
||||
const etagNoneMatchRes = _checkEtagNoneMatch(ifNoneMatchHeader?.toString(), contentMD5);
|
||||
const modifiedSinceRes = _checkModifiedSince(ifModifiedSinceHeader?.toString(), millis);
|
||||
const unmodifiedSinceRes = _checkUnmodifiedSince(ifUnmodifiedSinceHeader?.toString(), millis);
|
||||
const { modifiedSinceRes, unmodifiedSinceRes } =
|
||||
checkDateModifiedHeaders(headers, lastModified);
|
||||
// If-Unmodified-Since condition evaluates to false and If-Match
|
||||
// is not present, then return the error. Otherwise, If-Unmodified-Since is
|
||||
// silent when If-Match match, and when If-Match does not match, it's the
|
||||
|
|
|
@ -10,7 +10,7 @@ import * as routesUtils from './routesUtils';
|
|||
import routeWebsite from './routes/routeWebsite';
|
||||
import * as http from 'http';
|
||||
import StatsClient from '../metrics/StatsClient';
|
||||
|
||||
import { objectKeyByteLimit } from '../constants';
|
||||
import * as requestUtils from '../../lib/policyEvaluator/requestUtils';
|
||||
|
||||
const routeMap = {
|
||||
|
@ -64,8 +64,14 @@ function checkBucketAndKey(
|
|||
blacklistedPrefixes.object);
|
||||
if (!result.isValid) {
|
||||
log.debug('invalid object key', { objectKey });
|
||||
return errors.InvalidArgument.customizeDescription('Object key ' +
|
||||
`must not start with "${result.invalidPrefix}".`);
|
||||
if (result.invalidPrefix) {
|
||||
return errors.InvalidArgument.customizeDescription('Invalid ' +
|
||||
'prefix - object key cannot start with ' +
|
||||
`"${result.invalidPrefix}".`);
|
||||
}
|
||||
return errors.KeyTooLong.customizeDescription('Object key is too ' +
|
||||
'long. Maximum number of bytes allowed in keys is ' +
|
||||
`${objectKeyByteLimit}.`);
|
||||
}
|
||||
}
|
||||
if ((reqQuery.partNumber || reqQuery.uploadId)
|
||||
|
@ -210,7 +216,8 @@ export default function routes(
|
|||
// @ts-ignore
|
||||
logger.newRequestLogger());
|
||||
|
||||
if (!req.url!.startsWith('/_/healthcheck')) {
|
||||
if (!req.url!.startsWith('/_/healthcheck') &&
|
||||
!req.url!.startsWith('/_/report')) {
|
||||
log.info('received request', clientInfo);
|
||||
}
|
||||
|
||||
|
|
|
@ -56,6 +56,8 @@ export default function routerGET(
|
|||
call('bucketGetNotification');
|
||||
} else if (query.encryption !== undefined) {
|
||||
call('bucketGetEncryption');
|
||||
} else if (query.search !== undefined) {
|
||||
call('metadataSearch')
|
||||
} else {
|
||||
// GET bucket
|
||||
call('bucketGet');
|
||||
|
|
|
@ -56,6 +56,14 @@ export default function routePOST(
|
|||
corsHeaders));
|
||||
}
|
||||
|
||||
// POST Object restore
|
||||
if (query.restore !== undefined) {
|
||||
return api.callApiMethod('objectRestore', request, response,
|
||||
log, (err, statusCode, resHeaders) =>
|
||||
routesUtils.responseNoBody(err, resHeaders, response,
|
||||
statusCode, log));
|
||||
}
|
||||
|
||||
return routesUtils.responseNoBody(errors.NotImplemented, null, response,
|
||||
200, log);
|
||||
}
|
||||
|
|
|
@ -9,7 +9,7 @@ export default function routerWebsite(
|
|||
api: { callApiMethod: routesUtils.CallApiMethod },
|
||||
log: RequestLogger,
|
||||
statsClient?: StatsClient,
|
||||
dataRetrievalFn?: any,
|
||||
dataRetrievalParams?: any,
|
||||
) {
|
||||
const { bucketName, query } = request as any
|
||||
log.debug('routing request', { method: 'routerWebsite' });
|
||||
|
@ -29,7 +29,7 @@ export default function routerWebsite(
|
|||
if (redirectInfo) {
|
||||
if (err && redirectInfo.withError) {
|
||||
return routesUtils.redirectRequestOnError(err,
|
||||
'GET', redirectInfo, dataGetInfo, dataRetrievalFn,
|
||||
'GET', redirectInfo, dataGetInfo, dataRetrievalParams,
|
||||
response, resMetaHeaders, log)
|
||||
}
|
||||
// note that key might have been modified in websiteGet
|
||||
|
@ -43,7 +43,7 @@ export default function routerWebsite(
|
|||
// user has their own error page
|
||||
if (err && dataGetInfo) {
|
||||
return routesUtils.streamUserErrorPage(err, dataGetInfo,
|
||||
dataRetrievalFn, response, resMetaHeaders, log);
|
||||
dataRetrievalParams, response, resMetaHeaders, log);
|
||||
}
|
||||
// send default error html response
|
||||
if (err) {
|
||||
|
@ -53,7 +53,7 @@ export default function routerWebsite(
|
|||
}
|
||||
// no error, stream data
|
||||
return routesUtils.responseStreamData(null, query,
|
||||
resMetaHeaders, dataGetInfo, dataRetrievalFn, response,
|
||||
resMetaHeaders, dataGetInfo, dataRetrievalParams, response,
|
||||
undefined, log);
|
||||
});
|
||||
}
|
||||
|
@ -64,7 +64,7 @@ export default function routerWebsite(
|
|||
if (redirectInfo) {
|
||||
if (err && redirectInfo.withError) {
|
||||
return routesUtils.redirectRequestOnError(err,
|
||||
'HEAD', redirectInfo, null, dataRetrievalFn,
|
||||
'HEAD', redirectInfo, null, dataRetrievalParams,
|
||||
response, resMetaHeaders, log)
|
||||
}
|
||||
return routesUtils.redirectRequest(redirectInfo,
|
||||
|
|
|
@ -6,6 +6,8 @@ import { eachSeries } from 'async';
|
|||
import DataWrapper from '../storage/data/DataWrapper';
|
||||
import * as http from 'http';
|
||||
import StatsClient from '../metrics/StatsClient';
|
||||
import { objectKeyByteLimit } from '../constants';
|
||||
const jsutil = require('../jsutil');
|
||||
|
||||
export type CallApiMethod = (
|
||||
methodName: string,
|
||||
|
@ -144,6 +146,15 @@ const XMLResponseBackend = {
|
|||
'<Error>',
|
||||
`<Code>${errCode.message}</Code>`,
|
||||
`<Message>${errCode.description}</Message>`,
|
||||
);
|
||||
const invalidArguments = errCode.metadata.get('invalidArguments') || [];
|
||||
invalidArguments.forEach((invalidArgument, index) => {
|
||||
const counter = index + 1;
|
||||
const { ArgumentName, ArgumentValue } = invalidArgument as any;
|
||||
xml.push(`<ArgumentName${counter}>${ArgumentName}</ArgumentName${counter}>`);
|
||||
xml.push(`<ArgumentValue${counter}>${ArgumentValue}</ArgumentValue${counter}>`);
|
||||
});
|
||||
xml.push(
|
||||
'<Resource></Resource>',
|
||||
`<RequestId>${log.getSerializedUids()}</RequestId>`,
|
||||
'</Error>',
|
||||
|
@ -213,9 +224,18 @@ const JSONResponseBackend = {
|
|||
"requestId": "4442587FB7D0A2F9"
|
||||
}
|
||||
*/
|
||||
const invalidArguments = errCode.metadata.get('invalidArguments') || [];
|
||||
const invalids = invalidArguments.reduce((acc, invalidArgument, index) => {
|
||||
const counter = index + 1;
|
||||
const { ArgumentName, ArgumentValue } = invalidArgument as any;
|
||||
const name = `ArgumentName${counter}`;
|
||||
const value = `ArgumentValue${counter}`;
|
||||
return { ...acc, [name]: ArgumentName, [value]: ArgumentValue };
|
||||
}, {});
|
||||
const data = JSON.stringify({
|
||||
code: errCode.message,
|
||||
message: errCode.description,
|
||||
...invalids,
|
||||
resource: null,
|
||||
requestId: log.getSerializedUids(),
|
||||
});
|
||||
|
@ -362,12 +382,18 @@ function retrieveData(
|
|||
response.destroy();
|
||||
responseDestroyed = true;
|
||||
};
|
||||
|
||||
const _destroyReadable = (readable: http.IncomingMessage | null) => {
|
||||
// s3-data sends Readable stream only which does not implement destroy
|
||||
if (readable && readable.destroy) {
|
||||
readable.destroy();
|
||||
}
|
||||
};
|
||||
|
||||
// the S3-client might close the connection while we are processing it
|
||||
response.once('close', () => {
|
||||
responseDestroyed = true;
|
||||
if (currentStream) {
|
||||
currentStream.destroy();
|
||||
}
|
||||
_destroyReadable(currentStream);
|
||||
});
|
||||
|
||||
const {
|
||||
|
@ -384,6 +410,7 @@ function retrieveData(
|
|||
return eachSeries(locations,
|
||||
(current, next) => data.get(current, response, log,
|
||||
(err: any, readable: http.IncomingMessage) => {
|
||||
const cbOnce = jsutil.once(next);
|
||||
// NB: readable is of IncomingMessage type
|
||||
if (err) {
|
||||
log.error('failed to get object', {
|
||||
|
@ -391,7 +418,7 @@ function retrieveData(
|
|||
method: 'retrieveData',
|
||||
});
|
||||
_destroyResponse();
|
||||
return next(err);
|
||||
return cbOnce(err);
|
||||
}
|
||||
// response.isclosed is set by the S3 server. Might happen if
|
||||
// the S3-client closes the connection before the first request
|
||||
|
@ -400,24 +427,24 @@ function retrieveData(
|
|||
if (responseDestroyed || response.isclosed) {
|
||||
log.debug(
|
||||
'response destroyed before readable could stream');
|
||||
readable.destroy();
|
||||
_destroyReadable(readable);
|
||||
const responseErr = new Error();
|
||||
// @ts-ignore
|
||||
responseErr.code = 'ResponseError';
|
||||
responseErr.message = 'response closed by client request before all data sent';
|
||||
return next(responseErr);
|
||||
return cbOnce(responseErr);
|
||||
}
|
||||
// readable stream successfully consumed
|
||||
readable.on('end', () => {
|
||||
currentStream = null;
|
||||
log.debug('readable stream end reached');
|
||||
return next();
|
||||
return cbOnce();
|
||||
});
|
||||
// errors on server side with readable stream
|
||||
readable.on('error', err => {
|
||||
log.error('error piping data from source');
|
||||
_destroyResponse();
|
||||
return next(err);
|
||||
return cbOnce(err);
|
||||
});
|
||||
currentStream = readable;
|
||||
return readable.pipe(response, { end: false });
|
||||
|
@ -1125,6 +1152,9 @@ export function isValidObjectKey(objectKey: string, prefixBlacklist: string[]) {
|
|||
if (invalidPrefix) {
|
||||
return { isValid: false, invalidPrefix };
|
||||
}
|
||||
if (Buffer.byteLength(objectKey, 'utf8') > objectKeyByteLimit) {
|
||||
return { isValid: false };
|
||||
}
|
||||
return { isValid: true };
|
||||
}
|
||||
|
||||
|
|
|
@ -984,13 +984,14 @@ class DataWrapper {
|
|||
return this.client.delete(objectGetInfo, log.getSerializedUids(),
|
||||
err => {
|
||||
if (err) {
|
||||
if (err.is.ObjNotFound) {
|
||||
// TODO: sproxydclient and hdclient does not return standard Arsenal error yet.
|
||||
if (err.code === 404) {
|
||||
log.info('no such key in datastore', {
|
||||
objectGetInfo,
|
||||
implName: this.implName,
|
||||
moreRetries: 'no',
|
||||
});
|
||||
return cb(err);
|
||||
return cb(errors.ObjNotFound);
|
||||
}
|
||||
log.error('delete error from datastore', {
|
||||
error: err,
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
const https = require('https');
|
||||
const http = require('http');
|
||||
const { http, https } = require('httpagent');
|
||||
const url = require('url');
|
||||
const AWS = require('aws-sdk');
|
||||
const Sproxy = require('sproxydclient');
|
||||
|
@ -77,8 +76,8 @@ function parseLC(config, vault) {
|
|||
connectionAgent = new HttpsProxyAgent(options);
|
||||
} else {
|
||||
connectionAgent = sslEnabled ?
|
||||
new https.Agent(httpAgentConfig) :
|
||||
new http.Agent(httpAgentConfig);
|
||||
new https.Agent(httpAgentConfig, { maxSockets: false }) :
|
||||
new http.Agent(httpAgentConfig, { maxSockets: false });
|
||||
}
|
||||
const httpOptions = { agent: connectionAgent, timeout: 0 };
|
||||
const s3Params = {
|
||||
|
|
|
@ -1,6 +1,5 @@
|
|||
const url = require('url');
|
||||
|
||||
const azure = require('azure-storage');
|
||||
const { BlobServiceClient, StorageSharedKeyCredential, AnonymousCredential } = require('@azure/storage-blob');
|
||||
const { ClientSecretCredential } = require('@azure/identity');
|
||||
const errors = require('../../../errors').default;
|
||||
const azureMpuUtils = require('../../../s3middleware/azureHelpers/mpuUtils');
|
||||
const { validateAndFilterMpuParts } =
|
||||
|
@ -8,55 +7,103 @@ const { validateAndFilterMpuParts } =
|
|||
|
||||
const { createLogger, logHelper, translateAzureMetaHeaders } =
|
||||
require('./utils');
|
||||
const objectUtils = require('../../../s3middleware/objectUtils');
|
||||
|
||||
const constants = require('../../../constants');
|
||||
const packageVersion = require('../../../../package.json').version;
|
||||
|
||||
azure.Constants.USER_AGENT_PRODUCT_NAME = constants.productName;
|
||||
azure.Constants.USER_AGENT_PRODUCT_VERSION = packageVersion;
|
||||
|
||||
class AzureClient {
|
||||
static addQueryParams(endpoint, token) {
|
||||
const url = new URL(endpoint);
|
||||
const query = token.startsWith('?') ? token.slice(1) : token;
|
||||
if (!url.search) {
|
||||
url.search = `?${query}`;
|
||||
} else if (url.search === '?') {
|
||||
url.search += query;
|
||||
} else {
|
||||
url.search += `&${query}`;
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
constructor(config) {
|
||||
this._azureStorageEndpoint = config.azureStorageEndpoint;
|
||||
this._azureStorageCredentials = config.azureStorageCredentials;
|
||||
this._azureContainerName = config.azureContainerName;
|
||||
this._client = azure.createBlobService(
|
||||
this._azureStorageCredentials.storageAccountName,
|
||||
this._azureStorageCredentials.storageAccessKey,
|
||||
this._azureStorageEndpoint);
|
||||
this._client.enableGlobalHttpAgent = true;
|
||||
const cred = (credentialsConfig => {
|
||||
switch (credentialsConfig.authMethod) {
|
||||
case 'client-secret':
|
||||
return new ClientSecretCredential(
|
||||
credentialsConfig.tenantId,
|
||||
credentialsConfig.clientId,
|
||||
credentialsConfig.clientKey,
|
||||
);
|
||||
|
||||
case 'shared-access-signature':
|
||||
this._azureStorageEndpoint = AzureClient.addQueryParams(
|
||||
this._azureStorageEndpoint, credentialsConfig.sasToken);
|
||||
return new AnonymousCredential();
|
||||
|
||||
case 'shared-key':
|
||||
default:
|
||||
return new StorageSharedKeyCredential(
|
||||
credentialsConfig.storageAccountName,
|
||||
credentialsConfig.storageAccessKey,
|
||||
);
|
||||
}
|
||||
})(this._azureStorageCredentials);
|
||||
const proxyOptions = (() => {
|
||||
if (!config.proxy || !config.proxy.url) {
|
||||
return undefined;
|
||||
}
|
||||
// NOTE: config.proxy.certs is not supported
|
||||
const parsedUrl = new URL(config.proxy.url);
|
||||
return {
|
||||
host: parsedUrl.host,
|
||||
port: parsedUrl.port || 80,
|
||||
username: parsedUrl.username || undefined,
|
||||
password: parsedUrl.password || undefined,
|
||||
};
|
||||
})();
|
||||
this._client = new BlobServiceClient(this._azureStorageEndpoint, cred, {
|
||||
keepAliveOptions: {
|
||||
enable: false, // Enable use of global HTTP agent
|
||||
},
|
||||
proxyOptions,
|
||||
userAgentOptions: {
|
||||
userAgentPrefix: `${constants.productName}/${packageVersion} `,
|
||||
},
|
||||
}).getContainerClient(this._azureContainerName);
|
||||
this._dataStoreName = config.dataStoreName;
|
||||
this._bucketMatch = config.bucketMatch;
|
||||
if (config.proxy && config.proxy.url) {
|
||||
const parsedUrl = url.parse(config.proxy.url);
|
||||
if (!parsedUrl.port) {
|
||||
parsedUrl.port = 80;
|
||||
}
|
||||
const proxyParams = parsedUrl;
|
||||
if (config.proxy.certs) {
|
||||
Object.assign(proxyParams, config.proxy.certs);
|
||||
}
|
||||
this._client.setProxy(proxyParams);
|
||||
}
|
||||
}
|
||||
|
||||
_errorWrapper(s3Method, azureMethod, args, log, cb) {
|
||||
/**
|
||||
* Run azure method call.
|
||||
* @param {string} [s3Method] S3 method name
|
||||
* @param {string} [azureMethod] Azure method name
|
||||
* @param {ErrorWrapper~Command} [command] Actual command to run
|
||||
* @param {RequestLogger} [log] Logger
|
||||
* @param {ErrorWrapper~Cb} [cb] The final callback
|
||||
* @returns {void}
|
||||
*
|
||||
* @callback ErrorWrapper~Command
|
||||
* @param {azure.ContainerClient} [client] Azure client to use
|
||||
* @returns {Promise<any>}
|
||||
*
|
||||
* @callback ErrorWrapper~Cb
|
||||
* @param {azure.ArsenalError} [arsenalErr] Error returned by the command
|
||||
* @param {any} [result] Result of Azure SDK command
|
||||
* @returns {void}
|
||||
*/
|
||||
_errorWrapper(s3Method, azureMethod, command, log, cb) {
|
||||
if (log) {
|
||||
log.info(`calling azure ${azureMethod}`);
|
||||
}
|
||||
try {
|
||||
this._client[azureMethod].apply(this._client, args);
|
||||
} catch (err) {
|
||||
const error = errors.ServiceUnavailable;
|
||||
if (log) {
|
||||
log.error('error thrown by Azure Storage Client Library',
|
||||
{ error: err.message, stack: err.stack, s3Method,
|
||||
azureMethod, dataStoreName: this._dataStoreName });
|
||||
}
|
||||
cb(error.customizeDescription('Error from Azure ' +
|
||||
`method: ${azureMethod} on ${s3Method} S3 call: ` +
|
||||
`${err.message}`));
|
||||
log.info(`calling azure ${azureMethod} in ${s3Method}`);
|
||||
}
|
||||
command(this._client).then(
|
||||
result => cb(null, result),
|
||||
cb,
|
||||
);
|
||||
}
|
||||
|
||||
_createAzureKey(requestBucketName, requestObjectKey,
|
||||
|
@ -119,6 +166,32 @@ class AzureClient {
|
|||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Azure HTTP headers for content settings
|
||||
* @param {object} [properties] The blob properties to set.
|
||||
* @param {string} [properties.contentType] The MIME content type of the blob.
|
||||
* The default type is application/octet-stream.
|
||||
* @param {string} [properties.contentEncoding] The content encodings that have been applied
|
||||
* to the blob.
|
||||
* @param {string} [properties.contentLanguage] The natural languages used by this resource.
|
||||
* @param {string} [properties.cacheControl] The blob's cache control.
|
||||
* @param {string} [properties.contentDisposition] The blob's content disposition.
|
||||
* @param {string} [properties.contentMD5] The blob's MD5 hash.
|
||||
* @returns {BlobHTTPHeaders} The headers
|
||||
*/
|
||||
_getAzureContentSettingsHeaders(properties) {
|
||||
return {
|
||||
blobContentMD5: properties.contentMD5
|
||||
? objectUtils.getMD5Buffer(properties.contentMD5)
|
||||
: undefined,
|
||||
blobContentType: properties.contentType || undefined,
|
||||
blobCacheControl: properties.cacheControl || undefined,
|
||||
blobContentDisposition: properties.contentDisposition || undefined,
|
||||
blobContentEncoding: properties.contentEncoding || undefined,
|
||||
blobContentLanguage: properties.blobContentLanguage || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
put(stream, size, keyContext, reqUids, callback, skey, metadata) {
|
||||
const log = createLogger(reqUids);
|
||||
// before blob is put, make sure there is no ongoing MPU with same key
|
||||
|
@ -134,93 +207,106 @@ class AzureClient {
|
|||
const options = {
|
||||
metadata: translateAzureMetaHeaders(keyContext.metaHeaders,
|
||||
keyContext.tagging),
|
||||
contentSettings: {
|
||||
contentType: keyContext.contentType || undefined,
|
||||
cacheControl: keyContext.cacheControl || undefined,
|
||||
contentDisposition: keyContext.contentDisposition ||
|
||||
undefined,
|
||||
contentEncoding: keyContext.contentEncoding || undefined,
|
||||
},
|
||||
blobHTTPHeaders: this._getAzureContentSettingsHeaders(
|
||||
keyContext || {}),
|
||||
};
|
||||
if (size === 0) {
|
||||
return this._errorWrapper('put', 'createBlockBlobFromText',
|
||||
[this._azureContainerName, azureKey, '', options,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from Azure PUT data ' +
|
||||
'backend', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`Azure: ${err.message}`));
|
||||
}
|
||||
return callback(null, azureKey);
|
||||
}], log, callback);
|
||||
return this._errorWrapper('put', 'uploadData', async client => {
|
||||
try {
|
||||
await client.getBlockBlobClient(azureKey).upload('', 0, options);
|
||||
return azureKey;
|
||||
} catch (err) {
|
||||
logHelper(log, 'error', 'err from Azure PUT data backend',
|
||||
err, this._dataStoreName);
|
||||
throw errors.ServiceUnavailable.customizeDescription(
|
||||
`Error returned from Azure: ${err.message}`);
|
||||
}
|
||||
}, log, callback);
|
||||
}
|
||||
return this._errorWrapper('put', 'createBlockBlobFromStream',
|
||||
[this._azureContainerName, azureKey, stream, size, options,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from Azure PUT data ' +
|
||||
'backend', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`Azure: ${err.message}`));
|
||||
}
|
||||
return callback(null, azureKey);
|
||||
}], log, callback);
|
||||
return this._errorWrapper('put', 'createBlockBlobFromStream', async client => {
|
||||
try {
|
||||
await client.getBlockBlobClient(azureKey).upload(() => stream, size, options);
|
||||
return azureKey;
|
||||
} catch (err) {
|
||||
logHelper(log, 'error', 'err from Azure PUT data backend',
|
||||
err, this._dataStoreName);
|
||||
throw errors.ServiceUnavailable.customizeDescription(
|
||||
`Error returned from Azure: ${err.message}`);
|
||||
}
|
||||
}, log, callback);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Build BlobRequestConditions from azureStreamingOptions
|
||||
* @param {object} [objectGetInfoOptions] Azure streaming options
|
||||
* @param {object} [objectGetInfoOptions.accessConditions] Access conditions
|
||||
* @param {Date} [objectGetInfoOptions.accessConditions.DateUnModifiedSince] Filter objects not
|
||||
* modified since that date.
|
||||
* @returns {BlobRequestConditions} Request conditions
|
||||
*/
|
||||
_getAzureConditions(objectGetInfoOptions) {
|
||||
const accessConditions = objectGetInfoOptions.accessConditions || {};
|
||||
return {
|
||||
ifUnmodifiedSince: accessConditions.DateUnModifiedSince || undefined,
|
||||
};
|
||||
}
|
||||
|
||||
head(objectGetInfo, reqUids, callback) {
|
||||
const log = createLogger(reqUids);
|
||||
const { key, azureStreamingOptions } = objectGetInfo;
|
||||
return this._errorWrapper('head', 'getBlobProperties',
|
||||
[this._azureContainerName, key, azureStreamingOptions,
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
let logLevel;
|
||||
let retError;
|
||||
if (err.code === 'NotFound') {
|
||||
logLevel = 'info';
|
||||
retError = errors.LocationNotFound;
|
||||
} else {
|
||||
logLevel = 'error';
|
||||
retError = errors.ServiceUnavailable
|
||||
.customizeDescription(
|
||||
`Error returned from Azure: ${err.message}`);
|
||||
}
|
||||
logHelper(log, logLevel, 'err from Azure HEAD data backend',
|
||||
err, this._dataStoreName);
|
||||
return callback(retError);
|
||||
}
|
||||
return callback(null, data);
|
||||
}], log, callback);
|
||||
const { key } = objectGetInfo;
|
||||
return this._errorWrapper('head', 'getBlobProperties', async client => {
|
||||
try {
|
||||
const data = await client.getBlockBlobClient(key).getProperties();
|
||||
return data;
|
||||
} catch (err) {
|
||||
let logLevel;
|
||||
let retError;
|
||||
if (err.code === 'NotFound') {
|
||||
logLevel = 'info';
|
||||
retError = errors.LocationNotFound;
|
||||
} else {
|
||||
logLevel = 'error';
|
||||
retError = errors.ServiceUnavailable.customizeDescription(
|
||||
`Error returned from Azure: ${err.message}`);
|
||||
}
|
||||
logHelper(log, logLevel, 'err from Azure HEAD data backend',
|
||||
err, this._dataStoreName);
|
||||
throw retError;
|
||||
}
|
||||
}, log, callback);
|
||||
}
|
||||
|
||||
get(objectGetInfo, range, reqUids, callback) {
|
||||
const log = createLogger(reqUids);
|
||||
// for backwards compatibility
|
||||
const { key, response, azureStreamingOptions } = objectGetInfo;
|
||||
let streamingOptions;
|
||||
let rangeStart = 0;
|
||||
let rangeEnd = undefined;
|
||||
if (azureStreamingOptions) {
|
||||
// option coming from api.get()
|
||||
streamingOptions = azureStreamingOptions;
|
||||
rangeStart = (typeof azureStreamingOptions.rangeStart === 'string')
|
||||
? parseInt(azureStreamingOptions.rangeStart, 10)
|
||||
: azureStreamingOptions.rangeStart;
|
||||
rangeEnd = (typeof azureStreamingOptions.rangeEnd === 'string')
|
||||
? parseInt(azureStreamingOptions.rangeEnd, 10)
|
||||
: azureStreamingOptions.rangeEnd;
|
||||
} else if (range) {
|
||||
// option coming from multipleBackend.upload()
|
||||
const rangeStart = (typeof range[0] === 'number') ? range[0].toString() : undefined;
|
||||
const rangeEnd = range[1] ? range[1].toString() : undefined;
|
||||
streamingOptions = { rangeStart, rangeEnd };
|
||||
rangeStart = (typeof range[0] === 'number') ? range[0] : 0;
|
||||
rangeEnd = range[1] || undefined;
|
||||
}
|
||||
this._errorWrapper('get', 'getBlobToStream',
|
||||
[this._azureContainerName, key, response, streamingOptions,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err from Azure GET data backend',
|
||||
err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable);
|
||||
}
|
||||
return callback(null, response);
|
||||
}], log, callback);
|
||||
this._errorWrapper('get', 'getBlobToStream', async client => {
|
||||
try {
|
||||
const rsp = await client.getBlockBlobClient(key)
|
||||
.download(rangeStart, rangeEnd - rangeStart + 1 || undefined);
|
||||
rsp.readableStreamBody.pipe(response);
|
||||
return response;
|
||||
} catch (err) {
|
||||
logHelper(log, 'error', 'err from Azure GET data backend',
|
||||
err, this._dataStoreName);
|
||||
throw errors.ServiceUnavailable;
|
||||
}
|
||||
}, log, callback);
|
||||
}
|
||||
|
||||
delete(objectGetInfo, reqUids, callback) {
|
||||
|
@ -230,44 +316,46 @@ class AzureClient {
|
|||
objectGetInfo.key;
|
||||
let options;
|
||||
if (typeof objectGetInfo === 'object') {
|
||||
options = objectGetInfo.options;
|
||||
options = {
|
||||
conditions: this._getAzureConditions(objectGetInfo.options || {}),
|
||||
};
|
||||
}
|
||||
return this._errorWrapper('delete', 'deleteBlobIfExists',
|
||||
[this._azureContainerName, key, options,
|
||||
err => {
|
||||
if (err && err.statusCode === 412) {
|
||||
return callback(errors.PreconditionFailed);
|
||||
}
|
||||
if (err) {
|
||||
const log = createLogger(reqUids);
|
||||
logHelper(log, 'error', 'error deleting object from ' +
|
||||
'Azure datastore', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`Azure: ${err.message}`));
|
||||
}
|
||||
return callback();
|
||||
}], log, callback);
|
||||
return this._errorWrapper('delete', 'deleteBlobIfExists', async client => {
|
||||
try {
|
||||
await client.getBlockBlobClient(key).deleteIfExists(options);
|
||||
} catch (err) {
|
||||
if (err.statusCode === 412) {
|
||||
throw errors.PreconditionFailed;
|
||||
}
|
||||
const log = createLogger(reqUids);
|
||||
logHelper(log, 'error', 'error deleting object from Azure datastore',
|
||||
err, this._dataStoreName);
|
||||
throw errors.ServiceUnavailable.customizeDescription(
|
||||
`Error returned from Azure: ${err.message}`);
|
||||
}
|
||||
}, log, callback);
|
||||
}
|
||||
|
||||
healthcheck(location, callback, flightCheckOnStartUp) {
|
||||
const azureResp = {};
|
||||
const healthCheckAction = flightCheckOnStartUp ?
|
||||
'createContainerIfNotExists' : 'doesContainerExist';
|
||||
this._errorWrapper('checkAzureHealth', healthCheckAction,
|
||||
[this._azureContainerName, err => {
|
||||
/* eslint-disable no-param-reassign */
|
||||
if (err) {
|
||||
azureResp[location] = { error: err.message,
|
||||
external: true };
|
||||
return callback(null, azureResp);
|
||||
this._errorWrapper('healthcheck', 'checkAzureHealth', async client => {
|
||||
try {
|
||||
if (flightCheckOnStartUp) {
|
||||
await client.createIfNotExists();
|
||||
} else {
|
||||
await client.exists();
|
||||
}
|
||||
azureResp[location] = {
|
||||
message:
|
||||
'Congrats! You can access the Azure storage account',
|
||||
message: 'Congrats! You can access the Azure storage account',
|
||||
};
|
||||
return callback(null, azureResp);
|
||||
}], null, callback);
|
||||
} catch (err) {
|
||||
azureResp[location] = {
|
||||
error: err.message,
|
||||
external: true,
|
||||
};
|
||||
}
|
||||
return azureResp;
|
||||
}, null, callback);
|
||||
}
|
||||
|
||||
uploadPart(request, streamingV4Params, partStream, size, key, uploadId,
|
||||
|
@ -321,9 +409,7 @@ class AzureClient {
|
|||
completeMPU(jsonList, mdInfo, key, uploadId, bucket, metaHeaders,
|
||||
contentSettings, tagging, log, callback) {
|
||||
const azureKey = this._createAzureKey(bucket, key, this._bucketMatch);
|
||||
const commitList = {
|
||||
UncommittedBlocks: jsonList.uncommittedBlocks || [],
|
||||
};
|
||||
const commitList = jsonList.uncommittedBlocks || [];
|
||||
let filteredPartsObj;
|
||||
if (!jsonList.uncommittedBlocks) {
|
||||
const { storedParts, mpuOverviewKey, splitter } = mdInfo;
|
||||
|
@ -336,60 +422,56 @@ class AzureClient {
|
|||
// part.locations is always array of 1, which contains data info
|
||||
const subPartIds =
|
||||
azureMpuUtils.getSubPartIds(part.locations[0], uploadId);
|
||||
commitList.UncommittedBlocks.push(...subPartIds);
|
||||
commitList.push(...subPartIds);
|
||||
});
|
||||
}
|
||||
const options = {
|
||||
contentSettings,
|
||||
blobHTTPHeaders: this._getAzureContentSettingsHeaders(contentSettings || {}),
|
||||
metadata: translateAzureMetaHeaders(metaHeaders || {}, tagging),
|
||||
};
|
||||
return this._errorWrapper('completeMPU', 'commitBlocks',
|
||||
[this._azureContainerName, azureKey, commitList, options,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err completing MPU on Azure ' +
|
||||
'datastore', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`Azure: ${err.message}`));
|
||||
}
|
||||
const completeObjData = {
|
||||
key: azureKey,
|
||||
filteredPartsObj,
|
||||
};
|
||||
return callback(null, completeObjData);
|
||||
}], log, callback);
|
||||
return this._errorWrapper('completeMPU', 'commitBlocks', async client => {
|
||||
try {
|
||||
await client.getBlockBlobClient(azureKey).commitBlockList(commitList, options);
|
||||
return {
|
||||
key: azureKey,
|
||||
filteredPartsObj,
|
||||
};
|
||||
} catch (err) {
|
||||
logHelper(log, 'error', 'err completing MPU on Azure datastore',
|
||||
err, this._dataStoreName);
|
||||
throw errors.ServiceUnavailable.customizeDescription(
|
||||
`Error returned from Azure: ${err.message}`);
|
||||
}
|
||||
}, log, callback);
|
||||
}
|
||||
|
||||
objectPutTagging(key, bucket, objectMD, log, callback) {
|
||||
const azureKey = this._createAzureKey(bucket, key, this._bucketMatch);
|
||||
const azureMD = this._getMetaHeaders(objectMD);
|
||||
azureMD.tags = JSON.stringify(objectMD.tags);
|
||||
this._errorWrapper('objectPutTagging', 'setBlobMetadata',
|
||||
[this._azureContainerName, azureKey, azureMD,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err putting object tags to ' +
|
||||
'Azure backend', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable);
|
||||
}
|
||||
return callback();
|
||||
}], log, callback);
|
||||
this._errorWrapper('objectPutTagging', 'setBlobMetadata', async client => {
|
||||
try {
|
||||
await client.getBlockBlobClient(azureKey).setMetadata(azureMD);
|
||||
} catch (err) {
|
||||
logHelper(log, 'error', 'err putting object tags to Azure backend',
|
||||
err, this._dataStoreName);
|
||||
throw errors.ServiceUnavailable;
|
||||
}
|
||||
}, log, callback);
|
||||
}
|
||||
|
||||
objectDeleteTagging(key, bucketName, objectMD, log, callback) {
|
||||
const azureKey = this._createAzureKey(bucketName, key, this._bucketMatch);
|
||||
const azureMD = this._getMetaHeaders(objectMD);
|
||||
this._errorWrapper('objectDeleteTagging', 'setBlobMetadata',
|
||||
[this._azureContainerName, azureKey, azureMD,
|
||||
err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'err putting object tags to ' +
|
||||
'Azure backend', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable);
|
||||
}
|
||||
return callback();
|
||||
}], log, callback);
|
||||
this._errorWrapper('objectDeleteTagging', 'setBlobMetadata', async client => {
|
||||
try {
|
||||
await client.getBlockBlobClient(azureKey).setMetadata(azureMD);
|
||||
} catch (err) {
|
||||
logHelper(log, 'error', 'err putting object tags to Azure backend',
|
||||
err, this._dataStoreName);
|
||||
throw errors.ServiceUnavailable;
|
||||
}
|
||||
}, log, callback);
|
||||
}
|
||||
|
||||
copyObject(request, destLocationConstraintName, sourceKey,
|
||||
|
@ -406,54 +488,50 @@ class AzureClient {
|
|||
|
||||
let options;
|
||||
if (storeMetadataParams.metaHeaders) {
|
||||
options = { metadata:
|
||||
translateAzureMetaHeaders(storeMetadataParams.metaHeaders) };
|
||||
options = {
|
||||
metadata: translateAzureMetaHeaders(storeMetadataParams.metaHeaders),
|
||||
};
|
||||
}
|
||||
|
||||
this._errorWrapper('copyObject', 'startCopyBlob',
|
||||
[`${this._azureStorageEndpoint}` +
|
||||
`${sourceContainerName}/${sourceKey}`,
|
||||
this._azureContainerName, destAzureKey, options,
|
||||
(err, res) => {
|
||||
if (err) {
|
||||
if (err.code === 'CannotVerifyCopySource') {
|
||||
logHelper(log, 'error', 'Unable to access ' +
|
||||
`${sourceContainerName} Azure Container`, err,
|
||||
this._dataStoreName);
|
||||
return callback(errors.AccessDenied
|
||||
.customizeDescription('Error: Unable to access ' +
|
||||
`${sourceContainerName} Azure Container`),
|
||||
);
|
||||
}
|
||||
logHelper(log, 'error', 'error from data backend on ' +
|
||||
'copyObject', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`AWS: ${err.message}`),
|
||||
);
|
||||
// TODO: should we use syncCopyBlob() instead? or use poller.pollUntilDone() to wait until complete?
|
||||
this._errorWrapper('copyObject', 'startCopyBlob', async client => {
|
||||
let res;
|
||||
try {
|
||||
const poller = await client.getBlockBlobClient(destAzureKey).beginCopyFromURL(
|
||||
`${this._azureStorageEndpoint}${sourceContainerName}/${sourceKey}`,
|
||||
options,
|
||||
);
|
||||
|
||||
res = poller.getOperationState().result;
|
||||
if (res.copyProgress !== 'pending') {
|
||||
return destAzureKey;
|
||||
}
|
||||
if (res.copy.status === 'pending') {
|
||||
logHelper(log, 'error', 'Azure copy status is pending',
|
||||
} catch (err) {
|
||||
if (err.code === 'CannotVerifyCopySource') { // TOOD: may use a constant (or type) from SDK ??
|
||||
logHelper(log, 'error',
|
||||
`Unable to access ${sourceContainerName} Azure Container`,
|
||||
err, this._dataStoreName);
|
||||
const copyId = res.copy.id;
|
||||
this._client.abortCopyBlob(this._azureContainerName,
|
||||
destAzureKey, copyId, err => {
|
||||
if (err) {
|
||||
logHelper(log, 'error', 'error from data backend ' +
|
||||
'on abortCopyBlob', err, this._dataStoreName);
|
||||
return callback(errors.ServiceUnavailable
|
||||
.customizeDescription('Error returned from ' +
|
||||
`AWS on abortCopyBlob: ${err.message}`),
|
||||
);
|
||||
}
|
||||
return callback(errors.InvalidObjectState
|
||||
.customizeDescription('Error: Azure copy status was ' +
|
||||
'pending. It has been aborted successfully'),
|
||||
);
|
||||
});
|
||||
throw errors.AccessDenied.customizeDescription(
|
||||
`Error: Unable to access ${sourceContainerName} Azure Container`);
|
||||
}
|
||||
return callback(null, destAzureKey);
|
||||
}], log, callback);
|
||||
logHelper(log, 'error', 'error from data backend on copyObject',
|
||||
err, this._dataStoreName);
|
||||
throw errors.ServiceUnavailable.customizeDescription(
|
||||
`Error returned from AWS: ${err.message}`);
|
||||
}
|
||||
|
||||
logHelper(log, 'error', 'Azure copy status is pending', {}, this._dataStoreName);
|
||||
try {
|
||||
await client.getBlockBlobClient(destAzureKey).abortCopyFromURL(res.copyId);
|
||||
} catch (err) {
|
||||
logHelper(log, 'error', 'error from data backend on abortCopyBlob',
|
||||
err, this._dataStoreName);
|
||||
throw errors.ServiceUnavailable.customizeDescription(
|
||||
`Error returned from AWS on abortCopyBlob: ${err.message}`);
|
||||
}
|
||||
throw errors.InvalidObjectState.customizeDescription(
|
||||
'Error: Azure copy status was pending. It has been aborted successfully');
|
||||
}, log, callback);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -51,6 +51,36 @@ function _parseListEntries(entries) {
|
|||
});
|
||||
}
|
||||
|
||||
/** _parseLifecycleListEntries - parse the values returned in a lifeycle listing by metadata
|
||||
* @param {object[]} entries - Version or Content entries in a metadata listing
|
||||
* @param {string} entries[].key - metadata key
|
||||
* @param {string} entries[].value - stringified object metadata
|
||||
* @return {object} - mapped array with parsed value or JSON parsing err
|
||||
*/
|
||||
function _parseLifecycleListEntries(entries) {
|
||||
return entries.map(entry => {
|
||||
const tmp = JSON.parse(entry.value);
|
||||
return {
|
||||
key: entry.key,
|
||||
value: {
|
||||
Size: tmp['content-length'],
|
||||
ETag: tmp['content-md5'],
|
||||
VersionId: tmp.versionId,
|
||||
IsNull: tmp.isNull,
|
||||
LastModified: tmp['last-modified'],
|
||||
Owner: {
|
||||
DisplayName: tmp['owner-display-name'],
|
||||
ID: tmp['owner-id'],
|
||||
},
|
||||
StorageClass: tmp['x-amz-storage-class'],
|
||||
tags: tmp.tags,
|
||||
staleDate: tmp.staleDate,
|
||||
dataStoreName: tmp.dataStoreName,
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** parseListEntries - parse the values returned in a listing by metadata
|
||||
* @param {object[]} entries - Version or Content entries in a metadata listing
|
||||
* @param {string} entries[].key - metadata key
|
||||
|
@ -147,6 +177,42 @@ class MetadataWrapper {
|
|||
});
|
||||
}
|
||||
|
||||
updateBucketCapabilities(bucketName, bucketMD, capabilityName, capacityField, capability, log, cb) {
|
||||
log.debug('updating bucket capabilities in metadata');
|
||||
// When concurrency update is not supported, we update the whole bucket metadata
|
||||
if (!this.client.putBucketAttributesCapabilities) {
|
||||
return this.updateBucket(bucketName, bucketMD, log, cb);
|
||||
}
|
||||
return this.client.putBucketAttributesCapabilities(bucketName, capabilityName, capacityField, capability,
|
||||
log, err => {
|
||||
if (err) {
|
||||
log.debug('error from metadata', { implName: this.implName,
|
||||
error: err });
|
||||
return cb(err);
|
||||
}
|
||||
log.trace('bucket capabilities updated in metadata');
|
||||
return cb(err);
|
||||
});
|
||||
}
|
||||
|
||||
deleteBucketCapabilities(bucketName, bucketMD, capabilityName, capacityField, log, cb) {
|
||||
log.debug('deleting bucket capabilities in metadata');
|
||||
// When concurrency update is not supported, we update the whole bucket metadata
|
||||
if (!this.client.deleteBucketAttributesCapability) {
|
||||
return this.updateBucket(bucketName, bucketMD, log, cb);
|
||||
}
|
||||
return this.client.deleteBucketAttributesCapability(bucketName, capabilityName, capacityField,
|
||||
log, err => {
|
||||
if (err) {
|
||||
log.debug('error from metadata', { implName: this.implName,
|
||||
error: err });
|
||||
return cb(err);
|
||||
}
|
||||
log.trace('bucket capabilities deleted in metadata');
|
||||
return cb(err);
|
||||
});
|
||||
}
|
||||
|
||||
getBucket(bucketName, log, cb) {
|
||||
log.debug('getting bucket from metadata');
|
||||
this.client.getBucketAttributes(bucketName, log, (err, data) => {
|
||||
|
@ -213,6 +279,25 @@ class MetadataWrapper {
|
|||
});
|
||||
}
|
||||
|
||||
getObjectsMD(bucketName, objNamesWithParams, log, cb) {
|
||||
if (typeof this.client.getObjects !== 'function') {
|
||||
log.debug('backend does not support get object metadata with batching', {
|
||||
implName: this.implName,
|
||||
});
|
||||
return cb(errors.NotImplemented);
|
||||
}
|
||||
log.debug('getting objects from metadata', { objects: objNamesWithParams });
|
||||
return this.client.getObjects(bucketName, objNamesWithParams, log, (err, data) => {
|
||||
if (err) {
|
||||
log.debug('error getting objects from metadata', { implName: this.implName, objects: objNamesWithParams,
|
||||
err });
|
||||
return cb(err);
|
||||
}
|
||||
log.debug('objects retrieved from metadata', { objects: objNamesWithParams });
|
||||
return cb(err, data);
|
||||
});
|
||||
}
|
||||
|
||||
getObjectMD(bucketName, objName, params, log, cb) {
|
||||
log.debug('getting object from metadata');
|
||||
this.client.getObject(bucketName, objName, params, log, (err, data) => {
|
||||
|
@ -226,7 +311,7 @@ class MetadataWrapper {
|
|||
});
|
||||
}
|
||||
|
||||
deleteObjectMD(bucketName, objName, params, log, cb) {
|
||||
deleteObjectMD(bucketName, objName, params, log, cb, originOp = 's3:ObjectRemoved:Delete') {
|
||||
log.debug('deleting object from metadata');
|
||||
this.client.deleteObject(bucketName, objName, params, log, err => {
|
||||
if (err) {
|
||||
|
@ -236,7 +321,7 @@ class MetadataWrapper {
|
|||
}
|
||||
log.debug('object deleted from metadata');
|
||||
return cb(err);
|
||||
});
|
||||
}, originOp);
|
||||
}
|
||||
|
||||
listObject(bucketName, listingParams, log, cb) {
|
||||
|
@ -279,6 +364,29 @@ class MetadataWrapper {
|
|||
});
|
||||
}
|
||||
|
||||
listLifecycleObject(bucketName, listingParams, log, cb) {
|
||||
log.debug('getting object listing for lifecycle from metadata');
|
||||
this.client.listLifecycleObject(bucketName, listingParams, log, (err, data) => {
|
||||
if (err) {
|
||||
log.error('error from metadata', { implName: this.implName,
|
||||
err });
|
||||
return cb(err);
|
||||
}
|
||||
log.debug('object listing for lifecycle retrieved from metadata');
|
||||
// eslint-disable-next-line no-param-reassign
|
||||
data.Contents = parseListEntries(data.Contents, _parseLifecycleListEntries);
|
||||
if (data.Contents instanceof Error) {
|
||||
log.error('error parsing metadata listing for lifecycle', {
|
||||
error: data.Contents,
|
||||
listingType: listingParams.listingType,
|
||||
method: 'listLifecycleObject',
|
||||
});
|
||||
return cb(errors.InternalError);
|
||||
}
|
||||
return cb(null, data);
|
||||
});
|
||||
}
|
||||
|
||||
listMultipartUploads(bucketName, listingParams, log, cb) {
|
||||
this.client.listMultipartUploads(bucketName, listingParams, log,
|
||||
(err, data) => {
|
||||
|
@ -427,6 +535,139 @@ class MetadataWrapper {
|
|||
return cb();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Put bucket indexes
|
||||
*
|
||||
* indexSpec format:
|
||||
* [
|
||||
* { key:[ { key: "", order: 1 } ... ], name: <id 1>, ... , < backend options> },
|
||||
* ...
|
||||
* { key:[ { key: "", order: 1 } ... ], name: <id n>, ... },
|
||||
* ]
|
||||
*
|
||||
*
|
||||
* @param {String} bucketName bucket name
|
||||
* @param {Array<Object>} indexSpecs index specification
|
||||
* @param {Object} log logger
|
||||
* @param {Function} cb callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
putBucketIndexes(bucketName, indexSpecs, log, cb) {
|
||||
log.debug('put bucket indexes');
|
||||
|
||||
if (typeof this.client.putBucketIndexes !== 'function') {
|
||||
log.error('error from metadata', {
|
||||
method: 'putBucketIndexes',
|
||||
error: errors.NotImplemented,
|
||||
implName: this.implName,
|
||||
});
|
||||
return cb(errors.NotImplemented);
|
||||
}
|
||||
|
||||
return this.client.putBucketIndexes(bucketName, indexSpecs, log, err => {
|
||||
if (err) {
|
||||
log.debug('error from metadata', {
|
||||
method: 'putBucketIndexes',
|
||||
error: err,
|
||||
implName: this.implName,
|
||||
});
|
||||
return cb(err);
|
||||
}
|
||||
return cb(null);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Delete bucket indexes
|
||||
*
|
||||
* indexSpec format:
|
||||
* [
|
||||
* { key:[ { key: "", order: 1 } ... ], name: <id 1>, ... , < backend options> },
|
||||
* ...
|
||||
* { key:[ { key: "", order: 1 } ... ], name: <id n>, ... },
|
||||
* ]
|
||||
*
|
||||
*
|
||||
* @param {String} bucketName bucket name
|
||||
* @param {Array<Object>} indexSpecs index specification
|
||||
* @param {Object} log logger
|
||||
* @param {Function} cb callback
|
||||
* @return {undefined}
|
||||
*/
|
||||
deleteBucketIndexes(bucketName, indexSpecs, log, cb) {
|
||||
log.debug('delete bucket indexes');
|
||||
|
||||
if (typeof this.client.deleteBucketIndexes !== 'function') {
|
||||
log.error('error from metadata', {
|
||||
method: 'deleteBucketIndexes',
|
||||
error: errors.NotImplemented,
|
||||
implName: this.implName,
|
||||
});
|
||||
return cb(errors.NotImplemented);
|
||||
}
|
||||
|
||||
return this.client.deleteBucketIndexes(bucketName, indexSpecs, log, err => {
|
||||
if (err) {
|
||||
log.error('error from metadata', {
|
||||
method: 'deleteBucketIndexes',
|
||||
error: err,
|
||||
implName: this.implName,
|
||||
});
|
||||
return cb(err);
|
||||
}
|
||||
return cb(null);
|
||||
});
|
||||
}
|
||||
|
||||
getBucketIndexes(bucketName, log, cb) {
|
||||
log.debug('get bucket indexes');
|
||||
|
||||
if (typeof this.client.getBucketIndexes !== 'function') {
|
||||
log.debug('error from metadata', {
|
||||
method: 'getBucketIndexes',
|
||||
error: errors.NotImplemented,
|
||||
implName: this.implName,
|
||||
});
|
||||
return cb(errors.NotImplemented);
|
||||
}
|
||||
|
||||
return this.client.getBucketIndexes(bucketName, log, (err, res) => {
|
||||
if (err) {
|
||||
log.debug('error from metadata', {
|
||||
method: 'getBucketIndexes',
|
||||
error: err,
|
||||
implName: this.implName,
|
||||
});
|
||||
return cb(err);
|
||||
}
|
||||
return cb(null, res);
|
||||
});
|
||||
}
|
||||
|
||||
getIndexingJobs(log, cb) {
|
||||
if (typeof this.client.getIndexingJobs !== 'function') {
|
||||
log.debug('error from metadata', {
|
||||
method: 'getIndexingJobs',
|
||||
error: errors.NotImplemented,
|
||||
implName: this.implName,
|
||||
});
|
||||
return cb(errors.NotImplemented);
|
||||
}
|
||||
|
||||
return this.client.getIndexingJobs(log, (err, res) => {
|
||||
if (err) {
|
||||
log.debug('error from metadata', {
|
||||
method: 'getBucketIndexes',
|
||||
error: err,
|
||||
implName: this.implName,
|
||||
});
|
||||
return cb(err);
|
||||
}
|
||||
return cb(null, res);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = MetadataWrapper;
|
||||
|
|
|
@ -110,6 +110,17 @@ class BucketClientInterface {
|
|||
return null;
|
||||
}
|
||||
|
||||
listLifecycleObject(bucketName, params, log, cb) {
|
||||
this.client.listObject(bucketName, log.getSerializedUids(), params,
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
return cb(err);
|
||||
}
|
||||
return cb(null, JSON.parse(data));
|
||||
});
|
||||
return null;
|
||||
}
|
||||
|
||||
listMultipartUploads(bucketName, params, log, cb) {
|
||||
this.client.listObject(bucketName, log.getSerializedUids(), params,
|
||||
(err, data) => {
|
||||
|
|
|
@ -325,6 +325,10 @@ class BucketFileInterface {
|
|||
return this.internalListObject(bucketName, params, log, cb);
|
||||
}
|
||||
|
||||
listLifecycleObject(bucketName, params, log, cb) {
|
||||
return this.internalListObject(bucketName, params, log, cb);
|
||||
}
|
||||
|
||||
listMultipartUploads(bucketName, params, log, cb) {
|
||||
return this.internalListObject(bucketName, params, log, cb);
|
||||
}
|
||||
|
|
|
@ -318,6 +318,10 @@ const metastore = {
|
|||
});
|
||||
},
|
||||
|
||||
listLifecycleObject(bucketName, params, log, cb) {
|
||||
return process.nextTick(cb, errors.NotImplemented);
|
||||
},
|
||||
|
||||
listMultipartUploads(bucketName, listingParams, log, cb) {
|
||||
process.nextTick(() => {
|
||||
metastore.getBucketAttributes(bucketName, log, (err, bucket) => {
|
||||
|
|
|
@ -108,9 +108,26 @@ class ListRecordStream extends stream.Readable {
|
|||
if (value && value.tags) {
|
||||
value.tags = unescape(value.tags);
|
||||
}
|
||||
// updates overwrite the whole metadata,
|
||||
// so they are considered as puts
|
||||
let type = 'put';
|
||||
// When the object metadata contain the "deleted"
|
||||
// flag, it means that the operation is the update
|
||||
// we perform before the deletion of an object. We
|
||||
// perform the update to keep all the metadata in the
|
||||
// oplog. This update is what will be used by backbeat
|
||||
// as the delete operation so we put the type of operation
|
||||
// for this event to a delete.
|
||||
// Backbeat still receives the actual delete operations
|
||||
// but they are ignored as they don't contain any metadata.
|
||||
// The delete operations are kept in case we want to listen
|
||||
// to delete events comming from special collections other
|
||||
// than "bucket" collections.
|
||||
if (value && value.deleted) {
|
||||
type = 'delete';
|
||||
}
|
||||
entry = {
|
||||
type: 'put', // updates overwrite the whole metadata,
|
||||
// so they are considered as puts
|
||||
type,
|
||||
key: itemObj.o2._id,
|
||||
// updated value may be either stored directly in 'o'
|
||||
// attribute or in '$set' attribute (supposedly when
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -55,6 +55,22 @@ class MongoReadStream extends Readable {
|
|||
}
|
||||
}
|
||||
|
||||
if (options.lastModified) {
|
||||
query['value.last-modified'] = {};
|
||||
|
||||
if (options.lastModified.lt) {
|
||||
query['value.last-modified'].$lt = options.lastModified.lt;
|
||||
}
|
||||
}
|
||||
|
||||
if (options.dataStoreName) {
|
||||
query['value.dataStoreName'] = {};
|
||||
|
||||
if (options.dataStoreName.ne) {
|
||||
query['value.dataStoreName'].$ne = options.dataStoreName.ne;
|
||||
}
|
||||
}
|
||||
|
||||
if (!Object.keys(query._id).length) {
|
||||
delete query._id;
|
||||
}
|
||||
|
@ -69,7 +85,8 @@ class MongoReadStream extends Readable {
|
|||
Object.assign(query, searchOptions);
|
||||
}
|
||||
|
||||
this._cursor = c.find(query).sort({
|
||||
const projection = { 'value.location': 0 };
|
||||
this._cursor = c.find(query, { projection }).sort({
|
||||
_id: options.reverse ? -1 : 1,
|
||||
});
|
||||
if (options.limit && options.limit !== -1) {
|
||||
|
@ -85,15 +102,10 @@ class MongoReadStream extends Readable {
|
|||
return;
|
||||
}
|
||||
|
||||
this._cursor.next((err, doc) => {
|
||||
this._cursor.next().then(doc => {
|
||||
if (this._destroyed) {
|
||||
return;
|
||||
}
|
||||
if (err) {
|
||||
this.emit('error', err);
|
||||
return;
|
||||
}
|
||||
|
||||
let key = undefined;
|
||||
let value = undefined;
|
||||
|
||||
|
@ -117,6 +129,12 @@ class MongoReadStream extends Readable {
|
|||
value,
|
||||
});
|
||||
}
|
||||
}).catch(err => {
|
||||
if (this._destroyed) {
|
||||
return;
|
||||
}
|
||||
this.emit('error', err);
|
||||
return;
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -126,7 +144,7 @@ class MongoReadStream extends Readable {
|
|||
}
|
||||
this._destroyed = true;
|
||||
|
||||
this._cursor.close(err => {
|
||||
this._cursor.close().catch(err => {
|
||||
if (err) {
|
||||
this.emit('error', err);
|
||||
return;
|
||||
|
|
|
@ -185,6 +185,48 @@ function formatVersionKey(key, versionId, vFormat) {
|
|||
return formatVersionKeyV0(key, versionId);
|
||||
}
|
||||
|
||||
function indexFormatMongoArrayToObject(mongoIndexArray) {
|
||||
const indexObj = [];
|
||||
|
||||
for (const idx of mongoIndexArray) {
|
||||
const keys = [];
|
||||
let entries = [];
|
||||
|
||||
if (idx.key instanceof Map) {
|
||||
entries = idx.key.entries();
|
||||
} else {
|
||||
entries = Object.entries(idx.key);
|
||||
}
|
||||
|
||||
for (const k of entries) {
|
||||
keys.push({ key: k[0], order: k[1] });
|
||||
}
|
||||
|
||||
indexObj.push({ name: idx.name, keys });
|
||||
}
|
||||
|
||||
return indexObj;
|
||||
}
|
||||
|
||||
|
||||
function indexFormatObjectToMongoArray(indexObj) {
|
||||
const mongoIndexArray = [];
|
||||
|
||||
for (const idx of indexObj) {
|
||||
const key = new Map();
|
||||
|
||||
for (const k of idx.keys) {
|
||||
key.set(k.key, k.order);
|
||||
}
|
||||
|
||||
// copy all field except keys from idx
|
||||
// eslint-disable-next-line
|
||||
const { keys: _, ...toCopy } = idx;
|
||||
mongoIndexArray.push(Object.assign(toCopy, { name: idx.name, key }));
|
||||
}
|
||||
|
||||
return mongoIndexArray;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
credPrefix,
|
||||
|
@ -195,4 +237,6 @@ module.exports = {
|
|||
translateConditions,
|
||||
formatMasterKey,
|
||||
formatVersionKey,
|
||||
indexFormatMongoArrayToObject,
|
||||
indexFormatObjectToMongoArray,
|
||||
};
|
||||
|
|
|
@ -29,5 +29,4 @@ server.start(() => {
|
|||
logger.info('Metadata Proxy Server successfully started. ' +
|
||||
`Using the ${metadataWrapper.implName} backend`);
|
||||
});
|
||||
|
||||
```
|
||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue