diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index d77d0946..2347d6cc 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -532,6 +532,24 @@ jobs: echo "" done + test_root_node: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_root_node.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_switch_primary: runs-on: ubuntu-latest needs: build diff --git a/mon/lp-optimizer.js b/mon/lp-optimizer.js index e71a552c..63703532 100644 --- a/mon/lp-optimizer.js +++ b/mon/lp-optimizer.js @@ -77,7 +77,7 @@ async function optimize_initial({ osd_weights, combinator, pg_count, pg_size = 3 { if (osd !== NO_OSD) { - let osd_pg_count = osd_weights[osd]/total_weight*pg_effsize*pg_count; + let osd_pg_count = (osd_weights[osd]||0)/total_weight*pg_effsize*pg_count; lp += pg_per_osd[osd].join(' + ')+' <= '+osd_pg_count+';\n'; } } @@ -299,7 +299,7 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_weights, combinator )).join(' + '); const rm_osd_pg_count = (prev_pg_per_osd[osd]||[]) .reduce((a, [ old_pg_name, space ]) => (a + (all_pgs_hash[old_pg_name] ? space : 0)), 0); - const osd_pg_count = osd_weights[osd]*pg_effsize/total_weight*pg_count - rm_osd_pg_count; + const osd_pg_count = (osd_weights[osd]||0)*pg_effsize/total_weight*pg_count - rm_osd_pg_count; lp += osd_sum + ' <= ' + osd_pg_count + ';\n'; } } diff --git a/mon/mon.js b/mon/mon.js index 88488ed3..f28f1607 100644 --- a/mon/mon.js +++ b/mon/mon.js @@ -1197,16 +1197,37 @@ class Mon { return; } - pool_tree = pool_tree[root_node]; - const cur = [ ...(pool_tree||{}).children||[] ]; - for (let i = 0; i < cur.length; i++) + let included = [ ...(pool_tree[root_node] || {}).children||[] ]; + for (let i = 0; i < included.length; i++) { - if (cur.children) + if (included[i].children) { - cur.splice(i+1, 1, ...cur.children); + included.splice(i+1, 0, ...included[i].children); + } + } + let cur = pool_tree[root_node] || {}; + if (cur) + { + included.unshift(cur); + } + while (cur.id) + { + let parent = cur.parent||''; + if (pool_tree[parent]) + { + included.unshift(pool_tree[parent]); + pool_tree[parent] = { ...pool_tree[parent], children: [ cur ] }; + cur = pool_tree[parent]; + } + } + included = included.reduce((a, c) => { a[c.id||''] = true; return a; }, {}); + for (const item in pool_tree) + { + if (!included[item]) + { + delete pool_tree[item]; } } - return cur; } filter_osds_by_tags(orig_tree, tags) @@ -1336,7 +1357,7 @@ class Mon { return null; } - let pool_tree = osd_tree; + let pool_tree = { ...osd_tree }; this.filter_osds_by_root_node(pool_tree, pool_cfg.root_node); this.filter_osds_by_tags(pool_tree, pool_cfg.osd_tags); this.filter_osds_by_block_layout( @@ -1364,9 +1385,9 @@ class Mon osd_weights: Object.values(pool_tree).filter(item => item.level === 'osd').reduce((a, c) => { a[c.id] = c.size; return a; }, {}), combinator: !this.config.use_old_pg_combinator || pool_cfg.level_placement || pool_cfg.raw_placement // new algorithm: - ? new RuleCombinator(osd_tree, this.get_pg_rules(pool_id, pool_cfg), pool_cfg.max_osd_combinations) + ? new RuleCombinator(pool_tree, this.get_pg_rules(pool_id, pool_cfg), pool_cfg.max_osd_combinations) // old algorithm: - : new SimpleCombinator(flatten_tree(osd_tree[''].children, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations), + : new SimpleCombinator(flatten_tree(pool_tree[''].children, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations), pg_count: pool_cfg.pg_count, pg_size: pool_cfg.pg_size, pg_minsize: pool_cfg.pg_minsize, diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 36775989..f44fe279 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -45,6 +45,8 @@ IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh SCHEME=ec ./test_rebalance_verify.sh SCHEME=ec IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh +./test_root_node.sh + ./test_switch_primary.sh ./test_write.sh diff --git a/tests/test_root_node.sh b/tests/test_root_node.sh new file mode 100755 index 00000000..c6538276 --- /dev/null +++ b/tests/test_root_node.sh @@ -0,0 +1,31 @@ +#!/bin/bash -ex + +. `dirname $0`/common.sh + +TIME=$(date '+%s') +$ETCDCTL put /vitastor/config/global '{"placement_levels":{"rack":100,"host":101,"osd":102}}' +$ETCDCTL put /vitastor/config/node_placement '{"rack1":{"level":"rack"},"rack2":{"level":"rack"},"stor1":{"level":"host","parent":"rack1"},"stor2":{"level":"host","parent":"rack1"},"stor3":{"level":"host","parent":"rack2"},"stor4":{"level":"host","parent":"rack2"}}' +$ETCDCTL put /vitastor/osd/stats/1 '{"host":"stor1","size":1073741824,"time":"'$TIME'"}' +$ETCDCTL put /vitastor/osd/stats/2 '{"host":"stor1","size":1073741824,"time":"'$TIME'"}' +$ETCDCTL put /vitastor/osd/stats/3 '{"host":"stor2","size":1073741824,"time":"'$TIME'"}' +$ETCDCTL put /vitastor/osd/stats/4 '{"host":"stor2","size":1073741824,"time":"'$TIME'"}' +$ETCDCTL put /vitastor/osd/stats/5 '{"host":"stor3","size":1073741824,"time":"'$TIME'"}' +$ETCDCTL put /vitastor/osd/stats/6 '{"host":"stor3","size":1073741824,"time":"'$TIME'"}' +$ETCDCTL put /vitastor/osd/stats/7 '{"host":"stor4","size":1073741824,"time":"'$TIME'"}' +$ETCDCTL put /vitastor/osd/stats/8 '{"host":"stor4","size":1073741824,"time":"'$TIME'"}' +$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":2,"pg_count":16,"failure_domain":"host","root_node":"rack1"}}' + +node mon/mon-main.js --etcd_address $ETCD_URL --etcd_prefix "/vitastor" >>./testdata/mon.log 2>&1 & +MON_PID=$! + +sleep 2 + +etcdctl --endpoints=http://localhost:12379 get --prefix /vitastor/config/pgs --print-value-only + +if ! (etcdctl --endpoints=http://localhost:12379 get --prefix /vitastor/config/pgs --print-value-only | \ + jq -s -e '[ [ .[0].items["1"] | .[].osd_set | map(. | select(. != "" and (.|tonumber) < 5)) ][] | select((. | length) == 2) ] | length == 16'); then + format_error "Some PGs missing replicas" +fi + format_error "Some PGs missing replicas" + +format_green OK