Try new PG combinator in tests

mon/dsl_pgs.js

@@ -0,0 +1,438 @@
+const NO_OSD = 'Z';
+
+class RuleCombinator
+{
+    constructor(osd_tree, rules, max_combinations, ordered)
+    {
+        this.osd_tree = index_tree(Object.values(osd_tree).filter(o => o.id));
+        this.rules = rules;
+        this.max_combinations = max_combinations;
+        this.ordered = ordered;
+    }
+
+    random_combinations()
+    {
+        return random_custom_combinations(this.osd_tree, this.rules, this.max_combinations, this.ordered);
+    }
+
+    check_combinations(pgs)
+    {
+        return check_custom_combinations(this.osd_tree, this.rules, pgs);
+    }
+}
+
+// Convert alternative "level-index" format to rules
+// level_index = { [level: string]: string | string[] }
+// level_sequence = optional, levels from upper to lower, i.e. [ 'dc', 'host' ]
+// Example: level_index = { dc: "112233", host: "ABCDEF" }
+function parse_level_indexes(level_index, level_sequence)
+{
+    const rules = [];
+    const lvl_first = {};
+    for (const level in level_index)
+    {
+        const idx = level_index[level];
+        while (rules.length < idx.length)
+        {
+            rules.push([]);
+        }
+        const seen = {};
+        for (let i = 0; i < idx.length; i++)
+        {
+            if (!seen[idx[i]])
+            {
+                const other = Object.values(seen);
+                if (other.length)
+                {
+                    rules[i].push([ level, '!=', other ]);
+                }
+                seen[idx[i]] = i+1;
+            }
+            else
+            {
+                rules[i].push([ level, '=', seen[idx[i]] ]);
+            }
+        }
+        lvl_first[level] = seen;
+    }
+    if (level_sequence)
+    {
+        // Prune useless rules for the sake of prettiness
+        // For simplicity, call "upper" level DC and "lower" level host
+        const level_prio = Object.keys(level_sequence).reduce((a, c) => { a[level_sequence[c]] = c; return a; }, {});
+        for (let upper_i = 0; upper_i < level_sequence.length-1; upper_i++)
+        {
+            const upper_level = level_sequence[upper_i];
+            for (let i = 0; i < rules.length; i++)
+            {
+                const noteq = {};
+                for (let k = 0; k < level_index[upper_level].length; k++)
+                {
+                    // If upper_level[x] is different from upper_level[y]
+                    // then lower_level[x] is also different from lower_level[y]
+                    if (level_index[upper_level][k] != level_index[upper_level][i])
+                    {
+                        noteq[k+1] = true;
+                    }
+                }
+                for (let j = 0; j < rules[i].length; j++)
+                {
+                    if (level_prio[rules[i][j][0]] != null && level_prio[rules[i][j][0]] > upper_i && rules[i][j][1] == '!=')
+                    {
+                        rules[i][j][2] = rules[i][j][2].filter(other_host => !noteq[other_host]);
+                        if (!rules[i][j][2].length)
+                        {
+                            rules[i].splice(j--, 1);
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return rules;
+}
+
+// Parse rules in DSL format
+// dsl := item | item ("\n" | ",") items
+// item := "any" | rules
+// rules := rule | rule rules
+// rule := level operator arg
+// level := /\w+/
+// operator := "!=" | "=" | ">" | "?="
+// arg := value | "(" values ")"
+// values := value | value "," values
+// value := item_ref | constant_id
+// item_ref := /\d+/
+// constant_id := /"([^"]+)"/
+//
+// Output: [ level, operator, value ][][]
+function parse_pg_dsl(text)
+{
+    const tokens = [ ...text.matchAll(/\w+|!=|\?=|[>=\(\),\n]|"([^\"]+)"/g) ].map(t => [ t[0], t.index ]);
+    let positions = [ [] ];
+    let rules = positions[0];
+    for (let i = 0; i < tokens.length; )
+    {
+        if (tokens[i][0] === '\n' || tokens[i][0] === ',')
+        {
+            rules = [];
+            positions.push(rules);
+            i++;
+        }
+        else if (!rules.length && tokens[i][0] === 'any' && (i == tokens.length-1 || tokens[i+1][0] === ',' || tokens[i+1][0] === '\n'))
+        {
+            i++;
+        }
+        else
+        {
+            if (!/^\w/.exec(tokens[i][0]))
+            {
+                throw new Error('Unexpected '+tokens[i][0]+' at '+tokens[i][1]+' (level name expected)');
+            }
+            if (i > tokens.length-3)
+            {
+                throw new Error('Unexpected EOF (operator and value expected)');
+            }
+            if (/^\w/.exec(tokens[i+1][0]) || tokens[i+1][0] === ',' || tokens[i+1][0] === '\n')
+            {
+                throw new Error('Unexpected '+tokens[i+1][0]+' at '+tokens[i+1][1]+' (operator expected)');
+            }
+            if (!/^[\w"(]/.exec(tokens[i+2][0])) // "
+            {
+                throw new Error('Unexpected '+tokens[i+2][0]+' at '+tokens[i+2][1]+' (id, round brace, number or node ID expected)');
+            }
+            let rule = [ tokens[i][0], tokens[i+1][0], tokens[i+2][0] ];
+            i += 3;
+            if (rule[2][0] == '"')
+            {
+                rule[2] = { id: rule[2].substr(1, rule[2].length-2) };
+            }
+            else if (rule[2] === '(')
+            {
+                rule[2] = [];
+                while (true)
+                {
+                    if (i > tokens.length-1)
+                    {
+                        throw new Error('Unexpected EOF (expected list and a closing round brace)');
+                    }
+                    if (tokens[i][0] === ',')
+                    {
+                        i++;
+                    }
+                    else if (tokens[i][0] === ')')
+                    {
+                        i++;
+                        break;
+                    }
+                    else if (tokens[i][0][0] === '"')
+                    {
+                        rule[2].push({ id: tokens[i][0].substr(1, tokens[i][0].length-2) });
+                        i++;
+                    }
+                    else if (/^\d+$/.exec(tokens[i][0]))
+                    {
+                        const n = 0|tokens[i][0];
+                        if (!n)
+                        {
+                            throw new Error('Level reference cannot be 0 (refs count from 1) at '+tokens[i][1]);
+                        }
+                        else if (n > positions.length)
+                        {
+                            throw new Error('Forward references are forbidden at '+tokens[i][1]);
+                        }
+                        rule[2].push(n);
+                        i++;
+                    }
+                    else if (!/^\w/.exec(tokens[i][0]))
+                    {
+                        throw new Error('Unexpected '+tokens[i][0]+' at '+tokens[i][1]+' (number or node ID expected)');
+                    }
+                    else
+                    {
+                        rule[2].push({ id: tokens[i][0] });
+                        i++;
+                    }
+                }
+            }
+            else if (!/^\d+$/.exec(rule[2]))
+            {
+                rule[2] = { id: rule[2] };
+            }
+            else
+            {
+                rule[2] = 0|rule[2];
+                if (!rule[2])
+                {
+                    throw new Error('Level reference cannot be 0 (refs count from 1) at '+tokens[i-1][1]);
+                }
+                else if (rule[2] > positions.length)
+                {
+                    throw new Error('Forward references are forbidden at '+tokens[i-1][1]);
+                }
+            }
+            rules.push(rule);
+        }
+    }
+    return positions;
+}
+
+// osd_tree = index_tree() output
+// levels = { string: number }
+// rules = [ level, operator, value ][][]
+//   level = string
+//   operator = '=' | '!=' | '>' | '?='
+//   value = number|number[] | { id: string|string[] }
+// examples:
+// 1) simple 3 replicas with failure_domain=host:
+//    [ [], [ [ 'host', '!=', 1 ] ], [ [ 'host', '!=', [ 1, 2 ] ] ] ]
+//    in DSL form: any, host!=1, host!=(1,2)
+// 2) EC 4+2 in 3 DC:
+//    [ [], [ [ 'dc', '=', 1 ], [ 'host', '!=', 1 ] ],
+//      [ 'dc', '!=', 1 ], [ [ 'dc', '=', 3 ], [ 'host', '!=', 3 ] ],
+//      [ 'dc', '!=', [ 1, 3 ] ], [ [ 'dc', '=', 5 ], [ 'host', '!=', 5 ] ] ]
+//    in DSL form: any, dc=1 host!=1, dc!=1, dc=3 host!=3, dc!=(1,3), dc=5 host!=5
+// 3) 1 replica in fixed DC + 2 in random DCs:
+//    [ [ [ 'dc', '=', { id: 'meow' } ] ], [ [ 'dc', '!=', 1 ] ], [ [ 'dc', '!=', [ 1, 2 ] ] ] ]
+//    in DSL form: dc=meow, dc!=1, dc!=(1,2)
+// 4) 2 replicas in each DC (almost the same as (2)):
+//    DSL: any, dc=1 host!=1, dc!=1, dc=3 host!=3
+// Alternative simpler way to specify rules would be: [ DC: 112233 HOST: 123456 ]
+function random_custom_combinations(osd_tree, rules, count, ordered)
+{
+    const r = {};
+    const first = filter_tree_by_rules(osd_tree, rules[0], []);
+    let max_size = 0;
+    // All combinations for the first item (usually "any") to try to include each OSD at least once
+    for (const f of first)
+    {
+        const selected = [ f ];
+        for (let i = 1; i < rules.length; i++)
+        {
+            const filtered = filter_tree_by_rules(osd_tree, rules[i], selected);
+            selected.push(select_murmur3(filtered, 'p:'+f.id));
+        }
+        const size = selected.filter(s => s.id !== null).length;
+        max_size = max_size < size ? size : max_size;
+        const pg = selected.map(s => s.id === null ? NO_OSD : s.id);
+        if (!ordered)
+            pg.sort();
+        r['pg_'+pg.join('_')] = pg;
+    }
+    // Pseudo-random selection
+    for (let n = 0; n < count; n++)
+    {
+        const selected = [];
+        for (const item_rules of rules)
+        {
+            const filtered = selected.length ? filter_tree_by_rules(osd_tree, item_rules, selected) : first;
+            selected.push(select_murmur3(filtered, n));
+        }
+        const size = selected.filter(s => s.id !== null).length;
+        max_size = max_size < size ? size : max_size;
+        const pg = selected.map(s => s.id === null ? NO_OSD : s.id);
+        if (!ordered)
+            pg.sort();
+        r['pg_'+pg.join('_')] = pg;
+    }
+    // Exclude PGs with less successful selections than maximum
+    for (const k in r)
+    {
+        if (r[k].filter(s => s !== NO_OSD).length < max_size)
+        {
+            delete r[k];
+        }
+    }
+    return r;
+}
+
+function select_murmur3(filtered, prefix)
+{
+    if (!filtered.length)
+    {
+        return { levels: {}, id: null };
+    }
+    else
+    {
+        let i = 0, maxh = -1;
+        for (let j = 0; j < filtered.length; j++)
+        {
+            const h = murmur3(prefix+':'+filtered[j].id);
+            if (h > maxh)
+            {
+                i = j;
+                maxh = h;
+            }
+        }
+        return filtered[i];
+    }
+}
+
+function murmur3(s)
+{
+    let hash = 0x12345678;
+    for (let i = 0; i < s.length; i++)
+    {
+        hash ^= s.charCodeAt(i);
+        hash = (hash*0x5bd1e995) & 0xFFFFFFFF;
+        hash ^= (hash >> 15);
+    }
+    return hash;
+}
+
+function filter_tree_by_rules(osd_tree, rules, selected)
+{
+    let cur = osd_tree[''].children;
+    for (const rule of rules)
+    {
+        const val = (rule[2] instanceof Array ? rule[2] : [ rule[2] ])
+            .map(v => v instanceof Object ? v.id : selected[v-1].levels[rule[0]]);
+        let preferred = [], other = [];
+        for (let i = 0; i < cur.length; i++)
+        {
+            const item = cur[i];
+            const level_id = item.levels[rule[0]];
+            if (level_id)
+            {
+                if (rule[1] == '>' && val.filter(v => level_id <= v).length == 0 ||
+                    (rule[1] == '=' || rule[1] == '?=') && val.filter(v => level_id != v).length == 0 ||
+                    rule[1] == '!=' && val.filter(v => level_id == v).length == 0)
+                {
+                    // Include
+                    preferred.push(item);
+                }
+                else if (rule[1] == '?=' && val.filter(v => level_id != v).length > 0)
+                {
+                    // Non-preferred
+                    other.push(item);
+                }
+            }
+            else if (item.children)
+            {
+                // Descend
+                cur.splice(i+1, 0, ...item.children);
+            }
+        }
+        cur = preferred.length ? preferred : other;
+    }
+    // Get leaf items
+    for (let i = 0; i < cur.length; i++)
+    {
+        if (cur[i].children)
+        {
+            // Descend
+            cur.splice(i, 1, ...cur[i].children);
+            i--;
+        }
+    }
+    return cur;
+}
+
+// Convert from
+// node_list = { id: string|number, level: string, size?: number, parent?: string|number }[]
+// to
+// node_tree = { [node_id]: { id, level, size?, parent?, children?: child_node_id[], levels: { [level]: id, ... } } }
+function index_tree(node_list)
+{
+    const tree = { '': { children: [], levels: {} } };
+    for (const node of node_list)
+    {
+        tree[node.id] = { ...node, levels: {} };
+        delete tree[node.id].children;
+    }
+    for (const node of node_list)
+    {
+        const parent_id = node.parent && tree[node.parent] ? node.parent : '';
+        tree[parent_id].children = tree[parent_id].children || [];
+        tree[parent_id].children.push(tree[node.id]);
+    }
+    const cur = tree[''].children;
+    for (let i = 0; i < cur.length; i++)
+    {
+        cur[i].levels[cur[i].level] = cur[i].id;
+        if (cur[i].children)
+        {
+            for (const child of cur[i].children)
+            {
+                child.levels = { ...cur[i].levels, ...child.levels };
+            }
+            cur.splice(i, 1, ...cur[i].children);
+            i--;
+        }
+    }
+    return tree;
+}
+
+// selection = id[]
+// osd_tree = index_tree output
+// rules = parse_pg_dsl output
+function check_custom_combinations(osd_tree, rules, pgs)
+{
+    const res = [];
+    skip_pg: for (const pg of pgs)
+    {
+        let selected = pg.map(id => osd_tree[id] || null);
+        for (let i = 0; i < rules.length; i++)
+        {
+            const filtered = filter_tree_by_rules(osd_tree, rules[i], selected);
+            if (selected[i] === null && filtered.length ||
+                !filtered.filter(ok => selected[i].id === ok.id).length)
+            {
+                continue skip_pg;
+            }
+        }
+        res.push(pg);
+    }
+    return res;
+}
+
+module.exports = {
+    RuleCombinator,
+    NO_OSD,
+
+    index_tree,
+    parse_level_indexes,
+    parse_pg_dsl,
+    random_custom_combinations,
+    check_custom_combinations,
+};

mon/mon.js

@@ -6,6 +6,7 @@ const http = require('http');
 const crypto = require('crypto');
 const os = require('os');
 const WebSocket = require('ws');
+const { RuleCombinator, parse_level_indexes, parse_pg_dsl } = require('./dsl_pgs.js');
 const { SimpleCombinator } = require('./simple_pgs.js');
 const LPOptimizer = require('./lp-optimizer.js');
 const stableStringify = require('./stable-stringify.js');
@@ -64,6 +65,7 @@ const etcd_tree = {
             mon_stats_timeout: 1000, // ms. min: 100
             osd_out_time: 600, // seconds. min: 0
             placement_levels: { datacenter: 1, rack: 2, host: 3, osd: 4, ... },
+            force_new_pg_combinator: false,
             // client and osd
             tcp_header_buffer_size: 65536,
             use_sync_send_recv: false,
@@ -185,7 +187,10 @@ const etcd_tree = {
                 // number of parity chunks, required for EC
                 parity_chunks?: 1,
                 pg_count: 100,
-                failure_domain: 'host',
+                // default is failure_domain=host
+                failure_domain?: 'host',
+                level_distribution?: { host: '123' },
+                pg_rules?: 'any, dc=1 host!=1, dc=1 host!=(1,2)',
                 max_osd_combinations: 10000,
                 // block_size, bitmap_granularity, immediate_commit must match all OSDs used in that pool
                 block_size: 131072,
@@ -1089,7 +1094,6 @@ class Mon
         pool_cfg.pg_minsize = Math.floor(pool_cfg.pg_minsize);
         pool_cfg.parity_chunks = Math.floor(pool_cfg.parity_chunks) || undefined;
         pool_cfg.pg_count = Math.floor(pool_cfg.pg_count);
-        pool_cfg.failure_domain = pool_cfg.failure_domain || 'host';
         pool_cfg.max_osd_combinations = Math.floor(pool_cfg.max_osd_combinations) || 10000;
         if (!/^[1-9]\d*$/.exec(''+pool_id))
         {
@@ -1169,6 +1173,10 @@ class Mon
                 console.log('Pool '+pool_id+' has invalid primary_affinity_tags (must be a string or array of strings)');
             return false;
         }
+        if (!this.get_pg_rules(pool_id, pool_cfg, true))
+        {
+            return false;
+        }
         return true;
     }
 
@@ -1242,6 +1250,50 @@ class Mon
         return aff_osds;
     }
 
+    get_pg_rules(pool_id, pool_cfg, warn)
+    {
+        if (pool_cfg.level_distribution instanceof Object)
+        {
+            const levels = this.config.placement_levels||{};
+            levels.host = levels.host || 100;
+            levels.osd = levels.osd || 101;
+            for (const k in pool_cfg.level_distribution)
+            {
+                if (!levels[k] || typeof pool_cfg.level_distribution[k] !== 'string' &&
+                    (!pool_cfg.level_distribution[k] instanceof Array || pool_cfg.level_distribution[k].filter(s => typeof s !== 'string').length > 0))
+                {
+                    if (warn)
+                        console.log('Pool '+pool_id+' configuration is invalid: level_distribution should be { [level]: string or array of strings }');
+                    return null;
+                }
+            }
+            return parse_level_indexes(pool_cfg.level_distribution);
+        }
+        else if (typeof pool_cfg.pg_rules === 'string')
+        {
+            try
+            {
+                return parse_pg_dsl(pool_cfg.pg_rules);
+            }
+            catch (e)
+            {
+                if (warn)
+                    console.log('Pool '+pool_id+' configuration is invalid: invalid pg_rules: '+e.message);
+            }
+        }
+        else
+        {
+            let rules = [ [] ];
+            let prev = [ 1 ];
+            for (let i = 1; i < pool_cfg.pg_size; i++)
+            {
+                rules.push([ [ pool_cfg.failure_domain||'host', '!=', prev ] ]);
+                prev = [ ...prev, i+1 ];
+            }
+            return rules;
+        }
+    }
+
     async generate_pool_pgs(pool_id, osd_tree, levels)
     {
         const pool_cfg = this.state.config.pools[pool_id];
@@ -1275,7 +1327,11 @@ class Mon
         const old_pg_count = prev_pgs.length;
         const optimize_cfg = {
             osd_weights: Object.values(pool_tree).filter(item => item.level === 'osd').reduce((a, c) => { a[c.id] = c.size; return a; }, {}),
-            combinator: new SimpleCombinator(flatten_tree(osd_tree, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations),
+            combinator: 1 || this.config.force_new_pg_combinator || !pool_cfg.level_distribution && !pool_cfg.pg_rules
+                // new algorithm:
+                ? new RuleCombinator(osd_tree, this.get_pg_rules(pool_id, pool_cfg), pool_cfg.max_osd_combinations)
+                // old algorithm:
+                : new SimpleCombinator(flatten_tree(osd_tree, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations),
             pg_count: pool_cfg.pg_count,
             pg_size: pool_cfg.pg_size,
             pg_minsize: pool_cfg.pg_minsize,

mon/test-parse-dsl.js

@@ -0,0 +1,104 @@
+const { random_custom_combinations, index_tree, parse_level_indexes, parse_pg_dsl } = require('./dsl_pgs.js');
+
+function check(result, expected)
+{
+    console.dir(result, { depth: null });
+    if (JSON.stringify(result) !== JSON.stringify(expected))
+    {
+        process.stderr.write('Unexpected value, expected: ');
+        console.dir(expected, { depth: null });
+        process.exit(1);
+    }
+}
+
+check(
+    parse_pg_dsl("any, dc=1 host!=1, dc!=1, dc=3 host!=3, dc!=(1,3), dc=5 host!=5"),
+    [
+        [],
+        [ [ 'dc', '=', 1 ], [ 'host', '!=', 1 ] ],
+        [ [ 'dc', '!=', 1 ] ],
+        [ [ 'dc', '=', 3 ], [ 'host', '!=', 3 ] ],
+        [ [ 'dc', '!=', [ 1, 3 ] ] ],
+        [ [ 'dc', '=', 5 ], [ 'host', '!=', 5 ] ],
+    ]
+);
+
+check(
+    parse_pg_dsl("dc=meow, dc!=1, dc>2"),
+    [
+        [ [ 'dc', '=', { id: 'meow' } ] ],
+        [ [ 'dc', '!=', 1 ] ],
+        [ [ 'dc', '>', 2 ] ],
+    ]
+);
+
+check(
+    parse_level_indexes({ dc: '112233', host: 'ABCDEF' }),
+    [
+        [],
+        [ [ 'dc', '=', 1 ],         [ 'host', '!=', [ 1 ] ] ],
+        [ [ 'dc', '!=', [ 1 ] ],    [ 'host', '!=', [ 1, 2 ] ] ],
+        [ [ 'dc', '=', 3 ],         [ 'host', '!=', [ 1, 2, 3 ] ] ],
+        [ [ 'dc', '!=', [ 1, 3 ] ], [ 'host', '!=', [ 1, 2, 3, 4 ] ] ],
+        [ [ 'dc', '=', 5 ],         [ 'host', '!=', [ 1, 2, 3, 4, 5 ] ] ],
+    ]
+);
+
+check(
+    parse_level_indexes({ dc: '112233', host: 'ABCDEF' }, [ 'dc', 'host' ]),
+    [
+        [],
+        [ [ 'dc', '=', 1 ],         [ 'host', '!=', [ 1 ] ] ],
+        [ [ 'dc', '!=', [ 1 ] ] ],
+        [ [ 'dc', '=', 3 ],         [ 'host', '!=', [ 3 ] ] ],
+        [ [ 'dc', '!=', [ 1, 3 ] ] ],
+        [ [ 'dc', '=', 5 ],         [ 'host', '!=', [ 5 ] ] ],
+    ]
+);
+
+check(
+    parse_level_indexes({ dc: '112211223333', host: '123456789ABC' }),
+    [
+        [],
+        [ [ 'dc', '=', 1 ],         [ 'host', '!=', [ 1 ] ] ],
+        [ [ 'dc', '!=', [ 1 ] ],    [ 'host', '!=', [ 1, 2 ] ] ],
+        [ [ 'dc', '=', 3 ],         [ 'host', '!=', [ 1, 2, 3 ] ] ],
+        [ [ 'dc', '=', 1 ],         [ 'host', '!=', [ 1, 2, 3, 4 ] ] ],
+        [ [ 'dc', '=', 1 ],         [ 'host', '!=', [ 1, 2, 3, 4, 5 ] ] ],
+        [ [ 'dc', '=', 3 ],         [ 'host', '!=', [ 1, 2, 3, 4, 5, 6 ] ] ],
+        [ [ 'dc', '=', 3 ],         [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7 ] ] ],
+        [ [ 'dc', '!=', [ 1, 3 ] ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7, 8 ] ] ],
+        [ [ 'dc', '=', 9 ],         [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ] ],
+        [ [ 'dc', '=', 9 ],         [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ] ] ],
+        [ [ 'dc', '=', 9 ],         [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ] ] ],
+    ]
+);
+
+check(
+    parse_level_indexes({ dc: '112211223333', host: '123456789ABC' }, [ 'dc', 'host' ]),
+    [
+        [],
+        [ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1 ] ] ],
+        [ [ 'dc', '!=', [ 1 ] ] ],
+        [ [ 'dc', '=', 3 ], [ 'host', '!=', [ 3 ] ] ],
+        [ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1, 2 ] ] ],
+        [ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1, 2, 5 ] ] ],
+        [ [ 'dc', '=', 3 ], [ 'host', '!=', [ 3, 4 ] ] ],
+        [ [ 'dc', '=', 3 ], [ 'host', '!=', [ 3, 4, 7 ] ] ],
+        [ [ 'dc', '!=', [ 1, 3 ] ] ],
+        [ [ 'dc', '=', 9 ], [ 'host', '!=', [ 9 ] ] ],
+        [ [ 'dc', '=', 9 ], [ 'host', '!=', [ 9, 10 ] ] ],
+        [ [ 'dc', '=', 9 ], [ 'host', '!=', [ 9, 10, 11 ] ] ]
+    ]
+);
+
+check(
+    Object.keys(random_custom_combinations(index_tree([
+        { id: '1', size: 1, level: 'osd' },
+        { id: '2', size: 2, level: 'osd' },
+        { id: '3', size: 3, level: 'osd' }
+    ]), parse_level_indexes({ osd: '12' }), 10000)).sort(),
+    [ 'pg_1_2', 'pg_1_3', 'pg_2_3' ]
+);
+
+console.log('OK');