Compare commits

...

3 Commits

Author SHA1 Message Date
Vitaliy Filippov 17a80da61f Try new PG combinator in tests
Test / test_interrupted_rebalance (push) Successful in 4m13s Details
Test / test_snapshot_chain (push) Successful in 1m31s Details
Test / test_snapshot_down (push) Successful in 29s Details
Test / test_snapshot_down_ec (push) Successful in 30s Details
Test / test_snapshot_chain_ec (push) Successful in 2m8s Details
Test / test_splitbrain (push) Successful in 18s Details
Test / test_rebalance_verify_imm (push) Successful in 2m10s Details
Test / test_rebalance_verify (push) Successful in 2m47s Details
Test / test_rebalance_verify_ec (push) Successful in 2m43s Details
Test / test_write (push) Successful in 38s Details
Test / test_write_no_same (push) Successful in 13s Details
Test / test_rebalance_verify_ec_imm (push) Successful in 3m11s Details
Test / test_heal_pg_size_2 (push) Successful in 3m8s Details
Test / test_heal_csum_32k_dmj (push) Successful in 5m18s Details
Test / test_heal_ec (push) Successful in 5m33s Details
Test / test_heal_csum_32k_dj (push) Successful in 5m57s Details
Test / test_heal_csum_32k (push) Successful in 5m33s Details
Test / test_scrub (push) Successful in 1m2s Details
Test / test_scrub_zero_osd_2 (push) Successful in 1m10s Details
Test / test_scrub_xor (push) Successful in 48s Details
Test / test_heal_csum_4k_dmj (push) Successful in 5m41s Details
Test / test_heal_csum_4k_dj (push) Successful in 6m6s Details
Test / test_heal_csum_4k (push) Successful in 4m59s Details
Test / test_scrub_ec (push) Successful in 26s Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Successful in 1m37s Details
Test / test_change_pg_size (push) Failing after 24s Details
Test / test_scrub_pg_size_3 (push) Successful in 1m10s Details
Test / test_interrupted_rebalance_ec (push) Successful in 1m10s Details
Test / test_write_xor (push) Failing after 3m6s Details
Test / test_add_osd (push) Failing after 54s Details
2024-01-02 01:21:58 +03:00
Vitaliy Filippov 8972451690 Plug the new PG combinator into monitor 2024-01-02 01:21:58 +03:00
Vitaliy Filippov f4bfd2e64b Implement new DSL/rule-based PG generation algorithm 2024-01-02 01:21:58 +03:00
3 changed files with 592 additions and 3 deletions

438
mon/dsl_pgs.js Normal file
View File

@ -0,0 +1,438 @@
const NO_OSD = 'Z';
class RuleCombinator
{
constructor(osd_tree, rules, max_combinations, ordered)
{
this.osd_tree = index_tree(Object.values(osd_tree).filter(o => o.id));
this.rules = rules;
this.max_combinations = max_combinations;
this.ordered = ordered;
}
random_combinations()
{
return random_custom_combinations(this.osd_tree, this.rules, this.max_combinations, this.ordered);
}
check_combinations(pgs)
{
return check_custom_combinations(this.osd_tree, this.rules, pgs);
}
}
// Convert alternative "level-index" format to rules
// level_index = { [level: string]: string | string[] }
// level_sequence = optional, levels from upper to lower, i.e. [ 'dc', 'host' ]
// Example: level_index = { dc: "112233", host: "ABCDEF" }
function parse_level_indexes(level_index, level_sequence)
{
const rules = [];
const lvl_first = {};
for (const level in level_index)
{
const idx = level_index[level];
while (rules.length < idx.length)
{
rules.push([]);
}
const seen = {};
for (let i = 0; i < idx.length; i++)
{
if (!seen[idx[i]])
{
const other = Object.values(seen);
if (other.length)
{
rules[i].push([ level, '!=', other ]);
}
seen[idx[i]] = i+1;
}
else
{
rules[i].push([ level, '=', seen[idx[i]] ]);
}
}
lvl_first[level] = seen;
}
if (level_sequence)
{
// Prune useless rules for the sake of prettiness
// For simplicity, call "upper" level DC and "lower" level host
const level_prio = Object.keys(level_sequence).reduce((a, c) => { a[level_sequence[c]] = c; return a; }, {});
for (let upper_i = 0; upper_i < level_sequence.length-1; upper_i++)
{
const upper_level = level_sequence[upper_i];
for (let i = 0; i < rules.length; i++)
{
const noteq = {};
for (let k = 0; k < level_index[upper_level].length; k++)
{
// If upper_level[x] is different from upper_level[y]
// then lower_level[x] is also different from lower_level[y]
if (level_index[upper_level][k] != level_index[upper_level][i])
{
noteq[k+1] = true;
}
}
for (let j = 0; j < rules[i].length; j++)
{
if (level_prio[rules[i][j][0]] != null && level_prio[rules[i][j][0]] > upper_i && rules[i][j][1] == '!=')
{
rules[i][j][2] = rules[i][j][2].filter(other_host => !noteq[other_host]);
if (!rules[i][j][2].length)
{
rules[i].splice(j--, 1);
}
}
}
}
}
}
return rules;
}
// Parse rules in DSL format
// dsl := item | item ("\n" | ",") items
// item := "any" | rules
// rules := rule | rule rules
// rule := level operator arg
// level := /\w+/
// operator := "!=" | "=" | ">" | "?="
// arg := value | "(" values ")"
// values := value | value "," values
// value := item_ref | constant_id
// item_ref := /\d+/
// constant_id := /"([^"]+)"/
//
// Output: [ level, operator, value ][][]
function parse_pg_dsl(text)
{
const tokens = [ ...text.matchAll(/\w+|!=|\?=|[>=\(\),\n]|"([^\"]+)"/g) ].map(t => [ t[0], t.index ]);
let positions = [ [] ];
let rules = positions[0];
for (let i = 0; i < tokens.length; )
{
if (tokens[i][0] === '\n' || tokens[i][0] === ',')
{
rules = [];
positions.push(rules);
i++;
}
else if (!rules.length && tokens[i][0] === 'any' && (i == tokens.length-1 || tokens[i+1][0] === ',' || tokens[i+1][0] === '\n'))
{
i++;
}
else
{
if (!/^\w/.exec(tokens[i][0]))
{
throw new Error('Unexpected '+tokens[i][0]+' at '+tokens[i][1]+' (level name expected)');
}
if (i > tokens.length-3)
{
throw new Error('Unexpected EOF (operator and value expected)');
}
if (/^\w/.exec(tokens[i+1][0]) || tokens[i+1][0] === ',' || tokens[i+1][0] === '\n')
{
throw new Error('Unexpected '+tokens[i+1][0]+' at '+tokens[i+1][1]+' (operator expected)');
}
if (!/^[\w"(]/.exec(tokens[i+2][0])) // "
{
throw new Error('Unexpected '+tokens[i+2][0]+' at '+tokens[i+2][1]+' (id, round brace, number or node ID expected)');
}
let rule = [ tokens[i][0], tokens[i+1][0], tokens[i+2][0] ];
i += 3;
if (rule[2][0] == '"')
{
rule[2] = { id: rule[2].substr(1, rule[2].length-2) };
}
else if (rule[2] === '(')
{
rule[2] = [];
while (true)
{
if (i > tokens.length-1)
{
throw new Error('Unexpected EOF (expected list and a closing round brace)');
}
if (tokens[i][0] === ',')
{
i++;
}
else if (tokens[i][0] === ')')
{
i++;
break;
}
else if (tokens[i][0][0] === '"')
{
rule[2].push({ id: tokens[i][0].substr(1, tokens[i][0].length-2) });
i++;
}
else if (/^\d+$/.exec(tokens[i][0]))
{
const n = 0|tokens[i][0];
if (!n)
{
throw new Error('Level reference cannot be 0 (refs count from 1) at '+tokens[i][1]);
}
else if (n > positions.length)
{
throw new Error('Forward references are forbidden at '+tokens[i][1]);
}
rule[2].push(n);
i++;
}
else if (!/^\w/.exec(tokens[i][0]))
{
throw new Error('Unexpected '+tokens[i][0]+' at '+tokens[i][1]+' (number or node ID expected)');
}
else
{
rule[2].push({ id: tokens[i][0] });
i++;
}
}
}
else if (!/^\d+$/.exec(rule[2]))
{
rule[2] = { id: rule[2] };
}
else
{
rule[2] = 0|rule[2];
if (!rule[2])
{
throw new Error('Level reference cannot be 0 (refs count from 1) at '+tokens[i-1][1]);
}
else if (rule[2] > positions.length)
{
throw new Error('Forward references are forbidden at '+tokens[i-1][1]);
}
}
rules.push(rule);
}
}
return positions;
}
// osd_tree = index_tree() output
// levels = { string: number }
// rules = [ level, operator, value ][][]
// level = string
// operator = '=' | '!=' | '>' | '?='
// value = number|number[] | { id: string|string[] }
// examples:
// 1) simple 3 replicas with failure_domain=host:
// [ [], [ [ 'host', '!=', 1 ] ], [ [ 'host', '!=', [ 1, 2 ] ] ] ]
// in DSL form: any, host!=1, host!=(1,2)
// 2) EC 4+2 in 3 DC:
// [ [], [ [ 'dc', '=', 1 ], [ 'host', '!=', 1 ] ],
// [ 'dc', '!=', 1 ], [ [ 'dc', '=', 3 ], [ 'host', '!=', 3 ] ],
// [ 'dc', '!=', [ 1, 3 ] ], [ [ 'dc', '=', 5 ], [ 'host', '!=', 5 ] ] ]
// in DSL form: any, dc=1 host!=1, dc!=1, dc=3 host!=3, dc!=(1,3), dc=5 host!=5
// 3) 1 replica in fixed DC + 2 in random DCs:
// [ [ [ 'dc', '=', { id: 'meow' } ] ], [ [ 'dc', '!=', 1 ] ], [ [ 'dc', '!=', [ 1, 2 ] ] ] ]
// in DSL form: dc=meow, dc!=1, dc!=(1,2)
// 4) 2 replicas in each DC (almost the same as (2)):
// DSL: any, dc=1 host!=1, dc!=1, dc=3 host!=3
// Alternative simpler way to specify rules would be: [ DC: 112233 HOST: 123456 ]
function random_custom_combinations(osd_tree, rules, count, ordered)
{
const r = {};
const first = filter_tree_by_rules(osd_tree, rules[0], []);
let max_size = 0;
// All combinations for the first item (usually "any") to try to include each OSD at least once
for (const f of first)
{
const selected = [ f ];
for (let i = 1; i < rules.length; i++)
{
const filtered = filter_tree_by_rules(osd_tree, rules[i], selected);
selected.push(select_murmur3(filtered, 'p:'+f.id));
}
const size = selected.filter(s => s.id !== null).length;
max_size = max_size < size ? size : max_size;
const pg = selected.map(s => s.id === null ? NO_OSD : s.id);
if (!ordered)
pg.sort();
r['pg_'+pg.join('_')] = pg;
}
// Pseudo-random selection
for (let n = 0; n < count; n++)
{
const selected = [];
for (const item_rules of rules)
{
const filtered = selected.length ? filter_tree_by_rules(osd_tree, item_rules, selected) : first;
selected.push(select_murmur3(filtered, n));
}
const size = selected.filter(s => s.id !== null).length;
max_size = max_size < size ? size : max_size;
const pg = selected.map(s => s.id === null ? NO_OSD : s.id);
if (!ordered)
pg.sort();
r['pg_'+pg.join('_')] = pg;
}
// Exclude PGs with less successful selections than maximum
for (const k in r)
{
if (r[k].filter(s => s !== NO_OSD).length < max_size)
{
delete r[k];
}
}
return r;
}
function select_murmur3(filtered, prefix)
{
if (!filtered.length)
{
return { levels: {}, id: null };
}
else
{
let i = 0, maxh = -1;
for (let j = 0; j < filtered.length; j++)
{
const h = murmur3(prefix+':'+filtered[j].id);
if (h > maxh)
{
i = j;
maxh = h;
}
}
return filtered[i];
}
}
function murmur3(s)
{
let hash = 0x12345678;
for (let i = 0; i < s.length; i++)
{
hash ^= s[i].charCodeAt(i);
hash = (hash*0x5bd1e995) & 0xFFFFFFFF;
hash ^= (hash >> 15);
}
return hash;
}
function filter_tree_by_rules(osd_tree, rules, selected)
{
let cur = osd_tree[''].children;
for (const rule of rules)
{
const val = (rule[2] instanceof Array ? rule[2] : [ rule[2] ])
.map(v => v instanceof Object ? v.id : selected[v-1].levels[rule[0]]);
let preferred = [], other = [];
for (let i = 0; i < cur.length; i++)
{
const item = cur[i];
const level_id = item.levels[rule[0]];
if (level_id)
{
if (rule[1] == '>' && val.filter(v => level_id <= v).length == 0 ||
(rule[1] == '=' || rule[1] == '?=') && val.filter(v => level_id != v).length == 0 ||
rule[1] == '!=' && val.filter(v => level_id == v).length == 0)
{
// Include
preferred.push(item);
}
else if (rule[1] == '?=' && val.filter(v => level_id != v).length > 0)
{
// Non-preferred
other.push(item);
}
}
else if (item.children)
{
// Descend
cur.splice(i+1, 0, ...item.children);
}
}
cur = preferred.length ? preferred : other;
}
// Get leaf items
for (let i = 0; i < cur.length; i++)
{
if (cur[i].children)
{
// Descend
cur.splice(i, 1, ...cur[i].children);
i--;
}
}
return cur;
}
// Convert from
// node_list = { id: string|number, level: string, size?: number, parent?: string|number }[]
// to
// node_tree = { [node_id]: { id, level, size?, parent?, children?: child_node_id[], levels: { [level]: id, ... } } }
function index_tree(node_list)
{
const tree = { '': { children: [], levels: {} } };
for (const node of node_list)
{
tree[node.id] = { ...node, levels: {} };
delete tree[node.id].children;
}
for (const node of node_list)
{
const parent_id = node.parent && tree[node.parent] ? node.parent : '';
tree[parent_id].children = tree[parent_id].children || [];
tree[parent_id].children.push(tree[node.id]);
}
const cur = tree[''].children;
for (let i = 0; i < cur.length; i++)
{
cur[i].levels[cur[i].level] = cur[i].id;
if (cur[i].children)
{
for (const child of cur[i].children)
{
child.levels = { ...cur[i].levels, ...child.levels };
}
cur.splice(i, 1, ...cur[i].children);
i--;
}
}
return tree;
}
// selection = id[]
// osd_tree = index_tree output
// rules = parse_pg_dsl output
function check_custom_combinations(osd_tree, rules, pgs)
{
const res = [];
skip_pg: for (const pg of pgs)
{
let selected = pg.map(id => osd_tree[id] || null);
for (let i = 0; i < rules.length; i++)
{
const filtered = filter_tree_by_rules(osd_tree, rules[i], selected);
if (selected[i] === null && filtered.length ||
!filtered.filter(ok => selected[i].id === ok.id).length)
{
continue skip_pg;
}
}
res.push(pg);
}
return res;
}
module.exports = {
RuleCombinator,
NO_OSD,
index_tree,
parse_level_indexes,
parse_pg_dsl,
random_custom_combinations,
check_custom_combinations,
};

View File

@ -6,6 +6,7 @@ const http = require('http');
const crypto = require('crypto');
const os = require('os');
const WebSocket = require('ws');
const { RuleCombinator, parse_level_indexes, parse_pg_dsl } = require('./dsl_pgs.js');
const { SimpleCombinator } = require('./simple_pgs.js');
const LPOptimizer = require('./lp-optimizer.js');
const stableStringify = require('./stable-stringify.js');
@ -64,6 +65,7 @@ const etcd_tree = {
mon_stats_timeout: 1000, // ms. min: 100
osd_out_time: 600, // seconds. min: 0
placement_levels: { datacenter: 1, rack: 2, host: 3, osd: 4, ... },
force_new_pg_combinator: false,
// client and osd
tcp_header_buffer_size: 65536,
use_sync_send_recv: false,
@ -185,7 +187,10 @@ const etcd_tree = {
// number of parity chunks, required for EC
parity_chunks?: 1,
pg_count: 100,
failure_domain: 'host',
// default is failure_domain=host
failure_domain?: 'host',
level_distribution?: { host: '123' },
pg_rules?: 'any, dc=1 host!=1, dc=1 host!=(1,2)',
max_osd_combinations: 10000,
// block_size, bitmap_granularity, immediate_commit must match all OSDs used in that pool
block_size: 131072,
@ -1089,7 +1094,6 @@ class Mon
pool_cfg.pg_minsize = Math.floor(pool_cfg.pg_minsize);
pool_cfg.parity_chunks = Math.floor(pool_cfg.parity_chunks) || undefined;
pool_cfg.pg_count = Math.floor(pool_cfg.pg_count);
pool_cfg.failure_domain = pool_cfg.failure_domain || 'host';
pool_cfg.max_osd_combinations = Math.floor(pool_cfg.max_osd_combinations) || 10000;
if (!/^[1-9]\d*$/.exec(''+pool_id))
{
@ -1169,6 +1173,10 @@ class Mon
console.log('Pool '+pool_id+' has invalid primary_affinity_tags (must be a string or array of strings)');
return false;
}
if (!this.get_pg_rules(pool_id, pool_cfg, true))
{
return false;
}
return true;
}
@ -1242,6 +1250,50 @@ class Mon
return aff_osds;
}
get_pg_rules(pool_id, pool_cfg, warn)
{
if (pool_cfg.level_distribution instanceof Object)
{
const levels = this.config.placement_levels||{};
levels.host = levels.host || 100;
levels.osd = levels.osd || 101;
for (const k in pool_cfg.level_distribution)
{
if (!levels[k] || typeof pool_cfg.level_distribution[k] !== 'string' &&
(!pool_cfg.level_distribution[k] instanceof Array || pool_cfg.level_distribution[k].filter(s => typeof s !== 'string').length > 0))
{
if (warn)
console.log('Pool '+pool_id+' configuration is invalid: level_distribution should be { [level]: string or array of strings }');
return null;
}
}
return parse_level_indexes(pool_cfg.level_distribution);
}
else if (typeof pool_cfg.pg_rules === 'string')
{
try
{
return parse_pg_dsl(pool_cfg.pg_rules);
}
catch (e)
{
if (warn)
console.log('Pool '+pool_id+' configuration is invalid: invalid pg_rules: '+e.message);
}
}
else
{
let rules = [ [] ];
let prev = [ 1 ];
for (let i = 1; i < pool_cfg.pg_size; i++)
{
rules.push([ [ pool_cfg.failure_domain||'host', '!=', prev ] ]);
prev = [ ...prev, i+1 ];
}
return rules;
}
}
async generate_pool_pgs(pool_id, osd_tree, levels)
{
const pool_cfg = this.state.config.pools[pool_id];
@ -1275,7 +1327,11 @@ class Mon
const old_pg_count = prev_pgs.length;
const optimize_cfg = {
osd_weights: Object.values(pool_tree).filter(item => item.level === 'osd').reduce((a, c) => { a[c.id] = c.size; return a; }, {}),
combinator: new SimpleCombinator(flatten_tree(osd_tree, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations),
combinator: 1 || this.config.force_new_pg_combinator || !pool_cfg.level_distribution && !pool_cfg.pg_rules
// new algorithm:
? new RuleCombinator(osd_tree, this.get_pg_rules(pool_id, pool_cfg), pool_cfg.max_osd_combinations)
// old algorithm:
: new SimpleCombinator(flatten_tree(osd_tree, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations),
pg_count: pool_cfg.pg_count,
pg_size: pool_cfg.pg_size,
pg_minsize: pool_cfg.pg_minsize,

95
mon/test-parse-dsl.js Normal file
View File

@ -0,0 +1,95 @@
const { parse_level_indexes, parse_pg_dsl } = require('./dsl_pgs.js');
function check(result, expected)
{
console.dir(result, { depth: null });
if (JSON.stringify(result) !== JSON.stringify(expected))
{
process.stderr.write('Unexpected value, expected: ');
console.dir(expected, { depth: null });
process.exit(1);
}
}
check(
parse_pg_dsl("any, dc=1 host!=1, dc!=1, dc=3 host!=3, dc!=(1,3), dc=5 host!=5"),
[
[],
[ [ 'dc', '=', 1 ], [ 'host', '!=', 1 ] ],
[ [ 'dc', '!=', 1 ] ],
[ [ 'dc', '=', 3 ], [ 'host', '!=', 3 ] ],
[ [ 'dc', '!=', [ 1, 3 ] ] ],
[ [ 'dc', '=', 5 ], [ 'host', '!=', 5 ] ],
]
);
check(
parse_pg_dsl("dc=meow, dc!=1, dc>2"),
[
[ [ 'dc', '=', { id: 'meow' } ] ],
[ [ 'dc', '!=', 1 ] ],
[ [ 'dc', '>', 2 ] ],
]
);
check(
parse_level_indexes({ dc: '112233', host: 'ABCDEF' }),
[
[],
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1 ] ] ],
[ [ 'dc', '!=', [ 1 ] ], [ 'host', '!=', [ 1, 2 ] ] ],
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 1, 2, 3 ] ] ],
[ [ 'dc', '!=', [ 1, 3 ] ], [ 'host', '!=', [ 1, 2, 3, 4 ] ] ],
[ [ 'dc', '=', 5 ], [ 'host', '!=', [ 1, 2, 3, 4, 5 ] ] ],
]
);
check(
parse_level_indexes({ dc: '112233', host: 'ABCDEF' }, [ 'dc', 'host' ]),
[
[],
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1 ] ] ],
[ [ 'dc', '!=', [ 1 ] ] ],
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 3 ] ] ],
[ [ 'dc', '!=', [ 1, 3 ] ] ],
[ [ 'dc', '=', 5 ], [ 'host', '!=', [ 5 ] ] ],
]
);
check(
parse_level_indexes({ dc: '112211223333', host: '123456789ABC' }),
[
[],
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1 ] ] ],
[ [ 'dc', '!=', [ 1 ] ], [ 'host', '!=', [ 1, 2 ] ] ],
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 1, 2, 3 ] ] ],
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1, 2, 3, 4 ] ] ],
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1, 2, 3, 4, 5 ] ] ],
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6 ] ] ],
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7 ] ] ],
[ [ 'dc', '!=', [ 1, 3 ] ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7, 8 ] ] ],
[ [ 'dc', '=', 9 ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ] ],
[ [ 'dc', '=', 9 ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ] ] ],
[ [ 'dc', '=', 9 ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ] ] ],
]
);
check(
parse_level_indexes({ dc: '112211223333', host: '123456789ABC' }, [ 'dc', 'host' ]),
[
[],
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1 ] ] ],
[ [ 'dc', '!=', [ 1 ] ] ],
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 3 ] ] ],
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1, 2 ] ] ],
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1, 2, 5 ] ] ],
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 3, 4 ] ] ],
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 3, 4, 7 ] ] ],
[ [ 'dc', '!=', [ 1, 3 ] ] ],
[ [ 'dc', '=', 9 ], [ 'host', '!=', [ 9 ] ] ],
[ [ 'dc', '=', 9 ], [ 'host', '!=', [ 9, 10 ] ] ],
[ [ 'dc', '=', 9 ], [ 'host', '!=', [ 9, 10, 11 ] ] ]
]
);
console.log('OK');