vitalif
/
vitastor
5
4
Fork 9
Code Issues 1 Pull Requests 2 Actions Releases 69 Wiki Activity

Implement multi-level tree extractor for hierarchical failure domains

hier-failure-domains
Vitaliy Filippov 2022-12-25 14:30:11 +03:00
parent 431f780347
commit 57feb7f390
2 changed files with 91 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
mon/lp-optimizer.js
View File

@ -505,6 +505,7 @@ function put_aligned_pgs(aligned_pgs, int_pgs, prev_int_pgs, keygen)
// Convert multi-level osd_tree = { level: number|string, id?: string, size?: number, children?: osd_tree }[]
// levels = { string: number }
// to a two-level osd_tree suitable for all_combinations()
// FIXME: Replace with extract_tree_levels({ level: -Infinity, children: osd_tree }, [ failure_domain_level, osd_level ], levels)
function flatten_tree(osd_tree, levels, failure_domain_level, osd_level, domains = {}, i = { i: 1 })
{
osd_level = levels[osd_level] || osd_level;
@ -539,6 +540,49 @@ function extract_osds(osd_tree, levels, osd_level, osds = {})
return osds;
}
// Convert multi-level tree_node = { level: number|string, id?: string, size?: number, children?: tree_node[] }
// levels = { string: number }
// to a multi-level OSD tree suitable for random_hier_combinations()
// (or in case of just 2 levels - for all_combinations() / random_combinations())
//
// Example:
// tree_node = { level: 'dc', children: [ { level: 'rack', children: [ { level: 'host', children: [ { level: 'osd', size: 10 } ] } ] } ] }
// extract_levels = [ 'rack', 'osd' ]
// level_defs = { dc: 1, rack: 2, host: 3, osd: 4 }
//
// Result:
// { rack0: { osd1: 10 } }
function extract_tree_levels(tree_node, extract_levels, level_defs, new_tree = { idx: 1, items: {} })
{
const next_level = Number(level_defs[extract_levels[0]] || extract_levels[0]) || 0;
const level_name = level_defs[extract_levels[0]] ? extract_levels[0] : 'l'+extract_levels[0]+'_';
const is_leaf = extract_levels.length == 1;
if ((level_defs[tree_node.level] || tree_node.level) >= next_level)
{
if (!is_leaf)
{
// Insert a (possibly fake) level
const nt = { idx: 1, items: {} };
new_tree.items[level_name+(new_tree.idx++)] = nt.items;
extract_tree_levels(tree_node, extract_levels.slice(1), level_defs, nt);
}
else
{
// Insert a leaf node
const leaf_id = tree_node.id || (level_name+(new_tree.idx++));
new_tree.items[leaf_id] = tree_node.size;
}
}
else
{
for (const child_node of tree_node.children||[])
{
extract_tree_levels(child_node, extract_levels, level_defs, new_tree);
}
}
return new_tree.items;
}
// generate random PGs with hierarchical failure domains, i.e. for example 3 DC each with 2 HOSTS
// osd_tree = { level3_id: { level2_id: { level1_id: scalar_value } }, ... }
// osd_tree may contain arbitrary number of levels, but level count must be the same across the whole tree
@ -853,6 +897,7 @@ module.exports = {
pg_list_space_efficiency,
pg_per_osd_space_efficiency,
flatten_tree,
extract_tree_levels,
lp_solve,
make_int_pgs,

46
mon/test-random-hier.js Normal file
View File

@ -0,0 +1,46 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
const LPOptimizer = require('./lp-optimizer.js');
const osd_tree = {
100: { 110: { 111: 1, 112: 1 }, 120: { 121: 1, 122: 1 } },
200: { 210: { 211: 1, 212: 1 }, 220: { 221: 1, 222: 1 } },
300: { 310: { 311: 1, 312: 1 }, 320: { 321: 1, 322: 1 } },
400: { 410: { 411: 1, 412: 1 }, 420: { 421: 1, 422: 1 } },
500: { 510: { 511: 1, 512: 1 }, 520: { 521: 1, 522: 1 } },
};
const osd_tree2 = {
100: { 111: 1, 112: 1, 121: 1, 122: 1 },
200: { 211: 1, 212: 1, 221: 1, 222: 1 },
300: { 311: 1, 312: 1, 321: 1, 322: 1 },
400: { 411: 1, 412: 1, 421: 1, 422: 1 },
500: { 511: 1, 512: 1, 521: 1, 522: 1 },
};
async function run()
{
let r;
console.log(r = LPOptimizer.random_hier_combinations(osd_tree, [ 3, 2, 1 ], 10000, false, true));
console.log(r = LPOptimizer.random_hier_combinations(osd_tree2, [ 3, 2 ], 0, false, true));
// Will contain 'Z':
console.log(r = LPOptimizer.random_combinations(osd_tree2, 6, 0, true));
console.log(r = LPOptimizer.extract_tree_levels(
{ level: 'dc', children: [
{ level: 'rack', children: [
{ level: 'host', children: [
{ level: 'osd', id: 'OSD5', size: 10 },
] },
] },
{ level: 'osd', id: 'OSD10', size: 10 },
] },
[ 'rack', 'osd' ],
{ dc: 1, rack: 2, host: 3, osd: 4 }
));
if (JSON.stringify(r) != '{"rack1":{"OSD5":10},"rack2":{"OSD10":10}}')
throw new Error('extract_tree_levels failed');
console.log('OK');
}
run().catch(console.error);