diff --git a/lp/LPOptimizer.pm b/lp/LPOptimizer.pm deleted file mode 100644 index 4e181ec5..00000000 --- a/lp/LPOptimizer.pm +++ /dev/null @@ -1,445 +0,0 @@ -#!/usr/bin/perl -# Data distribution optimizer using linear programming (lp_solve) - -package LPOptimizer; - -use strict; -use IPC::Open2; - -sub make_single -{ - my ($osd_tree) = @_; - my $initial = all_combinations($osd_tree, 1)->[0]; - my $weight; - my $all_weights = { map { %$_ } values %$osd_tree }; - for my $osd (@$initial) - { - $weight = $all_weights->{$osd} if !$weight || $all_weights->{$osd} < $weight; - } - return [ - { set => $initial, weight => $weight }, - ]; -} - -sub optimize_initial -{ - my ($osd_tree, $pg_count) = @_; - my $all_weights = { map { %$_ } values %$osd_tree }; - my $total_weight = 0; - $total_weight += $all_weights->{$_} for keys %$all_weights; - my $pgs = all_combinations($osd_tree); - my $pg_per_osd = {}; - for my $pg (@$pgs) - { - push @{$pg_per_osd->{$_}}, "pg_".join("_", @$pg) for @$pg; - } - my $lp = ''; - $lp .= "max: ".join(" + ", map { "pg_".join("_", @$_) } @$pgs).";\n"; - for my $osd (keys %$pg_per_osd) - { - $lp .= join(" + ", @{$pg_per_osd->{$osd}})." <= ".int($all_weights->{$osd}/$total_weight*$pg_count + 0.5).";\n"; - } - for my $pg (@$pgs) - { - $lp .= "pg_".join("_", @$pg)." >= 0;\n"; - } - $lp .= "int ".join(", ", map { "pg_".join("_", @$_) } @$pgs).";\n"; - my ($score, $weights) = lp_solve($lp); - my $int_pgs = make_int_pgs($weights, $pg_count); - my $eff = pg_list_space_efficiency($int_pgs, $osd_tree); - return { score => $score, weights => $weights, int_pgs => $int_pgs, space => $eff, total_space => $total_weight }; -} - -sub make_int_pgs -{ - my ($weights, $pg_count) = @_; - my $total_weight = 0; - for my $pg_name (keys %$weights) - { - $total_weight += $weights->{$pg_name}; - } - my $int_pgs = []; - my $pg_left = $pg_count; - for my $pg_name (keys %$weights) - { - my $n = int($weights->{$pg_name} / $total_weight * $pg_left + 0.5); - for (my $i = 0; $i < $n; $i++) - { - push @$int_pgs, [ split /_/, substr($pg_name, 3) ]; - } - $total_weight -= $weights->{$pg_name}; - $pg_left -= $n; - } - return $int_pgs; -} - -sub lp_solve -{ - my ($lp) = @_; - my ($pid, $out, $in, $result); - $pid = open2($in, $out, 'lp_solve'); - print $out $lp; - close $out; - { - local $/ = undef; - $result = <$in>; - close $in; - } - my $score = 0; - my $weights = {}; - for my $line (split /\n/, $result) - { - if ($line =~ /^(^Value of objective function: ([\d\.]+)|Actual values of the variables:)\s*$/s) - { - if ($2) - { - $score = $2; - } - next; - } - my ($k, $v) = split /\s+/, $line; - if ($v != 0) - { - $weights->{$k} = $v; - } - } - return ($score, $weights); -} - -sub get_int_pg_weights -{ - my ($prev_int_pgs, $osd_tree) = @_; - my $space = pg_list_space_efficiency($prev_int_pgs, $osd_tree); - my $prev_weights = {}; - my $count = 0; - for my $pg (@$prev_int_pgs) - { - $prev_weights->{'pg_'.join('_', @$pg)}++; - $count++; - } - for my $pg_name (keys %$prev_weights) - { - $prev_weights->{$pg_name} *= $space / $count; - } - return $prev_weights; -} - -# Try to minimize data movement -sub optimize_change -{ - my ($prev_int_pgs, $osd_tree) = @_; - my $pg_count = scalar(@$prev_int_pgs); - my $prev_weights = {}; - my $prev_pg_per_osd = {}; - for my $pg (@$prev_int_pgs) - { - $prev_weights->{"pg_".join("_", @$pg)}++; - push @{$prev_pg_per_osd->{$_}}, "pg_".join("_", @$pg) for @$pg; - } - # Get all combinations - my $pgs = all_combinations($osd_tree); - my $pg_per_osd = {}; - for my $pg (@$pgs) - { - push @{$pg_per_osd->{$_}}, "pg_".join("_", @$pg) for @$pg; - } - # Penalize PGs based on their similarity to old PGs - my $intersect = {}; - for my $pg_name (keys %$prev_weights) - { - my @pg = split /_/, substr($pg_name, 3); - $intersect->{$pg[0].'::'} = $intersect->{':'.$pg[1].':'} = $intersect->{'::'.$pg[2]} = 2; - $intersect->{$pg[0].'::'.$pg[2]} = $intersect->{':'.$pg[1].':'.$pg[2]} = $intersect->{$pg[0].':'.$pg[1].':'} = 1; - } - my $move_weights = {}; - for my $pg (@$pgs) - { - $move_weights->{'pg_'.join('_', @$pg)} = - $intersect->{$pg->[0].'::'.$pg->[2]} || $intersect->{':'.$pg->[1].':'.$pg->[2]} || $intersect->{$pg->[0].':'.$pg->[1].':'} || - $intersect->{$pg->[0].'::'} || $intersect->{':'.$pg->[1].':'} || $intersect->{'::'.$pg->[2]} || - 3; - } - # Calculate total weight - old PG weights - my $pg_names = [ map { 'pg_'.join('_', @$_) } @$pgs ]; - my $all_weights = { map { %$_ } values %$osd_tree }; - my $total_weight = 0; - $total_weight += $all_weights->{$_} for keys %$all_weights; - # Generate the LP problem - my $lp = "min: ".join(" + ", map { $move_weights->{$_} . ' * ' . ($prev_weights->{$_} ? "add_$_" : "$_") } @$pg_names).";\n"; - $lp .= join(" + ", map { $prev_weights->{$_} ? "add_$_ - del_$_" : $_ } @$pg_names)." = 0;\n"; - for my $osd (keys %$pg_per_osd) - { - my @s; - for my $pg (@{$pg_per_osd->{$osd}}) - { - if ($prev_weights->{$pg}) - { - push @s, "add_$pg - del_$pg"; - } - else - { - push @s, $pg; - } - } - $lp .= join(" + ", @s)." <= ".int($all_weights->{$osd}*3/$total_weight*$pg_count - scalar(@{$prev_pg_per_osd->{$osd} || []})).";\n"; - } - my @sec; - for my $pg (@$pg_names) - { - if ($prev_weights->{$pg}) - { - push @sec, "add_$pg", "del_$pg"; - # Can't add or remove less than zero - $lp .= "add_$pg >= 0;\n"; - $lp .= "del_$pg >= 0;\n"; - # Can't remove more than the PG already has - $lp .= "add_$pg - del_$pg >= -".$prev_weights->{$pg}.";\n"; - } - else - { - push @sec, $pg; - $lp .= "$pg >= 0;\n"; - } - } - $lp .= "int ".join(", ", @sec).";\n"; - # Solve it - my ($score, $result) = lp_solve($lp); - # Generate the new distribution - my $weights = { %$prev_weights }; - for my $k (keys %$result) - { - if ($k =~ /^add_/s) - { - $weights->{substr($k, 4)} += $result->{$k}; - } - elsif ($k =~ /^del_/s) - { - $weights->{substr($k, 4)} -= $result->{$k}; - } - else - { - $weights->{$k} = $result->{$k}; - } - } - for my $k (keys %$weights) - { - delete $weights->{$k} if !$weights->{$k}; - } - my $int_pgs = make_int_pgs($weights, $pg_count); - # Align them with most similar previous PGs - my $new_pgs = align_pgs($prev_int_pgs, $int_pgs); - my $differs = 0; - my $osd_differs = 0; - for my $i (0..$#$new_pgs) - { - if (join('_', @{$new_pgs->[$i]}) ne join('_', @{$prev_int_pgs->[$i]})) - { - $differs++; - } - for my $j (0..2) - { - if ($new_pgs->[$i]->[$j] ne $prev_int_pgs->[$i]->[$j]) - { - $osd_differs++; - } - } - } - return { - prev_pgs => $prev_int_pgs, - score => $score, - weights => $weights, - int_pgs => $new_pgs, - differs => $differs, - osd_differs => $osd_differs, - space => pg_list_space_efficiency($new_pgs, $osd_tree), - total_space => $total_weight, - }; -} - -sub print_change_stats -{ - my ($retval) = @_; - my $new_pgs = $retval->{int_pgs}; - if ($retval->{prev_pgs}) - { - my $prev_int_pgs = $retval->{prev_pgs}; - for my $i (0..$#$new_pgs) - { - if (join('_', @{$new_pgs->[$i]}) ne join('_', @{$prev_int_pgs->[$i]})) - { - print "pg $i: ".join(' ', @{$prev_int_pgs->[$i]})." -> ".join(' ', @{$new_pgs->[$i]})."\n"; - } - } - printf("Data movement: ".$retval->{differs}." pgs, ".$retval->{osd_differs}." pg-osds = %.2f %%\n", $retval->{osd_differs} / @$prev_int_pgs / 3 * 100); - } - printf("Total space: %.2f TB, space efficiency: %.2f %%\n", $retval->{space} * 3, $retval->{space} * 3 / $retval->{total_space} * 100); -} - -sub align_pgs -{ - my ($prev_int_pgs, $int_pgs) = @_; - my $aligned_pgs = []; - put_aligned_pgs($aligned_pgs, $int_pgs, $prev_int_pgs, sub - { - my ($pg) = @_; - return (join(':', @$pg)); - }); - put_aligned_pgs($aligned_pgs, $int_pgs, $prev_int_pgs, sub - { - my ($pg) = @_; - return ($pg->[0].'::'.$pg->[2], ':'.$pg->[1].':'.$pg->[2], $pg->[0].':'.$pg->[1].':'); - }); - put_aligned_pgs($aligned_pgs, $int_pgs, $prev_int_pgs, sub - { - my ($pg) = @_; - return ($pg->[0].'::', ':'.$pg->[1].':', '::'.$pg->[2]); - }); - my $free_slots = [ grep { !$aligned_pgs->[$_] } 0..$#$prev_int_pgs ]; - for my $pg (@$int_pgs) - { - die "Can't place unaligned PG" if !@$free_slots; - $aligned_pgs->[shift @$free_slots] = $pg; - } - return $aligned_pgs; -} - -sub put_aligned_pgs -{ - my ($aligned_pgs, $int_pgs, $prev_int_pgs, $keygen) = @_; - my $prev_indexes = {}; - for (my $i = 0; $i < @$prev_int_pgs; $i++) - { - for my $k ($keygen->($prev_int_pgs->[$i])) - { - push @{$prev_indexes->{$k}}, $i; - } - } - PG: for (my $i = $#$int_pgs; $i >= 0; $i--) - { - my $pg = $int_pgs->[$i]; - my @keys = $keygen->($int_pgs->[$i]); - for my $k (@keys) - { - while (@{$prev_indexes->{$k} || []}) - { - my $idx = shift @{$prev_indexes->{$k}}; - if (!$aligned_pgs->[$idx]) - { - $aligned_pgs->[$idx] = $pg; - splice @$int_pgs, $i, 1; - next PG; - } - } - } - } -} - -sub all_combinations -{ - my ($osd_tree, $count, $ordered) = @_; - my $hosts = [ sort keys %$osd_tree ]; - my $osds = { map { $_ => [ sort keys %{$osd_tree->{$_}} ] } keys %$osd_tree }; - my $h = [ 0, 1, 2 ]; - my $o = [ 0, 0, 0 ]; - my $r = []; - while (!$count || $count < 0 || @$r < $count) - { - my $inc; - if ($h->[2] != $h->[1] && $h->[2] != $h->[0] && $h->[1] != $h->[0]) - { - push @$r, [ map { $osds->{$hosts->[$h->[$_]]}->[$o->[$_]] } 0..$#$h ]; - $inc = 2; - while ($inc >= 0) - { - $o->[$inc]++; - if ($o->[$inc] >= scalar @{$osds->{$hosts->[$h->[$inc]]}}) - { - $o->[$inc] = 0; - $inc--; - } - else - { - last; - } - } - } - else - { - $inc = -1; - } - if ($inc < 0) - { - # no osds left in current host combination, select next one - $o = [ 0, 0, 0 ]; - $h->[2]++; - if ($h->[2] >= scalar @$hosts) - { - $h->[1]++; - $h->[2] = $ordered ? $h->[1]+1 : 0; - if (($ordered ? $h->[2] : $h->[1]) >= scalar @$hosts) - { - $h->[0]++; - $h->[1] = $ordered ? $h->[0]+1 : 0; - $h->[2] = $ordered ? $h->[1]+1 : 0; - if (($ordered ? $h->[2] : $h->[0]) >= scalar @$hosts) - { - last; - } - } - } - } - } - return $r; -} - -sub pg_weights_space_efficiency -{ - my ($weights, $pg_count, $osd_tree) = @_; - my $per_osd = {}; - for my $pg_name (keys %$weights) - { - for my $osd (split /_/, substr($pg_name, 3)) - { - $per_osd->{$osd}++; - } - } - return pg_per_osd_space_efficiency($per_osd, $pg_count, $osd_tree); -} - -sub pg_list_space_efficiency -{ - my ($pgs, $osd_tree) = @_; - my $per_osd = {}; - for my $pg (@$pgs) - { - for my $osd (@$pg) - { - $per_osd->{$osd}++; - } - } - return pg_per_osd_space_efficiency($per_osd, scalar @$pgs, $osd_tree); -} - -sub pg_per_osd_space_efficiency -{ - my ($per_osd, $pg_count, $osd_tree) = @_; - my $all_weights = { map { %$_ } values %$osd_tree }; - # each PG gets randomly selected in 1/N cases - # => there are x PGs per OSD - # => an OSD is selected in x/N cases - # => total space * x/N <= OSD weight - # => total space <= OSD weight * N/x - my $space = undef; - for my $osd (keys %$per_osd) - { - my $space_estimate = $all_weights->{$osd} * $pg_count / $per_osd->{$osd}; - if (!defined $space || $space > $space_estimate) - { - $space = $space_estimate; - } - } - return $space; -} - -1; -__END__ diff --git a/lp/lp-optimizer.js b/lp/lp-optimizer.js index 69d35ab6..6aa9e863 100644 --- a/lp/lp-optimizer.js +++ b/lp/lp-optimizer.js @@ -15,6 +15,10 @@ async function lp_solve(text) { await new Promise(ok => finish_cb = ok); } + if (!stdout.trim()) + { + return null; + } let score = 0; let vars = {}; for (const line of stdout.split(/\n/)) @@ -28,7 +32,7 @@ async function lp_solve(text) } continue; } - else if (/This problem is infeasible/.exec(line)) + else if (/This problem is (infeasible|unbounded)/.exec(line)) { return null; } @@ -58,11 +62,17 @@ function make_single(osd_tree) ]; } -async function optimize_initial(osd_tree, pg_count) +async function optimize_initial(osd_tree, pg_count, max_combinations) { + max_combinations = max_combinations || 10000; const all_weights = Object.assign({}, ...Object.values(osd_tree)); const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c)); - const all_pgs = all_combinations(osd_tree, null, true); + let all_pgs = all_combinations(osd_tree, null, true); + if (all_pgs.length > max_combinations) + { + const prob = max_combinations/all_pgs.length; + all_pgs = all_pgs.filter(pg => Math.random() < prob); + } const pg_per_osd = {}; for (const pg of all_pgs) { @@ -76,14 +86,14 @@ async function optimize_initial(osd_tree, pg_count) lp += "max: "+all_pgs.map(pg => 'pg_'+pg.join('_')).join(' + ')+";\n"; for (const osd in pg_per_osd) { - const osd_pg_count = all_weights[osd]*3/total_weight*pg_count; - lp += pg_per_osd[osd].join(' + ')+' <= '+Math.round(osd_pg_count)+';\n'; + let osd_pg_count = all_weights[osd]*3/total_weight*pg_count; + lp += pg_per_osd[osd].join(' + ')+' <= '+osd_pg_count+';\n'; } for (const pg of all_pgs) { lp += 'pg_'+pg.join('_')+" >= 0;\n"; } - lp += "int "+all_pgs.map(pg => 'pg_'+pg.join('_')).join(', ')+";\n"; + lp += "sec "+all_pgs.map(pg => 'pg_'+pg.join('_')).join(', ')+";\n"; const lp_result = await lp_solve(lp); const int_pgs = make_int_pgs(lp_result.vars, pg_count); const eff = pg_list_space_efficiency(int_pgs, all_weights); @@ -129,8 +139,9 @@ function get_int_pg_weights(prev_int_pgs, osd_tree) } // Try to minimize data movement -async function optimize_change(prev_int_pgs, osd_tree) +async function optimize_change(prev_int_pgs, osd_tree, max_combinations) { + max_combinations = max_combinations || 10000; const pg_count = prev_int_pgs.length; const prev_weights = {}; const prev_pg_per_osd = {}; @@ -145,7 +156,21 @@ async function optimize_change(prev_int_pgs, osd_tree) } } // Get all combinations - const all_pgs = all_combinations(osd_tree); + let all_pgs = all_combinations(osd_tree, null, true); + if (all_pgs.length > max_combinations) + { + const intersecting = all_pgs.filter(pg => prev_weights['pg_'+pg.join('_')]); + if (intersecting.length > max_combinations) + { + const prob = max_combinations/intersecting.length; + all_pgs = intersecting.filter(pg => Math.random() < prob); + } + else + { + const prob = (max_combinations-intersecting.length)/all_pgs.length; + all_pgs = all_pgs.filter(pg => Math.random() < prob || prev_weights['pg_'+pg.join('_')]); + } + } const pg_per_osd = {}; for (const pg of all_pgs) { @@ -178,13 +203,15 @@ async function optimize_change(prev_int_pgs, osd_tree) const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c)); // Generate the LP problem let lp = ''; - lp += 'min: '+all_pg_names.map(pg_name => move_weights[pg_name] + ' * ' + (prev_weights[pg_name] ? 'add_' : '') + pg_name).join(' + ')+';\n'; - lp += all_pg_names.map(pg_name => prev_weights[pg_name] ? `add_${pg_name} - del_${pg_name}` : pg_name).join(' + ')+' = 0;\n'; + lp += 'max: '+all_pg_names.map(pg_name => ( + prev_weights[pg_name] ? `${4-move_weights[pg_name]}*add_${pg_name} - 4*del_${pg_name}` : `${4-move_weights[pg_name]}*${pg_name}` + )).join(' + ')+';\n'; for (const osd in pg_per_osd) { const osd_sum = (pg_per_osd[osd]||[]).map(pg_name => prev_weights[pg_name] ? `add_${pg_name} - del_${pg_name}` : pg_name).join(' + '); - const osd_pg_count = all_weights[osd]*3/total_weight*pg_count - (prev_pg_per_osd[osd]||[]).length; - lp += osd_sum + ' <= ' + Math.round(osd_pg_count) + ';\n'; + const rm_osd_pg_count = (prev_pg_per_osd[osd]||[]).filter(old_pg_name => move_weights[old_pg_name]).length; + let osd_pg_count = all_weights[osd]*3/total_weight*pg_count - rm_osd_pg_count; + lp += osd_sum + ' <= ' + osd_pg_count + ';\n'; } let pg_vars = []; for (const pg_name of all_pg_names) @@ -204,16 +231,23 @@ async function optimize_change(prev_int_pgs, osd_tree) lp += `${pg_name} >= 0;\n`; } } - lp += 'int '+pg_vars.join(', ')+';\n'; + lp += 'sec '+pg_vars.join(', ')+';\n'; // Solve it const lp_result = await lp_solve(lp); if (!lp_result) { console.log(lp); - throw new Error('Problem is infeasible - is it a bug?'); + throw new Error('Problem is infeasible or unbounded - is it a bug?'); } // Generate the new distribution const weights = { ...prev_weights }; + for (const k in prev_weights) + { + if (!move_weights[k]) + { + delete weights[k]; + } + } for (const k in lp_result.vars) { if (k.substr(0, 4) === 'add_') @@ -266,17 +300,20 @@ async function optimize_change(prev_int_pgs, osd_tree) }; } -function print_change_stats(retval) +function print_change_stats(retval, detailed) { const new_pgs = retval.int_pgs; const prev_int_pgs = retval.prev_pgs; if (prev_int_pgs) { - for (let i = 0; i < new_pgs.length; i++) + if (detailed) { - if (new_pgs[i].join('_') != prev_int_pgs[i].join('_')) + for (let i = 0; i < new_pgs.length; i++) { - console.log("pg "+i+": "+prev_int_pgs[i].join(' ')+" -> "+new_pgs[i].join(' ')); + if (new_pgs[i].join('_') != prev_int_pgs[i].join('_')) + { + console.log("pg "+i+": "+prev_int_pgs[i].join(' ')+" -> "+new_pgs[i].join(' ')); + } } } console.log( @@ -285,7 +322,7 @@ function print_change_stats(retval) ); } console.log( - "Total space: "+Math.round(retval.space*3*100)/100+" TB, space efficiency: "+ + "Total space (raw): "+Math.round(retval.space*3*100)/100+" TB, space efficiency: "+ Math.round(retval.space*3/retval.total_space*10000)/100+" %" ); } diff --git a/lp/test-optimize.js b/lp/test-optimize.js index 523b2bf9..4c8e9046 100644 --- a/lp/test-optimize.js +++ b/lp/test-optimize.js @@ -23,17 +23,37 @@ const osd_tree = { 5: 3.63869, 6: 3.63869, }, +/* 100: { + 1: 2.72800, + }, + 200: { + 2: 2.72900, + }, + 300: { + 3: 1.87000, + }, + 400: { + 4: 1.87000, + }, + 500: { + 5: 3.63869, + },*/ }; async function run() { - // Test: add 1 OSD of almost the same size. Ideal data movement could be 1/12 = 8.33%. Actual is ~11.72% + // Test: add 1 OSD of almost the same size. Ideal data movement could be 1/12 = 8.33%. Actual is ~13% // Space efficiency is ~99.5% in both cases. let prev = await LPOptimizer.optimize_initial(osd_tree, 256); - LPOptimizer.print_change_stats(prev); + LPOptimizer.print_change_stats(prev, false); + console.log('adding osd.8'); osd_tree[500][8] = 3.58589; let next = await LPOptimizer.optimize_change(prev.int_pgs, osd_tree); - LPOptimizer.print_change_stats(next); + LPOptimizer.print_change_stats(next, false); + console.log('removing osd.8'); + delete osd_tree[500][8]; + next = await LPOptimizer.optimize_change(next.int_pgs, osd_tree); + LPOptimizer.print_change_stats(next, false); } run().catch(console.error); diff --git a/lp/test-optimize.pl b/lp/test-optimize.pl deleted file mode 100644 index d4dcdb29..00000000 --- a/lp/test-optimize.pl +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/perl - -use strict; -use lib qw(.); -use LPOptimizer; - -my $osd_tree = { - 100 => { - 7 => 3.63869, - }, - 200 => { - 5 => 3.63869, - 6 => 3.63869, - }, - 300 => { - 10 => 3.46089, - 11 => 3.46089, - 12 => 3.46089, - }, - 400 => { - 1 => 3.49309, - 2 => 3.49309, - 3 => 3.49309, - }, - 500 => { - 4 => 3.58498, -# 8 => 3.58589, - 9 => 3.63869, - }, -}; - -my $prev = LPOptimizer::optimize_initial($osd_tree, 256); -LPOptimizer::print_change_stats($prev); -$osd_tree->{500}->{8} = 3.58589; -my $next = LPOptimizer::optimize_change($prev->{int_pgs}, $osd_tree); -LPOptimizer::print_change_stats($next);