WIP: [DNM] osd: run calc_object_states off the OSD thread #123

Draft
lei.xianqiang wants to merge 1 commits from lei.xianqiang/vitastor:calc_object_states_in_worker_threads into master
10 changed files with 1391 additions and 25 deletions

View File

@@ -13,6 +13,7 @@ with an OSD restart or, for some of them, even without restarting by updating co
- [bind_address](#bind_address)
- [bind_port](#bind_port)
- [osd_iothread_count](#osd_iothread_count)
- [pg_calc_thread_count](#pg_calc_thread_count)
- [etcd_report_interval](#etcd_report_interval)
- [etcd_stats_interval](#etcd_stats_interval)
- [run_primary](#run_primary)
@@ -101,6 +102,20 @@ switching overhead occurs. RDMA isn't affected by this option.
Because of latency, instead of enabling OSD I/O threads it's recommended to
just create multiple OSDs per disk, or use RDMA.
## pg_calc_thread_count
- Type: integer
- Default: 1
Number of threads that run PG peering's object-state calculation
(`calc_object_states`). A large PG used to stall `OSD_OP_PING` inside the
OSD loop until peers declared the OSD dead. The pool keeps that work off
the OSD thread. Results are installed on the OSD thread before the next
peering step.
0 does not create the pool. The calculation then runs on the OSD thread,
as it did before the pool existed. The value is read when the OSD starts.
## etcd_report_interval
- Type: seconds

View File

@@ -14,6 +14,7 @@
- [bind_address](#bind_address)
- [bind_port](#bind_port)
- [osd_iothread_count](#osd_iothread_count)
- [pg_calc_thread_count](#pg_calc_thread_count)
- [etcd_report_interval](#etcd_report_interval)
- [etcd_stats_interval](#etcd_stats_interval)
- [run_primary](#run_primary)
@@ -103,6 +104,20 @@
Из-за задержек вместо включения потоков ввода-вывода OSD рекомендуется
просто создавать по несколько OSD на каждом диске, или использовать RDMA.
## pg_calc_thread_count
- Тип: целое число
- Значение по умолчанию: 1
Число потоков, в которых OSD считает состояние объектов PG при peering
(`calc_object_states`). Большая PG раньше блокировала `OSD_OP_PING` в
главном цикле, и соседи объявляли OSD упавшим. Пул выносит этот расчёт
из главного потока. Результат устанавливается в главном потоке до
следующего шага peering.
0 не создаёт пул: расчёт выполняется в главном потоке OSD, как до
появления пула. Значение читается при старте OSD.
## etcd_report_interval
- Тип: секунды

View File

@@ -1,12 +1,18 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#include <stdexcept>
#include <stdio.h>
#include <stdlib.h>
#include <sys/socket.h>
#include <sys/poll.h>
#include <sys/eventfd.h>
#include <sys/mman.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
#include <arpa/inet.h>
#include <pthread.h>
#include <errno.h>
#include "addr_util.h"
#include "osd_primary.h"
@@ -83,6 +89,7 @@ void osd_t::start()
consumer.loop = [this]() { loop(); };
ringloop->register_consumer(&consumer);
start_pg_calc_worker();
}
osd_t::~osd_t()
@@ -116,6 +123,10 @@ osd_t::~osd_t()
// clients directly rather than through stop_client(): the stop protocol would
// fire repeer and PG-report hooks on the way out of a dying OSD
in_destruction = true;
// Join before clients die. Queued calcs are dropped without running; only the
// job already inside calc_object_states is finished, then its result is dropped
// before pgs (and their peering state) go away.
stop_pg_calc_worker();
while (!msgr.clients.empty())
msgr.destroy_client(msgr.clients.begin()->second);
if (bs)
@@ -351,6 +362,11 @@ void osd_t::parse_config(bool init)
this->disable_tcp = this->use_rdmacm && (config["disable_tcp"].bool_value() || config["disable_tcp"].uint64_value() != 0);
#endif
// OSD configuration
// 0 keeps calc_object_states on the OSD thread. A missing key is the default.
if (config["pg_calc_thread_count"].is_null())
pg_calc_thread_count = 1;
else
pg_calc_thread_count = (uint32_t)config["pg_calc_thread_count"].uint64_value();
etcd_report_interval = config["etcd_report_interval"].uint64_value();
if (etcd_report_interval <= 0)
etcd_report_interval = 5;
@@ -601,6 +617,9 @@ bool osd_t::shutdown()
void osd_t::loop()
{
// Apply calc_object_states results before peering so a woken loop can
// report PG state and still service pings in this iteration
drain_pg_calc();
handle_peers();
msgr.read_requests();
msgr.send_replies();
@@ -871,3 +890,259 @@ void osd_t::print_slow()
bs->dump_diagnostics();
}
}
static pg_calc_job_t *take_pg_calc_job(std::deque<pg_calc_job_t*> & q, pool_id_t pool_id, pg_num_t pg_num)
{
for (auto it = q.begin(); it != q.end(); ++it)
{
if ((*it)->pool_id == pool_id && (*it)->pg_num == pg_num)
{
pg_calc_job_t *job = *it;
q.erase(it);
return job;
}
}
return NULL;
}
void discard_pg_calc_job(pg_calc_job_t *job)
{
if (job->scratch)
{
pg_peering_state_t *ps = job->scratch->peering_state;
if (ps)
{
// LIST callbacks were cleared before the job was built
assert(ps->list_ops.empty());
for (auto & lr: ps->list_results)
if (lr.second.buf)
free(lr.second.buf);
delete ps;
job->scratch->peering_state = NULL;
}
delete job->scratch;
}
delete job;
}
pg_calc_worker_t::pg_calc_worker_t(int thread_count, bool hold, bool pause_before_calc, int *calc_runs):
hold(hold), pause_before_calc(pause_before_calc), calc_runs(calc_runs)
{
assert(thread_count > 0);
eventfd = ::eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
if (eventfd < 0)
throw std::runtime_error(std::string("eventfd: ") + strerror(errno));
try
{
threads.reserve(thread_count);
for (int i = 0; i < thread_count; i++)
threads.emplace_back(&pg_calc_worker_t::run, this);
}
catch (...)
{
stop();
throw;
}
}
pg_calc_worker_t::~pg_calc_worker_t()
{
stop();
}
void pg_calc_worker_t::push(pg_calc_job_t *job)
{
std::unique_lock<std::mutex> lk(mu);
queue.push_back(job);
// Wake every idle worker when the queue becomes non-empty. A later push
// in the same burst is seen by threads already leaving wait.
if (queue.size() == 1)
cond.notify_all();
}
pg_calc_job_t *pg_calc_worker_t::cancel(pool_id_t pool_id, pg_num_t pg_num)
{
std::unique_lock<std::mutex> lk(mu);
pg_calc_job_t *dropped = take_pg_calc_job(queue, pool_id, pg_num);
if (!dropped)
dropped = take_pg_calc_job(done, pool_id, pg_num);
return dropped;
}
std::deque<pg_calc_job_t*> pg_calc_worker_t::drain()
{
std::unique_lock<std::mutex> lk(mu);
std::deque<pg_calc_job_t*> jobs;
jobs.swap(done);
return jobs;
}
void pg_calc_worker_t::wait_not_busy()
{
std::unique_lock<std::mutex> lk(mu);
while (busy || queue.size())
cond.wait(lk);
}
bool pg_calc_worker_t::idle()
{
std::unique_lock<std::mutex> lk(mu);
return !busy && !queue.size() && !done.size();
}
void pg_calc_worker_t::release_hold()
{
std::unique_lock<std::mutex> lk(mu);
hold = false;
cond.notify_all();
}
void pg_calc_worker_t::unpause()
{
std::unique_lock<std::mutex> lk(mu);
pause_before_calc = false;
cond.notify_all();
}
void pg_calc_worker_t::wait_paused()
{
std::unique_lock<std::mutex> lk(mu);
while (!paused && !stopped)
cond.wait(lk);
}
size_t pg_calc_worker_t::queued()
{
std::unique_lock<std::mutex> lk(mu);
return queue.size();
}
void pg_calc_worker_t::stop()
{
std::deque<pg_calc_job_t*> dropped;
{
std::unique_lock<std::mutex> lk(mu);
if (stopped)
return;
stopped = true;
if (outer_loop_data)
{
outer_loop_data->callback = [](ring_data_t*){};
outer_loop_data = NULL;
}
// Drop every job that has not been popped. Join only waits for jobs
// already inside calc_object_states (or parked on pause_before_calc).
dropped.swap(queue);
cond.notify_all();
}
for (auto *job: dropped)
discard_pg_calc_job(job);
for (auto & th: threads)
if (th.joinable())
th.join();
{
std::unique_lock<std::mutex> lk(mu);
for (auto *job: done)
discard_pg_calc_job(job);
done.clear();
}
// Close after join. The worker writes this fd when a calculation finishes;
// msgr_iothread_t can close earlier because only the kernel writes its eventfd.
if (eventfd >= 0)
{
close(eventfd);
eventfd = -1;
}
}
void pg_calc_worker_t::add_to_ringloop(ring_loop_i *outer_loop)
{
assert(!this->outer_loop || this->outer_loop == outer_loop);
io_uring_sqe *sqe = outer_loop->get_sqe();
assert(sqe != NULL);
this->outer_loop = outer_loop;
this->outer_loop_data = ((ring_data_t*)sqe->user_data);
io_uring_prep_poll_add(sqe, eventfd, POLLIN);
outer_loop_data->callback = [this](ring_data_t *data)
{
if (data->res < 0)
{
throw std::runtime_error(std::string("eventfd poll failed: ") + strerror(-data->res));
}
outer_loop_data = NULL;
if (stopped)
{
return;
}
// Drain the counter. Results are applied from osd_t::loop, before
// peering, so the install stays on the main thread.
for (;;)
{
uint64_t ctr = 0;
ssize_t r = read(eventfd, &ctr, sizeof(ctr));
if (r < 0 && errno == EINTR)
continue;
break;
}
add_to_ringloop(this->outer_loop);
this->outer_loop->wakeup();
};
}
void pg_calc_worker_t::run()
{
pthread_setname_np(pthread_self(), "pg-calc");
while (true)
{
pg_calc_job_t *job = NULL;
{
std::unique_lock<std::mutex> lk(mu);
// stop abandons the queue. hold keeps jobs queued so cancel can drop them.
while (!stopped && (hold || !queue.size()))
cond.wait(lk);
if (stopped)
return;
job = queue.front();
queue.pop_front();
busy++;
if (pause_before_calc)
{
paused++;
cond.notify_all();
while (pause_before_calc && !stopped)
cond.wait(lk);
paused--;
if (stopped)
{
// Not inside calc_object_states yet: hand the job back for discard.
busy--;
done.push_back(job);
cond.notify_all();
return;
}
}
if (calc_runs)
(*calc_runs)++;
}
timespec t0, t1;
clock_gettime(CLOCK_MONOTONIC, &t0);
job->scratch->calc_object_states(job->log_level);
clock_gettime(CLOCK_MONOTONIC, &t1);
long ms = (long)((t1.tv_sec - t0.tv_sec) * 1000 + (t1.tv_nsec - t0.tv_nsec) / 1000000);
if (ms >= 100)
{
printf("[PG %u/%u] calc_object_states took %ld ms\n",
job->scratch->pool_id, job->scratch->pg_num, ms);
}
{
std::unique_lock<std::mutex> lk(mu);
busy--;
done.push_back(job);
cond.notify_all();
}
uint64_t one = 1;
ssize_t r = write(eventfd, &one, sizeof(one));
if (r < 0 && errno != EAGAIN)
fprintf(stderr, "pg calc eventfd write: %s\n", strerror(errno));
}
}

View File

@@ -15,6 +15,10 @@
#include <set>
#include <deque>
#include <vector>
#include <mutex>
#include <condition_variable>
#include <thread>
#include "blockstore.h"
#include "ringloop.h"
@@ -31,6 +35,66 @@
#define SELF_CLIENT 0
// One calc_object_states request. The worker owns scratch until the main
// thread accepts or discards the job.
struct pg_calc_job_t
{
pool_id_t pool_id = 0;
pg_num_t pg_num = 0;
uint64_t gen = 0;
int log_level = 0;
pg_t *scratch = NULL;
};
void discard_pg_calc_job(pg_calc_job_t *job);
// Thread pool for calc_object_states. Same shape as msgr_iothread_t: the
// constructor starts the threads, stop() joins them, and add_to_ringloop()
// arms a poll on eventfd so the OSD loop can drain results. Workers share
// one queue. Jobs still in the queue are dropped on stop; a job already
// popped runs to completion and its result is left in done for the caller
// to discard.
class pg_calc_worker_t
{
protected:
int eventfd = -1;
bool stopped = false;
// threads that have popped a job and not yet pushed it onto done
int busy = 0;
// threads waiting out pause_before_calc
int paused = 0;
// test gates, copied at start and only changed under mu
bool hold = false;
bool pause_before_calc = false;
int *calc_runs = NULL;
ring_loop_i *outer_loop = NULL;
ring_data_t *outer_loop_data = NULL;
std::mutex mu;
std::condition_variable cond;
std::deque<pg_calc_job_t*> queue;
std::deque<pg_calc_job_t*> done;
std::vector<std::thread> threads;
void run();
public:
pg_calc_worker_t(int thread_count, bool hold, bool pause_before_calc, int *calc_runs);
~pg_calc_worker_t();
void push(pg_calc_job_t *job);
// Drop a job that has not entered calc_object_states. NULL if the worker
// has already popped it.
pg_calc_job_t *cancel(pool_id_t pool_id, pg_num_t pg_num);
std::deque<pg_calc_job_t*> drain();
void wait_not_busy();
bool idle();
void release_hold();
void unpause();
void wait_paused();
size_t queued();
void stop();
void add_to_ringloop(ring_loop_i *outer_loop);
};
#define MAX_AUTOSYNC_INTERVAL 3600
#define DEFAULT_AUTOSYNC_INTERVAL 5
#define DEFAULT_AUTOSYNC_WRITES 128
@@ -196,6 +260,20 @@ class osd_t
int copies_to_delete_after_sync_count = 0;
uint64_t misplaced_objects = 0, degraded_objects = 0, incomplete_objects = 0, inconsistent_objects = 0, corrupted_objects = 0;
int peering_state = 0;
// Thread pool for calc_object_states (pg_calc_worker_t). pg_calc_thread_count
// is the pool size; 0 keeps the calculation on the OSD thread. Tests set
// pg_calc_sync so the calculation stays on the main thread: the mock event
// loop does not complete the eventfd poll. Must be set before start().
// pg_calc_hold: the worker does not pop, so a test can cancel a job that
// has not started. pg_calc_pause: after pop, stop before calc_object_states
// so a test can observe an in-flight job. Both must be set before start().
// When pg_calc_runs is set, it is incremented each time a calculation starts.
pg_calc_worker_t *pg_calc_worker = NULL;
int pg_calc_thread_count = 1;
bool pg_calc_sync = false;
bool pg_calc_hold = false;
bool pg_calc_pause = false;
int *pg_calc_runs = NULL;
std::map<object_id, osd_recovery_op_t> recovery_ops;
// OSD-owned in-flight ops (flush batches, peering LIST subops): their only
// owners are completion callbacks, so the destructor needs a handle on them
@@ -326,6 +404,21 @@ class osd_t
void rm_inflight(pg_t & pg);
void continue_pg(pg_t & pg);
bool continue_pg_peering(pg_t & pg);
void start_pg_calc_worker();
void stop_pg_calc_worker();
void drain_pg_calc();
void wait_pg_calc();
// Block until workers have finished. Results stay in the worker's done
// queue; the caller installs them with drain_pg_calc / wait_pg_calc.
void wait_pg_calc_not_busy();
void pg_calc_release_hold();
void pg_calc_unpause();
void wait_pg_calc_paused();
size_t pg_calc_queued();
void cancel_pg_calc(pg_t & pg);
void submit_pg_object_calc(pg_t & pg);
void accept_pg_calc(pg_calc_job_t *job);
void complete_pg_peering(pg_t & pg);
// flushing, recovery and backfill
void submit_pg_flush_ops(pg_t & pg);

View File

@@ -199,6 +199,8 @@ void osd_t::drop_dirty_pg_connections(pool_pg_num_t pg)
// Repeer on each connect/disconnect peer event
void osd_t::start_pg_peering(pg_t & pg)
{
// An in-flight calc still owns the previous peering_state. Drop its result.
cancel_pg_calc(pg);
pg.state = PG_PEERING;
this->peering_state |= OSD_PEERING_PGS;
reset_pg(pg);
@@ -335,6 +337,9 @@ bool osd_t::continue_pg_peering(pg_t & pg)
// Teardown: cancelled peering subops must not resume the peering machine
return false;
}
// peering_state is null while calc_object_states owns it on the worker thread
if (pg.calc_running || !pg.peering_state)
return false;
if (pg.peering_state->locked)
{
pg.peering_state->lists_done = true;
@@ -354,27 +359,11 @@ bool osd_t::continue_pg_peering(pg_t & pg)
}
if (pg.peering_state->lists_done)
{
pg.calc_object_states(log_level);
report_pg_state(pg);
schedule_scrub(pg);
inconsistent_objects += pg.inconsistent_objects.size();
incomplete_objects += pg.incomplete_objects.size();
misplaced_objects += pg.misplaced_objects.size();
// FIXME: degraded objects may currently include misplaced, too! Report them separately?
degraded_objects += pg.degraded_objects.size();
if (pg.state & PG_HAS_UNCLEAN)
this->peering_state = peering_state | OSD_FLUSHING_PGS;
else if (pg.state & (PG_HAS_DEGRADED | PG_HAS_MISPLACED))
{
this->peering_state = peering_state | OSD_RECOVERING;
if (pg.state & PG_HAS_DEGRADED)
{
// Restart recovery from degraded objects
this->recovery_last_degraded = true;
this->recovery_last_pg = {};
this->recovery_last_oid = {};
}
}
// Sort + walk of every object version. Run it off the OSD thread so a
// large PG cannot stall OSD_OP_PING past osd_ping_timeout.
submit_pg_object_calc(pg);
if (pg.calc_running)
return false;
return true;
}
return false;
@@ -392,7 +381,7 @@ void osd_t::relock_pg(pg_t & pg)
{
if (!enable_pg_locks || pg.disable_pg_locks && !pg.lock_peers.size())
{
if (pg.state & PG_PEERING)
if ((pg.state & PG_PEERING) && pg.peering_state)
pg.peering_state->locked = true;
continue_pg(pg);
return;
@@ -431,7 +420,7 @@ void osd_t::relock_pg(pg_t & pg)
}
if (!diff_osds.size())
{
if (pg.state & PG_PEERING)
if ((pg.state & PG_PEERING) && pg.peering_state)
pg.peering_state->locked = true;
continue_pg(pg);
return;
@@ -519,7 +508,9 @@ void osd_t::relock_pg(pg_t & pg)
};
msgr.outbox_push(op);
}
if (pg.state & PG_PEERING)
// calc_object_states owns peering_state on the worker. State can still be
// PG_PEERING in that window; don't touch the pointer until it is back.
if ((pg.state & PG_PEERING) && pg.peering_state)
{
pg.peering_state->locked = !diff_osds.size();
}
@@ -682,6 +673,7 @@ void osd_t::release_peering_state(pg_t & pg)
bool osd_t::stop_pg(pg_t & pg)
{
cancel_pg_calc(pg);
drop_dirty_pg_connections({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
release_peering_state(pg);
if (pg.state & (PG_STOPPING | PG_OFFLINE))
@@ -798,8 +790,225 @@ void osd_t::continue_pg(pg_t & pg)
{
start_pg_peering(pg);
}
else if ((pg.state & PG_PEERING) && pg.peering_state->locked)
else if ((pg.state & PG_PEERING) && pg.peering_state && pg.peering_state->locked)
{
continue_pg_peering(pg);
}
}
void osd_t::cancel_pg_calc(pg_t & pg)
{
if (!pg.calc_running)
return;
// Anything not yet inside calc_object_states is dropped here, so a repeer
// can LIST again without waiting out a stale sort. The job already popped
// by the worker is left to finish; calc_gen makes accept_pg_calc ignore it.
pg.calc_gen++;
if (!pg_calc_worker)
return;
pg_calc_job_t *dropped = pg_calc_worker->cancel(pg.pool_id, pg.pg_num);
if (!dropped)
return;
discard_pg_calc_job(dropped);
pg.calc_running = false;
pg.calc_inflight_gen = 0;
}
// Copies only what calc_object_states / walk() read. A field walk() starts
// reading later has to be copied here too: the worker never looks at the live
// PG, so a missing copy is a silent wrong peering result.
static pg_t *make_pg_calc_scratch(pg_t & pg)
{
pg_t *s = new pg_t();
s->scheme = pg.scheme;
s->pg_cursize = pg.pg_cursize;
s->pg_size = pg.pg_size;
s->pg_data_size = pg.pg_data_size;
s->pool_id = pg.pool_id;
s->pg_num = pg.pg_num;
s->reported_epoch = pg.reported_epoch;
s->state = pg.state;
s->target_set = pg.target_set;
s->cur_set = pg.cur_set;
s->cur_peers = pg.cur_peers;
s->all_peers = pg.all_peers;
s->peering_state = pg.peering_state;
pg.peering_state = NULL;
return s;
}
void osd_t::submit_pg_object_calc(pg_t & pg)
{
assert(!pg.calc_running);
assert(pg.peering_state);
pg.calc_gen++;
pg.calc_inflight_gen = pg.calc_gen;
pg.calc_running = true;
pg_calc_job_t *job = new pg_calc_job_t;
job->pool_id = pg.pool_id;
job->pg_num = pg.pg_num;
job->gen = pg.calc_gen;
job->log_level = log_level;
job->scratch = make_pg_calc_scratch(pg);
// Thread count 0, tests, and a missing worker run inline on this thread.
// A live pool returns the OSD loop to read_requests / OSD_OP_PING.
if (pg_calc_sync || !pg_calc_worker)
{
job->scratch->calc_object_states(job->log_level);
accept_pg_calc(job);
return;
}
pg_calc_worker->push(job);
}
void osd_t::complete_pg_peering(pg_t & pg)
{
report_pg_state(pg);
schedule_scrub(pg);
inconsistent_objects += pg.inconsistent_objects.size();
incomplete_objects += pg.incomplete_objects.size();
misplaced_objects += pg.misplaced_objects.size();
// FIXME: degraded objects may currently include misplaced, too! Report them separately?
degraded_objects += pg.degraded_objects.size();
if (pg.state & PG_HAS_UNCLEAN)
this->peering_state = peering_state | OSD_FLUSHING_PGS;
else if (pg.state & (PG_HAS_DEGRADED | PG_HAS_MISPLACED))
{
this->peering_state = peering_state | OSD_RECOVERING;
if (pg.state & PG_HAS_DEGRADED)
{
// Restart recovery from degraded objects
this->recovery_last_degraded = true;
this->recovery_last_pg = {};
this->recovery_last_oid = {};
}
}
}
void osd_t::accept_pg_calc(pg_calc_job_t *job)
{
auto pg_it = pgs.find({ .pool_id = job->pool_id, .pg_num = job->pg_num });
pg_t *pg = pg_it == pgs.end() ? NULL : &pg_it->second;
// calc_inflight_gen is the job this PG is still waiting on. A repeer bumps
// calc_gen so the result is dropped, but that in-flight job still owns
// calc_running. An older completion must not clear a newer job's flag.
bool waited = pg && pg->calc_running && pg->calc_inflight_gen == job->gen;
bool apply = waited && !in_destruction && pg->calc_gen == job->gen
&& (pg->state & PG_PEERING) && !pg->peering_state;
if (waited)
{
pg->calc_running = false;
pg->calc_inflight_gen = 0;
}
if (!apply)
{
discard_pg_calc_job(job);
if (pg && !in_destruction)
continue_pg(*pg);
return;
}
// reported_epoch may advance on the OSD thread while the worker is
// calculating from an older snapshot. Never let the installed result
// move epoch behind the value already persisted in etcd. Non-clean
// activation must use a new epoch strictly above that persisted value.
pg->epoch = std::max(job->scratch->epoch, pg->reported_epoch);
if (job->scratch->state != PG_ACTIVE && pg->epoch == pg->reported_epoch)
{
assert(pg->epoch != (((uint64_t)1 << PG_EPOCH_BITS)-1));
pg->epoch++;
}
pg->clean_count = job->scratch->clean_count;
pg->total_count = job->scratch->total_count;
pg->state = job->scratch->state;
// Object btrees store pointers into state_dict nodes. std::map::swap keeps
// those addresses. btree_map has a user-defined copy constructor and no move,
// so swap is what keeps the pointers valid.
pg->state_dict.swap(job->scratch->state_dict);
pg->inconsistent_objects.swap(job->scratch->inconsistent_objects);
pg->incomplete_objects.swap(job->scratch->incomplete_objects);
pg->misplaced_objects.swap(job->scratch->misplaced_objects);
pg->degraded_objects.swap(job->scratch->degraded_objects);
pg->flush_actions.swap(job->scratch->flush_actions);
pg->ver_override.swap(job->scratch->ver_override);
delete job->scratch;
job->scratch = NULL;
delete job;
complete_pg_peering(*pg);
}
void osd_t::drain_pg_calc()
{
if (!pg_calc_worker)
return;
for (auto *job: pg_calc_worker->drain())
accept_pg_calc(job);
}
void osd_t::wait_pg_calc()
{
if (!pg_calc_worker)
return;
for (;;)
{
pg_calc_worker->wait_not_busy();
drain_pg_calc();
if (pg_calc_worker->idle())
return;
}
}
void osd_t::wait_pg_calc_not_busy()
{
if (!pg_calc_worker)
return;
pg_calc_worker->wait_not_busy();
}
void osd_t::start_pg_calc_worker()
{
if (pg_calc_sync || pg_calc_thread_count <= 0 || pg_calc_worker)
return;
pg_calc_worker = new pg_calc_worker_t(pg_calc_thread_count, pg_calc_hold, pg_calc_pause, pg_calc_runs);
pg_calc_worker->add_to_ringloop(ringloop);
}
void osd_t::pg_calc_release_hold()
{
pg_calc_hold = false;
if (pg_calc_worker)
pg_calc_worker->release_hold();
}
void osd_t::pg_calc_unpause()
{
pg_calc_pause = false;
if (pg_calc_worker)
pg_calc_worker->unpause();
}
void osd_t::wait_pg_calc_paused()
{
if (!pg_calc_worker)
return;
pg_calc_worker->wait_paused();
}
size_t osd_t::pg_calc_queued()
{
if (!pg_calc_worker)
return 0;
return pg_calc_worker->queued();
}
void osd_t::stop_pg_calc_worker()
{
if (!pg_calc_worker)
return;
delete pg_calc_worker;
pg_calc_worker = NULL;
for (auto & p: pgs)
{
p.second.calc_running = false;
p.second.calc_inflight_gen = 0;
}
}

View File

@@ -413,6 +413,8 @@ retry:
}
// FIXME: Write at least some tests for this function
// May run on the pg-calc thread. Only touches this pg — the OSD hands it a
// scratch copy so the event loop never shares these maps with the worker.
void pg_t::calc_object_states(int log_level)
{
// Copy all object lists into one array

View File

@@ -127,6 +127,18 @@ struct pg_t
btree::btree_map<object_id, uint64_t> ver_override;
pg_peering_state_t *peering_state = NULL;
pg_flush_batch_t *flush_batch = NULL;
// calc_object_states runs on the calc pool so it cannot stall OSD_OP_PING
// in the main loop. While it runs, peering_state lives on a scratch PG and
// the main thread does not touch the object maps.
// stop / repeer bump calc_gen, so a result that is not yet installed is
// dropped. A job still in the queue or in done is discarded and
// calc_running is cleared, and the new peering can LIST. A job already
// popped cannot be stopped: calc_inflight_gen still names it, the result
// is discarded by generation, and only that completion clears its own
// calc_running. It does not clear the flag of a later calculation.
bool calc_running = false;
uint64_t calc_gen = 0;
uint64_t calc_inflight_gen = 0;
int inflight = 0; // including write_queue
std::multimap<object_id, osd_op_t*> write_queue;

View File

@@ -3074,6 +3074,650 @@ void test_pg_epoch_base_is_reported_epoch()
printf("test_pg_epoch_base_is_reported_epoch passed\n");
}
// calc_object_states must not run inside the OSD loop: a large object list used
// to block OSD_OP_PING until peers declared the OSD dead. The worker applies the
// same result on the next drain.
void test_calc_object_states_off_thread()
{
printf("test_calc_object_states_off_thread\n");
osd_test_fixture_t f;
f.configure_replicated_pool(/*pool_id*/ 1, /*pg_size*/ 2, /*pg_minsize*/ 1, /*pg_count*/ 1,
{ { 1, 2 } });
f.construct(json11::Json::object {
{ "osd_num", 1 },
{ "etcd_address", "127.0.0.1:2379" },
{ "immediate_commit", "none" },
{ "block_size", 131072 },
{ "bitmap_granularity", 4096 },
{ "no_recovery", true },
});
f.set_pg_calc_sync(false);
f.osd->start();
f.connect_peer(2);
f.reply_local_list({}, 0);
f.reply_peer_list(2, {}, 0);
f.ringloop->loop();
// The loop only queues the job. Waiting here is what installs PG_ACTIVE.
assert(f.pg(1, 1).state == PG_PEERING);
assert(f.pg(1, 1).calc_running);
f.wait_pg_calc();
assert(f.pg(1, 1).state == PG_ACTIVE);
assert(!f.pg(1, 1).calc_running);
assert(f.pg(1, 1).peering_state == NULL);
assert(f.pg(1, 1).total_count == 0);
printf("test_calc_object_states_off_thread passed\n");
}
static json11::Json::object calc_test_osd_config()
{
return json11::Json::object {
{ "osd_num", 1 },
{ "etcd_address", "127.0.0.1:2379" },
{ "immediate_commit", "none" },
{ "block_size", 131072 },
{ "bitmap_granularity", 4096 },
{ "no_recovery", true },
};
}
static void configure_calc_pool(osd_test_fixture_t & f)
{
f.configure_replicated_pool(/*pool_id*/ 1, /*pg_size*/ 2, /*pg_minsize*/ 1, /*pg_count*/ 1,
{ { 1, 2 } });
}
struct flush_snap_t
{
object_id oid;
osd_num_t osd_num = 0;
bool rollback = false, make_stable = false;
uint64_t rollback_to = 0, stable_to = 0;
};
inline bool operator==(const flush_snap_t & a, const flush_snap_t & b)
{
return a.oid == b.oid && a.osd_num == b.osd_num && a.rollback == b.rollback &&
a.make_stable == b.make_stable && a.rollback_to == b.rollback_to && a.stable_to == b.stable_to;
}
struct pg_calc_snap_t
{
int state = 0;
uint64_t epoch = 0, clean_count = 0, total_count = 0;
std::vector<object_id> degraded, misplaced, incomplete, inconsistent;
std::vector<flush_snap_t> flush;
std::vector<std::pair<object_id, uint64_t>> ver;
std::vector<std::pair<pg_osd_set_t, std::pair<uint64_t, uint64_t>>> dict;
std::vector<std::vector<osd_num_t>> read_targets;
};
static std::vector<object_id> object_keys(btree::btree_map<object_id, pg_osd_set_state_t*> & m)
{
std::vector<object_id> keys;
for (auto & kv: m)
keys.push_back(kv.first);
return keys;
}
static pg_calc_snap_t snap_pg(pg_t & pg)
{
pg_calc_snap_t s;
s.state = pg.state;
s.epoch = pg.epoch;
s.clean_count = pg.clean_count;
s.total_count = pg.total_count;
s.degraded = object_keys(pg.degraded_objects);
s.misplaced = object_keys(pg.misplaced_objects);
s.incomplete = object_keys(pg.incomplete_objects);
s.inconsistent = object_keys(pg.inconsistent_objects);
for (auto & kv: pg.flush_actions)
{
s.flush.push_back((flush_snap_t){
.oid = kv.first.oid,
.osd_num = kv.first.osd_num,
.rollback = kv.second.rollback,
.make_stable = kv.second.make_stable,
.rollback_to = kv.second.rollback_to,
.stable_to = kv.second.stable_to,
});
}
for (auto & kv: pg.ver_override)
s.ver.push_back({ kv.first, kv.second });
for (auto & kv: pg.state_dict)
{
s.dict.push_back({ kv.first, { kv.second.state, kv.second.object_count } });
s.read_targets.push_back(kv.second.read_target);
}
return s;
}
static bool same_osd_set(const pg_osd_set_t & a, const pg_osd_set_t & b)
{
if (a.size() != b.size())
return false;
for (size_t i = 0; i < a.size(); i++)
{
if (a[i].role != b[i].role || a[i].osd_num != b[i].osd_num || a[i].loc_bad != b[i].loc_bad)
return false;
}
return true;
}
static void assert_snaps_equal(const pg_calc_snap_t & a, const pg_calc_snap_t & b)
{
assert(a.state == b.state);
assert(a.epoch == b.epoch);
assert(a.clean_count == b.clean_count);
assert(a.total_count == b.total_count);
assert(a.degraded == b.degraded);
assert(a.misplaced == b.misplaced);
assert(a.incomplete == b.incomplete);
assert(a.inconsistent == b.inconsistent);
assert(a.flush == b.flush);
assert(a.ver == b.ver);
assert(a.dict.size() == b.dict.size());
assert(a.read_targets.size() == b.read_targets.size());
for (size_t i = 0; i < a.dict.size(); i++)
{
assert(same_osd_set(a.dict[i].first, b.dict[i].first));
assert(a.dict[i].second == b.dict[i].second);
assert(a.read_targets[i] == b.read_targets[i]);
}
}
static pg_calc_snap_t peer_one_object(bool async, bool both_copies, int thread_count = -1)
{
osd_test_fixture_t f;
configure_calc_pool(f);
inode_t inode = INODE_WITH_POOL(1, 1);
obj_ver_id obj = { .oid = { .inode = inode, .stripe = 0 }, .version = 1 };
auto cfg = calc_test_osd_config();
if (thread_count >= 0)
cfg["pg_calc_thread_count"] = thread_count;
if (async)
{
f.construct(cfg);
f.set_pg_calc_sync(false);
f.osd->start();
}
else
f.start(cfg);
f.connect_peer(2);
f.reply_local_list({ obj }, 1);
f.reply_peer_list(2, both_copies ? std::vector<obj_ver_id>{ obj } : std::vector<obj_ver_id>{}, 1);
f.ringloop->loop();
f.wait_pg_calc();
return snap_pg(f.pg(1, 1));
}
// The worker must install the same object map as the inline path.
void test_calc_object_states_matches_sync()
{
printf("test_calc_object_states_matches_sync\n");
assert_snaps_equal(peer_one_object(false, true), peer_one_object(true, true));
assert_snaps_equal(peer_one_object(false, false), peer_one_object(true, false));
assert(peer_one_object(false, false).degraded.size() == 1);
// Two workers share the queue and still install the same map.
assert_snaps_equal(peer_one_object(false, true), peer_one_object(true, true, 2));
assert_snaps_equal(peer_one_object(false, false), peer_one_object(true, false, 2));
printf("test_calc_object_states_matches_sync passed\n");
}
// pg_calc_thread_count 0 does not start the pool. calc_object_states runs
// on the OSD thread and peering finishes inside the same loop.
void test_calc_thread_count_zero()
{
printf("test_calc_thread_count_zero\n");
osd_test_fixture_t f;
configure_calc_pool(f);
auto cfg = calc_test_osd_config();
cfg["pg_calc_thread_count"] = 0;
f.construct(cfg);
f.set_pg_calc_sync(false);
f.osd->start();
f.connect_peer(2);
f.reply_local_list({}, 0);
f.reply_peer_list(2, {}, 0);
f.ringloop->loop();
assert(f.pg(1, 1).state == PG_ACTIVE);
assert(!f.pg(1, 1).calc_running);
assert(f.pg(1, 1).peering_state == NULL);
assert(f.pg_calc_queued() == 0);
printf("test_calc_thread_count_zero passed\n");
}
// A job still sitting in the queue is dropped. The new peering lists immediately,
// and calc_object_states for the cancelled generation never runs.
void test_cancel_queued_pg_calc()
{
printf("test_cancel_queued_pg_calc\n");
int runs = 0;
{
osd_test_fixture_t f;
configure_calc_pool(f);
f.construct(calc_test_osd_config());
f.set_pg_calc_sync(false);
f.set_pg_calc_hold(true);
f.set_pg_calc_runs(&runs);
f.osd->start();
f.connect_peer(2);
f.reply_local_list({}, 0);
f.reply_peer_list(2, {}, 0);
f.ringloop->loop();
assert(f.pg(1, 1).state == PG_PEERING);
assert(f.pg(1, 1).calc_running);
assert(f.pg(1, 1).calc_inflight_gen != 0);
assert(f.pg_calc_queued() == 1);
assert(runs == 0);
f.repeer_pg(1, 1);
assert(!f.pg(1, 1).calc_running);
assert(f.pg(1, 1).calc_inflight_gen == 0);
assert(f.pg_calc_queued() == 0);
assert(runs == 0);
assert(f.pg(1, 1).peering_state != NULL);
assert(f.pending_list_count() == 2);
f.reply_local_list({}, 0);
f.reply_peer_list(2, {}, 0);
f.ringloop->loop();
assert(f.pg(1, 1).calc_running);
assert(f.pg_calc_queued() == 1);
assert(runs == 0);
f.release_pg_calc_hold();
f.wait_pg_calc();
assert(runs == 1);
assert(f.pg(1, 1).state == PG_ACTIVE);
assert(!f.pg(1, 1).calc_running);
}
printf("test_cancel_queued_pg_calc passed\n");
}
// stop_pg drops a queued calc instead of leaving calc_running set.
void test_stop_drops_queued_pg_calc()
{
printf("test_stop_drops_queued_pg_calc\n");
int runs = 0;
{
osd_test_fixture_t f;
configure_calc_pool(f);
f.construct(calc_test_osd_config());
f.set_pg_calc_sync(false);
f.set_pg_calc_hold(true);
f.set_pg_calc_runs(&runs);
f.osd->start();
f.connect_peer(2);
f.reply_local_list({}, 0);
f.reply_peer_list(2, {}, 0);
f.ringloop->loop();
assert(f.pg_calc_queued() == 1);
f.stop_pg(1, 1);
// stop reports PG_OFFLINE, and report may apply the config again and
// start a fresh peering. Either way the queued calc is gone.
assert(!f.pg(1, 1).calc_running);
assert(f.pg(1, 1).calc_inflight_gen == 0);
assert(f.pg_calc_queued() == 0);
assert(!(f.pg(1, 1).state & PG_ACTIVE));
assert(runs == 0);
}
assert(runs == 0);
printf("test_stop_drops_queued_pg_calc passed\n");
}
// Destroying the OSD drops queued calcs without running them.
void test_shutdown_drops_queued_pg_calc()
{
printf("test_shutdown_drops_queued_pg_calc\n");
int runs = 0;
{
osd_test_fixture_t f;
configure_calc_pool(f);
f.construct(calc_test_osd_config());
f.set_pg_calc_sync(false);
f.set_pg_calc_hold(true);
f.set_pg_calc_runs(&runs);
f.osd->start();
f.connect_peer(2);
f.reply_local_list({}, 0);
f.reply_peer_list(2, {}, 0);
f.ringloop->loop();
assert(f.pg_calc_queued() == 1);
assert(runs == 0);
}
assert(runs == 0);
printf("test_shutdown_drops_queued_pg_calc passed\n");
}
// A job already popped keeps running, but its result is not installed, and the
// new peering does not LIST until that result has been dropped.
void test_cancel_busy_pg_calc()
{
printf("test_cancel_busy_pg_calc\n");
int runs = 0;
osd_test_fixture_t f;
configure_calc_pool(f);
f.construct(calc_test_osd_config());
f.set_pg_calc_sync(false);
f.set_pg_calc_pause(true);
f.set_pg_calc_runs(&runs);
f.osd->start();
f.connect_peer(2);
f.reply_local_list({}, 0);
f.reply_peer_list(2, {}, 0);
f.ringloop->loop();
f.wait_pg_calc_paused();
assert(f.pg(1, 1).calc_running);
assert(f.pg(1, 1).calc_inflight_gen != 0);
assert(f.pg_calc_queued() == 0);
assert(runs == 0);
uint64_t inflight = f.pg(1, 1).calc_inflight_gen;
f.repeer_pg(1, 1);
assert(f.pg(1, 1).calc_running);
assert(f.pg(1, 1).calc_inflight_gen == inflight);
assert(f.pg(1, 1).calc_gen != inflight);
assert(f.pg(1, 1).peering_state != NULL);
assert(f.pending_list_count() == 0);
assert(runs == 0);
f.unpause_pg_calc();
f.wait_pg_calc();
assert(runs == 1);
assert(!f.pg(1, 1).calc_running);
assert(f.pg(1, 1).state == PG_PEERING);
assert(f.pending_list_count() == 2);
f.reply_local_list({}, 0);
f.reply_peer_list(2, {}, 0);
f.ringloop->loop();
f.wait_pg_calc();
assert(runs == 2);
assert(f.pg(1, 1).state == PG_ACTIVE);
assert(!f.pg(1, 1).calc_running);
printf("test_cancel_busy_pg_calc passed\n");
}
static uint64_t calc_after_reported_epoch_advances(bool clean)
{
osd_test_fixture_t f;
configure_calc_pool(f);
f.st_cli->set("/vitastor/pg/history/1/1", json11::Json::object {
{ "osd_sets", json11::Json::array { json11::Json::array { 1, 2 } } },
{ "all_peers", json11::Json::array { 1, 2 } },
{ "epoch", 7 },
});
f.construct(calc_test_osd_config());
f.set_pg_calc_sync(false);
f.set_pg_calc_pause(true);
f.osd->start();
// Model a previous non-clean activation whose epoch 8 report is still
// waiting for its etcd watch confirmation when this peering starts.
f.pg(1, 1).epoch = 8;
f.connect_peer(2);
obj_ver_id obj = {
.oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 },
.version = 1,
};
f.reply_local_list(clean ? std::vector<obj_ver_id>{} : std::vector<obj_ver_id>{ obj }, clean ? 0 : 1);
f.reply_peer_list(2, {}, 0);
f.ringloop->loop();
f.wait_pg_calc_paused();
assert(f.pg(1, 1).reported_epoch == 7);
// The worker already owns a scratch PG with reported_epoch=7. Confirming
// the previous report now advances only the live PG to epoch 8.
f.confirm_pg_epoch(1, 1, 8);
assert(f.pg(1, 1).reported_epoch == 8);
f.unpause_pg_calc();
f.wait_pg_calc();
return f.pg(1, 1).epoch;
}
// Installing an asynchronous result must reconcile it with reported_epoch
// updates that happened after the scratch PG was created.
void test_calc_epoch_reconciles_reported_update()
{
printf("test_calc_epoch_reconciles_reported_update\n");
// A clean activation may reuse the persisted epoch, but never an older one.
assert(calc_after_reported_epoch_advances(true) == 8);
// A non-clean activation must allocate an epoch strictly above the persisted one.
assert(calc_after_reported_epoch_advances(false) == 9);
printf("test_calc_epoch_reconciles_reported_update passed\n");
}
// Replicated PG: v1 stable on both OSDs, v2 unstable on both. Peering keeps v1
// and rolls v2 back, which fills ver_override and flush_actions.
static pg_calc_snap_t peer_unstable_replicated(bool async)
{
osd_test_fixture_t f;
configure_calc_pool(f);
auto cfg = calc_test_osd_config();
cfg["no_rebalance"] = true;
inode_t inode = INODE_WITH_POOL(1, 1);
obj_ver_id v1 = { .oid = { .inode = inode, .stripe = 0 }, .version = 1 };
obj_ver_id v2 = { .oid = { .inode = inode, .stripe = 0 }, .version = 2 };
if (async)
{
f.construct(cfg);
f.set_pg_calc_sync(false);
f.osd->start();
}
else
f.start(cfg);
f.connect_peer(2);
f.reply_local_list({ v1, v2 }, 1);
f.reply_peer_list(2, { v1, v2 }, 1);
f.ringloop->loop();
f.wait_pg_calc();
return snap_pg(f.pg(1, 1));
}
// EC 2+2 stays incomplete until every member is connected, so the LIST the
// test answers is the full-set one.
static void configure_ec_calc_pool(osd_test_fixture_t & f)
{
f.configure_ec_pool(/*data_chunks*/ 2, /*parity_chunks*/ 2, /*primary_osd*/ 1);
f.st_cli->set("/vitastor/config/pools", json11::Json::object {
{ "1", json11::Json::object {
{ "name", "pool_1" },
{ "scheme", "ec" },
{ "pg_size", 4 },
{ "pg_minsize", 4 },
{ "parity_chunks", 2 },
{ "pg_count", 1 },
{ "failure_domain", "osd" },
{ "immediate_commit", "none" },
} },
});
}
static obj_ver_id ec_piece(inode_t inode, uint64_t role, uint64_t version)
{
return { .oid = { .inode = inode, .stripe = role }, .version = version };
}
static pg_calc_snap_t peer_ec(bool async, bool rollback)
{
osd_test_fixture_t f;
configure_ec_calc_pool(f);
auto cfg = calc_test_osd_config();
cfg["no_rebalance"] = true;
inode_t inode = INODE_WITH_POOL(1, 1);
if (async)
{
f.construct(cfg);
f.set_pg_calc_sync(false);
f.osd->start();
}
else
f.start(cfg);
f.connect_peer(2);
f.connect_peer(3);
f.connect_peer(4);
if (rollback)
{
// Stable v1 on every role. Unstable v2 only on roles 0 and 2, so it
// must not become the target; those two pieces roll back to v1.
f.reply_local_list({ ec_piece(inode, 0, 1), ec_piece(inode, 0, 2) }, 1);
f.reply_peer_list(2, { ec_piece(inode, 1, 1) }, 1);
f.reply_peer_list(3, { ec_piece(inode, 2, 1), ec_piece(inode, 2, 2) }, 1);
f.reply_peer_list(4, { ec_piece(inode, 3, 1) }, 1);
}
else
{
// v2 is the target on every role. OSD 2 also holds an older role-3
// piece; cur_set[3] is OSD 4, so that piece stays outdated.
f.reply_local_list({ ec_piece(inode, 0, 2) }, 1);
f.reply_peer_list(2, { ec_piece(inode, 1, 2), ec_piece(inode, 3, 1) }, 2);
f.reply_peer_list(3, { ec_piece(inode, 2, 2) }, 1);
f.reply_peer_list(4, { ec_piece(inode, 3, 2) }, 1);
}
f.ringloop->loop();
f.wait_pg_calc();
return snap_pg(f.pg(1, 1));
}
// The worker must install flush_actions, ver_override, and EC role placement
// the same way the inline path does. Those fields are not covered by a
// single clean replicated object.
void test_calc_object_states_matches_sync_extra()
{
printf("test_calc_object_states_matches_sync_extra\n");
inode_t inode = INODE_WITH_POOL(1, 1);
object_id oid = { .inode = inode, .stripe = 0 };
auto sync_u = peer_unstable_replicated(false);
auto async_u = peer_unstable_replicated(true);
assert_snaps_equal(sync_u, async_u);
assert(sync_u.state & PG_HAS_UNCLEAN);
assert(sync_u.ver.size() == 1 && sync_u.ver[0].first == oid && sync_u.ver[0].second == 1);
assert(sync_u.flush.size() == 2);
for (auto & fa: sync_u.flush)
assert(fa.rollback && !fa.make_stable && fa.rollback_to == 1);
auto sync_r = peer_ec(false, true);
auto async_r = peer_ec(true, true);
assert_snaps_equal(sync_r, async_r);
assert(sync_r.state & PG_HAS_UNCLEAN);
assert(!(sync_r.state & (PG_HAS_INCOMPLETE | PG_HAS_DEGRADED)));
assert(sync_r.clean_count == 1 && sync_r.total_count == 1);
assert(sync_r.ver.size() == 1 && sync_r.ver[0].second == 1);
assert(sync_r.flush.size() == 2);
for (auto & fa: sync_r.flush)
{
assert(fa.rollback && !fa.make_stable && fa.rollback_to == 1);
assert((fa.oid.stripe & STRIPE_MASK) == (fa.osd_num == 1 ? 0u : 2u));
}
auto sync_m = peer_ec(false, false);
auto async_m = peer_ec(true, false);
assert_snaps_equal(sync_m, async_m);
assert(sync_m.state & PG_HAS_MISPLACED);
assert(sync_m.misplaced.size() == 1);
assert(sync_m.read_targets.size() == 1);
assert(sync_m.read_targets[0].size() == 4 && sync_m.read_targets[0][3] == 4);
printf("test_calc_object_states_matches_sync_extra passed\n");
}
// A finished calc sits in the worker's done queue until the OSD thread
// drains it. Cancel must drop that result instead of installing it, and the
// new peering can LIST without waiting out another sort.
void test_cancel_done_pg_calc()
{
printf("test_cancel_done_pg_calc\n");
int runs = 0;
osd_test_fixture_t f;
configure_calc_pool(f);
f.construct(calc_test_osd_config());
f.set_pg_calc_sync(false);
f.set_pg_calc_runs(&runs);
f.osd->start();
f.connect_peer(2);
f.reply_local_list({}, 0);
f.reply_peer_list(2, {}, 0);
f.ringloop->loop();
assert(f.pg(1, 1).calc_running);
f.wait_pg_calc_not_busy();
assert(runs == 1);
assert(f.pg_calc_queued() == 0);
assert(f.pg(1, 1).calc_running);
assert(f.pg(1, 1).state == PG_PEERING);
f.repeer_pg(1, 1);
assert(runs == 1);
assert(!f.pg(1, 1).calc_running);
assert(f.pg(1, 1).calc_inflight_gen == 0);
assert(f.pg(1, 1).state == PG_PEERING);
assert(f.pg(1, 1).peering_state != NULL);
assert(f.pending_list_count() == 2);
f.reply_local_list({}, 0);
f.reply_peer_list(2, {}, 0);
f.ringloop->loop();
f.wait_pg_calc();
assert(runs == 2);
assert(f.pg(1, 1).state == PG_ACTIVE);
assert(!f.pg(1, 1).calc_running);
printf("test_cancel_done_pg_calc passed\n");
}
// Object-version epochs are copied onto the scratch PG. A reported_epoch that
// moves while that calc is in flight is reconciled on install: a higher
// version epoch is kept, and a non-clean activation still lands strictly
// above the persisted epoch.
static uint64_t calc_epoch_with_version(uint64_t version_epoch, uint64_t reported_after, bool clean)
{
osd_test_fixture_t f;
configure_calc_pool(f);
f.st_cli->set("/vitastor/pg/history/1/1", json11::Json::object {
{ "osd_sets", json11::Json::array { json11::Json::array { 1, 2 } } },
{ "all_peers", json11::Json::array { 1, 2 } },
{ "epoch", 7 },
});
f.construct(calc_test_osd_config());
f.set_pg_calc_sync(false);
f.set_pg_calc_pause(true);
f.osd->start();
f.connect_peer(2);
uint64_t ver = (version_epoch << (64 - PG_EPOCH_BITS)) | 1;
obj_ver_id obj = {
.oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 },
.version = ver,
};
f.reply_local_list({ obj }, 1);
f.reply_peer_list(2, clean ? std::vector<obj_ver_id>{ obj } : std::vector<obj_ver_id>{}, clean ? 1 : 0);
f.ringloop->loop();
f.wait_pg_calc_paused();
assert(f.pg(1, 1).reported_epoch == 7);
// The history hook only advances reported_epoch when the live epoch is
// already at least the value being confirmed.
f.pg(1, 1).epoch = reported_after;
f.confirm_pg_epoch(1, 1, reported_after);
assert(f.pg(1, 1).reported_epoch == reported_after);
f.unpause_pg_calc();
f.wait_pg_calc();
if (clean)
assert(f.pg(1, 1).state == PG_ACTIVE);
else
assert(f.pg(1, 1).state & PG_HAS_DEGRADED);
return f.pg(1, 1).epoch;
}
void test_calc_epoch_keeps_version_epoch()
{
printf("test_calc_epoch_keeps_version_epoch\n");
// Version epoch 10, reported moves 7 -> 12. Clean reuses 12.
assert(calc_epoch_with_version(10, 12, true) == 12);
// Non-clean scratch epoch is 11; reported catches it, so install bumps to 12.
assert(calc_epoch_with_version(10, 11, false) == 12);
// Version epoch 20 stays above a reported_epoch that only moved to 9.
assert(calc_epoch_with_version(20, 9, true) == 20);
assert(calc_epoch_with_version(20, 9, false) == 21);
printf("test_calc_epoch_keeps_version_epoch passed\n");
}
// A non-clean activation must bump the epoch above the persisted base, not above
// the version epochs of surviving copies
void test_pg_epoch_bump_above_reported_epoch()
@@ -3757,6 +4401,17 @@ int main(int narg, char *args[])
test_read_enoent_without_racing_op_marks_corrupted();
test_peering_outdated_copy_other_role();
test_pg_epoch_base_is_reported_epoch();
test_calc_object_states_off_thread();
test_calc_object_states_matches_sync();
test_calc_object_states_matches_sync_extra();
test_calc_thread_count_zero();
test_cancel_queued_pg_calc();
test_stop_drops_queued_pg_calc();
test_shutdown_drops_queued_pg_calc();
test_cancel_busy_pg_calc();
test_cancel_done_pg_calc();
test_calc_epoch_reconciles_reported_update();
test_calc_epoch_keeps_version_epoch();
test_pg_epoch_bump_above_reported_epoch();
test_chained_read_large_chain_info();
test_peering_partial_unstable_ec_rolls_back();

View File

@@ -180,6 +180,9 @@ struct osd_test_fixture_t
[this](blockstore_config_t & cfg) -> blockstore_i* {
return (bs = new capturing_bs_t(cfg));
});
// Mock ringloop does not poll eventfd, so a background calc would not be
// applied inside ringloop->loop(). Tests keep the synchronous path.
osd->pg_calc_sync = true;
}
void start(json11::Json::object osd_config)
@@ -270,6 +273,90 @@ struct osd_test_fixture_t
// Drive handle_peers so PGs see lists_done and finalize calc_object_states.
ringloop->wakeup();
ringloop->loop();
osd->wait_pg_calc();
}
void wait_pg_calc()
{
osd->wait_pg_calc();
}
// Workers have pushed finished jobs onto done. The OSD thread has not
// installed them yet, so cancel can still drop the result.
void wait_pg_calc_not_busy()
{
osd->wait_pg_calc_not_busy();
}
// Must be called after construct() and before start().
void set_pg_calc_sync(bool sync)
{
osd->pg_calc_sync = sync;
}
// Must be called after construct() and before start(). See osd_t::pg_calc_hold.
void set_pg_calc_hold(bool hold)
{
osd->pg_calc_hold = hold;
}
void set_pg_calc_pause(bool pause)
{
osd->pg_calc_pause = pause;
}
void set_pg_calc_runs(int *runs)
{
osd->pg_calc_runs = runs;
}
void release_pg_calc_hold()
{
osd->pg_calc_release_hold();
}
void unpause_pg_calc()
{
osd->pg_calc_unpause();
}
void wait_pg_calc_paused()
{
osd->wait_pg_calc_paused();
}
size_t pg_calc_queued()
{
return osd->pg_calc_queued();
}
void confirm_pg_epoch(pool_id_t pool_id, pg_num_t pg_num, uint64_t epoch)
{
osd->st_cli->pool_config.at(pool_id).pg_config.at(pg_num).epoch = epoch;
osd->on_change_pg_history_hook(pool_id, pg_num);
}
void repeer_pg(pool_id_t pool_id, pg_num_t pg_num)
{
osd->repeer_pg(osd->pgs.at({ .pool_id = pool_id, .pg_num = pg_num }));
}
void stop_pg(pool_id_t pool_id, pg_num_t pg_num)
{
osd->stop_pg(osd->pgs.at({ .pool_id = pool_id, .pg_num = pg_num }));
}
int pending_list_count()
{
int n = 0;
for (auto *op: bs->queued)
if (op->opcode == BS_OP_LIST)
n++;
for (auto & p: osd->msgr.osd_peers)
for (auto & kv: p.second->sent_ops)
if (kv.second->req.hdr.opcode == OSD_OP_SEC_LIST)
n++;
return n;
}
// Inline list response builder. `objects` are pairs (oid, version);

View File

@@ -2056,6 +2056,9 @@ void chaos_t::start_osd()
bs = new blockstore_impl_t(cfg, f.ringloop, f.tfd, true);
return bs;
});
// The chaos clock is simulated; peering calc stays on the calling thread
// so a step observes PG_ACTIVE without waiting on a real thread.
f.osd->pg_calc_sync = true;
}
catch(std::exception & ex)
{