WIP: [DNM] osd: run calc_object_states off the OSD thread #123
@@ -13,6 +13,7 @@ with an OSD restart or, for some of them, even without restarting by updating co
|
||||
- [bind_address](#bind_address)
|
||||
- [bind_port](#bind_port)
|
||||
- [osd_iothread_count](#osd_iothread_count)
|
||||
- [pg_calc_thread_count](#pg_calc_thread_count)
|
||||
- [etcd_report_interval](#etcd_report_interval)
|
||||
- [etcd_stats_interval](#etcd_stats_interval)
|
||||
- [run_primary](#run_primary)
|
||||
@@ -101,6 +102,20 @@ switching overhead occurs. RDMA isn't affected by this option.
|
||||
Because of latency, instead of enabling OSD I/O threads it's recommended to
|
||||
just create multiple OSDs per disk, or use RDMA.
|
||||
|
||||
## pg_calc_thread_count
|
||||
|
||||
- Type: integer
|
||||
- Default: 1
|
||||
|
||||
Number of threads that run PG peering's object-state calculation
|
||||
(`calc_object_states`). A large PG used to stall `OSD_OP_PING` inside the
|
||||
OSD loop until peers declared the OSD dead. The pool keeps that work off
|
||||
the OSD thread. Results are installed on the OSD thread before the next
|
||||
peering step.
|
||||
|
||||
0 does not create the pool. The calculation then runs on the OSD thread,
|
||||
as it did before the pool existed. The value is read when the OSD starts.
|
||||
|
||||
## etcd_report_interval
|
||||
|
||||
- Type: seconds
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
- [bind_address](#bind_address)
|
||||
- [bind_port](#bind_port)
|
||||
- [osd_iothread_count](#osd_iothread_count)
|
||||
- [pg_calc_thread_count](#pg_calc_thread_count)
|
||||
- [etcd_report_interval](#etcd_report_interval)
|
||||
- [etcd_stats_interval](#etcd_stats_interval)
|
||||
- [run_primary](#run_primary)
|
||||
@@ -103,6 +104,20 @@
|
||||
Из-за задержек вместо включения потоков ввода-вывода OSD рекомендуется
|
||||
просто создавать по несколько OSD на каждом диске, или использовать RDMA.
|
||||
|
||||
## pg_calc_thread_count
|
||||
|
||||
- Тип: целое число
|
||||
- Значение по умолчанию: 1
|
||||
|
||||
Число потоков, в которых OSD считает состояние объектов PG при peering
|
||||
(`calc_object_states`). Большая PG раньше блокировала `OSD_OP_PING` в
|
||||
главном цикле, и соседи объявляли OSD упавшим. Пул выносит этот расчёт
|
||||
из главного потока. Результат устанавливается в главном потоке до
|
||||
следующего шага peering.
|
||||
|
||||
0 не создаёт пул: расчёт выполняется в главном потоке OSD, как до
|
||||
появления пула. Значение читается при старте OSD.
|
||||
|
||||
## etcd_report_interval
|
||||
|
||||
- Тип: секунды
|
||||
|
||||
275
src/osd/osd.cpp
275
src/osd/osd.cpp
@@ -1,12 +1,18 @@
|
||||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include <stdexcept>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/poll.h>
|
||||
#include <sys/eventfd.h>
|
||||
#include <sys/mman.h>
|
||||
#include <netinet/in.h>
|
||||
#include <netinet/tcp.h>
|
||||
#include <arpa/inet.h>
|
||||
#include <pthread.h>
|
||||
#include <errno.h>
|
||||
|
||||
#include "addr_util.h"
|
||||
#include "osd_primary.h"
|
||||
@@ -83,6 +89,7 @@ void osd_t::start()
|
||||
|
||||
consumer.loop = [this]() { loop(); };
|
||||
ringloop->register_consumer(&consumer);
|
||||
start_pg_calc_worker();
|
||||
}
|
||||
|
||||
osd_t::~osd_t()
|
||||
@@ -116,6 +123,10 @@ osd_t::~osd_t()
|
||||
// clients directly rather than through stop_client(): the stop protocol would
|
||||
// fire repeer and PG-report hooks on the way out of a dying OSD
|
||||
in_destruction = true;
|
||||
// Join before clients die. Queued calcs are dropped without running; only the
|
||||
// job already inside calc_object_states is finished, then its result is dropped
|
||||
// before pgs (and their peering state) go away.
|
||||
stop_pg_calc_worker();
|
||||
while (!msgr.clients.empty())
|
||||
msgr.destroy_client(msgr.clients.begin()->second);
|
||||
if (bs)
|
||||
@@ -351,6 +362,11 @@ void osd_t::parse_config(bool init)
|
||||
this->disable_tcp = this->use_rdmacm && (config["disable_tcp"].bool_value() || config["disable_tcp"].uint64_value() != 0);
|
||||
#endif
|
||||
// OSD configuration
|
||||
// 0 keeps calc_object_states on the OSD thread. A missing key is the default.
|
||||
if (config["pg_calc_thread_count"].is_null())
|
||||
pg_calc_thread_count = 1;
|
||||
else
|
||||
pg_calc_thread_count = (uint32_t)config["pg_calc_thread_count"].uint64_value();
|
||||
etcd_report_interval = config["etcd_report_interval"].uint64_value();
|
||||
if (etcd_report_interval <= 0)
|
||||
etcd_report_interval = 5;
|
||||
@@ -601,6 +617,9 @@ bool osd_t::shutdown()
|
||||
|
||||
void osd_t::loop()
|
||||
{
|
||||
// Apply calc_object_states results before peering so a woken loop can
|
||||
// report PG state and still service pings in this iteration
|
||||
drain_pg_calc();
|
||||
handle_peers();
|
||||
msgr.read_requests();
|
||||
msgr.send_replies();
|
||||
@@ -871,3 +890,259 @@ void osd_t::print_slow()
|
||||
bs->dump_diagnostics();
|
||||
}
|
||||
}
|
||||
|
||||
static pg_calc_job_t *take_pg_calc_job(std::deque<pg_calc_job_t*> & q, pool_id_t pool_id, pg_num_t pg_num)
|
||||
{
|
||||
for (auto it = q.begin(); it != q.end(); ++it)
|
||||
{
|
||||
if ((*it)->pool_id == pool_id && (*it)->pg_num == pg_num)
|
||||
{
|
||||
pg_calc_job_t *job = *it;
|
||||
q.erase(it);
|
||||
return job;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void discard_pg_calc_job(pg_calc_job_t *job)
|
||||
{
|
||||
if (job->scratch)
|
||||
{
|
||||
pg_peering_state_t *ps = job->scratch->peering_state;
|
||||
if (ps)
|
||||
{
|
||||
// LIST callbacks were cleared before the job was built
|
||||
assert(ps->list_ops.empty());
|
||||
for (auto & lr: ps->list_results)
|
||||
if (lr.second.buf)
|
||||
free(lr.second.buf);
|
||||
delete ps;
|
||||
job->scratch->peering_state = NULL;
|
||||
}
|
||||
delete job->scratch;
|
||||
}
|
||||
delete job;
|
||||
}
|
||||
|
||||
pg_calc_worker_t::pg_calc_worker_t(int thread_count, bool hold, bool pause_before_calc, int *calc_runs):
|
||||
hold(hold), pause_before_calc(pause_before_calc), calc_runs(calc_runs)
|
||||
{
|
||||
assert(thread_count > 0);
|
||||
eventfd = ::eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
|
||||
if (eventfd < 0)
|
||||
throw std::runtime_error(std::string("eventfd: ") + strerror(errno));
|
||||
try
|
||||
{
|
||||
threads.reserve(thread_count);
|
||||
for (int i = 0; i < thread_count; i++)
|
||||
threads.emplace_back(&pg_calc_worker_t::run, this);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
stop();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
pg_calc_worker_t::~pg_calc_worker_t()
|
||||
{
|
||||
stop();
|
||||
}
|
||||
|
||||
void pg_calc_worker_t::push(pg_calc_job_t *job)
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
queue.push_back(job);
|
||||
// Wake every idle worker when the queue becomes non-empty. A later push
|
||||
// in the same burst is seen by threads already leaving wait.
|
||||
if (queue.size() == 1)
|
||||
cond.notify_all();
|
||||
}
|
||||
|
||||
pg_calc_job_t *pg_calc_worker_t::cancel(pool_id_t pool_id, pg_num_t pg_num)
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
pg_calc_job_t *dropped = take_pg_calc_job(queue, pool_id, pg_num);
|
||||
if (!dropped)
|
||||
dropped = take_pg_calc_job(done, pool_id, pg_num);
|
||||
return dropped;
|
||||
}
|
||||
|
||||
std::deque<pg_calc_job_t*> pg_calc_worker_t::drain()
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
std::deque<pg_calc_job_t*> jobs;
|
||||
jobs.swap(done);
|
||||
return jobs;
|
||||
}
|
||||
|
||||
void pg_calc_worker_t::wait_not_busy()
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
while (busy || queue.size())
|
||||
cond.wait(lk);
|
||||
}
|
||||
|
||||
bool pg_calc_worker_t::idle()
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
return !busy && !queue.size() && !done.size();
|
||||
}
|
||||
|
||||
void pg_calc_worker_t::release_hold()
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
hold = false;
|
||||
cond.notify_all();
|
||||
}
|
||||
|
||||
void pg_calc_worker_t::unpause()
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
pause_before_calc = false;
|
||||
cond.notify_all();
|
||||
}
|
||||
|
||||
void pg_calc_worker_t::wait_paused()
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
while (!paused && !stopped)
|
||||
cond.wait(lk);
|
||||
}
|
||||
|
||||
size_t pg_calc_worker_t::queued()
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
return queue.size();
|
||||
}
|
||||
|
||||
void pg_calc_worker_t::stop()
|
||||
{
|
||||
std::deque<pg_calc_job_t*> dropped;
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
if (stopped)
|
||||
return;
|
||||
stopped = true;
|
||||
if (outer_loop_data)
|
||||
{
|
||||
outer_loop_data->callback = [](ring_data_t*){};
|
||||
outer_loop_data = NULL;
|
||||
}
|
||||
// Drop every job that has not been popped. Join only waits for jobs
|
||||
// already inside calc_object_states (or parked on pause_before_calc).
|
||||
dropped.swap(queue);
|
||||
cond.notify_all();
|
||||
}
|
||||
for (auto *job: dropped)
|
||||
discard_pg_calc_job(job);
|
||||
for (auto & th: threads)
|
||||
if (th.joinable())
|
||||
th.join();
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
for (auto *job: done)
|
||||
discard_pg_calc_job(job);
|
||||
done.clear();
|
||||
}
|
||||
// Close after join. The worker writes this fd when a calculation finishes;
|
||||
// msgr_iothread_t can close earlier because only the kernel writes its eventfd.
|
||||
if (eventfd >= 0)
|
||||
{
|
||||
close(eventfd);
|
||||
eventfd = -1;
|
||||
}
|
||||
}
|
||||
|
||||
void pg_calc_worker_t::add_to_ringloop(ring_loop_i *outer_loop)
|
||||
{
|
||||
assert(!this->outer_loop || this->outer_loop == outer_loop);
|
||||
io_uring_sqe *sqe = outer_loop->get_sqe();
|
||||
assert(sqe != NULL);
|
||||
this->outer_loop = outer_loop;
|
||||
this->outer_loop_data = ((ring_data_t*)sqe->user_data);
|
||||
io_uring_prep_poll_add(sqe, eventfd, POLLIN);
|
||||
outer_loop_data->callback = [this](ring_data_t *data)
|
||||
{
|
||||
if (data->res < 0)
|
||||
{
|
||||
throw std::runtime_error(std::string("eventfd poll failed: ") + strerror(-data->res));
|
||||
}
|
||||
outer_loop_data = NULL;
|
||||
if (stopped)
|
||||
{
|
||||
return;
|
||||
}
|
||||
// Drain the counter. Results are applied from osd_t::loop, before
|
||||
// peering, so the install stays on the main thread.
|
||||
for (;;)
|
||||
{
|
||||
uint64_t ctr = 0;
|
||||
ssize_t r = read(eventfd, &ctr, sizeof(ctr));
|
||||
if (r < 0 && errno == EINTR)
|
||||
continue;
|
||||
break;
|
||||
}
|
||||
add_to_ringloop(this->outer_loop);
|
||||
this->outer_loop->wakeup();
|
||||
};
|
||||
}
|
||||
|
||||
void pg_calc_worker_t::run()
|
||||
{
|
||||
pthread_setname_np(pthread_self(), "pg-calc");
|
||||
while (true)
|
||||
{
|
||||
pg_calc_job_t *job = NULL;
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
// stop abandons the queue. hold keeps jobs queued so cancel can drop them.
|
||||
while (!stopped && (hold || !queue.size()))
|
||||
cond.wait(lk);
|
||||
if (stopped)
|
||||
return;
|
||||
job = queue.front();
|
||||
queue.pop_front();
|
||||
busy++;
|
||||
if (pause_before_calc)
|
||||
{
|
||||
paused++;
|
||||
cond.notify_all();
|
||||
while (pause_before_calc && !stopped)
|
||||
cond.wait(lk);
|
||||
paused--;
|
||||
if (stopped)
|
||||
{
|
||||
// Not inside calc_object_states yet: hand the job back for discard.
|
||||
busy--;
|
||||
done.push_back(job);
|
||||
cond.notify_all();
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (calc_runs)
|
||||
(*calc_runs)++;
|
||||
}
|
||||
timespec t0, t1;
|
||||
clock_gettime(CLOCK_MONOTONIC, &t0);
|
||||
job->scratch->calc_object_states(job->log_level);
|
||||
clock_gettime(CLOCK_MONOTONIC, &t1);
|
||||
long ms = (long)((t1.tv_sec - t0.tv_sec) * 1000 + (t1.tv_nsec - t0.tv_nsec) / 1000000);
|
||||
if (ms >= 100)
|
||||
{
|
||||
printf("[PG %u/%u] calc_object_states took %ld ms\n",
|
||||
job->scratch->pool_id, job->scratch->pg_num, ms);
|
||||
}
|
||||
{
|
||||
std::unique_lock<std::mutex> lk(mu);
|
||||
busy--;
|
||||
done.push_back(job);
|
||||
cond.notify_all();
|
||||
}
|
||||
uint64_t one = 1;
|
||||
ssize_t r = write(eventfd, &one, sizeof(one));
|
||||
if (r < 0 && errno != EAGAIN)
|
||||
fprintf(stderr, "pg calc eventfd write: %s\n", strerror(errno));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,10 @@
|
||||
|
||||
#include <set>
|
||||
#include <deque>
|
||||
#include <vector>
|
||||
#include <mutex>
|
||||
#include <condition_variable>
|
||||
#include <thread>
|
||||
|
||||
#include "blockstore.h"
|
||||
#include "ringloop.h"
|
||||
@@ -31,6 +35,66 @@
|
||||
|
||||
#define SELF_CLIENT 0
|
||||
|
||||
// One calc_object_states request. The worker owns scratch until the main
|
||||
// thread accepts or discards the job.
|
||||
struct pg_calc_job_t
|
||||
{
|
||||
pool_id_t pool_id = 0;
|
||||
pg_num_t pg_num = 0;
|
||||
uint64_t gen = 0;
|
||||
int log_level = 0;
|
||||
pg_t *scratch = NULL;
|
||||
};
|
||||
|
||||
void discard_pg_calc_job(pg_calc_job_t *job);
|
||||
|
||||
// Thread pool for calc_object_states. Same shape as msgr_iothread_t: the
|
||||
// constructor starts the threads, stop() joins them, and add_to_ringloop()
|
||||
// arms a poll on eventfd so the OSD loop can drain results. Workers share
|
||||
// one queue. Jobs still in the queue are dropped on stop; a job already
|
||||
// popped runs to completion and its result is left in done for the caller
|
||||
// to discard.
|
||||
class pg_calc_worker_t
|
||||
{
|
||||
protected:
|
||||
int eventfd = -1;
|
||||
bool stopped = false;
|
||||
// threads that have popped a job and not yet pushed it onto done
|
||||
int busy = 0;
|
||||
// threads waiting out pause_before_calc
|
||||
int paused = 0;
|
||||
// test gates, copied at start and only changed under mu
|
||||
bool hold = false;
|
||||
bool pause_before_calc = false;
|
||||
int *calc_runs = NULL;
|
||||
ring_loop_i *outer_loop = NULL;
|
||||
ring_data_t *outer_loop_data = NULL;
|
||||
std::mutex mu;
|
||||
std::condition_variable cond;
|
||||
std::deque<pg_calc_job_t*> queue;
|
||||
std::deque<pg_calc_job_t*> done;
|
||||
std::vector<std::thread> threads;
|
||||
|
||||
void run();
|
||||
public:
|
||||
pg_calc_worker_t(int thread_count, bool hold, bool pause_before_calc, int *calc_runs);
|
||||
~pg_calc_worker_t();
|
||||
|
||||
void push(pg_calc_job_t *job);
|
||||
// Drop a job that has not entered calc_object_states. NULL if the worker
|
||||
// has already popped it.
|
||||
pg_calc_job_t *cancel(pool_id_t pool_id, pg_num_t pg_num);
|
||||
std::deque<pg_calc_job_t*> drain();
|
||||
void wait_not_busy();
|
||||
bool idle();
|
||||
void release_hold();
|
||||
void unpause();
|
||||
void wait_paused();
|
||||
size_t queued();
|
||||
void stop();
|
||||
void add_to_ringloop(ring_loop_i *outer_loop);
|
||||
};
|
||||
|
||||
#define MAX_AUTOSYNC_INTERVAL 3600
|
||||
#define DEFAULT_AUTOSYNC_INTERVAL 5
|
||||
#define DEFAULT_AUTOSYNC_WRITES 128
|
||||
@@ -196,6 +260,20 @@ class osd_t
|
||||
int copies_to_delete_after_sync_count = 0;
|
||||
uint64_t misplaced_objects = 0, degraded_objects = 0, incomplete_objects = 0, inconsistent_objects = 0, corrupted_objects = 0;
|
||||
int peering_state = 0;
|
||||
// Thread pool for calc_object_states (pg_calc_worker_t). pg_calc_thread_count
|
||||
// is the pool size; 0 keeps the calculation on the OSD thread. Tests set
|
||||
// pg_calc_sync so the calculation stays on the main thread: the mock event
|
||||
// loop does not complete the eventfd poll. Must be set before start().
|
||||
// pg_calc_hold: the worker does not pop, so a test can cancel a job that
|
||||
// has not started. pg_calc_pause: after pop, stop before calc_object_states
|
||||
// so a test can observe an in-flight job. Both must be set before start().
|
||||
// When pg_calc_runs is set, it is incremented each time a calculation starts.
|
||||
pg_calc_worker_t *pg_calc_worker = NULL;
|
||||
int pg_calc_thread_count = 1;
|
||||
bool pg_calc_sync = false;
|
||||
bool pg_calc_hold = false;
|
||||
bool pg_calc_pause = false;
|
||||
int *pg_calc_runs = NULL;
|
||||
std::map<object_id, osd_recovery_op_t> recovery_ops;
|
||||
// OSD-owned in-flight ops (flush batches, peering LIST subops): their only
|
||||
// owners are completion callbacks, so the destructor needs a handle on them
|
||||
@@ -326,6 +404,21 @@ class osd_t
|
||||
void rm_inflight(pg_t & pg);
|
||||
void continue_pg(pg_t & pg);
|
||||
bool continue_pg_peering(pg_t & pg);
|
||||
void start_pg_calc_worker();
|
||||
void stop_pg_calc_worker();
|
||||
void drain_pg_calc();
|
||||
void wait_pg_calc();
|
||||
// Block until workers have finished. Results stay in the worker's done
|
||||
// queue; the caller installs them with drain_pg_calc / wait_pg_calc.
|
||||
void wait_pg_calc_not_busy();
|
||||
void pg_calc_release_hold();
|
||||
void pg_calc_unpause();
|
||||
void wait_pg_calc_paused();
|
||||
size_t pg_calc_queued();
|
||||
void cancel_pg_calc(pg_t & pg);
|
||||
void submit_pg_object_calc(pg_t & pg);
|
||||
void accept_pg_calc(pg_calc_job_t *job);
|
||||
void complete_pg_peering(pg_t & pg);
|
||||
|
||||
// flushing, recovery and backfill
|
||||
void submit_pg_flush_ops(pg_t & pg);
|
||||
|
||||
@@ -199,6 +199,8 @@ void osd_t::drop_dirty_pg_connections(pool_pg_num_t pg)
|
||||
// Repeer on each connect/disconnect peer event
|
||||
void osd_t::start_pg_peering(pg_t & pg)
|
||||
{
|
||||
// An in-flight calc still owns the previous peering_state. Drop its result.
|
||||
cancel_pg_calc(pg);
|
||||
pg.state = PG_PEERING;
|
||||
this->peering_state |= OSD_PEERING_PGS;
|
||||
reset_pg(pg);
|
||||
@@ -335,6 +337,9 @@ bool osd_t::continue_pg_peering(pg_t & pg)
|
||||
// Teardown: cancelled peering subops must not resume the peering machine
|
||||
return false;
|
||||
}
|
||||
// peering_state is null while calc_object_states owns it on the worker thread
|
||||
if (pg.calc_running || !pg.peering_state)
|
||||
return false;
|
||||
if (pg.peering_state->locked)
|
||||
{
|
||||
pg.peering_state->lists_done = true;
|
||||
@@ -354,27 +359,11 @@ bool osd_t::continue_pg_peering(pg_t & pg)
|
||||
}
|
||||
if (pg.peering_state->lists_done)
|
||||
{
|
||||
pg.calc_object_states(log_level);
|
||||
report_pg_state(pg);
|
||||
schedule_scrub(pg);
|
||||
inconsistent_objects += pg.inconsistent_objects.size();
|
||||
incomplete_objects += pg.incomplete_objects.size();
|
||||
misplaced_objects += pg.misplaced_objects.size();
|
||||
// FIXME: degraded objects may currently include misplaced, too! Report them separately?
|
||||
degraded_objects += pg.degraded_objects.size();
|
||||
if (pg.state & PG_HAS_UNCLEAN)
|
||||
this->peering_state = peering_state | OSD_FLUSHING_PGS;
|
||||
else if (pg.state & (PG_HAS_DEGRADED | PG_HAS_MISPLACED))
|
||||
{
|
||||
this->peering_state = peering_state | OSD_RECOVERING;
|
||||
if (pg.state & PG_HAS_DEGRADED)
|
||||
{
|
||||
// Restart recovery from degraded objects
|
||||
this->recovery_last_degraded = true;
|
||||
this->recovery_last_pg = {};
|
||||
this->recovery_last_oid = {};
|
||||
}
|
||||
}
|
||||
// Sort + walk of every object version. Run it off the OSD thread so a
|
||||
// large PG cannot stall OSD_OP_PING past osd_ping_timeout.
|
||||
submit_pg_object_calc(pg);
|
||||
if (pg.calc_running)
|
||||
return false;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
@@ -392,7 +381,7 @@ void osd_t::relock_pg(pg_t & pg)
|
||||
{
|
||||
if (!enable_pg_locks || pg.disable_pg_locks && !pg.lock_peers.size())
|
||||
{
|
||||
if (pg.state & PG_PEERING)
|
||||
if ((pg.state & PG_PEERING) && pg.peering_state)
|
||||
pg.peering_state->locked = true;
|
||||
continue_pg(pg);
|
||||
return;
|
||||
@@ -431,7 +420,7 @@ void osd_t::relock_pg(pg_t & pg)
|
||||
}
|
||||
if (!diff_osds.size())
|
||||
{
|
||||
if (pg.state & PG_PEERING)
|
||||
if ((pg.state & PG_PEERING) && pg.peering_state)
|
||||
pg.peering_state->locked = true;
|
||||
continue_pg(pg);
|
||||
return;
|
||||
@@ -519,7 +508,9 @@ void osd_t::relock_pg(pg_t & pg)
|
||||
};
|
||||
msgr.outbox_push(op);
|
||||
}
|
||||
if (pg.state & PG_PEERING)
|
||||
// calc_object_states owns peering_state on the worker. State can still be
|
||||
// PG_PEERING in that window; don't touch the pointer until it is back.
|
||||
if ((pg.state & PG_PEERING) && pg.peering_state)
|
||||
{
|
||||
pg.peering_state->locked = !diff_osds.size();
|
||||
}
|
||||
@@ -682,6 +673,7 @@ void osd_t::release_peering_state(pg_t & pg)
|
||||
|
||||
bool osd_t::stop_pg(pg_t & pg)
|
||||
{
|
||||
cancel_pg_calc(pg);
|
||||
drop_dirty_pg_connections({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
|
||||
release_peering_state(pg);
|
||||
if (pg.state & (PG_STOPPING | PG_OFFLINE))
|
||||
@@ -798,8 +790,225 @@ void osd_t::continue_pg(pg_t & pg)
|
||||
{
|
||||
start_pg_peering(pg);
|
||||
}
|
||||
else if ((pg.state & PG_PEERING) && pg.peering_state->locked)
|
||||
else if ((pg.state & PG_PEERING) && pg.peering_state && pg.peering_state->locked)
|
||||
{
|
||||
continue_pg_peering(pg);
|
||||
}
|
||||
}
|
||||
|
||||
void osd_t::cancel_pg_calc(pg_t & pg)
|
||||
{
|
||||
if (!pg.calc_running)
|
||||
return;
|
||||
// Anything not yet inside calc_object_states is dropped here, so a repeer
|
||||
// can LIST again without waiting out a stale sort. The job already popped
|
||||
// by the worker is left to finish; calc_gen makes accept_pg_calc ignore it.
|
||||
pg.calc_gen++;
|
||||
if (!pg_calc_worker)
|
||||
return;
|
||||
pg_calc_job_t *dropped = pg_calc_worker->cancel(pg.pool_id, pg.pg_num);
|
||||
if (!dropped)
|
||||
return;
|
||||
discard_pg_calc_job(dropped);
|
||||
pg.calc_running = false;
|
||||
pg.calc_inflight_gen = 0;
|
||||
}
|
||||
|
||||
// Copies only what calc_object_states / walk() read. A field walk() starts
|
||||
// reading later has to be copied here too: the worker never looks at the live
|
||||
// PG, so a missing copy is a silent wrong peering result.
|
||||
static pg_t *make_pg_calc_scratch(pg_t & pg)
|
||||
{
|
||||
pg_t *s = new pg_t();
|
||||
s->scheme = pg.scheme;
|
||||
s->pg_cursize = pg.pg_cursize;
|
||||
s->pg_size = pg.pg_size;
|
||||
s->pg_data_size = pg.pg_data_size;
|
||||
s->pool_id = pg.pool_id;
|
||||
s->pg_num = pg.pg_num;
|
||||
s->reported_epoch = pg.reported_epoch;
|
||||
s->state = pg.state;
|
||||
s->target_set = pg.target_set;
|
||||
s->cur_set = pg.cur_set;
|
||||
s->cur_peers = pg.cur_peers;
|
||||
s->all_peers = pg.all_peers;
|
||||
s->peering_state = pg.peering_state;
|
||||
pg.peering_state = NULL;
|
||||
return s;
|
||||
}
|
||||
|
||||
void osd_t::submit_pg_object_calc(pg_t & pg)
|
||||
{
|
||||
assert(!pg.calc_running);
|
||||
assert(pg.peering_state);
|
||||
pg.calc_gen++;
|
||||
pg.calc_inflight_gen = pg.calc_gen;
|
||||
pg.calc_running = true;
|
||||
pg_calc_job_t *job = new pg_calc_job_t;
|
||||
job->pool_id = pg.pool_id;
|
||||
job->pg_num = pg.pg_num;
|
||||
job->gen = pg.calc_gen;
|
||||
job->log_level = log_level;
|
||||
job->scratch = make_pg_calc_scratch(pg);
|
||||
// Thread count 0, tests, and a missing worker run inline on this thread.
|
||||
// A live pool returns the OSD loop to read_requests / OSD_OP_PING.
|
||||
if (pg_calc_sync || !pg_calc_worker)
|
||||
{
|
||||
job->scratch->calc_object_states(job->log_level);
|
||||
accept_pg_calc(job);
|
||||
return;
|
||||
}
|
||||
pg_calc_worker->push(job);
|
||||
}
|
||||
|
||||
void osd_t::complete_pg_peering(pg_t & pg)
|
||||
{
|
||||
report_pg_state(pg);
|
||||
schedule_scrub(pg);
|
||||
inconsistent_objects += pg.inconsistent_objects.size();
|
||||
incomplete_objects += pg.incomplete_objects.size();
|
||||
misplaced_objects += pg.misplaced_objects.size();
|
||||
// FIXME: degraded objects may currently include misplaced, too! Report them separately?
|
||||
degraded_objects += pg.degraded_objects.size();
|
||||
if (pg.state & PG_HAS_UNCLEAN)
|
||||
this->peering_state = peering_state | OSD_FLUSHING_PGS;
|
||||
else if (pg.state & (PG_HAS_DEGRADED | PG_HAS_MISPLACED))
|
||||
{
|
||||
this->peering_state = peering_state | OSD_RECOVERING;
|
||||
if (pg.state & PG_HAS_DEGRADED)
|
||||
{
|
||||
// Restart recovery from degraded objects
|
||||
this->recovery_last_degraded = true;
|
||||
this->recovery_last_pg = {};
|
||||
this->recovery_last_oid = {};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void osd_t::accept_pg_calc(pg_calc_job_t *job)
|
||||
{
|
||||
auto pg_it = pgs.find({ .pool_id = job->pool_id, .pg_num = job->pg_num });
|
||||
pg_t *pg = pg_it == pgs.end() ? NULL : &pg_it->second;
|
||||
// calc_inflight_gen is the job this PG is still waiting on. A repeer bumps
|
||||
// calc_gen so the result is dropped, but that in-flight job still owns
|
||||
// calc_running. An older completion must not clear a newer job's flag.
|
||||
bool waited = pg && pg->calc_running && pg->calc_inflight_gen == job->gen;
|
||||
bool apply = waited && !in_destruction && pg->calc_gen == job->gen
|
||||
&& (pg->state & PG_PEERING) && !pg->peering_state;
|
||||
if (waited)
|
||||
{
|
||||
pg->calc_running = false;
|
||||
pg->calc_inflight_gen = 0;
|
||||
}
|
||||
if (!apply)
|
||||
{
|
||||
discard_pg_calc_job(job);
|
||||
if (pg && !in_destruction)
|
||||
continue_pg(*pg);
|
||||
return;
|
||||
}
|
||||
// reported_epoch may advance on the OSD thread while the worker is
|
||||
// calculating from an older snapshot. Never let the installed result
|
||||
// move epoch behind the value already persisted in etcd. Non-clean
|
||||
// activation must use a new epoch strictly above that persisted value.
|
||||
pg->epoch = std::max(job->scratch->epoch, pg->reported_epoch);
|
||||
if (job->scratch->state != PG_ACTIVE && pg->epoch == pg->reported_epoch)
|
||||
{
|
||||
assert(pg->epoch != (((uint64_t)1 << PG_EPOCH_BITS)-1));
|
||||
pg->epoch++;
|
||||
}
|
||||
pg->clean_count = job->scratch->clean_count;
|
||||
pg->total_count = job->scratch->total_count;
|
||||
pg->state = job->scratch->state;
|
||||
// Object btrees store pointers into state_dict nodes. std::map::swap keeps
|
||||
// those addresses. btree_map has a user-defined copy constructor and no move,
|
||||
// so swap is what keeps the pointers valid.
|
||||
pg->state_dict.swap(job->scratch->state_dict);
|
||||
pg->inconsistent_objects.swap(job->scratch->inconsistent_objects);
|
||||
pg->incomplete_objects.swap(job->scratch->incomplete_objects);
|
||||
pg->misplaced_objects.swap(job->scratch->misplaced_objects);
|
||||
pg->degraded_objects.swap(job->scratch->degraded_objects);
|
||||
pg->flush_actions.swap(job->scratch->flush_actions);
|
||||
pg->ver_override.swap(job->scratch->ver_override);
|
||||
delete job->scratch;
|
||||
job->scratch = NULL;
|
||||
delete job;
|
||||
complete_pg_peering(*pg);
|
||||
}
|
||||
|
||||
void osd_t::drain_pg_calc()
|
||||
{
|
||||
if (!pg_calc_worker)
|
||||
return;
|
||||
for (auto *job: pg_calc_worker->drain())
|
||||
accept_pg_calc(job);
|
||||
}
|
||||
|
||||
void osd_t::wait_pg_calc()
|
||||
{
|
||||
if (!pg_calc_worker)
|
||||
return;
|
||||
for (;;)
|
||||
{
|
||||
pg_calc_worker->wait_not_busy();
|
||||
drain_pg_calc();
|
||||
if (pg_calc_worker->idle())
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void osd_t::wait_pg_calc_not_busy()
|
||||
{
|
||||
if (!pg_calc_worker)
|
||||
return;
|
||||
pg_calc_worker->wait_not_busy();
|
||||
}
|
||||
|
||||
void osd_t::start_pg_calc_worker()
|
||||
{
|
||||
if (pg_calc_sync || pg_calc_thread_count <= 0 || pg_calc_worker)
|
||||
return;
|
||||
pg_calc_worker = new pg_calc_worker_t(pg_calc_thread_count, pg_calc_hold, pg_calc_pause, pg_calc_runs);
|
||||
pg_calc_worker->add_to_ringloop(ringloop);
|
||||
}
|
||||
|
||||
void osd_t::pg_calc_release_hold()
|
||||
{
|
||||
pg_calc_hold = false;
|
||||
if (pg_calc_worker)
|
||||
pg_calc_worker->release_hold();
|
||||
}
|
||||
|
||||
void osd_t::pg_calc_unpause()
|
||||
{
|
||||
pg_calc_pause = false;
|
||||
if (pg_calc_worker)
|
||||
pg_calc_worker->unpause();
|
||||
}
|
||||
|
||||
void osd_t::wait_pg_calc_paused()
|
||||
{
|
||||
if (!pg_calc_worker)
|
||||
return;
|
||||
pg_calc_worker->wait_paused();
|
||||
}
|
||||
|
||||
size_t osd_t::pg_calc_queued()
|
||||
{
|
||||
if (!pg_calc_worker)
|
||||
return 0;
|
||||
return pg_calc_worker->queued();
|
||||
}
|
||||
|
||||
void osd_t::stop_pg_calc_worker()
|
||||
{
|
||||
if (!pg_calc_worker)
|
||||
return;
|
||||
delete pg_calc_worker;
|
||||
pg_calc_worker = NULL;
|
||||
for (auto & p: pgs)
|
||||
{
|
||||
p.second.calc_running = false;
|
||||
p.second.calc_inflight_gen = 0;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -413,6 +413,8 @@ retry:
|
||||
}
|
||||
|
||||
// FIXME: Write at least some tests for this function
|
||||
// May run on the pg-calc thread. Only touches this pg — the OSD hands it a
|
||||
// scratch copy so the event loop never shares these maps with the worker.
|
||||
void pg_t::calc_object_states(int log_level)
|
||||
{
|
||||
// Copy all object lists into one array
|
||||
|
||||
@@ -127,6 +127,18 @@ struct pg_t
|
||||
btree::btree_map<object_id, uint64_t> ver_override;
|
||||
pg_peering_state_t *peering_state = NULL;
|
||||
pg_flush_batch_t *flush_batch = NULL;
|
||||
// calc_object_states runs on the calc pool so it cannot stall OSD_OP_PING
|
||||
// in the main loop. While it runs, peering_state lives on a scratch PG and
|
||||
// the main thread does not touch the object maps.
|
||||
// stop / repeer bump calc_gen, so a result that is not yet installed is
|
||||
// dropped. A job still in the queue or in done is discarded and
|
||||
// calc_running is cleared, and the new peering can LIST. A job already
|
||||
// popped cannot be stopped: calc_inflight_gen still names it, the result
|
||||
// is discarded by generation, and only that completion clears its own
|
||||
// calc_running. It does not clear the flag of a later calculation.
|
||||
bool calc_running = false;
|
||||
uint64_t calc_gen = 0;
|
||||
uint64_t calc_inflight_gen = 0;
|
||||
|
||||
int inflight = 0; // including write_queue
|
||||
std::multimap<object_id, osd_op_t*> write_queue;
|
||||
|
||||
@@ -3074,6 +3074,650 @@ void test_pg_epoch_base_is_reported_epoch()
|
||||
printf("test_pg_epoch_base_is_reported_epoch passed\n");
|
||||
}
|
||||
|
||||
// calc_object_states must not run inside the OSD loop: a large object list used
|
||||
// to block OSD_OP_PING until peers declared the OSD dead. The worker applies the
|
||||
// same result on the next drain.
|
||||
void test_calc_object_states_off_thread()
|
||||
{
|
||||
printf("test_calc_object_states_off_thread\n");
|
||||
|
||||
osd_test_fixture_t f;
|
||||
f.configure_replicated_pool(/*pool_id*/ 1, /*pg_size*/ 2, /*pg_minsize*/ 1, /*pg_count*/ 1,
|
||||
{ { 1, 2 } });
|
||||
f.construct(json11::Json::object {
|
||||
{ "osd_num", 1 },
|
||||
{ "etcd_address", "127.0.0.1:2379" },
|
||||
{ "immediate_commit", "none" },
|
||||
{ "block_size", 131072 },
|
||||
{ "bitmap_granularity", 4096 },
|
||||
{ "no_recovery", true },
|
||||
});
|
||||
f.set_pg_calc_sync(false);
|
||||
f.osd->start();
|
||||
f.connect_peer(2);
|
||||
f.reply_local_list({}, 0);
|
||||
f.reply_peer_list(2, {}, 0);
|
||||
f.ringloop->loop();
|
||||
// The loop only queues the job. Waiting here is what installs PG_ACTIVE.
|
||||
assert(f.pg(1, 1).state == PG_PEERING);
|
||||
assert(f.pg(1, 1).calc_running);
|
||||
f.wait_pg_calc();
|
||||
assert(f.pg(1, 1).state == PG_ACTIVE);
|
||||
assert(!f.pg(1, 1).calc_running);
|
||||
assert(f.pg(1, 1).peering_state == NULL);
|
||||
assert(f.pg(1, 1).total_count == 0);
|
||||
|
||||
printf("test_calc_object_states_off_thread passed\n");
|
||||
}
|
||||
|
||||
static json11::Json::object calc_test_osd_config()
|
||||
{
|
||||
return json11::Json::object {
|
||||
{ "osd_num", 1 },
|
||||
{ "etcd_address", "127.0.0.1:2379" },
|
||||
{ "immediate_commit", "none" },
|
||||
{ "block_size", 131072 },
|
||||
{ "bitmap_granularity", 4096 },
|
||||
{ "no_recovery", true },
|
||||
};
|
||||
}
|
||||
|
||||
static void configure_calc_pool(osd_test_fixture_t & f)
|
||||
{
|
||||
f.configure_replicated_pool(/*pool_id*/ 1, /*pg_size*/ 2, /*pg_minsize*/ 1, /*pg_count*/ 1,
|
||||
{ { 1, 2 } });
|
||||
}
|
||||
|
||||
struct flush_snap_t
|
||||
{
|
||||
object_id oid;
|
||||
osd_num_t osd_num = 0;
|
||||
bool rollback = false, make_stable = false;
|
||||
uint64_t rollback_to = 0, stable_to = 0;
|
||||
};
|
||||
|
||||
inline bool operator==(const flush_snap_t & a, const flush_snap_t & b)
|
||||
{
|
||||
return a.oid == b.oid && a.osd_num == b.osd_num && a.rollback == b.rollback &&
|
||||
a.make_stable == b.make_stable && a.rollback_to == b.rollback_to && a.stable_to == b.stable_to;
|
||||
}
|
||||
|
||||
struct pg_calc_snap_t
|
||||
{
|
||||
int state = 0;
|
||||
uint64_t epoch = 0, clean_count = 0, total_count = 0;
|
||||
std::vector<object_id> degraded, misplaced, incomplete, inconsistent;
|
||||
std::vector<flush_snap_t> flush;
|
||||
std::vector<std::pair<object_id, uint64_t>> ver;
|
||||
std::vector<std::pair<pg_osd_set_t, std::pair<uint64_t, uint64_t>>> dict;
|
||||
std::vector<std::vector<osd_num_t>> read_targets;
|
||||
};
|
||||
|
||||
static std::vector<object_id> object_keys(btree::btree_map<object_id, pg_osd_set_state_t*> & m)
|
||||
{
|
||||
std::vector<object_id> keys;
|
||||
for (auto & kv: m)
|
||||
keys.push_back(kv.first);
|
||||
return keys;
|
||||
}
|
||||
|
||||
static pg_calc_snap_t snap_pg(pg_t & pg)
|
||||
{
|
||||
pg_calc_snap_t s;
|
||||
s.state = pg.state;
|
||||
s.epoch = pg.epoch;
|
||||
s.clean_count = pg.clean_count;
|
||||
s.total_count = pg.total_count;
|
||||
s.degraded = object_keys(pg.degraded_objects);
|
||||
s.misplaced = object_keys(pg.misplaced_objects);
|
||||
s.incomplete = object_keys(pg.incomplete_objects);
|
||||
s.inconsistent = object_keys(pg.inconsistent_objects);
|
||||
for (auto & kv: pg.flush_actions)
|
||||
{
|
||||
s.flush.push_back((flush_snap_t){
|
||||
.oid = kv.first.oid,
|
||||
.osd_num = kv.first.osd_num,
|
||||
.rollback = kv.second.rollback,
|
||||
.make_stable = kv.second.make_stable,
|
||||
.rollback_to = kv.second.rollback_to,
|
||||
.stable_to = kv.second.stable_to,
|
||||
});
|
||||
}
|
||||
for (auto & kv: pg.ver_override)
|
||||
s.ver.push_back({ kv.first, kv.second });
|
||||
for (auto & kv: pg.state_dict)
|
||||
{
|
||||
s.dict.push_back({ kv.first, { kv.second.state, kv.second.object_count } });
|
||||
s.read_targets.push_back(kv.second.read_target);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
static bool same_osd_set(const pg_osd_set_t & a, const pg_osd_set_t & b)
|
||||
{
|
||||
if (a.size() != b.size())
|
||||
return false;
|
||||
for (size_t i = 0; i < a.size(); i++)
|
||||
{
|
||||
if (a[i].role != b[i].role || a[i].osd_num != b[i].osd_num || a[i].loc_bad != b[i].loc_bad)
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static void assert_snaps_equal(const pg_calc_snap_t & a, const pg_calc_snap_t & b)
|
||||
{
|
||||
assert(a.state == b.state);
|
||||
assert(a.epoch == b.epoch);
|
||||
assert(a.clean_count == b.clean_count);
|
||||
assert(a.total_count == b.total_count);
|
||||
assert(a.degraded == b.degraded);
|
||||
assert(a.misplaced == b.misplaced);
|
||||
assert(a.incomplete == b.incomplete);
|
||||
assert(a.inconsistent == b.inconsistent);
|
||||
assert(a.flush == b.flush);
|
||||
assert(a.ver == b.ver);
|
||||
assert(a.dict.size() == b.dict.size());
|
||||
assert(a.read_targets.size() == b.read_targets.size());
|
||||
for (size_t i = 0; i < a.dict.size(); i++)
|
||||
{
|
||||
assert(same_osd_set(a.dict[i].first, b.dict[i].first));
|
||||
assert(a.dict[i].second == b.dict[i].second);
|
||||
assert(a.read_targets[i] == b.read_targets[i]);
|
||||
}
|
||||
}
|
||||
|
||||
static pg_calc_snap_t peer_one_object(bool async, bool both_copies, int thread_count = -1)
|
||||
{
|
||||
osd_test_fixture_t f;
|
||||
configure_calc_pool(f);
|
||||
inode_t inode = INODE_WITH_POOL(1, 1);
|
||||
obj_ver_id obj = { .oid = { .inode = inode, .stripe = 0 }, .version = 1 };
|
||||
auto cfg = calc_test_osd_config();
|
||||
if (thread_count >= 0)
|
||||
cfg["pg_calc_thread_count"] = thread_count;
|
||||
if (async)
|
||||
{
|
||||
f.construct(cfg);
|
||||
f.set_pg_calc_sync(false);
|
||||
f.osd->start();
|
||||
}
|
||||
else
|
||||
f.start(cfg);
|
||||
f.connect_peer(2);
|
||||
f.reply_local_list({ obj }, 1);
|
||||
f.reply_peer_list(2, both_copies ? std::vector<obj_ver_id>{ obj } : std::vector<obj_ver_id>{}, 1);
|
||||
f.ringloop->loop();
|
||||
f.wait_pg_calc();
|
||||
return snap_pg(f.pg(1, 1));
|
||||
}
|
||||
|
||||
// The worker must install the same object map as the inline path.
|
||||
void test_calc_object_states_matches_sync()
|
||||
{
|
||||
printf("test_calc_object_states_matches_sync\n");
|
||||
assert_snaps_equal(peer_one_object(false, true), peer_one_object(true, true));
|
||||
assert_snaps_equal(peer_one_object(false, false), peer_one_object(true, false));
|
||||
assert(peer_one_object(false, false).degraded.size() == 1);
|
||||
// Two workers share the queue and still install the same map.
|
||||
assert_snaps_equal(peer_one_object(false, true), peer_one_object(true, true, 2));
|
||||
assert_snaps_equal(peer_one_object(false, false), peer_one_object(true, false, 2));
|
||||
printf("test_calc_object_states_matches_sync passed\n");
|
||||
}
|
||||
|
||||
// pg_calc_thread_count 0 does not start the pool. calc_object_states runs
|
||||
// on the OSD thread and peering finishes inside the same loop.
|
||||
void test_calc_thread_count_zero()
|
||||
{
|
||||
printf("test_calc_thread_count_zero\n");
|
||||
osd_test_fixture_t f;
|
||||
configure_calc_pool(f);
|
||||
auto cfg = calc_test_osd_config();
|
||||
cfg["pg_calc_thread_count"] = 0;
|
||||
f.construct(cfg);
|
||||
f.set_pg_calc_sync(false);
|
||||
f.osd->start();
|
||||
f.connect_peer(2);
|
||||
f.reply_local_list({}, 0);
|
||||
f.reply_peer_list(2, {}, 0);
|
||||
f.ringloop->loop();
|
||||
assert(f.pg(1, 1).state == PG_ACTIVE);
|
||||
assert(!f.pg(1, 1).calc_running);
|
||||
assert(f.pg(1, 1).peering_state == NULL);
|
||||
assert(f.pg_calc_queued() == 0);
|
||||
printf("test_calc_thread_count_zero passed\n");
|
||||
}
|
||||
|
||||
// A job still sitting in the queue is dropped. The new peering lists immediately,
|
||||
// and calc_object_states for the cancelled generation never runs.
|
||||
void test_cancel_queued_pg_calc()
|
||||
{
|
||||
printf("test_cancel_queued_pg_calc\n");
|
||||
int runs = 0;
|
||||
{
|
||||
osd_test_fixture_t f;
|
||||
configure_calc_pool(f);
|
||||
f.construct(calc_test_osd_config());
|
||||
f.set_pg_calc_sync(false);
|
||||
f.set_pg_calc_hold(true);
|
||||
f.set_pg_calc_runs(&runs);
|
||||
f.osd->start();
|
||||
f.connect_peer(2);
|
||||
f.reply_local_list({}, 0);
|
||||
f.reply_peer_list(2, {}, 0);
|
||||
f.ringloop->loop();
|
||||
assert(f.pg(1, 1).state == PG_PEERING);
|
||||
assert(f.pg(1, 1).calc_running);
|
||||
assert(f.pg(1, 1).calc_inflight_gen != 0);
|
||||
assert(f.pg_calc_queued() == 1);
|
||||
assert(runs == 0);
|
||||
|
||||
f.repeer_pg(1, 1);
|
||||
assert(!f.pg(1, 1).calc_running);
|
||||
assert(f.pg(1, 1).calc_inflight_gen == 0);
|
||||
assert(f.pg_calc_queued() == 0);
|
||||
assert(runs == 0);
|
||||
assert(f.pg(1, 1).peering_state != NULL);
|
||||
assert(f.pending_list_count() == 2);
|
||||
|
||||
f.reply_local_list({}, 0);
|
||||
f.reply_peer_list(2, {}, 0);
|
||||
f.ringloop->loop();
|
||||
assert(f.pg(1, 1).calc_running);
|
||||
assert(f.pg_calc_queued() == 1);
|
||||
assert(runs == 0);
|
||||
f.release_pg_calc_hold();
|
||||
f.wait_pg_calc();
|
||||
assert(runs == 1);
|
||||
assert(f.pg(1, 1).state == PG_ACTIVE);
|
||||
assert(!f.pg(1, 1).calc_running);
|
||||
}
|
||||
printf("test_cancel_queued_pg_calc passed\n");
|
||||
}
|
||||
|
||||
// stop_pg drops a queued calc instead of leaving calc_running set.
|
||||
void test_stop_drops_queued_pg_calc()
|
||||
{
|
||||
printf("test_stop_drops_queued_pg_calc\n");
|
||||
int runs = 0;
|
||||
{
|
||||
osd_test_fixture_t f;
|
||||
configure_calc_pool(f);
|
||||
f.construct(calc_test_osd_config());
|
||||
f.set_pg_calc_sync(false);
|
||||
f.set_pg_calc_hold(true);
|
||||
f.set_pg_calc_runs(&runs);
|
||||
f.osd->start();
|
||||
f.connect_peer(2);
|
||||
f.reply_local_list({}, 0);
|
||||
f.reply_peer_list(2, {}, 0);
|
||||
f.ringloop->loop();
|
||||
assert(f.pg_calc_queued() == 1);
|
||||
f.stop_pg(1, 1);
|
||||
// stop reports PG_OFFLINE, and report may apply the config again and
|
||||
// start a fresh peering. Either way the queued calc is gone.
|
||||
assert(!f.pg(1, 1).calc_running);
|
||||
assert(f.pg(1, 1).calc_inflight_gen == 0);
|
||||
assert(f.pg_calc_queued() == 0);
|
||||
assert(!(f.pg(1, 1).state & PG_ACTIVE));
|
||||
assert(runs == 0);
|
||||
}
|
||||
assert(runs == 0);
|
||||
printf("test_stop_drops_queued_pg_calc passed\n");
|
||||
}
|
||||
|
||||
// Destroying the OSD drops queued calcs without running them.
|
||||
void test_shutdown_drops_queued_pg_calc()
|
||||
{
|
||||
printf("test_shutdown_drops_queued_pg_calc\n");
|
||||
int runs = 0;
|
||||
{
|
||||
osd_test_fixture_t f;
|
||||
configure_calc_pool(f);
|
||||
f.construct(calc_test_osd_config());
|
||||
f.set_pg_calc_sync(false);
|
||||
f.set_pg_calc_hold(true);
|
||||
f.set_pg_calc_runs(&runs);
|
||||
f.osd->start();
|
||||
f.connect_peer(2);
|
||||
f.reply_local_list({}, 0);
|
||||
f.reply_peer_list(2, {}, 0);
|
||||
f.ringloop->loop();
|
||||
assert(f.pg_calc_queued() == 1);
|
||||
assert(runs == 0);
|
||||
}
|
||||
assert(runs == 0);
|
||||
printf("test_shutdown_drops_queued_pg_calc passed\n");
|
||||
}
|
||||
|
||||
// A job already popped keeps running, but its result is not installed, and the
|
||||
// new peering does not LIST until that result has been dropped.
|
||||
void test_cancel_busy_pg_calc()
|
||||
{
|
||||
printf("test_cancel_busy_pg_calc\n");
|
||||
int runs = 0;
|
||||
osd_test_fixture_t f;
|
||||
configure_calc_pool(f);
|
||||
f.construct(calc_test_osd_config());
|
||||
f.set_pg_calc_sync(false);
|
||||
f.set_pg_calc_pause(true);
|
||||
f.set_pg_calc_runs(&runs);
|
||||
f.osd->start();
|
||||
f.connect_peer(2);
|
||||
f.reply_local_list({}, 0);
|
||||
f.reply_peer_list(2, {}, 0);
|
||||
f.ringloop->loop();
|
||||
f.wait_pg_calc_paused();
|
||||
assert(f.pg(1, 1).calc_running);
|
||||
assert(f.pg(1, 1).calc_inflight_gen != 0);
|
||||
assert(f.pg_calc_queued() == 0);
|
||||
assert(runs == 0);
|
||||
uint64_t inflight = f.pg(1, 1).calc_inflight_gen;
|
||||
|
||||
f.repeer_pg(1, 1);
|
||||
assert(f.pg(1, 1).calc_running);
|
||||
assert(f.pg(1, 1).calc_inflight_gen == inflight);
|
||||
assert(f.pg(1, 1).calc_gen != inflight);
|
||||
assert(f.pg(1, 1).peering_state != NULL);
|
||||
assert(f.pending_list_count() == 0);
|
||||
assert(runs == 0);
|
||||
|
||||
f.unpause_pg_calc();
|
||||
f.wait_pg_calc();
|
||||
assert(runs == 1);
|
||||
assert(!f.pg(1, 1).calc_running);
|
||||
assert(f.pg(1, 1).state == PG_PEERING);
|
||||
assert(f.pending_list_count() == 2);
|
||||
|
||||
f.reply_local_list({}, 0);
|
||||
f.reply_peer_list(2, {}, 0);
|
||||
f.ringloop->loop();
|
||||
f.wait_pg_calc();
|
||||
assert(runs == 2);
|
||||
assert(f.pg(1, 1).state == PG_ACTIVE);
|
||||
assert(!f.pg(1, 1).calc_running);
|
||||
printf("test_cancel_busy_pg_calc passed\n");
|
||||
}
|
||||
|
||||
static uint64_t calc_after_reported_epoch_advances(bool clean)
|
||||
{
|
||||
osd_test_fixture_t f;
|
||||
configure_calc_pool(f);
|
||||
f.st_cli->set("/vitastor/pg/history/1/1", json11::Json::object {
|
||||
{ "osd_sets", json11::Json::array { json11::Json::array { 1, 2 } } },
|
||||
{ "all_peers", json11::Json::array { 1, 2 } },
|
||||
{ "epoch", 7 },
|
||||
});
|
||||
f.construct(calc_test_osd_config());
|
||||
f.set_pg_calc_sync(false);
|
||||
f.set_pg_calc_pause(true);
|
||||
f.osd->start();
|
||||
// Model a previous non-clean activation whose epoch 8 report is still
|
||||
// waiting for its etcd watch confirmation when this peering starts.
|
||||
f.pg(1, 1).epoch = 8;
|
||||
f.connect_peer(2);
|
||||
obj_ver_id obj = {
|
||||
.oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 },
|
||||
.version = 1,
|
||||
};
|
||||
f.reply_local_list(clean ? std::vector<obj_ver_id>{} : std::vector<obj_ver_id>{ obj }, clean ? 0 : 1);
|
||||
f.reply_peer_list(2, {}, 0);
|
||||
f.ringloop->loop();
|
||||
f.wait_pg_calc_paused();
|
||||
assert(f.pg(1, 1).reported_epoch == 7);
|
||||
|
||||
// The worker already owns a scratch PG with reported_epoch=7. Confirming
|
||||
// the previous report now advances only the live PG to epoch 8.
|
||||
f.confirm_pg_epoch(1, 1, 8);
|
||||
assert(f.pg(1, 1).reported_epoch == 8);
|
||||
f.unpause_pg_calc();
|
||||
f.wait_pg_calc();
|
||||
return f.pg(1, 1).epoch;
|
||||
}
|
||||
|
||||
// Installing an asynchronous result must reconcile it with reported_epoch
|
||||
// updates that happened after the scratch PG was created.
|
||||
void test_calc_epoch_reconciles_reported_update()
|
||||
{
|
||||
printf("test_calc_epoch_reconciles_reported_update\n");
|
||||
// A clean activation may reuse the persisted epoch, but never an older one.
|
||||
assert(calc_after_reported_epoch_advances(true) == 8);
|
||||
// A non-clean activation must allocate an epoch strictly above the persisted one.
|
||||
assert(calc_after_reported_epoch_advances(false) == 9);
|
||||
printf("test_calc_epoch_reconciles_reported_update passed\n");
|
||||
}
|
||||
|
||||
// Replicated PG: v1 stable on both OSDs, v2 unstable on both. Peering keeps v1
|
||||
// and rolls v2 back, which fills ver_override and flush_actions.
|
||||
static pg_calc_snap_t peer_unstable_replicated(bool async)
|
||||
{
|
||||
osd_test_fixture_t f;
|
||||
configure_calc_pool(f);
|
||||
auto cfg = calc_test_osd_config();
|
||||
cfg["no_rebalance"] = true;
|
||||
inode_t inode = INODE_WITH_POOL(1, 1);
|
||||
obj_ver_id v1 = { .oid = { .inode = inode, .stripe = 0 }, .version = 1 };
|
||||
obj_ver_id v2 = { .oid = { .inode = inode, .stripe = 0 }, .version = 2 };
|
||||
if (async)
|
||||
{
|
||||
f.construct(cfg);
|
||||
f.set_pg_calc_sync(false);
|
||||
f.osd->start();
|
||||
}
|
||||
else
|
||||
f.start(cfg);
|
||||
f.connect_peer(2);
|
||||
f.reply_local_list({ v1, v2 }, 1);
|
||||
f.reply_peer_list(2, { v1, v2 }, 1);
|
||||
f.ringloop->loop();
|
||||
f.wait_pg_calc();
|
||||
return snap_pg(f.pg(1, 1));
|
||||
}
|
||||
|
||||
// EC 2+2 stays incomplete until every member is connected, so the LIST the
|
||||
// test answers is the full-set one.
|
||||
static void configure_ec_calc_pool(osd_test_fixture_t & f)
|
||||
{
|
||||
f.configure_ec_pool(/*data_chunks*/ 2, /*parity_chunks*/ 2, /*primary_osd*/ 1);
|
||||
f.st_cli->set("/vitastor/config/pools", json11::Json::object {
|
||||
{ "1", json11::Json::object {
|
||||
{ "name", "pool_1" },
|
||||
{ "scheme", "ec" },
|
||||
{ "pg_size", 4 },
|
||||
{ "pg_minsize", 4 },
|
||||
{ "parity_chunks", 2 },
|
||||
{ "pg_count", 1 },
|
||||
{ "failure_domain", "osd" },
|
||||
{ "immediate_commit", "none" },
|
||||
} },
|
||||
});
|
||||
}
|
||||
|
||||
static obj_ver_id ec_piece(inode_t inode, uint64_t role, uint64_t version)
|
||||
{
|
||||
return { .oid = { .inode = inode, .stripe = role }, .version = version };
|
||||
}
|
||||
|
||||
static pg_calc_snap_t peer_ec(bool async, bool rollback)
|
||||
{
|
||||
osd_test_fixture_t f;
|
||||
configure_ec_calc_pool(f);
|
||||
auto cfg = calc_test_osd_config();
|
||||
cfg["no_rebalance"] = true;
|
||||
inode_t inode = INODE_WITH_POOL(1, 1);
|
||||
if (async)
|
||||
{
|
||||
f.construct(cfg);
|
||||
f.set_pg_calc_sync(false);
|
||||
f.osd->start();
|
||||
}
|
||||
else
|
||||
f.start(cfg);
|
||||
f.connect_peer(2);
|
||||
f.connect_peer(3);
|
||||
f.connect_peer(4);
|
||||
if (rollback)
|
||||
{
|
||||
// Stable v1 on every role. Unstable v2 only on roles 0 and 2, so it
|
||||
// must not become the target; those two pieces roll back to v1.
|
||||
f.reply_local_list({ ec_piece(inode, 0, 1), ec_piece(inode, 0, 2) }, 1);
|
||||
f.reply_peer_list(2, { ec_piece(inode, 1, 1) }, 1);
|
||||
f.reply_peer_list(3, { ec_piece(inode, 2, 1), ec_piece(inode, 2, 2) }, 1);
|
||||
f.reply_peer_list(4, { ec_piece(inode, 3, 1) }, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
// v2 is the target on every role. OSD 2 also holds an older role-3
|
||||
// piece; cur_set[3] is OSD 4, so that piece stays outdated.
|
||||
f.reply_local_list({ ec_piece(inode, 0, 2) }, 1);
|
||||
f.reply_peer_list(2, { ec_piece(inode, 1, 2), ec_piece(inode, 3, 1) }, 2);
|
||||
f.reply_peer_list(3, { ec_piece(inode, 2, 2) }, 1);
|
||||
f.reply_peer_list(4, { ec_piece(inode, 3, 2) }, 1);
|
||||
}
|
||||
f.ringloop->loop();
|
||||
f.wait_pg_calc();
|
||||
return snap_pg(f.pg(1, 1));
|
||||
}
|
||||
|
||||
// The worker must install flush_actions, ver_override, and EC role placement
|
||||
// the same way the inline path does. Those fields are not covered by a
|
||||
// single clean replicated object.
|
||||
void test_calc_object_states_matches_sync_extra()
|
||||
{
|
||||
printf("test_calc_object_states_matches_sync_extra\n");
|
||||
inode_t inode = INODE_WITH_POOL(1, 1);
|
||||
object_id oid = { .inode = inode, .stripe = 0 };
|
||||
|
||||
auto sync_u = peer_unstable_replicated(false);
|
||||
auto async_u = peer_unstable_replicated(true);
|
||||
assert_snaps_equal(sync_u, async_u);
|
||||
assert(sync_u.state & PG_HAS_UNCLEAN);
|
||||
assert(sync_u.ver.size() == 1 && sync_u.ver[0].first == oid && sync_u.ver[0].second == 1);
|
||||
assert(sync_u.flush.size() == 2);
|
||||
for (auto & fa: sync_u.flush)
|
||||
assert(fa.rollback && !fa.make_stable && fa.rollback_to == 1);
|
||||
|
||||
auto sync_r = peer_ec(false, true);
|
||||
auto async_r = peer_ec(true, true);
|
||||
assert_snaps_equal(sync_r, async_r);
|
||||
assert(sync_r.state & PG_HAS_UNCLEAN);
|
||||
assert(!(sync_r.state & (PG_HAS_INCOMPLETE | PG_HAS_DEGRADED)));
|
||||
assert(sync_r.clean_count == 1 && sync_r.total_count == 1);
|
||||
assert(sync_r.ver.size() == 1 && sync_r.ver[0].second == 1);
|
||||
assert(sync_r.flush.size() == 2);
|
||||
for (auto & fa: sync_r.flush)
|
||||
{
|
||||
assert(fa.rollback && !fa.make_stable && fa.rollback_to == 1);
|
||||
assert((fa.oid.stripe & STRIPE_MASK) == (fa.osd_num == 1 ? 0u : 2u));
|
||||
}
|
||||
|
||||
auto sync_m = peer_ec(false, false);
|
||||
auto async_m = peer_ec(true, false);
|
||||
assert_snaps_equal(sync_m, async_m);
|
||||
assert(sync_m.state & PG_HAS_MISPLACED);
|
||||
assert(sync_m.misplaced.size() == 1);
|
||||
assert(sync_m.read_targets.size() == 1);
|
||||
assert(sync_m.read_targets[0].size() == 4 && sync_m.read_targets[0][3] == 4);
|
||||
printf("test_calc_object_states_matches_sync_extra passed\n");
|
||||
}
|
||||
|
||||
// A finished calc sits in the worker's done queue until the OSD thread
|
||||
// drains it. Cancel must drop that result instead of installing it, and the
|
||||
// new peering can LIST without waiting out another sort.
|
||||
void test_cancel_done_pg_calc()
|
||||
{
|
||||
printf("test_cancel_done_pg_calc\n");
|
||||
int runs = 0;
|
||||
osd_test_fixture_t f;
|
||||
configure_calc_pool(f);
|
||||
f.construct(calc_test_osd_config());
|
||||
f.set_pg_calc_sync(false);
|
||||
f.set_pg_calc_runs(&runs);
|
||||
f.osd->start();
|
||||
f.connect_peer(2);
|
||||
f.reply_local_list({}, 0);
|
||||
f.reply_peer_list(2, {}, 0);
|
||||
f.ringloop->loop();
|
||||
assert(f.pg(1, 1).calc_running);
|
||||
f.wait_pg_calc_not_busy();
|
||||
assert(runs == 1);
|
||||
assert(f.pg_calc_queued() == 0);
|
||||
assert(f.pg(1, 1).calc_running);
|
||||
assert(f.pg(1, 1).state == PG_PEERING);
|
||||
|
||||
f.repeer_pg(1, 1);
|
||||
assert(runs == 1);
|
||||
assert(!f.pg(1, 1).calc_running);
|
||||
assert(f.pg(1, 1).calc_inflight_gen == 0);
|
||||
assert(f.pg(1, 1).state == PG_PEERING);
|
||||
assert(f.pg(1, 1).peering_state != NULL);
|
||||
assert(f.pending_list_count() == 2);
|
||||
|
||||
f.reply_local_list({}, 0);
|
||||
f.reply_peer_list(2, {}, 0);
|
||||
f.ringloop->loop();
|
||||
f.wait_pg_calc();
|
||||
assert(runs == 2);
|
||||
assert(f.pg(1, 1).state == PG_ACTIVE);
|
||||
assert(!f.pg(1, 1).calc_running);
|
||||
printf("test_cancel_done_pg_calc passed\n");
|
||||
}
|
||||
|
||||
// Object-version epochs are copied onto the scratch PG. A reported_epoch that
|
||||
// moves while that calc is in flight is reconciled on install: a higher
|
||||
// version epoch is kept, and a non-clean activation still lands strictly
|
||||
// above the persisted epoch.
|
||||
static uint64_t calc_epoch_with_version(uint64_t version_epoch, uint64_t reported_after, bool clean)
|
||||
{
|
||||
osd_test_fixture_t f;
|
||||
configure_calc_pool(f);
|
||||
f.st_cli->set("/vitastor/pg/history/1/1", json11::Json::object {
|
||||
{ "osd_sets", json11::Json::array { json11::Json::array { 1, 2 } } },
|
||||
{ "all_peers", json11::Json::array { 1, 2 } },
|
||||
{ "epoch", 7 },
|
||||
});
|
||||
f.construct(calc_test_osd_config());
|
||||
f.set_pg_calc_sync(false);
|
||||
f.set_pg_calc_pause(true);
|
||||
f.osd->start();
|
||||
f.connect_peer(2);
|
||||
uint64_t ver = (version_epoch << (64 - PG_EPOCH_BITS)) | 1;
|
||||
obj_ver_id obj = {
|
||||
.oid = { .inode = INODE_WITH_POOL(1, 1), .stripe = 0 },
|
||||
.version = ver,
|
||||
};
|
||||
f.reply_local_list({ obj }, 1);
|
||||
f.reply_peer_list(2, clean ? std::vector<obj_ver_id>{ obj } : std::vector<obj_ver_id>{}, clean ? 1 : 0);
|
||||
f.ringloop->loop();
|
||||
f.wait_pg_calc_paused();
|
||||
assert(f.pg(1, 1).reported_epoch == 7);
|
||||
// The history hook only advances reported_epoch when the live epoch is
|
||||
// already at least the value being confirmed.
|
||||
f.pg(1, 1).epoch = reported_after;
|
||||
f.confirm_pg_epoch(1, 1, reported_after);
|
||||
assert(f.pg(1, 1).reported_epoch == reported_after);
|
||||
f.unpause_pg_calc();
|
||||
f.wait_pg_calc();
|
||||
if (clean)
|
||||
assert(f.pg(1, 1).state == PG_ACTIVE);
|
||||
else
|
||||
assert(f.pg(1, 1).state & PG_HAS_DEGRADED);
|
||||
return f.pg(1, 1).epoch;
|
||||
}
|
||||
|
||||
void test_calc_epoch_keeps_version_epoch()
|
||||
{
|
||||
printf("test_calc_epoch_keeps_version_epoch\n");
|
||||
// Version epoch 10, reported moves 7 -> 12. Clean reuses 12.
|
||||
assert(calc_epoch_with_version(10, 12, true) == 12);
|
||||
// Non-clean scratch epoch is 11; reported catches it, so install bumps to 12.
|
||||
assert(calc_epoch_with_version(10, 11, false) == 12);
|
||||
// Version epoch 20 stays above a reported_epoch that only moved to 9.
|
||||
assert(calc_epoch_with_version(20, 9, true) == 20);
|
||||
assert(calc_epoch_with_version(20, 9, false) == 21);
|
||||
printf("test_calc_epoch_keeps_version_epoch passed\n");
|
||||
}
|
||||
|
||||
// A non-clean activation must bump the epoch above the persisted base, not above
|
||||
// the version epochs of surviving copies
|
||||
void test_pg_epoch_bump_above_reported_epoch()
|
||||
@@ -3757,6 +4401,17 @@ int main(int narg, char *args[])
|
||||
test_read_enoent_without_racing_op_marks_corrupted();
|
||||
test_peering_outdated_copy_other_role();
|
||||
test_pg_epoch_base_is_reported_epoch();
|
||||
test_calc_object_states_off_thread();
|
||||
test_calc_object_states_matches_sync();
|
||||
test_calc_object_states_matches_sync_extra();
|
||||
test_calc_thread_count_zero();
|
||||
test_cancel_queued_pg_calc();
|
||||
test_stop_drops_queued_pg_calc();
|
||||
test_shutdown_drops_queued_pg_calc();
|
||||
test_cancel_busy_pg_calc();
|
||||
test_cancel_done_pg_calc();
|
||||
test_calc_epoch_reconciles_reported_update();
|
||||
test_calc_epoch_keeps_version_epoch();
|
||||
test_pg_epoch_bump_above_reported_epoch();
|
||||
test_chained_read_large_chain_info();
|
||||
test_peering_partial_unstable_ec_rolls_back();
|
||||
|
||||
@@ -180,6 +180,9 @@ struct osd_test_fixture_t
|
||||
[this](blockstore_config_t & cfg) -> blockstore_i* {
|
||||
return (bs = new capturing_bs_t(cfg));
|
||||
});
|
||||
// Mock ringloop does not poll eventfd, so a background calc would not be
|
||||
// applied inside ringloop->loop(). Tests keep the synchronous path.
|
||||
osd->pg_calc_sync = true;
|
||||
}
|
||||
|
||||
void start(json11::Json::object osd_config)
|
||||
@@ -270,6 +273,90 @@ struct osd_test_fixture_t
|
||||
// Drive handle_peers so PGs see lists_done and finalize calc_object_states.
|
||||
ringloop->wakeup();
|
||||
ringloop->loop();
|
||||
osd->wait_pg_calc();
|
||||
}
|
||||
|
||||
void wait_pg_calc()
|
||||
{
|
||||
osd->wait_pg_calc();
|
||||
}
|
||||
|
||||
// Workers have pushed finished jobs onto done. The OSD thread has not
|
||||
// installed them yet, so cancel can still drop the result.
|
||||
void wait_pg_calc_not_busy()
|
||||
{
|
||||
osd->wait_pg_calc_not_busy();
|
||||
}
|
||||
|
||||
// Must be called after construct() and before start().
|
||||
void set_pg_calc_sync(bool sync)
|
||||
{
|
||||
osd->pg_calc_sync = sync;
|
||||
}
|
||||
|
||||
// Must be called after construct() and before start(). See osd_t::pg_calc_hold.
|
||||
void set_pg_calc_hold(bool hold)
|
||||
{
|
||||
osd->pg_calc_hold = hold;
|
||||
}
|
||||
|
||||
void set_pg_calc_pause(bool pause)
|
||||
{
|
||||
osd->pg_calc_pause = pause;
|
||||
}
|
||||
|
||||
void set_pg_calc_runs(int *runs)
|
||||
{
|
||||
osd->pg_calc_runs = runs;
|
||||
}
|
||||
|
||||
void release_pg_calc_hold()
|
||||
{
|
||||
osd->pg_calc_release_hold();
|
||||
}
|
||||
|
||||
void unpause_pg_calc()
|
||||
{
|
||||
osd->pg_calc_unpause();
|
||||
}
|
||||
|
||||
void wait_pg_calc_paused()
|
||||
{
|
||||
osd->wait_pg_calc_paused();
|
||||
}
|
||||
|
||||
size_t pg_calc_queued()
|
||||
{
|
||||
return osd->pg_calc_queued();
|
||||
}
|
||||
|
||||
void confirm_pg_epoch(pool_id_t pool_id, pg_num_t pg_num, uint64_t epoch)
|
||||
{
|
||||
osd->st_cli->pool_config.at(pool_id).pg_config.at(pg_num).epoch = epoch;
|
||||
osd->on_change_pg_history_hook(pool_id, pg_num);
|
||||
}
|
||||
|
||||
void repeer_pg(pool_id_t pool_id, pg_num_t pg_num)
|
||||
{
|
||||
osd->repeer_pg(osd->pgs.at({ .pool_id = pool_id, .pg_num = pg_num }));
|
||||
}
|
||||
|
||||
void stop_pg(pool_id_t pool_id, pg_num_t pg_num)
|
||||
{
|
||||
osd->stop_pg(osd->pgs.at({ .pool_id = pool_id, .pg_num = pg_num }));
|
||||
}
|
||||
|
||||
int pending_list_count()
|
||||
{
|
||||
int n = 0;
|
||||
for (auto *op: bs->queued)
|
||||
if (op->opcode == BS_OP_LIST)
|
||||
n++;
|
||||
for (auto & p: osd->msgr.osd_peers)
|
||||
for (auto & kv: p.second->sent_ops)
|
||||
if (kv.second->req.hdr.opcode == OSD_OP_SEC_LIST)
|
||||
n++;
|
||||
return n;
|
||||
}
|
||||
|
||||
// Inline list response builder. `objects` are pairs (oid, version);
|
||||
|
||||
@@ -2056,6 +2056,9 @@ void chaos_t::start_osd()
|
||||
bs = new blockstore_impl_t(cfg, f.ringloop, f.tfd, true);
|
||||
return bs;
|
||||
});
|
||||
// The chaos clock is simulated; peering calc stays on the calling thread
|
||||
// so a step observes PG_ACTIVE without waiting on a real thread.
|
||||
f.osd->pg_calc_sync = true;
|
||||
}
|
||||
catch(std::exception & ex)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user