Sync only completed writes
Test / buildenv (push) Successful in 11s
Details
Test / build (push) Successful in 18s
Details
Test / test_cas (push) Successful in 10s
Details
Test / make_test (push) Successful in 38s
Details
Test / test_change_pg_size (push) Successful in 9s
Details
Test / test_change_pg_count (push) Successful in 43s
Details
Test / test_change_pg_count_ec (push) Successful in 38s
Details
Test / test_create_nomaxid (push) Successful in 8s
Details
Test / test_etcd_fail (push) Successful in 56s
Details
Test / test_add_osd (push) Successful in 2m34s
Details
Test / test_interrupted_rebalance_imm (push) Successful in 1m43s
Details
Test / test_failure_domain (push) Successful in 35s
Details
Test / test_snapshot (push) Successful in 22s
Details
Test / test_interrupted_rebalance_ec (push) Successful in 1m55s
Details
Test / test_interrupted_rebalance_ec_imm (push) Successful in 1m16s
Details
Test / test_minsize_1 (push) Successful in 16s
Details
Test / test_snapshot_ec (push) Successful in 23s
Details
Test / test_move_reappear (push) Successful in 21s
Details
Test / test_rm (push) Successful in 16s
Details
Test / test_snapshot_down (push) Successful in 25s
Details
Test / test_snapshot_down_ec (push) Successful in 35s
Details
Test / test_splitbrain (push) Successful in 24s
Details
Test / test_interrupted_rebalance (push) Successful in 5m14s
Details
Test / test_snapshot_chain (push) Successful in 2m50s
Details
Test / test_rebalance_verify_imm (push) Successful in 2m47s
Details
Test / test_rebalance_verify (push) Successful in 3m42s
Details
Test / test_switch_primary (push) Successful in 33s
Details
Test / test_write (push) Successful in 42s
Details
Test / test_write_xor (push) Successful in 44s
Details
Test / test_rebalance_verify_ec_imm (push) Successful in 2m52s
Details
Test / test_write_no_same (push) Successful in 15s
Details
Test / test_rebalance_verify_ec (push) Successful in 4m19s
Details
Test / test_heal_ec (push) Successful in 6m20s
Details
Test / test_heal_csum_32k (push) Successful in 3m29s
Details
Test / test_scrub (push) Successful in 1m24s
Details
Test / test_scrub_zero_osd_2 (push) Successful in 1m11s
Details
Test / test_heal_csum_4k_dmj (push) Successful in 4m23s
Details
Test / test_scrub_xor (push) Successful in 1m9s
Details
Test / test_heal_csum_4k_dj (push) Successful in 5m29s
Details
Test / test_heal_csum_4k (push) Successful in 5m36s
Details
Test / test_scrub_pg_size_3 (push) Successful in 1m53s
Details
Test / test_scrub_ec (push) Successful in 29s
Details
Test / test_heal_pg_size_2 (push) Successful in 3m9s
Details
Test / test_heal_csum_32k_dmj (push) Successful in 4m13s
Details
Test / test_heal_csum_32k_dj (push) Successful in 4m17s
Details
Test / test_snapshot_chain_ec (push) Successful in 1m25s
Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Failing after 24s
Details
Test / buildenv (push) Successful in 11s
Details
Test / build (push) Successful in 18s
Details
Test / test_cas (push) Successful in 10s
Details
Test / make_test (push) Successful in 38s
Details
Test / test_change_pg_size (push) Successful in 9s
Details
Test / test_change_pg_count (push) Successful in 43s
Details
Test / test_change_pg_count_ec (push) Successful in 38s
Details
Test / test_create_nomaxid (push) Successful in 8s
Details
Test / test_etcd_fail (push) Successful in 56s
Details
Test / test_add_osd (push) Successful in 2m34s
Details
Test / test_interrupted_rebalance_imm (push) Successful in 1m43s
Details
Test / test_failure_domain (push) Successful in 35s
Details
Test / test_snapshot (push) Successful in 22s
Details
Test / test_interrupted_rebalance_ec (push) Successful in 1m55s
Details
Test / test_interrupted_rebalance_ec_imm (push) Successful in 1m16s
Details
Test / test_minsize_1 (push) Successful in 16s
Details
Test / test_snapshot_ec (push) Successful in 23s
Details
Test / test_move_reappear (push) Successful in 21s
Details
Test / test_rm (push) Successful in 16s
Details
Test / test_snapshot_down (push) Successful in 25s
Details
Test / test_snapshot_down_ec (push) Successful in 35s
Details
Test / test_splitbrain (push) Successful in 24s
Details
Test / test_interrupted_rebalance (push) Successful in 5m14s
Details
Test / test_snapshot_chain (push) Successful in 2m50s
Details
Test / test_rebalance_verify_imm (push) Successful in 2m47s
Details
Test / test_rebalance_verify (push) Successful in 3m42s
Details
Test / test_switch_primary (push) Successful in 33s
Details
Test / test_write (push) Successful in 42s
Details
Test / test_write_xor (push) Successful in 44s
Details
Test / test_rebalance_verify_ec_imm (push) Successful in 2m52s
Details
Test / test_write_no_same (push) Successful in 15s
Details
Test / test_rebalance_verify_ec (push) Successful in 4m19s
Details
Test / test_heal_ec (push) Successful in 6m20s
Details
Test / test_heal_csum_32k (push) Successful in 3m29s
Details
Test / test_scrub (push) Successful in 1m24s
Details
Test / test_scrub_zero_osd_2 (push) Successful in 1m11s
Details
Test / test_heal_csum_4k_dmj (push) Successful in 4m23s
Details
Test / test_scrub_xor (push) Successful in 1m9s
Details
Test / test_heal_csum_4k_dj (push) Successful in 5m29s
Details
Test / test_heal_csum_4k (push) Successful in 5m36s
Details
Test / test_scrub_pg_size_3 (push) Successful in 1m53s
Details
Test / test_scrub_ec (push) Successful in 29s
Details
Test / test_heal_pg_size_2 (push) Successful in 3m9s
Details
Test / test_heal_csum_32k_dmj (push) Successful in 4m13s
Details
Test / test_heal_csum_32k_dj (push) Successful in 4m17s
Details
Test / test_snapshot_chain_ec (push) Successful in 1m25s
Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Failing after 24s
Details
Should be a final remaining fix to EC + non-capacitor (non-immediate-commit) write hangs :). First it was breaking non-EC ("instantly stable") writes because they sometimes complete out of order which was leading to the following error: terminate called after throwing an instance of 'std::runtime_error' what(): BUG: Unexpected dirty_entry 1000000000001:29480000 v65540 unstable state during flush: 0x151 But it is easily fixed by scanning previous and next dirty_entries in mark_stable.test-fix-ec-unknown-state-51
parent
1c322b33ed
commit
1cec62d25d
|
@ -163,20 +163,10 @@ void blockstore_impl_t::loop()
|
||||||
}
|
}
|
||||||
else if (op->opcode == BS_OP_SYNC)
|
else if (op->opcode == BS_OP_SYNC)
|
||||||
{
|
{
|
||||||
// wait for all small writes to be submitted
|
// sync only completed writes?
|
||||||
// wait for all big writes to complete, submit data device fsync
|
|
||||||
// wait for the data device fsync to complete, then submit journal writes for big writes
|
// wait for the data device fsync to complete, then submit journal writes for big writes
|
||||||
// then submit an fsync operation
|
// then submit an fsync operation
|
||||||
if (has_writes)
|
|
||||||
{
|
|
||||||
// Can't submit SYNC before previous writes
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
wr_st = continue_sync(op);
|
wr_st = continue_sync(op);
|
||||||
if (wr_st != 2)
|
|
||||||
{
|
|
||||||
has_writes = wr_st > 0 ? 1 : 2;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else if (op->opcode == BS_OP_STABLE)
|
else if (op->opcode == BS_OP_STABLE)
|
||||||
{
|
{
|
||||||
|
@ -283,7 +273,7 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
|
||||||
}
|
}
|
||||||
else if (PRIV(op)->wait_for == WAIT_JOURNAL)
|
else if (PRIV(op)->wait_for == WAIT_JOURNAL)
|
||||||
{
|
{
|
||||||
if (journal.used_start == PRIV(op)->wait_detail)
|
if (journal.used_start == PRIV(op)->wait_detail && !unstable_count_changed)
|
||||||
{
|
{
|
||||||
// do not submit
|
// do not submit
|
||||||
#ifdef BLOCKSTORE_DEBUG
|
#ifdef BLOCKSTORE_DEBUG
|
||||||
|
@ -291,6 +281,7 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
|
||||||
#endif
|
#endif
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
unstable_count_changed = false;
|
||||||
flusher->release_trim();
|
flusher->release_trim();
|
||||||
PRIV(op)->wait_for = 0;
|
PRIV(op)->wait_for = 0;
|
||||||
}
|
}
|
||||||
|
@ -362,6 +353,7 @@ void blockstore_impl_t::enqueue_op(blockstore_op_t *op)
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
unstable_writes.clear();
|
unstable_writes.clear();
|
||||||
|
unstable_count_changed = true;
|
||||||
op->callback = [old_callback](blockstore_op_t *op)
|
op->callback = [old_callback](blockstore_op_t *op)
|
||||||
{
|
{
|
||||||
obj_ver_id *vers = (obj_ver_id*)op->buf;
|
obj_ver_id *vers = (obj_ver_id*)op->buf;
|
||||||
|
|
|
@ -55,6 +55,7 @@
|
||||||
#define IS_JOURNAL(st) (((st) & 0x0F) == BS_ST_SMALL_WRITE)
|
#define IS_JOURNAL(st) (((st) & 0x0F) == BS_ST_SMALL_WRITE)
|
||||||
#define IS_BIG_WRITE(st) (((st) & 0x0F) == BS_ST_BIG_WRITE)
|
#define IS_BIG_WRITE(st) (((st) & 0x0F) == BS_ST_BIG_WRITE)
|
||||||
#define IS_DELETE(st) (((st) & 0x0F) == BS_ST_DELETE)
|
#define IS_DELETE(st) (((st) & 0x0F) == BS_ST_DELETE)
|
||||||
|
#define IS_INSTANT(st) (((st) & BS_ST_TYPE_MASK) == BS_ST_DELETE || ((st) & BS_ST_INSTANT))
|
||||||
|
|
||||||
#define BS_SUBMIT_CHECK_SQES(n) \
|
#define BS_SUBMIT_CHECK_SQES(n) \
|
||||||
if (ringloop->sqes_left() < (n))\
|
if (ringloop->sqes_left() < (n))\
|
||||||
|
@ -275,6 +276,7 @@ class blockstore_impl_t
|
||||||
std::vector<blockstore_op_t*> submit_queue;
|
std::vector<blockstore_op_t*> submit_queue;
|
||||||
std::vector<obj_ver_id> unsynced_big_writes, unsynced_small_writes;
|
std::vector<obj_ver_id> unsynced_big_writes, unsynced_small_writes;
|
||||||
int unsynced_big_write_count = 0, unstable_unsynced = 0;
|
int unsynced_big_write_count = 0, unstable_unsynced = 0;
|
||||||
|
bool unstable_count_changed = false;
|
||||||
int unsynced_queued_ops = 0;
|
int unsynced_queued_ops = 0;
|
||||||
allocator *data_alloc = NULL;
|
allocator *data_alloc = NULL;
|
||||||
uint64_t used_blocks = 0;
|
uint64_t used_blocks = 0;
|
||||||
|
@ -377,7 +379,7 @@ class blockstore_impl_t
|
||||||
// Stabilize
|
// Stabilize
|
||||||
int dequeue_stable(blockstore_op_t *op);
|
int dequeue_stable(blockstore_op_t *op);
|
||||||
int continue_stable(blockstore_op_t *op);
|
int continue_stable(blockstore_op_t *op);
|
||||||
void mark_stable(const obj_ver_id & ov, bool forget_dirty = false);
|
void mark_stable(obj_ver_id ov, bool forget_dirty = false);
|
||||||
void stabilize_object(object_id oid, uint64_t max_ver);
|
void stabilize_object(object_id oid, uint64_t max_ver);
|
||||||
blockstore_op_t* selective_sync(blockstore_op_t *op);
|
blockstore_op_t* selective_sync(blockstore_op_t *op);
|
||||||
int split_stab_op(blockstore_op_t *op, std::function<int(obj_ver_id v)> decider);
|
int split_stab_op(blockstore_op_t *op, std::function<int(obj_ver_id v)> decider);
|
||||||
|
|
|
@ -162,6 +162,7 @@ void blockstore_impl_t::mark_rolled_back(const obj_ver_id & ov)
|
||||||
unstable_writes.erase(unstab_it);
|
unstable_writes.erase(unstab_it);
|
||||||
else
|
else
|
||||||
unstab_it->second = max_unstable;
|
unstab_it->second = max_unstable;
|
||||||
|
unstable_count_changed = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -412,11 +412,40 @@ resume_4:
|
||||||
return 2;
|
return 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
void blockstore_impl_t::mark_stable(const obj_ver_id & v, bool forget_dirty)
|
void blockstore_impl_t::mark_stable(obj_ver_id v, bool forget_dirty)
|
||||||
{
|
{
|
||||||
auto dirty_it = dirty_db.find(v);
|
auto dirty_it = dirty_db.find(v);
|
||||||
if (dirty_it != dirty_db.end())
|
if (dirty_it != dirty_db.end())
|
||||||
{
|
{
|
||||||
|
if (IS_INSTANT(dirty_it->second.state))
|
||||||
|
{
|
||||||
|
// 'Instant' (non-EC) operations may complete and try to become stable out of order. Prevent it.
|
||||||
|
auto back_it = dirty_it;
|
||||||
|
while (back_it != dirty_db.begin())
|
||||||
|
{
|
||||||
|
back_it--;
|
||||||
|
if (back_it->first.oid != v.oid)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if (!IS_STABLE(back_it->second.state))
|
||||||
|
{
|
||||||
|
// There are preceding unstable versions, can't flush <v>
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
dirty_it++;
|
||||||
|
if (dirty_it == dirty_db.end() || dirty_it->first.oid != v.oid ||
|
||||||
|
!IS_SYNCED(dirty_it->second.state))
|
||||||
|
{
|
||||||
|
dirty_it--;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
v.version = dirty_it->first.version;
|
||||||
|
}
|
||||||
|
}
|
||||||
while (1)
|
while (1)
|
||||||
{
|
{
|
||||||
bool was_stable = IS_STABLE(dirty_it->second.state);
|
bool was_stable = IS_STABLE(dirty_it->second.state);
|
||||||
|
@ -508,5 +537,6 @@ void blockstore_impl_t::mark_stable(const obj_ver_id & v, bool forget_dirty)
|
||||||
unstab_it->second <= v.version)
|
unstab_it->second <= v.version)
|
||||||
{
|
{
|
||||||
unstable_writes.erase(unstab_it);
|
unstable_writes.erase(unstab_it);
|
||||||
|
unstable_count_changed = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -85,8 +85,7 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
|
||||||
left--;
|
left--;
|
||||||
auto & dirty_entry = dirty_db.at(sbw);
|
auto & dirty_entry = dirty_db.at(sbw);
|
||||||
uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len);
|
uint64_t dyn_size = dsk.dirty_dyn_size(dirty_entry.offset, dirty_entry.len);
|
||||||
if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size,
|
if (!space_check.check_available(op, 1, sizeof(journal_entry_big_write) + dyn_size, 0))
|
||||||
(unstable_writes.size()+unstable_unsynced)*journal.block_size))
|
|
||||||
{
|
{
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
|
@ -593,7 +593,7 @@ resume_4:
|
||||||
#endif
|
#endif
|
||||||
bool is_big = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE;
|
bool is_big = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE;
|
||||||
bool imm = is_big ? (immediate_commit == IMMEDIATE_ALL) : (immediate_commit != IMMEDIATE_NONE);
|
bool imm = is_big ? (immediate_commit == IMMEDIATE_ALL) : (immediate_commit != IMMEDIATE_NONE);
|
||||||
bool is_instant = ((dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_DELETE || (dirty_it->second.state & BS_ST_INSTANT));
|
bool is_instant = IS_INSTANT(dirty_it->second.state);
|
||||||
if (imm)
|
if (imm)
|
||||||
{
|
{
|
||||||
auto & unstab = unstable_writes[op->oid];
|
auto & unstab = unstable_writes[op->oid];
|
||||||
|
|
Loading…
Reference in New Issue