forked from vitalif/vitastor
Do not hang on inactive OSDs during delete, report and skip them instead
parent
f12b8e45a9
commit
8603b5cb1d
|
@ -28,6 +28,7 @@ struct rm_inode_t
|
||||||
cli_tool_t *parent = NULL;
|
cli_tool_t *parent = NULL;
|
||||||
inode_list_t *lister = NULL;
|
inode_list_t *lister = NULL;
|
||||||
std::vector<rm_pg_t*> lists;
|
std::vector<rm_pg_t*> lists;
|
||||||
|
std::vector<osd_num_t> inactive_osds;
|
||||||
uint64_t total_count = 0, total_done = 0, total_prev_pct = 0;
|
uint64_t total_count = 0, total_done = 0, total_prev_pct = 0;
|
||||||
uint64_t pgs_to_list = 0;
|
uint64_t pgs_to_list = 0;
|
||||||
bool lists_done = false;
|
bool lists_done = false;
|
||||||
|
@ -86,6 +87,16 @@ struct rm_inode_t
|
||||||
state = 100;
|
state = 100;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
inactive_osds = parent->cli->list_inode_get_inactive_osds(lister);
|
||||||
|
if (inactive_osds.size() && !parent->json_output)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Some data may remain after delete on OSDs which are currently down: ");
|
||||||
|
for (int i = 0; i < inactive_osds.size(); i++)
|
||||||
|
{
|
||||||
|
fprintf(stderr, i > 0 ? ", %lu" : "%lu", inactive_osds[i]);
|
||||||
|
}
|
||||||
|
fprintf(stderr, "\n");
|
||||||
|
}
|
||||||
pgs_to_list = parent->cli->list_pg_count(lister);
|
pgs_to_list = parent->cli->list_pg_count(lister);
|
||||||
parent->cli->list_inode_next(lister, parent->parallel_osds);
|
parent->cli->list_inode_next(lister, parent->parallel_osds);
|
||||||
}
|
}
|
||||||
|
@ -167,7 +178,7 @@ struct rm_inode_t
|
||||||
}
|
}
|
||||||
if (parent->progress && total_count > 0 && total_done*1000/total_count != total_prev_pct)
|
if (parent->progress && total_count > 0 && total_done*1000/total_count != total_prev_pct)
|
||||||
{
|
{
|
||||||
printf("\rRemoved %lu/%lu objects, %lu more PGs to list...", total_done, total_count, pgs_to_list);
|
fprintf(stderr, "\rRemoved %lu/%lu objects, %lu more PGs to list...", total_done, total_count, pgs_to_list);
|
||||||
total_prev_pct = total_done*1000/total_count;
|
total_prev_pct = total_done*1000/total_count;
|
||||||
}
|
}
|
||||||
if (lists_done && !lists.size())
|
if (lists_done && !lists.size())
|
||||||
|
@ -177,8 +188,17 @@ struct rm_inode_t
|
||||||
.text = error_count > 0 ? "Some blocks were not removed" : (
|
.text = error_count > 0 ? "Some blocks were not removed" : (
|
||||||
"Done, inode "+std::to_string(INODE_NO_POOL(inode))+" from pool "+
|
"Done, inode "+std::to_string(INODE_NO_POOL(inode))+" from pool "+
|
||||||
std::to_string(pool_id)+" removed"),
|
std::to_string(pool_id)+" removed"),
|
||||||
|
.data = json11::Json::object {
|
||||||
|
{ "removed_objects", total_done },
|
||||||
|
{ "total_objects", total_count },
|
||||||
|
{ "inactive_osds", inactive_osds },
|
||||||
|
},
|
||||||
};
|
};
|
||||||
state = 100;
|
state = 100;
|
||||||
|
if (parent->progress && total_count > 0)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "\n");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -130,6 +130,7 @@ public:
|
||||||
inode_list_t *list_inode_start(inode_t inode,
|
inode_list_t *list_inode_start(inode_t inode,
|
||||||
std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback);
|
std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback);
|
||||||
int list_pg_count(inode_list_t *lst);
|
int list_pg_count(inode_list_t *lst);
|
||||||
|
const std::vector<osd_num_t> & list_inode_get_inactive_osds(inode_list_t *lst);
|
||||||
void list_inode_next(inode_list_t *lst, int next_pgs);
|
void list_inode_next(inode_list_t *lst, int next_pgs);
|
||||||
//inline uint32_t get_bs_bitmap_granularity() { return st_cli.global_bitmap_granularity; }
|
//inline uint32_t get_bs_bitmap_granularity() { return st_cli.global_bitmap_granularity; }
|
||||||
//inline uint64_t get_bs_block_size() { return st_cli.global_block_size; }
|
//inline uint64_t get_bs_block_size() { return st_cli.global_block_size; }
|
||||||
|
|
|
@ -36,6 +36,7 @@ struct inode_list_t
|
||||||
inode_t inode = 0;
|
inode_t inode = 0;
|
||||||
int done_pgs = 0;
|
int done_pgs = 0;
|
||||||
int want = 0;
|
int want = 0;
|
||||||
|
std::vector<osd_num_t> inactive_osds;
|
||||||
std::vector<inode_list_pg_t*> pgs;
|
std::vector<inode_list_pg_t*> pgs;
|
||||||
std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback;
|
std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback;
|
||||||
};
|
};
|
||||||
|
@ -60,6 +61,7 @@ inode_list_t* cluster_client_t::list_inode_start(inode_t inode,
|
||||||
lst->inode = inode;
|
lst->inode = inode;
|
||||||
lst->callback = callback;
|
lst->callback = callback;
|
||||||
auto pool_cfg = st_cli.pool_config[pool_id];
|
auto pool_cfg = st_cli.pool_config[pool_id];
|
||||||
|
std::set<osd_num_t> inactive_osd_set;
|
||||||
for (auto & pg_item: pool_cfg.pg_config)
|
for (auto & pg_item: pool_cfg.pg_config)
|
||||||
{
|
{
|
||||||
auto & pg = pg_item.second;
|
auto & pg = pg_item.second;
|
||||||
|
@ -105,6 +107,8 @@ inode_list_t* cluster_client_t::list_inode_start(inode_t inode,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for (osd_num_t peer_osd: all_peers)
|
for (osd_num_t peer_osd: all_peers)
|
||||||
|
{
|
||||||
|
if (st_cli.peer_states.find(peer_osd) != st_cli.peer_states.end())
|
||||||
{
|
{
|
||||||
r->list_osds.push_back((inode_list_osd_t){
|
r->list_osds.push_back((inode_list_osd_t){
|
||||||
.pg = r,
|
.pg = r,
|
||||||
|
@ -112,6 +116,11 @@ inode_list_t* cluster_client_t::list_inode_start(inode_t inode,
|
||||||
.sent = false,
|
.sent = false,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
inactive_osd_set.insert(peer_osd);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -132,6 +141,7 @@ inode_list_t* cluster_client_t::list_inode_start(inode_t inode,
|
||||||
{
|
{
|
||||||
lst->pgs[i]->pos = i;
|
lst->pgs[i]->pos = i;
|
||||||
}
|
}
|
||||||
|
lst->inactive_osds.insert(lst->inactive_osds.end(), inactive_osd_set.begin(), inactive_osd_set.end());
|
||||||
lists.push_back(lst);
|
lists.push_back(lst);
|
||||||
return lst;
|
return lst;
|
||||||
}
|
}
|
||||||
|
@ -141,6 +151,11 @@ int cluster_client_t::list_pg_count(inode_list_t *lst)
|
||||||
return lst->pgs.size();
|
return lst->pgs.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const std::vector<osd_num_t> & cluster_client_t::list_inode_get_inactive_osds(inode_list_t *lst)
|
||||||
|
{
|
||||||
|
return lst->inactive_osds;
|
||||||
|
}
|
||||||
|
|
||||||
void cluster_client_t::list_inode_next(inode_list_t *lst, int next_pgs)
|
void cluster_client_t::list_inode_next(inode_list_t *lst, int next_pgs)
|
||||||
{
|
{
|
||||||
if (next_pgs >= 0)
|
if (next_pgs >= 0)
|
||||||
|
|
|
@ -647,12 +647,6 @@ void osd_t::continue_primary_del(osd_op_t *cur_op)
|
||||||
else if (op_data->st == 4) goto resume_4;
|
else if (op_data->st == 4) goto resume_4;
|
||||||
else if (op_data->st == 5) goto resume_5;
|
else if (op_data->st == 5) goto resume_5;
|
||||||
assert(op_data->st == 0);
|
assert(op_data->st == 0);
|
||||||
// Delete is forbidden even in active PGs if they're also degraded or have previous dead OSDs
|
|
||||||
if (pg.state & (PG_DEGRADED | PG_LEFT_ON_DEAD))
|
|
||||||
{
|
|
||||||
finish_op(cur_op, -EBUSY);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (!check_write_queue(cur_op, pg))
|
if (!check_write_queue(cur_op, pg))
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
|
|
Loading…
Reference in New Issue