2020-01-24 02:23:27 +03:00
|
|
|
#include <map>
|
2020-03-04 17:12:27 +03:00
|
|
|
#include <unordered_map>
|
2020-01-24 02:23:27 +03:00
|
|
|
#include <vector>
|
|
|
|
#include <algorithm>
|
|
|
|
|
2020-03-04 17:12:27 +03:00
|
|
|
#include "cpp-btree/btree_map.h"
|
|
|
|
|
2020-01-24 02:23:27 +03:00
|
|
|
#include "object_id.h"
|
2020-02-03 12:35:02 +03:00
|
|
|
#include "osd_ops.h"
|
2020-01-24 02:23:27 +03:00
|
|
|
|
|
|
|
// Placement group states
|
|
|
|
// Exactly one of these:
|
|
|
|
#define PG_OFFLINE (1<<0)
|
|
|
|
#define PG_PEERING (1<<1)
|
|
|
|
#define PG_INCOMPLETE (1<<2)
|
|
|
|
#define PG_ACTIVE (1<<3)
|
|
|
|
// Plus any of these:
|
2020-02-11 02:30:46 +03:00
|
|
|
#define PG_DEGRADED (1<<4)
|
2020-03-15 18:39:31 +03:00
|
|
|
#define PG_HAS_INCOMPLETE (1<<5)
|
2020-02-11 02:30:46 +03:00
|
|
|
#define PG_HAS_DEGRADED (1<<6)
|
|
|
|
#define PG_HAS_MISPLACED (1<<7)
|
|
|
|
#define PG_HAS_UNCLEAN (1<<8)
|
2020-01-24 02:23:27 +03:00
|
|
|
|
2020-03-15 18:39:31 +03:00
|
|
|
// FIXME: Safe default that doesn't depend on parity_block_size or pg_parity_size
|
2020-02-13 19:13:16 +03:00
|
|
|
#define STRIPE_MASK ((uint64_t)4096 - 1)
|
|
|
|
|
2020-01-24 02:23:27 +03:00
|
|
|
// OSD object states
|
|
|
|
#define OBJ_CLEAN 0x01
|
2020-03-22 20:36:29 +03:00
|
|
|
#define OBJ_DEGRADED 0x02
|
2020-01-24 02:23:27 +03:00
|
|
|
#define OBJ_INCOMPLETE 0x04
|
2020-03-22 20:36:29 +03:00
|
|
|
#define OBJ_MISPLACED 0x08
|
2020-01-27 01:34:24 +03:00
|
|
|
#define OBJ_NEEDS_STABLE 0x10000
|
|
|
|
#define OBJ_NEEDS_ROLLBACK 0x20000
|
2020-01-24 02:23:27 +03:00
|
|
|
#define OBJ_BUGGY 0x80000
|
|
|
|
|
|
|
|
struct pg_obj_loc_t
|
|
|
|
{
|
|
|
|
uint64_t role;
|
2020-02-03 12:35:02 +03:00
|
|
|
osd_num_t osd_num;
|
2020-03-23 15:44:29 +03:00
|
|
|
bool outdated;
|
2020-01-24 02:23:27 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
typedef std::vector<pg_obj_loc_t> pg_osd_set_t;
|
|
|
|
|
|
|
|
struct pg_osd_set_state_t
|
|
|
|
{
|
2020-02-11 02:30:46 +03:00
|
|
|
// (role -> osd_num_t) map, as in pg.target_set and pg.cur_set
|
2020-02-03 12:35:02 +03:00
|
|
|
std::vector<osd_num_t> read_target;
|
2020-02-11 02:30:46 +03:00
|
|
|
// full OSD set including additional OSDs where the object is misplaced
|
2020-01-24 02:23:27 +03:00
|
|
|
pg_osd_set_t osd_set;
|
|
|
|
uint64_t state = 0;
|
|
|
|
uint64_t object_count = 0;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct pg_list_result_t
|
|
|
|
{
|
2020-02-24 01:01:34 +03:00
|
|
|
obj_ver_id *buf = NULL;
|
2020-01-24 02:23:27 +03:00
|
|
|
uint64_t total_count;
|
|
|
|
uint64_t stable_count;
|
|
|
|
};
|
|
|
|
|
2020-02-11 02:30:46 +03:00
|
|
|
struct osd_op_t;
|
|
|
|
|
2020-01-24 02:23:27 +03:00
|
|
|
struct pg_peering_state_t
|
|
|
|
{
|
|
|
|
// osd_num -> list result
|
2020-03-04 17:12:27 +03:00
|
|
|
std::unordered_map<osd_num_t, osd_op_t*> list_ops;
|
|
|
|
std::unordered_map<osd_num_t, pg_list_result_t> list_results;
|
2020-01-24 02:23:27 +03:00
|
|
|
int list_done = 0;
|
|
|
|
};
|
|
|
|
|
2020-01-27 01:34:24 +03:00
|
|
|
struct obj_piece_id_t
|
|
|
|
{
|
|
|
|
object_id oid;
|
|
|
|
uint64_t osd_num;
|
|
|
|
};
|
|
|
|
|
2020-03-13 21:41:54 +03:00
|
|
|
struct flush_action_t
|
2020-01-27 01:34:24 +03:00
|
|
|
{
|
|
|
|
bool rollback = false, make_stable = false;
|
|
|
|
uint64_t stable_to = 0, rollback_to = 0;
|
2020-03-13 21:41:54 +03:00
|
|
|
bool submitted = false;
|
2020-01-27 01:34:24 +03:00
|
|
|
};
|
|
|
|
|
2020-03-15 18:39:31 +03:00
|
|
|
struct pg_flush_batch_t
|
|
|
|
{
|
|
|
|
std::map<osd_num_t, std::vector<obj_ver_id>> rollback_lists;
|
|
|
|
std::map<osd_num_t, std::vector<obj_ver_id>> stable_lists;
|
|
|
|
int flush_ops = 0, flush_done = 0;
|
|
|
|
int flush_objects = 0;
|
|
|
|
};
|
2020-03-13 21:41:54 +03:00
|
|
|
|
2020-01-24 02:23:27 +03:00
|
|
|
struct pg_t
|
|
|
|
{
|
|
|
|
int state;
|
2020-01-27 01:34:24 +03:00
|
|
|
uint64_t pg_cursize = 3, pg_size = 3, pg_minsize = 2;
|
2020-02-03 12:35:02 +03:00
|
|
|
pg_num_t pg_num;
|
2020-02-26 18:32:00 +03:00
|
|
|
uint64_t clean_count = 0, total_count = 0;
|
2020-02-11 02:30:46 +03:00
|
|
|
// target_set is the "correct" peer OSD set for this PG
|
2020-02-03 12:35:02 +03:00
|
|
|
std::vector<osd_num_t> target_set;
|
2020-02-11 02:30:46 +03:00
|
|
|
// cur_set is the current set of connected peer OSDs for this PG
|
|
|
|
// cur_set = (role => osd_num or UINT64_MAX if missing). role numbers begin with zero
|
|
|
|
std::vector<osd_num_t> cur_set;
|
|
|
|
// moved object map. by default, each object is considered to reside on the cur_set.
|
2020-01-24 02:23:27 +03:00
|
|
|
// this map stores all objects that differ.
|
2020-01-27 01:34:24 +03:00
|
|
|
// it may consume up to ~ (raw storage / object size) * 24 bytes in the worst case scenario
|
2020-01-24 02:23:27 +03:00
|
|
|
// which is up to ~192 MB per 1 TB in the worst case scenario
|
|
|
|
std::map<pg_osd_set_t, pg_osd_set_state_t> state_dict;
|
2020-03-04 17:12:27 +03:00
|
|
|
btree::btree_map<object_id, pg_osd_set_state_t*> obj_states;
|
2020-03-13 21:41:54 +03:00
|
|
|
std::map<obj_piece_id_t, flush_action_t> flush_actions;
|
2020-03-04 17:12:27 +03:00
|
|
|
btree::btree_map<object_id, uint64_t> ver_override;
|
2020-01-24 02:23:27 +03:00
|
|
|
pg_peering_state_t *peering_state = NULL;
|
2020-03-13 21:41:54 +03:00
|
|
|
pg_flush_batch_t *flush_batch = NULL;
|
|
|
|
int flush_actions_in_progress = 0;
|
2020-01-24 02:23:27 +03:00
|
|
|
|
2020-02-25 01:20:45 +03:00
|
|
|
std::multimap<object_id, osd_op_t*> write_queue;
|
|
|
|
|
2020-01-24 02:23:27 +03:00
|
|
|
void calc_object_states();
|
2020-03-14 22:19:45 +03:00
|
|
|
void print_state();
|
2020-01-24 02:23:27 +03:00
|
|
|
};
|
|
|
|
|
|
|
|
inline bool operator < (const pg_obj_loc_t &a, const pg_obj_loc_t &b)
|
|
|
|
{
|
2020-03-23 15:44:29 +03:00
|
|
|
return a.role < b.role ||
|
|
|
|
a.role == b.role && a.outdated < b.outdated ||
|
|
|
|
a.role == b.role && a.outdated == b.outdated && a.osd_num < b.osd_num;
|
2020-01-24 02:23:27 +03:00
|
|
|
}
|
|
|
|
|
2020-01-27 01:34:24 +03:00
|
|
|
inline bool operator == (const obj_piece_id_t & a, const obj_piece_id_t & b)
|
|
|
|
{
|
|
|
|
return a.oid == b.oid && a.osd_num == b.osd_num;
|
|
|
|
}
|
|
|
|
|
|
|
|
inline bool operator < (const obj_piece_id_t & a, const obj_piece_id_t & b)
|
|
|
|
{
|
|
|
|
return a.oid < b.oid || a.oid == b.oid && a.osd_num < b.osd_num;
|
|
|
|
}
|
|
|
|
|
2020-01-24 02:23:27 +03:00
|
|
|
namespace std
|
|
|
|
{
|
|
|
|
template<> struct hash<pg_osd_set_t>
|
|
|
|
{
|
|
|
|
inline size_t operator()(const pg_osd_set_t &s) const
|
|
|
|
{
|
|
|
|
size_t seed = 0;
|
|
|
|
for (auto e: s)
|
|
|
|
{
|
|
|
|
// Copy-pasted from spp::hash_combine()
|
|
|
|
seed ^= (e.role + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
|
|
|
|
seed ^= (e.osd_num + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
|
|
|
|
}
|
|
|
|
return seed;
|
|
|
|
}
|
|
|
|
};
|
2020-01-27 01:34:24 +03:00
|
|
|
|
|
|
|
template<> struct hash<obj_piece_id_t>
|
|
|
|
{
|
|
|
|
inline size_t operator()(const obj_piece_id_t &s) const
|
|
|
|
{
|
|
|
|
size_t seed = std::hash<object_id>()(s.oid);
|
|
|
|
// Copy-pasted from spp::hash_combine()
|
|
|
|
seed ^= (s.osd_num + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
|
|
|
|
return seed;
|
|
|
|
}
|
|
|
|
};
|
2020-01-24 02:23:27 +03:00
|
|
|
}
|