Fix PGs not stopping during sync, fix state reporting autovivification of erased PGs

osd_cluster.cpp

@@ -796,6 +796,7 @@ void osd_t::apply_pg_config()
                 .target_set = pg_cfg.target_set,
             };
             this->pg_state_dirty.insert(pg_num);
+            this->pgs[pg_num].print_state();
             if (pg_cfg.cur_primary == this->osd_num)
             {
                 // Add peers
@@ -830,15 +831,20 @@ void osd_t::report_pg_states()
         return;
     }
     etcd_reporting_pg_state = true;
-    std::vector<pg_num_t> reporting_pgs(pg_state_dirty.begin(), pg_state_dirty.end());
+    std::vector<pg_num_t> reporting_pgs;
-    pg_state_dirty.clear();
     json11::Json::array checks;
     json11::Json::array success;
     json11::Json::array failure;
-    for (auto pg_num: reporting_pgs)
+    for (auto it = pg_state_dirty.begin(); it != pg_state_dirty.end(); it++)
     {
-        auto & pg = this->pgs[pg_num];
+        auto pg_it = this->pgs.find(*it);
-        std::string state_key_base64 = base64_encode(etcd_prefix+"/pg/state/"+std::to_string(pg_num));
+        if (pg_it == this->pgs.end())
+        {
+            continue;
+        }
+        auto & pg = pg_it->second;
+        reporting_pgs.push_back(pg.pg_num);
+        std::string state_key_base64 = base64_encode(etcd_prefix+"/pg/state/"+std::to_string(pg.pg_num));
         if (pg.state == PG_STARTING)
         {
             // Check that the PG key does not exist
@@ -904,6 +910,7 @@ void osd_t::report_pg_states()
             } }
         });
     }
+    pg_state_dirty.clear();
     etcd_txn(json11::Json::object {
         { "compare", checks }, { "success", success }, { "failure", failure }
     }, ETCD_QUICK_TIMEOUT, [this, reporting_pgs](std::string err, json11::Json data)

osd_peering_pg.cpp

@@ -364,11 +364,13 @@ void pg_t::calc_object_states(int log_level)
 void pg_t::print_state()
 {
     printf(
-        "[PG %u] is %s%s%s%s%s%s%s%s%s (%lu objects)\n", pg_num,
+        "[PG %u] is %s%s%s%s%s%s%s%s%s%s%s (%lu objects)\n", pg_num,
+        (state & PG_STARTING) ? "starting" : "",
         (state & PG_OFFLINE) ? "offline" : "",
         (state & PG_PEERING) ? "peering" : "",
         (state & PG_INCOMPLETE) ? "incomplete" : "",
         (state & PG_ACTIVE) ? "active" : "",
+        (state & PG_STOPPING) ? "stopping" : "",
         (state & PG_DEGRADED) ? " + degraded" : "",
         (state & PG_HAS_INCOMPLETE) ? " + has_incomplete" : "",
         (state & PG_HAS_DEGRADED) ? " + has_degraded" : "",

osd_primary.cpp

@@ -49,9 +49,9 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
     if (cur_op->op_data && cur_op->op_data->pg_num > 0)
     {
         auto & pg = pgs[cur_op->op_data->pg_num];
-        int n = --pg.inflight;
+        pg.inflight--;
-        assert(n >= 0);
+        assert(pg.inflight >= 0);
-        if ((pg.state & PG_STOPPING) && n == 0 && !pg.flush_batch)
+        if ((pg.state & PG_STOPPING) && pg.inflight == 0 && !pg.flush_batch)
         {
             finish_stop_pg(pg);
         }
@@ -760,10 +760,6 @@ resume_5:
     op_data->st = 5;
     return;
 resume_6:
-    for (int i = 0; i < op_data->dirty_pg_count; i++)
-    {
-        pgs[op_data->dirty_pgs[i]].inflight--;
-    }
     if (op_data->errors > 0)
     {
         // Return objects back into the unstable write set
@@ -786,6 +782,15 @@ resume_6:
             }
         }
     }
+    for (int i = 0; i < op_data->dirty_pg_count; i++)
+    {
+        auto & pg = pgs.at(op_data->dirty_pgs[i]);
+        pg.inflight--;
+        if ((pg.state & PG_STOPPING) && pg.inflight == 0 && !pg.flush_batch)
+        {
+            finish_stop_pg(pg);
+        }
+    }
     // FIXME: Free those in the destructor?
     delete op_data->dirty_pgs;
     delete op_data->unstable_write_osds;