From cf9a641d66406b47085b424a978f40ae09b065c3 Mon Sep 17 00:00:00 2001
From: Vitaliy Filippov <vitalif@yourcmc.ru>
Date: Sun, 22 Mar 2020 02:13:10 +0300
Subject: [PATCH] Skip disconnected OSDs during sync

---
 src/osd.h                  |  2 +-
 src/osd_primary.cpp        |  5 ++++-
 src/osd_primary_subops.cpp | 18 +++++++++++++++---
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/src/osd.h b/src/osd.h
index 01a17619..c6f0217d 100644
--- a/src/osd.h
+++ b/src/osd.h
@@ -207,7 +207,7 @@ class osd_t
     void submit_primary_subops(int submit_type, uint64_t op_version, int pg_size, const uint64_t* osd_set, osd_op_t *cur_op);
     void submit_primary_del_subops(osd_op_t *cur_op, uint64_t *cur_set, uint64_t set_size, pg_osd_set_t & loc_set);
     void submit_primary_del_batch(osd_op_t *cur_op, obj_ver_osd_t *chunks_to_delete, int chunks_to_delete_count);
-    void submit_primary_sync_subops(osd_op_t *cur_op);
+    int submit_primary_sync_subops(osd_op_t *cur_op);
     void submit_primary_stab_subops(osd_op_t *cur_op);
 
     inline pg_num_t map_to_pg(object_id oid, uint64_t pg_stripe_size)
diff --git a/src/osd_primary.cpp b/src/osd_primary.cpp
index c0844536..3c6cbf0b 100644
--- a/src/osd_primary.cpp
+++ b/src/osd_primary.cpp
@@ -659,7 +659,10 @@ resume_2:
     if (immediate_commit != IMMEDIATE_ALL)
     {
         // SYNC
-        submit_primary_sync_subops(cur_op);
+        if (!submit_primary_sync_subops(cur_op))
+        {
+            goto resume_4;
+        }
 resume_3:
         op_data->st = 3;
         return;
diff --git a/src/osd_primary_subops.cpp b/src/osd_primary_subops.cpp
index a923996b..ed6b1a92 100644
--- a/src/osd_primary_subops.cpp
+++ b/src/osd_primary_subops.cpp
@@ -451,7 +451,7 @@ void osd_t::submit_primary_del_batch(osd_op_t *cur_op, obj_ver_osd_t *chunks_to_
     }
 }
 
-void osd_t::submit_primary_sync_subops(osd_op_t *cur_op)
+int osd_t::submit_primary_sync_subops(osd_op_t *cur_op)
 {
     osd_primary_op_data_t *op_data = cur_op->op_data;
     int n_osds = op_data->dirty_osd_count;
@@ -459,6 +459,7 @@ void osd_t::submit_primary_sync_subops(osd_op_t *cur_op)
     op_data->done = op_data->errors = 0;
     op_data->n_subops = n_osds;
     op_data->subops = subops;
+    std::map<uint64_t, int>::iterator peer_it;
     for (int i = 0; i < n_osds; i++)
     {
         osd_num_t sync_osd = op_data->dirty_osds[i];
@@ -475,10 +476,10 @@ void osd_t::submit_primary_sync_subops(osd_op_t *cur_op)
             });
             bs->enqueue_op(subops[i].bs_op);
         }
-        else
+        else if ((peer_it = c_cli.osd_peer_fds.find(sync_osd)) != c_cli.osd_peer_fds.end())
         {
             subops[i].op_type = OSD_OP_OUT;
-            subops[i].peer_fd = c_cli.osd_peer_fds.at(sync_osd);
+            subops[i].peer_fd = peer_it->second;
             subops[i].req.sec_sync = {
                 .header = {
                     .magic = SECONDARY_OSD_OP_MAGIC,
@@ -498,7 +499,18 @@ void osd_t::submit_primary_sync_subops(osd_op_t *cur_op)
             };
             c_cli.outbox_push(&subops[i]);
         }
+        else
+        {
+            op_data->done++;
+        }
     }
+    if (op_data->done >= op_data->n_subops)
+    {
+        delete[] op_data->subops;
+        op_data->subops = NULL;
+        return 0;
+    }
+    return 1;
 }
 
 void osd_t::submit_primary_stab_subops(osd_op_t *cur_op)