From 74d0196ba519b5ac495e4f94fe3dcb059b49fadd Mon Sep 17 00:00:00 2001
From: Vitaliy Filippov <vitalif@yourcmc.ru>
Date: Fri, 31 Jan 2020 13:46:23 +0300
Subject: [PATCH] Stripe read reconstruction

---
 Makefile        |  4 ++--
 osd_primary.cpp | 31 +++++++++++++++++++++++++++++--
 xor.h           |  4 ++--
 3 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index b9166d201..75e365914 100644
--- a/Makefile
+++ b/Makefile
@@ -16,7 +16,7 @@ ringloop.o: ringloop.cpp ringloop.h
 timerfd_interval.o: timerfd_interval.cpp timerfd_interval.h
 	g++ $(CXXFLAGS) -c -o $@ $<
 
-%.o: %.cpp allocator.h blockstore_flush.h blockstore.h blockstore_impl.h blockstore_init.h blockstore_journal.h crc32c.h ringloop.h xor.h timerfd_interval.h object_id.h
+%.o: %.cpp allocator.h blockstore_flush.h blockstore.h blockstore_impl.h blockstore_init.h blockstore_journal.h crc32c.h ringloop.h timerfd_interval.h object_id.h
 	g++ $(CXXFLAGS) -c -o $@ $<
 
 libblockstore.so: $(BLOCKSTORE_OBJS)
@@ -35,7 +35,7 @@ osd_peering.o: osd_peering.cpp osd.h osd_ops.h osd_peering_pg.h
 	g++ $(CXXFLAGS) -c -o $@ $<
 osd_peering_pg.o: osd_peering_pg.cpp object_id.h osd_peering_pg.h
 	g++ $(CXXFLAGS) -c -o $@ $<
-osd_primary.o: osd_primary.cpp osd.h osd_ops.h osd_peering_pg.h
+osd_primary.o: osd_primary.cpp osd.h osd_ops.h osd_peering_pg.h xor.h
 	g++ $(CXXFLAGS) -c -o $@ $<
 osd.o: osd.cpp osd.h osd_ops.h osd_peering_pg.h
 	g++ $(CXXFLAGS) -c -o $@ $<
diff --git a/osd_primary.cpp b/osd_primary.cpp
index c65b857c6..e96699ffd 100644
--- a/osd_primary.cpp
+++ b/osd_primary.cpp
@@ -1,4 +1,5 @@
 #include "osd.h"
+#include "xor.h"
 
 void osd_t::exec_primary_read(osd_op_t *cur_op)
 {
@@ -29,7 +30,6 @@ void osd_t::exec_primary_read(osd_op_t *cur_op)
         return;
     }
     // role -> start, end
-    void *buf = memalign(MEM_ALIGNMENT, cur_op->op.rw.len);
     uint64_t reads[pgs[pg_num].pg_minsize*2] = { 0 };
     for (int role = 0; role < pgs[pg_num].pg_minsize; role++)
     {
@@ -41,6 +41,8 @@ void osd_t::exec_primary_read(osd_op_t *cur_op)
     }
     if (pgs[pg_num].pg_cursize == 3)
     {
+        // Fast happy-path
+        void *buf = memalign(MEM_ALIGNMENT, cur_op->op.rw.len);
         
     }
     else
@@ -67,7 +69,7 @@ void osd_t::exec_primary_read(osd_op_t *cur_op)
                         else
                         {
                             real_reads[j*2] = reads[j*2] < reads[role*2] ? reads[j*2] : reads[role*2];
-                            real_reads[j*2] = reads[j*2+1] > reads[role*2+1] ? reads[j*2+1] : reads[role*2+1];
+                            real_reads[j*2+1] = reads[j*2+1] > reads[role*2+1] ? reads[j*2+1] : reads[role*2+1];
                         }
                         exist++;
                         if (exist >= pgs[pg_num].pg_minsize)
@@ -78,6 +80,31 @@ void osd_t::exec_primary_read(osd_op_t *cur_op)
                 }
             }
         }
+        uint64_t pos[pgs[pg_num].pg_size];
+        uint64_t buf_size = 0;
+        for (int role = 0; role < pgs[pg_num].pg_size; role++)
+        {
+            pos[role] = buf_size;
+            buf_size += real_reads[role*2+1];
+        }
+        void *buf = memalign(MEM_ALIGNMENT, buf_size);
+        // ...<SUBMIT READS AND GET REPLIES>...
+        
+        // Reconstruct missing stripes
+        for (int role = 0; role < pgs[pg_num].pg_minsize; role++)
+        {
+            if (reads[role*2+1] != 0 && pgs[pg_num].target_set[role] == UINT64_MAX)
+            {
+                int other = role == 0 ? 1 : 0;
+                int parity = pgs[pg_num].pg_size-1;
+                memxor(
+                    buf + pos[other] + (real_reads[other*2]-reads[role*2]),
+                    buf + pos[parity] + (real_reads[parity*2]-reads[role*2]),
+                    buf + pos[role], reads[role*2+1]-reads[role*2]
+                );
+            }
+        }
+        // Send buffer in parts to avoid copying
     }
 }
 
diff --git a/xor.h b/xor.h
index b98dcc518..c9d7ec721 100644
--- a/xor.h
+++ b/xor.h
@@ -2,11 +2,11 @@
 
 #include <stdint.h>
 
-inline void memxor(uint8_t *r1, uint8_t *r2, uint8_t *res, unsigned int len)
+inline void memxor(const void *r1, const void *r2, void *res, unsigned int len)
 {
     unsigned int i;
     for (i = 0; i < len; ++i)
     {
-        res[i] = r1[i] ^ r2[i];
+        ((uint8_t*)res)[i] = ((uint8_t*)r1)[i] ^ ((uint8_t*)r2)[i];
     }
 }