WIP/experimental LRC matrix generator

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8)
 
 project(vitastor)
 
-set(VERSION "0.6.17")
+set(VERSION "0.6.16")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.6.17
+VERSION ?= v0.6.16
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.6.17
+          image: vitalif/vitastor-csi:v0.6.16
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.6.17
+          image: vitalif/vitastor-csi:v0.6.16
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/example-pvc-block.yaml

@@ -1,13 +0,0 @@
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: test-vitastor-pvc-block
-spec:
-  storageClassName: vitastor
-  volumeMode: Block
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 10Gi

csi/deploy/example-test-pod-block.yaml

@@ -1,17 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: vitastor-test-block-pvc
-  namespace: default
-spec:
-  containers:
-  - name: vitastor-test-block-pvc
-    image: nginx
-    volumeDevices:
-      - name: data
-        devicePath: /dev/xvda
-  volumes:
-  - name: data
-    persistentVolumeClaim:
-      claimName: test-vitastor-pvc-block
-      readOnly: false

csi/deploy/example-test-pod.yaml

@@ -1,17 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: vitastor-test-nginx
-  namespace: default
-spec:
-  containers:
-   - name: vitastor-test-nginx
-     image: nginx
-     volumeMounts:
-       - mountPath: /usr/share/nginx/html/s3
-         name: data
-  volumes:
-   - name: data
-     persistentVolumeClaim:
-       claimName: test-vitastor-pvc
-       readOnly: false

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.6.17"
+    vitastorCSIDriverVersion = "0.6.16"
 )
 
 // Config struct fills the parameters of request or user input

csi/src/nodeserver.go

@@ -67,44 +67,29 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
     klog.Infof("received node publish volume request %+v", protosanitizer.StripSecrets(req))
 
     targetPath := req.GetTargetPath()
-    isBlock := req.GetVolumeCapability().GetBlock() != nil
 
     // Check that it's not already mounted
-    _, error := mount.IsNotMountPoint(ns.mounter, targetPath)
+    free, error := mount.IsNotMountPoint(ns.mounter, targetPath)
     if (error != nil)
     {
         if (os.IsNotExist(error))
         {
-            if (isBlock)
+            error := os.MkdirAll(targetPath, 0777)
+            if (error != nil)
             {
-                pathFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR, 0o600)
-                if (err != nil)
-                {
-                    klog.Errorf("failed to create block device mount target %s with error: %v", targetPath, err)
-                    return nil, status.Error(codes.Internal, err.Error())
-                }
-                err = pathFile.Close()
-                if (err != nil)
-                {
-                    klog.Errorf("failed to close %s with error: %v", targetPath, err)
-                    return nil, status.Error(codes.Internal, err.Error())
-                }
-            }
-            else
-            {
-                err := os.MkdirAll(targetPath, 0777)
-                if (err != nil)
-                {
-                    klog.Errorf("failed to create fs mount target %s with error: %v", targetPath, err)
-                    return nil, status.Error(codes.Internal, err.Error())
-                }
+                return nil, status.Error(codes.Internal, error.Error())
             }
+            free = true
         }
         else
         {
             return nil, status.Error(codes.Internal, error.Error())
         }
     }
+    if (!free)
+    {
+        return &csi.NodePublishVolumeResponse{}, nil
+    }
 
     ctxVars := make(map[string]string)
     err := json.Unmarshal([]byte(req.VolumeId), &ctxVars)
@@ -164,6 +149,7 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 
     // Format the device (ext4 or xfs)
     fsType := req.GetVolumeCapability().GetMount().GetFsType()
+    isBlock := req.GetVolumeCapability().GetBlock() != nil
     opt := req.GetVolumeCapability().GetMount().GetMountFlags()
     opt = append(opt, "_netdev")
     if ((req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY ||

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (0.6.17-1) unstable; urgency=medium
+vitastor (0.6.16-1) unstable; urgency=medium
 
   * RDMA support
   * Bugfixes

debian/vitastor.Dockerfile

@@ -33,8 +33,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.6.17; \
-    cd vitastor-0.6.17; \
+    cp -r /root/vitastor vitastor-0.6.16; \
+    cd vitastor-0.6.16; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -47,8 +47,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.17.orig.tar.xz vitastor-0.6.17; \
-    cd vitastor-0.6.17; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.16.orig.tar.xz vitastor-0.6.16; \
+    cd vitastor-0.6.16; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

mon/mon.js

@@ -224,19 +224,15 @@ const etcd_tree = {
             }, */
         },
         inodestats: {
-            /* <pool_id>: {
-                <inode_t>: {
-                    read: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
-                    write: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
-                    delete: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
-                },
+            /* <inode_t>: {
+                read: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
+                write: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
+                delete: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
             }, */
         },
         space: {
             /* <osd_num_t>: {
-                <pool_id>: {
-                    <inode_t>: uint64_t, // bytes
-                },
+                <inode_t>: uint64_t, // bytes
             }, */
         },
     },

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.6.17'
+VERSION = '0.6.16'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -25,4 +25,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.6.17/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.17$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.6.16/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.16$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -34,7 +34,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.17.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.16.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.17
+Version:        0.6.16
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.17.el7.tar.gz
+Source0:        vitastor-0.6.16.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -33,7 +33,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.17.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.16.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.17
+Version:        0.6.16
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.17.el8.tar.gz
+Source0:        vitastor-0.6.16.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -15,7 +15,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.6.17")
+add_definitions(-DVERSION="0.6.16")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)

src/allocator.cpp

@@ -25,7 +25,7 @@ allocator::allocator(uint64_t blocks)
     size = free = blocks;
     last_one_mask = (blocks % 64) == 0
         ? UINT64_MAX
-        : (((uint64_t)1 << (blocks % 64)) - 1);
+        : ((1l << (blocks % 64)) - 1);
     for (uint64_t i = 0; i < total; i++)
     {
         mask[i] = 0;
@@ -79,7 +79,7 @@ void allocator::set(uint64_t addr, bool value)
             }
             if (value)
             {
-                mask[last] = mask[last] | ((uint64_t)1 << bit);
+                mask[last] = mask[last] | (1l << bit);
                 if (mask[last] != (!is_last || cur_addr/64 < size/64
                     ? UINT64_MAX : last_one_mask))
                 {
@@ -88,7 +88,7 @@ void allocator::set(uint64_t addr, bool value)
             }
             else
             {
-                mask[last] = mask[last] & ~((uint64_t)1 << bit);
+                mask[last] = mask[last] & ~(1l << bit);
             }
             is_last = false;
             if (p2 > 1)

src/blockstore_init.cpp

@@ -131,7 +131,6 @@ resume_1:
     }
     // Skip superblock
     bs->meta_offset += bs->meta_block_size;
-    bs->meta_len -= bs->meta_block_size;
     prev_done = 0;
     done_len = 0;
     done_pos = 0;

src/cli_create.cpp

@@ -457,13 +457,13 @@ uint64_t parse_size(std::string size_str)
     if (type_char == 'k' || type_char == 'm' || type_char == 'g' || type_char == 't')
     {
         if (type_char == 'k')
-            mul = (uint64_t)1<<10;
+            mul = 1l<<10;
         else if (type_char == 'm')
-            mul = (uint64_t)1<<20;
+            mul = 1l<<20;
         else if (type_char == 'g')
-            mul = (uint64_t)1<<30;
+            mul = 1l<<30;
         else /*if (type_char == 't')*/
-            mul = (uint64_t)1<<40;
+            mul = 1l<<40;
         size_str = size_str.substr(0, size_str.length()-1);
     }
     uint64_t size = json11::Json(size_str).uint64_value() * mul;

src/cli_df.cpp

@@ -124,8 +124,8 @@ resume_1:
                 { "scheme_name", pool_cfg.scheme == POOL_SCHEME_REPLICATED
                     ? std::to_string(pool_cfg.pg_size)+"/"+std::to_string(pool_cfg.pg_minsize)
                     : "EC "+std::to_string(pool_cfg.pg_size-pool_cfg.parity_chunks)+"+"+std::to_string(pool_cfg.parity_chunks) },
-                { "used_raw", (uint64_t)(pool_stats[pool_cfg.id]["used_raw_tb"].number_value() * ((uint64_t)1<<40)) },
-                { "total_raw", (uint64_t)(pool_stats[pool_cfg.id]["total_raw_tb"].number_value() * ((uint64_t)1<<40)) },
+                { "used_raw", (uint64_t)(pool_stats[pool_cfg.id]["used_raw_tb"].number_value() * (1l<<40)) },
+                { "total_raw", (uint64_t)(pool_stats[pool_cfg.id]["total_raw_tb"].number_value() * (1l<<40)) },
                 { "max_available", pool_avail },
                 { "raw_to_usable", pool_stats[pool_cfg.id]["raw_to_usable"].number_value() },
                 { "space_efficiency", pool_stats[pool_cfg.id]["space_efficiency"].number_value() },

src/cli_ls.cpp

@@ -436,8 +436,8 @@ std::string print_table(json11::Json items, json11::Json header, bool use_esc)
     return str;
 }
 
-static uint64_t size_thresh[] = { (uint64_t)1024*1024*1024*1024, (uint64_t)1024*1024*1024, (uint64_t)1024*1024, 1024, 0 };
-static uint64_t size_thresh_d[] = { (uint64_t)1000000000000, (uint64_t)1000000000, (uint64_t)1000000, (uint64_t)1000, 0 };
+static uint64_t size_thresh[] = { 1024l*1024*1024*1024, 1024l*1024*1024, 1024l*1024, 1024, 0 };
+static uint64_t size_thresh_d[] = { 1000000000000l, 1000000000l, 1000000l, 1000l, 0 };
 static const int size_thresh_n = sizeof(size_thresh)/sizeof(size_thresh[0]);
 static const char *size_unit = "TGMKB";
 

src/cli_rm_data.cpp

@@ -193,7 +193,7 @@ std::function<bool(void)> cli_tool_t::start_rm(json11::Json cfg)
     remover->pool_id = cfg["pool"].uint64_value();
     if (remover->pool_id)
     {
-        remover->inode = (remover->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)) | (((uint64_t)remover->pool_id) << (64-POOL_ID_BITS));
+        remover->inode = (remover->inode & ((1l << (64-POOL_ID_BITS)) - 1)) | (((uint64_t)remover->pool_id) << (64-POOL_ID_BITS));
     }
     remover->pool_id = INODE_POOL(remover->inode);
     if (!remover->pool_id)

src/cli_status.cpp

@@ -217,7 +217,7 @@ resume_2:
             // JSON output
             printf("%s\n", json11::Json(json11::Json::object {
                 { "etcd_alive", etcd_alive },
-                { "etcd_count", (uint64_t)etcd_states.size() },
+                { "etcd_count", etcd_states.size() },
                 { "etcd_db_size", etcd_db_size },
                 { "mon_count", mon_count },
                 { "mon_master", mon_master },

src/cluster_client.cpp

@@ -9,7 +9,6 @@
 #define PART_SENT 1
 #define PART_DONE 2
 #define PART_ERROR 4
-#define PART_RETRY 8
 #define CACHE_DIRTY 1
 #define CACHE_FLUSHING 2
 #define CACHE_REPEATING 3
@@ -671,17 +670,14 @@ resume_2:
             if (!try_send(op, i))
             {
                 // We'll need to retry again
-                if (op->parts[i].flags & PART_RETRY)
+                op->up_wait = true;
+                if (!retry_timeout_id)
                 {
-                    op->up_wait = true;
-                    if (!retry_timeout_id)
+                    retry_timeout_id = tfd->set_timer(up_wait_retry_interval, false, [this](int)
                     {
-                        retry_timeout_id = tfd->set_timer(up_wait_retry_interval, false, [this](int)
-                        {
-                            retry_timeout_id = 0;
-                            continue_ops(true);
-                        });
-                    }
+                        retry_timeout_id = 0;
+                        continue_ops(true);
+                    });
                 }
                 op->state = 2;
             }
@@ -750,7 +746,7 @@ resume_3:
         {
             for (int i = 0; i < op->parts.size(); i++)
             {
-                op->parts[i].flags = PART_RETRY;
+                op->parts[i].flags = 0;
             }
             goto resume_2;
         }

src/etcd_state_client.cpp

@@ -968,7 +968,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
                     {
                         fprintf(
                             stderr, "Inode %lu/%lu parent_pool value is invalid, ignoring parent setting\n",
-                            inode_num >> (64-POOL_ID_BITS), inode_num & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)
+                            inode_num >> (64-POOL_ID_BITS), inode_num & ((1l << (64-POOL_ID_BITS)) - 1)
                         );
                         parent_inode_num = 0;
                     }

src/fio_cluster.cpp

@@ -214,14 +214,14 @@ static int sec_setup(struct thread_data *td)
 
     if (!o->image)
     {
-        if (!(o->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)))
+        if (!(o->inode & ((1l << (64-POOL_ID_BITS)) - 1)))
         {
             td_verror(td, EINVAL, "inode number is missing");
             return 1;
         }
         if (o->pool)
         {
-            o->inode = (o->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)) | (o->pool << (64-POOL_ID_BITS));
+            o->inode = (o->inode & ((1l << (64-POOL_ID_BITS)) - 1)) | (o->pool << (64-POOL_ID_BITS));
         }
         if (!(o->inode >> (64-POOL_ID_BITS)))
         {

src/lrc/Makefile

@@ -0,0 +1,2 @@
+mat: mat.c
+	gcc -O3 -I/usr/include/jerasure -o mat mat.c -lJerasure

src/lrc/mat.c

@@ -0,0 +1,277 @@
+#include <jerasure/reed_sol.h>
+#include <jerasure.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// Generate LRC matrix: (groups*local + global) code rows with (data_drives) columns
+// w should be >= log2(data_drives + groups*local + global), but not necessary 8/16/32
+int* reed_sol_vandermonde_lrc_matrix(int data_drives, int groups, int local, int global, int w)
+{
+    if (w < 0 || w > 32 || data_drives + groups*local + global > (1<<w))
+    {
+        return NULL;
+    }
+    int *lrc_matrix = (int*)malloc(sizeof(int) * (local*groups+global));
+    int *matrix = reed_sol_vandermonde_coding_matrix(data_drives, local+global, w);
+    for (int gr = 0; gr < groups; gr++)
+    {
+        for (int l = 0; l < local; l++)
+        {
+            for (int j = 0; j < data_drives; j++)
+            {
+                lrc_matrix[(gr*local+l)*data_drives + j] = (j / (data_drives/groups)) == gr ? matrix[l*data_drives + j] : 0;
+            }
+        }
+    }
+    for (int i = 0; i < global; i++)
+    {
+        for (int j = 0; j < data_drives; j++)
+        {
+            lrc_matrix[(groups*local+i)*data_drives + j] = matrix[(local+i)*data_drives + j];
+        }
+    }
+    free(matrix);
+    return lrc_matrix;
+}
+
+// Check if the generated LRC with given parameters is Maximally Reconstructible (MR-LRC)
+// Example of a MR-LRC: (8, 2, 1, 2, 6, 8)
+void check_mr_lrc(int data_drives, int groups, int local, int global, int matrix_w, int w, int print)
+{
+    
+}
+
+int main()
+{
+    int W = 8, MATRIX_W = 6;
+    int n = 8, groups = 2, local = 1, global = 2;
+    //n = 4, groups = 2, local = 1, global = 1;
+    int total_rows = n+groups*local+global;
+    int *matrix = reed_sol_vandermonde_lrc_matrix(n, groups, local, global, MATRIX_W);
+    int *lrc_matrix = (int*)malloc(sizeof(int) * total_rows*n);
+    // Fill identity+LRC matrix
+    for (int i = 0; i < n; i++)
+        for (int j = 0; j < n; j++)
+            lrc_matrix[i*n + j] = j == i ? 1 : 0;
+    memcpy(lrc_matrix + n*n, matrix, (total_rows-n)*n*sizeof(int));
+    free(matrix);
+    matrix = NULL;
+    // Print LRC matrix
+    for (int i = 0; i < total_rows; i++)
+    {
+        for (int j = 0; j < n; j++)
+        {
+            printf("%d ", lrc_matrix[i*n+j]);
+        }
+        printf("\n");
+    }
+    int impossible = 0, success = 0, failures = 0;
+    int *lost_per_group = (int*)malloc(sizeof(int) * groups);
+    for (int lost = local+global+1; lost <= groups*local+global; lost++)
+    //int lost = groups*local+global;
+    {
+        int *erased_matrix = (int*)malloc(sizeof(int) * (total_rows-lost)*n);
+        int *inverted_matrix = (int*)malloc(sizeof(int) * (total_rows-lost)*n);
+        int *p = (int*)malloc(sizeof(int) * (total_rows-lost));
+        for (int i = 0; i < n; i++)
+            p[i] = i;
+        int *p2 = (int*)malloc(sizeof(int) * n);
+        if (total_rows-lost > n)
+        {
+            p[n-1] = n; // skip combinations with all N data disks (0..n-1)
+            for (int i = n; i < total_rows-lost; i++)
+                p[i] = i+1;
+            p[total_rows-lost-1]--; // will be incremented on the first step
+        }
+        int inc = total_rows-lost-1;
+        while (1)
+        {
+            p[inc]++;
+            if (p[inc] >= n+groups*local+global)
+            {
+                if (inc == 0)
+                    break;
+                inc--;
+            }
+            else if (inc+1 < total_rows-lost)
+            {
+                p[inc+1] = p[inc];
+                inc++;
+            }
+            else
+            {
+                // Check if it should be recoverable
+                for (int gr = 0; gr < groups; gr++)
+                {
+                    lost_per_group[gr] = ((gr+1)*(n/groups) > n ? (n - gr*(n/groups)) : n/groups);
+                }
+                // Calculate count of data chunks lost in each group
+                for (int j = 0; j < total_rows-lost; j++)
+                {
+                    if (j < n)
+                    {
+                        lost_per_group[(p[j] / (n/groups))]--;
+                    }
+                }
+                // Every local parity chunk is supposed to restore 1 missing chunk inside its group
+                // So, subtract local parity chunk counts from each group lost chunk count
+                for (int j = 0; j < total_rows-lost; j++)
+                {
+                    if (p[j] >= n && p[j] < n+groups*local && lost_per_group[(p[j]-n)/local] > 0)
+                    {
+                        lost_per_group[(p[j]-n)/local]--;
+                    }
+                }
+                // Every global parity chunk is supposed to restore 1 chunk of all that are still missing
+                int still_missing = 0;
+                for (int gr = 0; gr < groups; gr++)
+                {
+                    still_missing += lost_per_group[gr];
+                }
+                for (int j = 0; j < total_rows-lost; j++)
+                {
+                    if (p[j] >= n+groups*local && still_missing > 0)
+                    {
+                        still_missing--;
+                    }
+                }
+                if (still_missing <= 0)
+                {
+                    // We hope it can be recoverable. Try to invert it
+                    int invert_ok = -1;
+                    if (total_rows-lost == n)
+                    {
+                        for (int i = 0; i < n; i++)
+                            for (int j = 0; j < n; j++)
+                                erased_matrix[i*n+j] = lrc_matrix[p[i]*n+j];
+                        invert_ok = jerasure_invert_matrix(erased_matrix, inverted_matrix, n, W);
+                    }
+                    else
+                    {
+                        // Check submatrices
+                        for (int i = 0; i < n; i++)
+                            p2[i] = i;
+                        p2[n-1]--;
+                        int inc2 = n-1;
+                        while (1)
+                        {
+                            p2[inc2]++;
+                            if (p2[inc2] >= total_rows-lost)
+                            {
+                                if (inc2 == 0)
+                                    break;
+                                inc2--;
+                            }
+                            else if (inc2+1 < n)
+                            {
+                                p2[inc2+1] = p2[inc2];
+                                inc2++;
+                            }
+                            else
+                            {
+                                for (int i = 0; i < n; i++)
+                                    for (int j = 0; j < n; j++)
+                                        erased_matrix[i*n+j] = lrc_matrix[p[p2[i]]*n+j];
+                                invert_ok = jerasure_invert_matrix(erased_matrix, inverted_matrix, n, W);
+                                if (invert_ok == 0)
+                                    break;
+                            }
+                        }
+                    }
+                    if (invert_ok < 0)
+                    {
+                        failures++;
+                        printf("\nFAIL: ");
+                        for (int i = 0; i < total_rows-lost; i++)
+                        {
+                            printf("%d ", p[i]);
+                        }
+                        printf("\nDIRECT:\n");
+                        for (int i = 0; i < total_rows-lost; i++)
+                        {
+                            for (int j = 0; j < n; j++)
+                                printf("%d ", lrc_matrix[p[i]*n+j]);
+                            printf("\n");
+                        }
+                        printf("INVERSE:\n");
+                        for (int i = 0; i < total_rows-lost; i++)
+                        {
+                            for (int j = 0; j < n; j++)
+                                printf("%d ", inverted_matrix[i*n+j]);
+                            printf("\n");
+                        }
+                    }
+                    else
+                    {
+                        success++;
+                        printf("OK: ");
+                        for (int i = 0; i < total_rows-lost; i++)
+                        {
+                            printf("%d ", p[i]);
+                        }
+                        printf("\n");
+                    }
+                }
+                else
+                {
+                    impossible++;
+                    printf("IMPOSSIBLE: ");
+                    for (int i = 0; i < total_rows-lost; i++)
+                    {
+                        printf("%d ", p[i]);
+                    }
+                    printf("\n");
+                }
+            }
+        }
+        free(p2);
+        free(p);
+        free(inverted_matrix);
+        free(erased_matrix);
+    }
+    free(lost_per_group);
+    printf("\n%d recovered, %d impossible, %d failures\n", success, impossible, failures);
+    return 0;
+}
+
+// 1 1 1 1 0 0 0 0
+// 0 0 0 0 1 1 1 1
+// 1 55 39 73 84 181 225 217
+// 1 172 70 235 143 34 200 101
+//
+// Can't recover
+// 1 2 4 5 8 9 10 11    -1
+// 2 3 4 6 8 9 10 11    -1
+// FULL:
+// 1 0 0 0 0 0 0 0 
+// 0 1 0 0 0 0 0 0 
+// 0 0 1 0 0 0 0 0 
+// 0 0 0 1 0 0 0 0 
+// 0 0 0 0 1 0 0 0 
+// 0 0 0 0 0 1 0 0 
+// 0 0 0 0 0 0 1 0 
+// 0 0 0 0 0 0 0 1 
+// 1 1 1 1 0 0 0 0 
+// 0 0 0 0 1 1 1 1 
+// 1 55 39 73 84 181 225 217 
+// 1 172 70 235 143 34 200 101 
+// FIRST UNRECOVERABLE:
+// 0 1 0 0 0 0 0 0 
+// 0 0 1 0 0 0 0 0 
+// 0 0 0 0 1 0 0 0 
+// 0 0 0 0 0 1 0 0 
+// 1 1 1 1 0 0 0 0 
+// 0 0 0 0 1 1 1 1 
+// 1 55 39 73 84 181 225 217 
+// 1 172 70 235 143 34 200 101 
+// SECOND UNRECOVERABLE:
+// 0 0 1 0 0 0 0 0 
+// 0 0 0 1 0 0 0 0 
+// 0 0 0 0 1 0 0 0 
+// 0 0 0 0 0 0 1 0 
+// 1 1 1 1 0 0 0 0 
+// 0 0 0 0 1 1 1 1 
+// 1 55 39 73 84 181 225 217 
+// 1 172 70 235 143 34 200 101 
+// Ho ho ho

src/nbd_proxy.cpp

@@ -189,7 +189,7 @@ public:
             uint64_t pool = cfg["pool"].uint64_value();
             if (pool)
             {
-                inode = (inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)) | (pool << (64-POOL_ID_BITS));
+                inode = (inode & ((1l << (64-POOL_ID_BITS)) - 1)) | (pool << (64-POOL_ID_BITS));
             }
             if (!(inode >> (64-POOL_ID_BITS)))
             {

src/osd_cluster.cpp

@@ -189,7 +189,7 @@ void osd_t::report_statistics()
     for (auto kv: bs->get_inode_space_stats())
     {
         pool_id_t pool_id = INODE_POOL(kv.first);
-        uint64_t only_inode_num = INODE_NO_POOL(kv.first);
+        uint64_t only_inode_num = (kv.first & ((1l << (64-POOL_ID_BITS)) - 1));
         if (!last_pool || pool_id != last_pool)
         {
             if (last_pool)
@@ -207,7 +207,7 @@ void osd_t::report_statistics()
     for (auto kv: inode_stats)
     {
         pool_id_t pool_id = INODE_POOL(kv.first);
-        uint64_t only_inode_num = (kv.first & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1));
+        uint64_t only_inode_num = (kv.first & ((1l << (64-POOL_ID_BITS)) - 1));
         if (!last_pool || pool_id != last_pool)
         {
             if (last_pool)

src/osd_id.h

@@ -9,7 +9,7 @@
 #define POOL_ID_MAX 0x10000
 #define POOL_ID_BITS 16
 #define INODE_POOL(inode) (pool_id_t)((inode) >> (64 - POOL_ID_BITS))
-#define INODE_NO_POOL(inode) (inode_t)(inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1))
+#define INODE_NO_POOL(inode) (inode_t)(inode & ((1l << (64-POOL_ID_BITS)) - 1))
 #define INODE_WITH_POOL(pool_id, inode) (((inode_t)(pool_id) << (64-POOL_ID_BITS)) | INODE_NO_POOL(inode))
 
 // Pool ID is 16 bits long

src/osd_peering_pg.cpp

@@ -437,7 +437,7 @@ void pg_t::calc_object_states(int log_level)
     st.walk();
     if (this->state & (PG_DEGRADED|PG_LEFT_ON_DEAD))
     {
-        assert(epoch != (((uint64_t)1 << PG_EPOCH_BITS)-1));
+        assert(epoch != ((1ul << PG_EPOCH_BITS)-1));
         epoch++;
     }
 }

src/osd_primary_write.cpp

@@ -144,9 +144,9 @@ resume_3:
     }
     else
     {
-        if ((op_data->fact_ver & ((uint64_t)1 << (64-PG_EPOCH_BITS) - 1)) == ((uint64_t)1 << (64-PG_EPOCH_BITS) - 1))
+        if ((op_data->fact_ver & (1ul<<(64-PG_EPOCH_BITS) - 1)) == (1ul<<(64-PG_EPOCH_BITS) - 1))
         {
-            assert(pg.epoch != (((uint64_t)1 << PG_EPOCH_BITS)-1));
+            assert(pg.epoch != ((1ul << PG_EPOCH_BITS)-1));
             pg.epoch++;
         }
         op_data->target_ver = op_data->fact_ver + 1;

src/qemu_driver.c

@@ -262,7 +262,7 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E
         client->pool = qdict_get_try_int(options, "pool", 0);
         if (client->pool)
         {
-            client->inode = (client->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)) | (client->pool << (64-POOL_ID_BITS));
+            client->inode = (client->inode & ((1l << (64-POOL_ID_BITS)) - 1)) | (client->pool << (64-POOL_ID_BITS));
         }
         client->size = qdict_get_try_int(options, "size", 0);
     }

src/test_shit.cpp

@@ -406,7 +406,7 @@ uint64_t crush(uint64_t key, int count, uint64_t *weights)
         seed = (key + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
         seed ^= (j + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
         seed = 2862933555777941757ull*seed + 3037000493ull; // LCPRNG
-        seed = -log(((double)seed) / ((uint64_t)1 << 32) / ((uint64_t)1 << 32)) * weights[j];
+        seed = -log(((double)seed) / (1ul << 32) / (1ul << 32)) * weights[j];
         if (seed > max)
         {
             max = seed;
@@ -439,8 +439,8 @@ void crush3(uint64_t key, int count, uint64_t *weights, uint64_t *r, uint64_t to
                 seed ^= (k2 + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
                 seed ^= (k3 + 0xc6a4a7935bd1e995 + (seed << 6) + (seed >> 2));
                 seed = 2862933555777941757ull*seed + 3037000493ull; // LCPRNG
-                //seed = ((double)seed) / ((uint64_t)1 << 32) / ((uint64_t)1 << 32) * (weights[k1] + weights[k2] + weights[k3]);
-                seed = ((double)seed) / ((uint64_t)1 << 32) / ((uint64_t)1 << 32) * (1 -
+                //seed = ((double)seed) / (1ul << 32) / (1ul << 32) * (weights[k1] + weights[k2] + weights[k3]);
+                seed = ((double)seed) / (1ul << 32) / (1ul << 32) * (1 -
                     (1 - 1.0*weights[k1]/total_weight)*
                     (1 - 1.0*weights[k2]/total_weight)*
                     (1 - 1.0*weights[k3]/total_weight)

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 0.6.17
+Version: 0.6.16
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

tests/test_add_osd.sh

@@ -4,7 +4,7 @@ PG_COUNT=16
 
 . `dirname $0`/run_3osds.sh
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD=libasan.so.5 \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -end_fsync=1 \
         -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
 

tests/test_change_pg_count.sh

@@ -37,7 +37,7 @@ if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '(
     format_error "FAILED: 16 PGS NOT UP"
 fi
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD=libasan.so.5 \
 fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
     -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M -cluster_log_level=10
 

tests/test_etcd_fail.sh

@@ -5,7 +5,7 @@ ETCD_COUNT=5
 
 . `dirname $0`/run_3osds.sh
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD=libasan.so.5 \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
         -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
 
@@ -26,7 +26,7 @@ kill_etcds()
 
 kill_etcds &
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD=libasan.so.5 \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
         -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10 -runtime=30
 

tests/test_interrupted_rebalance.sh

@@ -4,7 +4,7 @@
 
 IMG_SIZE=960
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD=libasan.so.5 \
 fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=16 -fsync=16 -rw=write \
     -etcd=$ETCD_URL -pool=1 -inode=2 -size=${IMG_SIZE}M -cluster_log_level=10
 

tests/test_move_reappear.sh

@@ -21,7 +21,7 @@ if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | le
     format_error "Failed to start the PG active+degraded"
 fi
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD=libasan.so.5 \
 fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
     -etcd=$ETCD_URL -pool=1 -inode=2 -size=32M -cluster_log_level=10
 

tests/test_rm.sh

@@ -3,7 +3,7 @@
 PG_COUNT=16
 . `dirname $0`/run_3osds.sh
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD=libasan.so.5 \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 \
         -end_fsync=1 -fsync=1 -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
 

tests/test_snapshot.sh

@@ -6,7 +6,7 @@
 
 $ETCDCTL put /vitastor/config/inode/1/2 '{"name":"testimg","size":'$((32*1024*1024))'}'
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD="libasan.so.5 build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
         -etcd=$ETCD_URL -pool=1 -inode=2 -size=32M -cluster_log_level=10
 
@@ -14,11 +14,11 @@ $ETCDCTL put /vitastor/config/inode/1/2 '{"name":"testimg@0","size":'$((32*1024*
 $ETCDCTL put /vitastor/config/inode/1/3 '{"parent_id":2,"name":"testimg","size":'$((32*1024*1024))'}'
 
 # Preload build/src/libfio_vitastor.so so libasan detects all symbols
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD="libasan.so.5 build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=32 -buffer_pattern=0xdeadface \
         -rw=randwrite -etcd=$ETCD_URL -image=testimg -number_ios=1024
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD="libasan.so.5 build/src/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -rw=read -etcd=$ETCD_URL -pool=1 -inode=3 -size=32M
 
 qemu-img convert -S 4096 -p \

tests/test_splitbrain.sh

@@ -2,7 +2,6 @@
 
 OSD_COUNT=2
 PG_SIZE=2
-PG_MINSIZE=1
 SCHEME=replicated
 
 . `dirname $0`/run_3osds.sh
@@ -14,7 +13,7 @@ sleep 2
 
 # Write
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD=libasan.so.5 \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=1 \
         -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10 -number_ios=100
 

tests/test_write.sh

@@ -7,20 +7,20 @@
 
 # Random writes without immediate_commit were stalling OSDs
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
-    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=68k -direct=1 -numjobs=16 -iodepth=4 \
+LD_PRELOAD=libasan.so.5 \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=124k -direct=1 -numjobs=16 -iodepth=4 \
         -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10
 
 # A lot of parallel syncs was crashing the primary OSD at some point
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD=libasan.so.5 \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -numjobs=64 -iodepth=1 -fsync=1 \
         -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -number_ios=100
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD=libasan.so.5 \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD=libasan.so.5 \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=32 -buffer_pattern=0xdeadface \
         -rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -number_ios=1024
 

tests/test_write_no_same.sh

@@ -11,7 +11,7 @@ GLOBAL_CONF='{"immediate_commit":"all"}'
 
 # Test basic write
 
-LD_PRELOAD="build/src/libfio_vitastor.so" \
+LD_PRELOAD=libasan.so.5 \
     fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=1G -cluster_log_level=10
 
 format_green OK