Implement CSI volume staging (one device per volume on each host)

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VERSION "1.2.0")
+set(VERSION "1.1.0")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v1.2.0
+VERSION ?= v1.1.0
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.2.0
+          image: vitalif/vitastor-csi:v1.1.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/005-csi-provisioner-rbac.yaml

@@ -35,13 +35,10 @@ rules:
     verbs: ["get", "list", "watch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshots"]
-    verbs: ["get", "list", "patch"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots/status"]
-    verbs: ["get", "list", "patch"]
+    verbs: ["get", "list"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents"]
-    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotclasses"]
     verbs: ["get", "list", "watch"]
@@ -56,7 +53,7 @@ rules:
     verbs: ["get", "list", "watch"]
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents/status"]
-    verbs: ["update", "patch"]
+    verbs: ["update"]
   - apiGroups: [""]
     resources: ["configmaps"]
     verbs: ["get"]

csi/deploy/007-csi-provisioner.yaml

@@ -23,11 +23,6 @@ metadata:
   name: csi-vitastor-provisioner
 spec:
   replicas: 3
-  strategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-      maxSurge: 0
   selector:
     matchLabels:
       app: csi-vitastor-provisioner
@@ -51,7 +46,7 @@ spec:
       priorityClassName: system-cluster-critical
       containers:
         - name: csi-provisioner
-          image: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
+          image: k8s.gcr.io/sig-storage/csi-provisioner:v2.2.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=5"
@@ -121,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.2.0
+          image: vitalif/vitastor-csi:v1.1.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/009-storage-class.yaml

@@ -17,4 +17,3 @@ parameters:
   # multiple etcdUrls may be specified, delimited by comma
   #etcdUrl: "http://192.168.7.2:2379"
   #etcdPrefix: "/vitastor"
-allowVolumeExpansion: true

csi/deploy/example-snapshot-class.yaml

@@ -1,7 +0,0 @@
-apiVersion: snapshot.storage.k8s.io/v1
-kind: VolumeSnapshotClass
-metadata:
-  name: vitastor-snapclass
-driver: csi.vitastor.io
-deletionPolicy: Delete
-parameters:

csi/deploy/example-snapshot-clone.yaml

@@ -1,16 +0,0 @@
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: test-vitastor-clone
-spec:
-  storageClassName: vitastor
-  dataSource:
-    name: snap1
-    kind: VolumeSnapshot
-    apiGroup: snapshot.storage.k8s.io
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 10Gi

csi/deploy/example-snapshot.yaml

@@ -1,8 +0,0 @@
-apiVersion: snapshot.storage.k8s.io/v1
-kind: VolumeSnapshot
-metadata:
-  name: snap1
-spec:
-  volumeSnapshotClassName: vitastor-snapclass
-  source:
-    persistentVolumeClaimName: test-vitastor-pvc

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.2.0"
+    vitastorCSIDriverVersion = "1.1.0"
 )
 
 // Config struct fills the parameters of request or user input

csi/src/controllerserver.go

@@ -180,34 +180,8 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
         return nil, status.Error(codes.InvalidArgument, "no etcdUrl in storage class configuration and no etcd_address in vitastor.conf")
     }
 
-    args := []string{ "create", volName, "-s", fmt.Sprintf("%v", volSize), "--pool", fmt.Sprintf("%v", poolId) }
-
-    // Support creation from snapshot
-    var src *csi.VolumeContentSource
-    if (req.VolumeContentSource.GetSnapshot() != nil)
-    {
-        snapId := req.VolumeContentSource.GetSnapshot().GetSnapshotId()
-        if (snapId != "")
-        {
-            snapVars := make(map[string]string)
-            err := json.Unmarshal([]byte(snapId), &snapVars)
-            if (err != nil)
-            {
-                return nil, status.Error(codes.Internal, "volume ID not in JSON format")
-            }
-            args = append(args, "--parent", snapVars["name"]+"@"+snapVars["snapshot"])
-            src = &csi.VolumeContentSource{
-                Type: &csi.VolumeContentSource_Snapshot{
-                    Snapshot: &csi.VolumeContentSource_SnapshotSource{
-                        SnapshotId: snapId,
-                    },
-                },
-            }
-        }
-    }
-
     // Create image using vitastor-cli
-    _, err := invokeCLI(ctxVars, args)
+    _, err := invokeCLI(ctxVars, []string{ "create", volName, "-s", fmt.Sprintf("%v", volSize), "--pool", fmt.Sprintf("%v", poolId) })
     if (err != nil)
     {
         if (strings.Index(err.Error(), "already exists") > 0)
@@ -235,7 +209,6 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
             // Ugly, but VolumeContext isn't passed to DeleteVolume :-(
             VolumeId: string(volumeIdJson),
             CapacityBytes: volSize,
-            ContentSource: src,
         },
     }, nil
 }
@@ -571,11 +544,16 @@ func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi
         {
             return nil, err
         }
-        inodeCfg, err = invokeList(ctxVars, volName, true)
+        inodeCfg, err := invokeList(ctxVars, volName, true)
         if (err != nil)
         {
             return nil, err
         }
+        return &csi.ControllerExpandVolumeResponse{
+            CapacityBytes: int64(inodeCfg[0].Size),
+            // Node expansion is required for FS
+            NodeExpansionRequired: req.VolumeCapability == nil || req.VolumeCapability.GetBlock() == nil,
+        }, nil
     }
 
     return &csi.ControllerExpandVolumeResponse{

csi/src/nodeserver.go

@@ -37,38 +37,8 @@ func NewNodeServer(driver *Driver) *NodeServer
     }
 }
 
-// NodeStageVolume mounts the volume to a staging path on the node.
-func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error)
+func (ns *NodeServer) checkMountPoint(targetPath string, isBlock bool)
 {
-    return &csi.NodeStageVolumeResponse{}, nil
-}
-
-// NodeUnstageVolume unstages the volume from the staging path
-func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error)
-{
-    return &csi.NodeUnstageVolumeResponse{}, nil
-}
-
-func Contains(list []string, s string) bool
-{
-    for i := 0; i < len(list); i++
-    {
-        if (list[i] == s)
-        {
-            return true
-        }
-    }
-    return false
-}
-
-// NodePublishVolume mounts the volume mounted to the staging path to the target path
-func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error)
-{
-    klog.Infof("received node publish volume request %+v", protosanitizer.StripSecrets(req))
-
-    targetPath := req.GetTargetPath()
-    isBlock := req.GetVolumeCapability().GetBlock() != nil
-
     // Check that it's not already mounted
     _, err := mount.IsNotMountPoint(ns.mounter, targetPath)
     if (err != nil)
@@ -81,13 +51,13 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
                 if (err != nil)
                 {
                     klog.Errorf("failed to create block device mount target %s with error: %v", targetPath, err)
-                    return nil, status.Error(codes.Internal, err.Error())
+                    return status.Error(codes.Internal, err.Error("failed to create block device mount target"))
                 }
                 err = pathFile.Close()
                 if (err != nil)
                 {
                     klog.Errorf("failed to close %s with error: %v", targetPath, err)
-                    return nil, status.Error(codes.Internal, err.Error())
+                    return status.Error(codes.Internal, err.Error("failed to create block device mount target"))
                 }
             }
             else
@@ -96,18 +66,35 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
                 if (err != nil)
                 {
                     klog.Errorf("failed to create fs mount target %s with error: %v", targetPath, err)
-                    return nil, status.Error(codes.Internal, err.Error())
+                    return status.Error(codes.Internal, err.Error("failed to create fs mount target"))
                 }
             }
         }
         else
         {
-            return nil, status.Error(codes.Internal, err.Error())
+            return status.Error(codes.Internal, err)
         }
     }
+    return nil
+}
+
+// NodeStageVolume mounts the volume to a staging path on the node.
+func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error)
+{
+    klog.Infof("received node stage volume request %+v", protosanitizer.StripSecrets(req))
+
+    targetPath := req.GetStagingTargetPath()
+    isBlock := req.GetVolumeCapability().GetBlock() != nil
+
+    // Check that it's not already mounted
+    err := ns.checkMountPoint(targetPath, isBlock)
+    if (err != nil)
+    {
+        return nil, err
+    }
 
     ctxVars := make(map[string]string)
-    err = json.Unmarshal([]byte(req.VolumeId), &ctxVars)
+    err := json.Unmarshal([]byte(req.VolumeId), &ctxVars)
     if (err != nil)
     {
         return nil, status.Error(codes.Internal, "volume ID not in JSON format")
@@ -121,7 +108,6 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
     }
 
     // Map NBD device
-    // FIXME: Check if already mapped
     args := []string{
         "map", "--etcd_address", strings.Join(etcdUrl, ","),
         "--etcd_prefix", etcdPrefix,
@@ -147,74 +133,70 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
     }
     devicePath := strings.TrimSpace(stdoutStr)
 
+    // Check existing format
     diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: utilexec.New()}
+    existingFormat, err := diskMounter.GetDiskFormat(devicePath)
+    if (err != nil)
+    {
+        klog.Errorf("failed to get disk format for path %s, error: %v", err)
+        // unmap NBD device
+        unmapOut, unmapErr := exec.Command("/usr/bin/vitastor-nbd", "unmap", devicePath).CombinedOutput()
+        if (unmapErr != nil)
+        {
+            klog.Errorf("failed to unmap NBD device %s: %s, error: %v", devicePath, unmapOut, unmapErr)
+        }
+        return nil, err
+    }
+
+    // Format the device (ext4 or xfs)
+    fsType := req.GetVolumeCapability().GetMount().GetFsType()
+    opt := req.GetVolumeCapability().GetMount().GetMountFlags()
+    opt = append(opt, "_netdev")
+    if ((req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY ||
+        req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY) &&
+        !Contains(opt, "ro"))
+    {
+        opt = append(opt, "ro")
+    }
+    if (fsType == "xfs")
+    {
+        opt = append(opt, "nouuid")
+    }
+    readOnly := Contains(opt, "ro")
+    if (existingFormat == "" && !readOnly)
+    {
+        args := []string{}
+        switch fsType
+        {
+            case "ext4":
+                args = []string{"-m0", "-Enodiscard,lazy_itable_init=1,lazy_journal_init=1", devicePath}
+            case "xfs":
+                args = []string{"-K", devicePath}
+        }
+        if (len(args) > 0)
+        {
+            cmdOut, cmdErr := diskMounter.Exec.Command("mkfs."+fsType, args...).CombinedOutput()
+            if (cmdErr != nil)
+            {
+                klog.Errorf("failed to run mkfs error: %v, output: %v", cmdErr, string(cmdOut))
+                // unmap NBD device
+                unmapOut, unmapErr := exec.Command("/usr/bin/vitastor-nbd", "unmap", devicePath).CombinedOutput()
+                if (unmapErr != nil)
+                {
+                    klog.Errorf("failed to unmap NBD device %s: %s, error: %v", devicePath, unmapOut, unmapErr)
+                }
+                return nil, status.Error(codes.Internal, cmdErr.Error())
+            }
+        }
+    }
     if (isBlock)
     {
-        err = diskMounter.Mount(devicePath, targetPath, "", []string{"bind"})
+        opt = append(opt, "bind")
+        err = diskMounter.Mount(devicePath, targetPath, fsType, opt)
     }
     else
     {
-        // Check existing format
-        existingFormat, err := diskMounter.GetDiskFormat(devicePath)
-        if (err != nil)
-        {
-            klog.Errorf("failed to get disk format for path %s, error: %v", err)
-            goto unmap
-        }
-
-        // Format the device (ext4 or xfs)
-        fsType := req.GetVolumeCapability().GetMount().GetFsType()
-        opt := req.GetVolumeCapability().GetMount().GetMountFlags()
-        opt = append(opt, "_netdev")
-        if ((req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY ||
-            req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY) &&
-            !Contains(opt, "ro"))
-        {
-            opt = append(opt, "ro")
-        }
-        if (fsType == "xfs")
-        {
-            opt = append(opt, "nouuid")
-        }
-        readOnly := Contains(opt, "ro")
-        if (existingFormat == "" && !readOnly)
-        {
-            var cmdOut []byte
-            switch fsType
-            {
-                case "ext4":
-                    args := []string{"-m0", "-Enodiscard,lazy_itable_init=1,lazy_journal_init=1", devicePath}
-                    cmdOut, err = diskMounter.Exec.Command("mkfs.ext4", args...).CombinedOutput()
-                case "xfs":
-                    cmdOut, err = diskMounter.Exec.Command("mkfs.xfs", "-K", devicePath).CombinedOutput()
-            }
-            if (err != nil)
-            {
-                klog.Errorf("failed to run mkfs error: %v, output: %v", err, string(cmdOut))
-                goto unmap
-            }
-        }
-
         err = diskMounter.FormatAndMount(devicePath, targetPath, fsType, opt)
-
-        // Try to run online resize on mount.
-        // FIXME: Implement online resize. It requires online resize support in vitastor-nbd.
-        if (err == nil && existingFormat != "" && !readOnly)
-        {
-            var cmdOut []byte
-            switch (fsType)
-            {
-                case "ext4":
-                    cmdOut, err = diskMounter.Exec.Command("resize2fs", devicePath).CombinedOutput()
-                case "xfs":
-                    cmdOut, err = diskMounter.Exec.Command("xfs_growfs", devicePath).CombinedOutput()
-            }
-            if (err != nil)
-            {
-                klog.Errorf("failed to run resizefs error: %v, output: %v", err, string(cmdOut))
-                goto unmap
-            }
-        }
     }
     if (err != nil)
     {
@@ -222,43 +204,103 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
             "failed to mount device path (%s) to path (%s) for volume (%s) error: %s",
             devicePath, targetPath, volName, err,
         )
-        goto unmap
+        // unmap NBD device
+        unmapOut, unmapErr := exec.Command("/usr/bin/vitastor-nbd", "unmap", devicePath).CombinedOutput()
+        if (unmapErr != nil)
+        {
+            klog.Errorf("failed to unmap NBD device %s: %s, error: %v", devicePath, unmapOut, unmapErr)
+        }
+        return nil, status.Error(codes.Internal, err.Error())
     }
-    return &csi.NodePublishVolumeResponse{}, nil
 
-unmap:
-    // unmap NBD device
-    unmapOut, unmapErr := exec.Command("/usr/bin/vitastor-nbd", "unmap", devicePath).CombinedOutput()
-    if (unmapErr != nil)
+    return &csi.NodeStageVolumeResponse{}, nil
+}
+
+// NodeUnstageVolume unstages the volume from the staging path
+func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error)
+{
+    klog.Infof("received node unstage volume request %+v", protosanitizer.StripSecrets(req))
+    err := ns.unmount(req.GetStagingTargetPath())
+    if (err != nil)
     {
-        klog.Errorf("failed to unmap NBD device %s: %s, error: %v", devicePath, unmapOut, unmapErr)
+        return nil, err
     }
-    return nil, status.Error(codes.Internal, err.Error())
+    return &csi.NodeUnstageVolumeResponse{}, nil
+}
+
+func Contains(list []string, s string) bool
+{
+    for i := 0; i < len(list); i++
+    {
+        if (list[i] == s)
+        {
+            return true
+        }
+    }
+    return false
+}
+
+// NodePublishVolume mounts the volume mounted to the staging path to the target path
+func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error)
+{
+    klog.Infof("received node publish volume request %+v", protosanitizer.StripSecrets(req))
+
+    stagingTargetPath := req.GetStagingTargetPath()
+    targetPath := req.GetTargetPath()
+    isBlock := req.GetVolumeCapability().GetBlock() != nil
+
+    err := ns.checkMountPoint(targetPath, isBlock)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: utilexec.New()}
+    err = diskMounter.Mount(stagingTargetPath, targetPath, "", []string{"bind"})
+    if (err != nil)
+    {
+        klog.Errorf(
+            "failed to bind staging path (%s) to path (%s) for volume (%s) error: %s",
+            stagingPath, targetPath, volName, err,
+        )
+        return nil, status.Error(codes.Internal, err.Error("failed to bind staging path"))
+    }
+
+    return &csi.NodePublishVolumeResponse{}, nil
 }
 
 // NodeUnpublishVolume unmounts the volume from the target path
 func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error)
 {
     klog.Infof("received node unpublish volume request %+v", protosanitizer.StripSecrets(req))
-    targetPath := req.GetTargetPath()
+    err := ns.unmount(req.GetTargetPath())
+    if (err != nil)
+    {
+        return nil, err
+    }
+    return &csi.NodeUnpublishVolumeResponse{}, nil
+}
+
+func (ns *NodeServer) unmount(targetPath string) error
+{
     devicePath, refCount, err := mount.GetDeviceNameFromMount(ns.mounter, targetPath)
     if (err != nil)
     {
         if (os.IsNotExist(err))
         {
-            return nil, status.Error(codes.NotFound, "Target path not found")
+            return status.Error(codes.NotFound, "Target path not found")
         }
-        return nil, status.Error(codes.Internal, err.Error())
+        return status.Error(codes.Internal, err.Error())
     }
     if (devicePath == "")
     {
-        return nil, status.Error(codes.NotFound, "Volume not mounted")
+        return status.Error(codes.NotFound, "Volume not mounted")
     }
     // unmount
     err = mount.CleanupMountPoint(targetPath, ns.mounter, false)
     if (err != nil)
     {
-        return nil, status.Error(codes.Internal, err.Error())
+        return status.Error(codes.Internal, err.Error())
     }
     // unmap NBD device
     if (refCount == 1)
@@ -269,7 +311,7 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
             klog.Errorf("failed to unmap NBD device %s: %s, error: %v", devicePath, unmapOut, unmapErr)
         }
     }
-    return &csi.NodeUnpublishVolumeResponse{}, nil
+    return nil
 }
 
 // NodeGetVolumeStats returns volume capacity statistics available for the volume

debian/changelog

@@ -1,10 +1,10 @@
-vitastor (1.2.0-1) unstable; urgency=medium
+vitastor (1.1.0-1) unstable; urgency=medium
 
   * Bugfixes
 
  -- Vitaliy Filippov <vitalif@yourcmc.ru>  Fri, 03 Jun 2022 02:09:44 +0300
 
-vitastor (1.2.0-1) unstable; urgency=medium
+vitastor (1.1.0-1) unstable; urgency=medium
 
   * Implement NFS proxy
   * Add documentation

debian/control

@@ -2,7 +2,7 @@ Source: vitastor
 Section: admin
 Priority: optional
 Maintainer: Vitaliy Filippov <vitalif@yourcmc.ru>
-Build-Depends: debhelper, liburing-dev (>= 0.6), g++ (>= 8), libstdc++6 (>= 8), linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev, libibverbs-dev, libisal-dev, cmake, pkg-config
+Build-Depends: debhelper, liburing-dev (>= 0.6), g++ (>= 8), libstdc++6 (>= 8), linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev, libibverbs-dev, libisal-dev
 Standards-Version: 4.5.0
 Homepage: https://vitastor.io/
 Rules-Requires-Root: no

debian/vitastor.Dockerfile

@@ -35,8 +35,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-1.2.0; \
-    cd vitastor-1.2.0; \
+    cp -r /root/vitastor vitastor-1.1.0; \
+    cd vitastor-1.1.0; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -49,8 +49,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.2.0.orig.tar.xz vitastor-1.2.0; \
-    cd vitastor-1.2.0; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.1.0.orig.tar.xz vitastor-1.1.0; \
+    cd vitastor-1.1.0; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

docs/installation/kubernetes.en.md

@@ -17,15 +17,4 @@ and apply all `NNN-*.yaml` manifests to your Kubernetes installation:
 for i in ./???-*.yaml; do kubectl apply -f $i; done
 ```
 
-After that you'll be able to create PersistentVolumes.
-
-## Features
-
-Vitastor CSI supports:
-- Kubernetes starting with 1.20 (or 1.17 for older vitastor-csi <= 1.1.0)
-- Filesystem RWO (ReadWriteOnce) volumes. Example: [PVC](../../csi/deploy/example-pvc.yaml), [pod](../../csi/deploy/example-test-pod.yaml)
-- Raw block RWX (ReadWriteMany) volumes. Example: [PVC](../../csi/deploy/example-pvc-block.yaml), [pod](../../csi/deploy/example-test-pod-block.yaml)
-- Volume expansion
-- Volume snapshots. Example: [snapshot class](../../csi/deploy/example-snapshot-class.yaml), [snapshot](../../csi/deploy/example-snapshot.yaml), [clone](../../csi/deploy/example-snapshot-clone.yaml)
-
-Remember that to use snapshots with CSI you also have to install [Snapshot Controller and CRDs](https://kubernetes-csi.github.io/docs/snapshot-controller.html#deployment).
+After that you'll be able to create PersistentVolumes. See example in [csi/deploy/example-pvc.yaml](../../csi/deploy/example-pvc.yaml).

docs/installation/kubernetes.ru.md

@@ -17,15 +17,4 @@
 for i in ./???-*.yaml; do kubectl apply -f $i; done
 ```
 
-После этого вы сможете создавать PersistentVolume.
-
-## Возможности
-
-CSI-плагин Vitastor поддерживает:
-- Версии Kubernetes, начиная с 1.20 (или с 1.17 для более старых vitastor-csi <= 1.1.0)
-- Файловые RWO (ReadWriteOnce) тома. Пример: [PVC](../../csi/deploy/example-pvc.yaml), [под](../../csi/deploy/example-test-pod.yaml)
-- Сырые блочные RWX (ReadWriteMany) тома. Пример: [PVC](../../csi/deploy/example-pvc-block.yaml), [под](../../csi/deploy/example-test-pod-block.yaml)
-- Расширение размера томов
-- Снимки томов. Пример: [класс снимков](../../csi/deploy/example-snapshot-class.yaml), [снимок](../../csi/deploy/example-snapshot.yaml), [клон снимка](../../csi/deploy/example-snapshot-clone.yaml)
-
-Не забывайте, что для использования снимков нужно сначала установить [контроллер снимков и CRD](https://kubernetes-csi.github.io/docs/snapshot-controller.html#deployment).
+После этого вы сможете создавать PersistentVolume. Пример смотрите в файле [csi/deploy/example-pvc.yaml](../../csi/deploy/example-pvc.yaml).

mon/mon.js

@@ -403,7 +403,6 @@ class Mon
         this.ws_alive = false;
         this.ws_keepalive_timer = null;
         this.on_stop_cb = () => this.on_stop(0).catch(console.error);
-        this.recheck_pgs_active = false;
     }
 
     parse_etcd_addresses(addrs)
@@ -693,27 +692,8 @@ class Mon
         });
     }
 
-    // Schedule save_last_clean() to to run after a small timeout (1s) (to not spam etcd)
-    schedule_save_last_clean()
-    {
-        if (!this.save_last_clean_timer)
-        {
-            this.save_last_clean_timer = setTimeout(() =>
-            {
-                this.save_last_clean_timer = null;
-                this.save_last_clean().catch(this.die);
-            }, this.config.mon_change_timeout || 1000);
-        }
-    }
-
     async save_last_clean()
     {
-        if (this.save_last_clean_running)
-        {
-            this.schedule_save_last_clean();
-            return;
-        }
-        this.save_last_clean_running = true;
         // last_clean_pgs is used to avoid extra data move when observing a series of changes in the cluster
         const new_clean_pgs = { items: {} };
     next_pool:
@@ -750,7 +730,6 @@ class Mon
                 value: b64(JSON.stringify(this.state.history.last_clean_pgs))
             } } ],
         }, this.etcd_start_timeout, 0);
-        this.save_last_clean_running = false;
     }
 
     get_mon_state()
@@ -1224,12 +1203,6 @@ class Mon
 
     async recheck_pgs()
     {
-        if (this.recheck_pgs_active)
-        {
-            this.schedule_recheck();
-            return;
-        }
-        this.recheck_pgs_active = true;
         // Take configuration and state, check it against the stored configuration hash
         // Recalculate PGs and save them to etcd if the configuration is changed
         // FIXME: Do not change anything if the distribution is good and random enough and no PGs are degraded
@@ -1251,7 +1224,6 @@ class Mon
                     // Pool deleted. Delete all PGs, but first stop them.
                     if (!await this.stop_all_pgs(pool_id))
                     {
-                        this.recheck_pgs_active = false;
                         this.schedule_recheck();
                         return;
                     }
@@ -1320,16 +1292,9 @@ class Mon
                         // PG count changed. Need to bring all PGs down.
                         if (!await this.stop_all_pgs(pool_id))
                         {
-                            this.recheck_pgs_active = false;
                             this.schedule_recheck();
                             return;
                         }
-                    }
-                    if (prev_pgs.length != pool_cfg.pg_count)
-                    {
-                        // Scale PG count
-                        // Do it even if old_pg_count is already equal to pool_cfg.pg_count,
-                        // because last_clean_pgs may still contain the old number of PGs
                         const new_pg_history = [];
                         PGUtil.scale_pg_count(prev_pgs, real_prev_pgs, pg_history, new_pg_history, pool_cfg.pg_count);
                         pg_history = new_pg_history;
@@ -1431,7 +1396,6 @@ class Mon
                 await this.save_pg_config(new_config_pgs);
             }
         }
-        this.recheck_pgs_active = false;
     }
 
     async save_pg_config(new_config_pgs, etcd_request = { compare: [], success: [] })
@@ -1481,6 +1445,7 @@ class Mon
     }
 
     // Schedule a recheck to run after a small timeout (1s)
+    // If already scheduled, cancel previous timer and schedule it again
     // This is required for multiple change events to trigger at most 1 recheck in 1s
     schedule_recheck()
     {

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.2.0",
+  "version": "1.1.0",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '1.2.0'
+VERSION = '1.1.0'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -24,4 +24,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-1.2.0/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.2.0$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-1.1.0/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.1.0$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.2.0.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.1.0.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.2.0
+Version:        1.1.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.2.0.el7.tar.gz
+Source0:        vitastor-1.1.0.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.2.0.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.1.0.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.2.0
+Version:        1.1.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.2.0.el8.tar.gz
+Source0:        vitastor-1.1.0.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.Dockerfile

@@ -18,7 +18,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.2.0.el9.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.1.0.el9.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.2.0
+Version:        1.1.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.2.0.el9.tar.gz
+Source0:        vitastor-1.1.0.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -16,7 +16,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="1.2.0")
+add_definitions(-DVERSION="1.1.0")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)

src/blockstore_flush.cpp

@@ -1372,8 +1372,7 @@ bool journal_flusher_co::trim_journal(int wait_base)
                     ? (uint32_t)JE_START_V1_SIZE : (uint32_t)JE_START_V2_SIZE),
                 .reserved = 0,
                 .journal_start = new_trim_pos,
-                .version = (uint64_t)(!bs->dsk.data_csum_type && ((journal_entry_start*)flusher->journal_superblock)->version == JOURNAL_VERSION_V1
-                    ? JOURNAL_VERSION_V1 : JOURNAL_VERSION_V2),
+                .version = JOURNAL_VERSION_V2,
                 .data_csum_type = bs->dsk.data_csum_type,
                 .csum_block_size = bs->dsk.csum_block_size,
             };

src/blockstore_init.cpp

@@ -553,7 +553,7 @@ resume_1:
         }
         if (je_start->size == JE_START_V0_SIZE ||
             (je_start->version != JOURNAL_VERSION_V1 || je_start->size != JE_START_V1_SIZE) &&
-            (je_start->version != JOURNAL_VERSION_V2 || je_start->size != JE_START_V2_SIZE && je_start->size != JE_START_V1_SIZE))
+            (je_start->version != JOURNAL_VERSION_V2 || je_start->size != JE_START_V2_SIZE))
         {
             fprintf(
                 stderr, "The code only supports journal versions 2 and 1, but it is %lu on disk."
@@ -562,8 +562,7 @@ resume_1:
             );
             exit(1);
         }
-        if (je_start->version == JOURNAL_VERSION_V1 ||
-            je_start->version == JOURNAL_VERSION_V2 && je_start->size == JE_START_V1_SIZE)
+        if (je_start->version == JOURNAL_VERSION_V1)
         {
             je_start->data_csum_type = 0;
             je_start->csum_block_size = 0;

src/blockstore_write.cpp

@@ -21,7 +21,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
         dyn = calloc_or_die(1, dyn_size+sizeof(int));
         *((int*)dyn) = 1;
     }
-    uint8_t *dyn_ptr = (alloc_dyn_data ? (uint8_t*)dyn+sizeof(int) : (uint8_t*)&dyn);
+    uint8_t *dyn_ptr = (uint8_t*)(alloc_dyn_data ? dyn+sizeof(int) : &dyn);
     uint64_t version = 1;
     if (dirty_db.size() > 0)
     {

src/cli.cpp

@@ -17,7 +17,7 @@
 static const char *exe_name = NULL;
 
 static const char* help_text =
-    "Vitastor command-line tool " VERSION "\n"
+    "Vitastor command-line tool\n"
     "(c) Vitaliy Filippov, 2019+ (VNPL-1.1)\n"
     "\n"
     "COMMANDS:\n"

src/cli_df.cpp

@@ -109,7 +109,7 @@ resume_1:
             }
             for (auto pg_per_pair: pg_per_osd)
             {
-                uint64_t pg_free = osd_free[pg_per_pair.first] * pool_cfg.real_pg_count / pg_per_pair.second;
+                uint64_t pg_free = osd_free[pg_per_pair.first] * pool_cfg.pg_count / pg_per_pair.second;
                 if (pool_avail > pg_free)
                 {
                     pool_avail = pg_free;
@@ -124,10 +124,8 @@ resume_1:
                 pool_avail *= (pool_cfg.pg_size - pool_cfg.parity_chunks);
             }
             pool_stats[pool_cfg.id] = json11::Json::object {
-                { "id", (uint64_t)pool_cfg.id },
                 { "name", pool_cfg.name },
                 { "pg_count", pool_cfg.pg_count },
-                { "real_pg_count", pool_cfg.real_pg_count },
                 { "scheme", pool_cfg.scheme == POOL_SCHEME_REPLICATED ? "replicated" : "ec" },
                 { "scheme_name", pool_cfg.scheme == POOL_SCHEME_REPLICATED
                     ? std::to_string(pool_cfg.pg_size)+"/"+std::to_string(pool_cfg.pg_minsize)
@@ -178,7 +176,7 @@ resume_1:
             { "title", "SCHEME" },
         });
         cols.push_back(json11::Json::object{
-            { "key", "pg_count_fmt" },
+            { "key", "pg_count" },
             { "title", "PGS" },
         });
         cols.push_back(json11::Json::object{
@@ -207,9 +205,6 @@ resume_1:
             double raw_to = kv.second["raw_to_usable"].number_value();
             if (raw_to < 0.000001 && raw_to > -0.000001)
                 raw_to = 1;
-            kv.second["pg_count_fmt"] = kv.second["real_pg_count"] == kv.second["pg_count"]
-                ? kv.second["real_pg_count"].as_string()
-                : kv.second["real_pg_count"].as_string()+"->"+kv.second["pg_count"].as_string();
             kv.second["total_fmt"] = format_size(kv.second["total_raw"].uint64_value() / raw_to);
             kv.second["used_fmt"] = format_size(kv.second["used_raw"].uint64_value() / raw_to);
             kv.second["max_avail_fmt"] = format_size(kv.second["max_available"].uint64_value());

src/cli_status.cpp

@@ -158,7 +158,12 @@ resume_2:
         for (auto & pool_pair: parent->cli->st_cli.pool_config)
         {
             auto & pool_cfg = pool_pair.second;
-            bool active = pool_cfg.real_pg_count > 0;
+            bool active = true;
+            if (pool_cfg.pg_config.size() != pool_cfg.pg_count)
+            {
+                active = false;
+                pgs_by_state["offline"] += pool_cfg.pg_count-pool_cfg.pg_config.size();
+            }
             pool_count++;
             for (auto pg_it = pool_cfg.pg_config.begin(); pg_it != pool_cfg.pg_config.end(); pg_it++)
             {

src/cluster_client.cpp

@@ -64,7 +64,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
 
 cluster_client_t::~cluster_client_t()
 {
-    msgr.repeer_pgs = [](osd_num_t){};
+    msgr.repeer_pgs = [this](osd_num_t){};
     if (ringloop)
     {
         ringloop->unregister_consumer(&consumer);
@@ -454,7 +454,7 @@ bool cluster_client_t::flush()
             wb->start_writebacks(this, 0);
             cluster_op_t *sync = new cluster_op_t;
             sync->opcode = OSD_OP_SYNC;
-            sync->callback = [](cluster_op_t *sync)
+            sync->callback = [this](cluster_op_t *sync)
             {
                 delete sync;
             };
@@ -465,7 +465,7 @@ bool cluster_client_t::flush()
     bool sync_done = false;
     cluster_op_t *sync = new cluster_op_t;
     sync->opcode = OSD_OP_SYNC;
-    sync->callback = [&sync_done](cluster_op_t *sync)
+    sync->callback = [this, &sync_done](cluster_op_t *sync)
     {
         delete sync;
         sync_done = true;

src/cluster_client_wb.cpp

@@ -263,7 +263,7 @@ void writeback_cache_t::flush_buffers(cluster_client_t *cli, dirty_buf_it_t from
     }
     assert(calc_len == op->len);
     writebacks_active++;
-    op->callback = [this, flush_id](cluster_op_t* op)
+    op->callback = [this, cli, flush_id](cluster_op_t* op)
     {
         // Buffer flushes should be always retried, regardless of the error,
         // so they should never result in an error here
@@ -383,7 +383,7 @@ static void copy_to_op(cluster_op_t *op, uint64_t offset, uint8_t *buf, uint64_t
             auto begin = (cur_offset < offset ? offset : cur_offset);
             auto end = (cur_offset+v.iov_len > offset+len ? offset+len : cur_offset+v.iov_len);
             memcpy(
-                (uint8_t*)v.iov_base + begin - cur_offset,
+                v.iov_base + begin - cur_offset,
                 buf + (cur_offset <= offset ? 0 : cur_offset-offset),
                 end - begin
             );

src/disk_tool.cpp

@@ -5,7 +5,7 @@
 #include "str_util.h"
 
 static const char *help_text =
-    "Vitastor disk management tool " VERSION "\n"
+    "Vitastor disk management tool\n"
     "(c) Vitaliy Filippov, 2022+ (VNPL-1.1)\n"
     "\n"
     "COMMANDS:\n"

src/messenger.cpp

@@ -490,14 +490,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
                     fprintf(stderr, "Connected to OSD %lu using RDMA\n", cl->osd_num);
                 }
                 cl->peer_state = PEER_RDMA;
-                tfd->set_fd_handler(cl->peer_fd, false, [this](int peer_fd, int epoll_events)
-                {
-                    // Do not miss the disconnection!
-                    if (epoll_events & EPOLLRDHUP)
-                    {
-                        handle_peer_epoll(peer_fd, epoll_events);
-                    }
-                });
+                tfd->set_fd_handler(cl->peer_fd, false, NULL);
                 // Add the initial receive request
                 try_recv_rdma(cl);
             }

src/msgr_send.cpp

@@ -3,7 +3,6 @@
 
 #define _XOPEN_SOURCE
 #include <limits.h>
-#include <sys/epoll.h>
 
 #include "messenger.h"
 
@@ -284,14 +283,7 @@ void osd_messenger_t::handle_send(int result, osd_client_t *cl)
                 fprintf(stderr, "Successfully connected with client %d using RDMA\n", cl->peer_fd);
             }
             cl->peer_state = PEER_RDMA;
-            tfd->set_fd_handler(cl->peer_fd, false, [this](int peer_fd, int epoll_events)
-            {
-                // Do not miss the disconnection!
-                if (epoll_events & EPOLLRDHUP)
-                {
-                    handle_peer_epoll(peer_fd, epoll_events);
-                }
-            });
+            tfd->set_fd_handler(cl->peer_fd, false, NULL);
             // Add the initial receive request
             try_recv_rdma(cl);
         }

src/osd_main.cpp

@@ -19,14 +19,6 @@ static void handle_sigint(int sig)
     exit(0);
 }
 
-static const char* help_text =
-    "Vitastor OSD (block object storage daemon) " VERSION "\n"
-    "(c) Vitaliy Filippov, 2019+ (VNPL-1.1)\n"
-    "\n"
-    "OSDs are usually started by vitastor-disk.\n"
-    "Manual usage: vitastor-osd [--option value] ...\n"
-;
-
 int main(int narg, char *args[])
 {
     setvbuf(stdout, NULL, _IONBF, 0);
@@ -45,16 +37,6 @@ int main(int narg, char *args[])
             char *opt = args[i]+2;
             config[std::string(opt)] = std::string(args[++i]);
         }
-        else if (!strcmp(args[i], "--help"))
-        {
-            printf("%s", help_text);
-            return 0;
-        }
-    }
-    if (!config.size())
-    {
-        printf("%s", help_text);
-        return 1;
     }
     signal(SIGINT, handle_sigint);
     signal(SIGTERM, handle_sigint);

src/osd_rmw.cpp

@@ -239,9 +239,8 @@ static void* get_jerasure_decoding_matrix(osd_rmw_stripe_t *stripes, int pg_size
 {
     int edd = 0;
     int erased[pg_size];
-    // we should distinguish stripes which are not at all and missing stripes
     for (int i = 0; i < pg_size; i++)
-        erased[i] = (stripes[i].read_end == 0 ? 2 : (stripes[i].missing ? 1 : 0));
+        erased[i] = (stripes[i].read_end == 0 || stripes[i].missing ? 1 : 0);
     for (int i = 0; i < pg_minsize; i++)
         if (stripes[i].read_end != 0 && stripes[i].missing)
             edd++;
@@ -254,7 +253,7 @@ static void* get_jerasure_decoding_matrix(osd_rmw_stripe_t *stripes, int pg_size
 #ifdef WITH_ISAL
         int smrow = 0;
         uint8_t *submatrix = (uint8_t*)malloc_or_die(pg_minsize*pg_minsize*2);
-        for (int i = 0; i < pg_size && smrow < pg_minsize; i++)
+        for (int i = 0; i < pg_size; i++)
         {
             if (!erased[i])
             {
@@ -280,7 +279,7 @@ static void* get_jerasure_decoding_matrix(osd_rmw_stripe_t *stripes, int pg_size
         smrow = 0;
         for (int i = 0; i < pg_minsize; i++)
         {
-            if (erased[i] == 1)
+            if (erased[i])
             {
                 memcpy(submatrix + pg_minsize*smrow, submatrix + (pg_minsize+i)*pg_minsize, pg_minsize);
                 smrow++;

src/osd_rmw_test.cpp

@@ -29,7 +29,6 @@ void test15(bool second);
 void test16();
 void test_recover_22_d2();
 void test_ec43_error_bruteforce();
-void test_recover_53_d5();
 
 int main(int narg, char *args[])
 {
@@ -68,8 +67,6 @@ int main(int narg, char *args[])
     test_recover_22_d2();
     // Error bruteforce
     test_ec43_error_bruteforce();
-    // Test 19
-    test_recover_53_d5();
     // End
     printf("all ok\n");
     return 0;
@@ -1115,7 +1112,7 @@ void test_recover_22_d2()
 
 /***
 
-18. EC 4+2 error location bruteforce
+EC 4+2 error location bruteforce
 
 ***/
 
@@ -1181,66 +1178,3 @@ void test_ec43_error_bruteforce()
     free(write_buf);
     use_ec(7, 4, false);
 }
-
-/***
-
-19. EC 5+3 recover 5th data block but not 4th
-
-***/
-
-void test_recover_53_d5()
-{
-    const int bmp = 128*1024 / 4096 / 8;
-    use_ec(8, 5, true);
-    osd_num_t osd_set[8] = { 1, 2, 3, 0, 0, 6, 7, 8 };
-    osd_rmw_stripe_t stripes[8] = {};
-    unsigned bitmaps[8] = { 0 };
-    // Read 512+128K
-    split_stripes(5, 128*1024, 512*1024, 128*1024, stripes);
-    assert(stripes[0].req_start == 0 && stripes[0].req_end == 0);
-    assert(stripes[1].req_start == 0 && stripes[1].req_end == 0);
-    assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
-    assert(stripes[3].req_start == 0 && stripes[3].req_end == 0);
-    assert(stripes[4].req_start == 0 && stripes[4].req_end == 128*1024);
-    uint8_t *data_buf = (uint8_t*)malloc_or_die(128*1024*8);
-    for (int i = 0; i < 8; i++)
-    {
-        stripes[i].read_start = stripes[i].req_start;
-        stripes[i].read_end = stripes[i].req_end;
-        stripes[i].read_buf = data_buf + i*128*1024;
-        stripes[i].bmp_buf = bitmaps + i;
-    }
-    // Read using parity
-    assert(extend_missing_stripes(stripes, osd_set, 5, 8) == 0);
-    assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
-    assert(stripes[1].read_start == 0 && stripes[1].read_end == 128*1024);
-    assert(stripes[2].read_start == 0 && stripes[2].read_end == 128*1024);
-    assert(stripes[3].read_start == 0 && stripes[3].read_end == 0);
-    assert(stripes[4].read_start == 0 && stripes[4].read_end == 128*1024);
-    assert(stripes[5].read_start == 0 && stripes[5].read_end == 128*1024);
-    assert(stripes[6].read_start == 0 && stripes[6].read_end == 128*1024);
-    assert(stripes[7].read_start == 0 && stripes[7].read_end == 0);
-    bitmaps[0] = 0xffffffff;
-    bitmaps[1] = 0xffffffff;
-    bitmaps[2] = 0xffffffff;
-    bitmaps[3] = 0;
-    bitmaps[4] = 0;
-    bitmaps[5] = 0xffffffff;
-    bitmaps[6] = 0x64646464;
-    bitmaps[7] = 0;
-    set_pattern(stripes[0].read_buf, 128*1024, 0x70a549add9a2280a);
-    set_pattern(stripes[1].read_buf, 128*1024, 0xa70a549add9a2280);
-    set_pattern(stripes[2].read_buf, 128*1024, 0x0a70a549add9a228);
-    set_pattern(stripes[3].read_buf, 128*1024, 0); // 0x80a70a549add9a22
-    set_pattern(stripes[4].read_buf, 128*1024, 0); // 0x280a70a549add9a2
-    set_pattern(stripes[5].read_buf, 128*1024, 0x7572c28f7a91eb22); // xor
-    set_pattern(stripes[6].read_buf, 128*1024, 0xb4542b32a560fe26); // 2nd EC chunk
-    set_pattern(stripes[7].read_buf, 128*1024, 0);
-    // Reconstruct
-    reconstruct_stripes_ec(stripes, 8, 5, bmp);
-    check_pattern(stripes[4].read_buf, 128*1024, 0x280a70a549add9a2);
-    assert(bitmaps[4] == 0xFFFFFFFF);
-    free(data_buf);
-    // Done
-    use_ec(8, 5, false);
-}

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.2.0
+Version: 1.1.0
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

tests/run_3osds.sh

@@ -29,7 +29,7 @@ start_osd_on()
 {
     local i=$1
     local dev=$2
-    build/src/vitastor-osd --osd_num $i --bind_address $ETCD_IP $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL \
+    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL \
         $(build/src/vitastor-disk simple-offsets --format options $OFFSET_ARGS $dev $OFFSET_ARGS 2>/dev/null) \
         >>./testdata/osd$i.log 2>&1 &
     eval OSD${i}_PID=$!