Return block NFS implementation back as an option too

- Do not try to allocate more DB blocks in an inode block until it's "confirmed" and "locked" by the first write
- Do not recheck for new zero DB blocks on first write into an inode block - a CAS failure means someone else is already writing into it
- Throw new allocation blocks away regardless of whether the known_version is 0 on a CAS failure

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VERSION "1.4.1")
+set(VERSION "1.4.2")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v1.4.1
+VERSION ?= v1.4.2
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.4.1
+          image: vitalif/vitastor-csi:v1.4.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.4.1
+          image: vitalif/vitastor-csi:v1.4.2
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.4.1"
+    vitastorCSIDriverVersion = "1.4.2"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.4.1-1) unstable; urgency=medium
+vitastor (1.4.2-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/vitastor.Dockerfile

@@ -35,8 +35,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-1.4.1; \
-    cd vitastor-1.4.1; \
+    cp -r /root/vitastor vitastor-1.4.2; \
+    cd vitastor-1.4.2; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -49,8 +49,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.4.1.orig.tar.xz vitastor-1.4.1; \
-    cd vitastor-1.4.1; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.4.2.orig.tar.xz vitastor-1.4.2; \
+    cd vitastor-1.4.2; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.4.1",
+  "version": "1.4.2",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

mon/vitastor-osd@.service

@@ -8,7 +8,9 @@ PartOf=vitastor.target
 LimitNOFILE=1048576
 LimitNPROC=1048576
 LimitMEMLOCK=infinity
-ExecStart=bash -c 'exec vitastor-disk exec-osd /dev/vitastor/osd%i-data >>/var/log/vitastor/osd%i.log 2>&1'
+# Use the following for direct logs to files
+#ExecStart=bash -c 'exec vitastor-disk exec-osd /dev/vitastor/osd%i-data >>/var/log/vitastor/osd%i.log 2>&1'
+ExecStart=vitastor-disk exec-osd /dev/vitastor/osd%i-data
 ExecStartPre=+vitastor-disk pre-exec /dev/vitastor/osd%i-data
 WorkingDirectory=/
 User=vitastor

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '1.4.1'
+VERSION = '1.4.2'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -24,4 +24,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-1.4.1/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.4.1$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-1.4.2/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.4.2$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -36,7 +36,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.4.1.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.4.2.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.4.1
+Version:        1.4.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.4.1.el7.tar.gz
+Source0:        vitastor-1.4.2.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.4.1.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.4.2.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.4.1
+Version:        1.4.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.4.1.el8.tar.gz
+Source0:        vitastor-1.4.2.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.Dockerfile

@@ -18,7 +18,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-1.4.1.el9.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-1.4.2.el9.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.4.1
+Version:        1.4.2
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.4.1.el9.tar.gz
+Source0:        vitastor-1.4.2.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -16,7 +16,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="1.4.1")
+add_definitions(-DVERSION="1.4.2")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})
@@ -209,9 +209,9 @@ target_link_libraries(vitastor-kv-stress
 # vitastor-nfs
 add_executable(vitastor-nfs
 	nfs_proxy.cpp
+	nfs_block.cpp
 	nfs_kv.cpp
 	nfs_kv_create.cpp
-	nfs_kv_fsstat.cpp
 	nfs_kv_getattr.cpp
 	nfs_kv_link.cpp
 	nfs_kv_lookup.cpp
@@ -221,6 +221,7 @@ add_executable(vitastor-nfs
 	nfs_kv_rename.cpp
 	nfs_kv_setattr.cpp
 	nfs_kv_write.cpp
+	nfs_fsstat.cpp
 	nfs_mount.cpp
 	nfs_portmap.cpp
 	sha256.c

src/nfs_block.h

@@ -0,0 +1,57 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+//
+// NFS proxy over Vitastor block images - header
+
+#pragma once
+
+struct nfs_dir_t
+{
+    uint64_t id;
+    uint64_t mod_rev;
+    timespec mtime;
+};
+
+struct extend_size_t
+{
+    inode_t inode;
+    uint64_t new_size;
+};
+
+inline bool operator < (const extend_size_t &a, const extend_size_t &b)
+{
+    return a.inode < b.inode || a.inode == b.inode && a.new_size < b.new_size;
+}
+
+struct extend_write_t
+{
+    rpc_op_t *rop;
+    int resize_res, write_res; // 1 = started, 0 = completed OK, -errno = completed with error
+};
+
+struct extend_inode_t
+{
+    uint64_t cur_extend = 0, next_extend = 0;
+};
+
+struct block_fs_state_t
+{
+    // filehandle = "S"+base64(sha256(full name with prefix)) or "roothandle" for mount root)
+    uint64_t next_dir_id = 2;
+    // filehandle => dir with name_prefix
+    std::map<std::string, std::string> dir_by_hash;
+    // dir with name_prefix => dir info
+    std::map<std::string, nfs_dir_t> dir_info;
+    // filehandle => inode ID
+    std::map<std::string, inode_t> inode_by_hash;
+    // inode ID => filehandle
+    std::map<inode_t, std::string> hash_by_inode;
+
+    // inode extend requests in progress
+    std::map<inode_t, extend_inode_t> extends;
+    std::multimap<extend_size_t, extend_write_t> extend_writes;
+
+    void init(nfs_proxy_t *proxy);
+};
+
+nfsstat3 vitastor_nfs_map_err(int err);

src/nfs_common.h

@@ -0,0 +1,22 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+//
+// NFS proxy - common functions
+
+#pragma once
+
+#include "nfs/nfs.h"
+
+void nfs_block_procs(nfs_client_t *self);
+void nfs_kv_procs(nfs_client_t *self);
+int nfs3_fsstat_proc(void *opaque, rpc_op_t *rop);
+int nfs3_fsinfo_proc(void *opaque, rpc_op_t *rop);
+int nfs3_pathconf_proc(void *opaque, rpc_op_t *rop);
+int nfs3_access_proc(void *opaque, rpc_op_t *rop);
+int nfs3_null_proc(void *opaque, rpc_op_t *rop);
+int nfs3_commit_proc(void *opaque, rpc_op_t *rop);
+int mount3_mnt_proc(void *opaque, rpc_op_t *rop);
+int mount3_dump_proc(void *opaque, rpc_op_t *rop);
+int mount3_umnt_proc(void *opaque, rpc_op_t *rop);
+int mount3_umntall_proc(void *opaque, rpc_op_t *rop);
+int mount3_export_proc(void *opaque, rpc_op_t *rop);

src/nfs_fsstat.cpp

@@ -0,0 +1,124 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+//
+// NFS proxy - common FSSTAT, FSINFO, PATHCONF
+
+#include <sys/time.h>
+
+#include "nfs_proxy.h"
+#include "nfs_kv.h"
+
+// Get file system statistics
+int nfs3_fsstat_proc(void *opaque, rpc_op_t *rop)
+{
+    nfs_client_t *self = (nfs_client_t*)opaque;
+    //FSSTAT3args *args = (FSSTAT3args*)rop->request;
+    if (self->parent->trace)
+        fprintf(stderr, "[%d] FSSTAT\n", self->nfs_fd);
+    FSSTAT3res *reply = (FSSTAT3res*)rop->reply;
+    uint64_t tbytes = 0, fbytes = 0;
+    auto pst_it = self->parent->pool_stats.find(self->parent->default_pool_id);
+    if (pst_it != self->parent->pool_stats.end())
+    {
+        auto ttb = pst_it->second["total_raw_tb"].number_value();
+        auto ftb = (pst_it->second["total_raw_tb"].number_value() - pst_it->second["used_raw_tb"].number_value());
+        tbytes = ttb / pst_it->second["raw_to_usable"].number_value() * ((uint64_t)2<<40);
+        fbytes = ftb / pst_it->second["raw_to_usable"].number_value() * ((uint64_t)2<<40);
+    }
+    *reply = (FSSTAT3res){
+        .status = NFS3_OK,
+        .resok = (FSSTAT3resok){
+            .obj_attributes = {
+                .attributes_follow = 0,
+                //.attributes = get_root_attributes(self),
+            },
+            .tbytes = tbytes, // total bytes
+            .fbytes = fbytes, // free bytes
+            .abytes = fbytes, // available bytes
+            .tfiles = (size3)1 << (63-POOL_ID_BITS), // maximum total files
+            .ffiles = (size3)1 << (63-POOL_ID_BITS), // free files
+            .afiles = (size3)1 << (63-POOL_ID_BITS), // available files
+            .invarsec = 0,
+        },
+    };
+    rpc_queue_reply(rop);
+    return 0;
+}
+
+int nfs3_fsinfo_proc(void *opaque, rpc_op_t *rop)
+{
+    nfs_client_t *self = (nfs_client_t*)opaque;
+    FSINFO3args *args = (FSINFO3args*)rop->request;
+    FSINFO3res *reply = (FSINFO3res*)rop->reply;
+    if (self->parent->trace)
+        fprintf(stderr, "[%d] FSINFO %s\n", self->nfs_fd, std::string(args->fsroot).c_str());
+    if (args->fsroot != NFS_ROOT_HANDLE)
+    {
+        *reply = (FSINFO3res){ .status = NFS3ERR_INVAL };
+    }
+    else
+    {
+        // Fill info
+        *reply = (FSINFO3res){
+            .status = NFS3_OK,
+            .resok = (FSINFO3resok){
+                .obj_attributes = {
+                    .attributes_follow = 0,
+                    //.attributes = get_root_attributes(self),
+                },
+                .rtmax = 128*1024*1024,
+                .rtpref = 128*1024*1024,
+                .rtmult = 4096,
+                .wtmax = 128*1024*1024,
+                .wtpref = 128*1024*1024,
+                .wtmult = 4096,
+                .dtpref = 128,
+                .maxfilesize = 0x7fffffffffffffff,
+                .time_delta = {
+                    .seconds = 1,
+                    .nseconds = 0,
+                },
+                .properties = FSF3_SYMLINK | FSF3_HOMOGENEOUS,
+            },
+        };
+    }
+    rpc_queue_reply(rop);
+    return 0;
+}
+
+int nfs3_pathconf_proc(void *opaque, rpc_op_t *rop)
+{
+    nfs_client_t *self = (nfs_client_t*)opaque;
+    PATHCONF3args *args = (PATHCONF3args*)rop->request;
+    PATHCONF3res *reply = (PATHCONF3res*)rop->reply;
+    if (self->parent->trace)
+        fprintf(stderr, "[%d] PATHCONF %s\n", self->nfs_fd, std::string(args->object).c_str());
+    if (args->object != NFS_ROOT_HANDLE)
+    {
+        *reply = (PATHCONF3res){ .status = NFS3ERR_INVAL };
+    }
+    else
+    {
+        // Fill info
+        *reply = (PATHCONF3res){
+            .status = NFS3_OK,
+            .resok = (PATHCONF3resok){
+                .obj_attributes = {
+                    // Without at least one reference to a non-constant value (local variable or something else),
+                    // with gcc 8 we get "internal compiler error: side-effects element in no-side-effects CONSTRUCTOR" here
+                    // FIXME: get rid of this after raising compiler requirement
+                    .attributes_follow = 0,
+                    //.attributes = get_root_attributes(self),
+                },
+                .linkmax = 0,
+                .name_max = 255,
+                .no_trunc = TRUE,
+                .chown_restricted = FALSE,
+                .case_insensitive = FALSE,
+                .case_preserving = TRUE,
+            },
+        };
+    }
+    rpc_queue_reply(rop);
+    return 0;
+}

src/nfs_kv.cpp

@@ -7,17 +7,9 @@
 
 #include "str_util.h"
 #include "nfs_proxy.h"
+#include "nfs_common.h"
 #include "nfs_kv.h"
 
-nfsstat3 vitastor_nfs_map_err(int err)
-{
-    return (err == EINVAL ? NFS3ERR_INVAL
-        : (err == ENOENT ? NFS3ERR_NOENT
-        : (err == ENOSPC ? NFS3ERR_NOSPC
-        : (err == EEXIST ? NFS3ERR_EXIST
-        : (err == EIO ? NFS3ERR_IO : (err ? NFS3ERR_IO : NFS3_OK))))));
-}
-
 nfstime3 nfstime_from_str(const std::string & s)
 {
     nfstime3 t;
@@ -142,3 +134,41 @@ bool kv_fh_valid(const std::string & fh)
 {
     return fh == NFS_ROOT_HANDLE || fh.size() == 9 && fh[0] == 'S' || fh.size() > 17 && fh[0] == 'I';
 }
+
+void nfs_kv_procs(nfs_client_t *self)
+{
+    struct rpc_service_proc_t pt[] = {
+        {NFS_PROGRAM, NFS_V3, NFS3_NULL,        nfs3_null_proc,           NULL,                            0,                        NULL,                           0,                       self},
+        {NFS_PROGRAM, NFS_V3, NFS3_GETATTR,     kv_nfs3_getattr_proc,     (xdrproc_t)xdr_GETATTR3args,     sizeof(GETATTR3args),     (xdrproc_t)xdr_GETATTR3res,     sizeof(GETATTR3res),     self},
+        {NFS_PROGRAM, NFS_V3, NFS3_SETATTR,     kv_nfs3_setattr_proc,     (xdrproc_t)xdr_SETATTR3args,     sizeof(SETATTR3args),     (xdrproc_t)xdr_SETATTR3res,     sizeof(SETATTR3res),     self},
+        {NFS_PROGRAM, NFS_V3, NFS3_LOOKUP,      kv_nfs3_lookup_proc,      (xdrproc_t)xdr_LOOKUP3args,      sizeof(LOOKUP3args),      (xdrproc_t)xdr_LOOKUP3res,      sizeof(LOOKUP3res),      self},
+        {NFS_PROGRAM, NFS_V3, NFS3_ACCESS,      nfs3_access_proc,         (xdrproc_t)xdr_ACCESS3args,      sizeof(ACCESS3args),      (xdrproc_t)xdr_ACCESS3res,      sizeof(ACCESS3res),      self},
+        {NFS_PROGRAM, NFS_V3, NFS3_READLINK,    kv_nfs3_readlink_proc,    (xdrproc_t)xdr_READLINK3args,    sizeof(READLINK3args),    (xdrproc_t)xdr_READLINK3res,    sizeof(READLINK3res),    self},
+        {NFS_PROGRAM, NFS_V3, NFS3_READ,        kv_nfs3_read_proc,        (xdrproc_t)xdr_READ3args,        sizeof(READ3args),        (xdrproc_t)xdr_READ3res,        sizeof(READ3res),        self},
+        {NFS_PROGRAM, NFS_V3, NFS3_WRITE,       kv_nfs3_write_proc,       (xdrproc_t)xdr_WRITE3args,       sizeof(WRITE3args),       (xdrproc_t)xdr_WRITE3res,       sizeof(WRITE3res),       self},
+        {NFS_PROGRAM, NFS_V3, NFS3_CREATE,      kv_nfs3_create_proc,      (xdrproc_t)xdr_CREATE3args,      sizeof(CREATE3args),      (xdrproc_t)xdr_CREATE3res,      sizeof(CREATE3res),      self},
+        {NFS_PROGRAM, NFS_V3, NFS3_MKDIR,       kv_nfs3_mkdir_proc,       (xdrproc_t)xdr_MKDIR3args,       sizeof(MKDIR3args),       (xdrproc_t)xdr_MKDIR3res,       sizeof(MKDIR3res),       self},
+        {NFS_PROGRAM, NFS_V3, NFS3_SYMLINK,     kv_nfs3_symlink_proc,     (xdrproc_t)xdr_SYMLINK3args,     sizeof(SYMLINK3args),     (xdrproc_t)xdr_SYMLINK3res,     sizeof(SYMLINK3res),     self},
+        {NFS_PROGRAM, NFS_V3, NFS3_MKNOD,       kv_nfs3_mknod_proc,       (xdrproc_t)xdr_MKNOD3args,       sizeof(MKNOD3args),       (xdrproc_t)xdr_MKNOD3res,       sizeof(MKNOD3res),       self},
+        {NFS_PROGRAM, NFS_V3, NFS3_REMOVE,      kv_nfs3_remove_proc,      (xdrproc_t)xdr_REMOVE3args,      sizeof(REMOVE3args),      (xdrproc_t)xdr_REMOVE3res,      sizeof(REMOVE3res),      self},
+        {NFS_PROGRAM, NFS_V3, NFS3_RMDIR,       kv_nfs3_rmdir_proc,       (xdrproc_t)xdr_RMDIR3args,       sizeof(RMDIR3args),       (xdrproc_t)xdr_RMDIR3res,       sizeof(RMDIR3res),       self},
+        {NFS_PROGRAM, NFS_V3, NFS3_RENAME,      kv_nfs3_rename_proc,      (xdrproc_t)xdr_RENAME3args,      sizeof(RENAME3args),      (xdrproc_t)xdr_RENAME3res,      sizeof(RENAME3res),      self},
+        {NFS_PROGRAM, NFS_V3, NFS3_LINK,        kv_nfs3_link_proc,        (xdrproc_t)xdr_LINK3args,        sizeof(LINK3args),        (xdrproc_t)xdr_LINK3res,        sizeof(LINK3res),        self},
+        {NFS_PROGRAM, NFS_V3, NFS3_READDIR,     kv_nfs3_readdir_proc,     (xdrproc_t)xdr_READDIR3args,     sizeof(READDIR3args),     (xdrproc_t)xdr_READDIR3res,     sizeof(READDIR3res),     self},
+        {NFS_PROGRAM, NFS_V3, NFS3_READDIRPLUS, kv_nfs3_readdirplus_proc, (xdrproc_t)xdr_READDIRPLUS3args, sizeof(READDIRPLUS3args), (xdrproc_t)xdr_READDIRPLUS3res, sizeof(READDIRPLUS3res), self},
+        {NFS_PROGRAM, NFS_V3, NFS3_FSSTAT,      nfs3_fsstat_proc,         (xdrproc_t)xdr_FSSTAT3args,      sizeof(FSSTAT3args),      (xdrproc_t)xdr_FSSTAT3res,      sizeof(FSSTAT3res),      self},
+        {NFS_PROGRAM, NFS_V3, NFS3_FSINFO,      nfs3_fsinfo_proc,         (xdrproc_t)xdr_FSINFO3args,      sizeof(FSINFO3args),      (xdrproc_t)xdr_FSINFO3res,      sizeof(FSINFO3res),      self},
+        {NFS_PROGRAM, NFS_V3, NFS3_PATHCONF,    nfs3_pathconf_proc,       (xdrproc_t)xdr_PATHCONF3args,    sizeof(PATHCONF3args),    (xdrproc_t)xdr_PATHCONF3res,    sizeof(PATHCONF3res),    self},
+        {NFS_PROGRAM, NFS_V3, NFS3_COMMIT,      nfs3_commit_proc,         (xdrproc_t)xdr_COMMIT3args,      sizeof(COMMIT3args),      (xdrproc_t)xdr_COMMIT3res,      sizeof(COMMIT3res),      self},
+        {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_NULL,    nfs3_null_proc,         NULL,                            0,                        NULL,                         0,                         self},
+        {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_MNT,     mount3_mnt_proc,        (xdrproc_t)xdr_nfs_dirpath,      sizeof(nfs_dirpath),      (xdrproc_t)xdr_nfs_mountres3, sizeof(nfs_mountres3),     self},
+        {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_DUMP,    mount3_dump_proc,       NULL,                            0,                        (xdrproc_t)xdr_nfs_mountlist, sizeof(nfs_mountlist),     self},
+        {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_UMNT,    mount3_umnt_proc,       (xdrproc_t)xdr_nfs_dirpath,      sizeof(nfs_dirpath),      NULL,                         0,                         self},
+        {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_UMNTALL, mount3_umntall_proc,    NULL,                            0,                        NULL,                         0,                         self},
+        {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_EXPORT,  mount3_export_proc,     NULL,                            0,                        (xdrproc_t)xdr_nfs_exports,   sizeof(nfs_exports),       self},
+    };
+    for (int i = 0; i < sizeof(pt)/sizeof(pt[0]); i++)
+    {
+        self->proc_table.insert(pt[i]);
+    }
+}

src/nfs_kv.h

@@ -1,3 +1,8 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+//
+// NFS proxy over VitastorKV database - header
+
 #pragma once
 
 #include "nfs/nfs.h"
@@ -31,7 +36,7 @@ struct shared_alloc_queue_t
     uint64_t size;
 };
 
-struct inode_extend_t
+struct kv_inode_extend_t
 {
     int refcnt = 0;
     uint64_t cur_extend = 0, next_extend = 0, done_extend = 0;
@@ -45,7 +50,7 @@ struct kv_fs_state_t
     std::vector<uint64_t> unallocated_ids;
     std::vector<shared_alloc_queue_t> allocating_shared;
     uint64_t cur_shared_inode = 0, cur_shared_offset = 0;
-    std::map<inode_t, inode_extend_t> extends;
+    std::map<inode_t, kv_inode_extend_t> extends;
 };
 
 struct shared_file_header_t
@@ -88,14 +93,3 @@ int kv_nfs3_rename_proc(void *opaque, rpc_op_t *rop);
 int kv_nfs3_link_proc(void *opaque, rpc_op_t *rop);
 int kv_nfs3_readdir_proc(void *opaque, rpc_op_t *rop);
 int kv_nfs3_readdirplus_proc(void *opaque, rpc_op_t *rop);
-int kv_nfs3_fsstat_proc(void *opaque, rpc_op_t *rop);
-int kv_nfs3_fsinfo_proc(void *opaque, rpc_op_t *rop);
-int kv_nfs3_pathconf_proc(void *opaque, rpc_op_t *rop);
-int nfs3_access_proc(void *opaque, rpc_op_t *rop);
-int nfs3_null_proc(void *opaque, rpc_op_t *rop);
-int nfs3_commit_proc(void *opaque, rpc_op_t *rop);
-int mount3_mnt_proc(void *opaque, rpc_op_t *rop);
-int mount3_dump_proc(void *opaque, rpc_op_t *rop);
-int mount3_umnt_proc(void *opaque, rpc_op_t *rop);
-int mount3_umntall_proc(void *opaque, rpc_op_t *rop);
-int mount3_export_proc(void *opaque, rpc_op_t *rop);

src/nfs_kv_fsstat.cpp

@@ -1,166 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 (see README.md for details)
-//
-// NFS proxy over VitastorKV database - FSSTAT, FSINFO, PATHCONF
-
-#include <sys/time.h>
-
-#include "nfs_proxy.h"
-#include "nfs_kv.h"
-
-// Get file system statistics
-int kv_nfs3_fsstat_proc(void *opaque, rpc_op_t *rop)
-{
-    nfs_client_t *self = (nfs_client_t*)opaque;
-    //FSSTAT3args *args = (FSSTAT3args*)rop->request;
-    if (self->parent->trace)
-        fprintf(stderr, "[%d] FSSTAT\n", self->nfs_fd);
-    FSSTAT3res *reply = (FSSTAT3res*)rop->reply;
-    uint64_t tbytes = 0, fbytes = 0;
-    auto pst_it = self->parent->pool_stats.find(self->parent->default_pool_id);
-    if (pst_it != self->parent->pool_stats.end())
-    {
-        auto ttb = pst_it->second["total_raw_tb"].number_value();
-        auto ftb = (pst_it->second["total_raw_tb"].number_value() - pst_it->second["used_raw_tb"].number_value());
-        tbytes = ttb / pst_it->second["raw_to_usable"].number_value() * ((uint64_t)2<<40);
-        fbytes = ftb / pst_it->second["raw_to_usable"].number_value() * ((uint64_t)2<<40);
-    }
-    *reply = (FSSTAT3res){
-        .status = NFS3_OK,
-        .resok = (FSSTAT3resok){
-            .obj_attributes = {
-                .attributes_follow = 0,
-                //.attributes = get_root_attributes(self),
-            },
-            .tbytes = tbytes, // total bytes
-            .fbytes = fbytes, // free bytes
-            .abytes = fbytes, // available bytes
-            .tfiles = (size3)(1 << 31), // maximum total files
-            .ffiles = (size3)(1 << 31), // free files
-            .afiles = (size3)(1 << 31), // available files
-            .invarsec = 0,
-        },
-    };
-    rpc_queue_reply(rop);
-    return 0;
-}
-
-int kv_nfs3_fsinfo_proc(void *opaque, rpc_op_t *rop)
-{
-    nfs_client_t *self = (nfs_client_t*)opaque;
-    FSINFO3args *args = (FSINFO3args*)rop->request;
-    FSINFO3res *reply = (FSINFO3res*)rop->reply;
-    if (self->parent->trace)
-        fprintf(stderr, "[%d] FSINFO %s\n", self->nfs_fd, std::string(args->fsroot).c_str());
-    if (args->fsroot != NFS_ROOT_HANDLE)
-    {
-        *reply = (FSINFO3res){ .status = NFS3ERR_INVAL };
-    }
-    else
-    {
-        // Fill info
-        *reply = (FSINFO3res){
-            .status = NFS3_OK,
-            .resok = (FSINFO3resok){
-                .obj_attributes = {
-                    .attributes_follow = 0,
-                    //.attributes = get_root_attributes(self),
-                },
-                .rtmax = 128*1024*1024,
-                .rtpref = 128*1024*1024,
-                .rtmult = 4096,
-                .wtmax = 128*1024*1024,
-                .wtpref = 128*1024*1024,
-                .wtmult = 4096,
-                .dtpref = 128,
-                .maxfilesize = 0x7fffffffffffffff,
-                .time_delta = {
-                    .seconds = 1,
-                    .nseconds = 0,
-                },
-                .properties = FSF3_SYMLINK | FSF3_HOMOGENEOUS,
-            },
-        };
-    }
-    rpc_queue_reply(rop);
-    return 0;
-}
-
-int kv_nfs3_pathconf_proc(void *opaque, rpc_op_t *rop)
-{
-    nfs_client_t *self = (nfs_client_t*)opaque;
-    PATHCONF3args *args = (PATHCONF3args*)rop->request;
-    PATHCONF3res *reply = (PATHCONF3res*)rop->reply;
-    if (self->parent->trace)
-        fprintf(stderr, "[%d] PATHCONF %s\n", self->nfs_fd, std::string(args->object).c_str());
-    if (args->object != NFS_ROOT_HANDLE)
-    {
-        *reply = (PATHCONF3res){ .status = NFS3ERR_INVAL };
-    }
-    else
-    {
-        // Fill info
-        *reply = (PATHCONF3res){
-            .status = NFS3_OK,
-            .resok = (PATHCONF3resok){
-                .obj_attributes = {
-                    // Without at least one reference to a non-constant value (local variable or something else),
-                    // with gcc 8 we get "internal compiler error: side-effects element in no-side-effects CONSTRUCTOR" here
-                    // FIXME: get rid of this after raising compiler requirement
-                    .attributes_follow = 0,
-                    //.attributes = get_root_attributes(self),
-                },
-                .linkmax = 0,
-                .name_max = 255,
-                .no_trunc = TRUE,
-                .chown_restricted = FALSE,
-                .case_insensitive = FALSE,
-                .case_preserving = TRUE,
-            },
-        };
-    }
-    rpc_queue_reply(rop);
-    return 0;
-}
-
-nfs_client_t::nfs_client_t()
-{
-    struct rpc_service_proc_t pt[] = {
-        {NFS_PROGRAM, NFS_V3, NFS3_NULL,        nfs3_null_proc,           NULL,                            0,                        NULL,                           0,                       this},
-        {NFS_PROGRAM, NFS_V3, NFS3_GETATTR,     kv_nfs3_getattr_proc,     (xdrproc_t)xdr_GETATTR3args,     sizeof(GETATTR3args),     (xdrproc_t)xdr_GETATTR3res,     sizeof(GETATTR3res),     this},
-        {NFS_PROGRAM, NFS_V3, NFS3_SETATTR,     kv_nfs3_setattr_proc,     (xdrproc_t)xdr_SETATTR3args,     sizeof(SETATTR3args),     (xdrproc_t)xdr_SETATTR3res,     sizeof(SETATTR3res),     this},
-        {NFS_PROGRAM, NFS_V3, NFS3_LOOKUP,      kv_nfs3_lookup_proc,      (xdrproc_t)xdr_LOOKUP3args,      sizeof(LOOKUP3args),      (xdrproc_t)xdr_LOOKUP3res,      sizeof(LOOKUP3res),      this},
-        {NFS_PROGRAM, NFS_V3, NFS3_ACCESS,      nfs3_access_proc,         (xdrproc_t)xdr_ACCESS3args,      sizeof(ACCESS3args),      (xdrproc_t)xdr_ACCESS3res,      sizeof(ACCESS3res),      this},
-        {NFS_PROGRAM, NFS_V3, NFS3_READLINK,    kv_nfs3_readlink_proc,    (xdrproc_t)xdr_READLINK3args,    sizeof(READLINK3args),    (xdrproc_t)xdr_READLINK3res,    sizeof(READLINK3res),    this},
-        {NFS_PROGRAM, NFS_V3, NFS3_READ,        kv_nfs3_read_proc,        (xdrproc_t)xdr_READ3args,        sizeof(READ3args),        (xdrproc_t)xdr_READ3res,        sizeof(READ3res),        this},
-        {NFS_PROGRAM, NFS_V3, NFS3_WRITE,       kv_nfs3_write_proc,       (xdrproc_t)xdr_WRITE3args,       sizeof(WRITE3args),       (xdrproc_t)xdr_WRITE3res,       sizeof(WRITE3res),       this},
-        {NFS_PROGRAM, NFS_V3, NFS3_CREATE,      kv_nfs3_create_proc,      (xdrproc_t)xdr_CREATE3args,      sizeof(CREATE3args),      (xdrproc_t)xdr_CREATE3res,      sizeof(CREATE3res),      this},
-        {NFS_PROGRAM, NFS_V3, NFS3_MKDIR,       kv_nfs3_mkdir_proc,       (xdrproc_t)xdr_MKDIR3args,       sizeof(MKDIR3args),       (xdrproc_t)xdr_MKDIR3res,       sizeof(MKDIR3res),       this},
-        {NFS_PROGRAM, NFS_V3, NFS3_SYMLINK,     kv_nfs3_symlink_proc,     (xdrproc_t)xdr_SYMLINK3args,     sizeof(SYMLINK3args),     (xdrproc_t)xdr_SYMLINK3res,     sizeof(SYMLINK3res),     this},
-        {NFS_PROGRAM, NFS_V3, NFS3_MKNOD,       kv_nfs3_mknod_proc,       (xdrproc_t)xdr_MKNOD3args,       sizeof(MKNOD3args),       (xdrproc_t)xdr_MKNOD3res,       sizeof(MKNOD3res),       this},
-        {NFS_PROGRAM, NFS_V3, NFS3_REMOVE,      kv_nfs3_remove_proc,      (xdrproc_t)xdr_REMOVE3args,      sizeof(REMOVE3args),      (xdrproc_t)xdr_REMOVE3res,      sizeof(REMOVE3res),      this},
-        {NFS_PROGRAM, NFS_V3, NFS3_RMDIR,       kv_nfs3_rmdir_proc,       (xdrproc_t)xdr_RMDIR3args,       sizeof(RMDIR3args),       (xdrproc_t)xdr_RMDIR3res,       sizeof(RMDIR3res),       this},
-        {NFS_PROGRAM, NFS_V3, NFS3_RENAME,      kv_nfs3_rename_proc,      (xdrproc_t)xdr_RENAME3args,      sizeof(RENAME3args),      (xdrproc_t)xdr_RENAME3res,      sizeof(RENAME3res),      this},
-        {NFS_PROGRAM, NFS_V3, NFS3_LINK,        kv_nfs3_link_proc,        (xdrproc_t)xdr_LINK3args,        sizeof(LINK3args),        (xdrproc_t)xdr_LINK3res,        sizeof(LINK3res),        this},
-        {NFS_PROGRAM, NFS_V3, NFS3_READDIR,     kv_nfs3_readdir_proc,     (xdrproc_t)xdr_READDIR3args,     sizeof(READDIR3args),     (xdrproc_t)xdr_READDIR3res,     sizeof(READDIR3res),     this},
-        {NFS_PROGRAM, NFS_V3, NFS3_READDIRPLUS, kv_nfs3_readdirplus_proc, (xdrproc_t)xdr_READDIRPLUS3args, sizeof(READDIRPLUS3args), (xdrproc_t)xdr_READDIRPLUS3res, sizeof(READDIRPLUS3res), this},
-        {NFS_PROGRAM, NFS_V3, NFS3_FSSTAT,      kv_nfs3_fsstat_proc,      (xdrproc_t)xdr_FSSTAT3args,      sizeof(FSSTAT3args),      (xdrproc_t)xdr_FSSTAT3res,      sizeof(FSSTAT3res),      this},
-        {NFS_PROGRAM, NFS_V3, NFS3_FSINFO,      kv_nfs3_fsinfo_proc,      (xdrproc_t)xdr_FSINFO3args,      sizeof(FSINFO3args),      (xdrproc_t)xdr_FSINFO3res,      sizeof(FSINFO3res),      this},
-        {NFS_PROGRAM, NFS_V3, NFS3_PATHCONF,    kv_nfs3_pathconf_proc,    (xdrproc_t)xdr_PATHCONF3args,    sizeof(PATHCONF3args),    (xdrproc_t)xdr_PATHCONF3res,    sizeof(PATHCONF3res),    this},
-        {NFS_PROGRAM, NFS_V3, NFS3_COMMIT,      nfs3_commit_proc,         (xdrproc_t)xdr_COMMIT3args,      sizeof(COMMIT3args),      (xdrproc_t)xdr_COMMIT3res,      sizeof(COMMIT3res),      this},
-        {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_NULL,    nfs3_null_proc,         NULL,                            0,                        NULL,                         0,                         this},
-        {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_MNT,     mount3_mnt_proc,        (xdrproc_t)xdr_nfs_dirpath,      sizeof(nfs_dirpath),      (xdrproc_t)xdr_nfs_mountres3, sizeof(nfs_mountres3),     this},
-        {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_DUMP,    mount3_dump_proc,       NULL,                            0,                        (xdrproc_t)xdr_nfs_mountlist, sizeof(nfs_mountlist),     this},
-        {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_UMNT,    mount3_umnt_proc,       (xdrproc_t)xdr_nfs_dirpath,      sizeof(nfs_dirpath),      NULL,                         0,                         this},
-        {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_UMNTALL, mount3_umntall_proc,    NULL,                            0,                        NULL,                         0,                         this},
-        {MOUNT_PROGRAM, MOUNT_V3, MOUNT3_EXPORT,  mount3_export_proc,     NULL,                            0,                        (xdrproc_t)xdr_nfs_exports,   sizeof(nfs_exports),       this},
-    };
-    for (int i = 0; i < sizeof(pt)/sizeof(pt[0]); i++)
-    {
-        proc_table.insert(pt[i]);
-    }
-}
-
-nfs_client_t::~nfs_client_t()
-{
-}

src/nfs_kv_write.cpp

@@ -40,7 +40,7 @@ struct nfs_kv_write_state
     uint64_t shared_inode = 0, shared_offset = 0;
     bool was_immediate = false;
     nfs_rmw_t rmw[2];
-    inode_extend_t *ext = NULL;
+    kv_inode_extend_t *ext = NULL;
 
     ~nfs_kv_write_state()
     {

src/nfs_mount.cpp

@@ -8,7 +8,14 @@
 #include "nfs_proxy.h"
 #include "nfs/nfs.h"
 
-nfsstat3 vitastor_nfs_map_err(int err);
+nfsstat3 vitastor_nfs_map_err(int err)
+{
+    return (err == EINVAL ? NFS3ERR_INVAL
+        : (err == ENOENT ? NFS3ERR_NOENT
+        : (err == ENOSPC ? NFS3ERR_NOSPC
+        : (err == EEXIST ? NFS3ERR_EXIST
+        : (err == EIO ? NFS3ERR_IO : (err ? NFS3ERR_IO : NFS3_OK))))));
+}
 
 int nfs3_null_proc(void *opaque, rpc_op_t *rop)
 {

src/nfs_proxy.cpp

@@ -22,6 +22,8 @@
 #include "str_util.h"
 #include "nfs_proxy.h"
 #include "nfs_kv.h"
+#include "nfs_block.h"
+#include "nfs_common.h"
 #include "http_client.h"
 #include "cli.h"
 
@@ -137,6 +139,11 @@ void nfs_proxy_t::run(json11::Json cfg)
     cmd->epmgr = epmgr;
     cmd->cli = cli;
     watch_stats();
+    if (!fs_kv_inode)
+    {
+        blockfs = new block_fs_state_t();
+        blockfs->init(this);
+    }
     // Load image metadata
     while (!cli->is_ready())
     {
@@ -423,6 +430,10 @@ void nfs_proxy_t::do_accept(int listen_fd)
         int one = 1;
         setsockopt(nfs_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
         auto cli = new nfs_client_t();
+        if (fs_kv_inode)
+            nfs_kv_procs(cli);
+        else
+            nfs_block_procs(cli);
         cli->parent = this;
         cli->nfs_fd = nfs_fd;
         for (auto & fn: pmap.proc_table)

src/nfs_proxy.h

@@ -15,6 +15,7 @@
 class cli_tool_t;
 
 struct kv_fs_state_t;
+struct block_fs_state_t;
 
 class nfs_proxy_t
 {
@@ -45,6 +46,7 @@ public:
     cli_tool_t *cmd = NULL;
     kv_dbw_t *db = NULL;
     kv_fs_state_t *kvfs = NULL;
+    block_fs_state_t *blockfs = NULL;
 
     std::vector<XDR*> xdr_pool;
 
@@ -108,9 +110,6 @@ public:
     std::vector<iovec> send_list, next_send_list;
     std::vector<rpc_op_t*> outbox, next_outbox;
 
-    nfs_client_t();
-    ~nfs_client_t();
-
     void select_read_buffer(unsigned wanted_size);
     void submit_read(unsigned wanted_size);
     void handle_read(int result);

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.4.1
+Version: 1.4.2
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}