Release 1.8.0

Bugfix release, would be 1.7.2, but etcd layout changes mandate it to be 1.8.0. :-)

- Change etcd layout: /config/pgs is now /pg/config, /pg/stats/* is now /pgstats/*.
  This is required to fix a rare PG history tracking issue caused by non-atomic
  delivery of etcd events sometimes resulting in `incomplete` objects in EC pools
  after mass OSD restarts. Upgrading can be performed freely, downgrade requires
  additional action: [1.8.0 to 1.7.1](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/docs/usage/admin.en.md#1-8-0-to-1-7-1)
- Fix a rare client hang on PG primary OSD switch
- Fix vitastor-nfs started using mount command sometimes not stopping automatically after unmount
- Fix vitastor-nfs mounts started using mount command sometimes hanging after daemonizing
- Fix merge/flatten into a pool with different object size (image migration between pools case)
- Do not print extra "PG disappeared after reload" verbose log messages for non-existing PGs
- Fix clustered Antietcd support and persistence filter
- Do not try to purge the same OSD multiple times if its multiple devices are passed to purge
- Various node.js binding fixes

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "1.7.1")
+set(VITASTOR_VERSION "1.8.0")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v1.7.1
+VITASTOR_VERSION ?= v1.8.0
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v1.7.1
+          image: vitalif/vitastor-csi:v1.8.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v1.7.1
+          image: vitalif/vitastor-csi:v1.8.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "1.7.1"
+    vitastorCSIDriverVersion = "1.8.0"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (1.7.1-1) unstable; urgency=medium
+vitastor (1.8.0-1) unstable; urgency=medium
 
   * Bugfixes
 

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {

node-binding/binding.gyp

@@ -10,7 +10,7 @@
         '<!(node -e "require(\'nan\')")'
       ],
       'cflags': [
-        '<!(pkg-config --cflags vitastor) -g'
+        '<!(pkg-config --cflags vitastor)'
       ],
       'libraries': [
         '<!(pkg-config --libs vitastor)',

node-binding/client.cc

@@ -34,6 +34,13 @@ public:
     Nan::Persistent<v8::Function> callback;
 };
 
+static uint64_t get_ui64(const v8::Local<v8::Value> & val)
+{
+    if (val->IsBigInt())
+        return val->ToBigInt(Nan::GetCurrentContext()).ToLocalChecked()->Uint64Value();
+    return Nan::To<int64_t>(val).FromJust();
+}
+
 //////////////////////////////////////////////////
 // NodeVitastor
 //////////////////////////////////////////////////
@@ -102,31 +109,14 @@ void NodeVitastor::on_io_readable(uv_poll_t* handle, int status, int revents)
     if (revents & UV_READABLE)
     {
         NodeVitastor* self = (NodeVitastor*)handle->data;
-        {
-            std::unique_lock<std::mutex> lock(self->mu);
-            vitastor_c_uring_handle_events(self->c);
-        }
-        self->run_postponed();
-    }
-}
-
-void NodeVitastor::run_postponed()
-{
-    std::vector<std::function<void()>> callbacks;
-    {
-        std::unique_lock<std::mutex> lock(mu);
-        callbacks.swap(postponed);
-    }
-    for (auto & cb: callbacks)
-    {
-        cb();
+        vitastor_c_uring_handle_events(self->c);
     }
 }
 
 NodeVitastorRequest* NodeVitastor::get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
 {
-    uint64_t offset = Nan::To<int64_t>(info[argpos+0]).FromJust();
-    uint64_t len = Nan::To<int64_t>(info[argpos+1]).FromJust();
+    uint64_t offset = get_ui64(info[argpos+0]);
+    uint64_t len = get_ui64(info[argpos+1]);
     uint8_t *buf = (uint8_t*)malloc(len);
     if (!buf)
     {
@@ -147,34 +137,34 @@ NodeVitastorRequest* NodeVitastor::get_read_request(const Nan::FunctionCallbackI
 NAN_METHOD(NodeVitastor::Read)
 {
     TRACE("NodeVitastor::Read");
+    if (info.Length() < 5)
+        Nan::ThrowError("Not enough arguments to read(pool, inode, offset, len, callback(err, buffer, version))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
-    uint64_t pool = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t inode = Nan::To<int64_t>(info[1]).FromJust();
+    uint64_t pool = get_ui64(info[0]);
+    uint64_t inode = get_ui64(info[1]);
 
     auto req = self->get_read_request(info, 2);
 
     self->Ref();
-    {
-        std::unique_lock<std::mutex> lock(self->mu);
-        vitastor_c_read(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, &req->iov, 1, postpone_read_finish, req);
-    }
-    self->run_postponed();
+    vitastor_c_read(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, &req->iov, 1, on_read_finish, req);
 }
 
 NodeVitastorRequest* NodeVitastor::get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
 {
-    uint64_t offset = Nan::To<int64_t>(info[argpos+0]).FromJust();
+    uint64_t offset = get_ui64(info[argpos+0]);
     const auto & bufarg = info[argpos+1];
     uint64_t version = 0;
-    if (!info[argpos+2].IsEmpty() && info[argpos+2]->IsObject())
+    if (!info[argpos+2].IsEmpty() &&
+        !info[argpos+2]->IsFunction() &&
+        info[argpos+2]->IsObject())
     {
         auto key = Nan::New<v8::String>("version").ToLocalChecked();
         auto params = info[argpos+2].As<v8::Object>();
         auto versionObj = Nan::Get(params, key).ToLocalChecked();
         if (!versionObj.IsEmpty())
-            version = Nan::To<int64_t>(versionObj).FromJust();
+            version = get_ui64(versionObj);
         argpos++;
     }
 
@@ -212,29 +202,29 @@ NodeVitastorRequest* NodeVitastor::get_write_request(const Nan::FunctionCallback
 NAN_METHOD(NodeVitastor::Write)
 {
     TRACE("NodeVitastor::Write");
+    if (info.Length() < 5)
+        Nan::ThrowError("Not enough arguments to write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
-    uint64_t pool = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t inode = Nan::To<int64_t>(info[1]).FromJust();
+    uint64_t pool = get_ui64(info[0]);
+    uint64_t inode = get_ui64(info[1]);
 
     auto req = self->get_write_request(info, 2);
 
     self->Ref();
-    {
-        std::unique_lock<std::mutex> lock(self->mu);
-        vitastor_c_write(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, req->version,
-            req->iov_list.size() ? req->iov_list.data() : &req->iov,
-            req->iov_list.size() ? req->iov_list.size() : 1,
-            postpone_write_finish, req);
-    }
-    self->run_postponed();
+    vitastor_c_write(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, req->version,
+        req->iov_list.size() ? req->iov_list.data() : &req->iov,
+        req->iov_list.size() ? req->iov_list.size() : 1,
+        on_write_finish, req);
 }
 
 // sync(callback(err))
 NAN_METHOD(NodeVitastor::Sync)
 {
     TRACE("NodeVitastor::Sync");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to sync(callback(err))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
@@ -242,34 +232,28 @@ NAN_METHOD(NodeVitastor::Sync)
     auto req = new NodeVitastorRequest(self, callback);
 
     self->Ref();
-    {
-        std::unique_lock<std::mutex> lock(self->mu);
-        vitastor_c_sync(self->c, postpone_write_finish, req);
-    }
-    self->run_postponed();
+    vitastor_c_sync(self->c, on_write_finish, req);
 }
 
 // read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))
 NAN_METHOD(NodeVitastor::ReadBitmap)
 {
     TRACE("NodeVitastor::ReadBitmap");
+    if (info.Length() < 6)
+        Nan::ThrowError("Not enough arguments to read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))");
 
     NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
 
-    uint64_t pool = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t inode = Nan::To<int64_t>(info[1]).FromJust();
-    uint64_t offset = Nan::To<int64_t>(info[2]).FromJust();
-    uint64_t len = Nan::To<int64_t>(info[3]).FromJust();
+    uint64_t pool = get_ui64(info[0]);
+    uint64_t inode = get_ui64(info[1]);
+    uint64_t offset = get_ui64(info[2]);
+    uint64_t len = get_ui64(info[3]);
     bool with_parents = Nan::To<bool>(info[4]).FromJust();
     v8::Local<v8::Function> callback = info[5].As<v8::Function>();
-    auto req = new NodeVitastorRequest(self, callback);
 
+    auto req = new NodeVitastorRequest(self, callback);
     self->Ref();
-    {
-        std::unique_lock<std::mutex> lock(self->mu);
-        vitastor_c_read_bitmap(self->c, ((pool << (64-POOL_ID_BITS)) | inode), offset, len, with_parents, postpone_read_bitmap_finish, req);
-    }
-    self->run_postponed();
+    vitastor_c_read_bitmap(self->c, ((pool << (64-POOL_ID_BITS)) | inode), offset, len, with_parents, on_read_bitmap_finish, req);
 }
 
 static void on_error(NodeVitastorRequest *req, Nan::Callback & nanCallback, long retval)
@@ -294,7 +278,7 @@ void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
         free(req->iov.iov_base);
         nanCallback.Call(0, NULL, req);
     }
-    else if (retval < 0)
+    else if (retval < 0 || (uint64_t)retval != req->len)
     {
         free(req->iov.iov_base);
         on_error(req, nanCallback, retval);
@@ -317,7 +301,16 @@ void NodeVitastor::on_write_finish(void *opaque, long retval)
     Nan::HandleScope scope;
     NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
     Nan::Callback nanCallback(Nan::New(req->callback));
-    on_error(req, nanCallback, retval);
+    if (retval < 0 || (uint64_t)retval != req->len)
+    {
+        on_error(req, nanCallback, retval);
+    }
+    else
+    {
+        v8::Local<v8::Value> args[1];
+        args[0] = Nan::Null();
+        nanCallback.Call(1, args, req);
+    }
     req->cli->Unref();
     delete req;
 }
@@ -343,24 +336,6 @@ void NodeVitastor::on_read_bitmap_finish(void *opaque, long retval, uint8_t *bit
     delete req;
 }
 
-void NodeVitastor::postpone_read_finish(void *opaque, long retval, uint64_t version)
-{
-    NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
-    req->cli->postponed.push_back([=]() { on_read_finish(opaque, retval, version); });
-}
-
-void NodeVitastor::postpone_write_finish(void *opaque, long retval)
-{
-    NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
-    req->cli->postponed.push_back([=]() { on_write_finish(opaque, retval); });
-}
-
-void NodeVitastor::postpone_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap)
-{
-    NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
-    req->cli->postponed.push_back([=]() { on_read_bitmap_finish(opaque, retval, bitmap); });
-}
-
 //NAN_METHOD(NodeVitastor::Destroy)
 //{
 //    TRACE("NodeVitastor::Destroy");
@@ -373,6 +348,8 @@ void NodeVitastor::postpone_read_bitmap_finish(void *opaque, long retval, uint8_
 NAN_METHOD(NodeVitastorImage::Create)
 {
     TRACE("NodeVitastorImage::Create");
+    if (info.Length() < 2)
+        Nan::ThrowError("Not enough arguments to Image(client, name)");
 
     v8::Local<v8::Object> parent = info[0].As<v8::Object>();
     std::string name = std::string(*Nan::Utf8String(info[1].As<v8::String>()));
@@ -386,7 +363,6 @@ NAN_METHOD(NodeVitastorImage::Create)
 
     img->Ref();
     cli->Ref();
-    std::unique_lock<std::mutex> lock(cli->mu);
     vitastor_c_watch_inode(cli->c, (char*)img->name.c_str(), on_watch_start, img);
 
     info.GetReturnValue().Set(info.This());
@@ -406,6 +382,8 @@ NodeVitastorImage::~NodeVitastorImage()
 NAN_METHOD(NodeVitastorImage::Read)
 {
     TRACE("NodeVitastorImage::Read");
+    if (info.Length() < 3)
+        Nan::ThrowError("Not enough arguments to read(offset, len, callback(err, buffer, version))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
@@ -420,6 +398,8 @@ NAN_METHOD(NodeVitastorImage::Read)
 NAN_METHOD(NodeVitastorImage::Write)
 {
     TRACE("NodeVitastorImage::Write");
+    if (info.Length() < 3)
+        Nan::ThrowError("Not enough arguments to write(offset, buffer, { version }?, callback(err))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
@@ -430,9 +410,12 @@ NAN_METHOD(NodeVitastorImage::Write)
     img->exec_or_wait(req);
 }
 
+// sync(callback(err))
 NAN_METHOD(NodeVitastorImage::Sync)
 {
     TRACE("NodeVitastorImage::Sync");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to sync(callback(err))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
@@ -448,11 +431,13 @@ NAN_METHOD(NodeVitastorImage::Sync)
 NAN_METHOD(NodeVitastorImage::ReadBitmap)
 {
     TRACE("NodeVitastorImage::ReadBitmap");
+    if (info.Length() < 4)
+        Nan::ThrowError("Not enough arguments to read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
-    uint64_t offset = Nan::To<int64_t>(info[0]).FromJust();
-    uint64_t len = Nan::To<int64_t>(info[1]).FromJust();
+    uint64_t offset = get_ui64(info[0]);
+    uint64_t len = get_ui64(info[1]);
     bool with_parents = Nan::To<bool>(info[2]).FromJust();
     v8::Local<v8::Function> callback = info[3].As<v8::Function>();
 
@@ -466,9 +451,12 @@ NAN_METHOD(NodeVitastorImage::ReadBitmap)
     img->exec_or_wait(req);
 }
 
+// get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))
 NAN_METHOD(NodeVitastorImage::GetInfo)
 {
     TRACE("NodeVitastorImage::GetInfo");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))");
 
     NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
 
@@ -497,50 +485,79 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
 {
     if (req->op == NODE_VITASTOR_READ)
     {
-        cli->Ref();
-        std::unique_lock<std::mutex> lock(cli->mu);
         uint64_t ino = vitastor_c_inode_get_num(watch);
-        vitastor_c_read(cli->c, ino, req->offset, req->len, &req->iov, 1, NodeVitastor::postpone_read_finish, req);
+        cli->Ref();
+        vitastor_c_read(cli->c, ino, req->offset, req->len, &req->iov, 1, NodeVitastor::on_read_finish, req);
     }
     else if (req->op == NODE_VITASTOR_WRITE)
     {
-        cli->Ref();
-        std::unique_lock<std::mutex> lock(cli->mu);
         uint64_t ino = vitastor_c_inode_get_num(watch);
+        cli->Ref();
         vitastor_c_write(cli->c, ino, req->offset, req->len, req->version,
             req->iov_list.size() ? req->iov_list.data() : &req->iov,
             req->iov_list.size() ? req->iov_list.size() : 1,
-            NodeVitastor::postpone_write_finish, req);
+            NodeVitastor::on_write_finish, req);
     }
     else if (req->op == NODE_VITASTOR_SYNC)
     {
-        cli->Ref();
-        std::unique_lock<std::mutex> lock(cli->mu);
         uint64_t ino = vitastor_c_inode_get_num(watch);
         uint32_t imm = vitastor_c_inode_get_immediate_commit(cli->c, ino);
+        cli->Ref();
         if (imm != IMMEDIATE_ALL)
         {
-            vitastor_c_sync(cli->c, NodeVitastor::postpone_write_finish, req);
+            vitastor_c_sync(cli->c, NodeVitastor::on_write_finish, req);
         }
         else
         {
-            NodeVitastor::postpone_write_finish(req, 0);
+            NodeVitastor::on_write_finish(req, 0);
         }
     }
     else if (req->op == NODE_VITASTOR_READ_BITMAP)
     {
-        cli->Ref();
-        std::unique_lock<std::mutex> lock(cli->mu);
         uint64_t ino = vitastor_c_inode_get_num(watch);
-        vitastor_c_read_bitmap(cli->c, ino, req->offset, req->len, req->with_parents, NodeVitastor::postpone_read_bitmap_finish, req);
+        cli->Ref();
+        vitastor_c_read_bitmap(cli->c, ino, req->offset, req->len, req->with_parents, NodeVitastor::on_read_bitmap_finish, req);
     }
     else if (req->op == NODE_VITASTOR_GET_INFO)
     {
-        v8::Local<v8::Object> res = Nan::New<v8::Object>();
-
-        fill_info(res);
+        uint64_t size = vitastor_c_inode_get_size(watch);
+        uint64_t num = vitastor_c_inode_get_num(watch);
+        uint32_t block_size = vitastor_c_inode_get_block_size(cli->c, num);
+        uint32_t bitmap_granularity = vitastor_c_inode_get_bitmap_granularity(cli->c, num);
+        int readonly = vitastor_c_inode_get_readonly(watch);
+        uint32_t immediate_commit = vitastor_c_inode_get_immediate_commit(cli->c, num);
+        uint64_t parent_id = vitastor_c_inode_get_parent_id(watch);
+        char *meta = vitastor_c_inode_get_meta(watch);
+        uint64_t mod_revision = vitastor_c_inode_get_mod_revision(watch);
 
         Nan::HandleScope scope;
+
+        v8::Local<v8::Object> res = Nan::New<v8::Object>();
+        Nan::Set(res, Nan::New<v8::String>("pool_id").ToLocalChecked(), Nan::New<v8::Number>(INODE_POOL(num)));
+        Nan::Set(res, Nan::New<v8::String>("inode_num").ToLocalChecked(), Nan::New<v8::Number>(INODE_NO_POOL(num)));
+        if (size < ((uint64_t)1<<53))
+            Nan::Set(res, Nan::New<v8::String>("size").ToLocalChecked(), Nan::New<v8::Number>(size));
+        else
+            Nan::Set(res, Nan::New<v8::String>("size").ToLocalChecked(), v8::BigInt::NewFromUnsigned(v8::Isolate::GetCurrent(), size));
+        if (parent_id)
+        {
+            Nan::Set(res, Nan::New<v8::String>("parent_pool_id").ToLocalChecked(), Nan::New<v8::Number>(INODE_POOL(parent_id)));
+            Nan::Set(res, Nan::New<v8::String>("parent_inode_num").ToLocalChecked(), Nan::New<v8::Number>(INODE_NO_POOL(parent_id)));
+        }
+        Nan::Set(res, Nan::New<v8::String>("readonly").ToLocalChecked(), Nan::New((bool)readonly));
+        if (meta)
+        {
+            Nan::JSON nanJSON;
+            Nan::Set(res, Nan::New<v8::String>("meta").ToLocalChecked(), nanJSON.Parse(Nan::New<v8::String>(meta).ToLocalChecked()).ToLocalChecked());
+        }
+        if (mod_revision < ((uint64_t)1<<53))
+            Nan::Set(res, Nan::New<v8::String>("mod_revision").ToLocalChecked(), Nan::New<v8::Number>(mod_revision));
+        else
+            Nan::Set(res, Nan::New<v8::String>("mod_revision").ToLocalChecked(), v8::BigInt::NewFromUnsigned(v8::Isolate::GetCurrent(), mod_revision));
+        Nan::Set(res, Nan::New<v8::String>("block_size").ToLocalChecked(), Nan::New(block_size));
+        Nan::Set(res, Nan::New<v8::String>("bitmap_granularity").ToLocalChecked(), Nan::New(bitmap_granularity));
+        Nan::Set(res, Nan::New<v8::String>("immediate_commit").ToLocalChecked(), Nan::New(immediate_commit));
+
         Nan::Callback nanCallback(Nan::New(req->callback));
         v8::Local<v8::Value> args[1];
         args[0] = res;
@@ -548,46 +565,6 @@ void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
 
         delete req;
     }
-    cli->run_postponed();
-}
-
-void NodeVitastorImage::fill_info(v8::Local<v8::Object> & res)
-{
-    std::unique_lock<std::mutex> lock(cli->mu);
-    uint64_t size = vitastor_c_inode_get_size(watch);
-    uint64_t num = vitastor_c_inode_get_num(watch);
-    uint32_t block_size = vitastor_c_inode_get_block_size(cli->c, num);
-    uint32_t bitmap_granularity = vitastor_c_inode_get_bitmap_granularity(cli->c, num);
-    int readonly = vitastor_c_inode_get_readonly(watch);
-    uint32_t immediate_commit = vitastor_c_inode_get_immediate_commit(cli->c, num);
-    uint64_t parent_id = vitastor_c_inode_get_parent_id(watch);
-    char *meta = vitastor_c_inode_get_meta(watch);
-    uint64_t mod_revision = vitastor_c_inode_get_mod_revision(watch);
-
-    Nan::Set(res, Nan::New<v8::String>("pool_id").ToLocalChecked(), Nan::New<v8::Number>(INODE_POOL(num)));
-    Nan::Set(res, Nan::New<v8::String>("inode_num").ToLocalChecked(), Nan::New<v8::Number>(INODE_NO_POOL(num)));
-    if (size < ((uint64_t)1<<53))
-        Nan::Set(res, Nan::New<v8::String>("size").ToLocalChecked(), Nan::New<v8::Number>(size));
-    else
-        Nan::Set(res, Nan::New<v8::String>("size").ToLocalChecked(), v8::BigInt::NewFromUnsigned(v8::Isolate::GetCurrent(), size));
-    if (parent_id)
-    {
-        Nan::Set(res, Nan::New<v8::String>("parent_pool_id").ToLocalChecked(), Nan::New<v8::Number>(INODE_POOL(parent_id)));
-        Nan::Set(res, Nan::New<v8::String>("parent_inode_num").ToLocalChecked(), Nan::New<v8::Number>(INODE_NO_POOL(parent_id)));
-    }
-    Nan::Set(res, Nan::New<v8::String>("readonly").ToLocalChecked(), Nan::New((bool)readonly));
-    if (meta)
-    {
-        Nan::JSON nanJSON;
-        Nan::Set(res, Nan::New<v8::String>("meta").ToLocalChecked(), nanJSON.Parse(Nan::New<v8::String>(meta).ToLocalChecked()).ToLocalChecked());
-    }
-    if (mod_revision < ((uint64_t)1<<53))
-        Nan::Set(res, Nan::New<v8::String>("mod_revision").ToLocalChecked(), Nan::New<v8::Number>(mod_revision));
-    else
-        Nan::Set(res, Nan::New<v8::String>("mod_revision").ToLocalChecked(), v8::BigInt::NewFromUnsigned(v8::Isolate::GetCurrent(), mod_revision));
-    Nan::Set(res, Nan::New<v8::String>("block_size").ToLocalChecked(), Nan::New(block_size));
-    Nan::Set(res, Nan::New<v8::String>("bitmap_granularity").ToLocalChecked(), Nan::New(bitmap_granularity));
-    Nan::Set(res, Nan::New<v8::String>("immediate_commit").ToLocalChecked(), Nan::New(immediate_commit));
 }
 
 void NodeVitastorImage::on_watch_start(void *opaque, long retval)
@@ -614,6 +591,8 @@ Nan::Persistent<v8::Function> NodeVitastorKV::listing_class;
 NAN_METHOD(NodeVitastorKV::Create)
 {
     TRACE("NodeVitastorKV::Create");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to new KV(client)");
 
     v8::Local<v8::Object> parent = info[0].As<v8::Object>();
     NodeVitastor *cli = Nan::ObjectWrap::Unwrap<NodeVitastor>(parent);
@@ -621,7 +600,6 @@ NAN_METHOD(NodeVitastorKV::Create)
     NodeVitastorKV *kv = new NodeVitastorKV();
     kv->cli = cli;
     {
-        std::unique_lock<std::mutex> lock(cli->mu);
         kv->dbw = new vitastorkv_dbw_t((cluster_client_t*)vitastor_c_get_internal_client(cli->c));
     }
 
@@ -640,10 +618,12 @@ NodeVitastorKV::~NodeVitastorKV()
 NAN_METHOD(NodeVitastorKV::Open)
 {
     TRACE("NodeVitastorKV::Open");
+    if (info.Length() < 4)
+        Nan::ThrowError("Not enough arguments to open(pool_id, inode_num, { ...config }, callback(err))");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
-    uint64_t inode_id = INODE_WITH_POOL(Nan::To<int64_t>(info[0]).FromJust(), Nan::To<int64_t>(info[1]).FromJust());
+    uint64_t inode_id = INODE_WITH_POOL(get_ui64(info[0]), get_ui64(info[1]));
 
     v8::Local<v8::Object> jsParams = info[2].As<v8::Object>();
     v8::Local<v8::Array> keys = Nan::GetOwnPropertyNames(jsParams).ToLocalChecked();
@@ -674,6 +654,8 @@ NAN_METHOD(NodeVitastorKV::Open)
 NAN_METHOD(NodeVitastorKV::Close)
 {
     TRACE("NodeVitastorKV::Close");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to close(callback(err))");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -695,6 +677,8 @@ NAN_METHOD(NodeVitastorKV::Close)
 NAN_METHOD(NodeVitastorKV::SetConfig)
 {
     TRACE("NodeVitastorKV::SetConfig");
+    if (info.Length() < 1)
+        Nan::ThrowError("Not enough arguments to set_config({ ...config })");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -751,6 +735,8 @@ void NodeVitastorKV::get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info,
 NAN_METHOD(NodeVitastorKV::Get)
 {
     TRACE("NodeVitastorKV::Get");
+    if (info.Length() < 2)
+        Nan::ThrowError("Not enough arguments to get(key, callback(err, value))");
     get_impl(info, false);
 }
 
@@ -758,6 +744,8 @@ NAN_METHOD(NodeVitastorKV::Get)
 NAN_METHOD(NodeVitastorKV::GetCached)
 {
     TRACE("NodeVitastorKV::GetCached");
+    if (info.Length() < 2)
+        Nan::ThrowError("Not enough arguments to get_cached(key, callback(err, value))");
     get_impl(info, true);
 }
 
@@ -776,10 +764,12 @@ static std::function<bool(int, const std::string &)> make_cas_callback(NodeVitas
     };
 }
 
-// set(key, value, callback(err), cas_compare(old_value))
+// set(key, value, callback(err), cas_compare(old_value)?)
 NAN_METHOD(NodeVitastorKV::Set)
 {
     TRACE("NodeVitastorKV::Set");
+    if (info.Length() < 3)
+        Nan::ThrowError("Not enough arguments to set(key, value, callback(err), cas_compare(old_value)?)");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -813,10 +803,12 @@ NAN_METHOD(NodeVitastorKV::Set)
     }, cas_cb);
 }
 
-// del(key, callback(err), cas_compare(old_value))
+// del(key, callback(err), cas_compare(old_value)?)
 NAN_METHOD(NodeVitastorKV::Del)
 {
     TRACE("NodeVitastorKV::Del");
+    if (info.Length() < 2)
+        Nan::ThrowError("Not enough arguments to del(key, callback(err), cas_compare(old_value)?)");
 
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
 
@@ -870,7 +862,6 @@ NAN_METHOD(NodeVitastorKV::List)
 {
     TRACE("NodeVitastorKV::Destroy");
     NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
-    std::unique_lock<std::mutex> lock(self->mu);
     if (!kv->dead)
         kv->Unref();
 }*/
@@ -896,10 +887,7 @@ NAN_METHOD(NodeVitastorKVListing::Create)
 
     NodeVitastorKVListing *list = new NodeVitastorKVListing();
     list->kv = kv;
-    {
-        std::unique_lock<std::mutex> lock(kv->cli->mu);
-        list->handle = list->kv->dbw->list_start(start_key);
-    }
+    list->handle = list->kv->dbw->list_start(start_key);
 
     list->Wrap(info.This());
     kv->Ref();
@@ -910,36 +898,54 @@ NodeVitastorKVListing::~NodeVitastorKVListing()
 {
     if (handle)
     {
-        std::unique_lock<std::mutex> lock(kv->cli->mu);
         kv->dbw->list_close(handle);
         handle = NULL;
     }
+    if (iter)
+    {
+        delete iter;
+        iter = NULL;
+    }
     kv->Unref();
 }
 
-// next(callback(err, value))
+// next(callback(err, value)?)
 NAN_METHOD(NodeVitastorKVListing::Next)
 {
     TRACE("NodeVitastorKVListing::Next");
 
     NodeVitastorKVListing* list = Nan::ObjectWrap::Unwrap<NodeVitastorKVListing>(info.This());
 
-    v8::Local<v8::Function> callback = info[0].As<v8::Function>();
-    auto req = new NodeVitastorRequest(list->kv->cli, callback);
+    if (info.Length() > 0)
+    {
+        v8::Local<v8::Function> callback = info[0].As<v8::Function>();
+        if (list->iter)
+        {
+            delete list->iter;
+        }
+        list->iter = new NodeVitastorRequest(list->kv->cli, callback);
+    }
     if (!list->handle)
     {
         // Already closed
-        Nan::Callback nanCallback(Nan::New(req->callback));
-        v8::Local<v8::Value> args[1];
-        args[0] = Nan::New<v8::Int32>(-EINVAL);
-        nanCallback.Call(1, args, req);
-        delete req;
+        if (list->iter)
+        {
+            auto req = list->iter;
+            list->iter = NULL;
+            Nan::Callback nanCallback(Nan::New(req->callback));
+            v8::Local<v8::Value> args[1];
+            args[0] = Nan::New<v8::Int32>(-EINVAL);
+            nanCallback.Call(1, args, req);
+            delete req;
+        }
         return;
     }
 
     list->kv->Ref();
-    list->kv->dbw->list_next(list->handle, [list, req](int res, const std::string & key, const std::string & value)
+    list->kv->dbw->list_next(list->handle, [list](int res, const std::string & key, const std::string & value)
     {
+        auto req = list->iter;
+        list->iter = NULL;
         Nan::HandleScope scope;
         Nan::Callback nanCallback(Nan::New(req->callback));
         v8::Local<v8::Value> args[3];
@@ -947,7 +953,10 @@ NAN_METHOD(NodeVitastorKVListing::Next)
         args[1] = !res ? v8::Local<v8::Value>(Nan::New<v8::String>(key).ToLocalChecked()) : v8::Local<v8::Value>(Nan::Null());
         args[2] = !res ? v8::Local<v8::Value>(Nan::New<v8::String>(value).ToLocalChecked()) : v8::Local<v8::Value>(Nan::Null());
         nanCallback.Call(3, args, req);
-        delete req;
+        if (list->iter)
+            delete req;
+        else
+            list->iter = req;
         list->kv->Unref();
     });
 }
@@ -961,8 +970,12 @@ NAN_METHOD(NodeVitastorKVListing::Close)
 
     if (list->handle)
     {
-        std::unique_lock<std::mutex> lock(list->kv->cli->mu);
         list->kv->dbw->list_close(list->handle);
         list->handle = NULL;
     }
+    if (list->iter)
+    {
+        delete list->iter;
+        list->iter = NULL;
+    }
 }

node-binding/client.h

@@ -4,8 +4,6 @@
 #ifndef NODE_VITASTOR_CLIENT_H
 #define NODE_VITASTOR_CLIENT_H
 
-#include <mutex>
-
 #include <nan.h>
 #include <vitastor_c.h>
 #include <vitastor_kv.h>
@@ -34,9 +32,6 @@ private:
     vitastor_c *c = NULL;
     int eventfd = -1;
     uv_poll_t poll_watcher;
-    // FIXME: Is it really needed?
-    std::mutex mu;
-    std::vector<std::function<void()>> postponed;
 
     NodeVitastor();
 
@@ -45,11 +40,6 @@ private:
     static void on_write_finish(void *opaque, long retval);
     static void on_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap);
 
-    void run_postponed();
-    static void postpone_read_finish(void *opaque, long retval, uint64_t version);
-    static void postpone_write_finish(void *opaque, long retval);
-    static void postpone_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap);
-
     NodeVitastorRequest* get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
     NodeVitastorRequest* get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
 
@@ -86,7 +76,6 @@ private:
     static void on_watch_start(void *opaque, long retval);
     void exec_request(NodeVitastorRequest *req);
     void exec_or_wait(NodeVitastorRequest *req);
-    void fill_info(v8::Local<v8::Object> & res);
 };
 
 class NodeVitastorKV: public Nan::ObjectWrap
@@ -106,9 +95,9 @@ public:
     static NAN_METHOD(Get);
     // get_cached(key, callback(err, value))
     static NAN_METHOD(GetCached);
-    // set(key, value, callback(err), cas_compare(old_value))
+    // set(key, value, callback(err), cas_compare(old_value)?)
     static NAN_METHOD(Set);
-    // del(key, callback(err), cas_compare(old_value))
+    // del(key, callback(err), cas_compare(old_value)?)
     static NAN_METHOD(Del);
     // list(start_key?)
     static NAN_METHOD(List);
@@ -131,7 +120,7 @@ class NodeVitastorKVListing: public Nan::ObjectWrap
 public:
     // constructor(node_vitastor_kv, start_key?)
     static NAN_METHOD(Create);
-    // next(callback(err, value))
+    // next(callback(err, value)?)
     static NAN_METHOD(Next);
     // close()
     static NAN_METHOD(Close);
@@ -141,6 +130,7 @@ public:
 private:
     NodeVitastorKV *kv = NULL;
     void *handle = NULL;
+    NodeVitastorRequest *iter = NULL;
 };
 
 #endif

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '1.7.1'
+VITASTOR_VERSION = '1.8.0'
 
 LOG = logging.getLogger(__name__)
 

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.7.1
+Version:        1.8.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.7.1.el7.tar.gz
+Source0:        vitastor-1.8.0.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.7.1
+Version:        1.8.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.7.1.el8.tar.gz
+Source0:        vitastor-1.8.0.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        1.7.1
+Version:        1.8.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-1.7.1.el9.tar.gz
+Source0:        vitastor-1.8.0.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="1.7.1")
+add_definitions(-DVITASTOR_VERSION="1.8.0")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/client/cluster_client.cpp

@@ -1286,7 +1286,11 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
         if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
         {
             copy_part_bitmap(op, part);
-            op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
+            if (op->inode == op->cur_inode)
+            {
+                // Read only returns the version of the uppermost layer
+                op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
+            }
         }
         else if (op->opcode == OSD_OP_WRITE)
         {

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 1.7.1
+Version: 1.8.0
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/nfs/nfs_proxy.cpp

@@ -522,17 +522,15 @@ void nfs_proxy_t::do_accept(int listen_fd)
         {
             cli->proc_table.insert(fn);
         }
+        rpc_clients[nfs_fd] = cli;
         epmgr->tfd->set_fd_handler(nfs_fd, true, [cli](int nfs_fd, int epoll_events)
         {
             // Handle incoming event
             if (epoll_events & EPOLLRDHUP)
             {
-                auto parent = cli->parent;
-                if (parent->trace)
+                if (cli->parent->trace)
                     fprintf(stderr, "Client %d disconnected\n", nfs_fd);
                 cli->stop();
-                parent->active_connections--;
-                parent->check_exit();
                 return;
             }
             cli->epoll_events |= epoll_events;
@@ -695,6 +693,8 @@ void nfs_client_t::handle_read(int result)
                         frag_num++;
                     }
                 }
+                // Increase client refcount while the RPC call is being processed
+                refs++;
                 // Handle full message
                 int referenced = handle_rpc_message(cur_buffer.buf, data+4, wanted-4*fragments);
                 cur_buffer.refs += referenced ? 1 : 0;
@@ -779,9 +779,13 @@ void nfs_client_t::stop()
     stopped = true;
     if (refs <= 0)
     {
+        auto parent = this->parent;
+        parent->rpc_clients.erase(nfs_fd);
+        parent->active_connections--;
         parent->epmgr->tfd->set_fd_handler(nfs_fd, true, NULL);
         close(nfs_fd);
         delete this;
+        parent->check_exit();
     }
 }
 
@@ -834,6 +838,10 @@ void nfs_client_t::handle_send(int result)
                         }
                     }
                     free(rop);
+                    if (deref())
+                    {
+                        return;
+                    }
                 }
                 result -= iov.iov_len;
                 done++;
@@ -1056,6 +1064,11 @@ int nfs_client_t::handle_rpc_message(void *base_buf, void *msg_buf, uint32_t msg
 
 void nfs_proxy_t::daemonize()
 {
+    // Stop all clients because client I/O sometimes breaks during daemonize
+    // I.e. the new process stops receiving events on the old FD
+    // It doesn't happen if we call sleep(1) here, but we don't want to call sleep(1)...
+    for (auto & clp: rpc_clients)
+        clp.second->stop();
     if (fork())
         exit(0);
     setsid();

src/nfs/nfs_proxy.h

@@ -21,6 +21,7 @@ class cli_tool_t;
 
 struct kv_fs_state_t;
 struct block_fs_state_t;
+class nfs_client_t;
 
 class nfs_proxy_t
 {
@@ -54,6 +55,7 @@ public:
     vitastorkv_dbw_t *db = NULL;
     kv_fs_state_t *kvfs = NULL;
     block_fs_state_t *blockfs = NULL;
+    std::map<int, nfs_client_t*> rpc_clients;
 
     std::vector<XDR*> xdr_pool;