Release 0.5.6

- Fix operation statistics
- Fix a rebalance hang introduced in 0.5.5
- Test PG count changes with actual data moving
- Fix a possible 'unexpected pg state: 0' error during PG count change

.dockerignore

@@ -1,5 +1,6 @@
 .git
 build
+packages
 mon/node_modules
 *.o
 *.so
@@ -15,3 +16,4 @@ fio
 qemu
 rpm/*.Dockerfile
 debian/*.Dockerfile
+Dockerfile

CMakeLists.txt

@@ -0,0 +1,5 @@
+cmake_minimum_required(VERSION 2.8)
+
+project(vitastor)
+
+add_subdirectory(src)

Make-gen.pl

@@ -1,46 +0,0 @@
-#!/usr/bin/perl
-
-use strict;
-
-my $deps = {};
-for my $line (split /\n/, `grep '^#include "' *.cpp *.h`)
-{
-    if ($line =~ /^([^:]+):\#include "([^"]+)"/s)
-    {
-        $deps->{$1}->{$2} = 1;
-    }
-}
-
-my $added;
-do
-{
-    $added = 0;
-    for my $file (keys %$deps)
-    {
-        for my $dep (keys %{$deps->{$file}})
-        {
-            if ($deps->{$dep})
-            {
-                for my $subdep (keys %{$deps->{$dep}})
-                {
-                    if (!$deps->{$file}->{$subdep})
-                    {
-                        $added = 1;
-                        $deps->{$file}->{$subdep} = 1;
-                    }
-                }
-            }
-        }
-    }
-} while ($added);
-
-for my $file (sort keys %$deps)
-{
-    if ($file =~ /\.cpp$/)
-    {
-        my $obj = $file;
-        $obj =~ s/\.cpp$/.o/s;
-        print "$obj: $file ".join(" ", sort keys %{$deps->{$file}})."\n";
-        print "\tg++ \$(CXXFLAGS) -c -o \$\@ \$\<\n";
-    }
-}

Makefile

@@ -1,195 +0,0 @@
-BINDIR ?= /usr/bin
-LIBDIR ?= /usr/lib/x86_64-linux-gnu
-QEMU_PLUGINDIR ?= /usr/lib/x86_64-linux-gnu/qemu
-
-BLOCKSTORE_OBJS := allocator.o blockstore.o blockstore_impl.o blockstore_init.o blockstore_open.o blockstore_journal.o blockstore_read.o \
-	blockstore_write.o blockstore_sync.o blockstore_stable.o blockstore_rollback.o blockstore_flush.o crc32c.o ringloop.o
-# -fsanitize=address
-CXXFLAGS := -g -O3 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fPIC -fdiagnostics-color=always -I/usr/include/jerasure
-all: libfio_blockstore.so osd libfio_sec_osd.so libfio_cluster.so stub_osd stub_uring_osd stub_bench osd_test dump_journal qemu_driver.so nbd_proxy rm_inode
-clean:
-	rm -f *.o libblockstore.so libfio_blockstore.so osd libfio_sec_osd.so libfio_cluster.so stub_osd stub_uring_osd stub_bench osd_test dump_journal qemu_driver.so nbd_proxy rm_inode
-
-install: all
-	mkdir -p $(DESTDIR)$(LIBDIR)/vitastor
-	install -m 0755 libfio_sec_osd.so $(DESTDIR)$(LIBDIR)/vitastor/
-	install -m 0755 libfio_cluster.so $(DESTDIR)$(LIBDIR)/vitastor/
-	install -m 0755 libfio_blockstore.so $(DESTDIR)$(LIBDIR)/vitastor/
-	install -m 0755 libblockstore.so $(DESTDIR)$(LIBDIR)/vitastor/
-	mkdir -p $(DESTDIR)$(BINDIR)
-	install -m 0755 osd $(DESTDIR)$(BINDIR)/vitastor-osd
-	install -m 0755 dump_journal $(DESTDIR)$(BINDIR)/vitastor-dump-journal
-	install -m 0755 nbd_proxy $(DESTDIR)$(BINDIR)/vitastor-nbd
-	install -m 0755 rm_inode $(DESTDIR)$(BINDIR)/vitastor-rm
-	mkdir -p $(DESTDIR)$(QEMU_PLUGINDIR)
-	install -m 0755 qemu_driver.so $(DESTDIR)$(QEMU_PLUGINDIR)/block-vitastor.so
-
-dump_journal: dump_journal.cpp crc32c.o blockstore_journal.h
-	g++ $(CXXFLAGS) -o $@ $< crc32c.o
-
-libblockstore.so: $(BLOCKSTORE_OBJS)
-	g++ $(CXXFLAGS) -o $@ -shared $(BLOCKSTORE_OBJS) -ltcmalloc_minimal -luring
-libfio_blockstore.so: ./libblockstore.so fio_engine.o json11.o
-	g++ $(CXXFLAGS) -Wl,-rpath,'$(LIBDIR)/vitastor',-rpath,'$$ORIGIN' -shared -o $@ fio_engine.o json11.o libblockstore.so -ltcmalloc_minimal -luring
-
-OSD_OBJS := osd.o osd_secondary.o msgr_receive.o msgr_send.o osd_peering.o osd_flush.o osd_peering_pg.o \
-	osd_primary.o osd_primary_subops.o etcd_state_client.o messenger.o osd_cluster.o http_client.o osd_ops.o pg_states.o \
-	osd_rmw.o json11.o base64.o timerfd_manager.o epoll_manager.o
-osd: ./libblockstore.so osd_main.cpp osd.h osd_ops.h $(OSD_OBJS)
-	g++ $(CXXFLAGS) -Wl,-rpath,'$(LIBDIR)/vitastor',-rpath,'$$ORIGIN' -o $@ osd_main.cpp $(OSD_OBJS) libblockstore.so -ltcmalloc_minimal -luring -lJerasure
-
-stub_osd: stub_osd.o rw_blocking.o
-	g++ $(CXXFLAGS) -o $@ stub_osd.o rw_blocking.o -ltcmalloc_minimal
-
-osd_rmw_test: osd_rmw_test.o
-	g++ $(CXXFLAGS) -o $@ osd_rmw_test.o -lJerasure -fsanitize=address
-
-STUB_URING_OSD_OBJS := stub_uring_osd.o epoll_manager.o messenger.o msgr_send.o msgr_receive.o ringloop.o timerfd_manager.o json11.o
-stub_uring_osd: $(STUB_URING_OSD_OBJS)
-	g++ $(CXXFLAGS) -o $@ -ltcmalloc_minimal $(STUB_URING_OSD_OBJS) -luring
-stub_bench: stub_bench.cpp osd_ops.h rw_blocking.o
-	g++ $(CXXFLAGS) -o $@ stub_bench.cpp rw_blocking.o -ltcmalloc_minimal
-osd_test: osd_test.cpp osd_ops.h rw_blocking.o
-	g++ $(CXXFLAGS) -o $@ osd_test.cpp rw_blocking.o -ltcmalloc_minimal
-osd_peering_pg_test: osd_peering_pg_test.cpp osd_peering_pg.o
-	g++ $(CXXFLAGS) -o $@ $< osd_peering_pg.o -ltcmalloc_minimal
-
-libfio_sec_osd.so: fio_sec_osd.o rw_blocking.o
-	g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o $@ fio_sec_osd.o rw_blocking.o
-
-FIO_CLUSTER_OBJS := cluster_client.o epoll_manager.o etcd_state_client.o \
-	messenger.o msgr_send.o msgr_receive.o ringloop.o json11.o http_client.o osd_ops.o pg_states.o timerfd_manager.o base64.o
-libfio_cluster.so: fio_cluster.o $(FIO_CLUSTER_OBJS)
-	g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o $@ $< $(FIO_CLUSTER_OBJS) -luring
-
-nbd_proxy: nbd_proxy.o $(FIO_CLUSTER_OBJS)
-	g++ $(CXXFLAGS) -ltcmalloc_minimal -o $@ $< $(FIO_CLUSTER_OBJS) -luring
-
-rm_inode: rm_inode.o $(FIO_CLUSTER_OBJS)
-	g++ $(CXXFLAGS) -ltcmalloc_minimal -o $@ $< $(FIO_CLUSTER_OBJS) -luring
-
-qemu_driver.o: qemu_driver.c qemu_proxy.h
-	gcc -I qemu/b/qemu `pkg-config glib-2.0 --cflags` \
-		-I qemu/include $(CXXFLAGS) -c -o $@ $<
-
-qemu_driver.so: qemu_driver.o qemu_proxy.o $(FIO_CLUSTER_OBJS)
-	g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o $@ $(FIO_CLUSTER_OBJS) qemu_driver.o qemu_proxy.o -luring
-
-test_blockstore: ./libblockstore.so test_blockstore.cpp timerfd_interval.o
-	g++ $(CXXFLAGS) -Wl,-rpath,'$(LIBDIR)/vitastor',-rpath,'$$ORIGIN' -o test_blockstore test_blockstore.cpp timerfd_interval.o libblockstore.so -ltcmalloc_minimal -luring
-test_shit: test_shit.cpp osd_peering_pg.o
-	g++ $(CXXFLAGS) -o test_shit test_shit.cpp -luring -lm
-test_allocator: test_allocator.cpp allocator.o
-	g++ $(CXXFLAGS) -o test_allocator test_allocator.cpp allocator.o
-
-crc32c.o: crc32c.c crc32c.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-json11.o: json11/json11.cpp
-	g++ $(CXXFLAGS) -c -o json11.o json11/json11.cpp
-
-# Autogenerated
-
-allocator.o: allocator.cpp allocator.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-base64.o: base64.cpp base64.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore.o: blockstore.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_flush.o: blockstore_flush.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_impl.o: blockstore_impl.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_init.o: blockstore_init.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_journal.o: blockstore_journal.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_open.o: blockstore_open.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_read.o: blockstore_read.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_rollback.o: blockstore_rollback.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_stable.o: blockstore_stable.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_sync.o: blockstore_sync.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_write.o: blockstore_write.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-cluster_client.o: cluster_client.cpp cluster_client.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-dump_journal.o: dump_journal.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-epoll_manager.o: epoll_manager.cpp epoll_manager.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-etcd_state_client.o: etcd_state_client.cpp base64.h etcd_state_client.h http_client.h json11/json11.hpp object_id.h osd_id.h osd_ops.h pg_states.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-fio_cluster.o: fio_cluster.cpp cluster_client.h epoll_manager.h etcd_state_client.h fio/arch/arch.h fio/fio.h fio/optgroup.h fio_headers.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-fio_engine.o: fio_engine.cpp blockstore.h fio/arch/arch.h fio/fio.h fio/optgroup.h fio_headers.h json11/json11.hpp object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-fio_sec_osd.o: fio_sec_osd.cpp fio/arch/arch.h fio/fio.h fio/optgroup.h fio_headers.h object_id.h osd_id.h osd_ops.h rw_blocking.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-http_client.o: http_client.cpp http_client.h json11/json11.hpp timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-messenger.o: messenger.cpp json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-msgr_receive.o: msgr_receive.cpp json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-msgr_send.o: msgr_send.cpp json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-nbd_proxy.o: nbd_proxy.cpp cluster_client.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd.o: osd.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_cluster.o: osd_cluster.cpp base64.h blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_flush.o: osd_flush.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_main.o: osd_main.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_ops.o: osd_ops.cpp object_id.h osd_id.h osd_ops.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_peering.o: osd_peering.cpp base64.h blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_peering_pg.o: osd_peering_pg.cpp cpp-btree/btree_map.h object_id.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_peering_pg_test.o: osd_peering_pg_test.cpp cpp-btree/btree_map.h object_id.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_primary.o: osd_primary.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h osd_primary.h osd_rmw.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_primary_subops.o: osd_primary_subops.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h osd_primary.h osd_rmw.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_rmw.o: osd_rmw.cpp malloc_or_die.h object_id.h osd_id.h osd_rmw.h xor.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_rmw_test.o: osd_rmw_test.cpp malloc_or_die.h object_id.h osd_id.h osd_rmw.cpp osd_rmw.h test_pattern.h xor.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_secondary.o: osd_secondary.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_test.o: osd_test.cpp object_id.h osd_id.h osd_ops.h rw_blocking.h test_pattern.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-pg_states.o: pg_states.cpp pg_states.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-qemu_proxy.o: qemu_proxy.cpp cluster_client.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h qemu_proxy.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-ringloop.o: ringloop.cpp ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-rm_inode.o: rm_inode.cpp cluster_client.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-rw_blocking.o: rw_blocking.cpp rw_blocking.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-stub_bench.o: stub_bench.cpp object_id.h osd_id.h osd_ops.h rw_blocking.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-stub_osd.o: stub_osd.cpp object_id.h osd_id.h osd_ops.h rw_blocking.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-stub_uring_osd.o: stub_uring_osd.cpp epoll_manager.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-test_allocator.o: test_allocator.cpp allocator.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-test_blockstore.o: test_blockstore.cpp blockstore.h object_id.h ringloop.h timerfd_interval.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-test_shit.o: test_shit.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-timerfd_interval.o: timerfd_interval.cpp ringloop.h timerfd_interval.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-timerfd_manager.o: timerfd_manager.cpp timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<

README.md

@@ -336,9 +336,8 @@ Vitastor with single-thread NBD on the same hardware:
 - You can also rebuild QEMU with a patch that makes LD_PRELOAD unnecessary to load vitastor driver.
   See `qemu-*.*-vitastor.patch`.
 - Install fio 3.7 or later, get its source and symlink it into `<vitastor>/fio`.
-- Build Vitastor with `make -j8`.
+- Build & install Vitastor with `mkdir build && cd build && cmake .. && make -j8 && make install`.
-- Run `make install` (optionally with `LIBDIR=/usr/lib64 QEMU_PLUGINDIR=/usr/lib64/qemu-kvm`
+  Pay attention to the `QEMU_PLUGINDIR` cmake option - it must be set to `qemu-kvm` on RHEL.
-  if you're using an RPM-based distro).
 
 ## Running
 
@@ -349,21 +348,16 @@ and calculate disk offsets almost by hand. This will be fixed in near future.
   with lazy fsync, but prepare for inferior single-thread latency.
 - Get a fast network (at least 10 Gbit/s).
 - Disable CPU powersaving: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
-- Start etcd with `--max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision` options.
+- Check `/usr/lib/vitastor/mon/make-units.sh` and `/usr/lib/vitastor/mon/make-osd.sh` and
-- Create global configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`
+  put desired values into the variables at the top of these files.
-  (if all your drives have capacitors).
+- Create systemd units for the monitor and etcd: `/usr/lib/vitastor/mon/make-units.sh`
-- Create pool configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":256,"failure_domain":"host"}}'`.
+- Create systemd units for your OSDs: `/usr/lib/vitastor/mon/make-osd.sh /dev/disk/by-partuuid/XXX [/dev/disk/by-partuuid/YYY ...]`
-  For jerasure pools the configuration should look like the following: `2:{"name":"ecpool","scheme":"jerasure","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}`.
+- You can edit the units and change OSD configuration. Notable configuration variables:
-- Calculate offsets for your drives with `node /usr/lib/vitastor/mon/simple-offsets.js --device /dev/sdX`.
-- Make systemd units for your OSDs. Look at `/usr/lib/vitastor/mon/make-units.sh` for example.
-  Notable configuration variables from the example:
   - `disable_data_fsync 1` - only safe with server-grade drives with capacitors.
   - `immediate_commit all` - use this if all your drives are server-grade.
   - `disable_device_lock 1` - only required if you run multiple OSDs on one block device.
-  - `flusher_count 16` - flusher is a micro-thread that removes old data from the journal.
+  - `flusher_count 256` - flusher is a micro-thread that removes old data from the journal.
-    More flushers mean more aggressive journal flushing which allows for more throughput
+    You don't have to worry about this parameter anymore, 256 is enough.
-    but slightly hurts latency under less load. Flushing will probably be improved in the future
-    because currently high queue depths sometimes lead to performance degradation.
   - `disk_alignment`, `journal_block_size`, `meta_block_size` should be set to the internal
     block size of your SSDs which is 4096 on most drives.
   - `journal_no_same_sector_overwrites true` prevents multiple overwrites of the same journal sector.
@@ -374,18 +368,22 @@ and calculate disk offsets almost by hand. This will be fixed in near future.
     setting is set, it is also required to raise `journal_sector_buffer_count` setting, which is the
     number of dirty journal sectors that may be written to at the same time.
 - `systemctl start vitastor.target` everywhere.
-- Start any number of monitors: `node /usr/lib/vitastor/mon/mon-main.js --etcd_url 'http://10.115.0.10:2379,http://10.115.0.11:2379,http://10.115.0.12:2379,http://10.115.0.13:2379' --etcd_prefix '/vitastor' --etcd_start_timeout 5`.
+- Create global configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`
+  (if all your drives have capacitors).
+- Create pool configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":256,"failure_domain":"host"}}'`.
+  For jerasure pools the configuration should look like the following: `2:{"name":"ecpool","scheme":"jerasure","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}`.
 - At this point, one of the monitors will configure PGs and OSDs will start them.
 - You can check PG states with `etcdctl --endpoints=... get --prefix /vitastor/pg/state`. All PGs should become 'active'.
-- Run tests with (for example): `fio -thread -ioengine=/usr/lib/x86_64-linux-gnu/vitastor/libfio_cluster.so -name=test -bs=4M -direct=1 -iodepth=16 -rw=write -etcd=10.115.0.10:2379/v3 -pool=1 -inode=1 -size=400G`.
+- Run tests with (for example): `fio -thread -ioengine=libfio_vitastor.so -name=test -bs=4M -direct=1 -iodepth=16 -rw=write -etcd=10.115.0.10:2379/v3 -pool=1 -inode=1 -size=400G`.
 - Upload VM disk image with qemu-img (for example):
   ```
-  LD_PRELOAD=/usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so qemu-img convert -f qcow2 debian10.qcow2 -p
+  qemu-img convert -f qcow2 debian10.qcow2 -p -O raw 'vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648'
-    -O raw 'vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648'
   ```
+  Note that the command requires to be run with `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so qemu-img ...`
+  if you use unmodified QEMU.
 - Run QEMU with (for example):
   ```
-  LD_PRELOAD=/usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so qemu-system-x86_64 -enable-kvm -m 1024
+  qemu-system-x86_64 -enable-kvm -m 1024
     -drive 'file=vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648',format=raw,if=none,id=drive-virtio-disk0,cache=none
     -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1,write-cache=off,physical_block_size=4096,logical_block_size=512
     -vnc 0.0.0.0:0
@@ -421,22 +419,27 @@ Copyright (c) Vitaliy Filippov (vitalif [at] yourcmc.ru), 2019+
 You can also find me in the Russian Telegram Ceph chat: https://t.me/ceph_ru
 
 All server-side code (OSD, Monitor and so on) is licensed under the terms of
-Vitastor Network Public License 1.0 (VNPL 1.0), a copyleft license based on
+Vitastor Network Public License 1.1 (VNPL 1.1), a copyleft license based on
 GNU GPLv3.0 with the additional "Network Interaction" clause which requires
 opensourcing all programs directly or indirectly interacting with Vitastor
-through a computer network ("Proxy Programs"). Proxy Programs may be made public
+through a computer network and expressly designed to be used in conjunction
-not only under the terms of the same license, but also under the terms of any
+with it ("Proxy Programs"). Proxy Programs may be made public not only under
-GPL-Compatible Free Software License, as listed by the Free Software Foundation.
+the terms of the same license, but also under the terms of any GPL-Compatible
+Free Software License, as listed by the Free Software Foundation.
 This is a stricter copyleft license than the Affero GPL.
 
+Please note that VNPL doesn't require you to open the code of proprietary
+software running inside a VM if it's not specially designed to be used with
+Vitastor.
+
 Basically, you can't use the software in a proprietary environment to provide
 its functionality to users without opensourcing all intermediary components
 standing between the user and Vitastor or purchasing a commercial license
 from the author 😀.
 
 Client libraries (cluster_client and so on) are dual-licensed under the same
-VNPL 1.0 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
+VNPL 1.1 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
 software like QEMU and fio.
 
-You can find the full text of VNPL-1.0 in the file [VNPL-1.0.txt](VNPL-1.0.txt).
+You can find the full text of VNPL-1.1 in the file [VNPL-1.1.txt](VNPL-1.1.txt).
 GPL 2.0 is also included in this repository as [GPL-2.0.txt](GPL-2.0.txt).

VNPL-1.1.txt

@@ -1,7 +1,7 @@
                      VITASTOR NETWORK PUBLIC LICENSE
-                       Version 1, 17 September 2020
+                     Version 1.1,  6 February 2021
 
- Copyright (C) 2020 Vitaliy Filippov <vitalif@yourcmc.ru>
+ Copyright (C) 2021 Vitaliy Filippov <vitalif@yourcmc.ru>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -540,12 +540,15 @@ License would be to refrain entirely from conveying the Program.
 
   13. Remote Network Interaction.
 
-  Notwithstanding any other provision of this License, if you provide
+  A "Proxy Program" means a separate program which is specially designed to
-any user an opportunity to interact with the covered work directly
+be used in conjunction with the covered work and interacts with it directly
-or indirectly through a computer network, an imitation of such network,
+or indirectly through any kind of API (application programming interfaces),
-or an additional program (hereinafter referred to as a "Proxy Program")
+a computer network, an imitation of such network, or another Proxy Program
-that, in turn, interacts with the covered work through a computer network,
+itself.
-an imitation of such network, or another Proxy Program itself,
+
+  Notwithstanding any other provision of this License, if you provide any user
+with an opportunity to interact with the covered work through a computer
+network, an imitation of such network, or any number of "Proxy Programs",
 you must prominently offer that user an opportunity to receive the
 Corresponding Source of the covered work and all Proxy Programs from a
 network server at no charge, through some standard or customary means of

copy-fio-includes.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-gcc -E -o fio_headers.i fio_headers.h
+gcc -I. -E -o fio_headers.i src/fio_headers.h
 
 rm -rf fio-copy
 for i in `grep -Po 'fio/[^"]+' fio_headers.i | sort | uniq`; do

copy-qemu-includes.sh

@@ -5,7 +5,7 @@
 #cd b/qemu; make qapi
 
 gcc -I qemu/b/qemu `pkg-config glib-2.0 --cflags` \
-    -I qemu/include -E -o qemu_driver.i qemu_driver.c
+    -I qemu/include -E -o qemu_driver.i src/qemu_driver.c
 
 rm -rf qemu-copy
 for i in `grep -Po 'qemu/[^"]+' qemu_driver.i | sort | uniq`; do

debian/build-vitastor-bullseye.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+sed 's/$REL/bullseye/' < vitastor.Dockerfile > ../Dockerfile
+cd ..
+mkdir -p packages
+sudo podman build -v `pwd`/packages:/root/packages -f Dockerfile .
+rm Dockerfile

debian/build-vitastor-buster.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+sed 's/$REL/buster/' < vitastor.Dockerfile > ../Dockerfile
+cd ..
+mkdir -p packages
+sudo podman build -v `pwd`/packages:/root/packages -f Dockerfile .
+rm Dockerfile

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (0.5.3-1) unstable; urgency=medium
+vitastor (0.5.6-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/copyright

@@ -5,16 +5,17 @@ Source: https://vitastor.io
 
 Files: *
 Copyright: 2019+ Vitaliy Filippov <vitalif@yourcmc.ru>
-License: Multiple licenses VNPL-1.0 and/or GPL-2.0+
+License: Multiple licenses VNPL-1.1 and/or GPL-2.0+
  All server-side code (OSD, Monitor and so on) is licensed under the terms of
- Vitastor Network Public License 1.0 (VNPL 1.0), a copyleft license based on
+ Vitastor Network Public License 1.1 (VNPL 1.1), a copyleft license based on
  GNU GPLv3.0 with the additional "Network Interaction" clause which requires
  opensourcing all programs directly or indirectly interacting with Vitastor
- through a computer network ("Proxy Programs"). Proxy Programs may be made public
+ through a computer network and expressly designed to be used in conjunction
- not only under the terms of the same license, but also under the terms of any
+ with it ("Proxy Programs"). Proxy Programs may be made public not only under
- GPL-Compatible Free Software License, as listed by the Free Software Foundation.
+ the terms of the same license, but also under the terms of any GPL-Compatible
+ Free Software License, as listed by the Free Software Foundation.
  This is a stricter copyleft license than the Affero GPL.
  .
  Client libraries (cluster_client and so on) are dual-licensed under the same
- VNPL 1.0 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
+ VNPL 1.1 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
  software like QEMU and fio.

debian/install

@@ -1,3 +1,3 @@
-VNPL-1.0.txt usr/share/doc/vitastor
+VNPL-1.1.txt usr/share/doc/vitastor
 GPL-2.0.txt usr/share/doc/vitastor
 mon usr/lib/vitastor

debian/patched-qemu.Dockerfile

@@ -1,13 +1,8 @@
 # Build patched QEMU for Debian Buster or Bullseye/Sid inside a container
-# cd ..; podman build --build-arg REL=bullseye -v `pwd`/build:/root/build -f debian/patched-qemu.Dockerfile .
+# cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/patched-qemu.Dockerfile .
-
-ARG REL=bullseye
 
 FROM debian:$REL
 
-# again, it doesn't work otherwise
-ARG REL=bullseye
-
 WORKDIR /root
 
 RUN if [ "$REL" = "buster" ]; then \
@@ -30,20 +25,20 @@ RUN apt-get --download-only source fio
 
 ADD qemu-5.0-vitastor.patch qemu-5.1-vitastor.patch /root/vitastor/
 RUN set -e; \
-    mkdir -p /root/build/qemu-$REL; \
+    mkdir -p /root/packages/qemu-$REL; \
-    rm -rf /root/build/qemu-$REL/*; \
+    rm -rf /root/packages/qemu-$REL/*; \
-    cd /root/build/qemu-$REL; \
+    cd /root/packages/qemu-$REL; \
     dpkg-source -x /root/qemu*.dsc; \
-    if [ -d /root/build/qemu-$REL/qemu-5.0 ]; then \
+    if [ -d /root/packages/qemu-$REL/qemu-5.0 ]; then \
-        cp /root/vitastor/qemu-5.0-vitastor.patch /root/build/qemu-$REL/qemu-5.0/debian/patches; \
+        cp /root/vitastor/qemu-5.0-vitastor.patch /root/packages/qemu-$REL/qemu-5.0/debian/patches; \
-        echo qemu-5.0-vitastor.patch >> /root/build/qemu-$REL/qemu-5.0/debian/patches/series; \
+        echo qemu-5.0-vitastor.patch >> /root/packages/qemu-$REL/qemu-5.0/debian/patches/series; \
     else \
-        cp /root/vitastor/qemu-5.1-vitastor.patch /root/build/qemu-$REL/qemu-*/debian/patches; \
+        cp /root/vitastor/qemu-5.1-vitastor.patch /root/packages/qemu-$REL/qemu-*/debian/patches; \
-        P=`ls -d /root/build/qemu-$REL/qemu-*/debian/patches`; \
+        P=`ls -d /root/packages/qemu-$REL/qemu-*/debian/patches`; \
         echo qemu-5.1-vitastor.patch >> $P/series; \
     fi; \
-    cd /root/build/qemu-$REL/qemu-*/; \
+    cd /root/packages/qemu-$REL/qemu-*/; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)(~bpo[\d\+]*)?\).*$/$1/')+vitastor1; \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v $V 'Plug Vitastor block driver'; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
-    rm -rf /root/build/qemu-$REL/qemu-*/
+    rm -rf /root/packages/qemu-$REL/qemu-*/

debian/vitastor.Dockerfile

@@ -1,13 +1,8 @@
 # Build Vitastor packages for Debian Buster or Bullseye/Sid inside a container
-# cd ..; podman build --build-arg REL=bullseye -v `pwd`/build:/root/build -f debian/vitastor.Dockerfile .
+# cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/vitastor.Dockerfile .
-
-ARG REL=bullseye
 
 FROM debian:$REL
 
-# again, it doesn't work otherwise
-ARG REL=bullseye
-
 WORKDIR /root
 
 RUN if [ "$REL" = "buster" ]; then \
@@ -27,7 +22,7 @@ RUN apt-get -y build-dep qemu
 RUN apt-get -y build-dep fio
 RUN apt-get --download-only source qemu
 RUN apt-get --download-only source fio
-RUN apt-get -y install libjerasure-dev
+RUN apt-get -y install libjerasure-dev cmake
 
 ADD . /root/vitastor
 RUN set -e -x; \
@@ -35,20 +30,20 @@ RUN set -e -x; \
     cd /root/fio-build/; \
     rm -rf /root/fio-build/*; \
     dpkg-source -x /root/fio*.dsc; \
-    cd /root/build/qemu-$REL/; \
+    cd /root/packages/qemu-$REL/; \
     rm -rf qemu*/; \
     dpkg-source -x qemu*.dsc; \
-    cd /root/build/qemu-$REL/qemu*/; \
+    cd /root/packages/qemu-$REL/qemu*/; \
     debian/rules b/configure-stamp; \
     cd b/qemu; \
-    make -j8 qapi; \
+    make -j8 qapi/qapi-builtin-types.h; \
-    mkdir -p /root/build/vitastor-$REL; \
+    mkdir -p /root/packages/vitastor-$REL; \
-    rm -rf /root/build/vitastor-$REL/*; \
+    rm -rf /root/packages/vitastor-$REL/*; \
-    cd /root/build/vitastor-$REL; \
+    cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.5.3; \
+    cp -r /root/vitastor vitastor-0.5.6; \
-    ln -s /root/build/qemu-$REL/qemu-*/ vitastor-0.5.3/qemu; \
+    ln -s /root/packages/qemu-$REL/qemu-*/ vitastor-0.5.6/qemu; \
-    ln -s /root/fio-build/fio-*/ vitastor-0.5.3/fio; \
+    ln -s /root/fio-build/fio-*/ vitastor-0.5.6/fio; \
-    cd vitastor-0.5.3; \
+    cd vitastor-0.5.6; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     QEMU=$(head -n1 qemu/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     sh copy-qemu-includes.sh; \
@@ -60,13 +55,13 @@ RUN set -e -x; \
     diff -NaurpbB a b > debian/patches/qemu-fio-headers.patch || true; \
     echo qemu-fio-headers.patch >> debian/patches/series; \
     rm -rf a b; \
-    rm -rf /root/build/qemu-$REL/qemu*/; \
+    rm -rf /root/packages/qemu-$REL/qemu*/; \
     echo "dep:fio=$FIO" > debian/substvars; \
     echo "dep:qemu=$QEMU" >> debian/substvars; \
-    cd /root/build/vitastor-$REL; \
+    cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.5.3.orig.tar.xz vitastor-0.5.3; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.5.6.orig.tar.xz vitastor-0.5.6; \
-    cd vitastor-0.5.3; \
+    cd vitastor-0.5.6; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
-    rm -rf /root/build/vitastor-$REL/vitastor-*/
+    rm -rf /root/packages/vitastor-$REL/vitastor-*/

lambda_size.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <iostream>
 #include <functional>

mon/PGUtil.js

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 module.exports = {
     scale_pg_count,
@@ -8,7 +8,7 @@ module.exports = {
 function scale_pg_count(prev_pgs, prev_pg_history, new_pg_history, new_pg_count)
 {
     const old_pg_count = prev_pgs.length;
-    // Add all possibly intersecting PGs into the history of new PGs
+    // Add all possibly intersecting PGs to the history of new PGs
     if (!(new_pg_count % old_pg_count))
     {
         // New PG count is a multiple of the old PG count
@@ -16,7 +16,7 @@ function scale_pg_count(prev_pgs, prev_pg_history, new_pg_history, new_pg_count)
         for (let i = 0; i < new_pg_count; i++)
         {
             const old_i = Math.floor(new_pg_count / mul);
-            new_pg_history[i] = JSON.parse(JSON.stringify(prev_pg_history[1+old_i]));
+            new_pg_history[i] = prev_pg_history[old_i] ? JSON.parse(JSON.stringify(prev_pg_history[old_i])) : undefined;
         }
     }
     else if (!(old_pg_count % new_pg_count))

mon/afr.js

@@ -1,31 +1,16 @@
 // Functions to calculate Annualized Failure Rate of your cluster
 // if you know AFR of your drives, number of drives, expected rebalance time
 // and replication factor
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see https://yourcmc.ru/git/vitalif/vitastor/src/branch/master/README.md for details) or AGPL-3.0
-
+// Author: Vitaliy Filippov, 2020+
-const { sprintf } = require('sprintf-js');
 
 module.exports = {
     cluster_afr_fullmesh,
     failure_rate_fullmesh,
     cluster_afr,
-    print_cluster_afr,
     c_n_k,
 };
 
-print_cluster_afr({ n_hosts: 4, n_drives: 6, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
-print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, capacity: 4000, speed: 0.1, replicas: 2 });
-print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
-print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, capacity: 4000, speed: 0.1, ec: [ 2, 1 ] });
-print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, ec: [ 2, 1 ] });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, capacity: 8000, speed: 0.02, replicas: 2 });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 2 });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, capacity: 8000, speed: 0.02, replicas: 3 });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3 });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100, degraded_replacement: 1 });
-
 /******** "FULL MESH": ASSUME EACH OSD COMMUNICATES WITH ALL OTHER OSDS ********/
 
 // Estimate AFR of the cluster
@@ -56,93 +41,38 @@ function failure_rate_fullmesh(n, a, f)
 /******** PGS: EACH OSD ONLY COMMUNICATES WITH <pgs> OTHER OSDs ********/
 
 // <n> hosts of <m> drives of <capacity> GB, each able to backfill at <speed> GB/s,
-// <k> replicas, <pgs> unique peer PGs per OSD
+// <k> replicas, <pgs> unique peer PGs per OSD (~50 for 100 PG-per-OSD in a big cluster)
 //
 // For each of n*m drives: P(drive fails in a year) * P(any of its peers fail in <l*365> next days).
 // More peers per OSD increase rebalance speed (more drives work together to resilver) if you
-// let them finish rebalance BEFORE replacing the failed drive.
+// let them finish rebalance BEFORE replacing the failed drive (degraded_replacement=false).
 // At the same time, more peers per OSD increase probability of any of them to fail!
+// osd_rm=true means that failed OSDs' data is rebalanced over all other hosts,
+// not over the same host as it's in Ceph by default (dead OSDs are marked 'out').
 //
 // Probability of all except one drives in a replica group to fail is (AFR^(k-1)).
 // So with <x> PGs it becomes ~ (x * (AFR*L/365)^(k-1)). Interesting but reasonable consequence
 // is that, with k=2, total failure rate doesn't depend on number of peers per OSD,
 // because it gets increased linearly by increased number of peers to fail
 // and decreased linearly by reduced rebalance time.
-function cluster_afr_pgs({ n_hosts, n_drives, afr_drive, capacity, speed, replicas, pgs = 1, degraded_replacement })
+function cluster_afr({ n_hosts, n_drives, afr_drive, afr_host, capacity, speed, ec, ec_data, ec_parity, replicas, pgs = 1, osd_rm, degraded_replacement, down_out_interval = 600 })
 {
-    pgs = Math.min(pgs, (n_hosts-1)*n_drives/(replicas-1));
+    const pg_size = (ec ? ec_data+ec_parity : replicas);
-    const l = capacity/(degraded_replacement ? 1 : pgs)/speed/86400/365;
+    pgs = Math.min(pgs, (n_hosts-1)*n_drives/(pg_size-1));
-    return 1 - (1 - afr_drive * (1-(1-(afr_drive*l)**(replicas-1))**pgs)) ** (n_hosts*n_drives);
+    const host_pgs = Math.min(pgs*n_drives, (n_hosts-1)*n_drives/(pg_size-1));
-}
+    const resilver_disk = n_drives == 1 || osd_rm ? pgs : (n_drives-1);
-
+    const disk_heal_time = (down_out_interval + capacity/(degraded_replacement ? 1 : resilver_disk)/speed)/86400/365;
-function cluster_afr_pgs_ec({ n_hosts, n_drives, afr_drive, capacity, speed, ec: [ ec_data, ec_parity ], pgs = 1, degraded_replacement })
+    const host_heal_time = (down_out_interval + n_drives*capacity/pgs/speed)/86400/365;
-{
+    const disk_heal_fail = ((afr_drive+afr_host/n_drives)*disk_heal_time);
-    const ec_total = ec_data+ec_parity;
+    const host_heal_fail = ((afr_drive+afr_host/n_drives)*host_heal_time);
-    pgs = Math.min(pgs, (n_hosts-1)*n_drives/(ec_total-1));
+    const disk_pg_fail = ec
-    const l = capacity/(degraded_replacement ? 1 : pgs)/speed/86400/365;
+        ? failure_rate_fullmesh(ec_data+ec_parity-1, disk_heal_fail, ec_parity)
-    return 1 - (1 - afr_drive * (1-(1-failure_rate_fullmesh(ec_total-1, afr_drive*l, ec_parity))**pgs)) ** (n_hosts*n_drives);
+        : disk_heal_fail**(replicas-1);
-}
+    const host_pg_fail = ec
-
+        ? failure_rate_fullmesh(ec_data+ec_parity-1, host_heal_fail, ec_parity)
-// Same as above, but also take server failures into account
+        : host_heal_fail**(replicas-1);
-function cluster_afr_pgs_hosts({ n_hosts, n_drives, afr_drive, afr_host, capacity, speed, replicas, pgs = 1, degraded_replacement })
+    return 1 - ((1 - afr_drive * (1-(1-disk_pg_fail)**pgs)) ** (n_hosts*n_drives))
-{
+        * ((1 - afr_host * (1-(1-host_pg_fail)**host_pgs)) ** n_hosts);
-    let otherhosts = Math.min(pgs, (n_hosts-1)/(replicas-1));
-    pgs = Math.min(pgs, (n_hosts-1)*n_drives/(replicas-1));
-    let pgh = Math.min(pgs*n_drives, (n_hosts-1)*n_drives/(replicas-1));
-    const ld = capacity/(degraded_replacement ? 1 : pgs)/speed/86400/365;
-    const lh = n_drives*capacity/pgs/speed/86400/365;
-    const p1 = ((afr_drive+afr_host*pgs/otherhosts)*lh);
-    const p2 = ((afr_drive+afr_host*pgs/otherhosts)*ld);
-    return 1 - ((1 - afr_host * (1-(1-p1**(replicas-1))**pgh)) ** n_hosts) *
-        ((1 - afr_drive * (1-(1-p2**(replicas-1))**pgs)) ** (n_hosts*n_drives));
-}
-
-function cluster_afr_pgs_ec_hosts({ n_hosts, n_drives, afr_drive, afr_host, capacity, speed, ec: [ ec_data, ec_parity ], pgs = 1, degraded_replacement })
-{
-    const ec_total = ec_data+ec_parity;
-    const otherhosts = Math.min(pgs, (n_hosts-1)/(ec_total-1));
-    pgs = Math.min(pgs, (n_hosts-1)*n_drives/(ec_total-1));
-    const pgh = Math.min(pgs*n_drives, (n_hosts-1)*n_drives/(ec_total-1));
-    const ld = capacity/(degraded_replacement ? 1 : pgs)/speed/86400/365;
-    const lh = n_drives*capacity/pgs/speed/86400/365;
-    const p1 = ((afr_drive+afr_host*pgs/otherhosts)*lh);
-    const p2 = ((afr_drive+afr_host*pgs/otherhosts)*ld);
-    return 1 - ((1 - afr_host * (1-(1-failure_rate_fullmesh(ec_total-1, p1, ec_parity))**pgh)) ** n_hosts) *
-        ((1 - afr_drive * (1-(1-failure_rate_fullmesh(ec_total-1, p2, ec_parity))**pgs)) ** (n_hosts*n_drives));
-}
-
-// Wrapper for 4 above functions
-function cluster_afr(config)
-{
-    if (config.ec && config.afr_host)
-    {
-        return cluster_afr_pgs_ec_hosts(config);
-    }
-    else if (config.ec)
-    {
-        return cluster_afr_pgs_ec(config);
-    }
-    else if (config.afr_host)
-    {
-        return cluster_afr_pgs_hosts(config);
-    }
-    else
-    {
-        return cluster_afr_pgs(config);
-    }
-}
-
-function print_cluster_afr(config)
-{
-    console.log(
-        `${config.n_hosts} nodes with ${config.n_drives} ${sprintf("%.1f", config.capacity/1000)}TB drives`+
-        `, capable to backfill at ${sprintf("%.1f", config.speed*1000)} MB/s, drive AFR ${sprintf("%.1f", config.afr_drive*100)}%`+
-        (config.afr_host ? `, host AFR ${sprintf("%.1f", config.afr_host*100)}%` : '')+
-        (config.ec ? `, EC ${config.ec[0]}+${config.ec[1]}` : `, ${config.replicas} replicas`)+
-        `, ${config.pgs||1} PG per OSD`+
-        (config.degraded_replacement ? `\n...and you don't let the rebalance finish before replacing drives` : '')
-    );
-    console.log('-> '+sprintf("%.7f%%", 100*cluster_afr(config))+'\n');
 }
 
 /******** UTILITY ********/

mon/afr_test.js

@@ -0,0 +1,28 @@
+const { sprintf } = require('sprintf-js');
+const { cluster_afr } = require('./afr.js');
+
+print_cluster_afr({ n_hosts: 4, n_drives: 6, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
+print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0, capacity: 4000, speed: 0.1, replicas: 2 });
+print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
+print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0, capacity: 4000, speed: 0.1, ec: true, ec_data: 2, ec_parity: 1 });
+print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, ec: true, ec_data: 2, ec_parity: 1 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0, capacity: 8000, speed: 0.02, replicas: 2 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 2 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0, capacity: 8000, speed: 0.02, replicas: 3 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100, degraded_replacement: 1 });
+
+function print_cluster_afr(config)
+{
+    console.log(
+        `${config.n_hosts} nodes with ${config.n_drives} ${sprintf("%.1f", config.capacity/1000)}TB drives`+
+        `, capable to backfill at ${sprintf("%.1f", config.speed*1000)} MB/s, drive AFR ${sprintf("%.1f", config.afr_drive*100)}%`+
+        (config.afr_host ? `, host AFR ${sprintf("%.1f", config.afr_host*100)}%` : '')+
+        (config.ec ? `, EC ${config.ec_data}+${config.ec_parity}` : `, ${config.replicas} replicas`)+
+        `, ${config.pgs||1} PG per OSD`+
+        (config.degraded_replacement ? `\n...and you don't let the rebalance finish before replacing drives` : '')
+    );
+    console.log('-> '+sprintf("%.7f%%", 100*cluster_afr(config))+'\n');
+}

mon/lp-optimizer.js

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 // Data distribution optimizer using linear programming (lp_solve)
 
@@ -58,7 +58,7 @@ async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize =
     }
     const all_weights = Object.assign({}, ...Object.values(osd_tree));
     const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c), 0);
-    const all_pgs = Object.values(random_combinations(osd_tree, pg_size, max_combinations));
+    const all_pgs = Object.values(random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1));
     const pg_per_osd = {};
     for (const pg of all_pgs)
     {
@@ -249,7 +249,7 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
         }
     }
     // Get all combinations
-    let all_pgs = random_combinations(osd_tree, pg_size, max_combinations);
+    let all_pgs = random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1);
     add_valid_previous(osd_tree, prev_weights, all_pgs);
     all_pgs = Object.values(all_pgs);
     const pg_per_osd = {};
@@ -275,6 +275,11 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
     lp += 'max: '+all_pg_names.map(pg_name => (
         prev_weights[pg_name] ? `${pg_size+1}*add_${pg_name} - ${pg_size+1}*del_${pg_name}` : `${pg_size+1-move_weights[pg_name]}*${pg_name}`
     )).join(' + ')+';\n';
+    lp += all_pg_names
+        .map(pg_name => (prev_weights[pg_name] ? `add_${pg_name} - del_${pg_name}` : `${pg_name}`))
+        .join(' + ')+' = '+(pg_count
+            - Object.keys(prev_weights).reduce((a, old_pg_name) => (a + (all_pgs_hash[old_pg_name] ? prev_weights[old_pg_name] : 0)), 0)
+        )+';\n';
     for (const osd in pg_per_osd)
     {
         if (osd !== NO_OSD)
@@ -488,7 +493,8 @@ function extract_osds(osd_tree, levels, osd_level, osds = {})
     return osds;
 }
 
-function random_combinations(osd_tree, pg_size, count)
+// ordered = don't treat (x,y) and (y,x) as equal
+function random_combinations(osd_tree, pg_size, count, ordered)
 {
     let seed = 0x5f020e43;
     let rng = () =>
@@ -516,25 +522,47 @@ function random_combinations(osd_tree, pg_size, count)
                 pg.push(osds[cur_hosts[next_host]][next_osd]);
                 cur_hosts.splice(next_host, 1);
             }
-            while (pg.length < pg_size)
+            const cyclic_pgs = [ pg ];
+            if (ordered)
             {
-                pg.push(NO_OSD);
+                for (let i = 1; i < pg.size; i++)
+                {
+                    cyclic_pgs.push([ ...pg.slice(i), ...pg.slice(0, i) ]);
+                }
+            }
+            for (const pg of cyclic_pgs)
+            {
+                while (pg.length < pg_size)
+                {
+                    pg.push(NO_OSD);
+                }
+                r['pg_'+pg.join('_')] = pg;
             }
-            r['pg_'+pg.join('_')] = pg;
         }
     }
     // Generate purely random combinations
-    restart: while (count > 0)
+    while (count > 0)
     {
         let host_idx = [];
-        for (let i = 0; i < pg_size && i < hosts.length; i++)
+        const cur_hosts = [ ...hosts.map((h, i) => i) ];
+        const max_hosts = pg_size < hosts.length ? pg_size : hosts.length;
+        if (ordered)
         {
-            let start = i > 0 ? host_idx[i-1]+1 : 0;
+            for (let i = 0; i < max_hosts; i++)
-            if (start >= hosts.length)
             {
-                continue restart;
+                const r = rng() % cur_hosts.length;
+                host_idx[i] = cur_hosts[r];
+                cur_hosts.splice(r, 1);
+            }
+        }
+        else
+        {
+            for (let i = 0; i < max_hosts; i++)
+            {
+                const r = rng() % (cur_hosts.length - (max_hosts - i - 1));
+                host_idx[i] = cur_hosts[r];
+                cur_hosts.splice(0, r+1);
             }
-            host_idx[i] = start + rng() % (hosts.length-start);
         }
         let pg = host_idx.map(h => osds[hosts[h]][rng() % osds[hosts[h]].length]);
         while (pg.length < pg_size)

mon/make-osd.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Very simple systemd unit generator for vitastor-osd services
+# Not the final solution yet, mostly for tests
+# Copyright (c) Vitaliy Filippov, 2019+
+# License: MIT
+
+# USAGE: ./make-osd.sh /dev/disk/by-partuuid/xxx [ /dev/disk/by-partuuid/yyy]...
+
+IP_SUBSTR="10.200.1."
+ETCD_HOSTS="etcd0=http://10.200.1.10:2380,etcd1=http://10.200.1.11:2380,etcd2=http://10.200.1.12:2380"
+
+set -e -x
+
+IP=`ip -json a s | jq -r '.[].addr_info[] | select(.local | startswith("'$IP_SUBSTR'")) | .local'`
+[ "$IP" != "" ] || exit 1
+ETCD_MON=$(echo $ETCD_HOSTS | perl -pe 's/:2380/:2379/g; s/etcd\d*=//g;')
+D=`dirname $0`
+
+# Create OSDs on all passed devices
+OSD_NUM=1
+for DEV in $*; do
+
+# Ugly :) -> node.js rework pending
+while true; do
+    ST=$(etcdctl --endpoints="$ETCD_MON" get --print-value-only /vitastor/osd/stats/$OSD_NUM)
+    if [ "$ST" = "" ]; then
+        break
+    fi
+    OSD_NUM=$((OSD_NUM+1))
+done
+etcdctl --endpoints="$ETCD_MON" put /vitastor/osd/stats/$OSD_NUM '{}'
+
+echo Creating OSD $OSD_NUM on $DEV
+
+OPT=`node $D/simple-offsets.js --device $DEV --format options | tr '\n' ' '`
+META=`echo $OPT | grep -Po '(?<=data_offset )\d+'`
+dd if=/dev/zero of=$DEV bs=1048576 count=$(((META+1048575)/1048576)) oflag=direct
+
+cat >/etc/systemd/system/vitastor-osd$OSD_NUM.service <<EOF
+[Unit]
+Description=Vitastor object storage daemon osd.$OSD_NUM
+After=network-online.target local-fs.target time-sync.target
+Wants=network-online.target local-fs.target time-sync.target
+PartOf=vitastor.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitMEMLOCK=infinity
+ExecStart=/usr/bin/vitastor-osd \\
+    --etcd_address $IP:2379/v3 \\
+    --bind_address $IP \\
+    --osd_num $OSD_NUM \\
+    --disable_data_fsync 1 \\
+    --immediate_commit all \\
+    --flusher_count 256 \\
+    --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
+    --journal_no_same_sector_overwrites true \\
+    --journal_sector_buffer_count 1024 \\
+    $OPT
+WorkingDirectory=/
+ExecStartPre=+chown vitastor:vitastor $DEV
+User=vitastor
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=vitastor.target
+EOF
+
+systemctl enable vitastor-osd$OSD_NUM
+
+done

mon/make-units.sh

@@ -1,19 +1,25 @@
 #!/bin/bash
-# Example startup script generator
+# Very simple systemd unit generator for etcd & vitastor-mon services
-# Of course this isn't a production solution yet, this is just for tests
+# Not the final solution yet, mostly for tests
 # Copyright (c) Vitaliy Filippov, 2019+
 # License: MIT
 
-IP=`ip -json a s | jq -r '.[].addr_info[] | select(.broadcast == "10.115.0.255") | .local'`
+# USAGE: ./make-units.sh
 
+IP_SUBSTR="10.200.1."
+ETCD_HOSTS="etcd0=http://10.200.1.10:2380,etcd1=http://10.200.1.11:2380,etcd2=http://10.200.1.12:2380"
+
+# determine IP
+IP=`ip -json a s | jq -r '.[].addr_info[] | select(.local | startswith("'$IP_SUBSTR'")) | .local'`
 [ "$IP" != "" ] || exit 1
+ETCD_NUM=${ETCD_HOSTS/$IP*/}
+[ "$ETCD_NUM" != "$ETCD_HOSTS" ] || exit 1
+ETCD_NUM=$(echo $ETCD_NUM | tr -d -c , | wc -c)
 
-BASE=${IP/*./}
+# etcd
-BASE=$((BASE-10))
-
 useradd etcd
 
-mkdir -p /var/lib/etcd$BASE.etcd
+mkdir -p /var/lib/etcd$ETCD_NUM.etcd
 cat >/etc/systemd/system/etcd.service <<EOF
 [Unit]
 Description=etcd for vitastor
@@ -22,19 +28,18 @@ Wants=network-online.target local-fs.target time-sync.target
 
 [Service]
 Restart=always
-ExecStart=/usr/local/bin/etcd -name etcd$BASE --data-dir /var/lib/etcd$BASE.etcd \\
+ExecStart=/usr/local/bin/etcd -name etcd$ETCD_NUM --data-dir /var/lib/etcd$ETCD_NUM.etcd \\
     --advertise-client-urls http://$IP:2379 --listen-client-urls http://$IP:2379 \\
     --initial-advertise-peer-urls http://$IP:2380 --listen-peer-urls http://$IP:2380 \\
-    --initial-cluster-token vitastor-etcd-1 --initial-cluster etcd0=http://10.115.0.10:2380,etcd1=http://10.115.0.11:2380,etcd2=http://10.115.0.12:2380,etcd3=http://10.115.0.13:2380 \\
+    --initial-cluster-token vitastor-etcd-1 --initial-cluster $ETCD_HOSTS \\
     --initial-cluster-state new --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision
-WorkingDirectory=/var/lib/etcd$BASE.etcd
+WorkingDirectory=/var/lib/etcd$ETCD_NUM.etcd
-ExecStartPre=+chown -R etcd /var/lib/etcd$BASE.etcd
+ExecStartPre=+chown -R etcd /var/lib/etcd$ETCD_NUM.etcd
 User=etcd
 PrivateTmp=false
 TasksMax=infinity
 Restart=always
 StartLimitInterval=0
-StartLimitIntervalSec=0
 RestartSec=10
 
 [Install]
@@ -48,9 +53,7 @@ systemctl start etcd
 useradd vitastor
 chmod 755 /root
 
-BASE=${IP/*./}
+# Vitastor target
-BASE=$(((BASE-10)*12))
-
 cat >/etc/systemd/system/vitastor.target <<EOF
 [Unit]
 Description=vitastor target
@@ -58,116 +61,25 @@ Description=vitastor target
 WantedBy=multi-user.target
 EOF
 
-i=1
+# Monitor unit
-for DEV in `ls /dev/disk/by-id/ | grep ata-INTEL_SSDSC2KB`; do
+ETCD_MON=$(echo $ETCD_HOSTS | perl -pe 's/:2380/:2379/g; s/etcd\d*=//g;')
-    dd if=/dev/zero of=/dev/disk/by-id/$DEV bs=1048576 count=$(((427814912+1048575)/1048576+2))
+cat >/etc/systemd/system/vitastor-mon.service <<EOF
-    dd if=/dev/zero of=/dev/disk/by-id/$DEV bs=1048576 count=$(((427814912+1048575)/1048576+2)) seek=$((1920377991168/1048576))
-cat >/etc/systemd/system/vitastor-osd$((BASE+i)).service <<EOF
 [Unit]
-Description=Vitastor object storage daemon osd.$((BASE+i))
+Description=Vitastor monitor
 After=network-online.target local-fs.target time-sync.target
 Wants=network-online.target local-fs.target time-sync.target
-PartOf=vitastor.target
 
 [Service]
-LimitNOFILE=1048576
+Restart=always
-LimitNPROC=1048576
+ExecStart=node /usr/lib/vitastor/mon/mon-main.js --etcd_url '$ETCD_MON' --etcd_prefix '/vitastor' --etcd_start_timeout 5
-LimitMEMLOCK=infinity
+WorkingDirectory=/
-ExecStart=/root/vitastor/osd \\
-    --etcd_address $IP:2379/v3 \\
-    --bind_address $IP \\
-    --osd_num $((BASE+i)) \\
-    --disable_data_fsync 1 \\
-    --disable_device_lock 1 \\
-    --immediate_commit all \\
-    --flusher_count 8 \\
-    --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
-    --journal_no_same_sector_overwrites true \\
-    --journal_sector_buffer_count 1024 \\
-    --journal_offset 0 \\
-    --meta_offset 16777216 \\
-    --data_offset 427814912 \\
-    --data_size $((1920377991168-427814912)) \\
-    --data_device /dev/disk/by-id/$DEV
-WorkingDirectory=/root/vitastor
-ExecStartPre=+chown vitastor:vitastor /dev/disk/by-id/$DEV
 User=vitastor
 PrivateTmp=false
 TasksMax=infinity
 Restart=always
 StartLimitInterval=0
-StartLimitIntervalSec=0
 RestartSec=10
 
 [Install]
 WantedBy=vitastor.target
 EOF
-    systemctl enable vitastor-osd$((BASE+i))
-    i=$((i+1))
-cat >/etc/systemd/system/vitastor-osd$((BASE+i)).service <<EOF
-[Unit]
-Description=Vitastor object storage daemon osd.$((BASE+i))
-After=network-online.target local-fs.target time-sync.target
-Wants=network-online.target local-fs.target time-sync.target
-PartOf=vitastor.target
-
-[Service]
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitMEMLOCK=infinity
-ExecStart=/root/vitastor/osd \\
-    --etcd_address $IP:2379/v3 \\
-    --bind_address $IP \\
-    --osd_num $((BASE+i)) \\
-    --disable_data_fsync 1 \\
-    --immediate_commit all \\
-    --flusher_count 8 \\
-    --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
-    --journal_no_same_sector_overwrites true \\
-    --journal_sector_buffer_count 1024 \\
-    --journal_offset 1920377991168 \\
-    --meta_offset $((1920377991168+16777216)) \\
-    --data_offset $((1920377991168+427814912)) \\
-    --data_size $((1920377991168-427814912)) \\
-    --data_device /dev/disk/by-id/$DEV
-WorkingDirectory=/root/vitastor
-ExecStartPre=+chown vitastor:vitastor /dev/disk/by-id/$DEV
-User=vitastor
-PrivateTmp=false
-TasksMax=infinity
-Restart=always
-StartLimitInterval=0
-StartLimitIntervalSec=0
-RestartSec=10
-
-[Install]
-WantedBy=vitastor.target
-EOF
-    systemctl enable vitastor-osd$((BASE+i))
-    i=$((i+1))
-done
-
-exit
-
-node mon-main.js --etcd_url 'http://10.115.0.10:2379,http://10.115.0.11:2379,http://10.115.0.12:2379,http://10.115.0.13:2379' --etcd_prefix '/vitastor' --etcd_start_timeout 5
-
-podman run -d --network host --restart always -v /var/lib/etcd0.etcd:/etcd0.etcd --name etcd quay.io/coreos/etcd:v3.4.13 etcd -name etcd0 \
-    -advertise-client-urls http://10.115.0.10:2379 -listen-client-urls http://10.115.0.10:2379 \
-    -initial-advertise-peer-urls http://10.115.0.10:2380 -listen-peer-urls http://10.115.0.10:2380 \
-    -initial-cluster-token vitastor-etcd-1 -initial-cluster etcd0=http://10.115.0.10:2380,etcd1=http://10.115.0.11:2380,etcd2=http://10.115.0.12:2380,etcd3=http://10.115.0.13:2380 \
-    -initial-cluster-state new --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision
-
-etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/global '{"immediate_commit":"all"}'
-
-etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":48,"failure_domain":"host"}}'
-
-#let pgs = {};
-#for (let n = 0; n < 48; n++) { let i = n/2 | 0; pgs[1+n] = { osd_set: [ (1+i%12+(i/12 | 0)*24), (1+12+i%12+(i/12 | 0)*24) ], primary: (1+(n%2)*12+i%12+(i/12 | 0)*24) }; };
-#console.log(JSON.stringify({ items: { 1: pgs } }));
-#etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/pgs ...
-
-#    --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
-#    --data_offset 427814912 \\
-
-#    --disk_alignment 4096 --journal_block_size 512 --meta_block_size 512 \\
-#    --data_offset 433434624 \\

mon/mon-main.js

@@ -1,7 +1,7 @@
 #!/usr/bin/node
 
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 const Mon = require('./mon.js');
 

mon/mon.js

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 const http = require('http');
 const crypto = require('crypto');
@@ -379,7 +379,7 @@ class Mon
                     }
                     if (this.verbose)
                     {
-                        console.log(e);
+                        console.log(JSON.stringify(e));
                     }
                 }
                 if (stats_changed)
@@ -811,6 +811,17 @@ class Mon
                         }
                         PGUtil.scale_pg_count(prev_pgs, this.state.pg.history[pool_id]||{}, pg_history, pool_cfg.pg_count);
                     }
+                    for (const pg of prev_pgs)
+                    {
+                        while (pg.length < pool_cfg.pg_size)
+                        {
+                            pg.push(0);
+                        }
+                        while (pg.length > pool_cfg.pg_size)
+                        {
+                            pg.pop();
+                        }
+                    }
                     optimize_result = await LPOptimizer.optimize_change({
                         prev_pgs,
                         osd_tree: pool_tree,
@@ -840,7 +851,7 @@ class Mon
                 this.save_new_pgs_txn(etcd_request, pool_id, up_osds, prev_pgs, optimize_result.int_pgs, pg_history);
             }
             this.state.config.pgs.hash = tree_hash;
-            await this.save_pg_config();
+            await this.save_pg_config(etcd_request);
         }
         else
         {
@@ -963,21 +974,21 @@ class Mon
             for (const op in st.op_stats||{})
             {
                 op_stats[op] = op_stats[op] || { count: 0n, usec: 0n, bytes: 0n };
-                op_stats[op].count += BigInt(st.op_stats.count||0);
+                op_stats[op].count += BigInt(st.op_stats[op].count||0);
-                op_stats[op].usec += BigInt(st.op_stats.usec||0);
+                op_stats[op].usec += BigInt(st.op_stats[op].usec||0);
-                op_stats[op].bytes += BigInt(st.op_stats.bytes||0);
+                op_stats[op].bytes += BigInt(st.op_stats[op].bytes||0);
             }
             for (const op in st.subop_stats||{})
             {
                 subop_stats[op] = subop_stats[op] || { count: 0n, usec: 0n };
-                subop_stats[op].count += BigInt(st.subop_stats.count||0);
+                subop_stats[op].count += BigInt(st.subop_stats[op].count||0);
-                subop_stats[op].usec += BigInt(st.subop_stats.usec||0);
+                subop_stats[op].usec += BigInt(st.subop_stats[op].usec||0);
             }
             for (const op in st.recovery_stats||{})
             {
                 recovery_stats[op] = recovery_stats[op] || { count: 0n, bytes: 0n };
-                recovery_stats[op].count += BigInt(st.recovery_stats.count||0);
+                recovery_stats[op].count += BigInt(st.recovery_stats[op].count||0);
-                recovery_stats[op].bytes += BigInt(st.recovery_stats.bytes||0);
+                recovery_stats[op].bytes += BigInt(st.recovery_stats[op].bytes||0);
             }
         }
         for (const op in op_stats)

mon/test-nonuniform.js

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 // Interesting real-world example coming from Ceph with EC and compression enabled.
 // EC parity chunks can't be compressed as efficiently as data chunks,

mon/test-optimize-simple.js

@@ -0,0 +1,25 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+const LPOptimizer = require('./lp-optimizer.js');
+
+async function run()
+{
+    const osd_tree = { a: { 1: 1 }, b: { 2: 1 }, c: { 3: 1 } };
+    let res;
+
+    console.log('16 PGs, size=3');
+    res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 3, pg_count: 16 });
+    LPOptimizer.print_change_stats(res, false);
+
+    console.log('\nReduce PG size to 2');
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs.map(pg => pg.slice(0, 2)), osd_tree, pg_size: 2 });
+    LPOptimizer.print_change_stats(res, false);
+
+    console.log('\nRemove OSD 3');
+    delete osd_tree['c'];
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2 });
+    LPOptimizer.print_change_stats(res, false);
+}
+
+run().catch(console.error);

mon/test-optimize-undersized.js

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 const LPOptimizer = require('./lp-optimizer.js');
 

mon/test-optimize.js

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 const LPOptimizer = require('./lp-optimizer.js');
 

rpm/build-tarball.sh

@@ -48,4 +48,4 @@ FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Ve
 QEMU=`rpm -qi qemu qemu-kvm | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
 perl -i -pe 's/(Requires:\s*qemu(?:-kvm)?)([^\n]+)?/$1 = '$QEMU'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.5.3/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.5.3$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.5.6/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.5.6$(rpm --eval '%dist').tar.gz *

rpm/qemu-el8.Dockerfile

@@ -1,5 +1,5 @@
 # Build packages for CentOS 8 inside a container
-# cd ..; podman build -t qemu-el8 -v `pwd`/build:/root/build -f rpm/qemu-el8.Dockerfile .
+# cd ..; podman build -t qemu-el8 -v `pwd`/packages:/root/packages -f rpm/qemu-el8.Dockerfile .
 
 FROM centos:8
 
@@ -14,8 +14,8 @@ RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=PowerTools --spec qemu-k
 ADD qemu-*-vitastor.patch /root/vitastor/
 
 RUN set -e; \
-    mkdir -p /root/build/qemu-el8; \
+    mkdir -p /root/packages/qemu-el8; \
-    rm -rf /root/build/qemu-el8/*; \
+    rm -rf /root/packages/qemu-el8/*; \
     rpm --nomd5 -i /root/qemu*.src.rpm; \
     cd ~/rpmbuild/SPECS; \
     PN=$(grep ^Patch qemu-kvm.spec | tail -n1 | perl -pe 's/Patch(\d+).*/$1/'); \
@@ -27,5 +27,5 @@ RUN set -e; \
     perl -i -pe 's/(^Release:\s*\d+)/$1.vitastor/' qemu-kvm.spec; \
     cp /root/vitastor/qemu-4.2-vitastor.patch ~/rpmbuild/SOURCES; \
     rpmbuild --nocheck -ba qemu-kvm.spec; \
-    cp ~/rpmbuild/RPMS/*/*qemu* /root/build/qemu-el8/; \
+    cp ~/rpmbuild/RPMS/*/*qemu* /root/packages/qemu-el8/; \
-    cp ~/rpmbuild/SRPMS/*qemu* /root/build/qemu-el8/
+    cp ~/rpmbuild/SRPMS/*qemu* /root/packages/qemu-el8/

rpm/vitastor-el7.Dockerfile

@@ -1,5 +1,5 @@
 # Build packages for CentOS 7 inside a container
-# cd ..; podman build -t vitastor-el7 -v `pwd`/build:/root/build -f rpm/vitastor-el7.Dockerfile .
+# cd ..; podman build -t vitastor-el7 -v `pwd`/packages:/root/packages -f rpm/vitastor-el7.Dockerfile .
 # localedef -i ru_RU -f UTF-8 ru_RU.UTF-8
 
 FROM centos:7
@@ -25,23 +25,23 @@ RUN set -e; \
     cd ~/rpmbuild/SPECS/; \
     . /opt/rh/devtoolset-9/enable; \
     rpmbuild -ba liburing.spec; \
-    mkdir -p /root/build/liburing-el7; \
+    mkdir -p /root/packages/liburing-el7; \
-    rm -rf /root/build/liburing-el7/*; \
+    rm -rf /root/packages/liburing-el7/*; \
-    cp ~/rpmbuild/RPMS/*/liburing* /root/build/liburing-el7/; \
+    cp ~/rpmbuild/RPMS/*/liburing* /root/packages/liburing-el7/; \
-    cp ~/rpmbuild/SRPMS/liburing* /root/build/liburing-el7/
+    cp ~/rpmbuild/SRPMS/liburing* /root/packages/liburing-el7/
 
-RUN rpm -i `ls /root/build/liburing-el7/liburing-*.x86_64.rpm | grep -v debug`
+RUN rpm -i `ls /root/packages/liburing-el7/liburing-*.x86_64.rpm | grep -v debug`
 
 ADD . /root/vitastor
 
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.5.3.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.5.6.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \
-    mkdir -p /root/build/vitastor-el7; \
+    mkdir -p /root/packages/vitastor-el7; \
-    rm -rf /root/build/vitastor-el7/*; \
+    rm -rf /root/packages/vitastor-el7/*; \
-    cp ~/rpmbuild/RPMS/*/vitastor* /root/build/vitastor-el7/; \
+    cp ~/rpmbuild/RPMS/*/vitastor* /root/packages/vitastor-el7/; \
-    cp ~/rpmbuild/SRPMS/vitastor* /root/build/vitastor-el7/
+    cp ~/rpmbuild/SRPMS/vitastor* /root/packages/vitastor-el7/

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.5.3
+Version:        0.5.6
-Release:        2%{?dist}
+Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
-License:        Vitastor Network Public License 1.0
+License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.5.3.el7.tar.gz
+Source0:        vitastor-0.5.6.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -14,6 +14,7 @@ BuildRequires:  rh-nodejs12
 BuildRequires:  rh-nodejs12-npm
 BuildRequires:  jerasure-devel
 BuildRequires:  gf-complete-devel
+BuildRequires:  cmake
 Requires:       fio = 3.7-1.el7
 Requires:       qemu-kvm = 2.0.0-1.el7.6
 Requires:       rh-nodejs12
@@ -35,12 +36,13 @@ size with configurable redundancy (replication or erasure codes/XOR).
 
 %build
 . /opt/rh/devtoolset-9/enable
-make %{?_smp_mflags} BINDIR=%_bindir LIBDIR=%_libdir QEMU_PLUGINDIR=%_libdir/qemu-kvm
+%cmake . -DQEMU_PLUGINDIR=qemu-kvm
+%make_build
 
 
 %install
 rm -rf $RPM_BUILD_ROOT
-%make_install BINDIR=%_bindir LIBDIR=%_libdir QEMU_PLUGINDIR=%_libdir/qemu-kvm
+%make_install
 . /opt/rh/rh-nodejs12/enable
 cd mon
 npm install
@@ -56,7 +58,11 @@ cp -r mon %buildroot/usr/lib/vitastor/mon
 %_bindir/vitastor-osd
 %_bindir/vitastor-rm
 %_libdir/qemu-kvm/block-vitastor.so
-%_libdir/vitastor
+%_libdir/libfio_vitastor.so
+%_libdir/libfio_vitastor_blk.so
+%_libdir/libfio_vitastor_sec.so
+%_libdir/libvitastor_blk.so
+%_libdir/libvitastor_client.so
 /usr/lib/vitastor
 
 

rpm/vitastor-el8.Dockerfile

@@ -1,5 +1,5 @@
 # Build packages for CentOS 8 inside a container
-# cd ..; podman build -t vitastor-el8 -v `pwd`/build:/root/build -f rpm/vitastor-el8.Dockerfile .
+# cd ..; podman build -t vitastor-el8 -v `pwd`/packages:/root/packages -f rpm/vitastor-el8.Dockerfile .
 
 FROM centos:8
 
@@ -13,8 +13,8 @@ RUN rm -rf /var/lib/dnf/*; dnf download --disablerepo='*' --enablerepo='vitastor
 RUN dnf download --source fio
 RUN rpm --nomd5 -i qemu*.src.rpm
 RUN rpm --nomd5 -i fio*.src.rpm
-RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=PowerTools --spec qemu-kvm.spec
+RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=powertools --spec qemu-kvm.spec
-RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=PowerTools --spec fio.spec
+RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=powertools --spec fio.spec && dnf install -y cmake
 
 ADD https://vitastor.io/rpms/liburing-el7/liburing-0.7-2.el7.src.rpm /root
 
@@ -23,23 +23,23 @@ RUN set -e; \
     cd ~/rpmbuild/SPECS/; \
     . /opt/rh/gcc-toolset-9/enable; \
     rpmbuild -ba liburing.spec; \
-    mkdir -p /root/build/liburing-el8; \
+    mkdir -p /root/packages/liburing-el8; \
-    rm -rf /root/build/liburing-el8/*; \
+    rm -rf /root/packages/liburing-el8/*; \
-    cp ~/rpmbuild/RPMS/*/liburing* /root/build/liburing-el8/; \
+    cp ~/rpmbuild/RPMS/*/liburing* /root/packages/liburing-el8/; \
-    cp ~/rpmbuild/SRPMS/liburing* /root/build/liburing-el8/
+    cp ~/rpmbuild/SRPMS/liburing* /root/packages/liburing-el8/
 
-RUN rpm -i `ls /root/build/liburing-el7/liburing-*.x86_64.rpm | grep -v debug`
+RUN rpm -i `ls /root/packages/liburing-el7/liburing-*.x86_64.rpm | grep -v debug`
 
 ADD . /root/vitastor
 
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.5.3.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.5.6.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \
-    mkdir -p /root/build/vitastor-el8; \
+    mkdir -p /root/packages/vitastor-el8; \
-    rm -rf /root/build/vitastor-el8/*; \
+    rm -rf /root/packages/vitastor-el8/*; \
-    cp ~/rpmbuild/RPMS/*/vitastor* /root/build/vitastor-el8/; \
+    cp ~/rpmbuild/RPMS/*/vitastor* /root/packages/vitastor-el8/; \
-    cp ~/rpmbuild/SRPMS/vitastor* /root/build/vitastor-el8/
+    cp ~/rpmbuild/SRPMS/vitastor* /root/packages/vitastor-el8/

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.5.3
+Version:        0.5.6
-Release:        2%{?dist}
+Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
-License:        Vitastor Network Public License 1.0
+License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.5.3.el8.tar.gz
+Source0:        vitastor-0.5.6.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -13,6 +13,7 @@ BuildRequires:  gcc-toolset-9-gcc-c++
 BuildRequires:  nodejs >= 10
 BuildRequires:  jerasure-devel
 BuildRequires:  gf-complete-devel
+BuildRequires:  cmake
 Requires:       fio = 3.7-3.el8
 Requires:       qemu-kvm = 4.2.0-29.el8.6
 Requires:       nodejs >= 10
@@ -33,12 +34,13 @@ size with configurable redundancy (replication or erasure codes/XOR).
 
 %build
 . /opt/rh/gcc-toolset-9/enable
-make %{?_smp_mflags} BINDIR=%_bindir LIBDIR=%_libdir QEMU_PLUGINDIR=%_libdir/qemu-kvm
+%cmake . -DQEMU_PLUGINDIR=qemu-kvm
+%make_build
 
 
 %install
 rm -rf $RPM_BUILD_ROOT
-%make_install BINDIR=%_bindir LIBDIR=%_libdir QEMU_PLUGINDIR=%_libdir/qemu-kvm
+%make_install
 cd mon
 npm install
 cd ..
@@ -53,7 +55,11 @@ cp -r mon %buildroot/usr/lib/vitastor
 %_bindir/vitastor-osd
 %_bindir/vitastor-rm
 %_libdir/qemu-kvm/block-vitastor.so
-%_libdir/vitastor
+%_libdir/libfio_vitastor.so
+%_libdir/libfio_vitastor_blk.so
+%_libdir/libfio_vitastor_sec.so
+%_libdir/libvitastor_blk.so
+%_libdir/libvitastor_client.so
 /usr/lib/vitastor
 
 

run_tests.sh

@@ -1,77 +0,0 @@
-#!/bin/bash
-
-if [ ! "$BASH_VERSION" ] ; then
-    echo "Use bash to run this script ($0)" 1>&2
-    exit 1
-fi
-
-format_error()
-{
-    echo $(echo -n -e "\033[1;31m")$1$(echo -n -e "\033[m")
-    $ETCDCTL get --prefix /vitastor > ./testdata/etcd-dump.txt
-    exit 1
-}
-format_green()
-{
-    echo $(echo -n -e "\033[1;32m")$1$(echo -n -e "\033[m")
-}
-
-set -e -x
-
-trap 'kill -9 $(jobs -p)' EXIT
-
-ETCD=${ETCD:-etcd}
-ETCD_PORT=${ETCD_PORT:-12379}
-
-rm -rf ./testdata
-mkdir -p ./testdata
-dd if=/dev/zero of=./testdata/test_osd1.bin bs=1024 count=1 seek=$((1024*1024-1))
-dd if=/dev/zero of=./testdata/test_osd2.bin bs=1024 count=1 seek=$((1024*1024-1))
-dd if=/dev/zero of=./testdata/test_osd3.bin bs=1024 count=1 seek=$((1024*1024-1))
-
-$ETCD -name etcd_test --data-dir ./testdata/etcd \
-    --advertise-client-urls http://127.0.0.1:$ETCD_PORT --listen-client-urls http://127.0.0.1:$ETCD_PORT \
-    --initial-advertise-peer-urls http://127.0.0.1:$((ETCD_PORT+1)) --listen-peer-urls http://127.0.0.1:$((ETCD_PORT+1)) \
-    --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision &>./testdata/etcd.log &
-ETCD_PID=$!
-ETCD_URL=127.0.0.1:$ETCD_PORT/v3
-ETCDCTL="${ETCD}ctl --endpoints=http://$ETCD_URL"
-
-./osd --osd_num 1 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log &
-OSD1_PID=$!
-./osd --osd_num 2 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log &
-OSD2_PID=$!
-./osd --osd_num 3 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log &
-OSD3_PID=$!
-
-cd mon
-npm install
-cd ..
-node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
-MON_PID=$!
-
-$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1,"pg_count":1,"failure_domain":"osd"}}'
-
-sleep 2
-
-if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and (.[0].items["1"]["1"].osd_set | sort) == ["1","2","3"]'); then
-    format_error "FAILED: 1 PG NOT CONFIGURED"
-fi
-
-if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then
-    format_error "FAILED: 1 PG NOT UP"
-fi
-
-echo leak:fio >> testdata/lsan-suppress.txt
-echo leak:tcmalloc >> testdata/lsan-suppress.txt
-echo leak:ceph >> testdata/lsan-suppress.txt
-echo leak:librbd >> testdata/lsan-suppress.txt
-echo leak:_M_mutate >> testdata/lsan-suppress.txt
-echo leak:_M_assign >> testdata/lsan-suppress.txt
-#LSAN_OPTIONS=suppressions=`pwd`/testdata/lsan-suppress.txt LD_PRELOAD=libasan.so.5 \
-#    fio -thread -name=test -ioengine=./libfio_sec_osd.so -bs=4k -fsync=128 `$ETCDCTL get /vitastor/osd/state/1 --print-value-only | jq -r '"-host="+.addresses[0]+" -port="+(.port|tostring)'` -rw=write -size=32M
-
-LSAN_OPTIONS=suppressions=`pwd`/testdata/lsan-suppress.txt LD_PRELOAD=libasan.so.5 \
-    fio -thread -name=test -ioengine=./libfio_cluster.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=1G -cluster_log_level=10
-
-format_green OK

src/CMakeLists.txt

@@ -0,0 +1,188 @@
+cmake_minimum_required(VERSION 2.8)
+
+project(vitastor)
+
+include(GNUInstallDirs)
+
+set(QEMU_PLUGINDIR qemu CACHE STRING "QEMU plugin directory suffix (qemu-kvm on RHEL)")
+set(WITH_ASAN false CACHE BOOL "Build with AddressSanitizer")
+if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
+	if(EXISTS "/etc/debian_version")
+		set(CMAKE_INSTALL_LIBDIR "lib/${CMAKE_LIBRARY_ARCHITECTURE}")
+	endif()
+	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+endif()
+
+add_definitions(-DVERSION="0.6-dev")
+add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith)
+if (${WITH_ASAN})
+	add_definitions(-fsanitize=address -fno-omit-frame-pointer)
+	add_link_options(-fsanitize=address -fno-omit-frame-pointer)
+endif (${WITH_ASAN})
+
+set(CMAKE_BUILD_TYPE RelWithDebInfo)
+string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
+string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}")
+string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
+string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
+string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}")
+string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
+string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
+string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}")
+string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
+string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
+string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}")
+string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
+
+find_package(PkgConfig)
+pkg_check_modules(LIBURING REQUIRED liburing)
+pkg_check_modules(GLIB REQUIRED glib-2.0)
+
+include_directories(
+	../
+	/usr/include/jerasure
+	${LIBURING_INCLUDE_DIRS}
+)
+
+# libvitastor_blk.so
+add_library(vitastor_blk SHARED
+	allocator.cpp blockstore.cpp blockstore_impl.cpp blockstore_init.cpp blockstore_open.cpp blockstore_journal.cpp blockstore_read.cpp
+	blockstore_write.cpp blockstore_sync.cpp blockstore_stable.cpp blockstore_rollback.cpp blockstore_flush.cpp crc32c.c ringloop.cpp
+)
+target_link_libraries(vitastor_blk
+	${LIBURING_LIBRARIES}
+	tcmalloc_minimal
+)
+
+# libfio_vitastor_blk.so
+add_library(fio_vitastor_blk SHARED
+	fio_engine.cpp
+	../json11/json11.cpp
+)
+target_link_libraries(fio_vitastor_blk
+	vitastor_blk
+)
+
+# vitastor-osd
+add_executable(vitastor-osd
+	osd_main.cpp osd.cpp osd_secondary.cpp msgr_receive.cpp msgr_send.cpp osd_peering.cpp osd_flush.cpp osd_peering_pg.cpp
+	osd_primary.cpp osd_primary_subops.cpp etcd_state_client.cpp messenger.cpp osd_cluster.cpp http_client.cpp osd_ops.cpp pg_states.cpp
+	osd_rmw.cpp base64.cpp timerfd_manager.cpp epoll_manager.cpp ../json11/json11.cpp
+)
+target_link_libraries(vitastor-osd
+	vitastor_blk
+	Jerasure
+)
+
+# libfio_vitastor_sec.so
+add_library(fio_vitastor_sec SHARED
+	fio_sec_osd.cpp
+	rw_blocking.cpp
+)
+target_link_libraries(fio_vitastor_sec
+	tcmalloc_minimal
+)
+
+# libvitastor_client.so
+add_library(vitastor_client SHARED
+	cluster_client.cpp epoll_manager.cpp etcd_state_client.cpp
+	messenger.cpp msgr_send.cpp msgr_receive.cpp ringloop.cpp ../json11/json11.cpp
+	http_client.cpp osd_ops.cpp pg_states.cpp timerfd_manager.cpp base64.cpp
+)
+target_link_libraries(vitastor_client
+	tcmalloc_minimal
+	${LIBURING_LIBRARIES}
+)
+
+# libfio_vitastor.so
+add_library(fio_vitastor SHARED
+	fio_cluster.cpp
+)
+target_link_libraries(fio_vitastor
+	vitastor_client
+)
+
+# vitastor-nbd
+add_executable(vitastor-nbd
+	nbd_proxy.cpp
+)
+target_link_libraries(vitastor-nbd
+	vitastor_client
+)
+
+# vitastor-rm
+add_executable(vitastor-rm
+	rm_inode.cpp
+)
+target_link_libraries(vitastor-rm
+	vitastor_client
+)
+
+# vitastor-dump-journal
+add_executable(vitastor-dump-journal
+	dump_journal.cpp crc32c.c
+)
+
+# qemu_driver.so
+add_library(qemu_proxy STATIC qemu_proxy.cpp)
+target_compile_options(qemu_proxy PUBLIC -fPIC)
+target_include_directories(qemu_proxy PUBLIC
+	../qemu/b/qemu
+	../qemu/include
+	${GLIB_INCLUDE_DIRS}
+)
+target_link_libraries(qemu_proxy
+	vitastor_client
+)
+add_library(qemu_vitastor SHARED
+	qemu_driver.c
+)
+target_link_libraries(qemu_vitastor
+	qemu_proxy
+)
+set_target_properties(qemu_vitastor PROPERTIES
+	PREFIX ""
+	OUTPUT_NAME "block-vitastor"
+)
+
+### Test stubs
+
+# stub_osd, stub_bench, osd_test
+add_executable(stub_osd stub_osd.cpp rw_blocking.cpp)
+target_link_libraries(stub_osd tcmalloc_minimal)
+add_executable(stub_bench stub_bench.cpp rw_blocking.cpp)
+target_link_libraries(stub_bench tcmalloc_minimal)
+add_executable(osd_test osd_test.cpp rw_blocking.cpp)
+target_link_libraries(osd_test tcmalloc_minimal)
+
+# osd_rmw_test
+add_executable(osd_rmw_test osd_rmw_test.cpp allocator.cpp)
+target_link_libraries(osd_rmw_test Jerasure tcmalloc_minimal)
+
+# stub_uring_osd
+add_executable(stub_uring_osd
+	stub_uring_osd.cpp epoll_manager.cpp messenger.cpp msgr_send.cpp msgr_receive.cpp ringloop.cpp timerfd_manager.cpp ../json11/json11.cpp
+)
+target_link_libraries(stub_uring_osd
+	${LIBURING_LIBRARIES}
+	tcmalloc_minimal
+)
+
+# osd_peering_pg_test
+add_executable(osd_peering_pg_test osd_peering_pg_test.cpp osd_peering_pg.cpp)
+target_link_libraries(osd_peering_pg_test tcmalloc_minimal)
+
+# test_allocator
+add_executable(test_allocator test_allocator.cpp allocator.cpp)
+
+## test_blockstore, test_shit
+#add_executable(test_blockstore test_blockstore.cpp timerfd_interval.cpp)
+#target_link_libraries(test_blockstore blockstore)
+#add_executable(test_shit test_shit.cpp osd_peering_pg.cpp)
+#target_link_libraries(test_shit ${LIBURING_LIBRARIES} m)
+
+### Install
+
+install(TARGETS vitastor-osd vitastor-dump-journal vitastor-nbd vitastor-rm RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(TARGETS fio_vitastor fio_vitastor_blk fio_vitastor_sec vitastor_blk vitastor_client LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(TARGETS qemu_vitastor LIBRARY DESTINATION /usr/${CMAKE_INSTALL_LIBDIR}/${QEMU_PLUGINDIR})

src/allocator.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <stdexcept>
 #include "allocator.h"

src/allocator.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 

src/base64.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "base64.h"
 

src/base64.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 #include <string>

src/blockstore.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 

src/blockstore.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 

src/blockstore_flush.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 
@@ -7,6 +7,8 @@ journal_flusher_t::journal_flusher_t(int flusher_count, blockstore_impl_t *bs)
 {
     this->bs = bs;
     this->flusher_count = flusher_count;
+    this->cur_flusher_count = 1;
+    this->target_flusher_count = 1;
     dequeuing = false;
     trimming = false;
     active_flushers = 0;
@@ -68,10 +70,24 @@ bool journal_flusher_t::is_active()
 
 void journal_flusher_t::loop()
 {
-    for (int i = 0; (active_flushers > 0 || dequeuing) && i < flusher_count; i++)
+    target_flusher_count = bs->write_iodepth*2;
+    if (target_flusher_count <= 0)
+        target_flusher_count = 1;
+    else if (target_flusher_count > flusher_count)
+        target_flusher_count = flusher_count;
+    if (target_flusher_count > cur_flusher_count)
+        cur_flusher_count = target_flusher_count;
+    else if (target_flusher_count < cur_flusher_count)
     {
-        co[i].loop();
+        while (target_flusher_count < cur_flusher_count)
+        {
+            if (co[cur_flusher_count-1].wait_state)
+                break;
+            cur_flusher_count--;
+        }
     }
+    for (int i = 0; (active_flushers > 0 || dequeuing) && i < cur_flusher_count; i++)
+        co[i].loop();
 }
 
 void journal_flusher_t::enqueue_flush(obj_ver_id ov)
@@ -223,6 +239,7 @@ bool journal_flusher_co::loop()
 resume_0:
     if (!flusher->flush_queue.size() || !flusher->dequeuing)
     {
+stop_flusher:
         if (flusher->trim_wanted > 0 && flusher->journal_trim_counter > 0)
         {
             // Attempt forced trim
@@ -327,9 +344,7 @@ resume_0:
 #ifdef BLOCKSTORE_DEBUG
                     printf("No older flushes, stopping\n");
 #endif
-                    flusher->dequeuing = false;
+                    goto stop_flusher;
-                    wait_state = 0;
-                    return true;
                 }
             }
         }

src/blockstore_flush.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 struct copy_buffer_t
 {
@@ -80,7 +80,7 @@ class journal_flusher_t
 {
     int trim_wanted = 0;
     bool dequeuing;
-    int flusher_count;
+    int flusher_count, cur_flusher_count, target_flusher_count;
     int flusher_start_threshold;
     journal_flusher_co *co;
     blockstore_impl_t *bs;

src/blockstore_impl.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 

src/blockstore_impl.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 
@@ -199,7 +199,10 @@ class blockstore_impl_t
     // Suitable only for server SSDs with capacitors, requires disabled data and journal fsyncs
     int immediate_commit = IMMEDIATE_NONE;
     bool inmemory_meta = false;
-    int flusher_count;
+    // Maximum flusher count
+    unsigned flusher_count;
+    // Maximum queue depth
+    unsigned max_write_iodepth = 128;
     /******* END OF OPTIONS *******/
 
     struct ring_consumer_t ring_consumer;
@@ -226,6 +229,7 @@ class blockstore_impl_t
 
     struct journal_t journal;
     journal_flusher_t *flusher;
+    int write_iodepth = 0;
 
     bool live = false, queue_stall = false;
     ring_loop_t *ringloop;

src/blockstore_init.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 

src/blockstore_init.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 

src/blockstore_journal.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 
@@ -20,7 +20,7 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
     int required = entries_required;
     while (1)
     {
-        int fits = bs->journal.no_same_sector_overwrites && bs->journal.sector_info[next_sector].written
+        int fits = bs->journal.no_same_sector_overwrites && next_pos == bs->journal.next_free && bs->journal.sector_info[next_sector].written
             ? 0
             : (bs->journal.block_size - next_in_pos) / size;
         if (fits > 0)
@@ -81,7 +81,8 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
             }
             // In fact, it's even more rare than "ran out of journal space", so print a warning
             printf(
-                "Ran out of journal sector buffers: %d/%lu buffers used (%d dirty), next buffer (%ld) is %s and flushed %lu times\n",
+                "Ran out of journal sector buffers: %d/%lu buffers used (%d dirty), next buffer (%ld)"
+                " is %s and flushed %lu times. Consider increasing \'journal_sector_buffer_count\'\n",
                 used, bs->journal.sector_count, dirty, next_sector,
                 bs->journal.sector_info[next_sector].dirty ? "dirty" : "not dirty",
                 bs->journal.sector_info[next_sector].flush_count
@@ -103,11 +104,8 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
     {
         // No space in the journal. Wait until used_start changes.
         printf(
-            "Ran out of journal space (free space: %lu bytes, sectors to write: %d)\n",
+            "Ran out of journal space (used_start=%08lx, next_free=%08lx, dirty_start=%08lx)\n",
-            (bs->journal.next_free >= bs->journal.used_start
+            bs->journal.used_start, bs->journal.next_free, bs->journal.dirty_start
-                ? bs->journal.len-bs->journal.block_size - (bs->journal.next_free-bs->journal.used_start)
-                : bs->journal.used_start - bs->journal.next_free),
-            sectors_to_write
         );
         PRIV(op)->wait_for = WAIT_JOURNAL;
         bs->flusher->request_trim();

src/blockstore_journal.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 

src/blockstore_open.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <sys/file.h>
 #include "blockstore_impl.h"
@@ -70,6 +70,7 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config)
     meta_block_size = strtoull(config["meta_block_size"].c_str(), NULL, 10);
     bitmap_granularity = strtoull(config["bitmap_granularity"].c_str(), NULL, 10);
     flusher_count = strtoull(config["flusher_count"].c_str(), NULL, 10);
+    max_write_iodepth = strtoull(config["max_write_iodepth"].c_str(), NULL, 10);
     // Validate
     if (!block_size)
     {
@@ -83,6 +84,10 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config)
     {
         flusher_count = 32;
     }
+    if (!max_write_iodepth)
+    {
+        max_write_iodepth = 128;
+    }
     if (!disk_alignment)
     {
         disk_alignment = 4096;

src/blockstore_read.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 

src/blockstore_rollback.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 

src/blockstore_stable.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 

src/blockstore_sync.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 

src/blockstore_write.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 
@@ -167,6 +167,10 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
             .version = op->version,
         }, e).first;
     }
+    if (write_iodepth >= max_write_iodepth)
+    {
+        return 0;
+    }
     if ((dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE)
     {
         blockstore_journal_check_t space_check(this);
@@ -191,6 +195,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
             FINISH_OP(op);
             return 1;
         }
+        write_iodepth++;
         BS_SUBMIT_GET_SQE(sqe, data);
         dirty_it->second.location = loc << block_order;
         dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SUBMITTED;
@@ -243,6 +248,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         {
             return 0;
         }
+        write_iodepth++;
         // There is sufficient space. Get SQE(s)
         struct io_uring_sqe *sqe1 = NULL;
         if (immediate_commit != IMMEDIATE_NONE ||
@@ -432,6 +438,7 @@ resume_4:
     }
     // Acknowledge write
     op->retval = op->len;
+    write_iodepth--;
     FINISH_OP(op);
     return 1;
 }
@@ -503,6 +510,7 @@ int blockstore_impl_t::dequeue_del(blockstore_op_t *op)
     {
         return 0;
     }
+    write_iodepth++;
     io_uring_sqe *sqe = NULL;
     if (immediate_commit != IMMEDIATE_NONE ||
         (journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_del) &&

src/cluster_client.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include <stdexcept>
 #include "cluster_client.h"

src/cluster_client.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/crc32c.h

@@ -8,4 +8,10 @@
 // unsigned __int64 _mm_crc32_u64 (unsigned __int64 crc, unsigned __int64 v)
 // unsigned int _mm_crc32_u8 (unsigned int crc, unsigned char v)
 
+#ifdef __cplusplus
+extern "C" {
+#endif
 uint32_t crc32c(uint32_t crc, const void *buf, size_t len);
+#ifdef __cplusplus
+};
+#endif

src/dump_journal.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #define _LARGEFILE64_SOURCE
 #include <sys/types.h>

src/epoll_manager.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include <sys/epoll.h>
 #include <sys/poll.h>

src/epoll_manager.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/etcd_state_client.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include "osd_ops.h"
 #include "pg_states.h"

src/etcd_state_client.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/fio_cluster.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 // FIO engine to test cluster I/O
 //

src/fio_engine.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 // FIO engine to test Blockstore
 //

src/fio_sec_osd.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 // FIO engine to test Blockstore through Secondary OSD interface
 //

src/http_client.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include <netinet/tcp.h>
 #include <sys/epoll.h>

src/http_client.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 #include <string>

src/malloc_or_die.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/messenger.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include <unistd.h>
 #include <fcntl.h>

src/messenger.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/msgr_receive.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include "messenger.h"
 

src/msgr_send.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #define _XOPEN_SOURCE
 #include <limits.h>

src/nbd_proxy.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 // Similar to qemu-nbd, but sets timeout and uses io_uring
 
 #include <linux/nbd.h>
@@ -111,7 +111,7 @@ public:
     {
         printf(
             "Vitastor NBD proxy\n"
-            "(c) Vitaliy Filippov, 2020 (VNPL-1.0)\n\n"
+            "(c) Vitaliy Filippov, 2020 (VNPL-1.1)\n\n"
             "USAGE:\n"
             "  %s map --etcd_address <etcd_address> --pool <pool> --inode <inode> --size <size in bytes>\n"
             "  %s unmap /dev/nbd0\n"

src/object_id.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/osd.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <sys/socket.h>
 #include <sys/poll.h>

src/osd.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 

src/osd_cluster.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "osd.h"
 #include "base64.h"
@@ -142,7 +142,7 @@ json11::Json osd_t::get_statistics()
     }
     st["host"] = self_state["host"];
     json11::Json::object op_stats, subop_stats;
-    for (int i = 0; i <= OSD_OP_MAX; i++)
+    for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
     {
         op_stats[osd_op_names[i]] = json11::Json::object {
             { "count", c_cli.stats.op_stat_count[i] },
@@ -150,7 +150,7 @@ json11::Json osd_t::get_statistics()
             { "bytes", c_cli.stats.op_stat_bytes[i] },
         };
     }
-    for (int i = 0; i <= OSD_OP_MAX; i++)
+    for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
     {
         subop_stats[osd_op_names[i]] = json11::Json::object {
             { "count", c_cli.stats.subop_stat_count[i] },
@@ -593,7 +593,10 @@ void osd_t::apply_pg_config()
                     }
                     else
                     {
-                        throw std::runtime_error("Unexpected PG "+std::to_string(pg_num)+" state: "+std::to_string(pg_it->second.state));
+                        throw std::runtime_error(
+                            "Unexpected PG "+std::to_string(pool_id)+"/"+std::to_string(pg_num)+
+                            " state: "+std::to_string(pg_it->second.state)
+                        );
                     }
                 }
                 auto & pg = this->pgs[{ .pool_id = pool_id, .pg_num = pg_num }];
@@ -802,11 +805,11 @@ void osd_t::report_pg_states()
                     if (pg_it->second.state == PG_OFFLINE)
                     {
                         // Remove offline PGs after reporting their state
-                        this->pgs.erase(pg_it);
                         if (pg_it->second.scheme == POOL_SCHEME_JERASURE)
                         {
                             use_jerasure(pg_it->second.pg_size, pg_it->second.pg_size-pg_it->second.parity_chunks, false);
                         }
+                        this->pgs.erase(pg_it);
                     }
                 }
             }

src/osd_flush.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "osd.h"
 
@@ -95,7 +95,7 @@ void osd_t::handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, p
     {
         // This flush batch is done
         std::vector<osd_op_t*> continue_ops;
-        auto & pg = pgs[pg_id];
+        auto & pg = pgs.at(pg_id);
         auto it = pg.flush_actions.begin(), prev_it = it;
         auto erase_start = it;
         while (1)

src/osd_id.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/osd_main.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "osd.h"
 

src/osd_ops.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include "osd_ops.h"
 

src/osd_ops.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/osd_peering.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <netinet/tcp.h>
 #include <sys/epoll.h>
@@ -473,7 +473,9 @@ bool osd_t::stop_pg(pg_t & pg)
         return false;
     }
     pg.state = pg.state & ~PG_ACTIVE | PG_STOPPING;
-    if (pg.inflight == 0 && !pg.flush_batch)
+    if (pg.inflight == 0 && !pg.flush_batch &&
+        // We must either forget all PG's unstable writes or wait for it to become clean
+        dirty_pgs.find({ .pool_id = pg.pool_id, .pg_num = pg.pg_num }) == dirty_pgs.end())
     {
         finish_stop_pg(pg);
     }

src/osd_peering_pg.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <unordered_map>
 #include "osd_peering_pg.h"

src/osd_peering_pg.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <map>
 #include <vector>

src/osd_peering_pg_test.cpp

@@ -1,8 +1,9 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #define _LARGEFILE64_SOURCE
 
+#include "malloc_or_die.h"
 #include "osd_peering_pg.h"
 #define STRIPE_SHIFT 12
 

src/osd_primary.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "osd_primary.h"
 
@@ -103,7 +103,7 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
     if (op_data->st == 1)      goto resume_1;
     else if (op_data->st == 2) goto resume_2;
     {
-        auto & pg = pgs[{ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num }];
+        auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
         for (int role = 0; role < op_data->pg_data_size; role++)
         {
             op_data->stripes[role].read_start = op_data->stripes[role].req_start;
@@ -211,7 +211,7 @@ void osd_t::continue_primary_write(osd_op_t *cur_op)
         return;
     }
     osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = pgs[{ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num }];
+    auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
     if (op_data->st == 1)      goto resume_1;
     else if (op_data->st == 2) goto resume_2;
     else if (op_data->st == 3) goto resume_3;
@@ -582,7 +582,7 @@ resume_2:
         int dpg = 0;
         for (auto dirty_pg_num: dirty_pgs)
         {
-            pgs[dirty_pg_num].inflight++;
+            pgs.at(dirty_pg_num).inflight++;
             op_data->dirty_pgs[dpg++] = dirty_pg_num;
         }
         dirty_pgs.clear();
@@ -639,7 +639,7 @@ resume_6:
                         .pool_id = INODE_POOL(w.oid.inode),
                         .pg_num = map_to_pg(w.oid, st_cli.pool_config.at(INODE_POOL(w.oid.inode)).pg_stripe_size),
                     };
-                    if (pgs[wpg].state & PG_ACTIVE)
+                    if (pgs.at(wpg).state & PG_ACTIVE)
                     {
                         uint64_t & dest = this->unstable_writes[(osd_object_id_t){
                             .osd_num = unstable_osd.osd_num,
@@ -750,7 +750,7 @@ void osd_t::continue_primary_del(osd_op_t *cur_op)
         return;
     }
     osd_primary_op_data_t *op_data = cur_op->op_data;
-    auto & pg = pgs[{ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num }];
+    auto & pg = pgs.at({ .pool_id = INODE_POOL(op_data->oid.inode), .pg_num = op_data->pg_num });
     if (op_data->st == 1)      goto resume_1;
     else if (op_data->st == 2) goto resume_2;
     else if (op_data->st == 3) goto resume_3;

src/osd_primary.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 

src/osd_primary_subops.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "osd_primary.h"
 
@@ -40,7 +40,7 @@ void osd_t::finish_op(osd_op_t *cur_op, int retval)
     {
         if (cur_op->op_data->pg_num > 0)
         {
-            auto & pg = pgs[{ .pool_id = INODE_POOL(cur_op->op_data->oid.inode), .pg_num = cur_op->op_data->pg_num }];
+            auto & pg = pgs.at({ .pool_id = INODE_POOL(cur_op->op_data->oid.inode), .pg_num = cur_op->op_data->pg_num });
             pg.inflight--;
             assert(pg.inflight >= 0);
             if ((pg.state & PG_STOPPING) && pg.inflight == 0 && !pg.flush_batch)

src/osd_rmw.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <stdexcept>
 #include <string.h>
@@ -11,7 +11,7 @@
 #include "osd_rmw.h"
 #include "malloc_or_die.h"
 
-#define OSD_JERASURE_W 32
+#define OSD_JERASURE_W 8
 
 static inline void extend_read(uint32_t start, uint32_t end, osd_rmw_stripe_t & stripe)
 {

src/osd_rmw.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 

src/osd_rmw_test.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #define RMW_DEBUG
 

src/osd_secondary.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "osd.h"
 

src/osd_test.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <sys/types.h>
 #include <sys/socket.h>

src/pg_states.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include "pg_states.h"
 

src/pg_states.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once