Release 0.5.6

- Fix operation statistics
- Fix a rebalance hang introduced in 0.5.5
- Test PG count changes with actual data moving
- Fix a possible 'unexpected pg state: 0' error during PG count change

.dockerignore

@@ -0,0 +1,19 @@
+.git
+build
+packages
+mon/node_modules
+*.o
+*.so
+osd
+stub_osd
+stub_uring_osd
+stub_bench
+osd_test
+dump_journal
+nbd_proxy
+rm_inode
+fio
+qemu
+rpm/*.Dockerfile
+debian/*.Dockerfile
+Dockerfile

CMakeLists.txt

@@ -0,0 +1,5 @@
+cmake_minimum_required(VERSION 2.8)
+
+project(vitastor)
+
+add_subdirectory(src)

Make-gen.pl

@@ -1,46 +0,0 @@
-#!/usr/bin/perl
-
-use strict;
-
-my $deps = {};
-for my $line (split /\n/, `grep '^#include "' *.cpp *.h`)
-{
-    if ($line =~ /^([^:]+):\#include "([^"]+)"/s)
-    {
-        $deps->{$1}->{$2} = 1;
-    }
-}
-
-my $added;
-do
-{
-    $added = 0;
-    for my $file (keys %$deps)
-    {
-        for my $dep (keys %{$deps->{$file}})
-        {
-            if ($deps->{$dep})
-            {
-                for my $subdep (keys %{$deps->{$dep}})
-                {
-                    if (!$deps->{$file}->{$subdep})
-                    {
-                        $added = 1;
-                        $deps->{$file}->{$subdep} = 1;
-                    }
-                }
-            }
-        }
-    }
-} while ($added);
-
-for my $file (sort keys %$deps)
-{
-    if ($file =~ /\.cpp$/)
-    {
-        my $obj = $file;
-        $obj =~ s/\.cpp$/.o/s;
-        print "$obj: $file ".join(" ", sort keys %{$deps->{$file}})."\n";
-        print "\tg++ \$(CXXFLAGS) -c -o \$\@ \$\<\n";
-    }
-}

Makefile

@@ -1,177 +0,0 @@
-BLOCKSTORE_OBJS := allocator.o blockstore.o blockstore_impl.o blockstore_init.o blockstore_open.o blockstore_journal.o blockstore_read.o \
-	blockstore_write.o blockstore_sync.o blockstore_stable.o blockstore_rollback.o blockstore_flush.o crc32c.o ringloop.o
-# -fsanitize=address
-CXXFLAGS := -g -O3 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fPIC -fdiagnostics-color=always
-all: libfio_blockstore.so osd libfio_sec_osd.so libfio_cluster.so stub_osd stub_uring_osd stub_bench osd_test dump_journal qemu_driver.so nbd_proxy rm_inode
-clean:
-	rm -f *.o
-
-dump_journal: dump_journal.cpp crc32c.o blockstore_journal.h
-	g++ $(CXXFLAGS) -o $@ $< crc32c.o
-
-libblockstore.so: $(BLOCKSTORE_OBJS)
-	g++ $(CXXFLAGS) -o $@ -shared $(BLOCKSTORE_OBJS) -ltcmalloc_minimal -luring
-libfio_blockstore.so: ./libblockstore.so fio_engine.o json11.o
-	g++ $(CXXFLAGS) -shared -o $@ fio_engine.o json11.o ./libblockstore.so -ltcmalloc_minimal -luring
-
-OSD_OBJS := osd.o osd_secondary.o msgr_receive.o msgr_send.o osd_peering.o osd_flush.o osd_peering_pg.o \
-	osd_primary.o osd_primary_subops.o etcd_state_client.o messenger.o osd_cluster.o http_client.o osd_ops.o pg_states.o \
-	osd_rmw.o json11.o base64.o timerfd_manager.o epoll_manager.o
-osd: ./libblockstore.so osd_main.cpp osd.h osd_ops.h $(OSD_OBJS)
-	g++ $(CXXFLAGS) -o $@ osd_main.cpp $(OSD_OBJS) ./libblockstore.so -ltcmalloc_minimal -luring
-
-stub_osd: stub_osd.o rw_blocking.o
-	g++ $(CXXFLAGS) -o $@ stub_osd.o rw_blocking.o -ltcmalloc_minimal
-
-osd_rmw_test: osd_rmw_test.o
-	g++ $(CXXFLAGS) -o $@ osd_rmw_test.o
-
-STUB_URING_OSD_OBJS := stub_uring_osd.o epoll_manager.o messenger.o msgr_send.o msgr_receive.o ringloop.o timerfd_manager.o json11.o
-stub_uring_osd: $(STUB_URING_OSD_OBJS)
-	g++ $(CXXFLAGS) -o $@ -ltcmalloc_minimal $(STUB_URING_OSD_OBJS) -luring
-stub_bench: stub_bench.cpp osd_ops.h rw_blocking.o
-	g++ $(CXXFLAGS) -o $@ stub_bench.cpp rw_blocking.o -ltcmalloc_minimal
-osd_test: osd_test.cpp osd_ops.h rw_blocking.o
-	g++ $(CXXFLAGS) -o $@ osd_test.cpp rw_blocking.o -ltcmalloc_minimal
-osd_peering_pg_test: osd_peering_pg_test.cpp osd_peering_pg.o
-	g++ $(CXXFLAGS) -o $@ $< osd_peering_pg.o -ltcmalloc_minimal
-
-libfio_sec_osd.so: fio_sec_osd.o rw_blocking.o
-	g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o $@ fio_sec_osd.o rw_blocking.o
-
-FIO_CLUSTER_OBJS := cluster_client.o epoll_manager.o etcd_state_client.o \
-	messenger.o msgr_send.o msgr_receive.o ringloop.o json11.o http_client.o osd_ops.o pg_states.o timerfd_manager.o base64.o
-libfio_cluster.so: fio_cluster.o $(FIO_CLUSTER_OBJS)
-	g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o $@ $< $(FIO_CLUSTER_OBJS) -luring
-
-nbd_proxy: nbd_proxy.o $(FIO_CLUSTER_OBJS)
-	g++ $(CXXFLAGS) -ltcmalloc_minimal -o $@ $< $(FIO_CLUSTER_OBJS) -luring
-
-rm_inode: rm_inode.o $(FIO_CLUSTER_OBJS)
-	g++ $(CXXFLAGS) -ltcmalloc_minimal -o $@ $< $(FIO_CLUSTER_OBJS) -luring
-
-qemu_driver.o: qemu_driver.c qemu_proxy.h
-	gcc -I qemu/b/qemu `pkg-config glib-2.0 --cflags` \
-		-I qemu/include $(CXXFLAGS) -c -o $@ $<
-
-qemu_driver.so: qemu_driver.o qemu_proxy.o $(FIO_CLUSTER_OBJS)
-	g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o $@ $< $(FIO_CLUSTER_OBJS) qemu_driver.o qemu_proxy.o -luring
-
-test_blockstore: ./libblockstore.so test_blockstore.cpp timerfd_interval.o
-	g++ $(CXXFLAGS) -o test_blockstore test_blockstore.cpp timerfd_interval.o ./libblockstore.so -ltcmalloc_minimal -luring
-test: test.cpp osd_peering_pg.o
-	g++ $(CXXFLAGS) -o test test.cpp osd_peering_pg.o -luring -lm
-test_allocator: test_allocator.cpp allocator.o
-	g++ $(CXXFLAGS) -o test_allocator test_allocator.cpp allocator.o
-
-crc32c.o: crc32c.c crc32c.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-json11.o: json11/json11.cpp
-	g++ $(CXXFLAGS) -c -o json11.o json11/json11.cpp
-
-# Autogenerated
-
-allocator.o: allocator.cpp allocator.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-base64.o: base64.cpp base64.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore.o: blockstore.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_flush.o: blockstore_flush.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_impl.o: blockstore_impl.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_init.o: blockstore_init.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_journal.o: blockstore_journal.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_open.o: blockstore_open.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_read.o: blockstore_read.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_rollback.o: blockstore_rollback.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_stable.o: blockstore_stable.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_sync.o: blockstore_sync.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-blockstore_write.o: blockstore_write.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-cluster_client.o: cluster_client.cpp cluster_client.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-dump_journal.o: dump_journal.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-epoll_manager.o: epoll_manager.cpp epoll_manager.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-etcd_state_client.o: etcd_state_client.cpp base64.h etcd_state_client.h http_client.h json11/json11.hpp object_id.h osd_id.h osd_ops.h pg_states.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-fio_cluster.o: fio_cluster.cpp cluster_client.h epoll_manager.h etcd_state_client.h fio/arch/arch.h fio/fio.h fio/optgroup.h fio_headers.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-fio_engine.o: fio_engine.cpp blockstore.h fio/arch/arch.h fio/fio.h fio/optgroup.h fio_headers.h json11/json11.hpp object_id.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-fio_sec_osd.o: fio_sec_osd.cpp fio/arch/arch.h fio/fio.h fio/optgroup.h fio_headers.h object_id.h osd_id.h osd_ops.h rw_blocking.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-http_client.o: http_client.cpp http_client.h json11/json11.hpp timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-messenger.o: messenger.cpp json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-msgr_receive.o: msgr_receive.cpp json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-msgr_send.o: msgr_send.cpp json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-nbd_proxy.o: nbd_proxy.cpp cluster_client.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd.o: osd.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_cluster.o: osd_cluster.cpp base64.h blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_flush.o: osd_flush.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_main.o: osd_main.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_ops.o: osd_ops.cpp object_id.h osd_id.h osd_ops.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_peering.o: osd_peering.cpp base64.h blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_peering_pg.o: osd_peering_pg.cpp cpp-btree/btree_map.h object_id.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_peering_pg_test.o: osd_peering_pg_test.cpp cpp-btree/btree_map.h object_id.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_primary.o: osd_primary.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h osd_primary.h osd_rmw.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_primary_subops.o: osd_primary_subops.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h osd_primary.h osd_rmw.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_rmw.o: osd_rmw.cpp malloc_or_die.h object_id.h osd_id.h osd_rmw.h xor.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_rmw_test.o: osd_rmw_test.cpp malloc_or_die.h object_id.h osd_id.h osd_rmw.cpp osd_rmw.h test_pattern.h xor.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_secondary.o: osd_secondary.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_test.o: osd_test.cpp object_id.h osd_id.h osd_ops.h rw_blocking.h test_pattern.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-pg_states.o: pg_states.cpp pg_states.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-qemu_proxy.o: qemu_proxy.cpp cluster_client.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h qemu_proxy.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-ringloop.o: ringloop.cpp ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-rm_inode.o: rm_inode.cpp cluster_client.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-rw_blocking.o: rw_blocking.cpp rw_blocking.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-stub_bench.o: stub_bench.cpp object_id.h osd_id.h osd_ops.h rw_blocking.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-stub_osd.o: stub_osd.cpp object_id.h osd_id.h osd_ops.h rw_blocking.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-stub_uring_osd.o: stub_uring_osd.cpp epoll_manager.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-test.o: test.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-test_allocator.o: test_allocator.cpp allocator.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-test_blockstore.o: test_blockstore.cpp blockstore.h object_id.h ringloop.h timerfd_interval.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-timerfd_interval.o: timerfd_interval.cpp ringloop.h timerfd_interval.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-timerfd_manager.o: timerfd_manager.cpp timerfd_manager.h
-	g++ $(CXXFLAGS) -c -o $@ $<

README.md

@@ -16,7 +16,8 @@ breaking changes in the future. However, the following is implemented:
 
 - Basic part: highly-available block storage with symmetric clustering and no SPOF
 - Performance ;-D
-- Two redundancy schemes: Replication and XOR n+1 (simplest case of EC)
+- Multiple redundancy schemes: Replication, XOR n+1, Reed-Solomon erasure codes
+  based on jerasure library with any number of data and parity drives in a group
 - Configuration via simple JSON data structures in etcd
 - Automatic data distribution over OSDs, with support for:
   - Mathematical optimization for better uniformity and less data movement
@@ -31,16 +32,14 @@ breaking changes in the future. However, the following is implemented:
 - QEMU driver (built out-of-tree)
 - Loadable fio engine for benchmarks (also built out-of-tree)
 - NBD proxy for kernel mounts
-- Inode removal tool (./rm_inode)
+- Inode removal tool (vitastor-rm)
+- Packaging for Debian and CentOS
 
 ## Roadmap
 
-- Packaging for Debian and, probably, CentOS too
 - OSD creation tool (OSDs currently have to be created by hand)
 - Other administrative tools
 - Per-inode I/O and space usage statistics
-- jerasure EC support with any number of data and parity drives in a group
-- Parallel usage of multiple network interfaces
 - Proxmox and OpenNebula plugins
 - iSCSI proxy
 - Inode metadata storage in etcd
@@ -50,6 +49,7 @@ breaking changes in the future. However, the following is implemented:
 - Checksums
 - SSD+HDD optimizations, possibly including tiered storage and soft journal flushes
 - RDMA and NVDIMM support
+- Web GUI
 - Compression (possibly)
 - Read caching using system page cache (possibly)
 
@@ -280,7 +280,34 @@ Vitastor with single-thread NBD on the same hardware:
 - Linear write (4M T1Q128): 1266 MB/s (compared to 2800 MB/s via fio)
 - Linear read (4M T1Q128): 975 MB/s (compared to 1500 MB/s via fio)
 
-## Building
+## Installation
+
+### Debian
+
+- Trust Vitastor package signing key:
+  `wget -q -O - https://vitastor.io/debian/pubkey | sudo apt-key add -`
+- Add Vitastor package repository to your /etc/apt/sources.list:
+  - Debian 11 (Bullseye/Sid): `deb https://vitastor.io/debian bullseye main`
+  - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
+- For Debian 10 (Buster) also enable backports repository:
+  `deb http://deb.debian.org/debian buster-backports main`
+- Install packages: `apt update; apt install vitastor lp-solve etcd linux-image-amd64`
+
+### CentOS
+
+- Add Vitastor package repository:
+  - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release-1.0-1.el7.noarch.rpm`
+  - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release-1.0-1.el8.noarch.rpm`
+- Enable EPEL: `yum/dnf install epel-release`
+- Enable additional CentOS repositories:
+  - CentOS 7: `yum install centos-release-scl`
+  - CentOS 8: `dnf install centos-release-advanced-virtualization`
+- Enable elrepo-kernel:
+  - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
+  - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
+- Install packages: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
+
+### Building from Source
 
 - Install Linux kernel 5.4 or newer, for io_uring support. 5.8 or later is highly recommended because
   there is at least one known io_uring hang with 5.4 and an HP SmartArray controller.
@@ -290,10 +317,10 @@ Vitastor with single-thread NBD on the same hardware:
   branch release-3.4, because there is a bug in upstream etcd which makes Vitastor OSDs fail to
   move PGs out of "starting" state if you have at least around ~500 PGs or so. The custom build
   will be unnecessary when etcd merges the fix: https://github.com/etcd-io/etcd/pull/12402.
-- Install node.js 12 or newer.
+- Install node.js 10 or newer.
-- Install gcc and g++ 9.x or later.
+- Install gcc and g++ 8.x or newer.
 - Clone https://yourcmc.ru/git/vitalif/vitastor/ with submodules.
-- Install QEMU 4.x or 5.x, get its source, begin to build it, stop the build and copy headers:
+- Install QEMU 3.0+, get its source, begin to build it, stop the build and copy headers:
    - `<qemu>/include` &rarr; `<vitastor>/qemu/include`
    - Debian:
       * Use qemu packages from the main repository
@@ -303,11 +330,14 @@ Vitastor with single-thread NBD on the same hardware:
       * Use qemu packages from the Advanced-Virtualization repository. To enable it, run
         `yum install centos-release-advanced-virtualization.noarch` and then `yum install qemu`
       * `<qemu>/config-host.h` &rarr; `<vitastor>/qemu/b/qemu/config-host.h`
-      * `<qemu>/qapi` &rarr; `<vitastor>/qemu/b/qemu/qapi`
+      * For QEMU 3.0+: `<qemu>/qapi` &rarr; `<vitastor>/qemu/b/qemu/qapi`
+      * For QEMU 2.0+: `<qemu>/qapi-types.h` &rarr; `<vitastor>/qemu/b/qemu/qapi-types.h`
    - `config-host.h` and `qapi` are required because they contain generated headers
-- Install fio 3.16 or later, get its source and symlink it into `<vitastor>/fio`.
+- You can also rebuild QEMU with a patch that makes LD_PRELOAD unnecessary to load vitastor driver.
-- Build Vitastor with `make -j8`.
+  See `qemu-*.*-vitastor.patch`.
-- Copy binaries somewhere.
+- Install fio 3.7 or later, get its source and symlink it into `<vitastor>/fio`.
+- Build & install Vitastor with `mkdir build && cd build && cmake .. && make -j8 && make install`.
+  Pay attention to the `QEMU_PLUGINDIR` cmake option - it must be set to `qemu-kvm` on RHEL.
 
 ## Running
 
@@ -318,20 +348,16 @@ and calculate disk offsets almost by hand. This will be fixed in near future.
   with lazy fsync, but prepare for inferior single-thread latency.
 - Get a fast network (at least 10 Gbit/s).
 - Disable CPU powersaving: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
-- Start etcd with `--max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision` options.
+- Check `/usr/lib/vitastor/mon/make-units.sh` and `/usr/lib/vitastor/mon/make-osd.sh` and
-- Create global configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`
+  put desired values into the variables at the top of these files.
-  (if all your drives have capacitors).
+- Create systemd units for the monitor and etcd: `/usr/lib/vitastor/mon/make-units.sh`
-- Create pool configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":256,"failure_domain":"host"}}'`.
+- Create systemd units for your OSDs: `/usr/lib/vitastor/mon/make-osd.sh /dev/disk/by-partuuid/XXX [/dev/disk/by-partuuid/YYY ...]`
-- Calculate offsets for your drives with `node ./mon/simple-offsets.js --device /dev/sdX`.
+- You can edit the units and change OSD configuration. Notable configuration variables:
-- Make systemd units for your OSDs. Look at `./mon/make-units.sh` for example.
-  Notable configuration variables from the example:
   - `disable_data_fsync 1` - only safe with server-grade drives with capacitors.
   - `immediate_commit all` - use this if all your drives are server-grade.
   - `disable_device_lock 1` - only required if you run multiple OSDs on one block device.
-  - `flusher_count 16` - flusher is a micro-thread that removes old data from the journal.
+  - `flusher_count 256` - flusher is a micro-thread that removes old data from the journal.
-    More flushers mean more aggressive journal flushing which allows for more throughput
+    You don't have to worry about this parameter anymore, 256 is enough.
-    but slightly hurts latency under less load. Flushing will probably be improved in the future
-    because currently high queue depths sometimes lead to performance degradation.
   - `disk_alignment`, `journal_block_size`, `meta_block_size` should be set to the internal
     block size of your SSDs which is 4096 on most drives.
   - `journal_no_same_sector_overwrites true` prevents multiple overwrites of the same journal sector.
@@ -342,35 +368,36 @@ and calculate disk offsets almost by hand. This will be fixed in near future.
     setting is set, it is also required to raise `journal_sector_buffer_count` setting, which is the
     number of dirty journal sectors that may be written to at the same time.
 - `systemctl start vitastor.target` everywhere.
-- Start any number of monitors: `cd mon; node mon-main.js --etcd_url 'http://10.115.0.10:2379,http://10.115.0.11:2379,http://10.115.0.12:2379,http://10.115.0.13:2379' --etcd_prefix '/vitastor' --etcd_start_timeout 5`.
+- Create global configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`
+  (if all your drives have capacitors).
+- Create pool configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":256,"failure_domain":"host"}}'`.
+  For jerasure pools the configuration should look like the following: `2:{"name":"ecpool","scheme":"jerasure","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}`.
 - At this point, one of the monitors will configure PGs and OSDs will start them.
 - You can check PG states with `etcdctl --endpoints=... get --prefix /vitastor/pg/state`. All PGs should become 'active'.
-- Run tests with (for example): `fio -thread -ioengine=./libfio_cluster.so -name=test -bs=4M -direct=1 -iodepth=16 -rw=write -etcd=10.115.0.10:2379/v3 -pool=1 -inode=1 -size=400G`.
+- Run tests with (for example): `fio -thread -ioengine=libfio_vitastor.so -name=test -bs=4M -direct=1 -iodepth=16 -rw=write -etcd=10.115.0.10:2379/v3 -pool=1 -inode=1 -size=400G`.
 - Upload VM disk image with qemu-img (for example):
   ```
-  LD_PRELOAD=./qemu_driver.so qemu-img convert -f qcow2 debian10.qcow2 -p
+  qemu-img convert -f qcow2 debian10.qcow2 -p -O raw 'vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648'
-    -O raw 'vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648'
   ```
+  Note that the command requires to be run with `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so qemu-img ...`
+  if you use unmodified QEMU.
 - Run QEMU with (for example):
   ```
-  LD_PRELOAD=./qemu_driver.so qemu-system-x86_64 -enable-kvm -m 1024
+  qemu-system-x86_64 -enable-kvm -m 1024
     -drive 'file=vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648',format=raw,if=none,id=drive-virtio-disk0,cache=none
     -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1,write-cache=off,physical_block_size=4096,logical_block_size=512
     -vnc 0.0.0.0:0
   ```
 - Remove inode with (for example):
   ```
-  ./rm_inode --etcd_address 10.115.0.10:2379/v3 --pool 1 --inode 1 --parallel_osds 16 --iodepth 32
+  vitastor-rm --etcd_address 10.115.0.10:2379/v3 --pool 1 --inode 1 --parallel_osds 16 --iodepth 32
   ```
 
 ## Known Problems
 
 - Object deletion requests may currently lead to 'incomplete' objects if your OSDs crash during
   deletion because proper handling of object cleanup in a cluster should be "three-phase"
-  and it's currently not implemented. Inode removal tool currently can't handle unclean
+  and it's currently not implemented. Just to repeat the removal again in this case.
-  objects, so incomplete objects become undeletable. This will be fixed in near future
-  by allowing the inode removal tool to delete unclean objects. With this problem fixed
-  you'll be able just to repeat the removal again.
 
 ## Implementation Principles
 
@@ -392,22 +419,27 @@ Copyright (c) Vitaliy Filippov (vitalif [at] yourcmc.ru), 2019+
 You can also find me in the Russian Telegram Ceph chat: https://t.me/ceph_ru
 
 All server-side code (OSD, Monitor and so on) is licensed under the terms of
-Vitastor Network Public License 1.0 (VNPL 1.0), a copyleft license based on
+Vitastor Network Public License 1.1 (VNPL 1.1), a copyleft license based on
 GNU GPLv3.0 with the additional "Network Interaction" clause which requires
 opensourcing all programs directly or indirectly interacting with Vitastor
-through a computer network ("Proxy Programs"). Proxy Programs may be made public
+through a computer network and expressly designed to be used in conjunction
-not only under the terms of the same license, but also under the terms of any
+with it ("Proxy Programs"). Proxy Programs may be made public not only under
-GPL-Compatible Free Software License, as listed by the Free Software Foundation.
+the terms of the same license, but also under the terms of any GPL-Compatible
+Free Software License, as listed by the Free Software Foundation.
 This is a stricter copyleft license than the Affero GPL.
 
+Please note that VNPL doesn't require you to open the code of proprietary
+software running inside a VM if it's not specially designed to be used with
+Vitastor.
+
 Basically, you can't use the software in a proprietary environment to provide
 its functionality to users without opensourcing all intermediary components
 standing between the user and Vitastor or purchasing a commercial license
 from the author 😀.
 
 Client libraries (cluster_client and so on) are dual-licensed under the same
-VNPL 1.0 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
+VNPL 1.1 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
 software like QEMU and fio.
 
-You can find the full text of VNPL-1.0 in the file [VNPL-1.0.txt](VNPL-1.0.txt).
+You can find the full text of VNPL-1.1 in the file [VNPL-1.1.txt](VNPL-1.1.txt).
 GPL 2.0 is also included in this repository as [GPL-2.0.txt](GPL-2.0.txt).

VNPL-1.1.txt

@@ -1,7 +1,7 @@
                      VITASTOR NETWORK PUBLIC LICENSE
-                       Version 1, 17 September 2020
+                     Version 1.1,  6 February 2021
 
- Copyright (C) 2020 Vitaliy Filippov <vitalif@yourcmc.ru>
+ Copyright (C) 2021 Vitaliy Filippov <vitalif@yourcmc.ru>
  Everyone is permitted to copy and distribute verbatim copies
  of this license document, but changing it is not allowed.
 
@@ -540,12 +540,15 @@ License would be to refrain entirely from conveying the Program.
 
   13. Remote Network Interaction.
 
-  Notwithstanding any other provision of this License, if you provide
+  A "Proxy Program" means a separate program which is specially designed to
-any user an opportunity to interact with the covered work directly
+be used in conjunction with the covered work and interacts with it directly
-or indirectly through a computer network, an imitation of such network,
+or indirectly through any kind of API (application programming interfaces),
-or an additional program (hereinafter referred to as a "Proxy Program")
+a computer network, an imitation of such network, or another Proxy Program
-that, in turn, interacts with the covered work through a computer network,
+itself.
-an imitation of such network, or another Proxy Program itself,
+
+  Notwithstanding any other provision of this License, if you provide any user
+with an opportunity to interact with the covered work through a computer
+network, an imitation of such network, or any number of "Proxy Programs",
 you must prominently offer that user an opportunity to receive the
 Corresponding Source of the covered work and all Proxy Programs from a
 network server at no charge, through some standard or customary means of

copy-fio-includes.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+gcc -I. -E -o fio_headers.i src/fio_headers.h
+
+rm -rf fio-copy
+for i in `grep -Po 'fio/[^"]+' fio_headers.i | sort | uniq`; do
+    j=${i##fio/}
+    p=$(dirname $j)
+    mkdir -p fio-copy/$p
+    cp $i fio-copy/$j
+done
+
+rm fio_headers.i

copy-qemu-includes.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+#cd qemu
+#debian/rules b/configure-stamp
+#cd b/qemu; make qapi
+
+gcc -I qemu/b/qemu `pkg-config glib-2.0 --cflags` \
+    -I qemu/include -E -o qemu_driver.i src/qemu_driver.c
+
+rm -rf qemu-copy
+for i in `grep -Po 'qemu/[^"]+' qemu_driver.i | sort | uniq`; do
+    j=${i##qemu/}
+    p=$(dirname $j)
+    mkdir -p qemu-copy/$p
+    cp $i qemu-copy/$j
+done
+
+rm qemu_driver.i

debian/build-vitastor-bullseye.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+sed 's/$REL/bullseye/' < vitastor.Dockerfile > ../Dockerfile
+cd ..
+mkdir -p packages
+sudo podman build -v `pwd`/packages:/root/packages -f Dockerfile .
+rm Dockerfile

debian/build-vitastor-buster.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+sed 's/$REL/buster/' < vitastor.Dockerfile > ../Dockerfile
+cd ..
+mkdir -p packages
+sudo podman build -v `pwd`/packages:/root/packages -f Dockerfile .
+rm Dockerfile

debian/changelog

@@ -0,0 +1,17 @@
+vitastor (0.5.6-1) unstable; urgency=medium
+
+  * Bugfixes
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Tue, 02 Feb 2021 23:01:24 +0300
+
+vitastor (0.5.1-1) unstable; urgency=medium
+
+  * Add jerasure support
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Sat, 05 Dec 2020 17:02:26 +0300
+
+vitastor (0.5-1) unstable; urgency=medium
+
+  * First packaging for Debian
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Thu, 05 Nov 2020 02:20:59 +0300

debian/compat

@@ -0,0 +1 @@
+13

debian/control

@@ -0,0 +1,17 @@
+Source: vitastor
+Section: admin
+Priority: optional
+Maintainer: Vitaliy Filippov <vitalif@yourcmc.ru>
+Build-Depends: debhelper, liburing-dev (>= 0.6), g++ (>= 8), libstdc++6 (>= 8), linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev
+Standards-Version: 4.5.0
+Homepage: https://vitastor.io/
+Rules-Requires-Root: no
+
+Package: vitastor
+Architecture: amd64
+Depends: ${shlibs:Depends}, ${misc:Depends}, fio (= ${dep:fio}), qemu (= ${dep:qemu}), nodejs (>= 10), node-sprintf-js, node-ws (>= 7), libjerasure2, lp-solve
+Description: Vitastor, a fast software-defined clustered block storage
+ Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
+ architecturally similar to Ceph which means strong consistency, primary-replication,
+ symmetric clustering and automatic data distribution over any number of drives of any
+ size with configurable redundancy (replication or erasure codes/XOR).

debian/copyright

@@ -0,0 +1,21 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: vitastor
+Upstream-Contact: Vitaliy Filippov <vitalif@yourcmc.ru>
+Source: https://vitastor.io
+
+Files: *
+Copyright: 2019+ Vitaliy Filippov <vitalif@yourcmc.ru>
+License: Multiple licenses VNPL-1.1 and/or GPL-2.0+
+ All server-side code (OSD, Monitor and so on) is licensed under the terms of
+ Vitastor Network Public License 1.1 (VNPL 1.1), a copyleft license based on
+ GNU GPLv3.0 with the additional "Network Interaction" clause which requires
+ opensourcing all programs directly or indirectly interacting with Vitastor
+ through a computer network and expressly designed to be used in conjunction
+ with it ("Proxy Programs"). Proxy Programs may be made public not only under
+ the terms of the same license, but also under the terms of any GPL-Compatible
+ Free Software License, as listed by the Free Software Foundation.
+ This is a stricter copyleft license than the Affero GPL.
+ .
+ Client libraries (cluster_client and so on) are dual-licensed under the same
+ VNPL 1.1 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
+ software like QEMU and fio.

debian/install

@@ -0,0 +1,3 @@
+VNPL-1.1.txt usr/share/doc/vitastor
+GPL-2.0.txt usr/share/doc/vitastor
+mon usr/lib/vitastor

debian/patched-qemu.Dockerfile

@@ -0,0 +1,44 @@
+# Build patched QEMU for Debian Buster or Bullseye/Sid inside a container
+# cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/patched-qemu.Dockerfile .
+
+FROM debian:$REL
+
+WORKDIR /root
+
+RUN if [ "$REL" = "buster" ]; then \
+        echo 'deb http://deb.debian.org/debian buster-backports main' >> /etc/apt/sources.list; \
+        echo >> /etc/apt/preferences; \
+        echo 'Package: *' >> /etc/apt/preferences; \
+        echo 'Pin: release a=buster-backports' >> /etc/apt/preferences; \
+        echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
+    fi; \
+    grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
+    echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
+    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
+
+RUN apt-get update
+RUN apt-get -y install qemu fio liburing1 liburing-dev libgoogle-perftools-dev devscripts
+RUN apt-get -y build-dep qemu
+RUN apt-get -y build-dep fio
+RUN apt-get --download-only source qemu
+RUN apt-get --download-only source fio
+
+ADD qemu-5.0-vitastor.patch qemu-5.1-vitastor.patch /root/vitastor/
+RUN set -e; \
+    mkdir -p /root/packages/qemu-$REL; \
+    rm -rf /root/packages/qemu-$REL/*; \
+    cd /root/packages/qemu-$REL; \
+    dpkg-source -x /root/qemu*.dsc; \
+    if [ -d /root/packages/qemu-$REL/qemu-5.0 ]; then \
+        cp /root/vitastor/qemu-5.0-vitastor.patch /root/packages/qemu-$REL/qemu-5.0/debian/patches; \
+        echo qemu-5.0-vitastor.patch >> /root/packages/qemu-$REL/qemu-5.0/debian/patches/series; \
+    else \
+        cp /root/vitastor/qemu-5.1-vitastor.patch /root/packages/qemu-$REL/qemu-*/debian/patches; \
+        P=`ls -d /root/packages/qemu-$REL/qemu-*/debian/patches`; \
+        echo qemu-5.1-vitastor.patch >> $P/series; \
+    fi; \
+    cd /root/packages/qemu-$REL/qemu-*/; \
+    V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)(~bpo[\d\+]*)?\).*$/$1/')+vitastor1; \
+    DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v $V 'Plug Vitastor block driver'; \
+    DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
+    rm -rf /root/packages/qemu-$REL/qemu-*/

debian/rules

@@ -0,0 +1,9 @@
+#!/usr/bin/make -f
+export DH_VERBOSE = 1
+
+%:
+	dh $@
+
+override_dh_installdeb:
+	cat debian/substvars >> debian/vitastor.substvars
+	dh_installdeb

debian/source/format

@@ -0,0 +1 @@
+3.0 (quilt)

debian/substvars

@@ -0,0 +1,2 @@
+dep:fio=3.16-1
+dep:qemu=1:5.1+dfsg-4+vitastor1

debian/vitastor.Dockerfile

@@ -0,0 +1,67 @@
+# Build Vitastor packages for Debian Buster or Bullseye/Sid inside a container
+# cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/vitastor.Dockerfile .
+
+FROM debian:$REL
+
+WORKDIR /root
+
+RUN if [ "$REL" = "buster" ]; then \
+        echo 'deb http://deb.debian.org/debian buster-backports main' >> /etc/apt/sources.list; \
+        echo >> /etc/apt/preferences; \
+        echo 'Package: *' >> /etc/apt/preferences; \
+        echo 'Pin: release a=buster-backports' >> /etc/apt/preferences; \
+        echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
+    fi; \
+    grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
+    echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
+    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
+
+RUN apt-get update
+RUN apt-get -y install qemu fio liburing1 liburing-dev libgoogle-perftools-dev devscripts
+RUN apt-get -y build-dep qemu
+RUN apt-get -y build-dep fio
+RUN apt-get --download-only source qemu
+RUN apt-get --download-only source fio
+RUN apt-get -y install libjerasure-dev cmake
+
+ADD . /root/vitastor
+RUN set -e -x; \
+    mkdir -p /root/fio-build/; \
+    cd /root/fio-build/; \
+    rm -rf /root/fio-build/*; \
+    dpkg-source -x /root/fio*.dsc; \
+    cd /root/packages/qemu-$REL/; \
+    rm -rf qemu*/; \
+    dpkg-source -x qemu*.dsc; \
+    cd /root/packages/qemu-$REL/qemu*/; \
+    debian/rules b/configure-stamp; \
+    cd b/qemu; \
+    make -j8 qapi/qapi-builtin-types.h; \
+    mkdir -p /root/packages/vitastor-$REL; \
+    rm -rf /root/packages/vitastor-$REL/*; \
+    cd /root/packages/vitastor-$REL; \
+    cp -r /root/vitastor vitastor-0.5.6; \
+    ln -s /root/packages/qemu-$REL/qemu-*/ vitastor-0.5.6/qemu; \
+    ln -s /root/fio-build/fio-*/ vitastor-0.5.6/fio; \
+    cd vitastor-0.5.6; \
+    FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
+    QEMU=$(head -n1 qemu/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
+    sh copy-qemu-includes.sh; \
+    sh copy-fio-includes.sh; \
+    rm qemu fio; \
+    mkdir -p a b debian/patches; \
+    mv qemu-copy b/qemu; \
+    mv fio-copy b/fio; \
+    diff -NaurpbB a b > debian/patches/qemu-fio-headers.patch || true; \
+    echo qemu-fio-headers.patch >> debian/patches/series; \
+    rm -rf a b; \
+    rm -rf /root/packages/qemu-$REL/qemu*/; \
+    echo "dep:fio=$FIO" > debian/substvars; \
+    echo "dep:qemu=$QEMU" >> debian/substvars; \
+    cd /root/packages/vitastor-$REL; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.5.6.orig.tar.xz vitastor-0.5.6; \
+    cd vitastor-0.5.6; \
+    V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
+    DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
+    DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
+    rm -rf /root/packages/vitastor-$REL/vitastor-*/

lambda_size.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <iostream>
 #include <functional>

mon/PGUtil.js

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 module.exports = {
     scale_pg_count,
@@ -8,7 +8,7 @@ module.exports = {
 function scale_pg_count(prev_pgs, prev_pg_history, new_pg_history, new_pg_count)
 {
     const old_pg_count = prev_pgs.length;
-    // Add all possibly intersecting PGs into the history of new PGs
+    // Add all possibly intersecting PGs to the history of new PGs
     if (!(new_pg_count % old_pg_count))
     {
         // New PG count is a multiple of the old PG count
@@ -16,7 +16,7 @@ function scale_pg_count(prev_pgs, prev_pg_history, new_pg_history, new_pg_count)
         for (let i = 0; i < new_pg_count; i++)
         {
             const old_i = Math.floor(new_pg_count / mul);
-            new_pg_history[i] = JSON.parse(JSON.stringify(prev_pg_history[1+old_i]));
+            new_pg_history[i] = prev_pg_history[old_i] ? JSON.parse(JSON.stringify(prev_pg_history[old_i])) : undefined;
         }
     }
     else if (!(old_pg_count % new_pg_count))

mon/afr.js

@@ -1,31 +1,16 @@
 // Functions to calculate Annualized Failure Rate of your cluster
 // if you know AFR of your drives, number of drives, expected rebalance time
 // and replication factor
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see https://yourcmc.ru/git/vitalif/vitastor/src/branch/master/README.md for details) or AGPL-3.0
-
+// Author: Vitaliy Filippov, 2020+
-const { sprintf } = require('sprintf-js');
 
 module.exports = {
     cluster_afr_fullmesh,
     failure_rate_fullmesh,
     cluster_afr,
-    print_cluster_afr,
     c_n_k,
 };
 
-print_cluster_afr({ n_hosts: 4, n_drives: 6, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
-print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, capacity: 4000, speed: 0.1, replicas: 2 });
-print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
-print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, capacity: 4000, speed: 0.1, ec: [ 2, 1 ] });
-print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, ec: [ 2, 1 ] });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, capacity: 8000, speed: 0.02, replicas: 2 });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 2 });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, capacity: 8000, speed: 0.02, replicas: 3 });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3 });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
-print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100, degraded_replacement: 1 });
-
 /******** "FULL MESH": ASSUME EACH OSD COMMUNICATES WITH ALL OTHER OSDS ********/
 
 // Estimate AFR of the cluster
@@ -56,93 +41,38 @@ function failure_rate_fullmesh(n, a, f)
 /******** PGS: EACH OSD ONLY COMMUNICATES WITH <pgs> OTHER OSDs ********/
 
 // <n> hosts of <m> drives of <capacity> GB, each able to backfill at <speed> GB/s,
-// <k> replicas, <pgs> unique peer PGs per OSD
+// <k> replicas, <pgs> unique peer PGs per OSD (~50 for 100 PG-per-OSD in a big cluster)
 //
 // For each of n*m drives: P(drive fails in a year) * P(any of its peers fail in <l*365> next days).
 // More peers per OSD increase rebalance speed (more drives work together to resilver) if you
-// let them finish rebalance BEFORE replacing the failed drive.
+// let them finish rebalance BEFORE replacing the failed drive (degraded_replacement=false).
 // At the same time, more peers per OSD increase probability of any of them to fail!
+// osd_rm=true means that failed OSDs' data is rebalanced over all other hosts,
+// not over the same host as it's in Ceph by default (dead OSDs are marked 'out').
 //
 // Probability of all except one drives in a replica group to fail is (AFR^(k-1)).
 // So with <x> PGs it becomes ~ (x * (AFR*L/365)^(k-1)). Interesting but reasonable consequence
 // is that, with k=2, total failure rate doesn't depend on number of peers per OSD,
 // because it gets increased linearly by increased number of peers to fail
 // and decreased linearly by reduced rebalance time.
-function cluster_afr_pgs({ n_hosts, n_drives, afr_drive, capacity, speed, replicas, pgs = 1, degraded_replacement })
+function cluster_afr({ n_hosts, n_drives, afr_drive, afr_host, capacity, speed, ec, ec_data, ec_parity, replicas, pgs = 1, osd_rm, degraded_replacement, down_out_interval = 600 })
 {
-    pgs = Math.min(pgs, (n_hosts-1)*n_drives/(replicas-1));
+    const pg_size = (ec ? ec_data+ec_parity : replicas);
-    const l = capacity/(degraded_replacement ? 1 : pgs)/speed/86400/365;
+    pgs = Math.min(pgs, (n_hosts-1)*n_drives/(pg_size-1));
-    return 1 - (1 - afr_drive * (1-(1-(afr_drive*l)**(replicas-1))**pgs)) ** (n_hosts*n_drives);
+    const host_pgs = Math.min(pgs*n_drives, (n_hosts-1)*n_drives/(pg_size-1));
-}
+    const resilver_disk = n_drives == 1 || osd_rm ? pgs : (n_drives-1);
-
+    const disk_heal_time = (down_out_interval + capacity/(degraded_replacement ? 1 : resilver_disk)/speed)/86400/365;
-function cluster_afr_pgs_ec({ n_hosts, n_drives, afr_drive, capacity, speed, ec: [ ec_data, ec_parity ], pgs = 1, degraded_replacement })
+    const host_heal_time = (down_out_interval + n_drives*capacity/pgs/speed)/86400/365;
-{
+    const disk_heal_fail = ((afr_drive+afr_host/n_drives)*disk_heal_time);
-    const ec_total = ec_data+ec_parity;
+    const host_heal_fail = ((afr_drive+afr_host/n_drives)*host_heal_time);
-    pgs = Math.min(pgs, (n_hosts-1)*n_drives/(ec_total-1));
+    const disk_pg_fail = ec
-    const l = capacity/(degraded_replacement ? 1 : pgs)/speed/86400/365;
+        ? failure_rate_fullmesh(ec_data+ec_parity-1, disk_heal_fail, ec_parity)
-    return 1 - (1 - afr_drive * (1-(1-failure_rate_fullmesh(ec_total-1, afr_drive*l, ec_parity))**pgs)) ** (n_hosts*n_drives);
+        : disk_heal_fail**(replicas-1);
-}
+    const host_pg_fail = ec
-
+        ? failure_rate_fullmesh(ec_data+ec_parity-1, host_heal_fail, ec_parity)
-// Same as above, but also take server failures into account
+        : host_heal_fail**(replicas-1);
-function cluster_afr_pgs_hosts({ n_hosts, n_drives, afr_drive, afr_host, capacity, speed, replicas, pgs = 1, degraded_replacement })
+    return 1 - ((1 - afr_drive * (1-(1-disk_pg_fail)**pgs)) ** (n_hosts*n_drives))
-{
+        * ((1 - afr_host * (1-(1-host_pg_fail)**host_pgs)) ** n_hosts);
-    let otherhosts = Math.min(pgs, (n_hosts-1)/(replicas-1));
-    pgs = Math.min(pgs, (n_hosts-1)*n_drives/(replicas-1));
-    let pgh = Math.min(pgs*n_drives, (n_hosts-1)*n_drives/(replicas-1));
-    const ld = capacity/(degraded_replacement ? 1 : pgs)/speed/86400/365;
-    const lh = n_drives*capacity/pgs/speed/86400/365;
-    const p1 = ((afr_drive+afr_host*pgs/otherhosts)*lh);
-    const p2 = ((afr_drive+afr_host*pgs/otherhosts)*ld);
-    return 1 - ((1 - afr_host * (1-(1-p1**(replicas-1))**pgh)) ** n_hosts) *
-        ((1 - afr_drive * (1-(1-p2**(replicas-1))**pgs)) ** (n_hosts*n_drives));
-}
-
-function cluster_afr_pgs_ec_hosts({ n_hosts, n_drives, afr_drive, afr_host, capacity, speed, ec: [ ec_data, ec_parity ], pgs = 1, degraded_replacement })
-{
-    const ec_total = ec_data+ec_parity;
-    const otherhosts = Math.min(pgs, (n_hosts-1)/(ec_total-1));
-    pgs = Math.min(pgs, (n_hosts-1)*n_drives/(ec_total-1));
-    const pgh = Math.min(pgs*n_drives, (n_hosts-1)*n_drives/(ec_total-1));
-    const ld = capacity/(degraded_replacement ? 1 : pgs)/speed/86400/365;
-    const lh = n_drives*capacity/pgs/speed/86400/365;
-    const p1 = ((afr_drive+afr_host*pgs/otherhosts)*lh);
-    const p2 = ((afr_drive+afr_host*pgs/otherhosts)*ld);
-    return 1 - ((1 - afr_host * (1-(1-failure_rate_fullmesh(ec_total-1, p1, ec_parity))**pgh)) ** n_hosts) *
-        ((1 - afr_drive * (1-(1-failure_rate_fullmesh(ec_total-1, p2, ec_parity))**pgs)) ** (n_hosts*n_drives));
-}
-
-// Wrapper for 4 above functions
-function cluster_afr(config)
-{
-    if (config.ec && config.afr_host)
-    {
-        return cluster_afr_pgs_ec_hosts(config);
-    }
-    else if (config.ec)
-    {
-        return cluster_afr_pgs_ec(config);
-    }
-    else if (config.afr_host)
-    {
-        return cluster_afr_pgs_hosts(config);
-    }
-    else
-    {
-        return cluster_afr_pgs(config);
-    }
-}
-
-function print_cluster_afr(config)
-{
-    console.log(
-        `${config.n_hosts} nodes with ${config.n_drives} ${sprintf("%.1f", config.capacity/1000)}TB drives`+
-        `, capable to backfill at ${sprintf("%.1f", config.speed*1000)} MB/s, drive AFR ${sprintf("%.1f", config.afr_drive*100)}%`+
-        (config.afr_host ? `, host AFR ${sprintf("%.1f", config.afr_host*100)}%` : '')+
-        (config.ec ? `, EC ${config.ec[0]}+${config.ec[1]}` : `, ${config.replicas} replicas`)+
-        `, ${config.pgs||1} PG per OSD`+
-        (config.degraded_replacement ? `\n...and you don't let the rebalance finish before replacing drives` : '')
-    );
-    console.log('-> '+sprintf("%.7f%%", 100*cluster_afr(config))+'\n');
 }
 
 /******** UTILITY ********/

mon/afr_test.js

@@ -0,0 +1,28 @@
+const { sprintf } = require('sprintf-js');
+const { cluster_afr } = require('./afr.js');
+
+print_cluster_afr({ n_hosts: 4, n_drives: 6, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
+print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0, capacity: 4000, speed: 0.1, replicas: 2 });
+print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
+print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0, capacity: 4000, speed: 0.1, ec: true, ec_data: 2, ec_parity: 1 });
+print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, ec: true, ec_data: 2, ec_parity: 1 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0, capacity: 8000, speed: 0.02, replicas: 2 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 2 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0, capacity: 8000, speed: 0.02, replicas: 3 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
+print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100, degraded_replacement: 1 });
+
+function print_cluster_afr(config)
+{
+    console.log(
+        `${config.n_hosts} nodes with ${config.n_drives} ${sprintf("%.1f", config.capacity/1000)}TB drives`+
+        `, capable to backfill at ${sprintf("%.1f", config.speed*1000)} MB/s, drive AFR ${sprintf("%.1f", config.afr_drive*100)}%`+
+        (config.afr_host ? `, host AFR ${sprintf("%.1f", config.afr_host*100)}%` : '')+
+        (config.ec ? `, EC ${config.ec_data}+${config.ec_parity}` : `, ${config.replicas} replicas`)+
+        `, ${config.pgs||1} PG per OSD`+
+        (config.degraded_replacement ? `\n...and you don't let the rebalance finish before replacing drives` : '')
+    );
+    console.log('-> '+sprintf("%.7f%%", 100*cluster_afr(config))+'\n');
+}

mon/lp-optimizer.js

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 // Data distribution optimizer using linear programming (lp_solve)
 
@@ -58,7 +58,7 @@ async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize =
     }
     const all_weights = Object.assign({}, ...Object.values(osd_tree));
     const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c), 0);
-    const all_pgs = Object.values(random_combinations(osd_tree, pg_size, max_combinations));
+    const all_pgs = Object.values(random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1));
     const pg_per_osd = {};
     for (const pg of all_pgs)
     {
@@ -249,7 +249,7 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
         }
     }
     // Get all combinations
-    let all_pgs = random_combinations(osd_tree, pg_size, max_combinations);
+    let all_pgs = random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1);
     add_valid_previous(osd_tree, prev_weights, all_pgs);
     all_pgs = Object.values(all_pgs);
     const pg_per_osd = {};
@@ -275,6 +275,11 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
     lp += 'max: '+all_pg_names.map(pg_name => (
         prev_weights[pg_name] ? `${pg_size+1}*add_${pg_name} - ${pg_size+1}*del_${pg_name}` : `${pg_size+1-move_weights[pg_name]}*${pg_name}`
     )).join(' + ')+';\n';
+    lp += all_pg_names
+        .map(pg_name => (prev_weights[pg_name] ? `add_${pg_name} - del_${pg_name}` : `${pg_name}`))
+        .join(' + ')+' = '+(pg_count
+            - Object.keys(prev_weights).reduce((a, old_pg_name) => (a + (all_pgs_hash[old_pg_name] ? prev_weights[old_pg_name] : 0)), 0)
+        )+';\n';
     for (const osd in pg_per_osd)
     {
         if (osd !== NO_OSD)
@@ -488,7 +493,8 @@ function extract_osds(osd_tree, levels, osd_level, osds = {})
     return osds;
 }
 
-function random_combinations(osd_tree, pg_size, count)
+// ordered = don't treat (x,y) and (y,x) as equal
+function random_combinations(osd_tree, pg_size, count, ordered)
 {
     let seed = 0x5f020e43;
     let rng = () =>
@@ -516,25 +522,47 @@ function random_combinations(osd_tree, pg_size, count)
                 pg.push(osds[cur_hosts[next_host]][next_osd]);
                 cur_hosts.splice(next_host, 1);
             }
-            while (pg.length < pg_size)
+            const cyclic_pgs = [ pg ];
+            if (ordered)
             {
-                pg.push(NO_OSD);
+                for (let i = 1; i < pg.size; i++)
+                {
+                    cyclic_pgs.push([ ...pg.slice(i), ...pg.slice(0, i) ]);
+                }
+            }
+            for (const pg of cyclic_pgs)
+            {
+                while (pg.length < pg_size)
+                {
+                    pg.push(NO_OSD);
+                }
+                r['pg_'+pg.join('_')] = pg;
             }
-            r['pg_'+pg.join('_')] = pg;
         }
     }
     // Generate purely random combinations
-    restart: while (count > 0)
+    while (count > 0)
     {
         let host_idx = [];
-        for (let i = 0; i < pg_size && i < hosts.length; i++)
+        const cur_hosts = [ ...hosts.map((h, i) => i) ];
+        const max_hosts = pg_size < hosts.length ? pg_size : hosts.length;
+        if (ordered)
         {
-            let start = i > 0 ? host_idx[i-1]+1 : 0;
+            for (let i = 0; i < max_hosts; i++)
-            if (start >= hosts.length)
             {
-                continue restart;
+                const r = rng() % cur_hosts.length;
+                host_idx[i] = cur_hosts[r];
+                cur_hosts.splice(r, 1);
+            }
+        }
+        else
+        {
+            for (let i = 0; i < max_hosts; i++)
+            {
+                const r = rng() % (cur_hosts.length - (max_hosts - i - 1));
+                host_idx[i] = cur_hosts[r];
+                cur_hosts.splice(0, r+1);
             }
-            host_idx[i] = start + rng() % (hosts.length-start);
         }
         let pg = host_idx.map(h => osds[hosts[h]][rng() % osds[hosts[h]].length]);
         while (pg.length < pg_size)

mon/make-osd.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Very simple systemd unit generator for vitastor-osd services
+# Not the final solution yet, mostly for tests
+# Copyright (c) Vitaliy Filippov, 2019+
+# License: MIT
+
+# USAGE: ./make-osd.sh /dev/disk/by-partuuid/xxx [ /dev/disk/by-partuuid/yyy]...
+
+IP_SUBSTR="10.200.1."
+ETCD_HOSTS="etcd0=http://10.200.1.10:2380,etcd1=http://10.200.1.11:2380,etcd2=http://10.200.1.12:2380"
+
+set -e -x
+
+IP=`ip -json a s | jq -r '.[].addr_info[] | select(.local | startswith("'$IP_SUBSTR'")) | .local'`
+[ "$IP" != "" ] || exit 1
+ETCD_MON=$(echo $ETCD_HOSTS | perl -pe 's/:2380/:2379/g; s/etcd\d*=//g;')
+D=`dirname $0`
+
+# Create OSDs on all passed devices
+OSD_NUM=1
+for DEV in $*; do
+
+# Ugly :) -> node.js rework pending
+while true; do
+    ST=$(etcdctl --endpoints="$ETCD_MON" get --print-value-only /vitastor/osd/stats/$OSD_NUM)
+    if [ "$ST" = "" ]; then
+        break
+    fi
+    OSD_NUM=$((OSD_NUM+1))
+done
+etcdctl --endpoints="$ETCD_MON" put /vitastor/osd/stats/$OSD_NUM '{}'
+
+echo Creating OSD $OSD_NUM on $DEV
+
+OPT=`node $D/simple-offsets.js --device $DEV --format options | tr '\n' ' '`
+META=`echo $OPT | grep -Po '(?<=data_offset )\d+'`
+dd if=/dev/zero of=$DEV bs=1048576 count=$(((META+1048575)/1048576)) oflag=direct
+
+cat >/etc/systemd/system/vitastor-osd$OSD_NUM.service <<EOF
+[Unit]
+Description=Vitastor object storage daemon osd.$OSD_NUM
+After=network-online.target local-fs.target time-sync.target
+Wants=network-online.target local-fs.target time-sync.target
+PartOf=vitastor.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitMEMLOCK=infinity
+ExecStart=/usr/bin/vitastor-osd \\
+    --etcd_address $IP:2379/v3 \\
+    --bind_address $IP \\
+    --osd_num $OSD_NUM \\
+    --disable_data_fsync 1 \\
+    --immediate_commit all \\
+    --flusher_count 256 \\
+    --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
+    --journal_no_same_sector_overwrites true \\
+    --journal_sector_buffer_count 1024 \\
+    $OPT
+WorkingDirectory=/
+ExecStartPre=+chown vitastor:vitastor $DEV
+User=vitastor
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=vitastor.target
+EOF
+
+systemctl enable vitastor-osd$OSD_NUM
+
+done

mon/make-units.sh

@@ -1,19 +1,25 @@
 #!/bin/bash
-# Example startup script generator
+# Very simple systemd unit generator for etcd & vitastor-mon services
-# Of course this isn't a production solution yet, this is just for tests
+# Not the final solution yet, mostly for tests
 # Copyright (c) Vitaliy Filippov, 2019+
 # License: MIT
 
-IP=`ip -json a s | jq -r '.[].addr_info[] | select(.broadcast == "10.115.0.255") | .local'`
+# USAGE: ./make-units.sh
 
+IP_SUBSTR="10.200.1."
+ETCD_HOSTS="etcd0=http://10.200.1.10:2380,etcd1=http://10.200.1.11:2380,etcd2=http://10.200.1.12:2380"
+
+# determine IP
+IP=`ip -json a s | jq -r '.[].addr_info[] | select(.local | startswith("'$IP_SUBSTR'")) | .local'`
 [ "$IP" != "" ] || exit 1
+ETCD_NUM=${ETCD_HOSTS/$IP*/}
+[ "$ETCD_NUM" != "$ETCD_HOSTS" ] || exit 1
+ETCD_NUM=$(echo $ETCD_NUM | tr -d -c , | wc -c)
 
-BASE=${IP/*./}
+# etcd
-BASE=$((BASE-10))
-
 useradd etcd
 
-mkdir -p /var/lib/etcd$BASE.etcd
+mkdir -p /var/lib/etcd$ETCD_NUM.etcd
 cat >/etc/systemd/system/etcd.service <<EOF
 [Unit]
 Description=etcd for vitastor
@@ -22,19 +28,18 @@ Wants=network-online.target local-fs.target time-sync.target
 
 [Service]
 Restart=always
-ExecStart=/usr/local/bin/etcd -name etcd$BASE --data-dir /var/lib/etcd$BASE.etcd \\
+ExecStart=/usr/local/bin/etcd -name etcd$ETCD_NUM --data-dir /var/lib/etcd$ETCD_NUM.etcd \\
     --advertise-client-urls http://$IP:2379 --listen-client-urls http://$IP:2379 \\
     --initial-advertise-peer-urls http://$IP:2380 --listen-peer-urls http://$IP:2380 \\
-    --initial-cluster-token vitastor-etcd-1 --initial-cluster etcd0=http://10.115.0.10:2380,etcd1=http://10.115.0.11:2380,etcd2=http://10.115.0.12:2380,etcd3=http://10.115.0.13:2380 \\
+    --initial-cluster-token vitastor-etcd-1 --initial-cluster $ETCD_HOSTS \\
     --initial-cluster-state new --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision
-WorkingDirectory=/var/lib/etcd$BASE.etcd
+WorkingDirectory=/var/lib/etcd$ETCD_NUM.etcd
-ExecStartPre=+chown -R etcd /var/lib/etcd$BASE.etcd
+ExecStartPre=+chown -R etcd /var/lib/etcd$ETCD_NUM.etcd
 User=etcd
 PrivateTmp=false
 TasksMax=infinity
 Restart=always
 StartLimitInterval=0
-StartLimitIntervalSec=0
 RestartSec=10
 
 [Install]
@@ -48,9 +53,7 @@ systemctl start etcd
 useradd vitastor
 chmod 755 /root
 
-BASE=${IP/*./}
+# Vitastor target
-BASE=$(((BASE-10)*12))
-
 cat >/etc/systemd/system/vitastor.target <<EOF
 [Unit]
 Description=vitastor target
@@ -58,116 +61,25 @@ Description=vitastor target
 WantedBy=multi-user.target
 EOF
 
-i=1
+# Monitor unit
-for DEV in `ls /dev/disk/by-id/ | grep ata-INTEL_SSDSC2KB`; do
+ETCD_MON=$(echo $ETCD_HOSTS | perl -pe 's/:2380/:2379/g; s/etcd\d*=//g;')
-    dd if=/dev/zero of=/dev/disk/by-id/$DEV bs=1048576 count=$(((427814912+1048575)/1048576+2))
+cat >/etc/systemd/system/vitastor-mon.service <<EOF
-    dd if=/dev/zero of=/dev/disk/by-id/$DEV bs=1048576 count=$(((427814912+1048575)/1048576+2)) seek=$((1920377991168/1048576))
-cat >/etc/systemd/system/vitastor-osd$((BASE+i)).service <<EOF
 [Unit]
-Description=Vitastor object storage daemon osd.$((BASE+i))
+Description=Vitastor monitor
 After=network-online.target local-fs.target time-sync.target
 Wants=network-online.target local-fs.target time-sync.target
-PartOf=vitastor.target
 
 [Service]
-LimitNOFILE=1048576
+Restart=always
-LimitNPROC=1048576
+ExecStart=node /usr/lib/vitastor/mon/mon-main.js --etcd_url '$ETCD_MON' --etcd_prefix '/vitastor' --etcd_start_timeout 5
-LimitMEMLOCK=infinity
+WorkingDirectory=/
-ExecStart=/root/vitastor/osd \\
-    --etcd_address $IP:2379/v3 \\
-    --bind_address $IP \\
-    --osd_num $((BASE+i)) \\
-    --disable_data_fsync 1 \\
-    --disable_device_lock 1 \\
-    --immediate_commit all \\
-    --flusher_count 8 \\
-    --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
-    --journal_no_same_sector_overwrites true \\
-    --journal_sector_buffer_count 1024 \\
-    --journal_offset 0 \\
-    --meta_offset 16777216 \\
-    --data_offset 427814912 \\
-    --data_size $((1920377991168-427814912)) \\
-    --data_device /dev/disk/by-id/$DEV
-WorkingDirectory=/root/vitastor
-ExecStartPre=+chown vitastor:vitastor /dev/disk/by-id/$DEV
 User=vitastor
 PrivateTmp=false
 TasksMax=infinity
 Restart=always
 StartLimitInterval=0
-StartLimitIntervalSec=0
 RestartSec=10
 
 [Install]
 WantedBy=vitastor.target
 EOF
-    systemctl enable vitastor-osd$((BASE+i))
-    i=$((i+1))
-cat >/etc/systemd/system/vitastor-osd$((BASE+i)).service <<EOF
-[Unit]
-Description=Vitastor object storage daemon osd.$((BASE+i))
-After=network-online.target local-fs.target time-sync.target
-Wants=network-online.target local-fs.target time-sync.target
-PartOf=vitastor.target
-
-[Service]
-LimitNOFILE=1048576
-LimitNPROC=1048576
-LimitMEMLOCK=infinity
-ExecStart=/root/vitastor/osd \\
-    --etcd_address $IP:2379/v3 \\
-    --bind_address $IP \\
-    --osd_num $((BASE+i)) \\
-    --disable_data_fsync 1 \\
-    --immediate_commit all \\
-    --flusher_count 8 \\
-    --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
-    --journal_no_same_sector_overwrites true \\
-    --journal_sector_buffer_count 1024 \\
-    --journal_offset 1920377991168 \\
-    --meta_offset $((1920377991168+16777216)) \\
-    --data_offset $((1920377991168+427814912)) \\
-    --data_size $((1920377991168-427814912)) \\
-    --data_device /dev/disk/by-id/$DEV
-WorkingDirectory=/root/vitastor
-ExecStartPre=+chown vitastor:vitastor /dev/disk/by-id/$DEV
-User=vitastor
-PrivateTmp=false
-TasksMax=infinity
-Restart=always
-StartLimitInterval=0
-StartLimitIntervalSec=0
-RestartSec=10
-
-[Install]
-WantedBy=vitastor.target
-EOF
-    systemctl enable vitastor-osd$((BASE+i))
-    i=$((i+1))
-done
-
-exit
-
-node mon-main.js --etcd_url 'http://10.115.0.10:2379,http://10.115.0.11:2379,http://10.115.0.12:2379,http://10.115.0.13:2379' --etcd_prefix '/vitastor' --etcd_start_timeout 5
-
-podman run -d --network host --restart always -v /var/lib/etcd0.etcd:/etcd0.etcd --name etcd quay.io/coreos/etcd:v3.4.13 etcd -name etcd0 \
-    -advertise-client-urls http://10.115.0.10:2379 -listen-client-urls http://10.115.0.10:2379 \
-    -initial-advertise-peer-urls http://10.115.0.10:2380 -listen-peer-urls http://10.115.0.10:2380 \
-    -initial-cluster-token vitastor-etcd-1 -initial-cluster etcd0=http://10.115.0.10:2380,etcd1=http://10.115.0.11:2380,etcd2=http://10.115.0.12:2380,etcd3=http://10.115.0.13:2380 \
-    -initial-cluster-state new --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision
-
-etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/global '{"immediate_commit":"all"}'
-
-etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":48,"failure_domain":"host"}}'
-
-#let pgs = {};
-#for (let n = 0; n < 48; n++) { let i = n/2 | 0; pgs[1+n] = { osd_set: [ (1+i%12+(i/12 | 0)*24), (1+12+i%12+(i/12 | 0)*24) ], primary: (1+(n%2)*12+i%12+(i/12 | 0)*24) }; };
-#console.log(JSON.stringify({ items: { 1: pgs } }));
-#etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/pgs ...
-
-#    --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
-#    --data_offset 427814912 \\
-
-#    --disk_alignment 4096 --journal_block_size 512 --meta_block_size 512 \\
-#    --data_offset 433434624 \\

mon/mon-main.js

@@ -1,7 +1,7 @@
 #!/usr/bin/node
 
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 const Mon = require('./mon.js');
 

mon/mon.js

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 const http = require('http');
 const crypto = require('crypto');
@@ -9,212 +9,215 @@ const LPOptimizer = require('./lp-optimizer.js');
 const stableStringify = require('./stable-stringify.js');
 const PGUtil = require('./PGUtil.js');
 
+// FIXME document all etcd keys and config variables in the form of JSON schema or similar
+const etcd_allow = new RegExp('^'+[
+    'config/global',
+    'config/node_placement',
+    'config/pools',
+    'config/osd/[1-9]\\d*',
+    'config/pgs',
+    'osd/state/[1-9]\\d*',
+    'osd/stats/[1-9]\\d*',
+    'mon/master',
+    'pg/state/[1-9]\\d*/[1-9]\\d*',
+    'pg/stats/[1-9]\\d*/[1-9]\\d*',
+    'pg/history/[1-9]\\d*/[1-9]\\d*',
+    'stats',
+].join('$|^')+'$');
+
+const etcd_tree = {
+    config: {
+        /* global: {
+            // mon
+            etcd_mon_ttl: 30, // min: 10
+            etcd_mon_timeout: 1000, // ms. min: 0
+            etcd_mon_retries: 5, // min: 0
+            mon_change_timeout: 1000, // ms. min: 100
+            mon_stats_timeout: 1000, // ms. min: 100
+            osd_out_time: 1800, // seconds. min: 0
+            placement_levels: { datacenter: 1, rack: 2, host: 3, osd: 4, ... },
+            // client and osd
+            use_sync_send_recv: false,
+            log_level: 0,
+            block_size: 131072,
+            disk_alignment: 4096,
+            bitmap_granularity: 4096,
+            immediate_commit: false, // 'all' or 'small'
+            client_dirty_limit: 33554432,
+            peer_connect_interval: 5, // seconds. min: 1
+            peer_connect_timeout: 5, // seconds. min: 1
+            up_wait_retry_interval: 500, // ms. min: 50
+            // osd
+            etcd_report_interval: 30, // min: 10
+            run_primary: true,
+            bind_address: "0.0.0.0",
+            bind_port: 0,
+            autosync_interval: 5,
+            client_queue_depth: 128, // unused
+            recovery_queue_depth: 4,
+            readonly: false,
+            print_stats_interval: 3,
+            // blockstore - fixed in superblock
+            block_size,
+            disk_alignment,
+            journal_block_size,
+            meta_block_size,
+            bitmap_granularity,
+            journal_device,
+            journal_offset,
+            journal_size,
+            disable_journal_fsync,
+            data_device,
+            data_offset,
+            data_size,
+            disable_data_fsync,
+            meta_device,
+            meta_offset,
+            disable_meta_fsync,
+            disable_device_lock,
+            // blockstore - configurable
+            flusher_count,
+            inmemory_metadata,
+            inmemory_journal,
+            journal_sector_buffer_count,
+            journal_no_same_sector_overwrites,
+        }, */
+        global: {},
+        /* node_placement: {
+            host1: { level: 'host', parent: 'rack1' },
+            ...
+        }, */
+        node_placement: {},
+        /* pools: {
+            <id>: {
+                name: 'testpool',
+                // jerasure uses Reed-Solomon-Vandermonde codes
+                scheme: 'replicated' | 'xor' | 'jerasure',
+                pg_size: 3,
+                pg_minsize: 2,
+                // number of parity chunks, required for jerasure
+                parity_chunks?: 1,
+                pg_count: 100,
+                failure_domain: 'host',
+                max_osd_combinations: 10000,
+                pg_stripe_size: 4194304,
+                root_node?: 'rack1',
+                // restrict pool to OSDs having all of these tags
+                osd_tags?: 'nvme' | [ 'nvme', ... ],
+            },
+            ...
+        }, */
+        pools: {},
+        osd: {
+            /* <id>: { reweight?: 1, tags?: [ 'nvme', ... ] }, ... */
+        },
+        /* pgs: {
+            hash: string,
+            items: {
+                <pool_id>: {
+                    <pg_id>: {
+                        osd_set: [ 1, 2, 3 ],
+                        primary: 1,
+                        pause: false,
+                    }
+                }
+            }
+        }, */
+        pgs: {},
+    },
+    osd: {
+        state: {
+            /* <osd_num_t>: {
+                state: "up",
+                addresses: string[],
+                host: string,
+                port: uint16_t,
+                primary_enabled: boolean,
+                blockstore_enabled: boolean,
+            }, */
+        },
+        stats: {
+            /* <osd_num_t>: {
+                time: number, // unix time
+                blockstore_ready: boolean,
+                size: uint64_t, // bytes
+                free: uint64_t, // bytes
+                host: string,
+                op_stats: {
+                    <string>: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
+                },
+                subop_stats: {
+                    <string>: { count: uint64_t, usec: uint64_t },
+                },
+                recovery_stats: {
+                    degraded: { count: uint64_t, bytes: uint64_t },
+                    misplaced: { count: uint64_t, bytes: uint64_t },
+                },
+            }, */
+        },
+    },
+    mon: {
+        master: {
+            /* ip: [ string ], */
+        },
+    },
+    pg: {
+        state: {
+            /* <pool_id>: {
+                <pg_id>: {
+                    primary: osd_num_t,
+                    state: ("starting"|"peering"|"incomplete"|"active"|"stopping"|"offline"|
+                        "degraded"|"has_incomplete"|"has_degraded"|"has_misplaced"|"has_unclean"|
+                        "has_invalid"|"left_on_dead")[],
+                }
+            }, */
+        },
+        stats: {
+            /* <pool_id>: {
+                <pg_id>: {
+                    object_count: uint64_t,
+                    clean_count: uint64_t,
+                    misplaced_count: uint64_t,
+                    degraded_count: uint64_t,
+                    incomplete_count: uint64_t,
+                    write_osd_set: osd_num_t[],
+                },
+            }, */
+        },
+        history: {
+            /* <pool_id>: {
+                <pg_id>: {
+                    osd_sets: osd_num_t[][],
+                    all_peers: osd_num_t[],
+                    epoch: uint32_t,
+                },
+            }, */
+        },
+    },
+    stats: {
+        /* op_stats: {
+            <string>: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
+        },
+        subop_stats: {
+            <string>: { count: uint64_t, usec: uint64_t },
+        },
+        recovery_stats: {
+            degraded: { count: uint64_t, bytes: uint64_t },
+            misplaced: { count: uint64_t, bytes: uint64_t },
+        },
+        object_counts: {
+            object: uint64_t,
+            clean: uint64_t,
+            misplaced: uint64_t,
+            degraded: uint64_t,
+            incomplete: uint64_t,
+        }, */
+    },
+};
+
 // FIXME Split into several files
 class Mon
 {
-    // FIXME document all etcd keys and config variables in the form of JSON schema or similar
-    static etcd_allow = new RegExp('^'+[
-        'config/global',
-        'config/node_placement',
-        'config/pools',
-        'config/osd/[1-9]\\d*',
-        'config/pgs',
-        'osd/state/[1-9]\\d*',
-        'osd/stats/[1-9]\\d*',
-        'mon/master',
-        'pg/state/[1-9]\\d*/[1-9]\\d*',
-        'pg/stats/[1-9]\\d*/[1-9]\\d*',
-        'pg/history/[1-9]\\d*/[1-9]\\d*',
-        'stats',
-    ].join('$|^')+'$')
-
-    static etcd_tree = {
-        config: {
-            /* global: {
-                // mon
-                etcd_mon_ttl: 30, // min: 10
-                etcd_mon_timeout: 1000, // ms. min: 0
-                etcd_mon_retries: 5, // min: 0
-                mon_change_timeout: 1000, // ms. min: 100
-                mon_stats_timeout: 1000, // ms. min: 100
-                osd_out_time: 1800, // seconds. min: 0
-                placement_levels: { datacenter: 1, rack: 2, host: 3, osd: 4, ... },
-                // client and osd
-                use_sync_send_recv: false,
-                log_level: 0,
-                block_size: 131072,
-                disk_alignment: 4096,
-                bitmap_granularity: 4096,
-                immediate_commit: false, // 'all' or 'small'
-                client_dirty_limit: 33554432,
-                peer_connect_interval: 5, // seconds. min: 1
-                peer_connect_timeout: 5, // seconds. min: 1
-                up_wait_retry_interval: 500, // ms. min: 50
-                // osd
-                etcd_report_interval: 30, // min: 10
-                run_primary: true,
-                bind_address: "0.0.0.0",
-                bind_port: 0,
-                autosync_interval: 5,
-                client_queue_depth: 128, // unused
-                recovery_queue_depth: 4,
-                readonly: false,
-                print_stats_interval: 3,
-                // blockstore - fixed in superblock
-                block_size,
-                disk_alignment,
-                journal_block_size,
-                meta_block_size,
-                bitmap_granularity,
-                journal_device,
-                journal_offset,
-                journal_size,
-                disable_journal_fsync,
-                data_device,
-                data_offset,
-                data_size,
-                disable_data_fsync,
-                meta_device,
-                meta_offset,
-                disable_meta_fsync,
-                disable_device_lock,
-                // blockstore - configurable
-                flusher_count,
-                inmemory_metadata,
-                inmemory_journal,
-                journal_sector_buffer_count,
-                journal_no_same_sector_overwrites,
-            }, */
-            global: {},
-            /* node_placement: {
-                host1: { level: 'host', parent: 'rack1' },
-                ...
-            }, */
-            node_placement: {},
-            /* pools: {
-                <id>: {
-                    name: 'testpool',
-                    scheme: 'xor',
-                    pg_size: 3,
-                    pg_minsize: 2,
-                    pg_count: 100,
-                    failure_domain: 'host',
-                    max_osd_combinations: 10000,
-                    pg_stripe_size: 4194304,
-                    root_node?: 'rack1',
-                    // restrict pool to OSDs having all of these tags
-                    osd_tags?: 'nvme' | [ 'nvme', ... ],
-                },
-                ...
-            }, */
-            pools: {},
-            osd: {
-                /* <id>: { reweight?: 1, tags?: [ 'nvme', ... ] }, ... */
-            },
-            /* pgs: {
-                hash: string,
-                items: {
-                    <pool_id>: {
-                        <pg_id>: {
-                            osd_set: [ 1, 2, 3 ],
-                            primary: 1,
-                            pause: false,
-                        }
-                    }
-                }
-            }, */
-            pgs: {},
-        },
-        osd: {
-            state: {
-                /* <osd_num_t>: {
-                    state: "up",
-                    addresses: string[],
-                    host: string,
-                    port: uint16_t,
-                    primary_enabled: boolean,
-                    blockstore_enabled: boolean,
-                }, */
-            },
-            stats: {
-                /* <osd_num_t>: {
-                    time: number, // unix time
-                    blockstore_ready: boolean,
-                    size: uint64_t, // bytes
-                    free: uint64_t, // bytes
-                    host: string,
-                    op_stats: {
-                        <string>: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
-                    },
-                    subop_stats: {
-                        <string>: { count: uint64_t, usec: uint64_t },
-                    },
-                    recovery_stats: {
-                        degraded: { count: uint64_t, bytes: uint64_t },
-                        misplaced: { count: uint64_t, bytes: uint64_t },
-                    },
-                }, */
-            },
-        },
-        mon: {
-            master: {
-                /* ip: [ string ], */
-            },
-        },
-        pg: {
-            state: {
-                /* <pool_id>: {
-                    <pg_id>: {
-                        primary: osd_num_t,
-                        state: ("starting"|"peering"|"incomplete"|"active"|"stopping"|"offline"|
-                            "degraded"|"has_incomplete"|"has_degraded"|"has_misplaced"|"has_unclean"|
-                            "has_invalid"|"left_on_dead")[],
-                    }
-                }, */
-            },
-            stats: {
-                /* <pool_id>: {
-                    <pg_id>: {
-                        object_count: uint64_t,
-                        clean_count: uint64_t,
-                        misplaced_count: uint64_t,
-                        degraded_count: uint64_t,
-                        incomplete_count: uint64_t,
-                        write_osd_set: osd_num_t[],
-                    },
-                }, */
-            },
-            history: {
-                /* <pool_id>: {
-                    <pg_id>: {
-                        osd_sets: osd_num_t[][],
-                        all_peers: osd_num_t[],
-                        epoch: uint32_t,
-                    },
-                }, */
-            },
-        },
-        stats: {
-            /* op_stats: {
-                <string>: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
-            },
-            subop_stats: {
-                <string>: { count: uint64_t, usec: uint64_t },
-            },
-            recovery_stats: {
-                degraded: { count: uint64_t, bytes: uint64_t },
-                misplaced: { count: uint64_t, bytes: uint64_t },
-            },
-            object_counts: {
-                object: uint64_t,
-                clean: uint64_t,
-                misplaced: uint64_t,
-                degraded: uint64_t,
-                incomplete: uint64_t,
-            }, */
-        },
-    }
-
     constructor(config)
     {
         // FIXME: Maybe prefer local etcd
@@ -250,7 +253,10 @@ class Mon
         const res = await this.etcd_call('/kv/txn', { success: [
             { requestRange: { key: b64(this.etcd_prefix+'/config/global') } }
         ] }, this.etcd_start_timeout, -1);
-        this.parse_kv(res.responses[0].response_range.kvs[0]);
+        if (res.responses[0].response_range.kvs)
+        {
+            this.parse_kv(res.responses[0].response_range.kvs[0]);
+        }
         this.check_config();
     }
 
@@ -331,6 +337,7 @@ class Mon
                 range_end: b64(this.etcd_prefix+'0'),
                 start_revision: ''+this.etcd_watch_revision,
                 watch_id: 1,
+                progress_notify: true,
             },
         }));
         this.ws.on('message', (msg) =>
@@ -372,7 +379,7 @@ class Mon
                     }
                     if (this.verbose)
                     {
-                        console.log(e);
+                        console.log(JSON.stringify(e));
                     }
                 }
                 if (stats_changed)
@@ -561,19 +568,15 @@ class Mon
                     { requestPut: { key: b64(this.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_cfg)) } },
                 ],
             }, this.config.etcd_mon_timeout, 0);
-            if (!res.succeeded)
+            return false;
-            {
-                return false;
-            }
-            this.state.config.pgs = new_cfg;
         }
         return !has_online;
     }
 
     save_new_pgs_txn(request, pool_id, up_osds, prev_pgs, new_pgs, pg_history)
     {
-        const replicated = this.state.config.pools[pool_id].scheme === 'replicated';
+        const replicated = new_pgs.length && this.state.config.pools[pool_id].scheme === 'replicated';
-        const pg_minsize = this.state.config.pools[pool_id].pg_minsize;
+        const pg_minsize = new_pgs.length && this.state.config.pools[pool_id].pg_minsize;
         const pg_items = {};
         new_pgs.map((osd_set, i) =>
         {
@@ -628,13 +631,21 @@ class Mon
             }
         }
         this.state.config.pgs.items = this.state.config.pgs.items || {};
-        this.state.config.pgs.items[pool_id] = pg_items;
+        if (!new_pgs.length)
+        {
+            delete this.state.config.pgs.items[pool_id];
+        }
+        else
+        {
+            this.state.config.pgs.items[pool_id] = pg_items;
+        }
     }
 
     validate_pool_cfg(pool_id, pool_cfg, warn)
     {
         pool_cfg.pg_size = Math.floor(pool_cfg.pg_size);
         pool_cfg.pg_minsize = Math.floor(pool_cfg.pg_minsize);
+        pool_cfg.parity_chunks = Math.floor(pool_cfg.parity_chunks) || undefined;
         pool_cfg.pg_count = Math.floor(pool_cfg.pg_count);
         pool_cfg.failure_domain = pool_cfg.failure_domain || 'host';
         pool_cfg.max_osd_combinations = Math.floor(pool_cfg.max_osd_combinations) || 10000;
@@ -644,8 +655,14 @@ class Mon
                 console.log('Pool ID '+pool_id+' is invalid');
             return false;
         }
-        if (!pool_cfg.pg_size || pool_cfg.pg_size < 1 ||
+        if (pool_cfg.scheme !== 'xor' && pool_cfg.scheme !== 'replicated' && pool_cfg.scheme !== 'jerasure')
-            pool_cfg.scheme === 'xor' && pool_cfg.pg_size < 3)
+        {
+            if (warn)
+                console.log('Pool '+pool_id+' has invalid coding scheme (one of "xor", "replicated" and "jerasure" required)');
+            return false;
+        }
+        if (!pool_cfg.pg_size || pool_cfg.pg_size < 1 || pool_cfg.pg_size > 256 ||
+            (pool_cfg.scheme === 'xor' || pool_cfg.scheme == 'jerasure') && pool_cfg.pg_size < 3)
         {
             if (warn)
                 console.log('Pool '+pool_id+' has invalid pg_size');
@@ -658,6 +675,18 @@ class Mon
                 console.log('Pool '+pool_id+' has invalid pg_minsize');
             return false;
         }
+        if (pool_cfg.scheme === 'xor' && pool_cfg.parity_chunks != 0 && pool_cfg.parity_chunks != 1)
+        {
+            if (warn)
+                console.log('Pool '+pool_id+' has invalid parity_chunks (must be 1)');
+            return false;
+        }
+        if (pool_cfg.scheme === 'jerasure' && (pool_cfg.parity_chunks < 1 || pool_cfg.parity_chunks > pool_cfg.pg_size-2))
+        {
+            if (warn)
+                console.log('Pool '+pool_id+' has invalid parity_chunks (must be between 1 and pg_size-2)');
+            return false;
+        }
         if (!pool_cfg.pg_count || pool_cfg.pg_count < 1)
         {
             if (warn)
@@ -670,12 +699,6 @@ class Mon
                 console.log('Pool '+pool_id+' has empty name');
             return false;
         }
-        if (pool_cfg.scheme !== 'xor' && pool_cfg.scheme !== 'replicated')
-        {
-            if (warn)
-                console.log('Pool '+pool_id+' has invalid coding scheme (only "xor" and "replicated" are allowed)');
-            return false;
-        }
         if (pool_cfg.max_osd_combinations < 100)
         {
             if (warn)
@@ -739,6 +762,24 @@ class Mon
         {
             // Something has changed
             const etcd_request = { compare: [], success: [] };
+            for (const pool_id in (this.state.config.pgs||{}).items||{})
+            {
+                if (!this.state.config.pools[pool_id])
+                {
+                    // Pool deleted. Delete all PGs, but first stop them.
+                    if (!await this.stop_all_pgs(pool_id))
+                    {
+                        this.schedule_recheck();
+                        return;
+                    }
+                    const prev_pgs = [];
+                    for (const pg in this.state.config.pgs.items[pool_id]||{})
+                    {
+                        prev_pgs[pg-1] = this.state.config.pgs.items[pool_id][pg].osd_set;
+                    }
+                    this.save_new_pgs_txn(etcd_request, pool_id, up_osds, prev_pgs, [], []);
+                }
+            }
             for (const pool_id in this.state.config.pools)
             {
                 const pool_cfg = this.state.config.pools[pool_id];
@@ -770,6 +811,17 @@ class Mon
                         }
                         PGUtil.scale_pg_count(prev_pgs, this.state.pg.history[pool_id]||{}, pg_history, pool_cfg.pg_count);
                     }
+                    for (const pg of prev_pgs)
+                    {
+                        while (pg.length < pool_cfg.pg_size)
+                        {
+                            pg.push(0);
+                        }
+                        while (pg.length > pool_cfg.pg_size)
+                        {
+                            pg.pop();
+                        }
+                    }
                     optimize_result = await LPOptimizer.optimize_change({
                         prev_pgs,
                         osd_tree: pool_tree,
@@ -799,7 +851,7 @@ class Mon
                 this.save_new_pgs_txn(etcd_request, pool_id, up_osds, prev_pgs, optimize_result.int_pgs, pg_history);
             }
             this.state.config.pgs.hash = tree_hash;
-            await this.save_pg_config();
+            await this.save_pg_config(etcd_request);
         }
         else
         {
@@ -922,21 +974,21 @@ class Mon
             for (const op in st.op_stats||{})
             {
                 op_stats[op] = op_stats[op] || { count: 0n, usec: 0n, bytes: 0n };
-                op_stats[op].count += BigInt(st.op_stats.count||0);
+                op_stats[op].count += BigInt(st.op_stats[op].count||0);
-                op_stats[op].usec += BigInt(st.op_stats.usec||0);
+                op_stats[op].usec += BigInt(st.op_stats[op].usec||0);
-                op_stats[op].bytes += BigInt(st.op_stats.bytes||0);
+                op_stats[op].bytes += BigInt(st.op_stats[op].bytes||0);
             }
             for (const op in st.subop_stats||{})
             {
                 subop_stats[op] = subop_stats[op] || { count: 0n, usec: 0n };
-                subop_stats[op].count += BigInt(st.subop_stats.count||0);
+                subop_stats[op].count += BigInt(st.subop_stats[op].count||0);
-                subop_stats[op].usec += BigInt(st.subop_stats.usec||0);
+                subop_stats[op].usec += BigInt(st.subop_stats[op].usec||0);
             }
             for (const op in st.recovery_stats||{})
             {
                 recovery_stats[op] = recovery_stats[op] || { count: 0n, bytes: 0n };
-                recovery_stats[op].count += BigInt(st.recovery_stats.count||0);
+                recovery_stats[op].count += BigInt(st.recovery_stats[op].count||0);
-                recovery_stats[op].bytes += BigInt(st.recovery_stats.bytes||0);
+                recovery_stats[op].bytes += BigInt(st.recovery_stats[op].bytes||0);
             }
         }
         for (const op in op_stats)
@@ -1228,4 +1280,7 @@ function sha1hex(str)
     return hash.digest('hex');
 }
 
+Mon.etcd_allow = etcd_allow;
+Mon.etcd_tree = etcd_tree;
+
 module.exports = Mon;

mon/simple-offsets.js

@@ -4,6 +4,7 @@
 // Simple tool to calculate journal and metadata offsets for a single device
 // Will be replaced by smarter tools in the future
 
+const fs = require('fs').promises;
 const child_process = require('child_process');
 
 async function run()
@@ -15,6 +16,7 @@ async function run()
         device_block_size: 4096,
         journal_offset: 0,
         device_size: 0,
+        format: 'text',
     };
     for (let i = 2; i < process.argv.length; i++)
     {
@@ -24,7 +26,22 @@ async function run()
             i++;
         }
     }
-    const device_size = Number(options.device_size || await system("blockdev --getsize64 "+options.device));
+    if (!options.device)
+    {
+        process.stderr.write('USAGE: nodejs '+process.argv[1]+' --device /dev/sdXXX\n');
+        process.exit(1);
+    }
+    options.device_size = Number(options.device_size);
+    let device_size = options.device_size;
+    if (!device_size)
+    {
+        const st = await fs.stat(options.device);
+        options.device_block_size = st.blksize;
+        if (st.isBlockDevice())
+            device_size = Number(await system("/sbin/blockdev --getsize64 "+options.device))
+        else
+            device_size = st.size;
+    }
     if (!device_size)
     {
         process.stderr.write('Failed to get device size\n');
@@ -32,25 +49,45 @@ async function run()
     }
     options.journal_offset = Math.ceil(options.journal_offset/options.device_block_size)*options.device_block_size;
     const meta_offset = options.journal_offset + Math.ceil(options.journal_size/options.device_block_size)*options.device_block_size;
-    const entries_per_block = Math.floor(options.device_block_size / (24 + options.object_size/options.bitmap_granularity/8));
+    const entries_per_block = Math.floor(options.device_block_size / (24 + 2*options.object_size/options.bitmap_granularity/8));
     const object_count = Math.floor((device_size-meta_offset)/options.object_size);
     const meta_size = Math.ceil(object_count / entries_per_block) * options.device_block_size;
     const data_offset = meta_offset + meta_size;
     const meta_size_fmt = (meta_size > 1024*1024*1024 ? Math.round(meta_size/1024/1024/1024*100)/100+" GB"
         : Math.round(meta_size/1024/1024*100)/100+" MB");
-    process.stdout.write(
+    if (options.format == 'text' || options.format == 'options')
-        `Metadata size: ${meta_size_fmt}\n`+
+    {
-        `Options for the OSD:\n`+
+        if (options.format == 'text')
-        `    --journal_offset ${options.journal_offset}\n`+
+        {
-        `    --meta_offset ${meta_offset}\n`+
+            process.stderr.write(
-        `    --data_offset ${data_offset}\n`+
+                `Metadata size: ${meta_size_fmt}\n`+
-        (options.device_size ? `    --data_size ${device_size-data_offset}\n` : '')
+                `Options for the OSD:\n`
-    );
+            );
+        }
+        process.stdout.write(
+            `    --data_device ${options.device}\n`+
+            `    --journal_offset ${options.journal_offset}\n`+
+            `    --meta_offset ${meta_offset}\n`+
+            `    --data_offset ${data_offset}\n`+
+            (options.device_size ? `    --data_size ${device_size-data_offset}\n` : '')
+        );
+    }
+    else if (options.format == 'env')
+    {
+        process.stdout.write(
+            `journal_offset=${options.journal_offset}\n`+
+            `meta_offset=${meta_offset}\n`+
+            `data_offset=${data_offset}\n`+
+            `data_size=${device_size-data_offset}\n`
+        );
+    }
+    else
+        process.stdout.write('Unknown format: '+options.format);
 }
 
 function system(cmd)
 {
-    return new Promise((ok, no) => child_process.exec(cmd, { maxBuffer: 64*1024*1024 }, (err, stdout, stderr) => (err ? no(err) : ok(stdout))));
+    return new Promise((ok, no) => child_process.exec(cmd, { maxBuffer: 64*1024*1024 }, (err, stdout, stderr) => (err ? no(err.message) : ok(stdout))));
 }
 
-run().catch(console.error);
+run().catch(err => { console.error(err); process.exit(1); });

mon/test-nonuniform.js

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 // Interesting real-world example coming from Ceph with EC and compression enabled.
 // EC parity chunks can't be compressed as efficiently as data chunks,

mon/test-optimize-simple.js

@@ -0,0 +1,25 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+const LPOptimizer = require('./lp-optimizer.js');
+
+async function run()
+{
+    const osd_tree = { a: { 1: 1 }, b: { 2: 1 }, c: { 3: 1 } };
+    let res;
+
+    console.log('16 PGs, size=3');
+    res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 3, pg_count: 16 });
+    LPOptimizer.print_change_stats(res, false);
+
+    console.log('\nReduce PG size to 2');
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs.map(pg => pg.slice(0, 2)), osd_tree, pg_size: 2 });
+    LPOptimizer.print_change_stats(res, false);
+
+    console.log('\nRemove OSD 3');
+    delete osd_tree['c'];
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2 });
+    LPOptimizer.print_change_stats(res, false);
+}
+
+run().catch(console.error);

mon/test-optimize-undersized.js

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 const LPOptimizer = require('./lp-optimizer.js');
 

mon/test-optimize.js

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 const LPOptimizer = require('./lp-optimizer.js');
 

qemu-3.1-vitastor.patch

@@ -0,0 +1,84 @@
+Index: qemu-3.1+dfsg/qapi/block-core.json
+===================================================================
+--- qemu-3.1+dfsg.orig/qapi/block-core.json
++++ qemu-3.1+dfsg/qapi/block-core.json
+@@ -2617,7 +2617,7 @@
+ ##
+ { 'enum': 'BlockdevDriver',
+   'data': [ 'blkdebug', 'blklogwrites', 'blkverify', 'bochs', 'cloop',
+-            'copy-on-read', 'dmg', 'file', 'ftp', 'ftps', 'gluster',
++            'copy-on-read', 'dmg', 'file', 'ftp', 'ftps', 'gluster', 'vitastor',
+             'host_cdrom', 'host_device', 'http', 'https', 'iscsi', 'luks',
+             'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels', 'qcow',
+             'qcow2', 'qed', 'quorum', 'raw', 'rbd', 'replication', 'sheepdog',
+@@ -3367,6 +3367,24 @@
+             '*tag': 'str' } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @etcd_host:   etcd connection address
++# @etcd_prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { 'inode': 'uint64',
++            'pool': 'uint64',
++            'size': 'uint64',
++            'etcd_host': 'str',
++            '*etcd_prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -3713,6 +3731,7 @@
+       'rbd':        'BlockdevOptionsRbd',
+       'replication':'BlockdevOptionsReplication',
+       'sheepdog':   'BlockdevOptionsSheepdog',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'ssh':        'BlockdevOptionsSsh',
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+@@ -4158,6 +4177,17 @@
+             '*block-state-zero':    'bool' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVpcSubformat:
+ #
+ # @dynamic: Growing image file
+@@ -4212,6 +4242,7 @@
+       'qed':            'BlockdevCreateOptionsQed',
+       'rbd':            'BlockdevCreateOptionsRbd',
+       'sheepdog':       'BlockdevCreateOptionsSheepdog',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
+Index: qemu-3.1+dfsg/scripts/modules/module_block.py
+===================================================================
+--- qemu-3.1+dfsg.orig/scripts/modules/module_block.py
++++ qemu-3.1+dfsg/scripts/modules/module_block.py
+@@ -88,6 +88,7 @@ def print_bottom(fheader):
+ output_file = sys.argv[1]
+ with open(output_file, 'w') as fheader:
+     print_top(fheader)
++    add_module(fheader, "vitastor", "vitastor", "vitastor")
+ 
+     for filename in sys.argv[2:]:
+         if os.path.isfile(filename):

qemu-4.2-vitastor.patch

@@ -0,0 +1,84 @@
+Index: qemu/qapi/block-core.json
+===================================================================
+--- qemu.orig/qapi/block-core.json	2020-11-07 22:57:38.932613674 +0000
++++ qemu.orig/qapi/block-core.json	2020-11-07 22:59:49.890722862 +0000
+@@ -2907,7 +2907,7 @@
+             'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels', 'qcow',
+             'qcow2', 'qed', 'quorum', 'raw', 'rbd',
+             { 'name': 'replication', 'if': 'defined(CONFIG_REPLICATION)' },
+-            'sheepdog',
++            'sheepdog', 'vitastor',
+             'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat', 'vxhs' ] }
+ 
+ ##
+@@ -3725,6 +3725,24 @@
+             '*tag': 'str' } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @etcd_host:   etcd connection address
++# @etcd_prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { 'inode': 'uint64',
++            'pool': 'uint64',
++            'size': 'uint64',
++            'etcd_host': 'str',
++            '*etcd_prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -4084,6 +4102,7 @@
+       'replication': { 'type': 'BlockdevOptionsReplication',
+                        'if': 'defined(CONFIG_REPLICATION)' },
+       'sheepdog':   'BlockdevOptionsSheepdog',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'ssh':        'BlockdevOptionsSsh',
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+@@ -4461,6 +4480,17 @@
+             '*cluster-size' :   'size' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -4722,6 +4752,7 @@
+       'qed':            'BlockdevCreateOptionsQed',
+       'rbd':            'BlockdevCreateOptionsRbd',
+       'sheepdog':       'BlockdevCreateOptionsSheepdog',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
+Index: qemu/scripts/modules/module_block.py
+===================================================================
+--- qemu.orig/scripts/modules/module_block.py	2020-11-07 22:57:38.936613739 +0000
++++ qemu/scripts/modules/module_block.py	2020-11-07 22:59:49.890722862 +0000
+@@ -86,6 +86,7 @@ def print_bottom(fheader):
+ output_file = sys.argv[1]
+ with open(output_file, 'w') as fheader:
+     print_top(fheader)
++    add_module(fheader, "vitastor", "vitastor", "vitastor")
+ 
+     for filename in sys.argv[2:]:
+         if os.path.isfile(filename):

qemu-5.0-vitastor.patch

@@ -0,0 +1,84 @@
+Index: qemu/qapi/block-core.json
+===================================================================
+--- qemu.orig/qapi/block-core.json
++++ qemu/qapi/block-core.json
+@@ -2798,7 +2798,7 @@
+             'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
+             'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
+             { 'name': 'replication', 'if': 'defined(CONFIG_REPLICATION)' },
+-            'sheepdog',
++            'sheepdog', 'vitastor',
+             'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat', 'vxhs' ] }
+ 
+ ##
+@@ -3635,6 +3635,24 @@
+             '*tag': 'str' } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @etcd_host:   etcd connection address
++# @etcd_prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { 'inode': 'uint64',
++            'pool': 'uint64',
++            'size': 'uint64',
++            'etcd_host': 'str',
++            '*etcd_prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -3995,6 +4013,7 @@
+       'replication': { 'type': 'BlockdevOptionsReplication',
+                        'if': 'defined(CONFIG_REPLICATION)' },
+       'sheepdog':   'BlockdevOptionsSheepdog',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'ssh':        'BlockdevOptionsSsh',
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+@@ -4365,6 +4384,17 @@
+             '*cluster-size' :   'size' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -4626,6 +4656,7 @@
+       'qed':            'BlockdevCreateOptionsQed',
+       'rbd':            'BlockdevCreateOptionsRbd',
+       'sheepdog':       'BlockdevCreateOptionsSheepdog',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
+Index: qemu/scripts/modules/module_block.py
+===================================================================
+--- qemu.orig/scripts/modules/module_block.py
++++ qemu/scripts/modules/module_block.py
+@@ -85,6 +85,7 @@ def print_bottom(fheader):
+ output_file = sys.argv[1]
+ with open(output_file, 'w') as fheader:
+     print_top(fheader)
++    add_module(fheader, "vitastor", "vitastor", "vitastor")
+ 
+     for filename in sys.argv[2:]:
+         if os.path.isfile(filename):

qemu-5.1-vitastor.patch

@@ -0,0 +1,84 @@
+Index: qemu-5.1+dfsg/qapi/block-core.json
+===================================================================
+--- qemu-5.1+dfsg.orig/qapi/block-core.json
++++ qemu-5.1+dfsg/qapi/block-core.json
+@@ -2807,7 +2807,7 @@
+             'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
+             'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
+             { 'name': 'replication', 'if': 'defined(CONFIG_REPLICATION)' },
+-            'sheepdog',
++            'sheepdog', 'vitastor',
+             'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat' ] }
+ 
+ ##
+@@ -3644,6 +3644,24 @@
+             '*tag': 'str' } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @etcd_host:   etcd connection address
++# @etcd_prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { 'inode': 'uint64',
++            'pool': 'uint64',
++            'size': 'uint64',
++            'etcd_host': 'str',
++            '*etcd_prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -3988,6 +4006,7 @@
+       'replication': { 'type': 'BlockdevOptionsReplication',
+                        'if': 'defined(CONFIG_REPLICATION)' },
+       'sheepdog':   'BlockdevOptionsSheepdog',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'ssh':        'BlockdevOptionsSsh',
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+@@ -4376,6 +4395,17 @@
+             '*cluster-size' :   'size' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -4637,6 +4667,7 @@
+       'qed':            'BlockdevCreateOptionsQed',
+       'rbd':            'BlockdevCreateOptionsRbd',
+       'sheepdog':       'BlockdevCreateOptionsSheepdog',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
+Index: qemu-5.1+dfsg/scripts/modules/module_block.py
+===================================================================
+--- qemu-5.1+dfsg.orig/scripts/modules/module_block.py
++++ qemu-5.1+dfsg/scripts/modules/module_block.py
+@@ -86,6 +86,7 @@ if __name__ == '__main__':
+     output_file = sys.argv[1]
+     with open(output_file, 'w') as fheader:
+         print_top(fheader)
++        add_module(fheader, "vitastor", "vitastor", "vitastor")
+ 
+         for filename in sys.argv[2:]:
+             if os.path.isfile(filename):

rpm/build-tarball.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# Vitastor depends on QEMU and FIO headers, but QEMU and FIO don't have -devel packages
+# So we have to copy their headers into the source tarball
+
+set -e
+
+VITASTOR=$(dirname $0)
+VITASTOR=$(realpath "$VITASTOR/..")
+
+if [ -d /opt/rh/gcc-toolset-9 ]; then
+    # CentOS 8
+    EL=8
+    . /opt/rh/gcc-toolset-9/enable
+else
+    # CentOS 7
+    EL=7
+    . /opt/rh/devtoolset-9/enable
+fi
+cd ~/rpmbuild/SPECS
+rpmbuild -bp fio.spec
+perl -i -pe 's/^make V=1/exit 0; make V=1/' qemu*.spec
+rpmbuild -bc qemu*.spec
+perl -i -pe 's/^exit 0; make V=1/make V=1/' qemu*.spec
+cd ~/rpmbuild/BUILD/qemu*/
+rm -rf $VITASTOR/qemu $VITASTOR/fio
+mkdir -p $VITASTOR/qemu/b/qemu
+make -j8 config-host.h
+cp config-host.h $VITASTOR/qemu/b/qemu
+cp -r include $VITASTOR/qemu
+if [ -f qapi-schema.json ]; then
+    # QEMU 2.0
+    make qapi-types.h
+    cp qapi-types.h $VITASTOR/qemu/b/qemu
+else
+    # QEMU 3.0+
+    make qapi
+    cp -r qapi $VITASTOR/qemu/b/qemu
+fi
+cd $VITASTOR
+sh copy-qemu-includes.sh
+rm -rf qemu
+mv qemu-copy qemu
+ln -s ~/rpmbuild/BUILD/fio*/ fio
+sh copy-fio-includes.sh
+rm fio
+mv fio-copy fio
+FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
+QEMU=`rpm -qi qemu qemu-kvm | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
+perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
+perl -i -pe 's/(Requires:\s*qemu(?:-kvm)?)([^\n]+)?/$1 = '$QEMU'/' $VITASTOR/rpm/vitastor-el$EL.spec
+tar --transform 's#^#vitastor-0.5.6/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.5.6$(rpm --eval '%dist').tar.gz *

rpm/qemu-el8.Dockerfile

@@ -0,0 +1,31 @@
+# Build packages for CentOS 8 inside a container
+# cd ..; podman build -t qemu-el8 -v `pwd`/packages:/root/packages -f rpm/qemu-el8.Dockerfile .
+
+FROM centos:8
+
+WORKDIR /root
+
+RUN rm -f /etc/yum.repos.d/CentOS-Media.repo
+RUN dnf -y install centos-release-advanced-virtualization epel-release dnf-plugins-core rpm-build
+RUN rm -rf /var/lib/dnf/*; dnf download --disablerepo='*' --enablerepo='centos-advanced-virtualization-source' --source qemu-kvm
+RUN rpm --nomd5 -i qemu*.src.rpm
+RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=PowerTools --spec qemu-kvm.spec
+
+ADD qemu-*-vitastor.patch /root/vitastor/
+
+RUN set -e; \
+    mkdir -p /root/packages/qemu-el8; \
+    rm -rf /root/packages/qemu-el8/*; \
+    rpm --nomd5 -i /root/qemu*.src.rpm; \
+    cd ~/rpmbuild/SPECS; \
+    PN=$(grep ^Patch qemu-kvm.spec | tail -n1 | perl -pe 's/Patch(\d+).*/$1/'); \
+    csplit qemu-kvm.spec "/^Patch$PN/"; \
+    cat xx00 > qemu-kvm.spec; \
+    head -n 1 xx01 >> qemu-kvm.spec; \
+    echo "Patch$((PN+1)): qemu-4.2-vitastor.patch" >> qemu-kvm.spec; \
+    tail -n +2 xx01 >> qemu-kvm.spec; \
+    perl -i -pe 's/(^Release:\s*\d+)/$1.vitastor/' qemu-kvm.spec; \
+    cp /root/vitastor/qemu-4.2-vitastor.patch ~/rpmbuild/SOURCES; \
+    rpmbuild --nocheck -ba qemu-kvm.spec; \
+    cp ~/rpmbuild/RPMS/*/*qemu* /root/packages/qemu-el8/; \
+    cp ~/rpmbuild/SRPMS/*qemu* /root/packages/qemu-el8/

rpm/qemu-kvm-el7.spec.patch

@@ -0,0 +1,257 @@
+--- qemu-kvm.spec.orig	2020-11-09 23:41:03.000000000 +0000
++++ qemu-kvm.spec	2020-12-06 10:44:24.207640963 +0000
+@@ -2,7 +2,7 @@
+ %global SLOF_gittagcommit 899d9883
+ 
+ %global have_usbredir 1
+-%global have_spice    1
++%global have_spice    0
+ %global have_opengl   1
+ %global have_fdt      0
+ %global have_gluster  1
+@@ -56,7 +56,7 @@ Requires: %{name}-block-curl = %{epoch}:
+ Requires: %{name}-block-gluster = %{epoch}:%{version}-%{release} \
+ %endif                                                           \
+ Requires: %{name}-block-iscsi = %{epoch}:%{version}-%{release}   \
+-Requires: %{name}-block-rbd = %{epoch}:%{version}-%{release}     \
++#Requires: %{name}-block-rbd = %{epoch}:%{version}-%{release}     \
+ Requires: %{name}-block-ssh = %{epoch}:%{version}-%{release}
+ 
+ # Macro to properly setup RHEL/RHEV conflict handling
+@@ -67,7 +67,7 @@ Obsoletes: %1-rhev
+ Summary: QEMU is a machine emulator and virtualizer
+ Name: qemu-kvm
+ Version: 4.2.0
+-Release: 29.vitastor%{?dist}.6
++Release: 30.vitastor%{?dist}.6
+ # Epoch because we pushed a qemu-1.0 package. AIUI this can't ever be dropped
+ Epoch: 15
+ License: GPLv2 and GPLv2+ and CC-BY
+@@ -99,8 +99,8 @@ Source30: kvm-s390x.conf
+ Source31: kvm-x86.conf
+ Source32: qemu-pr-helper.service
+ Source33: qemu-pr-helper.socket
+-Source34: 81-kvm-rhel.rules
+-Source35: udev-kvm-check.c
++#Source34: 81-kvm-rhel.rules
++#Source35: udev-kvm-check.c
+ Source36: README.tests
+ 
+ 
+@@ -825,7 +825,9 @@ Patch331: kvm-Drop-bogus-IPv6-messages.p
+ Patch333: kvm-virtiofsd-Whitelist-fchmod.patch
+ # For bz#1883869 - virtiofsd core dump in KATA Container [rhel-8.2.1.z]
+ Patch334: kvm-virtiofsd-avoid-proc-self-fd-tempdir.patch
+-Patch335: qemu-4.2-vitastor.patch
++Patch335: qemu-use-sphinx-1.2.patch
++Patch336: qemu-config-tcmalloc-warning.patch
++Patch337: qemu-4.2-vitastor.patch
+ 
+ BuildRequires: wget
+ BuildRequires: rpm-build
+@@ -842,7 +844,8 @@ BuildRequires: pciutils-devel
+ BuildRequires: libiscsi-devel
+ BuildRequires: ncurses-devel
+ BuildRequires: libattr-devel
+-BuildRequires: libusbx-devel >= 1.0.22
++BuildRequires: gperftools-devel
++BuildRequires: libusbx-devel >= 1.0.21
+ %if %{have_usbredir}
+ BuildRequires: usbredir-devel >= 0.7.1
+ %endif
+@@ -856,12 +859,12 @@ BuildRequires: virglrenderer-devel
+ # For smartcard NSS support
+ BuildRequires: nss-devel
+ %endif
+-BuildRequires: libseccomp-devel >= 2.4.0
++#Requires: libseccomp >= 2.4.0
+ # For network block driver
+ BuildRequires: libcurl-devel
+ BuildRequires: libssh-devel
+-BuildRequires: librados-devel
+-BuildRequires: librbd-devel
++#BuildRequires: librados-devel
++#BuildRequires: librbd-devel
+ %if %{have_gluster}
+ # For gluster block driver
+ BuildRequires: glusterfs-api-devel
+@@ -955,25 +958,25 @@ hardware for a full system such as a PC
+ 
+ %package -n qemu-kvm-core
+ Summary: qemu-kvm core components
++Requires: gperftools-libs
+ Requires: qemu-img = %{epoch}:%{version}-%{release}
+ %ifarch %{ix86} x86_64
+ Requires: seabios-bin >= 1.10.2-1
+ Requires: sgabios-bin
+-Requires: edk2-ovmf
+ %endif
+ %ifarch aarch64
+ Requires: edk2-aarch64
+ %endif
+ 
+ %ifnarch aarch64 s390x
+-Requires: seavgabios-bin >= 1.12.0-3
+-Requires: ipxe-roms-qemu >= 20170123-1
++Requires: seavgabios-bin >= 1.11.0-1
++Requires: ipxe-roms-qemu >= 20181214-1
++Requires: /usr/share/ipxe.efi
+ %endif
+ %ifarch %{power64}
+ Requires: SLOF >= %{SLOF_gittagdate}-1.git%{SLOF_gittagcommit}
+ %endif
+ Requires: %{name}-common = %{epoch}:%{version}-%{release}
+-Requires: libseccomp >= 2.4.0
+ # For compressed guest memory dumps
+ Requires: lzo snappy
+ %if %{have_kvm_setup}
+@@ -1085,15 +1088,15 @@ This package provides the additional iSC
+ Install this package if you want to access iSCSI volumes.
+ 
+ 
+-%package  block-rbd
+-Summary: QEMU Ceph/RBD block driver
+-Requires: %{name}-common%{?_isa} = %{epoch}:%{version}-%{release}
+-
+-%description block-rbd
+-This package provides the additional Ceph/RBD block driver for QEMU.
+-
+-Install this package if you want to access remote Ceph volumes
+-using the rbd protocol.
++#%package  block-rbd
++#Summary: QEMU Ceph/RBD block driver
++#Requires: %{name}-common%{?_isa} = %{epoch}:%{version}-%{release}
++#
++#%description block-rbd
++#This package provides the additional Ceph/RBD block driver for QEMU.
++#
++#Install this package if you want to access remote Ceph volumes
++#using the rbd protocol.
+ 
+ 
+ %package  block-ssh
+@@ -1117,12 +1120,14 @@ the Secure Shell (SSH) protocol.
+ # --build-id option is used for giving info to the debug packages.
+ buildldflags="VL_LDFLAGS=-Wl,--build-id"
+ 
+-%global block_drivers_list qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle
++#%global block_drivers_list qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle
++%global block_drivers_list qcow2,raw,file,host_device,nbd,iscsi,blkdebug,luks,null-co,nvme,copy-on-read,throttle
+ 
+ %if 0%{have_gluster}
+     %global block_drivers_list %{block_drivers_list},gluster
+ %endif
+ 
++[ -e /usr/bin/sphinx-build ] || ln -s sphinx-build-3 /usr/bin/sphinx-build
+ ./configure  \
+  --prefix="%{_prefix}" \
+  --libdir="%{_libdir}" \
+@@ -1152,15 +1157,15 @@ buildldflags="VL_LDFLAGS=-Wl,--build-id"
+ %else
+   --disable-numa \
+ %endif
+-  --enable-rbd \
++  --disable-rbd \
+ %if 0%{have_librdma}
+   --enable-rdma \
+ %else
+   --disable-rdma \
+ %endif
+   --disable-pvrdma \
+-  --enable-seccomp \
+-%if 0%{have_spice}
++  --disable-seccomp \
++%if %{have_spice}
+   --enable-spice \
+   --enable-smartcard \
+   --enable-virglrenderer \
+@@ -1179,7 +1184,7 @@ buildldflags="VL_LDFLAGS=-Wl,--build-id"
+ %else
+   --disable-usb-redir \
+ %endif
+-  --disable-tcmalloc \
++  --enable-tcmalloc \
+ %ifarch x86_64
+   --enable-libpmem \
+ %else
+@@ -1193,9 +1198,7 @@ buildldflags="VL_LDFLAGS=-Wl,--build-id"
+ %endif
+   --python=%{__python3} \
+   --target-list="%{buildarch}" \
+-  --block-drv-rw-whitelist=%{block_drivers_list} \
+   --audio-drv-list= \
+-  --block-drv-ro-whitelist=vmdk,vhdx,vpc,https,ssh \
+   --with-coroutine=ucontext \
+   --tls-priority=NORMAL \
+   --disable-bluez \
+@@ -1262,7 +1265,7 @@ buildldflags="VL_LDFLAGS=-Wl,--build-id"
+   --disable-sanitizers \
+   --disable-hvf \
+   --disable-whpx \
+-  --enable-malloc-trim \
++  --disable-malloc-trim \
+   --disable-membarrier \
+   --disable-vhost-crypto \
+   --disable-libxml2 \
+@@ -1308,7 +1311,7 @@ make V=1 %{?_smp_mflags} $buildldflags
+ cp -a %{kvm_target}-softmmu/qemu-system-%{kvm_target} qemu-kvm
+ 
+ gcc %{SOURCE6} $RPM_OPT_FLAGS $RPM_LD_FLAGS -o ksmctl
+-gcc %{SOURCE35} $RPM_OPT_FLAGS $RPM_LD_FLAGS -o udev-kvm-check
++#gcc %{SOURCE35} $RPM_OPT_FLAGS $RPM_LD_FLAGS -o udev-kvm-check
+ 
+ %install
+ %define _udevdir %(pkg-config --variable=udevdir udev)
+@@ -1343,8 +1346,8 @@ mkdir -p $RPM_BUILD_ROOT%{testsdir}/test
+ mkdir -p $RPM_BUILD_ROOT%{testsdir}/tests/qemu-iotests
+ mkdir -p $RPM_BUILD_ROOT%{testsdir}/scripts/qmp
+ 
+-install -p -m 0755 udev-kvm-check $RPM_BUILD_ROOT%{_udevdir}
+-install -p -m 0644 %{SOURCE34} $RPM_BUILD_ROOT%{_udevrulesdir}
++#install -p -m 0755 udev-kvm-check $RPM_BUILD_ROOT%{_udevdir}
++#install -p -m 0644 %{SOURCE34} $RPM_BUILD_ROOT%{_udevrulesdir}
+ 
+ install -m 0644 scripts/dump-guest-memory.py \
+                 $RPM_BUILD_ROOT%{_datadir}/%{name}
+@@ -1562,6 +1565,8 @@ rm -rf $RPM_BUILD_ROOT%{qemudocdir}/inte
+ # Remove spec
+ rm -rf $RPM_BUILD_ROOT%{qemudocdir}/specs
+ 
++%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')
++
+ %check
+ export DIFF=diff; make check V=1
+ 
+@@ -1645,8 +1650,8 @@ useradd -r -u 107 -g qemu -G kvm -d / -s
+ %config(noreplace) %{_sysconfdir}/sysconfig/ksm
+ %{_unitdir}/ksmtuned.service
+ %{_sbindir}/ksmtuned
+-%{_udevdir}/udev-kvm-check
+-%{_udevrulesdir}/81-kvm-rhel.rules
++#%{_udevdir}/udev-kvm-check
++#%{_udevrulesdir}/81-kvm-rhel.rules
+ %ghost %{_sysconfdir}/kvm
+ %config(noreplace) %{_sysconfdir}/ksmtuned.conf
+ %dir %{_sysconfdir}/%{name}
+@@ -1711,8 +1716,8 @@ useradd -r -u 107 -g qemu -G kvm -d / -s
+ %{_libexecdir}/vhost-user-gpu
+ %{_datadir}/%{name}/vhost-user/50-qemu-gpu.json
+ %endif
+-%{_libexecdir}/virtiofsd
+-%{_datadir}/%{name}/vhost-user/50-qemu-virtiofsd.json
++#%{_libexecdir}/virtiofsd
++#%{_datadir}/%{name}/vhost-user/50-qemu-virtiofsd.json
+ 
+ %files -n qemu-img
+ %defattr(-,root,root)
+@@ -1748,8 +1753,8 @@ useradd -r -u 107 -g qemu -G kvm -d / -s
+ %files block-iscsi
+ %{_libdir}/qemu-kvm/block-iscsi.so
+ 
+-%files block-rbd
+-%{_libdir}/qemu-kvm/block-rbd.so
++#%files block-rbd
++#%{_libdir}/qemu-kvm/block-rbd.so
+ 
+ %files block-ssh
+ %{_libdir}/qemu-kvm/block-ssh.so

rpm/qemu-kvm.spec.patch

@@ -0,0 +1,29 @@
+--- qemu-kvm.spec	2020-12-05 13:13:54.388623517 +0000
++++ qemu-kvm.spec	2020-12-05 13:13:58.728696598 +0000
+@@ -67,7 +67,7 @@ Obsoletes: %1-rhev
+ Summary: QEMU is a machine emulator and virtualizer
+ Name: qemu-kvm
+ Version: 4.2.0
+-Release: 29%{?dist}.6
++Release: 29.vitastor%{?dist}.6
+ # Epoch because we pushed a qemu-1.0 package. AIUI this can't ever be dropped
+ Epoch: 15
+ License: GPLv2 and GPLv2+ and CC-BY
+@@ -825,6 +825,7 @@ Patch331: kvm-Drop-bogus-IPv6-messages.p
+ Patch333: kvm-virtiofsd-Whitelist-fchmod.patch
+ # For bz#1883869 - virtiofsd core dump in KATA Container [rhel-8.2.1.z]
+ Patch334: kvm-virtiofsd-avoid-proc-self-fd-tempdir.patch
++Patch335: qemu-4.2-vitastor.patch
+ 
+ BuildRequires: wget
+ BuildRequires: rpm-build
+@@ -1192,9 +1193,7 @@ buildldflags="VL_LDFLAGS=-Wl,--build-id"
+ %endif
+   --python=%{__python3} \
+   --target-list="%{buildarch}" \
+-  --block-drv-rw-whitelist=%{block_drivers_list} \
+   --audio-drv-list= \
+-  --block-drv-ro-whitelist=vmdk,vhdx,vpc,https,ssh \
+   --with-coroutine=ucontext \
+   --tls-priority=NORMAL \
+   --disable-bluez \

rpm/vitastor-el7.Dockerfile

@@ -0,0 +1,47 @@
+# Build packages for CentOS 7 inside a container
+# cd ..; podman build -t vitastor-el7 -v `pwd`/packages:/root/packages -f rpm/vitastor-el7.Dockerfile .
+# localedef -i ru_RU -f UTF-8 ru_RU.UTF-8
+
+FROM centos:7
+
+WORKDIR /root
+
+RUN rm -f /etc/yum.repos.d/CentOS-Media.repo
+RUN yum -y --enablerepo=extras install centos-release-scl epel-release yum-utils rpm-build
+RUN yum -y install https://vitastor.io/rpms/centos/7/vitastor-release-1.0-1.el7.noarch.rpm
+RUN yum -y install devtoolset-9-gcc-c++ devtoolset-9-libatomic-devel gperftools-devel qemu-kvm fio rh-nodejs12 jerasure-devel gf-complete-devel
+RUN yumdownloader --disablerepo=centos-sclo-rh --source qemu-kvm
+RUN yumdownloader --disablerepo=centos-sclo-rh --source fio
+RUN rpm --nomd5 -i qemu*.src.rpm
+RUN rpm --nomd5 -i fio*.src.rpm
+RUN rm -f /etc/yum.repos.d/CentOS-Media.repo
+RUN cd ~/rpmbuild/SPECS && yum-builddep -y --enablerepo='*' --disablerepo=centos-sclo-rh --disablerepo=centos-sclo-rh-source --disablerepo=centos-sclo-sclo-testing qemu-kvm.spec
+RUN cd ~/rpmbuild/SPECS && yum-builddep -y --enablerepo='*' --disablerepo=centos-sclo-rh --disablerepo=centos-sclo-rh-source --disablerepo=centos-sclo-sclo-testing fio.spec
+
+ADD https://vitastor.io/rpms/liburing-el7/liburing-0.7-2.el7.src.rpm /root
+
+RUN set -e; \
+    rpm -i liburing*.src.rpm; \
+    cd ~/rpmbuild/SPECS/; \
+    . /opt/rh/devtoolset-9/enable; \
+    rpmbuild -ba liburing.spec; \
+    mkdir -p /root/packages/liburing-el7; \
+    rm -rf /root/packages/liburing-el7/*; \
+    cp ~/rpmbuild/RPMS/*/liburing* /root/packages/liburing-el7/; \
+    cp ~/rpmbuild/SRPMS/liburing* /root/packages/liburing-el7/
+
+RUN rpm -i `ls /root/packages/liburing-el7/liburing-*.x86_64.rpm | grep -v debug`
+
+ADD . /root/vitastor
+
+RUN set -e; \
+    cd /root/vitastor/rpm; \
+    sh build-tarball.sh; \
+    cp /root/vitastor-0.5.6.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
+    cd ~/rpmbuild/SPECS/; \
+    rpmbuild -ba vitastor.spec; \
+    mkdir -p /root/packages/vitastor-el7; \
+    rm -rf /root/packages/vitastor-el7/*; \
+    cp ~/rpmbuild/RPMS/*/vitastor* /root/packages/vitastor-el7/; \
+    cp ~/rpmbuild/SRPMS/vitastor* /root/packages/vitastor-el7/

rpm/vitastor-el7.spec

@@ -0,0 +1,69 @@
+Name:           vitastor
+Version:        0.5.6
+Release:        1%{?dist}
+Summary:        Vitastor, a fast software-defined clustered block storage
+
+License:        Vitastor Network Public License 1.1
+URL:            https://vitastor.io/
+Source0:        vitastor-0.5.6.el7.tar.gz
+
+BuildRequires:  liburing-devel >= 0.6
+BuildRequires:  gperftools-devel
+BuildRequires:  devtoolset-9-gcc-c++
+BuildRequires:  rh-nodejs12
+BuildRequires:  rh-nodejs12-npm
+BuildRequires:  jerasure-devel
+BuildRequires:  gf-complete-devel
+BuildRequires:  cmake
+Requires:       fio = 3.7-1.el7
+Requires:       qemu-kvm = 2.0.0-1.el7.6
+Requires:       rh-nodejs12
+Requires:       rh-nodejs12-npm
+Requires:       liburing >= 0.6
+Requires:       libJerasure2
+Requires:       lpsolve
+
+%description
+Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
+architecturally similar to Ceph which means strong consistency, primary-replication,
+symmetric clustering and automatic data distribution over any number of drives of any
+size with configurable redundancy (replication or erasure codes/XOR).
+
+
+%prep
+%setup -q
+
+
+%build
+. /opt/rh/devtoolset-9/enable
+%cmake . -DQEMU_PLUGINDIR=qemu-kvm
+%make_build
+
+
+%install
+rm -rf $RPM_BUILD_ROOT
+%make_install
+. /opt/rh/rh-nodejs12/enable
+cd mon
+npm install
+cd ..
+mkdir -p %buildroot/usr/lib/vitastor
+cp -r mon %buildroot/usr/lib/vitastor/mon
+
+
+%files
+%doc
+%_bindir/vitastor-dump-journal
+%_bindir/vitastor-nbd
+%_bindir/vitastor-osd
+%_bindir/vitastor-rm
+%_libdir/qemu-kvm/block-vitastor.so
+%_libdir/libfio_vitastor.so
+%_libdir/libfio_vitastor_blk.so
+%_libdir/libfio_vitastor_sec.so
+%_libdir/libvitastor_blk.so
+%_libdir/libvitastor_client.so
+/usr/lib/vitastor
+
+
+%changelog

rpm/vitastor-el8.Dockerfile

@@ -0,0 +1,45 @@
+# Build packages for CentOS 8 inside a container
+# cd ..; podman build -t vitastor-el8 -v `pwd`/packages:/root/packages -f rpm/vitastor-el8.Dockerfile .
+
+FROM centos:8
+
+WORKDIR /root
+
+RUN rm -f /etc/yum.repos.d/CentOS-Media.repo
+RUN dnf -y install centos-release-advanced-virtualization epel-release dnf-plugins-core
+RUN yum -y install https://vitastor.io/rpms/centos/8/vitastor-release-1.0-1.el8.noarch.rpm
+RUN dnf --enablerepo='centos-advanced-virtualization' -y install gcc-toolset-9 gcc-toolset-9-gcc-c++ gperftools-devel qemu-kvm fio nodejs rpm-build jerasure-devel gf-complete-devel
+RUN rm -rf /var/lib/dnf/*; dnf download --disablerepo='*' --enablerepo='vitastor' --source qemu-kvm
+RUN dnf download --source fio
+RUN rpm --nomd5 -i qemu*.src.rpm
+RUN rpm --nomd5 -i fio*.src.rpm
+RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=powertools --spec qemu-kvm.spec
+RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=powertools --spec fio.spec && dnf install -y cmake
+
+ADD https://vitastor.io/rpms/liburing-el7/liburing-0.7-2.el7.src.rpm /root
+
+RUN set -e; \
+    rpm -i liburing*.src.rpm; \
+    cd ~/rpmbuild/SPECS/; \
+    . /opt/rh/gcc-toolset-9/enable; \
+    rpmbuild -ba liburing.spec; \
+    mkdir -p /root/packages/liburing-el8; \
+    rm -rf /root/packages/liburing-el8/*; \
+    cp ~/rpmbuild/RPMS/*/liburing* /root/packages/liburing-el8/; \
+    cp ~/rpmbuild/SRPMS/liburing* /root/packages/liburing-el8/
+
+RUN rpm -i `ls /root/packages/liburing-el7/liburing-*.x86_64.rpm | grep -v debug`
+
+ADD . /root/vitastor
+
+RUN set -e; \
+    cd /root/vitastor/rpm; \
+    sh build-tarball.sh; \
+    cp /root/vitastor-0.5.6.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
+    cd ~/rpmbuild/SPECS/; \
+    rpmbuild -ba vitastor.spec; \
+    mkdir -p /root/packages/vitastor-el8; \
+    rm -rf /root/packages/vitastor-el8/*; \
+    cp ~/rpmbuild/RPMS/*/vitastor* /root/packages/vitastor-el8/; \
+    cp ~/rpmbuild/SRPMS/vitastor* /root/packages/vitastor-el8/

rpm/vitastor-el8.spec

@@ -0,0 +1,66 @@
+Name:           vitastor
+Version:        0.5.6
+Release:        1%{?dist}
+Summary:        Vitastor, a fast software-defined clustered block storage
+
+License:        Vitastor Network Public License 1.1
+URL:            https://vitastor.io/
+Source0:        vitastor-0.5.6.el8.tar.gz
+
+BuildRequires:  liburing-devel >= 0.6
+BuildRequires:  gperftools-devel
+BuildRequires:  gcc-toolset-9-gcc-c++
+BuildRequires:  nodejs >= 10
+BuildRequires:  jerasure-devel
+BuildRequires:  gf-complete-devel
+BuildRequires:  cmake
+Requires:       fio = 3.7-3.el8
+Requires:       qemu-kvm = 4.2.0-29.el8.6
+Requires:       nodejs >= 10
+Requires:       liburing >= 0.6
+Requires:       libJerasure2
+Requires:       lpsolve
+
+%description
+Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
+architecturally similar to Ceph which means strong consistency, primary-replication,
+symmetric clustering and automatic data distribution over any number of drives of any
+size with configurable redundancy (replication or erasure codes/XOR).
+
+
+%prep
+%setup -q
+
+
+%build
+. /opt/rh/gcc-toolset-9/enable
+%cmake . -DQEMU_PLUGINDIR=qemu-kvm
+%make_build
+
+
+%install
+rm -rf $RPM_BUILD_ROOT
+%make_install
+cd mon
+npm install
+cd ..
+mkdir -p %buildroot/usr/lib/vitastor
+cp -r mon %buildroot/usr/lib/vitastor
+
+
+%files
+%doc
+%_bindir/vitastor-dump-journal
+%_bindir/vitastor-nbd
+%_bindir/vitastor-osd
+%_bindir/vitastor-rm
+%_libdir/qemu-kvm/block-vitastor.so
+%_libdir/libfio_vitastor.so
+%_libdir/libfio_vitastor_blk.so
+%_libdir/libfio_vitastor_sec.so
+%_libdir/libvitastor_blk.so
+%_libdir/libvitastor_client.so
+/usr/lib/vitastor
+
+
+%changelog

src/CMakeLists.txt

@@ -0,0 +1,188 @@
+cmake_minimum_required(VERSION 2.8)
+
+project(vitastor)
+
+include(GNUInstallDirs)
+
+set(QEMU_PLUGINDIR qemu CACHE STRING "QEMU plugin directory suffix (qemu-kvm on RHEL)")
+set(WITH_ASAN false CACHE BOOL "Build with AddressSanitizer")
+if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
+	if(EXISTS "/etc/debian_version")
+		set(CMAKE_INSTALL_LIBDIR "lib/${CMAKE_LIBRARY_ARCHITECTURE}")
+	endif()
+	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+endif()
+
+add_definitions(-DVERSION="0.6-dev")
+add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith)
+if (${WITH_ASAN})
+	add_definitions(-fsanitize=address -fno-omit-frame-pointer)
+	add_link_options(-fsanitize=address -fno-omit-frame-pointer)
+endif (${WITH_ASAN})
+
+set(CMAKE_BUILD_TYPE RelWithDebInfo)
+string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
+string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}")
+string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
+string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
+string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}")
+string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
+string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
+string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}")
+string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
+string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
+string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}")
+string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
+
+find_package(PkgConfig)
+pkg_check_modules(LIBURING REQUIRED liburing)
+pkg_check_modules(GLIB REQUIRED glib-2.0)
+
+include_directories(
+	../
+	/usr/include/jerasure
+	${LIBURING_INCLUDE_DIRS}
+)
+
+# libvitastor_blk.so
+add_library(vitastor_blk SHARED
+	allocator.cpp blockstore.cpp blockstore_impl.cpp blockstore_init.cpp blockstore_open.cpp blockstore_journal.cpp blockstore_read.cpp
+	blockstore_write.cpp blockstore_sync.cpp blockstore_stable.cpp blockstore_rollback.cpp blockstore_flush.cpp crc32c.c ringloop.cpp
+)
+target_link_libraries(vitastor_blk
+	${LIBURING_LIBRARIES}
+	tcmalloc_minimal
+)
+
+# libfio_vitastor_blk.so
+add_library(fio_vitastor_blk SHARED
+	fio_engine.cpp
+	../json11/json11.cpp
+)
+target_link_libraries(fio_vitastor_blk
+	vitastor_blk
+)
+
+# vitastor-osd
+add_executable(vitastor-osd
+	osd_main.cpp osd.cpp osd_secondary.cpp msgr_receive.cpp msgr_send.cpp osd_peering.cpp osd_flush.cpp osd_peering_pg.cpp
+	osd_primary.cpp osd_primary_subops.cpp etcd_state_client.cpp messenger.cpp osd_cluster.cpp http_client.cpp osd_ops.cpp pg_states.cpp
+	osd_rmw.cpp base64.cpp timerfd_manager.cpp epoll_manager.cpp ../json11/json11.cpp
+)
+target_link_libraries(vitastor-osd
+	vitastor_blk
+	Jerasure
+)
+
+# libfio_vitastor_sec.so
+add_library(fio_vitastor_sec SHARED
+	fio_sec_osd.cpp
+	rw_blocking.cpp
+)
+target_link_libraries(fio_vitastor_sec
+	tcmalloc_minimal
+)
+
+# libvitastor_client.so
+add_library(vitastor_client SHARED
+	cluster_client.cpp epoll_manager.cpp etcd_state_client.cpp
+	messenger.cpp msgr_send.cpp msgr_receive.cpp ringloop.cpp ../json11/json11.cpp
+	http_client.cpp osd_ops.cpp pg_states.cpp timerfd_manager.cpp base64.cpp
+)
+target_link_libraries(vitastor_client
+	tcmalloc_minimal
+	${LIBURING_LIBRARIES}
+)
+
+# libfio_vitastor.so
+add_library(fio_vitastor SHARED
+	fio_cluster.cpp
+)
+target_link_libraries(fio_vitastor
+	vitastor_client
+)
+
+# vitastor-nbd
+add_executable(vitastor-nbd
+	nbd_proxy.cpp
+)
+target_link_libraries(vitastor-nbd
+	vitastor_client
+)
+
+# vitastor-rm
+add_executable(vitastor-rm
+	rm_inode.cpp
+)
+target_link_libraries(vitastor-rm
+	vitastor_client
+)
+
+# vitastor-dump-journal
+add_executable(vitastor-dump-journal
+	dump_journal.cpp crc32c.c
+)
+
+# qemu_driver.so
+add_library(qemu_proxy STATIC qemu_proxy.cpp)
+target_compile_options(qemu_proxy PUBLIC -fPIC)
+target_include_directories(qemu_proxy PUBLIC
+	../qemu/b/qemu
+	../qemu/include
+	${GLIB_INCLUDE_DIRS}
+)
+target_link_libraries(qemu_proxy
+	vitastor_client
+)
+add_library(qemu_vitastor SHARED
+	qemu_driver.c
+)
+target_link_libraries(qemu_vitastor
+	qemu_proxy
+)
+set_target_properties(qemu_vitastor PROPERTIES
+	PREFIX ""
+	OUTPUT_NAME "block-vitastor"
+)
+
+### Test stubs
+
+# stub_osd, stub_bench, osd_test
+add_executable(stub_osd stub_osd.cpp rw_blocking.cpp)
+target_link_libraries(stub_osd tcmalloc_minimal)
+add_executable(stub_bench stub_bench.cpp rw_blocking.cpp)
+target_link_libraries(stub_bench tcmalloc_minimal)
+add_executable(osd_test osd_test.cpp rw_blocking.cpp)
+target_link_libraries(osd_test tcmalloc_minimal)
+
+# osd_rmw_test
+add_executable(osd_rmw_test osd_rmw_test.cpp allocator.cpp)
+target_link_libraries(osd_rmw_test Jerasure tcmalloc_minimal)
+
+# stub_uring_osd
+add_executable(stub_uring_osd
+	stub_uring_osd.cpp epoll_manager.cpp messenger.cpp msgr_send.cpp msgr_receive.cpp ringloop.cpp timerfd_manager.cpp ../json11/json11.cpp
+)
+target_link_libraries(stub_uring_osd
+	${LIBURING_LIBRARIES}
+	tcmalloc_minimal
+)
+
+# osd_peering_pg_test
+add_executable(osd_peering_pg_test osd_peering_pg_test.cpp osd_peering_pg.cpp)
+target_link_libraries(osd_peering_pg_test tcmalloc_minimal)
+
+# test_allocator
+add_executable(test_allocator test_allocator.cpp allocator.cpp)
+
+## test_blockstore, test_shit
+#add_executable(test_blockstore test_blockstore.cpp timerfd_interval.cpp)
+#target_link_libraries(test_blockstore blockstore)
+#add_executable(test_shit test_shit.cpp osd_peering_pg.cpp)
+#target_link_libraries(test_shit ${LIBURING_LIBRARIES} m)
+
+### Install
+
+install(TARGETS vitastor-osd vitastor-dump-journal vitastor-nbd vitastor-rm RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(TARGETS fio_vitastor fio_vitastor_blk fio_vitastor_sec vitastor_blk vitastor_client LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(TARGETS qemu_vitastor LIBRARY DESTINATION /usr/${CMAKE_INSTALL_LIBDIR}/${QEMU_PLUGINDIR})

src/allocator.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <stdexcept>
 #include "allocator.h"

src/allocator.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 

src/base64.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "base64.h"
 

src/base64.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 #include <string>

src/blockstore.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 

src/blockstore.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 

src/blockstore_flush.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 
@@ -7,6 +7,8 @@ journal_flusher_t::journal_flusher_t(int flusher_count, blockstore_impl_t *bs)
 {
     this->bs = bs;
     this->flusher_count = flusher_count;
+    this->cur_flusher_count = 1;
+    this->target_flusher_count = 1;
     dequeuing = false;
     trimming = false;
     active_flushers = 0;
@@ -68,14 +70,31 @@ bool journal_flusher_t::is_active()
 
 void journal_flusher_t::loop()
 {
-    for (int i = 0; (active_flushers > 0 || dequeuing) && i < flusher_count; i++)
+    target_flusher_count = bs->write_iodepth*2;
+    if (target_flusher_count <= 0)
+        target_flusher_count = 1;
+    else if (target_flusher_count > flusher_count)
+        target_flusher_count = flusher_count;
+    if (target_flusher_count > cur_flusher_count)
+        cur_flusher_count = target_flusher_count;
+    else if (target_flusher_count < cur_flusher_count)
     {
-        co[i].loop();
+        while (target_flusher_count < cur_flusher_count)
+        {
+            if (co[cur_flusher_count-1].wait_state)
+                break;
+            cur_flusher_count--;
+        }
     }
+    for (int i = 0; (active_flushers > 0 || dequeuing) && i < cur_flusher_count; i++)
+        co[i].loop();
 }
 
 void journal_flusher_t::enqueue_flush(obj_ver_id ov)
 {
+#ifdef BLOCKSTORE_DEBUG
+    printf("enqueue_flush %lx:%lx v%lu\n", ov.oid.inode, ov.oid.stripe, ov.version);
+#endif
     auto it = flush_versions.find(ov.oid);
     if (it != flush_versions.end())
     {
@@ -94,8 +113,11 @@ void journal_flusher_t::enqueue_flush(obj_ver_id ov)
     }
 }
 
-void journal_flusher_t::unshift_flush(obj_ver_id ov)
+void journal_flusher_t::unshift_flush(obj_ver_id ov, bool force)
 {
+#ifdef BLOCKSTORE_DEBUG
+    printf("unshift_flush %lx:%lx v%lu\n", ov.oid.inode, ov.oid.stripe, ov.version);
+#endif
     auto it = flush_versions.find(ov.oid);
     if (it != flush_versions.end())
     {
@@ -105,15 +127,38 @@ void journal_flusher_t::unshift_flush(obj_ver_id ov)
     else
     {
         flush_versions[ov.oid] = ov.version;
-        flush_queue.push_front(ov.oid);
+        if (!force)
+            flush_queue.push_front(ov.oid);
     }
-    if (!dequeuing && (flush_queue.size() >= flusher_start_threshold || trim_wanted > 0))
+    if (force)
+        flush_queue.push_front(ov.oid);
+    if (force || !dequeuing && (flush_queue.size() >= flusher_start_threshold || trim_wanted > 0))
     {
         dequeuing = true;
         bs->ringloop->wakeup();
     }
 }
 
+void journal_flusher_t::remove_flush(object_id oid)
+{
+#ifdef BLOCKSTORE_DEBUG
+    printf("undo_flush %lx:%lx\n", oid.inode, oid.stripe);
+#endif
+    auto v_it = flush_versions.find(oid);
+    if (v_it != flush_versions.end())
+    {
+        flush_versions.erase(v_it);
+        for (auto q_it = flush_queue.begin(); q_it != flush_queue.end(); q_it++)
+        {
+            if (*q_it == oid)
+            {
+                flush_queue.erase(q_it);
+                break;
+            }
+        }
+    }
+}
+
 void journal_flusher_t::request_trim()
 {
     dequeuing = true;
@@ -194,6 +239,7 @@ bool journal_flusher_co::loop()
 resume_0:
     if (!flusher->flush_queue.size() || !flusher->dequeuing)
     {
+stop_flusher:
         if (flusher->trim_wanted > 0 && flusher->journal_trim_counter > 0)
         {
             // Attempt forced trim
@@ -298,9 +344,7 @@ resume_0:
 #ifdef BLOCKSTORE_DEBUG
                     printf("No older flushes, stopping\n");
 #endif
-                    flusher->dequeuing = false;
+                    goto stop_flusher;
-                    wait_state = 0;
-                    return true;
                 }
             }
         }
@@ -319,8 +363,8 @@ resume_1:
             return false;
         }
         // Writes and deletes shouldn't happen at the same time
-        assert(!(copy_count > 0 || has_writes) || !has_delete);
+        assert(!has_writes || !has_delete);
-        if (copy_count == 0 && !has_writes && !has_delete || has_delete && old_clean_loc == UINT64_MAX)
+        if (!has_writes && !has_delete || has_delete && old_clean_loc == UINT64_MAX)
         {
             // Nothing to flush
             bs->erase_dirty(dirty_start, std::next(dirty_end), clean_loc);
@@ -445,8 +489,8 @@ resume_1:
             clean_disk_entry *new_entry = (clean_disk_entry*)(meta_new.buf + meta_new.pos*bs->clean_entry_size);
             if (new_entry->oid.inode != 0 && new_entry->oid != cur.oid)
             {
-                printf("Fatal error (metadata corruption or bug): tried to overwrite non-zero metadata entry %lx (%lx:%lx) with %lx:%lx\n",
+                printf("Fatal error (metadata corruption or bug): tried to overwrite non-zero metadata entry %lu (%lx:%lx) with %lx:%lx\n",
-                    clean_loc, new_entry->oid.inode, new_entry->oid.stripe, cur.oid.inode, cur.oid.stripe);
+                    clean_loc >> bs->block_order, new_entry->oid.inode, new_entry->oid.stripe, cur.oid.inode, cur.oid.stripe);
                 exit(1);
             }
             new_entry->oid = cur.oid;
@@ -513,7 +557,7 @@ resume_1:
         if (repeat_it != flusher->sync_to_repeat.end() && repeat_it->second > cur.version)
         {
             // Requeue version
-            flusher->unshift_flush({ .oid = cur.oid, .version = repeat_it->second });
+            flusher->unshift_flush({ .oid = cur.oid, .version = repeat_it->second }, false);
         }
         flusher->sync_to_repeat.erase(repeat_it);
     trim_journal:
@@ -602,7 +646,7 @@ bool journal_flusher_co::scan_dirty(int wait_base)
         {
             char err[1024];
             snprintf(
-                err, 1024, "BUG: Unexpected dirty_entry %lx:%lx v%lu state during flush: %d",
+                err, 1024, "BUG: Unexpected dirty_entry %lx:%lx v%lu unstable state during flush: %d",
                 dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, dirty_it->second.state
             );
             throw std::runtime_error(err);

src/blockstore_flush.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 struct copy_buffer_t
 {
@@ -80,7 +80,7 @@ class journal_flusher_t
 {
     int trim_wanted = 0;
     bool dequeuing;
-    int flusher_count;
+    int flusher_count, cur_flusher_count, target_flusher_count;
     int flusher_start_threshold;
     journal_flusher_co *co;
     blockstore_impl_t *bs;
@@ -107,5 +107,6 @@ public:
     void request_trim();
     void release_trim();
     void enqueue_flush(obj_ver_id oid);
-    void unshift_flush(obj_ver_id oid);
+    void unshift_flush(obj_ver_id oid, bool force);
+    void remove_flush(object_id oid);
 };

src/blockstore_impl.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 
@@ -287,7 +287,7 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
     else if (PRIV(op)->wait_for == WAIT_JOURNAL_BUFFER)
     {
         int next = ((journal.cur_sector + 1) % journal.sector_count);
-        if (journal.sector_info[next].usage_count > 0 ||
+        if (journal.sector_info[next].flush_count > 0 ||
             journal.sector_info[next].dirty)
         {
             // do not submit

src/blockstore_impl.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 
@@ -30,12 +30,13 @@
 #define BS_ST_BIG_WRITE 0x02
 #define BS_ST_DELETE 0x03
 
-#define BS_ST_WAIT_BIG 0x10
+#define BS_ST_WAIT_DEL 0x10
-#define BS_ST_IN_FLIGHT 0x20
+#define BS_ST_WAIT_BIG 0x20
-#define BS_ST_SUBMITTED 0x30
+#define BS_ST_IN_FLIGHT 0x30
-#define BS_ST_WRITTEN 0x40
+#define BS_ST_SUBMITTED 0x40
-#define BS_ST_SYNCED 0x50
+#define BS_ST_WRITTEN 0x50
-#define BS_ST_STABLE 0x60
+#define BS_ST_SYNCED 0x60
+#define BS_ST_STABLE 0x70
 
 #define BS_ST_INSTANT 0x100
 
@@ -153,6 +154,8 @@ struct blockstore_op_private_t
 
     // Write
     struct iovec iov_zerofill[3];
+    // Warning: must not have a default value here because it's written to before calling constructor in blockstore_write.cpp O_o
+    uint64_t real_version;
 
     // Sync
     std::vector<obj_ver_id> sync_big_writes, sync_small_writes;
@@ -196,7 +199,10 @@ class blockstore_impl_t
     // Suitable only for server SSDs with capacitors, requires disabled data and journal fsyncs
     int immediate_commit = IMMEDIATE_NONE;
     bool inmemory_meta = false;
-    int flusher_count;
+    // Maximum flusher count
+    unsigned flusher_count;
+    // Maximum queue depth
+    unsigned max_write_iodepth = 128;
     /******* END OF OPTIONS *******/
 
     struct ring_consumer_t ring_consumer;
@@ -223,6 +229,7 @@ class blockstore_impl_t
 
     struct journal_t journal;
     journal_flusher_t *flusher;
+    int write_iodepth = 0;
 
     bool live = false, queue_stall = false;
     ring_loop_t *ringloop;

src/blockstore_init.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 
@@ -111,7 +111,7 @@ void blockstore_init_meta::handle_entries(void* entries, unsigned count, int blo
                 {
                     // free the previous block
 #ifdef BLOCKSTORE_DEBUG
-                    printf("Free block %lu (new location is %lu)\n", clean_it->second.location >> block_order, done_cnt+i >> block_order);
+                    printf("Free block %lu (new location is %lu)\n", clean_it->second.location >> block_order, done_cnt+i);
 #endif
                     bs->data_alloc->set(clean_it->second.location >> block_order, false);
                 }
@@ -557,11 +557,56 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
             {
 #ifdef BLOCKSTORE_DEBUG
                 printf(
-                    "je_big_write%s oid=%lx:%lx ver=%lu loc=%08lx\n",
+                    "je_big_write%s oid=%lx:%lx ver=%lu loc=%lu\n",
                     je->type == JE_BIG_WRITE_INSTANT ? "_instant" : "",
-                    je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location
+                    je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location >> bs->block_order
                 );
 #endif
+                auto dirty_it = bs->dirty_db.upper_bound((obj_ver_id){
+                    .oid = je->big_write.oid,
+                    .version = UINT64_MAX,
+                });
+                if (dirty_it != bs->dirty_db.begin() && bs->dirty_db.size() > 0)
+                {
+                    dirty_it--;
+                    if (dirty_it->first.oid == je->big_write.oid &&
+                        dirty_it->first.version >= je->big_write.version &&
+                        (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_DELETE)
+                    {
+                        // It is allowed to overwrite a deleted object with a
+                        // version number smaller than deletion version number,
+                        // because the presence of a BIG_WRITE entry means that
+                        // its data and metadata are already flushed.
+                        // We don't know if newer versions are flushed, but
+                        // the previous delete definitely is.
+                        // So we flush previous dirty entries, but retain the clean one.
+                        // This feature is required for writes happening shortly
+                        // after deletes.
+                        auto dirty_end = dirty_it;
+                        dirty_end++;
+                        while (1)
+                        {
+                            if (dirty_it == bs->dirty_db.begin())
+                            {
+                                break;
+                            }
+                            dirty_it--;
+                            if (dirty_it->first.oid != je->big_write.oid)
+                            {
+                                dirty_it++;
+                                break;
+                            }
+                        }
+                        auto clean_it = bs->clean_db.find(je->big_write.oid);
+                        bs->erase_dirty(
+                            dirty_it, dirty_end,
+                            clean_it != bs->clean_db.end() ? clean_it->second.location : UINT64_MAX
+                        );
+                        // Remove it from the flusher's queue, too
+                        // Otherwise it may end up referring to a small unstable write after reading the rest of the journal
+                        bs->flusher->remove_flush(je->big_write.oid);
+                    }
+                }
                 auto clean_it = bs->clean_db.find(je->big_write.oid);
                 if (clean_it == bs->clean_db.end() ||
                     clean_it->second.version < je->big_write.version)

src/blockstore_init.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 

src/blockstore_journal.cpp

@@ -1,12 +1,12 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 
 blockstore_journal_check_t::blockstore_journal_check_t(blockstore_impl_t *bs)
 {
     this->bs = bs;
-    sectors_required = 0;
+    sectors_to_write = 0;
     next_pos = bs->journal.next_free;
     next_sector = bs->journal.cur_sector;
     first_sector = -1;
@@ -20,23 +20,26 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
     int required = entries_required;
     while (1)
     {
-        int fits = bs->journal.no_same_sector_overwrites && bs->journal.sector_info[next_sector].written
+        int fits = bs->journal.no_same_sector_overwrites && next_pos == bs->journal.next_free && bs->journal.sector_info[next_sector].written
             ? 0
             : (bs->journal.block_size - next_in_pos) / size;
         if (fits > 0)
         {
+            if (fits > required)
+            {
+                fits = required;
+            }
             if (first_sector == -1)
             {
                 first_sector = next_sector;
             }
             required -= fits;
             next_in_pos += fits * size;
-            sectors_required++;
+            sectors_to_write++;
         }
         else if (bs->journal.sector_info[next_sector].dirty)
         {
-            // sectors_required is more like "sectors to write"
+            sectors_to_write++;
-            sectors_required++;
         }
         if (required <= 0)
         {
@@ -59,7 +62,7 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
                 " is too small for a batch of "+std::to_string(entries_required)+" entries of "+std::to_string(size)+" bytes"
             );
         }
-        if (bs->journal.sector_info[next_sector].usage_count > 0 ||
+        if (bs->journal.sector_info[next_sector].flush_count > 0 ||
             bs->journal.sector_info[next_sector].dirty)
         {
             // No memory buffer available. Wait for it.
@@ -71,17 +74,18 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
                     dirty++;
                     used++;
                 }
-                if (bs->journal.sector_info[i].usage_count > 0)
+                if (bs->journal.sector_info[i].flush_count > 0)
                 {
                     used++;
                 }
             }
             // In fact, it's even more rare than "ran out of journal space", so print a warning
             printf(
-                "Ran out of journal sector buffers: %d/%lu buffers used (%d dirty), next buffer (%ld) is %s and flushed %lu times\n",
+                "Ran out of journal sector buffers: %d/%lu buffers used (%d dirty), next buffer (%ld)"
+                " is %s and flushed %lu times. Consider increasing \'journal_sector_buffer_count\'\n",
                 used, bs->journal.sector_count, dirty, next_sector,
                 bs->journal.sector_info[next_sector].dirty ? "dirty" : "not dirty",
-                bs->journal.sector_info[next_sector].usage_count
+                bs->journal.sector_info[next_sector].flush_count
             );
             PRIV(op)->wait_for = WAIT_JOURNAL_BUFFER;
             return 0;
@@ -100,11 +104,8 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
     {
         // No space in the journal. Wait until used_start changes.
         printf(
-            "Ran out of journal space (free space: %lu bytes, sectors to write: %d)\n",
+            "Ran out of journal space (used_start=%08lx, next_free=%08lx, dirty_start=%08lx)\n",
-            (bs->journal.next_free >= bs->journal.used_start
+            bs->journal.used_start, bs->journal.next_free, bs->journal.dirty_start
-                ? bs->journal.len-bs->journal.block_size - (bs->journal.next_free-bs->journal.used_start)
-                : bs->journal.used_start - bs->journal.next_free),
-            sectors_required
         );
         PRIV(op)->wait_for = WAIT_JOURNAL;
         bs->flusher->request_trim();
@@ -116,22 +117,21 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
 
 journal_entry* prefill_single_journal_entry(journal_t & journal, uint16_t type, uint32_t size)
 {
-    if (journal.block_size - journal.in_sector_pos < size ||
+    if (!journal.entry_fits(size))
-        journal.no_same_sector_overwrites && journal.sector_info[journal.cur_sector].written)
     {
         assert(!journal.sector_info[journal.cur_sector].dirty);
         // Move to the next journal sector
-        journal.sector_info[journal.cur_sector].written = false;
+        if (journal.sector_info[journal.cur_sector].flush_count > 0)
-        if (journal.sector_info[journal.cur_sector].usage_count > 0)
         {
             // Also select next sector buffer in memory
             journal.cur_sector = ((journal.cur_sector + 1) % journal.sector_count);
-            assert(!journal.sector_info[journal.cur_sector].usage_count);
+            assert(!journal.sector_info[journal.cur_sector].flush_count);
         }
         else
         {
             journal.dirty_start = journal.next_free;
         }
+        journal.sector_info[journal.cur_sector].written = false;
         journal.sector_info[journal.cur_sector].offset = journal.next_free;
         journal.in_sector_pos = 0;
         journal.next_free = (journal.next_free+journal.block_size) < journal.len ? journal.next_free + journal.block_size : journal.block_size;
@@ -157,7 +157,7 @@ void prepare_journal_sector_write(journal_t & journal, int cur_sector, io_uring_
 {
     journal.sector_info[cur_sector].dirty = false;
     journal.sector_info[cur_sector].written = true;
-    journal.sector_info[cur_sector].usage_count++;
+    journal.sector_info[cur_sector].flush_count++;
     ring_data_t *data = ((ring_data_t*)sqe->user_data);
     data->iov = (struct iovec){
         (journal.inmemory

src/blockstore_journal.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 
@@ -133,7 +133,7 @@ inline uint32_t je_crc32(journal_entry *je)
 struct journal_sector_info_t
 {
     uint64_t offset;
-    uint64_t usage_count;
+    uint64_t flush_count;
     bool written;
     bool dirty;
 };
@@ -170,13 +170,18 @@ struct journal_t
     ~journal_t();
     bool trim();
     uint64_t get_trim_pos();
+    inline bool entry_fits(int size)
+    {
+        return !(block_size - in_sector_pos < size ||
+            no_same_sector_overwrites && sector_info[cur_sector].written);
+    }
 };
 
 struct blockstore_journal_check_t
 {
     blockstore_impl_t *bs;
     uint64_t next_pos, next_sector, next_in_pos;
-    int sectors_required, first_sector;
+    int sectors_to_write, first_sector;
     bool right_dir; // writing to the end or the beginning of the ring buffer
 
     blockstore_journal_check_t(blockstore_impl_t *bs);

src/blockstore_open.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <sys/file.h>
 #include "blockstore_impl.h"
@@ -70,6 +70,7 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config)
     meta_block_size = strtoull(config["meta_block_size"].c_str(), NULL, 10);
     bitmap_granularity = strtoull(config["bitmap_granularity"].c_str(), NULL, 10);
     flusher_count = strtoull(config["flusher_count"].c_str(), NULL, 10);
+    max_write_iodepth = strtoull(config["max_write_iodepth"].c_str(), NULL, 10);
     // Validate
     if (!block_size)
     {
@@ -83,6 +84,10 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config)
     {
         flusher_count = 32;
     }
+    if (!max_write_iodepth)
+    {
+        max_write_iodepth = 128;
+    }
     if (!disk_alignment)
     {
         disk_alignment = 4096;

src/blockstore_read.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 

src/blockstore_rollback.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 
@@ -75,44 +75,35 @@ skip_ov:
         return 0;
     }
     // There is sufficient space. Get SQEs
-    struct io_uring_sqe *sqe[space_check.sectors_required];
+    struct io_uring_sqe *sqe[space_check.sectors_to_write];
-    for (i = 0; i < space_check.sectors_required; i++)
+    for (i = 0; i < space_check.sectors_to_write; i++)
     {
         BS_SUBMIT_GET_SQE_DECL(sqe[i]);
     }
     // Prepare and submit journal entries
     auto cb = [this, op](ring_data_t *data) { handle_rollback_event(data, op); };
     int s = 0, cur_sector = -1;
-    if ((journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_rollback) &&
-        journal.sector_info[journal.cur_sector].dirty)
-    {
-        if (cur_sector == -1)
-            PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
-        cur_sector = journal.cur_sector;
-        prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
-    }
     for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
     {
+        if (!journal.entry_fits(sizeof(journal_entry_rollback)) &&
+            journal.sector_info[journal.cur_sector].dirty)
+        {
+            if (cur_sector == -1)
+                PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
+            prepare_journal_sector_write(journal, journal.cur_sector, sqe[s++], cb);
+            cur_sector = journal.cur_sector;
+        }
         journal_entry_rollback *je = (journal_entry_rollback*)
             prefill_single_journal_entry(journal, JE_ROLLBACK, sizeof(journal_entry_rollback));
-        journal.sector_info[journal.cur_sector].dirty = false;
         je->oid = v->oid;
         je->version = v->version;
         je->crc32 = je_crc32((journal_entry*)je);
         journal.crc32_last = je->crc32;
-        if (cur_sector != journal.cur_sector)
-        {
-            // Write previous sector. We should write the sector only after filling it,
-            // because otherwise we'll write a lot more sectors in the "no_same_sector_overwrite" mode
-            if (cur_sector != -1)
-                prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
-            else
-                PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
-            cur_sector = journal.cur_sector;
-        }
     }
-    if (cur_sector != -1)
+    prepare_journal_sector_write(journal, journal.cur_sector, sqe[s++], cb);
-        prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
+    assert(s == space_check.sectors_to_write);
+    if (cur_sector == -1)
+        PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
     PRIV(op)->max_flushed_journal_sector = 1 + journal.cur_sector;
     PRIV(op)->pending_ops = s;
     PRIV(op)->op_state = 1;
@@ -234,10 +225,38 @@ void blockstore_impl_t::handle_rollback_event(ring_data_t *data, blockstore_op_t
 
 void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start, blockstore_dirty_db_t::iterator dirty_end, uint64_t clean_loc)
 {
-    auto dirty_it = dirty_end;
+    if (dirty_end == dirty_start)
-    while (dirty_it != dirty_start)
     {
+        return;
+    }
+    auto dirty_it = dirty_end;
+    dirty_it--;
+    if (IS_DELETE(dirty_it->second.state))
+    {
+        object_id oid = dirty_it->first.oid;
+#ifdef BLOCKSTORE_DEBUG
+        printf("Unblock writes-after-delete %lx:%lx v%lx\n", oid.inode, oid.stripe, dirty_it->first.version);
+#endif
+        dirty_it = dirty_end;
+        // Unblock operations blocked by delete flushing
+        uint32_t next_state = BS_ST_IN_FLIGHT;
+        while (dirty_it != dirty_db.end() && dirty_it->first.oid == oid)
+        {
+            if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_DEL)
+            {
+                dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | next_state;
+                if (IS_BIG_WRITE(dirty_it->second.state))
+                {
+                    next_state = BS_ST_WAIT_BIG;
+                }
+            }
+            dirty_it++;
+        }
+        dirty_it = dirty_end;
         dirty_it--;
+    }
+    while (1)
+    {
         if (IS_BIG_WRITE(dirty_it->second.state) && dirty_it->second.location != clean_loc)
         {
 #ifdef BLOCKSTORE_DEBUG
@@ -256,6 +275,11 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
         {
             journal.used_sectors.erase(dirty_it->second.journal_sector);
         }
+        if (dirty_it == dirty_start)
+        {
+            break;
+        }
+        dirty_it--;
     }
     dirty_db.erase(dirty_start, dirty_end);
 }

src/blockstore_stable.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 
@@ -98,45 +98,36 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
         return 0;
     }
     // There is sufficient space. Get SQEs
-    struct io_uring_sqe *sqe[space_check.sectors_required];
+    struct io_uring_sqe *sqe[space_check.sectors_to_write];
-    for (i = 0; i < space_check.sectors_required; i++)
+    for (i = 0; i < space_check.sectors_to_write; i++)
     {
         BS_SUBMIT_GET_SQE_DECL(sqe[i]);
     }
     // Prepare and submit journal entries
     auto cb = [this, op](ring_data_t *data) { handle_stable_event(data, op); };
     int s = 0, cur_sector = -1;
-    if ((journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_stable) &&
-        journal.sector_info[journal.cur_sector].dirty)
-    {
-        if (cur_sector == -1)
-            PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
-        cur_sector = journal.cur_sector;
-        prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
-    }
     for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
     {
         // FIXME: Only stabilize versions that aren't stable yet
+        if (!journal.entry_fits(sizeof(journal_entry_stable)) &&
+            journal.sector_info[journal.cur_sector].dirty)
+        {
+            if (cur_sector == -1)
+                PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
+            prepare_journal_sector_write(journal, journal.cur_sector, sqe[s++], cb);
+            cur_sector = journal.cur_sector;
+        }
         journal_entry_stable *je = (journal_entry_stable*)
             prefill_single_journal_entry(journal, JE_STABLE, sizeof(journal_entry_stable));
-        journal.sector_info[journal.cur_sector].dirty = false;
         je->oid = v->oid;
         je->version = v->version;
         je->crc32 = je_crc32((journal_entry*)je);
         journal.crc32_last = je->crc32;
-        if (cur_sector != journal.cur_sector)
-        {
-            // Write previous sector. We should write the sector only after filling it,
-            // because otherwise we'll write a lot more sectors in the "no_same_sector_overwrite" mode
-            if (cur_sector != -1)
-                prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
-            else
-                PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
-            cur_sector = journal.cur_sector;
-        }
     }
-    if (cur_sector != -1)
+    prepare_journal_sector_write(journal, journal.cur_sector, sqe[s++], cb);
-        prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
+    assert(s == space_check.sectors_to_write);
+    if (cur_sector == -1)
+        PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
     PRIV(op)->max_flushed_journal_sector = 1 + journal.cur_sector;
     PRIV(op)->pending_ops = s;
     PRIV(op)->op_state = 1;
@@ -213,9 +204,6 @@ void blockstore_impl_t::mark_stable(const obj_ver_id & v)
                 break;
             }
         }
-#ifdef BLOCKSTORE_DEBUG
-        printf("enqueue_flush %lx:%lx v%lu\n", v.oid.inode, v.oid.stripe, v.version);
-#endif
         flusher->enqueue_flush(v);
     }
     auto unstab_it = unstable_writes.find(v.oid);

src/blockstore_sync.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 
@@ -112,30 +112,29 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
             return 0;
         }
         // Get SQEs. Don't bother about merging, submit each journal sector as a separate request
-        struct io_uring_sqe *sqe[space_check.sectors_required];
+        struct io_uring_sqe *sqe[space_check.sectors_to_write];
-        for (int i = 0; i < space_check.sectors_required; i++)
+        for (int i = 0; i < space_check.sectors_to_write; i++)
         {
             BS_SUBMIT_GET_SQE_DECL(sqe[i]);
         }
         // Prepare and submit journal entries
         auto it = PRIV(op)->sync_big_writes.begin();
         int s = 0, cur_sector = -1;
-        if ((journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_big_write) &&
-            journal.sector_info[journal.cur_sector].dirty)
-        {
-            if (cur_sector == -1)
-                PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
-            cur_sector = journal.cur_sector;
-            prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
-        }
         while (it != PRIV(op)->sync_big_writes.end())
         {
+            if (!journal.entry_fits(sizeof(journal_entry_big_write)) &&
+                journal.sector_info[journal.cur_sector].dirty)
+            {
+                if (cur_sector == -1)
+                    PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
+                prepare_journal_sector_write(journal, journal.cur_sector, sqe[s++], cb);
+                cur_sector = journal.cur_sector;
+            }
             journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry(
                 journal, (dirty_db[*it].state & BS_ST_INSTANT) ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE,
                 sizeof(journal_entry_big_write)
             );
             dirty_db[*it].journal_sector = journal.sector_info[journal.cur_sector].offset;
-            journal.sector_info[journal.cur_sector].dirty = false;
             journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
 #ifdef BLOCKSTORE_DEBUG
             printf(
@@ -152,19 +151,11 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
             je->crc32 = je_crc32((journal_entry*)je);
             journal.crc32_last = je->crc32;
             it++;
-            if (cur_sector != journal.cur_sector)
-            {
-                // Write previous sector. We should write the sector only after filling it,
-                // because otherwise we'll write a lot more sectors in the "no_same_sector_overwrite" mode
-                if (cur_sector != -1)
-                    prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
-                else
-                    PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
-                cur_sector = journal.cur_sector;
-            }
         }
-        if (cur_sector != -1)
+        prepare_journal_sector_write(journal, journal.cur_sector, sqe[s++], cb);
-            prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
+        assert(s == space_check.sectors_to_write);
+        if (cur_sector == -1)
+            PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
         PRIV(op)->max_flushed_journal_sector = 1 + journal.cur_sector;
         PRIV(op)->pending_ops = s;
         PRIV(op)->op_state = SYNC_JOURNAL_WRITE_SENT;

src/blockstore_write.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "blockstore_impl.h"
 
@@ -7,7 +7,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
 {
     // Check or assign version number
     bool found = false, deleted = false, is_del = (op->opcode == BS_OP_DELETE);
-    bool is_inflight_big = false;
+    bool wait_big = false, wait_del = false;
     uint64_t version = 1;
     if (dirty_db.size() > 0)
     {
@@ -21,7 +21,8 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
             found = true;
             version = dirty_it->first.version + 1;
             deleted = IS_DELETE(dirty_it->second.state);
-            is_inflight_big = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE
+            wait_del = ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_DEL);
+            wait_big = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE
                 ? !IS_SYNCED(dirty_it->second.state)
                 : ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG);
         }
@@ -38,23 +39,43 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
             deleted = true;
         }
     }
-    if (op->version == 0)
-    {
-        op->version = version;
-    }
-    else if (op->version < version)
-    {
-        // Invalid version requested
-        op->retval = -EEXIST;
-        return false;
-    }
     if (deleted && is_del)
     {
         // Already deleted
         op->retval = 0;
         return false;
     }
-    if (is_inflight_big && !is_del && !deleted && op->len < block_size &&
+    PRIV(op)->real_version = 0;
+    if (op->version == 0)
+    {
+        op->version = version;
+    }
+    else if (op->version < version)
+    {
+        // Implicit operations must be added like that: DEL [FLUSH] BIG [SYNC] SMALL SMALL
+        if (deleted || wait_del)
+        {
+            // It's allowed to write versions with low numbers over deletes
+            // However, we have to flush those deletes first as we use version number for ordering
+#ifdef BLOCKSTORE_DEBUG
+            printf("Write %lx:%lx v%lu over delete (real v%lu) offset=%u len=%u\n", op->oid.inode, op->oid.stripe, version, op->version, op->offset, op->len);
+#endif
+            wait_del = true;
+            PRIV(op)->real_version = op->version;
+            op->version = version;
+            flusher->unshift_flush((obj_ver_id){
+                .oid = op->oid,
+                .version = version-1,
+            }, true);
+        }
+        else
+        {
+            // Invalid version requested
+            op->retval = -EEXIST;
+            return false;
+        }
+    }
+    if (wait_big && !is_del && !deleted && op->len < block_size &&
         immediate_commit != IMMEDIATE_ALL)
     {
         // Issue an additional sync so that the previous big write can reach the journal
@@ -69,22 +90,31 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
 #ifdef BLOCKSTORE_DEBUG
     if (is_del)
         printf("Delete %lx:%lx v%lu\n", op->oid.inode, op->oid.stripe, op->version);
-    else
+    else if (!wait_del)
         printf("Write %lx:%lx v%lu offset=%u len=%u\n", op->oid.inode, op->oid.stripe, op->version, op->offset, op->len);
 #endif
-    // No strict need to add it into dirty_db here, it's just left
+    // FIXME No strict need to add it into dirty_db here, it's just left
     // from the previous implementation where reads waited for writes
+    uint32_t state;
+    if (is_del)
+        state = BS_ST_DELETE | BS_ST_IN_FLIGHT;
+    else
+    {
+        state = (op->len == block_size || deleted ? BS_ST_BIG_WRITE : BS_ST_SMALL_WRITE);
+        if (wait_del)
+            state |= BS_ST_WAIT_DEL;
+        else if (state == BS_ST_SMALL_WRITE && wait_big)
+            state |= BS_ST_WAIT_BIG;
+        else
+            state |= BS_ST_IN_FLIGHT;
+        if (op->opcode == BS_OP_WRITE_STABLE)
+            state |= BS_ST_INSTANT;
+    }
     dirty_db.emplace((obj_ver_id){
         .oid = op->oid,
         .version = op->version,
     }, (dirty_entry){
-        .state = (uint32_t)(
+        .state = state,
-            is_del
-                ? (BS_ST_DELETE | BS_ST_IN_FLIGHT)
-                : (op->opcode == BS_OP_WRITE_STABLE ? BS_ST_INSTANT : 0) | (op->len == block_size || deleted
-                    ? (BS_ST_BIG_WRITE | BS_ST_IN_FLIGHT)
-                    : (is_inflight_big ? (BS_ST_SMALL_WRITE | BS_ST_WAIT_BIG) : (BS_ST_SMALL_WRITE | BS_ST_IN_FLIGHT)))
-        ),
         .flags = 0,
         .location = 0,
         .offset = is_del ? 0 : op->offset,
@@ -106,12 +136,42 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         .version = op->version,
     });
     assert(dirty_it != dirty_db.end());
-    if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG)
+    if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) < BS_ST_IN_FLIGHT)
     {
         // Don't dequeue
         return 0;
     }
-    else if ((dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE)
+    if (PRIV(op)->real_version != 0)
+    {
+        // Restore original low version number for unblocked operations
+#ifdef BLOCKSTORE_DEBUG
+        printf("Restoring %lx:%lx version: v%lu -> v%lu\n", op->oid.inode, op->oid.stripe, op->version, PRIV(op)->real_version);
+#endif
+        auto prev_it = dirty_it;
+        prev_it--;
+        if (prev_it->first.oid == op->oid && prev_it->first.version >= PRIV(op)->real_version)
+        {
+            // Original version is still invalid
+            // FIXME Oops. Successive small writes will currently break in an unexpected way. Fix it
+            dirty_db.erase(dirty_it);
+            op->retval = -EEXIST;
+            FINISH_OP(op);
+            return 1;
+        }
+        op->version = PRIV(op)->real_version;
+        PRIV(op)->real_version = 0;
+        dirty_entry e = dirty_it->second;
+        dirty_db.erase(dirty_it);
+        dirty_it = dirty_db.emplace((obj_ver_id){
+            .oid = op->oid,
+            .version = op->version,
+        }, e).first;
+    }
+    if (write_iodepth >= max_write_iodepth)
+    {
+        return 0;
+    }
+    if ((dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE)
     {
         blockstore_journal_check_t space_check(this);
         if (!space_check.check_available(op, unsynced_big_writes.size() + 1, sizeof(journal_entry_big_write), JOURNAL_STABILIZE_RESERVATION))
@@ -129,10 +189,13 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
                 PRIV(op)->wait_for = WAIT_FREE;
                 return 0;
             }
+            // FIXME Oops. Successive small writes will currently break in an unexpected way. Fix it
+            dirty_db.erase(dirty_it);
             op->retval = -ENOSPC;
             FINISH_OP(op);
             return 1;
         }
+        write_iodepth++;
         BS_SUBMIT_GET_SQE(sqe, data);
         dirty_it->second.location = loc << block_order;
         dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SUBMITTED;
@@ -185,6 +248,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         {
             return 0;
         }
+        write_iodepth++;
         // There is sufficient space. Get SQE(s)
         struct io_uring_sqe *sqe1 = NULL;
         if (immediate_commit != IMMEDIATE_NONE ||
@@ -319,7 +383,6 @@ resume_2:
         sizeof(journal_entry_big_write)
     );
     dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset;
-    journal.sector_info[journal.cur_sector].dirty = false;
     journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
 #ifdef BLOCKSTORE_DEBUG
     printf(
@@ -344,7 +407,7 @@ resume_2:
 resume_4:
     // Switch object state
 #ifdef BLOCKSTORE_DEBUG
-    printf("Ack write %lx:%lx v%lu = %d\n", op->oid.inode, op->oid.stripe, op->version, dirty_it->second.state);
+    printf("Ack write %lx:%lx v%lu = state %x\n", op->oid.inode, op->oid.stripe, op->version, dirty_it->second.state);
 #endif
     bool imm = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE
         ? (immediate_commit == IMMEDIATE_ALL)
@@ -375,6 +438,7 @@ resume_4:
     }
     // Acknowledge write
     op->retval = op->len;
+    write_iodepth--;
     FINISH_OP(op);
     return 1;
 }
@@ -411,8 +475,8 @@ void blockstore_impl_t::release_journal_sectors(blockstore_op_t *op)
         uint64_t s = PRIV(op)->min_flushed_journal_sector;
         while (1)
         {
-            journal.sector_info[s-1].usage_count--;
+            journal.sector_info[s-1].flush_count--;
-            if (s != (1+journal.cur_sector) && journal.sector_info[s-1].usage_count == 0)
+            if (s != (1+journal.cur_sector) && journal.sector_info[s-1].flush_count == 0)
             {
                 // We know for sure that we won't write into this sector anymore
                 uint64_t new_ds = journal.sector_info[s-1].offset + journal.block_size;
@@ -446,6 +510,7 @@ int blockstore_impl_t::dequeue_del(blockstore_op_t *op)
     {
         return 0;
     }
+    write_iodepth++;
     io_uring_sqe *sqe = NULL;
     if (immediate_commit != IMMEDIATE_NONE ||
         (journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_del) &&
@@ -492,7 +557,10 @@ int blockstore_impl_t::dequeue_del(blockstore_op_t *op)
         prepare_journal_sector_write(journal, journal.cur_sector, sqe, cb);
         PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 1 + journal.cur_sector;
         PRIV(op)->pending_ops++;
-        // Remember small write as unsynced
+    }
+    else
+    {
+        // Remember delete as unsynced
         unsynced_small_writes.push_back((obj_ver_id){
             .oid = op->oid,
             .version = op->version,

src/cluster_client.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include <stdexcept>
 #include "cluster_client.h"
@@ -473,7 +473,7 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
     // Primary OSDs still operate individual stripes, but their size is multiplied by PG minsize in case of EC
     auto & pool_cfg = st_cli.pool_config[INODE_POOL(op->inode)];
     uint64_t pg_block_size = bs_block_size * (
-        pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_minsize
+        pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
     );
     uint64_t first_stripe = (op->offset / pg_block_size) * pg_block_size;
     uint64_t last_stripe = ((op->offset + op->len + pg_block_size - 1) / pg_block_size - 1) * pg_block_size;
@@ -488,7 +488,7 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
         uint64_t begin = (op->offset < stripe ? stripe : op->offset);
         uint64_t end = (op->offset + op->len) > (stripe + pg_block_size)
             ? (stripe + pg_block_size) : (op->offset + op->len);
-        op->parts[i] = {
+        op->parts[i] = (cluster_op_part_t){
             .parent = op,
             .offset = begin,
             .len = (uint32_t)(end - begin),
@@ -533,7 +533,7 @@ bool cluster_client_t::try_send(cluster_op_t *op, cluster_op_part_t *part)
             part->osd_num = primary_osd;
             part->sent = true;
             op->sent_count++;
-            part->op = {
+            part->op = (osd_op_t){
                 .op_type = OSD_OP_OUT,
                 .peer_fd = peer_fd,
                 .req = { .rw = {
@@ -694,7 +694,7 @@ void cluster_client_t::send_sync(cluster_op_t *op, cluster_op_part_t *part)
     assert(peer_it != msgr.osd_peer_fds.end());
     part->sent = true;
     op->sent_count++;
-    part->op = {
+    part->op = (osd_op_t){
         .op_type = OSD_OP_OUT,
         .peer_fd = peer_it->second,
         .req = {

src/cluster_client.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/crc32c.h

@@ -8,4 +8,10 @@
 // unsigned __int64 _mm_crc32_u64 (unsigned __int64 crc, unsigned __int64 v)
 // unsigned int _mm_crc32_u8 (unsigned int crc, unsigned char v)
 
+#ifdef __cplusplus
+extern "C" {
+#endif
 uint32_t crc32c(uint32_t crc, const void *buf, size_t len);
+#ifdef __cplusplus
+};
+#endif

src/dump_journal.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #define _LARGEFILE64_SOURCE
 #include <sys/types.h>
@@ -26,23 +26,32 @@ struct journal_dump_t
     uint64_t journal_offset;
     uint64_t journal_len;
     uint64_t journal_pos;
+    bool all;
+    bool started;
     int fd;
+    uint32_t crc32_last;
 
-    void dump_block(void *buf);
+    int dump_block(void *buf);
 };
 
 int main(int argc, char *argv[])
 {
-    if (argc < 5)
+    journal_dump_t self = { 0 };
+    int b = 1;
+    if (argc >= 2 && !strcmp(argv[1], "--all"))
     {
-        printf("USAGE: %s <journal_file> <journal_block_size> <offset> <size>\n", argv[0]);
+        self.all = true;
+        b = 2;
+    }
+    if (argc < b+4)
+    {
+        printf("USAGE: %s [--all] <journal_file> <journal_block_size> <offset> <size>\n", argv[0]);
         return 1;
     }
-    journal_dump_t self;
+    self.journal_device = argv[b];
-    self.journal_device = argv[1];
+    self.journal_block = strtoul(argv[b+1], NULL, 10);
-    self.journal_block = strtoul(argv[2], NULL, 10);
+    self.journal_offset = strtoull(argv[b+2], NULL, 10);
-    self.journal_offset = strtoull(argv[3], NULL, 10);
+    self.journal_len = strtoull(argv[b+3], NULL, 10);
-    self.journal_len = strtoull(argv[4], NULL, 10);
     if (self.journal_block < MEM_ALIGNMENT || (self.journal_block % MEM_ALIGNMENT) ||
         self.journal_block > 128*1024)
     {
@@ -57,30 +66,64 @@ int main(int argc, char *argv[])
     }
     void *data = memalign(MEM_ALIGNMENT, self.journal_block);
     self.journal_pos = 0;
-    while (self.journal_pos < self.journal_len)
+    if (self.all)
+    {
+        while (self.journal_pos < self.journal_len)
+        {
+            int r = pread(self.fd, data, self.journal_block, self.journal_offset+self.journal_pos);
+            assert(r == self.journal_block);
+            uint64_t s;
+            for (s = 0; s < self.journal_block; s += 8)
+            {
+                if (*((uint64_t*)(data+s)) != 0)
+                    break;
+            }
+            if (s == self.journal_block)
+            {
+                printf("offset %08lx: zeroes\n", self.journal_pos);
+                self.journal_pos += self.journal_block;
+            }
+            else if (((journal_entry*)data)->magic == JOURNAL_MAGIC)
+            {
+                printf("offset %08lx:\n", self.journal_pos);
+                self.dump_block(data);
+            }
+            else
+            {
+                printf("offset %08lx: no magic in the beginning, looks like random data (pattern=%lx)\n", self.journal_pos, *((uint64_t*)data));
+                self.journal_pos += self.journal_block;
+            }
+        }
+    }
+    else
     {
         int r = pread(self.fd, data, self.journal_block, self.journal_offset+self.journal_pos);
         assert(r == self.journal_block);
-        uint64_t s;
+        journal_entry *je = (journal_entry*)(data);
-        for (s = 0; s < self.journal_block; s += 8)
+        if (je->magic != JOURNAL_MAGIC || je->type != JE_START || je_crc32(je) != je->crc32)
         {
-            if (*((uint64_t*)(data+s)) != 0)
+            printf("offset %08lx: journal superblock is invalid\n", self.journal_pos);
-                break;
-        }
-        if (s == self.journal_block)
-        {
-            printf("offset %08lx: zeroes\n", self.journal_pos);
-            self.journal_pos += self.journal_block;
-        }
-        else if (((journal_entry*)data)->magic == JOURNAL_MAGIC)
-        {
-            printf("offset %08lx:\n", self.journal_pos);
-            self.dump_block(data);
         }
         else
         {
-            printf("offset %08lx: no magic in the beginning, looks like random data (pattern=%lx)\n", self.journal_pos, *((uint64_t*)data));
+            printf("offset %08lx:\n", self.journal_pos);
-            self.journal_pos += self.journal_block;
+            self.dump_block(data);
+            self.started = false;
+            self.journal_pos = je->start.journal_start;
+            while (1)
+            {
+                if (self.journal_pos >= self.journal_len)
+                    self.journal_pos = self.journal_block;
+                r = pread(self.fd, data, self.journal_block, self.journal_offset+self.journal_pos);
+                assert(r == self.journal_block);
+                printf("offset %08lx:\n", self.journal_pos);
+                r = self.dump_block(data);
+                if (r <= 0)
+                {
+                    printf("end of the journal\n");
+                    break;
+                }
+            }
         }
     }
     free(data);
@@ -88,7 +131,7 @@ int main(int argc, char *argv[])
     return 0;
 }
 
-void journal_dump_t::dump_block(void *buf)
+int journal_dump_t::dump_block(void *buf)
 {
     uint32_t pos = 0;
     journal_pos += journal_block;
@@ -97,12 +140,19 @@ void journal_dump_t::dump_block(void *buf)
     while (pos < journal_block)
     {
         journal_entry *je = (journal_entry*)(buf + pos);
-        if (je->magic != JOURNAL_MAGIC || je->type < JE_MIN || je->type > JE_MAX)
+        if (je->magic != JOURNAL_MAGIC || je->type < JE_MIN || je->type > JE_MAX ||
+            !all && started && je->crc32_prev != crc32_last)
         {
             break;
         }
-        const char *crc32_valid = je_crc32(je) == je->crc32 ? "(valid)" : "(invalid)";
+        bool crc32_valid = je_crc32(je) == je->crc32;
-        printf("entry % 3d: crc32=%08x %s prev=%08x ", entry, je->crc32, crc32_valid, je->crc32_prev);
+        if (!all && !crc32_valid)
+        {
+            break;
+        }
+        started = true;
+        crc32_last = je->crc32;
+        printf("entry % 3d: crc32=%08x %s prev=%08x ", entry, je->crc32, (crc32_valid ? "(valid)" : "(invalid)"), je->crc32_prev);
         if (je->type == JE_START)
         {
             printf("je_start start=%08lx\n", je->start.journal_start);
@@ -170,4 +220,5 @@ void journal_dump_t::dump_block(void *buf)
     {
         journal_pos = journal_len;
     }
+    return entry;
 }

src/epoll_manager.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include <sys/epoll.h>
 #include <sys/poll.h>
@@ -84,8 +84,12 @@ void epoll_manager_t::handle_epoll_events()
         nfds = epoll_wait(epoll_fd, events, MAX_EPOLL_EVENTS, 0);
         for (int i = 0; i < nfds; i++)
         {
-            auto & cb = epoll_handlers[events[i].data.fd];
+            auto cb_it = epoll_handlers.find(events[i].data.fd);
-            cb(events[i].data.fd, events[i].events);
+            if (cb_it != epoll_handlers.end())
+            {
+                auto & cb = cb_it->second;
+                cb(events[i].data.fd, events[i].events);
+            }
         }
     } while (nfds == MAX_EPOLL_EVENTS);
 }

src/epoll_manager.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/etcd_state_client.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include "osd_ops.h"
 #include "pg_states.h"
@@ -173,6 +173,7 @@ void etcd_state_client_t::start_etcd_watcher()
             { "range_end", base64_encode(etcd_prefix+"/config0") },
             { "start_revision", etcd_watch_revision+1 },
             { "watch_id", ETCD_CONFIG_WATCH_ID },
+            { "progress_notify", true },
         } }
     }).dump());
     etcd_watch_ws->post_message(WS_TEXT, json11::Json(json11::Json::object {
@@ -181,6 +182,7 @@ void etcd_state_client_t::start_etcd_watcher()
             { "range_end", base64_encode(etcd_prefix+"/osd/state0") },
             { "start_revision", etcd_watch_revision+1 },
             { "watch_id", ETCD_OSD_STATE_WATCH_ID },
+            { "progress_notify", true },
         } }
     }).dump());
     etcd_watch_ws->post_message(WS_TEXT, json11::Json(json11::Json::object {
@@ -189,6 +191,7 @@ void etcd_state_client_t::start_etcd_watcher()
             { "range_end", base64_encode(etcd_prefix+"/pg/state0") },
             { "start_revision", etcd_watch_revision+1 },
             { "watch_id", ETCD_PG_STATE_WATCH_ID },
+            { "progress_notify", true },
         } }
     }).dump());
     etcd_watch_ws->post_message(WS_TEXT, json11::Json(json11::Json::object {
@@ -197,6 +200,7 @@ void etcd_state_client_t::start_etcd_watcher()
             { "range_end", base64_encode(etcd_prefix+"/pg/history0") },
             { "start_revision", etcd_watch_revision+1 },
             { "watch_id", ETCD_PG_HISTORY_WATCH_ID },
+            { "progress_notify", true },
         } }
     }).dump());
 }
@@ -315,67 +319,99 @@ void etcd_state_client_t::parse_state(const std::string & key, const json11::Jso
         }
         for (auto & pool_item: value.object_items())
         {
+            pool_config_t pc;
+            // ID
             pool_id_t pool_id = stoull_full(pool_item.first);
             if (!pool_id || pool_id >= POOL_ID_MAX)
             {
                 printf("Pool ID %s is invalid (must be a number less than 0x%x), skipping pool\n", pool_item.first.c_str(), POOL_ID_MAX);
                 continue;
             }
-            if (pool_item.second["pg_size"].uint64_value() < 1 ||
+            pc.id = pool_id;
-                pool_item.second["scheme"] == "xor" && pool_item.second["pg_size"].uint64_value() < 3)
+            // Pool Name
-            {
+            pc.name = pool_item.second["name"].string_value();
-                printf("Pool %u has invalid pg_size, skipping pool\n", pool_id);
+            if (pc.name == "")
-                continue;
-            }
-            if (pool_item.second["pg_minsize"].uint64_value() < 1 ||
-                pool_item.second["pg_minsize"].uint64_value() > pool_item.second["pg_size"].uint64_value() ||
-                pool_item.second["pg_minsize"].uint64_value() < (pool_item.second["pg_size"].uint64_value() - 1))
-            {
-                printf("Pool %u has invalid pg_minsize, skipping pool\n", pool_id);
-                continue;
-            }
-            if (pool_item.second["pg_count"].uint64_value() < 1)
-            {
-                printf("Pool %u has invalid pg_count, skipping pool\n", pool_id);
-                continue;
-            }
-            if (pool_item.second["name"].string_value() == "")
             {
                 printf("Pool %u has empty name, skipping pool\n", pool_id);
                 continue;
             }
-            if (pool_item.second["scheme"] != "replicated" && pool_item.second["scheme"] != "xor")
+            // Failure Domain
+            pc.failure_domain = pool_item.second["failure_domain"].string_value();
+            // Coding Scheme
+            if (pool_item.second["scheme"] == "replicated")
+                pc.scheme = POOL_SCHEME_REPLICATED;
+            else if (pool_item.second["scheme"] == "xor")
+                pc.scheme = POOL_SCHEME_XOR;
+            else if (pool_item.second["scheme"] == "jerasure")
+                pc.scheme = POOL_SCHEME_JERASURE;
+            else
             {
-                printf("Pool %u has invalid coding scheme (only \"xor\" and \"replicated\" are allowed), skipping pool\n", pool_id);
+                printf("Pool %u has invalid coding scheme (one of \"xor\", \"replicated\" or \"jerasure\" required), skipping pool\n", pool_id);
                 continue;
             }
-            if (pool_item.second["max_osd_combinations"].uint64_value() > 0 &&
+            // PG Size
-                pool_item.second["max_osd_combinations"].uint64_value() < 100)
+            pc.pg_size = pool_item.second["pg_size"].uint64_value();
+            if (pc.pg_size < 1 ||
+                pool_item.second["pg_size"].uint64_value() < 3 &&
+                (pc.scheme == POOL_SCHEME_XOR || pc.scheme == POOL_SCHEME_JERASURE) ||
+                pool_item.second["pg_size"].uint64_value() > 256)
+            {
+                printf("Pool %u has invalid pg_size, skipping pool\n", pool_id);
+                continue;
+            }
+            // Parity Chunks
+            pc.parity_chunks = pool_item.second["parity_chunks"].uint64_value();
+            if (pc.scheme == POOL_SCHEME_XOR)
+            {
+                if (pc.parity_chunks > 1)
+                {
+                    printf("Pool %u has invalid parity_chunks (must be 1), skipping pool\n", pool_id);
+                    continue;
+                }
+                pc.parity_chunks = 1;
+            }
+            if (pc.scheme == POOL_SCHEME_JERASURE &&
+                (pc.parity_chunks < 1 || pc.parity_chunks > pc.pg_size-2))
+            {
+                printf("Pool %u has invalid parity_chunks (must be between 1 and pg_size-2), skipping pool\n", pool_id);
+                continue;
+            }
+            // PG MinSize
+            pc.pg_minsize = pool_item.second["pg_minsize"].uint64_value();
+            if (pc.pg_minsize < 1 || pc.pg_minsize > pc.pg_size ||
+                (pc.scheme == POOL_SCHEME_XOR || pc.scheme == POOL_SCHEME_JERASURE) &&
+                pc.pg_minsize < (pc.pg_size-pc.parity_chunks))
+            {
+                printf("Pool %u has invalid pg_minsize, skipping pool\n", pool_id);
+                continue;
+            }
+            // PG Count
+            pc.pg_count = pool_item.second["pg_count"].uint64_value();
+            if (pc.pg_count < 1)
+            {
+                printf("Pool %u has invalid pg_count, skipping pool\n", pool_id);
+                continue;
+            }
+            // Max OSD Combinations
+            pc.max_osd_combinations = pool_item.second["max_osd_combinations"].uint64_value();
+            if (!pc.max_osd_combinations)
+                pc.max_osd_combinations = 10000;
+            if (pc.max_osd_combinations > 0 && pc.max_osd_combinations < 100)
             {
                 printf("Pool %u has invalid max_osd_combinations (must be at least 100), skipping pool\n", pool_id);
                 continue;
             }
+            // PG Stripe Size
+            pc.pg_stripe_size = pool_item.second["pg_stripe_size"].uint64_value();
+            uint64_t min_stripe_size = bs_block_size * (pc.scheme == POOL_SCHEME_REPLICATED ? 1 : (pc.pg_size-pc.parity_chunks));
+            if (pc.pg_stripe_size < min_stripe_size)
+                pc.pg_stripe_size = min_stripe_size;
+            // Save
+            pc.real_pg_count = this->pool_config[pool_id].real_pg_count;
+            std::swap(pc.pg_config, this->pool_config[pool_id].pg_config);
+            std::swap(this->pool_config[pool_id], pc);
             auto & parsed_cfg = this->pool_config[pool_id];
             parsed_cfg.exists = true;
-            parsed_cfg.id = pool_id;
-            parsed_cfg.name = pool_item.second["name"].string_value();
-            parsed_cfg.scheme = pool_item.second["scheme"] == "replicated" ? POOL_SCHEME_REPLICATED : POOL_SCHEME_XOR;
-            parsed_cfg.pg_size = pool_item.second["pg_size"].uint64_value();
-            parsed_cfg.pg_minsize = pool_item.second["pg_minsize"].uint64_value();
-            parsed_cfg.pg_count = pool_item.second["pg_count"].uint64_value();
-            parsed_cfg.failure_domain = pool_item.second["failure_domain"].string_value();
-            parsed_cfg.pg_stripe_size = pool_item.second["pg_stripe_size"].uint64_value();
-            uint64_t min_stripe_size = bs_block_size *
-                (parsed_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : parsed_cfg.pg_minsize);
-            if (parsed_cfg.pg_stripe_size < min_stripe_size)
-            {
-                parsed_cfg.pg_stripe_size = min_stripe_size;
-            }
-            parsed_cfg.max_osd_combinations = pool_item.second["max_osd_combinations"].uint64_value();
-            if (!parsed_cfg.max_osd_combinations)
-            {
-                parsed_cfg.max_osd_combinations = 10000;
-            }
             for (auto & pg_item: parsed_cfg.pg_config)
             {
                 if (pg_item.second.target_set.size() != parsed_cfg.pg_size)

src/etcd_state_client.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 
@@ -43,7 +43,7 @@ struct pool_config_t
     pool_id_t id;
     std::string name;
     uint64_t scheme;
-    uint64_t pg_size, pg_minsize;
+    uint64_t pg_size, pg_minsize, parity_chunks;
     uint64_t pg_count;
     uint64_t real_pg_count;
     std::string failure_domain;

src/fio_cluster.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 // FIO engine to test cluster I/O
 //
@@ -93,7 +93,7 @@ static struct fio_option options[] = {
     {
         .name   = "cluster_log_level",
         .lname  = "cluster log level",
-        .type   = FIO_OPT_BOOL,
+        .type   = FIO_OPT_INT,
         .off1   = offsetof(struct sec_options, cluster_log),
         .help   = "Set log level for the Vitastor client",
         .def    = "0",
@@ -145,9 +145,7 @@ static void sec_cleanup(struct thread_data *td)
         delete bsd->cli;
         delete bsd->epmgr;
         delete bsd->ringloop;
-        bsd->cli = NULL;
+        delete bsd;
-        bsd->epmgr = NULL;
-        bsd->ringloop = NULL;
     }
 }
 

src/fio_engine.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 // FIO engine to test Blockstore
 //

src/fio_sec_osd.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 // FIO engine to test Blockstore through Secondary OSD interface
 //
@@ -140,6 +140,7 @@ static void sec_cleanup(struct thread_data *td)
     if (bsd)
     {
         close(bsd->connect_fd);
+        delete bsd;
     }
 }
 
@@ -312,6 +313,7 @@ static int sec_getevents(struct thread_data *td, unsigned int min, unsigned int
             exit(1);
         }
         io_u* io = it->second;
+        bsd->queue.erase(it);
         if (io->ddir == DDIR_READ)
         {
             if (reply.hdr.retval != io->xfer_buflen)

src/http_client.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include <netinet/tcp.h>
 #include <sys/epoll.h>

src/http_client.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 #include <string>

src/malloc_or_die.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/messenger.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include <unistd.h>
 #include <fcntl.h>
@@ -30,7 +30,7 @@ osd_messenger_t::~osd_messenger_t()
 {
     while (clients.size() > 0)
     {
-        stop_client(clients.begin()->first);
+        stop_client(clients.begin()->first, true);
     }
 }
 
@@ -111,12 +111,12 @@ void osd_messenger_t::try_connect_peer_addr(osd_num_t peer_osd, const char *peer
         timeout_id = tfd->set_timer(1000*peer_connect_timeout, false, [this, peer_fd](int timer_id)
         {
             osd_num_t peer_osd = clients.at(peer_fd)->osd_num;
-            stop_client(peer_fd);
+            stop_client(peer_fd, true);
             on_connect_peer(peer_osd, -EIO);
             return;
         });
     }
-    clients[peer_fd] = new osd_client_t({
+    clients[peer_fd] = new osd_client_t((osd_client_t){
         .peer_addr = addr,
         .peer_port = peer_port,
         .peer_fd = peer_fd,
@@ -149,7 +149,7 @@ void osd_messenger_t::handle_connect_epoll(int peer_fd)
     }
     if (result != 0)
     {
-        stop_client(peer_fd);
+        stop_client(peer_fd, true);
         on_connect_peer(peer_osd, -result);
         return;
     }
@@ -171,7 +171,7 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
     {
         // Stop client
         printf("[OSD %lu] client %d disconnected\n", this->osd_num, peer_fd);
-        stop_client(peer_fd);
+        stop_client(peer_fd, true);
     }
     else if (epoll_events & EPOLLIN)
     {
@@ -233,7 +233,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
     osd_op_t *op = new osd_op_t();
     op->op_type = OSD_OP_OUT;
     op->peer_fd = cl->peer_fd;
-    op->req = {
+    op->req = (osd_any_op_t){
         .show_conf = {
             .header = {
                 .magic = SECONDARY_OSD_OP_MAGIC,
@@ -309,7 +309,7 @@ void osd_messenger_t::cancel_op(osd_op_t *op)
     }
 }
 
-void osd_messenger_t::stop_client(int peer_fd)
+void osd_messenger_t::stop_client(int peer_fd, bool force)
 {
     assert(peer_fd != 0);
     auto it = clients.find(peer_fd);
@@ -334,6 +334,10 @@ void osd_messenger_t::stop_client(int peer_fd)
                 printf("[OSD %lu] Stopping client %d (regular client)\n", osd_num, peer_fd);
         }
     }
+    else if (!force)
+    {
+        return;
+    }
     cl->peer_state = PEER_STOPPED;
     clients.erase(it);
     tfd->set_fd_handler(peer_fd, false, NULL);
@@ -348,7 +352,14 @@ void osd_messenger_t::stop_client(int peer_fd)
     }
     if (cl->read_op)
     {
-        delete cl->read_op;
+        if (cl->read_op->callback)
+        {
+            cancel_op(cl->read_op);
+        }
+        else
+        {
+            delete cl->read_op;
+        }
         cl->read_op = NULL;
     }
     for (auto rit = read_ready_clients.begin(); rit != read_ready_clients.end(); rit++)
@@ -402,7 +413,7 @@ void osd_messenger_t::accept_connections(int listen_fd)
         fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
         int one = 1;
         setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
-        clients[peer_fd] = new osd_client_t({
+        clients[peer_fd] = new osd_client_t((osd_client_t){
             .peer_addr = addr,
             .peer_port = ntohs(addr.sin_port),
             .peer_fd = peer_fd,

src/messenger.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 
@@ -275,7 +275,7 @@ struct osd_messenger_t
 
 public:
     void connect_peer(uint64_t osd_num, json11::Json peer_state);
-    void stop_client(int peer_fd);
+    void stop_client(int peer_fd, bool force = false);
     void outbox_push(osd_op_t *cur_op);
     std::function<void(osd_op_t*)> exec_op;
     std::function<void(osd_num_t)> repeer_pgs;

src/msgr_receive.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include "messenger.h"
 
@@ -9,6 +9,10 @@ void osd_messenger_t::read_requests()
     {
         int peer_fd = read_ready_clients[i];
         osd_client_t *cl = clients[peer_fd];
+        if (cl->read_msg.msg_iovlen)
+        {
+            continue;
+        }
         if (cl->read_remaining < receive_buffer_size)
         {
             cl->read_iov.iov_base = cl->in_buf;
@@ -29,6 +33,7 @@ void osd_messenger_t::read_requests()
             io_uring_sqe* sqe = ringloop->get_sqe();
             if (!sqe)
             {
+                cl->read_msg.msg_iovlen = 0;
                 read_ready_clients.erase(read_ready_clients.begin(), read_ready_clients.begin() + i);
                 return;
             }
@@ -52,6 +57,7 @@ void osd_messenger_t::read_requests()
 bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
 {
     bool ret = false;
+    cl->read_msg.msg_iovlen = 0;
     cl->refs--;
     if (cl->peer_state == PEER_STOPPED)
     {
@@ -160,8 +166,14 @@ bool osd_messenger_t::handle_finished_read(osd_client_t *cl)
     {
         if (cl->read_op->req.hdr.magic == SECONDARY_OSD_REPLY_MAGIC)
             return handle_reply_hdr(cl);
-        else
+        else if (cl->read_op->req.hdr.magic == SECONDARY_OSD_OP_MAGIC)
             handle_op_hdr(cl);
+        else
+        {
+            printf("Received garbage: magic=%lx id=%lu opcode=%lx from %d\n", cl->read_op->req.hdr.magic, cl->read_op->req.hdr.id, cl->read_op->req.hdr.opcode, cl->peer_fd);
+            stop_client(cl->peer_fd);
+            return false;
+        }
     }
     else if (cl->read_state == CL_READ_DATA)
     {

src/msgr_send.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #define _XOPEN_SOURCE
 #include <limits.h>
@@ -46,7 +46,8 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
         to_send_list.push_back((iovec){ .iov_base = cur_op->req.buf, .iov_len = OSD_PACKET_SIZE });
         cl->sent_ops[cur_op->req.hdr.id] = cur_op;
     }
-    // Pre-defined send_lists
+    to_outbox.push_back(NULL);
+    // Operation data
     if ((cur_op->op_type == OSD_OP_IN
         ? (cur_op->req.hdr.opcode == OSD_OP_READ ||
         cur_op->req.hdr.opcode == OSD_OP_SEC_READ ||
@@ -58,17 +59,17 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
         cur_op->req.hdr.opcode == OSD_OP_SEC_STABILIZE ||
         cur_op->req.hdr.opcode == OSD_OP_SEC_ROLLBACK)) && cur_op->iov.count > 0)
     {
-        to_outbox.push_back(NULL);
         for (int i = 0; i < cur_op->iov.count; i++)
         {
             assert(cur_op->iov.buf[i].iov_base);
             to_send_list.push_back(cur_op->iov.buf[i]);
-            to_outbox.push_back(i == cur_op->iov.count-1 ? cur_op : NULL);
+            to_outbox.push_back(NULL);
         }
     }
-    else
+    if (cur_op->op_type == OSD_OP_IN)
     {
-        to_outbox.push_back(cur_op);
+        // To free it later
+        to_outbox[to_outbox.size()-1] = cur_op;
     }
     if (!ringloop)
     {
@@ -92,6 +93,10 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
 void osd_messenger_t::measure_exec(osd_op_t *cur_op)
 {
     // Measure execution latency
+    if (cur_op->req.hdr.opcode > OSD_OP_MAX)
+    {
+        return;
+    }
     timespec tv_end;
     clock_gettime(CLOCK_REALTIME, &tv_end);
     stats.op_stat_count[cur_op->req.hdr.opcode]++;
@@ -198,11 +203,8 @@ void osd_messenger_t::handle_send(int result, osd_client_t *cl)
             {
                 if (cl->outbox[done])
                 {
-                    // Operation fully sent
+                    // Reply fully sent
-                    if (cl->outbox[done]->op_type == OSD_OP_IN)
+                    delete cl->outbox[done];
-                    {
-                        delete cl->outbox[done];
-                    }
                 }
                 result -= iov.iov_len;
                 done++;

src/nbd_proxy.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 // Similar to qemu-nbd, but sets timeout and uses io_uring
 
 #include <linux/nbd.h>
@@ -17,6 +17,10 @@
 #include "epoll_manager.h"
 #include "cluster_client.h"
 
+#ifndef MSG_ZEROCOPY
+#define MSG_ZEROCOPY 0
+#endif
+
 const char *exe_name = NULL;
 
 class nbd_proxy
@@ -107,7 +111,7 @@ public:
     {
         printf(
             "Vitastor NBD proxy\n"
-            "(c) Vitaliy Filippov, 2020 (VNPL-1.0)\n\n"
+            "(c) Vitaliy Filippov, 2020 (VNPL-1.1)\n\n"
             "USAGE:\n"
             "  %s map --etcd_address <etcd_address> --pool <pool> --inode <inode> --size <size in bytes>\n"
             "  %s unmap /dev/nbd0\n"

src/object_id.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/osd.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <sys/socket.h>
 #include <sys/poll.h>

src/osd.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #pragma once
 

src/osd_cluster.cpp

@@ -1,9 +1,10 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "osd.h"
 #include "base64.h"
 #include "etcd_state_client.h"
+#include "osd_rmw.h"
 
 // Startup sequence:
 //   Start etcd watcher -> Load global OSD configuration -> Bind socket -> Acquire lease -> Report&lock OSD state
@@ -32,12 +33,26 @@ void osd_t::init_cluster()
             }
             pgs[{ 1, 1 }] = (pg_t){
                 .state = PG_PEERING,
+                .scheme = POOL_SCHEME_XOR,
                 .pg_cursize = 0,
+                .pg_size = 3,
+                .pg_minsize = 2,
+                .parity_chunks = 1,
                 .pool_id = 1,
                 .pg_num = 1,
                 .target_set = { 1, 2, 3 },
                 .cur_set = { 0, 0, 0 },
             };
+            st_cli.pool_config[1] = (pool_config_t){
+                .exists = true,
+                .id = 1,
+                .name = "testpool",
+                .scheme = POOL_SCHEME_XOR,
+                .pg_size = 3,
+                .pg_minsize = 2,
+                .pg_count = 1,
+                .real_pg_count = 1,
+            };
             report_pg_state(pgs[{ 1, 1 }]);
             pg_counts[1] = 1;
         }
@@ -127,7 +142,7 @@ json11::Json osd_t::get_statistics()
     }
     st["host"] = self_state["host"];
     json11::Json::object op_stats, subop_stats;
-    for (int i = 0; i <= OSD_OP_MAX; i++)
+    for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
     {
         op_stats[osd_op_names[i]] = json11::Json::object {
             { "count", c_cli.stats.op_stat_count[i] },
@@ -135,7 +150,7 @@ json11::Json osd_t::get_statistics()
             { "bytes", c_cli.stats.op_stat_bytes[i] },
         };
     }
-    for (int i = 0; i <= OSD_OP_MAX; i++)
+    for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
     {
         subop_stats[osd_op_names[i]] = json11::Json::object {
             { "count", c_cli.stats.subop_stat_count[i] },
@@ -369,6 +384,7 @@ void osd_t::create_osd_state()
         {
             st_cli.load_pgs();
         }
+        report_statistics();
     });
 }
 
@@ -479,7 +495,11 @@ void osd_t::apply_pg_count()
             }
             if (still_active > 0)
             {
-                printf("[OSD %lu] PG count change detected, but %d PG(s) are still active. This is not allowed. Exiting\n", this->osd_num, still_active);
+                printf(
+                    "[OSD %lu] PG count change detected for pool %u (new is %lu, old is %u),"
+                    " but %u PG(s) are still active. This is not allowed. Exiting\n",
+                    this->osd_num, pool_item.first, pool_item.second.real_pg_count, pg_counts[pool_item.first], still_active
+                );
                 force_stop(1);
                 return;
             }
@@ -573,7 +593,10 @@ void osd_t::apply_pg_config()
                     }
                     else
                     {
-                        throw std::runtime_error("Unexpected PG "+std::to_string(pg_num)+" state: "+std::to_string(pg_it->second.state));
+                        throw std::runtime_error(
+                            "Unexpected PG "+std::to_string(pool_id)+"/"+std::to_string(pg_num)+
+                            " state: "+std::to_string(pg_it->second.state)
+                        );
                     }
                 }
                 auto & pg = this->pgs[{ .pool_id = pool_id, .pg_num = pg_num }];
@@ -583,6 +606,7 @@ void osd_t::apply_pg_config()
                     .pg_cursize = 0,
                     .pg_size = pool_item.second.pg_size,
                     .pg_minsize = pool_item.second.pg_minsize,
+                    .parity_chunks = pool_item.second.parity_chunks,
                     .pool_id = pool_id,
                     .pg_num = pg_num,
                     .reported_epoch = pg_cfg.epoch,
@@ -590,6 +614,10 @@ void osd_t::apply_pg_config()
                     .all_peers = std::vector<osd_num_t>(all_peers.begin(), all_peers.end()),
                     .target_set = pg_cfg.target_set,
                 };
+                if (pg.scheme == POOL_SCHEME_JERASURE)
+                {
+                    use_jerasure(pg.pg_size, pg.pg_size-pg.parity_chunks, true);
+                }
                 this->pg_state_dirty.insert({ .pool_id = pool_id, .pg_num = pg_num });
                 pg.print_state();
                 if (pg_cfg.cur_primary == this->osd_num)
@@ -777,6 +805,10 @@ void osd_t::report_pg_states()
                     if (pg_it->second.state == PG_OFFLINE)
                     {
                         // Remove offline PGs after reporting their state
+                        if (pg_it->second.scheme == POOL_SCHEME_JERASURE)
+                        {
+                            use_jerasure(pg_it->second.pg_size, pg_it->second.pg_size-pg_it->second.parity_chunks, false);
+                        }
                         this->pgs.erase(pg_it);
                     }
                 }

src/osd_flush.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "osd.h"
 
@@ -95,7 +95,7 @@ void osd_t::handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, p
     {
         // This flush batch is done
         std::vector<osd_op_t*> continue_ops;
-        auto & pg = pgs[pg_id];
+        auto & pg = pgs.at(pg_id);
         auto it = pg.flush_actions.begin(), prev_it = it;
         auto erase_start = it;
         while (1)
@@ -166,7 +166,7 @@ void osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t
     {
         // local
         clock_gettime(CLOCK_REALTIME, &op->tv_begin);
-        op->bs_op = new blockstore_op_t({
+        op->bs_op = new blockstore_op_t((blockstore_op_t){
             .opcode = (uint64_t)(rollback ? BS_OP_ROLLBACK : BS_OP_STABLE),
             .callback = [this, op, pool_id, pg_num, fb](blockstore_op_t *bs_op)
             {
@@ -188,7 +188,7 @@ void osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t
         op->op_type = OSD_OP_OUT;
         op->iov.push_back(op->buf, count * sizeof(obj_ver_id));
         op->peer_fd = peer_fd;
-        op->req = {
+        op->req = (osd_any_op_t){
             .sec_stab = {
                 .header = {
                     .magic = SECONDARY_OSD_OP_MAGIC,
@@ -246,7 +246,7 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op)
 {
     op->osd_op = new osd_op_t();
     op->osd_op->op_type = OSD_OP_OUT;
-    op->osd_op->req = {
+    op->osd_op->req = (osd_any_op_t){
         .rw = {
             .header = {
                 .magic = SECONDARY_OSD_OP_MAGIC,

src/osd_id.h

@@ -1,10 +1,11 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 
 #define POOL_SCHEME_REPLICATED 1
 #define POOL_SCHEME_XOR 2
+#define POOL_SCHEME_JERASURE 3
 #define POOL_ID_MAX 0x10000
 #define POOL_ID_BITS 16
 #define INODE_POOL(inode) (pool_id_t)((inode) >> (64 - POOL_ID_BITS))

src/osd_main.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include "osd.h"
 

src/osd_ops.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #include "osd_ops.h"
 

src/osd_ops.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
 

src/osd_peering.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <netinet/tcp.h>
 #include <sys/epoll.h>
@@ -307,7 +307,7 @@ void osd_t::submit_sync_and_list_subop(osd_num_t role_osd, pg_peering_state_t *p
         osd_op_t *op = new osd_op_t();
         op->op_type = OSD_OP_OUT;
         op->peer_fd = cl->peer_fd;
-        op->req = {
+        op->req = (osd_any_op_t){
             .sec_sync = {
                 .header = {
                     .magic = SECONDARY_OSD_OP_MAGIC,
@@ -382,7 +382,7 @@ void osd_t::submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps)
         osd_op_t *op = new osd_op_t();
         op->op_type = OSD_OP_OUT;
         op->peer_fd = c_cli.osd_peer_fds[role_osd];
-        op->req = {
+        op->req = (osd_any_op_t){
             .sec_list = {
                 .header = {
                     .magic = SECONDARY_OSD_OP_MAGIC,
@@ -473,7 +473,9 @@ bool osd_t::stop_pg(pg_t & pg)
         return false;
     }
     pg.state = pg.state & ~PG_ACTIVE | PG_STOPPING;
-    if (pg.inflight == 0 && !pg.flush_batch)
+    if (pg.inflight == 0 && !pg.flush_batch &&
+        // We must either forget all PG's unstable writes or wait for it to become clean
+        dirty_pgs.find({ .pool_id = pg.pool_id, .pg_num = pg.pg_num }) == dirty_pgs.end())
     {
         finish_stop_pg(pg);
     }

src/osd_peering_pg.cpp

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <unordered_map>
 #include "osd_peering_pg.h"

src/osd_peering_pg.h

@@ -1,5 +1,5 @@
 // Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.0 (see README.md for details)
+// License: VNPL-1.1 (see README.md for details)
 
 #include <map>
 #include <vector>
@@ -75,7 +75,7 @@ struct pg_t
 {
     int state = 0;
     uint64_t scheme = 0;
-    uint64_t pg_cursize = 0, pg_size = 0, pg_minsize = 0;
+    uint64_t pg_cursize = 0, pg_size = 0, pg_minsize = 0, parity_chunks = 0;
     pool_id_t pool_id = 0;
     pg_num_t pg_num = 0;
     uint64_t clean_count = 0, total_count = 0;
@@ -94,7 +94,7 @@ struct pg_t
     std::vector<osd_num_t> cur_set;
     // same thing in state_dict-like format
     pg_osd_set_t cur_loc_set;
-    // moved object map. by default, each object is considered to reside on the cur_set.
+    // moved object map. by default, each object is considered to reside on cur_set.
     // this map stores all objects that differ.
     // it may consume up to ~ (raw storage / object size) * 24 bytes in the worst case scenario
     // which is up to ~192 MB per 1 TB in the worst case scenario