WIP/experimental LRC matrix generator

- Implement `vitastor-cli status` (print cluster status) command
- Add a new `make-osd-hybrid.js` script to quickly prepare a lot of hybrid (HDD+SSD) OSDs
- Implement snapshot deletion for Cinder driver (only works in a healthy cluster)
- Fix a huge :) bug causing reads to return all zeroes during rebalance. Add a test to prevent it in the future
- Disconnect NBD proxy correctly without leaving a zombie [vitastor-nbd] process in D state
- Fix a rare write hang appearing with small write throttling enabled

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8)
 
 project(vitastor)
 
-set(VERSION "0.6.15")
+set(VERSION "0.6.16")
 
 add_subdirectory(src)

README-ru.md

@@ -407,6 +407,7 @@ Vitastor с однопоточной NBD прокси на том же стен
 - На хостах мониторов:
   - Пропишите нужные вам значения в файле `/usr/lib/vitastor/mon/make-units.sh`
   - Создайте юниты systemd для etcd и мониторов: `/usr/lib/vitastor/mon/make-units.sh`
+- Запустите etcd и мониторы: `systemctl start etcd vitastor-mon`
 - Пропишите etcd_address и osd_network в `/etc/vitastor/vitastor.conf`. Например:
   ```
   {
@@ -414,7 +415,14 @@ Vitastor с однопоточной NBD прокси на том же стен
     "osd_network": "10.200.1.0/24"
   }
   ```
-- Создайте юниты systemd для OSD: `/usr/lib/vitastor/make-osd.sh /dev/disk/by-partuuid/XXX [/dev/disk/by-partuuid/YYY ...]`
+- Инициализуйте OSD:
+  - SSD: `/usr/lib/vitastor/make-osd.sh /dev/disk/by-partuuid/XXX [/dev/disk/by-partuuid/YYY ...]`
+  - Гибридные, HDD+SSD: `/usr/lib/vitastor/mon/make-osd-hybrid.js /dev/sda /dev/sdb ...` - передайте
+    все ваши SSD и HDD скрипту в командной строке подряд, скрипт автоматически выделит разделы под
+    журналы на SSD и данные на HDD. Скрипт пропускает HDD, на которых уже есть разделы
+    или вообще какие-то данные, поэтому если диски непустые, сначала очистите их с помощью
+    `wipefs -a`. SSD с таблицей разделов не пропускаются, но так как скрипт создаёт новые разделы
+    для журналов, на SSD должно быть доступно свободное нераспределённое место.
 - Вы можете менять параметры OSD в юнитах systemd или в `vitastor.conf`. Смысл некоторых параметров:
   - `disable_data_fsync 1` - отключает fsync, используется с SSD с конденсаторами.
   - `immediate_commit all` - используется с SSD с конденсаторами.
@@ -430,7 +438,6 @@ Vitastor с однопоточной NBD прокси на том же стен
     диски, используемые на одном из тестовых стендов - Intel D3-S4510 - очень сильно не любят такую
     перезапись, и для них была добавлена эта опция. Когда данный режим включён, также нужно поднимать
     значение `journal_sector_buffer_count`, так как иначе Vitastor не хватит буферов для записи в журнал.
-- Запустите все etcd: `systemctl start etcd`
 - Создайте глобальную конфигурацию в etcd: `etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`
   (если все ваши диски - серверные с конденсаторами).
 - Создайте пулы: `etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":256,"failure_domain":"host"}}'`.

README.md

@@ -360,6 +360,7 @@ and calculate disk offsets almost by hand. This will be fixed in near future.
 - On the monitor hosts:
   - Edit variables at the top of `/usr/lib/vitastor/mon/make-units.sh` to desired values.
   - Create systemd units for the monitor and etcd: `/usr/lib/vitastor/mon/make-units.sh`
+- Start etcd and monitors: `systemctl start etcd vitastor-mon`
 - Put etcd_address and osd_network into `/etc/vitastor/vitastor.conf`. Example:
   ```
   {
@@ -367,7 +368,13 @@ and calculate disk offsets almost by hand. This will be fixed in near future.
     "osd_network": "10.200.1.0/24"
   }
   ```
-- Create systemd units for your OSDs: `/usr/lib/vitastor/mon/make-osd.sh /dev/disk/by-partuuid/XXX [/dev/disk/by-partuuid/YYY ...]`
+- Initialize OSDs:
+  - Simplest, SSD-only: `/usr/lib/vitastor/mon/make-osd.sh /dev/disk/by-partuuid/XXX [/dev/disk/by-partuuid/YYY ...]`
+  - Hybrid, HDD+SSD: `/usr/lib/vitastor/mon/make-osd-hybrid.js /dev/sda /dev/sdb ...` - pass all your
+    devices (HDD and SSD) to this script - it will partition disks and initialize journals on its own.
+    This script skips HDDs which are already partitioned so if you want to use non-empty disks for
+    Vitastor you should first wipe them with `wipefs -a`. SSDs with GPT partition table are not skipped,
+    but some free unpartitioned space must be available because the script creates new partitions for journals.
 - You can change OSD configuration in units or in `vitastor.conf`. Notable configuration variables:
   - `disable_data_fsync 1` - only safe with server-grade drives with capacitors.
   - `immediate_commit all` - use this if all your drives are server-grade.

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.6.15
+VERSION ?= v0.6.16
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.6.15
+          image: vitalif/vitastor-csi:v0.6.16
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.6.15
+          image: vitalif/vitastor-csi:v0.6.16
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.6.15"
+    vitastorCSIDriverVersion = "0.6.16"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (0.6.15-1) unstable; urgency=medium
+vitastor (0.6.16-1) unstable; urgency=medium
 
   * RDMA support
   * Bugfixes

debian/vitastor.Dockerfile

@@ -33,8 +33,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.6.15; \
-    cd vitastor-0.6.15; \
+    cp -r /root/vitastor vitastor-0.6.16; \
+    cd vitastor-0.6.16; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -47,8 +47,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.15.orig.tar.xz vitastor-0.6.15; \
-    cd vitastor-0.6.15; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.16.orig.tar.xz vitastor-0.6.16; \
+    cd vitastor-0.6.16; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

mon/make-osd-hybrid.js

@@ -0,0 +1,414 @@
+#!/usr/bin/nodejs
+// systemd unit generator for hybrid (HDD+SSD) vitastor OSDs
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1
+
+// USAGE: nodejs make-osd-hybrid.js [--disable_ssd_cache 0] [--disable_hdd_cache 0] /dev/sda /dev/sdb /dev/sdc /dev/sdd ...
+// I.e. - just pass all HDDs and SSDs mixed, the script will decide where
+// to put journals on its own
+
+const fs = require('fs');
+const fsp = fs.promises;
+const child_process = require('child_process');
+
+const options = {
+    debug: 1,
+    journal_size: 1024*1024*1024,
+    min_meta_size: 1024*1024*1024,
+    object_size: 1024*1024,
+    bitmap_granularity: 4096,
+    device_block_size: 4096,
+    disable_ssd_cache: 1,
+    disable_hdd_cache: 1,
+};
+
+run().catch(console.fatal);
+
+async function run()
+{
+    const device_list = parse_options();
+    await system_or_die("mkdir -p /var/log/vitastor; chown vitastor /var/log/vitastor");
+    // Collect devices
+    const all_devices = await collect_devices(device_list);
+    const ssds = all_devices.filter(d => d.ssd);
+    const hdds = all_devices.filter(d => !d.ssd);
+    // Collect existing OSD units
+    const osd_units = await collect_osd_units();
+    // Count assigned HDD journals and unallocated space for each SSD
+    await check_journal_count(ssds, osd_units);
+    // Create new OSDs
+    await create_new_hybrid_osds(hdds, ssds, osd_units);
+    process.exit(0);
+}
+
+function parse_options()
+{
+    const devices = [];
+    const opt = {};
+    for (let i = 2; i < process.argv.length; i++)
+    {
+        const arg = process.argv[i];
+        if (arg == '--help' || arg == '-h')
+        {
+            opt.help = true;
+            break;
+        }
+        else if (arg.substr(0, 2) == '--')
+            opt[arg.substr(2)] = process.argv[++i];
+        else
+            devices.push(arg);
+    }
+    if (opt.help || !devices.length)
+    {
+        console.log(
+            'Prepare hybrid (HDD+SSD) Vitastor OSDs\n'+
+            '(c) Vitaliy Filippov, 2019+, license: VNPL-1.1\n\n'+
+            'USAGE: nodejs make-osd-hybrid.js [OPTIONS] /dev/sda /dev/sdb /dev/sdc ...\n'+
+            'Just pass all your SSDs and HDDs in any order, the script will distribute OSDs for you.\n\n'+
+            'OPTIONS (with defaults):\n'+
+            Object.keys(options).map(k => `  --${k} ${options[k]}`).join('\n')
+        );
+        process.exit(0);
+    }
+    for (const k in opt)
+        options[k] = opt[k];
+    return devices;
+}
+
+// Collect devices
+async function collect_devices(devices_to_check)
+{
+    const devices = [];
+    for (const dev of devices_to_check)
+    {
+        if (dev.substr(0, 5) != '/dev/')
+        {
+            console.log(`${dev} does not start with /dev/, skipping`);
+            continue;
+        }
+        if (!await file_exists('/sys/block/'+dev.substr(5)))
+        {
+            console.log(`${dev} is a partition, skipping`);
+            continue;
+        }
+        // Check if the device is an SSD
+        const rot = '/sys/block/'+dev.substr(5)+'/queue/rotational';
+        if (!await file_exists(rot))
+        {
+            console.log(`${dev} does not have ${rot} to check whether it's an SSD, skipping`);
+            continue;
+        }
+        const ssd = !parseInt(await fsp.readFile(rot, { encoding: 'utf-8' }));
+        // Check if the device has partition table
+        let [ has_partition_table, parts ] = await system(`sfdisk --dump ${dev} --json`);
+        if (has_partition_table != 0)
+        {
+            // Check if the device has any data
+            const [ has_data, out ] = await system(`blkid ${dev}`);
+            if (has_data == 0)
+            {
+                console.log(`${dev} contains data, skipping:\n  ${out.trim().replace(/\n/g, '\n  ')}`);
+                continue;
+            }
+        }
+        parts = parts ? JSON.parse(parts).partitiontable : null;
+        if (parts && parts.label != 'gpt')
+        {
+            console.log(`${dev} contains "${parts.label}" partition table, only GPT is supported, skipping`);
+            continue;
+        }
+        devices.push({
+            path: dev,
+            ssd,
+            parts,
+        });
+    }
+    return devices;
+}
+
+// Collect existing OSD units
+async function collect_osd_units()
+{
+    const units = [];
+    for (const unit of (await system("ls /etc/systemd/system/vitastor-osd*.service"))[1].trim().split('\n'))
+    {
+        if (!unit)
+        {
+            continue;
+        }
+        let cmd = /^ExecStart\s*=\s*(([^\n]*\\\n)*[^\n]*)/.exec(await fsp.readFile(unit, { encoding: 'utf-8' }));
+        if (!cmd)
+        {
+            console.log('ExecStart= not found in '+unit+', skipping')
+            continue;
+        }
+        let kv = {}, key;
+        cmd = cmd[1].replace(/^bash\s+-c\s+'/, '')
+            .replace(/>>\s*\S+2>\s*&1\s*'$/, '')
+            .replace(/\s*\\\n\s*/g, ' ')
+            .replace(/([^\s']+)|'([^']+)'/g, (m, m1, m2) =>
+            {
+                m1 = m1||m2;
+                if (key == null)
+                {
+                    if (m1.substr(0, 2) != '--')
+                    {
+                        console.log('Strange command line in '+unit+', stopping');
+                        process.exit(1);
+                    }
+                    key = m1.substr(2);
+                }
+                else
+                {
+                    kv[key] = m1;
+                    key = null;
+                }
+            });
+        units.push(kv);
+    }
+    return units;
+}
+
+// Count assigned HDD journals and unallocated space for each SSD
+async function check_journal_count(ssds, osd_units)
+{
+    const units_by_journal = osd_units.reduce((a, c) =>
+    {
+        if (c.journal_device)
+            a[c.journal_device] = c;
+        return a;
+    }, {});
+    for (const dev of ssds)
+    {
+        dev.journals = 0;
+        if (dev.parts)
+        {
+            for (const part of dev.parts.partitions)
+            {
+                if (part.uuid && units_by_journal['/dev/disk/by-partuuid/'+part.uuid.toLowerCase()])
+                {
+                    dev.journals++;
+                }
+            }
+            dev.free = free_from_parttable(dev.parts);
+        }
+        else
+        {
+            dev.free = parseInt(await system_or_die("blockdev --getsize64 "+dev.path));
+        }
+    }
+}
+
+async function create_new_hybrid_osds(hdds, ssds, osd_units)
+{
+    const units_by_disk = osd_units.reduce((a, c) => { a[c.data_device] = c; return a; }, {});
+    for (const dev of hdds)
+    {
+        if (!dev.parts)
+        {
+            // HDD is not partitioned yet, create a single partition
+            // + is the "default value" for sfdisk
+            await system_or_die('sfdisk '+dev.path, 'label: gpt\n\n+ +\n');
+            dev.parts = JSON.parse(await system_or_die('sfdisk --dump '+dev.path+' --json')).partitiontable;
+        }
+        if (dev.parts.partitions.length != 1)
+        {
+            console.log(dev.path+' has more than 1 partition, skipping');
+        }
+        else if ((dev.parts.partitions[0].start + dev.parts.partitions[0].size) != (1 + dev.parts.lastlba))
+        {
+            console.log(dev.path+'1 is not a whole-disk partition, skipping');
+        }
+        else if (!dev.parts.partitions[0].uuid)
+        {
+            console.log(dev.parts.partitions[0].node+' does not have UUID. Please repartition '+dev.path+' with GPT');
+        }
+        else if (!units_by_disk['/dev/disk/by-partuuid/'+dev.parts.partitions[0].uuid.toLowerCase()])
+        {
+            await create_hybrid_osd(dev, ssds);
+        }
+    }
+}
+
+async function create_hybrid_osd(dev, ssds)
+{
+    // Create a new OSD
+    // Calculate metadata size
+    const data_device = '/dev/disk/by-partuuid/'+dev.parts.partitions[0].uuid.toLowerCase();
+    const data_size = dev.parts.partitions[0].size * dev.parts.sectorsize;
+    const meta_entry_size = 24 + 2*options.object_size/options.bitmap_granularity/8;
+    const entries_per_block = Math.floor(options.device_block_size / meta_entry_size);
+    const object_count = Math.floor(data_size / options.object_size);
+    let meta_size = Math.ceil(1 + object_count / entries_per_block) * options.device_block_size;
+    // Leave some extra space for future metadata formats and round metadata area size to multiples of 1 MB
+    meta_size = 2*meta_size;
+    meta_size = Math.ceil(meta_size/1024/1024) * 1024*1024;
+    if (meta_size < options.min_meta_size)
+        meta_size = options.min_meta_size;
+    let journal_size = Math.ceil(options.journal_size/1024/1024) * 1024*1024;
+    // Pick an SSD for journal, balancing the number of journals across SSDs
+    let selected_ssd;
+    for (const ssd of ssds)
+        if (ssd.free >= (meta_size+journal_size) && (!selected_ssd || selected_ssd.journals > ssd.journals))
+            selected_ssd = ssd;
+    if (!selected_ssd)
+    {
+        console.error('Could not find free space for SSD journal and metadata for '+dev.path);
+        process.exit(1);
+    }
+    // Allocate an OSD number
+    const osd_num = (await system_or_die("vitastor-cli alloc-osd")).trim();
+    if (!osd_num)
+    {
+        console.error('Failed to run vitastor-cli alloc-osd');
+        process.exit(1);
+    }
+    console.log('Creating OSD '+osd_num+' on '+dev.path+' (HDD) with journal and metadata on '+selected_ssd.path+' (SSD)');
+    // Add two partitions: journal and metadata
+    const new_parts = await add_partitions(selected_ssd, [ journal_size, meta_size ]);
+    selected_ssd.journals++;
+    const journal_device = '/dev/disk/by-partuuid/'+new_parts[0].uuid.toLowerCase();
+    const meta_device = '/dev/disk/by-partuuid/'+new_parts[1].uuid.toLowerCase();
+    // Wait until the device symlinks appear
+    while (!await file_exists(journal_device))
+    {
+        await new Promise(ok => setTimeout(ok, 100));
+    }
+    while (!await file_exists(meta_device))
+    {
+        await new Promise(ok => setTimeout(ok, 100));
+    }
+    // Zero out metadata and journal
+    await system_or_die("dd if=/dev/zero of="+journal_device+" bs=1M count="+(journal_size/1024/1024)+" oflag=direct");
+    await system_or_die("dd if=/dev/zero of="+meta_device+" bs=1M count="+(meta_size/1024/1024)+" oflag=direct");
+    // Create unit file for the OSD
+    const has_scsi_cache_type = options.disable_ssd_cache &&
+        (await system("ls /sys/block/"+selected_ssd.path.substr(5)+"/device/scsi_disk/*/cache_type"))[0] == 0;
+    const write_through = options.disable_ssd_cache && (
+        has_scsi_cache_type || selected_ssd.path.substr(5, 4) == 'nvme'
+        && (await system_or_die("/sys/block/"+selected_ssd.path.substr(5)+"/queue/write_cache")).trim() == "write through");
+    await fsp.writeFile('/etc/systemd/system/vitastor-osd'+osd_num+'.service',
+`[Unit]
+Description=Vitastor object storage daemon osd.${osd_num}
+After=network-online.target local-fs.target time-sync.target
+Wants=network-online.target local-fs.target time-sync.target
+PartOf=vitastor.target
+
+[Service]
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitMEMLOCK=infinity
+ExecStart=bash -c '/usr/bin/vitastor-osd \\
+    --osd_num ${osd_num} ${write_through
+        ? "--disable_meta_fsync 1 --disable_journal_fsync 1 --immediate_commit "+(options.disable_hdd_cache ? "all" : "small")
+        : ""} \\
+    --throttle_small_writes 1 \\
+    --disk_alignment ${options.device_block_size} \\
+    --journal_block_size ${options.device_block_size} \\
+    --meta_block_size ${options.device_block_size} \\
+    --journal_no_same_sector_overwrites true \\
+    --journal_sector_buffer_count 1024 \\
+    --block_size ${options.object_size} \\
+    --data_device ${data_device} \\
+    --journal_device ${journal_device} \\
+    --meta_device ${meta_device} >>/var/log/vitastor/osd${osd_num}.log 2>&1'
+WorkingDirectory=/
+ExecStartPre=+chown vitastor:vitastor ${data_device}
+ExecStartPre=+chown vitastor:vitastor ${journal_device}
+ExecStartPre=+chown vitastor:vitastor ${meta_device}${
+    has_scsi_cache_type
+    ? "\nExecStartPre=+bash -c 'D=$$$(readlink "+journal_device+"); echo write through > $$$(dirname /sys/block/*/$$\${D##*/})/device/scsi_disk/*/cache_type'"
+    : ""}${
+    options.disable_hdd_cache
+    ? "\nExecStartPre=+bash -c 'D=$$$(readlink "+data_device+"); echo write through > $$$(dirname /sys/block/*/$$\${D##*/})/device/scsi_disk/*/cache_type'"
+    : ""}
+User=vitastor
+PrivateTmp=false
+TasksMax=infinity
+Restart=always
+StartLimitInterval=0
+RestartSec=10
+
+[Install]
+WantedBy=vitastor.target
+`);
+    await system_or_die("systemctl enable vitastor-osd"+osd_num);
+}
+
+async function add_partitions(dev, sizes)
+{
+    let script = 'label: gpt\n\n';
+    if (dev.parts)
+    {
+        // Old partitions
+        for (const part of dev.parts.partitions)
+        {
+            script += part.node+': '+Object.keys(part).map(k => k == 'node' ? '' : k+'='+part[k]).filter(k => k).join(', ')+'\n';
+        }
+    }
+    // New partitions
+    for (const size of sizes)
+    {
+        script += '+ '+Math.ceil(size/1024)+'KiB\n';
+    }
+    await system_or_die('sfdisk '+dev.path, script);
+    // Get new partition table and find the new partition
+    const newpt = JSON.parse(await system_or_die('sfdisk --dump '+dev.path+' --json')).partitiontable;
+    const old_nodes = dev.parts ? dev.parts.partitions.reduce((a, c) => { a[c.uuid] = true; return a; }, {}) : {};
+    const new_nodes = newpt.partitions.filter(part => !old_nodes[part.uuid]);
+    if (new_nodes.length != sizes.length)
+    {
+        console.error('Failed to partition '+dev.path+': new partitions not found in table');
+        process.exit(1);
+    }
+    dev.parts = newpt;
+    dev.free = free_from_parttable(newpt);
+    return new_nodes;
+}
+
+function free_from_parttable(pt)
+{
+    let free = pt.lastlba + 1 - pt.firstlba;
+    for (const part of pt.partitions)
+    {
+        free -= part.size;
+    }
+    free *= pt.sectorsize;
+    return free;
+}
+
+async function system_or_die(cmd, input = '')
+{
+    let [ exitcode, stdout, stderr ] = await system(cmd, input);
+    if (exitcode != 0)
+    {
+        console.error(cmd+' failed: '+stderr);
+        process.exit(1);
+    }
+    return stdout;
+}
+
+async function system(cmd, input = '')
+{
+    if (options.debug)
+    {
+        process.stderr.write('+ '+cmd+(input ? " <<EOF\n"+input.replace(/\s*$/, '\n')+"EOF" : '')+'\n');
+    }
+    const cp = child_process.spawn(cmd, { shell: true });
+    let stdout = '', stderr = '', finish_cb;
+    cp.stdout.on('data', buf => stdout += buf.toString());
+    cp.stderr.on('data', buf => stderr += buf.toString());
+    cp.on('exit', () => finish_cb && finish_cb());
+    cp.stdin.write(input);
+    cp.stdin.end();
+    if (cp.exitCode == null)
+    {
+        await new Promise(ok => finish_cb = ok);
+    }
+    return [ cp.exitCode, stdout, stderr ];
+}
+
+async function file_exists(filename)
+{
+    return new Promise((ok, no) => fs.access(filename, fs.constants.R_OK, err => ok(!err)));
+}

mon/make-osd.sh

@@ -25,6 +25,10 @@ OPT=$(vitastor-cli simple-offsets --format options $DEV | tr '\n' ' ')
 META=$(vitastor-cli simple-offsets --format json $DEV | jq .data_offset)
 dd if=/dev/zero of=$DEV bs=1048576 count=$(((META+1048575)/1048576)) oflag=direct
 
+mkdir -p /var/log/vitastor
+id vitastor &>/dev/null || useradd vitastor
+chown vitastor /var/log/vitastor
+
 cat >/etc/systemd/system/vitastor-osd$OSD_NUM.service <<EOF
 [Unit]
 Description=Vitastor object storage daemon osd.$OSD_NUM
@@ -36,14 +40,14 @@ PartOf=vitastor.target
 LimitNOFILE=1048576
 LimitNPROC=1048576
 LimitMEMLOCK=infinity
-ExecStart=/usr/bin/vitastor-osd \\
+ExecStart=bash -c '/usr/bin/vitastor-osd \\
     --osd_num $OSD_NUM \\
     --disable_data_fsync 1 \\
     --immediate_commit all \\
     --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
     --journal_no_same_sector_overwrites true \\
     --journal_sector_buffer_count 1024 \\
-    $OPT
+    $OPT >>/var/log/vitastor/osd$OSD_NUM.log 2>&1'
 WorkingDirectory=/
 ExecStartPre=+chown vitastor:vitastor $DEV
 User=vitastor

mon/mon.js

@@ -31,6 +31,7 @@ const etcd_allow = new RegExp('^'+[
     'osd/inodestats/[1-9]\\d*',
     'osd/space/[1-9]\\d*',
     'mon/master',
+    'mon/member/[a-f0-9]+',
     'pg/state/[1-9]\\d*/[1-9]\\d*',
     'pg/stats/[1-9]\\d*/[1-9]\\d*',
     'pg/history/[1-9]\\d*/[1-9]\\d*',
@@ -237,7 +238,10 @@ const etcd_tree = {
     },
     mon: {
         master: {
-            /* ip: [ string ], */
+            /* ip: [ string ], id: uint64_t */
+        },
+        standby: {
+            /* <uint64_t>: { ip: [ string ] }, */
         },
     },
     pg: {
@@ -268,7 +272,7 @@ const etcd_tree = {
                 <pg_id>: {
                     osd_sets: osd_num_t[][],
                     all_peers: osd_num_t[],
-                    epoch: uint32_t,
+                    epoch: uint64_t,
                 },
             }, */
         },
@@ -673,11 +677,25 @@ class Mon
         }, this.etcd_start_timeout, 0);
     }
 
+    get_mon_state()
+    {
+        return { ip: this.local_ips(), hostname: os.hostname() };
+    }
+
     async get_lease()
     {
         const max_ttl = this.config.etcd_mon_ttl + this.config.etcd_mon_timeout/1000*this.config.etcd_mon_retries;
-        const res = await this.etcd_call('/lease/grant', { TTL: max_ttl }, this.config.etcd_mon_timeout, -1);
+        // Get lease
+        let res = await this.etcd_call('/lease/grant', { TTL: max_ttl }, this.config.etcd_mon_timeout, -1);
         this.etcd_lease_id = res.ID;
+        // Register in /mon/member, just for the information
+        const state = this.get_mon_state();
+        res = await this.etcd_call('/kv/put', {
+            key: b64(this.etcd_prefix+'/mon/member/'+this.etcd_lease_id),
+            value: b64(JSON.stringify(state)),
+            lease: ''+this.etcd_lease_id
+        }, this.etcd_start_timeout, 0);
+        // Set refresh timer
         this.lease_timer = setInterval(async () =>
         {
             const res = await this.etcd_call('/lease/keepalive', { ID: this.etcd_lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries);
@@ -703,7 +721,7 @@ class Mon
 
     async become_master()
     {
-        const state = { ip: this.local_ips() };
+        const state = { ...this.get_mon_state(), id: ''+this.etcd_lease_id };
         while (1)
         {
             const res = await this.etcd_call('/kv/txn', {

mon/simple-offsets.js

@@ -49,7 +49,8 @@ async function run()
     }
     options.journal_offset = Math.ceil(options.journal_offset/options.device_block_size)*options.device_block_size;
     const meta_offset = options.journal_offset + Math.ceil(options.journal_size/options.device_block_size)*options.device_block_size;
-    const entries_per_block = Math.floor(options.device_block_size / (24 + 2*options.object_size/options.bitmap_granularity/8));
+    const meta_entry_size = 24 + 2*options.object_size/options.bitmap_granularity/8;
+    const entries_per_block = Math.floor(options.device_block_size / meta_entry_size);
     const object_count = Math.floor((device_size-meta_offset)/options.object_size);
     const meta_size = Math.ceil(1 + object_count / entries_per_block) * options.device_block_size;
     const data_offset = meta_offset + meta_size;

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.6.15'
+VERSION = '0.6.16'
 
 LOG = logging.getLogger(__name__)
 
@@ -355,7 +355,25 @@ class VitastorDriver(driver.CloneableImageVD,
     def revert_to_snapshot(self, context, volume, snapshot):
         """Revert a volume to a given snapshot."""
 
-        # FIXME Delete the image, then recreate it from the snapshot
+        vol_name = utils.convert_str(snapshot.volume_name)
+        snap_name = utils.convert_str(snapshot.name)
+
+        # Delete the image and recreate it from the snapshot
+        args = [ 'vitastor-cli', 'rm', vol_name, *(self._vitastor_args()) ]
+        try:
+            self._execute(*args)
+        except processutils.ProcessExecutionError as exc:
+            LOG.error("Failed to delete image "+vol_name+": "+exc)
+            raise exception.VolumeBackendAPIException(data = exc.stderr)
+        args = [
+            'vitastor-cli', 'create', '--parent', vol_name+'@'+snap_name,
+            vol_name, *(self._vitastor_args())
+        ]
+        try:
+            self._execute(*args)
+        except processutils.ProcessExecutionError as exc:
+            LOG.error("Failed to recreate image "+vol_name+" from "+vol_name+"@"+snap_name+": "+exc)
+            raise exception.VolumeBackendAPIException(data = exc.stderr)
 
     def delete_snapshot(self, snapshot):
         """Deletes a snapshot."""
@@ -363,24 +381,15 @@ class VitastorDriver(driver.CloneableImageVD,
         vol_name = utils.convert_str(snapshot.volume_name)
         snap_name = utils.convert_str(snapshot.name)
 
-        # Find the snapshot
-        resp = self._etcd_txn({ 'success': [
-            { 'request_range': { 'key': 'index/image/'+vol_name+'@'+snap_name } },
-        ] })
-        if len(resp['responses'][0]['kvs']) == 0:
-            raise exception.SnapshotNotFound(snapshot_id = snap_name)
-        inode_id = int(resp['responses'][0]['kvs'][0]['value']['id'])
-        pool_id = int(resp['responses'][0]['kvs'][0]['value']['pool_id'])
-        parents = {}
-        parents[(pool_id << 48) | (inode_id & 0xffffffffffff)] = True
-
-        # Check if there are child volumes
-        children = self._child_count(parents)
-        if children > 0:
-            raise exception.SnapshotIsBusy(snapshot_name = snap_name)
-
-        # FIXME: We can't delete snapshots because we can't merge layers yet
-        raise exception.VolumeBackendAPIException(data = 'Snapshot delete (layer merge) is not implemented yet')
+        args = [
+            'vitastor-cli', 'rm', vol_name+'@'+snap_name,
+            *(self._vitastor_args())
+        ]
+        try:
+            self._execute(*args)
+        except processutils.ProcessExecutionError as exc:
+            LOG.error("Failed to remove snapshot "+vol_name+'@'+snap_name+": "+exc)
+            raise exception.VolumeBackendAPIException(data = exc.stderr)
 
     def _child_count(self, parents):
         children = 0

rpm/build-tarball.sh

@@ -25,4 +25,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.6.15/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.15$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.6.16/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.16$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -34,7 +34,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.15.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.16.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.15
+Version:        0.6.16
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.15.el7.tar.gz
+Source0:        vitastor-0.6.16.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -33,7 +33,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.15.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.16.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.15
+Version:        0.6.16
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.15.el8.tar.gz
+Source0:        vitastor-0.6.16.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -15,7 +15,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.6.15")
+add_definitions(-DVERSION="0.6.16")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)
@@ -154,7 +154,7 @@ target_link_libraries(vitastor-nbd
 
 # vitastor-cli
 add_executable(vitastor-cli
-	cli.cpp cli_alloc_osd.cpp cli_simple_offsets.cpp cli_df.cpp
+	cli.cpp cli_alloc_osd.cpp cli_simple_offsets.cpp cli_status.cpp cli_df.cpp
 	cli_ls.cpp cli_create.cpp cli_modify.cpp cli_flatten.cpp cli_merge.cpp cli_rm_data.cpp cli_rm.cpp
 )
 target_link_libraries(vitastor-cli

src/blockstore_write.cpp

@@ -544,12 +544,13 @@ resume_4:
             if (ref_us > exec_us + throttle_threshold_us)
             {
                 // Pause reply
+                PRIV(op)->op_state = 5;
+                // Remember that the timer can in theory be called right here
                 tfd->set_timer_us(ref_us-exec_us, false, [this, op](int timer_id)
                 {
                     PRIV(op)->op_state++;
                     ringloop->wakeup();
                 });
-                PRIV(op)->op_state = 5;
                 return 1;
             }
         }

src/cli.cpp

@@ -86,6 +86,9 @@ void cli_tool_t::help()
         "(c) Vitaliy Filippov, 2019+ (VNPL-1.1)\n"
         "\n"
         "USAGE:\n"
+        "%s status\n"
+        "  Show cluster status\n"
+        "\n"
         "%s df\n"
         "  Show pool space statistics\n"
         "\n"
@@ -155,7 +158,7 @@ void cli_tool_t::help()
         "  --no-color          Disable colored output\n"
         "  --json              JSON output\n"
         ,
-        exe_name, exe_name, exe_name, exe_name, exe_name, exe_name,
+        exe_name, exe_name, exe_name, exe_name, exe_name, exe_name, exe_name,
         exe_name, exe_name, exe_name, exe_name, exe_name, exe_name
     );
     exit(0);
@@ -266,6 +269,11 @@ void cli_tool_t::run(json11::Json cfg)
         fprintf(stderr, "command is missing\n");
         exit(1);
     }
+    else if (cmd[0] == "status")
+    {
+        // Show cluster status
+        action_cb = start_status(cfg);
+    }
     else if (cmd[0] == "df")
     {
         // Show pool space stats
@@ -340,6 +348,8 @@ void cli_tool_t::run(json11::Json cfg)
     ringloop = new ring_loop_t(512);
     epmgr = new epoll_manager_t(ringloop);
     cli = new cluster_client_t(ringloop, epmgr->tfd, cfg);
+    // Smaller timeout by default for more interactiveness
+    cli->st_cli.etcd_slow_timeout = cli->st_cli.etcd_quick_timeout;
     cli->on_ready([this]()
     {
         // Initialize job

src/cli.h

@@ -51,6 +51,7 @@ public:
     friend struct snap_flattener_t;
     friend struct snap_remover_t;
 
+    std::function<bool(void)> start_status(json11::Json cfg);
     std::function<bool(void)> start_df(json11::Json);
     std::function<bool(void)> start_ls(json11::Json);
     std::function<bool(void)> start_create(json11::Json);
@@ -69,7 +70,7 @@ uint64_t parse_size(std::string size_str);
 
 std::string print_table(json11::Json items, json11::Json header, bool use_esc);
 
-std::string format_size(uint64_t size);
+std::string format_size(uint64_t size, bool nobytes = false);
 
 std::string format_lat(uint64_t lat);
 

src/cli_ls.cpp

@@ -437,22 +437,25 @@ std::string print_table(json11::Json items, json11::Json header, bool use_esc)
 }
 
 static uint64_t size_thresh[] = { 1024l*1024*1024*1024, 1024l*1024*1024, 1024l*1024, 1024, 0 };
+static uint64_t size_thresh_d[] = { 1000000000000l, 1000000000l, 1000000l, 1000l, 0 };
+static const int size_thresh_n = sizeof(size_thresh)/sizeof(size_thresh[0]);
 static const char *size_unit = "TGMKB";
 
-std::string format_size(uint64_t size)
+std::string format_size(uint64_t size, bool nobytes)
 {
+    uint64_t *thr = nobytes ? size_thresh_d : size_thresh;
     char buf[256];
-    for (int i = 0; i < sizeof(size_thresh)/sizeof(size_thresh[0]); i++)
+    for (int i = 0; i < size_thresh_n; i++)
     {
-        if (size >= size_thresh[i] || i >= sizeof(size_thresh)/sizeof(size_thresh[0])-1)
+        if (size >= thr[i] || i >= size_thresh_n-1)
         {
-            double value = size_thresh[i] ? (double)size/size_thresh[i] : size;
+            double value = thr[i] ? (double)size/thr[i] : size;
             int l = snprintf(buf, sizeof(buf), "%.1f", value);
             assert(l < sizeof(buf)-2);
             if (buf[l-1] == '0')
                 l -= 2;
-            buf[l] = ' ';
-            buf[l+1] = size_unit[i];
+            buf[l] = i == size_thresh_n-1 && nobytes ? 0 : ' ';
+            buf[l+1] = i == size_thresh_n-1 && nobytes ? 0 : size_unit[i];
             buf[l+2] = 0;
             break;
         }

src/cli_status.cpp

@@ -0,0 +1,295 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 (see README.md for details)
+
+#include "cli.h"
+#include "cluster_client.h"
+#include "base64.h"
+#include "pg_states.h"
+
+// Print cluster status:
+// etcd, mon, osd states
+// raw/used space, object states, pool states, pg states
+// client io, recovery io, rebalance io
+struct status_printer_t
+{
+    cli_tool_t *parent;
+
+    int state = 0;
+    json11::Json::array mon_members, osd_stats;
+    json11::Json agg_stats;
+    std::map<pool_id_t, json11::Json::object> pool_stats;
+    json11::Json::array etcd_states;
+
+    bool is_done()
+    {
+        return state == 100;
+    }
+
+    void loop()
+    {
+        if (state == 1)
+            goto resume_1;
+        else if (state == 2)
+            goto resume_2;
+        // etcd states
+        {
+            auto addrs = parent->cli->st_cli.get_addresses();
+            etcd_states.resize(addrs.size());
+            for (int i = 0; i < etcd_states.size(); i++)
+            {
+                parent->waiting++;
+                parent->cli->st_cli.etcd_call_oneshot(
+                    addrs[i], "/maintenance/status", json11::Json::object(),
+                    parent->cli->st_cli.etcd_quick_timeout, [this, i](std::string err, json11::Json res)
+                    {
+                        parent->waiting--;
+                        etcd_states[i] = err != "" ? json11::Json::object{ { "error", err } } : res;
+                        parent->ringloop->wakeup();
+                    }
+                );
+            }
+        }
+        state = 1;
+resume_1:
+        if (parent->waiting > 0)
+            return;
+        // Monitors, OSD states
+        parent->etcd_txn(json11::Json::object {
+            { "success", json11::Json::array {
+                json11::Json::object {
+                    { "request_range", json11::Json::object {
+                        { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/mon/") },
+                        { "range_end", base64_encode(parent->cli->st_cli.etcd_prefix+"/mon0") },
+                    } },
+                },
+                json11::Json::object {
+                    { "request_range", json11::Json::object {
+                        { "key", base64_encode(
+                            parent->cli->st_cli.etcd_prefix+"/osd/stats/"
+                        ) },
+                        { "range_end", base64_encode(
+                            parent->cli->st_cli.etcd_prefix+"/osd/stats0"
+                        ) },
+                    } },
+                },
+                json11::Json::object {
+                    { "request_range", json11::Json::object {
+                        { "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/stats") },
+                    } },
+                },
+            } },
+        });
+        state = 2;
+resume_2:
+        if (parent->waiting > 0)
+            return;
+        mon_members = parent->etcd_result["responses"][0]["response_range"]["kvs"].array_items();
+        osd_stats = parent->etcd_result["responses"][1]["response_range"]["kvs"].array_items();
+        if (parent->etcd_result["responses"][2]["response_range"]["kvs"].array_items().size() > 0)
+        {
+            agg_stats = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][2]["response_range"]["kvs"][0]).value;
+        }
+        int etcd_alive = 0;
+        uint64_t etcd_db_size = 0;
+        std::string etcd_detail;
+        for (int i = 0; i < etcd_states.size(); i++)
+        {
+            if (etcd_states[i]["error"].is_null())
+            {
+                etcd_alive++;
+                etcd_db_size = etcd_states[i]["dbSizeInUse"].uint64_value();
+            }
+        }
+        int mon_count = 0;
+        std::string mon_master;
+        for (int i = 0; i < mon_members.size(); i++)
+        {
+            auto kv = parent->cli->st_cli.parse_etcd_kv(mon_members[i]);
+            kv.key = kv.key.substr(parent->cli->st_cli.etcd_prefix.size());
+            if (kv.key.substr(0, 12) == "/mon/member/")
+                mon_count++;
+            else if (kv.key == "/mon/master")
+            {
+                if (kv.value["hostname"].is_string())
+                    mon_master = kv.value["hostname"].string_value();
+                else
+                    mon_master = kv.value["ip"][0].string_value();
+            }
+        }
+        int osd_count = 0, osd_up = 0;
+        uint64_t total_raw = 0, free_raw = 0, free_down_raw = 0, down_raw = 0;
+        for (int i = 0; i < osd_stats.size(); i++)
+        {
+            auto kv = parent->cli->st_cli.parse_etcd_kv(osd_stats[i]);
+            osd_num_t stat_osd_num = 0;
+            char null_byte = 0;
+            sscanf(kv.key.c_str() + parent->cli->st_cli.etcd_prefix.size(), "/osd/stats/%lu%c", &stat_osd_num, &null_byte);
+            if (!stat_osd_num || null_byte != 0)
+            {
+                fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
+                continue;
+            }
+            osd_count++;
+            total_raw += kv.value["size"].uint64_value();
+            free_raw += kv.value["free"].uint64_value();
+            auto peer_it = parent->cli->st_cli.peer_states.find(stat_osd_num);
+            if (peer_it != parent->cli->st_cli.peer_states.end())
+            {
+                osd_up++;
+            }
+            else
+            {
+                down_raw += kv.value["size"].uint64_value();
+                free_down_raw += kv.value["size"].uint64_value();
+            }
+        }
+        int pool_count = 0, pools_active = 0;
+        std::map<std::string, int> pgs_by_state;
+        std::string pgs_by_state_str;
+        for (auto & pool_pair: parent->cli->st_cli.pool_config)
+        {
+            auto & pool_cfg = pool_pair.second;
+            bool active = true;
+            if (pool_cfg.pg_config.size() != pool_cfg.pg_count)
+            {
+                active = false;
+                pgs_by_state["offline"] += pool_cfg.pg_count-pool_cfg.pg_config.size();
+            }
+            pool_count++;
+            for (auto pg_it = pool_cfg.pg_config.begin(); pg_it != pool_cfg.pg_config.end(); pg_it++)
+            {
+                if (!(pg_it->second.cur_state & PG_ACTIVE))
+                {
+                    active = false;
+                }
+                std::string pg_state_str;
+                for (int i = 0; i < pg_state_bit_count; i++)
+                {
+                    if (pg_it->second.cur_state & pg_state_bits[i])
+                    {
+                        pg_state_str += "+";
+                        pg_state_str += pg_state_names[i];
+                    }
+                }
+                if (pg_state_str.size())
+                    pgs_by_state[pg_state_str.substr(1)]++;
+                else
+                    pgs_by_state["offline"]++;
+            }
+            if (active)
+            {
+                pools_active++;
+            }
+        }
+        for (auto & kv: pgs_by_state)
+        {
+            if (pgs_by_state_str.size())
+            {
+                pgs_by_state_str += "\n           ";
+            }
+            pgs_by_state_str += std::to_string(kv.second)+" "+kv.first;
+        }
+        uint64_t object_size = parent->cli->get_bs_block_size();
+        std::string more_states;
+        uint64_t obj_n;
+        obj_n = agg_stats["object_counts"]["misplaced"].uint64_value();
+        if (obj_n > 0)
+            more_states += ", "+format_size(obj_n*object_size)+" misplaced";
+        obj_n = agg_stats["object_counts"]["degraded"].uint64_value();
+        if (obj_n > 0)
+            more_states += ", "+format_size(obj_n*object_size)+" degraded";
+        obj_n = agg_stats["object_counts"]["incomplete"].uint64_value();
+        if (obj_n > 0)
+            more_states += ", "+format_size(obj_n*object_size)+" incomplete";
+        std::string recovery_io;
+        {
+            uint64_t deg_bps = agg_stats["recovery_stats"]["degraded"]["bps"].uint64_value();
+            uint64_t deg_iops = agg_stats["recovery_stats"]["degraded"]["iops"].uint64_value();
+            uint64_t misp_bps = agg_stats["recovery_stats"]["misplaced"]["bps"].uint64_value();
+            uint64_t misp_iops = agg_stats["recovery_stats"]["misplaced"]["iops"].uint64_value();
+            if (deg_iops > 0 || deg_bps > 0)
+                recovery_io += "    recovery:  "+format_size(deg_bps)+"/s, "+format_size(deg_iops, true)+" op/s\n";
+            if (misp_iops > 0 || misp_bps > 0)
+                recovery_io += "    rebalance: "+format_size(misp_bps)+"/s, "+format_size(misp_iops, true)+" op/s\n";
+        }
+        if (parent->json_output)
+        {
+            // JSON output
+            printf("%s\n", json11::Json(json11::Json::object {
+                { "etcd_alive", etcd_alive },
+                { "etcd_count", etcd_states.size() },
+                { "etcd_db_size", etcd_db_size },
+                { "mon_count", mon_count },
+                { "mon_master", mon_master },
+                { "osd_up", osd_up },
+                { "osd_count", osd_count },
+                { "total_raw", total_raw },
+                { "free_raw", free_raw },
+                { "down_raw", down_raw },
+                { "free_down_raw", free_down_raw },
+                { "clean_data", agg_stats["object_counts"]["clean"].uint64_value() * object_size },
+                { "misplaced_data", agg_stats["object_counts"]["misplaced"].uint64_value() * object_size },
+                { "degraded_data", agg_stats["object_counts"]["degraded"].uint64_value() * object_size },
+                { "incomplete_data", agg_stats["object_counts"]["incomplete"].uint64_value() * object_size },
+                { "pool_count", pool_count },
+                { "active_pool_count", pools_active },
+                { "pg_states", pgs_by_state },
+                { "op_stats", agg_stats["op_stats"] },
+                { "recovery_stats", agg_stats["recovery_stats"] },
+                { "object_counts", agg_stats["object_counts"] },
+            }).dump().c_str());
+            state = 100;
+            return;
+        }
+        printf(
+            "  cluster:\n"
+            "    etcd: %d / %ld up, %s database size\n"
+            "    mon:  %d up%s\n"
+            "    osd:  %d / %d up\n"
+            "  \n"
+            "  data:\n"
+            "    raw:   %s used, %s / %s available%s\n"
+            "    state: %s clean%s\n"
+            "    pools: %d / %d active\n"
+            "    pgs:   %s\n"
+            "  \n"
+            "  io:\n"
+            "    client:%s %s/s rd, %s op/s rd, %s/s wr, %s op/s wr\n"
+            "%s",
+            etcd_alive, etcd_states.size(), format_size(etcd_db_size).c_str(),
+            mon_count, mon_master == "" ? "" : (", master "+mon_master).c_str(),
+            osd_up, osd_count,
+            format_size(total_raw-free_raw).c_str(),
+            format_size(free_raw-free_down_raw).c_str(),
+            format_size(total_raw-down_raw).c_str(),
+            (down_raw > 0 ? (", "+format_size(down_raw)+" down").c_str() : ""),
+            format_size(agg_stats["object_counts"]["clean"].uint64_value() * object_size).c_str(), more_states.c_str(),
+            pools_active, pool_count,
+            pgs_by_state_str.c_str(),
+            recovery_io.size() > 0 ? "   " : "",
+            format_size(agg_stats["op_stats"]["primary_read"]["bps"].uint64_value()).c_str(),
+            format_size(agg_stats["op_stats"]["primary_read"]["iops"].uint64_value(), true).c_str(),
+            format_size(agg_stats["op_stats"]["primary_write"]["bps"].uint64_value()).c_str(),
+            format_size(agg_stats["op_stats"]["primary_write"]["iops"].uint64_value(), true).c_str(),
+            recovery_io.c_str()
+        );
+        state = 100;
+    }
+};
+
+std::function<bool(void)> cli_tool_t::start_status(json11::Json cfg)
+{
+    json11::Json::array cmd = cfg["command"].array_items();
+    auto printer = new status_printer_t();
+    printer->parent = this;
+    return [printer]()
+    {
+        printer->loop();
+        if (printer->is_done())
+        {
+            delete printer;
+            return true;
+        }
+        return false;
+    };
+}

src/etcd_state_client.cpp

@@ -64,6 +64,42 @@ void etcd_state_client_t::etcd_txn_slow(json11::Json txn, std::function<void(std
     etcd_call("/kv/txn", txn, etcd_slow_timeout, max_etcd_attempts, 0, callback);
 }
 
+std::vector<std::string> etcd_state_client_t::get_addresses()
+{
+    auto addrs = etcd_local;
+    addrs.insert(addrs.end(), etcd_addresses.begin(), etcd_addresses.end());
+    return addrs;
+}
+
+void etcd_state_client_t::etcd_call_oneshot(std::string etcd_address, std::string api, json11::Json payload,
+    int timeout, std::function<void(std::string, json11::Json)> callback)
+{
+    std::string etcd_api_path;
+    int pos = etcd_address.find('/');
+    if (pos >= 0)
+    {
+        etcd_api_path = etcd_address.substr(pos);
+        etcd_address = etcd_address.substr(0, pos);
+    }
+    std::string req = payload.dump();
+    req = "POST "+etcd_api_path+api+" HTTP/1.1\r\n"
+        "Host: "+etcd_address+"\r\n"
+        "Content-Type: application/json\r\n"
+        "Content-Length: "+std::to_string(req.size())+"\r\n"
+        "Connection: close\r\n"
+        "\r\n"+req;
+    auto http_cli = http_init(tfd);
+    auto cb = [this, http_cli, callback](const http_response_t *response)
+    {
+        std::string err;
+        json11::Json data;
+        response->parse_json_response(err, data);
+        callback(err, data);
+        http_close(http_cli);
+    };
+    http_request(http_cli, etcd_address, req, { .timeout = timeout }, cb);
+}
+
 void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int timeout,
     int retries, int interval, std::function<void(std::string, json11::Json)> callback)
 {

src/etcd_state_client.h

@@ -112,6 +112,8 @@ public:
 
     json11::Json::object serialize_inode_cfg(inode_config_t *cfg);
     etcd_kv_t parse_etcd_kv(const json11::Json & kv_json);
+    std::vector<std::string> get_addresses();
+    void etcd_call_oneshot(std::string etcd_address, std::string api, json11::Json payload, int timeout, std::function<void(std::string, json11::Json)> callback);
     void etcd_call(std::string api, json11::Json payload, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback);
     void etcd_txn(json11::Json txn, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback);
     void etcd_txn_slow(json11::Json txn, std::function<void(std::string, json11::Json)> callback);

src/http_client.cpp

@@ -342,6 +342,8 @@ void http_co_t::handle_events()
             }
             else if (epoll_events & (EPOLLRDHUP|EPOLLERR))
             {
+                if (state == HTTP_CO_HEADERS_RECEIVED)
+                    std::swap(parsed.body, response);
                 close_connection();
                 run_cb_and_clear();
                 break;
@@ -465,6 +467,8 @@ again:
     {
         // < 0 means error, 0 means EOF
         epoll_events = epoll_events & ~EPOLLIN;
+        if (state == HTTP_CO_HEADERS_RECEIVED)
+            std::swap(parsed.body, response);
         close_connection();
         if (res < 0)
             parsed = { .error = std::string("recvmsg: ")+strerror(-res) };

src/lrc/Makefile

@@ -0,0 +1,2 @@
+mat: mat.c
+	gcc -O3 -I/usr/include/jerasure -o mat mat.c -lJerasure

src/lrc/mat.c

@@ -0,0 +1,277 @@
+#include <jerasure/reed_sol.h>
+#include <jerasure.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// Generate LRC matrix: (groups*local + global) code rows with (data_drives) columns
+// w should be >= log2(data_drives + groups*local + global), but not necessary 8/16/32
+int* reed_sol_vandermonde_lrc_matrix(int data_drives, int groups, int local, int global, int w)
+{
+    if (w < 0 || w > 32 || data_drives + groups*local + global > (1<<w))
+    {
+        return NULL;
+    }
+    int *lrc_matrix = (int*)malloc(sizeof(int) * (local*groups+global));
+    int *matrix = reed_sol_vandermonde_coding_matrix(data_drives, local+global, w);
+    for (int gr = 0; gr < groups; gr++)
+    {
+        for (int l = 0; l < local; l++)
+        {
+            for (int j = 0; j < data_drives; j++)
+            {
+                lrc_matrix[(gr*local+l)*data_drives + j] = (j / (data_drives/groups)) == gr ? matrix[l*data_drives + j] : 0;
+            }
+        }
+    }
+    for (int i = 0; i < global; i++)
+    {
+        for (int j = 0; j < data_drives; j++)
+        {
+            lrc_matrix[(groups*local+i)*data_drives + j] = matrix[(local+i)*data_drives + j];
+        }
+    }
+    free(matrix);
+    return lrc_matrix;
+}
+
+// Check if the generated LRC with given parameters is Maximally Reconstructible (MR-LRC)
+// Example of a MR-LRC: (8, 2, 1, 2, 6, 8)
+void check_mr_lrc(int data_drives, int groups, int local, int global, int matrix_w, int w, int print)
+{
+    
+}
+
+int main()
+{
+    int W = 8, MATRIX_W = 6;
+    int n = 8, groups = 2, local = 1, global = 2;
+    //n = 4, groups = 2, local = 1, global = 1;
+    int total_rows = n+groups*local+global;
+    int *matrix = reed_sol_vandermonde_lrc_matrix(n, groups, local, global, MATRIX_W);
+    int *lrc_matrix = (int*)malloc(sizeof(int) * total_rows*n);
+    // Fill identity+LRC matrix
+    for (int i = 0; i < n; i++)
+        for (int j = 0; j < n; j++)
+            lrc_matrix[i*n + j] = j == i ? 1 : 0;
+    memcpy(lrc_matrix + n*n, matrix, (total_rows-n)*n*sizeof(int));
+    free(matrix);
+    matrix = NULL;
+    // Print LRC matrix
+    for (int i = 0; i < total_rows; i++)
+    {
+        for (int j = 0; j < n; j++)
+        {
+            printf("%d ", lrc_matrix[i*n+j]);
+        }
+        printf("\n");
+    }
+    int impossible = 0, success = 0, failures = 0;
+    int *lost_per_group = (int*)malloc(sizeof(int) * groups);
+    for (int lost = local+global+1; lost <= groups*local+global; lost++)
+    //int lost = groups*local+global;
+    {
+        int *erased_matrix = (int*)malloc(sizeof(int) * (total_rows-lost)*n);
+        int *inverted_matrix = (int*)malloc(sizeof(int) * (total_rows-lost)*n);
+        int *p = (int*)malloc(sizeof(int) * (total_rows-lost));
+        for (int i = 0; i < n; i++)
+            p[i] = i;
+        int *p2 = (int*)malloc(sizeof(int) * n);
+        if (total_rows-lost > n)
+        {
+            p[n-1] = n; // skip combinations with all N data disks (0..n-1)
+            for (int i = n; i < total_rows-lost; i++)
+                p[i] = i+1;
+            p[total_rows-lost-1]--; // will be incremented on the first step
+        }
+        int inc = total_rows-lost-1;
+        while (1)
+        {
+            p[inc]++;
+            if (p[inc] >= n+groups*local+global)
+            {
+                if (inc == 0)
+                    break;
+                inc--;
+            }
+            else if (inc+1 < total_rows-lost)
+            {
+                p[inc+1] = p[inc];
+                inc++;
+            }
+            else
+            {
+                // Check if it should be recoverable
+                for (int gr = 0; gr < groups; gr++)
+                {
+                    lost_per_group[gr] = ((gr+1)*(n/groups) > n ? (n - gr*(n/groups)) : n/groups);
+                }
+                // Calculate count of data chunks lost in each group
+                for (int j = 0; j < total_rows-lost; j++)
+                {
+                    if (j < n)
+                    {
+                        lost_per_group[(p[j] / (n/groups))]--;
+                    }
+                }
+                // Every local parity chunk is supposed to restore 1 missing chunk inside its group
+                // So, subtract local parity chunk counts from each group lost chunk count
+                for (int j = 0; j < total_rows-lost; j++)
+                {
+                    if (p[j] >= n && p[j] < n+groups*local && lost_per_group[(p[j]-n)/local] > 0)
+                    {
+                        lost_per_group[(p[j]-n)/local]--;
+                    }
+                }
+                // Every global parity chunk is supposed to restore 1 chunk of all that are still missing
+                int still_missing = 0;
+                for (int gr = 0; gr < groups; gr++)
+                {
+                    still_missing += lost_per_group[gr];
+                }
+                for (int j = 0; j < total_rows-lost; j++)
+                {
+                    if (p[j] >= n+groups*local && still_missing > 0)
+                    {
+                        still_missing--;
+                    }
+                }
+                if (still_missing <= 0)
+                {
+                    // We hope it can be recoverable. Try to invert it
+                    int invert_ok = -1;
+                    if (total_rows-lost == n)
+                    {
+                        for (int i = 0; i < n; i++)
+                            for (int j = 0; j < n; j++)
+                                erased_matrix[i*n+j] = lrc_matrix[p[i]*n+j];
+                        invert_ok = jerasure_invert_matrix(erased_matrix, inverted_matrix, n, W);
+                    }
+                    else
+                    {
+                        // Check submatrices
+                        for (int i = 0; i < n; i++)
+                            p2[i] = i;
+                        p2[n-1]--;
+                        int inc2 = n-1;
+                        while (1)
+                        {
+                            p2[inc2]++;
+                            if (p2[inc2] >= total_rows-lost)
+                            {
+                                if (inc2 == 0)
+                                    break;
+                                inc2--;
+                            }
+                            else if (inc2+1 < n)
+                            {
+                                p2[inc2+1] = p2[inc2];
+                                inc2++;
+                            }
+                            else
+                            {
+                                for (int i = 0; i < n; i++)
+                                    for (int j = 0; j < n; j++)
+                                        erased_matrix[i*n+j] = lrc_matrix[p[p2[i]]*n+j];
+                                invert_ok = jerasure_invert_matrix(erased_matrix, inverted_matrix, n, W);
+                                if (invert_ok == 0)
+                                    break;
+                            }
+                        }
+                    }
+                    if (invert_ok < 0)
+                    {
+                        failures++;
+                        printf("\nFAIL: ");
+                        for (int i = 0; i < total_rows-lost; i++)
+                        {
+                            printf("%d ", p[i]);
+                        }
+                        printf("\nDIRECT:\n");
+                        for (int i = 0; i < total_rows-lost; i++)
+                        {
+                            for (int j = 0; j < n; j++)
+                                printf("%d ", lrc_matrix[p[i]*n+j]);
+                            printf("\n");
+                        }
+                        printf("INVERSE:\n");
+                        for (int i = 0; i < total_rows-lost; i++)
+                        {
+                            for (int j = 0; j < n; j++)
+                                printf("%d ", inverted_matrix[i*n+j]);
+                            printf("\n");
+                        }
+                    }
+                    else
+                    {
+                        success++;
+                        printf("OK: ");
+                        for (int i = 0; i < total_rows-lost; i++)
+                        {
+                            printf("%d ", p[i]);
+                        }
+                        printf("\n");
+                    }
+                }
+                else
+                {
+                    impossible++;
+                    printf("IMPOSSIBLE: ");
+                    for (int i = 0; i < total_rows-lost; i++)
+                    {
+                        printf("%d ", p[i]);
+                    }
+                    printf("\n");
+                }
+            }
+        }
+        free(p2);
+        free(p);
+        free(inverted_matrix);
+        free(erased_matrix);
+    }
+    free(lost_per_group);
+    printf("\n%d recovered, %d impossible, %d failures\n", success, impossible, failures);
+    return 0;
+}
+
+// 1 1 1 1 0 0 0 0
+// 0 0 0 0 1 1 1 1
+// 1 55 39 73 84 181 225 217
+// 1 172 70 235 143 34 200 101
+//
+// Can't recover
+// 1 2 4 5 8 9 10 11    -1
+// 2 3 4 6 8 9 10 11    -1
+// FULL:
+// 1 0 0 0 0 0 0 0 
+// 0 1 0 0 0 0 0 0 
+// 0 0 1 0 0 0 0 0 
+// 0 0 0 1 0 0 0 0 
+// 0 0 0 0 1 0 0 0 
+// 0 0 0 0 0 1 0 0 
+// 0 0 0 0 0 0 1 0 
+// 0 0 0 0 0 0 0 1 
+// 1 1 1 1 0 0 0 0 
+// 0 0 0 0 1 1 1 1 
+// 1 55 39 73 84 181 225 217 
+// 1 172 70 235 143 34 200 101 
+// FIRST UNRECOVERABLE:
+// 0 1 0 0 0 0 0 0 
+// 0 0 1 0 0 0 0 0 
+// 0 0 0 0 1 0 0 0 
+// 0 0 0 0 0 1 0 0 
+// 1 1 1 1 0 0 0 0 
+// 0 0 0 0 1 1 1 1 
+// 1 55 39 73 84 181 225 217 
+// 1 172 70 235 143 34 200 101 
+// SECOND UNRECOVERABLE:
+// 0 0 1 0 0 0 0 0 
+// 0 0 0 1 0 0 0 0 
+// 0 0 0 0 1 0 0 0 
+// 0 0 0 0 0 0 1 0 
+// 1 1 1 1 0 0 0 0 
+// 0 0 0 0 1 1 1 1 
+// 1 55 39 73 84 181 225 217 
+// 1 172 70 235 143 34 200 101 
+// Ho ho ho

src/nbd_proxy.cpp

@@ -54,6 +54,8 @@ protected:
     msghdr read_msg = { 0 }, send_msg = { 0 };
     iovec read_iov = { 0 };
 
+    std::string logfile = "/dev/null";
+
 public:
     ~nbd_proxy()
     {
@@ -278,6 +280,10 @@ public:
                 }
             }
         }
+        if (cfg["logfile"].is_string())
+        {
+            logfile = cfg["logfile"].string_value();
+        }
         if (bg)
         {
             daemonize();
@@ -363,14 +369,14 @@ public:
         setsid();
         if (fork())
             exit(0);
-        if (chdir("/") != 0)
-            fprintf(stderr, "Warning: Failed to chdir into /\n");
         close(0);
         close(1);
         close(2);
         open("/dev/null", O_RDONLY);
-        open("/dev/null", O_WRONLY);
-        open("/dev/null", O_WRONLY);
+        open(logfile.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0666);
+        open(logfile.c_str(), O_WRONLY|O_APPEND|O_CREAT, 0666);
+        if (chdir("/") != 0)
+            fprintf(stderr, "Warning: Failed to chdir into /\n");
     }
 
     json11::Json::object list_mapped()
@@ -684,6 +690,7 @@ protected:
             {
                 assert(result <= cur_left);
                 cur_left -= result;
+                cur_buf = (uint8_t*)cur_buf + result;
                 result = 0;
             }
             if (cur_left <= 0)
@@ -698,6 +705,12 @@ protected:
         if (read_state == CL_READ_HDR)
         {
             int req_type = be32toh(cur_req.type);
+            if (be32toh(cur_req.magic) == NBD_REQUEST_MAGIC && req_type == NBD_CMD_DISC)
+            {
+                // Disconnect
+                close(nbd_fd);
+                exit(0);
+            }
             if (be32toh(cur_req.magic) != NBD_REQUEST_MAGIC ||
                 req_type != NBD_CMD_READ && req_type != NBD_CMD_WRITE && req_type != NBD_CMD_FLUSH)
             {

src/osd_primary.cpp

@@ -194,18 +194,22 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
         // Determine version
         auto vo_it = pg.ver_override.find(op_data->oid);
         op_data->target_ver = vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX;
+        op_data->prev_set = pg.cur_set.data();
+        if (pg.state != PG_ACTIVE)
+        {
+            // PG may be degraded or have misplaced objects
+            op_data->prev_set = get_object_osd_set(pg, op_data->oid, pg.cur_set.data(), &op_data->object_state);
+        }
         if (pg.state == PG_ACTIVE || op_data->scheme == POOL_SCHEME_REPLICATED)
         {
             // Fast happy-path
             cur_op->buf = alloc_read_buffer(op_data->stripes, op_data->pg_data_size, 0);
-            submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, pg.cur_set.data(), cur_op);
+            submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
             op_data->st = 1;
         }
         else
         {
-            // PG may be degraded or have misplaced objects
-            uint64_t* cur_set = get_object_osd_set(pg, op_data->oid, pg.cur_set.data(), &op_data->object_state);
-            if (extend_missing_stripes(op_data->stripes, cur_set, op_data->pg_data_size, pg.pg_size) < 0)
+            if (extend_missing_stripes(op_data->stripes, op_data->prev_set, op_data->pg_data_size, pg.pg_size) < 0)
             {
                 finish_op(cur_op, -EIO);
                 return;
@@ -215,7 +219,7 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
             op_data->scheme = pg.scheme;
             op_data->degraded = 1;
             cur_op->buf = alloc_read_buffer(op_data->stripes, pg.pg_size, 0);
-            submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, cur_set, cur_op);
+            submit_primary_subops(SUBMIT_RMW_READ, op_data->target_ver, op_data->prev_set, cur_op);
             op_data->st = 1;
         }
     }

src/osd_primary_chain.cpp

@@ -400,18 +400,21 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
             stripes[role].read_end = stripes[role].req_end;
         }
         uint64_t *cur_set = pg.cur_set.data();
-        if (pg.state != PG_ACTIVE && op_data->scheme != POOL_SCHEME_REPLICATED)
+        if (pg.state != PG_ACTIVE)
         {
             pg_osd_set_state_t *object_state;
             cur_set = get_object_osd_set(pg, cur_oid, pg.cur_set.data(), &object_state);
-            if (extend_missing_stripes(stripes, cur_set, pg.pg_data_size, pg.pg_size) < 0)
+            if (op_data->scheme != POOL_SCHEME_REPLICATED)
             {
-                free(op_data->chain_reads);
-                op_data->chain_reads = NULL;
-                finish_op(cur_op, -EIO);
-                return -1;
+                if (extend_missing_stripes(stripes, cur_set, pg.pg_data_size, pg.pg_size) < 0)
+                {
+                    free(op_data->chain_reads);
+                    op_data->chain_reads = NULL;
+                    finish_op(cur_op, -EIO);
+                    return -1;
+                }
+                op_data->degraded = 1;
             }
-            op_data->degraded = 1;
         }
         if (op_data->scheme == POOL_SCHEME_REPLICATED)
         {
@@ -465,7 +468,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
         auto vo_it = pg.ver_override.find(cur_oid);
         uint64_t target_ver = vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX;
         uint64_t *cur_set = pg.cur_set.data();
-        if (pg.state != PG_ACTIVE && op_data->scheme != POOL_SCHEME_REPLICATED)
+        if (pg.state != PG_ACTIVE)
         {
             pg_osd_set_state_t *object_state;
             cur_set = get_object_osd_set(pg, cur_oid, pg.cur_set.data(), &object_state);

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 0.6.15
+Version: 0.6.16
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

tests/run_7osds.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+. `dirname $0`/common.sh
+
+if [ "$IMMEDIATE_COMMIT" != "" ]; then
+    NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 1"
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5,"immediate_commit":"all"}'
+else
+    NO_SAME="--journal_sector_buffer_count 1024 --log_level 1"
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5}'
+fi
+
+OSD_SIZE=1024
+OSD_COUNT=7
+OSD_ARGS=
+for i in $(seq 1 $OSD_COUNT); do
+    dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
+    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
+    eval OSD${i}_PID=$!
+done
+
+cd mon
+npm install
+cd ..
+node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log &
+MON_PID=$!
+
+if [ "$EC" != "" ]; then
+    POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1'
+    PG_SIZE=3
+else
+    POOLCFG='"scheme":"replicated","pg_size":2,"pg_minsize":2'
+    PG_SIZE=2
+fi
+$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":32,"failure_domain":"osd"}}'
+
+sleep 2
+
+if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == '$PG_SIZE') | length) == 32'); then
+    format_error "FAILED: 32 PGS NOT CONFIGURED"
+fi
+
+if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 32'); then
+    format_error "FAILED: 32 PGS NOT UP"
+fi
+
+try_reweight()
+{
+    osd=$1
+    w=$2
+    $ETCDCTL put /vitastor/config/osd/$osd '{"reweight":'$w'}'
+    sleep 3
+}
+
+wait_finish_rebalance()
+{
+    sec=$1
+    i=0
+    while [[ $i -lt $sec ]]; do
+        ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 32') && \
+            break
+        if [ $i -eq 60 ]; then
+            format_error "Rebalance couldn't finish in $sec seconds"
+        fi
+        sleep 1
+        i=$((i+1))
+    done
+}

tests/run_tests.sh

@@ -0,0 +1,43 @@
+#!/bin/bash -ex
+# Run all possible tests
+
+cd $(dirname $0)
+
+./test_add_osd.sh
+
+./test_cas.sh
+
+./test_change_pg_count.sh
+EC=1 ./test_change_pg_count.sh
+
+./test_change_pg_size.sh
+
+./test_etcd_fail.sh
+
+./test_failure_domain.sh
+
+./test_interrupted_rebalance.sh
+IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh
+EC=1 ./test_interrupted_rebalance.sh
+EC=1 IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh
+
+./test_minsize_1.sh
+
+./test_move_reappear.sh
+
+./test_rebalance_verify.sh
+IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
+EC=1 ./test_rebalance_verify.sh
+EC=1 IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
+
+./test_rm.sh
+
+./test_snapshot.sh
+SCHEME=replicated ./test_snapshot.sh
+
+./test_splitbrain.sh
+
+./test_write.sh
+SCHEME=replicated ./test_write.sh
+
+./test_write_no_same.sh

tests/test_change_pg_count.sh

@@ -78,7 +78,16 @@ try_change()
     fi
 
     # Check that no objects are lost !
-    nobj=`$ETCDCTL get --prefix '/vitastor/pg/stats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'`
+    # But note that reporting this information may take up to <etcd_report_interval+1> seconds
+    nobj=0
+    waittime=0
+    while [[ $nobj -ne $NOBJ && $waittime -lt 7 ]]; do
+        nobj=`$ETCDCTL get --prefix '/vitastor/pg/stats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'`
+        if [[ $nobj -ne $NOBJ ]]; then
+            waittime=$((waittime+1))
+            sleep 1
+        fi
+    done
     if [ "$nobj" -ne $NOBJ ]; then
         format_error "Data lost after changing PG count to $n: $NOBJ objects expected, but got $nobj"
     fi

tests/test_interrupted_rebalance.sh

@@ -1,41 +1,6 @@
 #!/bin/bash -ex
 
-. `dirname $0`/common.sh
-
-if [ "$IMMEDIATE_COMMIT" != "" ]; then
-    NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 1"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5,"immediate_commit":"all"}'
-else
-    NO_SAME="--journal_sector_buffer_count 1024 --log_level 1"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5}'
-fi
-
-OSD_SIZE=1024
-OSD_COUNT=7
-OSD_ARGS=
-for i in $(seq 1 $OSD_COUNT); do
-    dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
-    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
-    eval OSD${i}_PID=$!
-done
-
-cd mon
-npm install
-cd ..
-node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log &
-MON_PID=$!
-
-$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":32,"failure_domain":"osd"}}'
-
-sleep 2
-
-if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == 32'); then
-    format_error "FAILED: 32 PGS NOT CONFIGURED"
-fi
-
-if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 32'); then
-    format_error "FAILED: 32 PGS NOT UP"
-fi
+. `dirname $0`/run_7osds.sh
 
 IMG_SIZE=960
 
@@ -43,14 +8,6 @@ LD_PRELOAD=libasan.so.5 \
 fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=16 -fsync=16 -rw=write \
     -etcd=$ETCD_URL -pool=1 -inode=2 -size=${IMG_SIZE}M -cluster_log_level=10
 
-try_reweight()
-{
-    osd=$1
-    w=$2
-    $ETCDCTL put /vitastor/config/osd/$osd '{"reweight":'$w'}'
-    sleep 3
-}
-
 try_reweight 1 0
 
 try_reweight 2 0
@@ -72,14 +29,7 @@ try_reweight 4 1
 try_reweight 5 1
 
 # Wait for the rebalance to finish
-for i in {1..60}; do
-    ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 32') && \
-        break
-    if [ $i -eq 60 ]; then
-        format_error "Rebalance couldn't finish in 60 seconds"
-    fi
-    sleep 1
-done
+wait_finish_rebalance 60
 
 # Check that PGs never had degraded objects !
 if grep has_degraded ./testdata/mon.log; then

tests/test_rebalance_verify.sh

@@ -0,0 +1,57 @@
+#!/bin/bash -ex
+
+. `dirname $0`/run_7osds.sh
+
+IMG_SIZE=256
+
+$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
+
+NBD_DEV=$(sudo build/src/vitastor-nbd map --etcd_address $ETCD_URL --image testimg --logfile ./testdata/nbd.log &)
+
+trap "sudo build/src/vitastor-nbd unmap $NBD_DEV"'; kill -9 $(jobs -p)' EXIT
+
+sudo chown $(id -u) $NBD_DEV
+
+dd if=/dev/urandom of=./testdata/img1.bin bs=1M count=$IMG_SIZE
+
+dd if=./testdata/img1.bin of=$NBD_DEV bs=1M count=$IMG_SIZE oflag=direct
+
+verify() {
+    echo "Verifying before rebalance"
+    dd if=$NBD_DEV of=./testdata/img2.bin bs=1M count=$IMG_SIZE iflag=direct
+    diff ./testdata/img1.bin ./testdata/img2.bin
+
+    $ETCDCTL put /vitastor/config/osd/1 '{"reweight":'$1'}'
+    $ETCDCTL put /vitastor/config/osd/2 '{"reweight":'$1'}'
+    $ETCDCTL put /vitastor/config/osd/3 '{"reweight":'$1'}'
+    sleep 1
+
+    for i in {1..10000}; do
+        O=$(((RANDOM*RANDOM) % (IMG_SIZE*128)))
+        dd if=$NBD_DEV of=./testdata/img2.bin bs=4k seek=$O skip=$O count=1 iflag=direct conv=notrunc
+    done
+
+    echo "Verifying during rebalance"
+    diff ./testdata/img1.bin ./testdata/img2.bin
+
+    # Wait for the rebalance to finish
+    wait_finish_rebalance 60
+
+    echo "Verifying after rebalance"
+    dd if=$NBD_DEV of=./testdata/img2.bin bs=1M count=$IMG_SIZE iflag=direct
+    diff ./testdata/img1.bin ./testdata/img2.bin
+}
+
+# Verify with regular reads
+
+verify 0
+
+# Same with chained reads
+
+$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg0","size":'$((IMG_SIZE*1024*1024))'}'
+$ETCDCTL put /vitastor/config/inode/1/2 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))',"parent_id":1}'
+sleep 1
+
+verify 1
+
+format_green OK