Use real messenger in fio_sec_osd

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8)
 
 project(vitastor)
 
-set(VERSION "0.6.12")
+set(VERSION "0.6.10")
 
 add_subdirectory(src)

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.6.12
+VERSION ?= v0.6.10
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.6.12
+          image: vitalif/vitastor-csi:v0.6.10
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.6.12
+          image: vitalif/vitastor-csi:v0.6.10
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.6.12"
+    vitastorCSIDriverVersion = "0.6.10"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (0.6.12-1) unstable; urgency=medium
+vitastor (0.6.10-1) unstable; urgency=medium
 
   * RDMA support
   * Bugfixes

debian/vitastor.Dockerfile

@@ -33,8 +33,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.6.12; \
+    cp -r /root/vitastor vitastor-0.6.10; \
-    cd vitastor-0.6.12; \
+    cd vitastor-0.6.10; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -47,8 +47,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.12.orig.tar.xz vitastor-0.6.12; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.10.orig.tar.xz vitastor-0.6.10; \
-    cd vitastor-0.6.12; \
+    cd vitastor-0.6.10; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

mon/lp-optimizer.js

@@ -50,7 +50,7 @@ async function lp_solve(text)
     return { score, vars };
 }
 
-async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize = 2, max_combinations = 10000, parity_space = 1, ordered = false })
+async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize = 2, max_combinations = 10000, parity_space = 1, round_robin = false })
 {
     if (!pg_count || !osd_tree)
     {
@@ -92,7 +92,7 @@ async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize =
         console.log(lp);
         throw new Error('Problem is infeasible or unbounded - is it a bug?');
     }
-    const int_pgs = make_int_pgs(lp_result.vars, pg_count, ordered);
+    const int_pgs = make_int_pgs(lp_result.vars, pg_count, round_robin);
     const eff = pg_list_space_efficiency(int_pgs, all_weights, pg_minsize, parity_space);
     const res = {
         score: lp_result.score,
@@ -140,20 +140,20 @@ function make_int_pgs(weights, pg_count, round_robin)
     return int_pgs;
 }
 
-function calc_intersect_weights(old_pg_size, pg_size, pg_count, prev_weights, all_pgs, ordered)
+function calc_intersect_weights(pg_size, pg_count, prev_weights, all_pgs)
 {
     const move_weights = {};
-    if ((1 << old_pg_size) < pg_count)
+    if ((1 << pg_size) < pg_count)
     {
         const intersect = {};
         for (const pg_name in prev_weights)
         {
             const pg = pg_name.substr(3).split(/_/);
-            for (let omit = 1; omit < (1 << old_pg_size); omit++)
+            for (let omit = 1; omit < (1 << pg_size); omit++)
             {
                 let pg_omit = [ ...pg ];
-                let intersect_count = old_pg_size;
+                let intersect_count = pg_size;
-                for (let i = 0; i < old_pg_size; i++)
+                for (let i = 0; i < pg_size; i++)
                 {
                     if (omit & (1 << i))
                     {
@@ -161,8 +161,6 @@ function calc_intersect_weights(old_pg_size, pg_size, pg_count, prev_weights, al
                         intersect_count--;
                     }
                 }
-                if (!ordered)
-                    pg_omit = pg_omit.filter(n => n).sort();
                 pg_omit = pg_omit.join(':');
                 intersect[pg_omit] = Math.max(intersect[pg_omit] || 0, intersect_count);
             }
@@ -176,10 +174,10 @@ function calc_intersect_weights(old_pg_size, pg_size, pg_count, prev_weights, al
                 for (let i = 0; i < pg_size; i++)
                 {
                     if (omit & (1 << i))
+                    {
                         pg_omit[i] = '';
+                    }
                 }
-                if (!ordered)
-                    pg_omit = pg_omit.filter(n => n).sort();
                 pg_omit = pg_omit.join(':');
                 max_int = Math.max(max_int, intersect[pg_omit] || 0);
             }
@@ -188,18 +186,15 @@ function calc_intersect_weights(old_pg_size, pg_size, pg_count, prev_weights, al
     }
     else
     {
-        const prev_pg_hashed = Object.keys(prev_weights).map(pg_name => pg_name
+        const prev_pg_hashed = Object.keys(prev_weights).map(pg_name => pg_name.substr(3).split(/_/).reduce((a, c) => { a[c] = 1; return a; }, {}));
-            .substr(3).split(/_/).reduce((a, c, i) => { a[c] = i+1; return a; }, {}));
         for (const pg of all_pgs)
         {
             if (!prev_weights['pg_'+pg.join('_')])
             {
                 let max_int = 0;
-                for (const prev_hash of prev_pg_hashed)
+                for (const prev_hash in prev_pg_hashed)
                 {
-                    const intersect_count = ordered
+                    const intersect_count = pg.reduce((a, osd) => a + (prev_hash[osd] ? 1 : 0), 0);
-                        ? pg.reduce((a, osd, i) => a + (prev_hash[osd] == 1+i ? 1 : 0), 0)
-                        : pg.reduce((a, osd, i) => a + (prev_hash[osd] ? 1 : 0), 0);
                     if (max_int < intersect_count)
                     {
                         max_int = intersect_count;
@@ -248,7 +243,7 @@ function add_valid_previous(osd_tree, prev_weights, all_pgs)
 }
 
 // Try to minimize data movement
-async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3, pg_minsize = 2, max_combinations = 10000, parity_space = 1, ordered = false })
+async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3, pg_minsize = 2, max_combinations = 10000, parity_space = 1 })
 {
     if (!osd_tree)
     {
@@ -271,13 +266,9 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
             prev_pg_per_osd[osd].push([ pg_name, (i >= pg_minsize ? parity_space : 1) ]);
         }
     }
-    const old_pg_size = prev_int_pgs[0].length;
     // Get all combinations
     let all_pgs = random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1);
-    if (old_pg_size == pg_size)
+    add_valid_previous(osd_tree, prev_weights, all_pgs);
-    {
-        add_valid_previous(osd_tree, prev_weights, all_pgs);
-    }
     all_pgs = Object.values(all_pgs);
     const pg_per_osd = {};
     for (const pg of all_pgs)
@@ -291,7 +282,7 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
         }
     }
     // Penalize PGs based on their similarity to old PGs
-    const move_weights = calc_intersect_weights(old_pg_size, pg_size, pg_count, prev_weights, all_pgs, ordered);
+    const move_weights = calc_intersect_weights(pg_size, pg_count, prev_weights, all_pgs);
     // Calculate total weight - old PG weights
     const all_pg_names = all_pgs.map(pg => 'pg_'+pg.join('_'));
     const all_pgs_hash = all_pg_names.reduce((a, c) => { a[c] = true; return a; }, {});
@@ -382,35 +373,11 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
         {
             differs++;
         }
-    }
+        for (let j = 0; j < pg_size; j++)
-    if (ordered)
-    {
-        for (let i = 0; i < pg_count; i++)
         {
-            for (let j = 0; j < pg_size; j++)
+            if (new_pgs[i][j] != prev_int_pgs[i][j])
             {
-                if (new_pgs[i][j] != prev_int_pgs[i][j])
+                osd_differs++;
-                {
-                    osd_differs++;
-                }
-            }
-        }
-    }
-    else
-    {
-        for (let i = 0; i < pg_count; i++)
-        {
-            const old_map = prev_int_pgs[i].reduce((a, c) => { a[c] = (a[c]|0) + 1; return a; }, {});
-            for (let j = 0; j < pg_size; j++)
-            {
-                if ((0|old_map[new_pgs[i][j]]) > 0)
-                {
-                    old_map[new_pgs[i][j]]--;
-                }
-                else
-                {
-                    osd_differs++;
-                }
             }
         }
     }

mon/mon.js

@@ -84,12 +84,7 @@ const etcd_tree = {
             osd_ping_timeout: 5, // seconds. min: 1
             up_wait_retry_interval: 500, // ms. min: 50
             // osd
-            etcd_report_interval: 5, // seconds
+            etcd_report_interval: 5,
-            max_etcd_attempts: 5,
-            etcd_quick_timeout: 1000, // ms
-            etcd_slow_timeout: 5000, // ms
-            etcd_keepalive_timeout: 30, // seconds, default is min(30, etcd_report_interval*2)
-            etcd_ws_keepalive_interval: 30, // seconds
             run_primary: true,
             osd_network: null, // "192.168.7.0/24" or an array of masks
             bind_address: "0.0.0.0",
@@ -346,9 +341,6 @@ class Mon
         this.etcd_start_timeout = (config.etcd_start_timeout || 5) * 1000;
         this.state = JSON.parse(JSON.stringify(this.constructor.etcd_tree));
         this.signals_set = false;
-        this.ws = null;
-        this.ws_alive = false;
-        this.ws_keepalive_timer = null;
         this.on_stop_cb = () => this.on_stop(0).catch(console.error);
     }
 
@@ -391,7 +383,7 @@ class Mon
         for (const pool_id in this.state.config.pools)
         {
             if (!this.state.pool.stats[pool_id] ||
-                !Number(this.state.pool.stats[pool_id].pg_real_size))
+                !this.state.pool.stats[pool_id].pg_real_size)
             {
                 // Generate missing data in etcd
                 this.state.config.pgs.hash = null;
@@ -469,20 +461,8 @@ class Mon
 
     restart_watcher(cur_addr)
     {
-        if (this.ws)
-        {
-            this.ws.close();
-            this.ws = null;
-        }
-        if (this.ws_keepalive_timer)
-        {
-            clearInterval(this.ws_keepalive_timer);
-            this.ws_keepalive_timer = null;
-        }
         if (this.selected_etcd_url == cur_addr)
-        {
             this.selected_etcd_url = null;
-        }
         this.start_watcher(this.config.etcd_mon_retries).catch(this.die);
     }
 
@@ -502,7 +482,6 @@ class Mon
                 const timer_id = setTimeout(() =>
                 {
                     this.ws.close();
-                    this.ws = null;
                     ok(false);
                 }, this.config.etcd_mon_timeout);
                 this.ws = new WebSocket(base+'/watch');
@@ -531,20 +510,6 @@ class Mon
             this.die('Failed to open etcd watch websocket');
         }
         const cur_addr = this.selected_etcd_url;
-        this.ws_alive = true;
-        this.ws_keepalive_timer = setInterval(() =>
-        {
-            if (this.ws_alive)
-            {
-                this.ws_alive = false;
-                this.ws.send(JSON.stringify({ progress_request: {} }));
-            }
-            else
-            {
-                console.log('etcd websocket timed out, restarting it');
-                this.restart_watcher(cur_addr);
-            }
-        }, (Number(this.config.etcd_keepalive_interval) || 30)*1000);
         this.ws.on('error', () => this.restart_watcher(cur_addr));
         this.ws.send(JSON.stringify({
             create_request: {
@@ -557,7 +522,6 @@ class Mon
         }));
         this.ws.on('message', (msg) =>
         {
-            this.ws_alive = true;
             let data;
             try
             {
@@ -594,7 +558,7 @@ class Mon
                     console.log('Revision '+data.result.header.revision+' events: ');
                 }
                 this.etcd_watch_revision = BigInt(data.result.header.revision)+BigInt(1);
-                for (const e of data.result.events||[])
+                for (const e of data.result.events)
                 {
                     this.parse_kv(e.kv);
                     const key = e.kv.key.substr(this.etcd_prefix.length);
@@ -1133,7 +1097,7 @@ class Mon
                     pg_size: pool_cfg.pg_size,
                     pg_minsize: pool_cfg.pg_minsize,
                     max_combinations: pool_cfg.max_osd_combinations,
-                    ordered: pool_cfg.scheme != 'replicated',
+                    round_robin: pool_cfg.scheme != 'replicated',
                 };
                 let optimize_result;
                 if (old_pg_count > 0)
@@ -1156,6 +1120,10 @@ class Mon
                         {
                             pg.push(0);
                         }
+                        while (pg.length > pool_cfg.pg_size)
+                        {
+                            pg.pop();
+                        }
                     }
                     if (!this.state.config.pgs.hash)
                     {
@@ -1191,8 +1159,8 @@ class Mon
                 this.state.pool.stats[pool_id] = {
                     used_raw_tb: (this.state.pool.stats[pool_id]||{}).used_raw_tb || 0,
                     total_raw_tb: optimize_result.space,
-                    pg_real_size: pg_effsize || pool_cfg.pg_size,
+                    pg_real_size: pg_effsize,
-                    raw_to_usable: (pg_effsize || pool_cfg.pg_size) / (pool_cfg.scheme === 'replicated'
+                    raw_to_usable: pg_effsize / (pool_cfg.scheme === 'replicated'
                         ? 1 : (pool_cfg.pg_size - (pool_cfg.parity_chunks||0))),
                     space_efficiency: optimize_result.space/(optimize_result.total_space||1),
                 };

mon/test-optimize-simple.js

@@ -5,45 +5,21 @@ const LPOptimizer = require('./lp-optimizer.js');
 
 async function run()
 {
-    const osd_tree = {
+    const osd_tree = { a: { 1: 1 }, b: { 2: 1 }, c: { 3: 1 } };
-        100: { 1: 1 },
-        200: { 2: 1 },
-        300: { 3: 1 },
-    };
-
     let res;
 
     console.log('16 PGs, size=3');
-    res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 3, pg_count: 16, ordered: false });
+    res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 3, pg_count: 16 });
     LPOptimizer.print_change_stats(res, false);
-    assert(res.space == 3, 'Initial distribution');
+
-    console.log('\nChange size to 2');
+    console.log('\nReduce PG size to 2');
-    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2, ordered: false });
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs.map(pg => pg.slice(0, 2)), osd_tree, pg_size: 2 });
     LPOptimizer.print_change_stats(res, false);
-    assert(res.space >= 3*14/16 && res.osd_differs == 0, 'Redistribution');
+
     console.log('\nRemove OSD 3');
-    const no3_tree = { ...osd_tree };
+    delete osd_tree['c'];
-    delete no3_tree['300'];
+    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2 });
-    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: no3_tree, pg_size: 2, ordered: false });
     LPOptimizer.print_change_stats(res, false);
-    assert(res.space == 2, 'Redistribution after OSD removal');
-
-    console.log('\n16 PGs, size=3, ordered');
-    res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 3, pg_count: 16, ordered: true });
-    LPOptimizer.print_change_stats(res, false);
-    assert(res.space == 3, 'Initial distribution');
-    console.log('\nChange size to 2, ordered');
-    res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2, ordered: true });
-    LPOptimizer.print_change_stats(res, false);
-    assert(res.space >= 3*14/16 && res.osd_differs < 8, 'Redistribution');
-}
-
-function assert(cond, txt)
-{
-    if (!cond)
-    {
-        throw new Error((txt||'test')+' failed');
-    }
 }
 
 run().catch(console.error);

mon/test-optimize-undersized.js

@@ -45,45 +45,30 @@ async function run()
     console.log('Empty tree:');
     let res = await LPOptimizer.optimize_initial({ osd_tree: cur_tree, pg_size: 3, pg_count: 256 });
     LPOptimizer.print_change_stats(res, false);
-    assert(res.space == 0);
     console.log('\nAdding 1st failure domain:');
     cur_tree['dom1'] = osd_tree['dom1'];
     res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
     LPOptimizer.print_change_stats(res, false);
-    assert(res.space == 12 && res.total_space == 12);
     console.log('\nAdding 2nd failure domain:');
     cur_tree['dom2'] = osd_tree['dom2'];
     res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
     LPOptimizer.print_change_stats(res, false);
-    assert(res.space == 24 && res.total_space == 24);
     console.log('\nAdding 3rd failure domain:');
     cur_tree['dom3'] = osd_tree['dom3'];
     res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
     LPOptimizer.print_change_stats(res, false);
-    assert(res.space == 36 && res.total_space == 36);
     console.log('\nRemoving 3rd failure domain:');
     delete cur_tree['dom3'];
     res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
     LPOptimizer.print_change_stats(res, false);
-    assert(res.space == 24 && res.total_space == 24);
     console.log('\nRemoving 2nd failure domain:');
     delete cur_tree['dom2'];
     res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
     LPOptimizer.print_change_stats(res, false);
-    assert(res.space == 12 && res.total_space == 12);
     console.log('\nRemoving 1st failure domain:');
     delete cur_tree['dom1'];
     res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
     LPOptimizer.print_change_stats(res, false);
-    assert(res.space == 0);
-}
-
-function assert(cond, txt)
-{
-    if (!cond)
-    {
-        throw new Error((txt||'test')+' failed');
-    }
 }
 
 run().catch(console.error);

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.6.12'
+VERSION = '0.6.10'
 
 LOG = logging.getLogger(__name__)
 

rpm/build-tarball.sh

@@ -25,4 +25,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.6.12/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.12$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.6.10/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.10$(rpm --eval '%dist').tar.gz *

rpm/vitastor-el7.Dockerfile

@@ -34,7 +34,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.12.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.10.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.12
+Version:        0.6.10
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.12.el7.tar.gz
+Source0:        vitastor-0.6.10.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.Dockerfile

@@ -33,7 +33,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.6.12.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.6.10.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.6.12
+Version:        0.6.10
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.6.12.el8.tar.gz
+Source0:        vitastor-0.6.10.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -15,7 +15,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.6.12")
+add_definitions(-DVERSION="0.6.10")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)
@@ -111,11 +111,12 @@ if (${WITH_FIO})
 	# libfio_vitastor_sec.so
 	add_library(fio_vitastor_sec SHARED
 		fio_sec_osd.cpp
-		rw_blocking.cpp
-		addr_util.cpp
 	)
 	target_link_libraries(fio_vitastor_sec
+		vitastor_common
 		tcmalloc_minimal
+		${LIBURING_LIBRARIES}
+		${IBVERBS_LIBRARIES}
 	)
 endif (${WITH_FIO})
 

src/addr_util.cpp

@@ -1,7 +1,4 @@
 #include <arpa/inet.h>
-#include <net/if.h>
-#include <sys/types.h>
-#include <ifaddrs.h>
 #include <string.h>
 #include <stdio.h>
 
@@ -61,128 +58,3 @@ std::string addr_to_string(const sockaddr &addr)
         throw std::runtime_error(std::string("inet_ntop: ") + strerror(errno));
     return std::string(peer_str)+":"+std::to_string(port);
 }
-
-static bool cidr_match(const in_addr &addr, const in_addr &net, uint8_t bits)
-{
-    if (bits == 0)
-    {
-        // C99 6.5.7 (3): u32 << 32 is undefined behaviour
-        return true;
-    }
-    return !((addr.s_addr ^ net.s_addr) & htonl(0xFFFFFFFFu << (32 - bits)));
-}
-
-static bool cidr6_match(const in6_addr &address, const in6_addr &network, uint8_t bits)
-{
-    const uint32_t *a = address.s6_addr32;
-    const uint32_t *n = network.s6_addr32;
-    int bits_whole, bits_incomplete;
-    bits_whole = bits >> 5;         // number of whole u32
-    bits_incomplete = bits & 0x1F;  // number of bits in incomplete u32
-    if (bits_whole && memcmp(a, n, bits_whole << 2))
-        return false;
-    if (bits_incomplete)
-    {
-        uint32_t mask = htonl((0xFFFFFFFFu) << (32 - bits_incomplete));
-        if ((a[bits_whole] ^ n[bits_whole]) & mask)
-            return false;
-    }
-    return true;
-}
-
-struct addr_mask_t
-{
-    sa_family_t family;
-    in_addr ipv4;
-    in6_addr ipv6;
-    uint8_t bits;
-};
-
-std::vector<std::string> getifaddr_list(std::vector<std::string> mask_cfg, bool include_v6)
-{
-    std::vector<addr_mask_t> masks;
-    for (auto mask: mask_cfg)
-    {
-        unsigned bits = 0;
-        int p = mask.find('/');
-        if (p != std::string::npos)
-        {
-            char null_byte = 0;
-            if (sscanf(mask.c_str()+p+1, "%u%c", &bits, &null_byte) != 1 || bits > 128)
-            {
-                throw std::runtime_error((include_v6 ? "Invalid IPv4 address mask: " : "Invalid IP address mask: ") + mask);
-            }
-            mask = mask.substr(0, p);
-        }
-        in_addr ipv4;
-        in6_addr ipv6;
-        if (inet_pton(AF_INET, mask.c_str(), &ipv4) == 1)
-        {
-            if (bits > 32)
-            {
-                throw std::runtime_error((include_v6 ? "Invalid IPv4 address mask: " : "Invalid IP address mask: ") + mask);
-            }
-            masks.push_back((addr_mask_t){ .family = AF_INET, .ipv4 = ipv4, .bits = (uint8_t)bits });
-        }
-        else if (include_v6 && inet_pton(AF_INET6, mask.c_str(), &ipv6) == 1)
-        {
-            masks.push_back((addr_mask_t){ .family = AF_INET6, .ipv6 = ipv6, .bits = (uint8_t)bits });
-        }
-        else
-        {
-            throw std::runtime_error((include_v6 ? "Invalid IPv4 address mask: " : "Invalid IP address mask: ") + mask);
-        }
-    }
-    std::vector<std::string> addresses;
-    ifaddrs *list, *ifa;
-    if (getifaddrs(&list) == -1)
-    {
-        throw std::runtime_error(std::string("getifaddrs: ") + strerror(errno));
-    }
-    for (ifa = list; ifa != NULL; ifa = ifa->ifa_next)
-    {
-        if (!ifa->ifa_addr)
-        {
-            continue;
-        }
-        int family = ifa->ifa_addr->sa_family;
-        if ((family == AF_INET || family == AF_INET6 && include_v6) &&
-            (ifa->ifa_flags & (IFF_UP | IFF_RUNNING | IFF_LOOPBACK)) == (IFF_UP | IFF_RUNNING))
-        {
-            void *addr_ptr;
-            if (family == AF_INET)
-            {
-                addr_ptr = &((sockaddr_in *)ifa->ifa_addr)->sin_addr;
-            }
-            else
-            {
-                addr_ptr = &((sockaddr_in6 *)ifa->ifa_addr)->sin6_addr;
-            }
-            if (masks.size() > 0)
-            {
-                int i;
-                for (i = 0; i < masks.size(); i++)
-                {
-                    if (masks[i].family == family && (family == AF_INET
-                        ? cidr_match(*(in_addr*)addr_ptr, masks[i].ipv4, masks[i].bits)
-                        : cidr6_match(*(in6_addr*)addr_ptr, masks[i].ipv6, masks[i].bits)))
-                    {
-                        break;
-                    }
-                }
-                if (i >= masks.size())
-                {
-                    continue;
-                }
-            }
-            char addr[INET6_ADDRSTRLEN];
-            if (!inet_ntop(family, addr_ptr, addr, INET6_ADDRSTRLEN))
-            {
-                throw std::runtime_error(std::string("inet_ntop: ") + strerror(errno));
-            }
-            addresses.push_back(std::string(addr));
-        }
-    }
-    freeifaddrs(list);
-    return addresses;
-}

src/addr_util.h

@@ -2,8 +2,6 @@
 
 #include <sys/socket.h>
 #include <string>
-#include <vector>
 
 bool string_to_addr(std::string str, bool parse_port, int default_port, struct sockaddr *addr);
 std::string addr_to_string(const sockaddr &addr);
-std::vector<std::string> getifaddr_list(std::vector<std::string> mask_cfg = std::vector<std::string>(), bool include_v6 = false);

src/blockstore_flush.cpp

@@ -185,7 +185,7 @@ void journal_flusher_t::release_trim()
 void journal_flusher_t::dump_diagnostics()
 {
     const char *unflushable_type = "";
-    obj_ver_id unflushable = {};
+    obj_ver_id unflushable = { 0 };
     // Try to find out if there is a flushable object for information
     for (object_id cur_oid: flush_queue)
     {
@@ -486,8 +486,8 @@ resume_1:
         if (bs->clean_entry_bitmap_size)
         {
             new_clean_bitmap = (bs->inmemory_meta
-                ? (uint8_t*)meta_new.buf + meta_new.pos*bs->clean_entry_size + sizeof(clean_disk_entry)
+                ? meta_new.buf + meta_new.pos*bs->clean_entry_size + sizeof(clean_disk_entry)
-                : (uint8_t*)bs->clean_bitmap + (clean_loc >> bs->block_order)*(2*bs->clean_entry_bitmap_size));
+                : bs->clean_bitmap + (clean_loc >> bs->block_order)*(2*bs->clean_entry_bitmap_size));
             if (clean_init_bitmap)
             {
                 memset(new_clean_bitmap, 0, bs->clean_entry_bitmap_size);
@@ -533,7 +533,7 @@ resume_1:
                 return false;
             }
             // zero out old metadata entry
-            memset((uint8_t*)meta_old.buf + meta_old.pos*bs->clean_entry_size, 0, bs->clean_entry_size);
+            memset(meta_old.buf + meta_old.pos*bs->clean_entry_size, 0, bs->clean_entry_size);
             await_sqe(15);
             data->iov = (struct iovec){ meta_old.buf, bs->meta_block_size };
             data->callback = simple_callback_w;
@@ -544,7 +544,7 @@ resume_1:
         }
         if (has_delete)
         {
-            clean_disk_entry *new_entry = (clean_disk_entry*)((uint8_t*)meta_new.buf + meta_new.pos*bs->clean_entry_size);
+            clean_disk_entry *new_entry = (clean_disk_entry*)(meta_new.buf + meta_new.pos*bs->clean_entry_size);
             if (new_entry->oid.inode != 0 && new_entry->oid != cur.oid)
             {
                 printf("Fatal error (metadata corruption or bug): tried to delete metadata entry %lu (%lx:%lx v%lu) while deleting %lx:%lx\n",
@@ -553,11 +553,11 @@ resume_1:
                 exit(1);
             }
             // zero out new metadata entry
-            memset((uint8_t*)meta_new.buf + meta_new.pos*bs->clean_entry_size, 0, bs->clean_entry_size);
+            memset(meta_new.buf + meta_new.pos*bs->clean_entry_size, 0, bs->clean_entry_size);
         }
         else
         {
-            clean_disk_entry *new_entry = (clean_disk_entry*)((uint8_t*)meta_new.buf + meta_new.pos*bs->clean_entry_size);
+            clean_disk_entry *new_entry = (clean_disk_entry*)(meta_new.buf + meta_new.pos*bs->clean_entry_size);
             if (new_entry->oid.inode != 0 && new_entry->oid != cur.oid)
             {
                 printf("Fatal error (metadata corruption or bug): tried to overwrite non-zero metadata entry %lu (%lx:%lx v%lu) with %lx:%lx v%lu\n",
@@ -575,7 +575,7 @@ resume_1:
             if (bs->clean_entry_bitmap_size)
             {
                 void *bmp_ptr = bs->clean_entry_bitmap_size > sizeof(void*) ? dirty_end->second.bitmap : &dirty_end->second.bitmap;
-                memcpy((uint8_t*)(new_entry+1) + bs->clean_entry_bitmap_size, bmp_ptr, bs->clean_entry_bitmap_size);
+                memcpy((void*)(new_entry+1) + bs->clean_entry_bitmap_size, bmp_ptr, bs->clean_entry_bitmap_size);
             }
         }
         await_sqe(6);
@@ -762,7 +762,7 @@ bool journal_flusher_co::scan_dirty(int wait_base)
                         if (bs->journal.inmemory)
                         {
                             // Take it from memory
-                            memcpy(it->buf, (uint8_t*)bs->journal.buffer + submit_offset, submit_len);
+                            memcpy(it->buf, bs->journal.buffer + submit_offset, submit_len);
                         }
                         else
                         {
@@ -826,7 +826,7 @@ bool journal_flusher_co::modify_meta_read(uint64_t meta_loc, flusher_meta_write_
     wr.pos = ((meta_loc >> bs->block_order) % (bs->meta_block_size / bs->clean_entry_size));
     if (bs->inmemory_meta)
     {
-        wr.buf = (uint8_t*)bs->metadata_buffer + wr.sector;
+        wr.buf = bs->metadata_buffer + wr.sector;
         return true;
     }
     wr.it = flusher->meta_sectors.find(wr.sector);

src/blockstore_impl.cpp

@@ -372,7 +372,7 @@ void blockstore_impl_t::enqueue_op(blockstore_op_t *op)
                     };
                 }
                 unstable_writes.clear();
-                op->callback = [old_callback](blockstore_op_t *op)
+                op->callback = [this, old_callback](blockstore_op_t *op)
                 {
                     obj_ver_id *vers = (obj_ver_id*)op->buf;
                     delete[] vers;

src/blockstore_impl.h

@@ -280,7 +280,7 @@ class blockstore_impl_t
 
     friend class blockstore_init_meta;
     friend class blockstore_init_journal;
-    friend struct blockstore_journal_check_t;
+    friend class blockstore_journal_check_t;
     friend class journal_flusher_t;
     friend class journal_flusher_co;
 

src/blockstore_init.cpp

@@ -148,7 +148,7 @@ resume_1:
         {
             GET_SQE();
             data->iov = {
-                (uint8_t*)metadata_buffer + (bs->inmemory_meta
+                metadata_buffer + (bs->inmemory_meta
                     ? metadata_read
                     : (prev == 1 ? bs->metadata_buf_size : 0)),
                 bs->meta_len - metadata_read > bs->metadata_buf_size ? bs->metadata_buf_size : bs->meta_len - metadata_read,
@@ -169,13 +169,13 @@ resume_1:
         if (prev_done)
         {
             void *done_buf = bs->inmemory_meta
-                ? ((uint8_t*)metadata_buffer + done_pos)
+                ? (metadata_buffer + done_pos)
-                : ((uint8_t*)metadata_buffer + (prev_done == 2 ? bs->metadata_buf_size : 0));
+                : (metadata_buffer + (prev_done == 2 ? bs->metadata_buf_size : 0));
             unsigned count = bs->meta_block_size / bs->clean_entry_size;
             for (int sector = 0; sector < done_len; sector += bs->meta_block_size)
             {
                 // handle <count> entries
-                handle_entries((uint8_t*)done_buf + sector, count, bs->block_order);
+                handle_entries(done_buf + sector, count, bs->block_order);
                 done_cnt += count;
             }
             prev_done = 0;
@@ -215,7 +215,7 @@ void blockstore_init_meta::handle_entries(void* entries, unsigned count, int blo
 {
     for (unsigned i = 0; i < count; i++)
     {
-        clean_disk_entry *entry = (clean_disk_entry*)((uint8_t*)entries + i*bs->clean_entry_size);
+        clean_disk_entry *entry = (clean_disk_entry*)(entries + i*bs->clean_entry_size);
         if (!bs->inmemory_meta && bs->clean_entry_bitmap_size)
         {
             memcpy(bs->clean_bitmap + (done_cnt+i)*2*bs->clean_entry_bitmap_size, &entry->bitmap, 2*bs->clean_entry_bitmap_size);
@@ -440,7 +440,7 @@ resume_1:
                 if (!bs->journal.inmemory)
                     submitted_buf = memalign_or_die(MEM_ALIGNMENT, JOURNAL_BUFFER_SIZE);
                 else
-                    submitted_buf = (uint8_t*)bs->journal.buffer + journal_pos;
+                    submitted_buf = bs->journal.buffer + journal_pos;
                 data->iov = {
                     submitted_buf,
                     end - journal_pos < JOURNAL_BUFFER_SIZE ? end - journal_pos : JOURNAL_BUFFER_SIZE,
@@ -570,7 +570,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
     resume:
         while (pos < bs->journal.block_size)
         {
-            journal_entry *je = (journal_entry*)((uint8_t*)buf + proc_pos - done_pos + pos);
+            journal_entry *je = (journal_entry*)(buf + proc_pos - done_pos + pos);
             if (je->magic != JOURNAL_MAGIC || je_crc32(je) != je->crc32 ||
                 je->type < JE_MIN || je->type > JE_MAX || started && je->crc32_prev != crc32_last)
             {
@@ -619,7 +619,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                 if (location >= done_pos && location+je->small_write.len <= done_pos+len)
                 {
                     // data is within this buffer
-                    data_crc32 = crc32c(0, (uint8_t*)buf + location - done_pos, je->small_write.len);
+                    data_crc32 = crc32c(0, buf + location - done_pos, je->small_write.len);
                 }
                 else
                 {
@@ -634,7 +634,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                                 ? location+je->small_write.len : done[i].pos+done[i].len);
                             uint64_t part_begin = (location < done[i].pos ? done[i].pos : location);
                             covered += part_end - part_begin;
-                            data_crc32 = crc32c(data_crc32, (uint8_t*)done[i].buf + part_begin - done[i].pos, part_end - part_begin);
+                            data_crc32 = crc32c(data_crc32, done[i].buf + part_begin - done[i].pos, part_end - part_begin);
                         }
                     }
                     if (covered < je->small_write.len)
@@ -650,9 +650,9 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                     // interesting thing is that we must clear the corrupt entry if we're not readonly,
                     // because we don't write next entries in the same journal block
                     printf("Journal entry data is corrupt (data crc32 %x != %x)\n", data_crc32, je->small_write.crc32_data);
-                    memset((uint8_t*)buf + proc_pos - done_pos + pos, 0, bs->journal.block_size - pos);
+                    memset(buf + proc_pos - done_pos + pos, 0, bs->journal.block_size - pos);
                     bs->journal.next_free = prev_free;
-                    init_write_buf = (uint8_t*)buf + proc_pos - done_pos;
+                    init_write_buf = buf + proc_pos - done_pos;
                     init_write_sector = proc_pos;
                     return 0;
                 }
@@ -665,7 +665,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         .version = je->small_write.version,
                     };
                     void *bmp = NULL;
-                    void *bmp_from = (uint8_t*)je + sizeof(journal_entry_small_write);
+                    void *bmp_from = (void*)je + sizeof(journal_entry_small_write);
                     if (bs->clean_entry_bitmap_size <= sizeof(void*))
                     {
                         memcpy(&bmp, bmp_from, bs->clean_entry_bitmap_size);
@@ -745,7 +745,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
                         .version = je->big_write.version,
                     };
                     void *bmp = NULL;
-                    void *bmp_from = (uint8_t*)je + sizeof(journal_entry_big_write);
+                    void *bmp_from = (void*)je + sizeof(journal_entry_big_write);
                     if (bs->clean_entry_bitmap_size <= sizeof(void*))
                     {
                         memcpy(&bmp, bmp_from, bs->clean_entry_bitmap_size);

src/blockstore_init.h

@@ -6,7 +6,7 @@
 class blockstore_init_meta
 {
     blockstore_impl_t *bs;
-    int wait_state = 0;
+    int wait_state = 0, wait_count = 0;
     bool zero_on_init = false;
     void *metadata_buffer = NULL;
     uint64_t metadata_read = 0;

src/blockstore_journal.cpp

@@ -96,8 +96,7 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
         next_pos = next_pos + data_after;
         if (next_pos > bs->journal.len)
         {
-            if (right_dir)
+            next_pos = bs->journal.block_size + data_after;
-                next_pos = bs->journal.block_size + data_after;
             right_dir = false;
         }
     }
@@ -137,13 +136,13 @@ journal_entry* prefill_single_journal_entry(journal_t & journal, uint16_t type,
         journal.in_sector_pos = 0;
         journal.next_free = (journal.next_free+journal.block_size) < journal.len ? journal.next_free + journal.block_size : journal.block_size;
         memset(journal.inmemory
-            ? (uint8_t*)journal.buffer + journal.sector_info[journal.cur_sector].offset
+            ? journal.buffer + journal.sector_info[journal.cur_sector].offset
-            : (uint8_t*)journal.sector_buf + journal.block_size*journal.cur_sector, 0, journal.block_size);
+            : journal.sector_buf + journal.block_size*journal.cur_sector, 0, journal.block_size);
     }
     journal_entry *je = (struct journal_entry*)(
         (journal.inmemory
-            ? (uint8_t*)journal.buffer + journal.sector_info[journal.cur_sector].offset
+            ? journal.buffer + journal.sector_info[journal.cur_sector].offset
-            : (uint8_t*)journal.sector_buf + journal.block_size*journal.cur_sector) + journal.in_sector_pos
+            : journal.sector_buf + journal.block_size*journal.cur_sector) + journal.in_sector_pos
     );
     journal.in_sector_pos += size;
     je->magic = JOURNAL_MAGIC;
@@ -169,8 +168,8 @@ void blockstore_impl_t::prepare_journal_sector_write(int cur_sector, blockstore_
         journal.sector_info[cur_sector].flush_count++;
         data->iov = (struct iovec){
             (journal.inmemory
-                ? (uint8_t*)journal.buffer + journal.sector_info[cur_sector].offset
+                ? journal.buffer + journal.sector_info[cur_sector].offset
-                : (uint8_t*)journal.sector_buf + journal.block_size*cur_sector),
+                : journal.sector_buf + journal.block_size*cur_sector),
             journal.block_size
         };
         data->callback = [this, flush_id = journal.submit_id](ring_data_t *data) { handle_journal_write(data, flush_id); };

src/blockstore_read.cpp

@@ -24,7 +24,7 @@ int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_
     }
     if (journal.inmemory && IS_JOURNAL(item_state))
     {
-        memcpy(buf, (uint8_t*)journal.buffer + offset, len);
+        memcpy(buf, journal.buffer + offset, len);
         return 1;
     }
     BS_SUBMIT_GET_SQE(sqe, data);
@@ -75,7 +75,7 @@ int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op, uint64_t &fulfille
                 };
                 it = PRIV(read_op)->read_vec.insert(it, el);
                 if (!fulfill_read_push(read_op,
-                    (uint8_t*)read_op->buf + el.offset - read_op->offset,
+                    read_op->buf + el.offset - read_op->offset,
                     item_location + el.offset - item_start,
                     el.len, item_state, item_version))
                 {
@@ -102,7 +102,7 @@ uint8_t* blockstore_impl_t::get_clean_entry_bitmap(uint64_t block_loc, int offse
     {
         uint64_t sector = (meta_loc / (meta_block_size / clean_entry_size)) * meta_block_size;
         uint64_t pos = (meta_loc % (meta_block_size / clean_entry_size));
-        clean_entry_bitmap = ((uint8_t*)metadata_buffer + sector + pos*clean_entry_size + sizeof(clean_disk_entry) + offset);
+        clean_entry_bitmap = (uint8_t*)(metadata_buffer + sector + pos*clean_entry_size + sizeof(clean_disk_entry) + offset);
     }
     else
         clean_entry_bitmap = (uint8_t*)(clean_bitmap + meta_loc*2*clean_entry_bitmap_size + offset);

src/blockstore_write.cpp

@@ -102,7 +102,7 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
         // Issue an additional sync so that the previous big write can reach the journal
         blockstore_op_t *sync_op = new blockstore_op_t;
         sync_op->opcode = BS_OP_SYNC;
-        sync_op->callback = [](blockstore_op_t *sync_op)
+        sync_op->callback = [this, op](blockstore_op_t *sync_op)
         {
             delete sync_op;
         };
@@ -380,7 +380,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
             if (journal.inmemory)
             {
                 // Copy data
-                memcpy((uint8_t*)journal.buffer + journal.next_free, op->buf, op->len);
+                memcpy(journal.buffer + journal.next_free, op->buf, op->len);
             }
             BS_SUBMIT_GET_SQE(sqe2, data2);
             data2->iov = (struct iovec){ op->buf, op->len };
@@ -629,17 +629,13 @@ int blockstore_impl_t::dequeue_del(blockstore_op_t *op)
     {
         return 0;
     }
+    write_iodepth++;
     // Write current journal sector only if it's dirty and full, or in the immediate_commit mode
     BS_SUBMIT_CHECK_SQES(
         (immediate_commit != IMMEDIATE_NONE ||
             (journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_del) &&
             journal.sector_info[journal.cur_sector].dirty) ? 1 : 0
     );
-    if (write_iodepth >= max_write_iodepth)
-    {
-        return 0;
-    }
-    write_iodepth++;
     // Prepare journal sector write
     if (immediate_commit == IMMEDIATE_NONE)
     {

src/cli.cpp

@@ -177,7 +177,7 @@ void cli_tool_t::change_parent(inode_t cur, inode_t new_parent)
     new_cfg.parent_id = new_parent;
     json11::Json::object cur_cfg_json = cli->st_cli.serialize_inode_cfg(&new_cfg);
     waiting++;
-    cli->st_cli.etcd_txn_slow(json11::Json::object {
+    cli->st_cli.etcd_txn(json11::Json::object {
         { "compare", json11::Json::array {
             json11::Json::object {
                 { "target", "MOD" },
@@ -194,7 +194,7 @@ void cli_tool_t::change_parent(inode_t cur, inode_t new_parent)
                 } }
             },
         } },
-    }, [this, new_parent, cur, cur_name](std::string err, json11::Json res)
+    }, ETCD_SLOW_TIMEOUT, [this, new_parent, cur, cur_name](std::string err, json11::Json res)
     {
         if (err != "")
         {
@@ -229,22 +229,6 @@ void cli_tool_t::change_parent(inode_t cur, inode_t new_parent)
     });
 }
 
-void cli_tool_t::etcd_txn(json11::Json txn)
-{
-    waiting++;
-    cli->st_cli.etcd_txn_slow(txn, [this](std::string err, json11::Json res)
-    {
-        waiting--;
-        if (err != "")
-        {
-            fprintf(stderr, "Error reading from etcd: %s\n", err.c_str());
-            exit(1);
-        }
-        etcd_result = res;
-        ringloop->wakeup();
-    });
-}
-
 inode_config_t* cli_tool_t::get_inode_cfg(const std::string & name)
 {
     for (auto & ic: cli->st_cli.inode_config)

src/cli.h

@@ -34,7 +34,6 @@ public:
     cluster_client_t *cli = NULL;
 
     int waiting = 0;
-    json11::Json etcd_result;
     ring_consumer_t consumer;
     std::function<bool(void)> action_cb;
 
@@ -61,8 +60,6 @@ public:
     std::function<bool(void)> start_snap_rm(json11::Json);
     std::function<bool(void)> start_alloc_osd(json11::Json cfg, uint64_t *out = NULL);
     std::function<bool(void)> simple_offsets(json11::Json cfg);
-
-    void etcd_txn(json11::Json txn);
 };
 
 uint64_t parse_size(std::string size_str);

src/cli_alloc_osd.cpp

@@ -13,6 +13,7 @@ struct alloc_osd_t
 {
     cli_tool_t *parent;
 
+    json11::Json result;
     uint64_t new_id = 1;
 
     int state = 0;
@@ -28,7 +29,7 @@ struct alloc_osd_t
             goto resume_1;
         do
         {
-            parent->etcd_txn(json11::Json::object {
+            etcd_txn(json11::Json::object {
                 { "compare", json11::Json::array {
                     json11::Json::object {
                         { "target", "VERSION" },
@@ -62,10 +63,10 @@ struct alloc_osd_t
             state = 1;
             if (parent->waiting > 0)
                 return;
-            if (!parent->etcd_result["succeeded"].bool_value())
+            if (!result["succeeded"].bool_value())
             {
                 std::vector<osd_num_t> used;
-                for (auto kv: parent->etcd_result["responses"][0]["response_range"]["kvs"].array_items())
+                for (auto kv: result["responses"][0]["response_range"]["kvs"].array_items())
                 {
                     std::string key = base64_decode(kv["key"].string_value());
                     osd_num_t cur_osd;
@@ -97,9 +98,25 @@ struct alloc_osd_t
                     new_id = used[e-1]+1;
                 }
             }
-        } while (!parent->etcd_result["succeeded"].bool_value());
+        } while (!result["succeeded"].bool_value());
         state = 100;
     }
+
+    void etcd_txn(json11::Json txn)
+    {
+        parent->waiting++;
+        parent->cli->st_cli.etcd_txn(txn, ETCD_SLOW_TIMEOUT, [this](std::string err, json11::Json res)
+        {
+            parent->waiting--;
+            if (err != "")
+            {
+                fprintf(stderr, "Error reading from etcd: %s\n", err.c_str());
+                exit(1);
+            }
+            this->result = res;
+            parent->ringloop->wakeup();
+        });
+    }
 };
 
 std::function<bool(void)> cli_tool_t::start_alloc_osd(json11::Json cfg, uint64_t *out)

src/cli_create.cpp

@@ -31,6 +31,7 @@ struct image_creator_t
     inode_t new_parent_id = 0;
     inode_t new_id = 0, old_id = 0;
     uint64_t max_id_mod_rev = 0, cfg_mod_rev = 0, idx_mod_rev = 0;
+    json11::Json result;
 
     int state = 0;
 
@@ -124,26 +125,26 @@ struct image_creator_t
         }
         do
         {
-            parent->etcd_txn(json11::Json::object {
+            etcd_txn(json11::Json::object {
                 { "success", json11::Json::array { get_next_id() } }
             });
             state = 2;
 resume_2:
             if (parent->waiting > 0)
                 return;
-            extract_next_id(parent->etcd_result["responses"][0]);
+            extract_next_id(result["responses"][0]);
             attempt_create();
             state = 3;
 resume_3:
             if (parent->waiting > 0)
                 return;
-            if (!parent->etcd_result["succeeded"].bool_value() &&
+            if (!result["succeeded"].bool_value() &&
-                parent->etcd_result["responses"][0]["response_range"]["kvs"].array_items().size() > 0)
+                result["responses"][0]["response_range"]["kvs"].array_items().size() > 0)
             {
                 fprintf(stderr, "Image %s already exists\n", image_name.c_str());
                 exit(1);
             }
-        } while (!parent->etcd_result["succeeded"].bool_value());
+        } while (!result["succeeded"].bool_value());
         if (parent->progress)
         {
             printf("Image %s created\n", image_name.c_str());
@@ -195,13 +196,13 @@ resume_3:
 resume_4:
             if (parent->waiting > 0)
                 return;
-            if (!parent->etcd_result["succeeded"].bool_value() &&
+            if (!result["succeeded"].bool_value() &&
-                parent->etcd_result["responses"][0]["response_range"]["kvs"].array_items().size() > 0)
+                result["responses"][0]["response_range"]["kvs"].array_items().size() > 0)
             {
                 fprintf(stderr, "Snapshot %s@%s already exists\n", image_name.c_str(), new_snap.c_str());
                 exit(1);
             }
-        } while (!parent->etcd_result["succeeded"].bool_value());
+        } while (!result["succeeded"].bool_value());
         if (parent->progress)
         {
             printf("Snapshot %s@%s created\n", image_name.c_str(), new_snap.c_str());
@@ -245,7 +246,7 @@ resume_4:
             goto resume_2;
         else if (state == 3)
             goto resume_3;
-        parent->etcd_txn(json11::Json::object { { "success", json11::Json::array {
+        etcd_txn(json11::Json::object { { "success", json11::Json::array {
             get_next_id(),
             json11::Json::object {
                 { "request_range", json11::Json::object {
@@ -259,11 +260,11 @@ resume_4:
 resume_2:
         if (parent->waiting > 0)
             return;
-        extract_next_id(parent->etcd_result["responses"][0]);
+        extract_next_id(result["responses"][0]);
         old_id = 0;
         old_pool_id = 0;
         cfg_mod_rev = idx_mod_rev = 0;
-        if (parent->etcd_result["responses"][1]["response_range"]["kvs"].array_items().size() == 0)
+        if (result["responses"][1]["response_range"]["kvs"].array_items().size() == 0)
         {
             for (auto & ic: parent->cli->st_cli.inode_config)
             {
@@ -282,7 +283,7 @@ resume_2:
         {
             // FIXME: Parse kvs in etcd_state_client automatically
             {
-                auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][1]["response_range"]["kvs"][0]);
+                auto kv = parent->cli->st_cli.parse_etcd_kv(result["responses"][1]["response_range"]["kvs"][0]);
                 old_id = INODE_NO_POOL(kv.value["id"].uint64_value());
                 old_pool_id = (pool_id_t)kv.value["pool_id"].uint64_value();
                 idx_mod_rev = kv.mod_revision;
@@ -292,7 +293,7 @@ resume_2:
                     exit(1);
                 }
             }
-            parent->etcd_txn(json11::Json::object {
+            etcd_txn(json11::Json::object {
                 { "success", json11::Json::array {
                     json11::Json::object {
                         { "request_range", json11::Json::object {
@@ -309,7 +310,7 @@ resume_3:
             if (parent->waiting > 0)
                 return;
             {
-                auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
+                auto kv = parent->cli->st_cli.parse_etcd_kv(result["responses"][0]["response_range"]["kvs"][0]);
                 size = kv.value["size"].uint64_value();
                 new_parent_id = kv.value["parent_id"].uint64_value();
                 uint64_t parent_pool_id = kv.value["parent_pool_id"].uint64_value();
@@ -438,12 +439,28 @@ resume_3:
                 } },
             });
         };
-        parent->etcd_txn(json11::Json::object {
+        etcd_txn(json11::Json::object {
             { "compare", checks },
             { "success", success },
             { "failure", failure },
         });
     }
+
+    void etcd_txn(json11::Json txn)
+    {
+        parent->waiting++;
+        parent->cli->st_cli.etcd_txn(txn, ETCD_SLOW_TIMEOUT, [this](std::string err, json11::Json res)
+        {
+            parent->waiting--;
+            if (err != "")
+            {
+                fprintf(stderr, "Error reading from etcd: %s\n", err.c_str());
+                exit(1);
+            }
+            this->result = res;
+            parent->ringloop->wakeup();
+        });
+    }
 };
 
 uint64_t parse_size(std::string size_str)

src/cli_df.cpp

@@ -24,7 +24,8 @@ struct pool_lister_t
         if (state == 1)
             goto resume_1;
         // Space statistics - pool/stats/<pool>
-        parent->etcd_txn(json11::Json::object {
+        parent->waiting++;
+        parent->cli->st_cli.etcd_txn(json11::Json::object {
             { "success", json11::Json::array {
                 json11::Json::object {
                     { "request_range", json11::Json::object {
@@ -47,12 +48,21 @@ struct pool_lister_t
                     } },
                 },
             } },
+        }, ETCD_SLOW_TIMEOUT, [this](std::string err, json11::Json res)
+        {
+            parent->waiting--;
+            if (err != "")
+            {
+                fprintf(stderr, "Error reading from etcd: %s\n", err.c_str());
+                exit(1);
+            }
+            space_info = res;
+            parent->ringloop->wakeup();
         });
         state = 1;
 resume_1:
         if (parent->waiting > 0)
             return;
-        space_info = parent->etcd_result;
         std::map<pool_id_t, uint64_t> osd_free;
         for (auto & kv_item: space_info["responses"][0]["response_range"]["kvs"].array_items())
         {

src/cli_flatten.cpp

@@ -3,7 +3,6 @@
 
 #include "cli.h"
 #include "cluster_client.h"
-#include <sys/stat.h>
 
 // Flatten a layer: merge all parents into a layer and break the connection completely
 struct snap_flattener_t

src/cli_ls.cpp

@@ -84,7 +84,8 @@ struct image_lister_t
         // Space statistics
         // inode/stats/<pool>/<inode>::raw_used divided by pool/stats/<pool>::pg_real_size
         // multiplied by 1 or number of data drives
-        parent->etcd_txn(json11::Json::object {
+        parent->waiting++;
+        parent->cli->st_cli.etcd_txn(json11::Json::object {
             { "success", json11::Json::array {
                 json11::Json::object {
                     { "request_range", json11::Json::object {
@@ -111,12 +112,21 @@ struct image_lister_t
                     } },
                 },
             } },
+        }, ETCD_SLOW_TIMEOUT, [this](std::string err, json11::Json res)
+        {
+            parent->waiting--;
+            if (err != "")
+            {
+                fprintf(stderr, "Error reading from etcd: %s\n", err.c_str());
+                exit(1);
+            }
+            space_info = res;
+            parent->ringloop->wakeup();
         });
         state = 1;
 resume_1:
         if (parent->waiting > 0)
             return;
-        space_info = parent->etcd_result;
         std::map<pool_id_t, uint64_t> pool_pg_real_size;
         for (auto & kv_item: space_info["responses"][0]["response_range"]["kvs"].array_items())
         {

src/cli_merge.cpp

@@ -412,7 +412,7 @@ struct snap_merger_t
         uint64_t bitmap_size = target_block_size / gran;
         while (rwo->end < bitmap_size)
         {
-            auto bit = ((*((uint8_t*)rwo->op.bitmap_buf + (rwo->end >> 3))) & (1 << (rwo->end & 0x7)));
+            auto bit = ((*(uint8_t*)(rwo->op.bitmap_buf + (rwo->end >> 3))) & (1 << (rwo->end & 0x7)));
             if (!bit)
             {
                 if (rwo->end > rwo->start)
@@ -459,7 +459,7 @@ struct snap_merger_t
         subop->len = end-start;
         subop->version = version;
         subop->flags = OSD_OP_IGNORE_READONLY;
-        subop->iov.push_back((uint8_t*)rwo->buf+start, end-start);
+        subop->iov.push_back(rwo->buf+start, end-start);
         subop->callback = [this, rwo](cluster_op_t *subop)
         {
             rwo->todo--;
@@ -495,7 +495,7 @@ struct snap_merger_t
         subop->offset = offset;
         subop->len = 0;
         subop->flags = OSD_OP_IGNORE_READONLY;
-        subop->callback = [](cluster_op_t *subop)
+        subop->callback = [this](cluster_op_t *subop)
         {
             if (subop->retval != 0)
             {
@@ -519,10 +519,10 @@ struct snap_merger_t
                 deleted_unsynced++;
                 if (deleted_unsynced >= fsync_interval)
                 {
-                    uint64_t to = last_written_offset;
+                    uint64_t from = last_fsync_offset, to = last_written_offset;
                     cluster_op_t *subop = new cluster_op_t;
                     subop->opcode = OSD_OP_SYNC;
-                    subop->callback = [this, to](cluster_op_t *subop)
+                    subop->callback = [this, from, to](cluster_op_t *subop)
                     {
                         delete subop;
                         // We can now delete source data between <from> and <to>

src/cli_modify.cpp

@@ -170,19 +170,29 @@ resume_1:
                 } }
             });
         }
-        parent->etcd_txn(json11::Json::object {
+        parent->waiting++;
+        parent->cli->st_cli.etcd_txn(json11::Json::object {
             { "compare", checks },
             { "success", success },
+        }, ETCD_SLOW_TIMEOUT, [this](std::string err, json11::Json res)
+        {
+            if (err != "")
+            {
+                fprintf(stderr, "Error changing %s: %s\n", image_name.c_str(), err.c_str());
+                exit(1);
+            }
+            if (!res["succeeded"].bool_value())
+            {
+                fprintf(stderr, "Image %s was modified by someone else, please repeat your request\n", image_name.c_str());
+                exit(1);
+            }
+            parent->waiting--;
+            parent->ringloop->wakeup();
         });
         state = 2;
 resume_2:
         if (parent->waiting > 0)
             return;
-        if (!parent->etcd_result["succeeded"].bool_value())
-        {
-            fprintf(stderr, "Image %s was modified by someone else, please repeat your request\n", image_name.c_str());
-            exit(1);
-        }
         printf("Image %s modified\n", image_name.c_str());
         state = 100;
     }

src/cli_simple_offsets.cpp

@@ -8,7 +8,6 @@
 #include "cli.h"
 #include "cluster_client.h"
 #include "base64.h"
-#include <sys/stat.h>
 
 // Calculate offsets for a block device and print OSD command line parameters
 std::function<bool(void)> cli_tool_t::simple_offsets(json11::Json cfg)

src/cli_snap_rm.cpp

@@ -256,9 +256,9 @@ resume_9:
             });
         }
         parent->waiting++;
-        parent->cli->st_cli.etcd_txn_slow(json11::Json::object {
+        parent->cli->st_cli.etcd_txn(json11::Json::object {
             { "success", reads },
-        }, [this](std::string err, json11::Json data)
+        }, ETCD_SLOW_TIMEOUT, [this](std::string err, json11::Json data)
         {
             parent->waiting--;
             if (err != "")
@@ -414,10 +414,10 @@ resume_9:
             }
         }
         parent->waiting++;
-        parent->cli->st_cli.etcd_txn_slow(json11::Json::object {
+        parent->cli->st_cli.etcd_txn(json11::Json::object {
             { "compare", cmp },
             { "success", txn },
-        }, [this, target_name, child_name](std::string err, json11::Json res)
+        }, ETCD_SLOW_TIMEOUT, [this, target_name, child_name](std::string err, json11::Json res)
         {
             parent->waiting--;
             if (err != "")
@@ -454,7 +454,7 @@ resume_9:
             "/"+std::to_string(INODE_NO_POOL(cur))
         );
         parent->waiting++;
-        parent->cli->st_cli.etcd_txn_slow(json11::Json::object {
+        parent->cli->st_cli.etcd_txn(json11::Json::object {
             { "compare", json11::Json::array {
                 json11::Json::object {
                     { "target", "MOD" },
@@ -475,7 +475,7 @@ resume_9:
                     } },
                 },
             } },
-        }, [this, cur_name](std::string err, json11::Json res)
+        }, ETCD_SLOW_TIMEOUT, [this, cur_name](std::string err, json11::Json res)
         {
             parent->waiting--;
             if (err != "")

src/cluster_client.cpp

@@ -534,8 +534,8 @@ void cluster_client_t::copy_write(cluster_op_t *op, std::map<object_id, cluster_
             unsigned iov_len = (op->iov.buf[iov_idx].iov_len - iov_pos);
             if (iov_len <= cur_len)
             {
-                memcpy((uint8_t*)dirty_it->second.buf + pos - dirty_it->first.stripe,
+                memcpy(dirty_it->second.buf + pos - dirty_it->first.stripe,
-                    (uint8_t*)op->iov.buf[iov_idx].iov_base + iov_pos, iov_len);
+                    op->iov.buf[iov_idx].iov_base + iov_pos, iov_len);
                 pos += iov_len;
                 len -= iov_len;
                 cur_len -= iov_len;
@@ -544,8 +544,8 @@ void cluster_client_t::copy_write(cluster_op_t *op, std::map<object_id, cluster_
             }
             else
             {
-                memcpy((uint8_t*)dirty_it->second.buf + pos - dirty_it->first.stripe,
+                memcpy(dirty_it->second.buf + pos - dirty_it->first.stripe,
-                    (uint8_t*)op->iov.buf[iov_idx].iov_base + iov_pos, cur_len);
+                    op->iov.buf[iov_idx].iov_base + iov_pos, cur_len);
                 pos += cur_len;
                 len -= cur_len;
                 iov_pos += cur_len;
@@ -762,7 +762,7 @@ static void add_iov(int size, bool skip, cluster_op_t *op, int &iov_idx, size_t
         {
             if (!skip)
             {
-                iov.push_back((uint8_t*)op->iov.buf[iov_idx].iov_base + iov_pos, cur_left);
+                iov.push_back(op->iov.buf[iov_idx].iov_base + iov_pos, cur_left);
             }
             left -= cur_left;
             iov_pos = 0;
@@ -772,7 +772,7 @@ static void add_iov(int size, bool skip, cluster_op_t *op, int &iov_idx, size_t
         {
             if (!skip)
             {
-                iov.push_back((uint8_t*)op->iov.buf[iov_idx].iov_base + iov_pos, left);
+                iov.push_back(op->iov.buf[iov_idx].iov_base + iov_pos, left);
             }
             iov_pos += left;
             left = 0;
@@ -817,7 +817,7 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
                 // First allocation
                 memset(op->bitmap_buf, 0, object_bitmap_size);
             }
-            op->part_bitmaps = (uint8_t*)op->bitmap_buf + object_bitmap_size;
+            op->part_bitmaps = op->bitmap_buf + object_bitmap_size;
             op->bitmap_buf_size = bitmap_mem;
         }
     }
@@ -839,7 +839,7 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
             while (cur < end)
             {
                 unsigned bmp_loc = (cur - op->offset)/bs_bitmap_granularity;
-                bool skip = (((*((uint8_t*)op->bitmap_buf + bmp_loc/8)) >> (bmp_loc%8)) & 0x1);
+                bool skip = (((*(uint8_t*)(op->bitmap_buf + bmp_loc/8)) >> (bmp_loc%8)) & 0x1);
                 if (skip_prev != skip)
                 {
                     if (cur > prev)
@@ -944,7 +944,7 @@ bool cluster_client_t::try_send(cluster_op_t *op, int i)
                     .meta_revision = meta_rev,
                     .version = op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE ? op->version : 0,
                 } },
-                .bitmap = (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP ? (uint8_t*)op->part_bitmaps + pg_bitmap_size*i : NULL),
+                .bitmap = (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP ? op->part_bitmaps + pg_bitmap_size*i : NULL),
                 .bitmap_len = (unsigned)(op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP ? pg_bitmap_size : 0),
                 .callback = [this, part](osd_op_t *op_part)
                 {
@@ -1155,7 +1155,7 @@ void cluster_client_t::copy_part_bitmap(cluster_op_t *op, cluster_op_part_t *par
     if (!(object_offset & 0x7) && !(part_offset & 0x7) && (part_len >= 8))
     {
         // Copy bytes
-        mem_or((uint8_t*)op->bitmap_buf + object_offset/8, (uint8_t*)part->op.bitmap + part_offset/8, part_len/8);
+        mem_or(op->bitmap_buf + object_offset/8, part->op.bitmap + part_offset/8, part_len/8);
         object_offset += (part_len & ~0x7);
         part_offset += (part_len & ~0x7);
         part_len = (part_len & 0x7);
@@ -1163,8 +1163,8 @@ void cluster_client_t::copy_part_bitmap(cluster_op_t *op, cluster_op_part_t *par
     while (part_len > 0)
     {
         // Copy bits
-        (*((uint8_t*)op->bitmap_buf + (object_offset >> 3))) |= (
+        (*(uint8_t*)(op->bitmap_buf + (object_offset >> 3))) |= (
-            (((*((uint8_t*)part->op.bitmap + (part_offset >> 3))) >> (part_offset & 0x7)) & 0x1) << (object_offset & 0x7)
+            (((*(uint8_t*)(part->op.bitmap + (part_offset >> 3))) >> (part_offset & 0x7)) & 0x1) << (object_offset & 0x7)
         );
         part_offset++;
         object_offset++;

src/dump_journal.cpp

@@ -75,7 +75,7 @@ int main(int argc, char *argv[])
             uint64_t s;
             for (s = 0; s < self.journal_block; s += 8)
             {
-                if (*((uint64_t*)((uint8_t*)data+s)) != 0)
+                if (*((uint64_t*)(data+s)) != 0)
                     break;
             }
             if (s == self.journal_block)
@@ -139,7 +139,7 @@ int journal_dump_t::dump_block(void *buf)
     bool wrapped = false;
     while (pos < journal_block)
     {
-        journal_entry *je = (journal_entry*)((uint8_t*)buf + pos);
+        journal_entry *je = (journal_entry*)(buf + pos);
         if (je->magic != JOURNAL_MAGIC || je->type < JE_MIN || je->type > JE_MAX ||
             !all && started && je->crc32_prev != crc32_last)
         {

src/etcd_state_client.cpp

@@ -5,7 +5,6 @@
 #include "pg_states.h"
 #include "etcd_state_client.h"
 #ifndef __MOCK__
-#include "addr_util.h"
 #include "http_client.h"
 #include "base64.h"
 #endif
@@ -26,14 +25,9 @@ etcd_state_client_t::~etcd_state_client_t()
 #ifndef __MOCK__
     if (etcd_watch_ws)
     {
-        http_close(etcd_watch_ws);
+        etcd_watch_ws->close();
         etcd_watch_ws = NULL;
     }
-    if (keepalive_client)
-    {
-        http_close(keepalive_client);
-        keepalive_client = NULL;
-    }
 #endif
 }
 
@@ -54,18 +48,12 @@ etcd_kv_t etcd_state_client_t::parse_etcd_kv(const json11::Json & kv_json)
     return kv;
 }
 
-void etcd_state_client_t::etcd_txn(json11::Json txn, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback)
+void etcd_state_client_t::etcd_txn(json11::Json txn, int timeout, std::function<void(std::string, json11::Json)> callback)
 {
-    etcd_call("/kv/txn", txn, timeout, retries, interval, callback);
+    etcd_call("/kv/txn", txn, timeout, callback);
 }
 
-void etcd_state_client_t::etcd_txn_slow(json11::Json txn, std::function<void(std::string, json11::Json)> callback)
+void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int timeout, std::function<void(std::string, json11::Json)> callback)
-{
-    etcd_call("/kv/txn", txn, etcd_slow_timeout, max_etcd_attempts, 0, callback);
-}
-
-void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int timeout,
-    int retries, int interval, std::function<void(std::string, json11::Json)> callback)
 {
     if (!etcd_addresses.size() && !etcd_local.size())
     {
@@ -86,49 +74,14 @@ void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int t
         "Host: "+etcd_address+"\r\n"
         "Content-Type: application/json\r\n"
         "Content-Length: "+std::to_string(req.size())+"\r\n"
-        "Connection: keep-alive\r\n"
+        "Connection: close\r\n"
-        "Keep-Alive: timeout="+std::to_string(etcd_keepalive_timeout)+"\r\n"
         "\r\n"+req;
-    auto cb = [this, api, payload, timeout, retries, interval, callback,
+    http_request_json(tfd, etcd_address, req, timeout, [this, cur_addr = selected_etcd_address, callback](std::string err, json11::Json data)
-        cur_addr = selected_etcd_address](const http_response_t *response)
     {
-        std::string err;
+        if (err != "" && cur_addr == selected_etcd_address)
-        json11::Json data;
+            selected_etcd_address = "";
-        response->parse_json_response(err, data);
+        callback(err, data);
-        if (err != "")
+    });
-        {
-            if (cur_addr == selected_etcd_address)
-                selected_etcd_address = "";
-            if (retries > 0)
-            {
-                if (this->log_level > 0)
-                {
-                    printf(
-                        "Warning: etcd request failed: %s, retrying %d more times\n",
-                        err.c_str(), retries
-                    );
-                }
-                if (interval > 0)
-                {
-                    tfd->set_timer(interval, false, [this, api, payload, timeout, retries, interval, callback](int)
-                    {
-                        etcd_call(api, payload, timeout, retries-1, interval, callback);
-                    });
-                }
-                else
-                    etcd_call(api, payload, timeout, retries-1, interval, callback);
-            }
-            else
-                callback(err, data);
-        }
-        else
-            callback(err, data);
-    };
-    if (!keepalive_client)
-    {
-        keepalive_client = http_init(tfd);
-    }
-    http_request(keepalive_client, etcd_address, req, { .timeout = timeout, .keepalive = true }, cb);
 }
 
 void etcd_state_client_t::add_etcd_url(std::string addr)
@@ -202,33 +155,6 @@ void etcd_state_client_t::parse_config(const json11::Json & config)
         this->etcd_prefix = "/"+this->etcd_prefix;
     }
     this->log_level = config["log_level"].int64_value();
-    this->etcd_keepalive_timeout = config["etcd_keepalive_timeout"].uint64_value();
-    if (this->etcd_keepalive_timeout <= 0)
-    {
-        this->etcd_keepalive_timeout = config["etcd_report_interval"].uint64_value() * 2;
-        if (this->etcd_keepalive_timeout < 30)
-            this->etcd_keepalive_timeout = 30;
-    }
-    this->etcd_ws_keepalive_interval = config["etcd_ws_keepalive_interval"].uint64_value();
-    if (this->etcd_ws_keepalive_interval <= 0)
-    {
-        this->etcd_ws_keepalive_interval = 30;
-    }
-    this->max_etcd_attempts = config["max_etcd_attempts"].uint64_value();
-    if (this->max_etcd_attempts <= 0)
-    {
-        this->max_etcd_attempts = 5;
-    }
-    this->etcd_slow_timeout = config["etcd_slow_timeout"].uint64_value();
-    if (this->etcd_slow_timeout <= 0)
-    {
-        this->etcd_slow_timeout = 5000;
-    }
-    this->etcd_quick_timeout = config["etcd_quick_timeout"].uint64_value();
-    if (this->etcd_quick_timeout <= 0)
-    {
-        this->etcd_quick_timeout = 1000;
-    }
 }
 
 void etcd_state_client_t::pick_next_etcd()
@@ -243,16 +169,9 @@ void etcd_state_client_t::pick_next_etcd()
         std::vector<int> ns;
         for (int i = 0; i < etcd_addresses.size(); i++)
             ns.push_back(i);
-        if (!rand_initialized)
-        {
-            timespec tv;
-            clock_gettime(CLOCK_REALTIME, &tv);
-            srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
-            rand_initialized = true;
-        }
         while (ns.size())
         {
-            int i = lrand48() % ns.size();
+            int i = rand() % ns.size();
             addresses_to_try.push_back(etcd_addresses[ns[i]]);
             ns.erase(ns.begin()+i, ns.begin()+i+1);
         }
@@ -281,12 +200,10 @@ void etcd_state_client_t::start_etcd_watcher()
     ws_alive = 1;
     if (etcd_watch_ws)
     {
-        http_close(etcd_watch_ws);
+        etcd_watch_ws->close();
         etcd_watch_ws = NULL;
     }
-    if (this->log_level > 1)
+    etcd_watch_ws = open_websocket(tfd, etcd_address, etcd_api_path+"/watch", ETCD_SLOW_TIMEOUT,
-        printf("Trying to connect to etcd websocket at %s\n", etcd_address.c_str());
-    etcd_watch_ws = open_websocket(tfd, etcd_address, etcd_api_path+"/watch", etcd_slow_timeout,
         [this, cur_addr = selected_etcd_address](const http_response_t *msg)
     {
         if (msg->body.length())
@@ -302,8 +219,6 @@ void etcd_state_client_t::start_etcd_watcher()
             {
                 if (data["result"]["created"].bool_value())
                 {
-                    if (etcd_watches_initialised == 3 && this->log_level > 0)
-                        fprintf(stderr, "Successfully subscribed to etcd at %s\n", selected_etcd_address.c_str());
                     etcd_watches_initialised++;
                 }
                 if (data["result"]["canceled"].bool_value())
@@ -317,11 +232,8 @@ void etcd_state_client_t::start_etcd_watcher()
                         {
                             fprintf(stderr, "Revisions before %lu were compacted by etcd, reloading state\n",
                                 data["result"]["compact_revision"].uint64_value());
-                            if (etcd_watch_ws)
+                            etcd_watch_ws->close();
-                            {
+                            etcd_watch_ws = NULL;
-                                http_close(etcd_watch_ws);
-                                etcd_watch_ws = NULL;
-                            }
                             etcd_watch_revision = 0;
                             on_reload_hook();
                         }
@@ -372,20 +284,13 @@ void etcd_state_client_t::start_etcd_watcher()
         {
             if (cur_addr == selected_etcd_address)
             {
-                fprintf(stderr, "Disconnected from etcd %s\n", selected_etcd_address.c_str());
                 selected_etcd_address = "";
             }
-            else
+            etcd_watch_ws = NULL;
-                fprintf(stderr, "Disconnected from etcd\n");
-            if (etcd_watch_ws)
-            {
-                http_close(etcd_watch_ws);
-                etcd_watch_ws = NULL;
-            }
             if (etcd_watches_initialised == 0)
             {
-                // Connection not established, retry in <etcd_quick_timeout>
+                // Connection not established, retry in <ETCD_QUICK_TIMEOUT>
-                tfd->set_timer(etcd_quick_timeout, false, [this](int)
+                tfd->set_timer(ETCD_QUICK_TIMEOUT, false, [this](int)
                 {
                     start_etcd_watcher();
                 });
@@ -397,7 +302,7 @@ void etcd_state_client_t::start_etcd_watcher()
             }
         }
     });
-    http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
+    etcd_watch_ws->post_message(WS_TEXT, json11::Json(json11::Json::object {
         { "create_request", json11::Json::object {
             { "key", base64_encode(etcd_prefix+"/config/") },
             { "range_end", base64_encode(etcd_prefix+"/config0") },
@@ -406,7 +311,7 @@ void etcd_state_client_t::start_etcd_watcher()
             { "progress_notify", true },
         } }
     }).dump());
-    http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
+    etcd_watch_ws->post_message(WS_TEXT, json11::Json(json11::Json::object {
         { "create_request", json11::Json::object {
             { "key", base64_encode(etcd_prefix+"/osd/state/") },
             { "range_end", base64_encode(etcd_prefix+"/osd/state0") },
@@ -415,7 +320,7 @@ void etcd_state_client_t::start_etcd_watcher()
             { "progress_notify", true },
         } }
     }).dump());
-    http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
+    etcd_watch_ws->post_message(WS_TEXT, json11::Json(json11::Json::object {
         { "create_request", json11::Json::object {
             { "key", base64_encode(etcd_prefix+"/pg/state/") },
             { "range_end", base64_encode(etcd_prefix+"/pg/state0") },
@@ -424,7 +329,7 @@ void etcd_state_client_t::start_etcd_watcher()
             { "progress_notify", true },
         } }
     }).dump());
-    http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
+    etcd_watch_ws->post_message(WS_TEXT, json11::Json(json11::Json::object {
         { "create_request", json11::Json::object {
             { "key", base64_encode(etcd_prefix+"/pg/history/") },
             { "range_end", base64_encode(etcd_prefix+"/pg/history0") },
@@ -435,7 +340,7 @@ void etcd_state_client_t::start_etcd_watcher()
     }).dump());
     if (ws_keepalive_timer < 0)
     {
-        ws_keepalive_timer = tfd->set_timer(etcd_ws_keepalive_interval*1000, true, [this](int)
+        ws_keepalive_timer = tfd->set_timer(ETCD_KEEPALIVE_TIMEOUT, true, [this](int)
         {
             if (!etcd_watch_ws)
             {
@@ -443,21 +348,14 @@ void etcd_state_client_t::start_etcd_watcher()
             }
             else if (!ws_alive)
             {
-                if (this->log_level > 0)
+                etcd_watch_ws->close();
-                {
+                etcd_watch_ws = NULL;
-                    fprintf(stderr, "Websocket ping failed, disconnecting from etcd %s\n", selected_etcd_address.c_str());
-                }
-                if (etcd_watch_ws)
-                {
-                    http_close(etcd_watch_ws);
-                    etcd_watch_ws = NULL;
-                }
                 start_etcd_watcher();
             }
             else
             {
                 ws_alive = 0;
-                http_post_message(etcd_watch_ws, WS_TEXT, json11::Json(json11::Json::object {
+                etcd_watch_ws->post_message(WS_TEXT, json11::Json(json11::Json::object {
                     { "progress_request", json11::Json::object { } }
                 }).dump());
             }
@@ -469,12 +367,12 @@ void etcd_state_client_t::load_global_config()
 {
     etcd_call("/kv/range", json11::Json::object {
         { "key", base64_encode(etcd_prefix+"/config/global") }
-    }, etcd_slow_timeout, max_etcd_attempts, 0, [this](std::string err, json11::Json data)
+    }, ETCD_SLOW_TIMEOUT, [this](std::string err, json11::Json data)
     {
         if (err != "")
         {
             fprintf(stderr, "Error reading OSD configuration from etcd: %s\n", err.c_str());
-            tfd->set_timer(etcd_slow_timeout, false, [this](int timer_id)
+            tfd->set_timer(ETCD_SLOW_TIMEOUT, false, [this](int timer_id)
             {
                 load_global_config();
             });
@@ -542,13 +440,12 @@ void etcd_state_client_t::load_pgs()
     {
         req["compare"] = checks;
     }
-    etcd_txn_slow(req, [this](std::string err, json11::Json data)
+    etcd_txn(req, ETCD_SLOW_TIMEOUT, [this](std::string err, json11::Json data)
     {
         if (err != "")
         {
-            // Retry indefinitely
             fprintf(stderr, "Error loading PGs from etcd: %s\n", err.c_str());
-            tfd->set_timer(etcd_slow_timeout, false, [this](int timer_id)
+            tfd->set_timer(ETCD_SLOW_TIMEOUT, false, [this](int timer_id)
             {
                 load_pgs();
             });

src/etcd_state_client.h

@@ -12,6 +12,11 @@
 #define ETCD_PG_HISTORY_WATCH_ID 3
 #define ETCD_OSD_STATE_WATCH_ID 4
 
+#define MAX_ETCD_ATTEMPTS 5
+#define ETCD_SLOW_TIMEOUT 5000
+#define ETCD_QUICK_TIMEOUT 1000
+#define ETCD_KEEPALIVE_TIMEOUT 30000
+
 #define DEFAULT_BLOCK_SIZE 128*1024
 
 struct etcd_kv_t
@@ -66,7 +71,7 @@ struct inode_watch_t
     inode_config_t cfg;
 };
 
-struct http_co_t;
+struct websocket_t;
 
 struct etcd_state_client_t
 {
@@ -77,20 +82,13 @@ protected:
     std::string selected_etcd_address;
     std::vector<std::string> addresses_to_try;
     std::vector<inode_watch_t*> watches;
-    http_co_t *etcd_watch_ws = NULL, *keepalive_client = NULL;
+    websocket_t *etcd_watch_ws = NULL;
     int ws_keepalive_timer = -1;
     int ws_alive = 0;
-    bool rand_initialized = false;
     uint64_t bs_block_size = DEFAULT_BLOCK_SIZE;
     void add_etcd_url(std::string);
     void pick_next_etcd();
 public:
-    int etcd_keepalive_timeout = 30;
-    int etcd_ws_keepalive_interval = 30;
-    int max_etcd_attempts = 5;
-    int etcd_quick_timeout = 1000;
-    int etcd_slow_timeout = 5000;
-
     std::string etcd_prefix;
     int log_level = 0;
     timerfd_manager_t *tfd = NULL;
@@ -112,9 +110,8 @@ public:
 
     json11::Json::object serialize_inode_cfg(inode_config_t *cfg);
     etcd_kv_t parse_etcd_kv(const json11::Json & kv_json);
-    void etcd_call(std::string api, json11::Json payload, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback);
+    void etcd_call(std::string api, json11::Json payload, int timeout, std::function<void(std::string, json11::Json)> callback);
-    void etcd_txn(json11::Json txn, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback);
+    void etcd_txn(json11::Json txn, int timeout, std::function<void(std::string, json11::Json)> callback);
-    void etcd_txn_slow(json11::Json txn, std::function<void(std::string, json11::Json)> callback);
     void start_etcd_watcher();
     void load_global_config();
     void load_pgs();

src/fio_cluster.cpp

@@ -247,12 +247,6 @@ static int sec_setup(struct thread_data *td)
             vitastor_c_uring_wait_events(bsd->cli);
         }
         td->files[0]->real_file_size = vitastor_c_inode_get_size(bsd->watch);
-        if (!vitastor_c_inode_get_num(bsd->watch) ||
-            !td->files[0]->real_file_size)
-        {
-            td_verror(td, EINVAL, "image does not exist");
-            return 1;
-        }
     }
 
     bsd->trace = o->trace ? true : false;

src/fio_engine.cpp

@@ -26,9 +26,10 @@
 
 #include "blockstore.h"
 #include "epoll_manager.h"
-#include "json11/json11.hpp"
 #include "fio_headers.h"
 
+#include "json11/json11.hpp"
+
 struct bs_data
 {
     blockstore_t *bs;
@@ -149,6 +150,7 @@ static int bs_init(struct thread_data *td)
 static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
 {
     bs_data *bsd = (bs_data*)td->io_ops_data;
+    int n = bsd->op_n;
     if (io->ddir == DDIR_SYNC && bsd->last_sync)
     {
         return FIO_Q_COMPLETED;
@@ -176,7 +178,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         op->version = UINT64_MAX; // last unstable
         op->offset = io->offset % bsd->bs->get_block_size();
         op->len = io->xfer_buflen;
-        op->callback = [io](blockstore_op_t *op)
+        op->callback = [io, n](blockstore_op_t *op)
         {
             io->error = op->retval < 0 ? -op->retval : 0;
             bs_data *bsd = (bs_data*)io->engine_data;
@@ -198,7 +200,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         op->version = 0; // assign automatically
         op->offset = io->offset % bsd->bs->get_block_size();
         op->len = io->xfer_buflen;
-        op->callback = [io](blockstore_op_t *op)
+        op->callback = [io, n](blockstore_op_t *op)
         {
             io->error = op->retval < 0 ? -op->retval : 0;
             bs_data *bsd = (bs_data*)io->engine_data;
@@ -213,7 +215,7 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         break;
     case DDIR_SYNC:
         op->opcode = BS_OP_SYNC_STAB_ALL;
-        op->callback = [io](blockstore_op_t *op)
+        op->callback = [io, n](blockstore_op_t *op)
         {
             bs_data *bsd = (bs_data*)io->engine_data;
             io->error = op->retval < 0 ? -op->retval : 0;
@@ -228,7 +230,6 @@ static enum fio_q_status bs_queue(struct thread_data *td, struct io_u *io)
         break;
     default:
         io->error = EINVAL;
-        delete op;
         return FIO_Q_COMPLETED;
     }
 

src/fio_sec_osd.cpp

@@ -29,22 +29,23 @@
 #include <unordered_map>
 
 #include "addr_util.h"
-#include "rw_blocking.h"
+#include "epoll_manager.h"
+#include "ringloop.h"
+#include "messenger.h"
 #include "osd_ops.h"
 #include "fio_headers.h"
 
-struct op_buf_t
-{
-    osd_any_op_t buf;
-    io_u* fio_op;
-};
-
 struct sec_data
 {
+    epoll_manager_t *epmgr;
+    ring_loop_t *ringloop;
+    osd_messenger_t *msgr;
+    ring_consumer_t looper;
+    void *bitmap_buf;
     int connect_fd;
+    uint64_t op_id;
     /* block_size = 1 << block_order (128KB by default) */
     uint64_t block_order = 17, block_size = 1 << 17;
-    std::unordered_map<uint64_t, op_buf_t*> queue;
     bool last_sync = false;
     /* The list of completed io_u structs. */
     std::vector<io_u*> completed;
@@ -59,7 +60,6 @@ struct sec_options
     int single_primary = 0;
     int trace = 0;
     int block_order = 17;
-    int zerocopy_send = 0;
 };
 
 static struct fio_option options[] = {
@@ -110,16 +110,6 @@ static struct fio_option options[] = {
         .category = FIO_OPT_C_ENGINE,
         .group  = FIO_OPT_G_FILENAME,
     },
-    {
-        .name   = "zerocopy_send",
-        .lname  = "Use zero-copy send",
-        .type   = FIO_OPT_BOOL,
-        .off1   = offsetof(struct sec_options, zerocopy_send),
-        .help   = "Use zero-copy send (MSG_ZEROCOPY)",
-        .def    = "0",
-        .category = FIO_OPT_C_ENGINE,
-        .group  = FIO_OPT_G_FILENAME,
-    },
     {
         .name = NULL,
     },
@@ -128,9 +118,6 @@ static struct fio_option options[] = {
 static int sec_setup(struct thread_data *td)
 {
     sec_data *bsd;
-    //fio_file *f;
-    //int r;
-    //int64_t size;
 
     bsd = new sec_data;
     if (!bsd)
@@ -147,8 +134,6 @@ static int sec_setup(struct thread_data *td)
         td->o.open_files++;
     }
 
-    //f = td->files[0];
-    //f->real_file_size = size;
     return 0;
 }
 
@@ -157,6 +142,10 @@ static void sec_cleanup(struct thread_data *td)
     sec_data *bsd = (sec_data*)td->io_ops_data;
     if (bsd)
     {
+        delete bsd->msgr;
+        delete bsd->epmgr;
+        delete bsd->ringloop;
+        free(bsd->bitmap_buf);
         close(bsd->connect_fd);
         delete bsd;
     }
@@ -190,14 +179,45 @@ static int sec_init(struct thread_data *td)
     }
     int one = 1;
     setsockopt(bsd->connect_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
-    if (o->zerocopy_send)
+
+    fcntl(bsd->connect_fd, F_SETFL, fcntl(bsd->connect_fd, F_GETFL, 0) | O_NONBLOCK);
+
+    json11::Json cfg = json11::Json::object{ { "use_rdma", 0 } };
+
+    bsd->bitmap_buf = malloc(4096);
+
+    bsd->ringloop = new ring_loop_t(512);
+    bsd->epmgr = new epoll_manager_t(bsd->ringloop);
+    bsd->msgr = new osd_messenger_t();
+    bsd->msgr->tfd = bsd->epmgr->tfd;
+    bsd->msgr->ringloop = bsd->ringloop;
+    bsd->msgr->repeer_pgs = [](osd_num_t){};
+    bsd->msgr->parse_config(cfg);
+    bsd->msgr->init();
+
+    bsd->looper.loop = [bsd]()
     {
-        if (setsockopt(bsd->connect_fd, SOL_SOCKET, SO_ZEROCOPY, &one, sizeof(one)) < 0)
+        bsd->msgr->read_requests();
-        {
+        bsd->msgr->send_replies();
-            perror("setsockopt zerocopy");
+        bsd->ringloop->submit();
-            return 1;
+    };
-        }
+    bsd->ringloop->register_consumer(&bsd->looper);
-    }
+
+    int peer_fd = bsd->connect_fd;
+    bsd->msgr->clients[peer_fd] = new osd_client_t();
+    bsd->msgr->clients[peer_fd]->peer_addr = addr;
+    bsd->msgr->clients[peer_fd]->peer_port = ntohs(((sockaddr_in*)&addr)->sin_port);
+    bsd->msgr->clients[peer_fd]->peer_fd = peer_fd;
+    bsd->msgr->clients[peer_fd]->peer_state = PEER_CONNECTED;
+    bsd->msgr->clients[peer_fd]->connect_timeout_id = -1;
+    bsd->msgr->clients[peer_fd]->osd_num = 1;
+    bsd->msgr->clients[peer_fd]->in_buf = malloc_or_die(bsd->msgr->receive_buffer_size);
+    bsd->epmgr->tfd->set_fd_handler(peer_fd, true, [msgr = bsd->msgr](int peer_fd, int epoll_events)
+    {
+        // Either OUT (connected) or HUP
+        msgr->handle_peer_epoll(peer_fd, epoll_events);
+    });
+    bsd->msgr->osd_peer_fds[1] = peer_fd;
 
     // FIXME: read config (block size) from OSD
 
@@ -218,9 +238,12 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
     }
 
     io->engine_data = bsd;
-    op_buf_t *op_buf = new op_buf_t;
+
-    op_buf->fio_op = io;
+    osd_op_t *oo = new osd_op_t();
-    osd_any_op_t &op = op_buf->buf;
+    oo->op_type = OSD_OP_OUT;
+    oo->peer_fd = bsd->connect_fd;
+
+    osd_any_op_t & op = oo->req;
 
     op.hdr.magic = SECONDARY_OSD_OP_MAGIC;
     op.hdr.id = n;
@@ -237,6 +260,9 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
             op.sec_rw.version = UINT64_MAX; // last unstable
             op.sec_rw.offset = io->offset % bsd->block_size;
             op.sec_rw.len = io->xfer_buflen;
+            op.sec_rw.attr_len = 4;
+            oo->bitmap = bsd->bitmap_buf;
+            oo->bitmap_len = 4;
         }
         else
         {
@@ -245,6 +271,7 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
             op.rw.offset = io->offset;
             op.rw.len = io->xfer_buflen;
         }
+        oo->iov.push_back(io->xfer_buf, io->xfer_buflen);
         bsd->last_sync = false;
         break;
     case DDIR_WRITE:
@@ -266,6 +293,7 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
             op.rw.offset = io->offset;
             op.rw.len = io->xfer_buflen;
         }
+        oo->iov.push_back(io->xfer_buf, io->xfer_buflen);
         bsd->last_sync = false;
         break;
     case DDIR_SYNC:
@@ -284,10 +312,24 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
         break;
     default:
         io->error = EINVAL;
-        delete op_buf;
         return FIO_Q_COMPLETED;
     }
 
+    oo->callback = [td, io](osd_op_t *oo)
+    {
+        sec_options *opt = (sec_options*)td->eo;
+        sec_data *bsd = (sec_data*)td->io_ops_data;
+        if (opt->trace)
+        {
+            printf("--- %s # %ld %ld\n", io->ddir == DDIR_READ ? "READ" :
+                (io->ddir == DDIR_WRITE ? "WRITE" : "SYNC"), oo->reply.hdr.id, oo->reply.hdr.retval);
+        }
+        io->error = oo->reply.hdr.retval < 0 ? -oo->reply.hdr.retval : 0;
+        bsd->completed.push_back(io);
+        delete oo;
+    };
+    bsd->msgr->outbox_push(oo);
+
     if (opt->trace)
     {
         printf("+++ %s # %d\n", io->ddir == DDIR_READ ? "READ" :
@@ -297,20 +339,6 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
     io->error = 0;
     bsd->inflight++;
     bsd->op_n++;
-    bsd->queue[n] = op_buf;
-
-    iovec iov[2] = { { .iov_base = op.buf, .iov_len = OSD_PACKET_SIZE } };
-    int iovcnt = 1, wtotal = OSD_PACKET_SIZE;
-    if (io->ddir == DDIR_WRITE)
-    {
-        iov[iovcnt++] = { .iov_base = io->xfer_buf, .iov_len = io->xfer_buflen };
-        wtotal += io->xfer_buflen;
-    }
-    if (sendv_blocking(bsd->connect_fd, iov, iovcnt, opt->zerocopy_send ? MSG_ZEROCOPY : 0) != wtotal)
-    {
-        perror("sendmsg");
-        exit(1);
-    }
 
     if (io->error != 0)
         return FIO_Q_COMPLETED;
@@ -319,74 +347,13 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
 
 static int sec_getevents(struct thread_data *td, unsigned int min, unsigned int max, const struct timespec *t)
 {
-    sec_options *opt = (sec_options*)td->eo;
     sec_data *bsd = (sec_data*)td->io_ops_data;
-    // FIXME timeout, at least poll. Now it's the stupidest implementation possible
-    osd_any_reply_t reply;
     while (bsd->completed.size() < min)
     {
-        read_blocking(bsd->connect_fd, reply.buf, OSD_PACKET_SIZE);
+        bsd->ringloop->loop();
-        if (reply.hdr.magic != SECONDARY_OSD_REPLY_MAGIC)
+        if (bsd->completed.size() >= min)
-        {
+            break;
-            fprintf(stderr, "bad reply: magic = %lx instead of %lx\n", reply.hdr.magic, SECONDARY_OSD_REPLY_MAGIC);
+        bsd->ringloop->wait();
-            exit(1);
-        }
-        auto it = bsd->queue.find(reply.hdr.id);
-        if (it == bsd->queue.end())
-        {
-            fprintf(stderr, "bad reply: op id %lx missing in local queue\n", reply.hdr.id);
-            exit(1);
-        }
-        io_u* io = it->second->fio_op;
-        delete it->second;
-        bsd->queue.erase(it);
-        if (io->ddir == DDIR_READ)
-        {
-            if (reply.hdr.retval != io->xfer_buflen)
-            {
-                fprintf(stderr, "Short read: retval = %ld instead of %llu\n", reply.hdr.retval, io->xfer_buflen);
-                exit(1);
-            }
-            // Support bitmap
-            uint64_t bitmap = 0;
-            int iovcnt = 0;
-            iovec iov[2];
-            if (reply.sec_rw.attr_len > 0)
-            {
-                if (reply.sec_rw.attr_len <= 8)
-                    iov[iovcnt++] = { .iov_base = &bitmap, .iov_len = reply.sec_rw.attr_len };
-                else
-                    iov[iovcnt++] = { .iov_base = (void*)(bitmap = (uint64_t)malloc(reply.sec_rw.attr_len)), .iov_len = reply.sec_rw.attr_len };
-            }
-            iov[iovcnt++] = { .iov_base = io->xfer_buf, .iov_len = io->xfer_buflen };
-            readv_blocking(bsd->connect_fd, iov, iovcnt);
-            if (reply.sec_rw.attr_len > 8)
-            {
-                free((void*)bitmap);
-            }
-        }
-        else if (io->ddir == DDIR_WRITE)
-        {
-            if (reply.hdr.retval != io->xfer_buflen)
-            {
-                fprintf(stderr, "Short write: retval = %ld instead of %llu\n", reply.hdr.retval, io->xfer_buflen);
-                exit(1);
-            }
-        }
-        else if (io->ddir == DDIR_SYNC)
-        {
-            if (reply.hdr.retval != 0)
-            {
-                fprintf(stderr, "Sync failed: retval = %ld\n", reply.hdr.retval);
-                exit(1);
-            }
-        }
-        if (opt->trace)
-        {
-            printf("--- %s # %ld\n", io->ddir == DDIR_READ ? "READ" :
-                (io->ddir == DDIR_WRITE ? "WRITE" : "SYNC"), reply.hdr.id);
-        }
-        bsd->completed.push_back(io);
     }
     return bsd->completed.size();
 }

src/http_client.cpp

@@ -4,7 +4,9 @@
 #include <netinet/tcp.h>
 #include <sys/epoll.h>
 
+#include <net/if.h>
 #include <arpa/inet.h>
+#include <ifaddrs.h>
 
 #include <ctype.h>
 #include <unistd.h>
@@ -23,12 +25,11 @@
 static std::string trim(const std::string & in);
 static std::string ws_format_frame(int type, uint64_t size);
 static bool ws_parse_frame(std::string & buf, int & type, std::string & res);
-static void parse_http_headers(std::string & res, http_response_t *parsed);
 
+// FIXME: Use keepalive
 struct http_co_t
 {
     timerfd_manager_t *tfd;
-    std::function<void(const http_response_t*)> response_callback;
 
     int request_timeout = 0;
     std::string host;
@@ -36,12 +37,11 @@ struct http_co_t
     std::string ws_outbox;
     std::string response;
     bool want_streaming;
-    bool keepalive;
 
-    std::vector<std::function<void()>> keepalive_queue;
+    http_response_t parsed;
+    uint64_t target_response_size = 0;
 
     int state = 0;
-    std::string connected_host;
     int peer_fd = -1;
     int timeout_id = -1;
     int epoll_events = 0;
@@ -49,8 +49,10 @@ struct http_co_t
     std::vector<char> rbuf;
     iovec read_iov, send_iov;
     msghdr read_msg = { 0 }, send_msg = { 0 };
-    http_response_t parsed;
+
-    uint64_t target_response_size = 0;
+    std::function<void(const http_response_t*)> callback;
+
+    websocket_t ws;
 
     int onstack = 0;
     bool ended = false;
@@ -59,40 +61,66 @@ struct http_co_t
     inline void stackin() { onstack++; }
     inline void stackout() { onstack--; if (!onstack && ended) end(); }
     inline void end() { ended = true; if (!onstack) { delete this; } }
-    void run_cb_and_clear();
     void start_connection();
-    void close_connection();
     void handle_events();
     void handle_connect_result();
     void submit_read();
     void submit_send();
     bool handle_read();
     void post_message(int type, const std::string & msg);
-    void send_request(const std::string & host, const std::string & request,
-        const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
 };
 
-#define HTTP_CO_CLOSED 0
 #define HTTP_CO_CONNECTING 1
 #define HTTP_CO_SENDING_REQUEST 2
 #define HTTP_CO_REQUEST_SENT 3
 #define HTTP_CO_HEADERS_RECEIVED 4
 #define HTTP_CO_WEBSOCKET 5
 #define HTTP_CO_CHUNKED 6
-#define HTTP_CO_KEEPALIVE 7
 
 #define DEFAULT_TIMEOUT 5000
 
-http_co_t *http_init(timerfd_manager_t *tfd)
+void http_request(timerfd_manager_t *tfd, const std::string & host, const std::string & request,
+    const http_options_t & options, std::function<void(const http_response_t *response)> callback)
 {
     http_co_t *handler = new http_co_t();
+    handler->request_timeout = options.timeout < 0 ? 0 : (options.timeout == 0 ? DEFAULT_TIMEOUT : options.timeout);
+    handler->want_streaming = options.want_streaming;
     handler->tfd = tfd;
-    handler->state = HTTP_CO_CLOSED;
+    handler->host = host;
-    return handler;
+    handler->request = request;
+    handler->callback = callback;
+    handler->ws.co = handler;
+    handler->start_connection();
 }
 
-http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, const std::string & path,
+void http_request_json(timerfd_manager_t *tfd, const std::string & host, const std::string & request,
-    int timeout, std::function<void(const http_response_t *msg)> response_callback)
+    int timeout, std::function<void(std::string, json11::Json r)> callback)
+{
+    http_request(tfd, host, request, { .timeout = timeout }, [callback](const http_response_t* res)
+    {
+        if (res->error_code != 0)
+        {
+            callback("Error code: "+std::to_string(res->error_code)+" ("+std::string(strerror(res->error_code))+")", json11::Json());
+            return;
+        }
+        if (res->status_code != 200)
+        {
+            callback("HTTP "+std::to_string(res->status_code)+" "+res->status_line+" body: "+trim(res->body), json11::Json());
+            return;
+        }
+        std::string json_err;
+        json11::Json data = json11::Json::parse(res->body, json_err);
+        if (json_err != "")
+        {
+            callback("Bad JSON: "+json_err+" (response: "+trim(res->body)+")", json11::Json());
+            return;
+        }
+        callback(std::string(), data);
+    });
+}
+
+websocket_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, const std::string & path,
+    int timeout, std::function<void(const http_response_t *msg)> callback)
 {
     std::string request = "GET "+path+" HTTP/1.1\r\n"
         "Host: "+host+"\r\n"
@@ -102,154 +130,28 @@ http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, cons
         "Sec-WebSocket-Version: 13\r\n"
         "\r\n";
     http_co_t *handler = new http_co_t();
-    handler->tfd = tfd;
-    handler->state = HTTP_CO_CLOSED;
-    handler->host = host;
     handler->request_timeout = timeout < 0 ? -1 : (timeout == 0 ? DEFAULT_TIMEOUT : timeout);
     handler->want_streaming = false;
-    handler->keepalive = false;
+    handler->tfd = tfd;
+    handler->host = host;
     handler->request = request;
-    handler->response_callback = response_callback;
+    handler->callback = callback;
+    handler->ws.co = handler;
     handler->start_connection();
-    return handler;
+    return &handler->ws;
 }
 
-void http_request(http_co_t *handler, const std::string & host, const std::string & request,
+void websocket_t::post_message(int type, const std::string & msg)
-    const http_options_t & options, std::function<void(const http_response_t *response)> response_callback)
 {
-    handler->send_request(host, request, options, response_callback);
+    co->post_message(type, msg);
 }
 
-void http_co_t::run_cb_and_clear()
+void websocket_t::close()
 {
-    parsed.eof = true;
+    co->end();
-    std::function<void(const http_response_t*)> cb;
-    cb.swap(response_callback);
-    // Call callback after clearing it because otherwise we may hit reenterability problems
-    if (cb != NULL)
-        cb(&parsed);
-}
-
-void http_co_t::send_request(const std::string & host, const std::string & request,
-    const http_options_t & options, std::function<void(const http_response_t *response)> response_callback)
-{
-    stackin();
-    if (state == HTTP_CO_WEBSOCKET)
-    {
-        stackout();
-        throw std::runtime_error("Attempt to send HTTP request into a websocket or chunked stream");
-    }
-    else if (state != HTTP_CO_KEEPALIVE && state != HTTP_CO_CLOSED)
-    {
-        keepalive_queue.push_back([this, host, request, options, response_callback]()
-        {
-            this->send_request(host, request, options, response_callback);
-        });
-        stackout();
-        return;
-    }
-    if (state == HTTP_CO_KEEPALIVE && connected_host != host)
-    {
-        close_connection();
-    }
-    this->request_timeout = options.timeout < 0 ? 0 : (options.timeout == 0 ? DEFAULT_TIMEOUT : options.timeout);
-    this->want_streaming = options.want_streaming;
-    this->keepalive = options.keepalive;
-    this->host = host;
-    this->request = request;
-    this->response = "";
-    this->sent = 0;
-    this->response_callback = response_callback;
-    this->parsed = {};
-    if (request_timeout > 0)
-    {
-        timeout_id = tfd->set_timer(request_timeout, false, [this](int timer_id)
-        {
-            stackin();
-            close_connection();
-            parsed = { .error = "HTTP request timed out" };
-            run_cb_and_clear();
-            stackout();
-        });
-    }
-    if (state == HTTP_CO_KEEPALIVE)
-    {
-        state = HTTP_CO_SENDING_REQUEST;
-        submit_send();
-    }
-    else
-    {
-        start_connection();
-    }
-    stackout();
-}
-
-void http_post_message(http_co_t *handler, int type, const std::string & msg)
-{
-    handler->post_message(type, msg);
-}
-
-void http_co_t::post_message(int type, const std::string & msg)
-{
-    stackin();
-    if (state == HTTP_CO_WEBSOCKET)
-    {
-        request += ws_format_frame(type, msg.size());
-        request += msg;
-        submit_send();
-    }
-    else if (state == HTTP_CO_KEEPALIVE || state == HTTP_CO_CHUNKED)
-    {
-        throw std::runtime_error("Attempt to send websocket message on a regular HTTP connection");
-    }
-    else
-    {
-        ws_outbox += ws_format_frame(type, msg.size());
-        ws_outbox += msg;
-    }
-    stackout();
-}
-
-void http_close(http_co_t *handler)
-{
-    handler->end();
-}
-
-void http_response_t::parse_json_response(std::string & error, json11::Json & r) const
-{
-    if (this->error != "")
-    {
-        error = this->error;
-        r = json11::Json();
-    }
-    else if (status_code != 200)
-    {
-        error = "HTTP "+std::to_string(status_code)+" "+status_line+" body: "+trim(body);
-        r = json11::Json();
-    }
-    else
-    {
-        std::string json_err;
-        json11::Json data = json11::Json::parse(body, json_err);
-        if (json_err != "")
-        {
-            error = "Bad JSON: "+json_err+" (response: "+trim(body)+")";
-            r = json11::Json();
-        }
-        else
-        {
-            error = "";
-            r = data;
-        }
-    }
 }
 
 http_co_t::~http_co_t()
-{
-    close_connection();
-}
-
-void http_co_t::close_connection()
 {
     if (timeout_id >= 0)
     {
@@ -262,10 +164,22 @@ void http_co_t::close_connection()
         close(peer_fd);
         peer_fd = -1;
     }
-    state = HTTP_CO_CLOSED;
+    if (parsed.headers["transfer-encoding"] == "chunked")
-    connected_host = "";
+    {
-    response = "";
+        int prev = 0, pos = 0;
-    epoll_events = 0;
+        while ((pos = response.find("\r\n", prev)) >= prev)
+        {
+            uint64_t len = strtoull(response.c_str()+prev, NULL, 16);
+            parsed.body += response.substr(pos+2, len);
+            prev = pos+2+len+2;
+        }
+    }
+    else
+    {
+        std::swap(parsed.body, response);
+    }
+    parsed.eof = true;
+    callback(&parsed);
 }
 
 void http_co_t::start_connection()
@@ -274,29 +188,39 @@ void http_co_t::start_connection()
     struct sockaddr addr;
     if (!string_to_addr(host.c_str(), 1, 80, &addr))
     {
-        parsed = { .error = "Invalid address: "+host };
+        parsed.error_code = ENXIO;
-        run_cb_and_clear();
         stackout();
+        end();
         return;
     }
     peer_fd = socket(addr.sa_family, SOCK_STREAM, 0);
     if (peer_fd < 0)
     {
-        parsed = { .error = std::string("socket: ")+strerror(errno) };
+        parsed.error_code = errno;
-        run_cb_and_clear();
         stackout();
+        end();
         return;
     }
     fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
+    if (request_timeout > 0)
+    {
+        timeout_id = tfd->set_timer(request_timeout, false, [this](int timer_id)
+        {
+            if (response.length() == 0)
+            {
+                parsed.error_code = ETIME;
+            }
+            end();
+        });
+    }
     epoll_events = 0;
     // Finally call connect
     int r = ::connect(peer_fd, (sockaddr*)&addr, sizeof(addr));
     if (r < 0 && errno != EINPROGRESS)
     {
-        close_connection();
+        parsed.error_code = errno;
-        parsed = { .error = std::string("connect: ")+strerror(errno) };
-        run_cb_and_clear();
         stackout();
+        end();
         return;
     }
     tfd->set_fd_handler(peer_fd, true, [this](int peer_fd, int epoll_events)
@@ -304,7 +228,6 @@ void http_co_t::start_connection()
         this->epoll_events |= epoll_events;
         handle_events();
     });
-    connected_host = host;
     state = HTTP_CO_CONNECTING;
     stackout();
 }
@@ -327,8 +250,7 @@ void http_co_t::handle_events()
             }
             else if (epoll_events & (EPOLLRDHUP|EPOLLERR))
             {
-                close_connection();
+                end();
-                run_cb_and_clear();
                 break;
             }
         }
@@ -347,10 +269,9 @@ void http_co_t::handle_connect_result()
     }
     if (result != 0)
     {
-        close_connection();
+        parsed.error_code = result;
-        parsed = { .error = std::string("connect: ")+strerror(result) };
-        run_cb_and_clear();
         stackout();
+        end();
         return;
     }
     int one = 1;
@@ -365,51 +286,6 @@ void http_co_t::handle_connect_result()
     stackout();
 }
 
-void http_co_t::submit_send()
-{
-    stackin();
-    int res;
-again:
-    if (sent < request.size())
-    {
-        send_iov = (iovec){ .iov_base = (void*)(request.c_str()+sent), .iov_len = request.size()-sent };
-        send_msg.msg_iov = &send_iov;
-        send_msg.msg_iovlen = 1;
-        res = sendmsg(peer_fd, &send_msg, MSG_NOSIGNAL);
-        if (res < 0)
-        {
-            res = -errno;
-        }
-        if (res == -EAGAIN || res == -EINTR)
-        {
-            res = 0;
-        }
-        else if (res < 0)
-        {
-            close_connection();
-            parsed = { .error = std::string("sendmsg: ")+strerror(errno) };
-            run_cb_and_clear();
-            stackout();
-            return;
-        }
-        sent += res;
-        if (state == HTTP_CO_SENDING_REQUEST)
-        {
-            if (sent >= request.size())
-                state = HTTP_CO_REQUEST_SENT;
-            else
-                goto again;
-        }
-        else if (state == HTTP_CO_WEBSOCKET)
-        {
-            request = request.substr(sent);
-            sent = 0;
-            goto again;
-        }
-    }
-    stackout();
-}
-
 void http_co_t::submit_read()
 {
     stackin();
@@ -426,18 +302,16 @@ void http_co_t::submit_read()
     {
         res = -errno;
     }
-    if (res == -EAGAIN || res == -EINTR)
+    if (res == -EAGAIN)
     {
         epoll_events = epoll_events & ~EPOLLIN;
     }
     else if (res <= 0)
     {
         // < 0 means error, 0 means EOF
-        epoll_events = epoll_events & ~EPOLLIN;
+        if (!res)
-        close_connection();
+            epoll_events = epoll_events & ~EPOLLIN;
-        if (res < 0)
+        end();
-            parsed = { .error = std::string("recvmsg: ")+strerror(-res) };
-        run_cb_and_clear();
     }
     else
     {
@@ -447,6 +321,51 @@ void http_co_t::submit_read()
     stackout();
 }
 
+void http_co_t::submit_send()
+{
+    stackin();
+    int res;
+again:
+    if (sent < request.size())
+    {
+        send_iov = (iovec){ .iov_base = (void*)(request.c_str()+sent), .iov_len = request.size()-sent };
+        send_msg.msg_iov = &send_iov;
+        send_msg.msg_iovlen = 1;
+        res = sendmsg(peer_fd, &send_msg, MSG_NOSIGNAL);
+        if (res < 0)
+        {
+            res = -errno;
+        }
+        if (res == -EAGAIN)
+        {
+            res = 0;
+        }
+        else if (res < 0)
+        {
+            stackout();
+            end();
+            return;
+        }
+        sent += res;
+        if (state == HTTP_CO_SENDING_REQUEST)
+        {
+            if (sent >= request.size())
+            {
+                state = HTTP_CO_REQUEST_SENT;
+            }
+            else
+                goto again;
+        }
+        else if (state == HTTP_CO_WEBSOCKET)
+        {
+            request = request.substr(sent);
+            sent = 0;
+            goto again;
+        }
+    }
+    stackout();
+}
+
 bool http_co_t::handle_read()
 {
     stackin();
@@ -457,7 +376,6 @@ bool http_co_t::handle_read()
         {
             if (timeout_id >= 0)
             {
-                // Timeout is cleared when headers are received
                 tfd->clear_timer(timeout_id);
                 timeout_id = -1;
             }
@@ -485,26 +403,20 @@ bool http_co_t::handle_read()
                 if (!target_response_size)
                 {
                     // Sorry, unsupported response
-                    close_connection();
-                    parsed = { .error = "Response has neither Connection: close, nor Transfer-Encoding: chunked nor Content-Length headers" };
-                    run_cb_and_clear();
                     stackout();
+                    end();
                     return false;
                 }
             }
-            else
-            {
-                keepalive = false;
-            }
         }
     }
     if (state == HTTP_CO_HEADERS_RECEIVED && target_response_size > 0 && response.size() >= target_response_size)
     {
-        std::swap(parsed.body, response);
+        stackout();
-        response_callback(&parsed);
+        end();
-        parsed.eof = true;
+        return false;
     }
-    else if (state == HTTP_CO_CHUNKED && response.size() > 0)
+    if (state == HTTP_CO_CHUNKED && response.size() > 0)
     {
         int prev = 0, pos = 0;
         while ((pos = response.find("\r\n", prev)) >= prev)
@@ -527,49 +439,55 @@ bool http_co_t::handle_read()
         {
             response = response.substr(prev);
         }
-        if (want_streaming)
+        if (parsed.eof)
         {
-            // Streaming response
+            stackout();
-            response_callback(&parsed);
+            end();
+            return false;
+        }
+        if (want_streaming && parsed.body.size() > 0)
+        {
+            if (!ended)
+            {
+                // Don't deliver additional events after close()
+                callback(&parsed);
+            }
             parsed.body = "";
         }
-        if (parsed.eof && !want_streaming)
-        {
-            // Normal response
-            response_callback(&parsed);
-        }
     }
-    else if (state == HTTP_CO_WEBSOCKET && response.size() > 0)
+    if (state == HTTP_CO_WEBSOCKET && response.size() > 0)
     {
         while (ws_parse_frame(response, parsed.ws_msg_type, parsed.body))
         {
-            response_callback(&parsed);
+            if (!ended)
-            parsed.body = "";
-        }
-    }
-    if (parsed.eof)
-    {
-        response_callback = NULL;
-        parsed = {};
-        if (!keepalive)
-        {
-            close_connection();
-        }
-        else
-        {
-            state = HTTP_CO_KEEPALIVE;
-            if (keepalive_queue.size() > 0)
             {
-                auto next = keepalive_queue[0];
+                // Don't deliver additional events after close()
-                keepalive_queue.erase(keepalive_queue.begin(), keepalive_queue.begin()+1);
+                callback(&parsed);
-                next();
             }
+            parsed.body = "";
         }
     }
     stackout();
     return true;
 }
 
+void http_co_t::post_message(int type, const std::string & msg)
+{
+    stackin();
+    if (state == HTTP_CO_WEBSOCKET)
+    {
+        request += ws_format_frame(type, msg.size());
+        request += msg;
+        submit_send();
+    }
+    else
+    {
+        ws_outbox += ws_format_frame(type, msg.size());
+        ws_outbox += msg;
+    }
+    stackout();
+}
+
 uint64_t stoull_full(const std::string & str, int base)
 {
     if (isspace(str[0]))
@@ -585,7 +503,7 @@ uint64_t stoull_full(const std::string & str, int base)
     return r;
 }
 
-static void parse_http_headers(std::string & res, http_response_t *parsed)
+void parse_http_headers(std::string & res, http_response_t *parsed)
 {
     int pos = res.find("\r\n");
     pos = pos < 0 ? res.length() : pos+2;
@@ -634,13 +552,13 @@ static std::string ws_format_frame(int type, uint64_t size)
         res[p++] = size | /*mask*/0x80;
     else if (size < 65536)
     {
-        res[p++] = (char)(126 | /*mask*/0x80);
+        res[p++] = 126 | /*mask*/0x80;
         res[p++] = (size >> 8) & 0xFF;
         res[p++] = (size >> 0) & 0xFF;
     }
     else
     {
-        res[p++] = (char)(127 | /*mask*/0x80);
+        res[p++] = 127 | /*mask*/0x80;
         res[p++] = (size >> 56) & 0xFF;
         res[p++] = (size >> 48) & 0xFF;
         res[p++] = (size >> 40) & 0xFF;
@@ -707,6 +625,136 @@ static bool ws_parse_frame(std::string & buf, int & type, std::string & res)
     return true;
 }
 
+static bool cidr_match(const in_addr &addr, const in_addr &net, uint8_t bits)
+{
+    if (bits == 0)
+    {
+        // C99 6.5.7 (3): u32 << 32 is undefined behaviour
+        return true;
+    }
+    return !((addr.s_addr ^ net.s_addr) & htonl(0xFFFFFFFFu << (32 - bits)));
+}
+
+static bool cidr6_match(const in6_addr &address, const in6_addr &network, uint8_t bits)
+{
+    const uint32_t *a = address.s6_addr32;
+    const uint32_t *n = network.s6_addr32;
+    int bits_whole, bits_incomplete;
+    bits_whole = bits >> 5;         // number of whole u32
+    bits_incomplete = bits & 0x1F;  // number of bits in incomplete u32
+    if (bits_whole && memcmp(a, n, bits_whole << 2))
+        return false;
+    if (bits_incomplete)
+    {
+        uint32_t mask = htonl((0xFFFFFFFFu) << (32 - bits_incomplete));
+        if ((a[bits_whole] ^ n[bits_whole]) & mask)
+            return false;
+    }
+    return true;
+}
+
+struct addr_mask_t
+{
+    sa_family_t family;
+    in_addr ipv4;
+    in6_addr ipv6;
+    uint8_t bits;
+};
+
+std::vector<std::string> getifaddr_list(json11::Json mask_cfg, bool include_v6)
+{
+    std::vector<addr_mask_t> masks;
+    if (mask_cfg.is_string())
+    {
+        mask_cfg = json11::Json::array{ mask_cfg };
+    }
+    for (auto mask_json: mask_cfg.array_items())
+    {
+        std::string mask = mask_json.string_value();
+        unsigned bits = 0;
+        int p = mask.find('/');
+        if (p != std::string::npos)
+        {
+            char null_byte = 0;
+            if (sscanf(mask.c_str()+p+1, "%u%c", &bits, &null_byte) != 1 || bits > 128)
+            {
+                throw std::runtime_error((include_v6 ? "Invalid IPv4 address mask: " : "Invalid IP address mask: ") + mask);
+            }
+            mask = mask.substr(0, p);
+        }
+        in_addr ipv4;
+        in6_addr ipv6;
+        if (inet_pton(AF_INET, mask.c_str(), &ipv4) == 1)
+        {
+            if (bits > 32)
+            {
+                throw std::runtime_error((include_v6 ? "Invalid IPv4 address mask: " : "Invalid IP address mask: ") + mask);
+            }
+            masks.push_back((addr_mask_t){ .family = AF_INET, .ipv4 = ipv4, .bits = (uint8_t)bits });
+        }
+        else if (include_v6 && inet_pton(AF_INET6, mask.c_str(), &ipv6) == 1)
+        {
+            masks.push_back((addr_mask_t){ .family = AF_INET6, .ipv6 = ipv6, .bits = (uint8_t)bits });
+        }
+        else
+        {
+            throw std::runtime_error((include_v6 ? "Invalid IPv4 address mask: " : "Invalid IP address mask: ") + mask);
+        }
+    }
+    std::vector<std::string> addresses;
+    ifaddrs *list, *ifa;
+    if (getifaddrs(&list) == -1)
+    {
+        throw std::runtime_error(std::string("getifaddrs: ") + strerror(errno));
+    }
+    for (ifa = list; ifa != NULL; ifa = ifa->ifa_next)
+    {
+        if (!ifa->ifa_addr)
+        {
+            continue;
+        }
+        int family = ifa->ifa_addr->sa_family;
+        if ((family == AF_INET || family == AF_INET6 && include_v6) &&
+            (ifa->ifa_flags & (IFF_UP | IFF_RUNNING | IFF_LOOPBACK)) == (IFF_UP | IFF_RUNNING))
+        {
+            void *addr_ptr;
+            if (family == AF_INET)
+            {
+                addr_ptr = &((sockaddr_in *)ifa->ifa_addr)->sin_addr;
+            }
+            else
+            {
+                addr_ptr = &((sockaddr_in6 *)ifa->ifa_addr)->sin6_addr;
+            }
+            if (masks.size() > 0)
+            {
+                int i;
+                for (i = 0; i < masks.size(); i++)
+                {
+                    if (masks[i].family == family && (family == AF_INET
+                        ? cidr_match(*(in_addr*)addr_ptr, masks[i].ipv4, masks[i].bits)
+                        : cidr6_match(*(in6_addr*)addr_ptr, masks[i].ipv6, masks[i].bits)))
+                    {
+                        break;
+                    }
+                }
+                if (i >= masks.size())
+                {
+                    continue;
+                }
+            }
+            char addr[INET6_ADDRSTRLEN];
+            if (!inet_ntop(family, addr_ptr, addr, INET6_ADDRSTRLEN))
+            {
+                throw std::runtime_error(std::string("inet_ntop: ") + strerror(errno));
+            }
+            addresses.push_back(std::string(addr));
+        }
+    }
+    freeifaddrs(list);
+    return addresses;
+}
+
 std::string strtolower(const std::string & in)
 {
     std::string s = in;

src/http_client.h

@@ -21,34 +21,41 @@ struct http_options_t
 {
     int timeout;
     bool want_streaming;
-    bool keepalive;
 };
 
 struct http_response_t
 {
-    std::string error;
-
     bool eof = false;
+    int error_code = 0;
     int status_code = 0;
     std::string status_line;
     std::map<std::string, std::string> headers;
     int ws_msg_type = -1;
     std::string body;
-
-    void parse_json_response(std::string & error, json11::Json & r) const;
 };
 
-// Opened websocket or keepalive HTTP connection
 struct http_co_t;
 
-http_co_t* http_init(timerfd_manager_t *tfd);
+struct websocket_t
-http_co_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, const std::string & path,
+{
-    int timeout, std::function<void(const http_response_t *msg)> on_message);
+    http_co_t *co;
-void http_request(http_co_t *handler, const std::string & host, const std::string & request,
+    void post_message(int type, const std::string & msg);
-    const http_options_t & options, std::function<void(const http_response_t *response)> response_callback);
+    void close();
-void http_post_message(http_co_t *handler, int type, const std::string & msg);
+};
-void http_close(http_co_t *co);
+
+void parse_http_headers(std::string & res, http_response_t *parsed);
+
+std::vector<std::string> getifaddr_list(json11::Json mask_cfg = json11::Json(), bool include_v6 = true);
 
-// Utils
 uint64_t stoull_full(const std::string & str, int base = 10);
+
 std::string strtolower(const std::string & in);
+
+void http_request(timerfd_manager_t *tfd, const std::string & host, const std::string & request,
+    const http_options_t & options, std::function<void(const http_response_t *response)> callback);
+
+void http_request_json(timerfd_manager_t *tfd, const std::string & host, const std::string & request,
+    int timeout, std::function<void(std::string, json11::Json r)> callback);
+
+websocket_t* open_websocket(timerfd_manager_t *tfd, const std::string & host, const std::string & path,
+    int timeout, std::function<void(const http_response_t *msg)> callback);

src/messenger.cpp

@@ -4,7 +4,6 @@
 #include <unistd.h>
 #include <fcntl.h>
 #include <sys/socket.h>
-#include <sys/stat.h>
 #include <sys/epoll.h>
 #include <netinet/tcp.h>
 #include <stdexcept>
@@ -545,7 +544,7 @@ json11::Json osd_messenger_t::read_config(const json11::Json & config)
     int done = 0;
     while (done < st.st_size)
     {
-        int r = read(fd, (uint8_t*)buf.data()+done, st.st_size-done);
+        int r = read(fd, (void*)buf.data()+done, st.st_size-done);
         if (r < 0)
         {
             fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno));

src/messenger.h

@@ -120,7 +120,6 @@ struct osd_messenger_t
 protected:
     int keepalive_timer_id = -1;
 
-    uint32_t receive_buffer_size = 0;
     int peer_connect_interval = 0;
     int peer_connect_timeout = 0;
     int osd_idle_timeout = 0;
@@ -142,6 +141,8 @@ protected:
     std::vector<std::function<void()>> set_immediate;
 
 public:
+    uint32_t receive_buffer_size = 0;
+
     timerfd_manager_t *tfd;
     ring_loop_t *ringloop;
     // osd_num_t is only for logging and asserts
@@ -172,10 +173,11 @@ public:
     bool connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg);
 #endif
 
+    void handle_peer_epoll(int peer_fd, int epoll_events);
+
 protected:
     void try_connect_peer(uint64_t osd_num);
     void try_connect_peer_addr(osd_num_t peer_osd, const char *peer_host, int peer_port);
-    void handle_peer_epoll(int peer_fd, int epoll_events);
     void handle_connect_epoll(int peer_fd);
     void on_connect_peer(osd_num_t peer_osd, int peer_fd);
     void check_peer_config(osd_client_t *cl);

src/msgr_op.h

@@ -141,7 +141,7 @@ struct osd_op_buf_list_t
             else
             {
                 iov.iov_len -= result;
-                iov.iov_base = (uint8_t*)iov.iov_base + result;
+                iov.iov_base += result;
                 break;
             }
         }

src/msgr_rdma.cpp

@@ -58,9 +58,7 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(const char *ib_devname, uint8_t
     msgr_rdma_context_t *ctx = new msgr_rdma_context_t();
     ctx->mtu = mtu;
 
-    timespec tv;
+    srand48(time(NULL));
-    clock_gettime(CLOCK_REALTIME, &tv);
-    srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
     dev_list = ibv_get_device_list(NULL);
     if (!dev_list)
     {
@@ -391,7 +389,7 @@ bool osd_messenger_t::try_send_rdma(osd_client_t *cl)
         uint32_t len = (uint32_t)(op_size+iov.iov_len-rc->send_buf_pos < rc->max_msg
             ? iov.iov_len-rc->send_buf_pos : rc->max_msg-op_size);
         sge[op_sge++] = {
-            .addr = (uintptr_t)((uint8_t*)iov.iov_base+rc->send_buf_pos),
+            .addr = (uintptr_t)(iov.iov_base+rc->send_buf_pos),
             .length = len,
             .lkey = rc->ctx->mr->lkey,
         };
@@ -521,7 +519,7 @@ void osd_messenger_t::handle_rdma_events()
                     }
                     if (cl->rdma_conn->send_buf_pos > 0)
                     {
-                        cl->send_list[0].iov_base = (uint8_t*)cl->send_list[0].iov_base + cl->rdma_conn->send_buf_pos;
+                        cl->send_list[0].iov_base += cl->rdma_conn->send_buf_pos;
                         cl->send_list[0].iov_len -= cl->rdma_conn->send_buf_pos;
                         cl->rdma_conn->send_buf_pos = 0;
                     }

src/msgr_receive.cpp

@@ -67,7 +67,7 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
         }
         return false;
     }
-    if (result <= 0 && result != -EAGAIN && result != -EINTR)
+    if (result <= 0 && result != -EAGAIN)
     {
         // this is a client socket, so don't panic on error. just disconnect it
         if (result != 0)
@@ -77,7 +77,7 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
         stop_client(cl->peer_fd);
         return false;
     }
-    if (result == -EAGAIN || result == -EINTR || result < cl->read_iov.iov_len)
+    if (result == -EAGAIN || result < cl->read_iov.iov_len)
     {
         cl->read_ready--;
         if (cl->read_ready > 0)
@@ -142,13 +142,13 @@ bool osd_messenger_t::handle_read_buffer(osd_client_t *cl, void *curbuf, int rem
                 memcpy(cur->iov_base, curbuf, remain);
                 cl->read_remaining -= remain;
                 cur->iov_len -= remain;
-                cur->iov_base = (uint8_t*)cur->iov_base + remain;
+                cur->iov_base += remain;
                 remain = 0;
             }
             else
             {
                 memcpy(cur->iov_base, curbuf, cur->iov_len);
-                curbuf = (uint8_t*)curbuf + cur->iov_len;
+                curbuf += cur->iov_len;
                 cl->read_remaining -= cur->iov_len;
                 remain -= cur->iov_len;
                 cur->iov_len = 0;
@@ -390,7 +390,7 @@ void osd_messenger_t::handle_reply_ready(osd_op_t *op)
         (tv_end.tv_sec - op->tv_begin.tv_sec)*1000000 +
         (tv_end.tv_nsec - op->tv_begin.tv_nsec)/1000
     );
-    set_immediate.push_back([op]()
+    set_immediate.push_back([this, op]()
     {
         // Copy lambda to be unaffected by `delete op`
         std::function<void(osd_op_t*)>(op->callback)(op);

src/msgr_send.cpp

@@ -224,7 +224,7 @@ void osd_messenger_t::handle_send(int result, osd_client_t *cl)
         }
         return;
     }
-    if (result < 0 && result != -EAGAIN && result != -EINTR)
+    if (result < 0 && result != -EAGAIN)
     {
         // this is a client socket, so don't panic. just disconnect it
         fprintf(stderr, "Client %d socket write error: %d (%s). Disconnecting client\n", cl->peer_fd, -result, strerror(-result));
@@ -250,7 +250,7 @@ void osd_messenger_t::handle_send(int result, osd_client_t *cl)
             else
             {
                 iov.iov_len -= result;
-                iov.iov_base = (uint8_t*)iov.iov_base + result;
+                iov.iov_base += result;
                 break;
             }
         }

src/nbd_proxy.cpp

@@ -30,9 +30,6 @@ protected:
     std::string image_name;
     uint64_t inode = 0;
     uint64_t device_size = 0;
-    int nbd_timeout = 30;
-    int nbd_max_devices = 64;
-    int nbd_max_part = 3;
     inode_watch_t *watch = NULL;
 
     ring_loop_t *ringloop = NULL;
@@ -120,18 +117,9 @@ public:
             "Vitastor NBD proxy\n"
             "(c) Vitaliy Filippov, 2020-2021 (VNPL-1.1)\n\n"
             "USAGE:\n"
-            "  %s map [OPTIONS] (--image <image> | --pool <pool> --inode <inode> --size <size in bytes>)\n"
+            "  %s map [--etcd_address <etcd_address>] (--image <image> | --pool <pool> --inode <inode> --size <size in bytes>)\n"
             "  %s unmap /dev/nbd0\n"
-            "  %s ls [--json]\n"
+            "  %s ls [--json]\n",
-            "OPTIONS:\n"
-            "  All usual Vitastor config options like --etcd_address <etcd_address> plus NBD-specific:\n"
-            "  --nbd_timeout 30\n"
-            "    timeout in seconds after which the kernel will stop the device\n"
-            "    you can set it to 0, but beware that you won't be able to stop the device at all\n"
-            "    if vitastor-nbd process dies\n"
-            "  --nbd_max_devices 64 --nbd_max_part 3\n"
-            "    options for the \"nbd\" kernel module when modprobing it (nbds_max and max_part).\n"
-            "    note that maximum allowed (nbds_max)*(1+max_part) is 256.\n",
             exe_name, exe_name, exe_name
         );
         exit(0);
@@ -186,18 +174,6 @@ public:
                 exit(1);
             }
         }
-        if (cfg["nbd_max_devices"].is_number() || cfg["nbd_max_devices"].is_string())
-        {
-            nbd_max_devices = cfg["nbd_max_devices"].uint64_value();
-        }
-        if (cfg["nbd_max_part"].is_number() || cfg["nbd_max_part"].is_string())
-        {
-            nbd_max_part = cfg["nbd_max_part"].uint64_value();
-        }
-        if (cfg["nbd_timeout"].is_number() || cfg["nbd_timeout"].is_string())
-        {
-            nbd_timeout = cfg["nbd_timeout"].uint64_value();
-        }
         // Create client
         ringloop = new ring_loop_t(512);
         epmgr = new epoll_manager_t(ringloop);
@@ -214,12 +190,6 @@ public:
             }
             watch = cli->st_cli.watch_inode(image_name);
             device_size = watch->cfg.size;
-            if (!watch->cfg.num || !device_size)
-            {
-                // Image does not exist
-                fprintf(stderr, "Image %s does not exist\n", image_name.c_str());
-                exit(1);
-            }
         }
         // Initialize NBD
         int sockfd[2];
@@ -234,7 +204,7 @@ public:
         bool bg = cfg["foreground"].is_null();
         if (!cfg["dev_num"].is_null())
         {
-            if (run_nbd(sockfd, cfg["dev_num"].int64_value(), device_size, NBD_FLAG_SEND_FLUSH, nbd_timeout, bg) < 0)
+            if (run_nbd(sockfd, cfg["dev_num"].int64_value(), device_size, NBD_FLAG_SEND_FLUSH, 30, bg) < 0)
             {
                 perror("run_nbd");
                 exit(1);
@@ -308,7 +278,7 @@ public:
         stop = false;
         cluster_op_t *close_sync = new cluster_op_t;
         close_sync->opcode = OSD_OP_SYNC;
-        close_sync->callback = [&stop](cluster_op_t *op)
+        close_sync->callback = [this, &stop](cluster_op_t *op)
         {
             stop = true;
             delete op;
@@ -331,10 +301,7 @@ public:
             return;
         }
         int r;
-        // Kernel built-in default is 16 devices with up to 16 partitions per device which is a big shit
+        if ((r = system("modprobe nbd")) != 0)
-        // 64 also isn't too high, but the possible maximum is nbds_max=256 max_part=0 and it won't reserve
-        // any block device minor numbers for partitions
-        if ((r = system(("modprobe nbd nbds_max="+std::to_string(nbd_max_devices)+" max_part="+std::to_string(nbd_max_part)).c_str())) != 0)
         {
             if (r < 0)
                 perror("Failed to load NBD kernel module");
@@ -586,7 +553,7 @@ protected:
             }
             else
             {
-                send_list[to_eat].iov_base = (uint8_t*)send_list[to_eat].iov_base + result;
+                send_list[to_eat].iov_base += result;
                 send_list[to_eat].iov_len -= result;
                 break;
             }
@@ -660,8 +627,8 @@ protected:
                 memcpy(cur_buf, b, inc);
                 cur_left -= inc;
                 result -= inc;
-                cur_buf = (uint8_t*)cur_buf + inc;
+                cur_buf += inc;
-                b = (uint8_t*)b + inc;
+                b += inc;
             }
             else
             {
@@ -700,7 +667,7 @@ protected:
                 op->offset = be64toh(cur_req.from);
                 op->len = be32toh(cur_req.len);
                 buf = malloc_or_die(sizeof(nbd_reply) + op->len);
-                op->iov.push_back((uint8_t*)buf + sizeof(nbd_reply), op->len);
+                op->iov.push_back(buf + sizeof(nbd_reply), op->len);
             }
             else if (req_type == NBD_CMD_FLUSH)
             {
@@ -728,7 +695,7 @@ protected:
             if (req_type == NBD_CMD_WRITE)
             {
                 cur_op = op;
-                cur_buf = (uint8_t*)buf + sizeof(nbd_reply);
+                cur_buf = buf + sizeof(nbd_reply);
                 cur_left = op->len;
                 read_state = CL_READ_DATA;
             }
@@ -767,6 +734,5 @@ int main(int narg, const char *args[])
     exe_name = args[0];
     nbd_proxy *p = new nbd_proxy();
     p->exec(nbd_proxy::parse_args(narg, args));
-    delete p;
     return 0;
 }

src/osd.cpp

@@ -331,8 +331,8 @@ void osd_t::exec_op(osd_op_t *cur_op)
 
 void osd_t::reset_stats()
 {
-    msgr.stats = {};
+    msgr.stats = { 0 };
-    prev_stats = {};
+    prev_stats = { 0 };
     memset(recovery_stat_count, 0, sizeof(recovery_stat_count));
     memset(recovery_stat_bytes, 0, sizeof(recovery_stat_bytes));
 }
@@ -447,7 +447,7 @@ void osd_t::print_slow()
                 {
                     for (uint64_t i = 0; i < op->req.sec_stab.len; i += sizeof(obj_ver_id))
                     {
-                        obj_ver_id *ov = (obj_ver_id*)((uint8_t*)op->buf + i);
+                        obj_ver_id *ov = (obj_ver_id*)(op->buf + i);
                         bufprintf(i == 0 ? " %lx:%lx v%lu" : ", %lx:%lx v%lu", ov->oid.inode, ov->oid.stripe, ov->version);
                     }
                 }

src/osd.h

@@ -102,7 +102,7 @@ class osd_t
     bool no_rebalance = false;
     bool no_recovery = false;
     std::string bind_address;
-    int bind_port, listen_backlog = 128;
+    int bind_port, listen_backlog;
     // FIXME: Implement client queue depth limit
     int client_queue_depth = 128;
     bool allow_test_ops = false;
@@ -166,8 +166,8 @@ class osd_t
     osd_op_stats_t prev_stats;
     std::map<uint64_t, inode_stats_t> inode_stats;
     const char* recovery_stat_names[2] = { "degraded", "misplaced" };
-    uint64_t recovery_stat_count[2][2] = {};
+    uint64_t recovery_stat_count[2][2] = { 0 };
-    uint64_t recovery_stat_bytes[2][2] = {};
+    uint64_t recovery_stat_bytes[2][2] = { 0 };
 
     // cluster connection
     void parse_config(const json11::Json & config);

src/osd_cluster.cpp

@@ -6,7 +6,6 @@
 #include "etcd_state_client.h"
 #include "http_client.h"
 #include "osd_rmw.h"
-#include "addr_util.h"
 
 // Startup sequence:
 //   Start etcd watcher -> Load global OSD configuration -> Bind socket -> Acquire lease -> Report&lock OSD state
@@ -277,14 +276,14 @@ void osd_t::report_statistics()
             } }
         });
     }
-    st_cli.etcd_txn_slow(json11::Json::object { { "success", txn } }, [this](std::string err, json11::Json res)
+    st_cli.etcd_txn(json11::Json::object { { "success", txn } }, ETCD_SLOW_TIMEOUT, [this](std::string err, json11::Json res)
     {
         etcd_reporting_stats = false;
         if (err != "")
         {
             printf("[OSD %lu] Error reporting state to etcd: %s\n", this->osd_num, err.c_str());
             // Retry indefinitely
-            tfd->set_timer(st_cli.etcd_slow_timeout, false, [this](int timer_id)
+            tfd->set_timer(ETCD_SLOW_TIMEOUT, false, [this](int timer_id)
             {
                 report_statistics();
             });
@@ -355,13 +354,13 @@ void osd_t::acquire_lease()
 {
     // Maximum lease TTL is (report interval) + retries * (timeout + repeat interval)
     st_cli.etcd_call("/lease/grant", json11::Json::object {
-        { "TTL", etcd_report_interval+(st_cli.max_etcd_attempts*(2*st_cli.etcd_quick_timeout)+999)/1000 }
+        { "TTL", etcd_report_interval+(MAX_ETCD_ATTEMPTS*(2*ETCD_QUICK_TIMEOUT)+999)/1000 }
-    }, st_cli.etcd_quick_timeout, 0, 0, [this](std::string err, json11::Json data)
+    }, ETCD_QUICK_TIMEOUT, [this](std::string err, json11::Json data)
     {
         if (err != "" || data["ID"].string_value() == "")
         {
-            printf("Error acquiring a lease from etcd: %s, retrying\n", err.c_str());
+            printf("Error acquiring a lease from etcd: %s\n", err.c_str());
-            tfd->set_timer(st_cli.etcd_quick_timeout, false, [this](int timer_id)
+            tfd->set_timer(ETCD_QUICK_TIMEOUT, false, [this](int timer_id)
             {
                 acquire_lease();
             });
@@ -408,19 +407,19 @@ void osd_t::create_osd_state()
                 } }
             },
         } },
-    }, st_cli.etcd_quick_timeout, 0, 0, [this](std::string err, json11::Json data)
+    }, ETCD_QUICK_TIMEOUT, [this](std::string err, json11::Json data)
     {
         if (err != "")
         {
             etcd_failed_attempts++;
             printf("Error creating OSD state key: %s\n", err.c_str());
-            if (etcd_failed_attempts > st_cli.max_etcd_attempts)
+            if (etcd_failed_attempts > MAX_ETCD_ATTEMPTS)
             {
                 // Die
                 throw std::runtime_error("Cluster connection failed");
             }
             // Retry
-            tfd->set_timer(st_cli.etcd_quick_timeout, false, [this](int timer_id)
+            tfd->set_timer(ETCD_QUICK_TIMEOUT, false, [this](int timer_id)
             {
                 create_osd_state();
             });
@@ -452,7 +451,7 @@ void osd_t::renew_lease()
 {
     st_cli.etcd_call("/lease/keepalive", json11::Json::object {
         { "ID", etcd_lease_id }
-    }, st_cli.etcd_quick_timeout, 0, 0, [this](std::string err, json11::Json data)
+    }, ETCD_QUICK_TIMEOUT, [this](std::string err, json11::Json data)
     {
         if (err == "" && data["result"]["TTL"].string_value() == "")
         {
@@ -463,13 +462,13 @@ void osd_t::renew_lease()
         {
             etcd_failed_attempts++;
             printf("Error renewing etcd lease: %s\n", err.c_str());
-            if (etcd_failed_attempts > st_cli.max_etcd_attempts)
+            if (etcd_failed_attempts > MAX_ETCD_ATTEMPTS)
             {
                 // Die
                 throw std::runtime_error("Cluster connection failed");
             }
             // Retry
-            tfd->set_timer(st_cli.etcd_quick_timeout, false, [this](int timer_id)
+            tfd->set_timer(ETCD_QUICK_TIMEOUT, false, [this](int timer_id)
             {
                 renew_lease();
             });
@@ -488,7 +487,7 @@ void osd_t::force_stop(int exitcode)
     {
         st_cli.etcd_call("/kv/lease/revoke", json11::Json::object {
             { "ID", etcd_lease_id }
-        }, st_cli.etcd_quick_timeout, st_cli.max_etcd_attempts, 0, [this, exitcode](std::string err, json11::Json data)
+        }, ETCD_QUICK_TIMEOUT, [this, exitcode](std::string err, json11::Json data)
         {
             if (err != "")
             {
@@ -826,7 +825,7 @@ void osd_t::report_pg_states()
     etcd_reporting_pg_state = true;
     st_cli.etcd_txn(json11::Json::object {
         { "compare", checks }, { "success", success }, { "failure", failure }
-    }, st_cli.etcd_quick_timeout, 0, 0, [this, reporting_pgs](std::string err, json11::Json data)
+    }, ETCD_QUICK_TIMEOUT, [this, reporting_pgs](std::string err, json11::Json data)
     {
         etcd_reporting_pg_state = false;
         if (!data["succeeded"].bool_value())
@@ -858,13 +857,10 @@ void osd_t::report_pg_states()
                         if (null_byte == 0)
                         {
                             auto pg_it = pgs.find({ .pool_id = pool_id, .pg_num = pg_num });
-                            if (pg_it != pgs.end() && pg_it->second.state != PG_OFFLINE && pg_it->second.state != PG_STARTING &&
+                            if (pg_it != pgs.end() && pg_it->second.state != PG_OFFLINE && pg_it->second.state != PG_STARTING)
-                                kv.value["primary"].uint64_value() != 0 &&
-                                kv.value["primary"].uint64_value() != this->osd_num)
                             {
-                                // PG is somehow captured by another OSD
+                                // Live PG state update failed
-                                printf("BUG: OSD %lu captured our PG %u/%u. Race condition detected, exiting\n",
+                                printf("Failed to report state of pool %u PG %u which is live. Race condition detected, exiting\n", pool_id, pg_num);
-                                    kv.value["primary"].uint64_value(), pool_id, pg_num);
                                 force_stop(1);
                                 return;
                             }

src/osd_peering.cpp

@@ -27,9 +27,9 @@ void osd_t::handle_peers()
                     misplaced_objects += p.second.misplaced_objects.size();
                     // FIXME: degraded objects may currently include misplaced, too! Report them separately?
                     degraded_objects += p.second.degraded_objects.size();
-                    if (p.second.state & PG_HAS_UNCLEAN)
+                    if ((p.second.state & (PG_ACTIVE | PG_HAS_UNCLEAN)) == (PG_ACTIVE | PG_HAS_UNCLEAN))
                         peering_state = peering_state | OSD_FLUSHING_PGS;
-                    else if (p.second.state & PG_HAS_DEGRADED)
+                    else if (p.second.state & PG_ACTIVE)
                         peering_state = peering_state | OSD_RECOVERING;
                 }
                 else
@@ -176,17 +176,6 @@ void osd_t::start_pg_peering(pg_t & pg)
             msgr.stop_client(peer_fd);
         }
     }
-    // Try to connect with current peers if they're up, but we don't have connections to them
-    // Otherwise we may erroneously decide that the pg is incomplete :-)
-    for (auto pg_osd: pg.all_peers)
-    {
-        if (pg_osd != this->osd_num &&
-            msgr.osd_peer_fds.find(pg_osd) == msgr.osd_peer_fds.end() &&
-            msgr.wanted_peers.find(pg_osd) == msgr.wanted_peers.end())
-        {
-            msgr.connect_peer(pg_osd, st_cli.peer_states[pg_osd]);
-        }
-    }
     // Calculate current write OSD set
     pg.pg_cursize = 0;
     pg.cur_set.resize(pg.target_set.size());
@@ -218,6 +207,10 @@ void osd_t::start_pg_peering(pg_t & pg)
         {
             cur_peers.insert(pg_osd);
         }
+        else if (msgr.wanted_peers.find(pg_osd) == msgr.wanted_peers.end())
+        {
+            msgr.connect_peer(pg_osd, st_cli.peer_states[pg_osd]);
+        }
     }
     if (pg.target_history.size())
     {

src/osd_primary.cpp

@@ -96,11 +96,11 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
             (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 0 : pg_it->second.pg_size)
         )
     );
-    void *data_buf = (uint8_t*)op_data + sizeof(osd_primary_op_data_t);
+    void *data_buf = ((void*)op_data) + sizeof(osd_primary_op_data_t);
     op_data->pg_num = pg_num;
     op_data->oid = oid;
     op_data->stripes = (osd_rmw_stripe_t*)data_buf;
-    data_buf = (uint8_t*)data_buf + sizeof(osd_rmw_stripe_t) * stripe_count;
+    data_buf += sizeof(osd_rmw_stripe_t) * stripe_count;
     op_data->scheme = pool_cfg.scheme;
     op_data->pg_data_size = pg_data_size;
     op_data->pg_size = pg_it->second.pg_size;
@@ -110,17 +110,17 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op)
     for (int i = 0; i < stripe_count; i++)
     {
         op_data->stripes[i].bmp_buf = data_buf;
-        data_buf = (uint8_t*)data_buf + clean_entry_bitmap_size;
+        data_buf += clean_entry_bitmap_size;
     }
     op_data->chain_size = chain_size;
     if (chain_size > 0)
     {
         op_data->read_chain = (inode_t*)data_buf;
-        data_buf = (uint8_t*)data_buf + sizeof(inode_t) * chain_size;
+        data_buf += sizeof(inode_t) * chain_size;
         op_data->snapshot_bitmaps = data_buf;
-        data_buf = (uint8_t*)data_buf + chain_size * stripe_count * clean_entry_bitmap_size;
+        data_buf += chain_size * stripe_count * clean_entry_bitmap_size;
         op_data->missing_flags = (uint8_t*)data_buf;
-        data_buf = (uint8_t*)data_buf + chain_size * (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 0 : pg_it->second.pg_size);
+        data_buf += chain_size * (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 0 : pg_it->second.pg_size);
         // Copy chain
         int chain_num = 0;
         op_data->read_chain[chain_num++] = cur_op->req.rw.inode;
@@ -248,7 +248,7 @@ resume_2:
             {
                 // Send buffer in parts to avoid copying
                 cur_op->iov.push_back(
-                    (uint8_t*)stripes[role].read_buf + (stripes[role].req_start - stripes[role].read_start),
+                    stripes[role].read_buf + (stripes[role].req_start - stripes[role].read_start),
                     stripes[role].req_end - stripes[role].req_start
                 );
             }

src/osd_primary_chain.cpp

@@ -66,7 +66,7 @@ int osd_t::read_bitmaps(osd_op_t *cur_op, pg_t & pg, int base_state)
             auto read_version = (vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX);
             // Read bitmap synchronously from the local database
             bs->read_bitmap(
-                cur_oid, read_version, (uint8_t*)op_data->snapshot_bitmaps + chain_num*clean_entry_bitmap_size,
+                cur_oid, read_version, op_data->snapshot_bitmaps + chain_num*clean_entry_bitmap_size,
                 !chain_num ? &cur_op->reply.rw.version : NULL
             );
         }
@@ -96,15 +96,12 @@ resume_1:
                 {
                     if (op_data->missing_flags[chain_num*pg.pg_size + i])
                     {
-                        osd_rmw_stripe_t local_stripes[pg.pg_size];
+                        osd_rmw_stripe_t local_stripes[pg.pg_size] = { 0 };
                         for (i = 0; i < pg.pg_size; i++)
                         {
-                            local_stripes[i] = (osd_rmw_stripe_t){
+                            local_stripes[i].missing = op_data->missing_flags[chain_num*pg.pg_size + i] && true;
-                                .bmp_buf = (uint8_t*)op_data->snapshot_bitmaps + (chain_num*pg.pg_size + i)*clean_entry_bitmap_size,
+                            local_stripes[i].bmp_buf = op_data->snapshot_bitmaps + (chain_num*pg.pg_size + i)*clean_entry_bitmap_size;
-                                .read_start = 1,
+                            local_stripes[i].read_start = local_stripes[i].read_end = 1;
-                                .read_end = 1,
-                                .missing = op_data->missing_flags[chain_num*pg.pg_size + i] && true,
-                            };
                         }
                         if (pg.scheme == POOL_SCHEME_XOR)
                         {
@@ -149,7 +146,7 @@ int osd_t::collect_bitmap_requests(osd_op_t *cur_op, pg_t & pg, std::vector<bitm
                 .osd_num = read_target,
                 .oid = cur_oid,
                 .version = target_version,
-                .bmp_buf = (uint8_t*)op_data->snapshot_bitmaps + chain_num*clean_entry_bitmap_size,
+                .bmp_buf = op_data->snapshot_bitmaps + chain_num*clean_entry_bitmap_size,
             });
         }
         else
@@ -188,7 +185,7 @@ int osd_t::collect_bitmap_requests(osd_op_t *cur_op, pg_t & pg, std::vector<bitm
                             .stripe = cur_oid.stripe | i,
                         },
                         .version = target_version,
-                        .bmp_buf = (uint8_t*)op_data->snapshot_bitmaps + (chain_num*pg.pg_size + i)*clean_entry_bitmap_size,
+                        .bmp_buf = op_data->snapshot_bitmaps + (chain_num*pg.pg_size + i)*clean_entry_bitmap_size,
                     });
                     found++;
                 }
@@ -207,7 +204,6 @@ int osd_t::submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg)
     std::vector<bitmap_request_t> *bitmap_requests = new std::vector<bitmap_request_t>();
     if (collect_bitmap_requests(cur_op, pg, *bitmap_requests) < 0)
     {
-        delete bitmap_requests;
         return -1;
     }
     op_data->n_subops = 0;
@@ -270,15 +266,15 @@ int osd_t::submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg)
                     int requested_count = subop->req.sec_read_bmp.len / sizeof(obj_ver_id);
                     if (subop->reply.hdr.retval == requested_count * (8 + clean_entry_bitmap_size))
                     {
-                        void *cur_buf = (uint8_t*)subop->buf + 8;
+                        void *cur_buf = subop->buf + 8;
                         for (int j = prev; j <= i; j++)
                         {
                             memcpy((*bitmap_requests)[j].bmp_buf, cur_buf, clean_entry_bitmap_size);
                             if ((*bitmap_requests)[j].oid.inode == cur_op->req.rw.inode)
                             {
-                                memcpy(&cur_op->reply.rw.version, (uint8_t*)cur_buf-8, 8);
+                                memcpy(&cur_op->reply.rw.version, cur_buf-8, 8);
                             }
-                            cur_buf = (uint8_t*)cur_buf + 8 + clean_entry_bitmap_size;
+                            cur_buf += 8 + clean_entry_bitmap_size;
                         }
                     }
                     if ((cur_op->op_data->errors + cur_op->op_data->done + 1) >= cur_op->op_data->n_subops)
@@ -367,7 +363,7 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
         + sizeof(osd_rmw_stripe_t) * stripe_count * op_data->chain_size
     );
     osd_rmw_stripe_t *chain_stripes = (osd_rmw_stripe_t*)(
-        (uint8_t*)op_data->chain_reads + sizeof(osd_chain_read_t) * op_data->chain_read_count
+        ((void*)op_data->chain_reads) + sizeof(osd_chain_read_t) * op_data->chain_read_count
     );
     // Now process each subrequest as a separate read, including reconstruction if needed
     // Prepare reads
@@ -429,8 +425,8 @@ int osd_t::submit_chained_read_requests(pg_t & pg, osd_op_t *cur_op)
             if (stripes[role].read_end > 0)
             {
                 stripes[role].read_buf = cur_buf;
-                stripes[role].bmp_buf = (uint8_t*)op_data->snapshot_bitmaps + (chain_reads[cri].chain_pos*stripe_count + role)*clean_entry_bitmap_size;
+                stripes[role].bmp_buf = op_data->snapshot_bitmaps + (chain_reads[cri].chain_pos*stripe_count + role)*clean_entry_bitmap_size;
-                cur_buf = (uint8_t*)cur_buf + stripes[role].read_end - stripes[role].read_start;
+                cur_buf += stripes[role].read_end - stripes[role].read_start;
             }
         }
     }
@@ -478,7 +474,7 @@ void osd_t::send_chained_read_results(pg_t & pg, osd_op_t *cur_op)
     osd_primary_op_data_t *op_data = cur_op->op_data;
     int stripe_count = (pg.scheme == POOL_SCHEME_REPLICATED ? 1 : pg.pg_size);
     osd_rmw_stripe_t *chain_stripes = (osd_rmw_stripe_t*)(
-        (uint8_t*)op_data->chain_reads + sizeof(osd_chain_read_t) * op_data->chain_read_count
+        ((void*)op_data->chain_reads) + sizeof(osd_chain_read_t) * op_data->chain_read_count
     );
     // Reconstruct parts if needed
     if (op_data->degraded)
@@ -548,7 +544,7 @@ void osd_t::send_chained_read_results(pg_t & pg, osd_op_t *cur_op)
                             role_end = bs_block_size;
                         assert(stripes[role].read_buf);
                         cur_op->iov.push_back(
-                            (uint8_t*)stripes[role].read_buf + (role_start - stripes[role].read_start),
+                            stripes[role].read_buf + (role_start - stripes[role].read_start),
                             role_end - role_start
                         );
                         sent += role_end - role_start;

src/osd_primary_sync.cpp

@@ -86,7 +86,7 @@ resume_2:
             sizeof(obj_ver_osd_t)*this->copies_to_delete_after_sync_count
         );
         op_data->dirty_pgs = (pool_pg_num_t*)dirty_buf;
-        op_data->dirty_osds = (osd_num_t*)((uint8_t*)dirty_buf + sizeof(pool_pg_num_t)*dirty_pgs.size());
+        op_data->dirty_osds = (osd_num_t*)(dirty_buf + sizeof(pool_pg_num_t)*dirty_pgs.size());
         op_data->dirty_pg_count = dirty_pgs.size();
         op_data->dirty_osd_count = dirty_osds.size();
         if (this->copies_to_delete_after_sync_count)

src/osd_primary_write.cpp

@@ -113,7 +113,7 @@ resume_3:
             op_data->stripes[0].write_end != bs_block_size))
         {
             memcpy(
-                (uint8_t*)op_data->stripes[0].read_buf + op_data->stripes[0].req_start,
+                op_data->stripes[0].read_buf + op_data->stripes[0].req_start,
                 op_data->stripes[0].write_buf,
                 op_data->stripes[0].req_end - op_data->stripes[0].req_start
             );

src/osd_rmw.cpp

@@ -103,8 +103,8 @@ void reconstruct_stripes_xor(osd_rmw_stripe_t *stripes, int pg_size, uint32_t bi
                         assert(stripes[role].read_start >= stripes[prev].read_start &&
                             stripes[role].read_start >= stripes[other].read_start);
                         memxor(
-                            (uint8_t*)stripes[prev].read_buf + (stripes[role].read_start - stripes[prev].read_start),
+                            stripes[prev].read_buf + (stripes[role].read_start - stripes[prev].read_start),
-                            (uint8_t*)stripes[other].read_buf + (stripes[role].read_start - stripes[other].read_start),
+                            stripes[other].read_buf + (stripes[role].read_start - stripes[other].read_start),
                             stripes[role].read_buf, stripes[role].read_end - stripes[role].read_start
                         );
                         memxor(stripes[prev].bmp_buf, stripes[other].bmp_buf, stripes[role].bmp_buf, bitmap_size);
@@ -115,7 +115,7 @@ void reconstruct_stripes_xor(osd_rmw_stripe_t *stripes, int pg_size, uint32_t bi
                         assert(stripes[role].read_start >= stripes[other].read_start);
                         memxor(
                             stripes[role].read_buf,
-                            (uint8_t*)stripes[other].read_buf + (stripes[role].read_start - stripes[other].read_start),
+                            stripes[other].read_buf + (stripes[role].read_start - stripes[other].read_start),
                             stripes[role].read_buf, stripes[role].read_end - stripes[role].read_start
                         );
                         memxor(stripes[role].bmp_buf, stripes[other].bmp_buf, stripes[role].bmp_buf, bitmap_size);
@@ -202,9 +202,10 @@ reed_sol_matrix_t* get_jerasure_matrix(int pg_size, int pg_minsize)
 int* get_jerasure_decoding_matrix(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize)
 {
     int edd = 0;
-    int erased[pg_size];
+    int erased[pg_size] = { 0 };
     for (int i = 0; i < pg_size; i++)
-        erased[i] = (stripes[i].read_end == 0 || stripes[i].missing ? 1 : 0);
+        if (stripes[i].read_end == 0 || stripes[i].missing)
+            erased[i] = 1;
     for (int i = 0; i < pg_minsize; i++)
         if (stripes[i].read_end != 0 && stripes[i].missing)
             edd++;
@@ -240,9 +241,7 @@ void reconstruct_stripes_jerasure(osd_rmw_stripe_t *stripes, int pg_size, int pg
         return;
     }
     int *decoding_matrix = dm_ids + pg_minsize;
-    char *data_ptrs[pg_size];
+    char *data_ptrs[pg_size] = { 0 };
-    for (int role = 0; role < pg_size; role++)
-        data_ptrs[role] = NULL;
     for (int role = 0; role < pg_minsize; role++)
     {
         if (stripes[role].read_end != 0 && stripes[role].missing)
@@ -255,7 +254,7 @@ void reconstruct_stripes_jerasure(osd_rmw_stripe_t *stripes, int pg_size, int pg
                     {
                         assert(stripes[other].read_start <= stripes[role].read_start);
                         assert(stripes[other].read_end >= stripes[role].read_end);
-                        data_ptrs[other] = (char*)stripes[other].read_buf + (stripes[role].read_start - stripes[other].read_start);
+                        data_ptrs[other] = (char*)(stripes[other].read_buf + (stripes[role].read_start - stripes[other].read_start));
                     }
                 }
                 data_ptrs[role] = (char*)stripes[role].read_buf;
@@ -331,7 +330,7 @@ void* alloc_read_buffer(osd_rmw_stripe_t *stripes, int read_pg_size, uint64_t ad
     {
         if (stripes[role].read_end != 0)
         {
-            stripes[role].read_buf = (uint8_t*)buf + buf_pos;
+            stripes[role].read_buf = buf + buf_pos;
             buf_pos += stripes[role].read_end - stripes[role].read_start;
         }
     }
@@ -447,12 +446,12 @@ void* calc_rmw(void *request_buf, osd_rmw_stripe_t *stripes, uint64_t *read_osd_
     {
         if (stripes[role].req_end != 0)
         {
-            stripes[role].write_buf = (uint8_t*)request_buf + in_pos;
+            stripes[role].write_buf = request_buf + in_pos;
             in_pos += stripes[role].req_end - stripes[role].req_start;
         }
         else if (role >= pg_minsize && write_osd_set[role] != 0 && end != 0)
         {
-            stripes[role].write_buf = (uint8_t*)rmw_buf + buf_pos;
+            stripes[role].write_buf = rmw_buf + buf_pos;
             buf_pos += end - start;
         }
     }
@@ -477,13 +476,13 @@ static void get_old_new_buffers(osd_rmw_stripe_t & stripe, uint32_t wr_start, ui
     if (ne && (!oe || ns <= os))
     {
         // NEW or NEW->OLD
-        bufs[nbufs++] = { .buf = (uint8_t*)stripe.write_buf + ns - stripe.req_start, .len = ne-ns };
+        bufs[nbufs++] = { .buf = stripe.write_buf + ns - stripe.req_start, .len = ne-ns };
         if (os < ne)
             os = ne;
         if (oe > os)
         {
             // NEW->OLD
-            bufs[nbufs++] = { .buf = (uint8_t*)stripe.read_buf + os - stripe.read_start, .len = oe-os };
+            bufs[nbufs++] = { .buf = stripe.read_buf + os - stripe.read_start, .len = oe-os };
         }
     }
     else if (oe)
@@ -492,18 +491,18 @@ static void get_old_new_buffers(osd_rmw_stripe_t & stripe, uint32_t wr_start, ui
         if (ne)
         {
             // OLD->NEW or OLD->NEW->OLD
-            bufs[nbufs++] = { .buf = (uint8_t*)stripe.read_buf + os - stripe.read_start, .len = ns-os };
+            bufs[nbufs++] = { .buf = stripe.read_buf + os - stripe.read_start, .len = ns-os };
-            bufs[nbufs++] = { .buf = (uint8_t*)stripe.write_buf + ns - stripe.req_start, .len = ne-ns };
+            bufs[nbufs++] = { .buf = stripe.write_buf + ns - stripe.req_start, .len = ne-ns };
             if (oe > ne)
             {
                 // OLD->NEW->OLD
-                bufs[nbufs++] = { .buf = (uint8_t*)stripe.read_buf + ne - stripe.read_start, .len = oe-ne };
+                bufs[nbufs++] = { .buf = stripe.read_buf + ne - stripe.read_start, .len = oe-ne };
             }
         }
         else
         {
             // OLD
-            bufs[nbufs++] = { .buf = (uint8_t*)stripe.read_buf + os - stripe.read_start, .len = oe-os };
+            bufs[nbufs++] = { .buf = stripe.read_buf + os - stripe.read_start, .len = oe-os };
         }
     }
 }
@@ -518,7 +517,7 @@ static void xor_multiple_buffers(buf_len_t *xor1, int n1, buf_len_t *xor2, int n
     {
         // We know for sure that ranges overlap
         uint32_t end = std::min(end1, end2);
-        memxor((uint8_t*)xor1[i1].buf + pos-start1, (uint8_t*)xor2[i2].buf + pos-start2, (uint8_t*)dest+pos, end-pos);
+        memxor(xor1[i1].buf + pos-start1, xor2[i2].buf + pos-start2, dest+pos, end-pos);
         pos = end;
         if (pos >= end1)
         {
@@ -587,7 +586,7 @@ static void calc_rmw_parity_copy_mod(osd_rmw_stripe_t *stripes, int pg_size, int
             {
                 // Copy modified chunk into the read buffer to write it back
                 memcpy(
-                    (uint8_t*)stripes[role].read_buf + stripes[role].req_start,
+                    stripes[role].read_buf + stripes[role].req_start,
                     stripes[role].write_buf,
                     stripes[role].req_end - stripes[role].req_start
                 );
@@ -610,7 +609,7 @@ static void calc_rmw_parity_copy_parity(osd_rmw_stripe_t *stripes, int pg_size,
             {
                 // Copy new parity into the read buffer to write it back
                 memcpy(
-                    (uint8_t*)stripes[role].read_buf + start,
+                    stripes[role].read_buf + start,
                     stripes[role].write_buf,
                     end - start
                 );
@@ -699,15 +698,9 @@ void calc_rmw_parity_jerasure(osd_rmw_stripe_t *stripes, int pg_size, int pg_min
         {
             // Calculate new coding chunks
             buf_len_t bufs[pg_size][3];
-            int nbuf[pg_size], curbuf[pg_size];
+            int nbuf[pg_size] = { 0 }, curbuf[pg_size] = { 0 };
             uint32_t positions[pg_size];
-            void *data_ptrs[pg_size];
+            void *data_ptrs[pg_size] = { 0 };
-            for (int i = 0; i < pg_size; i++)
-            {
-                data_ptrs[i] = NULL;
-                nbuf[i] = 0;
-                curbuf[i] = 0;
-            }
             for (int i = 0; i < pg_minsize; i++)
             {
                 get_old_new_buffers(stripes[i], start, end, bufs[i], nbuf[i]);
@@ -726,7 +719,7 @@ void calc_rmw_parity_jerasure(osd_rmw_stripe_t *stripes, int pg_size, int pg_min
                 {
                     assert(curbuf[i] < nbuf[i]);
                     assert(bufs[i][curbuf[i]].buf);
-                    data_ptrs[i] = (uint8_t*)bufs[i][curbuf[i]].buf + pos-positions[i];
+                    data_ptrs[i] = bufs[i][curbuf[i]].buf + pos-positions[i];
                     uint32_t this_end = bufs[i][curbuf[i]].len + positions[i];
                     if (next_end > this_end)
                         next_end = this_end;

src/osd_rmw_test.cpp

@@ -90,7 +90,7 @@ void dump_stripes(osd_rmw_stripe_t *stripes, int pg_size)
 void test1()
 {
     osd_num_t osd_set[3] = { 1, 0, 3 };
-    osd_rmw_stripe_t stripes[3] = {};
+    osd_rmw_stripe_t stripes[3] = { 0 };
     // Test 1.1
     split_stripes(2, 128*1024, 128*1024-4096, 8192, stripes);
     assert(stripes[0].req_start == 128*1024-4096 && stripes[0].req_end == 128*1024);
@@ -129,7 +129,7 @@ void test4()
     const uint32_t bmp = 4;
     unsigned bitmaps[3] = { 0 };
     osd_num_t osd_set[3] = { 1, 0, 3 };
-    osd_rmw_stripe_t stripes[3] = {};
+    osd_rmw_stripe_t stripes[3] = { 0 };
     // Test 4.1
     split_stripes(2, 128*1024, 128*1024-4096, 8192, stripes);
     for (int i = 0; i < 3; i++)
@@ -142,11 +142,11 @@ void test4()
     assert(stripes[0].write_start == 128*1024-4096 && stripes[0].write_end == 128*1024);
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
-    assert(stripes[0].read_buf == (uint8_t*)rmw_buf+128*1024);
+    assert(stripes[0].read_buf == rmw_buf+128*1024);
-    assert(stripes[1].read_buf == (uint8_t*)rmw_buf+128*1024*2);
+    assert(stripes[1].read_buf == rmw_buf+128*1024*2);
-    assert(stripes[2].read_buf == (uint8_t*)rmw_buf+128*1024*3-4096);
+    assert(stripes[2].read_buf == rmw_buf+128*1024*3-4096);
     assert(stripes[0].write_buf == write_buf);
-    assert(stripes[1].write_buf == (uint8_t*)write_buf+4096);
+    assert(stripes[1].write_buf == write_buf+4096);
     assert(stripes[2].write_buf == rmw_buf);
     // Test 4.2
     set_pattern(write_buf, 8192, PATTERN0);
@@ -183,7 +183,7 @@ void test4()
 void test5()
 {
     osd_num_t osd_set[3] = { 1, 0, 3 };
-    osd_rmw_stripe_t stripes[3] = {};
+    osd_rmw_stripe_t stripes[3] = { 0 };
     // Test 5.1
     split_stripes(2, 128*1024, 0, 64*1024*3, stripes);
     assert(stripes[0].req_start == 0 && stripes[0].req_end == 128*1024);
@@ -198,11 +198,11 @@ void test5()
     assert(stripes[0].write_start == 0 && stripes[0].write_end == 128*1024);
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 64*1024);
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
-    assert(stripes[0].read_buf == (uint8_t*)rmw_buf+128*1024);
+    assert(stripes[0].read_buf == rmw_buf+128*1024);
-    assert(stripes[1].read_buf == (uint8_t*)rmw_buf+64*3*1024);
+    assert(stripes[1].read_buf == rmw_buf+64*3*1024);
-    assert(stripes[2].read_buf == (uint8_t*)rmw_buf+64*4*1024);
+    assert(stripes[2].read_buf == rmw_buf+64*4*1024);
     assert(stripes[0].write_buf == write_buf);
-    assert(stripes[1].write_buf == (uint8_t*)write_buf+128*1024);
+    assert(stripes[1].write_buf == write_buf+128*1024);
     assert(stripes[2].write_buf == rmw_buf);
     free(rmw_buf);
     free(write_buf);
@@ -224,7 +224,7 @@ void test5()
 void test6()
 {
     osd_num_t osd_set[3] = { 1, 2, 3 };
-    osd_rmw_stripe_t stripes[3] = {};
+    osd_rmw_stripe_t stripes[3] = { 0 };
     // Test 6.1
     split_stripes(2, 128*1024, 0, 64*1024*3, stripes);
     void *write_buf = malloc(64*1024*3);
@@ -236,10 +236,10 @@ void test6()
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 64*1024);
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
     assert(stripes[0].read_buf == 0);
-    assert(stripes[1].read_buf == (uint8_t*)rmw_buf+128*1024);
+    assert(stripes[1].read_buf == rmw_buf+128*1024);
     assert(stripes[2].read_buf == 0);
     assert(stripes[0].write_buf == write_buf);
-    assert(stripes[1].write_buf == (uint8_t*)write_buf+128*1024);
+    assert(stripes[1].write_buf == write_buf+128*1024);
     assert(stripes[2].write_buf == rmw_buf);
     free(rmw_buf);
     free(write_buf);
@@ -267,7 +267,7 @@ void test7()
 {
     osd_num_t osd_set[3] = { 1, 0, 3 };
     osd_num_t write_osd_set[3] = { 1, 2, 3 };
-    osd_rmw_stripe_t stripes[3] = {};
+    osd_rmw_stripe_t stripes[3] = { 0 };
     // Test 7.1
     split_stripes(2, 128*1024, 128*1024-4096, 8192, stripes);
     void *write_buf = malloc(8192);
@@ -278,11 +278,11 @@ void test7()
     assert(stripes[0].write_start == 128*1024-4096 && stripes[0].write_end == 128*1024);
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
-    assert(stripes[0].read_buf == (uint8_t*)rmw_buf+128*1024);
+    assert(stripes[0].read_buf == rmw_buf+128*1024);
-    assert(stripes[1].read_buf == (uint8_t*)rmw_buf+128*1024*2);
+    assert(stripes[1].read_buf == rmw_buf+128*1024*2);
-    assert(stripes[2].read_buf == (uint8_t*)rmw_buf+128*1024*3);
+    assert(stripes[2].read_buf == rmw_buf+128*1024*3);
     assert(stripes[0].write_buf == write_buf);
-    assert(stripes[1].write_buf == (uint8_t*)write_buf+4096);
+    assert(stripes[1].write_buf == write_buf+4096);
     assert(stripes[2].write_buf == rmw_buf);
     // Test 7.2
     set_pattern(write_buf, 8192, PATTERN0);
@@ -320,7 +320,7 @@ void test8()
 {
     osd_num_t osd_set[3] = { 0, 2, 3 };
     osd_num_t write_osd_set[3] = { 1, 2, 3 };
-    osd_rmw_stripe_t stripes[3] = {};
+    osd_rmw_stripe_t stripes[3] = { 0 };
     // Test 8.1
     split_stripes(2, 128*1024, 0, 128*1024+4096, stripes);
     void *write_buf = malloc(128*1024+4096);
@@ -332,10 +332,10 @@ void test8()
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
     assert(stripes[0].read_buf == NULL);
-    assert(stripes[1].read_buf == (uint8_t*)rmw_buf+128*1024);
+    assert(stripes[1].read_buf == rmw_buf+128*1024);
     assert(stripes[2].read_buf == NULL);
     assert(stripes[0].write_buf == write_buf);
-    assert(stripes[1].write_buf == (uint8_t*)write_buf+128*1024);
+    assert(stripes[1].write_buf == write_buf+128*1024);
     assert(stripes[2].write_buf == rmw_buf);
     // Test 8.2
     set_pattern(write_buf, 128*1024+4096, PATTERN0);
@@ -345,7 +345,7 @@ void test8()
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);     // recheck again
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024); // recheck again
     assert(stripes[0].write_buf == write_buf);                               // recheck again
-    assert(stripes[1].write_buf == (uint8_t*)write_buf+128*1024);                      // recheck again
+    assert(stripes[1].write_buf == write_buf+128*1024);                      // recheck again
     assert(stripes[2].write_buf == rmw_buf);                                 // recheck again
     check_pattern(stripes[2].write_buf, 4096, 0); // new parity
     check_pattern(stripes[2].write_buf+4096, 128*1024-4096, PATTERN0^PATTERN1); // new parity
@@ -375,7 +375,7 @@ void test9()
 {
     osd_num_t osd_set[3] = { 0, 2, 3 };
     osd_num_t write_osd_set[3] = { 1, 2, 3 };
-    osd_rmw_stripe_t stripes[3] = {};
+    osd_rmw_stripe_t stripes[3] = { 0 };
     // Test 9.0
     split_stripes(2, 128*1024, 64*1024, 0, stripes);
     assert(stripes[0].req_start == 0 && stripes[0].req_end == 0);
@@ -391,8 +391,8 @@ void test9()
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 0);
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 0);
     assert(stripes[0].read_buf == rmw_buf);
-    assert(stripes[1].read_buf == (uint8_t*)rmw_buf+128*1024);
+    assert(stripes[1].read_buf == rmw_buf+128*1024);
-    assert(stripes[2].read_buf == (uint8_t*)rmw_buf+128*1024*2);
+    assert(stripes[2].read_buf == rmw_buf+128*1024*2);
     assert(stripes[0].write_buf == NULL);
     assert(stripes[1].write_buf == NULL);
     assert(stripes[2].write_buf == NULL);
@@ -430,7 +430,7 @@ void test10()
 {
     osd_num_t osd_set[3] = { 1, 0, 0 };
     osd_num_t write_osd_set[3] = { 1, 2, 3 };
-    osd_rmw_stripe_t stripes[3] = {};
+    osd_rmw_stripe_t stripes[3] = { 0 };
     // Test 10.0
     split_stripes(2, 128*1024, 0, 256*1024, stripes);
     assert(stripes[0].req_start == 0 && stripes[0].req_end == 128*1024);
@@ -450,7 +450,7 @@ void test10()
     assert(stripes[1].read_buf == NULL);
     assert(stripes[2].read_buf == NULL);
     assert(stripes[0].write_buf == write_buf);
-    assert(stripes[1].write_buf == (uint8_t*)write_buf+128*1024);
+    assert(stripes[1].write_buf == write_buf+128*1024);
     assert(stripes[2].write_buf == rmw_buf);
     // Test 10.2
     set_pattern(stripes[0].write_buf, 128*1024, PATTERN1);
@@ -460,7 +460,7 @@ void test10()
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 128*1024);
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
     assert(stripes[0].write_buf == write_buf);
-    assert(stripes[1].write_buf == (uint8_t*)write_buf+128*1024);
+    assert(stripes[1].write_buf == write_buf+128*1024);
     assert(stripes[2].write_buf == rmw_buf);
     check_pattern(stripes[2].write_buf, 128*1024, PATTERN1^PATTERN2);
     free(rmw_buf);
@@ -486,7 +486,7 @@ void test11()
 {
     osd_num_t osd_set[3] = { 1, 0, 0 };
     osd_num_t write_osd_set[3] = { 1, 2, 3 };
-    osd_rmw_stripe_t stripes[3] = {};
+    osd_rmw_stripe_t stripes[3] = { 0 };
     // Test 11.0
     split_stripes(2, 128*1024, 128*1024, 256*1024, stripes);
     assert(stripes[0].req_start == 0 && stripes[0].req_end == 0);
@@ -502,7 +502,7 @@ void test11()
     assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 128*1024);
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
-    assert(stripes[0].read_buf == (uint8_t*)rmw_buf+128*1024);
+    assert(stripes[0].read_buf == rmw_buf+128*1024);
     assert(stripes[1].read_buf == NULL);
     assert(stripes[2].read_buf == NULL);
     assert(stripes[0].write_buf == NULL);
@@ -542,7 +542,7 @@ void test12()
 {
     osd_num_t osd_set[3] = { 1, 2, 0 };
     osd_num_t write_osd_set[3] = { 1, 2, 3 };
-    osd_rmw_stripe_t stripes[3] = {};
+    osd_rmw_stripe_t stripes[3] = { 0 };
     // Test 12.0
     split_stripes(2, 128*1024, 0, 0, stripes);
     assert(stripes[0].req_start == 0 && stripes[0].req_end == 0);
@@ -557,8 +557,8 @@ void test12()
     assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 0);
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
-    assert(stripes[0].read_buf == (uint8_t*)rmw_buf+128*1024);
+    assert(stripes[0].read_buf == rmw_buf+128*1024);
-    assert(stripes[1].read_buf == (uint8_t*)rmw_buf+2*128*1024);
+    assert(stripes[1].read_buf == rmw_buf+2*128*1024);
     assert(stripes[2].read_buf == NULL);
     assert(stripes[0].write_buf == NULL);
     assert(stripes[1].write_buf == NULL);
@@ -597,7 +597,7 @@ void test13()
     use_jerasure(4, 2, true);
     osd_num_t osd_set[4] = { 1, 2, 0, 0 };
     osd_num_t write_osd_set[4] = { 1, 2, 3, 4 };
-    osd_rmw_stripe_t stripes[4] = {};
+    osd_rmw_stripe_t stripes[4] = { 0 };
     // Test 13.0
     void *write_buf = malloc_or_die(8192);
     split_stripes(2, 128*1024, 128*1024-4096, 8192, stripes);
@@ -616,14 +616,14 @@ void test13()
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
     assert(stripes[3].write_start == 0 && stripes[3].write_end == 128*1024);
-    assert(stripes[0].read_buf == (uint8_t*)rmw_buf+2*128*1024);
+    assert(stripes[0].read_buf == rmw_buf+2*128*1024);
-    assert(stripes[1].read_buf == (uint8_t*)rmw_buf+3*128*1024-4096);
+    assert(stripes[1].read_buf == rmw_buf+3*128*1024-4096);
     assert(stripes[2].read_buf == NULL);
     assert(stripes[3].read_buf == NULL);
     assert(stripes[0].write_buf == write_buf);
-    assert(stripes[1].write_buf == (uint8_t*)write_buf+4096);
+    assert(stripes[1].write_buf == write_buf+4096);
     assert(stripes[2].write_buf == rmw_buf);
-    assert(stripes[3].write_buf == (uint8_t*)rmw_buf+128*1024);
+    assert(stripes[3].write_buf == rmw_buf+128*1024);
     // Test 13.2 - encode
     set_pattern(write_buf, 8192, PATTERN3);
     set_pattern(stripes[0].read_buf, 128*1024-4096, PATTERN1);
@@ -634,9 +634,9 @@ void test13()
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
     assert(stripes[3].write_start == 0 && stripes[3].write_end == 128*1024);
     assert(stripes[0].write_buf == write_buf);
-    assert(stripes[1].write_buf == (uint8_t*)write_buf+4096);
+    assert(stripes[1].write_buf == write_buf+4096);
     assert(stripes[2].write_buf == rmw_buf);
-    assert(stripes[3].write_buf == (uint8_t*)rmw_buf+128*1024);
+    assert(stripes[3].write_buf == rmw_buf+128*1024);
     // Test 13.3 - full decode and verify
     osd_num_t read_osd_set[4] = { 0, 0, 3, 4 };
     memset(stripes, 0, sizeof(stripes));
@@ -658,11 +658,11 @@ void test13()
     void *read_buf = alloc_read_buffer(stripes, 4, 0);
     assert(read_buf);
     assert(stripes[0].read_buf == read_buf);
-    assert(stripes[1].read_buf == (uint8_t*)read_buf+128*1024);
+    assert(stripes[1].read_buf == read_buf+128*1024);
-    assert(stripes[2].read_buf == (uint8_t*)read_buf+2*128*1024);
+    assert(stripes[2].read_buf == read_buf+2*128*1024);
-    assert(stripes[3].read_buf == (uint8_t*)read_buf+3*128*1024);
+    assert(stripes[3].read_buf == read_buf+3*128*1024);
-    memcpy((uint8_t*)read_buf+2*128*1024, rmw_buf, 128*1024);
+    memcpy(read_buf+2*128*1024, rmw_buf, 128*1024);
-    memcpy((uint8_t*)read_buf+3*128*1024, (uint8_t*)rmw_buf+128*1024, 128*1024);
+    memcpy(read_buf+3*128*1024, rmw_buf+128*1024, 128*1024);
     reconstruct_stripes_jerasure(stripes, 4, 2, 0);
     check_pattern(stripes[0].read_buf, 128*1024-4096, PATTERN1);
     check_pattern(stripes[0].read_buf+128*1024-4096, 4096, PATTERN3);
@@ -690,10 +690,10 @@ void test13()
     assert(read_buf);
     assert(stripes[0].read_buf == read_buf);
     assert(stripes[1].read_buf == NULL);
-    assert(stripes[2].read_buf == (uint8_t*)read_buf+128*1024);
+    assert(stripes[2].read_buf == read_buf+128*1024);
-    assert(stripes[3].read_buf == (uint8_t*)read_buf+2*128*1024);
+    assert(stripes[3].read_buf == read_buf+2*128*1024);
-    memcpy((uint8_t*)read_buf+128*1024, rmw_buf, 128*1024);
+    memcpy(read_buf+128*1024, rmw_buf, 128*1024);
-    memcpy((uint8_t*)read_buf+2*128*1024, (uint8_t*)rmw_buf+128*1024, 128*1024);
+    memcpy(read_buf+2*128*1024, rmw_buf+128*1024, 128*1024);
     reconstruct_stripes_jerasure(stripes, 4, 2, 0);
     check_pattern(stripes[0].read_buf, 128*1024-4096, PATTERN1);
     check_pattern(stripes[0].read_buf+128*1024-4096, 4096, PATTERN3);
@@ -725,7 +725,7 @@ void test14()
     use_jerasure(3, 2, true);
     osd_num_t osd_set[3] = { 1, 2, 0 };
     osd_num_t write_osd_set[3] = { 1, 2, 3 };
-    osd_rmw_stripe_t stripes[3] = {};
+    osd_rmw_stripe_t stripes[3] = { 0 };
     unsigned bitmaps[3] = { 0 };
     // Test 13.0
     void *write_buf = malloc_or_die(8192);
@@ -744,11 +744,11 @@ void test14()
     assert(stripes[0].write_start == 128*1024-4096 && stripes[0].write_end == 128*1024);
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
-    assert(stripes[0].read_buf == (uint8_t*)rmw_buf+128*1024);
+    assert(stripes[0].read_buf == rmw_buf+128*1024);
-    assert(stripes[1].read_buf == (uint8_t*)rmw_buf+2*128*1024-4096);
+    assert(stripes[1].read_buf == rmw_buf+2*128*1024-4096);
     assert(stripes[2].read_buf == NULL);
     assert(stripes[0].write_buf == write_buf);
-    assert(stripes[1].write_buf == (uint8_t*)write_buf+4096);
+    assert(stripes[1].write_buf == write_buf+4096);
     assert(stripes[2].write_buf == rmw_buf);
     // Test 13.2 - encode
     set_pattern(write_buf, 8192, PATTERN3);
@@ -765,7 +765,7 @@ void test14()
     assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
     assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
     assert(stripes[0].write_buf == write_buf);
-    assert(stripes[1].write_buf == (uint8_t*)write_buf+4096);
+    assert(stripes[1].write_buf == write_buf+4096);
     assert(stripes[2].write_buf == rmw_buf);
     // Test 13.3 - decode and verify
     osd_num_t read_osd_set[4] = { 0, 2, 3 };
@@ -788,8 +788,8 @@ void test14()
         stripes[i].bmp_buf = bitmaps+i;
     assert(read_buf);
     assert(stripes[0].read_buf == read_buf);
-    assert(stripes[1].read_buf == (uint8_t*)read_buf+128*1024);
+    assert(stripes[1].read_buf == read_buf+128*1024);
-    assert(stripes[2].read_buf == (uint8_t*)read_buf+2*128*1024);
+    assert(stripes[2].read_buf == read_buf+2*128*1024);
     set_pattern(stripes[1].read_buf, 4096, PATTERN3);
     set_pattern(stripes[1].read_buf+4096, 128*1024-4096, PATTERN2);
     memcpy(stripes[2].read_buf, rmw_buf, 128*1024);

src/osd_secondary.cpp

@@ -54,8 +54,8 @@ void osd_t::exec_secondary(osd_op_t *cur_op)
             void *cur_buf = reply_buf;
             for (int i = 0; i < n; i++)
             {
-                bs->read_bitmap(ov[i].oid, ov[i].version, (uint8_t*)cur_buf + sizeof(uint64_t), (uint64_t*)cur_buf);
+                bs->read_bitmap(ov[i].oid, ov[i].version, cur_buf + sizeof(uint64_t), (uint64_t*)cur_buf);
-                cur_buf = (uint8_t*)cur_buf + (8 + clean_entry_bitmap_size);
+                cur_buf += (8 + clean_entry_bitmap_size);
             }
             free(cur_op->buf);
             cur_op->buf = reply_buf;
@@ -159,7 +159,7 @@ void osd_t::exec_show_config(osd_op_t *cur_op)
         { "readonly", readonly },
         { "immediate_commit", (immediate_commit == IMMEDIATE_ALL ? "all" :
             (immediate_commit == IMMEDIATE_SMALL ? "small" : "none")) },
-        { "lease_timeout", etcd_report_interval+(st_cli.max_etcd_attempts*(2*st_cli.etcd_quick_timeout)+999)/1000 },
+        { "lease_timeout", etcd_report_interval+(MAX_ETCD_ATTEMPTS*(2*ETCD_QUICK_TIMEOUT)+999)/1000 },
     };
 #ifdef WITH_RDMA
     if (msgr.is_rdma_enabled())

src/rw_blocking.cpp

@@ -3,10 +3,7 @@
 
 #include <errno.h>
 #include <stdlib.h>
-#include <stdint.h>
 #include <stdio.h>
-#include <sys/types.h>
-#include <sys/socket.h>
 
 #include "rw_blocking.h"
 
@@ -23,7 +20,7 @@ int read_blocking(int fd, void *read_buf, size_t remaining)
                 // EOF
                 return done;
             }
-            else if (errno != EINTR && errno != EAGAIN && errno != EPIPE)
+            else if (errno != EAGAIN && errno != EPIPE)
             {
                 perror("read");
                 exit(1);
@@ -31,7 +28,7 @@ int read_blocking(int fd, void *read_buf, size_t remaining)
             continue;
         }
         done += r;
-        read_buf = (uint8_t*)read_buf + r;
+        read_buf += r;
     }
     return done;
 }
@@ -44,7 +41,7 @@ int write_blocking(int fd, void *write_buf, size_t remaining)
         size_t r = write(fd, write_buf, remaining-done);
         if (r < 0)
         {
-            if (errno != EINTR && errno != EAGAIN && errno != EPIPE)
+            if (errno != EAGAIN && errno != EPIPE)
             {
                 perror("write");
                 exit(1);
@@ -52,7 +49,7 @@ int write_blocking(int fd, void *write_buf, size_t remaining)
             continue;
         }
         done += r;
-        write_buf = (uint8_t*)write_buf + r;
+        write_buf += r;
     }
     return done;
 }
@@ -66,7 +63,7 @@ int readv_blocking(int fd, iovec *iov, int iovcnt)
         ssize_t r = readv(fd, iov+v, iovcnt-v);
         if (r < 0)
         {
-            if (errno != EINTR && errno != EAGAIN && errno != EPIPE)
+            if (errno != EAGAIN && errno != EPIPE)
             {
                 perror("writev");
                 exit(1);
@@ -79,7 +76,7 @@ int readv_blocking(int fd, iovec *iov, int iovcnt)
             if (iov[v].iov_len > r)
             {
                 iov[v].iov_len -= r;
-                iov[v].iov_base = (uint8_t*)iov[v].iov_base + r;
+                iov[v].iov_base += r;
                 break;
             }
             else
@@ -101,7 +98,7 @@ int writev_blocking(int fd, iovec *iov, int iovcnt)
         ssize_t r = writev(fd, iov+v, iovcnt-v);
         if (r < 0)
         {
-            if (errno != EINTR && errno != EAGAIN && errno != EPIPE)
+            if (errno != EAGAIN && errno != EPIPE)
             {
                 perror("writev");
                 exit(1);
@@ -114,45 +111,7 @@ int writev_blocking(int fd, iovec *iov, int iovcnt)
             if (iov[v].iov_len > r)
             {
                 iov[v].iov_len -= r;
-                iov[v].iov_base = (uint8_t*)iov[v].iov_base + r;
+                iov[v].iov_base += r;
-                break;
-            }
-            else
-            {
-                r -= iov[v].iov_len;
-                v++;
-            }
-        }
-    }
-    return done;
-}
-
-int sendv_blocking(int fd, iovec *iov, int iovcnt, int flags)
-{
-    struct msghdr msg = { 0 };
-    int v = 0;
-    int done = 0;
-    while (v < iovcnt)
-    {
-        msg.msg_iov = iov+v;
-        msg.msg_iovlen = iovcnt-v;
-        ssize_t r = sendmsg(fd, &msg, flags);
-        if (r < 0)
-        {
-            if (errno != EINTR && errno != EAGAIN && errno != EPIPE)
-            {
-                perror("sendmsg");
-                exit(1);
-            }
-            continue;
-        }
-        done += r;
-        while (v < iovcnt)
-        {
-            if (iov[v].iov_len > r)
-            {
-                iov[v].iov_len -= r;
-                iov[v].iov_base = (uint8_t*)iov[v].iov_base + r;
                 break;
             }
             else

src/rw_blocking.h

@@ -10,4 +10,3 @@ int read_blocking(int fd, void *read_buf, size_t remaining);
 int write_blocking(int fd, void *write_buf, size_t remaining);
 int readv_blocking(int fd, iovec *iov, int iovcnt);
 int writev_blocking(int fd, iovec *iov, int iovcnt);
-int sendv_blocking(int fd, iovec *iov, int iovcnt, int flags);

src/stub_osd.cpp

@@ -112,7 +112,7 @@ int bind_stub(std::string bind_address, int bind_port)
 void run_stub(int peer_fd)
 {
     osd_any_op_t op;
-    osd_any_reply_t reply = {};
+    osd_any_reply_t reply = { 0 };
     void *buf = NULL;
     while (1)
     {
@@ -166,4 +166,5 @@ void run_stub(int peer_fd)
             break;
         }
     }
+    free(buf);
 }

src/stub_uring_osd.cpp

@@ -110,10 +110,8 @@ void stub_exec_op(osd_messenger_t *msgr, osd_op_t *op)
     if (op->req.hdr.opcode == OSD_OP_SEC_READ)
     {
         op->reply.hdr.retval = op->req.sec_rw.len;
-        op->buf = memalign_or_die(MEM_ALIGNMENT, op->req.sec_rw.len);
+        op->buf = malloc(op->req.sec_rw.len);
         op->iov.push_back(op->buf, op->req.sec_rw.len);
-        op->reply.sec_rw.attr_len = 4;
-        op->bitmap = op->buf;
     }
     else if (op->req.hdr.opcode == OSD_OP_SEC_WRITE || op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
     {

src/test_pattern.h

@@ -11,5 +11,5 @@
 #define PATTERN2 0xffe3bad5f578a78e
 #define PATTERN3 0x426bd7854eb08509
 
-#define set_pattern(buf, len, pattern) for (uint64_t i = 0; i < len; i += 8) { *(uint64_t*)((uint8_t*)buf + i) = pattern; }
+#define set_pattern(buf, len, pattern) for (uint64_t i = 0; i < len; i += 8) { *(uint64_t*)((void*)buf + i) = pattern; }
-#define check_pattern(buf, len, pattern) { uint64_t bad = UINT64_MAX; for (uint64_t i = 0; i < len; i += 8) { if ((*(uint64_t*)((uint8_t*)buf + i)) != (pattern)) { bad = i; break; } } if (bad != UINT64_MAX) { printf("mismatch at %lx\n", bad); } assert(bad == UINT64_MAX); }
+#define check_pattern(buf, len, pattern) { uint64_t bad = UINT64_MAX; for (uint64_t i = 0; i < len; i += 8) { if ((*(uint64_t*)(buf + i)) != (pattern)) { bad = i; break; } } if (bad != UINT64_MAX) { printf("mismatch at %lx\n", bad); } assert(bad == UINT64_MAX); }

src/timerfd_manager.cpp

@@ -94,7 +94,7 @@ again:
     if (!timers.size())
     {
         nearest = -1;
-        itimerspec exp = {};
+        itimerspec exp = { 0 };
         if (timerfd_settime(timerfd, 0, &exp, NULL))
         {
             throw std::runtime_error(std::string("timerfd_settime: ") + strerror(errno));

src/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 0.6.12
+Version: 0.6.10
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

tests/common.sh

@@ -23,39 +23,19 @@ trap 'kill -9 $(jobs -p)' EXIT
 ETCD=${ETCD:-etcd}
 ETCD_IP=${ETCD_IP:-127.0.0.1}
 ETCD_PORT=${ETCD_PORT:-12379}
-ETCD_COUNT=${ETCD_COUNT:-1}
 
 if [ "$KEEP_DATA" = "" ]; then
     rm -rf ./testdata
     mkdir -p ./testdata
 fi
 
-ETCD_URL="http://$ETCD_IP:$ETCD_PORT"
+$ETCD -name etcd_test --data-dir ./testdata/etcd \
-ETCD_CLUSTER="etcd1=http://$ETCD_IP:$((ETCD_PORT+1))"
+    --advertise-client-urls http://$ETCD_IP:$ETCD_PORT --listen-client-urls http://$ETCD_IP:$ETCD_PORT \
-for i in $(seq 2 $ETCD_COUNT); do
+    --initial-advertise-peer-urls http://$ETCD_IP:$((ETCD_PORT+1)) --listen-peer-urls http://$ETCD_IP:$((ETCD_PORT+1)) \
-    ETCD_URL="$ETCD_URL,http://$ETCD_IP:$((ETCD_PORT+2*i-2))"
+    --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision &>./testdata/etcd.log &
-    ETCD_CLUSTER="$ETCD_CLUSTER,etcd$i=http://$ETCD_IP:$((ETCD_PORT+2*i-1))"
+ETCD_PID=$!
-done
+ETCD_URL=$ETCD_IP:$ETCD_PORT/v3
-ETCDCTL="${ETCD}ctl --endpoints=$ETCD_URL"
+ETCDCTL="${ETCD}ctl --endpoints=http://$ETCD_URL"
-
-start_etcd()
-{
-    local i=$1
-    $ETCD -name etcd$i --data-dir ./testdata/etcd$i \
-        --advertise-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) --listen-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) \
-        --initial-advertise-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) --listen-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) \
-        --initial-cluster-token vitastor-tests-etcd --initial-cluster-state new \
-        --initial-cluster "$ETCD_CLUSTER" \
-        --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision &>./testdata/etcd$i.log &
-    eval ETCD${i}_PID=$!
-}
-
-for i in $(seq 1 $ETCD_COUNT); do
-    start_etcd $i
-done
-if [ $ETCD_COUNT -gt 1 ]; then
-    sleep 1
-fi
 
 echo leak:fio >> testdata/lsan-suppress.txt
 echo leak:tcmalloc >> testdata/lsan-suppress.txt

tests/run_3osds.sh

@@ -5,7 +5,6 @@
 OSD_SIZE=${OSD_SIZE:-1024}
 PG_COUNT=${PG_COUNT:-1}
 PG_SIZE=${PG_SIZE:-3}
-PG_MINSIZE=${PG_MINSIZE:-2}
 OSD_COUNT=${OSD_COUNT:-3}
 SCHEME=${SCHEME:-ec}
 
@@ -18,7 +17,7 @@ done
 cd mon
 npm install
 cd ..
-node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
+node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
 MON_PID=$!
 
 if [ -n "$GLOBAL_CONF" ]; then
@@ -26,12 +25,12 @@ if [ -n "$GLOBAL_CONF" ]; then
 fi
 
 if [ "$SCHEME" = "replicated" ]; then
-    $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$PG_COUNT',"failure_domain":"osd"}}'
+    $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":'$PG_SIZE',"pg_minsize":'$((PG_SIZE-1))',"pg_count":'$PG_COUNT',"failure_domain":"osd"}}'
 else
-    $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"parity_chunks":1,"pg_count":'$PG_COUNT',"failure_domain":"osd"}}'
+    $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":'$PG_SIZE',"pg_minsize":'$((PG_SIZE-1))',"parity_chunks":1,"pg_count":'$PG_COUNT',"failure_domain":"osd"}}'
 fi
 
-sleep 3
+sleep 2
 
 if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and ([ .[0].items["1"][] | select(((.osd_set | select(. != 0) | sort | unique) | length) == '$PG_SIZE') ] | length) == '$PG_COUNT); then
     format_error "FAILED: $PG_COUNT PG(s) NOT CONFIGURED"

tests/test_add_osd.sh

@@ -1,36 +0,0 @@
-#!/bin/bash -ex
-
-PG_COUNT=16
-
-. `dirname $0`/run_3osds.sh
-
-LD_PRELOAD=libasan.so.5 \
-    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -end_fsync=1 \
-        -rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
-
-for i in 4; do
-    dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
-    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
-    eval OSD${i}_PID=$!
-done
-
-sleep 2
-
-for i in {1..10}; do
-    ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
-        jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4"])') && \
-        ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$PG_COUNT'') && \
-        break
-    sleep 1
-done
-
-if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only |\
-    jq -s -e '([ .[0].items["1"] | map(.osd_set)[][] ] | sort | unique == ["1","2","3","4"])'); then
-    format_error "FAILED: OSD NOT ADDED INTO DISTRIBUTION"
-fi
-
-if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$PG_COUNT''); then
-    format_error "FAILED: $PG_COUNT PGS NOT ACTIVE"
-fi
-
-format_green OK

tests/test_change_pg_count.sh

@@ -22,7 +22,7 @@ done
 cd mon
 npm install
 cd ..
-node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log &
+node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log &
 MON_PID=$!
 
 $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":16,"failure_domain":"osd"}}'

tests/test_change_pg_size.sh

@@ -14,7 +14,7 @@ done
 cd mon
 npm install
 cd ..
-node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
+node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
 MON_PID=$!
 
 $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":3,"pg_minsize":2,"pg_count":16,"failure_domain":"osd"}}'

tests/test_etcd_fail.sh

@@ -1,33 +0,0 @@
-#!/bin/bash -ex
-
-# Run 5 etcds and kill them while operating
-ETCD_COUNT=5
-
-. `dirname $0`/run_3osds.sh
-
-LD_PRELOAD=libasan.so.5 \
-    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
-        -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10
-
-kill_etcds()
-{
-    sleep 5
-    kill $ETCD1_PID $ETCD2_PID
-    sleep 5
-    kill $ETCD3_PID
-    start_etcd 1
-    sleep 5
-    kill $ETCD4_PID
-    start_etcd 2
-    sleep 5
-    kill $ETCD5_PID
-    start_etcd 3
-}
-
-kill_etcds &
-
-LD_PRELOAD=libasan.so.5 \
-    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \
-        -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10 -runtime=30
-
-format_green OK

tests/test_failure_domain.sh

@@ -18,7 +18,7 @@ $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicate
 cd mon
 npm install
 cd ..
-node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
+node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
 MON_PID=$!
 
 sleep 2

tests/test_interrupted_rebalance.sh

@@ -22,7 +22,7 @@ done
 cd mon
 npm install
 cd ..
-node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log &
+node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log &
 MON_PID=$!
 
 $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":32,"failure_domain":"osd"}}'

tests/test_minsize_1.sh

@@ -1,17 +0,0 @@
-#!/bin/bash -ex
-
-PG_MINSIZE=1
-SCHEME=replicated
-
-. `dirname $0`/run_3osds.sh
-
-kill -INT $OSD1_PID
-kill -INT $OSD2_PID
-
-sleep 5
-
-if ! ($ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active", "degraded"]) ] | length == '$PG_COUNT); then
-    format_error "FAILED: $PG_COUNT PG(s) NOT ACTIVE+DEGRADED"
-fi
-
-format_green OK

tests/test_vm_cont.sh

@@ -14,7 +14,7 @@ for i in $(seq 1 $OSD_COUNT); do
     eval OSD${i}_PID=$!
 done
 
-node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
+node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
 MON_PID=$!
 
 sleep 3