From 891250d355baf1cc41223eeb4747c0ae0824335f Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Mon, 14 Jun 2021 22:42:49 +0300 Subject: [PATCH 01/12] Implement CAS writes From now on, reads will return the server-side object version numbers and writes and deletes will have an additional "version" parameter which, if set to a non-zero value, will be atomically compared with the current version of the object plus 1 and the modification will fail if it doesn't match. This feature opens the road to correct online flattening of snapshot layers and other interesting things. --- src/cluster_client.cpp | 22 +++++++++++++++++----- src/cluster_client.h | 3 +++ src/messenger.cpp | 2 +- src/osd_ops.h | 5 +++++ src/osd_primary.cpp | 11 ++++++++++- src/osd_primary_chain.cpp | 14 ++++++++++++-- src/osd_primary_write.cpp | 8 +++++++- 7 files changed, 55 insertions(+), 10 deletions(-) diff --git a/src/cluster_client.cpp b/src/cluster_client.cpp index 6bc2ff7e..7cf6419a 100644 --- a/src/cluster_client.cpp +++ b/src/cluster_client.cpp @@ -633,6 +633,13 @@ resume_1: // Slice the operation into parts slice_rw(op); op->needs_reslice = false; + if (op->opcode == OSD_OP_WRITE && op->version && op->parts.size() > 1) + { + // Atomic writes to multiple stripes are unsupported + op->retval = -EINVAL; + erase_op(op); + return 1; + } resume_2: // Send unsent parts, if they're not subject to change op->state = 3; @@ -922,6 +929,7 @@ bool cluster_client_t::try_send(cluster_op_t *op, int i) .offset = part->offset, .len = part->len, .meta_revision = meta_rev, + .version = op->opcode == OSD_OP_WRITE ? op->version : 0, } }, .bitmap = op->opcode == OSD_OP_WRITE ? NULL : op->part_bitmaps + pg_bitmap_size*i, .bitmap_len = (unsigned)(op->opcode == OSD_OP_WRITE ? 0 : pg_bitmap_size), @@ -1072,10 +1080,6 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part) if (part->op.reply.hdr.retval != expected) { // Operation failed, retry - fprintf( - stderr, "%s operation failed on OSD %lu: retval=%ld (expected %d), dropping connection\n", - osd_op_names[part->op.req.hdr.opcode], part->osd_num, part->op.reply.hdr.retval, expected - ); if (part->op.reply.hdr.retval == -EPIPE) { // Mark op->up_wait = true before stopping the client @@ -1094,7 +1098,14 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part) // Don't overwrite other errors with -EPIPE op->retval = part->op.reply.hdr.retval; } - msgr.stop_client(part->op.peer_fd); + if (op->retval != -EINTR && op->retval != -EIO) + { + fprintf( + stderr, "%s operation failed on OSD %lu: retval=%ld (expected %d), dropping connection\n", + osd_op_names[part->op.req.hdr.opcode], part->osd_num, part->op.reply.hdr.retval, expected + ); + msgr.stop_client(part->op.peer_fd); + } part->flags |= PART_ERROR; } else @@ -1106,6 +1117,7 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part) if (op->opcode == OSD_OP_READ) { copy_part_bitmap(op, part); + op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0; } } if (op->inflight_count == 0) diff --git a/src/cluster_client.h b/src/cluster_client.h index dcf3db32..456e6dd9 100644 --- a/src/cluster_client.h +++ b/src/cluster_client.h @@ -31,6 +31,9 @@ struct cluster_op_t uint64_t inode; uint64_t offset; uint64_t len; + // for reads and writes within a single object (stripe), + // reads can return current version and writes can use "CAS" semantics + uint64_t version = 0; int retval; osd_op_buf_list_t iov; std::function callback; diff --git a/src/messenger.cpp b/src/messenger.cpp index d25bfe1a..747402b6 100644 --- a/src/messenger.cpp +++ b/src/messenger.cpp @@ -261,7 +261,7 @@ void osd_messenger_t::try_connect_peer_addr(osd_num_t peer_osd, const char *peer { osd_num_t peer_osd = clients.at(peer_fd)->osd_num; stop_client(peer_fd, true); - on_connect_peer(peer_osd, -EIO); + on_connect_peer(peer_osd, -EPIPE); return; }); } diff --git a/src/osd_ops.h b/src/osd_ops.h index e8078b71..98770701 100644 --- a/src/osd_ops.h +++ b/src/osd_ops.h @@ -191,6 +191,9 @@ struct __attribute__((__packed__)) osd_op_rw_t uint32_t flags; // inode metadata revision uint64_t meta_revision; + // object version for atomic "CAS" (compare-and-set) writes + // writes and deletes fail with -EINTR if object version differs from (version-1) + uint64_t version; }; struct __attribute__((__packed__)) osd_reply_rw_t @@ -199,6 +202,8 @@ struct __attribute__((__packed__)) osd_reply_rw_t // for reads: bitmap length uint32_t bitmap_len; uint32_t pad0; + // for reads: object version + uint64_t version; }; // sync to the primary OSD diff --git a/src/osd_primary.cpp b/src/osd_primary.cpp index 3a3e685e..52c463bb 100644 --- a/src/osd_primary.cpp +++ b/src/osd_primary.cpp @@ -222,6 +222,7 @@ resume_2: finish_op(cur_op, op_data->epipe > 0 ? -EPIPE : -EIO); return; } + cur_op->reply.rw.version = op_data->fact_ver; cur_op->reply.rw.bitmap_len = op_data->pg_data_size * clean_entry_bitmap_size; if (op_data->degraded) { @@ -343,6 +344,12 @@ resume_3: pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO); return; } + // Check CAS version + if (cur_op->req.rw.version && op_data->fact_ver != (cur_op->req.rw.version-1)) + { + cur_op->reply.hdr.retval = -EINTR; + goto continue_others; + } // Save version override for parallel reads pg.ver_override[op_data->oid] = op_data->fact_ver; // Submit deletes @@ -370,6 +377,8 @@ resume_5: free_object_state(pg, &op_data->object_state); } pg.total_count--; + cur_op->reply.hdr.retval = 0; +continue_others: osd_op_t *next_op = NULL; auto next_it = pg.write_queue.find(op_data->oid); if (next_it != pg.write_queue.end() && next_it->second == cur_op) @@ -378,7 +387,7 @@ resume_5: if (next_it != pg.write_queue.end() && next_it->first == op_data->oid) next_op = next_it->second; } - finish_op(cur_op, cur_op->req.rw.len); + finish_op(cur_op, cur_op->reply.hdr.retval); if (next_op) { // Continue next write to the same object diff --git a/src/osd_primary_chain.cpp b/src/osd_primary_chain.cpp index e9a2fbfb..64f60845 100644 --- a/src/osd_primary_chain.cpp +++ b/src/osd_primary_chain.cpp @@ -65,7 +65,10 @@ int osd_t::read_bitmaps(osd_op_t *cur_op, pg_t & pg, int base_state) auto vo_it = pg.ver_override.find(cur_oid); auto read_version = (vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX); // Read bitmap synchronously from the local database - bs->read_bitmap(cur_oid, read_version, op_data->snapshot_bitmaps + chain_num*clean_entry_bitmap_size, NULL); + bs->read_bitmap( + cur_oid, read_version, op_data->snapshot_bitmaps + chain_num*clean_entry_bitmap_size, + !chain_num ? &cur_op->reply.rw.version : NULL + ); } } else @@ -228,7 +231,10 @@ int osd_t::submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg) // Read bitmap synchronously from the local database for (int j = prev; j <= i; j++) { - bs->read_bitmap((*bitmap_requests)[j].oid, (*bitmap_requests)[j].version, (*bitmap_requests)[j].bmp_buf, NULL); + bs->read_bitmap( + (*bitmap_requests)[j].oid, (*bitmap_requests)[j].version, (*bitmap_requests)[j].bmp_buf, + (*bitmap_requests)[j].oid.inode == cur_op->req.rw.inode ? &cur_op->reply.rw.version : NULL + ); } } else @@ -264,6 +270,10 @@ int osd_t::submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg) for (int j = prev; j <= i; j++) { memcpy((*bitmap_requests)[j].bmp_buf, cur_buf, clean_entry_bitmap_size); + if ((*bitmap_requests)[j].oid.inode == cur_op->req.rw.inode) + { + memcpy(&cur_op->reply.rw.version, cur_buf-8, 8); + } cur_buf += 8 + clean_entry_bitmap_size; } } diff --git a/src/osd_primary_write.cpp b/src/osd_primary_write.cpp index d1e08f81..386d3af6 100644 --- a/src/osd_primary_write.cpp +++ b/src/osd_primary_write.cpp @@ -96,6 +96,12 @@ resume_3: pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO); return; } + // Check CAS version + if (cur_op->req.rw.version && op_data->fact_ver != (cur_op->req.rw.version-1)) + { + cur_op->reply.hdr.retval = -EINTR; + goto continue_others; + } if (op_data->scheme == POOL_SCHEME_REPLICATED) { // Set bitmap bits @@ -265,7 +271,7 @@ continue_others: next_op = next_it->second; } // finish_op would invalidate next_it if it cleared pg.write_queue, but it doesn't do that :) - finish_op(cur_op, cur_op->req.rw.len); + finish_op(cur_op, cur_op->reply.hdr.retval); if (next_op) { // Continue next write to the same object From 9c45d43e74e8d359c9c3fd4e2e9dfe0a46275ec4 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Mon, 14 Jun 2021 23:11:50 +0300 Subject: [PATCH 02/12] Extract common 3 OSD code from several test scripts --- tests/run_3osds.sh | 43 +++++++++++++++++++++++++++++++++++++ tests/test_snapshot.sh | 36 +------------------------------ tests/test_vm_start.sh | 36 ++----------------------------- tests/test_write.sh | 36 +------------------------------ tests/test_write_no_same.sh | 36 +++---------------------------- 5 files changed, 50 insertions(+), 137 deletions(-) create mode 100644 tests/run_3osds.sh diff --git a/tests/run_3osds.sh b/tests/run_3osds.sh new file mode 100644 index 00000000..a23780ff --- /dev/null +++ b/tests/run_3osds.sh @@ -0,0 +1,43 @@ +#!/bin/bash -ex + +. `dirname $0`/common.sh + +OSD_SIZE=${OSD_SIZE:-1024} + +dd if=/dev/zero of=./testdata/test_osd1.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1)) +dd if=/dev/zero of=./testdata/test_osd2.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1)) +dd if=/dev/zero of=./testdata/test_osd3.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1)) + +build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log & +OSD1_PID=$! +build/src/vitastor-osd --osd_num 2 --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log & +OSD2_PID=$! +build/src/vitastor-osd --osd_num 3 --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log & +OSD3_PID=$! + +cd mon +npm install +cd .. +node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & +MON_PID=$! + +if [ -n "$GLOBAL_CONF" ]; then + $ETCDCTL put /vitastor/config/global "$GLOBAL_CONF" +fi + +$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1,"pg_count":1,"failure_domain":"osd"}}' + +sleep 2 + +if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and (.[0].items["1"]["1"].osd_set | sort) == ["1","2","3"]'); then + format_error "FAILED: 1 PG NOT CONFIGURED" +fi + +if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then + format_error "FAILED: 1 PG NOT UP" +fi + +if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then + sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so + sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so +fi diff --git a/tests/test_snapshot.sh b/tests/test_snapshot.sh index 6f33d1f0..4723640f 100755 --- a/tests/test_snapshot.sh +++ b/tests/test_snapshot.sh @@ -1,40 +1,6 @@ #!/bin/bash -ex -. `dirname $0`/common.sh - -dd if=/dev/zero of=./testdata/test_osd1.bin bs=1024 count=1 seek=$((1024*1024-1)) -dd if=/dev/zero of=./testdata/test_osd2.bin bs=1024 count=1 seek=$((1024*1024-1)) -dd if=/dev/zero of=./testdata/test_osd3.bin bs=1024 count=1 seek=$((1024*1024-1)) - -build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log & -OSD1_PID=$! -build/src/vitastor-osd --osd_num 2 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log & -OSD2_PID=$! -build/src/vitastor-osd --osd_num 3 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log & -OSD3_PID=$! - -cd mon -npm install -cd .. -node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & -MON_PID=$! - -$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1,"pg_count":1,"failure_domain":"osd"}}' - -sleep 2 - -if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and (.[0].items["1"]["1"].osd_set | sort) == ["1","2","3"]'); then - format_error "FAILED: 1 PG NOT CONFIGURED" -fi - -if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then - format_error "FAILED: 1 PG NOT UP" -fi - -if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then - sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so - sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so -fi +. `dirname $0`/run_3osds.sh # Test basic write and snapshot diff --git a/tests/test_vm_start.sh b/tests/test_vm_start.sh index f2b1f3e2..5e3ec485 100755 --- a/tests/test_vm_start.sh +++ b/tests/test_vm_start.sh @@ -1,40 +1,8 @@ #!/bin/bash -ex -. `dirname $0`/common.sh +OSD_SIZE=2048 -dd if=/dev/zero of=./testdata/test_osd1.bin bs=2048 count=1 seek=$((1024*1024-1)) -dd if=/dev/zero of=./testdata/test_osd2.bin bs=2048 count=1 seek=$((1024*1024-1)) -dd if=/dev/zero of=./testdata/test_osd3.bin bs=2048 count=1 seek=$((1024*1024-1)) - -build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log & -OSD1_PID=$! -build/src/vitastor-osd --osd_num 2 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log & -OSD2_PID=$! -build/src/vitastor-osd --osd_num 3 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log & -OSD3_PID=$! - -cd mon -npm install -cd .. -node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & -MON_PID=$! - -$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1,"pg_count":1,"failure_domain":"osd"}}' - -sleep 2 - -if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and (.[0].items["1"]["1"].osd_set | sort) == ["1","2","3"]'); then - format_error "FAILED: 1 PG NOT CONFIGURED" -fi - -if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then - format_error "FAILED: 1 PG NOT UP" -fi - -if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then - sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so - sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so -fi +. `dirname $0`/run_3osds.sh $ETCDCTL put /vitastor/config/inode/1/1 '{"name":"debian9","size":'$((2048*1024*1024))'}' diff --git a/tests/test_write.sh b/tests/test_write.sh index c8553e6b..b41b9f57 100755 --- a/tests/test_write.sh +++ b/tests/test_write.sh @@ -1,44 +1,10 @@ #!/bin/bash -ex -. `dirname $0`/common.sh - -dd if=/dev/zero of=./testdata/test_osd1.bin bs=1024 count=1 seek=$((1024*1024-1)) -dd if=/dev/zero of=./testdata/test_osd2.bin bs=1024 count=1 seek=$((1024*1024-1)) -dd if=/dev/zero of=./testdata/test_osd3.bin bs=1024 count=1 seek=$((1024*1024-1)) - -build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log & -OSD1_PID=$! -build/src/vitastor-osd --osd_num 2 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log & -OSD2_PID=$! -build/src/vitastor-osd --osd_num 3 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log & -OSD3_PID=$! - -cd mon -npm install -cd .. -node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & -MON_PID=$! - -$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1,"pg_count":1,"failure_domain":"osd"}}' - -sleep 2 - -if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and (.[0].items["1"]["1"].osd_set | sort) == ["1","2","3"]'); then - format_error "FAILED: 1 PG NOT CONFIGURED" -fi - -if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then - format_error "FAILED: 1 PG NOT UP" -fi +. `dirname $0`/run_3osds.sh #LD_PRELOAD=libasan.so.5 \ # fio -thread -name=test -ioengine=build/src/libfio_vitastor_sec.so -bs=4k -fsync=128 `$ETCDCTL get /vitastor/osd/state/1 --print-value-only | jq -r '"-host="+.addresses[0]+" -port="+(.port|tostring)'` -rw=write -size=32M -if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then - sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so - sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so -fi - # A lot of parallel syncs was crashing the primary OSD at some point LD_PRELOAD=libasan.so.5 \ diff --git a/tests/test_write_no_same.sh b/tests/test_write_no_same.sh index fa70728a..588c3b7c 100755 --- a/tests/test_write_no_same.sh +++ b/tests/test_write_no_same.sh @@ -1,40 +1,10 @@ #!/bin/bash -ex # Test the `no_same_sector_overwrites` mode -. `dirname $0`/common.sh +OSD_ARGS="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all" +GLOBAL_CONF='{"immediate_commit":"all"}' -dd if=/dev/zero of=./testdata/test_osd1.bin bs=1024 count=1 seek=$((1024*1024-1)) -dd if=/dev/zero of=./testdata/test_osd2.bin bs=1024 count=1 seek=$((1024*1024-1)) -dd if=/dev/zero of=./testdata/test_osd3.bin bs=1024 count=1 seek=$((1024*1024-1)) - -NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all" - -build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 $NO_SAME --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log & -OSD1_PID=$! -build/src/vitastor-osd --osd_num 2 --bind_address 127.0.0.1 $NO_SAME --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log & -OSD2_PID=$! -build/src/vitastor-osd --osd_num 3 --bind_address 127.0.0.1 $NO_SAME --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log & -OSD3_PID=$! - -cd mon -npm install -cd .. -node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & -MON_PID=$! - -$ETCDCTL put /vitastor/config/global '{"immediate_commit":"all"}' - -$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1,"pg_count":1,"failure_domain":"osd"}}' - -sleep 2 - -if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and (.[0].items["1"]["1"].osd_set | sort) == ["1","2","3"]'); then - format_error "FAILED: 1 PG NOT CONFIGURED" -fi - -if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then - format_error "FAILED: 1 PG NOT UP" -fi +. `dirname $0`/run_3osds.sh #LSAN_OPTIONS=report_objects=true:suppressions=`pwd`/testdata/lsan-suppress.txt LD_PRELOAD=libasan.so.5 \ # fio -thread -name=test -ioengine=build/src/libfio_vitastor_sec.so -bs=4k -fsync=128 `$ETCDCTL get /vitastor/osd/state/1 --print-value-only | jq -r '"-host="+.addresses[0]+" -port="+(.port|tostring)'` -rw=write -size=32M From 3de553ecd7d70f78b1436b5ea6e7159c787d140f Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Tue, 15 Jun 2021 00:07:20 +0300 Subject: [PATCH 03/12] Add a test for CAS write operation --- src/CMakeLists.txt | 8 +++ src/test_cas.cpp | 135 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_cas.sh | 7 +++ 3 files changed, 150 insertions(+) create mode 100644 src/test_cas.cpp create mode 100755 tests/test_cas.sh diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cc5d4e58..0a644574 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -200,6 +200,14 @@ target_link_libraries(osd_peering_pg_test tcmalloc_minimal) # test_allocator add_executable(test_allocator test_allocator.cpp allocator.cpp) +# test_cas +add_executable(test_cas + test_cas.cpp +) +target_link_libraries(test_cas + vitastor_client +) + # test_cluster_client add_executable(test_cluster_client test_cluster_client.cpp diff --git a/src/test_cas.cpp b/src/test_cas.cpp new file mode 100644 index 00000000..6c888e1d --- /dev/null +++ b/src/test_cas.cpp @@ -0,0 +1,135 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 (see README.md for details) + +#include +#include + +#include "epoll_manager.h" +#include "cluster_client.h" + +void send_read(cluster_client_t *cli, uint64_t inode, std::function cb) +{ + cluster_op_t *op = new cluster_op_t(); + op->opcode = OSD_OP_READ; + op->inode = inode; + op->offset = 0; + op->len = 4096; + op->iov.push_back(malloc_or_die(op->len), op->len); + op->callback = [cb](cluster_op_t *op) + { + uint64_t version = op->version; + int retval = op->retval; + if (retval == op->len) + retval = 0; + free(op->iov.buf[0].iov_base); + delete op; + if (cb != NULL) + cb(retval, version); + }; + cli->execute(op); +} + +void send_write(cluster_client_t *cli, uint64_t inode, int byte, uint64_t version, std::function cb) +{ + cluster_op_t *op = new cluster_op_t(); + op->opcode = OSD_OP_WRITE; + op->inode = inode; + op->offset = 0; + op->len = 4096; + op->version = version; + op->iov.push_back(malloc_or_die(op->len), op->len); + memset(op->iov.buf[0].iov_base, byte, op->len); + op->callback = [cb](cluster_op_t *op) + { + int retval = op->retval; + if (retval == op->len) + retval = 0; + free(op->iov.buf[0].iov_base); + delete op; + if (cb != NULL) + cb(retval); + }; + cli->execute(op); +} + +int main(int narg, char *args[]) +{ + json11::Json::object cfgo; + for (int i = 1; i < narg; i++) + { + if (args[i][0] == '-' && args[i][1] == '-') + { + const char *opt = args[i]+2; + cfgo[opt] = i == narg-1 ? "1" : args[++i]; + } + } + json11::Json cfg(cfgo); + uint64_t inode = (cfg["pool_id"].uint64_value() << (64-POOL_ID_BITS)) + | cfg["inode_id"].uint64_value(); + uint64_t base_ver = 0; + // Create client + auto ringloop = new ring_loop_t(512); + auto epmgr = new epoll_manager_t(ringloop); + auto cli = new cluster_client_t(ringloop, epmgr->tfd, cfg); + cli->on_ready([&]() + { + send_read(cli, inode, [&](int r, uint64_t v) + { + if (r < 0) + { + fprintf(stderr, "Initial read operation failed\n"); + exit(1); + } + base_ver = v; + // CAS v=1 = compare with zero, non-existing object + send_write(cli, inode, 0x01, base_ver+1, [&](int r) + { + if (r < 0) + { + fprintf(stderr, "CAS for non-existing object failed\n"); + exit(1); + } + // Check that read returns the new version + send_read(cli, inode, [&](int r, uint64_t v) + { + if (r < 0) + { + fprintf(stderr, "Read operation failed after write\n"); + exit(1); + } + if (v != base_ver+1) + { + fprintf(stderr, "Read operation failed to return the new version number\n"); + exit(1); + } + // CAS v=2 = compare with v=1, existing object + send_write(cli, inode, 0x02, base_ver+2, [&](int r) + { + if (r < 0) + { + fprintf(stderr, "CAS for existing object failed\n"); + exit(1); + } + // CAS v=2 again = compare with v=1, but version is 2. Must fail with -EINTR + send_write(cli, inode, 0x03, base_ver+2, [&](int r) + { + if (r != -EINTR) + { + fprintf(stderr, "CAS conflict detection failed\n"); + exit(1); + } + printf("Basic CAS test succeeded\n"); + exit(0); + }); + }); + }); + }); + }); + }); + while (1) + { + ringloop->loop(); + ringloop->wait(); + } + return 0; +} diff --git a/tests/test_cas.sh b/tests/test_cas.sh new file mode 100755 index 00000000..2cc980d3 --- /dev/null +++ b/tests/test_cas.sh @@ -0,0 +1,7 @@ +#!/bin/bash -ex + +. `dirname $0`/run_3osds.sh + +build/src/test_cas --pool_id 1 --inode_id 1 --etcd_address $ETCD_URL + +format_green OK From 511a89948b2821db535c0d91aab5badab577a856 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 19 Jun 2021 00:23:28 +0300 Subject: [PATCH 04/12] Rework qemu_proxy into a C wrapper library with public header --- rpm/vitastor-el7.spec | 1 + rpm/vitastor-el8.spec | 1 + src/CMakeLists.txt | 112 +++++++++++-------- src/fio_cluster.cpp | 128 +++++++++------------- src/qemu_driver.c | 32 +++--- src/qemu_proxy.cpp | 177 ------------------------------ src/qemu_proxy.h | 35 ------ src/vitastor_c.cpp | 245 ++++++++++++++++++++++++++++++++++++++++++ src/vitastor_c.h | 54 ++++++++++ 9 files changed, 434 insertions(+), 351 deletions(-) delete mode 100644 src/qemu_proxy.cpp delete mode 100644 src/qemu_proxy.h create mode 100644 src/vitastor_c.cpp create mode 100644 src/vitastor_c.h diff --git a/rpm/vitastor-el7.spec b/rpm/vitastor-el7.spec index 3b0fa823..8b50fe7f 100644 --- a/rpm/vitastor-el7.spec +++ b/rpm/vitastor-el7.spec @@ -64,6 +64,7 @@ cp -r mon %buildroot/usr/lib/vitastor/mon %_libdir/libfio_vitastor_sec.so %_libdir/libvitastor_blk.so* %_libdir/libvitastor_client.so* +%_includedir/vitastor_c.h /usr/lib/vitastor diff --git a/rpm/vitastor-el8.spec b/rpm/vitastor-el8.spec index 106b1ab1..7ac6ccd4 100644 --- a/rpm/vitastor-el8.spec +++ b/rpm/vitastor-el8.spec @@ -61,6 +61,7 @@ cp -r mon %buildroot/usr/lib/vitastor %_libdir/libfio_vitastor_sec.so %_libdir/libvitastor_blk.so* %_libdir/libvitastor_client.so* +%_includedir/vitastor_c.h /usr/lib/vitastor diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0a644574..9ac90913 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,8 @@ project(vitastor) include(GNUInstallDirs) +set(WITH_QEMU true CACHE BOOL "Build QEMU driver") +set(WITH_FIO true CACHE BOOL "Build FIO driver") set(QEMU_PLUGINDIR qemu CACHE STRING "QEMU plugin directory suffix (qemu-kvm on RHEL)") set(WITH_ASAN false CACHE BOOL "Build with AddressSanitizer") if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$") @@ -36,7 +38,9 @@ string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_RELWITHDEBINFO "${CM find_package(PkgConfig) pkg_check_modules(LIBURING REQUIRED liburing) -pkg_check_modules(GLIB REQUIRED glib-2.0) +if (${WITH_QEMU}) + pkg_check_modules(GLIB REQUIRED glib-2.0) +endif (${WITH_QEMU}) pkg_check_modules(IBVERBS libibverbs) if (IBVERBS_LIBRARIES) add_definitions(-DWITH_RDMA) @@ -62,14 +66,16 @@ target_link_libraries(vitastor_blk ) set_target_properties(vitastor_blk PROPERTIES VERSION ${VERSION} SOVERSION 0) -# libfio_vitastor_blk.so -add_library(fio_vitastor_blk SHARED - fio_engine.cpp - ../json11/json11.cpp -) -target_link_libraries(fio_vitastor_blk - vitastor_blk -) +if (${WITH_FIO}) + # libfio_vitastor_blk.so + add_library(fio_vitastor_blk SHARED + fio_engine.cpp + ../json11/json11.cpp + ) + target_link_libraries(fio_vitastor_blk + vitastor_blk + ) +endif (${WITH_FIO}) # libvitastor_common.a set(MSGR_RDMA "") @@ -96,19 +102,23 @@ target_link_libraries(vitastor-osd ${IBVERBS_LIBRARIES} ) -# libfio_vitastor_sec.so -add_library(fio_vitastor_sec SHARED - fio_sec_osd.cpp - rw_blocking.cpp -) -target_link_libraries(fio_vitastor_sec - tcmalloc_minimal -) +if (${WITH_FIO}) + # libfio_vitastor_sec.so + add_library(fio_vitastor_sec SHARED + fio_sec_osd.cpp + rw_blocking.cpp + ) + target_link_libraries(fio_vitastor_sec + tcmalloc_minimal + ) +endif (${WITH_FIO}) # libvitastor_client.so add_library(vitastor_client SHARED cluster_client.cpp + vitastor_c.cpp ) +set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "vitastor_c.h") target_link_libraries(vitastor_client vitastor_common tcmalloc_minimal @@ -117,13 +127,15 @@ target_link_libraries(vitastor_client ) set_target_properties(vitastor_client PROPERTIES VERSION ${VERSION} SOVERSION 0) -# libfio_vitastor.so -add_library(fio_vitastor SHARED - fio_cluster.cpp -) -target_link_libraries(fio_vitastor - vitastor_client -) +if (${WITH_FIO}) + # libfio_vitastor.so + add_library(fio_vitastor SHARED + fio_cluster.cpp + ) + target_link_libraries(fio_vitastor + vitastor_client + ) +endif (${WITH_FIO}) # vitastor-nbd add_executable(vitastor-nbd @@ -146,27 +158,24 @@ add_executable(vitastor-dump-journal dump_journal.cpp crc32c.c ) -# qemu_driver.so -add_library(qemu_proxy STATIC qemu_proxy.cpp) -target_compile_options(qemu_proxy PUBLIC -fPIC) -target_include_directories(qemu_proxy PUBLIC - ../qemu/b/qemu - ../qemu/include - ${GLIB_INCLUDE_DIRS} -) -target_link_libraries(qemu_proxy - vitastor_client -) -add_library(qemu_vitastor SHARED - qemu_driver.c -) -target_link_libraries(qemu_vitastor - qemu_proxy -) -set_target_properties(qemu_vitastor PROPERTIES - PREFIX "" - OUTPUT_NAME "block-vitastor" -) +if (${WITH_QEMU}) + # qemu_driver.so + add_library(qemu_vitastor SHARED + qemu_driver.c + ) + target_include_directories(qemu_vitastor PUBLIC + ../qemu/b/qemu + ../qemu/include + ${GLIB_INCLUDE_DIRS} + ) + target_link_libraries(qemu_vitastor + vitastor_client + ) + set_target_properties(qemu_vitastor PROPERTIES + PREFIX "" + OUTPUT_NAME "block-vitastor" + ) +endif (${WITH_QEMU}) ### Test stubs @@ -226,5 +235,14 @@ target_include_directories(test_cluster_client PUBLIC ${CMAKE_SOURCE_DIR}/src/mo ### Install install(TARGETS vitastor-osd vitastor-dump-journal vitastor-nbd vitastor-rm RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -install(TARGETS fio_vitastor fio_vitastor_blk fio_vitastor_sec vitastor_blk vitastor_client LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) -install(TARGETS qemu_vitastor LIBRARY DESTINATION /usr/${CMAKE_INSTALL_LIBDIR}/${QEMU_PLUGINDIR}) +install( + TARGETS vitastor_blk vitastor_client + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) +if (${WITH_FIO}) + install(TARGETS fio_vitastor fio_vitastor_blk fio_vitastor_sec LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) +endif (${WITH_FIO}) +if (${WITH_QEMU}) + install(TARGETS qemu_vitastor LIBRARY DESTINATION /usr/${CMAKE_INSTALL_LIBDIR}/${QEMU_PLUGINDIR}) +endif (${WITH_QEMU}) diff --git a/src/fio_cluster.cpp b/src/fio_cluster.cpp index 0797be40..78419eaa 100644 --- a/src/fio_cluster.cpp +++ b/src/fio_cluster.cpp @@ -25,20 +25,17 @@ #include -#include "epoll_manager.h" -#include "cluster_client.h" +#include "vitastor_c.h" #include "fio_headers.h" struct sec_data { - ring_loop_t *ringloop = NULL; - epoll_manager_t *epmgr = NULL; - cluster_client_t *cli = NULL; - inode_watch_t *watch = NULL; + vitastor_c *cli = NULL; + void *watch = NULL; bool last_sync = false; /* The list of completed io_u structs. */ std::vector completed; - uint64_t op_n = 0, inflight = 0; + uint64_t inflight = 0; bool trace = false; }; @@ -189,6 +186,12 @@ static struct fio_option options[] = { }, }; +static void watch_callback(long retval, void *opaque) +{ + struct sec_data *bsd = (struct sec_data*)opaque; + bsd->watch = (void*)retval; +} + static int sec_setup(struct thread_data *td) { sec_options *o = (sec_options*)td->eo; @@ -209,27 +212,6 @@ static int sec_setup(struct thread_data *td) td->o.open_files++; } - json11::Json::object cfg; - if (o->config_path) - cfg["config_path"] = std::string(o->config_path); - if (o->etcd_host) - cfg["etcd_address"] = std::string(o->etcd_host); - if (o->etcd_prefix) - cfg["etcd_prefix"] = std::string(o->etcd_prefix); - if (o->rdma_device) - cfg["rdma_device"] = std::string(o->rdma_device); - if (o->rdma_port_num) - cfg["rdma_port_num"] = o->rdma_port_num; - if (o->rdma_gid_index) - cfg["rdma_gid_index"] = o->rdma_gid_index; - if (o->rdma_mtu) - cfg["rdma_mtu"] = o->rdma_mtu; - if (o->cluster_log) - cfg["log_level"] = o->cluster_log; - if (o->use_rdma != -1) - cfg["use_rdma"] = o->use_rdma; - json11::Json cfg_json(cfg); - if (!o->image) { if (!(o->inode & ((1l << (64-POOL_ID_BITS)) - 1))) @@ -251,20 +233,20 @@ static int sec_setup(struct thread_data *td) { o->inode = 0; } - bsd->ringloop = new ring_loop_t(512); - bsd->epmgr = new epoll_manager_t(bsd->ringloop); - bsd->cli = new cluster_client_t(bsd->ringloop, bsd->epmgr->tfd, cfg_json); + bsd->cli = vitastor_c_create_uring(o->config_path, o->etcd_host, o->etcd_prefix, + o->use_rdma, o->rdma_device, o->rdma_port_num, o->rdma_gid_index, o->rdma_mtu, o->cluster_log); if (o->image) { - while (!bsd->cli->is_ready()) + bsd->watch = NULL; + vitastor_c_watch_inode(bsd->cli, o->image, watch_callback, bsd); + while (true) { - bsd->ringloop->loop(); - if (bsd->cli->is_ready()) + vitastor_c_uring_handle_events(bsd->cli); + if (bsd->watch) break; - bsd->ringloop->wait(); + vitastor_c_uring_wait_events(bsd->cli); } - bsd->watch = bsd->cli->st_cli.watch_inode(std::string(o->image)); - td->files[0]->real_file_size = bsd->watch->cfg.size; + td->files[0]->real_file_size = vitastor_c_inode_get_size(bsd->watch); } bsd->trace = o->trace ? true : false; @@ -279,11 +261,9 @@ static void sec_cleanup(struct thread_data *td) { if (bsd->watch) { - bsd->cli->st_cli.close_watch(bsd->watch); + vitastor_c_close_watch(bsd->cli, bsd->watch); } - delete bsd->cli; - delete bsd->epmgr; - delete bsd->ringloop; + vitastor_c_destroy(bsd->cli); delete bsd; } } @@ -294,12 +274,26 @@ static int sec_init(struct thread_data *td) return 0; } +static void io_callback(long retval, void *opaque) +{ + struct io_u *io = (struct io_u*)opaque; + io->error = retval < 0 ? -retval : 0; + sec_data *bsd = (sec_data*)io->engine_data; + bsd->inflight--; + bsd->completed.push_back(io); + if (bsd->trace) + { + printf("--- %s 0x%lx retval=%ld\n", io->ddir == DDIR_READ ? "READ" : + (io->ddir == DDIR_WRITE ? "WRITE" : "SYNC"), (uint64_t)io, retval); + } +} + /* Begin read or write request. */ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io) { sec_options *opt = (sec_options*)td->eo; sec_data *bsd = (sec_data*)td->io_ops_data; - int n = bsd->op_n; + struct iovec iov; fio_ro_check(td, io); if (io->ddir == DDIR_SYNC && bsd->last_sync) @@ -308,32 +302,29 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io) } io->engine_data = bsd; - cluster_op_t *op = new cluster_op_t; + io->error = 0; + bsd->inflight++; - op->inode = opt->image ? bsd->watch->cfg.num : opt->inode; + uint64_t inode = opt->image ? vitastor_c_inode_get_num(bsd->watch) : opt->inode; switch (io->ddir) { case DDIR_READ: - op->opcode = OSD_OP_READ; - op->offset = io->offset; - op->len = io->xfer_buflen; - op->iov.push_back(io->xfer_buf, io->xfer_buflen); + iov = { .iov_base = io->xfer_buf, .iov_len = io->xfer_buflen }; + vitastor_c_read(bsd->cli, inode, io->offset, io->xfer_buflen, &iov, 1, io_callback, io); bsd->last_sync = false; break; case DDIR_WRITE: - if (opt->image && bsd->watch->cfg.readonly) + if (opt->image && vitastor_c_inode_get_readonly(bsd->watch)) { io->error = EROFS; return FIO_Q_COMPLETED; } - op->opcode = OSD_OP_WRITE; - op->offset = io->offset; - op->len = io->xfer_buflen; - op->iov.push_back(io->xfer_buf, io->xfer_buflen); + iov = { .iov_base = io->xfer_buf, .iov_len = io->xfer_buflen }; + vitastor_c_write(bsd->cli, inode, io->offset, io->xfer_buflen, &iov, 1, io_callback, io); bsd->last_sync = false; break; case DDIR_SYNC: - op->opcode = OSD_OP_SYNC; + vitastor_c_sync(bsd->cli, io_callback, io); bsd->last_sync = true; break; default: @@ -341,39 +332,20 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io) return FIO_Q_COMPLETED; } - op->callback = [io, n](cluster_op_t *op) - { - io->error = op->retval < 0 ? -op->retval : 0; - sec_data *bsd = (sec_data*)io->engine_data; - bsd->inflight--; - bsd->completed.push_back(io); - if (bsd->trace) - { - printf("--- %s n=%d retval=%d\n", io->ddir == DDIR_READ ? "READ" : - (io->ddir == DDIR_WRITE ? "WRITE" : "SYNC"), n, op->retval); - } - delete op; - }; - if (opt->trace) { if (io->ddir == DDIR_SYNC) { - printf("+++ SYNC # %d\n", n); + printf("+++ SYNC 0x%lx\n", (uint64_t)io); } else { - printf("+++ %s # %d 0x%llx+%llx\n", + printf("+++ %s 0x%lx 0x%llx+%llx\n", io->ddir == DDIR_READ ? "READ" : "WRITE", - n, io->offset, io->xfer_buflen); + (uint64_t)io, io->offset, io->xfer_buflen); } } - io->error = 0; - bsd->inflight++; - bsd->op_n++; - bsd->cli->execute(op); - if (io->error != 0) return FIO_Q_COMPLETED; return FIO_Q_QUEUED; @@ -384,10 +356,10 @@ static int sec_getevents(struct thread_data *td, unsigned int min, unsigned int sec_data *bsd = (sec_data*)td->io_ops_data; while (true) { - bsd->ringloop->loop(); + vitastor_c_uring_handle_events(bsd->cli); if (bsd->completed.size() >= min) break; - bsd->ringloop->wait(); + vitastor_c_uring_wait_events(bsd->cli); } return bsd->completed.size(); } diff --git a/src/qemu_driver.c b/src/qemu_driver.c index 07944878..4e647f26 100644 --- a/src/qemu_driver.c +++ b/src/qemu_driver.c @@ -26,7 +26,7 @@ #define qobject_unref QDECREF #endif -#include "qemu_proxy.h" +#include "vitastor_c.h" void qemu_module_dummy(void) { @@ -48,6 +48,7 @@ typedef struct VitastorClient uint64_t pool; uint64_t size; long readonly; + int use_rdma; char *rdma_device; int rdma_port_num; int rdma_gid_index; @@ -132,6 +133,7 @@ static void vitastor_parse_filename(const char *filename, QDict *options, Error if (!strcmp(name, "inode") || !strcmp(name, "pool") || !strcmp(name, "size") || + !strcmp(name, "use_rdma") || !strcmp(name, "rdma_port_num") || !strcmp(name, "rdma_gid_index") || !strcmp(name, "rdma_mtu")) @@ -181,7 +183,7 @@ static void coroutine_fn vitastor_co_get_metadata(VitastorRPC *task) task->co = qemu_coroutine_self(); qemu_mutex_lock(&client->mutex); - vitastor_proxy_watch_metadata(client->proxy, client->image, vitastor_co_generic_bh_cb, task); + vitastor_c_watch_inode(client->proxy, client->image, vitastor_co_generic_bh_cb, task); qemu_mutex_unlock(&client->mutex); while (!task->complete) @@ -198,13 +200,14 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E client->config_path = g_strdup(qdict_get_try_str(options, "config_path")); client->etcd_host = g_strdup(qdict_get_try_str(options, "etcd_host")); client->etcd_prefix = g_strdup(qdict_get_try_str(options, "etcd_prefix")); + client->use_rdma = qdict_get_try_int(options, "use_rdma", -1); client->rdma_device = g_strdup(qdict_get_try_str(options, "rdma_device")); client->rdma_port_num = qdict_get_try_int(options, "rdma_port_num", 0); client->rdma_gid_index = qdict_get_try_int(options, "rdma_gid_index", 0); client->rdma_mtu = qdict_get_try_int(options, "rdma_mtu", 0); - client->proxy = vitastor_proxy_create( - bdrv_get_aio_context(bs), client->config_path, client->etcd_host, client->etcd_prefix, - client->rdma_device, client->rdma_port_num, client->rdma_gid_index, client->rdma_mtu + client->proxy = vitastor_c_create_qemu( + (QEMUSetFDHandler*)aio_set_fd_handler, bdrv_get_aio_context(bs), client->config_path, client->etcd_host, client->etcd_prefix, + client->use_rdma, client->rdma_device, client->rdma_port_num, client->rdma_gid_index, client->rdma_mtu, 0 ); client->image = g_strdup(qdict_get_try_str(options, "image")); client->readonly = (flags & BDRV_O_RDWR) ? 1 : 0; @@ -224,9 +227,9 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E } BDRV_POLL_WHILE(bs, !task.complete); client->watch = (void*)task.ret; - client->readonly = client->readonly || vitastor_proxy_get_readonly(client->watch); - client->size = vitastor_proxy_get_size(client->watch); - if (!vitastor_proxy_get_inode_num(client->watch)) + client->readonly = client->readonly || vitastor_c_inode_get_readonly(client->watch); + client->size = vitastor_c_inode_get_size(client->watch); + if (!vitastor_c_inode_get_num(client->watch)) { error_setg(errp, "image does not exist"); vitastor_close(bs); @@ -255,6 +258,7 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E } bs->total_sectors = client->size / BDRV_SECTOR_SIZE; //client->aio_context = bdrv_get_aio_context(bs); + qdict_del(options, "use_rdma"); qdict_del(options, "rdma_mtu"); qdict_del(options, "rdma_gid_index"); qdict_del(options, "rdma_port_num"); @@ -272,7 +276,7 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E static void vitastor_close(BlockDriverState *bs) { VitastorClient *client = bs->opaque; - vitastor_proxy_destroy(client->proxy); + vitastor_c_destroy(client->proxy); qemu_mutex_destroy(&client->mutex); if (client->config_path) g_free(client->config_path); @@ -410,9 +414,9 @@ static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs, uint64_t offset vitastor_co_init_task(bs, &task); task.iov = iov; - uint64_t inode = client->watch ? vitastor_proxy_get_inode_num(client->watch) : client->inode; + uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode; qemu_mutex_lock(&client->mutex); - vitastor_proxy_rw(0, client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_generic_bh_cb, &task); + vitastor_c_read(client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_generic_bh_cb, &task); qemu_mutex_unlock(&client->mutex); while (!task.complete) @@ -430,9 +434,9 @@ static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs, uint64_t offse vitastor_co_init_task(bs, &task); task.iov = iov; - uint64_t inode = client->watch ? vitastor_proxy_get_inode_num(client->watch) : client->inode; + uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode; qemu_mutex_lock(&client->mutex); - vitastor_proxy_rw(1, client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_generic_bh_cb, &task); + vitastor_c_write(client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_generic_bh_cb, &task); qemu_mutex_unlock(&client->mutex); while (!task.complete) @@ -462,7 +466,7 @@ static int coroutine_fn vitastor_co_flush(BlockDriverState *bs) vitastor_co_init_task(bs, &task); qemu_mutex_lock(&client->mutex); - vitastor_proxy_sync(client->proxy, vitastor_co_generic_bh_cb, &task); + vitastor_c_sync(client->proxy, vitastor_co_generic_bh_cb, &task); qemu_mutex_unlock(&client->mutex); while (!task.complete) diff --git a/src/qemu_proxy.cpp b/src/qemu_proxy.cpp deleted file mode 100644 index 28bebc45..00000000 --- a/src/qemu_proxy.cpp +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (c) Vitaliy Filippov, 2019+ -// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details) - -// C-C++ proxy for the QEMU driver -// (QEMU headers don't compile with g++) - -#include - -#include "cluster_client.h" - -typedef void* AioContext; -#include "qemu_proxy.h" - -extern "C" -{ - // QEMU - typedef void IOHandler(void *opaque); - void aio_set_fd_handler(AioContext *ctx, int fd, int is_external, IOHandler *fd_read, IOHandler *fd_write, void *poll_fn, void *opaque); -} - -struct QemuProxyData -{ - int fd; - std::function callback; -}; - -class QemuProxy -{ - std::map handlers; - -public: - - timerfd_manager_t *tfd; - cluster_client_t *cli; - AioContext *ctx; - - QemuProxy(AioContext *ctx, const char *config_path, const char *etcd_host, const char *etcd_prefix, - const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu) - { - this->ctx = ctx; - json11::Json::object cfg; - if (config_path) - cfg["config_path"] = std::string(config_path); - if (etcd_host) - cfg["etcd_address"] = std::string(etcd_host); - if (etcd_prefix) - cfg["etcd_prefix"] = std::string(etcd_prefix); - if (rdma_device) - cfg["rdma_device"] = std::string(rdma_device); - if (rdma_port_num) - cfg["rdma_port_num"] = rdma_port_num; - if (rdma_gid_index) - cfg["rdma_gid_index"] = rdma_gid_index; - if (rdma_mtu) - cfg["rdma_mtu"] = rdma_mtu; - json11::Json cfg_json(cfg); - tfd = new timerfd_manager_t([this](int fd, bool wr, std::function callback) { set_fd_handler(fd, wr, callback); }); - cli = new cluster_client_t(NULL, tfd, cfg_json); - } - - ~QemuProxy() - { - delete cli; - delete tfd; - } - - void set_fd_handler(int fd, bool wr, std::function callback) - { - if (callback != NULL) - { - handlers[fd] = { .fd = fd, .callback = callback }; - aio_set_fd_handler(ctx, fd, false, &QemuProxy::read_handler, wr ? &QemuProxy::write_handler : NULL, NULL, &handlers[fd]); - } - else - { - handlers.erase(fd); - aio_set_fd_handler(ctx, fd, false, NULL, NULL, NULL, NULL); - } - } - - static void read_handler(void *opaque) - { - QemuProxyData *data = (QemuProxyData *)opaque; - data->callback(data->fd, EPOLLIN); - } - - static void write_handler(void *opaque) - { - QemuProxyData *data = (QemuProxyData *)opaque; - data->callback(data->fd, EPOLLOUT); - } -}; - -extern "C" { - -void* vitastor_proxy_create(AioContext *ctx, const char *config_path, const char *etcd_host, const char *etcd_prefix, - const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu) -{ - QemuProxy *p = new QemuProxy(ctx, config_path, etcd_host, etcd_prefix, rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu); - return p; -} - -void vitastor_proxy_destroy(void *client) -{ - QemuProxy *p = (QemuProxy*)client; - delete p; -} - -void vitastor_proxy_rw(int write, void *client, uint64_t inode, uint64_t offset, uint64_t len, - iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque) -{ - QemuProxy *p = (QemuProxy*)client; - cluster_op_t *op = new cluster_op_t; - op->opcode = write ? OSD_OP_WRITE : OSD_OP_READ; - op->inode = inode; - op->offset = offset; - op->len = len; - for (int i = 0; i < iovcnt; i++) - { - op->iov.push_back(iov[i].iov_base, iov[i].iov_len); - } - op->callback = [cb, opaque](cluster_op_t *op) - { - cb(op->retval, opaque); - delete op; - }; - p->cli->execute(op); -} - -void vitastor_proxy_sync(void *client, VitastorIOHandler cb, void *opaque) -{ - QemuProxy *p = (QemuProxy*)client; - cluster_op_t *op = new cluster_op_t; - op->opcode = OSD_OP_SYNC; - op->callback = [cb, opaque](cluster_op_t *op) - { - cb(op->retval, opaque); - delete op; - }; - p->cli->execute(op); -} - -void vitastor_proxy_watch_metadata(void *client, char *image, VitastorIOHandler cb, void *opaque) -{ - QemuProxy *p = (QemuProxy*)client; - p->cli->on_ready([=]() - { - auto watch = p->cli->st_cli.watch_inode(std::string(image)); - cb((long)watch, opaque); - }); -} - -void vitastor_proxy_close_watch(void *client, void *watch) -{ - QemuProxy *p = (QemuProxy*)client; - p->cli->st_cli.close_watch((inode_watch_t*)watch); -} - -uint64_t vitastor_proxy_get_size(void *watch_ptr) -{ - inode_watch_t *watch = (inode_watch_t*)watch_ptr; - return watch->cfg.size; -} - -uint64_t vitastor_proxy_get_inode_num(void *watch_ptr) -{ - inode_watch_t *watch = (inode_watch_t*)watch_ptr; - return watch->cfg.num; -} - -int vitastor_proxy_get_readonly(void *watch_ptr) -{ - inode_watch_t *watch = (inode_watch_t*)watch_ptr; - return watch->cfg.readonly; -} - -} diff --git a/src/qemu_proxy.h b/src/qemu_proxy.h deleted file mode 100644 index 7940c819..00000000 --- a/src/qemu_proxy.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Vitaliy Filippov, 2019+ -// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details) - -#ifndef VITASTOR_QEMU_PROXY_H -#define VITASTOR_QEMU_PROXY_H - -#ifndef POOL_ID_BITS -#define POOL_ID_BITS 16 -#endif -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -// Our exports -typedef void VitastorIOHandler(long retval, void *opaque); -void* vitastor_proxy_create(AioContext *ctx, const char *config_path, const char *etcd_host, const char *etcd_prefix, - const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu); -void vitastor_proxy_destroy(void *client); -void vitastor_proxy_rw(int write, void *client, uint64_t inode, uint64_t offset, uint64_t len, - struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque); -void vitastor_proxy_sync(void *client, VitastorIOHandler cb, void *opaque); -void vitastor_proxy_watch_metadata(void *client, char *image, VitastorIOHandler cb, void *opaque); -void vitastor_proxy_close_watch(void *client, void *watch); -uint64_t vitastor_proxy_get_size(void *watch); -uint64_t vitastor_proxy_get_inode_num(void *watch); -int vitastor_proxy_get_readonly(void *watch); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/vitastor_c.cpp b/src/vitastor_c.cpp new file mode 100644 index 00000000..9f6cfa6d --- /dev/null +++ b/src/vitastor_c.cpp @@ -0,0 +1,245 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details) + +// Simplified C client library for QEMU, fio and other external drivers +// Also acts as a C-C++ proxy for the QEMU driver (QEMU headers don't compile with g++) + +#include + +#include "ringloop.h" +#include "epoll_manager.h" +#include "cluster_client.h" + +#include "vitastor_c.h" + +struct vitastor_qemu_fd_t +{ + int fd; + std::function callback; +}; + +struct vitastor_c +{ + std::map handlers; + ring_loop_t *ringloop = NULL; + epoll_manager_t *epmgr = NULL; + timerfd_manager_t *tfd = NULL; + cluster_client_t *cli = NULL; + + QEMUSetFDHandler *aio_set_fd_handler = NULL; + void *aio_ctx = NULL; +}; + +extern "C" { + +static json11::Json vitastor_c_common_config(const char *config_path, const char *etcd_host, const char *etcd_prefix, + int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level) +{ + json11::Json::object cfg; + if (config_path) + cfg["config_path"] = std::string(config_path); + if (etcd_host) + cfg["etcd_address"] = std::string(etcd_host); + if (etcd_prefix) + cfg["etcd_prefix"] = std::string(etcd_prefix); + // -1 means unspecified + if (use_rdma >= 0) + cfg["use_rdma"] = use_rdma > 0; + if (rdma_device) + cfg["rdma_device"] = std::string(rdma_device); + if (rdma_port_num) + cfg["rdma_port_num"] = rdma_port_num; + if (rdma_gid_index) + cfg["rdma_gid_index"] = rdma_gid_index; + if (rdma_mtu) + cfg["rdma_mtu"] = rdma_mtu; + if (log_level) + cfg["log_level"] = log_level; + return json11::Json(cfg); +} + +static void vitastor_c_read_handler(void *opaque) +{ + vitastor_qemu_fd_t *data = (vitastor_qemu_fd_t *)opaque; + data->callback(data->fd, EPOLLIN); +} + +static void vitastor_c_write_handler(void *opaque) +{ + vitastor_qemu_fd_t *data = (vitastor_qemu_fd_t *)opaque; + data->callback(data->fd, EPOLLOUT); +} + +vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context, + const char *config_path, const char *etcd_host, const char *etcd_prefix, + bool use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level) +{ + json11::Json cfg_json = vitastor_c_common_config( + config_path, etcd_host, etcd_prefix, use_rdma, + rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level + ); + vitastor_c *self = new vitastor_c; + self->aio_set_fd_handler = aio_set_fd_handler; + self->aio_ctx = aio_context; + self->tfd = new timerfd_manager_t([self](int fd, bool wr, std::function callback) + { + if (callback != NULL) + { + self->handlers[fd] = { .fd = fd, .callback = callback }; + self->aio_set_fd_handler(self->aio_ctx, fd, false, + vitastor_c_read_handler, wr ? vitastor_c_write_handler : NULL, NULL, &self->handlers[fd]); + } + else + { + self->handlers.erase(fd); + self->aio_set_fd_handler(self->aio_ctx, fd, false, NULL, NULL, NULL, NULL); + } + }); + self->cli = new cluster_client_t(NULL, self->tfd, cfg_json); + return self; +} + +vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_host, const char *etcd_prefix, + int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level) +{ + json11::Json cfg_json = vitastor_c_common_config( + config_path, etcd_host, etcd_prefix, use_rdma, + rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level + ); + vitastor_c *self = new vitastor_c; + self->ringloop = new ring_loop_t(512); + self->epmgr = new epoll_manager_t(self->ringloop); + self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json); + return self; +} + +vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len) +{ + json11::Json::object cfg; + for (int i = 0; i < options_len-1; i += 2) + { + cfg[options[i]] = std::string(options[i+1]); + } + json11::Json cfg_json(cfg); + vitastor_c *self = new vitastor_c; + self->ringloop = new ring_loop_t(512); + self->epmgr = new epoll_manager_t(self->ringloop); + self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json); + return self; +} + +void vitastor_c_destroy(vitastor_c *client) +{ + delete client->cli; + if (client->epmgr) + delete client->epmgr; + else + delete client->tfd; + if (client->ringloop) + delete client->ringloop; + delete client; +} + +int vitastor_c_is_ready(vitastor_c *client) +{ + return client->cli->is_ready(); +} + +void vitastor_c_uring_wait_ready(vitastor_c *client) +{ + while (!client->cli->is_ready()) + { + client->ringloop->loop(); + if (client->cli->is_ready()) + break; + client->ringloop->wait(); + } +} + +void vitastor_c_uring_handle_events(vitastor_c *client) +{ + client->ringloop->loop(); +} + +void vitastor_c_uring_wait_events(vitastor_c *client) +{ + client->ringloop->wait(); +} + +static inline void vitastor_c_rw(bool write, vitastor_c *p, uint64_t inode, uint64_t offset, uint64_t len, + struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque) +{ + cluster_op_t *op = new cluster_op_t; + op->opcode = write ? OSD_OP_WRITE : OSD_OP_READ; + op->inode = inode; + op->offset = offset; + op->len = len; + for (int i = 0; i < iovcnt; i++) + { + op->iov.push_back(iov[i].iov_base, iov[i].iov_len); + } + op->callback = [cb, opaque](cluster_op_t *op) + { + cb(op->retval, opaque); + delete op; + }; + p->cli->execute(op); +} + +void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, + struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque) +{ + vitastor_c_rw(0, client, inode, offset, len, iov, iovcnt, cb, opaque); +} + +void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, + struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque) +{ + vitastor_c_rw(1, client, inode, offset, len, iov, iovcnt, cb, opaque); +} + +void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque) +{ + cluster_op_t *op = new cluster_op_t; + op->opcode = OSD_OP_SYNC; + op->callback = [cb, opaque](cluster_op_t *op) + { + cb(op->retval, opaque); + delete op; + }; + client->cli->execute(op); +} + +void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler cb, void *opaque) +{ + client->cli->on_ready([=]() + { + auto watch = client->cli->st_cli.watch_inode(std::string(image)); + cb((long)watch, opaque); + }); +} + +void vitastor_c_close_watch(vitastor_c *client, void *handle) +{ + client->cli->st_cli.close_watch((inode_watch_t*)handle); +} + +uint64_t vitastor_c_inode_get_size(void *handle) +{ + inode_watch_t *watch = (inode_watch_t*)handle; + return watch->cfg.size; +} + +uint64_t vitastor_c_inode_get_num(void *handle) +{ + inode_watch_t *watch = (inode_watch_t*)handle; + return watch->cfg.num; +} + +int vitastor_c_inode_get_readonly(void *handle) +{ + inode_watch_t *watch = (inode_watch_t*)handle; + return watch->cfg.readonly; +} + +} diff --git a/src/vitastor_c.h b/src/vitastor_c.h new file mode 100644 index 00000000..7c816dff --- /dev/null +++ b/src/vitastor_c.h @@ -0,0 +1,54 @@ +// Copyright (c) Vitaliy Filippov, 2019+ +// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details) + +// Simplified C client library for QEMU, fio and other external drivers + +#ifndef VITASTOR_QEMU_PROXY_H +#define VITASTOR_QEMU_PROXY_H + +#ifndef POOL_ID_BITS +#define POOL_ID_BITS 16 +#endif +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct vitastor_c; +typedef struct vitastor_c vitastor_c; + +typedef void VitastorIOHandler(long retval, void *opaque); + +// QEMU +typedef void IOHandler(void *opaque); +typedef void QEMUSetFDHandler(void *ctx, int fd, int is_external, IOHandler *fd_read, IOHandler *fd_write, void *poll_fn, void *opaque); + +vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context, + const char *config_path, const char *etcd_host, const char *etcd_prefix, + bool use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level); +vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_host, const char *etcd_prefix, + int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level); +vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len); +void vitastor_c_destroy(vitastor_c *client); +int vitastor_c_is_ready(vitastor_c *client); +void vitastor_c_uring_wait_ready(vitastor_c *client); +void vitastor_c_uring_handle_events(vitastor_c *client); +void vitastor_c_uring_wait_events(vitastor_c *client); +void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, + struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque); +void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, + struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque); +void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque); +void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler cb, void *opaque); +void vitastor_c_close_watch(vitastor_c *client, void *handle); +uint64_t vitastor_c_inode_get_size(void *handle); +uint64_t vitastor_c_inode_get_num(void *handle); +int vitastor_c_inode_get_readonly(void *handle); + +#ifdef __cplusplus +} +#endif + +#endif From 6ca8afffe59f84bddfd60909975d508c8e1aeb82 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 19 Jun 2021 01:00:52 +0300 Subject: [PATCH 05/12] Add CAS version parameter to the C wrapper --- src/fio_cluster.cpp | 15 ++++++++++----- src/qemu_driver.c | 14 ++++++++++---- src/vitastor_c.cpp | 39 ++++++++++++++++++++++++--------------- src/vitastor_c.h | 7 ++++--- 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/src/fio_cluster.cpp b/src/fio_cluster.cpp index 78419eaa..edc82822 100644 --- a/src/fio_cluster.cpp +++ b/src/fio_cluster.cpp @@ -186,10 +186,10 @@ static struct fio_option options[] = { }, }; -static void watch_callback(long retval, void *opaque) +static void watch_callback(void *opaque, long watch) { struct sec_data *bsd = (struct sec_data*)opaque; - bsd->watch = (void*)retval; + bsd->watch = (void*)watch; } static int sec_setup(struct thread_data *td) @@ -274,7 +274,7 @@ static int sec_init(struct thread_data *td) return 0; } -static void io_callback(long retval, void *opaque) +static void io_callback(void *opaque, long retval) { struct io_u *io = (struct io_u*)opaque; io->error = retval < 0 ? -retval : 0; @@ -288,6 +288,11 @@ static void io_callback(long retval, void *opaque) } } +static void read_callback(void *opaque, long retval, uint64_t version) +{ + io_callback(opaque, retval); +} + /* Begin read or write request. */ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io) { @@ -310,7 +315,7 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io) { case DDIR_READ: iov = { .iov_base = io->xfer_buf, .iov_len = io->xfer_buflen }; - vitastor_c_read(bsd->cli, inode, io->offset, io->xfer_buflen, &iov, 1, io_callback, io); + vitastor_c_read(bsd->cli, inode, io->offset, io->xfer_buflen, &iov, 1, read_callback, io); bsd->last_sync = false; break; case DDIR_WRITE: @@ -320,7 +325,7 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io) return FIO_Q_COMPLETED; } iov = { .iov_base = io->xfer_buf, .iov_len = io->xfer_buflen }; - vitastor_c_write(bsd->cli, inode, io->offset, io->xfer_buflen, &iov, 1, io_callback, io); + vitastor_c_write(bsd->cli, inode, io->offset, io->xfer_buflen, 0, &iov, 1, io_callback, io); bsd->last_sync = false; break; case DDIR_SYNC: diff --git a/src/qemu_driver.c b/src/qemu_driver.c index 4e647f26..5c282da1 100644 --- a/src/qemu_driver.c +++ b/src/qemu_driver.c @@ -66,7 +66,8 @@ typedef struct VitastorRPC } VitastorRPC; static void vitastor_co_init_task(BlockDriverState *bs, VitastorRPC *task); -static void vitastor_co_generic_bh_cb(long retval, void *opaque); +static void vitastor_co_generic_bh_cb(void *opaque, long retval); +static void vitastor_co_read_cb(void *opaque, long retval, uint64_t version); static void vitastor_close(BlockDriverState *bs); static char *qemu_rbd_next_tok(char *src, char delim, char **p) @@ -391,7 +392,7 @@ static void vitastor_co_init_task(BlockDriverState *bs, VitastorRPC *task) }; } -static void vitastor_co_generic_bh_cb(long retval, void *opaque) +static void vitastor_co_generic_bh_cb(void *opaque, long retval) { VitastorRPC *task = opaque; task->ret = retval; @@ -407,6 +408,11 @@ static void vitastor_co_generic_bh_cb(long retval, void *opaque) } } +static void vitastor_co_read_cb(void *opaque, long retval, uint64_t version) +{ + vitastor_co_generic_bh_cb(opaque, retval); +} + static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes, QEMUIOVector *iov, int flags) { VitastorClient *client = bs->opaque; @@ -416,7 +422,7 @@ static int coroutine_fn vitastor_co_preadv(BlockDriverState *bs, uint64_t offset uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode; qemu_mutex_lock(&client->mutex); - vitastor_c_read(client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_generic_bh_cb, &task); + vitastor_c_read(client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_read_cb, &task); qemu_mutex_unlock(&client->mutex); while (!task.complete) @@ -436,7 +442,7 @@ static int coroutine_fn vitastor_co_pwritev(BlockDriverState *bs, uint64_t offse uint64_t inode = client->watch ? vitastor_c_inode_get_num(client->watch) : client->inode; qemu_mutex_lock(&client->mutex); - vitastor_c_write(client->proxy, inode, offset, bytes, iov->iov, iov->niov, vitastor_co_generic_bh_cb, &task); + vitastor_c_write(client->proxy, inode, offset, bytes, 0, iov->iov, iov->niov, vitastor_co_generic_bh_cb, &task); qemu_mutex_unlock(&client->mutex); while (!task.complete) diff --git a/src/vitastor_c.cpp b/src/vitastor_c.cpp index 9f6cfa6d..9300888e 100644 --- a/src/vitastor_c.cpp +++ b/src/vitastor_c.cpp @@ -166,11 +166,11 @@ void vitastor_c_uring_wait_events(vitastor_c *client) client->ringloop->wait(); } -static inline void vitastor_c_rw(bool write, vitastor_c *p, uint64_t inode, uint64_t offset, uint64_t len, - struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque) +void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, + struct iovec *iov, int iovcnt, VitastorReadHandler cb, void *opaque) { cluster_op_t *op = new cluster_op_t; - op->opcode = write ? OSD_OP_WRITE : OSD_OP_READ; + op->opcode = OSD_OP_READ; op->inode = inode; op->offset = offset; op->len = len; @@ -180,22 +180,31 @@ static inline void vitastor_c_rw(bool write, vitastor_c *p, uint64_t inode, uint } op->callback = [cb, opaque](cluster_op_t *op) { - cb(op->retval, opaque); + cb(opaque, op->retval, op->version); delete op; }; - p->cli->execute(op); + client->cli->execute(op); } -void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, +void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version, struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque) { - vitastor_c_rw(0, client, inode, offset, len, iov, iovcnt, cb, opaque); -} - -void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, - struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque) -{ - vitastor_c_rw(1, client, inode, offset, len, iov, iovcnt, cb, opaque); + cluster_op_t *op = new cluster_op_t; + op->opcode = OSD_OP_WRITE; + op->inode = inode; + op->offset = offset; + op->len = len; + op->version = check_version; + for (int i = 0; i < iovcnt; i++) + { + op->iov.push_back(iov[i].iov_base, iov[i].iov_len); + } + op->callback = [cb, opaque](cluster_op_t *op) + { + cb(opaque, op->retval); + delete op; + }; + client->cli->execute(op); } void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque) @@ -204,7 +213,7 @@ void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque) op->opcode = OSD_OP_SYNC; op->callback = [cb, opaque](cluster_op_t *op) { - cb(op->retval, opaque); + cb(opaque, op->retval); delete op; }; client->cli->execute(op); @@ -215,7 +224,7 @@ void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler c client->cli->on_ready([=]() { auto watch = client->cli->st_cli.watch_inode(std::string(image)); - cb((long)watch, opaque); + cb(opaque, (long)watch); }); } diff --git a/src/vitastor_c.h b/src/vitastor_c.h index 7c816dff..f964c642 100644 --- a/src/vitastor_c.h +++ b/src/vitastor_c.h @@ -19,7 +19,8 @@ extern "C" { struct vitastor_c; typedef struct vitastor_c vitastor_c; -typedef void VitastorIOHandler(long retval, void *opaque); +typedef void VitastorReadHandler(void *opaque, long retval, uint64_t version); +typedef void VitastorIOHandler(void *opaque, long retval); // QEMU typedef void IOHandler(void *opaque); @@ -37,8 +38,8 @@ void vitastor_c_uring_wait_ready(vitastor_c *client); void vitastor_c_uring_handle_events(vitastor_c *client); void vitastor_c_uring_wait_events(vitastor_c *client); void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, - struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque); -void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, + struct iovec *iov, int iovcnt, VitastorReadHandler cb, void *opaque); +void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, uint64_t check_version, struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque); void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque); void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler cb, void *opaque); From aad7792d3feb2da20b0c1e3e22db5a08cd3324e0 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 20 Jun 2021 00:23:03 +0300 Subject: [PATCH 06/12] Check for loops in parent inode chains --- src/cluster_client.cpp | 7 +++++-- src/osd_primary.cpp | 9 +++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/cluster_client.cpp b/src/cluster_client.cpp index 7cf6419a..9dc098da 100644 --- a/src/cluster_client.cpp +++ b/src/cluster_client.cpp @@ -695,13 +695,16 @@ resume_3: // Check parent inode auto ino_it = st_cli.inode_config.find(op->cur_inode); while (ino_it != st_cli.inode_config.end() && ino_it->second.parent_id && - INODE_POOL(ino_it->second.parent_id) == INODE_POOL(op->cur_inode)) + INODE_POOL(ino_it->second.parent_id) == INODE_POOL(op->cur_inode) && + // Check for loops + ino_it->second.parent_id != op->inode) { // Skip parents from the same pool ino_it = st_cli.inode_config.find(ino_it->second.parent_id); } if (ino_it != st_cli.inode_config.end() && - ino_it->second.parent_id) + ino_it->second.parent_id && + ino_it->second.parent_id != op->inode) { // Continue reading from the parent inode op->cur_inode = ino_it->second.parent_id; diff --git a/src/osd_primary.cpp b/src/osd_primary.cpp index 52c463bb..edc7c387 100644 --- a/src/osd_primary.cpp +++ b/src/osd_primary.cpp @@ -67,7 +67,9 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op) } // Find parents from the same pool. Optimized reads only work within pools while (inode_it != st_cli.inode_config.end() && inode_it->second.parent_id && - INODE_POOL(inode_it->second.parent_id) == pg_it->second.pool_id) + INODE_POOL(inode_it->second.parent_id) == pg_it->second.pool_id && + // Check for loops + inode_it->second.parent_id != cur_op->req.rw.inode) { chain_size++; inode_it = st_cli.inode_config.find(inode_it->second.parent_id); @@ -123,7 +125,10 @@ bool osd_t::prepare_primary_rw(osd_op_t *cur_op) int chain_num = 0; op_data->read_chain[chain_num++] = cur_op->req.rw.inode; auto inode_it = st_cli.inode_config.find(cur_op->req.rw.inode); - while (inode_it != st_cli.inode_config.end() && inode_it->second.parent_id) + while (inode_it != st_cli.inode_config.end() && inode_it->second.parent_id && + INODE_POOL(inode_it->second.parent_id) == pg_it->second.pool_id && + // Check for loops + inode_it->second.parent_id != cur_op->req.rw.inode) { op_data->read_chain[chain_num++] = inode_it->second.parent_id; inode_it = st_cli.inode_config.find(inode_it->second.parent_id); From dfdf5c1f9cf078b7b042c1b383a5b1baf2fccd16 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 20 Jun 2021 00:23:56 +0300 Subject: [PATCH 07/12] Fix comments in mon.js --- mon/mon.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/mon/mon.js b/mon/mon.js index d8ea083c..ff213002 100644 --- a/mon/mon.js +++ b/mon/mon.js @@ -42,7 +42,7 @@ const etcd_tree = { config: { /* global: { // WARNING: NOT ALL OF THESE ARE ACTUALLY CONFIGURABLE HERE - // THIS IS JUST A POOR'S MAN CONFIG DOCUMENTATION + // THIS IS JUST A POOR MAN'S CONFIG DOCUMENTATION // etcd connection config_path: "/etc/vitastor/vitastor.conf", etcd_address: "10.0.115.10:2379/v3", @@ -257,11 +257,13 @@ const etcd_tree = { }, inode: { stats: { - /* : { - raw_used: uint64_t, // raw used bytes on OSDs - read: { count: uint64_t, usec: uint64_t, bytes: uint64_t }, - write: { count: uint64_t, usec: uint64_t, bytes: uint64_t }, - delete: { count: uint64_t, usec: uint64_t, bytes: uint64_t }, + /* : { + : { + raw_used: uint64_t, // raw used bytes on OSDs + read: { count: uint64_t, usec: uint64_t, bytes: uint64_t }, + write: { count: uint64_t, usec: uint64_t, bytes: uint64_t }, + delete: { count: uint64_t, usec: uint64_t, bytes: uint64_t }, + }, }, */ }, }, From 24f19c4b80901cdc575512ddef2d5585e6658011 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 27 Jun 2021 00:58:56 +0300 Subject: [PATCH 08/12] Add libvirt 7.0 patch --- cinder-driver/libvirt-7.0-vitastor.diff | 501 ++++++++++++++++++++++++ 1 file changed, 501 insertions(+) create mode 100644 cinder-driver/libvirt-7.0-vitastor.diff diff --git a/cinder-driver/libvirt-7.0-vitastor.diff b/cinder-driver/libvirt-7.0-vitastor.diff new file mode 100644 index 00000000..48723334 --- /dev/null +++ b/cinder-driver/libvirt-7.0-vitastor.diff @@ -0,0 +1,501 @@ +diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h +index 089e1e0..d7e7ef4 100644 +--- a/include/libvirt/libvirt-storage.h ++++ b/include/libvirt/libvirt-storage.h +@@ -245,6 +245,7 @@ typedef enum { + VIR_CONNECT_LIST_STORAGE_POOLS_ZFS = 1 << 17, + VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE = 1 << 18, + VIR_CONNECT_LIST_STORAGE_POOLS_ISCSI_DIRECT = 1 << 19, ++ VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR = 1 << 20, + } virConnectListAllStoragePoolsFlags; + + int virConnectListAllStoragePools(virConnectPtr conn, +diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c +index 01b7187..5b81e37 100644 +--- a/src/conf/domain_conf.c ++++ b/src/conf/domain_conf.c +@@ -31392,6 +31392,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSourcePtr src, + + case VIR_STORAGE_POOL_MPATH: + case VIR_STORAGE_POOL_RBD: ++ case VIR_STORAGE_POOL_VITASTOR: + case VIR_STORAGE_POOL_SHEEPDOG: + case VIR_STORAGE_POOL_GLUSTER: + case VIR_STORAGE_POOL_LAST: +diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c +index 0c50529..fe97574 100644 +--- a/src/conf/storage_conf.c ++++ b/src/conf/storage_conf.c +@@ -60,7 +60,7 @@ VIR_ENUM_IMPL(virStoragePool, + "logical", "disk", "iscsi", + "iscsi-direct", "scsi", "mpath", + "rbd", "sheepdog", "gluster", +- "zfs", "vstorage", ++ "zfs", "vstorage", "vitastor", + ); + + VIR_ENUM_IMPL(virStoragePoolFormatFileSystem, +@@ -249,6 +249,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = { + .formatToString = virStorageFileFormatTypeToString, + } + }, ++ {.poolType = VIR_STORAGE_POOL_VITASTOR, ++ .poolOptions = { ++ .flags = (VIR_STORAGE_POOL_SOURCE_HOST | ++ VIR_STORAGE_POOL_SOURCE_NETWORK | ++ VIR_STORAGE_POOL_SOURCE_NAME), ++ }, ++ .volOptions = { ++ .defaultFormat = VIR_STORAGE_FILE_RAW, ++ .formatFromString = virStorageVolumeFormatFromString, ++ .formatToString = virStorageFileFormatTypeToString, ++ } ++ }, + {.poolType = VIR_STORAGE_POOL_SHEEPDOG, + .poolOptions = { + .flags = (VIR_STORAGE_POOL_SOURCE_HOST | +@@ -551,6 +563,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt, + _("element 'name' is mandatory for RBD pool")); + goto cleanup; + } ++ if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) { ++ virReportError(VIR_ERR_XML_ERROR, "%s", ++ _("element 'name' is mandatory for Vitastor pool")); ++ return -1; ++ } + + if (options->formatFromString) { + g_autofree char *format = NULL; +@@ -1217,6 +1234,7 @@ virStoragePoolDefFormatBuf(virBufferPtr buf, + /* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor + * files, so they don't have a target */ + if (def->type != VIR_STORAGE_POOL_RBD && ++ def->type != VIR_STORAGE_POOL_VITASTOR && + def->type != VIR_STORAGE_POOL_SHEEPDOG && + def->type != VIR_STORAGE_POOL_GLUSTER && + def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) { +diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h +index ffd406e..8868a05 100644 +--- a/src/conf/storage_conf.h ++++ b/src/conf/storage_conf.h +@@ -110,6 +110,7 @@ typedef enum { + VIR_STORAGE_POOL_GLUSTER, /* Gluster device */ + VIR_STORAGE_POOL_ZFS, /* ZFS */ + VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */ ++ VIR_STORAGE_POOL_VITASTOR, /* Vitastor */ + + VIR_STORAGE_POOL_LAST, + } virStoragePoolType; +@@ -474,6 +475,7 @@ VIR_ENUM_DECL(virStoragePartedFs); + VIR_CONNECT_LIST_STORAGE_POOLS_SCSI | \ + VIR_CONNECT_LIST_STORAGE_POOLS_MPATH | \ + VIR_CONNECT_LIST_STORAGE_POOLS_RBD | \ ++ VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \ + VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \ + VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER | \ + VIR_CONNECT_LIST_STORAGE_POOLS_ZFS | \ +diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c +index 9fe8b3f..bf595b0 100644 +--- a/src/conf/virstorageobj.c ++++ b/src/conf/virstorageobj.c +@@ -1491,6 +1491,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload, + return 1; + break; + ++ case VIR_STORAGE_POOL_VITASTOR: + case VIR_STORAGE_POOL_RBD: + case VIR_STORAGE_POOL_LAST: + break; +@@ -1990,6 +1991,8 @@ virStoragePoolObjMatch(virStoragePoolObjPtr obj, + (obj->def->type == VIR_STORAGE_POOL_MPATH)) || + (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) && + (obj->def->type == VIR_STORAGE_POOL_RBD)) || ++ (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) && ++ (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) || + (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) && + (obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) || + (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) && +diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c +index 2a7cdca..f756be1 100644 +--- a/src/libvirt-storage.c ++++ b/src/libvirt-storage.c +@@ -92,6 +92,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool) + * VIR_CONNECT_LIST_STORAGE_POOLS_SCSI + * VIR_CONNECT_LIST_STORAGE_POOLS_MPATH + * VIR_CONNECT_LIST_STORAGE_POOLS_RBD ++ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR + * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG + * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER + * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS +diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c +index f9c6da2..81cb67d 100644 +--- a/src/qemu/qemu_block.c ++++ b/src/qemu/qemu_block.c +@@ -938,6 +938,30 @@ qemuBlockStorageSourceGetRBDProps(virStorageSourcePtr src, + } + + ++static virJSONValuePtr ++qemuBlockStorageSourceGetVitastorProps(virStorageSource *src, ++ bool onlytarget) ++{ ++ qemuDomainStorageSourcePrivatePtr srcPriv = QEMU_DOMAIN_STORAGE_SOURCE_PRIVATE(src); ++ g_autoptr(virJSONValue) servers = NULL; ++ virJSONValuePtr ret = NULL; ++ ++ if (src->nhosts > 0 && ++ !(servers = qemuBlockStorageSourceBuildHostsJSONInetSocketAddress(src))) ++ return NULL; ++ ++ if (virJSONValueObjectCreate(&ret, ++ "A:server", &servers, ++ "s:etcd_prefix", src->query, ++ "S:config_path", src->configFile, ++ "s:image", src->path, ++ NULL) < 0) ++ return NULL; ++ ++ return ret; ++} ++ ++ + static virJSONValuePtr + qemuBlockStorageSourceGetSheepdogProps(virStorageSourcePtr src) + { +@@ -1224,6 +1248,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSourcePtr src, + return NULL; + break; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: ++ driver = "vitastor"; ++ if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src, onlytarget))) ++ return NULL; ++ break; ++ + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + driver = "sheepdog"; + if (!(fileprops = qemuBlockStorageSourceGetSheepdogProps(src))) +@@ -2183,6 +2213,7 @@ qemuBlockGetBackingStoreString(virStorageSourcePtr src, + + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_RBD: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_VXHS: + case VIR_STORAGE_NET_PROTOCOL_NFS: + case VIR_STORAGE_NET_PROTOCOL_SSH: +@@ -2560,6 +2591,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSourcePtr src, + return -1; + break; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: ++ driver = "vitastor"; ++ if (!(location = qemuBlockStorageSourceGetVitastorProps(src, false))) ++ return -1; ++ break; ++ + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + driver = "sheepdog"; + if (!(location = qemuBlockStorageSourceGetSheepdogProps(src))) +diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c +index 6f970a3..10b39ca 100644 +--- a/src/qemu/qemu_command.c ++++ b/src/qemu/qemu_command.c +@@ -1034,6 +1034,43 @@ qemuBuildNetworkDriveStr(virStorageSourcePtr src, + ret = virBufferContentAndReset(&buf); + break; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: ++ if (strchr(src->path, ':')) { ++ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, ++ _("':' not allowed in Vitastor source volume name '%s'"), ++ src->path); ++ return NULL; ++ } ++ ++ virBufferStrcat(&buf, "vitastor:image=", src->path, NULL); ++ ++ if (src->nhosts > 0) { ++ virBufferAddLit(&buf, ":etcd_host="); ++ for (i = 0; i < src->nhosts; i++) { ++ if (i) ++ virBufferAddLit(&buf, ","); ++ ++ /* assume host containing : is ipv6 */ ++ if (strchr(src->hosts[i].name, ':')) ++ virBufferEscape(&buf, '\\', ":", "[%s]", ++ src->hosts[i].name); ++ else ++ virBufferAsprintf(&buf, "%s", src->hosts[i].name); ++ ++ if (src->hosts[i].port) ++ virBufferAsprintf(&buf, "\\:%u", src->hosts[i].port); ++ } ++ } ++ ++ if (src->configFile) ++ virBufferEscape(&buf, '\\', ":", ":config_path=%s", src->configFile); ++ ++ if (src->query) ++ virBufferEscape(&buf, '\\', ":", ":etcd_prefix=%s", src->query); ++ ++ ret = virBufferContentAndReset(&buf); ++ break; ++ + case VIR_STORAGE_NET_PROTOCOL_VXHS: + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("VxHS protocol does not support URI syntax")); +diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c +index 0765dc7..c69b1f1 100644 +--- a/src/qemu/qemu_domain.c ++++ b/src/qemu/qemu_domain.c +@@ -9704,6 +9704,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSourcePtr src, + break; + + case VIR_STORAGE_NET_PROTOCOL_RBD: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_GLUSTER: + case VIR_STORAGE_NET_PROTOCOL_ISCSI: +diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c +index ee333c3..674aa58 100644 +--- a/src/qemu/qemu_snapshot.c ++++ b/src/qemu/qemu_snapshot.c +@@ -403,6 +403,7 @@ qemuSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDefPtr snapdisk, + case VIR_STORAGE_NET_PROTOCOL_NONE: + case VIR_STORAGE_NET_PROTOCOL_NBD: + case VIR_STORAGE_NET_PROTOCOL_RBD: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_GLUSTER: + case VIR_STORAGE_NET_PROTOCOL_ISCSI: +@@ -493,6 +494,7 @@ qemuSnapshotPrepareDiskExternalActive(virDomainObjPtr vm, + case VIR_STORAGE_NET_PROTOCOL_NONE: + case VIR_STORAGE_NET_PROTOCOL_NBD: + case VIR_STORAGE_NET_PROTOCOL_RBD: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_ISCSI: + case VIR_STORAGE_NET_PROTOCOL_HTTP: +@@ -623,6 +625,7 @@ qemuSnapshotPrepareDiskInternal(virDomainDiskDefPtr disk, + case VIR_STORAGE_NET_PROTOCOL_NONE: + case VIR_STORAGE_NET_PROTOCOL_NBD: + case VIR_STORAGE_NET_PROTOCOL_RBD: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_GLUSTER: + case VIR_STORAGE_NET_PROTOCOL_ISCSI: +diff --git a/src/util/virstoragefile.c b/src/util/virstoragefile.c +index 0d3c2af..36e3afc 100644 +--- a/src/util/virstoragefile.c ++++ b/src/util/virstoragefile.c +@@ -91,6 +91,7 @@ VIR_ENUM_IMPL(virStorageNetProtocol, + "ssh", + "vxhs", + "nfs", ++ "vitastor", + ); + + VIR_ENUM_IMPL(virStorageNetHostTransport, +@@ -2880,6 +2881,75 @@ virStorageSourceParseRBDColonString(const char *rbdstr, + } + + ++static int ++virStorageSourceParseVitastorColonString(const char *colonstr, ++ virStorageSourcePtr src) ++{ ++ char *p, *e, *next; ++ g_autofree char *options = NULL; ++ ++ /* optionally skip the "vitastor:" prefix if provided */ ++ if (STRPREFIX(colonstr, "vitastor:")) ++ colonstr += strlen("vitastor:"); ++ ++ options = g_strdup(colonstr); ++ ++ p = options; ++ while (*p) { ++ /* find : delimiter or end of string */ ++ for (e = p; *e && *e != ':'; ++e) { ++ if (*e == '\\') { ++ e++; ++ if (*e == '\0') ++ break; ++ } ++ } ++ if (*e == '\0') { ++ next = e; /* last kv pair */ ++ } else { ++ next = e + 1; ++ *e = '\0'; ++ } ++ ++ if (STRPREFIX(p, "image=")) { ++ src->path = g_strdup(p + strlen("image=")); ++ } else if (STRPREFIX(p, "etcd_prefix=")) { ++ src->query = g_strdup(p + strlen("etcd_prefix=")); ++ } else if (STRPREFIX(p, "config_file=")) { ++ src->configFile = g_strdup(p + strlen("config_file=")); ++ } else if (STRPREFIX(p, "etcd_host=")) { ++ char *h, *sep; ++ ++ h = p + strlen("etcd_host="); ++ while (h < e) { ++ for (sep = h; sep < e; ++sep) { ++ if (*sep == '\\' && (sep[1] == ',' || ++ sep[1] == ';' || ++ sep[1] == ' ')) { ++ *sep = '\0'; ++ sep += 2; ++ break; ++ } ++ } ++ ++ if (virStorageSourceRBDAddHost(src, h) < 0) ++ return -1; ++ ++ h = sep; ++ } ++ } ++ ++ p = next; ++ } ++ ++ if (!src->path) { ++ return -1; ++ } ++ ++ return 0; ++} ++ ++ + static int + virStorageSourceParseNBDColonString(const char *nbdstr, + virStorageSourcePtr src) +@@ -2992,6 +3062,11 @@ virStorageSourceParseBackingColon(virStorageSourcePtr src, + return -1; + break; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: ++ if (virStorageSourceParseVitastorColonString(path, src) < 0) ++ return -1; ++ break; ++ + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_LAST: + case VIR_STORAGE_NET_PROTOCOL_NONE: +@@ -3581,6 +3656,54 @@ virStorageSourceParseBackingJSONRBD(virStorageSourcePtr src, + return 0; + } + ++static int ++virStorageSourceParseBackingJSONVitastor(virStorageSourcePtr src, ++ virJSONValuePtr json, ++ const char *jsonstr G_GNUC_UNUSED, ++ int opaque G_GNUC_UNUSED) ++{ ++ const char *filename; ++ const char *image = virJSONValueObjectGetString(json, "image"); ++ const char *conf = virJSONValueObjectGetString(json, "config_path"); ++ const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd_prefix"); ++ virJSONValuePtr servers = virJSONValueObjectGetArray(json, "server"); ++ size_t nservers; ++ size_t i; ++ ++ src->type = VIR_STORAGE_TYPE_NETWORK; ++ src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR; ++ ++ /* legacy syntax passed via 'filename' option */ ++ if ((filename = virJSONValueObjectGetString(json, "filename"))) ++ return virStorageSourceParseVitastorColonString(filename, src); ++ ++ if (!image) { ++ virReportError(VIR_ERR_INVALID_ARG, "%s", ++ _("missing image name in Vitastor backing volume " ++ "JSON specification")); ++ return -1; ++ } ++ ++ src->path = g_strdup(image); ++ src->configFile = g_strdup(conf); ++ src->query = g_strdup(etcd_prefix); ++ ++ if (servers) { ++ nservers = virJSONValueArraySize(servers); ++ ++ src->hosts = g_new0(virStorageNetHostDef, nservers); ++ src->nhosts = nservers; ++ ++ for (i = 0; i < nservers; i++) { ++ if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i, ++ virJSONValueArrayGet(servers, i)) < 0) ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ + static int + virStorageSourceParseBackingJSONRaw(virStorageSourcePtr src, + virJSONValuePtr json, +@@ -3759,6 +3882,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = { + {"sheepdog", false, virStorageSourceParseBackingJSONSheepdog, 0}, + {"ssh", false, virStorageSourceParseBackingJSONSSH, 0}, + {"rbd", false, virStorageSourceParseBackingJSONRBD, 0}, ++ {"vitastor", false, virStorageSourceParseBackingJSONVitastor, 0}, + {"raw", true, virStorageSourceParseBackingJSONRaw, 0}, + {"nfs", false, virStorageSourceParseBackingJSONNFS, 0}, + {"vxhs", false, virStorageSourceParseBackingJSONVxHS, 0}, +@@ -4503,6 +4627,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol) + case VIR_STORAGE_NET_PROTOCOL_GLUSTER: + return 24007; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_RBD: + /* we don't provide a default for RBD */ + return 0; +diff --git a/src/util/virstoragefile.h b/src/util/virstoragefile.h +index 5689c39..3eb4e3c 100644 +--- a/src/util/virstoragefile.h ++++ b/src/util/virstoragefile.h +@@ -136,6 +136,7 @@ typedef enum { + VIR_STORAGE_NET_PROTOCOL_SSH, + VIR_STORAGE_NET_PROTOCOL_VXHS, + VIR_STORAGE_NET_PROTOCOL_NFS, ++ VIR_STORAGE_NET_PROTOCOL_VITASTOR, + + VIR_STORAGE_NET_PROTOCOL_LAST + } virStorageNetProtocol; +diff --git a/tests/storagepoolcapsschemadata/poolcaps-fs.xml b/tests/storagepoolcapsschemadata/poolcaps-fs.xml +index eee75af..8bd0a57 100644 +--- a/tests/storagepoolcapsschemadata/poolcaps-fs.xml ++++ b/tests/storagepoolcapsschemadata/poolcaps-fs.xml +@@ -204,4 +204,11 @@ + + + ++ ++ ++ ++ ++ ++ ++ + +diff --git a/tests/storagepoolcapsschemadata/poolcaps-full.xml b/tests/storagepoolcapsschemadata/poolcaps-full.xml +index 805950a..852df0d 100644 +--- a/tests/storagepoolcapsschemadata/poolcaps-full.xml ++++ b/tests/storagepoolcapsschemadata/poolcaps-full.xml +@@ -204,4 +204,11 @@ + + + ++ ++ ++ ++ ++ ++ ++ + From 1d00c17d68cb972568acacd1757a2d55960f0615 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 27 Jun 2021 10:58:12 +0300 Subject: [PATCH 09/12] Add libvirt 7.5 patch --- cinder-driver/libvirt-7.0-vitastor.diff | 44 +- cinder-driver/libvirt-7.5-vitastor.diff | 577 ++++++++++++++++++++++++ 2 files changed, 611 insertions(+), 10 deletions(-) create mode 100644 cinder-driver/libvirt-7.5-vitastor.diff diff --git a/cinder-driver/libvirt-7.0-vitastor.diff b/cinder-driver/libvirt-7.0-vitastor.diff index 48723334..282fb5fa 100644 --- a/cinder-driver/libvirt-7.0-vitastor.diff +++ b/cinder-driver/libvirt-7.0-vitastor.diff @@ -128,18 +128,16 @@ index 2a7cdca..f756be1 100644 * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c -index f9c6da2..81cb67d 100644 +index f9c6da2..92093e4 100644 --- a/src/qemu/qemu_block.c +++ b/src/qemu/qemu_block.c -@@ -938,6 +938,30 @@ qemuBlockStorageSourceGetRBDProps(virStorageSourcePtr src, +@@ -938,6 +938,28 @@ qemuBlockStorageSourceGetRBDProps(virStorageSourcePtr src, } +static virJSONValuePtr -+qemuBlockStorageSourceGetVitastorProps(virStorageSource *src, -+ bool onlytarget) ++qemuBlockStorageSourceGetVitastorProps(virStorageSource *src) +{ -+ qemuDomainStorageSourcePrivatePtr srcPriv = QEMU_DOMAIN_STORAGE_SOURCE_PRIVATE(src); + g_autoptr(virJSONValue) servers = NULL; + virJSONValuePtr ret = NULL; + @@ -162,20 +160,20 @@ index f9c6da2..81cb67d 100644 static virJSONValuePtr qemuBlockStorageSourceGetSheepdogProps(virStorageSourcePtr src) { -@@ -1224,6 +1248,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSourcePtr src, +@@ -1224,6 +1246,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSourcePtr src, return NULL; break; + case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + driver = "vitastor"; -+ if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src, onlytarget))) ++ if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src))) + return NULL; + break; + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: driver = "sheepdog"; if (!(fileprops = qemuBlockStorageSourceGetSheepdogProps(src))) -@@ -2183,6 +2213,7 @@ qemuBlockGetBackingStoreString(virStorageSourcePtr src, +@@ -2183,6 +2211,7 @@ qemuBlockGetBackingStoreString(virStorageSourcePtr src, case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: case VIR_STORAGE_NET_PROTOCOL_RBD: @@ -183,13 +181,13 @@ index f9c6da2..81cb67d 100644 case VIR_STORAGE_NET_PROTOCOL_VXHS: case VIR_STORAGE_NET_PROTOCOL_NFS: case VIR_STORAGE_NET_PROTOCOL_SSH: -@@ -2560,6 +2591,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSourcePtr src, +@@ -2560,6 +2589,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSourcePtr src, return -1; break; + case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + driver = "vitastor"; -+ if (!(location = qemuBlockStorageSourceGetVitastorProps(src, false))) ++ if (!(location = qemuBlockStorageSourceGetVitastorProps(src))) + return -1; + break; + @@ -284,6 +282,18 @@ index ee333c3..674aa58 100644 case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: case VIR_STORAGE_NET_PROTOCOL_GLUSTER: case VIR_STORAGE_NET_PROTOCOL_ISCSI: +diff --git a/src/test/test_driver.c b/src/test/test_driver.c +index 29c4c86..a27ad94 100644 +--- a/src/test/test_driver.c ++++ b/src/test/test_driver.c +@@ -7096,6 +7096,7 @@ testStorageVolumeTypeForPool(int pooltype) + case VIR_STORAGE_POOL_ISCSI_DIRECT: + case VIR_STORAGE_POOL_GLUSTER: + case VIR_STORAGE_POOL_RBD: ++ case VIR_STORAGE_POOL_VITASTOR: + return VIR_STORAGE_VOL_NETWORK; + case VIR_STORAGE_POOL_LOGICAL: + case VIR_STORAGE_POOL_DISK: diff --git a/src/util/virstoragefile.c b/src/util/virstoragefile.c index 0d3c2af..36e3afc 100644 --- a/src/util/virstoragefile.c @@ -499,3 +509,17 @@ index 805950a..852df0d 100644 + + +diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c +index 7835fa6..8841fcf 100644 +--- a/tools/virsh-pool.c ++++ b/tools/virsh-pool.c +@@ -1237,6 +1237,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd G_GNUC_UNUSED) + case VIR_STORAGE_POOL_VSTORAGE: + flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE; + break; ++ case VIR_STORAGE_POOL_VITASTOR: ++ flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR; ++ break; + case VIR_STORAGE_POOL_LAST: + break; + } diff --git a/cinder-driver/libvirt-7.5-vitastor.diff b/cinder-driver/libvirt-7.5-vitastor.diff new file mode 100644 index 00000000..eab7d06c --- /dev/null +++ b/cinder-driver/libvirt-7.5-vitastor.diff @@ -0,0 +1,577 @@ +diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h +index 089e1e0..d7e7ef4 100644 +--- a/include/libvirt/libvirt-storage.h ++++ b/include/libvirt/libvirt-storage.h +@@ -245,6 +245,7 @@ typedef enum { + VIR_CONNECT_LIST_STORAGE_POOLS_ZFS = 1 << 17, + VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE = 1 << 18, + VIR_CONNECT_LIST_STORAGE_POOLS_ISCSI_DIRECT = 1 << 19, ++ VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR = 1 << 20, + } virConnectListAllStoragePoolsFlags; + + int virConnectListAllStoragePools(virConnectPtr conn, +diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c +index d78f846..97f4d6d 100644 +--- a/src/conf/domain_conf.c ++++ b/src/conf/domain_conf.c +@@ -30775,6 +30775,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src, + + case VIR_STORAGE_POOL_MPATH: + case VIR_STORAGE_POOL_RBD: ++ case VIR_STORAGE_POOL_VITASTOR: + case VIR_STORAGE_POOL_SHEEPDOG: + case VIR_STORAGE_POOL_GLUSTER: + case VIR_STORAGE_POOL_LAST: +diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c +index 2aa9a3d..166ca1f 100644 +--- a/src/conf/storage_conf.c ++++ b/src/conf/storage_conf.c +@@ -60,7 +60,7 @@ VIR_ENUM_IMPL(virStoragePool, + "logical", "disk", "iscsi", + "iscsi-direct", "scsi", "mpath", + "rbd", "sheepdog", "gluster", +- "zfs", "vstorage", ++ "zfs", "vstorage", "vitastor", + ); + + VIR_ENUM_IMPL(virStoragePoolFormatFileSystem, +@@ -246,6 +246,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = { + .formatToString = virStorageFileFormatTypeToString, + } + }, ++ {.poolType = VIR_STORAGE_POOL_VITASTOR, ++ .poolOptions = { ++ .flags = (VIR_STORAGE_POOL_SOURCE_HOST | ++ VIR_STORAGE_POOL_SOURCE_NETWORK | ++ VIR_STORAGE_POOL_SOURCE_NAME), ++ }, ++ .volOptions = { ++ .defaultFormat = VIR_STORAGE_FILE_RAW, ++ .formatFromString = virStorageVolumeFormatFromString, ++ .formatToString = virStorageFileFormatTypeToString, ++ } ++ }, + {.poolType = VIR_STORAGE_POOL_SHEEPDOG, + .poolOptions = { + .flags = (VIR_STORAGE_POOL_SOURCE_HOST | +@@ -546,6 +558,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt, + _("element 'name' is mandatory for RBD pool")); + return -1; + } ++ if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) { ++ virReportError(VIR_ERR_XML_ERROR, "%s", ++ _("element 'name' is mandatory for Vitastor pool")); ++ return -1; ++ } + + if (options->formatFromString) { + g_autofree char *format = NULL; +@@ -1182,6 +1199,7 @@ virStoragePoolDefFormatBuf(virBuffer *buf, + /* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor + * files, so they don't have a target */ + if (def->type != VIR_STORAGE_POOL_RBD && ++ def->type != VIR_STORAGE_POOL_VITASTOR && + def->type != VIR_STORAGE_POOL_SHEEPDOG && + def->type != VIR_STORAGE_POOL_GLUSTER && + def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) { +diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h +index 76efaac..928149a 100644 +--- a/src/conf/storage_conf.h ++++ b/src/conf/storage_conf.h +@@ -106,6 +106,7 @@ typedef enum { + VIR_STORAGE_POOL_GLUSTER, /* Gluster device */ + VIR_STORAGE_POOL_ZFS, /* ZFS */ + VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */ ++ VIR_STORAGE_POOL_VITASTOR, /* Vitastor */ + + VIR_STORAGE_POOL_LAST, + } virStoragePoolType; +@@ -465,6 +466,7 @@ VIR_ENUM_DECL(virStoragePartedFs); + VIR_CONNECT_LIST_STORAGE_POOLS_SCSI | \ + VIR_CONNECT_LIST_STORAGE_POOLS_MPATH | \ + VIR_CONNECT_LIST_STORAGE_POOLS_RBD | \ ++ VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \ + VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \ + VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER | \ + VIR_CONNECT_LIST_STORAGE_POOLS_ZFS | \ +diff --git a/src/conf/storage_source_conf.c b/src/conf/storage_source_conf.c +index 5ca06fa..05ded49 100644 +--- a/src/conf/storage_source_conf.c ++++ b/src/conf/storage_source_conf.c +@@ -85,6 +85,7 @@ VIR_ENUM_IMPL(virStorageNetProtocol, + "ssh", + "vxhs", + "nfs", ++ "vitastor", + ); + + +@@ -1262,6 +1263,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol) + case VIR_STORAGE_NET_PROTOCOL_GLUSTER: + return 24007; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_RBD: + /* we don't provide a default for RBD */ + return 0; +diff --git a/src/conf/storage_source_conf.h b/src/conf/storage_source_conf.h +index 389c7b5..dbf02e3 100644 +--- a/src/conf/storage_source_conf.h ++++ b/src/conf/storage_source_conf.h +@@ -127,6 +127,7 @@ typedef enum { + VIR_STORAGE_NET_PROTOCOL_SSH, + VIR_STORAGE_NET_PROTOCOL_VXHS, + VIR_STORAGE_NET_PROTOCOL_NFS, ++ VIR_STORAGE_NET_PROTOCOL_VITASTOR, + + VIR_STORAGE_NET_PROTOCOL_LAST + } virStorageNetProtocol; +diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c +index 24957d6..4520a73 100644 +--- a/src/conf/virstorageobj.c ++++ b/src/conf/virstorageobj.c +@@ -1487,6 +1487,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload, + return 1; + break; + ++ case VIR_STORAGE_POOL_VITASTOR: + case VIR_STORAGE_POOL_RBD: + case VIR_STORAGE_POOL_LAST: + break; +@@ -1986,6 +1987,8 @@ virStoragePoolObjMatch(virStoragePoolObj *obj, + (obj->def->type == VIR_STORAGE_POOL_MPATH)) || + (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) && + (obj->def->type == VIR_STORAGE_POOL_RBD)) || ++ (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) && ++ (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) || + (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) && + (obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) || + (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) && +diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c +index 2a7cdca..f756be1 100644 +--- a/src/libvirt-storage.c ++++ b/src/libvirt-storage.c +@@ -92,6 +92,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool) + * VIR_CONNECT_LIST_STORAGE_POOLS_SCSI + * VIR_CONNECT_LIST_STORAGE_POOLS_MPATH + * VIR_CONNECT_LIST_STORAGE_POOLS_RBD ++ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR + * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG + * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER + * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS +diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c +index 56cb9ab..dfb31b9 100644 +--- a/src/libxl/libxl_conf.c ++++ b/src/libxl/libxl_conf.c +@@ -972,6 +972,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSource *src, + case VIR_STORAGE_NET_PROTOCOL_SSH: + case VIR_STORAGE_NET_PROTOCOL_VXHS: + case VIR_STORAGE_NET_PROTOCOL_NFS: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_LAST: + case VIR_STORAGE_NET_PROTOCOL_NONE: + virReportError(VIR_ERR_NO_SUPPORT, +diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c +index c0905b0..c172378 100644 +--- a/src/libxl/xen_xl.c ++++ b/src/libxl/xen_xl.c +@@ -1540,6 +1540,7 @@ xenFormatXLDiskSrcNet(virStorageSource *src) + case VIR_STORAGE_NET_PROTOCOL_SSH: + case VIR_STORAGE_NET_PROTOCOL_VXHS: + case VIR_STORAGE_NET_PROTOCOL_NFS: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_LAST: + case VIR_STORAGE_NET_PROTOCOL_NONE: + virReportError(VIR_ERR_NO_SUPPORT, +diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c +index 6627d04..277b04e 100644 +--- a/src/qemu/qemu_block.c ++++ b/src/qemu/qemu_block.c +@@ -928,6 +928,28 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src, + } + + ++static virJSONValue * ++qemuBlockStorageSourceGetVitastorProps(virStorageSource *src) ++{ ++ g_autoptr(virJSONValue) servers = NULL; ++ virJSONValue *ret = NULL; ++ ++ if (src->nhosts > 0 && ++ !(servers = qemuBlockStorageSourceBuildHostsJSONInetSocketAddress(src))) ++ return NULL; ++ ++ if (virJSONValueObjectCreate(&ret, ++ "A:etcd_address", &servers, ++ "s:etcd_prefix", src->query, ++ "S:config_path", src->configFile, ++ "s:image", src->path, ++ NULL) < 0) ++ return NULL; ++ ++ return ret; ++} ++ ++ + static virJSONValue * + qemuBlockStorageSourceGetSheepdogProps(virStorageSource *src) + { +@@ -1218,6 +1240,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src, + return NULL; + break; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: ++ driver = "vitastor"; ++ if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src))) ++ return NULL; ++ break; ++ + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + driver = "sheepdog"; + if (!(fileprops = qemuBlockStorageSourceGetSheepdogProps(src))) +@@ -2231,6 +2259,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src, + + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_RBD: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_VXHS: + case VIR_STORAGE_NET_PROTOCOL_NFS: + case VIR_STORAGE_NET_PROTOCOL_SSH: +@@ -2608,6 +2637,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src, + return -1; + break; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: ++ driver = "vitastor"; ++ if (!(location = qemuBlockStorageSourceGetVitastorProps(src))) ++ return -1; ++ break; ++ + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + driver = "sheepdog"; + if (!(location = qemuBlockStorageSourceGetSheepdogProps(src))) +diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c +index ea51369..8258632 100644 +--- a/src/qemu/qemu_command.c ++++ b/src/qemu/qemu_command.c +@@ -1074,6 +1074,43 @@ qemuBuildNetworkDriveStr(virStorageSource *src, + ret = virBufferContentAndReset(&buf); + break; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: ++ if (strchr(src->path, ':')) { ++ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, ++ _("':' not allowed in Vitastor source volume name '%s'"), ++ src->path); ++ return NULL; ++ } ++ ++ virBufferStrcat(&buf, "vitastor:image=", src->path, NULL); ++ ++ if (src->nhosts > 0) { ++ virBufferAddLit(&buf, ":etcd_host="); ++ for (i = 0; i < src->nhosts; i++) { ++ if (i) ++ virBufferAddLit(&buf, ","); ++ ++ /* assume host containing : is ipv6 */ ++ if (strchr(src->hosts[i].name, ':')) ++ virBufferEscape(&buf, '\\', ":", "[%s]", ++ src->hosts[i].name); ++ else ++ virBufferAsprintf(&buf, "%s", src->hosts[i].name); ++ ++ if (src->hosts[i].port) ++ virBufferAsprintf(&buf, "\\:%u", src->hosts[i].port); ++ } ++ } ++ ++ if (src->configFile) ++ virBufferEscape(&buf, '\\', ":", ":config_path=%s", src->configFile); ++ ++ if (src->query) ++ virBufferEscape(&buf, '\\', ":", ":etcd_prefix=%s", src->query); ++ ++ ret = virBufferContentAndReset(&buf); ++ break; ++ + case VIR_STORAGE_NET_PROTOCOL_VXHS: + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("VxHS protocol does not support URI syntax")); +diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c +index fc60e15..88bcb26 100644 +--- a/src/qemu/qemu_domain.c ++++ b/src/qemu/qemu_domain.c +@@ -10027,6 +10027,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src, + break; + + case VIR_STORAGE_NET_PROTOCOL_RBD: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_GLUSTER: + case VIR_STORAGE_NET_PROTOCOL_ISCSI: +diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c +index 4e74ddd..14e5f2e 100644 +--- a/src/qemu/qemu_snapshot.c ++++ b/src/qemu/qemu_snapshot.c +@@ -402,6 +402,7 @@ qemuSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDef *snapdisk, + case VIR_STORAGE_NET_PROTOCOL_NONE: + case VIR_STORAGE_NET_PROTOCOL_NBD: + case VIR_STORAGE_NET_PROTOCOL_RBD: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_GLUSTER: + case VIR_STORAGE_NET_PROTOCOL_ISCSI: +@@ -494,6 +495,7 @@ qemuSnapshotPrepareDiskExternalActive(virDomainObj *vm, + case VIR_STORAGE_NET_PROTOCOL_NONE: + case VIR_STORAGE_NET_PROTOCOL_NBD: + case VIR_STORAGE_NET_PROTOCOL_RBD: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_ISCSI: + case VIR_STORAGE_NET_PROTOCOL_HTTP: +@@ -647,6 +649,7 @@ qemuSnapshotPrepareDiskInternal(virDomainDiskDef *disk, + case VIR_STORAGE_NET_PROTOCOL_NONE: + case VIR_STORAGE_NET_PROTOCOL_NBD: + case VIR_STORAGE_NET_PROTOCOL_RBD: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_GLUSTER: + case VIR_STORAGE_NET_PROTOCOL_ISCSI: +diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c +index c2ff4b8..70d0689 100644 +--- a/src/storage/storage_driver.c ++++ b/src/storage/storage_driver.c +@@ -1644,6 +1644,7 @@ storageVolLookupByPathCallback(virStoragePoolObj *obj, + + case VIR_STORAGE_POOL_GLUSTER: + case VIR_STORAGE_POOL_RBD: ++ case VIR_STORAGE_POOL_VITASTOR: + case VIR_STORAGE_POOL_SHEEPDOG: + case VIR_STORAGE_POOL_ZFS: + case VIR_STORAGE_POOL_LAST: +diff --git a/src/storage_file/storage_source_backingstore.c b/src/storage_file/storage_source_backingstore.c +index e48ae72..d7a9b72 100644 +--- a/src/storage_file/storage_source_backingstore.c ++++ b/src/storage_file/storage_source_backingstore.c +@@ -284,6 +284,75 @@ virStorageSourceParseRBDColonString(const char *rbdstr, + } + + ++static int ++virStorageSourceParseVitastorColonString(const char *colonstr, ++ virStorageSource *src) ++{ ++ char *p, *e, *next; ++ g_autofree char *options = NULL; ++ ++ /* optionally skip the "vitastor:" prefix if provided */ ++ if (STRPREFIX(colonstr, "vitastor:")) ++ colonstr += strlen("vitastor:"); ++ ++ options = g_strdup(colonstr); ++ ++ p = options; ++ while (*p) { ++ /* find : delimiter or end of string */ ++ for (e = p; *e && *e != ':'; ++e) { ++ if (*e == '\\') { ++ e++; ++ if (*e == '\0') ++ break; ++ } ++ } ++ if (*e == '\0') { ++ next = e; /* last kv pair */ ++ } else { ++ next = e + 1; ++ *e = '\0'; ++ } ++ ++ if (STRPREFIX(p, "image=")) { ++ src->path = g_strdup(p + strlen("image=")); ++ } else if (STRPREFIX(p, "etcd_prefix=")) { ++ src->query = g_strdup(p + strlen("etcd_prefix=")); ++ } else if (STRPREFIX(p, "config_file=")) { ++ src->configFile = g_strdup(p + strlen("config_file=")); ++ } else if (STRPREFIX(p, "etcd_host=")) { ++ char *h, *sep; ++ ++ h = p + strlen("etcd_host="); ++ while (h < e) { ++ for (sep = h; sep < e; ++sep) { ++ if (*sep == '\\' && (sep[1] == ',' || ++ sep[1] == ';' || ++ sep[1] == ' ')) { ++ *sep = '\0'; ++ sep += 2; ++ break; ++ } ++ } ++ ++ if (virStorageSourceRBDAddHost(src, h) < 0) ++ return -1; ++ ++ h = sep; ++ } ++ } ++ ++ p = next; ++ } ++ ++ if (!src->path) { ++ return -1; ++ } ++ ++ return 0; ++} ++ ++ + static int + virStorageSourceParseNBDColonString(const char *nbdstr, + virStorageSource *src) +@@ -396,6 +465,11 @@ virStorageSourceParseBackingColon(virStorageSource *src, + return -1; + break; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: ++ if (virStorageSourceParseVitastorColonString(path, src) < 0) ++ return -1; ++ break; ++ + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_LAST: + case VIR_STORAGE_NET_PROTOCOL_NONE: +@@ -984,6 +1058,54 @@ virStorageSourceParseBackingJSONRBD(virStorageSource *src, + return 0; + } + ++static int ++virStorageSourceParseBackingJSONVitastor(virStorageSource *src, ++ virJSONValue *json, ++ const char *jsonstr G_GNUC_UNUSED, ++ int opaque G_GNUC_UNUSED) ++{ ++ const char *filename; ++ const char *image = virJSONValueObjectGetString(json, "image"); ++ const char *conf = virJSONValueObjectGetString(json, "config_path"); ++ const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd_prefix"); ++ virJSONValue *servers = virJSONValueObjectGetArray(json, "server"); ++ size_t nservers; ++ size_t i; ++ ++ src->type = VIR_STORAGE_TYPE_NETWORK; ++ src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR; ++ ++ /* legacy syntax passed via 'filename' option */ ++ if ((filename = virJSONValueObjectGetString(json, "filename"))) ++ return virStorageSourceParseVitastorColonString(filename, src); ++ ++ if (!image) { ++ virReportError(VIR_ERR_INVALID_ARG, "%s", ++ _("missing image name in Vitastor backing volume " ++ "JSON specification")); ++ return -1; ++ } ++ ++ src->path = g_strdup(image); ++ src->configFile = g_strdup(conf); ++ src->query = g_strdup(etcd_prefix); ++ ++ if (servers) { ++ nservers = virJSONValueArraySize(servers); ++ ++ src->hosts = g_new0(virStorageNetHostDef, nservers); ++ src->nhosts = nservers; ++ ++ for (i = 0; i < nservers; i++) { ++ if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i, ++ virJSONValueArrayGet(servers, i)) < 0) ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ + static int + virStorageSourceParseBackingJSONRaw(virStorageSource *src, + virJSONValue *json, +@@ -1162,6 +1284,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = { + {"sheepdog", false, virStorageSourceParseBackingJSONSheepdog, 0}, + {"ssh", false, virStorageSourceParseBackingJSONSSH, 0}, + {"rbd", false, virStorageSourceParseBackingJSONRBD, 0}, ++ {"vitastor", false, virStorageSourceParseBackingJSONVitastor, 0}, + {"raw", true, virStorageSourceParseBackingJSONRaw, 0}, + {"nfs", false, virStorageSourceParseBackingJSONNFS, 0}, + {"vxhs", false, virStorageSourceParseBackingJSONVxHS, 0}, +diff --git a/src/test/test_driver.c b/src/test/test_driver.c +index ef0ddab..2173dc3 100644 +--- a/src/test/test_driver.c ++++ b/src/test/test_driver.c +@@ -7131,6 +7131,7 @@ testStorageVolumeTypeForPool(int pooltype) + case VIR_STORAGE_POOL_ISCSI_DIRECT: + case VIR_STORAGE_POOL_GLUSTER: + case VIR_STORAGE_POOL_RBD: ++ case VIR_STORAGE_POOL_VITASTOR: + return VIR_STORAGE_VOL_NETWORK; + case VIR_STORAGE_POOL_LOGICAL: + case VIR_STORAGE_POOL_DISK: +diff --git a/tests/storagepoolcapsschemadata/poolcaps-fs.xml b/tests/storagepoolcapsschemadata/poolcaps-fs.xml +index eee75af..8bd0a57 100644 +--- a/tests/storagepoolcapsschemadata/poolcaps-fs.xml ++++ b/tests/storagepoolcapsschemadata/poolcaps-fs.xml +@@ -204,4 +204,11 @@ + + + ++ ++ ++ ++ ++ ++ ++ + +diff --git a/tests/storagepoolcapsschemadata/poolcaps-full.xml b/tests/storagepoolcapsschemadata/poolcaps-full.xml +index 805950a..852df0d 100644 +--- a/tests/storagepoolcapsschemadata/poolcaps-full.xml ++++ b/tests/storagepoolcapsschemadata/poolcaps-full.xml +@@ -204,4 +204,11 @@ + + + ++ ++ ++ ++ ++ ++ ++ + +diff --git a/tests/storagepoolxml2argvtest.c b/tests/storagepoolxml2argvtest.c +index 449b745..7f95cc8 100644 +--- a/tests/storagepoolxml2argvtest.c ++++ b/tests/storagepoolxml2argvtest.c +@@ -68,6 +68,7 @@ testCompareXMLToArgvFiles(bool shouldFail, + case VIR_STORAGE_POOL_GLUSTER: + case VIR_STORAGE_POOL_ZFS: + case VIR_STORAGE_POOL_VSTORAGE: ++ case VIR_STORAGE_POOL_VITASTOR: + case VIR_STORAGE_POOL_LAST: + default: + VIR_TEST_DEBUG("pool type '%s' has no xml2argv test", defTypeStr); +diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c +index 18f3839..c8e1436 100644 +--- a/tools/virsh-pool.c ++++ b/tools/virsh-pool.c +@@ -1231,6 +1231,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd G_GNUC_UNUSED) + case VIR_STORAGE_POOL_VSTORAGE: + flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE; + break; ++ case VIR_STORAGE_POOL_VITASTOR: ++ flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR; ++ break; + case VIR_STORAGE_POOL_LAST: + break; + } From 24301b116c2e49cc2837e9d1adc42db641366c44 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sun, 27 Jun 2021 18:43:29 +0300 Subject: [PATCH 10/12] Add libvirt 5.0 patch --- cinder-driver/libvirt-5.0-vitastor.diff | 526 ++++++++++++++++++++++++ 1 file changed, 526 insertions(+) create mode 100644 cinder-driver/libvirt-5.0-vitastor.diff diff --git a/cinder-driver/libvirt-5.0-vitastor.diff b/cinder-driver/libvirt-5.0-vitastor.diff new file mode 100644 index 00000000..af30809c --- /dev/null +++ b/cinder-driver/libvirt-5.0-vitastor.diff @@ -0,0 +1,526 @@ +commit 367f518e76c36e1d815b41f5a0f623b71877e13c +Author: Vitaliy Filippov +Date: Sun Jun 27 12:52:40 2021 +0300 + + Add Vitastor support + +diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h +index 4bf2b5f..dbc011b 100644 +--- a/include/libvirt/libvirt-storage.h ++++ b/include/libvirt/libvirt-storage.h +@@ -240,6 +240,7 @@ typedef enum { + VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER = 1 << 16, + VIR_CONNECT_LIST_STORAGE_POOLS_ZFS = 1 << 17, + VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE = 1 << 18, ++ VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR = 1 << 20, + } virConnectListAllStoragePoolsFlags; + + int virConnectListAllStoragePools(virConnectPtr conn, +diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c +index 222bb8c..7687a5d 100644 +--- a/src/conf/domain_conf.c ++++ b/src/conf/domain_conf.c +@@ -30930,6 +30930,7 @@ virDomainDiskTranslateSourcePool(virDomainDiskDefPtr def) + + case VIR_STORAGE_POOL_MPATH: + case VIR_STORAGE_POOL_RBD: ++ case VIR_STORAGE_POOL_VITASTOR: + case VIR_STORAGE_POOL_SHEEPDOG: + case VIR_STORAGE_POOL_GLUSTER: + case VIR_STORAGE_POOL_LAST: +diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c +index 55db7a9..7cbe937 100644 +--- a/src/conf/storage_conf.c ++++ b/src/conf/storage_conf.c +@@ -58,7 +58,7 @@ VIR_ENUM_IMPL(virStoragePool, + "logical", "disk", "iscsi", + "iscsi-direct", "scsi", "mpath", + "rbd", "sheepdog", "gluster", +- "zfs", "vstorage") ++ "zfs", "vstorage", "vitastor") + + VIR_ENUM_IMPL(virStoragePoolFormatFileSystem, + VIR_STORAGE_POOL_FS_LAST, +@@ -232,6 +232,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = { + .formatToString = virStorageFileFormatTypeToString, + } + }, ++ {.poolType = VIR_STORAGE_POOL_VITASTOR, ++ .poolOptions = { ++ .flags = (VIR_STORAGE_POOL_SOURCE_HOST | ++ VIR_STORAGE_POOL_SOURCE_NETWORK | ++ VIR_STORAGE_POOL_SOURCE_NAME), ++ }, ++ .volOptions = { ++ .defaultFormat = VIR_STORAGE_FILE_RAW, ++ .formatFromString = virStorageVolumeFormatFromString, ++ .formatToString = virStorageFileFormatTypeToString, ++ } ++ }, + {.poolType = VIR_STORAGE_POOL_SHEEPDOG, + .poolOptions = { + .flags = (VIR_STORAGE_POOL_SOURCE_HOST | +@@ -434,6 +446,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt, + _("element 'name' is mandatory for RBD pool")); + goto cleanup; + } ++ if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) { ++ virReportError(VIR_ERR_XML_ERROR, "%s", ++ _("element 'name' is mandatory for Vitastor pool")); ++ return -1; ++ } + + if (options->formatFromString) { + char *format = virXPathString("string(./format/@type)", ctxt); +@@ -1009,6 +1026,7 @@ virStoragePoolDefFormatBuf(virBufferPtr buf, + /* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor + * files, so they don't have a target */ + if (def->type != VIR_STORAGE_POOL_RBD && ++ def->type != VIR_STORAGE_POOL_VITASTOR && + def->type != VIR_STORAGE_POOL_SHEEPDOG && + def->type != VIR_STORAGE_POOL_GLUSTER && + def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) { +diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h +index dc0aa2a..ed4983d 100644 +--- a/src/conf/storage_conf.h ++++ b/src/conf/storage_conf.h +@@ -91,6 +91,7 @@ typedef enum { + VIR_STORAGE_POOL_GLUSTER, /* Gluster device */ + VIR_STORAGE_POOL_ZFS, /* ZFS */ + VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */ ++ VIR_STORAGE_POOL_VITASTOR, /* Vitastor */ + + VIR_STORAGE_POOL_LAST, + } virStoragePoolType; +@@ -422,6 +423,7 @@ VIR_ENUM_DECL(virStoragePartedFs) + VIR_CONNECT_LIST_STORAGE_POOLS_SCSI | \ + VIR_CONNECT_LIST_STORAGE_POOLS_MPATH | \ + VIR_CONNECT_LIST_STORAGE_POOLS_RBD | \ ++ VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \ + VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \ + VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER | \ + VIR_CONNECT_LIST_STORAGE_POOLS_ZFS | \ +diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c +index 6ea6a97..3ba45b9 100644 +--- a/src/conf/virstorageobj.c ++++ b/src/conf/virstorageobj.c +@@ -1478,6 +1478,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload, + return 1; + break; + ++ case VIR_STORAGE_POOL_VITASTOR: + case VIR_STORAGE_POOL_RBD: + case VIR_STORAGE_POOL_LAST: + break; +@@ -1971,6 +1972,8 @@ virStoragePoolObjMatch(virStoragePoolObjPtr obj, + (obj->def->type == VIR_STORAGE_POOL_MPATH)) || + (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) && + (obj->def->type == VIR_STORAGE_POOL_RBD)) || ++ (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) && ++ (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) || + (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) && + (obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) || + (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) && +diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c +index 2ea3e94..d5d2273 100644 +--- a/src/libvirt-storage.c ++++ b/src/libvirt-storage.c +@@ -92,6 +92,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool) + * VIR_CONNECT_LIST_STORAGE_POOLS_SCSI + * VIR_CONNECT_LIST_STORAGE_POOLS_MPATH + * VIR_CONNECT_LIST_STORAGE_POOLS_RBD ++ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR + * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG + * + * Returns the number of storage pools found or -1 and sets @pools to +diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c +index 73e988a..ab7bb81 100644 +--- a/src/libxl/libxl_conf.c ++++ b/src/libxl/libxl_conf.c +@@ -905,6 +905,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSourcePtr src, + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_SSH: + case VIR_STORAGE_NET_PROTOCOL_VXHS: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_LAST: + case VIR_STORAGE_NET_PROTOCOL_NONE: + virReportError(VIR_ERR_NO_SUPPORT, +diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c +index cbf0aa4..a485979 100644 +--- a/src/qemu/qemu_block.c ++++ b/src/qemu/qemu_block.c +@@ -959,6 +959,29 @@ qemuBlockStorageSourceGetRBDProps(virStorageSourcePtr src) + } + + ++static virJSONValuePtr ++qemuBlockStorageSourceGetVitastorProps(virStorageSource *src) ++{ ++ virJSONValuePtr servers = NULL; ++ virJSONValuePtr ret = NULL; ++ ++ if (src->nhosts > 0 && ++ !(servers = qemuBlockStorageSourceBuildHostsJSONInetSocketAddress(src))) ++ return NULL; ++ ++ if (virJSONValueObjectCreate(&ret, ++ "s:driver", "vitastor", ++ "A:server", &servers, ++ "s:etcd_prefix", src->relPath, ++ "S:config_path", src->configFile, ++ "s:image", src->path, ++ NULL) < 0) ++ return NULL; ++ ++ return ret; ++} ++ ++ + static virJSONValuePtr + qemuBlockStorageSourceGetSheepdogProps(virStorageSourcePtr src) + { +@@ -1174,6 +1197,11 @@ qemuBlockStorageSourceGetBackendProps(virStorageSourcePtr src, + return NULL; + break; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: ++ if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src))) ++ return NULL; ++ break; ++ + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + if (!(fileprops = qemuBlockStorageSourceGetSheepdogProps(src))) + return NULL; +diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c +index 822d5f8..e375cef 100644 +--- a/src/qemu/qemu_command.c ++++ b/src/qemu/qemu_command.c +@@ -975,6 +975,43 @@ qemuBuildNetworkDriveStr(virStorageSourcePtr src, + ret = virBufferContentAndReset(&buf); + break; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: ++ if (strchr(src->path, ':')) { ++ virReportError(VIR_ERR_CONFIG_UNSUPPORTED, ++ _("':' not allowed in Vitastor source volume name '%s'"), ++ src->path); ++ return NULL; ++ } ++ ++ virBufferStrcat(&buf, "vitastor:image=", src->path, NULL); ++ ++ if (src->nhosts > 0) { ++ virBufferAddLit(&buf, ":etcd_host="); ++ for (i = 0; i < src->nhosts; i++) { ++ if (i) ++ virBufferAddLit(&buf, ","); ++ ++ /* assume host containing : is ipv6 */ ++ if (strchr(src->hosts[i].name, ':')) ++ virBufferEscape(&buf, '\\', ":", "[%s]", ++ src->hosts[i].name); ++ else ++ virBufferAsprintf(&buf, "%s", src->hosts[i].name); ++ ++ if (src->hosts[i].port) ++ virBufferAsprintf(&buf, "\\:%u", src->hosts[i].port); ++ } ++ } ++ ++ if (src->configFile) ++ virBufferEscape(&buf, '\\', ":", ":config_path=%s", src->configFile); ++ ++ if (src->relPath) ++ virBufferEscape(&buf, '\\', ":", ":etcd_prefix=%s", src->relPath); ++ ++ ret = virBufferContentAndReset(&buf); ++ break; ++ + case VIR_STORAGE_NET_PROTOCOL_VXHS: + virReportError(VIR_ERR_INTERNAL_ERROR, "%s", + _("VxHS protocol does not support URI syntax")); +diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c +index ec6b340..f399efa 100644 +--- a/src/qemu/qemu_domain.c ++++ b/src/qemu/qemu_domain.c +@@ -10881,6 +10881,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSourcePtr src, + break; + + case VIR_STORAGE_NET_PROTOCOL_RBD: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_GLUSTER: + case VIR_STORAGE_NET_PROTOCOL_ISCSI: +diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c +index 1d96170..2d24396 100644 +--- a/src/qemu/qemu_driver.c ++++ b/src/qemu/qemu_driver.c +@@ -14687,6 +14687,7 @@ qemuDomainSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDefPtr snapdi + case VIR_STORAGE_NET_PROTOCOL_TFTP: + case VIR_STORAGE_NET_PROTOCOL_SSH: + case VIR_STORAGE_NET_PROTOCOL_VXHS: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_LAST: + virReportError(VIR_ERR_INTERNAL_ERROR, + _("external inactive snapshots are not supported on " +@@ -14764,6 +14765,7 @@ qemuDomainSnapshotPrepareDiskExternalActive(virDomainSnapshotDiskDefPtr snapdisk + case VIR_STORAGE_NET_PROTOCOL_TFTP: + case VIR_STORAGE_NET_PROTOCOL_SSH: + case VIR_STORAGE_NET_PROTOCOL_VXHS: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_LAST: + virReportError(VIR_ERR_INTERNAL_ERROR, + _("external active snapshots are not supported on " +@@ -14887,6 +14889,7 @@ qemuDomainSnapshotPrepareDiskInternal(virDomainDiskDefPtr disk, + case VIR_STORAGE_NET_PROTOCOL_TFTP: + case VIR_STORAGE_NET_PROTOCOL_SSH: + case VIR_STORAGE_NET_PROTOCOL_VXHS: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_LAST: + virReportError(VIR_ERR_INTERNAL_ERROR, + _("internal inactive snapshots are not supported on " +diff --git a/src/qemu/qemu_parse_command.c b/src/qemu/qemu_parse_command.c +index c4650f0..551da41 100644 +--- a/src/qemu/qemu_parse_command.c ++++ b/src/qemu/qemu_parse_command.c +@@ -2184,6 +2184,7 @@ qemuParseCommandLine(virFileCachePtr capsCache, + case VIR_STORAGE_NET_PROTOCOL_TFTP: + case VIR_STORAGE_NET_PROTOCOL_SSH: + case VIR_STORAGE_NET_PROTOCOL_LAST: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_NONE: + /* ignored for now */ + break; +diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c +index 4a13e90..33301c7 100644 +--- a/src/storage/storage_driver.c ++++ b/src/storage/storage_driver.c +@@ -1568,6 +1568,7 @@ storageVolLookupByPathCallback(virStoragePoolObjPtr obj, + case VIR_STORAGE_POOL_RBD: + case VIR_STORAGE_POOL_SHEEPDOG: + case VIR_STORAGE_POOL_ZFS: ++ case VIR_STORAGE_POOL_VITASTOR: + case VIR_STORAGE_POOL_LAST: + ignore_value(VIR_STRDUP(stable_path, data->path)); + break; +diff --git a/src/util/virstoragefile.c b/src/util/virstoragefile.c +index bd4b027..b323cd6 100644 +--- a/src/util/virstoragefile.c ++++ b/src/util/virstoragefile.c +@@ -84,7 +84,8 @@ VIR_ENUM_IMPL(virStorageNetProtocol, VIR_STORAGE_NET_PROTOCOL_LAST, + "ftps", + "tftp", + "ssh", +- "vxhs") ++ "vxhs", ++ "vitastor") + + VIR_ENUM_IMPL(virStorageNetHostTransport, VIR_STORAGE_NET_HOST_TRANS_LAST, + "tcp", +@@ -2839,6 +2840,83 @@ virStorageSourceParseRBDColonString(const char *rbdstr, + } + + ++static int ++virStorageSourceParseVitastorColonString(const char *colonstr, ++ virStorageSourcePtr src) ++{ ++ char *p, *e, *next; ++ char *options = NULL; ++ ++ /* optionally skip the "vitastor:" prefix if provided */ ++ if (STRPREFIX(colonstr, "vitastor:")) ++ colonstr += strlen("vitastor:"); ++ ++ if (VIR_STRDUP(options, colonstr) < 0) ++ return -1; ++ ++ p = options; ++ while (*p) { ++ /* find : delimiter or end of string */ ++ for (e = p; *e && *e != ':'; ++e) { ++ if (*e == '\\') { ++ e++; ++ if (*e == '\0') ++ break; ++ } ++ } ++ if (*e == '\0') { ++ next = e; /* last kv pair */ ++ } else { ++ next = e + 1; ++ *e = '\0'; ++ } ++ ++ if (STRPREFIX(p, "image=")) { ++ if (VIR_STRDUP(src->path, p + strlen("image=")) < 0) ++ return -1; ++ } else if (STRPREFIX(p, "etcd_prefix=")) { ++ if (VIR_STRDUP(src->relPath, p + strlen("etcd_prefix=")) < 0) ++ return -1; ++ } else if (STRPREFIX(p, "config_file=")) { ++ if (VIR_STRDUP(src->configFile, p + strlen("config_file=")) < 0) ++ return -1; ++ } else if (STRPREFIX(p, "etcd_host=")) { ++ char *h, *sep; ++ ++ h = p + strlen("etcd_host="); ++ while (h < e) { ++ for (sep = h; sep < e; ++sep) { ++ if (*sep == '\\' && (sep[1] == ',' || ++ sep[1] == ';' || ++ sep[1] == ' ')) { ++ *sep = '\0'; ++ sep += 2; ++ break; ++ } ++ } ++ ++ if (virStorageSourceRBDAddHost(src, h) < 0) ++ goto error; ++ ++ h = sep; ++ } ++ } ++ ++ p = next; ++ } ++ ++ if (!src->path) { ++ goto error; ++ } ++ ++ return 0; ++ ++error: ++ VIR_FREE(options); ++ return -1; ++} ++ ++ + static int + virStorageSourceParseNBDColonString(const char *nbdstr, + virStorageSourcePtr src) +@@ -2942,6 +3020,11 @@ virStorageSourceParseBackingColon(virStorageSourcePtr src, + goto cleanup; + break; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: ++ if (virStorageSourceParseVitastorColonString(path, src) < 0) ++ return -1; ++ break; ++ + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_LAST: + case VIR_STORAGE_NET_PROTOCOL_NONE: +@@ -3441,6 +3524,56 @@ virStorageSourceParseBackingJSONRBD(virStorageSourcePtr src, + return ret; + } + ++static int ++virStorageSourceParseBackingJSONVitastor(virStorageSourcePtr src, ++ virJSONValuePtr json, ++ int opaque ATTRIBUTE_UNUSED) ++{ ++ const char *filename; ++ const char *image = virJSONValueObjectGetString(json, "image"); ++ const char *conf = virJSONValueObjectGetString(json, "config_path"); ++ const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd_prefix"); ++ virJSONValuePtr servers = virJSONValueObjectGetArray(json, "server"); ++ size_t nservers; ++ size_t i; ++ ++ src->type = VIR_STORAGE_TYPE_NETWORK; ++ src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR; ++ ++ /* legacy syntax passed via 'filename' option */ ++ if ((filename = virJSONValueObjectGetString(json, "filename"))) ++ return virStorageSourceParseVitastorColonString(filename, src); ++ ++ if (!image) { ++ virReportError(VIR_ERR_INVALID_ARG, "%s", ++ _("missing image name in Vitastor backing volume " ++ "JSON specification")); ++ return -1; ++ } ++ ++ if (VIR_STRDUP(src->path, image) < 0 || ++ VIR_STRDUP(src->configFile, conf) < 0 || ++ VIR_STRDUP(src->relPath, etcd_prefix) < 0) ++ return -1; ++ ++ if (servers) { ++ nservers = virJSONValueArraySize(servers); ++ ++ if (VIR_ALLOC_N(src->hosts, nservers) < 0) ++ return -1; ++ ++ src->nhosts = nservers; ++ ++ for (i = 0; i < nservers; i++) { ++ if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i, ++ virJSONValueArrayGet(servers, i)) < 0) ++ return -1; ++ } ++ } ++ ++ return 0; ++} ++ + static int + virStorageSourceParseBackingJSONRaw(virStorageSourcePtr src, + virJSONValuePtr json, +@@ -3507,6 +3640,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = { + {"sheepdog", virStorageSourceParseBackingJSONSheepdog, 0}, + {"ssh", virStorageSourceParseBackingJSONSSH, 0}, + {"rbd", virStorageSourceParseBackingJSONRBD, 0}, ++ {"vitastor", virStorageSourceParseBackingJSONVitastor, 0}, + {"raw", virStorageSourceParseBackingJSONRaw, 0}, + {"vxhs", virStorageSourceParseBackingJSONVxHS, 0}, + }; +@@ -4276,6 +4410,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol) + case VIR_STORAGE_NET_PROTOCOL_GLUSTER: + return 24007; + ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_RBD: + /* we don't provide a default for RBD */ + return 0; +diff --git a/src/util/virstoragefile.h b/src/util/virstoragefile.h +index 1d6161a..8d83bf3 100644 +--- a/src/util/virstoragefile.h ++++ b/src/util/virstoragefile.h +@@ -134,6 +134,7 @@ typedef enum { + VIR_STORAGE_NET_PROTOCOL_TFTP, + VIR_STORAGE_NET_PROTOCOL_SSH, + VIR_STORAGE_NET_PROTOCOL_VXHS, ++ VIR_STORAGE_NET_PROTOCOL_VITASTOR, + + VIR_STORAGE_NET_PROTOCOL_LAST + } virStorageNetProtocol; +diff --git a/src/xenconfig/xen_xl.c b/src/xenconfig/xen_xl.c +index accfc3a..a18f9c3 100644 +--- a/src/xenconfig/xen_xl.c ++++ b/src/xenconfig/xen_xl.c +@@ -1535,6 +1535,7 @@ xenFormatXLDiskSrcNet(virStorageSourcePtr src) + case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: + case VIR_STORAGE_NET_PROTOCOL_SSH: + case VIR_STORAGE_NET_PROTOCOL_VXHS: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_LAST: + case VIR_STORAGE_NET_PROTOCOL_NONE: + virReportError(VIR_ERR_NO_SUPPORT, +diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c +index 70ca39b..9caef51 100644 +--- a/tools/virsh-pool.c ++++ b/tools/virsh-pool.c +@@ -1219,6 +1219,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd ATTRIBUTE_UNUSED) + case VIR_STORAGE_POOL_VSTORAGE: + flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE; + break; ++ case VIR_STORAGE_POOL_VITASTOR: ++ flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR; ++ break; + case VIR_STORAGE_POOL_LAST: + break; + } From eb0a3adafca3f2741d0dc71c5cbb9fe3956f7d7d Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Mon, 28 Jun 2021 01:20:55 +0300 Subject: [PATCH 11/12] Patch libvirt schema, add an example to test libvirt --- cinder-driver/libvirt-5.0-vitastor.diff | 50 ++++++++++++++++++++++- cinder-driver/libvirt-7.0-vitastor.diff | 54 +++++++++++++++++++++++++ cinder-driver/libvirt-7.5-vitastor.diff | 54 +++++++++++++++++++++++++ cinder-driver/libvirt-example.xml | 32 +++++++++++++++ 4 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 cinder-driver/libvirt-example.xml diff --git a/cinder-driver/libvirt-5.0-vitastor.diff b/cinder-driver/libvirt-5.0-vitastor.diff index af30809c..5f9475f6 100644 --- a/cinder-driver/libvirt-5.0-vitastor.diff +++ b/cinder-driver/libvirt-5.0-vitastor.diff @@ -1,9 +1,57 @@ -commit 367f518e76c36e1d815b41f5a0f623b71877e13c +commit 74b70c3e9482850c0f141f52ef2510466d68050b Author: Vitaliy Filippov Date: Sun Jun 27 12:52:40 2021 +0300 Add Vitastor support +diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng +index aa50eac..082b4f8 100644 +--- a/docs/schemas/domaincommon.rng ++++ b/docs/schemas/domaincommon.rng +@@ -1728,6 +1728,35 @@ + + + ++ ++ ++ ++ ++ vitastor ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -1851,6 +1880,7 @@ + + + ++ + + + diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h index 4bf2b5f..dbc011b 100644 --- a/include/libvirt/libvirt-storage.h diff --git a/cinder-driver/libvirt-7.0-vitastor.diff b/cinder-driver/libvirt-7.0-vitastor.diff index 282fb5fa..53e89673 100644 --- a/cinder-driver/libvirt-7.0-vitastor.diff +++ b/cinder-driver/libvirt-7.0-vitastor.diff @@ -1,3 +1,57 @@ +commit 46c76fe10d9e327fcefbf22b3173095626bee083 +Author: Vitaliy Filippov +Date: Mon Jun 28 01:18:41 2021 +0300 + + Add Vitastor support + +diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng +index 7dc419b..875433b 100644 +--- a/docs/schemas/domaincommon.rng ++++ b/docs/schemas/domaincommon.rng +@@ -1827,6 +1827,35 @@ + + + ++ ++ ++ ++ ++ vitastor ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -2083,6 +2112,7 @@ + + + ++ + + + diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h index 089e1e0..d7e7ef4 100644 --- a/include/libvirt/libvirt-storage.h diff --git a/cinder-driver/libvirt-7.5-vitastor.diff b/cinder-driver/libvirt-7.5-vitastor.diff index eab7d06c..fb50dca6 100644 --- a/cinder-driver/libvirt-7.5-vitastor.diff +++ b/cinder-driver/libvirt-7.5-vitastor.diff @@ -1,3 +1,57 @@ +commit 804f2fb24aedd32e238f84a7865e8a454e012e9c +Author: Vitaliy Filippov +Date: Mon Jun 28 01:20:19 2021 +0300 + + Add Vitastor support + +diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng +index 5ea14b6..a9df168 100644 +--- a/docs/schemas/domaincommon.rng ++++ b/docs/schemas/domaincommon.rng +@@ -1859,6 +1859,35 @@ + + + ++ ++ ++ ++ ++ vitastor ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -2115,6 +2144,7 @@ + + + ++ + + + diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h index 089e1e0..d7e7ef4 100644 --- a/include/libvirt/libvirt-storage.h diff --git a/cinder-driver/libvirt-example.xml b/cinder-driver/libvirt-example.xml new file mode 100644 index 00000000..0b288ab2 --- /dev/null +++ b/cinder-driver/libvirt-example.xml @@ -0,0 +1,32 @@ + + + debian9 + 96f277fb-fd9c-49da-bf21-a5cfd54eb162 + 524288 + 524288 + 1 + + hvm + + + + /usr/bin/qemu-system-x86_64 + + + + + + + + + + + + + + + + + + + From 30bb6026818b66bbe05bde38d70673e4633313e9 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Mon, 28 Jun 2021 22:00:23 +0300 Subject: [PATCH 12/12] Add _VITASTOR to missing switches in libvirt 7.0 patch --- cinder-driver/libvirt-7.0-vitastor.diff | 50 ++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/cinder-driver/libvirt-7.0-vitastor.diff b/cinder-driver/libvirt-7.0-vitastor.diff index 53e89673..e94b8f50 100644 --- a/cinder-driver/libvirt-7.0-vitastor.diff +++ b/cinder-driver/libvirt-7.0-vitastor.diff @@ -1,4 +1,4 @@ -commit 46c76fe10d9e327fcefbf22b3173095626bee083 +commit 41854e5059a1ba0b8e2918ce872e1ba78d3ecd6a Author: Vitaliy Filippov Date: Mon Jun 28 01:18:41 2021 +0300 @@ -181,6 +181,30 @@ index 2a7cdca..f756be1 100644 * VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG * VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER * VIR_CONNECT_LIST_STORAGE_POOLS_ZFS +diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c +index 6a8ae27..a735bc6 100644 +--- a/src/libxl/libxl_conf.c ++++ b/src/libxl/libxl_conf.c +@@ -942,6 +942,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSourcePtr src, + case VIR_STORAGE_NET_PROTOCOL_SSH: + case VIR_STORAGE_NET_PROTOCOL_VXHS: + case VIR_STORAGE_NET_PROTOCOL_NFS: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_LAST: + case VIR_STORAGE_NET_PROTOCOL_NONE: + virReportError(VIR_ERR_NO_SUPPORT, +diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c +index 17b93d0..c5a0084 100644 +--- a/src/libxl/xen_xl.c ++++ b/src/libxl/xen_xl.c +@@ -1601,6 +1601,7 @@ xenFormatXLDiskSrcNet(virStorageSourcePtr src) + case VIR_STORAGE_NET_PROTOCOL_SSH: + case VIR_STORAGE_NET_PROTOCOL_VXHS: + case VIR_STORAGE_NET_PROTOCOL_NFS: ++ case VIR_STORAGE_NET_PROTOCOL_VITASTOR: + case VIR_STORAGE_NET_PROTOCOL_LAST: + case VIR_STORAGE_NET_PROTOCOL_NONE: + virReportError(VIR_ERR_NO_SUPPORT, diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c index f9c6da2..92093e4 100644 --- a/src/qemu/qemu_block.c @@ -336,6 +360,18 @@ index ee333c3..674aa58 100644 case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: case VIR_STORAGE_NET_PROTOCOL_GLUSTER: case VIR_STORAGE_NET_PROTOCOL_ISCSI: +diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c +index 16bc53a..1e5d820 100644 +--- a/src/storage/storage_driver.c ++++ b/src/storage/storage_driver.c +@@ -1645,6 +1645,7 @@ storageVolLookupByPathCallback(virStoragePoolObjPtr obj, + + case VIR_STORAGE_POOL_GLUSTER: + case VIR_STORAGE_POOL_RBD: ++ case VIR_STORAGE_POOL_VITASTOR: + case VIR_STORAGE_POOL_SHEEPDOG: + case VIR_STORAGE_POOL_ZFS: + case VIR_STORAGE_POOL_LAST: diff --git a/src/test/test_driver.c b/src/test/test_driver.c index 29c4c86..a27ad94 100644 --- a/src/test/test_driver.c @@ -563,6 +599,18 @@ index 805950a..852df0d 100644 + + +diff --git a/tests/storagepoolxml2argvtest.c b/tests/storagepoolxml2argvtest.c +index 967d1f2..1e8ff7a 100644 +--- a/tests/storagepoolxml2argvtest.c ++++ b/tests/storagepoolxml2argvtest.c +@@ -68,6 +68,7 @@ testCompareXMLToArgvFiles(bool shouldFail, + case VIR_STORAGE_POOL_GLUSTER: + case VIR_STORAGE_POOL_ZFS: + case VIR_STORAGE_POOL_VSTORAGE: ++ case VIR_STORAGE_POOL_VITASTOR: + case VIR_STORAGE_POOL_LAST: + default: + VIR_TEST_DEBUG("pool type '%s' has no xml2argv test", defTypeStr); diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c index 7835fa6..8841fcf 100644 --- a/tools/virsh-pool.c