From b66160a7ad0e842a17b452b77d4489feae416426 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 2 Jul 2021 22:47:01 +0300 Subject: [PATCH 01/10] Aggregate per-pool statistics in mon --- mon/lp-optimizer.js | 1 + mon/mon.js | 73 ++++++++++++++++++++++++++++++++++++++++++--- src/qemu_driver.c | 1 + 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/mon/lp-optimizer.js b/mon/lp-optimizer.js index 5a90cc3f..d251b9c9 100644 --- a/mon/lp-optimizer.js +++ b/mon/lp-optimizer.js @@ -244,6 +244,7 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3, { return null; } + // FIXME: use parity_chunks with parity_space instead of pg_minsize const pg_effsize = Math.min(pg_minsize, Object.keys(osd_tree).length) + Math.max(0, Math.min(pg_size, Object.keys(osd_tree).length) - pg_minsize) * parity_space; const pg_count = prev_int_pgs.length; diff --git a/mon/mon.js b/mon/mon.js index ff213002..f972a62b 100644 --- a/mon/mon.js +++ b/mon/mon.js @@ -36,6 +36,8 @@ const etcd_allow = new RegExp('^'+[ 'history/last_clean_pgs', 'inode/stats/[1-9]\\d*/[1-9]\\d*', 'stats', + 'index/image/.*', + 'index/maxid/[1-9]\\d*', ].join('$|^')+'$'); const etcd_tree = { @@ -267,6 +269,16 @@ const etcd_tree = { }, */ }, }, + pool: { + stats: { + /* : { + used_raw_tb: float, // used raw space in the pool + total_raw_tb: float, // maximum amount of space in the pool + raw_to_usable: float, // raw to usable ratio + space_efficiency: float, // 0..1 + } */ + }, + }, stats: { /* op_stats: { : { count: uint64_t, usec: uint64_t, bytes: uint64_t }, @@ -289,6 +301,17 @@ const etcd_tree = { history: { last_clean_pgs: {}, }, + index: { + image: { + /* : { + id: uint64_t, + pool_id: uint64_t, + }, */ + }, + maxid: { + /* : uint64_t, */ + }, + }, }; // FIXME Split into several files @@ -363,6 +386,11 @@ class Mon { this.config.mon_stats_timeout = 100; } + this.config.mon_stats_interval = Number(this.config.mon_stats_interval) || 5000; + if (this.config.mon_stats_interval < 100) + { + this.config.mon_stats_interval = 100; + } // After this number of seconds, a dead OSD will be removed from PG distribution this.config.osd_out_time = Number(this.config.osd_out_time) || 0; if (!this.config.osd_out_time) @@ -1027,6 +1055,17 @@ class Mon } }); } LPOptimizer.print_change_stats(optimize_result); + const pg_effsize = Math.min(pool_cfg.pg_size, Object.keys(pool_tree).length); + this.state.pool.stats[pool_id] = { + used_raw_tb: (this.state.pool.stats[pool_id]||{}).used_raw_tb || 0, + total_raw_tb: optimize_result.space, + raw_to_usable: pg_effsize / (pool_cfg.pg_size - (pool_cfg.parity_chunks||0)), + space_efficiency: optimize_result.space/(optimize_result.total_space||1), + }; + etcd_request.success.push({ requestPut: { + key: b64(this.etcd_prefix+'/pool/stats/'+pool_id), + value: b64(JSON.stringify(this.state.pool.stats[pool_id])), + } }); this.save_new_pgs_txn(etcd_request, pool_id, up_osds, real_prev_pgs, optimize_result.int_pgs, pg_history); } this.state.config.pgs.hash = tree_hash; @@ -1133,7 +1172,7 @@ class Mon }, this.config.mon_change_timeout || 1000); } - sum_stats() + sum_op_stats() { const op_stats = {}, subop_stats = {}, recovery_stats = {}; for (const osd in this.state.osd.stats) @@ -1194,18 +1233,31 @@ class Mon write: { count: 0n, usec: 0n, bytes: 0n }, delete: { count: 0n, usec: 0n, bytes: 0n }, }); + for (const pool_id in this.state.config.pools) + { + this.state.pool.stats[pool_id] = this.state.pool.stats[pool_id] || {}; + this.state.pool.stats[pool_id].used_raw_tb = 0n; + } for (const osd_num in this.state.osd.space) { for (const pool_id in this.state.osd.space[osd_num]) { + this.state.pool.stats[pool_id] = this.state.pool.stats[pool_id] || { used_raw_tb: 0n }; inode_stats[pool_id] = inode_stats[pool_id] || {}; for (const inode_num in this.state.osd.space[osd_num][pool_id]) { + const u = BigInt(this.state.osd.space[osd_num][pool_id][inode_num]||0); inode_stats[pool_id][inode_num] = inode_stats[pool_id][inode_num] || inode_stub(); - inode_stats[pool_id][inode_num].raw_used += BigInt(this.state.osd.space[osd_num][pool_id][inode_num]||0); + inode_stats[pool_id][inode_num].raw_used += u; + this.state.pool.stats[pool_id].used_raw_tb += u; } } } + for (const pool_id in this.state.config.pools) + { + const used = this.state.pool.stats[pool_id].used_raw_tb; + this.state.pool.stats[pool_id].used_raw_tb = Number(used)/1024/1024/1024/1024; + } for (const osd_num in this.state.osd.inodestats) { const ist = this.state.osd.inodestats[osd_num]; @@ -1277,7 +1329,7 @@ class Mon async update_total_stats() { const txn = []; - const stats = this.sum_stats(); + const stats = this.sum_op_stats(); const object_counts = this.sum_object_counts(); const inode_stats = this.sum_inode_stats(); this.fix_stat_overflows(stats, (this.prev_stats = this.prev_stats || {})); @@ -1296,6 +1348,13 @@ class Mon } }); } } + for (const pool_id in this.state.pool.stats) + { + txn.push({ requestPut: { + key: b64(this.etcd_prefix+'/pool/stats/'+pool_id), + value: b64(JSON.stringify(this.state.pool.stats[pool_id])), + } }); + } if (txn.length) { await this.etcd_call('/kv/txn', { success: txn }, this.config.etcd_mon_timeout, 0); @@ -1309,11 +1368,17 @@ class Mon clearTimeout(this.stats_timer); this.stats_timer = null; } + let sleep = (this.stats_update_next||0) - Date.now(); + if (sleep < this.config.mon_stats_timeout) + { + sleep = this.config.mon_stats_timeout; + } this.stats_timer = setTimeout(() => { this.stats_timer = null; + this.stats_update_next = Date.now() + this.config.mon_stats_interval; this.update_total_stats().catch(console.error); - }, this.config.mon_stats_timeout || 1000); + }, sleep); } parse_kv(kv) diff --git a/src/qemu_driver.c b/src/qemu_driver.c index 5c282da1..93f2c660 100644 --- a/src/qemu_driver.c +++ b/src/qemu_driver.c @@ -199,6 +199,7 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E int64_t ret = 0; qemu_mutex_init(&client->mutex); client->config_path = g_strdup(qdict_get_try_str(options, "config_path")); + // FIXME: Rename to etcd_address client->etcd_host = g_strdup(qdict_get_try_str(options, "etcd_host")); client->etcd_prefix = g_strdup(qdict_get_try_str(options, "etcd_prefix")); client->use_rdma = qdict_get_try_int(options, "use_rdma", -1); From b52dd6843afa0082566f41c91cc1b4aeafa932b6 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 2 Jul 2021 23:10:32 +0300 Subject: [PATCH 02/10] Rename qemu_rbd_unescape and qemu_rbd_next_tok to *_vitastor_* --- src/qemu_driver.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/qemu_driver.c b/src/qemu_driver.c index 93f2c660..4ee4af27 100644 --- a/src/qemu_driver.c +++ b/src/qemu_driver.c @@ -70,7 +70,7 @@ static void vitastor_co_generic_bh_cb(void *opaque, long retval); static void vitastor_co_read_cb(void *opaque, long retval, uint64_t version); static void vitastor_close(BlockDriverState *bs); -static char *qemu_rbd_next_tok(char *src, char delim, char **p) +static char *qemu_vitastor_next_tok(char *src, char delim, char **p) { char *end; *p = NULL; @@ -89,7 +89,7 @@ static char *qemu_rbd_next_tok(char *src, char delim, char **p) return src; } -static void qemu_rbd_unescape(char *src) +static void qemu_vitastor_unescape(char *src) { char *p; for (p = src; *src; ++src, ++p) @@ -122,15 +122,15 @@ static void vitastor_parse_filename(const char *filename, QDict *options, Error while (p) { char *name, *value; - name = qemu_rbd_next_tok(p, '=', &p); + name = qemu_vitastor_next_tok(p, '=', &p); if (!p) { error_setg(errp, "conf option %s has no value", name); break; } - qemu_rbd_unescape(name); - value = qemu_rbd_next_tok(p, ':', &p); - qemu_rbd_unescape(value); + qemu_vitastor_unescape(name); + value = qemu_vitastor_next_tok(p, ':', &p); + qemu_vitastor_unescape(value); if (!strcmp(name, "inode") || !strcmp(name, "pool") || !strcmp(name, "size") || From bb31050ab5b13c9aa8e9da377d1dc442ea2d2f39 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Thu, 8 Jul 2021 01:08:27 +0300 Subject: [PATCH 03/10] Add missing image, config_path options to QEMU QAPI --- qemu-3.1-vitastor.patch | 16 ++++++++++------ qemu-4.2-vitastor.patch | 16 ++++++++++------ qemu-5.0-vitastor.patch | 16 ++++++++++------ qemu-5.1-vitastor.patch | 16 ++++++++++------ 4 files changed, 40 insertions(+), 24 deletions(-) diff --git a/qemu-3.1-vitastor.patch b/qemu-3.1-vitastor.patch index 42268b5d..35880d21 100644 --- a/qemu-3.1-vitastor.patch +++ b/qemu-3.1-vitastor.patch @@ -11,7 +11,7 @@ Index: qemu-3.1+dfsg/qapi/block-core.json 'host_cdrom', 'host_device', 'http', 'https', 'iscsi', 'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels', 'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd', 'replication', 'sheepdog', -@@ -3367,6 +3367,24 @@ +@@ -3367,6 +3367,28 @@ '*tag': 'str' } } ## @@ -19,17 +19,21 @@ Index: qemu-3.1+dfsg/qapi/block-core.json +# +# Driver specific block device options for vitastor +# ++# @image: Image name +# @inode: Inode number +# @pool: Pool ID +# @size: Desired image size in bytes -+# @etcd_host: etcd connection address ++# @config_path: Path to Vitastor configuration ++# @etcd_host: etcd connection address(es) +# @etcd_prefix: etcd key/value prefix +## +{ 'struct': 'BlockdevOptionsVitastor', -+ 'data': { 'inode': 'uint64', -+ 'pool': 'uint64', -+ 'size': 'uint64', -+ 'etcd_host': 'str', ++ 'data': { '*inode': 'uint64', ++ '*pool': 'uint64', ++ '*size': 'uint64', ++ '*image': 'str', ++ '*config_path': 'str', ++ '*etcd_host': 'str', + '*etcd_prefix': 'str' } } + +## diff --git a/qemu-4.2-vitastor.patch b/qemu-4.2-vitastor.patch index b09f7b6a..4617f106 100644 --- a/qemu-4.2-vitastor.patch +++ b/qemu-4.2-vitastor.patch @@ -11,7 +11,7 @@ Index: qemu/qapi/block-core.json 'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat', 'vxhs' ] } ## -@@ -3725,6 +3725,24 @@ +@@ -3725,6 +3725,28 @@ '*tag': 'str' } } ## @@ -19,17 +19,21 @@ Index: qemu/qapi/block-core.json +# +# Driver specific block device options for vitastor +# ++# @image: Image name +# @inode: Inode number +# @pool: Pool ID +# @size: Desired image size in bytes -+# @etcd_host: etcd connection address ++# @config_path: Path to Vitastor configuration ++# @etcd_host: etcd connection address(es) +# @etcd_prefix: etcd key/value prefix +## +{ 'struct': 'BlockdevOptionsVitastor', -+ 'data': { 'inode': 'uint64', -+ 'pool': 'uint64', -+ 'size': 'uint64', -+ 'etcd_host': 'str', ++ 'data': { '*inode': 'uint64', ++ '*pool': 'uint64', ++ '*size': 'uint64', ++ '*image': 'str', ++ '*config_path': 'str', ++ '*etcd_host': 'str', + '*etcd_prefix': 'str' } } + +## diff --git a/qemu-5.0-vitastor.patch b/qemu-5.0-vitastor.patch index 0f7cba0b..30b84a59 100644 --- a/qemu-5.0-vitastor.patch +++ b/qemu-5.0-vitastor.patch @@ -11,7 +11,7 @@ Index: qemu/qapi/block-core.json 'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat', 'vxhs' ] } ## -@@ -3635,6 +3635,24 @@ +@@ -3635,6 +3635,28 @@ '*tag': 'str' } } ## @@ -19,17 +19,21 @@ Index: qemu/qapi/block-core.json +# +# Driver specific block device options for vitastor +# ++# @image: Image name +# @inode: Inode number +# @pool: Pool ID +# @size: Desired image size in bytes -+# @etcd_host: etcd connection address ++# @config_path: Path to Vitastor configuration ++# @etcd_host: etcd connection address(es) +# @etcd_prefix: etcd key/value prefix +## +{ 'struct': 'BlockdevOptionsVitastor', -+ 'data': { 'inode': 'uint64', -+ 'pool': 'uint64', -+ 'size': 'uint64', -+ 'etcd_host': 'str', ++ 'data': { '*inode': 'uint64', ++ '*pool': 'uint64', ++ '*size': 'uint64', ++ '*image': 'str', ++ '*config_path': 'str', ++ '*etcd_host': 'str', + '*etcd_prefix': 'str' } } + +## diff --git a/qemu-5.1-vitastor.patch b/qemu-5.1-vitastor.patch index 3f6eb165..6c52fd99 100644 --- a/qemu-5.1-vitastor.patch +++ b/qemu-5.1-vitastor.patch @@ -11,7 +11,7 @@ Index: qemu-5.1+dfsg/qapi/block-core.json 'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat' ] } ## -@@ -3644,6 +3644,24 @@ +@@ -3644,6 +3644,28 @@ '*tag': 'str' } } ## @@ -19,17 +19,21 @@ Index: qemu-5.1+dfsg/qapi/block-core.json +# +# Driver specific block device options for vitastor +# ++# @image: Image name +# @inode: Inode number +# @pool: Pool ID +# @size: Desired image size in bytes -+# @etcd_host: etcd connection address ++# @config_path: Path to Vitastor configuration ++# @etcd_host: etcd connection address(es) +# @etcd_prefix: etcd key/value prefix +## +{ 'struct': 'BlockdevOptionsVitastor', -+ 'data': { 'inode': 'uint64', -+ 'pool': 'uint64', -+ 'size': 'uint64', -+ 'etcd_host': 'str', ++ 'data': { '*inode': 'uint64', ++ '*pool': 'uint64', ++ '*size': 'uint64', ++ '*image': 'str', ++ '*config_path': 'str', ++ '*etcd_host': 'str', + '*etcd_prefix': 'str' } } + +## From bff413584d71f502482a541c5b5ccbdb5c017b2d Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 9 Jul 2021 01:09:21 +0300 Subject: [PATCH 04/10] Fix qemuBlockStorageSourceGetVitastorProps --- cinder-driver/libvirt-5.0-vitastor.diff | 61 ++++++++++++++++++----- cinder-driver/libvirt-7.0-vitastor.diff | 64 ++++++++++++++++++------- cinder-driver/libvirt-7.5-vitastor.diff | 64 ++++++++++++++++++------- 3 files changed, 142 insertions(+), 47 deletions(-) diff --git a/cinder-driver/libvirt-5.0-vitastor.diff b/cinder-driver/libvirt-5.0-vitastor.diff index 5f9475f6..cf77268d 100644 --- a/cinder-driver/libvirt-5.0-vitastor.diff +++ b/cinder-driver/libvirt-5.0-vitastor.diff @@ -1,4 +1,4 @@ -commit 74b70c3e9482850c0f141f52ef2510466d68050b +commit bd283191b3e7a4c6d1c100d3d96e348a1ebffe55 Author: Vitaliy Filippov Date: Sun Jun 27 12:52:40 2021 +0300 @@ -65,10 +65,32 @@ index 4bf2b5f..dbc011b 100644 int virConnectListAllStoragePools(virConnectPtr conn, diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c -index 222bb8c..7687a5d 100644 +index 222bb8c..685d255 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c -@@ -30930,6 +30930,7 @@ virDomainDiskTranslateSourcePool(virDomainDiskDefPtr def) +@@ -8653,6 +8653,10 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node, + goto cleanup; + } + ++ if (src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR) { ++ src->relPath = virXMLPropString(node, "query"); ++ } ++ + if ((haveTLS = virXMLPropString(node, "tls")) && + (src->haveTLS = virTristateBoolTypeFromString(haveTLS)) <= 0) { + virReportError(VIR_ERR_XML_ERROR, +@@ -23849,6 +23853,10 @@ virDomainDiskSourceFormatNetwork(virBufferPtr attrBuf, + + virBufferEscapeString(attrBuf, " name='%s'", path ? path : src->path); + ++ if (src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR && src->relPath != NULL) { ++ virBufferEscapeString(attrBuf, " query='%s'", src->relPath); ++ } ++ + VIR_FREE(path); + + if (src->haveTLS != VIR_TRISTATE_BOOL_ABSENT && +@@ -30930,6 +30938,7 @@ virDomainDiskTranslateSourcePool(virDomainDiskDefPtr def) case VIR_STORAGE_POOL_MPATH: case VIR_STORAGE_POOL_RBD: @@ -194,32 +216,45 @@ index 73e988a..ab7bb81 100644 case VIR_STORAGE_NET_PROTOCOL_NONE: virReportError(VIR_ERR_NO_SUPPORT, diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c -index cbf0aa4..a485979 100644 +index cbf0aa4..096700d 100644 --- a/src/qemu/qemu_block.c +++ b/src/qemu/qemu_block.c -@@ -959,6 +959,29 @@ qemuBlockStorageSourceGetRBDProps(virStorageSourcePtr src) +@@ -959,6 +959,42 @@ qemuBlockStorageSourceGetRBDProps(virStorageSourcePtr src) } +static virJSONValuePtr +qemuBlockStorageSourceGetVitastorProps(virStorageSource *src) +{ -+ virJSONValuePtr servers = NULL; + virJSONValuePtr ret = NULL; ++ virStorageNetHostDefPtr host; ++ size_t i; ++ virBuffer buf = VIR_BUFFER_INITIALIZER; ++ char *etcd = NULL; + -+ if (src->nhosts > 0 && -+ !(servers = qemuBlockStorageSourceBuildHostsJSONInetSocketAddress(src))) -+ return NULL; ++ for (i = 0; i < src->nhosts; i++) { ++ host = src->hosts + i; ++ if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) { ++ goto cleanup; ++ } ++ virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port); ++ } ++ if (src->nhosts > 0) { ++ etcd = virBufferContentAndReset(&buf); ++ } + + if (virJSONValueObjectCreate(&ret, + "s:driver", "vitastor", -+ "A:server", &servers, -+ "s:etcd_prefix", src->relPath, ++ "S:etcd_host", etcd, ++ "S:etcd_prefix", src->relPath, + "S:config_path", src->configFile, + "s:image", src->path, + NULL) < 0) -+ return NULL; ++ goto cleanup; + ++cleanup: ++ VIR_FREE(etcd); ++ virBufferFreeAndReset(&buf); + return ret; +} + @@ -227,7 +262,7 @@ index cbf0aa4..a485979 100644 static virJSONValuePtr qemuBlockStorageSourceGetSheepdogProps(virStorageSourcePtr src) { -@@ -1174,6 +1197,11 @@ qemuBlockStorageSourceGetBackendProps(virStorageSourcePtr src, +@@ -1174,6 +1210,11 @@ qemuBlockStorageSourceGetBackendProps(virStorageSourcePtr src, return NULL; break; diff --git a/cinder-driver/libvirt-7.0-vitastor.diff b/cinder-driver/libvirt-7.0-vitastor.diff index e94b8f50..1d5777de 100644 --- a/cinder-driver/libvirt-7.0-vitastor.diff +++ b/cinder-driver/libvirt-7.0-vitastor.diff @@ -1,6 +1,6 @@ -commit 41854e5059a1ba0b8e2918ce872e1ba78d3ecd6a +commit 41cdfe8317d98f70aadedfdbb381effed2641bdd Author: Vitaliy Filippov -Date: Mon Jun 28 01:18:41 2021 +0300 +Date: Fri Jul 9 01:31:57 2021 +0300 Add Vitastor support @@ -65,10 +65,20 @@ index 089e1e0..d7e7ef4 100644 int virConnectListAllStoragePools(virConnectPtr conn, diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c -index 01b7187..5b81e37 100644 +index 01b7187..c6e9702 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c -@@ -31392,6 +31392,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSourcePtr src, +@@ -8261,7 +8261,8 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node, + src->configFile = virXPathString("string(./config/@file)", ctxt); + + if (src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTP || +- src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS) ++ src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS || ++ src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR) + src->query = virXMLPropString(node, "query"); + + if (virDomainStorageNetworkParseHosts(node, ctxt, &src->hosts, &src->nhosts) < 0) +@@ -31392,6 +31393,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSourcePtr src, case VIR_STORAGE_POOL_MPATH: case VIR_STORAGE_POOL_RBD: @@ -206,26 +216,36 @@ index 17b93d0..c5a0084 100644 case VIR_STORAGE_NET_PROTOCOL_NONE: virReportError(VIR_ERR_NO_SUPPORT, diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c -index f9c6da2..92093e4 100644 +index f9c6da2..922dde5 100644 --- a/src/qemu/qemu_block.c +++ b/src/qemu/qemu_block.c -@@ -938,6 +938,28 @@ qemuBlockStorageSourceGetRBDProps(virStorageSourcePtr src, +@@ -938,6 +938,38 @@ qemuBlockStorageSourceGetRBDProps(virStorageSourcePtr src, } +static virJSONValuePtr +qemuBlockStorageSourceGetVitastorProps(virStorageSource *src) +{ -+ g_autoptr(virJSONValue) servers = NULL; + virJSONValuePtr ret = NULL; ++ virStorageNetHostDefPtr host; ++ size_t i; ++ g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; ++ g_autofree char *etcd = NULL; + -+ if (src->nhosts > 0 && -+ !(servers = qemuBlockStorageSourceBuildHostsJSONInetSocketAddress(src))) -+ return NULL; ++ for (i = 0; i < src->nhosts; i++) { ++ host = src->hosts + i; ++ if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) { ++ return NULL; ++ } ++ virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port); ++ } ++ if (src->nhosts > 0) { ++ etcd = virBufferContentAndReset(&buf); ++ } + + if (virJSONValueObjectCreate(&ret, -+ "A:server", &servers, -+ "s:etcd_prefix", src->query, ++ "S:etcd_host", etcd, ++ "S:etcd_prefix", src->query, + "S:config_path", src->configFile, + "s:image", src->path, + NULL) < 0) @@ -238,7 +258,7 @@ index f9c6da2..92093e4 100644 static virJSONValuePtr qemuBlockStorageSourceGetSheepdogProps(virStorageSourcePtr src) { -@@ -1224,6 +1246,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSourcePtr src, +@@ -1224,6 +1256,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSourcePtr src, return NULL; break; @@ -251,7 +271,7 @@ index f9c6da2..92093e4 100644 case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: driver = "sheepdog"; if (!(fileprops = qemuBlockStorageSourceGetSheepdogProps(src))) -@@ -2183,6 +2211,7 @@ qemuBlockGetBackingStoreString(virStorageSourcePtr src, +@@ -2183,6 +2221,7 @@ qemuBlockGetBackingStoreString(virStorageSourcePtr src, case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: case VIR_STORAGE_NET_PROTOCOL_RBD: @@ -259,7 +279,7 @@ index f9c6da2..92093e4 100644 case VIR_STORAGE_NET_PROTOCOL_VXHS: case VIR_STORAGE_NET_PROTOCOL_NFS: case VIR_STORAGE_NET_PROTOCOL_SSH: -@@ -2560,6 +2589,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSourcePtr src, +@@ -2560,6 +2599,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSourcePtr src, return -1; break; @@ -321,10 +341,20 @@ index 6f970a3..10b39ca 100644 virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("VxHS protocol does not support URI syntax")); diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c -index 0765dc7..c69b1f1 100644 +index 0765dc7..4cff344 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c -@@ -9704,6 +9704,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSourcePtr src, +@@ -4610,7 +4610,8 @@ qemuDomainValidateStorageSource(virStorageSourcePtr src, + if (src->query && + (actualType != VIR_STORAGE_TYPE_NETWORK || + (src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTPS && +- src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP))) { ++ src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP && ++ src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR))) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("query is supported only with HTTP(S) protocols")); + return -1; +@@ -9704,6 +9705,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSourcePtr src, break; case VIR_STORAGE_NET_PROTOCOL_RBD: diff --git a/cinder-driver/libvirt-7.5-vitastor.diff b/cinder-driver/libvirt-7.5-vitastor.diff index fb50dca6..620fbed0 100644 --- a/cinder-driver/libvirt-7.5-vitastor.diff +++ b/cinder-driver/libvirt-7.5-vitastor.diff @@ -1,4 +1,4 @@ -commit 804f2fb24aedd32e238f84a7865e8a454e012e9c +commit c6e1958a1b4974828e8e5852beb252ce6594e670 Author: Vitaliy Filippov Date: Mon Jun 28 01:20:19 2021 +0300 @@ -65,10 +65,20 @@ index 089e1e0..d7e7ef4 100644 int virConnectListAllStoragePools(virConnectPtr conn, diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c -index d78f846..97f4d6d 100644 +index d78f846..f7222e3 100644 --- a/src/conf/domain_conf.c +++ b/src/conf/domain_conf.c -@@ -30775,6 +30775,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src, +@@ -8251,7 +8251,8 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node, + src->configFile = virXPathString("string(./config/@file)", ctxt); + + if (src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTP || +- src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS) ++ src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS || ++ src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR) + src->query = virXMLPropString(node, "query"); + + if (virDomainStorageNetworkParseHosts(node, ctxt, &src->hosts, &src->nhosts) < 0) +@@ -30775,6 +30776,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src, case VIR_STORAGE_POOL_MPATH: case VIR_STORAGE_POOL_RBD: @@ -238,26 +248,36 @@ index c0905b0..c172378 100644 case VIR_STORAGE_NET_PROTOCOL_NONE: virReportError(VIR_ERR_NO_SUPPORT, diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c -index 6627d04..277b04e 100644 +index 6627d04..c33f428 100644 --- a/src/qemu/qemu_block.c +++ b/src/qemu/qemu_block.c -@@ -928,6 +928,28 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src, +@@ -928,6 +928,38 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src, } +static virJSONValue * +qemuBlockStorageSourceGetVitastorProps(virStorageSource *src) +{ -+ g_autoptr(virJSONValue) servers = NULL; -+ virJSONValue *ret = NULL; ++ virJSONValuePtr ret = NULL; ++ virStorageNetHostDefPtr host; ++ size_t i; ++ g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER; ++ g_autofree char *etcd = NULL; + -+ if (src->nhosts > 0 && -+ !(servers = qemuBlockStorageSourceBuildHostsJSONInetSocketAddress(src))) -+ return NULL; ++ for (i = 0; i < src->nhosts; i++) { ++ host = src->hosts + i; ++ if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) { ++ return NULL; ++ } ++ virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port); ++ } ++ if (src->nhosts > 0) { ++ etcd = virBufferContentAndReset(&buf); ++ } + + if (virJSONValueObjectCreate(&ret, -+ "A:etcd_address", &servers, -+ "s:etcd_prefix", src->query, ++ "S:etcd_host", etcd, ++ "S:etcd_prefix", src->query, + "S:config_path", src->configFile, + "s:image", src->path, + NULL) < 0) @@ -270,7 +290,7 @@ index 6627d04..277b04e 100644 static virJSONValue * qemuBlockStorageSourceGetSheepdogProps(virStorageSource *src) { -@@ -1218,6 +1240,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src, +@@ -1218,6 +1250,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src, return NULL; break; @@ -283,7 +303,7 @@ index 6627d04..277b04e 100644 case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: driver = "sheepdog"; if (!(fileprops = qemuBlockStorageSourceGetSheepdogProps(src))) -@@ -2231,6 +2259,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src, +@@ -2231,6 +2269,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src, case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG: case VIR_STORAGE_NET_PROTOCOL_RBD: @@ -291,7 +311,7 @@ index 6627d04..277b04e 100644 case VIR_STORAGE_NET_PROTOCOL_VXHS: case VIR_STORAGE_NET_PROTOCOL_NFS: case VIR_STORAGE_NET_PROTOCOL_SSH: -@@ -2608,6 +2637,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src, +@@ -2608,6 +2647,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src, return -1; break; @@ -353,10 +373,20 @@ index ea51369..8258632 100644 virReportError(VIR_ERR_INTERNAL_ERROR, "%s", _("VxHS protocol does not support URI syntax")); diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c -index fc60e15..88bcb26 100644 +index fc60e15..5ab410d 100644 --- a/src/qemu/qemu_domain.c +++ b/src/qemu/qemu_domain.c -@@ -10027,6 +10027,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src, +@@ -4829,7 +4829,8 @@ qemuDomainValidateStorageSource(virStorageSource *src, + if (src->query && + (actualType != VIR_STORAGE_TYPE_NETWORK || + (src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTPS && +- src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP))) { ++ src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP && ++ src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR))) { + virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s", + _("query is supported only with HTTP(S) protocols")); + return -1; +@@ -10027,6 +10028,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src, break; case VIR_STORAGE_NET_PROTOCOL_RBD: From 7a45c5f86c5c513b637475d8df30fee3f7c26bfe Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 9 Jul 2021 12:29:39 +0300 Subject: [PATCH 05/10] buster-backports has broken mesa --- debian/patched-qemu.Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/patched-qemu.Dockerfile b/debian/patched-qemu.Dockerfile index e81e7bb1..3709c29b 100644 --- a/debian/patched-qemu.Dockerfile +++ b/debian/patched-qemu.Dockerfile @@ -11,6 +11,10 @@ RUN if [ "$REL" = "buster" ]; then \ echo 'Package: *' >> /etc/apt/preferences; \ echo 'Pin: release a=buster-backports' >> /etc/apt/preferences; \ echo 'Pin-Priority: 500' >> /etc/apt/preferences; \ + echo >> /etc/apt/preferences; \ + echo 'Package: libglvnd* libgles* libglx* libgl1 libegl* libopengl* mesa*' >> /etc/apt/preferences; \ + echo 'Pin: release a=buster-backports' >> /etc/apt/preferences; \ + echo 'Pin-Priority: 50' >> /etc/apt/preferences; \ fi; \ grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \ echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \ @@ -20,6 +24,8 @@ RUN apt-get update RUN apt-get -y install qemu fio liburing1 liburing-dev libgoogle-perftools-dev devscripts RUN apt-get -y build-dep qemu RUN apt-get -y build-dep fio +# To build a custom version +#RUN cp /root/packages/qemu-orig/* /root RUN apt-get --download-only source qemu RUN apt-get --download-only source fio From 4be761254cb614eda7a633fb1a365058c0a0d4f7 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 9 Jul 2021 21:51:19 +0300 Subject: [PATCH 06/10] Move patches to patches/ --- README-ru.md | 2 +- README.md | 2 +- debian/patched-qemu.Dockerfile | 6 +++--- {cinder-driver => patches}/libvirt-5.0-vitastor.diff | 0 {cinder-driver => patches}/libvirt-7.0-vitastor.diff | 0 {cinder-driver => patches}/libvirt-7.5-vitastor.diff | 0 {cinder-driver => patches}/libvirt-example.xml | 0 qemu-3.1-vitastor.patch => patches/qemu-3.1-vitastor.patch | 0 qemu-4.2-vitastor.patch => patches/qemu-4.2-vitastor.patch | 0 qemu-5.0-vitastor.patch => patches/qemu-5.0-vitastor.patch | 0 qemu-5.1-vitastor.patch => patches/qemu-5.1-vitastor.patch | 0 rpm/qemu-el8.Dockerfile | 4 ++-- 12 files changed, 7 insertions(+), 7 deletions(-) rename {cinder-driver => patches}/libvirt-5.0-vitastor.diff (100%) rename {cinder-driver => patches}/libvirt-7.0-vitastor.diff (100%) rename {cinder-driver => patches}/libvirt-7.5-vitastor.diff (100%) rename {cinder-driver => patches}/libvirt-example.xml (100%) rename qemu-3.1-vitastor.patch => patches/qemu-3.1-vitastor.patch (100%) rename qemu-4.2-vitastor.patch => patches/qemu-4.2-vitastor.patch (100%) rename qemu-5.0-vitastor.patch => patches/qemu-5.0-vitastor.patch (100%) rename qemu-5.1-vitastor.patch => patches/qemu-5.1-vitastor.patch (100%) diff --git a/README-ru.md b/README-ru.md index 5824f4e9..cca03578 100644 --- a/README-ru.md +++ b/README-ru.md @@ -370,7 +370,7 @@ Vitastor с однопоточной NBD прокси на том же стен - Установите gcc и g++ 8.x или новее. - Склонируйте данный репозиторий с подмодулями: `git clone https://yourcmc.ru/git/vitalif/vitastor/`. - Желательно пересобрать QEMU с патчем, который делает необязательным запуск через LD_PRELOAD. - См `qemu-*.*-vitastor.patch` - выберите версию, наиболее близкую вашей версии QEMU. + См `patches/qemu-*.*-vitastor.patch` - выберите версию, наиболее близкую вашей версии QEMU. - Установите QEMU 3.0 или новее, возьмите исходные коды установленного пакета, начните его пересборку, через некоторое время остановите её и скопируйте следующие заголовки: - `/include` → `/qemu/include` diff --git a/README.md b/README.md index c3ab9e2a..34b74174 100644 --- a/README.md +++ b/README.md @@ -338,7 +338,7 @@ Vitastor with single-thread NBD on the same hardware: * For QEMU 2.0+: `/qapi-types.h` → `/qemu/b/qemu/qapi-types.h` - `config-host.h` and `qapi` are required because they contain generated headers - You can also rebuild QEMU with a patch that makes LD_PRELOAD unnecessary to load vitastor driver. - See `qemu-*.*-vitastor.patch`. + See `patches/qemu-*.*-vitastor.patch`. - Install fio 3.7 or later, get its source and symlink it into `/fio`. - Build & install Vitastor with `mkdir build && cd build && cmake .. && make -j8 && make install`. Pay attention to the `QEMU_PLUGINDIR` cmake option - it must be set to `qemu-kvm` on RHEL. diff --git a/debian/patched-qemu.Dockerfile b/debian/patched-qemu.Dockerfile index 3709c29b..e43f2068 100644 --- a/debian/patched-qemu.Dockerfile +++ b/debian/patched-qemu.Dockerfile @@ -29,17 +29,17 @@ RUN apt-get -y build-dep fio RUN apt-get --download-only source qemu RUN apt-get --download-only source fio -ADD qemu-5.0-vitastor.patch qemu-5.1-vitastor.patch /root/vitastor/ +ADD patches/qemu-5.0-vitastor.patch patches/qemu-5.1-vitastor.patch /root/vitastor/patches/ RUN set -e; \ mkdir -p /root/packages/qemu-$REL; \ rm -rf /root/packages/qemu-$REL/*; \ cd /root/packages/qemu-$REL; \ dpkg-source -x /root/qemu*.dsc; \ if [ -d /root/packages/qemu-$REL/qemu-5.0 ]; then \ - cp /root/vitastor/qemu-5.0-vitastor.patch /root/packages/qemu-$REL/qemu-5.0/debian/patches; \ + cp /root/vitastor/patches/qemu-5.0-vitastor.patch /root/packages/qemu-$REL/qemu-5.0/debian/patches; \ echo qemu-5.0-vitastor.patch >> /root/packages/qemu-$REL/qemu-5.0/debian/patches/series; \ else \ - cp /root/vitastor/qemu-5.1-vitastor.patch /root/packages/qemu-$REL/qemu-*/debian/patches; \ + cp /root/vitastor/patches/qemu-5.1-vitastor.patch /root/packages/qemu-$REL/qemu-*/debian/patches; \ P=`ls -d /root/packages/qemu-$REL/qemu-*/debian/patches`; \ echo qemu-5.1-vitastor.patch >> $P/series; \ fi; \ diff --git a/cinder-driver/libvirt-5.0-vitastor.diff b/patches/libvirt-5.0-vitastor.diff similarity index 100% rename from cinder-driver/libvirt-5.0-vitastor.diff rename to patches/libvirt-5.0-vitastor.diff diff --git a/cinder-driver/libvirt-7.0-vitastor.diff b/patches/libvirt-7.0-vitastor.diff similarity index 100% rename from cinder-driver/libvirt-7.0-vitastor.diff rename to patches/libvirt-7.0-vitastor.diff diff --git a/cinder-driver/libvirt-7.5-vitastor.diff b/patches/libvirt-7.5-vitastor.diff similarity index 100% rename from cinder-driver/libvirt-7.5-vitastor.diff rename to patches/libvirt-7.5-vitastor.diff diff --git a/cinder-driver/libvirt-example.xml b/patches/libvirt-example.xml similarity index 100% rename from cinder-driver/libvirt-example.xml rename to patches/libvirt-example.xml diff --git a/qemu-3.1-vitastor.patch b/patches/qemu-3.1-vitastor.patch similarity index 100% rename from qemu-3.1-vitastor.patch rename to patches/qemu-3.1-vitastor.patch diff --git a/qemu-4.2-vitastor.patch b/patches/qemu-4.2-vitastor.patch similarity index 100% rename from qemu-4.2-vitastor.patch rename to patches/qemu-4.2-vitastor.patch diff --git a/qemu-5.0-vitastor.patch b/patches/qemu-5.0-vitastor.patch similarity index 100% rename from qemu-5.0-vitastor.patch rename to patches/qemu-5.0-vitastor.patch diff --git a/qemu-5.1-vitastor.patch b/patches/qemu-5.1-vitastor.patch similarity index 100% rename from qemu-5.1-vitastor.patch rename to patches/qemu-5.1-vitastor.patch diff --git a/rpm/qemu-el8.Dockerfile b/rpm/qemu-el8.Dockerfile index 0d4cc78a..a3d79419 100644 --- a/rpm/qemu-el8.Dockerfile +++ b/rpm/qemu-el8.Dockerfile @@ -11,7 +11,7 @@ RUN rm -rf /var/lib/dnf/*; dnf download --disablerepo='*' --enablerepo='centos-a RUN rpm --nomd5 -i qemu*.src.rpm RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=PowerTools --spec qemu-kvm.spec -ADD qemu-*-vitastor.patch /root/vitastor/ +ADD patches/qemu-*-vitastor.patch /root/vitastor/patches/ RUN set -e; \ mkdir -p /root/packages/qemu-el8; \ @@ -25,7 +25,7 @@ RUN set -e; \ echo "Patch$((PN+1)): qemu-4.2-vitastor.patch" >> qemu-kvm.spec; \ tail -n +2 xx01 >> qemu-kvm.spec; \ perl -i -pe 's/(^Release:\s*\d+)/$1.vitastor/' qemu-kvm.spec; \ - cp /root/vitastor/qemu-4.2-vitastor.patch ~/rpmbuild/SOURCES; \ + cp /root/vitastor/patches/qemu-4.2-vitastor.patch ~/rpmbuild/SOURCES; \ rpmbuild --nocheck -ba qemu-kvm.spec; \ cp ~/rpmbuild/RPMS/*/*qemu* /root/packages/qemu-el8/; \ cp ~/rpmbuild/SRPMS/*qemu* /root/packages/qemu-el8/ From e90bbe6385872a0622602c9db9bb0804e95a100b Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 10 Jul 2021 01:03:36 +0300 Subject: [PATCH 07/10] Implement OpenStack Cinder driver for Vitastor It can't delete snapshots yet because Vitastor layer merge isn't implemented yet. You can only delete volumes with all snapshots. This will be fixed in the near future. --- patches/cinder-vitastor.py | 948 ++++++++++++++++++++++++++++++++++++ patches/devstack-local.conf | 23 + patches/nova-20.diff | 287 +++++++++++ 3 files changed, 1258 insertions(+) create mode 100644 patches/cinder-vitastor.py create mode 100644 patches/devstack-local.conf create mode 100644 patches/nova-20.diff diff --git a/patches/cinder-vitastor.py b/patches/cinder-vitastor.py new file mode 100644 index 00000000..22771d67 --- /dev/null +++ b/patches/cinder-vitastor.py @@ -0,0 +1,948 @@ +# Vitastor Driver for OpenStack Cinder +# +# -------------------------------------------- +# Install as cinder/volume/drivers/vitastor.py +# -------------------------------------------- +# +# Copyright 2020 Vitaliy Filippov +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Cinder Vitastor Driver""" + +import binascii +import base64 +import errno +import json +import math +import os +import tempfile + +from castellan import key_manager +from oslo_config import cfg +from oslo_log import log as logging +from oslo_service import loopingcall +from oslo_concurrency import processutils +from oslo_utils import encodeutils +from oslo_utils import excutils +from oslo_utils import fileutils +from oslo_utils import units +import six +from six.moves.urllib import request + +from cinder import exception +from cinder.i18n import _ +from cinder.image import image_utils +from cinder import interface +from cinder import objects +from cinder.objects import fields +from cinder import utils +from cinder.volume import configuration +from cinder.volume import driver +from cinder.volume import volume_utils + +VERSION = '0.6.4' + +LOG = logging.getLogger(__name__) + +VITASTOR_OPTS = [ + cfg.StrOpt( + 'vitastor_config_path', + default='/etc/vitastor/vitastor.conf', + help='Vitastor configuration file path' + ), + cfg.StrOpt( + 'vitastor_etcd_address', + default='', + help='Vitastor etcd address(es)'), + cfg.StrOpt( + 'vitastor_etcd_prefix', + default='/vitastor', + help='Vitastor etcd prefix' + ), + cfg.StrOpt( + 'vitastor_pool_id', + default='', + help='Vitastor pool ID to use for volumes' + ), + # FIXME exclusive_cinder_pool ? +] + +CONF = cfg.CONF +CONF.register_opts(VITASTOR_OPTS, group = configuration.SHARED_CONF_GROUP) + +class VitastorDriverException(exception.VolumeDriverException): + message = _("Vitastor Cinder driver failure: %(reason)s") + +@interface.volumedriver +class VitastorDriver(driver.CloneableImageVD, + driver.ManageableVD, driver.ManageableSnapshotsVD, + driver.BaseVD): + """Implements Vitastor volume commands.""" + + cfg = {} + _etcd_urls = [] + + def __init__(self, active_backend_id = None, *args, **kwargs): + super(VitastorDriver, self).__init__(*args, **kwargs) + self.configuration.append_config_values(VITASTOR_OPTS) + + @classmethod + def get_driver_options(cls): + additional_opts = cls._get_oslo_driver_opts( + 'reserved_percentage', + 'max_over_subscription_ratio', + 'volume_dd_blocksize' + ) + return VITASTOR_OPTS + additional_opts + + def do_setup(self, context): + """Performs initialization steps that could raise exceptions.""" + super(VitastorDriver, self).do_setup(context) + # Make sure configuration is in UTF-8 + for attr in [ 'config_path', 'etcd_address', 'etcd_prefix', 'pool_id' ]: + val = self.configuration.safe_get('vitastor_'+attr) + if val is not None: + self.cfg[attr] = utils.convert_str(val) + self.cfg = self._load_config(self.cfg) + + def _load_config(self, cfg): + # Try to load configuration file + try: + f = open(cfg['config_path'] or '/etc/vitastor/vitastor.conf') + conf = json.loads(f.read()) + f.close() + for k in conf: + cfg[k] = cfg.get(k, conf[k]) + except: + pass + if isinstance(cfg['etcd_address'], str): + cfg['etcd_address'] = cfg['etcd_address'].split(',') + # Sanitize etcd URLs + for i, etcd_url in enumerate(cfg['etcd_address']): + ssl = False + if etcd_url.lower().startswith('http://'): + etcd_url = etcd_url[7:] + elif etcd_url.lower().startswith('https://'): + etcd_url = etcd_url[8:] + ssl = True + if etcd_url.find('/') < 0: + etcd_url += '/v3' + if ssl: + etcd_url = 'https://'+etcd_url + else: + etcd_url = 'http://'+etcd_url + cfg['etcd_address'][i] = etcd_url + return cfg + + def check_for_setup_error(self): + """Returns an error if prerequisites aren't met.""" + + def _encode_etcd_key(self, key): + if not isinstance(key, bytes): + key = str(key).encode('utf-8') + return base64.b64encode(self.cfg['etcd_prefix'].encode('utf-8')+b'/'+key).decode('utf-8') + + def _encode_etcd_value(self, value): + if not isinstance(value, bytes): + value = str(value).encode('utf-8') + return base64.b64encode(value).decode('utf-8') + + def _encode_etcd_requests(self, obj): + for v in obj: + for rt in v: + if 'key' in v[rt]: + v[rt]['key'] = self._encode_etcd_key(v[rt]['key']) + if 'range_end' in v[rt]: + v[rt]['range_end'] = self._encode_etcd_key(v[rt]['range_end']) + if 'value' in v[rt]: + v[rt]['value'] = self._encode_etcd_value(v[rt]['value']) + + def _etcd_txn(self, params): + if 'compare' in params: + for v in params['compare']: + if 'key' in v: + v['key'] = self._encode_etcd_key(v['key']) + if 'failure' in params: + self._encode_etcd_requests(params['failure']) + if 'success' in params: + self._encode_etcd_requests(params['success']) + body = json.dumps(params).encode('utf-8') + headers = { + 'Content-Type': 'application/json' + } + err = None + for etcd_url in self.cfg['etcd_address']: + try: + resp = request.urlopen(request.Request(etcd_url+'/kv/txn', body, headers), timeout = 5) + data = json.loads(resp.read()) + if 'responses' not in data: + data['responses'] = [] + for i, resp in enumerate(data['responses']): + if 'response_range' in resp: + if 'kvs' not in resp['response_range']: + resp['response_range']['kvs'] = [] + for kv in resp['response_range']['kvs']: + kv['key'] = base64.b64decode(kv['key'].encode('utf-8')).decode('utf-8') + if kv['key'].startswith(self.cfg['etcd_prefix']+'/'): + kv['key'] = kv['key'][len(self.cfg['etcd_prefix'])+1 : ] + kv['value'] = json.loads(base64.b64decode(kv['value'].encode('utf-8'))) + if len(resp.keys()) != 1: + LOG.exception('unknown responses['+str(i)+'] format: '+json.dumps(resp)) + else: + resp = data['responses'][i] = resp[list(resp.keys())[0]] + return data + except Exception as e: + LOG.exception('error calling etcd transaction: '+body.decode('utf-8')+'\nerror: '+str(e)) + err = e + raise err + + def _etcd_foreach(self, prefix, add_fn): + total = 0 + batch = 1000 + begin = prefix+'/' + while True: + resp = self._etcd_txn({ 'success': [ + { 'request_range': { + 'key': begin, + 'range_end': prefix+'0', + 'limit': batch+1, + } }, + ] }) + i = 0 + while i < batch and i < len(resp['responses'][0]['kvs']): + kv = resp['responses'][0]['kvs'][i] + add_fn(kv) + i += 1 + if len(resp['responses'][0]['kvs']) <= batch: + break + begin = resp['responses'][0]['kvs'][batch]['key'] + return total + + def _update_volume_stats(self): + location_info = json.dumps({ + 'config': self.configuration.vitastor_config_path, + 'etcd_address': self.configuration.vitastor_etcd_address, + 'etcd_prefix': self.configuration.vitastor_etcd_prefix, + 'pool_id': self.configuration.vitastor_pool_id, + }) + + stats = { + 'vendor_name': 'Vitastor', + 'driver_version': self.VERSION, + 'storage_protocol': 'vitastor', + 'total_capacity_gb': 'unknown', + 'free_capacity_gb': 'unknown', + # FIXME check if safe_get is required + 'reserved_percentage': self.configuration.safe_get('reserved_percentage'), + 'multiattach': True, + 'thin_provisioning_support': True, + 'max_over_subscription_ratio': self.configuration.safe_get('max_over_subscription_ratio'), + 'location_info': location_info, + 'backend_state': 'down', + 'volume_backend_name': self.configuration.safe_get('volume_backend_name') or 'vitastor', + 'replication_enabled': False, + } + + try: + pool_stats = self._etcd_txn({ 'success': [ + { 'request_range': { 'key': 'pool/stats/'+str(self.cfg['pool_id']) } } + ] }) + total_provisioned = 0 + def add_total(kv): + nonlocal total_provisioned + if kv['key'].find('@') >= 0: + total_provisioned += kv['value']['size'] + self._etcd_foreach('config/inode/'+str(self.cfg['pool_id']), lambda kv: add_total(kv)) + stats['provisioned_capacity_gb'] = round(total_provisioned/1024.0/1024.0/1024.0, 2) + pool_stats = pool_stats['responses'][0]['kvs'] + if len(pool_stats): + pool_stats = pool_stats[0] + stats['free_capacity_gb'] = round(1024.0*(pool_stats['total_raw_tb']-pool_stats['used_raw_tb'])/pool_stats['raw_to_usable'], 2) + stats['total_capacity_gb'] = round(1024.0*pool_stats['total_raw_tb'], 2) + stats['backend_state'] = 'up' + except Exception as e: + # just log and return unknown capacities + LOG.exception('error getting vitastor pool stats: '+str(e)) + + self._stats = stats + + def _next_id(self, resp): + if len(resp['kvs']) == 0: + return (1, 0) + else: + return (1 + resp['kvs'][0]['value'], resp['kvs'][0]['mod_revision']) + + def create_volume(self, volume): + """Creates a logical volume.""" + + size = int(volume.size) * units.Gi + # FIXME: Check if convert_str is really required + vol_name = utils.convert_str(volume.name) + if vol_name.find('@') >= 0 or vol_name.find('/') >= 0: + raise exception.VolumeBackendAPIException(data = '@ and / are forbidden in volume and snapshot names') + + LOG.debug("creating volume '%s'", vol_name) + + self._create_image(vol_name, { 'size': size }) + + if volume.encryption_key_id: + self._create_encrypted_volume(volume, volume.obj_context) + + volume_update = {} + return volume_update + + def _create_encrypted_volume(self, volume, context): + """Create a new LUKS encrypted image directly in Vitastor.""" + vol_name = utils.convert_str(volume.name) + f, opts = self._encrypt_opts(volume, context) + # FIXME: Check if it works at all :-) + self._execute( + 'qemu-img', 'convert', '-f', 'luks', *opts, + 'vitastor:image='+vol_name.replace(':', '\\:')+self._qemu_args(), + '%sM' % (volume.size * 1024) + ) + f.close() + + def _encrypt_opts(self, volume, context): + encryption = volume_utils.check_encryption_provider(self.db, volume, context) + # Fetch the key associated with the volume and decode the passphrase + keymgr = key_manager.API(CONF) + key = keymgr.get(context, encryption['encryption_key_id']) + passphrase = binascii.hexlify(key.get_encoded()).decode('utf-8') + # Decode the dm-crypt style cipher spec into something qemu-img can use + cipher_spec = image_utils.decode_cipher(encryption['cipher'], encryption['key_size']) + tmp_dir = volume_utils.image_conversion_dir() + f = tempfile.NamedTemporaryFile(prefix = 'luks_', dir = tmp_dir) + f.write(passphrase) + f.flush() + return (f, [ + '--object', 'secret,id=luks_sec,format=raw,file=%(passfile)s' % {'passfile': f.name}, + '-o', 'key-secret=luks_sec,cipher-alg=%(cipher_alg)s,cipher-mode=%(cipher_mode)s,ivgen-alg=%(ivgen_alg)s' % cipher_spec, + ]) + + def create_snapshot(self, snapshot): + """Creates a volume snapshot.""" + + vol_name = utils.convert_str(snapshot.volume_name) + snap_name = utils.convert_str(snapshot.name) + if snap_name.find('@') >= 0 or snap_name.find('/') >= 0: + raise exception.VolumeBackendAPIException(data = '@ and / are forbidden in volume and snapshot names') + self._create_snapshot(vol_name, vol_name+'@'+snap_name) + + def snapshot_revert_use_temp_snapshot(self): + """Disable the use of a temporary snapshot on revert.""" + return False + + def revert_to_snapshot(self, context, volume, snapshot): + """Revert a volume to a given snapshot.""" + + # FIXME Delete the image, then recreate it from the snapshot + + def delete_snapshot(self, snapshot): + """Deletes a snapshot.""" + + vol_name = utils.convert_str(snapshot.volume_name) + snap_name = utils.convert_str(snapshot.name) + + # Find the snapshot + resp = self._etcd_txn({ 'success': [ + { 'request_range': { 'key': 'index/image/'+vol_name+'@'+snap_name } }, + ] }) + if len(resp['responses'][0]['kvs']) == 0: + raise exception.SnapshotNotFound(snapshot_id = snap_name) + inode_id = int(resp['responses'][0]['kvs'][0]['value']['id']) + pool_id = int(resp['responses'][0]['kvs'][0]['value']['pool_id']) + parents = {} + parents[(pool_id << 48) | (inode_id & 0xffffffffffff)] = True + + # Check if there are child volumes + children = self._child_count(parents) + if children > 0: + raise exception.SnapshotIsBusy(snapshot_name = snap_name) + + # FIXME: We can't delete snapshots because we can't merge layers yet + raise exception.VolumeBackendAPIException(data = 'Snapshot delete (layer merge) is not implemented yet') + + def _child_count(self, parents): + children = 0 + def add_child(kv): + nonlocal children + children += self._check_parent(kv, parents) + self._etcd_foreach('config/inode', lambda kv: add_child(kv)) + return children + + def _check_parent(self, kv, parents): + if 'parent_id' not in kv['value']: + return 0 + parent_id = kv['value']['parent_id'] + _, _, pool_id, inode_id = kv['key'].split('/') + parent_pool_id = pool_id + if 'parent_pool_id' in kv['value'] and kv['value']['parent_pool_id']: + parent_pool_id = kv['value']['parent_pool_id'] + inode = (int(pool_id) << 48) | (int(inode_id) & 0xffffffffffff) + parent = (int(parent_pool_id) << 48) | (int(parent_id) & 0xffffffffffff) + if parent in parents and inode not in parents: + return 1 + return 0 + + def create_cloned_volume(self, volume, src_vref): + """Create a cloned volume from another volume.""" + + size = int(volume.size) * units.Gi + src_name = utils.convert_str(src_vref.name) + dest_name = utils.convert_str(volume.name) + if dest_name.find('@') >= 0 or dest_name.find('/') >= 0: + raise exception.VolumeBackendAPIException(data = '@ and / are forbidden in volume and snapshot names') + + # FIXME Do full copy if requested (cfg.disable_clone) + + if src_vref.admin_metadata.get('readonly') == 'True': + # source volume is a volume-image cache entry or other readonly volume + # clone without intermediate snapshot + src = self._get_image(src_name) + LOG.debug("creating image '%s' from '%s'", dest_name, src_name) + new_cfg = self._create_image(dest_name, { + 'size': size, + 'parent_id': src['idx']['id'], + 'parent_pool_id': src['idx']['pool_id'], + }) + return {} + + clone_snap = "%s@%s.clone_snap" % (src_name, dest_name) + make_img = True + if (volume.display_name and + volume.display_name.startswith('image-') and + src_vref.project_id != volume.project_id): + # idiotic openstack creates image-volume cache entries + # as clones of normal VM volumes... :-X prevent it :-D + clone_snap = dest_name + make_img = False + + LOG.debug("creating layer '%s' under '%s'", clone_snap, src_name) + new_cfg = self._create_snapshot(src_name, clone_snap, True) + if make_img: + # Then create a clone from it + new_cfg = self._create_image(dest_name, { + 'size': size, + 'parent_id': new_cfg['parent_id'], + 'parent_pool_id': new_cfg['parent_pool_id'], + }) + + return {} + + def create_volume_from_snapshot(self, volume, snapshot): + """Creates a cloned volume from an existing snapshot.""" + + vol_name = utils.convert_str(volume.name) + snap_name = utils.convert_str(snapshot.name) + + snap = self._get_image(vol_name+'@'+snap_name) + if not snap: + raise exception.SnapshotNotFound(snapshot_id = snap_name) + snap_inode_id = int(resp['responses'][0]['kvs'][0]['value']['id']) + snap_pool_id = int(resp['responses'][0]['kvs'][0]['value']['pool_id']) + + size = snap['cfg']['size'] + if int(volume.size): + size = int(volume.size) * units.Gi + new_cfg = self._create_image(vol_name, { + 'size': size, + 'parent_id': snap['idx']['id'], + 'parent_pool_id': snap['idx']['pool_id'], + }) + + return {} + + def _vitastor_args(self): + args = [] + for k in [ 'config_path', 'etcd_address', 'etcd_prefix' ]: + v = self.configuration.safe_get('vitastor_'+k) + if v: + args.extend(['--'+k, v]) + return args + + def _qemu_args(self): + args = '' + for k in [ 'config_path', 'etcd_address', 'etcd_prefix' ]: + v = self.configuration.safe_get('vitastor_'+k) + kk = k + if kk == 'etcd_address': + # FIXME use etcd_address in qemu driver + kk = 'etcd_host' + if v: + args += ':'+kk+'='+v.replace(':', '\\:') + return args + + def delete_volume(self, volume): + """Deletes a logical volume.""" + + vol_name = utils.convert_str(volume.name) + + # Find the volume and all its snapshots + range_end = b'index/image/' + vol_name.encode('utf-8') + range_end = range_end[0 : len(range_end)-1] + six.int2byte(range_end[len(range_end)-1] + 1) + resp = self._etcd_txn({ 'success': [ + { 'request_range': { 'key': 'index/image/'+vol_name, 'range_end': range_end } }, + ] }) + if len(resp['responses'][0]['kvs']) == 0: + # already deleted + LOG.info("volume %s no longer exists in backend", vol_name) + return + layers = resp['responses'][0]['kvs'] + layer_ids = {} + for kv in layers: + inode_id = int(kv['value']['id']) + pool_id = int(kv['value']['pool_id']) + inode_pool_id = (pool_id << 48) | (inode_id & 0xffffffffffff) + layer_ids[inode_pool_id] = True + + # Check if the volume has clones and raise 'busy' if so + children = self._child_count(layer_ids) + if children > 0: + raise exception.VolumeIsBusy(volume_name = vol_name) + + # Clear data + for kv in layers: + args = [ + 'vitastor-rm', '--pool', str(kv['value']['pool_id']), + '--inode', str(kv['value']['id']), '--progress', '0', + *(self._vitastor_args()) + ] + try: + self._execute(*args) + except processutils.ProcessExecutionError as exc: + LOG.error("Failed to remove layer "+kv['key']+": "+exc) + raise exception.VolumeBackendAPIException(data = exc.stderr) + + # Delete all layers from etcd + requests = [] + for kv in layers: + requests.append({ 'request_delete_range': { 'key': kv['key'] } }) + requests.append({ 'request_delete_range': { 'key': 'config/inode/'+str(kv['value']['pool_id'])+'/'+str(kv['value']['id']) } }) + self._etcd_txn({ 'success': requests }) + + def retype(self, context, volume, new_type, diff, host): + """Change extra type specifications for a volume.""" + + # FIXME Maybe (in the future) support multiple pools as different types + return True, {} + + def ensure_export(self, context, volume): + """Synchronously recreates an export for a logical volume.""" + pass + + def create_export(self, context, volume, connector): + """Exports the volume.""" + pass + + def remove_export(self, context, volume): + """Removes an export for a logical volume.""" + pass + + def _create_image(self, vol_name, cfg): + pool_s = str(self.cfg['pool_id']) + image_id = 0 + while image_id == 0: + # check if the image already exists and find a free ID + resp = self._etcd_txn({ 'success': [ + { 'request_range': { 'key': 'index/image/'+vol_name } }, + { 'request_range': { 'key': 'index/maxid/'+pool_s } }, + ] }) + if len(resp['responses'][0]['kvs']) > 0: + # already exists + raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' already exists') + image_id, id_mod = self._next_id(resp['responses'][1]) + # try to create the image + resp = self._etcd_txn({ 'compare': [ + { 'target': 'MOD', 'mod_revision': id_mod, 'key': 'index/maxid/'+pool_s }, + { 'target': 'VERSION', 'version': 0, 'key': 'index/image/'+vol_name }, + { 'target': 'VERSION', 'version': 0, 'key': 'config/inode/'+pool_s+'/'+str(image_id) }, + ], 'success': [ + { 'request_put': { 'key': 'index/maxid/'+pool_s, 'value': image_id } }, + { 'request_put': { 'key': 'index/image/'+vol_name, 'value': json.dumps({ + 'id': image_id, 'pool_id': self.cfg['pool_id'] + }) } }, + { 'request_put': { 'key': 'config/inode/'+pool_s+'/'+str(image_id), 'value': json.dumps({ + **cfg, 'name': vol_name, + }) } }, + ] }) + if not resp.get('succeeded'): + # repeat + image_id = 0 + + def _create_snapshot(self, vol_name, snap_vol_name, allow_existing = False): + while True: + # check if the image already exists and snapshot doesn't + resp = self._etcd_txn({ 'success': [ + { 'request_range': { 'key': 'index/image/'+vol_name } }, + { 'request_range': { 'key': 'index/image/'+snap_vol_name } }, + ] }) + if len(resp['responses'][0]['kvs']) == 0: + raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' does not exist') + if len(resp['responses'][1]['kvs']) > 0: + if allow_existing: + snap_idx = resp['responses'][1]['kvs'][0]['value'] + resp = self._etcd_txn({ 'success': [ + { 'request_range': { 'key': 'config/inode/'+str(snap_idx['pool_id'])+'/'+str(snap_idx['id']) } }, + ] }) + if len(resp['responses'][0]['kvs']) == 0: + raise exception.VolumeBackendAPIException(data = + 'Volume '+snap_vol_name+' is already indexed, but does not exist' + ) + return resp['responses'][0]['kvs'][0]['value'] + raise exception.VolumeBackendAPIException( + data = 'Volume '+snap_vol_name+' already exists' + ) + vol_idx = resp['responses'][0]['kvs'][0]['value'] + vol_idx_mod = resp['responses'][0]['kvs'][0]['mod_revision'] + # get image inode config and find a new ID + resp = self._etcd_txn({ 'success': [ + { 'request_range': { 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']) } }, + { 'request_range': { 'key': 'index/maxid/'+str(self.cfg['pool_id']) } }, + ] }) + if len(resp['responses'][0]['kvs']) == 0: + raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' does not exist') + vol_cfg = resp['responses'][0]['kvs'][0]['value'] + vol_mod = resp['responses'][0]['kvs'][0]['mod_revision'] + new_id, id_mod = self._next_id(resp['responses'][1]) + # try to redirect image to the new inode + new_cfg = { + **vol_cfg, 'name': vol_name, 'parent_id': vol_idx['id'], 'parent_pool_id': vol_idx['pool_id'] + } + resp = self._etcd_txn({ 'compare': [ + { 'target': 'MOD', 'mod_revision': vol_idx_mod, 'key': 'index/image/'+vol_name }, + { 'target': 'MOD', 'mod_revision': vol_mod, 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']) }, + { 'target': 'MOD', 'mod_revision': id_mod, 'key': 'index/maxid/'+str(self.cfg['pool_id']) }, + { 'target': 'VERSION', 'version': 0, 'key': 'index/image/'+snap_vol_name }, + { 'target': 'VERSION', 'version': 0, 'key': 'config/inode/'+str(self.cfg['pool_id'])+'/'+str(new_id) }, + ], 'success': [ + { 'request_put': { 'key': 'index/maxid/'+str(self.cfg['pool_id']), 'value': new_id } }, + { 'request_put': { 'key': 'index/image/'+vol_name, 'value': json.dumps({ + 'id': new_id, 'pool_id': self.cfg['pool_id'] + }) } }, + { 'request_put': { 'key': 'config/inode/'+str(self.cfg['pool_id'])+'/'+str(new_id), 'value': json.dumps(new_cfg) } }, + { 'request_put': { 'key': 'index/image/'+snap_vol_name, 'value': json.dumps({ + 'id': vol_idx['id'], 'pool_id': vol_idx['pool_id'] + }) } }, + { 'request_put': { 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']), 'value': json.dumps({ + **vol_cfg, 'name': snap_vol_name, 'readonly': True + }) } } + ] }) + if resp.get('succeeded'): + return new_cfg + + def initialize_connection(self, volume, connector): + data = { + 'driver_volume_type': 'vitastor', + 'data': { + 'config_path': self.configuration.vitastor_config_path, + 'etcd_address': self.configuration.vitastor_etcd_address, + 'etcd_prefix': self.configuration.vitastor_etcd_prefix, + 'name': volume.name, + 'logical_block_size': 512, + 'physical_block_size': 4096, + } + } + LOG.debug('connection data: %s', data) + return data + + def terminate_connection(self, volume, connector, **kwargs): + pass + + def clone_image(self, context, volume, image_location, image_meta, image_service): + if image_location: + # Note: image_location[0] is glance image direct_url. + # image_location[1] contains the list of all locations (including + # direct_url) or None if show_multiple_locations is False in + # glance configuration. + if image_location[1]: + url_locations = [location['url'] for location in image_location[1]] + else: + url_locations = [image_location[0]] + # iterate all locations to look for a cloneable one. + for url_location in url_locations: + if url_location and url_location.startswith('cinder://'): + # The idea is to use cinder:// Glance volumes as base images + base_vol = self.db.volume_get(context, url_location[len('cinder://') : ]) + if not base_vol or base_vol.volume_type_id != volume.volume_type_id: + continue + size = int(volume.size) * units.Gi + dest_name = utils.convert_str(volume.name) + # Find or create the base snapshot + snap_cfg = self._create_snapshot(base_vol.name, base_vol.name+'@.clone_snap', True) + # Then create a clone from it + new_cfg = self._create_image(dest_name, { + 'size': size, + 'parent_id': snap_cfg['parent_id'], + 'parent_pool_id': snap_cfg['parent_pool_id'], + }) + return ({}, True) + return ({}, False) + + def copy_image_to_encrypted_volume(self, context, volume, image_service, image_id): + self.copy_image_to_volume(context, volume, image_service, image_id, encrypted = True) + + def copy_image_to_volume(self, context, volume, image_service, image_id, encrypted = False): + tmp_dir = volume_utils.image_conversion_dir() + with tempfile.NamedTemporaryFile(dir = tmp_dir) as tmp: + image_utils.fetch_to_raw( + context, image_service, image_id, tmp.name, + self.configuration.volume_dd_blocksize, size = volume.size + ) + out_format = [ '-O', 'raw' ] + if encrypted: + key_file, opts = self._encrypt_opts(volume, context) + out_format = [ '-O', 'luks', *opts ] + dest_name = utils.convert_str(volume.name) + self._try_execute( + 'qemu-img', 'convert', '-f', 'raw', tmp.name, *out_format, + 'vitastor:image='+dest_name.replace(':', '\\:')+self._qemu_args() + ) + if encrypted: + key_file.close() + + def copy_volume_to_image(self, context, volume, image_service, image_meta): + tmp_dir = volume_utils.image_conversion_dir() + tmp_file = os.path.join(tmp_dir, volume.name + '-' + image_meta['id']) + with fileutils.remove_path_on_error(tmp_file): + vol_name = utils.convert_str(volume.name) + self._try_execute( + 'qemu-img', 'convert', '-f', 'raw', + 'vitastor:image='+vol_name.replace(':', '\\:')+self._qemu_args(), + '-O', 'raw', tmp_file + ) + # FIXME: Copy directly if the destination image is also in Vitastor + volume_utils.upload_volume(context, image_service, image_meta, tmp_file, volume) + os.unlink(tmp_file) + + def _get_image(self, vol_name): + # find the image + resp = self._etcd_txn({ 'success': [ + { 'request_range': { 'key': 'index/image/'+vol_name } }, + ] }) + if len(resp['responses'][0]['kvs']) == 0: + return None + vol_idx = resp['responses'][0]['kvs'][0]['value'] + vol_idx_mod = resp['responses'][0]['kvs'][0]['mod_revision'] + # get image inode config + resp = self._etcd_txn({ 'success': [ + { 'request_range': { 'key': 'config/inode/'+str(vol_idx['pool_id'])+'/'+str(vol_idx['id']) } }, + ] }) + if len(resp['responses'][0]['kvs']) == 0: + return None + vol_cfg = resp['responses'][0]['kvs'][0]['value'] + vol_cfg_mod = resp['responses'][0]['kvs'][0]['mod_revision'] + return { + 'cfg': vol_cfg, + 'cfg_mod': vol_cfg_mod, + 'idx': vol_idx, + 'idx_mod': vol_idx_mod, + } + + def extend_volume(self, volume, new_size): + """Extend an existing volume.""" + vol_name = utils.convert_str(volume.name) + while True: + vol = self._get_image(vol_name) + if not vol: + raise exception.VolumeBackendAPIException(data = 'Volume '+vol_name+' does not exist') + # change size + size = int(new_size) * units.Gi + if size == vol['cfg']['size']: + break + resp = self._etcd_txn({ 'compare': [ { + 'target': 'MOD', + 'mod_revision': vol['cfg_mod'], + 'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']), + } ], 'success': [ + { 'request_put': { + 'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']), + 'value': json.dumps({ **vol['cfg'], 'size': size }), + } }, + ] }) + if resp.get('succeeded'): + break + LOG.debug( + "Extend volume from %(old_size)s GB to %(new_size)s GB.", + {'old_size': volume.size, 'new_size': new_size} + ) + + def _add_manageable_volume(self, kv, manageable_volumes, cinder_ids): + cfg = kv['value'] + if kv['key'].find('@') >= 0: + # snapshot + return + image_id = volume_utils.extract_id_from_volume_name(cfg['name']) + image_info = { + 'reference': {'source-name': image_name}, + 'size': int(math.ceil(float(cfg['size']) / units.Gi)), + 'cinder_id': None, + 'extra_info': None, + } + if image_id in cinder_ids: + image_info['cinder_id'] = image_id + image_info['safe_to_manage'] = False + image_info['reason_not_safe'] = 'already managed' + else: + image_info['safe_to_manage'] = True + image_info['reason_not_safe'] = None + manageable_volumes.append(image_info) + + def get_manageable_volumes(self, cinder_volumes, marker, limit, offset, sort_keys, sort_dirs): + manageable_volumes = [] + cinder_ids = [resource['id'] for resource in cinder_volumes] + + # List all volumes + # FIXME: It's possible to use pagination in our case, but.. do we want it? + self._etcd_foreach('config/inode/'+str(self.cfg['pool_id']), + lambda kv: self._add_manageable_volume(kv, manageable_volumes, cinder_ids)) + + return volume_utils.paginate_entries_list( + manageable_volumes, marker, limit, offset, sort_keys, sort_dirs) + + def _get_existing_name(existing_ref): + if not isinstance(existing_ref, dict): + existing_ref = {"source-name": existing_ref} + if 'source-name' not in existing_ref: + reason = _('Reference must contain source-name element.') + raise exception.ManageExistingInvalidReference(existing_ref=existing_ref, reason=reason) + src_name = utils.convert_str(existing_ref['source-name']) + if not src_name: + reason = _('Reference must contain source-name element.') + raise exception.ManageExistingInvalidReference(existing_ref=existing_ref, reason=reason) + return src_name + + def manage_existing_get_size(self, volume, existing_ref): + """Return size of an existing image for manage_existing. + + :param volume: volume ref info to be set + :param existing_ref: {'source-name': } + """ + src_name = self._get_existing_name(existing_ref) + vol = self._get_image(src_name) + if not vol: + raise exception.VolumeBackendAPIException(data = 'Volume '+src_name+' does not exist') + return int(math.ceil(float(vol['cfg']['size']) / units.Gi)) + + def manage_existing(self, volume, existing_ref): + """Manages an existing image. + + Renames the image name to match the expected name for the volume. + + :param volume: volume ref info to be set + :param existing_ref: {'source-name': } + """ + from_name = self._get_existing_name(existing_ref) + to_name = utils.convert_str(volume.name) + self._rename(from_name, to_name) + + def _rename(self, from_name, to_name): + while True: + vol = self._get_image(from_name) + if not vol: + raise exception.VolumeBackendAPIException(data = 'Volume '+from_name+' does not exist') + to = self._get_image(to_name) + if to: + raise exception.VolumeBackendAPIException(data = 'Volume '+to_name+' already exists') + resp = self._etcd_txn({ 'compare': [ + { 'target': 'MOD', 'mod_revision': vol['idx_mod'], 'key': 'index/image/'+vol['cfg']['name'] }, + { 'target': 'MOD', 'mod_revision': vol['cfg_mod'], 'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']) }, + { 'target': 'VERSION', 'version': 0, 'key': 'index/image/'+to_name }, + ], 'success': [ + { 'request_delete_range': { 'key': 'index/image/'+vol['cfg']['name'] } }, + { 'request_put': { 'key': 'index/image/'+to_name, 'value': json.dumps(vol['idx']) } }, + { 'request_put': { 'key': 'config/inode/'+str(vol['idx']['pool_id'])+'/'+str(vol['idx']['id']), + 'value': json.dumps({ **vol['cfg'], 'name': to_name }) } }, + ] }) + if resp.get('succeeded'): + break + + def unmanage(self, volume): + pass + + def _add_manageable_snapshot(self, kv, manageable_snapshots, cinder_ids): + cfg = kv['value'] + dog = kv['key'].find('@') + if dog < 0: + # snapshot + return + image_name = kv['key'][0 : dog] + snap_name = kv['key'][dog+1 : ] + snapshot_id = volume_utils.extract_id_from_snapshot_name(snap_name) + snapshot_info = { + 'reference': {'source-name': snap_name}, + 'size': int(math.ceil(float(cfg['size']) / units.Gi)), + 'cinder_id': None, + 'extra_info': None, + 'safe_to_manage': False, + 'reason_not_safe': None, + 'source_reference': {'source-name': image_name} + } + if snapshot_id in cinder_ids: + # Exclude snapshots already managed. + snapshot_info['reason_not_safe'] = ('already managed') + snapshot_info['cinder_id'] = snapshot_id + elif snap_name.endswith('.clone_snap'): + # Exclude clone snapshot. + snapshot_info['reason_not_safe'] = ('used for clone snap') + else: + snapshot_info['safe_to_manage'] = True + manageable_snapshots.append(snapshot_info) + + def get_manageable_snapshots(self, cinder_snapshots, marker, limit, offset, sort_keys, sort_dirs): + """List manageable snapshots in Vitastor.""" + manageable_snapshots = [] + cinder_snapshot_ids = [resource['id'] for resource in cinder_snapshots] + # List all volumes + # FIXME: It's possible to use pagination in our case, but.. do we want it? + self._etcd_foreach('config/inode/'+str(self.cfg['pool_id']), + lambda kv: self._add_manageable_volume(kv, manageable_snapshots, cinder_snapshot_ids)) + return volume_utils.paginate_entries_list( + manageable_snapshots, marker, limit, offset, sort_keys, sort_dirs) + + def manage_existing_snapshot_get_size(self, snapshot, existing_ref): + """Return size of an existing image for manage_existing. + + :param snapshot: snapshot ref info to be set + :param existing_ref: {'source-name': } + """ + vol_name = utils.convert_str(snapshot.volume_name) + snap_name = self._get_existing_name(existing_ref) + vol = self._get_image(vol_name+'@'+snap_name) + if not vol: + raise exception.ManageExistingInvalidReference( + existing_ref=snapshot_name, reason='Specified snapshot does not exist.' + ) + return int(math.ceil(float(vol['cfg']['size']) / units.Gi)) + + def manage_existing_snapshot(self, snapshot, existing_ref): + """Manages an existing snapshot. + + Renames the snapshot name to match the expected name for the snapshot. + Error checking done by manage_existing_get_size is not repeated. + + :param snapshot: snapshot ref info to be set + :param existing_ref: {'source-name': } + """ + vol_name = utils.convert_str(snapshot.volume_name) + snap_name = self._get_existing_name(existing_ref) + from_name = vol_name+'@'+snap_name + to_name = vol_name+'@'+utils.convert_str(snapshot.name) + self._rename(from_name, to_name) + + def unmanage_snapshot(self, snapshot): + """Removes the specified snapshot from Cinder management.""" + pass + + def _dumps(self, obj): + return json.dumps(obj, separators=(',', ':'), sort_keys=True) diff --git a/patches/devstack-local.conf b/patches/devstack-local.conf new file mode 100644 index 00000000..17fd04ca --- /dev/null +++ b/patches/devstack-local.conf @@ -0,0 +1,23 @@ +# Devstack configuration for bridged networking + +[[local|localrc]] +ADMIN_PASSWORD=secret +DATABASE_PASSWORD=$ADMIN_PASSWORD +RABBIT_PASSWORD=$ADMIN_PASSWORD +SERVICE_PASSWORD=$ADMIN_PASSWORD +HOST_IP=10.0.2.15 +Q_USE_SECGROUP=True +FLOATING_RANGE="10.0.2.0/24" +IPV4_ADDRS_SAFE_TO_USE="10.0.5.0/24" +Q_FLOATING_ALLOCATION_POOL=start=10.0.2.50,end=10.0.2.100 +PUBLIC_NETWORK_GATEWAY=10.0.2.2 +PUBLIC_INTERFACE=ens3 +Q_USE_PROVIDERNET_FOR_PUBLIC=True +Q_AGENT=linuxbridge +Q_ML2_PLUGIN_MECHANISM_DRIVERS=linuxbridge +LB_PHYSICAL_INTERFACE=ens3 +PUBLIC_PHYSICAL_NETWORK=default +LB_INTERFACE_MAPPINGS=default:ens3 +Q_SERVICE_PLUGIN_CLASSES= +Q_ML2_PLUGIN_TYPE_DRIVERS=flat +Q_ML2_PLUGIN_EXT_DRIVERS= diff --git a/patches/nova-20.diff b/patches/nova-20.diff new file mode 100644 index 00000000..268ac205 --- /dev/null +++ b/patches/nova-20.diff @@ -0,0 +1,287 @@ +diff --git a/nova/virt/image/model.py b/nova/virt/image/model.py +index 971f7e9c07..70ed70d5e2 100644 +--- a/nova/virt/image/model.py ++++ b/nova/virt/image/model.py +@@ -129,3 +129,22 @@ class RBDImage(Image): + self.user = user + self.password = password + self.servers = servers ++ ++ ++class VitastorImage(Image): ++ """Class for images in a remote Vitastor cluster""" ++ ++ def __init__(self, name, etcd_address = None, etcd_prefix = None, config_path = None): ++ """Create a new Vitastor image object ++ ++ :param name: name of the image ++ :param etcd_address: etcd URL(s) (optional) ++ :param etcd_prefix: etcd prefix (optional) ++ :param config_path: path to the configuration (optional) ++ """ ++ super(RBDImage, self).__init__(FORMAT_RAW) ++ ++ self.name = name ++ self.etcd_address = etcd_address ++ self.etcd_prefix = etcd_prefix ++ self.config_path = config_path +diff --git a/nova/virt/images.py b/nova/virt/images.py +index 5358f3766a..ebe3d6effb 100644 +--- a/nova/virt/images.py ++++ b/nova/virt/images.py +@@ -41,7 +41,7 @@ IMAGE_API = glance.API() + + def qemu_img_info(path, format=None): + """Return an object containing the parsed output from qemu-img info.""" +- if not os.path.exists(path) and not path.startswith('rbd:'): ++ if not os.path.exists(path) and not path.startswith('rbd:') and not path.startswith('vitastor:'): + raise exception.DiskNotFound(location=path) + + info = nova.privsep.qemu.unprivileged_qemu_img_info(path, format=format) +@@ -50,7 +50,7 @@ def qemu_img_info(path, format=None): + + def privileged_qemu_img_info(path, format=None, output_format='json'): + """Return an object containing the parsed output from qemu-img info.""" +- if not os.path.exists(path) and not path.startswith('rbd:'): ++ if not os.path.exists(path) and not path.startswith('rbd:') and not path.startswith('vitastor:'): + raise exception.DiskNotFound(location=path) + + info = nova.privsep.qemu.privileged_qemu_img_info(path, format=format) +diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py +index f9475776b3..51573fe41d 100644 +--- a/nova/virt/libvirt/config.py ++++ b/nova/virt/libvirt/config.py +@@ -1060,6 +1060,8 @@ class LibvirtConfigGuestDisk(LibvirtConfigGuestDevice): + self.driver_iommu = False + self.source_path = None + self.source_protocol = None ++ self.source_query = None ++ self.source_config = None + self.source_name = None + self.source_hosts = [] + self.source_ports = [] +@@ -1186,7 +1188,8 @@ class LibvirtConfigGuestDisk(LibvirtConfigGuestDevice): + elif self.source_type == "mount": + dev.append(etree.Element("source", dir=self.source_path)) + elif self.source_type == "network" and self.source_protocol: +- source = etree.Element("source", protocol=self.source_protocol) ++ source = etree.Element("source", protocol=self.source_protocol, ++ query=self.source_query, config=self.source_config) + if self.source_name is not None: + source.set('name', self.source_name) + hosts_info = zip(self.source_hosts, self.source_ports) +diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py +index 391231c527..34dc60dcdd 100644 +--- a/nova/virt/libvirt/driver.py ++++ b/nova/virt/libvirt/driver.py +@@ -179,6 +179,7 @@ VOLUME_DRIVERS = { + 'local': 'nova.virt.libvirt.volume.volume.LibvirtVolumeDriver', + 'fake': 'nova.virt.libvirt.volume.volume.LibvirtFakeVolumeDriver', + 'rbd': 'nova.virt.libvirt.volume.net.LibvirtNetVolumeDriver', ++ 'vitastor': 'nova.virt.libvirt.volume.vitastor.LibvirtVitastorVolumeDriver', + 'nfs': 'nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver', + 'smbfs': 'nova.virt.libvirt.volume.smbfs.LibvirtSMBFSVolumeDriver', + 'fibre_channel': 'nova.virt.libvirt.volume.fibrechannel.LibvirtFibreChannelVolumeDriver', # noqa:E501 +@@ -385,10 +386,10 @@ class LibvirtDriver(driver.ComputeDriver): + # This prevents the risk of one test setting a capability + # which bleeds over into other tests. + +- # LVM and RBD require raw images. If we are not configured to ++ # LVM, RBD, Vitastor require raw images. If we are not configured to + # force convert images into raw format, then we _require_ raw + # images only. +- raw_only = ('rbd', 'lvm') ++ raw_only = ('rbd', 'lvm', 'vitastor') + requires_raw_image = (CONF.libvirt.images_type in raw_only and + not CONF.force_raw_images) + requires_ploop_image = CONF.libvirt.virt_type == 'parallels' +@@ -775,12 +776,12 @@ class LibvirtDriver(driver.ComputeDriver): + # Some imagebackends are only able to import raw disk images, + # and will fail if given any other format. See the bug + # https://bugs.launchpad.net/nova/+bug/1816686 for more details. +- if CONF.libvirt.images_type in ('rbd',): ++ if CONF.libvirt.images_type in ('rbd', 'vitastor'): + if not CONF.force_raw_images: + msg = _("'[DEFAULT]/force_raw_images = False' is not " +- "allowed with '[libvirt]/images_type = rbd'. " ++ "allowed with '[libvirt]/images_type = rbd' or 'vitastor'. " + "Please check the two configs and if you really " +- "do want to use rbd as images_type, set " ++ "do want to use rbd or vitastor as images_type, set " + "force_raw_images to True.") + raise exception.InvalidConfiguration(msg) + +@@ -2603,6 +2604,16 @@ class LibvirtDriver(driver.ComputeDriver): + if connection_info['data'].get('auth_enabled'): + username = connection_info['data']['auth_username'] + path = f"rbd:{volume_name}:id={username}" ++ elif connection_info['driver_volume_type'] == 'vitastor': ++ volume_name = connection_info['data']['name'] ++ path = 'vitastor:image='+volume_name.replace(':', '\\:') ++ for k in [ 'config_path', 'etcd_address', 'etcd_prefix' ]: ++ if k in connection_info['data']: ++ kk = k ++ if kk == 'etcd_address': ++ # FIXME use etcd_address in qemu driver ++ kk = 'etcd_host' ++ path += ":"+kk+"="+connection_info['data'][k].replace(':', '\\:') + else: + path = 'unknown' + raise exception.DiskNotFound(location='unknown') +@@ -2827,8 +2838,8 @@ class LibvirtDriver(driver.ComputeDriver): + + image_format = CONF.libvirt.snapshot_image_format or source_type + +- # NOTE(bfilippov): save lvm and rbd as raw +- if image_format == 'lvm' or image_format == 'rbd': ++ # NOTE(bfilippov): save lvm and rbd and vitastor as raw ++ if image_format == 'lvm' or image_format == 'rbd' or image_format == 'vitastor': + image_format = 'raw' + + metadata = self._create_snapshot_metadata(instance.image_meta, +@@ -2899,7 +2910,7 @@ class LibvirtDriver(driver.ComputeDriver): + expected_state=task_states.IMAGE_UPLOADING) + + # TODO(nic): possibly abstract this out to the root_disk +- if source_type == 'rbd' and live_snapshot: ++ if (source_type == 'rbd' or source_type == 'vitastor') and live_snapshot: + # Standard snapshot uses qemu-img convert from RBD which is + # not safe to run with live_snapshot. + live_snapshot = False +@@ -4099,7 +4110,7 @@ class LibvirtDriver(driver.ComputeDriver): + # cleanup rescue volume + lvm.remove_volumes([lvmdisk for lvmdisk in self._lvm_disks(instance) + if lvmdisk.endswith('.rescue')]) +- if CONF.libvirt.images_type == 'rbd': ++ if CONF.libvirt.images_type == 'rbd' or CONF.libvirt.images_type == 'vitastor': + filter_fn = lambda disk: (disk.startswith(instance.uuid) and + disk.endswith('.rescue')) + rbd_utils.RBDDriver().cleanup_volumes(filter_fn) +@@ -4356,6 +4367,8 @@ class LibvirtDriver(driver.ComputeDriver): + # TODO(mikal): there is a bug here if images_type has + # changed since creation of the instance, but I am pretty + # sure that this bug already exists. ++ if CONF.libvirt.images_type == 'vitastor': ++ return 'vitastor' + return 'rbd' if CONF.libvirt.images_type == 'rbd' else 'raw' + + @staticmethod +@@ -4764,10 +4777,10 @@ class LibvirtDriver(driver.ComputeDriver): + finally: + # NOTE(mikal): if the config drive was imported into RBD, + # then we no longer need the local copy +- if CONF.libvirt.images_type == 'rbd': ++ if CONF.libvirt.images_type == 'rbd' or CONF.libvirt.images_type == 'vitastor': + LOG.info('Deleting local config drive %(path)s ' +- 'because it was imported into RBD.', +- {'path': config_disk_local_path}, ++ 'because it was imported into %(type).', ++ {'path': config_disk_local_path, 'type': CONF.libvirt.images_type}, + instance=instance) + os.unlink(config_disk_local_path) + +diff --git a/nova/virt/libvirt/utils.py b/nova/virt/libvirt/utils.py +index da2a6e8b8a..52c02e72f1 100644 +--- a/nova/virt/libvirt/utils.py ++++ b/nova/virt/libvirt/utils.py +@@ -340,6 +340,10 @@ def find_disk(guest: libvirt_guest.Guest) -> ty.Tuple[str, ty.Optional[str]]: + disk_path = disk.source_name + if disk_path: + disk_path = 'rbd:' + disk_path ++ elif not disk_path and disk.source_protocol == 'vitastor': ++ disk_path = disk.source_name ++ if disk_path: ++ disk_path = 'vitastor:' + disk_path + + if not disk_path: + raise RuntimeError(_("Can't retrieve root device path " +@@ -354,6 +358,8 @@ def get_disk_type_from_path(path: str) -> ty.Optional[str]: + return 'lvm' + elif path.startswith('rbd:'): + return 'rbd' ++ elif path.startswith('vitastor:'): ++ return 'vitastor' + elif (os.path.isdir(path) and + os.path.exists(os.path.join(path, "DiskDescriptor.xml"))): + return 'ploop' +diff --git a/nova/virt/libvirt/volume/vitastor.py b/nova/virt/libvirt/volume/vitastor.py +new file mode 100644 +index 0000000000..0256df62c1 +--- /dev/null ++++ b/nova/virt/libvirt/volume/vitastor.py +@@ -0,0 +1,75 @@ ++# Copyright (c) 2021+, Vitaliy Filippov ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may ++# not use this file except in compliance with the License. You may obtain ++# a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations ++# under the License. ++ ++from os_brick import exception as os_brick_exception ++from os_brick import initiator ++from os_brick.initiator import connector ++from oslo_log import log as logging ++ ++import nova.conf ++from nova import utils ++from nova.virt.libvirt.volume import volume as libvirt_volume ++ ++ ++CONF = nova.conf.CONF ++LOG = logging.getLogger(__name__) ++ ++ ++class LibvirtVitastorVolumeDriver(libvirt_volume.LibvirtBaseVolumeDriver): ++ """Driver to attach Vitastor volumes to libvirt.""" ++ def __init__(self, host): ++ super(LibvirtVitastorVolumeDriver, self).__init__(host, is_block_dev=False) ++ ++ def connect_volume(self, connection_info, instance): ++ pass ++ ++ def disconnect_volume(self, connection_info, instance): ++ pass ++ ++ def get_config(self, connection_info, disk_info): ++ """Returns xml for libvirt.""" ++ conf = super(LibvirtVitastorVolumeDriver, self).get_config(connection_info, disk_info) ++ conf.source_type = 'network' ++ conf.source_protocol = 'vitastor' ++ conf.source_name = connection_info['data'].get('name') ++ conf.source_query = connection_info['data'].get('etcd_prefix') or None ++ conf.source_config = connection_info['data'].get('config_path') or None ++ conf.source_hosts = [] ++ conf.source_ports = [] ++ addresses = connection_info['data'].get('etcd_address', '') ++ if addresses: ++ if not isinstance(addresses, list): ++ addresses = addresses.split(',') ++ for addr in addresses: ++ if addr.startswith('https://'): ++ raise NotImplementedError('Vitastor block driver does not support SSL for etcd communication yet') ++ if addr.startswith('http://'): ++ addr = addr[7:] ++ addr = addr.rstrip('/') ++ if addr.endswith('/v3'): ++ addr = addr[0:-3] ++ p = addr.find('/') ++ if p > 0: ++ raise NotImplementedError('libvirt does not support custom URL paths for Vitastor etcd yet. Use /etc/vitastor/vitastor.conf') ++ p = addr.find(':') ++ port = '2379' ++ if p > 0: ++ port = addr[p+1:] ++ addr = addr[0:p] ++ conf.source_hosts.append(addr) ++ conf.source_ports.append(port) ++ return conf ++ ++ def extend_volume(self, connection_info, instance, requested_size): ++ raise NotImplementedError From b66a079892718e7b29968cabd93e2b541ee86f25 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 10 Jul 2021 01:11:20 +0300 Subject: [PATCH 08/10] State basic OpenStack support --- README-ru.md | 4 +++- README.md | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README-ru.md b/README-ru.md index cca03578..70ba4f2c 100644 --- a/README-ru.md +++ b/README-ru.md @@ -48,12 +48,14 @@ Vitastor на данный момент находится в статусе п - Сглаживание производительности случайной записи в SSD+HDD конфигурациях - Поддержка RDMA/RoCEv2 через libibverbs - CSI-плагин для Kubernetes +- Базовая поддержка OpenStack: драйвер Cinder, патчи для Nova и libvirt ## Планы развития +- Поддержка удаления снапшотов (слияния слоёв) - Более корректные скрипты разметки дисков и автоматического запуска OSD - Другие инструменты администрирования -- Плагины для OpenStack, OpenNebula, Proxmox и других облачных систем +- Плагины для OpenNebula, Proxmox и других облачных систем - iSCSI-прокси - Более быстрое переключение при отказах - Фоновая проверка целостности без контрольных сумм (сверка реплик) diff --git a/README.md b/README.md index 34b74174..5f73eb0d 100644 --- a/README.md +++ b/README.md @@ -42,12 +42,14 @@ breaking changes in the future. However, the following is implemented: - Write throttling to smooth random write workloads in SSD+HDD configurations - RDMA/RoCEv2 support via libibverbs - CSI plugin for Kubernetes +- Basic OpenStack support: Cinder driver, Nova and libvirt patches ## Roadmap +- Snapshot deletion (layer merge) support - Better OSD creation and auto-start tools - Other administrative tools -- Plugins for OpenStack, OpenNebula, Proxmox and other cloud systems +- Plugins for OpenNebula, Proxmox and other cloud systems - iSCSI proxy - Faster failover - Scrubbing without checksums (verification of replicas) From 8b2a4c95392ee2fb06b1bc38b79482ce0610c337 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 10 Jul 2021 11:01:14 +0300 Subject: [PATCH 09/10] Fix centos builds (yum-builddep stopped working in el7, cmake in el8..) --- rpm/vitastor-el7.Dockerfile | 4 ++-- rpm/vitastor-el8.Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rpm/vitastor-el7.Dockerfile b/rpm/vitastor-el7.Dockerfile index 6df2805c..b7458f3c 100644 --- a/rpm/vitastor-el7.Dockerfile +++ b/rpm/vitastor-el7.Dockerfile @@ -15,8 +15,8 @@ RUN yumdownloader --disablerepo=centos-sclo-rh --source fio RUN rpm --nomd5 -i qemu*.src.rpm RUN rpm --nomd5 -i fio*.src.rpm RUN rm -f /etc/yum.repos.d/CentOS-Media.repo -RUN cd ~/rpmbuild/SPECS && yum-builddep -y --enablerepo='*' --disablerepo=centos-sclo-rh --disablerepo=centos-sclo-rh-source --disablerepo=centos-sclo-sclo-testing qemu-kvm.spec -RUN cd ~/rpmbuild/SPECS && yum-builddep -y --enablerepo='*' --disablerepo=centos-sclo-rh --disablerepo=centos-sclo-rh-source --disablerepo=centos-sclo-sclo-testing fio.spec +RUN cd ~/rpmbuild/SPECS && yum-builddep -y qemu-kvm.spec +RUN cd ~/rpmbuild/SPECS && yum-builddep -y fio.spec RUN yum -y install rdma-core-devel ADD https://vitastor.io/rpms/liburing-el7/liburing-0.7-2.el7.src.rpm /root diff --git a/rpm/vitastor-el8.Dockerfile b/rpm/vitastor-el8.Dockerfile index ce519336..96a96a69 100644 --- a/rpm/vitastor-el8.Dockerfile +++ b/rpm/vitastor-el8.Dockerfile @@ -15,7 +15,7 @@ RUN rpm --nomd5 -i qemu*.src.rpm RUN rpm --nomd5 -i fio*.src.rpm RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=powertools --spec qemu-kvm.spec RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=powertools --spec fio.spec && dnf install -y cmake -RUN yum -y install libibverbs-devel +RUN yum -y install libibverbs-devel libarchive ADD https://vitastor.io/rpms/liburing-el7/liburing-0.7-2.el7.src.rpm /root From cb282d25e07b25ac15fc0ac31ee04f983175ff11 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 10 Jul 2021 01:14:34 +0300 Subject: [PATCH 10/10] Release 0.6.5 - Basic support for OpenStack: Cinder driver, patches for Nova and libvirt - Add missing "image" and "config_path" QEMU options - Calculate aggregate per-pool statistics in monitor - Implement writes with Check-And-Set semantics - Add a C wrapper library with public header --- CMakeLists.txt | 2 +- csi/Makefile | 2 +- csi/deploy/004-csi-nodeplugin.yaml | 2 +- csi/deploy/007-csi-provisioner.yaml | 2 +- csi/src/config.go | 2 +- debian/changelog | 2 +- debian/vitastor.Dockerfile | 12 ++++++------ patches/cinder-vitastor.py | 2 +- rpm/build-tarball.sh | 2 +- rpm/vitastor-el7.Dockerfile | 2 +- rpm/vitastor-el7.spec | 4 ++-- rpm/vitastor-el8.Dockerfile | 2 +- rpm/vitastor-el8.spec | 4 ++-- src/CMakeLists.txt | 2 +- 14 files changed, 21 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3edbb140..84c6f237 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8) project(vitastor) -set(VERSION "0.6.4") +set(VERSION "0.6.5") add_subdirectory(src) diff --git a/csi/Makefile b/csi/Makefile index 0d017ad6..23e12a80 100644 --- a/csi/Makefile +++ b/csi/Makefile @@ -1,4 +1,4 @@ -VERSION ?= v0.6.4 +VERSION ?= v0.6.5 all: build push diff --git a/csi/deploy/004-csi-nodeplugin.yaml b/csi/deploy/004-csi-nodeplugin.yaml index 78b8af78..b2703250 100644 --- a/csi/deploy/004-csi-nodeplugin.yaml +++ b/csi/deploy/004-csi-nodeplugin.yaml @@ -49,7 +49,7 @@ spec: capabilities: add: ["SYS_ADMIN"] allowPrivilegeEscalation: true - image: vitalif/vitastor-csi:v0.6.4 + image: vitalif/vitastor-csi:v0.6.5 args: - "--node=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" diff --git a/csi/deploy/007-csi-provisioner.yaml b/csi/deploy/007-csi-provisioner.yaml index 65786518..4c9350f6 100644 --- a/csi/deploy/007-csi-provisioner.yaml +++ b/csi/deploy/007-csi-provisioner.yaml @@ -116,7 +116,7 @@ spec: privileged: true capabilities: add: ["SYS_ADMIN"] - image: vitalif/vitastor-csi:v0.6.4 + image: vitalif/vitastor-csi:v0.6.5 args: - "--node=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" diff --git a/csi/src/config.go b/csi/src/config.go index cb56a43e..f91b3527 100644 --- a/csi/src/config.go +++ b/csi/src/config.go @@ -5,7 +5,7 @@ package vitastor const ( vitastorCSIDriverName = "csi.vitastor.io" - vitastorCSIDriverVersion = "0.6.4" + vitastorCSIDriverVersion = "0.6.5" ) // Config struct fills the parameters of request or user input diff --git a/debian/changelog b/debian/changelog index 4c84a0c6..d063e65e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -vitastor (0.6.4-1) unstable; urgency=medium +vitastor (0.6.5-1) unstable; urgency=medium * RDMA support * Bugfixes diff --git a/debian/vitastor.Dockerfile b/debian/vitastor.Dockerfile index 7f224bb7..136b18b2 100644 --- a/debian/vitastor.Dockerfile +++ b/debian/vitastor.Dockerfile @@ -40,10 +40,10 @@ RUN set -e -x; \ mkdir -p /root/packages/vitastor-$REL; \ rm -rf /root/packages/vitastor-$REL/*; \ cd /root/packages/vitastor-$REL; \ - cp -r /root/vitastor vitastor-0.6.4; \ - ln -s /root/packages/qemu-$REL/qemu-*/ vitastor-0.6.4/qemu; \ - ln -s /root/fio-build/fio-*/ vitastor-0.6.4/fio; \ - cd vitastor-0.6.4; \ + cp -r /root/vitastor vitastor-0.6.5; \ + ln -s /root/packages/qemu-$REL/qemu-*/ vitastor-0.6.5/qemu; \ + ln -s /root/fio-build/fio-*/ vitastor-0.6.5/fio; \ + cd vitastor-0.6.5; \ FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \ QEMU=$(head -n1 qemu/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \ sh copy-qemu-includes.sh; \ @@ -59,8 +59,8 @@ RUN set -e -x; \ echo "dep:fio=$FIO" > debian/substvars; \ echo "dep:qemu=$QEMU" >> debian/substvars; \ cd /root/packages/vitastor-$REL; \ - tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.4.orig.tar.xz vitastor-0.6.4; \ - cd vitastor-0.6.4; \ + tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.6.5.orig.tar.xz vitastor-0.6.5; \ + cd vitastor-0.6.5; \ V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \ DEBFULLNAME="Vitaliy Filippov " dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \ DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \ diff --git a/patches/cinder-vitastor.py b/patches/cinder-vitastor.py index 22771d67..8284db88 100644 --- a/patches/cinder-vitastor.py +++ b/patches/cinder-vitastor.py @@ -50,7 +50,7 @@ from cinder.volume import configuration from cinder.volume import driver from cinder.volume import volume_utils -VERSION = '0.6.4' +VERSION = '0.6.5' LOG = logging.getLogger(__name__) diff --git a/rpm/build-tarball.sh b/rpm/build-tarball.sh index 3074b6ee..602a8fae 100755 --- a/rpm/build-tarball.sh +++ b/rpm/build-tarball.sh @@ -48,4 +48,4 @@ FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Ve QEMU=`rpm -qi qemu qemu-kvm | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'` perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec perl -i -pe 's/(Requires:\s*qemu(?:-kvm)?)([^\n]+)?/$1 = '$QEMU'/' $VITASTOR/rpm/vitastor-el$EL.spec -tar --transform 's#^#vitastor-0.6.4/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.4$(rpm --eval '%dist').tar.gz * +tar --transform 's#^#vitastor-0.6.5/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.6.5$(rpm --eval '%dist').tar.gz * diff --git a/rpm/vitastor-el7.Dockerfile b/rpm/vitastor-el7.Dockerfile index b7458f3c..91fdac14 100644 --- a/rpm/vitastor-el7.Dockerfile +++ b/rpm/vitastor-el7.Dockerfile @@ -38,7 +38,7 @@ ADD . /root/vitastor RUN set -e; \ cd /root/vitastor/rpm; \ sh build-tarball.sh; \ - cp /root/vitastor-0.6.4.el7.tar.gz ~/rpmbuild/SOURCES; \ + cp /root/vitastor-0.6.5.el7.tar.gz ~/rpmbuild/SOURCES; \ cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \ cd ~/rpmbuild/SPECS/; \ rpmbuild -ba vitastor.spec; \ diff --git a/rpm/vitastor-el7.spec b/rpm/vitastor-el7.spec index 8b50fe7f..7e8bd8ec 100644 --- a/rpm/vitastor-el7.spec +++ b/rpm/vitastor-el7.spec @@ -1,11 +1,11 @@ Name: vitastor -Version: 0.6.4 +Version: 0.6.5 Release: 1%{?dist} Summary: Vitastor, a fast software-defined clustered block storage License: Vitastor Network Public License 1.1 URL: https://vitastor.io/ -Source0: vitastor-0.6.4.el7.tar.gz +Source0: vitastor-0.6.5.el7.tar.gz BuildRequires: liburing-devel >= 0.6 BuildRequires: gperftools-devel diff --git a/rpm/vitastor-el8.Dockerfile b/rpm/vitastor-el8.Dockerfile index 96a96a69..f0fdf2dc 100644 --- a/rpm/vitastor-el8.Dockerfile +++ b/rpm/vitastor-el8.Dockerfile @@ -36,7 +36,7 @@ ADD . /root/vitastor RUN set -e; \ cd /root/vitastor/rpm; \ sh build-tarball.sh; \ - cp /root/vitastor-0.6.4.el8.tar.gz ~/rpmbuild/SOURCES; \ + cp /root/vitastor-0.6.5.el8.tar.gz ~/rpmbuild/SOURCES; \ cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \ cd ~/rpmbuild/SPECS/; \ rpmbuild -ba vitastor.spec; \ diff --git a/rpm/vitastor-el8.spec b/rpm/vitastor-el8.spec index 7ac6ccd4..bdacb57f 100644 --- a/rpm/vitastor-el8.spec +++ b/rpm/vitastor-el8.spec @@ -1,11 +1,11 @@ Name: vitastor -Version: 0.6.4 +Version: 0.6.5 Release: 1%{?dist} Summary: Vitastor, a fast software-defined clustered block storage License: Vitastor Network Public License 1.1 URL: https://vitastor.io/ -Source0: vitastor-0.6.4.el8.tar.gz +Source0: vitastor-0.6.5.el8.tar.gz BuildRequires: liburing-devel >= 0.6 BuildRequires: gperftools-devel diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9ac90913..e2d2e72d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,7 +15,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$") set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}") endif() -add_definitions(-DVERSION="0.6.4") +add_definitions(-DVERSION="0.6.5") add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -I ${CMAKE_SOURCE_DIR}/src) if (${WITH_ASAN}) add_definitions(-fsanitize=address -fno-omit-frame-pointer)