Move base64 handling into antietcd.js

README.md

@@ -48,8 +48,9 @@ Antietcd doesn't background itself, so use systemd or start-stop-daemon to run i
 
 ```
 node_modules/.bin/anticli [OPTIONS] put <key> [<value>]
-node_modules/.bin/anticli [OPTIONS] get <key> [-p|--prefix] [-v|--print-value-only] [-k|--keys-only]
+node_modules/.bin/anticli [OPTIONS] get <key> [-p|--prefix] [-v|--print-value-only] [-k|--keys-only] [--no-temp]
 node_modules/.bin/anticli [OPTIONS] del <key> [-p|--prefix]
+node_modules/.bin/anticli [OPTIONS] load [--with-lease] < dump.json
 ```
 
 For `put`, if `<value>` is not specified, it will be read from STDIN.
@@ -70,6 +71,9 @@ Options:
 <dt>--timeout 1000</dt>
 <dd>Specify request timeout in milliseconds</dd>
 
+<dt>--json or --write-out=json</dt>
+<dd>Print raw response in JSON</dd>
+
 </dl>
 
 ## Options
@@ -100,6 +104,9 @@ Specify &lt;ca&gt; = &lt;cert&gt; if your certificate is self-signed.</dd>
 <dt>--ws_keepalive_interval 30000</dt>
 <dd>Client websocket ping (keepalive) interval in milliseconds</dd>
 
+<dt>--use_base64 1</dt>
+<dd>Use base64 encoding of keys and values, like in etcd (enabled by default).</dd>
+
 </dl>
 
 ### Persistence

anticli.js

@@ -4,6 +4,7 @@
 // (c) Vitaliy Filippov, 2024
 // License: Mozilla Public License 2.0 or Vitastor Network Public License 1.1
 
+const fs = require('fs');
 const fsp = require('fs').promises;
 const http = require('http');
 const https = require('https');
@@ -15,13 +16,14 @@ License: Mozilla Public License 2.0 or Vitastor Network Public License 1.1
 Usage:
 
 anticli.js [OPTIONS] put <key> [<value>]
-anticli.js [OPTIONS] get <key> [-p|--prefix] [-v|--print-value-only] [-k|--keys-only]
+anticli.js [OPTIONS] get <key> [-p|--prefix] [-v|--print-value-only] [-k|--keys-only] [--no-temp]
 anticli.js [OPTIONS] del <key> [-p|--prefix]
+anticli.js [OPTIONS] load [--with-lease] < dump.json
 
 Options:
 
 [--endpoints|-e http://node1:2379,http://node2:2379,http://node3:2379]
-[--cert cert.pem] [--key key.pem] [--timeout 1000]
+[--cert cert.pem] [--key key.pem] [--timeout 1000] [--json]
 `;
 
 class AntiEtcdCli
@@ -54,6 +56,19 @@ class AntiEtcdCli
             {
                 options['keys_only'] = true;
             }
+            else if (arg == '--with_lease')
+            {
+                options['with_lease'] = true;
+            }
+            else if (arg == '--write_out' && args[i+1] == 'json')
+            {
+                i++;
+                options['json'] = true;
+            }
+            else if (arg == '--json' || arg == '--write_out=json')
+            {
+                options['json'] = true;
+            }
             else if (arg[0] == '-' && arg[1] !== '-')
             {
                 process.stderr.write('Unknown option '+arg);
@@ -68,9 +83,9 @@ class AntiEtcdCli
                 cmd.push(arg);
             }
         }
-        if (!cmd.length || cmd[0] != 'get' && cmd[0] != 'put' && cmd[0] != 'del')
+        if (!cmd.length || cmd[0] != 'get' && cmd[0] != 'put' && cmd[0] != 'del' && cmd[0] != 'load')
         {
-            process.stderr.write('Supported commands: get, put, del. Use --help to see details\n');
+            process.stderr.write('Supported commands: get, put, del, load. Use --help to see details\n');
             process.exit(1);
         }
         return [ cmd, options ];
@@ -102,12 +117,51 @@ class AntiEtcdCli
         {
             await this.del(cmd.slice(1));
         }
+        else if (cmd[0] == 'load')
+        {
+            await this.load();
+        }
         // wait until output is fully flushed
         await new Promise(ok => process.stdout.write('', ok));
         await new Promise(ok => process.stderr.write('', ok));
         process.exit(0);
     }
 
+    async load()
+    {
+        const dump = JSON.parse(await new Promise((ok, no) => fs.readFile(0, { encoding: 'utf-8' }, (err, res) => err ? no(err) : ok(res))));
+        if (!dump.responses && !dump.kvs)
+        {
+            console.error('dump should be /kv/txn or /kv/range response in json format');
+            process.exit(1);
+        }
+        const success = [];
+        for (const r of (dump.responses
+            ? dump.responses.map(r => r.response_range).filter(r => r)
+            : [ dump ]))
+        {
+            for (const kv of r.kvs)
+            {
+                if (kv.value == null)
+                {
+                    console.error('dump should contain values');
+                    process.exit(1);
+                }
+                success.push({ request_put: { key: kv.key, value: kv.value, lease: this.options.with_lease ? kv.lease||undefined : undefined } });
+            }
+        }
+        const res = await this.request('/v3/kv/txn', { success });
+        if (this.options.json)
+        {
+            process.stdout.write(JSON.stringify(res));
+            return;
+        }
+        if (res.succeeded)
+        {
+            process.stdout.write('OK, loaded '+success.length+' values\n');
+        }
+    }
+
     async get(keys)
     {
         if (this.options.prefix)
@@ -116,6 +170,22 @@ class AntiEtcdCli
         }
         const txn = { success: keys.map(key => ({ request_range: this.options.prefix ? { key: b64(key+'/'), range_end: b64(key+'0') } : { key: b64(key) } })) };
         const res = await this.request('/v3/kv/txn', txn);
+        if (this.options.notemp)
+        {
+            // Skip temporary values (values with lease)
+            for (const r of res.responses||[])
+            {
+                if (r.response_range)
+                {
+                    r.response_range.kvs = r.response_range.kvs.filter(kv => !kv.lease);
+                }
+            }
+        }
+        if (this.options.json)
+        {
+            process.stdout.write(JSON.stringify(keys.length == 1 ? res.responses[0].response_range : res));
+            return;
+        }
         for (const r of res.responses||[])
         {
             if (r.response_range)
@@ -139,7 +209,7 @@ class AntiEtcdCli
     {
         if (value === undefined)
         {
-            value = await fsp.readFile(0, { encoding: 'utf-8' });
+            value = await new Promise((ok, no) => fs.readFile(0, { encoding: 'utf-8' }, (err, res) => err ? no(err) : ok(res)));
         }
         const res = await this.request('/v3/kv/put', { key: b64(key), value: b64(value) });
         if (res.header)
@@ -175,18 +245,18 @@ class AntiEtcdCli
             {
                 if (res.json.error)
                 {
-                    process.stderr.write(cur_url+': '+res.json.error);
+                    process.stderr.write(cur_url+': '+res.json.error+'\n');
                     process.exit(1);
                 }
                 return res.json;
             }
             if (res.body)
             {
-                process.stderr.write(cur_url+': '+res.body);
+                process.stderr.write(cur_url+': '+res.body+'\n');
             }
             if (res.error)
             {
-                process.stderr.write(cur_url+': '+res.error);
+                process.stderr.write(cur_url+': '+res.error+'\n');
                 if (!res.response || !res.response.statusCode)
                 {
                     // This URL is unavailable

antietcd-app.js

@@ -43,6 +43,8 @@ HTTP:
     Require TLS client certificates signed by <ca> or by default CA to connect.
 --ws_keepalive_interval 30000
     Client websocket ping (keepalive) interval in milliseconds
+--use_base64 1
+    Use base64 encoding of keys and values, like in etcd (enabled by default).
 
 Persistence:
 
@@ -105,7 +107,14 @@ function parse()
             options[arg.substr(2)] = process.argv[++i];
         }
     }
+    if ('stale_read' in options)
+    {
         options['stale_read'] = options['stale_read'] === '1' || options['stale_read'] === 'yes' || options['stale_read'] === 'true';
+    }
+    if ('use_base64' in options)
+    {
+        options['use_base64'] = options['use_base64'] === '1' || options['use_base64'] === 'yes' || options['use_base64'] === 'true';
+    }
     if (options['persist_filter'])
     {
         options['persist_filter'] = require(options['persist_filter'])(options);

antietcd.js

@@ -15,13 +15,21 @@ const ws = require('ws');
 const EtcTree = require('./etctree.js');
 const AntiPersistence = require('./antipersistence.js');
 const AntiCluster = require('./anticluster.js');
-const { runCallbacks, RequestError } = require('./common.js');
+const { runCallbacks, de64, b64, RequestError } = require('./common.js');
 
 class AntiEtcd extends EventEmitter
 {
     constructor(cfg)
     {
         super();
+        if (!('use_base64' in cfg))
+        {
+            cfg.use_base64 = 1;
+        }
+        if (!('stale_read' in cfg))
+        {
+            cfg.stale_read = 1;
+        }
         this.clients = {};
         this.client_id = 1;
         this.etctree = new EtcTree(true);
@@ -411,7 +419,7 @@ class AntiEtcd extends EventEmitter
     // public watch API
     async create_watch(params, callback)
     {
-        const watch = this.etctree.api_create_watch({ ...params, watch_id: null }, callback);
+        const watch = this.etctree.api_create_watch(this._encodeWatch(params), (msg) => callback(this._encodeMsg(msg)));
         if (!watch.created)
         {
             throw new RequestError(400, 'Requested watch revision is compacted', { compact_revision: watch.compact_revision });
@@ -435,24 +443,64 @@ class AntiEtcd extends EventEmitter
     // internal handlers
     async _handle_kv_txn(data)
     {
-        return await this.etctree.api_txn(data);
+        if (this.cfg.use_base64)
+        {
+            for (const item of data.compare||[])
+            {
+                if (item.key != null)
+                    item.key = de64(item.key);
+            }
+            for (const items of [ data.success, data.failure ])
+            {
+                for (const item of items||[])
+                {
+                    const req = item.request_range || item.requestRange ||
+                        item.request_put || item.requestPut ||
+                        item.request_delete_range || item.requestDeleteRange;
+                    if (req.key != null)
+                        req.key = de64(req.key);
+                    if (req.range_end != null)
+                        req.range_end = de64(req.range_end);
+                    if (req.value != null)
+                        req.value = de64(req.value);
+                }
+            }
+        }
+        const result = await this.etctree.api_txn(data);
+        if (this.cfg.use_base64)
+        {
+            for (const item of result.responses||[])
+            {
+                if (item.response_range)
+                {
+                    for (const kv of item.response_range.kvs)
+                    {
+                        if (kv.key != null)
+                            kv.key = b64(kv.key);
+                        if (kv.value != null)
+                            kv.value = b64(kv.value);
+                    }
+                }
+            }
+        }
+        return result;
     }
 
     async _handle_kv_range(data)
     {
-        const r = await this.etctree.api_txn({ success: [ { request_range: data } ] });
+        const r = await this._handle_kv_txn({ success: [ { request_range: data } ] });
         return { header: r.header, ...r.responses[0].response_range };
     }
 
     async _handle_kv_put(data)
     {
-        const r = await this.etctree.api_txn({ success: [ { request_put: data } ] });
+        const r = await this._handle_kv_txn({ success: [ { request_put: data } ] });
         return { header: r.header, ...r.responses[0].response_put };
     }
 
     async _handle_kv_deleterange(data)
     {
-        const r = await this.etctree.api_txn({ success: [ { request_delete_range: data } ] });
+        const r = await this._handle_kv_txn({ success: [ { request_delete_range: data } ] });
         return { header: r.header, ...r.responses[0].response_delete_range };
     }
 
@@ -494,9 +542,7 @@ class AntiEtcd extends EventEmitter
             const create_request = msg.create_request;
             if (!create_request.watch_id || !client.watches[create_request.watch_id])
             {
-                const watch = this.etctree.api_create_watch(
+                const watch = this.etctree.api_create_watch(this._encodeWatch(create_request), (msg) => socket.send(JSON.stringify(this._encodeMsg(msg))));
-                    { ...create_request, watch_id: null }, (msg) => socket.send(JSON.stringify(msg))
-                );
                 if (!watch.created)
                 {
                     socket.send(JSON.stringify({ result: { header: { revision: this.etctree.mod_revision }, watch_id: create_request.watch_id, ...watch } }));
@@ -533,6 +579,35 @@ class AntiEtcd extends EventEmitter
         }
     }
 
+    _encodeWatch(create_request)
+    {
+        const req = { ...create_request, watch_id: null };
+        if (this.cfg.use_base64)
+        {
+            if (req.key != null)
+                req.key = de64(req.key);
+            if (req.range_end != null)
+                req.range_end = de64(req.range_end);
+        }
+        return req;
+    }
+
+    _encodeMsg(msg)
+    {
+        if (this.cfg.use_base64 && msg.result && msg.result.events)
+        {
+            return { ...msg, result: { ...msg.result, events: msg.result.events.map(ev => ({
+                ...ev,
+                kv: !ev.kv ? ev.kv : {
+                    ...ev.kv,
+                    key: b64(ev.kv.key),
+                    value: b64(ev.kv.value),
+                },
+            })) } };
+        }
+        return msg;
+    }
+
     _unsubscribeClient(client_id)
     {
         if (!this.clients[client_id])

antipersistence.js

@@ -8,7 +8,7 @@ const zlib = require('zlib');
 
 const stableStringify = require('./stable-stringify.js');
 const EtcTree = require('./etctree.js');
-const { de64, runCallbacks } = require('./common.js');
+const { runCallbacks } = require('./common.js');
 
 class AntiPersistence
 {
@@ -60,20 +60,18 @@ class AntiPersistence
                 if (ev.lease)
                 {
                     // Values with lease are never persisted
-                    const key = de64(ev.key);
+                    if (this.prev_value[ev.key] !== undefined)
-                    if (this.prev_value[key] !== undefined)
                     {
-                        delete this.prev_value[key];
+                        delete this.prev_value[ev.key];
                         changed = true;
                     }
                 }
                 else
                 {
-                    const key = de64(ev.key);
+                    const filtered = this.cfg.persist_filter(ev.key, ev.value == null ? undefined : ev.value);
-                    const filtered = this.cfg.persist_filter(key, ev.value == null ? undefined : de64(ev.value));
+                    if (!EtcTree.eq(filtered, this.prev_value[ev.key]))
-                    if (!EtcTree.eq(filtered, this.prev_value[key]))
                     {
-                        this.prev_value[key] = filtered;
+                        this.prev_value[ev.key] = filtered;
                         changed = true;
                     }
                 }

common.js

@@ -19,6 +19,13 @@ function de64(k)
     return Buffer.from(k, 'base64').toString();
 }
 
+function b64(k)
+{
+    if (k == null) // null or undefined
+        return k;
+    return Buffer.from(k).toString('base64');
+}
+
 function runCallbacks(obj, key, new_value)
 {
     const cbs = obj[key];
@@ -35,5 +42,6 @@ function runCallbacks(obj, key, new_value)
 module.exports = {
     RequestError,
     de64,
+    b64,
     runCallbacks,
 };

etctree.js

@@ -19,7 +19,7 @@ const { RequestError } = require('./common.js');
 
 class EtcTree
 {
-    constructor(use_base64)
+    constructor()
     {
         this.state = {};
         this.leases = {};
@@ -27,7 +27,6 @@ class EtcTree
         this.watcher_id = 0;
         this.mod_revision = 0;
         this.compact_revision = 0;
-        this.use_base64 = use_base64;
         this.replicate = null;
         this.paused = false;
         this.active_immediate = [];
@@ -51,23 +50,9 @@ class EtcTree
         this.replicate = replicate;
     }
 
-    de64(k)
-    {
-        if (k == null) // null or undefined
-            return k;
-        return this.use_base64 ? Buffer.from(k, 'base64').toString() : k;
-    }
-
-    b64(k)
-    {
-        if (k == null) // null or undefined
-            return k;
-        return this.use_base64 ? Buffer.from(k).toString('base64') : k;
-    }
-
     _check(chk)
     {
-        const parts = this._key_parts(this.de64(chk.key));
+        const parts = this._key_parts(chk.key);
         const { cur } = this._get_subtree(parts, false, false);
         let check_value, ref_value;
         if (chk.target === 'MOD')
@@ -118,9 +103,9 @@ class EtcTree
 
     _get_range(req)
     {
-        const key = this.de64(req.key);
+        const key = req.key;
-        const end = this.de64(req.range_end);
+        const end = req.range_end;
-        if (end != null && (key[key.length-1] != '/' || end[end.length-1] != '0' ||
+        if (end != null && (key !== '' && end !== '') && (key[key.length-1] != '/' || end[end.length-1] != '0' ||
             end.substr(0, end.length-1) !== key.substr(0, key.length-1)))
         {
             throw new RequestError(501, 'Non-directory range queries are unsupported');
@@ -279,7 +264,7 @@ class EtcTree
             {
                 cur_old.value = cur_new.value;
                 const key_watchers = (cur_old.key_watchers ? [ ...watchers, ...(cur_old.key_watchers||[]) ] : watchers);
-                const notify = { watchers: key_watchers, key, value: cur_new.value, mod_revision: cur_new.mod_revision };
+                const notify = { watchers: key_watchers, key: this.b64(key), value: this.b64(cur_new.value), mod_revision: cur_new.mod_revision };
                 if (cur_new.lease)
                 {
                     notify.lease = cur_new.lease;
@@ -444,7 +429,7 @@ class EtcTree
         }
         for (const key in this.leases[id].keys)
         {
-            this._delete_range({ key: this.use_base64 ? this.b64(key) : key }, next_revision, notifications);
+            this._delete_range({ key }, next_revision, notifications);
         }
         delete this.leases[id];
     }
@@ -597,9 +582,9 @@ class EtcTree
         {
             const ev = {
                 type: cur.value == null ? 'DELETE' : 'PUT',
-                kv: cur.value == null ? { key: this.b64(prefix === null ? '' : prefix) } : {
+                kv: cur.value == null ? { key: (prefix === null ? '' : prefix) } : {
-                    key: this.b64(prefix),
+                    key: prefix,
-                    value: this.b64(cur.value),
+                    value: cur.value,
                     mod_revision: cur.mod_revision,
                 },
             };
@@ -727,9 +712,9 @@ class EtcTree
     _put(request_put, cur_revision, notifications)
     {
         // FIXME: prev_kv, ignore_value(?), ignore_lease(?)
-        const parts = this._key_parts(this.de64(request_put.key));
+        const parts = this._key_parts(request_put.key);
         const key = parts.join('/');
-        const value = this.de64(request_put.value);
+        const value = request_put.value;
         const { cur, watchers } = this._get_subtree(parts, true, true);
         if (cur.key_watchers)
         {
@@ -762,7 +747,7 @@ class EtcTree
                 cur.create_revision = cur_revision;
             }
             cur.value = value;
-            const notify = { watchers, key: this.b64(key), value: this.b64(value), mod_revision: cur.mod_revision };
+            const notify = { watchers, key, value, mod_revision: cur.mod_revision };
             if (cur.lease)
             {
                 notify.lease = cur.lease;
@@ -793,10 +778,10 @@ class EtcTree
         }
         if (cur.value != null)
         {
-            const item = { key: this.b64(prefix === null ? '' : prefix) };
+            const item = { key: (prefix === null ? '' : prefix) };
             if (!req.keys_only)
             {
-                item.value = this.b64(cur.value);
+                item.value = cur.value;
                 item.mod_revision = cur.mod_revision;
                 //item.create_revision = cur.create_revision;
                 //item.version = cur.version;
@@ -833,7 +818,7 @@ class EtcTree
             this.mod_revision = cur_revision;
             notifications.push({
                 watchers: cur.key_watchers ? [ ...watchers, ...cur.key_watchers ] : watchers,
-                key: this.b64(prefix === null ? '' : prefix),
+                key: (prefix === null ? '' : prefix),
                 mod_revision: cur_revision,
             });
         }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "antietcd",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "Simplistic etcd replacement based on TinyRaft",
   "main": "antietcd.js",
   "scripts": {