From 77155ab7bd6dc271a580da9d455c6da248ac17db Mon Sep 17 00:00:00 2001
From: Vitaliy Filippov <vitalif@yourcmc.ru>
Date: Mon, 8 May 2023 20:38:56 +0300
Subject: [PATCH] Add self-restart support to monitor (mainly for tests)

---
 mon/mon-main.js |  2 +-
 mon/mon.js      | 43 ++++++++++++++++++++++++++++++++++---------
 2 files changed, 35 insertions(+), 10 deletions(-)

diff --git a/mon/mon-main.js b/mon/mon-main.js
index ff90b8ce..2d32e989 100755
--- a/mon/mon-main.js
+++ b/mon/mon-main.js
@@ -13,7 +13,7 @@ for (let i = 2; i < process.argv.length; i++)
     {
         console.error('USAGE: '+process.argv[0]+' '+process.argv[1]+' [--verbose 1]'+
             ' [--etcd_address "http://127.0.0.1:2379,..."] [--config_path /etc/vitastor/vitastor.conf]'+
-            ' [--etcd_prefix "/vitastor"] [--etcd_start_timeout 5]');
+            ' [--etcd_prefix "/vitastor"] [--etcd_start_timeout 5] [--restart_interval 5]');
         process.exit();
     }
     else if (process.argv[i].substr(0, 2) == '--')
diff --git a/mon/mon.js b/mon/mon.js
index d58cc31e..e0b88420 100644
--- a/mon/mon.js
+++ b/mon/mon.js
@@ -561,7 +561,7 @@ class Mon
         }
         if (!this.ws)
         {
-            this.die('Failed to open etcd watch websocket');
+            await this.die('Failed to open etcd watch websocket');
         }
         const cur_addr = this.selected_etcd_url;
         this.ws_alive = true;
@@ -728,7 +728,7 @@ class Mon
             const res = await this.etcd_call('/lease/keepalive', { ID: this.etcd_lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries);
             if (!res.result.TTL)
             {
-                this.die('Lease expired');
+                await this.die('Lease expired');
             }
         }, this.config.etcd_mon_timeout);
         if (!this.signals_set)
@@ -741,9 +741,32 @@ class Mon
 
     async on_stop(status)
     {
-        clearInterval(this.lease_timer);
-        await this.etcd_call('/lease/revoke', { ID: this.etcd_lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries);
-        process.exit(status);
+        if (this.ws_keepalive_timer)
+        {
+            clearInterval(this.ws_keepalive_timer);
+            this.ws_keepalive_timer = null;
+        }
+        if (this.lease_timer)
+        {
+            clearInterval(this.lease_timer);
+            this.lease_timer = null;
+        }
+        if (this.etcd_lease_id)
+        {
+            const lease_id = this.etcd_lease_id;
+            this.etcd_lease_id = null;
+            await this.etcd_call('/lease/revoke', { ID: lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries);
+        }
+        if (!status || !this.initConfig.restart_interval)
+        {
+            process.exit(status);
+        }
+        else
+        {
+            console.log('Restarting after '+this.initConfig.restart_interval+' seconds');
+            await new Promise(ok => setTimeout(ok, this.initConfig.restart_interval*1000));
+            await this.start();
+        }
     }
 
     async become_master()
@@ -1771,14 +1794,13 @@ class Mon
                 return res.json;
             }
         }
-        this.die();
+        await this.die();
     }
 
-    _die(err)
+    async _die(err)
     {
-        // In fact we can just try to rejoin
         console.error(new Error(err || 'Cluster connection failed'));
-        process.exit(1);
+        await this.on_stop(1);
     }
 
     local_ips(all)
@@ -1823,6 +1845,7 @@ function POST(url, body, timeout)
             clearTimeout(timer_id);
             let res_body = '';
             res.setEncoding('utf8');
+            res.on('error', no);
             res.on('data', chunk => { res_body += chunk; });
             res.on('end', () =>
             {
@@ -1842,6 +1865,8 @@ function POST(url, body, timeout)
                 }
             });
         });
+        req.on('error', no);
+        req.on('close', () => no(new Error('Connection closed prematurely')));
         req.write(body_text);
         req.end();
     });