Fix vitastor-cli fix hanging when an OSD is down

The client queues a request to an OSD which is down until the OSD comes back, so fix hung forever when the primary of an object's PG or one of --bad-osds was down. Refuse to start while one of --bad-osds is down, skip objects whose PG primary is down as it is done for inactive PGs, and exit with EIO when some objects were skipped or a request failed, so that a partial repair is not taken for a complete one.
Fix vitastor-cli describe hanging when an OSD is down
2026-10-25 16:10:29 +03:00 · 2026-10-25 16:10:29 +03:00 · 2026-10-25 16:10:29 +03:00
11 changed files with 203 additions and 250 deletions
--- a/docs/usage/cli.en.md
+++ b/docs/usage/cli.en.md
@@ -243,16 +243,15 @@ Flatten a layer, i.e. merge data and detach it from parents.

 ## rm-data

-`vitastor-cli rm-data --pool <id|name> --inode <inode> [--wait-list] [--min-offset <offset>]`
+`vitastor-cli rm-data --pool <pool> --inode <inode> [--wait-list] [--min-offset <offset>]`

-Remove inode data without changing metadata. `<inode>` may be decimal or 0x hex.
+Remove inode data without changing metadata.

 ```
 --wait-list   Retrieve full objects listings before starting to remove objects.
              Requires more memory, but allows to show correct removal progress.
 --min-offset  Purge only data starting with specified offset.
 --max-offset  Purge only data before specified offset.
-              Offsets are in bytes or with a K/M/G/T unit suffix, decimal or 0x hex.
 --client_wait_up_timeout 16  Timeout for waiting until PGs are up in seconds.
 ```

--- a/docs/usage/cli.ru.md
+++ b/docs/usage/cli.ru.md
@@ -252,16 +252,15 @@ vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M] \

 ## rm-data

-`vitastor-cli rm-data --pool <id|name> --inode <inode> [--wait-list] [--min-offset <offset>]`
+`vitastor-cli rm-data --pool <pool> --inode <inode> [--wait-list] [--min-offset <offset>]`

-Удалить данные инода, не меняя метаданные образов. `<inode>` можно задать в десятичном виде или в hex с 0x.
+Удалить данные инода, не меняя метаданные образов.

 ```
 --wait-list   Сначала запросить полный листинг объектов, а потом начать удалять.
              Требует больше памяти, но позволяет правильно печатать прогресс удаления.
 --min-offset  Удалять только данные, начиная с заданного смещения.
 --max-offset  Удалять только данные до (исключительно) заданного смещения.
-              Смещения задаются в байтах или с суффиксом K/M/G/T, в десятичном виде или в hex с 0x.
 --client_wait_up_timeout 16  Время ожидания поднятия PG в секундах.
 ```

--- a/src/cmd/cli.cpp
+++ b/src/cmd/cli.cpp
@@ -116,13 +116,12 @@ static const char* help_text =
    "vitastor-cli flatten <layer>\n"
    "  Flatten a layer, i.e. merge data and detach it from parents.\n"
    "\n"
-    "vitastor-cli rm-data --pool <id|name> --inode <inode> [--wait-list] [--min-offset <offset>]\n"
-    "  Remove inode data without changing metadata. <inode> may be decimal or 0x hex.\n"
+    "vitastor-cli rm-data --pool <pool> --inode <inode> [--wait-list] [--min-offset <offset>]\n"
+    "  Remove inode data without changing metadata.\n"
    "  --wait-list   Retrieve full objects listings before starting to remove objects.\n"
    "                Requires more memory, but allows to show correct removal progress.\n"
    "  --min-offset  Purge only data starting with specified offset.\n"
    "  --max-offset  Purge only data before specified offset.\n"
-    "                Offsets are in bytes or with a K/M/G/T unit suffix, decimal or 0x hex.\n"
    "  --client_wait_up_timeout 16  Timeout for waiting until PGs are up in seconds.\n"
    "\n"
    "vitastor-cli merge-data <from> <to> [--target <target>]\n"
@@ -688,7 +687,7 @@ static int run(cli_tool_t *p, json11::Json::object cfg)
    {
        printf("%s\n", result.data.dump().c_str());
    }
-    else if (p->json_output && result.err)
+    else if (p->json_output && result.err && !p->json_printed)
    {
        printf("%s\n", json11::Json(json11::Json::object {
            { "error_code", result.err },
--- a/src/cmd/cli.h
+++ b/src/cmd/cli.h
@@ -38,6 +38,7 @@ public:
    bool json_output = false;
    int log_level = 0;
    bool is_command_line = false;
+    bool json_printed = false;
    bool color = false;

    std::shared_ptr<user_info_t> user; // for http mode
--- a/src/cmd/cli_describe.cpp
+++ b/src/cmd/cli_describe.cpp
@@ -33,6 +33,13 @@ std::vector<uint64_t> parse_uint64_list(json11::Json val)
    return ret;
 }

+bool is_osd_up(cli_tool_t *parent, osd_num_t osd_num)
+{
+    auto & peer_states = parent->cli->st_cli->peer_states;
+    auto peer_it = peer_states.find(osd_num);
+    return peer_it != peer_states.end() && !peer_it->second.is_null();
+}
+
 struct cli_describe_t
 {
    uint64_t object_state = 0;
@@ -41,6 +48,8 @@ struct cli_describe_t
    std::vector<osd_num_t> only_osds;
    uint64_t min_inode = 0, max_inode = 0;
    uint64_t min_offset = 0, max_offset = 0;
+    std::map<osd_num_t, std::string> skipped_pgs;
+    bool incomplete = false;

    cli_tool_t *parent = NULL;
    int state = 0;
@@ -83,19 +92,18 @@ struct cli_describe_t
        }
        only_pg = cfg["pg"].uint64_value();
        min_inode = cfg["inode"].uint64_value();
+        min_offset = stoull_full(cfg["min_offset"].string_value(), 0);
+        max_offset = stoull_full(cfg["max_offset"].string_value(), 0);
        if (min_inode)
        {
            if (!INODE_POOL(min_inode))
                min_inode |= (uint64_t)only_pool << (64-POOL_ID_BITS);
            max_inode = min_inode;
-            min_offset = max_offset = 0;
        }
        else
        {
            min_inode = stoull_full(cfg["min_inode"].string_value(), 0); // to support 0x...
            max_inode = stoull_full(cfg["max_inode"].string_value(), 0);
-            min_offset = stoull_full(cfg["min_offset"].string_value(), 0);
-            max_offset = stoull_full(cfg["max_offset"].string_value(), 0);
            if (!min_inode && !max_inode && only_pool)
            {
                min_inode = (uint64_t)only_pool << (64-POOL_ID_BITS);
@@ -103,6 +111,9 @@ struct cli_describe_t
                    (((uint64_t)1 << (64-POOL_ID_BITS)) - 1);
            }
        }
+        // The OSD ends the listing at (max_inode, max_offset) inclusive
+        if (max_inode && cfg["max_offset"].is_null())
+            max_offset = UINT64_MAX;
        only_osds = parse_uint64_list(cfg["osds"]);
        object_state = stoull_full(cfg["object_state"].string_value(), 0);
        if (!object_state && cfg["object_state"].is_string())
@@ -158,15 +169,49 @@ struct cli_describe_t
                if (pp.first >= min_pool && (!max_pool || pp.first <= max_pool))
                {
                    for (auto & pgp: pp.second.pg_config)
-                        only_osds.push_back(pgp.second.cur_primary);
+                    {
+                        if (only_pool && only_pg && (pp.first != only_pool || pgp.first != only_pg))
+                            continue;
+                        if (is_osd_up(parent, pgp.second.cur_primary))
+                            only_osds.push_back(pgp.second.cur_primary);
+                        else
+                        {
+                            auto & pgs = skipped_pgs[pgp.second.cur_primary];
+                            pgs += (pgs == "" ? "" : ", ")+std::to_string(pp.first)+"/"+std::to_string(pgp.first);
+                        }
+                    }
                }
            }
        }
+        else
+        {
+            for (auto osd_num: only_osds)
+            {
+                if (!is_osd_up(parent, osd_num))
+                    skipped_pgs[osd_num] = "";
+            }
+            only_osds.erase(std::remove_if(only_osds.begin(), only_osds.end(), [this](osd_num_t osd_num)
+            {
+                return !is_osd_up(parent, osd_num);
+            }), only_osds.end());
+        }
+        // Requests to an OSD which is down are never answered
+        for (auto & sk: skipped_pgs)
+        {
+            if (!sk.first)
+                fprintf(stderr, "No primary OSD for PG %s, skipping\n", sk.second.c_str());
+            else if (sk.second != "")
+                fprintf(stderr, "OSD %ju is down, skipping PG %s\n", sk.first, sk.second.c_str());
+            else
+                fprintf(stderr, "OSD %ju is down, skipping it\n", sk.first);
+            incomplete = true;
+        }
        remove_duplicates(only_osds);
        parent->cli->init_msgr();
        if (parent->json_output && parent->is_command_line)
        {
            printf("[\n");
+            parent->json_printed = true;
        }
        for (int i = 0; i < only_osds.size(); i++)
        {
@@ -190,6 +235,7 @@ struct cli_describe_t
            {
                if (op->reply.hdr.retval < 0)
                {
+                    incomplete = true;
                    fprintf(
                        stderr, "Failed to describe objects on OSD %ju (retval=%jd)\n",
                        osd_num, op->reply.hdr.retval
@@ -197,6 +243,7 @@ struct cli_describe_t
                }
                else if (op->reply.describe.result_bytes != op->reply.hdr.retval * sizeof(osd_reply_describe_item_t))
                {
+                    incomplete = true;
                    fprintf(
                        stderr, "Invalid response size from OSD %ju (expected %ju bytes, got %ju bytes)\n",
                        osd_num, op->reply.hdr.retval * sizeof(osd_reply_describe_item_t), op->reply.describe.result_bytes
@@ -263,6 +310,11 @@ struct cli_describe_t
        {
            result.data = describe_items;
        }
+        if (incomplete)
+        {
+            result.err = EIO;
+            result.text = "The listing is incomplete, see the messages above";
+        }
        state = 100;
        describe_items.clear();
    }
--- a/src/cmd/cli_fix.cpp
+++ b/src/cmd/cli_fix.cpp
@@ -21,6 +21,7 @@ struct cli_fix_t
    json11::Json options;
    cli_result_t result;
    json11::Json::array fix_result;
+    bool incomplete = false;

    bool is_done()
    {
@@ -123,6 +124,20 @@ struct cli_fix_t
            state = 100;
            return;
        }
+        {
+            std::string down_osds;
+            for (auto osd_num: bad_osds)
+            {
+                if (!is_osd_up(parent, osd_num))
+                    down_osds += (down_osds == "" ? "" : ", ")+std::to_string(osd_num);
+            }
+            if (down_osds != "")
+            {
+                result = (cli_result_t){ .err = EAGAIN, .text = "Bad OSD(s) "+down_osds+" are down, can't remove object copies from them" };
+                state = 100;
+                return;
+            }
+        }
        remove_duplicates(objects);
        parent->cli->init_msgr();
    resume_1:
@@ -138,6 +153,7 @@ struct cli_fix_t
            if (pool_cfg_it == parent->cli->st_cli->pool_config.end())
            {
                fprintf(stderr, "Object %jx:%jx is from unknown pool\n", obj.inode, obj.stripe);
+                incomplete = true;
                continue;
            }
            auto & pool_cfg = pool_cfg_it->second;
@@ -150,9 +166,20 @@ struct cli_fix_t
                    stderr, "Object %jx:%jx is from PG %u/%u which is not currently active\n",
                    obj.inode, obj.stripe, pool_cfg_it->first, pg_num
                );
+                incomplete = true;
                continue;
            }
            osd_num_t primary_osd = pg_it->second.cur_primary;
+            if (!is_osd_up(parent, primary_osd))
+            {
+                // Requests to an OSD which is down are never answered
+                fprintf(
+                    stderr, "Object %jx:%jx is from PG %u/%u whose primary OSD %ju is down\n",
+                    obj.inode, obj.stripe, pool_cfg_it->first, pg_num, primary_osd
+                );
+                incomplete = true;
+                continue;
+            }
            // Describe -> Remove some copies -> Scrub again
            osd_op_t *op = new osd_op_t;
            op->req = (osd_any_op_t){
@@ -172,6 +199,7 @@ struct cli_fix_t
                if (op->reply.hdr.retval < 0 || op->reply.describe.result_bytes != op->reply.hdr.retval * sizeof(osd_reply_describe_item_t))
                {
                    fprintf(stderr, "Failed to describe objects on OSD %ju (retval=%jd)\n", primary_osd, op->reply.hdr.retval);
+                    incomplete = true;
                    parent->waiting--;
                    loop();
                }
@@ -208,6 +236,7 @@ struct cli_fix_t
                                (*rm_count)--;
                                if (rm_op->reply.hdr.retval < 0)
                                {
+                                    incomplete = true;
                                    fprintf(
                                        stderr, "Failed to remove object %jx:%jx from OSD %ju (retval=%jd)\n",
                                        rm_op->req.sec_del.oid.inode, rm_op->req.sec_del.oid.stripe,
@@ -252,6 +281,7 @@ struct cli_fix_t
                                    {
                                        if (scrub_op->reply.hdr.retval < 0 && scrub_op->reply.hdr.retval != -ENOENT)
                                        {
+                                            incomplete = true;
                                            fprintf(
                                                stderr, "Failed to scrub %jx:%jx on OSD %ju (retval=%jd)\n",
                                                obj.inode, obj.stripe, primary_osd, scrub_op->reply.hdr.retval
@@ -289,6 +319,11 @@ struct cli_fix_t
        {
            result.data = fix_result;
        }
+        if (incomplete)
+        {
+            result.err = EIO;
+            result.text = "Some objects were not fixed, see the messages above";
+        }
        state = 100;
    }
 };
--- a/src/cmd/cli_fix.h
+++ b/src/cmd/cli_fix.h
@@ -7,6 +7,7 @@
 #include <algorithm>

 std::vector<uint64_t> parse_uint64_list(json11::Json val);
+bool is_osd_up(cli_tool_t *parent, osd_num_t osd_num);

 template<class T> void remove_duplicates(std::vector<T> & ret)
 {
--- a/src/cmd/cli_rm_data.cpp
+++ b/src/cmd/cli_rm_data.cpp
@@ -1,7 +1,6 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.1 (see README.md for details)

-#include <ctype.h>
 #include "cli.h"
 #include "cluster_client.h"

@@ -24,7 +23,6 @@ struct rm_inode_t
 {
    uint64_t inode = 0;
    pool_id_t pool_id = 0;
-    std::string pool_name;
    uint64_t min_offset = 0;
    uint64_t max_offset = 0;
    bool down_ok = false;
@@ -304,25 +302,6 @@ struct rm_inode_t
            goto resume_1;
        if (state == 100)
            return;
-        if (pool_name != "")
-        {
-            pool_id = 0;
-            for (auto & pp: parent->cli->st_cli->pool_config)
-            {
-                if (pp.second.name == pool_name)
-                {
-                    pool_id = pp.first;
-                    break;
-                }
-            }
-            if (!pool_id)
-            {
-                result = (cli_result_t){ .err = ENOENT, .text = "Pool "+pool_name+" does not exist" };
-                state = 100;
-                return;
-            }
-            inode = INODE_WITH_POOL(pool_id, inode);
-        }
        if (!pool_id)
        {
            result = (cli_result_t){ .err = EINVAL, .text = "Pool is not specified" };
@@ -338,103 +317,20 @@ struct rm_inode_t
    }
 };

-// Decimal or 0x hex, without sign, octal or overflow
-static bool parse_rm_number(std::string s, uint64_t & value)
-{
-    int base = 10;
-    if (s.size() > 2 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X'))
-    {
-        s = s.substr(2);
-        base = 16;
-    }
-    if (s == "" || s.find_first_not_of(base == 16 ? "0123456789abcdefABCDEF" : "0123456789") != std::string::npos)
-        return false;
-    errno = 0;
-    value = strtoull(s.c_str(), NULL, base);
-    return !errno;
-}
-
-// Command-line values are strings with optional K/M/G/T suffixes, internal callers pass numbers
-static bool parse_rm_offset(const json11::Json & value, uint64_t & offset)
-{
-    if (!value.is_string())
-    {
-        offset = value.uint64_value();
-        return true;
-    }
-    std::string s = value.string_value();
-    int shift = 0;
-    switch (s.size() ? tolower(s.back()) : 0)
-    {
-        case 'k': shift = 10; break;
-        case 'm': shift = 20; break;
-        case 'g': shift = 30; break;
-        case 't': shift = 40; break;
-    }
-    if (shift)
-        s.pop_back();
-    if (!parse_rm_number(s, offset) || offset > (UINT64_MAX >> shift))
-        return false;
-    offset <<= shift;
-    return true;
-}
-
-static bool parse_rm_inode(const json11::Json & value, uint64_t & inode)
-{
-    if (!value.is_string())
-    {
-        inode = value.uint64_value();
-        return true;
-    }
-    return parse_rm_number(value.string_value(), inode);
-}
-
-// A non-numeric string is a pool name, resolved after loading the pool configuration
-static bool parse_rm_pool(const json11::Json & value, uint64_t & pool_id, std::string & pool_name)
-{
-    pool_id = value.uint64_value();
-    if (value.is_string() && !pool_id && value.string_value() != "0")
-        pool_name = value.string_value();
-    return pool_id < POOL_ID_MAX && (!value.is_string() || value.string_value() != "");
-}
-
 std::function<bool(cli_result_t &)> cli_tool_t::start_rm_data(json11::Json cfg)
 {
-    uint64_t inode = 0, pool_id = 0, min_offset = 0, max_offset = 0;
-    std::string pool_name, error;
-    if (!parse_rm_inode(cfg["inode"], inode))
-        error = "Invalid --inode value: "+cfg["inode"].as_string();
-    else if (!parse_rm_pool(cfg["pool"], pool_id, pool_name))
-        error = "Invalid --pool value: "+cfg["pool"].as_string();
-    else if (!parse_rm_offset(cfg["min_offset"], min_offset))
-        error = "Invalid --min-offset value: "+cfg["min_offset"].as_string();
-    else if (!parse_rm_offset(cfg["max_offset"], max_offset))
-        error = "Invalid --max-offset value: "+cfg["max_offset"].as_string();
-    else if (!max_offset && cfg["max_offset"].is_string())
-        error = "--max-offset 0 is invalid, omit it to remove data up to the end";
-    else if (max_offset && min_offset >= max_offset)
-        error = "--min-offset must be less than --max-offset";
-    if (error != "")
-    {
-        return [error](cli_result_t & result)
-        {
-            result = (cli_result_t){ .err = EINVAL, .text = error };
-            return true;
-        };
-    }
    auto remover = new rm_inode_t();
    remover->parent = this;
-    remover->inode = inode;
-    remover->pool_id = pool_id;
-    remover->pool_name = pool_name;
+    remover->inode = cfg["inode"].uint64_value();
+    remover->pool_id = cfg["pool"].uint64_value();
    if (remover->pool_id)
    {
        remover->inode = (remover->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)) | (((uint64_t)remover->pool_id) << (64-POOL_ID_BITS));
    }
    remover->down_ok = cfg["down_ok"].bool_value();
    remover->pool_id = INODE_POOL(remover->inode);
-    remover->min_offset = min_offset;
-    remover->max_offset = max_offset;
+    remover->min_offset = cfg["min_offset"].uint64_value();
+    remover->max_offset = cfg["max_offset"].uint64_value();
    return [remover](cli_result_t & result)
    {
        remover->loop();
--- a/tests/run_tests.sh
+++ b/tests/run_tests.sh
@@ -65,6 +65,7 @@ OLD=1 ./test_move_reappear.sh
 ./test_rm.sh

 ./test_rm_degraded.sh
+./test_describe.sh

 ./test_snapshot_chain.sh
 SCHEME=ec ./test_snapshot_chain.sh
--- a/tests/test_describe.sh
+++ b/tests/test_describe.sh
@@ -0,0 +1,98 @@
+#!/bin/bash -ex
+
+# Test vitastor-cli describe and fix on degraded objects and with down OSDs
+
+PG_COUNT=4
+PG_SIZE=3
+PG_MINSIZE=2
+# Keep degraded objects until the checks are done
+GLOBAL_CONFIG=',"no_recovery":true'
+. `dirname $0`/run_3osds.sh
+
+$VITASTOR_CLI create -s 32M testimg
+
+# Data written while OSD 3 is down is degraded when it comes back
+kill -9 $OSD3_PID
+$ETCDCTL del /vitastor/osd/state/3
+wait_condition 60 "$ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '[ .[] | select(.state | index(\"degraded\")) ] | length == $PG_COUNT'" "PG peering without OSD 3"
+$VITASTOR_CLI dd if=/dev/urandom oimg=testimg bs=1M count=32
+start_osd 3
+wait_condition 60 "$ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '[ .[] | select((.state | index(\"active\")) and (.state | index(\"has_degraded\"))) ] | length == $PG_COUNT'" "PG peering with OSD 3"
+
+INODE_ID=$($VITASTOR_CLI ls --json | jq -r '.[] | select(.name == "testimg") | .inode_id')
+INODE=$(printf '0x%x' $INODE_ID)
+INODE_NUM=$((INODE_ID & 0xffffffffffff))
+
+# --inode lists the whole inode, not only its first object
+all=$($VITASTOR_CLI describe --json | jq -c '[ .[] | select(.inode == "'$INODE'") | .stripe ] | unique')
+if [[ $(echo "$all" | jq length) -le 1 ]]; then
+    format_error "describe should list more than one degraded object of testimg: $all"
+fi
+one=$($VITASTOR_CLI describe --pool 1 --inode $INODE_NUM --json | jq -c '[ .[].stripe ] | unique')
+if [[ "$one" != "$all" ]]; then
+    format_error "describe --inode lists objects $one instead of $all"
+fi
+
+# --min-offset and --max-offset narrow --inode, and --max-inode includes its whole inode
+second=$(echo "$all" | jq -r '.[1]')
+part=$($VITASTOR_CLI describe --pool 1 --inode $INODE_NUM --min-offset $second --max-offset $second --json | jq -c '[ .[].stripe ] | unique')
+if [[ "$part" != "[\"$second\"]" ]]; then
+    format_error "describe --inode --min-offset $second --max-offset $second lists objects $part"
+fi
+range=$($VITASTOR_CLI describe --min-inode $INODE --max-inode $INODE --json | jq -c '[ .[].stripe ] | unique')
+if [[ "$range" != "$all" ]]; then
+    format_error "describe --min-inode $INODE --max-inode $INODE lists objects $range instead of $all"
+fi
+
+# describe and fix must not wait for an OSD which is down.
+# Stop the monitor so that PGs keep the down OSD as their primary,
+# and stop a primary OSD which has the latest data, so other PGs stay degraded.
+VICTIM=$($VITASTOR_CLI ls-pgs --json | jq -r '[ .[].cur_primary | select(. == 1 or . == 2) ] | first')
+VICTIM_PG=$($VITASTOR_CLI ls-pgs --json | jq -r '[ .[] | select(.cur_primary == '$VICTIM') | .pg_num ] | first')
+OTHER_PG=$($VITASTOR_CLI ls-pgs --json | jq -r '[ .[] | select(.cur_primary != '$VICTIM') | .pg_num ] | first')
+OTHER_PG_COUNT=$($VITASTOR_CLI ls-pgs --json | jq '[ .[] | select(.cur_primary != '$VICTIM') ] | length')
+if [[ "$VICTIM" = null || "$OTHER_PG_COUNT" = 0 ]]; then
+    format_error "Expected PGs with primary OSD 1 or 2 and PGs with other primary OSDs"
+fi
+VICTIM_OBJ=$($VITASTOR_CLI describe --pool 1 --pg $VICTIM_PG --json | jq -r '.[0] | .inode+":"+.stripe')
+OTHER_OBJ=$($VITASTOR_CLI describe --pool 1 --pg $OTHER_PG --json | jq -r '.[0] | .inode+":"+.stripe')
+
+kill -9 $MON_PID
+eval kill -9 \$OSD${VICTIM}_PID
+$ETCDCTL del /vitastor/osd/state/$VICTIM
+wait_condition 60 "$ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '[ .[] | select(.primary != $VICTIM and (.state | index(\"degraded\"))) ] | length == $OTHER_PG_COUNT'" "PG peering without OSD $VICTIM"
+# Keep PG 1/$VICTIM_PG active with the down primary, as it is until the OSD's etcd lease expires
+$ETCDCTL put /vitastor/pg/state/1/$VICTIM_PG '{"peers":[1,2,3],"primary":'$VICTIM',"state":["active"]}'
+
+# A listing with skipped OSDs or PGs is incomplete: EIO, but still valid JSON on stdout
+for args in "" "--osds 1,2,3" "--pool 1 --pg $VICTIM_PG"; do
+    rc=0
+    timeout 30 $VITASTOR_CLI describe $args --json > ./testdata/describe.json 2> ./testdata/describe.err || rc=$?
+    [[ $rc == 5 ]] || format_error "describe $args exited with $rc instead of EIO with OSD $VICTIM down"
+    grep -q "OSD $VICTIM is down, skipping \(it\|PG 1/$VICTIM_PG\)" ./testdata/describe.err || \
+        format_error "describe $args should warn about OSD $VICTIM: $(cat ./testdata/describe.err)"
+    grep -q "The listing is incomplete" ./testdata/describe.err || \
+        format_error "describe $args should say that the listing is incomplete: $(cat ./testdata/describe.err)"
+    count=$(jq length ./testdata/describe.json) || format_error "describe $args printed invalid JSON"
+    if [[ "$args" == --pool* ]]; then
+        [[ $count == 0 ]] || format_error "describe $args should list nothing"
+    else
+        [[ $count -gt 0 ]] || format_error "describe $args should still list objects from the other OSDs"
+    fi
+done
+
+if timeout 30 $VITASTOR_CLI fix --bad-osds $VICTIM --objects $OTHER_OBJ 2> ./testdata/fix.err; then
+    format_error "fix should refuse to work with bad OSD $VICTIM down"
+fi
+grep -q "Bad OSD(s) $VICTIM are down" ./testdata/fix.err || format_error "fix printed: $(cat ./testdata/fix.err)"
+
+# An object which is skipped is not fixed: EIO
+rc=0
+timeout 30 $VITASTOR_CLI fix --bad-osds 3 --objects $VICTIM_OBJ 2> ./testdata/fix.err || rc=$?
+[[ $rc == 5 ]] || format_error "fix exited with $rc instead of EIO on an object from PG 1/$VICTIM_PG with primary OSD $VICTIM down"
+grep -q "is from PG 1/$VICTIM_PG whose primary OSD $VICTIM is down" ./testdata/fix.err || \
+    format_error "fix should skip the object from PG 1/$VICTIM_PG: $(cat ./testdata/fix.err)"
+grep -q "Some objects were not fixed" ./testdata/fix.err || \
+    format_error "fix should say that it skipped objects: $(cat ./testdata/fix.err)"
+
+format_green OK
--- a/tests/test_rm.sh
+++ b/tests/test_rm.sh
@@ -24,132 +24,4 @@ sz=$(stat -c %s ./testdata/bin/testimg.bin)

 [[ "$sz" = 67108864 ]] || format_error "File size is $sz bytes after resize to 64 M"

-# test rm-data offset parsing and validation
-
-$VITASTOR_CLI create -s 32M rmimg
-dd if=/dev/urandom of=./testdata/bin/rmimg.bin bs=1M count=32
-$VITASTOR_CLI dd iodepth=4 bs=1M if=./testdata/bin/rmimg.bin oimg=rmimg
-
-check_rmimg()
-{
-    rm -f ./testdata/bin/check.bin
-    $VITASTOR_CLI dd iodepth=4 bs=1M conv=nosparse iimg=rmimg of=./testdata/bin/check.bin
-    cmp ./testdata/bin/check.bin ./testdata/bin/rmimg.bin || format_error "$1"
-}
-
-zero_rmimg()
-{
-    dd if=/dev/zero of=./testdata/bin/rmimg.bin bs=1M seek=$1 count=$2 conv=notrunc
-}
-
-check_rmimg "Image data differs after write"
-
-RM_DATA="$VITASTOR_CLI rm-data --pool 1 --inode $($VITASTOR_CLI ls --json rmimg | jq -r '.[0].inode_num')"
-
-# Bad offsets and empty ranges must fail without removing anything
-while IFS='|' read -r -u 3 args err; do
-    if out=$($RM_DATA $args 2>&1); then
-        format_error "rm-data $args should fail"
-    fi
-    [[ "$out" = "$err" ]] || format_error "rm-data $args failed with an unexpected error: $out"
-    check_rmimg "rm-data $args removed data"
-done 3<<EOF
--min-offset 8M --max-offset 10X|Invalid --max-offset value: 10X
--min-offset 8M --max-offset 1.5G|Invalid --max-offset value: 1.5G
--min-offset 8M --max-offset 0x|Invalid --max-offset value: 0x
--min-offset 8X --max-offset 16M|Invalid --min-offset value: 8X
--min-offset 1.5M|Invalid --min-offset value: 1.5M
--min-offset 8M --max-offset -8M|Invalid --max-offset value: -8M
--min-offset -1|Invalid --min-offset value: -1
--min-offset 16777217T|Invalid --min-offset value: 16777217T
--min-offset 8M --max-offset 16777217T|Invalid --max-offset value: 16777217T
--min-offset 18446744073709551616|Invalid --min-offset value: 18446744073709551616
--min-offset 8M --max-offset 0x10000000000000000|Invalid --max-offset value: 0x10000000000000000
--min-offset 16M --max-offset 8M|--min-offset must be less than --max-offset
--min-offset 8M --max-offset 8M|--min-offset must be less than --max-offset
--min-offset 16M --max-offset 0|--max-offset 0 is invalid, omit it to remove data up to the end
--max-offset 0M|--max-offset 0 is invalid, omit it to remove data up to the end
-EOF
-
-$RM_DATA --min-offset 010M --max-offset 012M
-zero_rmimg 10 2
-check_rmimg "rm-data --min-offset 010M --max-offset 012M removed wrong data"
-
-$RM_DATA --min-offset 8M --max-offset 10M
-zero_rmimg 8 2
-check_rmimg "rm-data --min-offset 8M --max-offset 10M removed wrong data"
-
-$RM_DATA --min-offset $((20*1024*1024)) --max-offset $((24*1024*1024))
-zero_rmimg 20 4
-check_rmimg "rm-data with offsets in bytes removed wrong data"
-
-$RM_DATA --min-offset 0x1a00000 --max-offset 0x1c00000
-zero_rmimg 26 2
-check_rmimg "rm-data with hex offsets removed wrong data"
-
-$RM_DATA --min-offset 30M
-zero_rmimg 30 2
-check_rmimg "rm-data --min-offset 30M removed wrong data"
-
-# test rm-data pool names and hex inode numbers
-
-$VITASTOR_CLI create-pool testpool2 -s 2 -n 4 --failure_domain osd
-wait_pool_up 60 2 2 4
-
-$VITASTOR_CLI create -s 32M -p testpool2 rmimg2
-dd if=/dev/urandom of=./testdata/bin/rmimg2.bin bs=1M count=32
-$VITASTOR_CLI dd iodepth=4 bs=1M if=./testdata/bin/rmimg2.bin oimg=rmimg2
-
-zero_rmimg2()
-{
-    dd if=/dev/zero of=./testdata/bin/rmimg2.bin bs=1M seek=$1 count=$2 conv=notrunc
-}
-
-check_rmimg2()
-{
-    check_rmimg "$1"
-    rm -f ./testdata/bin/check.bin
-    $VITASTOR_CLI dd iodepth=4 bs=1M conv=nosparse iimg=rmimg2 of=./testdata/bin/check.bin
-    cmp ./testdata/bin/check.bin ./testdata/bin/rmimg2.bin || format_error "$1"
-}
-
-check_rmimg2 "Image data differs after write"
-
-INODE1=$($VITASTOR_CLI ls --json rmimg | jq -r '.[0].inode_num')
-INODE2=$($VITASTOR_CLI ls --json rmimg2 | jq -r '.[0].inode_num')
-[[ "$($VITASTOR_CLI ls --json rmimg2 | jq -r '.[0].pool_id')" = 2 ]] || format_error "rmimg2 is not in pool 2"
-
-# Bad inode numbers and unknown pools must fail without removing anything
-while IFS='|' read -r -u 3 args err; do
-    if out=$($VITASTOR_CLI rm-data $args 2>&1); then
-        format_error "rm-data $args should fail"
-    fi
-    [[ "$out" = "$err" ]] || format_error "rm-data $args failed with an unexpected error: $out"
-    check_rmimg2 "rm-data $args removed data"
-done 3<<EOF
--pool 2 --inode ${INODE2}x|Invalid --inode value: ${INODE2}x
--pool 2 --inode 0x|Invalid --inode value: 0x
--pool 2 --inode 0x0x$INODE2|Invalid --inode value: 0x0x$INODE2
--pool 2 --inode -1|Invalid --inode value: -1
--pool 70000 --inode $INODE2|Invalid --pool value: 70000
--pool nosuchpool --inode $(( (1<<48) | INODE1 ))|Pool nosuchpool does not exist
-EOF
-
-$VITASTOR_CLI rm-data --pool testpool2 --inode $INODE2 --min-offset 8M --max-offset 12M
-zero_rmimg2 8 4
-check_rmimg2 "rm-data --pool testpool2 --inode $INODE2 removed wrong data"
-
-$VITASTOR_CLI rm-data --pool 2 --inode $(printf 0x%x $INODE2) --min-offset 16M --max-offset 20M
-zero_rmimg2 16 4
-check_rmimg2 "rm-data --pool 2 --inode $(printf 0x%x $INODE2) removed wrong data"
-
-$VITASTOR_CLI rm-data --inode $(printf 0x%x $(( (2<<48) | INODE2 ))) --min-offset 20M --max-offset 24M
-zero_rmimg2 20 4
-check_rmimg2 "rm-data --inode $(printf 0x%x $(( (2<<48) | INODE2 ))) removed wrong data"
-
-# --pool overrides pool bits of --inode
-$VITASTOR_CLI rm-data --pool testpool --inode $(( (2<<48) | INODE1 )) --min-offset 0 --max-offset 4M
-zero_rmimg 0 4
-check_rmimg2 "rm-data --pool testpool --inode $(( (2<<48) | INODE1 )) removed wrong data"
-
 format_green OK