From a409598b160c188a8b2ee754dce6941afa5def2d Mon Sep 17 00:00:00 2001
From: Vitaliy Filippov <vitalif@yourcmc.ru>
Date: Tue, 9 May 2023 20:16:34 +0300
Subject: [PATCH] Wait for free space again, but count on big_write flushes
 instead of just flusher activity

---
 src/blockstore_impl.cpp     | 12 ++++++++++++
 src/blockstore_impl.h       |  3 +++
 src/blockstore_rollback.cpp |  5 +++++
 src/blockstore_stable.cpp   |  2 ++
 src/blockstore_write.cpp    |  7 +++++++
 5 files changed, 29 insertions(+)

diff --git a/src/blockstore_impl.cpp b/src/blockstore_impl.cpp
index 32952740..253d5257 100644
--- a/src/blockstore_impl.cpp
+++ b/src/blockstore_impl.cpp
@@ -307,6 +307,18 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
         }
         PRIV(op)->wait_for = 0;
     }
+    else if (PRIV(op)->wait_for == WAIT_FREE)
+    {
+        if (!data_alloc->get_free_count() && big_to_flush > 0)
+        {
+#ifdef BLOCKSTORE_DEBUG
+            printf("Still waiting for free space on the data device\n");
+#endif
+            return;
+        }
+        flusher->release_trim();
+        PRIV(op)->wait_for = 0;
+    }
     else
     {
         throw std::runtime_error("BUG: op->wait_for value is unexpected");
diff --git a/src/blockstore_impl.h b/src/blockstore_impl.h
index c7d7d6ef..95b3d405 100644
--- a/src/blockstore_impl.h
+++ b/src/blockstore_impl.h
@@ -160,6 +160,8 @@ struct __attribute__((__packed__)) dirty_entry
 #define WAIT_JOURNAL 3
 // Suspend operation until the next journal sector buffer is free
 #define WAIT_JOURNAL_BUFFER 4
+// Suspend operation until there is some free space on the data device
+#define WAIT_FREE 5
 
 struct fulfill_read_t
 {
@@ -263,6 +265,7 @@ class blockstore_impl_t
 
     struct journal_t journal;
     journal_flusher_t *flusher;
+    int big_to_flush = 0;
     int write_iodepth = 0;
 
     bool live = false, queue_stall = false;
diff --git a/src/blockstore_rollback.cpp b/src/blockstore_rollback.cpp
index d0db8a99..a6c7a77f 100644
--- a/src/blockstore_rollback.cpp
+++ b/src/blockstore_rollback.cpp
@@ -201,6 +201,11 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
     }
     while (1)
     {
+        if ((IS_BIG_WRITE(dirty_it->second.state) || IS_DELETE(dirty_it->second.state)) &&
+            IS_STABLE(dirty_it->second.state))
+        {
+            big_to_flush--;
+        }
         if (IS_BIG_WRITE(dirty_it->second.state) && dirty_it->second.location != clean_loc &&
             dirty_it->second.location != UINT64_MAX)
         {
diff --git a/src/blockstore_stable.cpp b/src/blockstore_stable.cpp
index 4b6f52c1..fb3b7098 100644
--- a/src/blockstore_stable.cpp
+++ b/src/blockstore_stable.cpp
@@ -446,6 +446,7 @@ void blockstore_impl_t::mark_stable(const obj_ver_id & v, bool forget_dirty)
                     {
                         inode_space_stats[dirty_it->first.oid.inode] += dsk.data_block_size;
                     }
+                    big_to_flush++;
                 }
                 else if (IS_DELETE(dirty_it->second.state))
                 {
@@ -454,6 +455,7 @@ void blockstore_impl_t::mark_stable(const obj_ver_id & v, bool forget_dirty)
                         sp -= dsk.data_block_size;
                     else
                         inode_space_stats.erase(dirty_it->first.oid.inode);
+                    big_to_flush++;
                 }
             }
             if (forget_dirty && (IS_BIG_WRITE(dirty_it->second.state) ||
diff --git a/src/blockstore_write.cpp b/src/blockstore_write.cpp
index 3457eb03..e023e0f5 100644
--- a/src/blockstore_write.cpp
+++ b/src/blockstore_write.cpp
@@ -271,6 +271,13 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         if (loc == UINT64_MAX)
         {
             // no space
+            if (big_to_flush > 0)
+            {
+                // hope that some space will be available after flush
+                flusher->request_trim();
+                PRIV(op)->wait_for = WAIT_FREE;
+                return 0;
+            }
             cancel_all_writes(op, dirty_it, -ENOSPC);
             return 2;
         }