forked from vitalif/vitastor
Compare commits
156 Commits
Author | SHA1 | Date |
---|---|---|
Vitaliy Filippov | a8d744ca0e | |
Vitaliy Filippov | b5ff44fb6f | |
Vitaliy Filippov | f918bc4543 | |
Vitaliy Filippov | 6875a838e0 | |
Vitaliy Filippov | 20781abd3d | |
Vitaliy Filippov | 1f02f645c0 | |
Vitaliy Filippov | ee44f64927 | |
Vitaliy Filippov | abf0611d93 | |
Vitaliy Filippov | edbf0eb040 | |
Vitaliy Filippov | 09725038e7 | |
Vitaliy Filippov | 18f71b059a | |
Vitaliy Filippov | 2db2ed22ea | |
Vitaliy Filippov | aa7699da24 | |
Vitaliy Filippov | 853ecba780 | |
Vitaliy Filippov | 2f9c76b8fc | |
Vitaliy Filippov | 8da7f26459 | |
Vitaliy Filippov | 9998b50c7e | |
Vitaliy Filippov | 0422d94a70 | |
Vitaliy Filippov | ff2208ae70 | |
Vitaliy Filippov | ae54dddb0c | |
Vitaliy Filippov | bfc175fe0f | |
Vitaliy Filippov | 07e10210b6 | |
Vitaliy Filippov | 221b728fc9 | |
Vitaliy Filippov | 6625aaae00 | |
Vitaliy Filippov | 7e6e1a5a82 | |
Vitaliy Filippov | 435045751d | |
Vitaliy Filippov | c5fb1d5987 | |
Vitaliy Filippov | 9f59381bea | |
Vitaliy Filippov | 9ac7e75178 | |
Vitaliy Filippov | 88671cf745 | |
Vitaliy Filippov | fe1749c427 | |
Vitaliy Filippov | ceb9c28de7 | |
Vitaliy Filippov | 299d7d7c95 | |
Vitaliy Filippov | d1526b415f | |
Vitaliy Filippov | f49fd53d55 | |
Vitaliy Filippov | dd76eda5e5 | |
Vitaliy Filippov | 87dbd8fa57 | |
Vitaliy Filippov | b44f49aab2 | |
Vitaliy Filippov | 036555638e | |
Vitaliy Filippov | af5155fcd9 | |
Vitaliy Filippov | 0d2efbecc9 | |
Vitaliy Filippov | e62e8b6bae | |
Vitaliy Filippov | c4ba24c305 | |
Vitaliy Filippov | 19e47a0279 | |
Vitaliy Filippov | bd178ac20f | |
Vitaliy Filippov | 7006875a24 | |
Vitaliy Filippov | ad577c4aac | |
Vitaliy Filippov | 836635c518 | |
Vitaliy Filippov | 88a03f4e98 | |
Vitaliy Filippov | 2a5036669d | |
Vitaliy Filippov | 2e0c853180 | |
Vitaliy Filippov | e91ff2a9ec | |
Vitaliy Filippov | 086667f568 | |
Vitaliy Filippov | 73ce20e246 | |
Vitaliy Filippov | 1be94da437 | |
Vitaliy Filippov | 80e12358a2 | |
Vitaliy Filippov | 36c935ace6 | |
Vitaliy Filippov | 0d8b5e2ef9 | |
Vitaliy Filippov | 98f1e2c277 | |
Vitaliy Filippov | 21e7686037 | |
Vitaliy Filippov | ab21a1908b | |
Vitaliy Filippov | 30d1ccd43e | |
Vitaliy Filippov | 8bdd6d8d78 | |
Vitaliy Filippov | 09b3e4e789 | |
Vitaliy Filippov | 07912fd670 | |
Vitaliy Filippov | bc742ccf8c | |
Vitaliy Filippov | 314b20437b | |
Vitaliy Filippov | 29bac892ad | |
Vitaliy Filippov | cf7547faf3 | |
Vitaliy Filippov | ab90ed747f | |
Vitaliy Filippov | 29d8ac8b1b | |
Vitaliy Filippov | 97795ea1b1 | |
Vitaliy Filippov | 24e7075f08 | |
Vitaliy Filippov | 6155b23a7e | |
Vitaliy Filippov | 7d49706c07 | |
Vitaliy Filippov | 46e79f3306 | |
Vitaliy Filippov | 41fd14e024 | |
Vitaliy Filippov | bb2d9a3afe | |
Vitaliy Filippov | e899ed2c25 | |
Vitaliy Filippov | e21b14b72c | |
Vitaliy Filippov | 5af8eddaa9 | |
Vitaliy Filippov | 4f5a94c07a | |
Vitaliy Filippov | e16b87ecc8 | |
Vitaliy Filippov | fcb4aa0a11 | |
Vitaliy Filippov | 12adfa470c | |
Vitaliy Filippov | 7f15e0c084 | |
Vitaliy Filippov | 08d4bef419 | |
Vitaliy Filippov | 2d73b19a6c | |
Vitaliy Filippov | 69c87009e9 | |
Vitaliy Filippov | c974cb539c | |
Vitaliy Filippov | 00e98f64f3 | |
Vitaliy Filippov | 91a70dfb1b | |
Vitaliy Filippov | 178388ac8c | |
Vitaliy Filippov | bf9a175efc | |
Vitaliy Filippov | 08aed962de | |
Vitaliy Filippov | 8c65e890b9 | |
Vitaliy Filippov | 8cda70b889 | |
Vitaliy Filippov | 61ab22403a | |
Vitaliy Filippov | 16da663a66 | |
Vitaliy Filippov | 4a2dcf7b6b | |
Vitaliy Filippov | 8d48cc56b0 | |
Vitaliy Filippov | 9f58f01425 | |
Vitaliy Filippov | b9e7d31aa1 | |
Vitaliy Filippov | 2d9f09dcb6 | |
Vitaliy Filippov | 7cc59260c5 | |
Vitaliy Filippov | ca0a11ec85 | |
Vitaliy Filippov | 51c0b5afee | |
Vitaliy Filippov | e1e01d042e | |
Vitaliy Filippov | 534a4a657e | |
Vitaliy Filippov | 9b5d8b9ad4 | |
Vitaliy Filippov | e66ed47515 | |
Vitaliy Filippov | 036c6d4c42 | |
Vitaliy Filippov | 4cb79a3bf8 | |
Vitaliy Filippov | 3bf53754c2 | |
Vitaliy Filippov | 6023cac361 | |
Vitaliy Filippov | 915d04c446 | |
Vitaliy Filippov | 21e06ea40d | |
Vitaliy Filippov | 9ef7f865b0 | |
Vitaliy Filippov | 9dd20a31aa | |
Vitaliy Filippov | 28be049909 | |
Vitaliy Filippov | 78fbaacf1f | |
Vitaliy Filippov | 1526c5a213 | |
Vitaliy Filippov | c7cc414c90 | |
Vitaliy Filippov | f4ea313707 | |
Vitaliy Filippov | b88b76f316 | |
Vitaliy Filippov | 4a17a61d1f | |
Vitaliy Filippov | ccabbbfbcb | |
Vitaliy Filippov | 26dac57083 | |
Vitaliy Filippov | 44a53d8352 | |
Vitaliy Filippov | 9d80bd2d98 | |
Vitaliy Filippov | 322a38a144 | |
Vitaliy Filippov | 1018764c91 | |
Vitaliy Filippov | a45e0e5e67 | |
Vitaliy Filippov | 44656fbf67 | |
Vitaliy Filippov | 089f138e0c | |
Vitaliy Filippov | bcc8e697f9 | |
Vitaliy Filippov | a4c46ba745 | |
Vitaliy Filippov | 5596ad8997 | |
Vitaliy Filippov | 59c29b0cee | |
Vitaliy Filippov | 959089b919 | |
Vitaliy Filippov | d3e7749616 | |
Vitaliy Filippov | b56f8820ec | |
Vitaliy Filippov | 4bd2bd48eb | |
Vitaliy Filippov | a3fc9f8d7d | |
Vitaliy Filippov | 530975aed7 | |
Vitaliy Filippov | 1446aad107 | |
Vitaliy Filippov | 46479e2456 | |
Vitaliy Filippov | e41bee72a5 | |
Vitaliy Filippov | 2e0f223ddb | |
Vitaliy Filippov | 3be7bc29d8 | |
Vitaliy Filippov | 0c43ff9daf | |
Vitaliy Filippov | 64d471cf53 | |
Vitaliy Filippov | 809b2ad8cd | |
Vitaliy Filippov | 550d4af151 | |
Vitaliy Filippov | cf0f23ab8e | |
Vitaliy Filippov | a516fefa8c |
|
@ -0,0 +1,19 @@
|
|||
.git
|
||||
build
|
||||
packages
|
||||
mon/node_modules
|
||||
*.o
|
||||
*.so
|
||||
osd
|
||||
stub_osd
|
||||
stub_uring_osd
|
||||
stub_bench
|
||||
osd_test
|
||||
dump_journal
|
||||
nbd_proxy
|
||||
rm_inode
|
||||
fio
|
||||
qemu
|
||||
rpm/*.Dockerfile
|
||||
debian/*.Dockerfile
|
||||
Dockerfile
|
|
@ -0,0 +1,18 @@
|
|||
*.o
|
||||
*.so
|
||||
package-lock.json
|
||||
fio
|
||||
qemu
|
||||
osd
|
||||
stub_osd
|
||||
stub_uring_osd
|
||||
stub_bench
|
||||
osd_test
|
||||
osd_peering_pg_test
|
||||
dump_journal
|
||||
nbd_proxy
|
||||
rm_inode
|
||||
test_allocator
|
||||
test_blockstore
|
||||
test_shit
|
||||
osd_rmw_test
|
|
@ -0,0 +1,5 @@
|
|||
cmake_minimum_required(VERSION 2.8)
|
||||
|
||||
project(vitastor)
|
||||
|
||||
add_subdirectory(src)
|
|
@ -0,0 +1,27 @@
|
|||
Copyright (c) Vitaliy Filippov (vitalif [at] yourcmc.ru), 2019+
|
||||
|
||||
All server-side code (OSD, Monitor and so on) is licensed under the terms of
|
||||
Vitastor Network Public License 1.1 (VNPL 1.1), a copyleft license based on
|
||||
GNU GPLv3.0 with the additional "Network Interaction" clause which requires
|
||||
opensourcing all programs directly or indirectly interacting with Vitastor
|
||||
through a computer network and expressly designed to be used in conjunction
|
||||
with it ("Proxy Programs"). Proxy Programs may be made public not only under
|
||||
the terms of the same license, but also under the terms of any GPL-Compatible
|
||||
Free Software License, as listed by the Free Software Foundation.
|
||||
This is a stricter copyleft license than the Affero GPL.
|
||||
|
||||
Please note that VNPL doesn't require you to open the code of proprietary
|
||||
software running inside a VM if it's not specially designed to be used with
|
||||
Vitastor.
|
||||
|
||||
Basically, you can't use the software in a proprietary environment to provide
|
||||
its functionality to users without opensourcing all intermediary components
|
||||
standing between the user and Vitastor or purchasing a commercial license
|
||||
from the author 😀.
|
||||
|
||||
Client libraries (cluster_client and so on) are dual-licensed under the same
|
||||
VNPL 1.1 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
|
||||
software like QEMU and fio.
|
||||
|
||||
You can find the full text of VNPL-1.1 in the file [VNPL-1.1.txt](VNPL-1.1.txt).
|
||||
GPL 2.0 is also included in this repository as [GPL-2.0.txt](GPL-2.0.txt).
|
46
Make-gen.pl
46
Make-gen.pl
|
@ -1,46 +0,0 @@
|
|||
#!/usr/bin/perl
|
||||
|
||||
use strict;
|
||||
|
||||
my $deps = {};
|
||||
for my $line (split /\n/, `grep '^#include "' *.cpp *.h`)
|
||||
{
|
||||
if ($line =~ /^([^:]+):\#include "([^"]+)"/s)
|
||||
{
|
||||
$deps->{$1}->{$2} = 1;
|
||||
}
|
||||
}
|
||||
|
||||
my $added;
|
||||
do
|
||||
{
|
||||
$added = 0;
|
||||
for my $file (keys %$deps)
|
||||
{
|
||||
for my $dep (keys %{$deps->{$file}})
|
||||
{
|
||||
if ($deps->{$dep})
|
||||
{
|
||||
for my $subdep (keys %{$deps->{$dep}})
|
||||
{
|
||||
if (!$deps->{$file}->{$subdep})
|
||||
{
|
||||
$added = 1;
|
||||
$deps->{$file}->{$subdep} = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} while ($added);
|
||||
|
||||
for my $file (sort keys %$deps)
|
||||
{
|
||||
if ($file =~ /\.cpp$/)
|
||||
{
|
||||
my $obj = $file;
|
||||
$obj =~ s/\.cpp$/.o/s;
|
||||
print "$obj: $file ".join(" ", sort keys %{$deps->{$file}})."\n";
|
||||
print "\tg++ \$(CXXFLAGS) -c -o \$\@ \$\<\n";
|
||||
}
|
||||
}
|
177
Makefile
177
Makefile
|
@ -1,177 +0,0 @@
|
|||
BLOCKSTORE_OBJS := allocator.o blockstore.o blockstore_impl.o blockstore_init.o blockstore_open.o blockstore_journal.o blockstore_read.o \
|
||||
blockstore_write.o blockstore_sync.o blockstore_stable.o blockstore_rollback.o blockstore_flush.o crc32c.o ringloop.o
|
||||
# -fsanitize=address
|
||||
CXXFLAGS := -g -O3 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fPIC -fdiagnostics-color=always
|
||||
all: libfio_blockstore.so osd libfio_sec_osd.so libfio_cluster.so stub_osd stub_uring_osd stub_bench osd_test dump_journal qemu_driver.so nbd_proxy rm_inode
|
||||
clean:
|
||||
rm -f *.o
|
||||
|
||||
dump_journal: dump_journal.cpp crc32c.o blockstore_journal.h
|
||||
g++ $(CXXFLAGS) -o $@ $< crc32c.o
|
||||
|
||||
libblockstore.so: $(BLOCKSTORE_OBJS)
|
||||
g++ $(CXXFLAGS) -o $@ -shared $(BLOCKSTORE_OBJS) -ltcmalloc_minimal -luring
|
||||
libfio_blockstore.so: ./libblockstore.so fio_engine.o json11.o
|
||||
g++ $(CXXFLAGS) -shared -o $@ fio_engine.o json11.o ./libblockstore.so -ltcmalloc_minimal -luring
|
||||
|
||||
OSD_OBJS := osd.o osd_secondary.o msgr_receive.o msgr_send.o osd_peering.o osd_flush.o osd_peering_pg.o \
|
||||
osd_primary.o osd_primary_subops.o etcd_state_client.o messenger.o osd_cluster.o http_client.o osd_ops.o pg_states.o \
|
||||
osd_rmw.o json11.o base64.o timerfd_manager.o epoll_manager.o
|
||||
osd: ./libblockstore.so osd_main.cpp osd.h osd_ops.h $(OSD_OBJS)
|
||||
g++ $(CXXFLAGS) -o $@ osd_main.cpp $(OSD_OBJS) ./libblockstore.so -ltcmalloc_minimal -luring
|
||||
|
||||
stub_osd: stub_osd.o rw_blocking.o
|
||||
g++ $(CXXFLAGS) -o $@ stub_osd.o rw_blocking.o -ltcmalloc_minimal
|
||||
|
||||
osd_rmw_test: osd_rmw_test.o
|
||||
g++ $(CXXFLAGS) -o $@ osd_rmw_test.o
|
||||
|
||||
STUB_URING_OSD_OBJS := stub_uring_osd.o epoll_manager.o messenger.o msgr_send.o msgr_receive.o ringloop.o timerfd_manager.o json11.o
|
||||
stub_uring_osd: $(STUB_URING_OSD_OBJS)
|
||||
g++ $(CXXFLAGS) -o $@ -ltcmalloc_minimal $(STUB_URING_OSD_OBJS) -luring
|
||||
stub_bench: stub_bench.cpp osd_ops.h rw_blocking.o
|
||||
g++ $(CXXFLAGS) -o $@ stub_bench.cpp rw_blocking.o -ltcmalloc_minimal
|
||||
osd_test: osd_test.cpp osd_ops.h rw_blocking.o
|
||||
g++ $(CXXFLAGS) -o $@ osd_test.cpp rw_blocking.o -ltcmalloc_minimal
|
||||
osd_peering_pg_test: osd_peering_pg_test.cpp osd_peering_pg.o
|
||||
g++ $(CXXFLAGS) -o $@ $< osd_peering_pg.o -ltcmalloc_minimal
|
||||
|
||||
libfio_sec_osd.so: fio_sec_osd.o rw_blocking.o
|
||||
g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o $@ fio_sec_osd.o rw_blocking.o
|
||||
|
||||
FIO_CLUSTER_OBJS := cluster_client.o epoll_manager.o etcd_state_client.o \
|
||||
messenger.o msgr_send.o msgr_receive.o ringloop.o json11.o http_client.o osd_ops.o pg_states.o timerfd_manager.o base64.o
|
||||
libfio_cluster.so: fio_cluster.o $(FIO_CLUSTER_OBJS)
|
||||
g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o $@ $< $(FIO_CLUSTER_OBJS) -luring
|
||||
|
||||
nbd_proxy: nbd_proxy.o $(FIO_CLUSTER_OBJS)
|
||||
g++ $(CXXFLAGS) -ltcmalloc_minimal -o $@ $< $(FIO_CLUSTER_OBJS) -luring
|
||||
|
||||
rm_inode: rm_inode.o $(FIO_CLUSTER_OBJS)
|
||||
g++ $(CXXFLAGS) -ltcmalloc_minimal -o $@ $< $(FIO_CLUSTER_OBJS) -luring
|
||||
|
||||
qemu_driver.o: qemu_driver.c qemu_proxy.h
|
||||
gcc -I qemu/b/qemu `pkg-config glib-2.0 --cflags` \
|
||||
-I qemu/include $(CXXFLAGS) -c -o $@ $<
|
||||
|
||||
qemu_driver.so: qemu_driver.o qemu_proxy.o $(FIO_CLUSTER_OBJS)
|
||||
g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o $@ $< $(FIO_CLUSTER_OBJS) qemu_driver.o qemu_proxy.o -luring
|
||||
|
||||
test_blockstore: ./libblockstore.so test_blockstore.cpp timerfd_interval.o
|
||||
g++ $(CXXFLAGS) -o test_blockstore test_blockstore.cpp timerfd_interval.o ./libblockstore.so -ltcmalloc_minimal -luring
|
||||
test: test.cpp osd_peering_pg.o
|
||||
g++ $(CXXFLAGS) -o test test.cpp osd_peering_pg.o -luring -lm
|
||||
test_allocator: test_allocator.cpp allocator.o
|
||||
g++ $(CXXFLAGS) -o test_allocator test_allocator.cpp allocator.o
|
||||
|
||||
crc32c.o: crc32c.c crc32c.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
json11.o: json11/json11.cpp
|
||||
g++ $(CXXFLAGS) -c -o json11.o json11/json11.cpp
|
||||
|
||||
# Autogenerated
|
||||
|
||||
allocator.o: allocator.cpp allocator.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
base64.o: base64.cpp base64.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
blockstore.o: blockstore.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
blockstore_flush.o: blockstore_flush.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
blockstore_impl.o: blockstore_impl.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
blockstore_init.o: blockstore_init.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
blockstore_journal.o: blockstore_journal.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
blockstore_open.o: blockstore_open.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
blockstore_read.o: blockstore_read.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
blockstore_rollback.o: blockstore_rollback.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
blockstore_stable.o: blockstore_stable.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
blockstore_sync.o: blockstore_sync.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
blockstore_write.o: blockstore_write.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
cluster_client.o: cluster_client.cpp cluster_client.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
dump_journal.o: dump_journal.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
epoll_manager.o: epoll_manager.cpp epoll_manager.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
etcd_state_client.o: etcd_state_client.cpp base64.h etcd_state_client.h http_client.h json11/json11.hpp object_id.h osd_id.h osd_ops.h pg_states.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
fio_cluster.o: fio_cluster.cpp cluster_client.h epoll_manager.h etcd_state_client.h fio/arch/arch.h fio/fio.h fio/optgroup.h fio_headers.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
fio_engine.o: fio_engine.cpp blockstore.h fio/arch/arch.h fio/fio.h fio/optgroup.h fio_headers.h json11/json11.hpp object_id.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
fio_sec_osd.o: fio_sec_osd.cpp fio/arch/arch.h fio/fio.h fio/optgroup.h fio_headers.h object_id.h osd_id.h osd_ops.h rw_blocking.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
http_client.o: http_client.cpp http_client.h json11/json11.hpp timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
messenger.o: messenger.cpp json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
msgr_receive.o: msgr_receive.cpp json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
msgr_send.o: msgr_send.cpp json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
nbd_proxy.o: nbd_proxy.cpp cluster_client.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd.o: osd.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_cluster.o: osd_cluster.cpp base64.h blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_flush.o: osd_flush.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_main.o: osd_main.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_ops.o: osd_ops.cpp object_id.h osd_id.h osd_ops.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_peering.o: osd_peering.cpp base64.h blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_peering_pg.o: osd_peering_pg.cpp cpp-btree/btree_map.h object_id.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_peering_pg_test.o: osd_peering_pg_test.cpp cpp-btree/btree_map.h object_id.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_primary.o: osd_primary.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h osd_primary.h osd_rmw.h pg_states.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_primary_subops.o: osd_primary_subops.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h osd_primary.h osd_rmw.h pg_states.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_rmw.o: osd_rmw.cpp malloc_or_die.h object_id.h osd_id.h osd_rmw.h xor.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_rmw_test.o: osd_rmw_test.cpp malloc_or_die.h object_id.h osd_id.h osd_rmw.cpp osd_rmw.h test_pattern.h xor.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_secondary.o: osd_secondary.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
osd_test.o: osd_test.cpp object_id.h osd_id.h osd_ops.h rw_blocking.h test_pattern.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
pg_states.o: pg_states.cpp pg_states.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
qemu_proxy.o: qemu_proxy.cpp cluster_client.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h qemu_proxy.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
ringloop.o: ringloop.cpp ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
rm_inode.o: rm_inode.cpp cluster_client.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
rw_blocking.o: rw_blocking.cpp rw_blocking.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
stub_bench.o: stub_bench.cpp object_id.h osd_id.h osd_ops.h rw_blocking.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
stub_osd.o: stub_osd.cpp object_id.h osd_id.h osd_ops.h rw_blocking.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
stub_uring_osd.o: stub_uring_osd.cpp epoll_manager.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
test.o: test.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
test_allocator.o: test_allocator.cpp allocator.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
test_blockstore.o: test_blockstore.cpp blockstore.h object_id.h ringloop.h timerfd_interval.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
timerfd_interval.o: timerfd_interval.cpp ringloop.h timerfd_interval.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
||||
timerfd_manager.o: timerfd_manager.cpp timerfd_manager.h
|
||||
g++ $(CXXFLAGS) -c -o $@ $<
|
|
@ -0,0 +1,491 @@
|
|||
## Vitastor
|
||||
|
||||
[Read English version](README.md)
|
||||
|
||||
## Идея
|
||||
|
||||
Я всего лишь хочу сделать качественную блочную SDS!
|
||||
|
||||
Vitastor - распределённая блочная SDS, прямой аналог Ceph RBD и внутренних СХД популярных
|
||||
облачных провайдеров. Однако, в отличие от них, Vitastor быстрый и при этом простой.
|
||||
Только пока маленький :-).
|
||||
|
||||
Архитектурная схожесть с Ceph означает заложенную на уровне алгоритмов записи строгую консистентность,
|
||||
репликацию через первичный OSD, симметричную кластеризацию без единой точки отказа
|
||||
и автоматическое распределение данных по любому числу дисков любого размера с настраиваемыми схемами
|
||||
избыточности - репликацией или с произвольными кодами коррекции ошибок.
|
||||
|
||||
## Возможности
|
||||
|
||||
Vitastor на данный момент находится в статусе предварительного выпуска, расширенные
|
||||
возможности пока отсутствуют, а в будущих версиях вероятны "ломающие" изменения.
|
||||
|
||||
Однако следующее уже реализовано:
|
||||
|
||||
- Базовая часть - надёжное кластерное блочное хранилище без единой точки отказа
|
||||
- Производительность ;-D
|
||||
- Несколько схем отказоустойчивости: репликация, XOR n+1 (1 диск чётности), коды коррекции ошибок
|
||||
Рида-Соломона на основе библиотеки jerasure с любым числом дисков данных и чётности в группе
|
||||
- Конфигурация через простые человекочитаемые JSON-структуры в etcd
|
||||
- Автоматическое распределение данных по OSD, с поддержкой:
|
||||
- Математической оптимизации для лучшей равномерности распределения и минимизации перемещений данных
|
||||
- Нескольких пулов с разными схемами избыточности
|
||||
- Дерева распределения, выбора OSD по тегам / классам устройств (только SSD, только HDD) и по поддереву
|
||||
- Настраиваемых доменов отказа (диск/сервер/стойка и т.п.)
|
||||
- Восстановление деградированных блоков
|
||||
- Ребаланс, то есть перемещение данных между OSD (дисками)
|
||||
- Поддержка "ленивого" fsync (fsync не на каждую операцию)
|
||||
- Сбор статистики ввода/вывода в etcd
|
||||
- Клиентская библиотека режима пользователя для ввода/вывода
|
||||
- Драйвер диска для QEMU (собирается вне дерева исходников QEMU)
|
||||
- Драйвер диска для утилиты тестирования производительности fio (также собирается вне дерева исходников fio)
|
||||
- NBD-прокси для монтирования образов ядром ("блочное устройство в режиме пользователя")
|
||||
- Утилита удаления образов/инодов (vitastor-rm)
|
||||
- Пакеты для Debian и CentOS
|
||||
- Статистика операций ввода/вывода и занятого места в разрезе инодов
|
||||
- Именование инодов через хранение их метаданных в etcd
|
||||
- Снапшоты и copy-on-write клоны
|
||||
|
||||
## Планы разработки
|
||||
|
||||
- Более корректные скрипты разметки дисков и автоматического запуска OSD
|
||||
- Другие инструменты администрирования
|
||||
- Плагины для OpenStack, Kubernetes, OpenNebula, Proxmox и других облачных систем
|
||||
- iSCSI-прокси
|
||||
- Таймауты операций и более быстрое выявление отказов
|
||||
- Фоновая проверка целостности без контрольных сумм (сверка реплик)
|
||||
- Контрольные суммы
|
||||
- Оптимизации для гибридных SSD+HDD хранилищ
|
||||
- Поддержка RDMA и NVDIMM
|
||||
- Web-интерфейс
|
||||
- Возможно, сжатие
|
||||
- Возможно, поддержка кэширования данных через системный page cache
|
||||
|
||||
## Архитектура
|
||||
|
||||
Так же, как и в Ceph, в Vitastor:
|
||||
|
||||
- Есть пулы (pools), PG, OSD, мониторы, домены отказа, дерево распределения (аналог crush-дерева).
|
||||
- Образы делятся на блоки фиксированного размера (объекты), и эти объекты распределяются по OSD.
|
||||
- У OSD есть журнал и метаданные и они тоже могут размещаться на отдельных быстрых дисках.
|
||||
- Все операции записи тоже транзакционны. В Vitastor, правда, есть режим отложенного/ленивого fsync
|
||||
(коммита), в котором fsync не вызывается на каждую операцию записи, что делает его более
|
||||
пригодным для использования на "плохих" (десктопных) SSD. Однако все операции записи
|
||||
в любом случае атомарны.
|
||||
- Клиентская библиотека тоже старается ждать восстановления после любого отказа кластера, то есть,
|
||||
вы тоже можете перезагрузить хоть весь кластер разом, и клиенты только на время зависнут,
|
||||
но не отключатся.
|
||||
|
||||
Некоторые базовые термины для тех, кто не знаком с Ceph:
|
||||
|
||||
- OSD (Object Storage Daemon) - процесс, который хранит данные на одном диске и обрабатывает
|
||||
запросы чтения/записи от клиентов.
|
||||
- Пул (Pool) - контейнер для данных, имеющих одну и ту же схему избыточности и правила распределения по OSD.
|
||||
- PG (Placement Group) - группа объектов, хранимых на одном и том же наборе реплик (OSD).
|
||||
Несколько PG могут храниться на одном и том же наборе реплик, но объекты одной PG
|
||||
в норме не хранятся на разных наборах OSD.
|
||||
- Монитор - демон, хранящий состояние кластера.
|
||||
- Домен отказа (Failure Domain) - группа OSD, которым вы разрешаете "упасть" всем вместе.
|
||||
Иными словами, это группа OSD, в которые СХД не помещает разные копии одного и того же
|
||||
блока данных. Например, если домен отказа - сервер, то на двух дисках одного сервера
|
||||
никогда не окажется 2 и более копий одного и того же блока данных, а значит, даже
|
||||
если в этом сервере откажут все диски, это будет равносильно потере только 1 копии
|
||||
любого блока данных.
|
||||
- Дерево распределения (Placement Tree / CRUSH Tree) - иерархическая группировка OSD
|
||||
в узлы, которые далее можно использовать как домены отказа. То есть, диск (OSD) входит в
|
||||
сервер, сервер входит в стойку, стойка входит в ряд, ряд в датацентр и т.п.
|
||||
|
||||
Чем Vitastor отличается от Ceph:
|
||||
|
||||
- Vitastor в первую очередь сфокусирован на SSD. Также Vitastor, вероятно, должен неплохо работать
|
||||
с комбинацией SSD и HDD через bcache, а в будущем, возможно, будут добавлены и нативные способы
|
||||
оптимизации под SSD+HDD. Однако хранилище на основе одних лишь жёстких дисков, вообще без SSD,
|
||||
не в приоритете, поэтому оптимизации под этот кейс могут вообще не состояться.
|
||||
- OSD Vitastor однопоточный и всегда таким останется, так как это самый оптимальный способ работы.
|
||||
Если вам не хватает 1 ядра на 1 диск, просто делите диск на разделы и запускайте на нём несколько OSD.
|
||||
Но, скорее всего, вам хватит и 1 ядра - Vitastor не так прожорлив к ресурсам CPU, как Ceph.
|
||||
- Журнал и метаданные всегда размещаются в памяти, благодаря чему никогда не тратится лишнее время
|
||||
на чтение метаданных с диска. Размер метаданных линейно зависит от размера диска и блока данных,
|
||||
который задаётся в конфигурации кластера и по умолчанию составляет 128 КБ. С блоком 128 КБ метаданные
|
||||
занимают примерно 512 МБ памяти на 1 ТБ дискового пространства (и это всё равно меньше, чем нужно Ceph-у).
|
||||
Журнал вообще не должен быть большим, например, тесты производительности в данном документе проводились
|
||||
с журналом размером всего 16 МБ. Большой журнал, вероятно, даже вреден, т.к. "грязные" записи (записи,
|
||||
не сброшенные из журнала) тоже занимают память и могут немного замедлять работу.
|
||||
- В Vitastor нет внутреннего copy-on-write. Я считаю, что реализация CoW-хранилища гораздо сложнее,
|
||||
поэтому сложнее добиться устойчиво хороших результатов. Возможно, в один прекрасный день
|
||||
я придумаю красивый алгоритм для CoW-хранилища, но пока нет - внутреннего CoW в Vitastor не будет.
|
||||
Всё это не относится к "внешнему" CoW (снапшотам и клонам).
|
||||
- Базовый слой Vitastor - простое блочное хранилище с блоками фиксированного размера, а не сложное
|
||||
объектное хранилище с расширенными возможностями, как в Ceph (RADOS).
|
||||
- В Vitastor есть режим "ленивых fsync", в котором OSD группирует запросы записи перед сбросом их
|
||||
на диск, что позволяет получить лучшую производительность с дешёвыми настольными SSD без конденсаторов
|
||||
("Advanced Power Loss Protection" / "Capacitor-Based Power Loss Protection").
|
||||
Тем не менее, такой режим всё равно медленнее использования нормальных серверных SSD и мгновенного
|
||||
fsync, так как приводит к дополнительным операциям передачи данных по сети, поэтому рекомендуется
|
||||
всё-таки использовать хорошие серверные диски, тем более, стоят они почти так же, как десктопные.
|
||||
- PG эфемерны. Это означает, что они не хранятся на дисках и существуют только в памяти работающих OSD.
|
||||
- Процессы восстановления оперируют отдельными объектами, а не целыми PG.
|
||||
- PGLOG-ов нет.
|
||||
- "Мониторы" не хранят данные. Конфигурация и состояние кластера хранятся в etcd в простых человекочитаемых
|
||||
JSON-структурах. Мониторы Vitastor только следят за состоянием кластера и управляют перемещением данных.
|
||||
В этом смысле монитор Vitastor не является критичным компонентом системы и больше похож на Ceph-овский
|
||||
менеджер (MGR). Монитор Vitastor написан на node.js.
|
||||
- Распределение PG не основано на консистентных хешах. Вместо этого все маппинги PG хранятся прямо в etcd
|
||||
(ибо нет никакой проблемы сохранить несколько сотен-тысяч записей в памяти, а не считать каждый раз хеши).
|
||||
Перераспределение PG по OSD выполняется через математическую оптимизацию,
|
||||
а конкретно, сведение задачи к ЛП (задаче линейного программирования) и решение оной с помощью утилиты
|
||||
lp_solve. Такой подход позволяет обычно выравнивать распределение места почти идеально - равномерность
|
||||
обычно составляет 96-99%, в отличие от Ceph, где на голом CRUSH-е без балансировщика обычно выходит 80-90%.
|
||||
Также это позволяет минимизировать объём перемещения данных и случайность связей между OSD, а также менять
|
||||
распределение вручную, не боясь сломать логику перебалансировки. В таком подходе есть и потенциальный
|
||||
недостаток - есть предположение, что в очень большом кластере он может сломаться - однако вплоть до
|
||||
нескольких сотен OSD подход точно работает нормально. Ну и, собственно, при необходимости легко
|
||||
реализовать и консистентные хеши.
|
||||
- Отдельный слой, подобный слою "CRUSH-правил", отсутствует. Вы настраиваете схемы отказоустойчивости,
|
||||
домены отказа и правила выбора OSD напрямую в конфигурации пулов.
|
||||
|
||||
## Понимание сути производительности систем хранения
|
||||
|
||||
Вкратце: для быстрой хранилки задержки важнее, чем пиковые iops-ы.
|
||||
|
||||
Лучшая возможная задержка достигается при тестировании в 1 поток с глубиной очереди 1,
|
||||
что приблизительно означает минимально нагруженное состояние кластера. В данном случае
|
||||
IOPS = 1/задержка. Ни числом серверов, ни дисков, ни серверных процессов/потоков
|
||||
задержка не масштабируется... Она зависит только от того, насколько быстро один
|
||||
серверный процесс (и клиент) обрабатывают одну операцию.
|
||||
|
||||
Почему задержки важны? Потому, что некоторые приложения *не могут* использовать глубину
|
||||
очереди больше 1, ибо их задача не параллелизуется. Важный пример - это все СУБД
|
||||
с поддержкой консистентности (ACID), потому что все они обеспечивают её через
|
||||
журналирование, а журналы пишутся последовательно и с fsync() после каждой операции.
|
||||
|
||||
fsync, кстати - это ещё одна очень важная вещь, про которую почти всегда забывают в тестах.
|
||||
Смысл в том, что все современные диски имеют кэши/буферы записи и не гарантируют, что
|
||||
данные реально физически записываются на носитель до того, как вы делаете fsync(),
|
||||
который транслируется в команду сброса кэша операционной системой.
|
||||
|
||||
Дешёвые SSD для настольных ПК и ноутбуков очень быстрые без fsync - NVMe диски, например,
|
||||
могут обработать порядка 80000 операций записи в секунду с глубиной очереди 1 без fsync.
|
||||
Однако с fsync, когда они реально вынуждены писать каждый блок данных во флеш-память,
|
||||
они выжимают лишь 1000-2000 операций записи в секунду (число практически постоянное
|
||||
для всех моделей SSD).
|
||||
|
||||
Серверные SSD часто имеют суперконденсаторы, работающие как встроенный источник
|
||||
бесперебойного питания и дающие дискам успеть сбросить их DRAM-кэш в постоянную
|
||||
флеш-память при отключении питания. Благодаря этому диски с чистой совестью
|
||||
*игнорируют fsync*, так как точно знают, что данные из кэша доедут до постоянной
|
||||
памяти.
|
||||
|
||||
Все наиболее известные программные СХД, например, Ceph и внутренние СХД, используемые
|
||||
такими облачными провайдерами, как Amazon, Google, Яндекс, медленные в смысле задержки.
|
||||
В лучшем случае они дают задержки от 0.3мс на чтение и 0.6мс на запись 4 КБ блоками
|
||||
даже при условии использования наилучшего возможного железа.
|
||||
|
||||
И это в эпоху SSD, когда вы можете пойти на рынок и купить там SSD, задержка которого
|
||||
на чтение будет 0.1мс, а на запись - 0.04мс, за 100$ или даже дешевле.
|
||||
|
||||
Когда мне нужно быстро протестировать производительность дисковой подсистемы, я
|
||||
использую следующие 6 команд, с небольшими вариациями:
|
||||
|
||||
- Линейная запись:
|
||||
`fio -ioengine=libaio -direct=1 -invalidate=1 -name=test -bs=4M -iodepth=32 -rw=write -runtime=60 -filename=/dev/sdX`
|
||||
- Линейное чтение:
|
||||
`fio -ioengine=libaio -direct=1 -invalidate=1 -name=test -bs=4M -iodepth=32 -rw=read -runtime=60 -filename=/dev/sdX`
|
||||
- Запись в 1 поток (T1Q1):
|
||||
`fio -ioengine=libaio -direct=1 -invalidate=1 -name=test -bs=4k -iodepth=1 -fsync=1 -rw=randwrite -runtime=60 -filename=/dev/sdX`
|
||||
- Чтение в 1 поток (T1Q1):
|
||||
`fio -ioengine=libaio -direct=1 -invalidate=1 -name=test -bs=4k -iodepth=1 -rw=randread -runtime=60 -filename=/dev/sdX`
|
||||
- Параллельная запись (numjobs используется, когда 1 ядро CPU не может насытить диск):
|
||||
`fio -ioengine=libaio -direct=1 -invalidate=1 -name=test -bs=4k -iodepth=128 [-numjobs=4 -group_reporting] -rw=randwrite -runtime=60 -filename=/dev/sdX`
|
||||
- Параллельное чтение (numjobs - аналогично):
|
||||
`fio -ioengine=libaio -direct=1 -invalidate=1 -name=test -bs=4k -iodepth=128 [-numjobs=4 -group_reporting] -rw=randread -runtime=60 -filename=/dev/sdX`
|
||||
|
||||
## Теоретическая максимальная производительность Vitastor
|
||||
|
||||
При использовании репликации:
|
||||
- Задержка чтения в 1 поток (T1Q1): 1 сетевой RTT + 1 чтение с диска.
|
||||
- Запись+fsync в 1 поток:
|
||||
- С мгновенным сбросом: 2 RTT + 1 запись.
|
||||
- С отложенным ("ленивым") сбросом: 4 RTT + 1 запись + 1 fsync.
|
||||
- Параллельное чтение: сумма IOPS всех дисков либо производительность сети, если в сеть упрётся раньше.
|
||||
- Параллельная запись: сумма IOPS всех дисков / число реплик / WA либо производительность сети, если в сеть упрётся раньше.
|
||||
|
||||
При использовании кодов коррекции ошибок (EC):
|
||||
- Задержка чтения в 1 поток (T1Q1): 1.5 RTT + 1 чтение.
|
||||
- Запись+fsync в 1 поток:
|
||||
- С мгновенным сбросом: 3.5 RTT + 1 чтение + 2 записи.
|
||||
- С отложенным ("ленивым") сбросом: 5.5 RTT + 1 чтение + 2 записи + 2 fsync.
|
||||
- Под 0.5 на самом деле подразумевается (k-1)/k, где k - число дисков данных,
|
||||
что означает, что дополнительное обращение по сети не нужно, когда операция
|
||||
чтения обслуживается локально.
|
||||
- Параллельное чтение: сумма IOPS всех дисков либо производительность сети, если в сеть упрётся раньше.
|
||||
- Параллельная запись: сумма IOPS всех дисков / общее число дисков данных и чётности / WA либо производительность сети, если в сеть упрётся раньше.
|
||||
Примечание: IOPS дисков в данном случае надо брать в смешанном режиме чтения/записи в пропорции, аналогичной формулам выше.
|
||||
|
||||
WA (мультипликатор записи) для 4 КБ блоков в Vitastor обычно составляет 3-5:
|
||||
1. Запись метаданных в журнал
|
||||
2. Запись блока данных в журнал
|
||||
3. Запись метаданных в БД
|
||||
4. Ещё одна запись метаданных в журнал при использовании EC
|
||||
5. Запись блока данных на диск данных
|
||||
|
||||
Если вы найдёте SSD, хорошо работающий с 512-байтными блоками данных (Optane?),
|
||||
то 1, 3 и 4 можно снизить до 512 байт (1/8 от размера данных) и получить WA всего 2.375.
|
||||
|
||||
Кроме того, WA снижается при использовании отложенного/ленивого сброса при параллельной
|
||||
нагрузке, т.к. блоки журнала записываются на диск только когда они заполняются или явным
|
||||
образом запрашивается fsync.
|
||||
|
||||
## Пример сравнения с Ceph
|
||||
|
||||
Железо - 4 сервера, в каждом:
|
||||
- 6x SATA SSD Intel D3-4510 3.84 TB
|
||||
- 2x Xeon Gold 6242 (16 cores @ 2.8 GHz)
|
||||
- 384 GB RAM
|
||||
- 1x 25 GbE сетевая карта (Mellanox ConnectX-4 LX), подключённая к свитчу Juniper QFX5200
|
||||
|
||||
Экономия энергии CPU отключена. В тестах и Vitastor, и Ceph развёрнуто по 2 OSD на 1 SSD.
|
||||
|
||||
Все результаты ниже относятся к случайной нагрузке 4 КБ блоками (если явно не указано обратное).
|
||||
|
||||
Производительность голых дисков:
|
||||
- T1Q1 запись ~27000 iops (задержка ~0.037ms)
|
||||
- T1Q1 чтение ~9800 iops (задержка ~0.101ms)
|
||||
- T1Q32 запись ~60000 iops
|
||||
- T1Q32 чтение ~81700 iops
|
||||
|
||||
Ceph 15.2.4 (Bluestore):
|
||||
- T1Q1 запись ~1000 iops (задержка ~1ms)
|
||||
- T1Q1 чтение ~1750 iops (задержка ~0.57ms)
|
||||
- T8Q64 запись ~100000 iops, потребление CPU процессами OSD около 40 ядер на каждом сервере
|
||||
- T8Q64 чтение ~480000 iops, потребление CPU процессами OSD около 40 ядер на каждом сервере
|
||||
|
||||
Тесты в 8 потоков проводились на 8 400GB RBD образах со всех хостов (с каждого хоста запускалось 2 процесса fio).
|
||||
Это нужно потому, что в Ceph несколько RBD-клиентов, пишущих в 1 образ, очень сильно замедляются.
|
||||
|
||||
Настройки RocksDB и Bluestore в Ceph не менялись, единственным изменением было отключение cephx_sign_messages.
|
||||
|
||||
На самом деле, результаты теста не такие уж и плохие для Ceph (могло быть хуже).
|
||||
Собственно говоря, эти серверы как раз хорошо сбалансированы для Ceph - 6 SATA SSD как раз
|
||||
утилизируют 25-гигабитную сеть, а без 2 мощных процессоров Ceph-у бы не хватило ядер,
|
||||
чтобы выдать пристойный результат. Собственно, что и показывает жор 40 ядер в процессе
|
||||
параллельного теста.
|
||||
|
||||
Vitastor:
|
||||
- T1Q1 запись: 7087 iops (задержка 0.14ms)
|
||||
- T1Q1 чтение: 6838 iops (задержка 0.145ms)
|
||||
- T2Q64 запись: 162000 iops, потребление CPU - 3 ядра на каждом сервере
|
||||
- T8Q64 чтение: 895000 iops, потребление CPU - 4 ядра на каждом сервере
|
||||
- Линейная запись (4M T1Q32): 2800 МБ/с
|
||||
- Линейное чтение (4M T1Q32): 1500 МБ/с
|
||||
|
||||
Тест на чтение в 8 потоков проводился на 1 большом образе (3.2 ТБ) со всех хостов (опять же, по 2 fio с каждого).
|
||||
В Vitastor никакой разницы между 1 образом и 8-ю нет. Естественно, примерно 1/4 запросов чтения
|
||||
в такой конфигурации, как и в тестах Ceph выше, обслуживалась с локальной машины. Если проводить
|
||||
тест так, чтобы все операции всегда обращались к первичным OSD по сети - тест сильнее упирался
|
||||
в сеть и результат составлял примерно 689000 iops.
|
||||
|
||||
Настройки Vitastor: `--disable_data_fsync true --immediate_commit all --flusher_count 8
|
||||
--disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096
|
||||
--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024
|
||||
--journal_size 16777216`.
|
||||
|
||||
### EC/XOR 2+1
|
||||
|
||||
Vitastor:
|
||||
- T1Q1 запись: 2808 iops (задержка ~0.355ms)
|
||||
- T1Q1 чтение: 6190 iops (задержка ~0.16ms)
|
||||
- T2Q64 запись: 85500 iops, потребление CPU - 3.4 ядра на каждом сервере
|
||||
- T8Q64 чтение: 812000 iops, потребление CPU - 4.7 ядра на каждом сервере
|
||||
- Линейная запись (4M T1Q32): 3200 МБ/с
|
||||
- Линейное чтение (4M T1Q32): 1800 МБ/с
|
||||
|
||||
Ceph:
|
||||
- T1Q1 запись: 730 iops (задержка ~1.37ms latency)
|
||||
- T1Q1 чтение: 1500 iops с холодным кэшем метаданных (задержка ~0.66ms), 2300 iops через 2 минуты прогрева (задержка ~0.435ms)
|
||||
- T4Q128 запись (4 RBD images): 45300 iops, потребление CPU - 30 ядер на каждом сервере
|
||||
- T8Q64 чтение (4 RBD images): 278600 iops, потребление CPU - 40 ядер на каждом сервере
|
||||
- Линейная запись (4M T1Q32): 1950 МБ/с в пустой образ, 2500 МБ/с в заполненный образ
|
||||
- Линейное чтение (4M T1Q32): 2400 МБ/с
|
||||
|
||||
### NBD
|
||||
|
||||
NBD - на данный момент единственный способ монтировать Vitastor ядром Linux, но он
|
||||
приводит к дополнительным копированиям данных, поэтому немного ухудшает производительность,
|
||||
правда, в основном - линейную, а случайная затрагивается слабо.
|
||||
|
||||
NBD расшифровывается как "сетевое блочное устройство", но на самом деле оно также
|
||||
работает просто как аналог FUSE для блочных устройств, то есть, представляет собой
|
||||
"блочное устройство в пространстве пользователя".
|
||||
|
||||
Vitastor с однопоточной NBD прокси на том же стенде:
|
||||
- T1Q1 запись: 6000 iops (задержка 0.166ms)
|
||||
- T1Q1 чтение: 5518 iops (задержка 0.18ms)
|
||||
- T1Q128 запись: 94400 iops
|
||||
- T1Q128 чтение: 103000 iops
|
||||
- Линейная запись (4M T1Q128): 1266 МБ/с (в сравнении с 2800 МБ/с через fio)
|
||||
- Линейное чтение (4M T1Q128): 975 МБ/с (в сравнении с 1500 МБ/с через fio)
|
||||
|
||||
## Установка
|
||||
|
||||
### Debian
|
||||
|
||||
- Добавьте ключ репозитория Vitastor:
|
||||
`wget -q -O - https://vitastor.io/debian/pubkey | sudo apt-key add -`
|
||||
- Добавьте репозиторий Vitastor в /etc/apt/sources.list:
|
||||
- Debian 11 (Bullseye/Sid): `deb https://vitastor.io/debian bullseye main`
|
||||
- Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
|
||||
- Для Debian 10 (Buster) также включите репозиторий backports:
|
||||
`deb http://deb.debian.org/debian buster-backports main`
|
||||
- Установите пакеты: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu`
|
||||
|
||||
### CentOS
|
||||
|
||||
- Добавьте в систему репозиторий Vitastor:
|
||||
- CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release-1.0-1.el7.noarch.rpm`
|
||||
- CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release-1.0-1.el8.noarch.rpm`
|
||||
- Включите EPEL: `yum/dnf install epel-release`
|
||||
- Включите дополнительные репозитории CentOS:
|
||||
- CentOS 7: `yum install centos-release-scl`
|
||||
- CentOS 8: `dnf install centos-release-advanced-virtualization`
|
||||
- Включите elrepo-kernel:
|
||||
- CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
|
||||
- CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
|
||||
- Установите пакеты: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
|
||||
|
||||
### Установка из исходников
|
||||
|
||||
- Установите ядро 5.4 или более новое, для поддержки io_uring. Желательно 5.8 или даже новее,
|
||||
так как в 5.4 есть как минимум 1 известный баг, ведущий к зависанию с io_uring и контроллером HP SmartArray.
|
||||
- Установите liburing 0.4 или более новый и его заголовки.
|
||||
- Установите lp_solve.
|
||||
- Установите etcd. Внимание: вам нужна версия с исправлением отсюда: https://github.com/vitalif/etcd/,
|
||||
из ветки release-3.4, так как в etcd есть баг, который [будет](https://github.com/etcd-io/etcd/pull/12402)
|
||||
исправлен только в 3.4.15. Баг приводит к неспособности Vitastor запустить PG, когда их хотя бы 500 штук.
|
||||
- Установите node.js 10 или новее.
|
||||
- Установите gcc и g++ 8.x или новее.
|
||||
- Склонируйте данный репозиторий с подмодулями: `git clone https://yourcmc.ru/git/vitalif/vitastor/`.
|
||||
- Желательно пересобрать QEMU с патчем, который делает необязательным запуск через LD_PRELOAD.
|
||||
См `qemu-*.*-vitastor.patch` - выберите версию, наиболее близкую вашей версии QEMU.
|
||||
- Установите QEMU 3.0 или новее, возьмите исходные коды установленного пакета, начните его пересборку,
|
||||
через некоторое время остановите её и скопируйте следующие заголовки:
|
||||
- `<qemu>/include` → `<vitastor>/qemu/include`
|
||||
- Debian:
|
||||
* Берите qemu из основного репозитория
|
||||
* `<qemu>/b/qemu/config-host.h` → `<vitastor>/qemu/b/qemu/config-host.h`
|
||||
* `<qemu>/b/qemu/qapi` → `<vitastor>/qemu/b/qemu/qapi`
|
||||
- CentOS 8:
|
||||
* Берите qemu из репозитория Advanced-Virtualization. Чтобы включить его, запустите
|
||||
`yum install centos-release-advanced-virtualization.noarch` и далее `yum install qemu`
|
||||
* `<qemu>/config-host.h` → `<vitastor>/qemu/b/qemu/config-host.h`
|
||||
* Для QEMU 3.0+: `<qemu>/qapi` → `<vitastor>/qemu/b/qemu/qapi`
|
||||
* Для QEMU 2.0+: `<qemu>/qapi-types.h` → `<vitastor>/qemu/b/qemu/qapi-types.h`
|
||||
- `config-host.h` и `qapi` нужны, т.к. в них содержатся автогенерируемые заголовки
|
||||
- Установите fio 3.7 или новее, возьмите исходники пакета и сделайте на них симлинк с `<vitastor>/fio`.
|
||||
- Соберите и установите Vitastor командой `mkdir build && cd build && cmake .. && make -j8 && make install`.
|
||||
Обратите внимание на переменную cmake `QEMU_PLUGINDIR` - под RHEL её нужно установить равной `qemu-kvm`.
|
||||
|
||||
## Запуск
|
||||
|
||||
Внимание: процедура пока что достаточно нетривиальная, задавать конфигурацию и смещения
|
||||
на диске нужно почти вручную. Это будет исправлено в ближайшем будущем.
|
||||
|
||||
- Желательны SATA SSD или NVMe диски с конденсаторами (серверные SSD). Можно использовать и
|
||||
десктопные SSD, включив режим отложенного fsync, но производительность однопоточной записи
|
||||
в этом случае пострадает.
|
||||
- Быстрая сеть, минимум 10 гбит/с
|
||||
- Для наилучшей производительности нужно отключить энергосбережение CPU: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
|
||||
- Пропишите нужные вам значения вверху файлов `/usr/lib/vitastor/mon/make-units.sh` и `/usr/lib/vitastor/mon/make-osd.sh`.
|
||||
- Создайте юниты systemd для etcd и мониторов: `/usr/lib/vitastor/mon/make-units.sh`
|
||||
- Создайте юниты для OSD: `/usr/lib/vitastor/mon/make-osd.sh /dev/disk/by-partuuid/XXX [/dev/disk/by-partuuid/YYY ...]`
|
||||
- Вы можете поменять параметры OSD в юнитах systemd. Смысл некоторых параметров:
|
||||
- `disable_data_fsync 1` - отключает fsync, используется с SSD с конденсаторами.
|
||||
- `immediate_commit all` - используется с SSD с конденсаторами.
|
||||
- `disable_device_lock 1` - отключает блокировку файла устройства, нужно, только если вы запускаете
|
||||
несколько OSD на одном блочном устройстве.
|
||||
- `flusher_count 256` - "flusher" - микропоток, удаляющий старые данные из журнала.
|
||||
Не волнуйтесь об этой настройке, 256 теперь достаточно практически всегда.
|
||||
- `disk_alignment`, `journal_block_size`, `meta_block_size` следует установить равными размеру
|
||||
внутреннего блока SSD. Это почти всегда 4096.
|
||||
- `journal_no_same_sector_overwrites true` запрещает перезапись одного и того же сектора журнала подряд
|
||||
много раз в процессе записи. Большинство (99%) SSD не нуждаются в данной опции. Однако выяснилось, что
|
||||
диски, используемые на одном из тестовых стендов - Intel D3-S4510 - очень сильно не любят такую
|
||||
перезапись, и для них была добавлена эта опция. Когда данный режим включён, также нужно поднимать
|
||||
значение `journal_sector_buffer_count`, так как иначе Vitastor не хватит буферов для записи в журнал.
|
||||
- Запустите все etcd: `systemctl start etcd`
|
||||
- Создайте глобальную конфигурацию в etcd: `etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`
|
||||
(если все ваши диски - серверные с конденсаторами).
|
||||
- Создайте пулы: `etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":256,"failure_domain":"host"}}'`.
|
||||
Для jerasure EC-пулов конфигурация должна выглядеть так: `2:{"name":"ecpool","scheme":"jerasure","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}`.
|
||||
- Запустите все OSD: `systemctl start vitastor.target`
|
||||
- Ваш кластер должен быть готов - один из мониторов должен уже сконфигурировать PG, а OSD должны запустить их.
|
||||
- Вы можете проверить состояние PG прямо в etcd: `etcdctl --endpoints=... get --prefix /vitastor/pg/state`. Все PG должны быть 'active'.
|
||||
- Пример команды для запуска тестов: `fio -thread -ioengine=libfio_vitastor.so -name=test -bs=4M -direct=1 -iodepth=16 -rw=write -etcd=10.115.0.10:2379/v3 -pool=1 -inode=1 -size=400G`.
|
||||
- Пример команды для заливки образа ВМ в vitastor через qemu-img:
|
||||
```
|
||||
qemu-img convert -f qcow2 debian10.qcow2 -p -O raw 'vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648'
|
||||
```
|
||||
Если вы используете немодифицированный QEMU, данной команде потребуется переменная окружения `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so`.
|
||||
- Пример команды запуска QEMU:
|
||||
```
|
||||
qemu-system-x86_64 -enable-kvm -m 1024
|
||||
-drive 'file=vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648',format=raw,if=none,id=drive-virtio-disk0,cache=none
|
||||
-device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1,write-cache=off,physical_block_size=4096,logical_block_size=512
|
||||
-vnc 0.0.0.0:0
|
||||
```
|
||||
- Пример команды удаления образа (инода) из Vitastor:
|
||||
```
|
||||
vitastor-rm --etcd_address 10.115.0.10:2379/v3 --pool 1 --inode 1 --parallel_osds 16 --iodepth 32
|
||||
```
|
||||
|
||||
## Известные проблемы
|
||||
|
||||
- Запросы удаления объектов могут в данный момент приводить к "неполным" объектам в EC-пулах,
|
||||
если в процессе удаления произойдут отказы OSD или серверов, потому что правильная обработка
|
||||
запросов удаления в кластере должна быть "трёхфазной", а это пока не реализовано. Если вы
|
||||
столкнётесь с такой ситуацией, просто повторите запрос удаления.
|
||||
|
||||
## Принципы реализации
|
||||
|
||||
- Я люблю архитектурно простые решения. Vitastor проектируется именно так и я намерен
|
||||
и далее следовать данному принципу.
|
||||
- Если вы пришли сюда за идеальным кодом на C++, вы, вероятно, не по адресу. "Общепринятые"
|
||||
практики написания C++ кода меня не очень волнуют, так как зачастую, опять-таки, ведут к
|
||||
излишним усложнениям и код получается красивый... но медленный.
|
||||
- По той же причине в коде иногда можно встретить велосипеды типа собственного упрощённого
|
||||
HTTP-клиента для работы с etcd. Зато эти велосипеды маленькие и компактные и не требуют
|
||||
использования десятка внешних библиотек.
|
||||
- node.js для монитора - не случайный выбор. Он очень быстрый, имеет встроенную событийную
|
||||
машину, приятный нейтральный C-подобный язык программирования и развитую инфраструктуру.
|
||||
|
||||
## Автор и лицензия
|
||||
|
||||
Автор: Виталий Филиппов (vitalif [at] yourcmc.ru), 2019+
|
||||
|
||||
Заходите в Telegram-чат Vitastor: https://t.me/vitastor
|
||||
|
||||
Лицензия: VNPL 1.1 на серверный код и двойная VNPL 1.1 + GPL 2.0+ на клиентский.
|
||||
|
||||
VNPL - "сетевой копилефт", собственная свободная копилефт-лицензия
|
||||
Vitastor Network Public License 1.1, основанная на GNU GPL 3.0 с дополнительным
|
||||
условием "Сетевого взаимодействия", требующим распространять все программы,
|
||||
специально разработанные для использования вместе с Vitastor и взаимодействующие
|
||||
с ним по сети, под лицензией VNPL или под любой другой свободной лицензией.
|
||||
|
||||
Идея VNPL - расширение действия копилефта не только на модули, явным образом
|
||||
связываемые с кодом Vitastor, но также на модули, оформленные в виде микросервисов
|
||||
и взаимодействующие с ним по сети.
|
||||
|
||||
Таким образом, если вы хотите построить на основе Vitastor сервис, содержаший
|
||||
компоненты с закрытым кодом, взаимодействующие с Vitastor, вам нужна коммерческая
|
||||
лицензия от автора 😀.
|
||||
|
||||
На Windows и любое другое ПО, не разработанное *специально* для использования
|
||||
вместе с Vitastor, никакие ограничения не накладываются.
|
||||
|
||||
Клиентские библиотеки распространяются на условиях двойной лицензии VNPL 1.0
|
||||
и также на условиях GNU GPL 2.0 или более поздней версии. Так сделано в целях
|
||||
совместимости с таким ПО, как QEMU и fio.
|
||||
|
||||
Вы можете найти полный текст VNPL 1.1 в файле [VNPL-1.1.txt](VNPL-1.1.txt),
|
||||
а GPL 2.0 в файле [GPL-2.0.txt](GPL-2.0.txt).
|
134
README.md
134
README.md
|
@ -1,5 +1,7 @@
|
|||
## Vitastor
|
||||
|
||||
[Читать на русском](README-ru.md)
|
||||
|
||||
## The Idea
|
||||
|
||||
Make Software-Defined Block Storage Great Again.
|
||||
|
@ -16,7 +18,8 @@ breaking changes in the future. However, the following is implemented:
|
|||
|
||||
- Basic part: highly-available block storage with symmetric clustering and no SPOF
|
||||
- Performance ;-D
|
||||
- Two redundancy schemes: Replication and XOR n+1 (simplest case of EC)
|
||||
- Multiple redundancy schemes: Replication, XOR n+1, Reed-Solomon erasure codes
|
||||
based on jerasure library with any number of data and parity drives in a group
|
||||
- Configuration via simple JSON data structures in etcd
|
||||
- Automatic data distribution over OSDs, with support for:
|
||||
- Mathematical optimization for better uniformity and less data movement
|
||||
|
@ -31,25 +34,24 @@ breaking changes in the future. However, the following is implemented:
|
|||
- QEMU driver (built out-of-tree)
|
||||
- Loadable fio engine for benchmarks (also built out-of-tree)
|
||||
- NBD proxy for kernel mounts
|
||||
- Inode removal tool (./rm_inode)
|
||||
- Inode removal tool (vitastor-rm)
|
||||
- Packaging for Debian and CentOS
|
||||
- Per-inode I/O and space usage statistics
|
||||
- Inode metadata storage in etcd
|
||||
- Snapshots and copy-on-write image clones
|
||||
|
||||
## Roadmap
|
||||
|
||||
- Packaging for Debian and, probably, CentOS too
|
||||
- OSD creation tool (OSDs currently have to be created by hand)
|
||||
- Better OSD creation and auto-start tools
|
||||
- Other administrative tools
|
||||
- Per-inode I/O and space usage statistics
|
||||
- jerasure EC support with any number of data and parity drives in a group
|
||||
- Parallel usage of multiple network interfaces
|
||||
- Proxmox and OpenNebula plugins
|
||||
- Plugins for OpenStack, Kubernetes, OpenNebula, Proxmox and other cloud systems
|
||||
- iSCSI proxy
|
||||
- Inode metadata storage in etcd
|
||||
- Snapshots and copy-on-write image clones
|
||||
- Operation timeouts and better failure detection
|
||||
- Scrubbing without checksums (verification of replicas)
|
||||
- Checksums
|
||||
- SSD+HDD optimizations, possibly including tiered storage and soft journal flushes
|
||||
- RDMA and NVDIMM support
|
||||
- Web GUI
|
||||
- Compression (possibly)
|
||||
- Read caching using system page cache (possibly)
|
||||
|
||||
|
@ -280,7 +282,34 @@ Vitastor with single-thread NBD on the same hardware:
|
|||
- Linear write (4M T1Q128): 1266 MB/s (compared to 2800 MB/s via fio)
|
||||
- Linear read (4M T1Q128): 975 MB/s (compared to 1500 MB/s via fio)
|
||||
|
||||
## Building
|
||||
## Installation
|
||||
|
||||
### Debian
|
||||
|
||||
- Trust Vitastor package signing key:
|
||||
`wget -q -O - https://vitastor.io/debian/pubkey | sudo apt-key add -`
|
||||
- Add Vitastor package repository to your /etc/apt/sources.list:
|
||||
- Debian 11 (Bullseye/Sid): `deb https://vitastor.io/debian bullseye main`
|
||||
- Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
|
||||
- For Debian 10 (Buster) also enable backports repository:
|
||||
`deb http://deb.debian.org/debian buster-backports main`
|
||||
- Install packages: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu`
|
||||
|
||||
### CentOS
|
||||
|
||||
- Add Vitastor package repository:
|
||||
- CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release-1.0-1.el7.noarch.rpm`
|
||||
- CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release-1.0-1.el8.noarch.rpm`
|
||||
- Enable EPEL: `yum/dnf install epel-release`
|
||||
- Enable additional CentOS repositories:
|
||||
- CentOS 7: `yum install centos-release-scl`
|
||||
- CentOS 8: `dnf install centos-release-advanced-virtualization`
|
||||
- Enable elrepo-kernel:
|
||||
- CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
|
||||
- CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
|
||||
- Install packages: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
|
||||
|
||||
### Building from Source
|
||||
|
||||
- Install Linux kernel 5.4 or newer, for io_uring support. 5.8 or later is highly recommended because
|
||||
there is at least one known io_uring hang with 5.4 and an HP SmartArray controller.
|
||||
|
@ -290,10 +319,10 @@ Vitastor with single-thread NBD on the same hardware:
|
|||
branch release-3.4, because there is a bug in upstream etcd which makes Vitastor OSDs fail to
|
||||
move PGs out of "starting" state if you have at least around ~500 PGs or so. The custom build
|
||||
will be unnecessary when etcd merges the fix: https://github.com/etcd-io/etcd/pull/12402.
|
||||
- Install node.js 12 or newer.
|
||||
- Install gcc and g++ 9.x or later.
|
||||
- Install node.js 10 or newer.
|
||||
- Install gcc and g++ 8.x or newer.
|
||||
- Clone https://yourcmc.ru/git/vitalif/vitastor/ with submodules.
|
||||
- Install QEMU 4.x or 5.x, get its source, begin to build it, stop the build and copy headers:
|
||||
- Install QEMU 3.0+, get its source, begin to build it, stop the build and copy headers:
|
||||
- `<qemu>/include` → `<vitastor>/qemu/include`
|
||||
- Debian:
|
||||
* Use qemu packages from the main repository
|
||||
|
@ -303,11 +332,14 @@ Vitastor with single-thread NBD on the same hardware:
|
|||
* Use qemu packages from the Advanced-Virtualization repository. To enable it, run
|
||||
`yum install centos-release-advanced-virtualization.noarch` and then `yum install qemu`
|
||||
* `<qemu>/config-host.h` → `<vitastor>/qemu/b/qemu/config-host.h`
|
||||
* `<qemu>/qapi` → `<vitastor>/qemu/b/qemu/qapi`
|
||||
* For QEMU 3.0+: `<qemu>/qapi` → `<vitastor>/qemu/b/qemu/qapi`
|
||||
* For QEMU 2.0+: `<qemu>/qapi-types.h` → `<vitastor>/qemu/b/qemu/qapi-types.h`
|
||||
- `config-host.h` and `qapi` are required because they contain generated headers
|
||||
- Install fio 3.16 or later, get its source and symlink it into `<vitastor>/fio`.
|
||||
- Build Vitastor with `make -j8`.
|
||||
- Copy binaries somewhere.
|
||||
- You can also rebuild QEMU with a patch that makes LD_PRELOAD unnecessary to load vitastor driver.
|
||||
See `qemu-*.*-vitastor.patch`.
|
||||
- Install fio 3.7 or later, get its source and symlink it into `<vitastor>/fio`.
|
||||
- Build & install Vitastor with `mkdir build && cd build && cmake .. && make -j8 && make install`.
|
||||
Pay attention to the `QEMU_PLUGINDIR` cmake option - it must be set to `qemu-kvm` on RHEL.
|
||||
|
||||
## Running
|
||||
|
||||
|
@ -318,20 +350,16 @@ and calculate disk offsets almost by hand. This will be fixed in near future.
|
|||
with lazy fsync, but prepare for inferior single-thread latency.
|
||||
- Get a fast network (at least 10 Gbit/s).
|
||||
- Disable CPU powersaving: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
|
||||
- Start etcd with `--max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision` options.
|
||||
- Create global configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`
|
||||
(if all your drives have capacitors).
|
||||
- Create pool configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":256,"failure_domain":"host"}}'`.
|
||||
- Calculate offsets for your drives with `node ./mon/simple-offsets.js --device /dev/sdX`.
|
||||
- Make systemd units for your OSDs. Look at `./mon/make-units.sh` for example.
|
||||
Notable configuration variables from the example:
|
||||
- Check `/usr/lib/vitastor/mon/make-units.sh` and `/usr/lib/vitastor/mon/make-osd.sh` and
|
||||
put desired values into the variables at the top of these files.
|
||||
- Create systemd units for the monitor and etcd: `/usr/lib/vitastor/mon/make-units.sh`
|
||||
- Create systemd units for your OSDs: `/usr/lib/vitastor/mon/make-osd.sh /dev/disk/by-partuuid/XXX [/dev/disk/by-partuuid/YYY ...]`
|
||||
- You can edit the units and change OSD configuration. Notable configuration variables:
|
||||
- `disable_data_fsync 1` - only safe with server-grade drives with capacitors.
|
||||
- `immediate_commit all` - use this if all your drives are server-grade.
|
||||
- `disable_device_lock 1` - only required if you run multiple OSDs on one block device.
|
||||
- `flusher_count 16` - flusher is a micro-thread that removes old data from the journal.
|
||||
More flushers mean more aggressive journal flushing which allows for more throughput
|
||||
but slightly hurts latency under less load. Flushing will probably be improved in the future
|
||||
because currently high queue depths sometimes lead to performance degradation.
|
||||
- `flusher_count 256` - flusher is a micro-thread that removes old data from the journal.
|
||||
You don't have to worry about this parameter anymore, 256 is enough.
|
||||
- `disk_alignment`, `journal_block_size`, `meta_block_size` should be set to the internal
|
||||
block size of your SSDs which is 4096 on most drives.
|
||||
- `journal_no_same_sector_overwrites true` prevents multiple overwrites of the same journal sector.
|
||||
|
@ -342,39 +370,42 @@ and calculate disk offsets almost by hand. This will be fixed in near future.
|
|||
setting is set, it is also required to raise `journal_sector_buffer_count` setting, which is the
|
||||
number of dirty journal sectors that may be written to at the same time.
|
||||
- `systemctl start vitastor.target` everywhere.
|
||||
- Start any number of monitors: `cd mon; node mon-main.js --etcd_url 'http://10.115.0.10:2379,http://10.115.0.11:2379,http://10.115.0.12:2379,http://10.115.0.13:2379' --etcd_prefix '/vitastor' --etcd_start_timeout 5`.
|
||||
- Create global configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`
|
||||
(if all your drives have capacitors).
|
||||
- Create pool configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":256,"failure_domain":"host"}}'`.
|
||||
For jerasure pools the configuration should look like the following: `2:{"name":"ecpool","scheme":"jerasure","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}`.
|
||||
- At this point, one of the monitors will configure PGs and OSDs will start them.
|
||||
- You can check PG states with `etcdctl --endpoints=... get --prefix /vitastor/pg/state`. All PGs should become 'active'.
|
||||
- Run tests with (for example): `fio -thread -ioengine=./libfio_cluster.so -name=test -bs=4M -direct=1 -iodepth=16 -rw=write -etcd=10.115.0.10:2379/v3 -pool=1 -inode=1 -size=400G`.
|
||||
- Run tests with (for example): `fio -thread -ioengine=libfio_vitastor.so -name=test -bs=4M -direct=1 -iodepth=16 -rw=write -etcd=10.115.0.10:2379/v3 -pool=1 -inode=1 -size=400G`.
|
||||
- Upload VM disk image with qemu-img (for example):
|
||||
```
|
||||
LD_PRELOAD=./qemu_driver.so qemu-img convert -f qcow2 debian10.qcow2 -p
|
||||
-O raw 'vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648'
|
||||
qemu-img convert -f qcow2 debian10.qcow2 -p -O raw 'vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648'
|
||||
```
|
||||
Note that the command requires to be run with `LD_PRELOAD=/usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so qemu-img ...`
|
||||
if you use unmodified QEMU.
|
||||
- Run QEMU with (for example):
|
||||
```
|
||||
LD_PRELOAD=./qemu_driver.so qemu-system-x86_64 -enable-kvm -m 1024
|
||||
qemu-system-x86_64 -enable-kvm -m 1024
|
||||
-drive 'file=vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648',format=raw,if=none,id=drive-virtio-disk0,cache=none
|
||||
-device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1,write-cache=off,physical_block_size=4096,logical_block_size=512
|
||||
-vnc 0.0.0.0:0
|
||||
```
|
||||
- Remove inode with (for example):
|
||||
```
|
||||
./rm_inode --etcd_address 10.115.0.10:2379/v3 --pool 1 --inode 1 --parallel_osds 16 --iodepth 32
|
||||
vitastor-rm --etcd_address 10.115.0.10:2379/v3 --pool 1 --inode 1 --parallel_osds 16 --iodepth 32
|
||||
```
|
||||
|
||||
## Known Problems
|
||||
|
||||
- Object deletion requests may currently lead to 'incomplete' objects if your OSDs crash during
|
||||
deletion because proper handling of object cleanup in a cluster should be "three-phase"
|
||||
and it's currently not implemented. Inode removal tool currently can't handle unclean
|
||||
objects, so incomplete objects become undeletable. This will be fixed in near future
|
||||
by allowing the inode removal tool to delete unclean objects. With this problem fixed
|
||||
you'll be able just to repeat the removal again.
|
||||
- Object deletion requests may currently lead to 'incomplete' objects in EC pools
|
||||
if your OSDs crash during deletion because proper handling of object cleanup
|
||||
in a cluster should be "three-phase" and it's currently not implemented.
|
||||
Just repeat the removal request again in this case.
|
||||
|
||||
## Implementation Principles
|
||||
|
||||
- I like simple and stupid solutions, so expect Vitastor to stay simple.
|
||||
- I like architecturally simple solutions. Vitastor is and will always be designed
|
||||
exactly like that.
|
||||
- I also like reinventing the wheel to some extent, like writing my own HTTP client
|
||||
for etcd interaction instead of using prebuilt libraries, because in this case
|
||||
I'm confident about what my code does and what it doesn't do.
|
||||
|
@ -389,25 +420,30 @@ and calculate disk offsets almost by hand. This will be fixed in near future.
|
|||
|
||||
Copyright (c) Vitaliy Filippov (vitalif [at] yourcmc.ru), 2019+
|
||||
|
||||
You can also find me in the Russian Telegram Ceph chat: https://t.me/ceph_ru
|
||||
Join Vitastor Telegram Chat: https://t.me/vitastor
|
||||
|
||||
All server-side code (OSD, Monitor and so on) is licensed under the terms of
|
||||
Vitastor Network Public License 1.0 (VNPL 1.0), a copyleft license based on
|
||||
Vitastor Network Public License 1.1 (VNPL 1.1), a copyleft license based on
|
||||
GNU GPLv3.0 with the additional "Network Interaction" clause which requires
|
||||
opensourcing all programs directly or indirectly interacting with Vitastor
|
||||
through a computer network ("Proxy Programs"). Proxy Programs may be made public
|
||||
not only under the terms of the same license, but also under the terms of any
|
||||
GPL-Compatible Free Software License, as listed by the Free Software Foundation.
|
||||
through a computer network and expressly designed to be used in conjunction
|
||||
with it ("Proxy Programs"). Proxy Programs may be made public not only under
|
||||
the terms of the same license, but also under the terms of any GPL-Compatible
|
||||
Free Software License, as listed by the Free Software Foundation.
|
||||
This is a stricter copyleft license than the Affero GPL.
|
||||
|
||||
Please note that VNPL doesn't require you to open the code of proprietary
|
||||
software running inside a VM if it's not specially designed to be used with
|
||||
Vitastor.
|
||||
|
||||
Basically, you can't use the software in a proprietary environment to provide
|
||||
its functionality to users without opensourcing all intermediary components
|
||||
standing between the user and Vitastor or purchasing a commercial license
|
||||
from the author 😀.
|
||||
|
||||
Client libraries (cluster_client and so on) are dual-licensed under the same
|
||||
VNPL 1.0 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
|
||||
VNPL 1.1 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
|
||||
software like QEMU and fio.
|
||||
|
||||
You can find the full text of VNPL-1.0 in the file [VNPL-1.0.txt](VNPL-1.0.txt).
|
||||
You can find the full text of VNPL-1.1 in the file [VNPL-1.1.txt](VNPL-1.1.txt).
|
||||
GPL 2.0 is also included in this repository as [GPL-2.0.txt](GPL-2.0.txt).
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
VITASTOR NETWORK PUBLIC LICENSE
|
||||
Version 1, 17 September 2020
|
||||
Version 1.1, 6 February 2021
|
||||
|
||||
Copyright (C) 2020 Vitaliy Filippov <vitalif@yourcmc.ru>
|
||||
Copyright (C) 2021 Vitaliy Filippov <vitalif@yourcmc.ru>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
|
@ -540,12 +540,15 @@ License would be to refrain entirely from conveying the Program.
|
|||
|
||||
13. Remote Network Interaction.
|
||||
|
||||
Notwithstanding any other provision of this License, if you provide
|
||||
any user an opportunity to interact with the covered work directly
|
||||
or indirectly through a computer network, an imitation of such network,
|
||||
or an additional program (hereinafter referred to as a "Proxy Program")
|
||||
that, in turn, interacts with the covered work through a computer network,
|
||||
an imitation of such network, or another Proxy Program itself,
|
||||
A "Proxy Program" means a separate program which is specially designed to
|
||||
be used in conjunction with the covered work and interacts with it directly
|
||||
or indirectly through any kind of API (application programming interfaces),
|
||||
a computer network, an imitation of such network, or another Proxy Program
|
||||
itself.
|
||||
|
||||
Notwithstanding any other provision of this License, if you provide any user
|
||||
with an opportunity to interact with the covered work through a computer
|
||||
network, an imitation of such network, or any number of "Proxy Programs",
|
||||
you must prominently offer that user an opportunity to receive the
|
||||
Corresponding Source of the covered work and all Proxy Programs from a
|
||||
network server at no charge, through some standard or customary means of
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
|
||||
gcc -I. -E -o fio_headers.i src/fio_headers.h
|
||||
|
||||
rm -rf fio-copy
|
||||
for i in `grep -Po 'fio/[^"]+' fio_headers.i | sort | uniq`; do
|
||||
j=${i##fio/}
|
||||
p=$(dirname $j)
|
||||
mkdir -p fio-copy/$p
|
||||
cp $i fio-copy/$j
|
||||
done
|
||||
|
||||
rm fio_headers.i
|
|
@ -0,0 +1,18 @@
|
|||
#!/bin/bash
|
||||
|
||||
#cd qemu
|
||||
#debian/rules b/configure-stamp
|
||||
#cd b/qemu; make qapi
|
||||
|
||||
gcc -I qemu/b/qemu `pkg-config glib-2.0 --cflags` \
|
||||
-I qemu/include -E -o qemu_driver.i src/qemu_driver.c
|
||||
|
||||
rm -rf qemu-copy
|
||||
for i in `grep -Po 'qemu/[^"]+' qemu_driver.i | sort | uniq`; do
|
||||
j=${i##qemu/}
|
||||
p=$(dirname $j)
|
||||
mkdir -p qemu-copy/$p
|
||||
cp $i qemu-copy/$j
|
||||
done
|
||||
|
||||
rm qemu_driver.i
|
|
@ -0,0 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
sed 's/$REL/bullseye/g' < vitastor.Dockerfile > ../Dockerfile
|
||||
cd ..
|
||||
mkdir -p packages
|
||||
sudo podman build -v `pwd`/packages:/root/packages -f Dockerfile .
|
||||
rm Dockerfile
|
|
@ -0,0 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
sed 's/$REL/buster/g' < vitastor.Dockerfile > ../Dockerfile
|
||||
cd ..
|
||||
mkdir -p packages
|
||||
sudo podman build -v `pwd`/packages:/root/packages -f Dockerfile .
|
||||
rm Dockerfile
|
|
@ -0,0 +1,17 @@
|
|||
vitastor (0.5.10-1) unstable; urgency=medium
|
||||
|
||||
* Bugfixes
|
||||
|
||||
-- Vitaliy Filippov <vitalif@yourcmc.ru> Tue, 02 Feb 2021 23:01:24 +0300
|
||||
|
||||
vitastor (0.5.1-1) unstable; urgency=medium
|
||||
|
||||
* Add jerasure support
|
||||
|
||||
-- Vitaliy Filippov <vitalif@yourcmc.ru> Sat, 05 Dec 2020 17:02:26 +0300
|
||||
|
||||
vitastor (0.5-1) unstable; urgency=medium
|
||||
|
||||
* First packaging for Debian
|
||||
|
||||
-- Vitaliy Filippov <vitalif@yourcmc.ru> Thu, 05 Nov 2020 02:20:59 +0300
|
|
@ -0,0 +1 @@
|
|||
13
|
|
@ -0,0 +1,17 @@
|
|||
Source: vitastor
|
||||
Section: admin
|
||||
Priority: optional
|
||||
Maintainer: Vitaliy Filippov <vitalif@yourcmc.ru>
|
||||
Build-Depends: debhelper, liburing-dev (>= 0.6), g++ (>= 8), libstdc++6 (>= 8), linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev
|
||||
Standards-Version: 4.5.0
|
||||
Homepage: https://vitastor.io/
|
||||
Rules-Requires-Root: no
|
||||
|
||||
Package: vitastor
|
||||
Architecture: amd64
|
||||
Depends: ${shlibs:Depends}, ${misc:Depends}, fio (= ${dep:fio}), qemu (= ${dep:qemu}), nodejs (>= 10), node-sprintf-js, node-ws (>= 7), libjerasure2, lp-solve
|
||||
Description: Vitastor, a fast software-defined clustered block storage
|
||||
Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
|
||||
architecturally similar to Ceph which means strong consistency, primary-replication,
|
||||
symmetric clustering and automatic data distribution over any number of drives of any
|
||||
size with configurable redundancy (replication or erasure codes/XOR).
|
|
@ -0,0 +1,21 @@
|
|||
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
|
||||
Upstream-Name: vitastor
|
||||
Upstream-Contact: Vitaliy Filippov <vitalif@yourcmc.ru>
|
||||
Source: https://vitastor.io
|
||||
|
||||
Files: *
|
||||
Copyright: 2019+ Vitaliy Filippov <vitalif@yourcmc.ru>
|
||||
License: Multiple licenses VNPL-1.1 and/or GPL-2.0+
|
||||
All server-side code (OSD, Monitor and so on) is licensed under the terms of
|
||||
Vitastor Network Public License 1.1 (VNPL 1.1), a copyleft license based on
|
||||
GNU GPLv3.0 with the additional "Network Interaction" clause which requires
|
||||
opensourcing all programs directly or indirectly interacting with Vitastor
|
||||
through a computer network and expressly designed to be used in conjunction
|
||||
with it ("Proxy Programs"). Proxy Programs may be made public not only under
|
||||
the terms of the same license, but also under the terms of any GPL-Compatible
|
||||
Free Software License, as listed by the Free Software Foundation.
|
||||
This is a stricter copyleft license than the Affero GPL.
|
||||
.
|
||||
Client libraries (cluster_client and so on) are dual-licensed under the same
|
||||
VNPL 1.1 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
|
||||
software like QEMU and fio.
|
|
@ -0,0 +1,3 @@
|
|||
VNPL-1.1.txt usr/share/doc/vitastor
|
||||
GPL-2.0.txt usr/share/doc/vitastor
|
||||
mon usr/lib/vitastor
|
|
@ -0,0 +1,44 @@
|
|||
# Build patched QEMU for Debian Buster or Bullseye/Sid inside a container
|
||||
# cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/patched-qemu.Dockerfile .
|
||||
|
||||
FROM debian:$REL
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
RUN if [ "$REL" = "buster" ]; then \
|
||||
echo 'deb http://deb.debian.org/debian buster-backports main' >> /etc/apt/sources.list; \
|
||||
echo >> /etc/apt/preferences; \
|
||||
echo 'Package: *' >> /etc/apt/preferences; \
|
||||
echo 'Pin: release a=buster-backports' >> /etc/apt/preferences; \
|
||||
echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
|
||||
fi; \
|
||||
grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
|
||||
echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
|
||||
echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get -y install qemu fio liburing1 liburing-dev libgoogle-perftools-dev devscripts
|
||||
RUN apt-get -y build-dep qemu
|
||||
RUN apt-get -y build-dep fio
|
||||
RUN apt-get --download-only source qemu
|
||||
RUN apt-get --download-only source fio
|
||||
|
||||
ADD qemu-5.0-vitastor.patch qemu-5.1-vitastor.patch /root/vitastor/
|
||||
RUN set -e; \
|
||||
mkdir -p /root/packages/qemu-$REL; \
|
||||
rm -rf /root/packages/qemu-$REL/*; \
|
||||
cd /root/packages/qemu-$REL; \
|
||||
dpkg-source -x /root/qemu*.dsc; \
|
||||
if [ -d /root/packages/qemu-$REL/qemu-5.0 ]; then \
|
||||
cp /root/vitastor/qemu-5.0-vitastor.patch /root/packages/qemu-$REL/qemu-5.0/debian/patches; \
|
||||
echo qemu-5.0-vitastor.patch >> /root/packages/qemu-$REL/qemu-5.0/debian/patches/series; \
|
||||
else \
|
||||
cp /root/vitastor/qemu-5.1-vitastor.patch /root/packages/qemu-$REL/qemu-*/debian/patches; \
|
||||
P=`ls -d /root/packages/qemu-$REL/qemu-*/debian/patches`; \
|
||||
echo qemu-5.1-vitastor.patch >> $P/series; \
|
||||
fi; \
|
||||
cd /root/packages/qemu-$REL/qemu-*/; \
|
||||
V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)(~bpo[\d\+]*)?\).*$/$1/')+vitastor1; \
|
||||
DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v $V 'Plug Vitastor block driver'; \
|
||||
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
|
||||
rm -rf /root/packages/qemu-$REL/qemu-*/
|
|
@ -0,0 +1,9 @@
|
|||
#!/usr/bin/make -f
|
||||
export DH_VERBOSE = 1
|
||||
|
||||
%:
|
||||
dh $@
|
||||
|
||||
override_dh_installdeb:
|
||||
cat debian/substvars >> debian/vitastor.substvars
|
||||
dh_installdeb
|
|
@ -0,0 +1 @@
|
|||
3.0 (quilt)
|
|
@ -0,0 +1,2 @@
|
|||
dep:fio=3.16-1
|
||||
dep:qemu=1:5.1+dfsg-4+vitastor1
|
|
@ -0,0 +1,67 @@
|
|||
# Build Vitastor packages for Debian Buster or Bullseye/Sid inside a container
|
||||
# cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/vitastor.Dockerfile .
|
||||
|
||||
FROM debian:$REL
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
RUN if [ "$REL" = "buster" ]; then \
|
||||
echo 'deb http://deb.debian.org/debian buster-backports main' >> /etc/apt/sources.list; \
|
||||
echo >> /etc/apt/preferences; \
|
||||
echo 'Package: *' >> /etc/apt/preferences; \
|
||||
echo 'Pin: release a=buster-backports' >> /etc/apt/preferences; \
|
||||
echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
|
||||
fi; \
|
||||
grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
|
||||
echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
|
||||
echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get -y install qemu fio liburing1 liburing-dev libgoogle-perftools-dev devscripts
|
||||
RUN apt-get -y build-dep qemu
|
||||
RUN apt-get -y build-dep fio
|
||||
RUN apt-get --download-only source qemu
|
||||
RUN apt-get --download-only source fio
|
||||
RUN apt-get -y install libjerasure-dev cmake
|
||||
|
||||
ADD . /root/vitastor
|
||||
RUN set -e -x; \
|
||||
mkdir -p /root/fio-build/; \
|
||||
cd /root/fio-build/; \
|
||||
rm -rf /root/fio-build/*; \
|
||||
dpkg-source -x /root/fio*.dsc; \
|
||||
cd /root/packages/qemu-$REL/; \
|
||||
rm -rf qemu*/; \
|
||||
dpkg-source -x qemu*.dsc; \
|
||||
cd /root/packages/qemu-$REL/qemu*/; \
|
||||
debian/rules b/configure-stamp; \
|
||||
cd b/qemu; \
|
||||
make -j8 qapi/qapi-builtin-types.h; \
|
||||
mkdir -p /root/packages/vitastor-$REL; \
|
||||
rm -rf /root/packages/vitastor-$REL/*; \
|
||||
cd /root/packages/vitastor-$REL; \
|
||||
cp -r /root/vitastor vitastor-0.5.10; \
|
||||
ln -s /root/packages/qemu-$REL/qemu-*/ vitastor-0.5.10/qemu; \
|
||||
ln -s /root/fio-build/fio-*/ vitastor-0.5.10/fio; \
|
||||
cd vitastor-0.5.10; \
|
||||
FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
|
||||
QEMU=$(head -n1 qemu/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
|
||||
sh copy-qemu-includes.sh; \
|
||||
sh copy-fio-includes.sh; \
|
||||
rm qemu fio; \
|
||||
mkdir -p a b debian/patches; \
|
||||
mv qemu-copy b/qemu; \
|
||||
mv fio-copy b/fio; \
|
||||
diff -NaurpbB a b > debian/patches/qemu-fio-headers.patch || true; \
|
||||
echo qemu-fio-headers.patch >> debian/patches/series; \
|
||||
rm -rf a b; \
|
||||
rm -rf /root/packages/qemu-$REL/qemu*/; \
|
||||
echo "dep:fio=$FIO" > debian/substvars; \
|
||||
echo "dep:qemu=$QEMU" >> debian/substvars; \
|
||||
cd /root/packages/vitastor-$REL; \
|
||||
tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.5.10.orig.tar.xz vitastor-0.5.10; \
|
||||
cd vitastor-0.5.10; \
|
||||
V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
|
||||
DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
|
||||
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
|
||||
rm -rf /root/packages/vitastor-$REL/vitastor-*/
|
|
@ -1,51 +0,0 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
|
||||
#include <iostream>
|
||||
#include <functional>
|
||||
#include <array>
|
||||
#include <cstdlib> // for malloc() and free()
|
||||
using namespace std;
|
||||
|
||||
// replace operator new and delete to log allocations
|
||||
void* operator new(std::size_t n)
|
||||
{
|
||||
cout << "Allocating " << n << " bytes" << endl;
|
||||
return malloc(n);
|
||||
}
|
||||
|
||||
void operator delete(void* p) throw()
|
||||
{
|
||||
free(p);
|
||||
}
|
||||
|
||||
class test
|
||||
{
|
||||
public:
|
||||
std::string s;
|
||||
void a(std::function<void()> & f, const char *str)
|
||||
{
|
||||
auto l = [this, str]() { cout << str << " ? " << s << " from this\n"; };
|
||||
cout << "Assigning lambda3 of size " << sizeof(l) << endl;
|
||||
f = l;
|
||||
}
|
||||
};
|
||||
|
||||
int main()
|
||||
{
|
||||
std::array<char, 16> arr1;
|
||||
auto lambda1 = [arr1](){};
|
||||
cout << "Assigning lambda1 of size " << sizeof(lambda1) << endl;
|
||||
std::function<void()> f1 = lambda1;
|
||||
|
||||
std::array<char, 17> arr2;
|
||||
auto lambda2 = [arr2](){};
|
||||
cout << "Assigning lambda2 of size " << sizeof(lambda2) << endl;
|
||||
std::function<void()> f2 = lambda2;
|
||||
|
||||
test t;
|
||||
std::function<void()> f3;
|
||||
t.s = "str";
|
||||
t.a(f3, "huyambda");
|
||||
f3();
|
||||
}
|
114
mon/PGUtil.js
114
mon/PGUtil.js
|
@ -1,22 +1,59 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
module.exports = {
|
||||
scale_pg_count,
|
||||
};
|
||||
|
||||
function add_pg_history(new_pg_history, new_pg, prev_pgs, prev_pg_history, old_pg)
|
||||
{
|
||||
if (!new_pg_history[new_pg])
|
||||
{
|
||||
new_pg_history[new_pg] = {
|
||||
osd_sets: {},
|
||||
all_peers: {},
|
||||
epoch: 0,
|
||||
};
|
||||
}
|
||||
const nh = new_pg_history[new_pg], oh = prev_pg_history[old_pg];
|
||||
nh.osd_sets[prev_pgs[old_pg].join(' ')] = prev_pgs[old_pg];
|
||||
if (oh && oh.osd_sets && oh.osd_sets.length)
|
||||
{
|
||||
for (const pg of oh.osd_sets)
|
||||
{
|
||||
nh.osd_sets[pg.join(' ')] = pg;
|
||||
}
|
||||
}
|
||||
if (oh && oh.all_peers && oh.all_peers.length)
|
||||
{
|
||||
for (const osd_num of oh.all_peers)
|
||||
{
|
||||
nh.all_peers[osd_num] = Number(osd_num);
|
||||
}
|
||||
}
|
||||
if (oh && oh.epoch)
|
||||
{
|
||||
nh.epoch = nh.epoch < oh.epoch ? oh.epoch : nh.epoch;
|
||||
}
|
||||
}
|
||||
|
||||
function finish_pg_history(merged_history)
|
||||
{
|
||||
merged_history.osd_sets = Object.values(merged_history.osd_sets);
|
||||
merged_history.all_peers = Object.values(merged_history.all_peers);
|
||||
}
|
||||
|
||||
function scale_pg_count(prev_pgs, prev_pg_history, new_pg_history, new_pg_count)
|
||||
{
|
||||
const old_pg_count = prev_pgs.length;
|
||||
// Add all possibly intersecting PGs into the history of new PGs
|
||||
// Add all possibly intersecting PGs to the history of new PGs
|
||||
if (!(new_pg_count % old_pg_count))
|
||||
{
|
||||
// New PG count is a multiple of the old PG count
|
||||
const mul = (new_pg_count / old_pg_count);
|
||||
// New PG count is a multiple of old PG count
|
||||
for (let i = 0; i < new_pg_count; i++)
|
||||
{
|
||||
const old_i = Math.floor(new_pg_count / mul);
|
||||
new_pg_history[i] = JSON.parse(JSON.stringify(prev_pg_history[1+old_i]));
|
||||
add_pg_history(new_pg_history, i, prev_pgs, prev_pg_history, i % old_pg_count);
|
||||
finish_pg_history(new_pg_history[i]);
|
||||
}
|
||||
}
|
||||
else if (!(old_pg_count % new_pg_count))
|
||||
|
@ -25,68 +62,26 @@ function scale_pg_count(prev_pgs, prev_pg_history, new_pg_history, new_pg_count)
|
|||
const mul = (old_pg_count / new_pg_count);
|
||||
for (let i = 0; i < new_pg_count; i++)
|
||||
{
|
||||
new_pg_history[i] = {
|
||||
osd_sets: [],
|
||||
all_peers: [],
|
||||
epoch: 0,
|
||||
};
|
||||
for (let j = 0; j < mul; j++)
|
||||
{
|
||||
new_pg_history[i].osd_sets.push(prev_pgs[i*mul]);
|
||||
const hist = prev_pg_history[1+i*mul+j];
|
||||
if (hist && hist.osd_sets && hist.osd_sets.length)
|
||||
{
|
||||
Array.prototype.push.apply(new_pg_history[i].osd_sets, hist.osd_sets);
|
||||
}
|
||||
if (hist && hist.all_peers && hist.all_peers.length)
|
||||
{
|
||||
Array.prototype.push.apply(new_pg_history[i].all_peers, hist.all_peers);
|
||||
}
|
||||
if (hist && hist.epoch)
|
||||
{
|
||||
new_pg_history[i].epoch = new_pg_history[i].epoch < hist.epoch ? hist.epoch : new_pg_history[i].epoch;
|
||||
}
|
||||
add_pg_history(new_pg_history, i, prev_pgs, prev_pg_history, i+j*new_pg_count);
|
||||
}
|
||||
finish_pg_history(new_pg_history[i]);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Any PG may intersect with any PG after non-multiple PG count change
|
||||
// So, merge ALL PGs history
|
||||
let all_sets = {};
|
||||
let all_peers = {};
|
||||
let max_epoch = 0;
|
||||
for (const pg of prev_pgs)
|
||||
let merged_history = {};
|
||||
for (let i = 0; i < old_pg_count; i++)
|
||||
{
|
||||
all_sets[pg.join(' ')] = pg;
|
||||
add_pg_history(merged_history, 1, prev_pgs, prev_pg_history, i);
|
||||
}
|
||||
for (const pg in prev_pg_history)
|
||||
{
|
||||
const hist = prev_pg_history[pg];
|
||||
if (hist && hist.osd_sets)
|
||||
{
|
||||
for (const pg of hist.osd_sets)
|
||||
{
|
||||
all_sets[pg.join(' ')] = pg;
|
||||
}
|
||||
}
|
||||
if (hist && hist.all_peers)
|
||||
{
|
||||
for (const osd_num of hist.all_peers)
|
||||
{
|
||||
all_peers[osd_num] = Number(osd_num);
|
||||
}
|
||||
}
|
||||
if (hist && hist.epoch)
|
||||
{
|
||||
max_epoch = max_epoch < hist.epoch ? hist.epoch : max_epoch;
|
||||
}
|
||||
}
|
||||
all_sets = Object.values(all_sets);
|
||||
all_peers = Object.values(all_peers);
|
||||
finish_pg_history(merged_history[1]);
|
||||
for (let i = 0; i < new_pg_count; i++)
|
||||
{
|
||||
new_pg_history[i] = { osd_sets: all_sets, all_peers, epoch: max_epoch };
|
||||
new_pg_history[i] = { ...merged_history[1] };
|
||||
}
|
||||
}
|
||||
// Mark history keys for removed PGs as removed
|
||||
|
@ -94,19 +89,16 @@ function scale_pg_count(prev_pgs, prev_pg_history, new_pg_history, new_pg_count)
|
|||
{
|
||||
new_pg_history[i] = null;
|
||||
}
|
||||
// Just for the lp_solve optimizer - pick a "previous" PG for each "new" one
|
||||
if (old_pg_count < new_pg_count)
|
||||
{
|
||||
for (let i = new_pg_count-1; i >= 0; i--)
|
||||
for (let i = old_pg_count; i < new_pg_count; i++)
|
||||
{
|
||||
prev_pgs[i] = prev_pgs[Math.floor(i/new_pg_count*old_pg_count)];
|
||||
prev_pgs[i] = prev_pgs[i % old_pg_count];
|
||||
}
|
||||
}
|
||||
else if (old_pg_count > new_pg_count)
|
||||
{
|
||||
for (let i = 0; i < new_pg_count; i++)
|
||||
{
|
||||
prev_pgs[i] = prev_pgs[Math.round(i/new_pg_count*old_pg_count)];
|
||||
}
|
||||
prev_pgs.splice(new_pg_count, old_pg_count-new_pg_count);
|
||||
}
|
||||
}
|
||||
|
|
116
mon/afr.js
116
mon/afr.js
|
@ -1,31 +1,16 @@
|
|||
// Functions to calculate Annualized Failure Rate of your cluster
|
||||
// if you know AFR of your drives, number of drives, expected rebalance time
|
||||
// and replication factor
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
|
||||
const { sprintf } = require('sprintf-js');
|
||||
// License: VNPL-1.1 (see https://yourcmc.ru/git/vitalif/vitastor/src/branch/master/README.md for details) or AGPL-3.0
|
||||
// Author: Vitaliy Filippov, 2020+
|
||||
|
||||
module.exports = {
|
||||
cluster_afr_fullmesh,
|
||||
failure_rate_fullmesh,
|
||||
cluster_afr,
|
||||
print_cluster_afr,
|
||||
c_n_k,
|
||||
};
|
||||
|
||||
print_cluster_afr({ n_hosts: 4, n_drives: 6, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
|
||||
print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, capacity: 4000, speed: 0.1, replicas: 2 });
|
||||
print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
|
||||
print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, capacity: 4000, speed: 0.1, ec: [ 2, 1 ] });
|
||||
print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, ec: [ 2, 1 ] });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, capacity: 8000, speed: 0.02, replicas: 2 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 2 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, capacity: 8000, speed: 0.02, replicas: 3 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100, degraded_replacement: 1 });
|
||||
|
||||
/******** "FULL MESH": ASSUME EACH OSD COMMUNICATES WITH ALL OTHER OSDS ********/
|
||||
|
||||
// Estimate AFR of the cluster
|
||||
|
@ -56,93 +41,38 @@ function failure_rate_fullmesh(n, a, f)
|
|||
/******** PGS: EACH OSD ONLY COMMUNICATES WITH <pgs> OTHER OSDs ********/
|
||||
|
||||
// <n> hosts of <m> drives of <capacity> GB, each able to backfill at <speed> GB/s,
|
||||
// <k> replicas, <pgs> unique peer PGs per OSD
|
||||
// <k> replicas, <pgs> unique peer PGs per OSD (~50 for 100 PG-per-OSD in a big cluster)
|
||||
//
|
||||
// For each of n*m drives: P(drive fails in a year) * P(any of its peers fail in <l*365> next days).
|
||||
// More peers per OSD increase rebalance speed (more drives work together to resilver) if you
|
||||
// let them finish rebalance BEFORE replacing the failed drive.
|
||||
// let them finish rebalance BEFORE replacing the failed drive (degraded_replacement=false).
|
||||
// At the same time, more peers per OSD increase probability of any of them to fail!
|
||||
// osd_rm=true means that failed OSDs' data is rebalanced over all other hosts,
|
||||
// not over the same host as it's in Ceph by default (dead OSDs are marked 'out').
|
||||
//
|
||||
// Probability of all except one drives in a replica group to fail is (AFR^(k-1)).
|
||||
// So with <x> PGs it becomes ~ (x * (AFR*L/365)^(k-1)). Interesting but reasonable consequence
|
||||
// is that, with k=2, total failure rate doesn't depend on number of peers per OSD,
|
||||
// because it gets increased linearly by increased number of peers to fail
|
||||
// and decreased linearly by reduced rebalance time.
|
||||
function cluster_afr_pgs({ n_hosts, n_drives, afr_drive, capacity, speed, replicas, pgs = 1, degraded_replacement })
|
||||
function cluster_afr({ n_hosts, n_drives, afr_drive, afr_host, capacity, speed, ec, ec_data, ec_parity, replicas, pgs = 1, osd_rm, degraded_replacement, down_out_interval = 600 })
|
||||
{
|
||||
pgs = Math.min(pgs, (n_hosts-1)*n_drives/(replicas-1));
|
||||
const l = capacity/(degraded_replacement ? 1 : pgs)/speed/86400/365;
|
||||
return 1 - (1 - afr_drive * (1-(1-(afr_drive*l)**(replicas-1))**pgs)) ** (n_hosts*n_drives);
|
||||
}
|
||||
|
||||
function cluster_afr_pgs_ec({ n_hosts, n_drives, afr_drive, capacity, speed, ec: [ ec_data, ec_parity ], pgs = 1, degraded_replacement })
|
||||
{
|
||||
const ec_total = ec_data+ec_parity;
|
||||
pgs = Math.min(pgs, (n_hosts-1)*n_drives/(ec_total-1));
|
||||
const l = capacity/(degraded_replacement ? 1 : pgs)/speed/86400/365;
|
||||
return 1 - (1 - afr_drive * (1-(1-failure_rate_fullmesh(ec_total-1, afr_drive*l, ec_parity))**pgs)) ** (n_hosts*n_drives);
|
||||
}
|
||||
|
||||
// Same as above, but also take server failures into account
|
||||
function cluster_afr_pgs_hosts({ n_hosts, n_drives, afr_drive, afr_host, capacity, speed, replicas, pgs = 1, degraded_replacement })
|
||||
{
|
||||
let otherhosts = Math.min(pgs, (n_hosts-1)/(replicas-1));
|
||||
pgs = Math.min(pgs, (n_hosts-1)*n_drives/(replicas-1));
|
||||
let pgh = Math.min(pgs*n_drives, (n_hosts-1)*n_drives/(replicas-1));
|
||||
const ld = capacity/(degraded_replacement ? 1 : pgs)/speed/86400/365;
|
||||
const lh = n_drives*capacity/pgs/speed/86400/365;
|
||||
const p1 = ((afr_drive+afr_host*pgs/otherhosts)*lh);
|
||||
const p2 = ((afr_drive+afr_host*pgs/otherhosts)*ld);
|
||||
return 1 - ((1 - afr_host * (1-(1-p1**(replicas-1))**pgh)) ** n_hosts) *
|
||||
((1 - afr_drive * (1-(1-p2**(replicas-1))**pgs)) ** (n_hosts*n_drives));
|
||||
}
|
||||
|
||||
function cluster_afr_pgs_ec_hosts({ n_hosts, n_drives, afr_drive, afr_host, capacity, speed, ec: [ ec_data, ec_parity ], pgs = 1, degraded_replacement })
|
||||
{
|
||||
const ec_total = ec_data+ec_parity;
|
||||
const otherhosts = Math.min(pgs, (n_hosts-1)/(ec_total-1));
|
||||
pgs = Math.min(pgs, (n_hosts-1)*n_drives/(ec_total-1));
|
||||
const pgh = Math.min(pgs*n_drives, (n_hosts-1)*n_drives/(ec_total-1));
|
||||
const ld = capacity/(degraded_replacement ? 1 : pgs)/speed/86400/365;
|
||||
const lh = n_drives*capacity/pgs/speed/86400/365;
|
||||
const p1 = ((afr_drive+afr_host*pgs/otherhosts)*lh);
|
||||
const p2 = ((afr_drive+afr_host*pgs/otherhosts)*ld);
|
||||
return 1 - ((1 - afr_host * (1-(1-failure_rate_fullmesh(ec_total-1, p1, ec_parity))**pgh)) ** n_hosts) *
|
||||
((1 - afr_drive * (1-(1-failure_rate_fullmesh(ec_total-1, p2, ec_parity))**pgs)) ** (n_hosts*n_drives));
|
||||
}
|
||||
|
||||
// Wrapper for 4 above functions
|
||||
function cluster_afr(config)
|
||||
{
|
||||
if (config.ec && config.afr_host)
|
||||
{
|
||||
return cluster_afr_pgs_ec_hosts(config);
|
||||
}
|
||||
else if (config.ec)
|
||||
{
|
||||
return cluster_afr_pgs_ec(config);
|
||||
}
|
||||
else if (config.afr_host)
|
||||
{
|
||||
return cluster_afr_pgs_hosts(config);
|
||||
}
|
||||
else
|
||||
{
|
||||
return cluster_afr_pgs(config);
|
||||
}
|
||||
}
|
||||
|
||||
function print_cluster_afr(config)
|
||||
{
|
||||
console.log(
|
||||
`${config.n_hosts} nodes with ${config.n_drives} ${sprintf("%.1f", config.capacity/1000)}TB drives`+
|
||||
`, capable to backfill at ${sprintf("%.1f", config.speed*1000)} MB/s, drive AFR ${sprintf("%.1f", config.afr_drive*100)}%`+
|
||||
(config.afr_host ? `, host AFR ${sprintf("%.1f", config.afr_host*100)}%` : '')+
|
||||
(config.ec ? `, EC ${config.ec[0]}+${config.ec[1]}` : `, ${config.replicas} replicas`)+
|
||||
`, ${config.pgs||1} PG per OSD`+
|
||||
(config.degraded_replacement ? `\n...and you don't let the rebalance finish before replacing drives` : '')
|
||||
);
|
||||
console.log('-> '+sprintf("%.7f%%", 100*cluster_afr(config))+'\n');
|
||||
const pg_size = (ec ? ec_data+ec_parity : replicas);
|
||||
pgs = Math.min(pgs, (n_hosts-1)*n_drives/(pg_size-1));
|
||||
const host_pgs = Math.min(pgs*n_drives, (n_hosts-1)*n_drives/(pg_size-1));
|
||||
const resilver_disk = n_drives == 1 || osd_rm ? pgs : (n_drives-1);
|
||||
const disk_heal_time = (down_out_interval + capacity/(degraded_replacement ? 1 : resilver_disk)/speed)/86400/365;
|
||||
const host_heal_time = (down_out_interval + n_drives*capacity/pgs/speed)/86400/365;
|
||||
const disk_heal_fail = ((afr_drive+afr_host/n_drives)*disk_heal_time);
|
||||
const host_heal_fail = ((afr_drive+afr_host/n_drives)*host_heal_time);
|
||||
const disk_pg_fail = ec
|
||||
? failure_rate_fullmesh(ec_data+ec_parity-1, disk_heal_fail, ec_parity)
|
||||
: disk_heal_fail**(replicas-1);
|
||||
const host_pg_fail = ec
|
||||
? failure_rate_fullmesh(ec_data+ec_parity-1, host_heal_fail, ec_parity)
|
||||
: host_heal_fail**(replicas-1);
|
||||
return 1 - ((1 - afr_drive * (1-(1-disk_pg_fail)**pgs)) ** (n_hosts*n_drives))
|
||||
* ((1 - afr_host * (1-(1-host_pg_fail)**host_pgs)) ** n_hosts);
|
||||
}
|
||||
|
||||
/******** UTILITY ********/
|
||||
|
|
|
@ -0,0 +1,28 @@
|
|||
const { sprintf } = require('sprintf-js');
|
||||
const { cluster_afr } = require('./afr.js');
|
||||
|
||||
print_cluster_afr({ n_hosts: 4, n_drives: 6, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
|
||||
print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0, capacity: 4000, speed: 0.1, replicas: 2 });
|
||||
print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, replicas: 2 });
|
||||
print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0, capacity: 4000, speed: 0.1, ec: true, ec_data: 2, ec_parity: 1 });
|
||||
print_cluster_afr({ n_hosts: 4, n_drives: 3, afr_drive: 0.03, afr_host: 0.05, capacity: 4000, speed: 0.1, ec: true, ec_data: 2, ec_parity: 1 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0, capacity: 8000, speed: 0.02, replicas: 2 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 2 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0, capacity: 8000, speed: 0.02, replicas: 3 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100 });
|
||||
print_cluster_afr({ n_hosts: 10, n_drives: 10, afr_drive: 0.1, afr_host: 0.05, capacity: 8000, speed: 0.02, replicas: 3, pgs: 100, degraded_replacement: 1 });
|
||||
|
||||
function print_cluster_afr(config)
|
||||
{
|
||||
console.log(
|
||||
`${config.n_hosts} nodes with ${config.n_drives} ${sprintf("%.1f", config.capacity/1000)}TB drives`+
|
||||
`, capable to backfill at ${sprintf("%.1f", config.speed*1000)} MB/s, drive AFR ${sprintf("%.1f", config.afr_drive*100)}%`+
|
||||
(config.afr_host ? `, host AFR ${sprintf("%.1f", config.afr_host*100)}%` : '')+
|
||||
(config.ec ? `, EC ${config.ec_data}+${config.ec_parity}` : `, ${config.replicas} replicas`)+
|
||||
`, ${config.pgs||1} PG per OSD`+
|
||||
(config.degraded_replacement ? `\n...and you don't let the rebalance finish before replacing drives` : '')
|
||||
);
|
||||
console.log('-> '+sprintf("%.7f%%", 100*cluster_afr(config))+'\n');
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
// Data distribution optimizer using linear programming (lp_solve)
|
||||
|
||||
|
@ -58,7 +58,7 @@ async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize =
|
|||
}
|
||||
const all_weights = Object.assign({}, ...Object.values(osd_tree));
|
||||
const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c), 0);
|
||||
const all_pgs = Object.values(random_combinations(osd_tree, pg_size, max_combinations));
|
||||
const all_pgs = Object.values(random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1));
|
||||
const pg_per_osd = {};
|
||||
for (const pg of all_pgs)
|
||||
{
|
||||
|
@ -249,7 +249,7 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
|
|||
}
|
||||
}
|
||||
// Get all combinations
|
||||
let all_pgs = random_combinations(osd_tree, pg_size, max_combinations);
|
||||
let all_pgs = random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1);
|
||||
add_valid_previous(osd_tree, prev_weights, all_pgs);
|
||||
all_pgs = Object.values(all_pgs);
|
||||
const pg_per_osd = {};
|
||||
|
@ -275,6 +275,11 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
|
|||
lp += 'max: '+all_pg_names.map(pg_name => (
|
||||
prev_weights[pg_name] ? `${pg_size+1}*add_${pg_name} - ${pg_size+1}*del_${pg_name}` : `${pg_size+1-move_weights[pg_name]}*${pg_name}`
|
||||
)).join(' + ')+';\n';
|
||||
lp += all_pg_names
|
||||
.map(pg_name => (prev_weights[pg_name] ? `add_${pg_name} - del_${pg_name}` : `${pg_name}`))
|
||||
.join(' + ')+' = '+(pg_count
|
||||
- Object.keys(prev_weights).reduce((a, old_pg_name) => (a + (all_pgs_hash[old_pg_name] ? prev_weights[old_pg_name] : 0)), 0)
|
||||
)+';\n';
|
||||
for (const osd in pg_per_osd)
|
||||
{
|
||||
if (osd !== NO_OSD)
|
||||
|
@ -488,7 +493,8 @@ function extract_osds(osd_tree, levels, osd_level, osds = {})
|
|||
return osds;
|
||||
}
|
||||
|
||||
function random_combinations(osd_tree, pg_size, count)
|
||||
// ordered = don't treat (x,y) and (y,x) as equal
|
||||
function random_combinations(osd_tree, pg_size, count, ordered)
|
||||
{
|
||||
let seed = 0x5f020e43;
|
||||
let rng = () =>
|
||||
|
@ -516,25 +522,47 @@ function random_combinations(osd_tree, pg_size, count)
|
|||
pg.push(osds[cur_hosts[next_host]][next_osd]);
|
||||
cur_hosts.splice(next_host, 1);
|
||||
}
|
||||
while (pg.length < pg_size)
|
||||
const cyclic_pgs = [ pg ];
|
||||
if (ordered)
|
||||
{
|
||||
pg.push(NO_OSD);
|
||||
for (let i = 1; i < pg.size; i++)
|
||||
{
|
||||
cyclic_pgs.push([ ...pg.slice(i), ...pg.slice(0, i) ]);
|
||||
}
|
||||
}
|
||||
for (const pg of cyclic_pgs)
|
||||
{
|
||||
while (pg.length < pg_size)
|
||||
{
|
||||
pg.push(NO_OSD);
|
||||
}
|
||||
r['pg_'+pg.join('_')] = pg;
|
||||
}
|
||||
r['pg_'+pg.join('_')] = pg;
|
||||
}
|
||||
}
|
||||
// Generate purely random combinations
|
||||
restart: while (count > 0)
|
||||
while (count > 0)
|
||||
{
|
||||
let host_idx = [];
|
||||
for (let i = 0; i < pg_size && i < hosts.length; i++)
|
||||
const cur_hosts = [ ...hosts.map((h, i) => i) ];
|
||||
const max_hosts = pg_size < hosts.length ? pg_size : hosts.length;
|
||||
if (ordered)
|
||||
{
|
||||
let start = i > 0 ? host_idx[i-1]+1 : 0;
|
||||
if (start >= hosts.length)
|
||||
for (let i = 0; i < max_hosts; i++)
|
||||
{
|
||||
continue restart;
|
||||
const r = rng() % cur_hosts.length;
|
||||
host_idx[i] = cur_hosts[r];
|
||||
cur_hosts.splice(r, 1);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (let i = 0; i < max_hosts; i++)
|
||||
{
|
||||
const r = rng() % (cur_hosts.length - (max_hosts - i - 1));
|
||||
host_idx[i] = cur_hosts[r];
|
||||
cur_hosts.splice(0, r+1);
|
||||
}
|
||||
host_idx[i] = start + rng() % (hosts.length-start);
|
||||
}
|
||||
let pg = host_idx.map(h => osds[hosts[h]][rng() % osds[hosts[h]].length]);
|
||||
while (pg.length < pg_size)
|
||||
|
|
|
@ -0,0 +1,76 @@
|
|||
#!/bin/bash
|
||||
# Very simple systemd unit generator for vitastor-osd services
|
||||
# Not the final solution yet, mostly for tests
|
||||
# Copyright (c) Vitaliy Filippov, 2019+
|
||||
# License: MIT
|
||||
|
||||
# USAGE: ./make-osd.sh /dev/disk/by-partuuid/xxx [ /dev/disk/by-partuuid/yyy]...
|
||||
|
||||
IP_SUBSTR="10.200.1."
|
||||
ETCD_HOSTS="etcd0=http://10.200.1.10:2380,etcd1=http://10.200.1.11:2380,etcd2=http://10.200.1.12:2380"
|
||||
|
||||
set -e -x
|
||||
|
||||
IP=`ip -json a s | jq -r '.[].addr_info[] | select(.local | startswith("'$IP_SUBSTR'")) | .local'`
|
||||
[ "$IP" != "" ] || exit 1
|
||||
ETCD_MON=$(echo $ETCD_HOSTS | perl -pe 's/:2380/:2379/g; s/etcd\d*=//g;')
|
||||
D=`dirname $0`
|
||||
|
||||
# Create OSDs on all passed devices
|
||||
OSD_NUM=1
|
||||
for DEV in $*; do
|
||||
|
||||
# Ugly :) -> node.js rework pending
|
||||
while true; do
|
||||
ST=$(etcdctl --endpoints="$ETCD_MON" get --print-value-only /vitastor/osd/stats/$OSD_NUM)
|
||||
if [ "$ST" = "" ]; then
|
||||
break
|
||||
fi
|
||||
OSD_NUM=$((OSD_NUM+1))
|
||||
done
|
||||
etcdctl --endpoints="$ETCD_MON" put /vitastor/osd/stats/$OSD_NUM '{}'
|
||||
|
||||
echo Creating OSD $OSD_NUM on $DEV
|
||||
|
||||
OPT=`node $D/simple-offsets.js --device $DEV --format options | tr '\n' ' '`
|
||||
META=`echo $OPT | grep -Po '(?<=data_offset )\d+'`
|
||||
dd if=/dev/zero of=$DEV bs=1048576 count=$(((META+1048575)/1048576)) oflag=direct
|
||||
|
||||
cat >/etc/systemd/system/vitastor-osd$OSD_NUM.service <<EOF
|
||||
[Unit]
|
||||
Description=Vitastor object storage daemon osd.$OSD_NUM
|
||||
After=network-online.target local-fs.target time-sync.target
|
||||
Wants=network-online.target local-fs.target time-sync.target
|
||||
PartOf=vitastor.target
|
||||
|
||||
[Service]
|
||||
LimitNOFILE=1048576
|
||||
LimitNPROC=1048576
|
||||
LimitMEMLOCK=infinity
|
||||
ExecStart=/usr/bin/vitastor-osd \\
|
||||
--etcd_address $IP:2379/v3 \\
|
||||
--bind_address $IP \\
|
||||
--osd_num $OSD_NUM \\
|
||||
--disable_data_fsync 1 \\
|
||||
--immediate_commit all \\
|
||||
--flusher_count 256 \\
|
||||
--disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
|
||||
--journal_no_same_sector_overwrites true \\
|
||||
--journal_sector_buffer_count 1024 \\
|
||||
$OPT
|
||||
WorkingDirectory=/
|
||||
ExecStartPre=+chown vitastor:vitastor $DEV
|
||||
User=vitastor
|
||||
PrivateTmp=false
|
||||
TasksMax=infinity
|
||||
Restart=always
|
||||
StartLimitInterval=0
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=vitastor.target
|
||||
EOF
|
||||
|
||||
systemctl enable vitastor-osd$OSD_NUM
|
||||
|
||||
done
|
|
@ -1,19 +1,25 @@
|
|||
#!/bin/bash
|
||||
# Example startup script generator
|
||||
# Of course this isn't a production solution yet, this is just for tests
|
||||
# Very simple systemd unit generator for etcd & vitastor-mon services
|
||||
# Not the final solution yet, mostly for tests
|
||||
# Copyright (c) Vitaliy Filippov, 2019+
|
||||
# License: MIT
|
||||
|
||||
IP=`ip -json a s | jq -r '.[].addr_info[] | select(.broadcast == "10.115.0.255") | .local'`
|
||||
# USAGE: ./make-units.sh
|
||||
|
||||
IP_SUBSTR="10.200.1."
|
||||
ETCD_HOSTS="etcd0=http://10.200.1.10:2380,etcd1=http://10.200.1.11:2380,etcd2=http://10.200.1.12:2380"
|
||||
|
||||
# determine IP
|
||||
IP=`ip -json a s | jq -r '.[].addr_info[] | select(.local | startswith("'$IP_SUBSTR'")) | .local'`
|
||||
[ "$IP" != "" ] || exit 1
|
||||
ETCD_NUM=${ETCD_HOSTS/$IP*/}
|
||||
[ "$ETCD_NUM" != "$ETCD_HOSTS" ] || exit 1
|
||||
ETCD_NUM=$(echo $ETCD_NUM | tr -d -c , | wc -c)
|
||||
|
||||
BASE=${IP/*./}
|
||||
BASE=$((BASE-10))
|
||||
|
||||
# etcd
|
||||
useradd etcd
|
||||
|
||||
mkdir -p /var/lib/etcd$BASE.etcd
|
||||
mkdir -p /var/lib/etcd$ETCD_NUM.etcd
|
||||
cat >/etc/systemd/system/etcd.service <<EOF
|
||||
[Unit]
|
||||
Description=etcd for vitastor
|
||||
|
@ -22,19 +28,18 @@ Wants=network-online.target local-fs.target time-sync.target
|
|||
|
||||
[Service]
|
||||
Restart=always
|
||||
ExecStart=/usr/local/bin/etcd -name etcd$BASE --data-dir /var/lib/etcd$BASE.etcd \\
|
||||
ExecStart=/usr/local/bin/etcd -name etcd$ETCD_NUM --data-dir /var/lib/etcd$ETCD_NUM.etcd \\
|
||||
--advertise-client-urls http://$IP:2379 --listen-client-urls http://$IP:2379 \\
|
||||
--initial-advertise-peer-urls http://$IP:2380 --listen-peer-urls http://$IP:2380 \\
|
||||
--initial-cluster-token vitastor-etcd-1 --initial-cluster etcd0=http://10.115.0.10:2380,etcd1=http://10.115.0.11:2380,etcd2=http://10.115.0.12:2380,etcd3=http://10.115.0.13:2380 \\
|
||||
--initial-cluster-token vitastor-etcd-1 --initial-cluster $ETCD_HOSTS \\
|
||||
--initial-cluster-state new --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision
|
||||
WorkingDirectory=/var/lib/etcd$BASE.etcd
|
||||
ExecStartPre=+chown -R etcd /var/lib/etcd$BASE.etcd
|
||||
WorkingDirectory=/var/lib/etcd$ETCD_NUM.etcd
|
||||
ExecStartPre=+chown -R etcd /var/lib/etcd$ETCD_NUM.etcd
|
||||
User=etcd
|
||||
PrivateTmp=false
|
||||
TasksMax=infinity
|
||||
Restart=always
|
||||
StartLimitInterval=0
|
||||
StartLimitIntervalSec=0
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
|
@ -48,9 +53,7 @@ systemctl start etcd
|
|||
useradd vitastor
|
||||
chmod 755 /root
|
||||
|
||||
BASE=${IP/*./}
|
||||
BASE=$(((BASE-10)*12))
|
||||
|
||||
# Vitastor target
|
||||
cat >/etc/systemd/system/vitastor.target <<EOF
|
||||
[Unit]
|
||||
Description=vitastor target
|
||||
|
@ -58,116 +61,25 @@ Description=vitastor target
|
|||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
i=1
|
||||
for DEV in `ls /dev/disk/by-id/ | grep ata-INTEL_SSDSC2KB`; do
|
||||
dd if=/dev/zero of=/dev/disk/by-id/$DEV bs=1048576 count=$(((427814912+1048575)/1048576+2))
|
||||
dd if=/dev/zero of=/dev/disk/by-id/$DEV bs=1048576 count=$(((427814912+1048575)/1048576+2)) seek=$((1920377991168/1048576))
|
||||
cat >/etc/systemd/system/vitastor-osd$((BASE+i)).service <<EOF
|
||||
# Monitor unit
|
||||
ETCD_MON=$(echo $ETCD_HOSTS | perl -pe 's/:2380/:2379/g; s/etcd\d*=//g;')
|
||||
cat >/etc/systemd/system/vitastor-mon.service <<EOF
|
||||
[Unit]
|
||||
Description=Vitastor object storage daemon osd.$((BASE+i))
|
||||
Description=Vitastor monitor
|
||||
After=network-online.target local-fs.target time-sync.target
|
||||
Wants=network-online.target local-fs.target time-sync.target
|
||||
PartOf=vitastor.target
|
||||
|
||||
[Service]
|
||||
LimitNOFILE=1048576
|
||||
LimitNPROC=1048576
|
||||
LimitMEMLOCK=infinity
|
||||
ExecStart=/root/vitastor/osd \\
|
||||
--etcd_address $IP:2379/v3 \\
|
||||
--bind_address $IP \\
|
||||
--osd_num $((BASE+i)) \\
|
||||
--disable_data_fsync 1 \\
|
||||
--disable_device_lock 1 \\
|
||||
--immediate_commit all \\
|
||||
--flusher_count 8 \\
|
||||
--disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
|
||||
--journal_no_same_sector_overwrites true \\
|
||||
--journal_sector_buffer_count 1024 \\
|
||||
--journal_offset 0 \\
|
||||
--meta_offset 16777216 \\
|
||||
--data_offset 427814912 \\
|
||||
--data_size $((1920377991168-427814912)) \\
|
||||
--data_device /dev/disk/by-id/$DEV
|
||||
WorkingDirectory=/root/vitastor
|
||||
ExecStartPre=+chown vitastor:vitastor /dev/disk/by-id/$DEV
|
||||
Restart=always
|
||||
ExecStart=node /usr/lib/vitastor/mon/mon-main.js --etcd_url '$ETCD_MON' --etcd_prefix '/vitastor' --etcd_start_timeout 5
|
||||
WorkingDirectory=/
|
||||
User=vitastor
|
||||
PrivateTmp=false
|
||||
TasksMax=infinity
|
||||
Restart=always
|
||||
StartLimitInterval=0
|
||||
StartLimitIntervalSec=0
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=vitastor.target
|
||||
EOF
|
||||
systemctl enable vitastor-osd$((BASE+i))
|
||||
i=$((i+1))
|
||||
cat >/etc/systemd/system/vitastor-osd$((BASE+i)).service <<EOF
|
||||
[Unit]
|
||||
Description=Vitastor object storage daemon osd.$((BASE+i))
|
||||
After=network-online.target local-fs.target time-sync.target
|
||||
Wants=network-online.target local-fs.target time-sync.target
|
||||
PartOf=vitastor.target
|
||||
|
||||
[Service]
|
||||
LimitNOFILE=1048576
|
||||
LimitNPROC=1048576
|
||||
LimitMEMLOCK=infinity
|
||||
ExecStart=/root/vitastor/osd \\
|
||||
--etcd_address $IP:2379/v3 \\
|
||||
--bind_address $IP \\
|
||||
--osd_num $((BASE+i)) \\
|
||||
--disable_data_fsync 1 \\
|
||||
--immediate_commit all \\
|
||||
--flusher_count 8 \\
|
||||
--disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
|
||||
--journal_no_same_sector_overwrites true \\
|
||||
--journal_sector_buffer_count 1024 \\
|
||||
--journal_offset 1920377991168 \\
|
||||
--meta_offset $((1920377991168+16777216)) \\
|
||||
--data_offset $((1920377991168+427814912)) \\
|
||||
--data_size $((1920377991168-427814912)) \\
|
||||
--data_device /dev/disk/by-id/$DEV
|
||||
WorkingDirectory=/root/vitastor
|
||||
ExecStartPre=+chown vitastor:vitastor /dev/disk/by-id/$DEV
|
||||
User=vitastor
|
||||
PrivateTmp=false
|
||||
TasksMax=infinity
|
||||
Restart=always
|
||||
StartLimitInterval=0
|
||||
StartLimitIntervalSec=0
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=vitastor.target
|
||||
EOF
|
||||
systemctl enable vitastor-osd$((BASE+i))
|
||||
i=$((i+1))
|
||||
done
|
||||
|
||||
exit
|
||||
|
||||
node mon-main.js --etcd_url 'http://10.115.0.10:2379,http://10.115.0.11:2379,http://10.115.0.12:2379,http://10.115.0.13:2379' --etcd_prefix '/vitastor' --etcd_start_timeout 5
|
||||
|
||||
podman run -d --network host --restart always -v /var/lib/etcd0.etcd:/etcd0.etcd --name etcd quay.io/coreos/etcd:v3.4.13 etcd -name etcd0 \
|
||||
-advertise-client-urls http://10.115.0.10:2379 -listen-client-urls http://10.115.0.10:2379 \
|
||||
-initial-advertise-peer-urls http://10.115.0.10:2380 -listen-peer-urls http://10.115.0.10:2380 \
|
||||
-initial-cluster-token vitastor-etcd-1 -initial-cluster etcd0=http://10.115.0.10:2380,etcd1=http://10.115.0.11:2380,etcd2=http://10.115.0.12:2380,etcd3=http://10.115.0.13:2380 \
|
||||
-initial-cluster-state new --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision
|
||||
|
||||
etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/global '{"immediate_commit":"all"}'
|
||||
|
||||
etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":48,"failure_domain":"host"}}'
|
||||
|
||||
#let pgs = {};
|
||||
#for (let n = 0; n < 48; n++) { let i = n/2 | 0; pgs[1+n] = { osd_set: [ (1+i%12+(i/12 | 0)*24), (1+12+i%12+(i/12 | 0)*24) ], primary: (1+(n%2)*12+i%12+(i/12 | 0)*24) }; };
|
||||
#console.log(JSON.stringify({ items: { 1: pgs } }));
|
||||
#etcdctl --endpoints http://10.115.0.10:2379 put /vitastor/config/pgs ...
|
||||
|
||||
# --disk_alignment 4096 --journal_block_size 4096 --meta_block_size 4096 \\
|
||||
# --data_offset 427814912 \\
|
||||
|
||||
# --disk_alignment 4096 --journal_block_size 512 --meta_block_size 512 \\
|
||||
# --data_offset 433434624 \\
|
||||
|
|
|
@ -0,0 +1,23 @@
|
|||
const fsp = require('fs').promises;
|
||||
|
||||
async function merge(file1, file2, out)
|
||||
{
|
||||
if (!out)
|
||||
{
|
||||
console.error('USAGE: nodejs merge.js layer1 layer2 output');
|
||||
process.exit();
|
||||
}
|
||||
const layer1 = await fsp.readFile(file1);
|
||||
const layer2 = await fsp.readFile(file2);
|
||||
const zero = Buffer.alloc(4096);
|
||||
for (let i = 0; i < layer2.length; i += 4096)
|
||||
{
|
||||
if (zero.compare(layer2, i, i+4096) != 0)
|
||||
{
|
||||
layer2.copy(layer1, i, i, i+4096);
|
||||
}
|
||||
}
|
||||
await fsp.writeFile(out, layer1);
|
||||
}
|
||||
|
||||
merge(process.argv[2], process.argv[3], process.argv[4]);
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/node
|
||||
|
||||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
const Mon = require('./mon.js');
|
||||
|
||||
|
|
940
mon/mon.js
940
mon/mon.js
File diff suppressed because it is too large
Load Diff
|
@ -4,6 +4,7 @@
|
|||
// Simple tool to calculate journal and metadata offsets for a single device
|
||||
// Will be replaced by smarter tools in the future
|
||||
|
||||
const fs = require('fs').promises;
|
||||
const child_process = require('child_process');
|
||||
|
||||
async function run()
|
||||
|
@ -15,6 +16,7 @@ async function run()
|
|||
device_block_size: 4096,
|
||||
journal_offset: 0,
|
||||
device_size: 0,
|
||||
format: 'text',
|
||||
};
|
||||
for (let i = 2; i < process.argv.length; i++)
|
||||
{
|
||||
|
@ -24,7 +26,22 @@ async function run()
|
|||
i++;
|
||||
}
|
||||
}
|
||||
const device_size = Number(options.device_size || await system("blockdev --getsize64 "+options.device));
|
||||
if (!options.device)
|
||||
{
|
||||
process.stderr.write('USAGE: nodejs '+process.argv[1]+' --device /dev/sdXXX\n');
|
||||
process.exit(1);
|
||||
}
|
||||
options.device_size = Number(options.device_size);
|
||||
let device_size = options.device_size;
|
||||
if (!device_size)
|
||||
{
|
||||
const st = await fs.stat(options.device);
|
||||
options.device_block_size = st.blksize;
|
||||
if (st.isBlockDevice())
|
||||
device_size = Number(await system("/sbin/blockdev --getsize64 "+options.device))
|
||||
else
|
||||
device_size = st.size;
|
||||
}
|
||||
if (!device_size)
|
||||
{
|
||||
process.stderr.write('Failed to get device size\n');
|
||||
|
@ -32,25 +49,45 @@ async function run()
|
|||
}
|
||||
options.journal_offset = Math.ceil(options.journal_offset/options.device_block_size)*options.device_block_size;
|
||||
const meta_offset = options.journal_offset + Math.ceil(options.journal_size/options.device_block_size)*options.device_block_size;
|
||||
const entries_per_block = Math.floor(options.device_block_size / (24 + options.object_size/options.bitmap_granularity/8));
|
||||
const entries_per_block = Math.floor(options.device_block_size / (24 + 2*options.object_size/options.bitmap_granularity/8));
|
||||
const object_count = Math.floor((device_size-meta_offset)/options.object_size);
|
||||
const meta_size = Math.ceil(object_count / entries_per_block) * options.device_block_size;
|
||||
const data_offset = meta_offset + meta_size;
|
||||
const meta_size_fmt = (meta_size > 1024*1024*1024 ? Math.round(meta_size/1024/1024/1024*100)/100+" GB"
|
||||
: Math.round(meta_size/1024/1024*100)/100+" MB");
|
||||
process.stdout.write(
|
||||
`Metadata size: ${meta_size_fmt}\n`+
|
||||
`Options for the OSD:\n`+
|
||||
` --journal_offset ${options.journal_offset}\n`+
|
||||
` --meta_offset ${meta_offset}\n`+
|
||||
` --data_offset ${data_offset}\n`+
|
||||
(options.device_size ? ` --data_size ${device_size-data_offset}\n` : '')
|
||||
);
|
||||
if (options.format == 'text' || options.format == 'options')
|
||||
{
|
||||
if (options.format == 'text')
|
||||
{
|
||||
process.stderr.write(
|
||||
`Metadata size: ${meta_size_fmt}\n`+
|
||||
`Options for the OSD:\n`
|
||||
);
|
||||
}
|
||||
process.stdout.write(
|
||||
` --data_device ${options.device}\n`+
|
||||
` --journal_offset ${options.journal_offset}\n`+
|
||||
` --meta_offset ${meta_offset}\n`+
|
||||
` --data_offset ${data_offset}\n`+
|
||||
(options.device_size ? ` --data_size ${device_size-data_offset}\n` : '')
|
||||
);
|
||||
}
|
||||
else if (options.format == 'env')
|
||||
{
|
||||
process.stdout.write(
|
||||
`journal_offset=${options.journal_offset}\n`+
|
||||
`meta_offset=${meta_offset}\n`+
|
||||
`data_offset=${data_offset}\n`+
|
||||
`data_size=${device_size-data_offset}\n`
|
||||
);
|
||||
}
|
||||
else
|
||||
process.stdout.write('Unknown format: '+options.format);
|
||||
}
|
||||
|
||||
function system(cmd)
|
||||
{
|
||||
return new Promise((ok, no) => child_process.exec(cmd, { maxBuffer: 64*1024*1024 }, (err, stdout, stderr) => (err ? no(err) : ok(stdout))));
|
||||
return new Promise((ok, no) => child_process.exec(cmd, { maxBuffer: 64*1024*1024 }, (err, stdout, stderr) => (err ? no(err.message) : ok(stdout))));
|
||||
}
|
||||
|
||||
run().catch(console.error);
|
||||
run().catch(err => { console.error(err); process.exit(1); });
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
// Interesting real-world example coming from Ceph with EC and compression enabled.
|
||||
// EC parity chunks can't be compressed as efficiently as data chunks,
|
||||
|
|
|
@ -0,0 +1,25 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
const LPOptimizer = require('./lp-optimizer.js');
|
||||
|
||||
async function run()
|
||||
{
|
||||
const osd_tree = { a: { 1: 1 }, b: { 2: 1 }, c: { 3: 1 } };
|
||||
let res;
|
||||
|
||||
console.log('16 PGs, size=3');
|
||||
res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 3, pg_count: 16 });
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
|
||||
console.log('\nReduce PG size to 2');
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs.map(pg => pg.slice(0, 2)), osd_tree, pg_size: 2 });
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
|
||||
console.log('\nRemove OSD 3');
|
||||
delete osd_tree['c'];
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2 });
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
}
|
||||
|
||||
run().catch(console.error);
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
const LPOptimizer = require('./lp-optimizer.js');
|
||||
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
const LPOptimizer = require('./lp-optimizer.js');
|
||||
|
||||
|
|
|
@ -0,0 +1,84 @@
|
|||
Index: qemu-3.1+dfsg/qapi/block-core.json
|
||||
===================================================================
|
||||
--- qemu-3.1+dfsg.orig/qapi/block-core.json
|
||||
+++ qemu-3.1+dfsg/qapi/block-core.json
|
||||
@@ -2617,7 +2617,7 @@
|
||||
##
|
||||
{ 'enum': 'BlockdevDriver',
|
||||
'data': [ 'blkdebug', 'blklogwrites', 'blkverify', 'bochs', 'cloop',
|
||||
- 'copy-on-read', 'dmg', 'file', 'ftp', 'ftps', 'gluster',
|
||||
+ 'copy-on-read', 'dmg', 'file', 'ftp', 'ftps', 'gluster', 'vitastor',
|
||||
'host_cdrom', 'host_device', 'http', 'https', 'iscsi', 'luks',
|
||||
'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels', 'qcow',
|
||||
'qcow2', 'qed', 'quorum', 'raw', 'rbd', 'replication', 'sheepdog',
|
||||
@@ -3367,6 +3367,24 @@
|
||||
'*tag': 'str' } }
|
||||
|
||||
##
|
||||
+# @BlockdevOptionsVitastor:
|
||||
+#
|
||||
+# Driver specific block device options for vitastor
|
||||
+#
|
||||
+# @inode: Inode number
|
||||
+# @pool: Pool ID
|
||||
+# @size: Desired image size in bytes
|
||||
+# @etcd_host: etcd connection address
|
||||
+# @etcd_prefix: etcd key/value prefix
|
||||
+##
|
||||
+{ 'struct': 'BlockdevOptionsVitastor',
|
||||
+ 'data': { 'inode': 'uint64',
|
||||
+ 'pool': 'uint64',
|
||||
+ 'size': 'uint64',
|
||||
+ 'etcd_host': 'str',
|
||||
+ '*etcd_prefix': 'str' } }
|
||||
+
|
||||
+##
|
||||
# @ReplicationMode:
|
||||
#
|
||||
# An enumeration of replication modes.
|
||||
@@ -3713,6 +3731,7 @@
|
||||
'rbd': 'BlockdevOptionsRbd',
|
||||
'replication':'BlockdevOptionsReplication',
|
||||
'sheepdog': 'BlockdevOptionsSheepdog',
|
||||
+ 'vitastor': 'BlockdevOptionsVitastor',
|
||||
'ssh': 'BlockdevOptionsSsh',
|
||||
'throttle': 'BlockdevOptionsThrottle',
|
||||
'vdi': 'BlockdevOptionsGenericFormat',
|
||||
@@ -4158,6 +4177,17 @@
|
||||
'*block-state-zero': 'bool' } }
|
||||
|
||||
##
|
||||
+# @BlockdevCreateOptionsVitastor:
|
||||
+#
|
||||
+# Driver specific image creation options for Vitastor.
|
||||
+#
|
||||
+# @size: Size of the virtual disk in bytes
|
||||
+##
|
||||
+{ 'struct': 'BlockdevCreateOptionsVitastor',
|
||||
+ 'data': { 'location': 'BlockdevOptionsVitastor',
|
||||
+ 'size': 'size' } }
|
||||
+
|
||||
+##
|
||||
# @BlockdevVpcSubformat:
|
||||
#
|
||||
# @dynamic: Growing image file
|
||||
@@ -4212,6 +4242,7 @@
|
||||
'qed': 'BlockdevCreateOptionsQed',
|
||||
'rbd': 'BlockdevCreateOptionsRbd',
|
||||
'sheepdog': 'BlockdevCreateOptionsSheepdog',
|
||||
+ 'vitastor': 'BlockdevCreateOptionsVitastor',
|
||||
'ssh': 'BlockdevCreateOptionsSsh',
|
||||
'vdi': 'BlockdevCreateOptionsVdi',
|
||||
'vhdx': 'BlockdevCreateOptionsVhdx',
|
||||
Index: qemu-3.1+dfsg/scripts/modules/module_block.py
|
||||
===================================================================
|
||||
--- qemu-3.1+dfsg.orig/scripts/modules/module_block.py
|
||||
+++ qemu-3.1+dfsg/scripts/modules/module_block.py
|
||||
@@ -88,6 +88,7 @@ def print_bottom(fheader):
|
||||
output_file = sys.argv[1]
|
||||
with open(output_file, 'w') as fheader:
|
||||
print_top(fheader)
|
||||
+ add_module(fheader, "vitastor", "vitastor", "vitastor")
|
||||
|
||||
for filename in sys.argv[2:]:
|
||||
if os.path.isfile(filename):
|
|
@ -0,0 +1,84 @@
|
|||
Index: qemu/qapi/block-core.json
|
||||
===================================================================
|
||||
--- qemu.orig/qapi/block-core.json 2020-11-07 22:57:38.932613674 +0000
|
||||
+++ qemu.orig/qapi/block-core.json 2020-11-07 22:59:49.890722862 +0000
|
||||
@@ -2907,7 +2907,7 @@
|
||||
'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels', 'qcow',
|
||||
'qcow2', 'qed', 'quorum', 'raw', 'rbd',
|
||||
{ 'name': 'replication', 'if': 'defined(CONFIG_REPLICATION)' },
|
||||
- 'sheepdog',
|
||||
+ 'sheepdog', 'vitastor',
|
||||
'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat', 'vxhs' ] }
|
||||
|
||||
##
|
||||
@@ -3725,6 +3725,24 @@
|
||||
'*tag': 'str' } }
|
||||
|
||||
##
|
||||
+# @BlockdevOptionsVitastor:
|
||||
+#
|
||||
+# Driver specific block device options for vitastor
|
||||
+#
|
||||
+# @inode: Inode number
|
||||
+# @pool: Pool ID
|
||||
+# @size: Desired image size in bytes
|
||||
+# @etcd_host: etcd connection address
|
||||
+# @etcd_prefix: etcd key/value prefix
|
||||
+##
|
||||
+{ 'struct': 'BlockdevOptionsVitastor',
|
||||
+ 'data': { 'inode': 'uint64',
|
||||
+ 'pool': 'uint64',
|
||||
+ 'size': 'uint64',
|
||||
+ 'etcd_host': 'str',
|
||||
+ '*etcd_prefix': 'str' } }
|
||||
+
|
||||
+##
|
||||
# @ReplicationMode:
|
||||
#
|
||||
# An enumeration of replication modes.
|
||||
@@ -4084,6 +4102,7 @@
|
||||
'replication': { 'type': 'BlockdevOptionsReplication',
|
||||
'if': 'defined(CONFIG_REPLICATION)' },
|
||||
'sheepdog': 'BlockdevOptionsSheepdog',
|
||||
+ 'vitastor': 'BlockdevOptionsVitastor',
|
||||
'ssh': 'BlockdevOptionsSsh',
|
||||
'throttle': 'BlockdevOptionsThrottle',
|
||||
'vdi': 'BlockdevOptionsGenericFormat',
|
||||
@@ -4461,6 +4480,17 @@
|
||||
'*cluster-size' : 'size' } }
|
||||
|
||||
##
|
||||
+# @BlockdevCreateOptionsVitastor:
|
||||
+#
|
||||
+# Driver specific image creation options for Vitastor.
|
||||
+#
|
||||
+# @size: Size of the virtual disk in bytes
|
||||
+##
|
||||
+{ 'struct': 'BlockdevCreateOptionsVitastor',
|
||||
+ 'data': { 'location': 'BlockdevOptionsVitastor',
|
||||
+ 'size': 'size' } }
|
||||
+
|
||||
+##
|
||||
# @BlockdevVmdkSubformat:
|
||||
#
|
||||
# Subformat options for VMDK images
|
||||
@@ -4722,6 +4752,7 @@
|
||||
'qed': 'BlockdevCreateOptionsQed',
|
||||
'rbd': 'BlockdevCreateOptionsRbd',
|
||||
'sheepdog': 'BlockdevCreateOptionsSheepdog',
|
||||
+ 'vitastor': 'BlockdevCreateOptionsVitastor',
|
||||
'ssh': 'BlockdevCreateOptionsSsh',
|
||||
'vdi': 'BlockdevCreateOptionsVdi',
|
||||
'vhdx': 'BlockdevCreateOptionsVhdx',
|
||||
Index: qemu/scripts/modules/module_block.py
|
||||
===================================================================
|
||||
--- qemu.orig/scripts/modules/module_block.py 2020-11-07 22:57:38.936613739 +0000
|
||||
+++ qemu/scripts/modules/module_block.py 2020-11-07 22:59:49.890722862 +0000
|
||||
@@ -86,6 +86,7 @@ def print_bottom(fheader):
|
||||
output_file = sys.argv[1]
|
||||
with open(output_file, 'w') as fheader:
|
||||
print_top(fheader)
|
||||
+ add_module(fheader, "vitastor", "vitastor", "vitastor")
|
||||
|
||||
for filename in sys.argv[2:]:
|
||||
if os.path.isfile(filename):
|
|
@ -0,0 +1,84 @@
|
|||
Index: qemu/qapi/block-core.json
|
||||
===================================================================
|
||||
--- qemu.orig/qapi/block-core.json
|
||||
+++ qemu/qapi/block-core.json
|
||||
@@ -2798,7 +2798,7 @@
|
||||
'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
|
||||
'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
|
||||
{ 'name': 'replication', 'if': 'defined(CONFIG_REPLICATION)' },
|
||||
- 'sheepdog',
|
||||
+ 'sheepdog', 'vitastor',
|
||||
'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat', 'vxhs' ] }
|
||||
|
||||
##
|
||||
@@ -3635,6 +3635,24 @@
|
||||
'*tag': 'str' } }
|
||||
|
||||
##
|
||||
+# @BlockdevOptionsVitastor:
|
||||
+#
|
||||
+# Driver specific block device options for vitastor
|
||||
+#
|
||||
+# @inode: Inode number
|
||||
+# @pool: Pool ID
|
||||
+# @size: Desired image size in bytes
|
||||
+# @etcd_host: etcd connection address
|
||||
+# @etcd_prefix: etcd key/value prefix
|
||||
+##
|
||||
+{ 'struct': 'BlockdevOptionsVitastor',
|
||||
+ 'data': { 'inode': 'uint64',
|
||||
+ 'pool': 'uint64',
|
||||
+ 'size': 'uint64',
|
||||
+ 'etcd_host': 'str',
|
||||
+ '*etcd_prefix': 'str' } }
|
||||
+
|
||||
+##
|
||||
# @ReplicationMode:
|
||||
#
|
||||
# An enumeration of replication modes.
|
||||
@@ -3995,6 +4013,7 @@
|
||||
'replication': { 'type': 'BlockdevOptionsReplication',
|
||||
'if': 'defined(CONFIG_REPLICATION)' },
|
||||
'sheepdog': 'BlockdevOptionsSheepdog',
|
||||
+ 'vitastor': 'BlockdevOptionsVitastor',
|
||||
'ssh': 'BlockdevOptionsSsh',
|
||||
'throttle': 'BlockdevOptionsThrottle',
|
||||
'vdi': 'BlockdevOptionsGenericFormat',
|
||||
@@ -4365,6 +4384,17 @@
|
||||
'*cluster-size' : 'size' } }
|
||||
|
||||
##
|
||||
+# @BlockdevCreateOptionsVitastor:
|
||||
+#
|
||||
+# Driver specific image creation options for Vitastor.
|
||||
+#
|
||||
+# @size: Size of the virtual disk in bytes
|
||||
+##
|
||||
+{ 'struct': 'BlockdevCreateOptionsVitastor',
|
||||
+ 'data': { 'location': 'BlockdevOptionsVitastor',
|
||||
+ 'size': 'size' } }
|
||||
+
|
||||
+##
|
||||
# @BlockdevVmdkSubformat:
|
||||
#
|
||||
# Subformat options for VMDK images
|
||||
@@ -4626,6 +4656,7 @@
|
||||
'qed': 'BlockdevCreateOptionsQed',
|
||||
'rbd': 'BlockdevCreateOptionsRbd',
|
||||
'sheepdog': 'BlockdevCreateOptionsSheepdog',
|
||||
+ 'vitastor': 'BlockdevCreateOptionsVitastor',
|
||||
'ssh': 'BlockdevCreateOptionsSsh',
|
||||
'vdi': 'BlockdevCreateOptionsVdi',
|
||||
'vhdx': 'BlockdevCreateOptionsVhdx',
|
||||
Index: qemu/scripts/modules/module_block.py
|
||||
===================================================================
|
||||
--- qemu.orig/scripts/modules/module_block.py
|
||||
+++ qemu/scripts/modules/module_block.py
|
||||
@@ -85,6 +85,7 @@ def print_bottom(fheader):
|
||||
output_file = sys.argv[1]
|
||||
with open(output_file, 'w') as fheader:
|
||||
print_top(fheader)
|
||||
+ add_module(fheader, "vitastor", "vitastor", "vitastor")
|
||||
|
||||
for filename in sys.argv[2:]:
|
||||
if os.path.isfile(filename):
|
|
@ -0,0 +1,84 @@
|
|||
Index: qemu-5.1+dfsg/qapi/block-core.json
|
||||
===================================================================
|
||||
--- qemu-5.1+dfsg.orig/qapi/block-core.json
|
||||
+++ qemu-5.1+dfsg/qapi/block-core.json
|
||||
@@ -2807,7 +2807,7 @@
|
||||
'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
|
||||
'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
|
||||
{ 'name': 'replication', 'if': 'defined(CONFIG_REPLICATION)' },
|
||||
- 'sheepdog',
|
||||
+ 'sheepdog', 'vitastor',
|
||||
'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat' ] }
|
||||
|
||||
##
|
||||
@@ -3644,6 +3644,24 @@
|
||||
'*tag': 'str' } }
|
||||
|
||||
##
|
||||
+# @BlockdevOptionsVitastor:
|
||||
+#
|
||||
+# Driver specific block device options for vitastor
|
||||
+#
|
||||
+# @inode: Inode number
|
||||
+# @pool: Pool ID
|
||||
+# @size: Desired image size in bytes
|
||||
+# @etcd_host: etcd connection address
|
||||
+# @etcd_prefix: etcd key/value prefix
|
||||
+##
|
||||
+{ 'struct': 'BlockdevOptionsVitastor',
|
||||
+ 'data': { 'inode': 'uint64',
|
||||
+ 'pool': 'uint64',
|
||||
+ 'size': 'uint64',
|
||||
+ 'etcd_host': 'str',
|
||||
+ '*etcd_prefix': 'str' } }
|
||||
+
|
||||
+##
|
||||
# @ReplicationMode:
|
||||
#
|
||||
# An enumeration of replication modes.
|
||||
@@ -3988,6 +4006,7 @@
|
||||
'replication': { 'type': 'BlockdevOptionsReplication',
|
||||
'if': 'defined(CONFIG_REPLICATION)' },
|
||||
'sheepdog': 'BlockdevOptionsSheepdog',
|
||||
+ 'vitastor': 'BlockdevOptionsVitastor',
|
||||
'ssh': 'BlockdevOptionsSsh',
|
||||
'throttle': 'BlockdevOptionsThrottle',
|
||||
'vdi': 'BlockdevOptionsGenericFormat',
|
||||
@@ -4376,6 +4395,17 @@
|
||||
'*cluster-size' : 'size' } }
|
||||
|
||||
##
|
||||
+# @BlockdevCreateOptionsVitastor:
|
||||
+#
|
||||
+# Driver specific image creation options for Vitastor.
|
||||
+#
|
||||
+# @size: Size of the virtual disk in bytes
|
||||
+##
|
||||
+{ 'struct': 'BlockdevCreateOptionsVitastor',
|
||||
+ 'data': { 'location': 'BlockdevOptionsVitastor',
|
||||
+ 'size': 'size' } }
|
||||
+
|
||||
+##
|
||||
# @BlockdevVmdkSubformat:
|
||||
#
|
||||
# Subformat options for VMDK images
|
||||
@@ -4637,6 +4667,7 @@
|
||||
'qed': 'BlockdevCreateOptionsQed',
|
||||
'rbd': 'BlockdevCreateOptionsRbd',
|
||||
'sheepdog': 'BlockdevCreateOptionsSheepdog',
|
||||
+ 'vitastor': 'BlockdevCreateOptionsVitastor',
|
||||
'ssh': 'BlockdevCreateOptionsSsh',
|
||||
'vdi': 'BlockdevCreateOptionsVdi',
|
||||
'vhdx': 'BlockdevCreateOptionsVhdx',
|
||||
Index: qemu-5.1+dfsg/scripts/modules/module_block.py
|
||||
===================================================================
|
||||
--- qemu-5.1+dfsg.orig/scripts/modules/module_block.py
|
||||
+++ qemu-5.1+dfsg/scripts/modules/module_block.py
|
||||
@@ -86,6 +86,7 @@ if __name__ == '__main__':
|
||||
output_file = sys.argv[1]
|
||||
with open(output_file, 'w') as fheader:
|
||||
print_top(fheader)
|
||||
+ add_module(fheader, "vitastor", "vitastor", "vitastor")
|
||||
|
||||
for filename in sys.argv[2:]:
|
||||
if os.path.isfile(filename):
|
|
@ -0,0 +1,51 @@
|
|||
#!/bin/bash
|
||||
# Vitastor depends on QEMU and FIO headers, but QEMU and FIO don't have -devel packages
|
||||
# So we have to copy their headers into the source tarball
|
||||
|
||||
set -e
|
||||
|
||||
VITASTOR=$(dirname $0)
|
||||
VITASTOR=$(realpath "$VITASTOR/..")
|
||||
|
||||
if [ -d /opt/rh/gcc-toolset-9 ]; then
|
||||
# CentOS 8
|
||||
EL=8
|
||||
. /opt/rh/gcc-toolset-9/enable
|
||||
else
|
||||
# CentOS 7
|
||||
EL=7
|
||||
. /opt/rh/devtoolset-9/enable
|
||||
fi
|
||||
cd ~/rpmbuild/SPECS
|
||||
rpmbuild -bp fio.spec
|
||||
perl -i -pe 's/^make V=1/exit 0; make V=1/' qemu*.spec
|
||||
rpmbuild -bc qemu*.spec
|
||||
perl -i -pe 's/^exit 0; make V=1/make V=1/' qemu*.spec
|
||||
cd ~/rpmbuild/BUILD/qemu*/
|
||||
rm -rf $VITASTOR/qemu $VITASTOR/fio
|
||||
mkdir -p $VITASTOR/qemu/b/qemu
|
||||
make -j8 config-host.h
|
||||
cp config-host.h $VITASTOR/qemu/b/qemu
|
||||
cp -r include $VITASTOR/qemu
|
||||
if [ -f qapi-schema.json ]; then
|
||||
# QEMU 2.0
|
||||
make qapi-types.h
|
||||
cp qapi-types.h $VITASTOR/qemu/b/qemu
|
||||
else
|
||||
# QEMU 3.0+
|
||||
make qapi
|
||||
cp -r qapi $VITASTOR/qemu/b/qemu
|
||||
fi
|
||||
cd $VITASTOR
|
||||
sh copy-qemu-includes.sh
|
||||
rm -rf qemu
|
||||
mv qemu-copy qemu
|
||||
ln -s ~/rpmbuild/BUILD/fio*/ fio
|
||||
sh copy-fio-includes.sh
|
||||
rm fio
|
||||
mv fio-copy fio
|
||||
FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
|
||||
QEMU=`rpm -qi qemu qemu-kvm | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
|
||||
perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
|
||||
perl -i -pe 's/(Requires:\s*qemu(?:-kvm)?)([^\n]+)?/$1 = '$QEMU'/' $VITASTOR/rpm/vitastor-el$EL.spec
|
||||
tar --transform 's#^#vitastor-0.5.10/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.5.10$(rpm --eval '%dist').tar.gz *
|
|
@ -0,0 +1,31 @@
|
|||
# Build packages for CentOS 8 inside a container
|
||||
# cd ..; podman build -t qemu-el8 -v `pwd`/packages:/root/packages -f rpm/qemu-el8.Dockerfile .
|
||||
|
||||
FROM centos:8
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
RUN rm -f /etc/yum.repos.d/CentOS-Media.repo
|
||||
RUN dnf -y install centos-release-advanced-virtualization epel-release dnf-plugins-core rpm-build
|
||||
RUN rm -rf /var/lib/dnf/*; dnf download --disablerepo='*' --enablerepo='centos-advanced-virtualization-source' --source qemu-kvm
|
||||
RUN rpm --nomd5 -i qemu*.src.rpm
|
||||
RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=PowerTools --spec qemu-kvm.spec
|
||||
|
||||
ADD qemu-*-vitastor.patch /root/vitastor/
|
||||
|
||||
RUN set -e; \
|
||||
mkdir -p /root/packages/qemu-el8; \
|
||||
rm -rf /root/packages/qemu-el8/*; \
|
||||
rpm --nomd5 -i /root/qemu*.src.rpm; \
|
||||
cd ~/rpmbuild/SPECS; \
|
||||
PN=$(grep ^Patch qemu-kvm.spec | tail -n1 | perl -pe 's/Patch(\d+).*/$1/'); \
|
||||
csplit qemu-kvm.spec "/^Patch$PN/"; \
|
||||
cat xx00 > qemu-kvm.spec; \
|
||||
head -n 1 xx01 >> qemu-kvm.spec; \
|
||||
echo "Patch$((PN+1)): qemu-4.2-vitastor.patch" >> qemu-kvm.spec; \
|
||||
tail -n +2 xx01 >> qemu-kvm.spec; \
|
||||
perl -i -pe 's/(^Release:\s*\d+)/$1.vitastor/' qemu-kvm.spec; \
|
||||
cp /root/vitastor/qemu-4.2-vitastor.patch ~/rpmbuild/SOURCES; \
|
||||
rpmbuild --nocheck -ba qemu-kvm.spec; \
|
||||
cp ~/rpmbuild/RPMS/*/*qemu* /root/packages/qemu-el8/; \
|
||||
cp ~/rpmbuild/SRPMS/*qemu* /root/packages/qemu-el8/
|
|
@ -0,0 +1,257 @@
|
|||
--- qemu-kvm.spec.orig 2020-11-09 23:41:03.000000000 +0000
|
||||
+++ qemu-kvm.spec 2020-12-06 10:44:24.207640963 +0000
|
||||
@@ -2,7 +2,7 @@
|
||||
%global SLOF_gittagcommit 899d9883
|
||||
|
||||
%global have_usbredir 1
|
||||
-%global have_spice 1
|
||||
+%global have_spice 0
|
||||
%global have_opengl 1
|
||||
%global have_fdt 0
|
||||
%global have_gluster 1
|
||||
@@ -56,7 +56,7 @@ Requires: %{name}-block-curl = %{epoch}:
|
||||
Requires: %{name}-block-gluster = %{epoch}:%{version}-%{release} \
|
||||
%endif \
|
||||
Requires: %{name}-block-iscsi = %{epoch}:%{version}-%{release} \
|
||||
-Requires: %{name}-block-rbd = %{epoch}:%{version}-%{release} \
|
||||
+#Requires: %{name}-block-rbd = %{epoch}:%{version}-%{release} \
|
||||
Requires: %{name}-block-ssh = %{epoch}:%{version}-%{release}
|
||||
|
||||
# Macro to properly setup RHEL/RHEV conflict handling
|
||||
@@ -67,7 +67,7 @@ Obsoletes: %1-rhev
|
||||
Summary: QEMU is a machine emulator and virtualizer
|
||||
Name: qemu-kvm
|
||||
Version: 4.2.0
|
||||
-Release: 29.vitastor%{?dist}.6
|
||||
+Release: 30.vitastor%{?dist}.6
|
||||
# Epoch because we pushed a qemu-1.0 package. AIUI this can't ever be dropped
|
||||
Epoch: 15
|
||||
License: GPLv2 and GPLv2+ and CC-BY
|
||||
@@ -99,8 +99,8 @@ Source30: kvm-s390x.conf
|
||||
Source31: kvm-x86.conf
|
||||
Source32: qemu-pr-helper.service
|
||||
Source33: qemu-pr-helper.socket
|
||||
-Source34: 81-kvm-rhel.rules
|
||||
-Source35: udev-kvm-check.c
|
||||
+#Source34: 81-kvm-rhel.rules
|
||||
+#Source35: udev-kvm-check.c
|
||||
Source36: README.tests
|
||||
|
||||
|
||||
@@ -825,7 +825,9 @@ Patch331: kvm-Drop-bogus-IPv6-messages.p
|
||||
Patch333: kvm-virtiofsd-Whitelist-fchmod.patch
|
||||
# For bz#1883869 - virtiofsd core dump in KATA Container [rhel-8.2.1.z]
|
||||
Patch334: kvm-virtiofsd-avoid-proc-self-fd-tempdir.patch
|
||||
-Patch335: qemu-4.2-vitastor.patch
|
||||
+Patch335: qemu-use-sphinx-1.2.patch
|
||||
+Patch336: qemu-config-tcmalloc-warning.patch
|
||||
+Patch337: qemu-4.2-vitastor.patch
|
||||
|
||||
BuildRequires: wget
|
||||
BuildRequires: rpm-build
|
||||
@@ -842,7 +844,8 @@ BuildRequires: pciutils-devel
|
||||
BuildRequires: libiscsi-devel
|
||||
BuildRequires: ncurses-devel
|
||||
BuildRequires: libattr-devel
|
||||
-BuildRequires: libusbx-devel >= 1.0.22
|
||||
+BuildRequires: gperftools-devel
|
||||
+BuildRequires: libusbx-devel >= 1.0.21
|
||||
%if %{have_usbredir}
|
||||
BuildRequires: usbredir-devel >= 0.7.1
|
||||
%endif
|
||||
@@ -856,12 +859,12 @@ BuildRequires: virglrenderer-devel
|
||||
# For smartcard NSS support
|
||||
BuildRequires: nss-devel
|
||||
%endif
|
||||
-BuildRequires: libseccomp-devel >= 2.4.0
|
||||
+#Requires: libseccomp >= 2.4.0
|
||||
# For network block driver
|
||||
BuildRequires: libcurl-devel
|
||||
BuildRequires: libssh-devel
|
||||
-BuildRequires: librados-devel
|
||||
-BuildRequires: librbd-devel
|
||||
+#BuildRequires: librados-devel
|
||||
+#BuildRequires: librbd-devel
|
||||
%if %{have_gluster}
|
||||
# For gluster block driver
|
||||
BuildRequires: glusterfs-api-devel
|
||||
@@ -955,25 +958,25 @@ hardware for a full system such as a PC
|
||||
|
||||
%package -n qemu-kvm-core
|
||||
Summary: qemu-kvm core components
|
||||
+Requires: gperftools-libs
|
||||
Requires: qemu-img = %{epoch}:%{version}-%{release}
|
||||
%ifarch %{ix86} x86_64
|
||||
Requires: seabios-bin >= 1.10.2-1
|
||||
Requires: sgabios-bin
|
||||
-Requires: edk2-ovmf
|
||||
%endif
|
||||
%ifarch aarch64
|
||||
Requires: edk2-aarch64
|
||||
%endif
|
||||
|
||||
%ifnarch aarch64 s390x
|
||||
-Requires: seavgabios-bin >= 1.12.0-3
|
||||
-Requires: ipxe-roms-qemu >= 20170123-1
|
||||
+Requires: seavgabios-bin >= 1.11.0-1
|
||||
+Requires: ipxe-roms-qemu >= 20181214-1
|
||||
+Requires: /usr/share/ipxe.efi
|
||||
%endif
|
||||
%ifarch %{power64}
|
||||
Requires: SLOF >= %{SLOF_gittagdate}-1.git%{SLOF_gittagcommit}
|
||||
%endif
|
||||
Requires: %{name}-common = %{epoch}:%{version}-%{release}
|
||||
-Requires: libseccomp >= 2.4.0
|
||||
# For compressed guest memory dumps
|
||||
Requires: lzo snappy
|
||||
%if %{have_kvm_setup}
|
||||
@@ -1085,15 +1088,15 @@ This package provides the additional iSC
|
||||
Install this package if you want to access iSCSI volumes.
|
||||
|
||||
|
||||
-%package block-rbd
|
||||
-Summary: QEMU Ceph/RBD block driver
|
||||
-Requires: %{name}-common%{?_isa} = %{epoch}:%{version}-%{release}
|
||||
-
|
||||
-%description block-rbd
|
||||
-This package provides the additional Ceph/RBD block driver for QEMU.
|
||||
-
|
||||
-Install this package if you want to access remote Ceph volumes
|
||||
-using the rbd protocol.
|
||||
+#%package block-rbd
|
||||
+#Summary: QEMU Ceph/RBD block driver
|
||||
+#Requires: %{name}-common%{?_isa} = %{epoch}:%{version}-%{release}
|
||||
+#
|
||||
+#%description block-rbd
|
||||
+#This package provides the additional Ceph/RBD block driver for QEMU.
|
||||
+#
|
||||
+#Install this package if you want to access remote Ceph volumes
|
||||
+#using the rbd protocol.
|
||||
|
||||
|
||||
%package block-ssh
|
||||
@@ -1117,12 +1120,14 @@ the Secure Shell (SSH) protocol.
|
||||
# --build-id option is used for giving info to the debug packages.
|
||||
buildldflags="VL_LDFLAGS=-Wl,--build-id"
|
||||
|
||||
-%global block_drivers_list qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle
|
||||
+#%global block_drivers_list qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle
|
||||
+%global block_drivers_list qcow2,raw,file,host_device,nbd,iscsi,blkdebug,luks,null-co,nvme,copy-on-read,throttle
|
||||
|
||||
%if 0%{have_gluster}
|
||||
%global block_drivers_list %{block_drivers_list},gluster
|
||||
%endif
|
||||
|
||||
+[ -e /usr/bin/sphinx-build ] || ln -s sphinx-build-3 /usr/bin/sphinx-build
|
||||
./configure \
|
||||
--prefix="%{_prefix}" \
|
||||
--libdir="%{_libdir}" \
|
||||
@@ -1152,15 +1157,15 @@ buildldflags="VL_LDFLAGS=-Wl,--build-id"
|
||||
%else
|
||||
--disable-numa \
|
||||
%endif
|
||||
- --enable-rbd \
|
||||
+ --disable-rbd \
|
||||
%if 0%{have_librdma}
|
||||
--enable-rdma \
|
||||
%else
|
||||
--disable-rdma \
|
||||
%endif
|
||||
--disable-pvrdma \
|
||||
- --enable-seccomp \
|
||||
-%if 0%{have_spice}
|
||||
+ --disable-seccomp \
|
||||
+%if %{have_spice}
|
||||
--enable-spice \
|
||||
--enable-smartcard \
|
||||
--enable-virglrenderer \
|
||||
@@ -1179,7 +1184,7 @@ buildldflags="VL_LDFLAGS=-Wl,--build-id"
|
||||
%else
|
||||
--disable-usb-redir \
|
||||
%endif
|
||||
- --disable-tcmalloc \
|
||||
+ --enable-tcmalloc \
|
||||
%ifarch x86_64
|
||||
--enable-libpmem \
|
||||
%else
|
||||
@@ -1193,9 +1198,7 @@ buildldflags="VL_LDFLAGS=-Wl,--build-id"
|
||||
%endif
|
||||
--python=%{__python3} \
|
||||
--target-list="%{buildarch}" \
|
||||
- --block-drv-rw-whitelist=%{block_drivers_list} \
|
||||
--audio-drv-list= \
|
||||
- --block-drv-ro-whitelist=vmdk,vhdx,vpc,https,ssh \
|
||||
--with-coroutine=ucontext \
|
||||
--tls-priority=NORMAL \
|
||||
--disable-bluez \
|
||||
@@ -1262,7 +1265,7 @@ buildldflags="VL_LDFLAGS=-Wl,--build-id"
|
||||
--disable-sanitizers \
|
||||
--disable-hvf \
|
||||
--disable-whpx \
|
||||
- --enable-malloc-trim \
|
||||
+ --disable-malloc-trim \
|
||||
--disable-membarrier \
|
||||
--disable-vhost-crypto \
|
||||
--disable-libxml2 \
|
||||
@@ -1308,7 +1311,7 @@ make V=1 %{?_smp_mflags} $buildldflags
|
||||
cp -a %{kvm_target}-softmmu/qemu-system-%{kvm_target} qemu-kvm
|
||||
|
||||
gcc %{SOURCE6} $RPM_OPT_FLAGS $RPM_LD_FLAGS -o ksmctl
|
||||
-gcc %{SOURCE35} $RPM_OPT_FLAGS $RPM_LD_FLAGS -o udev-kvm-check
|
||||
+#gcc %{SOURCE35} $RPM_OPT_FLAGS $RPM_LD_FLAGS -o udev-kvm-check
|
||||
|
||||
%install
|
||||
%define _udevdir %(pkg-config --variable=udevdir udev)
|
||||
@@ -1343,8 +1346,8 @@ mkdir -p $RPM_BUILD_ROOT%{testsdir}/test
|
||||
mkdir -p $RPM_BUILD_ROOT%{testsdir}/tests/qemu-iotests
|
||||
mkdir -p $RPM_BUILD_ROOT%{testsdir}/scripts/qmp
|
||||
|
||||
-install -p -m 0755 udev-kvm-check $RPM_BUILD_ROOT%{_udevdir}
|
||||
-install -p -m 0644 %{SOURCE34} $RPM_BUILD_ROOT%{_udevrulesdir}
|
||||
+#install -p -m 0755 udev-kvm-check $RPM_BUILD_ROOT%{_udevdir}
|
||||
+#install -p -m 0644 %{SOURCE34} $RPM_BUILD_ROOT%{_udevrulesdir}
|
||||
|
||||
install -m 0644 scripts/dump-guest-memory.py \
|
||||
$RPM_BUILD_ROOT%{_datadir}/%{name}
|
||||
@@ -1562,6 +1565,8 @@ rm -rf $RPM_BUILD_ROOT%{qemudocdir}/inte
|
||||
# Remove spec
|
||||
rm -rf $RPM_BUILD_ROOT%{qemudocdir}/specs
|
||||
|
||||
+%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g')
|
||||
+
|
||||
%check
|
||||
export DIFF=diff; make check V=1
|
||||
|
||||
@@ -1645,8 +1650,8 @@ useradd -r -u 107 -g qemu -G kvm -d / -s
|
||||
%config(noreplace) %{_sysconfdir}/sysconfig/ksm
|
||||
%{_unitdir}/ksmtuned.service
|
||||
%{_sbindir}/ksmtuned
|
||||
-%{_udevdir}/udev-kvm-check
|
||||
-%{_udevrulesdir}/81-kvm-rhel.rules
|
||||
+#%{_udevdir}/udev-kvm-check
|
||||
+#%{_udevrulesdir}/81-kvm-rhel.rules
|
||||
%ghost %{_sysconfdir}/kvm
|
||||
%config(noreplace) %{_sysconfdir}/ksmtuned.conf
|
||||
%dir %{_sysconfdir}/%{name}
|
||||
@@ -1711,8 +1716,8 @@ useradd -r -u 107 -g qemu -G kvm -d / -s
|
||||
%{_libexecdir}/vhost-user-gpu
|
||||
%{_datadir}/%{name}/vhost-user/50-qemu-gpu.json
|
||||
%endif
|
||||
-%{_libexecdir}/virtiofsd
|
||||
-%{_datadir}/%{name}/vhost-user/50-qemu-virtiofsd.json
|
||||
+#%{_libexecdir}/virtiofsd
|
||||
+#%{_datadir}/%{name}/vhost-user/50-qemu-virtiofsd.json
|
||||
|
||||
%files -n qemu-img
|
||||
%defattr(-,root,root)
|
||||
@@ -1748,8 +1753,8 @@ useradd -r -u 107 -g qemu -G kvm -d / -s
|
||||
%files block-iscsi
|
||||
%{_libdir}/qemu-kvm/block-iscsi.so
|
||||
|
||||
-%files block-rbd
|
||||
-%{_libdir}/qemu-kvm/block-rbd.so
|
||||
+#%files block-rbd
|
||||
+#%{_libdir}/qemu-kvm/block-rbd.so
|
||||
|
||||
%files block-ssh
|
||||
%{_libdir}/qemu-kvm/block-ssh.so
|
|
@ -0,0 +1,29 @@
|
|||
--- qemu-kvm.spec 2020-12-05 13:13:54.388623517 +0000
|
||||
+++ qemu-kvm.spec 2020-12-05 13:13:58.728696598 +0000
|
||||
@@ -67,7 +67,7 @@ Obsoletes: %1-rhev
|
||||
Summary: QEMU is a machine emulator and virtualizer
|
||||
Name: qemu-kvm
|
||||
Version: 4.2.0
|
||||
-Release: 29%{?dist}.6
|
||||
+Release: 29.vitastor%{?dist}.6
|
||||
# Epoch because we pushed a qemu-1.0 package. AIUI this can't ever be dropped
|
||||
Epoch: 15
|
||||
License: GPLv2 and GPLv2+ and CC-BY
|
||||
@@ -825,6 +825,7 @@ Patch331: kvm-Drop-bogus-IPv6-messages.p
|
||||
Patch333: kvm-virtiofsd-Whitelist-fchmod.patch
|
||||
# For bz#1883869 - virtiofsd core dump in KATA Container [rhel-8.2.1.z]
|
||||
Patch334: kvm-virtiofsd-avoid-proc-self-fd-tempdir.patch
|
||||
+Patch335: qemu-4.2-vitastor.patch
|
||||
|
||||
BuildRequires: wget
|
||||
BuildRequires: rpm-build
|
||||
@@ -1192,9 +1193,7 @@ buildldflags="VL_LDFLAGS=-Wl,--build-id"
|
||||
%endif
|
||||
--python=%{__python3} \
|
||||
--target-list="%{buildarch}" \
|
||||
- --block-drv-rw-whitelist=%{block_drivers_list} \
|
||||
--audio-drv-list= \
|
||||
- --block-drv-ro-whitelist=vmdk,vhdx,vpc,https,ssh \
|
||||
--with-coroutine=ucontext \
|
||||
--tls-priority=NORMAL \
|
||||
--disable-bluez \
|
|
@ -0,0 +1,47 @@
|
|||
# Build packages for CentOS 7 inside a container
|
||||
# cd ..; podman build -t vitastor-el7 -v `pwd`/packages:/root/packages -f rpm/vitastor-el7.Dockerfile .
|
||||
# localedef -i ru_RU -f UTF-8 ru_RU.UTF-8
|
||||
|
||||
FROM centos:7
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
RUN rm -f /etc/yum.repos.d/CentOS-Media.repo
|
||||
RUN yum -y --enablerepo=extras install centos-release-scl epel-release yum-utils rpm-build
|
||||
RUN yum -y install https://vitastor.io/rpms/centos/7/vitastor-release-1.0-1.el7.noarch.rpm
|
||||
RUN yum -y install devtoolset-9-gcc-c++ devtoolset-9-libatomic-devel gperftools-devel qemu-kvm fio rh-nodejs12 jerasure-devel gf-complete-devel
|
||||
RUN yumdownloader --disablerepo=centos-sclo-rh --source qemu-kvm
|
||||
RUN yumdownloader --disablerepo=centos-sclo-rh --source fio
|
||||
RUN rpm --nomd5 -i qemu*.src.rpm
|
||||
RUN rpm --nomd5 -i fio*.src.rpm
|
||||
RUN rm -f /etc/yum.repos.d/CentOS-Media.repo
|
||||
RUN cd ~/rpmbuild/SPECS && yum-builddep -y --enablerepo='*' --disablerepo=centos-sclo-rh --disablerepo=centos-sclo-rh-source --disablerepo=centos-sclo-sclo-testing qemu-kvm.spec
|
||||
RUN cd ~/rpmbuild/SPECS && yum-builddep -y --enablerepo='*' --disablerepo=centos-sclo-rh --disablerepo=centos-sclo-rh-source --disablerepo=centos-sclo-sclo-testing fio.spec
|
||||
|
||||
ADD https://vitastor.io/rpms/liburing-el7/liburing-0.7-2.el7.src.rpm /root
|
||||
|
||||
RUN set -e; \
|
||||
rpm -i liburing*.src.rpm; \
|
||||
cd ~/rpmbuild/SPECS/; \
|
||||
. /opt/rh/devtoolset-9/enable; \
|
||||
rpmbuild -ba liburing.spec; \
|
||||
mkdir -p /root/packages/liburing-el7; \
|
||||
rm -rf /root/packages/liburing-el7/*; \
|
||||
cp ~/rpmbuild/RPMS/*/liburing* /root/packages/liburing-el7/; \
|
||||
cp ~/rpmbuild/SRPMS/liburing* /root/packages/liburing-el7/
|
||||
|
||||
RUN rpm -i `ls /root/packages/liburing-el7/liburing-*.x86_64.rpm | grep -v debug`
|
||||
|
||||
ADD . /root/vitastor
|
||||
|
||||
RUN set -e; \
|
||||
cd /root/vitastor/rpm; \
|
||||
sh build-tarball.sh; \
|
||||
cp /root/vitastor-0.5.10.el7.tar.gz ~/rpmbuild/SOURCES; \
|
||||
cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
|
||||
cd ~/rpmbuild/SPECS/; \
|
||||
rpmbuild -ba vitastor.spec; \
|
||||
mkdir -p /root/packages/vitastor-el7; \
|
||||
rm -rf /root/packages/vitastor-el7/*; \
|
||||
cp ~/rpmbuild/RPMS/*/vitastor* /root/packages/vitastor-el7/; \
|
||||
cp ~/rpmbuild/SRPMS/vitastor* /root/packages/vitastor-el7/
|
|
@ -0,0 +1,69 @@
|
|||
Name: vitastor
|
||||
Version: 0.5.10
|
||||
Release: 1%{?dist}
|
||||
Summary: Vitastor, a fast software-defined clustered block storage
|
||||
|
||||
License: Vitastor Network Public License 1.1
|
||||
URL: https://vitastor.io/
|
||||
Source0: vitastor-0.5.10.el7.tar.gz
|
||||
|
||||
BuildRequires: liburing-devel >= 0.6
|
||||
BuildRequires: gperftools-devel
|
||||
BuildRequires: devtoolset-9-gcc-c++
|
||||
BuildRequires: rh-nodejs12
|
||||
BuildRequires: rh-nodejs12-npm
|
||||
BuildRequires: jerasure-devel
|
||||
BuildRequires: gf-complete-devel
|
||||
BuildRequires: cmake
|
||||
Requires: fio = 3.7-1.el7
|
||||
Requires: qemu-kvm = 2.0.0-1.el7.6
|
||||
Requires: rh-nodejs12
|
||||
Requires: rh-nodejs12-npm
|
||||
Requires: liburing >= 0.6
|
||||
Requires: libJerasure2
|
||||
Requires: lpsolve
|
||||
|
||||
%description
|
||||
Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
|
||||
architecturally similar to Ceph which means strong consistency, primary-replication,
|
||||
symmetric clustering and automatic data distribution over any number of drives of any
|
||||
size with configurable redundancy (replication or erasure codes/XOR).
|
||||
|
||||
|
||||
%prep
|
||||
%setup -q
|
||||
|
||||
|
||||
%build
|
||||
. /opt/rh/devtoolset-9/enable
|
||||
%cmake . -DQEMU_PLUGINDIR=qemu-kvm
|
||||
%make_build
|
||||
|
||||
|
||||
%install
|
||||
rm -rf $RPM_BUILD_ROOT
|
||||
%make_install
|
||||
. /opt/rh/rh-nodejs12/enable
|
||||
cd mon
|
||||
npm install
|
||||
cd ..
|
||||
mkdir -p %buildroot/usr/lib/vitastor
|
||||
cp -r mon %buildroot/usr/lib/vitastor/mon
|
||||
|
||||
|
||||
%files
|
||||
%doc
|
||||
%_bindir/vitastor-dump-journal
|
||||
%_bindir/vitastor-nbd
|
||||
%_bindir/vitastor-osd
|
||||
%_bindir/vitastor-rm
|
||||
%_libdir/qemu-kvm/block-vitastor.so
|
||||
%_libdir/libfio_vitastor.so
|
||||
%_libdir/libfio_vitastor_blk.so
|
||||
%_libdir/libfio_vitastor_sec.so
|
||||
%_libdir/libvitastor_blk.so
|
||||
%_libdir/libvitastor_client.so
|
||||
/usr/lib/vitastor
|
||||
|
||||
|
||||
%changelog
|
|
@ -0,0 +1,45 @@
|
|||
# Build packages for CentOS 8 inside a container
|
||||
# cd ..; podman build -t vitastor-el8 -v `pwd`/packages:/root/packages -f rpm/vitastor-el8.Dockerfile .
|
||||
|
||||
FROM centos:8
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
RUN rm -f /etc/yum.repos.d/CentOS-Media.repo
|
||||
RUN dnf -y install centos-release-advanced-virtualization epel-release dnf-plugins-core
|
||||
RUN yum -y install https://vitastor.io/rpms/centos/8/vitastor-release-1.0-1.el8.noarch.rpm
|
||||
RUN dnf --enablerepo='centos-advanced-virtualization' -y install gcc-toolset-9 gcc-toolset-9-gcc-c++ gperftools-devel qemu-kvm fio nodejs rpm-build jerasure-devel gf-complete-devel
|
||||
RUN rm -rf /var/lib/dnf/*; dnf download --disablerepo='*' --enablerepo='vitastor' --source qemu-kvm
|
||||
RUN dnf download --source fio
|
||||
RUN rpm --nomd5 -i qemu*.src.rpm
|
||||
RUN rpm --nomd5 -i fio*.src.rpm
|
||||
RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=powertools --spec qemu-kvm.spec
|
||||
RUN cd ~/rpmbuild/SPECS && dnf builddep -y --enablerepo=powertools --spec fio.spec && dnf install -y cmake
|
||||
|
||||
ADD https://vitastor.io/rpms/liburing-el7/liburing-0.7-2.el7.src.rpm /root
|
||||
|
||||
RUN set -e; \
|
||||
rpm -i liburing*.src.rpm; \
|
||||
cd ~/rpmbuild/SPECS/; \
|
||||
. /opt/rh/gcc-toolset-9/enable; \
|
||||
rpmbuild -ba liburing.spec; \
|
||||
mkdir -p /root/packages/liburing-el8; \
|
||||
rm -rf /root/packages/liburing-el8/*; \
|
||||
cp ~/rpmbuild/RPMS/*/liburing* /root/packages/liburing-el8/; \
|
||||
cp ~/rpmbuild/SRPMS/liburing* /root/packages/liburing-el8/
|
||||
|
||||
RUN rpm -i `ls /root/packages/liburing-el7/liburing-*.x86_64.rpm | grep -v debug`
|
||||
|
||||
ADD . /root/vitastor
|
||||
|
||||
RUN set -e; \
|
||||
cd /root/vitastor/rpm; \
|
||||
sh build-tarball.sh; \
|
||||
cp /root/vitastor-0.5.10.el8.tar.gz ~/rpmbuild/SOURCES; \
|
||||
cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
|
||||
cd ~/rpmbuild/SPECS/; \
|
||||
rpmbuild -ba vitastor.spec; \
|
||||
mkdir -p /root/packages/vitastor-el8; \
|
||||
rm -rf /root/packages/vitastor-el8/*; \
|
||||
cp ~/rpmbuild/RPMS/*/vitastor* /root/packages/vitastor-el8/; \
|
||||
cp ~/rpmbuild/SRPMS/vitastor* /root/packages/vitastor-el8/
|
|
@ -0,0 +1,66 @@
|
|||
Name: vitastor
|
||||
Version: 0.5.10
|
||||
Release: 1%{?dist}
|
||||
Summary: Vitastor, a fast software-defined clustered block storage
|
||||
|
||||
License: Vitastor Network Public License 1.1
|
||||
URL: https://vitastor.io/
|
||||
Source0: vitastor-0.5.10.el8.tar.gz
|
||||
|
||||
BuildRequires: liburing-devel >= 0.6
|
||||
BuildRequires: gperftools-devel
|
||||
BuildRequires: gcc-toolset-9-gcc-c++
|
||||
BuildRequires: nodejs >= 10
|
||||
BuildRequires: jerasure-devel
|
||||
BuildRequires: gf-complete-devel
|
||||
BuildRequires: cmake
|
||||
Requires: fio = 3.7-3.el8
|
||||
Requires: qemu-kvm = 4.2.0-29.el8.6
|
||||
Requires: nodejs >= 10
|
||||
Requires: liburing >= 0.6
|
||||
Requires: libJerasure2
|
||||
Requires: lpsolve
|
||||
|
||||
%description
|
||||
Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
|
||||
architecturally similar to Ceph which means strong consistency, primary-replication,
|
||||
symmetric clustering and automatic data distribution over any number of drives of any
|
||||
size with configurable redundancy (replication or erasure codes/XOR).
|
||||
|
||||
|
||||
%prep
|
||||
%setup -q
|
||||
|
||||
|
||||
%build
|
||||
. /opt/rh/gcc-toolset-9/enable
|
||||
%cmake . -DQEMU_PLUGINDIR=qemu-kvm
|
||||
%make_build
|
||||
|
||||
|
||||
%install
|
||||
rm -rf $RPM_BUILD_ROOT
|
||||
%make_install
|
||||
cd mon
|
||||
npm install
|
||||
cd ..
|
||||
mkdir -p %buildroot/usr/lib/vitastor
|
||||
cp -r mon %buildroot/usr/lib/vitastor
|
||||
|
||||
|
||||
%files
|
||||
%doc
|
||||
%_bindir/vitastor-dump-journal
|
||||
%_bindir/vitastor-nbd
|
||||
%_bindir/vitastor-osd
|
||||
%_bindir/vitastor-rm
|
||||
%_libdir/qemu-kvm/block-vitastor.so
|
||||
%_libdir/libfio_vitastor.so
|
||||
%_libdir/libfio_vitastor_blk.so
|
||||
%_libdir/libfio_vitastor_sec.so
|
||||
%_libdir/libvitastor_blk.so
|
||||
%_libdir/libvitastor_client.so
|
||||
/usr/lib/vitastor
|
||||
|
||||
|
||||
%changelog
|
|
@ -0,0 +1,188 @@
|
|||
cmake_minimum_required(VERSION 2.8)
|
||||
|
||||
project(vitastor)
|
||||
|
||||
include(GNUInstallDirs)
|
||||
|
||||
set(QEMU_PLUGINDIR qemu CACHE STRING "QEMU plugin directory suffix (qemu-kvm on RHEL)")
|
||||
set(WITH_ASAN false CACHE BOOL "Build with AddressSanitizer")
|
||||
if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
|
||||
if(EXISTS "/etc/debian_version")
|
||||
set(CMAKE_INSTALL_LIBDIR "lib/${CMAKE_LIBRARY_ARCHITECTURE}")
|
||||
endif()
|
||||
set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
|
||||
endif()
|
||||
|
||||
add_definitions(-DVERSION="0.6-dev")
|
||||
add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith)
|
||||
if (${WITH_ASAN})
|
||||
add_definitions(-fsanitize=address -fno-omit-frame-pointer)
|
||||
add_link_options(-fsanitize=address -fno-omit-frame-pointer)
|
||||
endif (${WITH_ASAN})
|
||||
|
||||
set(CMAKE_BUILD_TYPE RelWithDebInfo)
|
||||
string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
|
||||
string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
|
||||
string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}")
|
||||
string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
|
||||
string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
|
||||
string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}")
|
||||
string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
|
||||
|
||||
find_package(PkgConfig)
|
||||
pkg_check_modules(LIBURING REQUIRED liburing)
|
||||
pkg_check_modules(GLIB REQUIRED glib-2.0)
|
||||
|
||||
include_directories(
|
||||
../
|
||||
/usr/include/jerasure
|
||||
${LIBURING_INCLUDE_DIRS}
|
||||
)
|
||||
|
||||
# libvitastor_blk.so
|
||||
add_library(vitastor_blk SHARED
|
||||
allocator.cpp blockstore.cpp blockstore_impl.cpp blockstore_init.cpp blockstore_open.cpp blockstore_journal.cpp blockstore_read.cpp
|
||||
blockstore_write.cpp blockstore_sync.cpp blockstore_stable.cpp blockstore_rollback.cpp blockstore_flush.cpp crc32c.c ringloop.cpp
|
||||
)
|
||||
target_link_libraries(vitastor_blk
|
||||
${LIBURING_LIBRARIES}
|
||||
tcmalloc_minimal
|
||||
)
|
||||
|
||||
# libfio_vitastor_blk.so
|
||||
add_library(fio_vitastor_blk SHARED
|
||||
fio_engine.cpp
|
||||
../json11/json11.cpp
|
||||
)
|
||||
target_link_libraries(fio_vitastor_blk
|
||||
vitastor_blk
|
||||
)
|
||||
|
||||
# vitastor-osd
|
||||
add_executable(vitastor-osd
|
||||
osd_main.cpp osd.cpp osd_secondary.cpp msgr_receive.cpp msgr_send.cpp osd_peering.cpp osd_flush.cpp osd_peering_pg.cpp
|
||||
osd_primary.cpp osd_primary_subops.cpp etcd_state_client.cpp messenger.cpp osd_cluster.cpp http_client.cpp osd_ops.cpp pg_states.cpp
|
||||
osd_rmw.cpp base64.cpp timerfd_manager.cpp epoll_manager.cpp ../json11/json11.cpp
|
||||
)
|
||||
target_link_libraries(vitastor-osd
|
||||
vitastor_blk
|
||||
Jerasure
|
||||
)
|
||||
|
||||
# libfio_vitastor_sec.so
|
||||
add_library(fio_vitastor_sec SHARED
|
||||
fio_sec_osd.cpp
|
||||
rw_blocking.cpp
|
||||
)
|
||||
target_link_libraries(fio_vitastor_sec
|
||||
tcmalloc_minimal
|
||||
)
|
||||
|
||||
# libvitastor_client.so
|
||||
add_library(vitastor_client SHARED
|
||||
cluster_client.cpp epoll_manager.cpp etcd_state_client.cpp
|
||||
messenger.cpp msgr_send.cpp msgr_receive.cpp ringloop.cpp ../json11/json11.cpp
|
||||
http_client.cpp osd_ops.cpp pg_states.cpp timerfd_manager.cpp base64.cpp
|
||||
)
|
||||
target_link_libraries(vitastor_client
|
||||
tcmalloc_minimal
|
||||
${LIBURING_LIBRARIES}
|
||||
)
|
||||
|
||||
# libfio_vitastor.so
|
||||
add_library(fio_vitastor SHARED
|
||||
fio_cluster.cpp
|
||||
)
|
||||
target_link_libraries(fio_vitastor
|
||||
vitastor_client
|
||||
)
|
||||
|
||||
# vitastor-nbd
|
||||
add_executable(vitastor-nbd
|
||||
nbd_proxy.cpp
|
||||
)
|
||||
target_link_libraries(vitastor-nbd
|
||||
vitastor_client
|
||||
)
|
||||
|
||||
# vitastor-rm
|
||||
add_executable(vitastor-rm
|
||||
rm_inode.cpp
|
||||
)
|
||||
target_link_libraries(vitastor-rm
|
||||
vitastor_client
|
||||
)
|
||||
|
||||
# vitastor-dump-journal
|
||||
add_executable(vitastor-dump-journal
|
||||
dump_journal.cpp crc32c.c
|
||||
)
|
||||
|
||||
# qemu_driver.so
|
||||
add_library(qemu_proxy STATIC qemu_proxy.cpp)
|
||||
target_compile_options(qemu_proxy PUBLIC -fPIC)
|
||||
target_include_directories(qemu_proxy PUBLIC
|
||||
../qemu/b/qemu
|
||||
../qemu/include
|
||||
${GLIB_INCLUDE_DIRS}
|
||||
)
|
||||
target_link_libraries(qemu_proxy
|
||||
vitastor_client
|
||||
)
|
||||
add_library(qemu_vitastor SHARED
|
||||
qemu_driver.c
|
||||
)
|
||||
target_link_libraries(qemu_vitastor
|
||||
qemu_proxy
|
||||
)
|
||||
set_target_properties(qemu_vitastor PROPERTIES
|
||||
PREFIX ""
|
||||
OUTPUT_NAME "block-vitastor"
|
||||
)
|
||||
|
||||
### Test stubs
|
||||
|
||||
# stub_osd, stub_bench, osd_test
|
||||
add_executable(stub_osd stub_osd.cpp rw_blocking.cpp)
|
||||
target_link_libraries(stub_osd tcmalloc_minimal)
|
||||
add_executable(stub_bench stub_bench.cpp rw_blocking.cpp)
|
||||
target_link_libraries(stub_bench tcmalloc_minimal)
|
||||
add_executable(osd_test osd_test.cpp rw_blocking.cpp)
|
||||
target_link_libraries(osd_test tcmalloc_minimal)
|
||||
|
||||
# osd_rmw_test
|
||||
add_executable(osd_rmw_test osd_rmw_test.cpp allocator.cpp)
|
||||
target_link_libraries(osd_rmw_test Jerasure tcmalloc_minimal)
|
||||
|
||||
# stub_uring_osd
|
||||
add_executable(stub_uring_osd
|
||||
stub_uring_osd.cpp epoll_manager.cpp messenger.cpp msgr_send.cpp msgr_receive.cpp ringloop.cpp timerfd_manager.cpp ../json11/json11.cpp
|
||||
)
|
||||
target_link_libraries(stub_uring_osd
|
||||
${LIBURING_LIBRARIES}
|
||||
tcmalloc_minimal
|
||||
)
|
||||
|
||||
# osd_peering_pg_test
|
||||
add_executable(osd_peering_pg_test osd_peering_pg_test.cpp osd_peering_pg.cpp)
|
||||
target_link_libraries(osd_peering_pg_test tcmalloc_minimal)
|
||||
|
||||
# test_allocator
|
||||
add_executable(test_allocator test_allocator.cpp allocator.cpp)
|
||||
|
||||
## test_blockstore, test_shit
|
||||
#add_executable(test_blockstore test_blockstore.cpp timerfd_interval.cpp)
|
||||
#target_link_libraries(test_blockstore blockstore)
|
||||
#add_executable(test_shit test_shit.cpp osd_peering_pg.cpp)
|
||||
#target_link_libraries(test_shit ${LIBURING_LIBRARIES} m)
|
||||
|
||||
### Install
|
||||
|
||||
install(TARGETS vitastor-osd vitastor-dump-journal vitastor-nbd vitastor-rm RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
|
||||
install(TARGETS fio_vitastor fio_vitastor_blk fio_vitastor_sec vitastor_blk vitastor_client LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
|
||||
install(TARGETS qemu_vitastor LIBRARY DESTINATION /usr/${CMAKE_INSTALL_LIBDIR}/${QEMU_PLUGINDIR})
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include <stdexcept>
|
||||
#include "allocator.h"
|
||||
|
@ -13,19 +13,19 @@ allocator::allocator(uint64_t blocks)
|
|||
{
|
||||
throw std::invalid_argument("blocks");
|
||||
}
|
||||
uint64_t p2 = 1, total = 1;
|
||||
uint64_t p2 = 1;
|
||||
total = 0;
|
||||
while (p2 * 64 < blocks)
|
||||
{
|
||||
p2 = p2 * 64;
|
||||
total += p2;
|
||||
p2 = p2 * 64;
|
||||
}
|
||||
total -= p2;
|
||||
total += (blocks+63) / 64;
|
||||
mask = new uint64_t[2 + total];
|
||||
mask = new uint64_t[total];
|
||||
size = free = blocks;
|
||||
last_one_mask = (blocks % 64) == 0
|
||||
? UINT64_MAX
|
||||
: ~(UINT64_MAX << (64 - blocks % 64));
|
||||
: ((1l << (blocks % 64)) - 1);
|
||||
for (uint64_t i = 0; i < total; i++)
|
||||
{
|
||||
mask[i] = 0;
|
||||
|
@ -99,6 +99,10 @@ uint64_t allocator::find_free()
|
|||
uint64_t p2 = 1, offset = 0, addr = 0, f, i;
|
||||
while (p2 < size)
|
||||
{
|
||||
if (offset+addr >= total)
|
||||
{
|
||||
return UINT64_MAX;
|
||||
}
|
||||
uint64_t m = mask[offset + addr];
|
||||
for (i = 0, f = 1; i < 64; i++, f <<= 1)
|
||||
{
|
||||
|
@ -113,11 +117,6 @@ uint64_t allocator::find_free()
|
|||
return UINT64_MAX;
|
||||
}
|
||||
addr = (addr * 64) | i;
|
||||
if (addr >= size)
|
||||
{
|
||||
// No space
|
||||
return UINT64_MAX;
|
||||
}
|
||||
offset += p2;
|
||||
p2 = p2 * 64;
|
||||
}
|
||||
|
@ -128,3 +127,35 @@ uint64_t allocator::get_free_count()
|
|||
{
|
||||
return free;
|
||||
}
|
||||
|
||||
void bitmap_set(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity)
|
||||
{
|
||||
if (start == 0)
|
||||
{
|
||||
if (len == 32*bitmap_granularity)
|
||||
{
|
||||
*((uint32_t*)bitmap) = UINT32_MAX;
|
||||
return;
|
||||
}
|
||||
else if (len == 64*bitmap_granularity)
|
||||
{
|
||||
*((uint64_t*)bitmap) = UINT64_MAX;
|
||||
return;
|
||||
}
|
||||
}
|
||||
unsigned bit_start = start / bitmap_granularity;
|
||||
unsigned bit_end = ((start + len) + bitmap_granularity - 1) / bitmap_granularity;
|
||||
while (bit_start < bit_end)
|
||||
{
|
||||
if (!(bit_start & 7) && bit_end >= bit_start+8)
|
||||
{
|
||||
((uint8_t*)bitmap)[bit_start / 8] = UINT8_MAX;
|
||||
bit_start += 8;
|
||||
}
|
||||
else
|
||||
{
|
||||
((uint8_t*)bitmap)[bit_start / 8] |= 1 << (bit_start % 8);
|
||||
bit_start++;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
||||
|
@ -8,6 +8,7 @@
|
|||
// Hierarchical bitmap allocator
|
||||
class allocator
|
||||
{
|
||||
uint64_t total;
|
||||
uint64_t size;
|
||||
uint64_t free;
|
||||
uint64_t last_one_mask;
|
||||
|
@ -19,3 +20,5 @@ public:
|
|||
uint64_t find_free();
|
||||
uint64_t get_free_count();
|
||||
};
|
||||
|
||||
void bitmap_set(void *bitmap, uint64_t start, uint64_t len, uint64_t bitmap_granularity);
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "base64.h"
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
#include <string>
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "blockstore_impl.h"
|
||||
|
||||
|
@ -35,12 +35,7 @@ bool blockstore_t::is_safe_to_stop()
|
|||
|
||||
void blockstore_t::enqueue_op(blockstore_op_t *op)
|
||||
{
|
||||
impl->enqueue_op(op, false);
|
||||
}
|
||||
|
||||
void blockstore_t::enqueue_op_first(blockstore_op_t *op)
|
||||
{
|
||||
impl->enqueue_op(op, true);
|
||||
impl->enqueue_op(op);
|
||||
}
|
||||
|
||||
std::unordered_map<object_id, uint64_t> & blockstore_t::get_unstable_writes()
|
||||
|
@ -48,6 +43,11 @@ std::unordered_map<object_id, uint64_t> & blockstore_t::get_unstable_writes()
|
|||
return impl->unstable_writes;
|
||||
}
|
||||
|
||||
std::map<uint64_t, uint64_t> & blockstore_t::get_inode_space_stats()
|
||||
{
|
||||
return impl->inode_space_stats;
|
||||
}
|
||||
|
||||
uint32_t blockstore_t::get_block_size()
|
||||
{
|
||||
return impl->get_block_size();
|
||||
|
@ -63,7 +63,7 @@ uint64_t blockstore_t::get_free_block_count()
|
|||
return impl->get_free_block_count();
|
||||
}
|
||||
|
||||
uint32_t blockstore_t::get_disk_alignment()
|
||||
uint32_t blockstore_t::get_bitmap_granularity()
|
||||
{
|
||||
return impl->get_disk_alignment();
|
||||
return impl->get_bitmap_granularity();
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
||||
|
@ -27,6 +27,7 @@
|
|||
#define DEFAULT_ORDER 17
|
||||
#define MIN_BLOCK_SIZE 4*1024
|
||||
#define MAX_BLOCK_SIZE 128*1024*1024
|
||||
#define DEFAULT_BITMAP_GRANULARITY 4096
|
||||
|
||||
#define BS_OP_MIN 1
|
||||
#define BS_OP_READ 1
|
||||
|
@ -64,6 +65,8 @@ Input:
|
|||
- offset, len = offset and length within object. length may be zero, in that case
|
||||
read operation only returns the version / write operation only bumps the version
|
||||
- buf = pre-allocated buffer for data (read) / with data (write). may be NULL if len == 0.
|
||||
- bitmap = pointer to the new 'external' object bitmap data. Its part which is respective to the
|
||||
write request is copied into the metadata area bitwise and stored there.
|
||||
|
||||
Output:
|
||||
- retval = number of bytes actually read/written or negative error number (-EINVAL or -ENOSPC)
|
||||
|
@ -141,6 +144,7 @@ struct blockstore_op_t
|
|||
uint32_t offset;
|
||||
uint32_t len;
|
||||
void *buf;
|
||||
void *bitmap;
|
||||
int retval;
|
||||
|
||||
uint8_t private_data[BS_OP_PRIVATE_DATA_SIZE];
|
||||
|
@ -175,17 +179,16 @@ public:
|
|||
// Submission
|
||||
void enqueue_op(blockstore_op_t *op);
|
||||
|
||||
// Insert operation into the beginning of the queue
|
||||
// Intended for the OSD syncer "thread" to be able to stabilize something when the journal is full
|
||||
void enqueue_op_first(blockstore_op_t *op);
|
||||
|
||||
// Unstable writes are added here (map of object_id -> version)
|
||||
std::unordered_map<object_id, uint64_t> & get_unstable_writes();
|
||||
|
||||
// Get per-inode space usage statistics
|
||||
std::map<uint64_t, uint64_t> & get_inode_space_stats();
|
||||
|
||||
// FIXME rename to object_size
|
||||
uint32_t get_block_size();
|
||||
uint64_t get_block_count();
|
||||
uint64_t get_free_block_count();
|
||||
|
||||
uint32_t get_disk_alignment();
|
||||
uint32_t get_bitmap_granularity();
|
||||
};
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "blockstore_impl.h"
|
||||
|
||||
|
@ -7,6 +7,8 @@ journal_flusher_t::journal_flusher_t(int flusher_count, blockstore_impl_t *bs)
|
|||
{
|
||||
this->bs = bs;
|
||||
this->flusher_count = flusher_count;
|
||||
this->cur_flusher_count = 1;
|
||||
this->target_flusher_count = 1;
|
||||
dequeuing = false;
|
||||
trimming = false;
|
||||
active_flushers = 0;
|
||||
|
@ -68,14 +70,31 @@ bool journal_flusher_t::is_active()
|
|||
|
||||
void journal_flusher_t::loop()
|
||||
{
|
||||
for (int i = 0; (active_flushers > 0 || dequeuing) && i < flusher_count; i++)
|
||||
target_flusher_count = bs->write_iodepth*2;
|
||||
if (target_flusher_count <= 0)
|
||||
target_flusher_count = 1;
|
||||
else if (target_flusher_count > flusher_count)
|
||||
target_flusher_count = flusher_count;
|
||||
if (target_flusher_count > cur_flusher_count)
|
||||
cur_flusher_count = target_flusher_count;
|
||||
else if (target_flusher_count < cur_flusher_count)
|
||||
{
|
||||
co[i].loop();
|
||||
while (target_flusher_count < cur_flusher_count)
|
||||
{
|
||||
if (co[cur_flusher_count-1].wait_state)
|
||||
break;
|
||||
cur_flusher_count--;
|
||||
}
|
||||
}
|
||||
for (int i = 0; (active_flushers > 0 || dequeuing) && i < cur_flusher_count; i++)
|
||||
co[i].loop();
|
||||
}
|
||||
|
||||
void journal_flusher_t::enqueue_flush(obj_ver_id ov)
|
||||
{
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("enqueue_flush %lx:%lx v%lu\n", ov.oid.inode, ov.oid.stripe, ov.version);
|
||||
#endif
|
||||
auto it = flush_versions.find(ov.oid);
|
||||
if (it != flush_versions.end())
|
||||
{
|
||||
|
@ -94,8 +113,11 @@ void journal_flusher_t::enqueue_flush(obj_ver_id ov)
|
|||
}
|
||||
}
|
||||
|
||||
void journal_flusher_t::unshift_flush(obj_ver_id ov)
|
||||
void journal_flusher_t::unshift_flush(obj_ver_id ov, bool force)
|
||||
{
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("unshift_flush %lx:%lx v%lu\n", ov.oid.inode, ov.oid.stripe, ov.version);
|
||||
#endif
|
||||
auto it = flush_versions.find(ov.oid);
|
||||
if (it != flush_versions.end())
|
||||
{
|
||||
|
@ -105,15 +127,38 @@ void journal_flusher_t::unshift_flush(obj_ver_id ov)
|
|||
else
|
||||
{
|
||||
flush_versions[ov.oid] = ov.version;
|
||||
flush_queue.push_front(ov.oid);
|
||||
if (!force)
|
||||
flush_queue.push_front(ov.oid);
|
||||
}
|
||||
if (!dequeuing && (flush_queue.size() >= flusher_start_threshold || trim_wanted > 0))
|
||||
if (force)
|
||||
flush_queue.push_front(ov.oid);
|
||||
if (force || !dequeuing && (flush_queue.size() >= flusher_start_threshold || trim_wanted > 0))
|
||||
{
|
||||
dequeuing = true;
|
||||
bs->ringloop->wakeup();
|
||||
}
|
||||
}
|
||||
|
||||
void journal_flusher_t::remove_flush(object_id oid)
|
||||
{
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("undo_flush %lx:%lx\n", oid.inode, oid.stripe);
|
||||
#endif
|
||||
auto v_it = flush_versions.find(oid);
|
||||
if (v_it != flush_versions.end())
|
||||
{
|
||||
flush_versions.erase(v_it);
|
||||
for (auto q_it = flush_queue.begin(); q_it != flush_queue.end(); q_it++)
|
||||
{
|
||||
if (*q_it == oid)
|
||||
{
|
||||
flush_queue.erase(q_it);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void journal_flusher_t::request_trim()
|
||||
{
|
||||
dequeuing = true;
|
||||
|
@ -194,6 +239,7 @@ bool journal_flusher_co::loop()
|
|||
resume_0:
|
||||
if (!flusher->flush_queue.size() || !flusher->dequeuing)
|
||||
{
|
||||
stop_flusher:
|
||||
if (flusher->trim_wanted > 0 && flusher->journal_trim_counter > 0)
|
||||
{
|
||||
// Attempt forced trim
|
||||
|
@ -298,9 +344,7 @@ resume_0:
|
|||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("No older flushes, stopping\n");
|
||||
#endif
|
||||
flusher->dequeuing = false;
|
||||
wait_state = 0;
|
||||
return true;
|
||||
goto stop_flusher;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -319,8 +363,8 @@ resume_1:
|
|||
return false;
|
||||
}
|
||||
// Writes and deletes shouldn't happen at the same time
|
||||
assert(!(copy_count > 0 || has_writes) || !has_delete);
|
||||
if (copy_count == 0 && !has_writes && !has_delete || has_delete && old_clean_loc == UINT64_MAX)
|
||||
assert(!has_writes || !has_delete);
|
||||
if (!has_writes && !has_delete || has_delete && old_clean_loc == UINT64_MAX)
|
||||
{
|
||||
// Nothing to flush
|
||||
bs->erase_dirty(dirty_start, std::next(dirty_end), clean_loc);
|
||||
|
@ -382,18 +426,18 @@ resume_1:
|
|||
{
|
||||
new_clean_bitmap = (bs->inmemory_meta
|
||||
? meta_new.buf + meta_new.pos*bs->clean_entry_size + sizeof(clean_disk_entry)
|
||||
: bs->clean_bitmap + (clean_loc >> bs->block_order)*bs->clean_entry_bitmap_size);
|
||||
: bs->clean_bitmap + (clean_loc >> bs->block_order)*(2*bs->clean_entry_bitmap_size));
|
||||
if (clean_init_bitmap)
|
||||
{
|
||||
memset(new_clean_bitmap, 0, bs->clean_entry_bitmap_size);
|
||||
bitmap_set(new_clean_bitmap, clean_bitmap_offset, clean_bitmap_len);
|
||||
bitmap_set(new_clean_bitmap, clean_bitmap_offset, clean_bitmap_len, bs->bitmap_granularity);
|
||||
}
|
||||
}
|
||||
for (it = v.begin(); it != v.end(); it++)
|
||||
{
|
||||
if (new_clean_bitmap)
|
||||
{
|
||||
bitmap_set(new_clean_bitmap, it->offset, it->len);
|
||||
bitmap_set(new_clean_bitmap, it->offset, it->len, bs->bitmap_granularity);
|
||||
}
|
||||
await_sqe(4);
|
||||
data->iov = (struct iovec){ it->buf, (size_t)it->len };
|
||||
|
@ -427,6 +471,7 @@ resume_1:
|
|||
wait_state = 5;
|
||||
return false;
|
||||
}
|
||||
// zero out old metadata entry
|
||||
memset(meta_old.buf + meta_old.pos*bs->clean_entry_size, 0, bs->clean_entry_size);
|
||||
await_sqe(15);
|
||||
data->iov = (struct iovec){ meta_old.buf, bs->meta_block_size };
|
||||
|
@ -438,6 +483,7 @@ resume_1:
|
|||
}
|
||||
if (has_delete)
|
||||
{
|
||||
// zero out new metadata entry
|
||||
memset(meta_new.buf + meta_new.pos*bs->clean_entry_size, 0, bs->clean_entry_size);
|
||||
}
|
||||
else
|
||||
|
@ -445,8 +491,8 @@ resume_1:
|
|||
clean_disk_entry *new_entry = (clean_disk_entry*)(meta_new.buf + meta_new.pos*bs->clean_entry_size);
|
||||
if (new_entry->oid.inode != 0 && new_entry->oid != cur.oid)
|
||||
{
|
||||
printf("Fatal error (metadata corruption or bug): tried to overwrite non-zero metadata entry %lx (%lx:%lx) with %lx:%lx\n",
|
||||
clean_loc, new_entry->oid.inode, new_entry->oid.stripe, cur.oid.inode, cur.oid.stripe);
|
||||
printf("Fatal error (metadata corruption or bug): tried to overwrite non-zero metadata entry %lu (%lx:%lx) with %lx:%lx\n",
|
||||
clean_loc >> bs->block_order, new_entry->oid.inode, new_entry->oid.stripe, cur.oid.inode, cur.oid.stripe);
|
||||
exit(1);
|
||||
}
|
||||
new_entry->oid = cur.oid;
|
||||
|
@ -455,6 +501,12 @@ resume_1:
|
|||
{
|
||||
memcpy(&new_entry->bitmap, new_clean_bitmap, bs->clean_entry_bitmap_size);
|
||||
}
|
||||
// copy latest external bitmap/attributes
|
||||
if (bs->clean_entry_bitmap_size)
|
||||
{
|
||||
void *bmp_ptr = bs->clean_entry_bitmap_size > sizeof(void*) ? dirty_end->second.bitmap : &dirty_end->second.bitmap;
|
||||
memcpy((void*)(new_entry+1) + bs->clean_entry_bitmap_size, bmp_ptr, bs->clean_entry_bitmap_size);
|
||||
}
|
||||
}
|
||||
await_sqe(6);
|
||||
data->iov = (struct iovec){ meta_new.buf, bs->meta_block_size };
|
||||
|
@ -513,7 +565,7 @@ resume_1:
|
|||
if (repeat_it != flusher->sync_to_repeat.end() && repeat_it->second > cur.version)
|
||||
{
|
||||
// Requeue version
|
||||
flusher->unshift_flush({ .oid = cur.oid, .version = repeat_it->second });
|
||||
flusher->unshift_flush({ .oid = cur.oid, .version = repeat_it->second }, false);
|
||||
}
|
||||
flusher->sync_to_repeat.erase(repeat_it);
|
||||
trim_journal:
|
||||
|
@ -602,7 +654,7 @@ bool journal_flusher_co::scan_dirty(int wait_base)
|
|||
{
|
||||
char err[1024];
|
||||
snprintf(
|
||||
err, 1024, "BUG: Unexpected dirty_entry %lx:%lx v%lu state during flush: %d",
|
||||
err, 1024, "BUG: Unexpected dirty_entry %lx:%lx v%lu unstable state during flush: %d",
|
||||
dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, dirty_it->second.state
|
||||
);
|
||||
throw std::runtime_error(err);
|
||||
|
@ -779,31 +831,34 @@ bool journal_flusher_co::fsync_batch(bool fsync_meta, int wait_base)
|
|||
sync_found:
|
||||
cur_sync->ready_count++;
|
||||
flusher->syncing_flushers++;
|
||||
if (flusher->syncing_flushers >= flusher->flusher_count || !flusher->flush_queue.size())
|
||||
resume_1:
|
||||
if (!cur_sync->state)
|
||||
{
|
||||
// Sync batch is ready. Do it.
|
||||
await_sqe(0);
|
||||
data->iov = { 0 };
|
||||
data->callback = simple_callback_w;
|
||||
my_uring_prep_fsync(sqe, fsync_meta ? bs->meta_fd : bs->data_fd, IORING_FSYNC_DATASYNC);
|
||||
cur_sync->state = 1;
|
||||
wait_count++;
|
||||
resume_1:
|
||||
if (wait_count > 0)
|
||||
if (flusher->syncing_flushers >= flusher->cur_flusher_count || !flusher->flush_queue.size())
|
||||
{
|
||||
// Sync batch is ready. Do it.
|
||||
await_sqe(0);
|
||||
data->iov = { 0 };
|
||||
data->callback = simple_callback_w;
|
||||
my_uring_prep_fsync(sqe, fsync_meta ? bs->meta_fd : bs->data_fd, IORING_FSYNC_DATASYNC);
|
||||
cur_sync->state = 1;
|
||||
wait_count++;
|
||||
resume_2:
|
||||
if (wait_count > 0)
|
||||
{
|
||||
wait_state = 2;
|
||||
return false;
|
||||
}
|
||||
// Sync completed. All previous coroutines waiting for it must be resumed
|
||||
cur_sync->state = 2;
|
||||
bs->ringloop->wakeup();
|
||||
}
|
||||
else
|
||||
{
|
||||
// Wait until someone else sends and completes a sync.
|
||||
wait_state = 1;
|
||||
return false;
|
||||
}
|
||||
// Sync completed. All previous coroutines waiting for it must be resumed
|
||||
cur_sync->state = 2;
|
||||
bs->ringloop->wakeup();
|
||||
}
|
||||
// Wait until someone else sends and completes a sync.
|
||||
resume_2:
|
||||
if (!cur_sync->state)
|
||||
{
|
||||
wait_state = 2;
|
||||
return false;
|
||||
}
|
||||
flusher->syncing_flushers--;
|
||||
cur_sync->ready_count--;
|
||||
|
@ -814,35 +869,3 @@ bool journal_flusher_co::fsync_batch(bool fsync_meta, int wait_base)
|
|||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void journal_flusher_co::bitmap_set(void *bitmap, uint64_t start, uint64_t len)
|
||||
{
|
||||
if (start == 0)
|
||||
{
|
||||
if (len == 32*bs->bitmap_granularity)
|
||||
{
|
||||
*((uint32_t*)bitmap) = UINT32_MAX;
|
||||
return;
|
||||
}
|
||||
else if (len == 64*bs->bitmap_granularity)
|
||||
{
|
||||
*((uint64_t*)bitmap) = UINT64_MAX;
|
||||
return;
|
||||
}
|
||||
}
|
||||
unsigned bit_start = start / bs->bitmap_granularity;
|
||||
unsigned bit_end = ((start + len) + bs->bitmap_granularity - 1) / bs->bitmap_granularity;
|
||||
while (bit_start < bit_end)
|
||||
{
|
||||
if (!(bit_start & 7) && bit_end >= bit_start+8)
|
||||
{
|
||||
((uint8_t*)bitmap)[bit_start / 8] = UINT8_MAX;
|
||||
bit_start += 8;
|
||||
}
|
||||
else
|
||||
{
|
||||
((uint8_t*)bitmap)[bit_start / 8] |= 1 << (bit_start % 8);
|
||||
bit_start++;
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
struct copy_buffer_t
|
||||
{
|
||||
|
@ -69,7 +69,6 @@ class journal_flusher_co
|
|||
bool modify_meta_read(uint64_t meta_loc, flusher_meta_write_t &wr, int wait_base);
|
||||
void update_clean_db();
|
||||
bool fsync_batch(bool fsync_meta, int wait_base);
|
||||
void bitmap_set(void *bitmap, uint64_t start, uint64_t len);
|
||||
public:
|
||||
journal_flusher_co();
|
||||
bool loop();
|
||||
|
@ -80,7 +79,7 @@ class journal_flusher_t
|
|||
{
|
||||
int trim_wanted = 0;
|
||||
bool dequeuing;
|
||||
int flusher_count;
|
||||
int flusher_count, cur_flusher_count, target_flusher_count;
|
||||
int flusher_start_threshold;
|
||||
journal_flusher_co *co;
|
||||
blockstore_impl_t *bs;
|
||||
|
@ -107,5 +106,6 @@ public:
|
|||
void request_trim();
|
||||
void release_trim();
|
||||
void enqueue_flush(obj_ver_id oid);
|
||||
void unshift_flush(obj_ver_id oid);
|
||||
void unshift_flush(obj_ver_id oid, bool force);
|
||||
void remove_flush(object_id oid);
|
||||
};
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "blockstore_impl.h"
|
||||
|
||||
|
@ -10,9 +10,9 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_t *
|
|||
ring_consumer.loop = [this]() { loop(); };
|
||||
ringloop->register_consumer(&ring_consumer);
|
||||
initialized = 0;
|
||||
zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, block_size);
|
||||
data_fd = meta_fd = journal.fd = -1;
|
||||
parse_config(config);
|
||||
zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, block_size);
|
||||
try
|
||||
{
|
||||
open_data();
|
||||
|
@ -101,26 +101,14 @@ void blockstore_impl_t::loop()
|
|||
{
|
||||
// try to submit ops
|
||||
unsigned initial_ring_space = ringloop->space_left();
|
||||
// FIXME: rework this "sync polling"
|
||||
auto cur_sync = in_progress_syncs.begin();
|
||||
while (cur_sync != in_progress_syncs.end())
|
||||
// has_writes == 0 - no writes before the current queue item
|
||||
// has_writes == 1 - some writes in progress
|
||||
// has_writes == 2 - tried to submit some writes, but failed
|
||||
int has_writes = 0, op_idx = 0, new_idx = 0;
|
||||
for (; op_idx < submit_queue.size(); op_idx++, new_idx++)
|
||||
{
|
||||
if (continue_sync(*cur_sync) != 2)
|
||||
{
|
||||
// List is unmodified
|
||||
cur_sync++;
|
||||
}
|
||||
else
|
||||
{
|
||||
cur_sync = in_progress_syncs.begin();
|
||||
}
|
||||
}
|
||||
auto cur = submit_queue.begin();
|
||||
int has_writes = 0;
|
||||
while (cur != submit_queue.end())
|
||||
{
|
||||
auto op_ptr = cur;
|
||||
auto op = *(cur++);
|
||||
auto op = submit_queue[op_idx];
|
||||
submit_queue[new_idx] = op;
|
||||
// FIXME: This needs some simplification
|
||||
// Writes should not block reads if the ring is not full and reads don't depend on them
|
||||
// In all other cases we should stop submission
|
||||
|
@ -142,10 +130,13 @@ void blockstore_impl_t::loop()
|
|||
}
|
||||
unsigned ring_space = ringloop->space_left();
|
||||
unsigned prev_sqe_pos = ringloop->save();
|
||||
bool dequeue_op = false;
|
||||
// 0 = can't submit
|
||||
// 1 = in progress
|
||||
// 2 = can be removed from queue
|
||||
int wr_st = 0;
|
||||
if (op->opcode == BS_OP_READ)
|
||||
{
|
||||
dequeue_op = dequeue_read(op);
|
||||
wr_st = dequeue_read(op);
|
||||
}
|
||||
else if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE)
|
||||
{
|
||||
|
@ -154,8 +145,8 @@ void blockstore_impl_t::loop()
|
|||
// Some writes already could not be submitted
|
||||
continue;
|
||||
}
|
||||
dequeue_op = dequeue_write(op);
|
||||
has_writes = dequeue_op ? 1 : 2;
|
||||
wr_st = dequeue_write(op);
|
||||
has_writes = wr_st > 0 ? 1 : 2;
|
||||
}
|
||||
else if (op->opcode == BS_OP_DELETE)
|
||||
{
|
||||
|
@ -164,8 +155,8 @@ void blockstore_impl_t::loop()
|
|||
// Some writes already could not be submitted
|
||||
continue;
|
||||
}
|
||||
dequeue_op = dequeue_del(op);
|
||||
has_writes = dequeue_op ? 1 : 2;
|
||||
wr_st = dequeue_del(op);
|
||||
has_writes = wr_st > 0 ? 1 : 2;
|
||||
}
|
||||
else if (op->opcode == BS_OP_SYNC)
|
||||
{
|
||||
|
@ -178,29 +169,31 @@ void blockstore_impl_t::loop()
|
|||
// Can't submit SYNC before previous writes
|
||||
continue;
|
||||
}
|
||||
dequeue_op = dequeue_sync(op);
|
||||
wr_st = continue_sync(op, false);
|
||||
if (wr_st != 2)
|
||||
{
|
||||
has_writes = wr_st > 0 ? 1 : 2;
|
||||
}
|
||||
}
|
||||
else if (op->opcode == BS_OP_STABLE)
|
||||
{
|
||||
dequeue_op = dequeue_stable(op);
|
||||
wr_st = dequeue_stable(op);
|
||||
}
|
||||
else if (op->opcode == BS_OP_ROLLBACK)
|
||||
{
|
||||
dequeue_op = dequeue_rollback(op);
|
||||
wr_st = dequeue_rollback(op);
|
||||
}
|
||||
else if (op->opcode == BS_OP_LIST)
|
||||
{
|
||||
// LIST doesn't need to be blocked by previous modifications,
|
||||
// it only needs to include all in-progress writes as they're guaranteed
|
||||
// to be readable and stabilizable/rollbackable by subsequent operations
|
||||
// LIST doesn't need to be blocked by previous modifications
|
||||
process_list(op);
|
||||
dequeue_op = true;
|
||||
wr_st = 2;
|
||||
}
|
||||
if (dequeue_op)
|
||||
if (wr_st == 2)
|
||||
{
|
||||
submit_queue.erase(op_ptr);
|
||||
new_idx--;
|
||||
}
|
||||
else
|
||||
if (wr_st == 0)
|
||||
{
|
||||
ringloop->restore(prev_sqe_pos);
|
||||
if (PRIV(op)->wait_for == WAIT_SQE)
|
||||
|
@ -211,6 +204,14 @@ void blockstore_impl_t::loop()
|
|||
}
|
||||
}
|
||||
}
|
||||
if (op_idx != new_idx)
|
||||
{
|
||||
while (op_idx < submit_queue.size())
|
||||
{
|
||||
submit_queue[new_idx++] = submit_queue[op_idx++];
|
||||
}
|
||||
submit_queue.resize(new_idx);
|
||||
}
|
||||
if (!readonly)
|
||||
{
|
||||
flusher->loop();
|
||||
|
@ -233,7 +234,7 @@ bool blockstore_impl_t::is_safe_to_stop()
|
|||
{
|
||||
// It's safe to stop blockstore when there are no in-flight operations,
|
||||
// no in-progress syncs and flusher isn't doing anything
|
||||
if (submit_queue.size() > 0 || in_progress_syncs.size() > 0 || !readonly && flusher->is_active())
|
||||
if (submit_queue.size() > 0 || !readonly && flusher->is_active())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
@ -287,7 +288,7 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
|
|||
else if (PRIV(op)->wait_for == WAIT_JOURNAL_BUFFER)
|
||||
{
|
||||
int next = ((journal.cur_sector + 1) % journal.sector_count);
|
||||
if (journal.sector_info[next].usage_count > 0 ||
|
||||
if (journal.sector_info[next].flush_count > 0 ||
|
||||
journal.sector_info[next].dirty)
|
||||
{
|
||||
// do not submit
|
||||
|
@ -300,7 +301,7 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
|
|||
}
|
||||
else if (PRIV(op)->wait_for == WAIT_FREE)
|
||||
{
|
||||
if (!data_alloc->get_free_count() && !flusher->is_active())
|
||||
if (!data_alloc->get_free_count() && flusher->is_active())
|
||||
{
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("Still waiting for free space on the data device\n");
|
||||
|
@ -315,7 +316,7 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
|
|||
}
|
||||
}
|
||||
|
||||
void blockstore_impl_t::enqueue_op(blockstore_op_t *op, bool first)
|
||||
void blockstore_impl_t::enqueue_op(blockstore_op_t *op)
|
||||
{
|
||||
if (op->opcode < BS_OP_MIN || op->opcode > BS_OP_MAX ||
|
||||
((op->opcode == BS_OP_READ || op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE) && (
|
||||
|
@ -323,8 +324,7 @@ void blockstore_impl_t::enqueue_op(blockstore_op_t *op, bool first)
|
|||
op->len > block_size-op->offset ||
|
||||
(op->len % disk_alignment)
|
||||
)) ||
|
||||
readonly && op->opcode != BS_OP_READ && op->opcode != BS_OP_LIST ||
|
||||
first && (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE))
|
||||
readonly && op->opcode != BS_OP_READ && op->opcode != BS_OP_LIST)
|
||||
{
|
||||
// Basic verification not passed
|
||||
op->retval = -EINVAL;
|
||||
|
@ -374,25 +374,12 @@ void blockstore_impl_t::enqueue_op(blockstore_op_t *op, bool first)
|
|||
std::function<void (blockstore_op_t*)>(op->callback)(op);
|
||||
return;
|
||||
}
|
||||
if (op->opcode == BS_OP_SYNC && immediate_commit == IMMEDIATE_ALL)
|
||||
{
|
||||
op->retval = 0;
|
||||
std::function<void (blockstore_op_t*)>(op->callback)(op);
|
||||
return;
|
||||
}
|
||||
// Call constructor without allocating memory. We'll call destructor before returning op back
|
||||
new ((void*)op->private_data) blockstore_op_private_t;
|
||||
PRIV(op)->wait_for = 0;
|
||||
PRIV(op)->op_state = 0;
|
||||
PRIV(op)->pending_ops = 0;
|
||||
if (!first)
|
||||
{
|
||||
submit_queue.push_back(op);
|
||||
}
|
||||
else
|
||||
{
|
||||
submit_queue.push_front(op);
|
||||
}
|
||||
submit_queue.push_back(op);
|
||||
ringloop->wakeup();
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
||||
|
@ -30,12 +30,13 @@
|
|||
#define BS_ST_BIG_WRITE 0x02
|
||||
#define BS_ST_DELETE 0x03
|
||||
|
||||
#define BS_ST_WAIT_BIG 0x10
|
||||
#define BS_ST_IN_FLIGHT 0x20
|
||||
#define BS_ST_SUBMITTED 0x30
|
||||
#define BS_ST_WRITTEN 0x40
|
||||
#define BS_ST_SYNCED 0x50
|
||||
#define BS_ST_STABLE 0x60
|
||||
#define BS_ST_WAIT_DEL 0x10
|
||||
#define BS_ST_WAIT_BIG 0x20
|
||||
#define BS_ST_IN_FLIGHT 0x30
|
||||
#define BS_ST_SUBMITTED 0x40
|
||||
#define BS_ST_WRITTEN 0x50
|
||||
#define BS_ST_SYNCED 0x60
|
||||
#define BS_ST_STABLE 0x70
|
||||
|
||||
#define BS_ST_INSTANT 0x100
|
||||
|
||||
|
@ -76,7 +77,8 @@
|
|||
|
||||
#include "blockstore_journal.h"
|
||||
|
||||
// 24 bytes + block bitmap per "clean" entry on disk with fixed metadata tables
|
||||
// 32 bytes = 24 bytes + block bitmap (4 bytes by default) + external attributes (also bitmap, 4 bytes by default)
|
||||
// per "clean" entry on disk with fixed metadata tables
|
||||
// FIXME: maybe add crc32's to metadata
|
||||
struct __attribute__((__packed__)) clean_disk_entry
|
||||
{
|
||||
|
@ -92,7 +94,7 @@ struct __attribute__((__packed__)) clean_entry
|
|||
uint64_t location;
|
||||
};
|
||||
|
||||
// 56 = 24 + 32 bytes per dirty entry in memory (obj_ver_id => dirty_entry)
|
||||
// 64 = 24 + 40 bytes per dirty entry in memory (obj_ver_id => dirty_entry)
|
||||
struct __attribute__((__packed__)) dirty_entry
|
||||
{
|
||||
uint32_t state;
|
||||
|
@ -101,6 +103,7 @@ struct __attribute__((__packed__)) dirty_entry
|
|||
uint32_t offset; // data offset within object (stripe)
|
||||
uint32_t len; // data length
|
||||
uint64_t journal_sector; // journal sector used for this entry
|
||||
void* bitmap; // either external bitmap itself when it fits, or a pointer to it when it doesn't
|
||||
};
|
||||
|
||||
// - Sync must be submitted after previous writes/deletes (not before!)
|
||||
|
@ -153,12 +156,12 @@ struct blockstore_op_private_t
|
|||
|
||||
// Write
|
||||
struct iovec iov_zerofill[3];
|
||||
// Warning: must not have a default value here because it's written to before calling constructor in blockstore_write.cpp O_o
|
||||
uint64_t real_version;
|
||||
|
||||
// Sync
|
||||
std::vector<obj_ver_id> sync_big_writes, sync_small_writes;
|
||||
int sync_small_checked, sync_big_checked;
|
||||
std::list<blockstore_op_t*>::iterator in_progress_ptr;
|
||||
int prev_sync_count;
|
||||
};
|
||||
|
||||
// https://github.com/algorithm-ninja/cpp-btree
|
||||
|
@ -196,7 +199,10 @@ class blockstore_impl_t
|
|||
// Suitable only for server SSDs with capacitors, requires disabled data and journal fsyncs
|
||||
int immediate_commit = IMMEDIATE_NONE;
|
||||
bool inmemory_meta = false;
|
||||
int flusher_count;
|
||||
// Maximum flusher count
|
||||
unsigned flusher_count;
|
||||
// Maximum queue depth
|
||||
unsigned max_write_iodepth = 128;
|
||||
/******* END OF OPTIONS *******/
|
||||
|
||||
struct ring_consumer_t ring_consumer;
|
||||
|
@ -204,9 +210,8 @@ class blockstore_impl_t
|
|||
blockstore_clean_db_t clean_db;
|
||||
uint8_t *clean_bitmap = NULL;
|
||||
blockstore_dirty_db_t dirty_db;
|
||||
std::list<blockstore_op_t*> submit_queue; // FIXME: funny thing is that vector is better here
|
||||
std::vector<blockstore_op_t*> submit_queue;
|
||||
std::vector<obj_ver_id> unsynced_big_writes, unsynced_small_writes;
|
||||
std::list<blockstore_op_t*> in_progress_syncs; // ...and probably here, too
|
||||
allocator *data_alloc = NULL;
|
||||
uint8_t *zero_object;
|
||||
|
||||
|
@ -223,6 +228,7 @@ class blockstore_impl_t
|
|||
|
||||
struct journal_t journal;
|
||||
journal_flusher_t *flusher;
|
||||
int write_iodepth = 0;
|
||||
|
||||
bool live = false, queue_stall = false;
|
||||
ring_loop_t *ringloop;
|
||||
|
@ -245,6 +251,7 @@ class blockstore_impl_t
|
|||
void open_data();
|
||||
void open_meta();
|
||||
void open_journal();
|
||||
uint8_t* get_clean_entry_bitmap(uint64_t block_loc, int offset);
|
||||
|
||||
// Asynchronous init
|
||||
int initialized;
|
||||
|
@ -264,6 +271,7 @@ class blockstore_impl_t
|
|||
|
||||
// Write
|
||||
bool enqueue_write(blockstore_op_t *op);
|
||||
void cancel_all_writes(blockstore_op_t *op, blockstore_dirty_db_t::iterator dirty_it, int retval);
|
||||
int dequeue_write(blockstore_op_t *op);
|
||||
int dequeue_del(blockstore_op_t *op);
|
||||
int continue_write(blockstore_op_t *op);
|
||||
|
@ -271,11 +279,9 @@ class blockstore_impl_t
|
|||
void handle_write_event(ring_data_t *data, blockstore_op_t *op);
|
||||
|
||||
// Sync
|
||||
int dequeue_sync(blockstore_op_t *op);
|
||||
int continue_sync(blockstore_op_t *op, bool queue_has_in_progress_sync);
|
||||
void handle_sync_event(ring_data_t *data, blockstore_op_t *op);
|
||||
int continue_sync(blockstore_op_t *op);
|
||||
void ack_one_sync(blockstore_op_t *op);
|
||||
int ack_sync(blockstore_op_t *op);
|
||||
void ack_sync(blockstore_op_t *op);
|
||||
|
||||
// Stabilize
|
||||
int dequeue_stable(blockstore_op_t *op);
|
||||
|
@ -315,13 +321,16 @@ public:
|
|||
bool is_stalled();
|
||||
|
||||
// Submission
|
||||
void enqueue_op(blockstore_op_t *op, bool first = false);
|
||||
void enqueue_op(blockstore_op_t *op);
|
||||
|
||||
// Unstable writes are added here (map of object_id -> version)
|
||||
std::unordered_map<object_id, uint64_t> unstable_writes;
|
||||
|
||||
// Space usage statistics
|
||||
std::map<uint64_t, uint64_t> inode_space_stats;
|
||||
|
||||
inline uint32_t get_block_size() { return block_size; }
|
||||
inline uint64_t get_block_count() { return block_count; }
|
||||
inline uint64_t get_free_block_count() { return data_alloc->get_free_count(); }
|
||||
inline uint32_t get_disk_alignment() { return disk_alignment; }
|
||||
inline uint32_t get_bitmap_granularity() { return disk_alignment; }
|
||||
};
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "blockstore_impl.h"
|
||||
|
||||
|
@ -100,7 +100,7 @@ void blockstore_init_meta::handle_entries(void* entries, unsigned count, int blo
|
|||
clean_disk_entry *entry = (clean_disk_entry*)(entries + i*bs->clean_entry_size);
|
||||
if (!bs->inmemory_meta && bs->clean_entry_bitmap_size)
|
||||
{
|
||||
memcpy(bs->clean_bitmap + (done_cnt+i)*bs->clean_entry_bitmap_size, &entry->bitmap, bs->clean_entry_bitmap_size);
|
||||
memcpy(bs->clean_bitmap + (done_cnt+i)*2*bs->clean_entry_bitmap_size, &entry->bitmap, 2*bs->clean_entry_bitmap_size);
|
||||
}
|
||||
if (entry->oid.inode > 0)
|
||||
{
|
||||
|
@ -111,10 +111,14 @@ void blockstore_init_meta::handle_entries(void* entries, unsigned count, int blo
|
|||
{
|
||||
// free the previous block
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("Free block %lu (new location is %lu)\n", clean_it->second.location >> block_order, done_cnt+i >> block_order);
|
||||
printf("Free block %lu (new location is %lu)\n", clean_it->second.location >> block_order, done_cnt+i);
|
||||
#endif
|
||||
bs->data_alloc->set(clean_it->second.location >> block_order, false);
|
||||
}
|
||||
else
|
||||
{
|
||||
bs->inode_space_stats[entry->oid.inode] += bs->block_size;
|
||||
}
|
||||
entries_loaded++;
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("Allocate block (clean entry) %lu: %lx:%lx v%lu\n", done_cnt+i, entry->oid.inode, entry->oid.stripe, entry->version);
|
||||
|
@ -530,6 +534,21 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
|
|||
.oid = je->small_write.oid,
|
||||
.version = je->small_write.version,
|
||||
};
|
||||
void *bmp = (void*)je + sizeof(journal_entry_small_write);
|
||||
if (bs->clean_entry_bitmap_size <= sizeof(void*))
|
||||
{
|
||||
memcpy(&bmp, bmp, bs->clean_entry_bitmap_size);
|
||||
}
|
||||
else if (!bs->journal.inmemory)
|
||||
{
|
||||
// FIXME Using large blockstore objects and not keeping journal in memory
|
||||
// will result in a lot of small allocations for entry bitmaps. This can
|
||||
// only be fixed by using a patched map with dynamic entry size, but not
|
||||
// the btree_map, because it doesn't keep iterators valid all the time.
|
||||
void *bmp_cp = malloc_or_die(bs->clean_entry_bitmap_size);
|
||||
memcpy(bmp_cp, bmp, bs->clean_entry_bitmap_size);
|
||||
bmp = bmp_cp;
|
||||
}
|
||||
bs->dirty_db.emplace(ov, (dirty_entry){
|
||||
.state = (BS_ST_SMALL_WRITE | BS_ST_SYNCED),
|
||||
.flags = 0,
|
||||
|
@ -537,6 +556,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
|
|||
.offset = je->small_write.offset,
|
||||
.len = je->small_write.len,
|
||||
.journal_sector = proc_pos,
|
||||
.bitmap = bmp,
|
||||
});
|
||||
bs->journal.used_sectors[proc_pos]++;
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
|
@ -557,11 +577,56 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
|
|||
{
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf(
|
||||
"je_big_write%s oid=%lx:%lx ver=%lu loc=%08lx\n",
|
||||
"je_big_write%s oid=%lx:%lx ver=%lu loc=%lu\n",
|
||||
je->type == JE_BIG_WRITE_INSTANT ? "_instant" : "",
|
||||
je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location
|
||||
je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location >> bs->block_order
|
||||
);
|
||||
#endif
|
||||
auto dirty_it = bs->dirty_db.upper_bound((obj_ver_id){
|
||||
.oid = je->big_write.oid,
|
||||
.version = UINT64_MAX,
|
||||
});
|
||||
if (dirty_it != bs->dirty_db.begin() && bs->dirty_db.size() > 0)
|
||||
{
|
||||
dirty_it--;
|
||||
if (dirty_it->first.oid == je->big_write.oid &&
|
||||
dirty_it->first.version >= je->big_write.version &&
|
||||
(dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_DELETE)
|
||||
{
|
||||
// It is allowed to overwrite a deleted object with a
|
||||
// version number smaller than deletion version number,
|
||||
// because the presence of a BIG_WRITE entry means that
|
||||
// its data and metadata are already flushed.
|
||||
// We don't know if newer versions are flushed, but
|
||||
// the previous delete definitely is.
|
||||
// So we flush previous dirty entries, but retain the clean one.
|
||||
// This feature is required for writes happening shortly
|
||||
// after deletes.
|
||||
auto dirty_end = dirty_it;
|
||||
dirty_end++;
|
||||
while (1)
|
||||
{
|
||||
if (dirty_it == bs->dirty_db.begin())
|
||||
{
|
||||
break;
|
||||
}
|
||||
dirty_it--;
|
||||
if (dirty_it->first.oid != je->big_write.oid)
|
||||
{
|
||||
dirty_it++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
auto clean_it = bs->clean_db.find(je->big_write.oid);
|
||||
bs->erase_dirty(
|
||||
dirty_it, dirty_end,
|
||||
clean_it != bs->clean_db.end() ? clean_it->second.location : UINT64_MAX
|
||||
);
|
||||
// Remove it from the flusher's queue, too
|
||||
// Otherwise it may end up referring to a small unstable write after reading the rest of the journal
|
||||
bs->flusher->remove_flush(je->big_write.oid);
|
||||
}
|
||||
}
|
||||
auto clean_it = bs->clean_db.find(je->big_write.oid);
|
||||
if (clean_it == bs->clean_db.end() ||
|
||||
clean_it->second.version < je->big_write.version)
|
||||
|
@ -571,6 +636,21 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
|
|||
.oid = je->big_write.oid,
|
||||
.version = je->big_write.version,
|
||||
};
|
||||
void *bmp = (void*)je + sizeof(journal_entry_big_write);
|
||||
if (bs->clean_entry_bitmap_size <= sizeof(void*))
|
||||
{
|
||||
memcpy(&bmp, bmp, bs->clean_entry_bitmap_size);
|
||||
}
|
||||
else if (!bs->journal.inmemory)
|
||||
{
|
||||
// FIXME Using large blockstore objects and not keeping journal in memory
|
||||
// will result in a lot of small allocations for entry bitmaps. This can
|
||||
// only be fixed by using a patched map with dynamic entry size, but not
|
||||
// the btree_map, because it doesn't keep iterators valid all the time.
|
||||
void *bmp_cp = malloc_or_die(bs->clean_entry_bitmap_size);
|
||||
memcpy(bmp_cp, bmp, bs->clean_entry_bitmap_size);
|
||||
bmp = bmp_cp;
|
||||
}
|
||||
bs->dirty_db.emplace(ov, (dirty_entry){
|
||||
.state = (BS_ST_BIG_WRITE | BS_ST_SYNCED),
|
||||
.flags = 0,
|
||||
|
@ -578,6 +658,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
|
|||
.offset = je->big_write.offset,
|
||||
.len = je->big_write.len,
|
||||
.journal_sector = proc_pos,
|
||||
.bitmap = bmp,
|
||||
});
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("Allocate block %lu\n", je->big_write.location >> bs->block_order);
|
||||
|
@ -628,7 +709,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
|
|||
printf("je_delete oid=%lx:%lx ver=%lu\n", je->del.oid.inode, je->del.oid.stripe, je->del.version);
|
||||
#endif
|
||||
auto clean_it = bs->clean_db.find(je->del.oid);
|
||||
if (clean_it == bs->clean_db.end() ||
|
||||
if (clean_it != bs->clean_db.end() &&
|
||||
clean_it->second.version < je->del.version)
|
||||
{
|
||||
// oid, version
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
|
@ -1,12 +1,12 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "blockstore_impl.h"
|
||||
|
||||
blockstore_journal_check_t::blockstore_journal_check_t(blockstore_impl_t *bs)
|
||||
{
|
||||
this->bs = bs;
|
||||
sectors_required = 0;
|
||||
sectors_to_write = 0;
|
||||
next_pos = bs->journal.next_free;
|
||||
next_sector = bs->journal.cur_sector;
|
||||
first_sector = -1;
|
||||
|
@ -20,23 +20,26 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
|
|||
int required = entries_required;
|
||||
while (1)
|
||||
{
|
||||
int fits = bs->journal.no_same_sector_overwrites && bs->journal.sector_info[next_sector].written
|
||||
int fits = bs->journal.no_same_sector_overwrites && next_pos == bs->journal.next_free && bs->journal.sector_info[next_sector].written
|
||||
? 0
|
||||
: (bs->journal.block_size - next_in_pos) / size;
|
||||
if (fits > 0)
|
||||
{
|
||||
if (fits > required)
|
||||
{
|
||||
fits = required;
|
||||
}
|
||||
if (first_sector == -1)
|
||||
{
|
||||
first_sector = next_sector;
|
||||
}
|
||||
required -= fits;
|
||||
next_in_pos += fits * size;
|
||||
sectors_required++;
|
||||
sectors_to_write++;
|
||||
}
|
||||
else if (bs->journal.sector_info[next_sector].dirty)
|
||||
{
|
||||
// sectors_required is more like "sectors to write"
|
||||
sectors_required++;
|
||||
sectors_to_write++;
|
||||
}
|
||||
if (required <= 0)
|
||||
{
|
||||
|
@ -59,7 +62,7 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
|
|||
" is too small for a batch of "+std::to_string(entries_required)+" entries of "+std::to_string(size)+" bytes"
|
||||
);
|
||||
}
|
||||
if (bs->journal.sector_info[next_sector].usage_count > 0 ||
|
||||
if (bs->journal.sector_info[next_sector].flush_count > 0 ||
|
||||
bs->journal.sector_info[next_sector].dirty)
|
||||
{
|
||||
// No memory buffer available. Wait for it.
|
||||
|
@ -71,17 +74,18 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
|
|||
dirty++;
|
||||
used++;
|
||||
}
|
||||
if (bs->journal.sector_info[i].usage_count > 0)
|
||||
if (bs->journal.sector_info[i].flush_count > 0)
|
||||
{
|
||||
used++;
|
||||
}
|
||||
}
|
||||
// In fact, it's even more rare than "ran out of journal space", so print a warning
|
||||
printf(
|
||||
"Ran out of journal sector buffers: %d/%lu buffers used (%d dirty), next buffer (%ld) is %s and flushed %lu times\n",
|
||||
"Ran out of journal sector buffers: %d/%lu buffers used (%d dirty), next buffer (%ld)"
|
||||
" is %s and flushed %lu times. Consider increasing \'journal_sector_buffer_count\'\n",
|
||||
used, bs->journal.sector_count, dirty, next_sector,
|
||||
bs->journal.sector_info[next_sector].dirty ? "dirty" : "not dirty",
|
||||
bs->journal.sector_info[next_sector].usage_count
|
||||
bs->journal.sector_info[next_sector].flush_count
|
||||
);
|
||||
PRIV(op)->wait_for = WAIT_JOURNAL_BUFFER;
|
||||
return 0;
|
||||
|
@ -100,11 +104,8 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
|
|||
{
|
||||
// No space in the journal. Wait until used_start changes.
|
||||
printf(
|
||||
"Ran out of journal space (free space: %lu bytes, sectors to write: %d)\n",
|
||||
(bs->journal.next_free >= bs->journal.used_start
|
||||
? bs->journal.len-bs->journal.block_size - (bs->journal.next_free-bs->journal.used_start)
|
||||
: bs->journal.used_start - bs->journal.next_free),
|
||||
sectors_required
|
||||
"Ran out of journal space (used_start=%08lx, next_free=%08lx, dirty_start=%08lx)\n",
|
||||
bs->journal.used_start, bs->journal.next_free, bs->journal.dirty_start
|
||||
);
|
||||
PRIV(op)->wait_for = WAIT_JOURNAL;
|
||||
bs->flusher->request_trim();
|
||||
|
@ -116,22 +117,21 @@ int blockstore_journal_check_t::check_available(blockstore_op_t *op, int entries
|
|||
|
||||
journal_entry* prefill_single_journal_entry(journal_t & journal, uint16_t type, uint32_t size)
|
||||
{
|
||||
if (journal.block_size - journal.in_sector_pos < size ||
|
||||
journal.no_same_sector_overwrites && journal.sector_info[journal.cur_sector].written)
|
||||
if (!journal.entry_fits(size))
|
||||
{
|
||||
assert(!journal.sector_info[journal.cur_sector].dirty);
|
||||
// Move to the next journal sector
|
||||
journal.sector_info[journal.cur_sector].written = false;
|
||||
if (journal.sector_info[journal.cur_sector].usage_count > 0)
|
||||
if (journal.sector_info[journal.cur_sector].flush_count > 0)
|
||||
{
|
||||
// Also select next sector buffer in memory
|
||||
journal.cur_sector = ((journal.cur_sector + 1) % journal.sector_count);
|
||||
assert(!journal.sector_info[journal.cur_sector].usage_count);
|
||||
assert(!journal.sector_info[journal.cur_sector].flush_count);
|
||||
}
|
||||
else
|
||||
{
|
||||
journal.dirty_start = journal.next_free;
|
||||
}
|
||||
journal.sector_info[journal.cur_sector].written = false;
|
||||
journal.sector_info[journal.cur_sector].offset = journal.next_free;
|
||||
journal.in_sector_pos = 0;
|
||||
journal.next_free = (journal.next_free+journal.block_size) < journal.len ? journal.next_free + journal.block_size : journal.block_size;
|
||||
|
@ -157,7 +157,7 @@ void prepare_journal_sector_write(journal_t & journal, int cur_sector, io_uring_
|
|||
{
|
||||
journal.sector_info[cur_sector].dirty = false;
|
||||
journal.sector_info[cur_sector].written = true;
|
||||
journal.sector_info[cur_sector].usage_count++;
|
||||
journal.sector_info[cur_sector].flush_count++;
|
||||
ring_data_t *data = ((ring_data_t*)sqe->user_data);
|
||||
data->iov = (struct iovec){
|
||||
(journal.inmemory
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
||||
|
@ -54,6 +54,9 @@ struct __attribute__((__packed__)) journal_entry_small_write
|
|||
// data_offset is its offset within journal
|
||||
uint64_t data_offset;
|
||||
uint32_t crc32_data;
|
||||
// small_write and big_write entries are followed by the "external" bitmap
|
||||
// its size is dynamic and included in journal entry's <size> field
|
||||
uint8_t bitmap[];
|
||||
};
|
||||
|
||||
struct __attribute__((__packed__)) journal_entry_big_write
|
||||
|
@ -68,6 +71,9 @@ struct __attribute__((__packed__)) journal_entry_big_write
|
|||
uint32_t offset;
|
||||
uint32_t len;
|
||||
uint64_t location;
|
||||
// small_write and big_write entries are followed by the "external" bitmap
|
||||
// its size is dynamic and included in journal entry's <size> field
|
||||
uint8_t bitmap[];
|
||||
};
|
||||
|
||||
struct __attribute__((__packed__)) journal_entry_stable
|
||||
|
@ -133,7 +139,7 @@ inline uint32_t je_crc32(journal_entry *je)
|
|||
struct journal_sector_info_t
|
||||
{
|
||||
uint64_t offset;
|
||||
uint64_t usage_count;
|
||||
uint64_t flush_count;
|
||||
bool written;
|
||||
bool dirty;
|
||||
};
|
||||
|
@ -170,13 +176,18 @@ struct journal_t
|
|||
~journal_t();
|
||||
bool trim();
|
||||
uint64_t get_trim_pos();
|
||||
inline bool entry_fits(int size)
|
||||
{
|
||||
return !(block_size - in_sector_pos < size ||
|
||||
no_same_sector_overwrites && sector_info[cur_sector].written);
|
||||
}
|
||||
};
|
||||
|
||||
struct blockstore_journal_check_t
|
||||
{
|
||||
blockstore_impl_t *bs;
|
||||
uint64_t next_pos, next_sector, next_in_pos;
|
||||
int sectors_required, first_sector;
|
||||
int sectors_to_write, first_sector;
|
||||
bool right_dir; // writing to the end or the beginning of the ring buffer
|
||||
|
||||
blockstore_journal_check_t(blockstore_impl_t *bs);
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include <sys/file.h>
|
||||
#include "blockstore_impl.h"
|
||||
|
@ -70,6 +70,7 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config)
|
|||
meta_block_size = strtoull(config["meta_block_size"].c_str(), NULL, 10);
|
||||
bitmap_granularity = strtoull(config["bitmap_granularity"].c_str(), NULL, 10);
|
||||
flusher_count = strtoull(config["flusher_count"].c_str(), NULL, 10);
|
||||
max_write_iodepth = strtoull(config["max_write_iodepth"].c_str(), NULL, 10);
|
||||
// Validate
|
||||
if (!block_size)
|
||||
{
|
||||
|
@ -83,13 +84,17 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config)
|
|||
{
|
||||
flusher_count = 32;
|
||||
}
|
||||
if (!max_write_iodepth)
|
||||
{
|
||||
max_write_iodepth = 128;
|
||||
}
|
||||
if (!disk_alignment)
|
||||
{
|
||||
disk_alignment = 4096;
|
||||
}
|
||||
else if (disk_alignment % MEM_ALIGNMENT)
|
||||
{
|
||||
throw std::runtime_error("disk_alingment must be a multiple of "+std::to_string(MEM_ALIGNMENT));
|
||||
throw std::runtime_error("disk_alignment must be a multiple of "+std::to_string(MEM_ALIGNMENT));
|
||||
}
|
||||
if (!journal_block_size)
|
||||
{
|
||||
|
@ -113,7 +118,7 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config)
|
|||
}
|
||||
if (!bitmap_granularity)
|
||||
{
|
||||
bitmap_granularity = 4096;
|
||||
bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
|
||||
}
|
||||
else if (bitmap_granularity % disk_alignment)
|
||||
{
|
||||
|
@ -165,7 +170,7 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config)
|
|||
}
|
||||
// init some fields
|
||||
clean_entry_bitmap_size = block_size / bitmap_granularity / 8;
|
||||
clean_entry_size = sizeof(clean_disk_entry) + clean_entry_bitmap_size;
|
||||
clean_entry_size = sizeof(clean_disk_entry) + 2*clean_entry_bitmap_size;
|
||||
journal.block_size = journal_block_size;
|
||||
journal.next_free = journal_block_size;
|
||||
journal.used_start = journal_block_size;
|
||||
|
@ -232,7 +237,7 @@ void blockstore_impl_t::calc_lengths()
|
|||
}
|
||||
else if (clean_entry_bitmap_size)
|
||||
{
|
||||
clean_bitmap = (uint8_t*)malloc(block_count * clean_entry_bitmap_size);
|
||||
clean_bitmap = (uint8_t*)malloc(block_count * 2*clean_entry_bitmap_size);
|
||||
if (!clean_bitmap)
|
||||
throw std::runtime_error("Failed to allocate memory for the metadata sparse write bitmap");
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "blockstore_impl.h"
|
||||
|
||||
|
@ -94,6 +94,21 @@ endwhile:
|
|||
return 1;
|
||||
}
|
||||
|
||||
uint8_t* blockstore_impl_t::get_clean_entry_bitmap(uint64_t block_loc, int offset)
|
||||
{
|
||||
uint8_t *clean_entry_bitmap;
|
||||
uint64_t meta_loc = block_loc >> block_order;
|
||||
if (inmemory_meta)
|
||||
{
|
||||
uint64_t sector = (meta_loc / (meta_block_size / clean_entry_size)) * meta_block_size;
|
||||
uint64_t pos = (meta_loc % (meta_block_size / clean_entry_size));
|
||||
clean_entry_bitmap = (uint8_t*)(metadata_buffer + sector + pos*clean_entry_size + sizeof(clean_disk_entry) + offset);
|
||||
}
|
||||
else
|
||||
clean_entry_bitmap = (uint8_t*)(clean_bitmap + meta_loc*2*clean_entry_bitmap_size + offset);
|
||||
return clean_entry_bitmap;
|
||||
}
|
||||
|
||||
int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
|
||||
{
|
||||
auto clean_it = clean_db.find(read_op->oid);
|
||||
|
@ -112,7 +127,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
|
|||
read_op->version = 0;
|
||||
read_op->retval = read_op->len;
|
||||
FINISH_OP(read_op);
|
||||
return 1;
|
||||
return 2;
|
||||
}
|
||||
uint64_t fulfilled = 0;
|
||||
PRIV(read_op)->pending_ops = 0;
|
||||
|
@ -134,6 +149,11 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
|
|||
if (!result_version)
|
||||
{
|
||||
result_version = dirty_it->first.version;
|
||||
if (read_op->bitmap)
|
||||
{
|
||||
void *bmp_ptr = (clean_entry_bitmap_size > sizeof(void*) ? dirty_it->second.bitmap : &dirty_it->second.bitmap);
|
||||
memcpy(read_op->bitmap, bmp_ptr, clean_entry_bitmap_size);
|
||||
}
|
||||
}
|
||||
if (!fulfill_read(read_op, fulfilled, dirty.offset, dirty.offset + dirty.len,
|
||||
dirty.state, dirty_it->first.version, dirty.location + (IS_JOURNAL(dirty.state) ? 0 : dirty.offset)))
|
||||
|
@ -155,6 +175,11 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
|
|||
if (!result_version)
|
||||
{
|
||||
result_version = clean_it->second.version;
|
||||
if (read_op->bitmap)
|
||||
{
|
||||
void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, clean_entry_bitmap_size);
|
||||
memcpy(read_op->bitmap, bmp_ptr, clean_entry_bitmap_size);
|
||||
}
|
||||
}
|
||||
if (fulfilled < read_op->len)
|
||||
{
|
||||
|
@ -169,18 +194,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
|
|||
}
|
||||
else
|
||||
{
|
||||
uint64_t meta_loc = clean_it->second.location >> block_order;
|
||||
uint8_t *clean_entry_bitmap;
|
||||
if (inmemory_meta)
|
||||
{
|
||||
uint64_t sector = (meta_loc / (meta_block_size / clean_entry_size)) * meta_block_size;
|
||||
uint64_t pos = (meta_loc % (meta_block_size / clean_entry_size));
|
||||
clean_entry_bitmap = (uint8_t*)(metadata_buffer + sector + pos*clean_entry_size + sizeof(clean_disk_entry));
|
||||
}
|
||||
else
|
||||
{
|
||||
clean_entry_bitmap = (uint8_t*)(clean_bitmap + meta_loc*clean_entry_bitmap_size);
|
||||
}
|
||||
uint8_t *clean_entry_bitmap = get_clean_entry_bitmap(clean_it->second.location, 0);
|
||||
uint64_t bmp_start = 0, bmp_end = 0, bmp_size = block_size/bitmap_granularity;
|
||||
while (bmp_start < bmp_size)
|
||||
{
|
||||
|
@ -191,8 +205,8 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
|
|||
if (bmp_end > bmp_start)
|
||||
{
|
||||
// fill with zeroes
|
||||
fulfill_read(read_op, fulfilled, bmp_start * bitmap_granularity,
|
||||
bmp_end * bitmap_granularity, (BS_ST_DELETE | BS_ST_STABLE), 0, 0);
|
||||
assert(fulfill_read(read_op, fulfilled, bmp_start * bitmap_granularity,
|
||||
bmp_end * bitmap_granularity, (BS_ST_DELETE | BS_ST_STABLE), 0, 0));
|
||||
}
|
||||
bmp_start = bmp_end;
|
||||
while (clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7)) && bmp_end < bmp_size)
|
||||
|
@ -218,7 +232,7 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
|
|||
else if (fulfilled < read_op->len)
|
||||
{
|
||||
// fill remaining parts with zeroes
|
||||
fulfill_read(read_op, fulfilled, 0, block_size, (BS_ST_DELETE | BS_ST_STABLE), 0, 0);
|
||||
assert(fulfill_read(read_op, fulfilled, 0, block_size, (BS_ST_DELETE | BS_ST_STABLE), 0, 0));
|
||||
}
|
||||
assert(fulfilled == read_op->len);
|
||||
read_op->version = result_version;
|
||||
|
@ -232,10 +246,10 @@ int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
|
|||
}
|
||||
read_op->retval = read_op->len;
|
||||
FINISH_OP(read_op);
|
||||
return 1;
|
||||
return 2;
|
||||
}
|
||||
read_op->retval = 0;
|
||||
return 1;
|
||||
return 2;
|
||||
}
|
||||
|
||||
void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op)
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "blockstore_impl.h"
|
||||
|
||||
|
@ -50,7 +50,7 @@ skip_ov:
|
|||
{
|
||||
op->retval = -EBUSY;
|
||||
FINISH_OP(op);
|
||||
return 1;
|
||||
return 2;
|
||||
}
|
||||
if (dirty_it == dirty_db.begin())
|
||||
{
|
||||
|
@ -66,7 +66,7 @@ skip_ov:
|
|||
// Already rolled back
|
||||
op->retval = 0;
|
||||
FINISH_OP(op);
|
||||
return 1;
|
||||
return 2;
|
||||
}
|
||||
// Check journal space
|
||||
blockstore_journal_check_t space_check(this);
|
||||
|
@ -75,44 +75,35 @@ skip_ov:
|
|||
return 0;
|
||||
}
|
||||
// There is sufficient space. Get SQEs
|
||||
struct io_uring_sqe *sqe[space_check.sectors_required];
|
||||
for (i = 0; i < space_check.sectors_required; i++)
|
||||
struct io_uring_sqe *sqe[space_check.sectors_to_write];
|
||||
for (i = 0; i < space_check.sectors_to_write; i++)
|
||||
{
|
||||
BS_SUBMIT_GET_SQE_DECL(sqe[i]);
|
||||
}
|
||||
// Prepare and submit journal entries
|
||||
auto cb = [this, op](ring_data_t *data) { handle_rollback_event(data, op); };
|
||||
int s = 0, cur_sector = -1;
|
||||
if ((journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_rollback) &&
|
||||
journal.sector_info[journal.cur_sector].dirty)
|
||||
{
|
||||
if (cur_sector == -1)
|
||||
PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
cur_sector = journal.cur_sector;
|
||||
prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
|
||||
}
|
||||
for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
|
||||
{
|
||||
if (!journal.entry_fits(sizeof(journal_entry_rollback)) &&
|
||||
journal.sector_info[journal.cur_sector].dirty)
|
||||
{
|
||||
if (cur_sector == -1)
|
||||
PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
prepare_journal_sector_write(journal, journal.cur_sector, sqe[s++], cb);
|
||||
cur_sector = journal.cur_sector;
|
||||
}
|
||||
journal_entry_rollback *je = (journal_entry_rollback*)
|
||||
prefill_single_journal_entry(journal, JE_ROLLBACK, sizeof(journal_entry_rollback));
|
||||
journal.sector_info[journal.cur_sector].dirty = false;
|
||||
je->oid = v->oid;
|
||||
je->version = v->version;
|
||||
je->crc32 = je_crc32((journal_entry*)je);
|
||||
journal.crc32_last = je->crc32;
|
||||
if (cur_sector != journal.cur_sector)
|
||||
{
|
||||
// Write previous sector. We should write the sector only after filling it,
|
||||
// because otherwise we'll write a lot more sectors in the "no_same_sector_overwrite" mode
|
||||
if (cur_sector != -1)
|
||||
prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
|
||||
else
|
||||
PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
cur_sector = journal.cur_sector;
|
||||
}
|
||||
}
|
||||
if (cur_sector != -1)
|
||||
prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
|
||||
prepare_journal_sector_write(journal, journal.cur_sector, sqe[s++], cb);
|
||||
assert(s == space_check.sectors_to_write);
|
||||
if (cur_sector == -1)
|
||||
PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
PRIV(op)->max_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
PRIV(op)->pending_ops = s;
|
||||
PRIV(op)->op_state = 1;
|
||||
|
@ -135,11 +126,8 @@ resume_2:
|
|||
resume_3:
|
||||
if (!disable_journal_fsync)
|
||||
{
|
||||
io_uring_sqe *sqe = get_sqe();
|
||||
if (!sqe)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
io_uring_sqe *sqe;
|
||||
BS_SUBMIT_GET_SQE_DECL(sqe);
|
||||
ring_data_t *data = ((ring_data_t*)sqe->user_data);
|
||||
my_uring_prep_fsync(sqe, journal.fd, IORING_FSYNC_DATASYNC);
|
||||
data->iov = { 0 };
|
||||
|
@ -160,7 +148,7 @@ resume_5:
|
|||
// Acknowledge op
|
||||
op->retval = 0;
|
||||
FINISH_OP(op);
|
||||
return 1;
|
||||
return 2;
|
||||
}
|
||||
|
||||
void blockstore_impl_t::mark_rolled_back(const obj_ver_id & ov)
|
||||
|
@ -175,10 +163,7 @@ void blockstore_impl_t::mark_rolled_back(const obj_ver_id & ov)
|
|||
auto rm_start = it;
|
||||
auto rm_end = it;
|
||||
it--;
|
||||
while (it->first.oid == ov.oid &&
|
||||
it->first.version > ov.version &&
|
||||
!IS_IN_FLIGHT(it->second.state) &&
|
||||
!IS_STABLE(it->second.state))
|
||||
while (1)
|
||||
{
|
||||
if (it->first.oid != ov.oid)
|
||||
break;
|
||||
|
@ -188,7 +173,7 @@ void blockstore_impl_t::mark_rolled_back(const obj_ver_id & ov)
|
|||
max_unstable = it->first.version;
|
||||
break;
|
||||
}
|
||||
else if (IS_STABLE(it->second.state))
|
||||
else if (IS_IN_FLIGHT(it->second.state) || IS_STABLE(it->second.state))
|
||||
break;
|
||||
// Remove entry
|
||||
rm_start = it;
|
||||
|
@ -199,14 +184,14 @@ void blockstore_impl_t::mark_rolled_back(const obj_ver_id & ov)
|
|||
if (rm_start != rm_end)
|
||||
{
|
||||
erase_dirty(rm_start, rm_end, UINT64_MAX);
|
||||
}
|
||||
auto unstab_it = unstable_writes.find(ov.oid);
|
||||
if (unstab_it != unstable_writes.end())
|
||||
{
|
||||
if (max_unstable == 0)
|
||||
unstable_writes.erase(unstab_it);
|
||||
else
|
||||
unstab_it->second = max_unstable;
|
||||
auto unstab_it = unstable_writes.find(ov.oid);
|
||||
if (unstab_it != unstable_writes.end())
|
||||
{
|
||||
if (max_unstable == 0)
|
||||
unstable_writes.erase(unstab_it);
|
||||
else
|
||||
unstab_it->second = max_unstable;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -225,19 +210,44 @@ void blockstore_impl_t::handle_rollback_event(ring_data_t *data, blockstore_op_t
|
|||
if (PRIV(op)->pending_ops == 0)
|
||||
{
|
||||
PRIV(op)->op_state++;
|
||||
if (!continue_rollback(op))
|
||||
{
|
||||
submit_queue.push_front(op);
|
||||
}
|
||||
ringloop->wakeup();
|
||||
}
|
||||
}
|
||||
|
||||
void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start, blockstore_dirty_db_t::iterator dirty_end, uint64_t clean_loc)
|
||||
{
|
||||
auto dirty_it = dirty_end;
|
||||
while (dirty_it != dirty_start)
|
||||
if (dirty_end == dirty_start)
|
||||
{
|
||||
return;
|
||||
}
|
||||
auto dirty_it = dirty_end;
|
||||
dirty_it--;
|
||||
if (IS_DELETE(dirty_it->second.state))
|
||||
{
|
||||
object_id oid = dirty_it->first.oid;
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("Unblock writes-after-delete %lx:%lx v%lx\n", oid.inode, oid.stripe, dirty_it->first.version);
|
||||
#endif
|
||||
dirty_it = dirty_end;
|
||||
// Unblock operations blocked by delete flushing
|
||||
uint32_t next_state = BS_ST_IN_FLIGHT;
|
||||
while (dirty_it != dirty_db.end() && dirty_it->first.oid == oid)
|
||||
{
|
||||
if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_DEL)
|
||||
{
|
||||
dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | next_state;
|
||||
if (IS_BIG_WRITE(dirty_it->second.state))
|
||||
{
|
||||
next_state = BS_ST_WAIT_BIG;
|
||||
}
|
||||
}
|
||||
dirty_it++;
|
||||
}
|
||||
dirty_it = dirty_end;
|
||||
dirty_it--;
|
||||
}
|
||||
while (1)
|
||||
{
|
||||
if (IS_BIG_WRITE(dirty_it->second.state) && dirty_it->second.location != clean_loc)
|
||||
{
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
|
@ -256,6 +266,16 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
|
|||
{
|
||||
journal.used_sectors.erase(dirty_it->second.journal_sector);
|
||||
}
|
||||
if (clean_entry_bitmap_size > sizeof(void*))
|
||||
{
|
||||
free(dirty_it->second.bitmap);
|
||||
dirty_it->second.bitmap = NULL;
|
||||
}
|
||||
if (dirty_it == dirty_start)
|
||||
{
|
||||
break;
|
||||
}
|
||||
dirty_it--;
|
||||
}
|
||||
dirty_db.erase(dirty_start, dirty_end);
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "blockstore_impl.h"
|
||||
|
||||
|
@ -60,7 +60,7 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
|
|||
// No such object version
|
||||
op->retval = -ENOENT;
|
||||
FINISH_OP(op);
|
||||
return 1;
|
||||
return 2;
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -77,7 +77,7 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
|
|||
// Object not synced yet. Caller must sync it first
|
||||
op->retval = -EBUSY;
|
||||
FINISH_OP(op);
|
||||
return 1;
|
||||
return 2;
|
||||
}
|
||||
else if (!IS_STABLE(dirty_it->second.state))
|
||||
{
|
||||
|
@ -89,7 +89,7 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
|
|||
// Already stable
|
||||
op->retval = 0;
|
||||
FINISH_OP(op);
|
||||
return 1;
|
||||
return 2;
|
||||
}
|
||||
// Check journal space
|
||||
blockstore_journal_check_t space_check(this);
|
||||
|
@ -98,45 +98,36 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
|
|||
return 0;
|
||||
}
|
||||
// There is sufficient space. Get SQEs
|
||||
struct io_uring_sqe *sqe[space_check.sectors_required];
|
||||
for (i = 0; i < space_check.sectors_required; i++)
|
||||
struct io_uring_sqe *sqe[space_check.sectors_to_write];
|
||||
for (i = 0; i < space_check.sectors_to_write; i++)
|
||||
{
|
||||
BS_SUBMIT_GET_SQE_DECL(sqe[i]);
|
||||
}
|
||||
// Prepare and submit journal entries
|
||||
auto cb = [this, op](ring_data_t *data) { handle_stable_event(data, op); };
|
||||
int s = 0, cur_sector = -1;
|
||||
if ((journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_stable) &&
|
||||
journal.sector_info[journal.cur_sector].dirty)
|
||||
{
|
||||
if (cur_sector == -1)
|
||||
PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
cur_sector = journal.cur_sector;
|
||||
prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
|
||||
}
|
||||
for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
|
||||
{
|
||||
// FIXME: Only stabilize versions that aren't stable yet
|
||||
if (!journal.entry_fits(sizeof(journal_entry_stable)) &&
|
||||
journal.sector_info[journal.cur_sector].dirty)
|
||||
{
|
||||
if (cur_sector == -1)
|
||||
PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
prepare_journal_sector_write(journal, journal.cur_sector, sqe[s++], cb);
|
||||
cur_sector = journal.cur_sector;
|
||||
}
|
||||
journal_entry_stable *je = (journal_entry_stable*)
|
||||
prefill_single_journal_entry(journal, JE_STABLE, sizeof(journal_entry_stable));
|
||||
journal.sector_info[journal.cur_sector].dirty = false;
|
||||
je->oid = v->oid;
|
||||
je->version = v->version;
|
||||
je->crc32 = je_crc32((journal_entry*)je);
|
||||
journal.crc32_last = je->crc32;
|
||||
if (cur_sector != journal.cur_sector)
|
||||
{
|
||||
// Write previous sector. We should write the sector only after filling it,
|
||||
// because otherwise we'll write a lot more sectors in the "no_same_sector_overwrite" mode
|
||||
if (cur_sector != -1)
|
||||
prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
|
||||
else
|
||||
PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
cur_sector = journal.cur_sector;
|
||||
}
|
||||
}
|
||||
if (cur_sector != -1)
|
||||
prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
|
||||
prepare_journal_sector_write(journal, journal.cur_sector, sqe[s++], cb);
|
||||
assert(s == space_check.sectors_to_write);
|
||||
if (cur_sector == -1)
|
||||
PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
PRIV(op)->max_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
PRIV(op)->pending_ops = s;
|
||||
PRIV(op)->op_state = 1;
|
||||
|
@ -159,11 +150,8 @@ resume_2:
|
|||
resume_3:
|
||||
if (!disable_journal_fsync)
|
||||
{
|
||||
io_uring_sqe *sqe = get_sqe();
|
||||
if (!sqe)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
io_uring_sqe *sqe;
|
||||
BS_SUBMIT_GET_SQE_DECL(sqe);
|
||||
ring_data_t *data = ((ring_data_t*)sqe->user_data);
|
||||
my_uring_prep_fsync(sqe, journal.fd, IORING_FSYNC_DATASYNC);
|
||||
data->iov = { 0 };
|
||||
|
@ -185,7 +173,7 @@ resume_5:
|
|||
// Acknowledge op
|
||||
op->retval = 0;
|
||||
FINISH_OP(op);
|
||||
return 1;
|
||||
return 2;
|
||||
}
|
||||
|
||||
void blockstore_impl_t::mark_stable(const obj_ver_id & v)
|
||||
|
@ -198,6 +186,15 @@ void blockstore_impl_t::mark_stable(const obj_ver_id & v)
|
|||
if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_SYNCED)
|
||||
{
|
||||
dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_STABLE;
|
||||
// Allocations and deletions are counted when they're stabilized
|
||||
if (IS_BIG_WRITE(dirty_it->second.state))
|
||||
{
|
||||
inode_space_stats[dirty_it->first.oid.inode] += block_size;
|
||||
}
|
||||
else if (IS_DELETE(dirty_it->second.state))
|
||||
{
|
||||
inode_space_stats[dirty_it->first.oid.inode] -= block_size;
|
||||
}
|
||||
}
|
||||
else if (IS_STABLE(dirty_it->second.state))
|
||||
{
|
||||
|
@ -213,9 +210,6 @@ void blockstore_impl_t::mark_stable(const obj_ver_id & v)
|
|||
break;
|
||||
}
|
||||
}
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("enqueue_flush %lx:%lx v%lu\n", v.oid.inode, v.oid.stripe, v.version);
|
||||
#endif
|
||||
flusher->enqueue_flush(v);
|
||||
}
|
||||
auto unstab_it = unstable_writes.find(v.oid);
|
||||
|
@ -240,9 +234,6 @@ void blockstore_impl_t::handle_stable_event(ring_data_t *data, blockstore_op_t *
|
|||
if (PRIV(op)->pending_ops == 0)
|
||||
{
|
||||
PRIV(op)->op_state++;
|
||||
if (!continue_stable(op))
|
||||
{
|
||||
submit_queue.push_front(op);
|
||||
}
|
||||
ringloop->wakeup();
|
||||
}
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "blockstore_impl.h"
|
||||
|
||||
|
@ -12,8 +12,15 @@
|
|||
#define SYNC_JOURNAL_SYNC_SENT 7
|
||||
#define SYNC_DONE 8
|
||||
|
||||
int blockstore_impl_t::dequeue_sync(blockstore_op_t *op)
|
||||
int blockstore_impl_t::continue_sync(blockstore_op_t *op, bool queue_has_in_progress_sync)
|
||||
{
|
||||
if (immediate_commit == IMMEDIATE_ALL)
|
||||
{
|
||||
// We can return immediately because sync is only dequeued after all previous writes
|
||||
op->retval = 0;
|
||||
FINISH_OP(op);
|
||||
return 2;
|
||||
}
|
||||
if (PRIV(op)->op_state == 0)
|
||||
{
|
||||
stop_sync_submitted = false;
|
||||
|
@ -29,34 +36,15 @@ int blockstore_impl_t::dequeue_sync(blockstore_op_t *op)
|
|||
PRIV(op)->op_state = SYNC_HAS_SMALL;
|
||||
else
|
||||
PRIV(op)->op_state = SYNC_DONE;
|
||||
// Always add sync to in_progress_syncs because we clear unsynced_big_writes and unsynced_small_writes
|
||||
PRIV(op)->prev_sync_count = in_progress_syncs.size();
|
||||
PRIV(op)->in_progress_ptr = in_progress_syncs.insert(in_progress_syncs.end(), op);
|
||||
}
|
||||
continue_sync(op);
|
||||
// Always dequeue because we always add syncs to in_progress_syncs
|
||||
return 1;
|
||||
}
|
||||
|
||||
int blockstore_impl_t::continue_sync(blockstore_op_t *op)
|
||||
{
|
||||
auto cb = [this, op](ring_data_t *data) { handle_sync_event(data, op); };
|
||||
if (PRIV(op)->op_state == SYNC_HAS_SMALL)
|
||||
{
|
||||
// No big writes, just fsync the journal
|
||||
for (; PRIV(op)->sync_small_checked < PRIV(op)->sync_small_writes.size(); PRIV(op)->sync_small_checked++)
|
||||
{
|
||||
if (IS_IN_FLIGHT(dirty_db[PRIV(op)->sync_small_writes[PRIV(op)->sync_small_checked]].state))
|
||||
{
|
||||
// Wait for small inflight writes to complete
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
if (journal.sector_info[journal.cur_sector].dirty)
|
||||
{
|
||||
// Write out the last journal sector if it happens to be dirty
|
||||
BS_SUBMIT_GET_ONLY_SQE(sqe);
|
||||
prepare_journal_sector_write(journal, journal.cur_sector, sqe, cb);
|
||||
prepare_journal_sector_write(journal, journal.cur_sector, sqe, [this, op](ring_data_t *data) { handle_sync_event(data, op); });
|
||||
PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
PRIV(op)->pending_ops = 1;
|
||||
PRIV(op)->op_state = SYNC_JOURNAL_WRITE_SENT;
|
||||
|
@ -69,21 +57,13 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
|
|||
}
|
||||
if (PRIV(op)->op_state == SYNC_HAS_BIG)
|
||||
{
|
||||
for (; PRIV(op)->sync_big_checked < PRIV(op)->sync_big_writes.size(); PRIV(op)->sync_big_checked++)
|
||||
{
|
||||
if (IS_IN_FLIGHT(dirty_db[PRIV(op)->sync_big_writes[PRIV(op)->sync_big_checked]].state))
|
||||
{
|
||||
// Wait for big inflight writes to complete
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
// 1st step: fsync data
|
||||
if (!disable_data_fsync)
|
||||
{
|
||||
BS_SUBMIT_GET_SQE(sqe, data);
|
||||
my_uring_prep_fsync(sqe, data_fd, IORING_FSYNC_DATASYNC);
|
||||
data->iov = { 0 };
|
||||
data->callback = cb;
|
||||
data->callback = [this, op](ring_data_t *data) { handle_sync_event(data, op); };
|
||||
PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 0;
|
||||
PRIV(op)->pending_ops = 1;
|
||||
PRIV(op)->op_state = SYNC_DATA_SYNC_SENT;
|
||||
|
@ -96,14 +76,6 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
|
|||
}
|
||||
if (PRIV(op)->op_state == SYNC_DATA_SYNC_DONE)
|
||||
{
|
||||
for (; PRIV(op)->sync_small_checked < PRIV(op)->sync_small_writes.size(); PRIV(op)->sync_small_checked++)
|
||||
{
|
||||
if (IS_IN_FLIGHT(dirty_db[PRIV(op)->sync_small_writes[PRIV(op)->sync_small_checked]].state))
|
||||
{
|
||||
// Wait for small inflight writes to complete
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
// 2nd step: Data device is synced, prepare & write journal entries
|
||||
// Check space in the journal and journal memory buffers
|
||||
blockstore_journal_check_t space_check(this);
|
||||
|
@ -112,30 +84,29 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
|
|||
return 0;
|
||||
}
|
||||
// Get SQEs. Don't bother about merging, submit each journal sector as a separate request
|
||||
struct io_uring_sqe *sqe[space_check.sectors_required];
|
||||
for (int i = 0; i < space_check.sectors_required; i++)
|
||||
struct io_uring_sqe *sqe[space_check.sectors_to_write];
|
||||
for (int i = 0; i < space_check.sectors_to_write; i++)
|
||||
{
|
||||
BS_SUBMIT_GET_SQE_DECL(sqe[i]);
|
||||
}
|
||||
// Prepare and submit journal entries
|
||||
auto it = PRIV(op)->sync_big_writes.begin();
|
||||
int s = 0, cur_sector = -1;
|
||||
if ((journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_big_write) &&
|
||||
journal.sector_info[journal.cur_sector].dirty)
|
||||
{
|
||||
if (cur_sector == -1)
|
||||
PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
cur_sector = journal.cur_sector;
|
||||
prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
|
||||
}
|
||||
while (it != PRIV(op)->sync_big_writes.end())
|
||||
{
|
||||
if (!journal.entry_fits(sizeof(journal_entry_big_write)) &&
|
||||
journal.sector_info[journal.cur_sector].dirty)
|
||||
{
|
||||
if (cur_sector == -1)
|
||||
PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
prepare_journal_sector_write(journal, journal.cur_sector, sqe[s++], [this, op](ring_data_t *data) { handle_sync_event(data, op); });
|
||||
cur_sector = journal.cur_sector;
|
||||
}
|
||||
journal_entry_big_write *je = (journal_entry_big_write*)prefill_single_journal_entry(
|
||||
journal, (dirty_db[*it].state & BS_ST_INSTANT) ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE,
|
||||
sizeof(journal_entry_big_write)
|
||||
);
|
||||
dirty_db[*it].journal_sector = journal.sector_info[journal.cur_sector].offset;
|
||||
journal.sector_info[journal.cur_sector].dirty = false;
|
||||
journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf(
|
||||
|
@ -152,19 +123,11 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
|
|||
je->crc32 = je_crc32((journal_entry*)je);
|
||||
journal.crc32_last = je->crc32;
|
||||
it++;
|
||||
if (cur_sector != journal.cur_sector)
|
||||
{
|
||||
// Write previous sector. We should write the sector only after filling it,
|
||||
// because otherwise we'll write a lot more sectors in the "no_same_sector_overwrite" mode
|
||||
if (cur_sector != -1)
|
||||
prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
|
||||
else
|
||||
PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
cur_sector = journal.cur_sector;
|
||||
}
|
||||
}
|
||||
if (cur_sector != -1)
|
||||
prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
|
||||
prepare_journal_sector_write(journal, journal.cur_sector, sqe[s++], [this, op](ring_data_t *data) { handle_sync_event(data, op); });
|
||||
assert(s == space_check.sectors_to_write);
|
||||
if (cur_sector == -1)
|
||||
PRIV(op)->min_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
PRIV(op)->max_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
PRIV(op)->pending_ops = s;
|
||||
PRIV(op)->op_state = SYNC_JOURNAL_WRITE_SENT;
|
||||
|
@ -177,7 +140,7 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
|
|||
BS_SUBMIT_GET_SQE(sqe, data);
|
||||
my_uring_prep_fsync(sqe, journal.fd, IORING_FSYNC_DATASYNC);
|
||||
data->iov = { 0 };
|
||||
data->callback = cb;
|
||||
data->callback = [this, op](ring_data_t *data) { handle_sync_event(data, op); };
|
||||
PRIV(op)->pending_ops = 1;
|
||||
PRIV(op)->op_state = SYNC_JOURNAL_SYNC_SENT;
|
||||
return 1;
|
||||
|
@ -187,9 +150,10 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
|
|||
PRIV(op)->op_state = SYNC_DONE;
|
||||
}
|
||||
}
|
||||
if (PRIV(op)->op_state == SYNC_DONE)
|
||||
if (PRIV(op)->op_state == SYNC_DONE && !queue_has_in_progress_sync)
|
||||
{
|
||||
return ack_sync(op);
|
||||
ack_sync(op);
|
||||
return 2;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
@ -221,42 +185,16 @@ void blockstore_impl_t::handle_sync_event(ring_data_t *data, blockstore_op_t *op
|
|||
else if (PRIV(op)->op_state == SYNC_JOURNAL_SYNC_SENT)
|
||||
{
|
||||
PRIV(op)->op_state = SYNC_DONE;
|
||||
ack_sync(op);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw std::runtime_error("BUG: unexpected sync op state");
|
||||
}
|
||||
ringloop->wakeup();
|
||||
}
|
||||
}
|
||||
|
||||
int blockstore_impl_t::ack_sync(blockstore_op_t *op)
|
||||
{
|
||||
if (PRIV(op)->op_state == SYNC_DONE && PRIV(op)->prev_sync_count == 0)
|
||||
{
|
||||
// Remove dependency of subsequent syncs
|
||||
auto it = PRIV(op)->in_progress_ptr;
|
||||
int done_syncs = 1;
|
||||
++it;
|
||||
// Acknowledge sync
|
||||
ack_one_sync(op);
|
||||
while (it != in_progress_syncs.end())
|
||||
{
|
||||
auto & next_sync = *it++;
|
||||
PRIV(next_sync)->prev_sync_count -= done_syncs;
|
||||
if (PRIV(next_sync)->prev_sync_count == 0 && PRIV(next_sync)->op_state == SYNC_DONE)
|
||||
{
|
||||
done_syncs++;
|
||||
// Acknowledge next_sync
|
||||
ack_one_sync(next_sync);
|
||||
}
|
||||
}
|
||||
return 2;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void blockstore_impl_t::ack_one_sync(blockstore_op_t *op)
|
||||
void blockstore_impl_t::ack_sync(blockstore_op_t *op)
|
||||
{
|
||||
// Handle states
|
||||
for (auto it = PRIV(op)->sync_big_writes.begin(); it != PRIV(op)->sync_big_writes.end(); it++)
|
||||
|
@ -304,7 +242,6 @@ void blockstore_impl_t::ack_one_sync(blockstore_op_t *op)
|
|||
}
|
||||
}
|
||||
}
|
||||
in_progress_syncs.erase(PRIV(op)->in_progress_ptr);
|
||||
op->retval = 0;
|
||||
FINISH_OP(op);
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "blockstore_impl.h"
|
||||
|
||||
|
@ -7,8 +7,13 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
|
|||
{
|
||||
// Check or assign version number
|
||||
bool found = false, deleted = false, is_del = (op->opcode == BS_OP_DELETE);
|
||||
bool is_inflight_big = false;
|
||||
bool wait_big = false, wait_del = false;
|
||||
void *bmp = NULL;
|
||||
uint64_t version = 1;
|
||||
if (!is_del && clean_entry_bitmap_size > sizeof(void*))
|
||||
{
|
||||
bmp = calloc_or_die(1, clean_entry_bitmap_size);
|
||||
}
|
||||
if (dirty_db.size() > 0)
|
||||
{
|
||||
auto dirty_it = dirty_db.upper_bound((obj_ver_id){
|
||||
|
@ -21,9 +26,14 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
|
|||
found = true;
|
||||
version = dirty_it->first.version + 1;
|
||||
deleted = IS_DELETE(dirty_it->second.state);
|
||||
is_inflight_big = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE
|
||||
wait_del = ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_DEL);
|
||||
wait_big = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE
|
||||
? !IS_SYNCED(dirty_it->second.state)
|
||||
: ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG);
|
||||
if (clean_entry_bitmap_size > sizeof(void*))
|
||||
memcpy(bmp, dirty_it->second.bitmap, clean_entry_bitmap_size);
|
||||
else
|
||||
bmp = dirty_it->second.bitmap;
|
||||
}
|
||||
}
|
||||
if (!found)
|
||||
|
@ -32,29 +42,55 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
|
|||
if (clean_it != clean_db.end())
|
||||
{
|
||||
version = clean_it->second.version + 1;
|
||||
void *bmp_ptr = get_clean_entry_bitmap(clean_it->second.location, clean_entry_bitmap_size);
|
||||
memcpy((clean_entry_bitmap_size > sizeof(void*) ? bmp : &bmp), bmp_ptr, clean_entry_bitmap_size);
|
||||
}
|
||||
else
|
||||
{
|
||||
deleted = true;
|
||||
}
|
||||
}
|
||||
if (op->version == 0)
|
||||
{
|
||||
op->version = version;
|
||||
}
|
||||
else if (op->version < version)
|
||||
{
|
||||
// Invalid version requested
|
||||
op->retval = -EEXIST;
|
||||
return false;
|
||||
}
|
||||
if (deleted && is_del)
|
||||
{
|
||||
// Already deleted
|
||||
op->retval = 0;
|
||||
return false;
|
||||
}
|
||||
if (is_inflight_big && !is_del && !deleted && op->len < block_size &&
|
||||
PRIV(op)->real_version = 0;
|
||||
if (op->version == 0)
|
||||
{
|
||||
op->version = version;
|
||||
}
|
||||
else if (op->version < version)
|
||||
{
|
||||
// Implicit operations must be added like that: DEL [FLUSH] BIG [SYNC] SMALL SMALL
|
||||
if (deleted || wait_del)
|
||||
{
|
||||
// It's allowed to write versions with low numbers over deletes
|
||||
// However, we have to flush those deletes first as we use version number for ordering
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("Write %lx:%lx v%lu over delete (real v%lu) offset=%u len=%u\n", op->oid.inode, op->oid.stripe, version, op->version, op->offset, op->len);
|
||||
#endif
|
||||
wait_del = true;
|
||||
PRIV(op)->real_version = op->version;
|
||||
op->version = version;
|
||||
flusher->unshift_flush((obj_ver_id){
|
||||
.oid = op->oid,
|
||||
.version = version-1,
|
||||
}, true);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Invalid version requested
|
||||
op->retval = -EEXIST;
|
||||
if (!is_del && clean_entry_bitmap_size > sizeof(void*))
|
||||
{
|
||||
free(bmp);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (wait_big && !is_del && !deleted && op->len < block_size &&
|
||||
immediate_commit != IMMEDIATE_ALL)
|
||||
{
|
||||
// Issue an additional sync so that the previous big write can reach the journal
|
||||
|
@ -69,31 +105,88 @@ bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
|
|||
#ifdef BLOCKSTORE_DEBUG
|
||||
if (is_del)
|
||||
printf("Delete %lx:%lx v%lu\n", op->oid.inode, op->oid.stripe, op->version);
|
||||
else
|
||||
else if (!wait_del)
|
||||
printf("Write %lx:%lx v%lu offset=%u len=%u\n", op->oid.inode, op->oid.stripe, op->version, op->offset, op->len);
|
||||
#endif
|
||||
// No strict need to add it into dirty_db here, it's just left
|
||||
// FIXME No strict need to add it into dirty_db here, it's just left
|
||||
// from the previous implementation where reads waited for writes
|
||||
uint32_t state;
|
||||
if (is_del)
|
||||
state = BS_ST_DELETE | BS_ST_IN_FLIGHT;
|
||||
else
|
||||
{
|
||||
state = (op->len == block_size || deleted ? BS_ST_BIG_WRITE : BS_ST_SMALL_WRITE);
|
||||
if (wait_del)
|
||||
state |= BS_ST_WAIT_DEL;
|
||||
else if (state == BS_ST_SMALL_WRITE && wait_big)
|
||||
state |= BS_ST_WAIT_BIG;
|
||||
else
|
||||
state |= BS_ST_IN_FLIGHT;
|
||||
if (op->opcode == BS_OP_WRITE_STABLE)
|
||||
state |= BS_ST_INSTANT;
|
||||
if (op->bitmap)
|
||||
{
|
||||
// Only allow to overwrite part of the object bitmap respective to the write's offset/len
|
||||
uint8_t *bmp_ptr = (uint8_t*)(clean_entry_bitmap_size > sizeof(void*) ? bmp : &bmp);
|
||||
uint32_t bit = op->offset/bitmap_granularity;
|
||||
uint32_t bits_left = op->len/bitmap_granularity;
|
||||
while (!(bit % 8) && bits_left > 8)
|
||||
{
|
||||
// Copy bytes
|
||||
bmp_ptr[bit/8] = ((uint8_t*)op->bitmap)[bit/8];
|
||||
bit += 8;
|
||||
bits_left -= 8;
|
||||
}
|
||||
while (bits_left > 0)
|
||||
{
|
||||
// Copy bits
|
||||
bmp_ptr[bit/8] = (bmp_ptr[bit/8] & ~(1 << (bit%8)))
|
||||
| (((uint8_t*)op->bitmap)[bit/8] & (1 << bit%8));
|
||||
bit++;
|
||||
bits_left--;
|
||||
}
|
||||
}
|
||||
}
|
||||
dirty_db.emplace((obj_ver_id){
|
||||
.oid = op->oid,
|
||||
.version = op->version,
|
||||
}, (dirty_entry){
|
||||
.state = (uint32_t)(
|
||||
is_del
|
||||
? (BS_ST_DELETE | BS_ST_IN_FLIGHT)
|
||||
: (op->opcode == BS_OP_WRITE_STABLE ? BS_ST_INSTANT : 0) | (op->len == block_size || deleted
|
||||
? (BS_ST_BIG_WRITE | BS_ST_IN_FLIGHT)
|
||||
: (is_inflight_big ? (BS_ST_SMALL_WRITE | BS_ST_WAIT_BIG) : (BS_ST_SMALL_WRITE | BS_ST_IN_FLIGHT)))
|
||||
),
|
||||
.state = state,
|
||||
.flags = 0,
|
||||
.location = 0,
|
||||
.offset = is_del ? 0 : op->offset,
|
||||
.len = is_del ? 0 : op->len,
|
||||
.journal_sector = 0,
|
||||
.bitmap = bmp,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
void blockstore_impl_t::cancel_all_writes(blockstore_op_t *op, blockstore_dirty_db_t::iterator dirty_it, int retval)
|
||||
{
|
||||
while (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid)
|
||||
{
|
||||
if (clean_entry_bitmap_size > sizeof(void*))
|
||||
free(dirty_it->second.bitmap);
|
||||
dirty_db.erase(dirty_it++);
|
||||
}
|
||||
bool found = false;
|
||||
for (auto other_op: submit_queue)
|
||||
{
|
||||
if (!found && other_op == op)
|
||||
found = true;
|
||||
else if (found && other_op->oid == op->oid &&
|
||||
(other_op->opcode == BS_OP_WRITE || other_op->opcode == BS_OP_WRITE_STABLE))
|
||||
{
|
||||
// Mark operations to cancel them
|
||||
PRIV(other_op)->real_version = UINT64_MAX;
|
||||
other_op->retval = retval;
|
||||
}
|
||||
}
|
||||
op->retval = retval;
|
||||
FINISH_OP(op);
|
||||
}
|
||||
|
||||
// First step of the write algorithm: dequeue operation and submit initial write(s)
|
||||
int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
|
||||
{
|
||||
|
@ -106,12 +199,46 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
|
|||
.version = op->version,
|
||||
});
|
||||
assert(dirty_it != dirty_db.end());
|
||||
if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) == BS_ST_WAIT_BIG)
|
||||
if ((dirty_it->second.state & BS_ST_WORKFLOW_MASK) < BS_ST_IN_FLIGHT)
|
||||
{
|
||||
// Don't dequeue
|
||||
return 0;
|
||||
}
|
||||
else if ((dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE)
|
||||
if (PRIV(op)->real_version != 0)
|
||||
{
|
||||
if (PRIV(op)->real_version == UINT64_MAX)
|
||||
{
|
||||
// This is the flag value used to cancel operations
|
||||
FINISH_OP(op);
|
||||
return 2;
|
||||
}
|
||||
// Restore original low version number for unblocked operations
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("Restoring %lx:%lx version: v%lu -> v%lu\n", op->oid.inode, op->oid.stripe, op->version, PRIV(op)->real_version);
|
||||
#endif
|
||||
auto prev_it = dirty_it;
|
||||
prev_it--;
|
||||
if (prev_it->first.oid == op->oid && prev_it->first.version >= PRIV(op)->real_version)
|
||||
{
|
||||
// Original version is still invalid
|
||||
// All subsequent writes to the same object must be canceled too
|
||||
cancel_all_writes(op, dirty_it, -EEXIST);
|
||||
return 2;
|
||||
}
|
||||
op->version = PRIV(op)->real_version;
|
||||
PRIV(op)->real_version = 0;
|
||||
dirty_entry e = dirty_it->second;
|
||||
dirty_db.erase(dirty_it);
|
||||
dirty_it = dirty_db.emplace((obj_ver_id){
|
||||
.oid = op->oid,
|
||||
.version = op->version,
|
||||
}, e).first;
|
||||
}
|
||||
if (write_iodepth >= max_write_iodepth)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
if ((dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE)
|
||||
{
|
||||
blockstore_journal_check_t space_check(this);
|
||||
if (!space_check.check_available(op, unsynced_big_writes.size() + 1, sizeof(journal_entry_big_write), JOURNAL_STABILIZE_RESERVATION))
|
||||
|
@ -129,10 +256,10 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
|
|||
PRIV(op)->wait_for = WAIT_FREE;
|
||||
return 0;
|
||||
}
|
||||
op->retval = -ENOSPC;
|
||||
FINISH_OP(op);
|
||||
return 1;
|
||||
cancel_all_writes(op, dirty_it, -ENOSPC);
|
||||
return 2;
|
||||
}
|
||||
write_iodepth++;
|
||||
BS_SUBMIT_GET_SQE(sqe, data);
|
||||
dirty_it->second.location = loc << block_order;
|
||||
dirty_it->second.state = (dirty_it->second.state & ~BS_ST_WORKFLOW_MASK) | BS_ST_SUBMITTED;
|
||||
|
@ -185,6 +312,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
|
|||
{
|
||||
return 0;
|
||||
}
|
||||
write_iodepth++;
|
||||
// There is sufficient space. Get SQE(s)
|
||||
struct io_uring_sqe *sqe1 = NULL;
|
||||
if (immediate_commit != IMMEDIATE_NONE ||
|
||||
|
@ -217,7 +345,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
|
|||
// Then pre-fill journal entry
|
||||
journal_entry_small_write *je = (journal_entry_small_write*)prefill_single_journal_entry(
|
||||
journal, op->opcode == BS_OP_WRITE_STABLE ? JE_SMALL_WRITE_INSTANT : JE_SMALL_WRITE,
|
||||
sizeof(journal_entry_small_write)
|
||||
sizeof(journal_entry_small_write) + clean_entry_bitmap_size
|
||||
);
|
||||
dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset;
|
||||
journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
|
||||
|
@ -236,6 +364,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
|
|||
je->len = op->len;
|
||||
je->data_offset = journal.next_free;
|
||||
je->crc32_data = crc32c(0, op->buf, op->len);
|
||||
memcpy((void*)(je+1), (clean_entry_bitmap_size > sizeof(void*) ? dirty_it->second.bitmap : &dirty_it->second.bitmap), clean_entry_bitmap_size);
|
||||
je->crc32 = je_crc32((journal_entry*)je);
|
||||
journal.crc32_last = je->crc32;
|
||||
if (immediate_commit != IMMEDIATE_NONE)
|
||||
|
@ -282,7 +411,7 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
|
|||
if (!PRIV(op)->pending_ops)
|
||||
{
|
||||
PRIV(op)->op_state = 4;
|
||||
continue_write(op);
|
||||
return continue_write(op);
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -296,30 +425,29 @@ int blockstore_impl_t::continue_write(blockstore_op_t *op)
|
|||
{
|
||||
io_uring_sqe *sqe = NULL;
|
||||
journal_entry_big_write *je;
|
||||
int op_state = PRIV(op)->op_state;
|
||||
if (op_state != 2 && op_state != 4)
|
||||
{
|
||||
// In progress
|
||||
return 1;
|
||||
}
|
||||
auto dirty_it = dirty_db.find((obj_ver_id){
|
||||
.oid = op->oid,
|
||||
.version = op->version,
|
||||
});
|
||||
assert(dirty_it != dirty_db.end());
|
||||
if (PRIV(op)->op_state == 2)
|
||||
if (op_state == 2)
|
||||
goto resume_2;
|
||||
else if (PRIV(op)->op_state == 4)
|
||||
else if (op_state == 4)
|
||||
goto resume_4;
|
||||
else
|
||||
return 1;
|
||||
resume_2:
|
||||
// Only for the immediate_commit mode: prepare and submit big_write journal entry
|
||||
sqe = get_sqe();
|
||||
if (!sqe)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
BS_SUBMIT_GET_SQE_DECL(sqe);
|
||||
je = (journal_entry_big_write*)prefill_single_journal_entry(
|
||||
journal, op->opcode == BS_OP_WRITE_STABLE ? JE_BIG_WRITE_INSTANT : JE_BIG_WRITE,
|
||||
sizeof(journal_entry_big_write)
|
||||
sizeof(journal_entry_big_write) + clean_entry_bitmap_size
|
||||
);
|
||||
dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset;
|
||||
journal.sector_info[journal.cur_sector].dirty = false;
|
||||
journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf(
|
||||
|
@ -333,6 +461,7 @@ resume_2:
|
|||
je->offset = op->offset;
|
||||
je->len = op->len;
|
||||
je->location = dirty_it->second.location;
|
||||
memcpy((void*)(je+1), (clean_entry_bitmap_size > sizeof(void*) ? dirty_it->second.bitmap : &dirty_it->second.bitmap), clean_entry_bitmap_size);
|
||||
je->crc32 = je_crc32((journal_entry*)je);
|
||||
journal.crc32_last = je->crc32;
|
||||
prepare_journal_sector_write(journal, journal.cur_sector, sqe,
|
||||
|
@ -344,7 +473,7 @@ resume_2:
|
|||
resume_4:
|
||||
// Switch object state
|
||||
#ifdef BLOCKSTORE_DEBUG
|
||||
printf("Ack write %lx:%lx v%lu = %d\n", op->oid.inode, op->oid.stripe, op->version, dirty_it->second.state);
|
||||
printf("Ack write %lx:%lx v%lu = state %x\n", op->oid.inode, op->oid.stripe, op->version, dirty_it->second.state);
|
||||
#endif
|
||||
bool imm = (dirty_it->second.state & BS_ST_TYPE_MASK) == BS_ST_BIG_WRITE
|
||||
? (immediate_commit == IMMEDIATE_ALL)
|
||||
|
@ -375,8 +504,9 @@ resume_4:
|
|||
}
|
||||
// Acknowledge write
|
||||
op->retval = op->len;
|
||||
write_iodepth--;
|
||||
FINISH_OP(op);
|
||||
return 1;
|
||||
return 2;
|
||||
}
|
||||
|
||||
void blockstore_impl_t::handle_write_event(ring_data_t *data, blockstore_op_t *op)
|
||||
|
@ -395,10 +525,7 @@ void blockstore_impl_t::handle_write_event(ring_data_t *data, blockstore_op_t *o
|
|||
{
|
||||
release_journal_sectors(op);
|
||||
PRIV(op)->op_state++;
|
||||
if (!continue_write(op))
|
||||
{
|
||||
submit_queue.push_front(op);
|
||||
}
|
||||
ringloop->wakeup();
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -411,8 +538,8 @@ void blockstore_impl_t::release_journal_sectors(blockstore_op_t *op)
|
|||
uint64_t s = PRIV(op)->min_flushed_journal_sector;
|
||||
while (1)
|
||||
{
|
||||
journal.sector_info[s-1].usage_count--;
|
||||
if (s != (1+journal.cur_sector) && journal.sector_info[s-1].usage_count == 0)
|
||||
journal.sector_info[s-1].flush_count--;
|
||||
if (s != (1+journal.cur_sector) && journal.sector_info[s-1].flush_count == 0)
|
||||
{
|
||||
// We know for sure that we won't write into this sector anymore
|
||||
uint64_t new_ds = journal.sector_info[s-1].offset + journal.block_size;
|
||||
|
@ -436,6 +563,10 @@ void blockstore_impl_t::release_journal_sectors(blockstore_op_t *op)
|
|||
|
||||
int blockstore_impl_t::dequeue_del(blockstore_op_t *op)
|
||||
{
|
||||
if (PRIV(op)->op_state)
|
||||
{
|
||||
return continue_write(op);
|
||||
}
|
||||
auto dirty_it = dirty_db.find((obj_ver_id){
|
||||
.oid = op->oid,
|
||||
.version = op->version,
|
||||
|
@ -446,6 +577,7 @@ int blockstore_impl_t::dequeue_del(blockstore_op_t *op)
|
|||
{
|
||||
return 0;
|
||||
}
|
||||
write_iodepth++;
|
||||
io_uring_sqe *sqe = NULL;
|
||||
if (immediate_commit != IMMEDIATE_NONE ||
|
||||
(journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_del) &&
|
||||
|
@ -492,7 +624,10 @@ int blockstore_impl_t::dequeue_del(blockstore_op_t *op)
|
|||
prepare_journal_sector_write(journal, journal.cur_sector, sqe, cb);
|
||||
PRIV(op)->min_flushed_journal_sector = PRIV(op)->max_flushed_journal_sector = 1 + journal.cur_sector;
|
||||
PRIV(op)->pending_ops++;
|
||||
// Remember small write as unsynced
|
||||
}
|
||||
else
|
||||
{
|
||||
// Remember delete as unsynced
|
||||
unsynced_small_writes.push_back((obj_ver_id){
|
||||
.oid = op->oid,
|
||||
.version = op->version,
|
||||
|
@ -501,7 +636,7 @@ int blockstore_impl_t::dequeue_del(blockstore_op_t *op)
|
|||
if (!PRIV(op)->pending_ops)
|
||||
{
|
||||
PRIV(op)->op_state = 4;
|
||||
continue_write(op);
|
||||
return continue_write(op);
|
||||
}
|
||||
else
|
||||
{
|
|
@ -1,20 +1,20 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#include <stdexcept>
|
||||
#include "cluster_client.h"
|
||||
|
||||
#define SCRAP_BUFFER_SIZE 4*1024*1024
|
||||
|
||||
cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json & config)
|
||||
{
|
||||
this->ringloop = ringloop;
|
||||
this->tfd = tfd;
|
||||
|
||||
log_level = config["log_level"].int64_value();
|
||||
this->config = config;
|
||||
|
||||
msgr.osd_num = 0;
|
||||
msgr.tfd = tfd;
|
||||
msgr.ringloop = ringloop;
|
||||
msgr.log_level = log_level;
|
||||
msgr.repeer_pgs = [this](osd_num_t peer_osd)
|
||||
{
|
||||
if (msgr.osd_peer_fds.find(peer_osd) != msgr.osd_peer_fds.end())
|
||||
|
@ -67,8 +67,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
|
|||
msgr.stop_client(op->peer_fd);
|
||||
delete op;
|
||||
};
|
||||
msgr.use_sync_send_recv = config["use_sync_send_recv"].bool_value() ||
|
||||
config["use_sync_send_recv"].uint64_value();
|
||||
msgr.init();
|
||||
|
||||
st_cli.tfd = tfd;
|
||||
st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); };
|
||||
|
@ -79,6 +78,9 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
|
|||
st_cli.parse_config(config);
|
||||
st_cli.load_global_config();
|
||||
|
||||
scrap_buffer_size = SCRAP_BUFFER_SIZE;
|
||||
scrap_buffer = malloc_or_die(scrap_buffer_size);
|
||||
|
||||
if (ringloop)
|
||||
{
|
||||
consumer.loop = [this]()
|
||||
|
@ -97,13 +99,21 @@ cluster_client_t::~cluster_client_t()
|
|||
{
|
||||
ringloop->unregister_consumer(&consumer);
|
||||
}
|
||||
free(scrap_buffer);
|
||||
}
|
||||
|
||||
void cluster_client_t::stop()
|
||||
cluster_op_t::~cluster_op_t()
|
||||
{
|
||||
while (msgr.clients.size() > 0)
|
||||
if (buf)
|
||||
{
|
||||
msgr.stop_client(msgr.clients.begin()->first);
|
||||
free(buf);
|
||||
buf = NULL;
|
||||
}
|
||||
if (bitmap_buf)
|
||||
{
|
||||
free(bitmap_buf);
|
||||
part_bitmaps = NULL;
|
||||
bitmap_buf = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -144,20 +154,16 @@ static uint32_t is_power_of_two(uint64_t value)
|
|||
void cluster_client_t::on_load_config_hook(json11::Json::object & config)
|
||||
{
|
||||
bs_block_size = config["block_size"].uint64_value();
|
||||
bs_disk_alignment = config["disk_alignment"].uint64_value();
|
||||
bs_bitmap_granularity = config["bitmap_granularity"].uint64_value();
|
||||
if (!bs_block_size)
|
||||
{
|
||||
bs_block_size = DEFAULT_BLOCK_SIZE;
|
||||
}
|
||||
if (!bs_disk_alignment)
|
||||
{
|
||||
bs_disk_alignment = DEFAULT_DISK_ALIGNMENT;
|
||||
}
|
||||
if (!bs_bitmap_granularity)
|
||||
{
|
||||
bs_bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
|
||||
}
|
||||
bs_bitmap_size = bs_block_size / bs_bitmap_granularity / 8;
|
||||
uint32_t block_order;
|
||||
if ((block_order = is_power_of_two(bs_block_size)) >= 64 || bs_block_size < MIN_BLOCK_SIZE || bs_block_size >= MAX_BLOCK_SIZE)
|
||||
{
|
||||
|
@ -185,16 +191,8 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & config)
|
|||
{
|
||||
up_wait_retry_interval = 50;
|
||||
}
|
||||
msgr.peer_connect_interval = config["peer_connect_interval"].uint64_value();
|
||||
if (!msgr.peer_connect_interval)
|
||||
{
|
||||
msgr.peer_connect_interval = DEFAULT_PEER_CONNECT_INTERVAL;
|
||||
}
|
||||
msgr.peer_connect_timeout = config["peer_connect_timeout"].uint64_value();
|
||||
if (!msgr.peer_connect_timeout)
|
||||
{
|
||||
msgr.peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT;
|
||||
}
|
||||
msgr.parse_config(config);
|
||||
msgr.parse_config(this->config);
|
||||
st_cli.load_pgs();
|
||||
}
|
||||
|
||||
|
@ -228,21 +226,21 @@ void cluster_client_t::on_change_hook(json11::Json::object & changes)
|
|||
// And now they have to be resliced!
|
||||
for (auto op: cur_ops)
|
||||
{
|
||||
if (INODE_POOL(op->inode) == pool_item.first)
|
||||
if (INODE_POOL(op->cur_inode) == pool_item.first)
|
||||
{
|
||||
op->needs_reslice = true;
|
||||
}
|
||||
}
|
||||
for (auto op: unsynced_writes)
|
||||
{
|
||||
if (INODE_POOL(op->inode) == pool_item.first)
|
||||
if (INODE_POOL(op->cur_inode) == pool_item.first)
|
||||
{
|
||||
op->needs_reslice = true;
|
||||
}
|
||||
}
|
||||
for (auto op: syncing_writes)
|
||||
{
|
||||
if (INODE_POOL(op->inode) == pool_item.first)
|
||||
if (INODE_POOL(op->cur_inode) == pool_item.first)
|
||||
{
|
||||
op->needs_reslice = true;
|
||||
}
|
||||
|
@ -261,6 +259,11 @@ void cluster_client_t::on_change_osd_state_hook(uint64_t peer_osd)
|
|||
}
|
||||
}
|
||||
|
||||
bool cluster_client_t::is_ready()
|
||||
{
|
||||
return pgs_loaded;
|
||||
}
|
||||
|
||||
void cluster_client_t::on_ready(std::function<void(void)> fn)
|
||||
{
|
||||
if (pgs_loaded)
|
||||
|
@ -312,7 +315,7 @@ void cluster_client_t::execute(cluster_op_t *op)
|
|||
op->retval = 0;
|
||||
if (op->opcode != OSD_OP_SYNC && op->opcode != OSD_OP_READ && op->opcode != OSD_OP_WRITE ||
|
||||
(op->opcode == OSD_OP_READ || op->opcode == OSD_OP_WRITE) && (!op->inode || !op->len ||
|
||||
op->offset % bs_disk_alignment || op->len % bs_disk_alignment))
|
||||
op->offset % bs_bitmap_granularity || op->len % bs_bitmap_granularity))
|
||||
{
|
||||
op->retval = -EINVAL;
|
||||
std::function<void(cluster_op_t*)>(op->callback)(op);
|
||||
|
@ -323,34 +326,81 @@ void cluster_client_t::execute(cluster_op_t *op)
|
|||
execute_sync(op);
|
||||
return;
|
||||
}
|
||||
if (op->opcode == OSD_OP_WRITE && !immediate_commit)
|
||||
op->cur_inode = op->inode;
|
||||
if (op->opcode == OSD_OP_WRITE)
|
||||
{
|
||||
if (next_writes.size() > 0)
|
||||
auto ino_it = st_cli.inode_config.find(op->inode);
|
||||
if (ino_it != st_cli.inode_config.end() && ino_it->second.readonly)
|
||||
{
|
||||
assert(cur_sync);
|
||||
next_writes.push_back(op);
|
||||
op->retval = -EINVAL;
|
||||
std::function<void(cluster_op_t*)>(op->callback)(op);
|
||||
return;
|
||||
}
|
||||
if (queued_bytes >= client_dirty_limit)
|
||||
if (!immediate_commit)
|
||||
{
|
||||
// Push an extra SYNC operation to flush previous writes
|
||||
next_writes.push_back(op);
|
||||
cluster_op_t *sync_op = new cluster_op_t;
|
||||
sync_op->is_internal = true;
|
||||
sync_op->opcode = OSD_OP_SYNC;
|
||||
sync_op->callback = [](cluster_op_t* sync_op) {};
|
||||
execute_sync(sync_op);
|
||||
return;
|
||||
if (next_writes.size() > 0)
|
||||
{
|
||||
assert(cur_sync);
|
||||
next_writes.push_back(op);
|
||||
return;
|
||||
}
|
||||
if (queued_bytes >= client_dirty_limit)
|
||||
{
|
||||
// Push an extra SYNC operation to flush previous writes
|
||||
next_writes.push_back(op);
|
||||
cluster_op_t *sync_op = new cluster_op_t;
|
||||
sync_op->is_internal = true;
|
||||
sync_op->opcode = OSD_OP_SYNC;
|
||||
sync_op->callback = [](cluster_op_t* sync_op) {};
|
||||
execute_sync(sync_op);
|
||||
return;
|
||||
}
|
||||
queued_bytes += op->len;
|
||||
op = copy_write(op);
|
||||
unsynced_writes.push_back(op);
|
||||
}
|
||||
queued_bytes += op->len;
|
||||
}
|
||||
cur_ops.insert(op);
|
||||
continue_rw(op);
|
||||
}
|
||||
|
||||
cluster_op_t *cluster_client_t::copy_write(cluster_op_t *op)
|
||||
{
|
||||
// Save operation for replay when one of PGs goes out of sync
|
||||
// (primary OSD drops our connection in this case)
|
||||
cluster_op_t *op_copy = new cluster_op_t();
|
||||
op_copy->is_internal = true;
|
||||
op_copy->orig_op = op;
|
||||
op_copy->opcode = op->opcode;
|
||||
op_copy->inode = op->inode;
|
||||
op_copy->cur_inode = op->inode;
|
||||
op_copy->offset = op->offset;
|
||||
op_copy->len = op->len;
|
||||
op_copy->buf = malloc_or_die(op->len);
|
||||
op_copy->iov.push_back(op_copy->buf, op->len);
|
||||
op_copy->callback = [](cluster_op_t* op_copy)
|
||||
{
|
||||
if (op_copy->orig_op)
|
||||
{
|
||||
// Acknowledge write and forget the original pointer
|
||||
op_copy->orig_op->retval = op_copy->retval;
|
||||
std::function<void(cluster_op_t*)>(op_copy->orig_op->callback)(op_copy->orig_op);
|
||||
op_copy->orig_op = NULL;
|
||||
}
|
||||
};
|
||||
void *cur_buf = op_copy->buf;
|
||||
for (int i = 0; i < op->iov.count; i++)
|
||||
{
|
||||
memcpy(cur_buf, op->iov.buf[i].iov_base, op->iov.buf[i].iov_len);
|
||||
cur_buf += op->iov.buf[i].iov_len;
|
||||
}
|
||||
return op_copy;
|
||||
}
|
||||
|
||||
// FIXME Reimplement it using "coroutine emulation"
|
||||
void cluster_client_t::continue_rw(cluster_op_t *op)
|
||||
{
|
||||
pool_id_t pool_id = INODE_POOL(op->inode);
|
||||
pool_id_t pool_id = INODE_POOL(op->cur_inode);
|
||||
if (!pool_id)
|
||||
{
|
||||
op->retval = -EINVAL;
|
||||
|
@ -363,40 +413,6 @@ void cluster_client_t::continue_rw(cluster_op_t *op)
|
|||
// Postpone operations to unknown pools
|
||||
return;
|
||||
}
|
||||
if (op->opcode == OSD_OP_WRITE && !immediate_commit && !op->is_internal)
|
||||
{
|
||||
// Save operation for replay when PG goes out of sync
|
||||
// (primary OSD drops our connection in this case)
|
||||
cluster_op_t *op_copy = new cluster_op_t();
|
||||
op_copy->is_internal = true;
|
||||
op_copy->orig_op = op;
|
||||
op_copy->opcode = op->opcode;
|
||||
op_copy->inode = op->inode;
|
||||
op_copy->offset = op->offset;
|
||||
op_copy->len = op->len;
|
||||
op_copy->buf = malloc_or_die(op->len);
|
||||
op_copy->iov.push_back(op_copy->buf, op->len);
|
||||
op_copy->callback = [](cluster_op_t* op_copy)
|
||||
{
|
||||
if (op_copy->orig_op)
|
||||
{
|
||||
// Acknowledge write and forget the original pointer
|
||||
op_copy->orig_op->retval = op_copy->retval;
|
||||
std::function<void(cluster_op_t*)>(op_copy->orig_op->callback)(op_copy->orig_op);
|
||||
op_copy->orig_op = NULL;
|
||||
}
|
||||
};
|
||||
void *cur_buf = op_copy->buf;
|
||||
for (int i = 0; i < op->iov.count; i++)
|
||||
{
|
||||
memcpy(cur_buf, op->iov.buf[i].iov_base, op->iov.buf[i].iov_len);
|
||||
cur_buf += op->iov.buf[i].iov_len;
|
||||
}
|
||||
unsynced_writes.push_back(op_copy);
|
||||
cur_ops.erase(op);
|
||||
cur_ops.insert(op_copy);
|
||||
op = op_copy;
|
||||
}
|
||||
if (!op->parts.size())
|
||||
{
|
||||
// Slice the operation into parts
|
||||
|
@ -405,11 +421,11 @@ void cluster_client_t::continue_rw(cluster_op_t *op)
|
|||
if (!op->needs_reslice)
|
||||
{
|
||||
// Send unsent parts, if they're not subject to change
|
||||
for (auto & op_part: op->parts)
|
||||
for (int i = 0; i < op->parts.size(); i++)
|
||||
{
|
||||
if (!op_part.sent && !op_part.done)
|
||||
if (!op->parts[i].sent && !op->parts[i].done)
|
||||
{
|
||||
try_send(op, &op_part);
|
||||
try_send(op, i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -420,15 +436,37 @@ void cluster_client_t::continue_rw(cluster_op_t *op)
|
|||
// Finished successfully
|
||||
// Even if the PG count has changed in meanwhile we treat it as success
|
||||
// because if some operations were invalid for the new PG count we'd get errors
|
||||
bool is_read = op->opcode == OSD_OP_READ;
|
||||
if (is_read)
|
||||
{
|
||||
// Check parent inode
|
||||
auto ino_it = st_cli.inode_config.find(op->cur_inode);
|
||||
if (ino_it != st_cli.inode_config.end() &&
|
||||
ino_it->second.parent_id)
|
||||
{
|
||||
// Continue reading from the parent inode
|
||||
// FIXME: This obviously requires optimizations for long snapshot chains
|
||||
op->cur_inode = ino_it->second.parent_id;
|
||||
op->parts.clear();
|
||||
op->done_count = 0;
|
||||
op->needs_reslice = true;
|
||||
continue_rw(op);
|
||||
return;
|
||||
}
|
||||
}
|
||||
cur_ops.erase(op);
|
||||
op->retval = op->len;
|
||||
std::function<void(cluster_op_t*)>(op->callback)(op);
|
||||
continue_sync();
|
||||
if (!is_read)
|
||||
{
|
||||
continue_sync();
|
||||
}
|
||||
return;
|
||||
}
|
||||
else if (op->retval != 0 && op->retval != -EPIPE)
|
||||
{
|
||||
// Fatal error (not -EPIPE)
|
||||
bool is_read = op->opcode == OSD_OP_READ;
|
||||
cur_ops.erase(op);
|
||||
if (!immediate_commit && op->opcode == OSD_OP_WRITE)
|
||||
{
|
||||
|
@ -445,11 +483,12 @@ void cluster_client_t::continue_rw(cluster_op_t *op)
|
|||
std::function<void(cluster_op_t*)>(op->callback)(op);
|
||||
if (del)
|
||||
{
|
||||
if (op->buf)
|
||||
free(op->buf);
|
||||
delete op;
|
||||
}
|
||||
continue_sync();
|
||||
if (!is_read)
|
||||
{
|
||||
continue_sync();
|
||||
}
|
||||
return;
|
||||
}
|
||||
else
|
||||
|
@ -467,60 +506,145 @@ void cluster_client_t::continue_rw(cluster_op_t *op)
|
|||
}
|
||||
}
|
||||
|
||||
static void add_iov(int size, bool skip, cluster_op_t *op, int &iov_idx, size_t &iov_pos, osd_op_buf_list_t &iov, void *scrap, int scrap_len)
|
||||
{
|
||||
int left = size;
|
||||
while (left > 0 && iov_idx < op->iov.count)
|
||||
{
|
||||
int cur_left = op->iov.buf[iov_idx].iov_len - iov_pos;
|
||||
if (cur_left < left)
|
||||
{
|
||||
if (!skip)
|
||||
{
|
||||
iov.push_back(op->iov.buf[iov_idx].iov_base + iov_pos, cur_left);
|
||||
}
|
||||
left -= cur_left;
|
||||
iov_pos = 0;
|
||||
iov_idx++;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!skip)
|
||||
{
|
||||
iov.push_back(op->iov.buf[iov_idx].iov_base + iov_pos, left);
|
||||
}
|
||||
iov_pos += left;
|
||||
left = 0;
|
||||
}
|
||||
}
|
||||
assert(left == 0);
|
||||
if (skip && scrap_len > 0)
|
||||
{
|
||||
// All skipped ranges are read into the same useless buffer
|
||||
left = size;
|
||||
while (left > 0)
|
||||
{
|
||||
int cur_left = scrap_len < left ? scrap_len : left;
|
||||
iov.push_back(scrap, cur_left);
|
||||
left -= cur_left;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void cluster_client_t::slice_rw(cluster_op_t *op)
|
||||
{
|
||||
// Slice the request into individual object stripe requests
|
||||
// Primary OSDs still operate individual stripes, but their size is multiplied by PG minsize in case of EC
|
||||
auto & pool_cfg = st_cli.pool_config[INODE_POOL(op->inode)];
|
||||
uint64_t pg_block_size = bs_block_size * (
|
||||
pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_minsize
|
||||
auto & pool_cfg = st_cli.pool_config[INODE_POOL(op->cur_inode)];
|
||||
uint32_t pg_data_size = (
|
||||
pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
|
||||
);
|
||||
uint64_t pg_block_size = bs_block_size * pg_data_size;
|
||||
uint64_t first_stripe = (op->offset / pg_block_size) * pg_block_size;
|
||||
uint64_t last_stripe = ((op->offset + op->len + pg_block_size - 1) / pg_block_size - 1) * pg_block_size;
|
||||
op->retval = 0;
|
||||
op->parts.resize((last_stripe - first_stripe) / pg_block_size + 1);
|
||||
if (op->opcode == OSD_OP_READ)
|
||||
{
|
||||
// Allocate memory for the bitmap
|
||||
unsigned object_bitmap_size = ((op->len / bs_bitmap_granularity + 7) / 8);
|
||||
object_bitmap_size = (object_bitmap_size < 8 ? 8 : object_bitmap_size);
|
||||
unsigned bitmap_mem = object_bitmap_size + (bs_bitmap_size * pg_data_size) * op->parts.size();
|
||||
if (op->bitmap_buf_size < bitmap_mem)
|
||||
{
|
||||
op->bitmap_buf = realloc_or_die(op->bitmap_buf, bitmap_mem);
|
||||
if (!op->bitmap_buf_size)
|
||||
{
|
||||
// First allocation
|
||||
memset(op->bitmap_buf, 0, object_bitmap_size);
|
||||
}
|
||||
op->part_bitmaps = op->bitmap_buf + object_bitmap_size;
|
||||
op->bitmap_buf_size = bitmap_mem;
|
||||
}
|
||||
}
|
||||
int iov_idx = 0;
|
||||
size_t iov_pos = 0;
|
||||
int i = 0;
|
||||
for (uint64_t stripe = first_stripe; stripe <= last_stripe; stripe += pg_block_size)
|
||||
{
|
||||
pg_num_t pg_num = (op->inode + stripe/pool_cfg.pg_stripe_size) % pool_cfg.real_pg_count + 1;
|
||||
pg_num_t pg_num = (op->cur_inode + stripe/pool_cfg.pg_stripe_size) % pool_cfg.real_pg_count + 1; // like map_to_pg()
|
||||
uint64_t begin = (op->offset < stripe ? stripe : op->offset);
|
||||
uint64_t end = (op->offset + op->len) > (stripe + pg_block_size)
|
||||
? (stripe + pg_block_size) : (op->offset + op->len);
|
||||
op->parts[i] = {
|
||||
.parent = op,
|
||||
.offset = begin,
|
||||
.len = (uint32_t)(end - begin),
|
||||
.pg_num = pg_num,
|
||||
.sent = false,
|
||||
.done = false,
|
||||
};
|
||||
int left = end-begin;
|
||||
while (left > 0 && iov_idx < op->iov.count)
|
||||
op->parts[i].iov.reset();
|
||||
if (op->cur_inode != op->inode)
|
||||
{
|
||||
if (op->iov.buf[iov_idx].iov_len - iov_pos < left)
|
||||
// Read remaining parts from upper layers
|
||||
uint64_t prev = begin, cur = begin;
|
||||
bool skip_prev = true;
|
||||
while (cur < end)
|
||||
{
|
||||
op->parts[i].iov.push_back(op->iov.buf[iov_idx].iov_base + iov_pos, op->iov.buf[iov_idx].iov_len - iov_pos);
|
||||
left -= (op->iov.buf[iov_idx].iov_len - iov_pos);
|
||||
iov_pos = 0;
|
||||
iov_idx++;
|
||||
unsigned bmp_loc = (cur - op->offset)/bs_bitmap_granularity;
|
||||
bool skip = (((*(uint8_t*)(op->bitmap_buf + bmp_loc/8)) >> (bmp_loc%8)) & 0x1);
|
||||
if (skip_prev != skip)
|
||||
{
|
||||
if (cur > prev)
|
||||
{
|
||||
if (prev == begin && skip_prev)
|
||||
{
|
||||
begin = cur;
|
||||
// Just advance iov_idx & iov_pos
|
||||
add_iov(cur-prev, true, op, iov_idx, iov_pos, op->parts[i].iov, NULL, 0);
|
||||
}
|
||||
else
|
||||
add_iov(cur-prev, skip_prev, op, iov_idx, iov_pos, op->parts[i].iov, scrap_buffer, scrap_buffer_size);
|
||||
}
|
||||
skip_prev = skip;
|
||||
prev = cur;
|
||||
}
|
||||
cur += bs_bitmap_granularity;
|
||||
}
|
||||
assert(cur > prev);
|
||||
if (skip_prev)
|
||||
{
|
||||
// Just advance iov_idx & iov_pos
|
||||
add_iov(end-prev, true, op, iov_idx, iov_pos, op->parts[i].iov, NULL, 0);
|
||||
end = prev;
|
||||
}
|
||||
else
|
||||
{
|
||||
op->parts[i].iov.push_back(op->iov.buf[iov_idx].iov_base + iov_pos, left);
|
||||
iov_pos += left;
|
||||
left = 0;
|
||||
}
|
||||
add_iov(cur-prev, skip_prev, op, iov_idx, iov_pos, op->parts[i].iov, scrap_buffer, scrap_buffer_size);
|
||||
if (end == begin)
|
||||
op->done_count++;
|
||||
}
|
||||
assert(left == 0);
|
||||
else
|
||||
{
|
||||
add_iov(end-begin, false, op, iov_idx, iov_pos, op->parts[i].iov, NULL, 0);
|
||||
}
|
||||
op->parts[i].parent = op;
|
||||
op->parts[i].offset = begin;
|
||||
op->parts[i].len = (uint32_t)(end - begin);
|
||||
op->parts[i].pg_num = pg_num;
|
||||
op->parts[i].osd_num = 0;
|
||||
op->parts[i].sent = end <= begin;
|
||||
op->parts[i].done = end <= begin;
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
bool cluster_client_t::try_send(cluster_op_t *op, cluster_op_part_t *part)
|
||||
bool cluster_client_t::try_send(cluster_op_t *op, int i)
|
||||
{
|
||||
auto & pool_cfg = st_cli.pool_config[INODE_POOL(op->inode)];
|
||||
cluster_op_part_t *part = &op->parts[i];
|
||||
auto & pool_cfg = st_cli.pool_config[INODE_POOL(op->cur_inode)];
|
||||
auto pg_it = pool_cfg.pg_config.find(part->pg_num);
|
||||
if (pg_it != pool_cfg.pg_config.end() &&
|
||||
!pg_it->second.pause && pg_it->second.cur_primary)
|
||||
|
@ -533,7 +657,10 @@ bool cluster_client_t::try_send(cluster_op_t *op, cluster_op_part_t *part)
|
|||
part->osd_num = primary_osd;
|
||||
part->sent = true;
|
||||
op->sent_count++;
|
||||
part->op = {
|
||||
uint64_t pg_bitmap_size = bs_bitmap_size * (
|
||||
pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
|
||||
);
|
||||
part->op = (osd_op_t){
|
||||
.op_type = OSD_OP_OUT,
|
||||
.peer_fd = peer_fd,
|
||||
.req = { .rw = {
|
||||
|
@ -542,10 +669,12 @@ bool cluster_client_t::try_send(cluster_op_t *op, cluster_op_part_t *part)
|
|||
.id = op_id++,
|
||||
.opcode = op->opcode,
|
||||
},
|
||||
.inode = op->inode,
|
||||
.inode = op->cur_inode,
|
||||
.offset = part->offset,
|
||||
.len = part->len,
|
||||
} },
|
||||
.bitmap = op->opcode == OSD_OP_WRITE ? NULL : op->part_bitmaps + pg_bitmap_size*i,
|
||||
.bitmap_len = (unsigned)(op->opcode == OSD_OP_WRITE ? 0 : pg_bitmap_size),
|
||||
.callback = [this, part](osd_op_t *op_part)
|
||||
{
|
||||
handle_op_part(part);
|
||||
|
@ -671,8 +800,6 @@ void cluster_client_t::finish_sync()
|
|||
assert(op->sent_count == 0);
|
||||
if (op->is_internal)
|
||||
{
|
||||
if (op->buf)
|
||||
free(op->buf);
|
||||
delete op;
|
||||
}
|
||||
}
|
||||
|
@ -694,7 +821,7 @@ void cluster_client_t::send_sync(cluster_op_t *op, cluster_op_part_t *part)
|
|||
assert(peer_it != msgr.osd_peer_fds.end());
|
||||
part->sent = true;
|
||||
op->sent_count++;
|
||||
part->op = {
|
||||
part->op = (osd_op_t){
|
||||
.op_type = OSD_OP_OUT,
|
||||
.peer_fd = peer_it->second,
|
||||
.req = {
|
||||
|
@ -712,6 +839,16 @@ void cluster_client_t::send_sync(cluster_op_t *op, cluster_op_part_t *part)
|
|||
msgr.outbox_push(&part->op);
|
||||
}
|
||||
|
||||
static inline void mem_or(void *res, const void *r2, unsigned int len)
|
||||
{
|
||||
unsigned int i;
|
||||
for (i = 0; i < len; ++i)
|
||||
{
|
||||
// Hope the compiler vectorizes this
|
||||
((uint8_t*)res)[i] = ((uint8_t*)res)[i] | ((uint8_t*)r2)[i];
|
||||
}
|
||||
}
|
||||
|
||||
void cluster_client_t::handle_op_part(cluster_op_part_t *part)
|
||||
{
|
||||
cluster_op_t *op = part->parent;
|
||||
|
@ -749,6 +886,35 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
|
|||
// OK
|
||||
part->done = true;
|
||||
op->done_count++;
|
||||
if (op->opcode == OSD_OP_READ)
|
||||
{
|
||||
// Copy (OR) bitmap
|
||||
auto & pool_cfg = st_cli.pool_config[INODE_POOL(op->cur_inode)];
|
||||
uint32_t pg_block_size = bs_block_size * (
|
||||
pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
|
||||
);
|
||||
uint32_t object_offset = (part->op.req.rw.offset - op->offset) / bs_bitmap_granularity;
|
||||
uint32_t part_offset = (part->op.req.rw.offset % pg_block_size) / bs_bitmap_granularity;
|
||||
uint32_t part_len = part->op.req.rw.len / bs_bitmap_granularity;
|
||||
if (!(object_offset & 0x7) && !(part_offset & 0x7) && (part_len >= 8))
|
||||
{
|
||||
// Copy bytes
|
||||
mem_or(op->bitmap_buf + object_offset/8, part->op.bitmap + part_offset/8, part_len/8);
|
||||
object_offset += (part_len & ~0x7);
|
||||
part_offset += (part_len & ~0x7);
|
||||
part_len = (part_len & 0x7);
|
||||
}
|
||||
while (part_len > 0)
|
||||
{
|
||||
// Copy bits
|
||||
(*(uint8_t*)(op->bitmap_buf + (object_offset >> 3))) |= (
|
||||
(((*(uint8_t*)(part->op.bitmap + (part_offset >> 3))) >> (part_offset & 0x7)) & 0x1) << (object_offset & 0x7)
|
||||
);
|
||||
part_offset++;
|
||||
object_offset++;
|
||||
part_len--;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (op->sent_count == 0)
|
||||
{
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
||||
|
@ -8,8 +8,6 @@
|
|||
|
||||
#define MIN_BLOCK_SIZE 4*1024
|
||||
#define MAX_BLOCK_SIZE 128*1024*1024
|
||||
#define DEFAULT_DISK_ALIGNMENT 4096
|
||||
#define DEFAULT_BITMAP_GRANULARITY 4096
|
||||
#define DEFAULT_CLIENT_DIRTY_LIMIT 32*1024*1024
|
||||
|
||||
struct cluster_op_t;
|
||||
|
@ -36,7 +34,9 @@ struct cluster_op_t
|
|||
int retval;
|
||||
osd_op_buf_list_t iov;
|
||||
std::function<void(cluster_op_t*)> callback;
|
||||
~cluster_op_t();
|
||||
protected:
|
||||
uint64_t cur_inode; // for snapshot reads
|
||||
void *buf = NULL;
|
||||
cluster_op_t *orig_op = NULL;
|
||||
bool is_internal = false;
|
||||
|
@ -44,6 +44,8 @@ protected:
|
|||
bool up_wait = false;
|
||||
int sent_count = 0, done_count = 0;
|
||||
std::vector<cluster_op_part_t> parts;
|
||||
void *bitmap_buf = NULL, *part_bitmaps = NULL;
|
||||
unsigned bitmap_buf_size = 0;
|
||||
friend class cluster_client_t;
|
||||
};
|
||||
|
||||
|
@ -53,8 +55,7 @@ class cluster_client_t
|
|||
ring_loop_t *ringloop;
|
||||
|
||||
uint64_t bs_block_size = 0;
|
||||
uint64_t bs_disk_alignment = 0;
|
||||
uint64_t bs_bitmap_granularity = 0;
|
||||
uint32_t bs_bitmap_granularity = 0, bs_bitmap_size = 0;
|
||||
std::map<pool_id_t, uint64_t> pg_counts;
|
||||
bool immediate_commit = false;
|
||||
// FIXME: Implement inmemory_commit mode. Note that it requires to return overlapping reads from memory.
|
||||
|
@ -75,6 +76,8 @@ class cluster_client_t
|
|||
std::vector<cluster_op_t*> next_writes;
|
||||
std::vector<cluster_op_t*> offline_ops;
|
||||
uint64_t queued_bytes = 0;
|
||||
void *scrap_buffer = NULL;
|
||||
unsigned scrap_buffer_size = 0;
|
||||
|
||||
bool pgs_loaded = false;
|
||||
std::vector<std::function<void(void)>> on_ready_hooks;
|
||||
|
@ -82,12 +85,13 @@ class cluster_client_t
|
|||
public:
|
||||
etcd_state_client_t st_cli;
|
||||
osd_messenger_t msgr;
|
||||
json11::Json config;
|
||||
|
||||
cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json & config);
|
||||
~cluster_client_t();
|
||||
void execute(cluster_op_t *op);
|
||||
bool is_ready();
|
||||
void on_ready(std::function<void(void)> fn);
|
||||
void stop();
|
||||
|
||||
protected:
|
||||
void continue_ops(bool up_retry = false);
|
||||
|
@ -95,9 +99,10 @@ protected:
|
|||
void on_load_pgs_hook(bool success);
|
||||
void on_change_hook(json11::Json::object & changes);
|
||||
void on_change_osd_state_hook(uint64_t peer_osd);
|
||||
cluster_op_t *copy_write(cluster_op_t *op);
|
||||
void continue_rw(cluster_op_t *op);
|
||||
void slice_rw(cluster_op_t *op);
|
||||
bool try_send(cluster_op_t *op, cluster_op_part_t *part);
|
||||
bool try_send(cluster_op_t *op, int i);
|
||||
void execute_sync(cluster_op_t *op);
|
||||
void continue_sync();
|
||||
void finish_sync();
|
|
@ -8,4 +8,10 @@
|
|||
// unsigned __int64 _mm_crc32_u64 (unsigned __int64 crc, unsigned __int64 v)
|
||||
// unsigned int _mm_crc32_u8 (unsigned int crc, unsigned char v)
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
uint32_t crc32c(uint32_t crc, const void *buf, size_t len);
|
||||
#ifdef __cplusplus
|
||||
};
|
||||
#endif
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#define _LARGEFILE64_SOURCE
|
||||
#include <sys/types.h>
|
||||
|
@ -26,23 +26,32 @@ struct journal_dump_t
|
|||
uint64_t journal_offset;
|
||||
uint64_t journal_len;
|
||||
uint64_t journal_pos;
|
||||
bool all;
|
||||
bool started;
|
||||
int fd;
|
||||
uint32_t crc32_last;
|
||||
|
||||
void dump_block(void *buf);
|
||||
int dump_block(void *buf);
|
||||
};
|
||||
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
if (argc < 5)
|
||||
journal_dump_t self = { 0 };
|
||||
int b = 1;
|
||||
if (argc >= 2 && !strcmp(argv[1], "--all"))
|
||||
{
|
||||
printf("USAGE: %s <journal_file> <journal_block_size> <offset> <size>\n", argv[0]);
|
||||
self.all = true;
|
||||
b = 2;
|
||||
}
|
||||
if (argc < b+4)
|
||||
{
|
||||
printf("USAGE: %s [--all] <journal_file> <journal_block_size> <offset> <size>\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
journal_dump_t self;
|
||||
self.journal_device = argv[1];
|
||||
self.journal_block = strtoul(argv[2], NULL, 10);
|
||||
self.journal_offset = strtoull(argv[3], NULL, 10);
|
||||
self.journal_len = strtoull(argv[4], NULL, 10);
|
||||
self.journal_device = argv[b];
|
||||
self.journal_block = strtoul(argv[b+1], NULL, 10);
|
||||
self.journal_offset = strtoull(argv[b+2], NULL, 10);
|
||||
self.journal_len = strtoull(argv[b+3], NULL, 10);
|
||||
if (self.journal_block < MEM_ALIGNMENT || (self.journal_block % MEM_ALIGNMENT) ||
|
||||
self.journal_block > 128*1024)
|
||||
{
|
||||
|
@ -57,30 +66,64 @@ int main(int argc, char *argv[])
|
|||
}
|
||||
void *data = memalign(MEM_ALIGNMENT, self.journal_block);
|
||||
self.journal_pos = 0;
|
||||
while (self.journal_pos < self.journal_len)
|
||||
if (self.all)
|
||||
{
|
||||
while (self.journal_pos < self.journal_len)
|
||||
{
|
||||
int r = pread(self.fd, data, self.journal_block, self.journal_offset+self.journal_pos);
|
||||
assert(r == self.journal_block);
|
||||
uint64_t s;
|
||||
for (s = 0; s < self.journal_block; s += 8)
|
||||
{
|
||||
if (*((uint64_t*)(data+s)) != 0)
|
||||
break;
|
||||
}
|
||||
if (s == self.journal_block)
|
||||
{
|
||||
printf("offset %08lx: zeroes\n", self.journal_pos);
|
||||
self.journal_pos += self.journal_block;
|
||||
}
|
||||
else if (((journal_entry*)data)->magic == JOURNAL_MAGIC)
|
||||
{
|
||||
printf("offset %08lx:\n", self.journal_pos);
|
||||
self.dump_block(data);
|
||||
}
|
||||
else
|
||||
{
|
||||
printf("offset %08lx: no magic in the beginning, looks like random data (pattern=%lx)\n", self.journal_pos, *((uint64_t*)data));
|
||||
self.journal_pos += self.journal_block;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
int r = pread(self.fd, data, self.journal_block, self.journal_offset+self.journal_pos);
|
||||
assert(r == self.journal_block);
|
||||
uint64_t s;
|
||||
for (s = 0; s < self.journal_block; s += 8)
|
||||
journal_entry *je = (journal_entry*)(data);
|
||||
if (je->magic != JOURNAL_MAGIC || je->type != JE_START || je_crc32(je) != je->crc32)
|
||||
{
|
||||
if (*((uint64_t*)(data+s)) != 0)
|
||||
break;
|
||||
}
|
||||
if (s == self.journal_block)
|
||||
{
|
||||
printf("offset %08lx: zeroes\n", self.journal_pos);
|
||||
self.journal_pos += self.journal_block;
|
||||
}
|
||||
else if (((journal_entry*)data)->magic == JOURNAL_MAGIC)
|
||||
{
|
||||
printf("offset %08lx:\n", self.journal_pos);
|
||||
self.dump_block(data);
|
||||
printf("offset %08lx: journal superblock is invalid\n", self.journal_pos);
|
||||
}
|
||||
else
|
||||
{
|
||||
printf("offset %08lx: no magic in the beginning, looks like random data (pattern=%lx)\n", self.journal_pos, *((uint64_t*)data));
|
||||
self.journal_pos += self.journal_block;
|
||||
printf("offset %08lx:\n", self.journal_pos);
|
||||
self.dump_block(data);
|
||||
self.started = false;
|
||||
self.journal_pos = je->start.journal_start;
|
||||
while (1)
|
||||
{
|
||||
if (self.journal_pos >= self.journal_len)
|
||||
self.journal_pos = self.journal_block;
|
||||
r = pread(self.fd, data, self.journal_block, self.journal_offset+self.journal_pos);
|
||||
assert(r == self.journal_block);
|
||||
printf("offset %08lx:\n", self.journal_pos);
|
||||
r = self.dump_block(data);
|
||||
if (r <= 0)
|
||||
{
|
||||
printf("end of the journal\n");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
free(data);
|
||||
|
@ -88,7 +131,7 @@ int main(int argc, char *argv[])
|
|||
return 0;
|
||||
}
|
||||
|
||||
void journal_dump_t::dump_block(void *buf)
|
||||
int journal_dump_t::dump_block(void *buf)
|
||||
{
|
||||
uint32_t pos = 0;
|
||||
journal_pos += journal_block;
|
||||
|
@ -97,12 +140,19 @@ void journal_dump_t::dump_block(void *buf)
|
|||
while (pos < journal_block)
|
||||
{
|
||||
journal_entry *je = (journal_entry*)(buf + pos);
|
||||
if (je->magic != JOURNAL_MAGIC || je->type < JE_MIN || je->type > JE_MAX)
|
||||
if (je->magic != JOURNAL_MAGIC || je->type < JE_MIN || je->type > JE_MAX ||
|
||||
!all && started && je->crc32_prev != crc32_last)
|
||||
{
|
||||
break;
|
||||
}
|
||||
const char *crc32_valid = je_crc32(je) == je->crc32 ? "(valid)" : "(invalid)";
|
||||
printf("entry % 3d: crc32=%08x %s prev=%08x ", entry, je->crc32, crc32_valid, je->crc32_prev);
|
||||
bool crc32_valid = je_crc32(je) == je->crc32;
|
||||
if (!all && !crc32_valid)
|
||||
{
|
||||
break;
|
||||
}
|
||||
started = true;
|
||||
crc32_last = je->crc32;
|
||||
printf("entry % 3d: crc32=%08x %s prev=%08x ", entry, je->crc32, (crc32_valid ? "(valid)" : "(invalid)"), je->crc32_prev);
|
||||
if (je->type == JE_START)
|
||||
{
|
||||
printf("je_start start=%08lx\n", je->start.journal_start);
|
||||
|
@ -170,4 +220,5 @@ void journal_dump_t::dump_block(void *buf)
|
|||
{
|
||||
journal_pos = journal_len;
|
||||
}
|
||||
return entry;
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#include <sys/epoll.h>
|
||||
#include <sys/poll.h>
|
||||
|
@ -84,8 +84,12 @@ void epoll_manager_t::handle_epoll_events()
|
|||
nfds = epoll_wait(epoll_fd, events, MAX_EPOLL_EVENTS, 0);
|
||||
for (int i = 0; i < nfds; i++)
|
||||
{
|
||||
auto & cb = epoll_handlers[events[i].data.fd];
|
||||
cb(events[i].data.fd, events[i].events);
|
||||
auto cb_it = epoll_handlers.find(events[i].data.fd);
|
||||
if (cb_it != epoll_handlers.end())
|
||||
{
|
||||
auto & cb = cb_it->second;
|
||||
cb(events[i].data.fd, events[i].events);
|
||||
}
|
||||
}
|
||||
} while (nfds == MAX_EPOLL_EVENTS);
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#include "osd_ops.h"
|
||||
#include "pg_states.h"
|
||||
|
@ -7,6 +7,21 @@
|
|||
#include "http_client.h"
|
||||
#include "base64.h"
|
||||
|
||||
etcd_state_client_t::~etcd_state_client_t()
|
||||
{
|
||||
for (auto watch: watches)
|
||||
{
|
||||
delete watch;
|
||||
}
|
||||
watches.clear();
|
||||
etcd_watches_initialised = -1;
|
||||
if (etcd_watch_ws)
|
||||
{
|
||||
etcd_watch_ws->close();
|
||||
etcd_watch_ws = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
json_kv_t etcd_state_client_t::parse_etcd_kv(const json11::Json & kv_json)
|
||||
{
|
||||
json_kv_t kv;
|
||||
|
@ -46,6 +61,23 @@ void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int t
|
|||
http_request_json(tfd, etcd_address, req, timeout, callback);
|
||||
}
|
||||
|
||||
void etcd_state_client_t::add_etcd_url(std::string addr)
|
||||
{
|
||||
if (addr.length() > 0)
|
||||
{
|
||||
if (strtolower(addr.substr(0, 7)) == "http://")
|
||||
addr = addr.substr(7);
|
||||
else if (strtolower(addr.substr(0, 8)) == "https://")
|
||||
{
|
||||
printf("HTTPS is unsupported for etcd. Either use plain HTTP or setup a local proxy for etcd interaction\n");
|
||||
exit(1);
|
||||
}
|
||||
if (addr.find('/') < 0)
|
||||
addr += "/v3";
|
||||
this->etcd_addresses.push_back(addr);
|
||||
}
|
||||
}
|
||||
|
||||
void etcd_state_client_t::parse_config(json11::Json & config)
|
||||
{
|
||||
this->etcd_addresses.clear();
|
||||
|
@ -55,13 +87,7 @@ void etcd_state_client_t::parse_config(json11::Json & config)
|
|||
while (1)
|
||||
{
|
||||
int pos = ea.find(',');
|
||||
std::string addr = pos >= 0 ? ea.substr(0, pos) : ea;
|
||||
if (addr.length() > 0)
|
||||
{
|
||||
if (addr.find('/') < 0)
|
||||
addr += "/v3";
|
||||
this->etcd_addresses.push_back(addr);
|
||||
}
|
||||
add_etcd_url(pos >= 0 ? ea.substr(0, pos) : ea);
|
||||
if (pos >= 0)
|
||||
ea = ea.substr(pos+1);
|
||||
else
|
||||
|
@ -72,13 +98,7 @@ void etcd_state_client_t::parse_config(json11::Json & config)
|
|||
{
|
||||
for (auto & ea: config["etcd_address"].array_items())
|
||||
{
|
||||
std::string addr = ea.string_value();
|
||||
if (addr != "")
|
||||
{
|
||||
if (addr.find('/') < 0)
|
||||
addr += "/v3";
|
||||
this->etcd_addresses.push_back(addr);
|
||||
}
|
||||
add_etcd_url(ea.string_value());
|
||||
}
|
||||
}
|
||||
this->etcd_prefix = config["etcd_prefix"].string_value();
|
||||
|
@ -160,7 +180,7 @@ void etcd_state_client_t::start_etcd_watcher()
|
|||
start_etcd_watcher();
|
||||
});
|
||||
}
|
||||
else
|
||||
else if (etcd_watches_initialised > 0)
|
||||
{
|
||||
// Connection was live, retry immediately
|
||||
start_etcd_watcher();
|
||||
|
@ -173,6 +193,7 @@ void etcd_state_client_t::start_etcd_watcher()
|
|||
{ "range_end", base64_encode(etcd_prefix+"/config0") },
|
||||
{ "start_revision", etcd_watch_revision+1 },
|
||||
{ "watch_id", ETCD_CONFIG_WATCH_ID },
|
||||
{ "progress_notify", true },
|
||||
} }
|
||||
}).dump());
|
||||
etcd_watch_ws->post_message(WS_TEXT, json11::Json(json11::Json::object {
|
||||
|
@ -181,6 +202,7 @@ void etcd_state_client_t::start_etcd_watcher()
|
|||
{ "range_end", base64_encode(etcd_prefix+"/osd/state0") },
|
||||
{ "start_revision", etcd_watch_revision+1 },
|
||||
{ "watch_id", ETCD_OSD_STATE_WATCH_ID },
|
||||
{ "progress_notify", true },
|
||||
} }
|
||||
}).dump());
|
||||
etcd_watch_ws->post_message(WS_TEXT, json11::Json(json11::Json::object {
|
||||
|
@ -189,6 +211,7 @@ void etcd_state_client_t::start_etcd_watcher()
|
|||
{ "range_end", base64_encode(etcd_prefix+"/pg/state0") },
|
||||
{ "start_revision", etcd_watch_revision+1 },
|
||||
{ "watch_id", ETCD_PG_STATE_WATCH_ID },
|
||||
{ "progress_notify", true },
|
||||
} }
|
||||
}).dump());
|
||||
etcd_watch_ws->post_message(WS_TEXT, json11::Json(json11::Json::object {
|
||||
|
@ -197,6 +220,7 @@ void etcd_state_client_t::start_etcd_watcher()
|
|||
{ "range_end", base64_encode(etcd_prefix+"/pg/history0") },
|
||||
{ "start_revision", etcd_watch_revision+1 },
|
||||
{ "watch_id", ETCD_PG_HISTORY_WATCH_ID },
|
||||
{ "progress_notify", true },
|
||||
} }
|
||||
}).dump());
|
||||
}
|
||||
|
@ -247,6 +271,12 @@ void etcd_state_client_t::load_pgs()
|
|||
{ "key", base64_encode(etcd_prefix+"/config/pgs") },
|
||||
} }
|
||||
},
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(etcd_prefix+"/config/inode/") },
|
||||
{ "range_end", base64_encode(etcd_prefix+"/config/inode0") },
|
||||
} }
|
||||
},
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(etcd_prefix+"/pg/history/") },
|
||||
|
@ -315,67 +345,99 @@ void etcd_state_client_t::parse_state(const std::string & key, const json11::Jso
|
|||
}
|
||||
for (auto & pool_item: value.object_items())
|
||||
{
|
||||
pool_config_t pc;
|
||||
// ID
|
||||
pool_id_t pool_id = stoull_full(pool_item.first);
|
||||
if (!pool_id || pool_id >= POOL_ID_MAX)
|
||||
{
|
||||
printf("Pool ID %s is invalid (must be a number less than 0x%x), skipping pool\n", pool_item.first.c_str(), POOL_ID_MAX);
|
||||
continue;
|
||||
}
|
||||
if (pool_item.second["pg_size"].uint64_value() < 1 ||
|
||||
pool_item.second["scheme"] == "xor" && pool_item.second["pg_size"].uint64_value() < 3)
|
||||
{
|
||||
printf("Pool %u has invalid pg_size, skipping pool\n", pool_id);
|
||||
continue;
|
||||
}
|
||||
if (pool_item.second["pg_minsize"].uint64_value() < 1 ||
|
||||
pool_item.second["pg_minsize"].uint64_value() > pool_item.second["pg_size"].uint64_value() ||
|
||||
pool_item.second["pg_minsize"].uint64_value() < (pool_item.second["pg_size"].uint64_value() - 1))
|
||||
{
|
||||
printf("Pool %u has invalid pg_minsize, skipping pool\n", pool_id);
|
||||
continue;
|
||||
}
|
||||
if (pool_item.second["pg_count"].uint64_value() < 1)
|
||||
{
|
||||
printf("Pool %u has invalid pg_count, skipping pool\n", pool_id);
|
||||
continue;
|
||||
}
|
||||
if (pool_item.second["name"].string_value() == "")
|
||||
pc.id = pool_id;
|
||||
// Pool Name
|
||||
pc.name = pool_item.second["name"].string_value();
|
||||
if (pc.name == "")
|
||||
{
|
||||
printf("Pool %u has empty name, skipping pool\n", pool_id);
|
||||
continue;
|
||||
}
|
||||
if (pool_item.second["scheme"] != "replicated" && pool_item.second["scheme"] != "xor")
|
||||
// Failure Domain
|
||||
pc.failure_domain = pool_item.second["failure_domain"].string_value();
|
||||
// Coding Scheme
|
||||
if (pool_item.second["scheme"] == "replicated")
|
||||
pc.scheme = POOL_SCHEME_REPLICATED;
|
||||
else if (pool_item.second["scheme"] == "xor")
|
||||
pc.scheme = POOL_SCHEME_XOR;
|
||||
else if (pool_item.second["scheme"] == "jerasure")
|
||||
pc.scheme = POOL_SCHEME_JERASURE;
|
||||
else
|
||||
{
|
||||
printf("Pool %u has invalid coding scheme (only \"xor\" and \"replicated\" are allowed), skipping pool\n", pool_id);
|
||||
printf("Pool %u has invalid coding scheme (one of \"xor\", \"replicated\" or \"jerasure\" required), skipping pool\n", pool_id);
|
||||
continue;
|
||||
}
|
||||
if (pool_item.second["max_osd_combinations"].uint64_value() > 0 &&
|
||||
pool_item.second["max_osd_combinations"].uint64_value() < 100)
|
||||
// PG Size
|
||||
pc.pg_size = pool_item.second["pg_size"].uint64_value();
|
||||
if (pc.pg_size < 1 ||
|
||||
pool_item.second["pg_size"].uint64_value() < 3 &&
|
||||
(pc.scheme == POOL_SCHEME_XOR || pc.scheme == POOL_SCHEME_JERASURE) ||
|
||||
pool_item.second["pg_size"].uint64_value() > 256)
|
||||
{
|
||||
printf("Pool %u has invalid pg_size, skipping pool\n", pool_id);
|
||||
continue;
|
||||
}
|
||||
// Parity Chunks
|
||||
pc.parity_chunks = pool_item.second["parity_chunks"].uint64_value();
|
||||
if (pc.scheme == POOL_SCHEME_XOR)
|
||||
{
|
||||
if (pc.parity_chunks > 1)
|
||||
{
|
||||
printf("Pool %u has invalid parity_chunks (must be 1), skipping pool\n", pool_id);
|
||||
continue;
|
||||
}
|
||||
pc.parity_chunks = 1;
|
||||
}
|
||||
if (pc.scheme == POOL_SCHEME_JERASURE &&
|
||||
(pc.parity_chunks < 1 || pc.parity_chunks > pc.pg_size-2))
|
||||
{
|
||||
printf("Pool %u has invalid parity_chunks (must be between 1 and pg_size-2), skipping pool\n", pool_id);
|
||||
continue;
|
||||
}
|
||||
// PG MinSize
|
||||
pc.pg_minsize = pool_item.second["pg_minsize"].uint64_value();
|
||||
if (pc.pg_minsize < 1 || pc.pg_minsize > pc.pg_size ||
|
||||
(pc.scheme == POOL_SCHEME_XOR || pc.scheme == POOL_SCHEME_JERASURE) &&
|
||||
pc.pg_minsize < (pc.pg_size-pc.parity_chunks))
|
||||
{
|
||||
printf("Pool %u has invalid pg_minsize, skipping pool\n", pool_id);
|
||||
continue;
|
||||
}
|
||||
// PG Count
|
||||
pc.pg_count = pool_item.second["pg_count"].uint64_value();
|
||||
if (pc.pg_count < 1)
|
||||
{
|
||||
printf("Pool %u has invalid pg_count, skipping pool\n", pool_id);
|
||||
continue;
|
||||
}
|
||||
// Max OSD Combinations
|
||||
pc.max_osd_combinations = pool_item.second["max_osd_combinations"].uint64_value();
|
||||
if (!pc.max_osd_combinations)
|
||||
pc.max_osd_combinations = 10000;
|
||||
if (pc.max_osd_combinations > 0 && pc.max_osd_combinations < 100)
|
||||
{
|
||||
printf("Pool %u has invalid max_osd_combinations (must be at least 100), skipping pool\n", pool_id);
|
||||
continue;
|
||||
}
|
||||
// PG Stripe Size
|
||||
pc.pg_stripe_size = pool_item.second["pg_stripe_size"].uint64_value();
|
||||
uint64_t min_stripe_size = bs_block_size * (pc.scheme == POOL_SCHEME_REPLICATED ? 1 : (pc.pg_size-pc.parity_chunks));
|
||||
if (pc.pg_stripe_size < min_stripe_size)
|
||||
pc.pg_stripe_size = min_stripe_size;
|
||||
// Save
|
||||
pc.real_pg_count = this->pool_config[pool_id].real_pg_count;
|
||||
std::swap(pc.pg_config, this->pool_config[pool_id].pg_config);
|
||||
std::swap(this->pool_config[pool_id], pc);
|
||||
auto & parsed_cfg = this->pool_config[pool_id];
|
||||
parsed_cfg.exists = true;
|
||||
parsed_cfg.id = pool_id;
|
||||
parsed_cfg.name = pool_item.second["name"].string_value();
|
||||
parsed_cfg.scheme = pool_item.second["scheme"] == "replicated" ? POOL_SCHEME_REPLICATED : POOL_SCHEME_XOR;
|
||||
parsed_cfg.pg_size = pool_item.second["pg_size"].uint64_value();
|
||||
parsed_cfg.pg_minsize = pool_item.second["pg_minsize"].uint64_value();
|
||||
parsed_cfg.pg_count = pool_item.second["pg_count"].uint64_value();
|
||||
parsed_cfg.failure_domain = pool_item.second["failure_domain"].string_value();
|
||||
parsed_cfg.pg_stripe_size = pool_item.second["pg_stripe_size"].uint64_value();
|
||||
uint64_t min_stripe_size = bs_block_size *
|
||||
(parsed_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : parsed_cfg.pg_minsize);
|
||||
if (parsed_cfg.pg_stripe_size < min_stripe_size)
|
||||
{
|
||||
parsed_cfg.pg_stripe_size = min_stripe_size;
|
||||
}
|
||||
parsed_cfg.max_osd_combinations = pool_item.second["max_osd_combinations"].uint64_value();
|
||||
if (!parsed_cfg.max_osd_combinations)
|
||||
{
|
||||
parsed_cfg.max_osd_combinations = 10000;
|
||||
}
|
||||
for (auto & pg_item: parsed_cfg.pg_config)
|
||||
{
|
||||
if (pg_item.second.target_set.size() != parsed_cfg.pg_size)
|
||||
|
@ -561,4 +623,105 @@ void etcd_state_client_t::parse_state(const std::string & key, const json11::Jso
|
|||
}
|
||||
}
|
||||
}
|
||||
else if (key.substr(0, etcd_prefix.length()+14) == etcd_prefix+"/config/inode/")
|
||||
{
|
||||
// <etcd_prefix>/config/inode/%d/%d
|
||||
uint64_t pool_id = 0;
|
||||
uint64_t inode_num = 0;
|
||||
char null_byte = 0;
|
||||
sscanf(key.c_str() + etcd_prefix.length()+14, "%lu/%lu%c", &pool_id, &inode_num, &null_byte);
|
||||
if (!pool_id || pool_id >= POOL_ID_MAX || !inode_num || (inode_num >> (64-POOL_ID_BITS)) || null_byte != 0)
|
||||
{
|
||||
printf("Bad etcd key %s, ignoring\n", key.c_str());
|
||||
}
|
||||
else
|
||||
{
|
||||
inode_num |= (pool_id << (64-POOL_ID_BITS));
|
||||
auto it = this->inode_config.find(inode_num);
|
||||
if (it != this->inode_config.end() && it->second.name != "")
|
||||
{
|
||||
auto n_it = this->inode_by_name.find(it->second.name);
|
||||
if (n_it->second == inode_num)
|
||||
{
|
||||
this->inode_by_name.erase(n_it);
|
||||
for (auto w: watches)
|
||||
{
|
||||
if (w->name == it->second.name)
|
||||
{
|
||||
w->cfg = { 0 };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!value.is_object())
|
||||
{
|
||||
this->inode_config.erase(inode_num);
|
||||
}
|
||||
else
|
||||
{
|
||||
inode_t parent_inode_num = value["parent_id"].uint64_value();
|
||||
if (parent_inode_num && !(parent_inode_num >> (64-POOL_ID_BITS)))
|
||||
{
|
||||
uint64_t parent_pool_id = value["parent_pool"].uint64_value();
|
||||
if (!parent_pool_id)
|
||||
parent_inode_num |= pool_id << (64-POOL_ID_BITS);
|
||||
else if (parent_pool_id >= POOL_ID_MAX)
|
||||
{
|
||||
printf(
|
||||
"Inode %lu/%lu parent_pool value is invalid, ignoring parent setting\n",
|
||||
inode_num >> (64-POOL_ID_BITS), inode_num & ((1l << (64-POOL_ID_BITS)) - 1)
|
||||
);
|
||||
parent_inode_num = 0;
|
||||
}
|
||||
else
|
||||
parent_inode_num |= parent_pool_id << (64-POOL_ID_BITS);
|
||||
}
|
||||
inode_config_t cfg = (inode_config_t){
|
||||
.num = inode_num,
|
||||
.name = value["name"].string_value(),
|
||||
.size = value["size"].uint64_value(),
|
||||
.parent_id = parent_inode_num,
|
||||
.readonly = value["readonly"].bool_value(),
|
||||
};
|
||||
this->inode_config[inode_num] = cfg;
|
||||
if (cfg.name != "")
|
||||
{
|
||||
this->inode_by_name[cfg.name] = inode_num;
|
||||
for (auto w: watches)
|
||||
{
|
||||
if (w->name == value["name"].string_value())
|
||||
{
|
||||
w->cfg = cfg;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inode_watch_t* etcd_state_client_t::watch_inode(std::string name)
|
||||
{
|
||||
inode_watch_t *watch = new inode_watch_t;
|
||||
watch->name = name;
|
||||
watches.push_back(watch);
|
||||
auto it = inode_by_name.find(name);
|
||||
if (it != inode_by_name.end())
|
||||
{
|
||||
watch->cfg = inode_config[it->second];
|
||||
}
|
||||
return watch;
|
||||
}
|
||||
|
||||
void etcd_state_client_t::close_watch(inode_watch_t* watch)
|
||||
{
|
||||
for (int i = 0; i < watches.size(); i++)
|
||||
{
|
||||
if (watches[i] == watch)
|
||||
{
|
||||
watches.erase(watches.begin()+i, watches.begin()+i+1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
delete watch;
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
||||
|
@ -43,7 +43,7 @@ struct pool_config_t
|
|||
pool_id_t id;
|
||||
std::string name;
|
||||
uint64_t scheme;
|
||||
uint64_t pg_size, pg_minsize;
|
||||
uint64_t pg_size, pg_minsize, parity_chunks;
|
||||
uint64_t pg_count;
|
||||
uint64_t real_pg_count;
|
||||
std::string failure_domain;
|
||||
|
@ -52,8 +52,29 @@ struct pool_config_t
|
|||
std::map<pg_num_t, pg_config_t> pg_config;
|
||||
};
|
||||
|
||||
struct inode_config_t
|
||||
{
|
||||
uint64_t num;
|
||||
std::string name;
|
||||
uint64_t size;
|
||||
inode_t parent_id;
|
||||
bool readonly;
|
||||
};
|
||||
|
||||
struct inode_watch_t
|
||||
{
|
||||
std::string name;
|
||||
inode_config_t cfg;
|
||||
};
|
||||
|
||||
struct etcd_state_client_t
|
||||
{
|
||||
protected:
|
||||
std::vector<inode_watch_t*> watches;
|
||||
websocket_t *etcd_watch_ws = NULL;
|
||||
uint64_t bs_block_size = 0;
|
||||
void add_etcd_url(std::string);
|
||||
public:
|
||||
std::vector<std::string> etcd_addresses;
|
||||
std::string etcd_prefix;
|
||||
int log_level = 0;
|
||||
|
@ -61,10 +82,10 @@ struct etcd_state_client_t
|
|||
|
||||
int etcd_watches_initialised = 0;
|
||||
uint64_t etcd_watch_revision = 0;
|
||||
websocket_t *etcd_watch_ws = NULL;
|
||||
uint64_t bs_block_size = 0;
|
||||
std::map<pool_id_t, pool_config_t> pool_config;
|
||||
std::map<osd_num_t, json11::Json> peer_states;
|
||||
std::map<inode_t, inode_config_t> inode_config;
|
||||
std::map<std::string, inode_t> inode_by_name;
|
||||
|
||||
std::function<void(json11::Json::object &)> on_change_hook;
|
||||
std::function<void(json11::Json::object &)> on_load_config_hook;
|
||||
|
@ -81,4 +102,7 @@ struct etcd_state_client_t
|
|||
void load_pgs();
|
||||
void parse_state(const std::string & key, const json11::Json & value);
|
||||
void parse_config(json11::Json & config);
|
||||
inode_watch_t* watch_inode(std::string name);
|
||||
void close_watch(inode_watch_t* watch);
|
||||
~etcd_state_client_t();
|
||||
};
|
|
@ -1,22 +1,22 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
// FIO engine to test cluster I/O
|
||||
//
|
||||
// Random write:
|
||||
//
|
||||
// fio -thread -ioengine=./libfio_cluster.so -name=test -bs=4k -direct=1 -fsync=16 -iodepth=16 -rw=randwrite \
|
||||
// -etcd=127.0.0.1:2379 [-etcd_prefix=/vitastor] -pool=1 -inode=1 -size=1000M
|
||||
// -etcd=127.0.0.1:2379 [-etcd_prefix=/vitastor] (-image=testimg | -pool=1 -inode=1 -size=1000M)
|
||||
//
|
||||
// Linear write:
|
||||
//
|
||||
// fio -thread -ioengine=./libfio_cluster.so -name=test -bs=128k -direct=1 -fsync=32 -iodepth=32 -rw=write \
|
||||
// -etcd=127.0.0.1:2379 [-etcd_prefix=/vitastor] -pool=1 -inode=1 -size=1000M
|
||||
// -etcd=127.0.0.1:2379 [-etcd_prefix=/vitastor] -image=testimg
|
||||
//
|
||||
// Random read (run with -iodepth=32 or -iodepth=1):
|
||||
//
|
||||
// fio -thread -ioengine=./libfio_cluster.so -name=test -bs=4k -direct=1 -iodepth=32 -rw=randread \
|
||||
// -etcd=127.0.0.1:2379 [-etcd_prefix=/vitastor] -pool=1 -inode=1 -size=1000M
|
||||
// -etcd=127.0.0.1:2379 [-etcd_prefix=/vitastor] -image=testimg
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <sys/socket.h>
|
||||
|
@ -35,6 +35,7 @@ struct sec_data
|
|||
ring_loop_t *ringloop = NULL;
|
||||
epoll_manager_t *epmgr = NULL;
|
||||
cluster_client_t *cli = NULL;
|
||||
inode_watch_t *watch = NULL;
|
||||
bool last_sync = false;
|
||||
/* The list of completed io_u structs. */
|
||||
std::vector<io_u*> completed;
|
||||
|
@ -47,6 +48,7 @@ struct sec_options
|
|||
int __pad;
|
||||
char *etcd_host = NULL;
|
||||
char *etcd_prefix = NULL;
|
||||
char *image = NULL;
|
||||
uint64_t pool = 0;
|
||||
uint64_t inode = 0;
|
||||
int cluster_log = 0;
|
||||
|
@ -64,7 +66,7 @@ static struct fio_option options[] = {
|
|||
.group = FIO_OPT_G_FILENAME,
|
||||
},
|
||||
{
|
||||
.name = "etcd",
|
||||
.name = "etcd_prefix",
|
||||
.lname = "etcd key prefix",
|
||||
.type = FIO_OPT_STR_STORE,
|
||||
.off1 = offsetof(struct sec_options, etcd_prefix),
|
||||
|
@ -72,6 +74,15 @@ static struct fio_option options[] = {
|
|||
.category = FIO_OPT_C_ENGINE,
|
||||
.group = FIO_OPT_G_FILENAME,
|
||||
},
|
||||
{
|
||||
.name = "image",
|
||||
.lname = "Vitastor image name",
|
||||
.type = FIO_OPT_STR_STORE,
|
||||
.off1 = offsetof(struct sec_options, image),
|
||||
.help = "Vitastor image name to run tests on",
|
||||
.category = FIO_OPT_C_ENGINE,
|
||||
.group = FIO_OPT_G_FILENAME,
|
||||
},
|
||||
{
|
||||
.name = "pool",
|
||||
.lname = "pool number for the inode",
|
||||
|
@ -86,14 +97,14 @@ static struct fio_option options[] = {
|
|||
.lname = "inode to run tests on",
|
||||
.type = FIO_OPT_INT,
|
||||
.off1 = offsetof(struct sec_options, inode),
|
||||
.help = "inode to run tests on (1 by default)",
|
||||
.help = "inode number to run tests on",
|
||||
.category = FIO_OPT_C_ENGINE,
|
||||
.group = FIO_OPT_G_FILENAME,
|
||||
},
|
||||
{
|
||||
.name = "cluster_log_level",
|
||||
.lname = "cluster log level",
|
||||
.type = FIO_OPT_BOOL,
|
||||
.type = FIO_OPT_INT,
|
||||
.off1 = offsetof(struct sec_options, cluster_log),
|
||||
.help = "Set log level for the Vitastor client",
|
||||
.def = "0",
|
||||
|
@ -117,8 +128,15 @@ static struct fio_option options[] = {
|
|||
|
||||
static int sec_setup(struct thread_data *td)
|
||||
{
|
||||
sec_options *o = (sec_options*)td->eo;
|
||||
sec_data *bsd;
|
||||
|
||||
if (!o->etcd_host)
|
||||
{
|
||||
td_verror(td, EINVAL, "etcd address is missing");
|
||||
return 1;
|
||||
}
|
||||
|
||||
bsd = new sec_data;
|
||||
if (!bsd)
|
||||
{
|
||||
|
@ -134,6 +152,51 @@ static int sec_setup(struct thread_data *td)
|
|||
td->o.open_files++;
|
||||
}
|
||||
|
||||
json11::Json cfg = json11::Json::object {
|
||||
{ "etcd_address", std::string(o->etcd_host) },
|
||||
{ "etcd_prefix", std::string(o->etcd_prefix ? o->etcd_prefix : "/vitastor") },
|
||||
{ "log_level", o->cluster_log },
|
||||
};
|
||||
|
||||
if (!o->image)
|
||||
{
|
||||
if (!(o->inode & ((1l << (64-POOL_ID_BITS)) - 1)))
|
||||
{
|
||||
td_verror(td, EINVAL, "inode number is missing");
|
||||
return 1;
|
||||
}
|
||||
if (o->pool)
|
||||
{
|
||||
o->inode = (o->inode & ((1l << (64-POOL_ID_BITS)) - 1)) | (o->pool << (64-POOL_ID_BITS));
|
||||
}
|
||||
if (!(o->inode >> (64-POOL_ID_BITS)))
|
||||
{
|
||||
td_verror(td, EINVAL, "pool is missing");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
o->inode = 0;
|
||||
}
|
||||
bsd->ringloop = new ring_loop_t(512);
|
||||
bsd->epmgr = new epoll_manager_t(bsd->ringloop);
|
||||
bsd->cli = new cluster_client_t(bsd->ringloop, bsd->epmgr->tfd, cfg);
|
||||
if (o->image)
|
||||
{
|
||||
while (!bsd->cli->is_ready())
|
||||
{
|
||||
bsd->ringloop->loop();
|
||||
if (bsd->cli->is_ready())
|
||||
break;
|
||||
bsd->ringloop->wait();
|
||||
}
|
||||
bsd->watch = bsd->cli->st_cli.watch_inode(std::string(o->image));
|
||||
td->files[0]->real_file_size = bsd->watch->cfg.size;
|
||||
}
|
||||
|
||||
bsd->trace = o->trace ? true : false;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -142,40 +205,20 @@ static void sec_cleanup(struct thread_data *td)
|
|||
sec_data *bsd = (sec_data*)td->io_ops_data;
|
||||
if (bsd)
|
||||
{
|
||||
if (bsd->watch)
|
||||
{
|
||||
bsd->cli->st_cli.close_watch(bsd->watch);
|
||||
}
|
||||
delete bsd->cli;
|
||||
delete bsd->epmgr;
|
||||
delete bsd->ringloop;
|
||||
bsd->cli = NULL;
|
||||
bsd->epmgr = NULL;
|
||||
bsd->ringloop = NULL;
|
||||
delete bsd;
|
||||
}
|
||||
}
|
||||
|
||||
/* Connect to the server from each thread. */
|
||||
static int sec_init(struct thread_data *td)
|
||||
{
|
||||
sec_options *o = (sec_options*)td->eo;
|
||||
sec_data *bsd = (sec_data*)td->io_ops_data;
|
||||
|
||||
json11::Json cfg = json11::Json::object {
|
||||
{ "etcd_address", std::string(o->etcd_host) },
|
||||
{ "etcd_prefix", std::string(o->etcd_prefix ? o->etcd_prefix : "/vitastor") },
|
||||
{ "log_level", o->cluster_log },
|
||||
};
|
||||
|
||||
if (o->pool)
|
||||
o->inode = (o->inode & ((1l << (64-POOL_ID_BITS)) - 1)) | (o->pool << (64-POOL_ID_BITS));
|
||||
if (!(o->inode >> (64-POOL_ID_BITS)))
|
||||
{
|
||||
td_verror(td, EINVAL, "pool is missing");
|
||||
return 1;
|
||||
}
|
||||
bsd->ringloop = new ring_loop_t(512);
|
||||
bsd->epmgr = new epoll_manager_t(bsd->ringloop);
|
||||
bsd->cli = new cluster_client_t(bsd->ringloop, bsd->epmgr->tfd, cfg);
|
||||
|
||||
bsd->trace = o->trace ? true : false;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -195,19 +238,23 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
|
|||
io->engine_data = bsd;
|
||||
cluster_op_t *op = new cluster_op_t;
|
||||
|
||||
op->inode = opt->image ? bsd->watch->cfg.num : opt->inode;
|
||||
switch (io->ddir)
|
||||
{
|
||||
case DDIR_READ:
|
||||
op->opcode = OSD_OP_READ;
|
||||
op->inode = opt->inode;
|
||||
op->offset = io->offset;
|
||||
op->len = io->xfer_buflen;
|
||||
op->iov.push_back(io->xfer_buf, io->xfer_buflen);
|
||||
bsd->last_sync = false;
|
||||
break;
|
||||
case DDIR_WRITE:
|
||||
if (opt->image && bsd->watch->cfg.readonly)
|
||||
{
|
||||
io->error = EROFS;
|
||||
return FIO_Q_COMPLETED;
|
||||
}
|
||||
op->opcode = OSD_OP_WRITE;
|
||||
op->inode = opt->inode;
|
||||
op->offset = io->offset;
|
||||
op->len = io->xfer_buflen;
|
||||
op->iov.push_back(io->xfer_buf, io->xfer_buflen);
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
// FIO engine to test Blockstore
|
||||
//
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
// FIO engine to test Blockstore through Secondary OSD interface
|
||||
//
|
||||
|
@ -140,6 +140,7 @@ static void sec_cleanup(struct thread_data *td)
|
|||
if (bsd)
|
||||
{
|
||||
close(bsd->connect_fd);
|
||||
delete bsd;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -312,6 +313,7 @@ static int sec_getevents(struct thread_data *td, unsigned int min, unsigned int
|
|||
exit(1);
|
||||
}
|
||||
io_u* io = it->second;
|
||||
bsd->queue.erase(it);
|
||||
if (io->ddir == DDIR_READ)
|
||||
{
|
||||
if (reply.hdr.retval != io->xfer_buflen)
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#include <netinet/tcp.h>
|
||||
#include <sys/epoll.h>
|
||||
|
@ -22,7 +22,6 @@
|
|||
#define READ_BUFFER_SIZE 9000
|
||||
|
||||
static int extract_port(std::string & host);
|
||||
static std::string strtolower(const std::string & in);
|
||||
static std::string trim(const std::string & in);
|
||||
static std::string ws_format_frame(int type, uint64_t size);
|
||||
static bool ws_parse_frame(std::string & buf, int & type, std::string & res);
|
||||
|
@ -673,7 +672,7 @@ static int extract_port(std::string & host)
|
|||
return port;
|
||||
}
|
||||
|
||||
static std::string strtolower(const std::string & in)
|
||||
std::string strtolower(const std::string & in)
|
||||
{
|
||||
std::string s = in;
|
||||
for (int i = 0; i < s.length(); i++)
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
#include <string>
|
||||
|
@ -49,6 +49,8 @@ std::vector<std::string> getifaddr_list(bool include_v6 = false);
|
|||
|
||||
uint64_t stoull_full(const std::string & str, int base = 10);
|
||||
|
||||
std::string strtolower(const std::string & in);
|
||||
|
||||
void http_request(timerfd_manager_t *tfd, const std::string & host, const std::string & request,
|
||||
const http_options_t & options, std::function<void(const http_response_t *response)> callback);
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
|
@ -26,14 +26,106 @@ osd_op_t::~osd_op_t()
|
|||
}
|
||||
}
|
||||
|
||||
void osd_messenger_t::init()
|
||||
{
|
||||
keepalive_timer_id = tfd->set_timer(1000, true, [this](int)
|
||||
{
|
||||
for (auto cl_it = clients.begin(); cl_it != clients.end();)
|
||||
{
|
||||
auto cl = (cl_it++)->second;
|
||||
if (!cl->osd_num)
|
||||
{
|
||||
// Do not run keepalive on regular clients
|
||||
continue;
|
||||
}
|
||||
if (cl->ping_time_remaining > 0)
|
||||
{
|
||||
cl->ping_time_remaining--;
|
||||
if (!cl->ping_time_remaining)
|
||||
{
|
||||
// Ping timed out, stop the client
|
||||
stop_client(cl->peer_fd, true);
|
||||
}
|
||||
}
|
||||
else if (cl->idle_time_remaining > 0)
|
||||
{
|
||||
cl->idle_time_remaining--;
|
||||
if (!cl->idle_time_remaining)
|
||||
{
|
||||
// Connection is idle for <osd_idle_time>, send ping
|
||||
osd_op_t *op = new osd_op_t();
|
||||
op->op_type = OSD_OP_OUT;
|
||||
op->peer_fd = cl->peer_fd;
|
||||
op->req = (osd_any_op_t){
|
||||
.hdr = {
|
||||
.magic = SECONDARY_OSD_OP_MAGIC,
|
||||
.id = this->next_subop_id++,
|
||||
.opcode = OSD_OP_PING,
|
||||
},
|
||||
};
|
||||
op->callback = [this, cl](osd_op_t *op)
|
||||
{
|
||||
int fail_fd = (op->reply.hdr.retval != 0 ? op->peer_fd : -1);
|
||||
cl->ping_time_remaining = 0;
|
||||
delete op;
|
||||
if (fail_fd >= 0)
|
||||
{
|
||||
stop_client(fail_fd, true);
|
||||
}
|
||||
};
|
||||
outbox_push(op);
|
||||
cl->ping_time_remaining = osd_ping_timeout;
|
||||
cl->idle_time_remaining = osd_idle_timeout;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
cl->idle_time_remaining = osd_idle_timeout;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
osd_messenger_t::~osd_messenger_t()
|
||||
{
|
||||
if (keepalive_timer_id >= 0)
|
||||
{
|
||||
tfd->clear_timer(keepalive_timer_id);
|
||||
keepalive_timer_id = -1;
|
||||
}
|
||||
while (clients.size() > 0)
|
||||
{
|
||||
stop_client(clients.begin()->first);
|
||||
stop_client(clients.begin()->first, true);
|
||||
}
|
||||
}
|
||||
|
||||
void osd_messenger_t::parse_config(const json11::Json & config)
|
||||
{
|
||||
this->use_sync_send_recv = config["use_sync_send_recv"].bool_value() ||
|
||||
config["use_sync_send_recv"].uint64_value();
|
||||
this->peer_connect_interval = config["peer_connect_interval"].uint64_value();
|
||||
if (!this->peer_connect_interval)
|
||||
{
|
||||
this->peer_connect_interval = DEFAULT_PEER_CONNECT_INTERVAL;
|
||||
}
|
||||
this->peer_connect_timeout = config["peer_connect_timeout"].uint64_value();
|
||||
if (!this->peer_connect_timeout)
|
||||
{
|
||||
this->peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT;
|
||||
}
|
||||
this->osd_idle_timeout = config["osd_idle_timeout"].uint64_value();
|
||||
if (!this->osd_idle_timeout)
|
||||
{
|
||||
this->osd_idle_timeout = DEFAULT_OSD_PING_TIMEOUT;
|
||||
}
|
||||
this->osd_ping_timeout = config["osd_ping_timeout"].uint64_value();
|
||||
if (!this->osd_ping_timeout)
|
||||
{
|
||||
this->osd_ping_timeout = DEFAULT_OSD_PING_TIMEOUT;
|
||||
}
|
||||
this->log_level = config["log_level"].uint64_value();
|
||||
}
|
||||
|
||||
void osd_messenger_t::connect_peer(uint64_t peer_osd, json11::Json peer_state)
|
||||
{
|
||||
if (wanted_peers.find(peer_osd) == wanted_peers.end())
|
||||
|
@ -111,12 +203,12 @@ void osd_messenger_t::try_connect_peer_addr(osd_num_t peer_osd, const char *peer
|
|||
timeout_id = tfd->set_timer(1000*peer_connect_timeout, false, [this, peer_fd](int timer_id)
|
||||
{
|
||||
osd_num_t peer_osd = clients.at(peer_fd)->osd_num;
|
||||
stop_client(peer_fd);
|
||||
stop_client(peer_fd, true);
|
||||
on_connect_peer(peer_osd, -EIO);
|
||||
return;
|
||||
});
|
||||
}
|
||||
clients[peer_fd] = new osd_client_t({
|
||||
clients[peer_fd] = new osd_client_t((osd_client_t){
|
||||
.peer_addr = addr,
|
||||
.peer_port = peer_port,
|
||||
.peer_fd = peer_fd,
|
||||
|
@ -149,7 +241,7 @@ void osd_messenger_t::handle_connect_epoll(int peer_fd)
|
|||
}
|
||||
if (result != 0)
|
||||
{
|
||||
stop_client(peer_fd);
|
||||
stop_client(peer_fd, true);
|
||||
on_connect_peer(peer_osd, -result);
|
||||
return;
|
||||
}
|
||||
|
@ -171,7 +263,7 @@ void osd_messenger_t::handle_peer_epoll(int peer_fd, int epoll_events)
|
|||
{
|
||||
// Stop client
|
||||
printf("[OSD %lu] client %d disconnected\n", this->osd_num, peer_fd);
|
||||
stop_client(peer_fd);
|
||||
stop_client(peer_fd, true);
|
||||
}
|
||||
else if (epoll_events & EPOLLIN)
|
||||
{
|
||||
|
@ -233,7 +325,7 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
|
|||
osd_op_t *op = new osd_op_t();
|
||||
op->op_type = OSD_OP_OUT;
|
||||
op->peer_fd = cl->peer_fd;
|
||||
op->req = {
|
||||
op->req = (osd_any_op_t){
|
||||
.show_conf = {
|
||||
.header = {
|
||||
.magic = SECONDARY_OSD_OP_MAGIC,
|
||||
|
@ -309,7 +401,7 @@ void osd_messenger_t::cancel_op(osd_op_t *op)
|
|||
}
|
||||
}
|
||||
|
||||
void osd_messenger_t::stop_client(int peer_fd)
|
||||
void osd_messenger_t::stop_client(int peer_fd, bool force)
|
||||
{
|
||||
assert(peer_fd != 0);
|
||||
auto it = clients.find(peer_fd);
|
||||
|
@ -334,6 +426,10 @@ void osd_messenger_t::stop_client(int peer_fd)
|
|||
printf("[OSD %lu] Stopping client %d (regular client)\n", osd_num, peer_fd);
|
||||
}
|
||||
}
|
||||
else if (!force)
|
||||
{
|
||||
return;
|
||||
}
|
||||
cl->peer_state = PEER_STOPPED;
|
||||
clients.erase(it);
|
||||
tfd->set_fd_handler(peer_fd, false, NULL);
|
||||
|
@ -348,7 +444,14 @@ void osd_messenger_t::stop_client(int peer_fd)
|
|||
}
|
||||
if (cl->read_op)
|
||||
{
|
||||
delete cl->read_op;
|
||||
if (cl->read_op->callback)
|
||||
{
|
||||
cancel_op(cl->read_op);
|
||||
}
|
||||
else
|
||||
{
|
||||
delete cl->read_op;
|
||||
}
|
||||
cl->read_op = NULL;
|
||||
}
|
||||
for (auto rit = read_ready_clients.begin(); rit != read_ready_clients.end(); rit++)
|
||||
|
@ -402,7 +505,7 @@ void osd_messenger_t::accept_connections(int listen_fd)
|
|||
fcntl(peer_fd, F_SETFL, fcntl(peer_fd, F_GETFL, 0) | O_NONBLOCK);
|
||||
int one = 1;
|
||||
setsockopt(peer_fd, SOL_TCP, TCP_NODELAY, &one, sizeof(one));
|
||||
clients[peer_fd] = new osd_client_t({
|
||||
clients[peer_fd] = new osd_client_t((osd_client_t){
|
||||
.peer_addr = addr,
|
||||
.peer_port = ntohs(addr.sin_port),
|
||||
.peer_fd = peer_fd,
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
||||
|
@ -34,6 +34,8 @@
|
|||
|
||||
#define DEFAULT_PEER_CONNECT_INTERVAL 5
|
||||
#define DEFAULT_PEER_CONNECT_TIMEOUT 5
|
||||
#define DEFAULT_OSD_PING_TIMEOUT 5
|
||||
#define DEFAULT_BITMAP_GRANULARITY 4096
|
||||
|
||||
// Kind of a vector with small-list-optimisation
|
||||
struct osd_op_buf_list_t
|
||||
|
@ -173,13 +175,17 @@ struct osd_primary_op_data_t;
|
|||
|
||||
struct osd_op_t
|
||||
{
|
||||
timespec tv_begin;
|
||||
timespec tv_begin = { 0 }, tv_end = { 0 };
|
||||
uint64_t op_type = OSD_OP_IN;
|
||||
int peer_fd;
|
||||
osd_any_op_t req;
|
||||
osd_any_reply_t reply;
|
||||
blockstore_op_t *bs_op = NULL;
|
||||
void *buf = NULL;
|
||||
// bitmap, bitmap_len, bmp_data are only meaningful for reads
|
||||
void *bitmap = NULL;
|
||||
unsigned bitmap_len = 0;
|
||||
unsigned bmp_data = 0;
|
||||
void *rmw_buf = NULL;
|
||||
osd_primary_op_data_t* op_data = NULL;
|
||||
std::function<void(osd_op_t*)> callback;
|
||||
|
@ -198,6 +204,8 @@ struct osd_client_t
|
|||
int peer_fd;
|
||||
int peer_state;
|
||||
int connect_timeout_id = -1;
|
||||
int ping_time_remaining = 0;
|
||||
int idle_time_remaining = 0;
|
||||
osd_num_t osd_num = 0;
|
||||
|
||||
void *in_buf = NULL;
|
||||
|
@ -251,6 +259,7 @@ struct osd_messenger_t
|
|||
{
|
||||
timerfd_manager_t *tfd;
|
||||
ring_loop_t *ringloop;
|
||||
int keepalive_timer_id = -1;
|
||||
|
||||
// osd_num_t is only for logging and asserts
|
||||
osd_num_t osd_num;
|
||||
|
@ -258,6 +267,8 @@ struct osd_messenger_t
|
|||
int receive_buffer_size = 64*1024;
|
||||
int peer_connect_interval = DEFAULT_PEER_CONNECT_INTERVAL;
|
||||
int peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT;
|
||||
int osd_idle_timeout = DEFAULT_OSD_PING_TIMEOUT;
|
||||
int osd_ping_timeout = DEFAULT_OSD_PING_TIMEOUT;
|
||||
int log_level = 0;
|
||||
bool use_sync_send_recv = false;
|
||||
|
||||
|
@ -274,8 +285,10 @@ struct osd_messenger_t
|
|||
osd_op_stats_t stats;
|
||||
|
||||
public:
|
||||
void init();
|
||||
void parse_config(const json11::Json & config);
|
||||
void connect_peer(uint64_t osd_num, json11::Json peer_state);
|
||||
void stop_client(int peer_fd);
|
||||
void stop_client(int peer_fd, bool force = false);
|
||||
void outbox_push(osd_op_t *cur_op);
|
||||
std::function<void(osd_op_t*)> exec_op;
|
||||
std::function<void(osd_num_t)> repeer_pgs;
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#include "messenger.h"
|
||||
|
||||
|
@ -9,6 +9,10 @@ void osd_messenger_t::read_requests()
|
|||
{
|
||||
int peer_fd = read_ready_clients[i];
|
||||
osd_client_t *cl = clients[peer_fd];
|
||||
if (cl->read_msg.msg_iovlen)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
if (cl->read_remaining < receive_buffer_size)
|
||||
{
|
||||
cl->read_iov.iov_base = cl->in_buf;
|
||||
|
@ -29,6 +33,7 @@ void osd_messenger_t::read_requests()
|
|||
io_uring_sqe* sqe = ringloop->get_sqe();
|
||||
if (!sqe)
|
||||
{
|
||||
cl->read_msg.msg_iovlen = 0;
|
||||
read_ready_clients.erase(read_ready_clients.begin(), read_ready_clients.begin() + i);
|
||||
return;
|
||||
}
|
||||
|
@ -52,6 +57,7 @@ void osd_messenger_t::read_requests()
|
|||
bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
|
||||
{
|
||||
bool ret = false;
|
||||
cl->read_msg.msg_iovlen = 0;
|
||||
cl->refs--;
|
||||
if (cl->peer_state == PEER_STOPPED)
|
||||
{
|
||||
|
@ -160,8 +166,14 @@ bool osd_messenger_t::handle_finished_read(osd_client_t *cl)
|
|||
{
|
||||
if (cl->read_op->req.hdr.magic == SECONDARY_OSD_REPLY_MAGIC)
|
||||
return handle_reply_hdr(cl);
|
||||
else
|
||||
else if (cl->read_op->req.hdr.magic == SECONDARY_OSD_OP_MAGIC)
|
||||
handle_op_hdr(cl);
|
||||
else
|
||||
{
|
||||
printf("Received garbage: magic=%lx id=%lu opcode=%lx from %d\n", cl->read_op->req.hdr.magic, cl->read_op->req.hdr.id, cl->read_op->req.hdr.opcode, cl->peer_fd);
|
||||
stop_client(cl->peer_fd);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else if (cl->read_state == CL_READ_DATA)
|
||||
{
|
||||
|
@ -190,22 +202,34 @@ void osd_messenger_t::handle_op_hdr(osd_client_t *cl)
|
|||
osd_op_t *cur_op = cl->read_op;
|
||||
if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ)
|
||||
{
|
||||
if (cur_op->req.sec_rw.len > 0)
|
||||
cur_op->buf = memalign_or_die(MEM_ALIGNMENT, cur_op->req.sec_rw.len);
|
||||
cl->read_remaining = 0;
|
||||
}
|
||||
else if (cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
|
||||
cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE)
|
||||
{
|
||||
if (cur_op->req.sec_rw.attr_len > 0)
|
||||
{
|
||||
if (cur_op->req.sec_rw.attr_len > sizeof(unsigned))
|
||||
cur_op->bitmap = cur_op->rmw_buf = malloc_or_die(cur_op->req.sec_rw.attr_len);
|
||||
else
|
||||
cur_op->bitmap = &cur_op->bmp_data;
|
||||
cl->recv_list.push_back(cur_op->bitmap, cur_op->req.sec_rw.attr_len);
|
||||
}
|
||||
if (cur_op->req.sec_rw.len > 0)
|
||||
{
|
||||
cur_op->buf = memalign_or_die(MEM_ALIGNMENT, cur_op->req.sec_rw.len);
|
||||
cl->read_remaining = cur_op->req.sec_rw.len;
|
||||
cl->recv_list.push_back(cur_op->buf, cur_op->req.sec_rw.len);
|
||||
}
|
||||
cl->read_remaining = cur_op->req.sec_rw.len + cur_op->req.sec_rw.attr_len;
|
||||
}
|
||||
else if (cur_op->req.hdr.opcode == OSD_OP_SEC_STABILIZE ||
|
||||
cur_op->req.hdr.opcode == OSD_OP_SEC_ROLLBACK)
|
||||
{
|
||||
if (cur_op->req.sec_stab.len > 0)
|
||||
{
|
||||
cur_op->buf = memalign_or_die(MEM_ALIGNMENT, cur_op->req.sec_stab.len);
|
||||
cl->recv_list.push_back(cur_op->buf, cur_op->req.sec_stab.len);
|
||||
}
|
||||
cl->read_remaining = cur_op->req.sec_stab.len;
|
||||
}
|
||||
else if (cur_op->req.hdr.opcode == OSD_OP_READ)
|
||||
|
@ -215,13 +239,15 @@ void osd_messenger_t::handle_op_hdr(osd_client_t *cl)
|
|||
else if (cur_op->req.hdr.opcode == OSD_OP_WRITE)
|
||||
{
|
||||
if (cur_op->req.rw.len > 0)
|
||||
{
|
||||
cur_op->buf = memalign_or_die(MEM_ALIGNMENT, cur_op->req.rw.len);
|
||||
cl->recv_list.push_back(cur_op->buf, cur_op->req.rw.len);
|
||||
}
|
||||
cl->read_remaining = cur_op->req.rw.len;
|
||||
}
|
||||
if (cl->read_remaining > 0)
|
||||
{
|
||||
// Read data
|
||||
cl->recv_list.push_back(cur_op->buf, cl->read_remaining);
|
||||
cl->read_state = CL_READ_DATA;
|
||||
}
|
||||
else
|
||||
|
@ -247,12 +273,12 @@ bool osd_messenger_t::handle_reply_hdr(osd_client_t *cl)
|
|||
osd_op_t *op = req_it->second;
|
||||
memcpy(op->reply.buf, cl->read_op->req.buf, OSD_PACKET_SIZE);
|
||||
cl->sent_ops.erase(req_it);
|
||||
if ((op->reply.hdr.opcode == OSD_OP_SEC_READ || op->reply.hdr.opcode == OSD_OP_READ) &&
|
||||
op->reply.hdr.retval > 0)
|
||||
if (op->reply.hdr.opcode == OSD_OP_SEC_READ || op->reply.hdr.opcode == OSD_OP_READ)
|
||||
{
|
||||
// Read data. In this case we assume that the buffer is preallocated by the caller (!)
|
||||
assert(op->iov.count > 0);
|
||||
if (op->reply.hdr.retval != (op->reply.hdr.opcode == OSD_OP_SEC_READ ? op->req.sec_rw.len : op->req.rw.len))
|
||||
unsigned bmp_len = (op->reply.hdr.opcode == OSD_OP_SEC_READ ? op->reply.sec_rw.attr_len : op->reply.rw.bitmap_len);
|
||||
if (op->reply.hdr.retval != (op->reply.hdr.opcode == OSD_OP_SEC_READ ? op->req.sec_rw.len : op->req.rw.len) ||
|
||||
bmp_len > op->bitmap_len)
|
||||
{
|
||||
// Check reply length to not overflow the buffer
|
||||
printf("Client %d read reply of different length\n", cl->peer_fd);
|
||||
|
@ -260,11 +286,23 @@ bool osd_messenger_t::handle_reply_hdr(osd_client_t *cl)
|
|||
stop_client(cl->peer_fd);
|
||||
return false;
|
||||
}
|
||||
cl->recv_list.append(op->iov);
|
||||
if (bmp_len > 0)
|
||||
{
|
||||
cl->recv_list.push_back(op->bitmap, bmp_len);
|
||||
}
|
||||
if (op->reply.hdr.retval > 0)
|
||||
{
|
||||
assert(op->iov.count > 0);
|
||||
cl->recv_list.append(op->iov);
|
||||
}
|
||||
cl->read_remaining = op->reply.hdr.retval + bmp_len;
|
||||
if (cl->read_remaining == 0)
|
||||
{
|
||||
goto reuse;
|
||||
}
|
||||
delete cl->read_op;
|
||||
cl->read_op = op;
|
||||
cl->read_state = CL_READ_REPLY_DATA;
|
||||
cl->read_remaining = op->reply.hdr.retval;
|
||||
}
|
||||
else if (op->reply.hdr.opcode == OSD_OP_SEC_LIST && op->reply.hdr.retval > 0)
|
||||
{
|
||||
|
@ -288,6 +326,7 @@ bool osd_messenger_t::handle_reply_hdr(osd_client_t *cl)
|
|||
}
|
||||
else
|
||||
{
|
||||
reuse:
|
||||
// It's fine to reuse cl->read_op for the next reply
|
||||
handle_reply_ready(op);
|
||||
cl->recv_list.push_back(cl->read_op->req.buf, OSD_PACKET_SIZE);
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#define _XOPEN_SOURCE
|
||||
#include <limits.h>
|
||||
|
@ -46,7 +46,29 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
|
|||
to_send_list.push_back((iovec){ .iov_base = cur_op->req.buf, .iov_len = OSD_PACKET_SIZE });
|
||||
cl->sent_ops[cur_op->req.hdr.id] = cur_op;
|
||||
}
|
||||
// Pre-defined send_lists
|
||||
to_outbox.push_back(NULL);
|
||||
// Bitmap
|
||||
if (cur_op->op_type == OSD_OP_IN &&
|
||||
cur_op->req.hdr.opcode == OSD_OP_SEC_READ &&
|
||||
cur_op->reply.sec_rw.attr_len > 0)
|
||||
{
|
||||
to_send_list.push_back((iovec){
|
||||
.iov_base = cur_op->bitmap,
|
||||
.iov_len = cur_op->reply.sec_rw.attr_len,
|
||||
});
|
||||
to_outbox.push_back(NULL);
|
||||
}
|
||||
else if (cur_op->op_type == OSD_OP_OUT &&
|
||||
(cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE || cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE) &&
|
||||
cur_op->req.sec_rw.attr_len > 0)
|
||||
{
|
||||
to_send_list.push_back((iovec){
|
||||
.iov_base = cur_op->bitmap,
|
||||
.iov_len = cur_op->req.sec_rw.attr_len,
|
||||
});
|
||||
to_outbox.push_back(NULL);
|
||||
}
|
||||
// Operation data
|
||||
if ((cur_op->op_type == OSD_OP_IN
|
||||
? (cur_op->req.hdr.opcode == OSD_OP_READ ||
|
||||
cur_op->req.hdr.opcode == OSD_OP_SEC_READ ||
|
||||
|
@ -58,17 +80,17 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
|
|||
cur_op->req.hdr.opcode == OSD_OP_SEC_STABILIZE ||
|
||||
cur_op->req.hdr.opcode == OSD_OP_SEC_ROLLBACK)) && cur_op->iov.count > 0)
|
||||
{
|
||||
to_outbox.push_back(NULL);
|
||||
for (int i = 0; i < cur_op->iov.count; i++)
|
||||
{
|
||||
assert(cur_op->iov.buf[i].iov_base);
|
||||
to_send_list.push_back(cur_op->iov.buf[i]);
|
||||
to_outbox.push_back(i == cur_op->iov.count-1 ? cur_op : NULL);
|
||||
to_outbox.push_back(NULL);
|
||||
}
|
||||
}
|
||||
else
|
||||
if (cur_op->op_type == OSD_OP_IN)
|
||||
{
|
||||
to_outbox.push_back(cur_op);
|
||||
// To free it later
|
||||
to_outbox[to_outbox.size()-1] = cur_op;
|
||||
}
|
||||
if (!ringloop)
|
||||
{
|
||||
|
@ -92,8 +114,14 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
|
|||
void osd_messenger_t::measure_exec(osd_op_t *cur_op)
|
||||
{
|
||||
// Measure execution latency
|
||||
timespec tv_end;
|
||||
clock_gettime(CLOCK_REALTIME, &tv_end);
|
||||
if (cur_op->req.hdr.opcode > OSD_OP_MAX)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if (!cur_op->tv_end.tv_sec)
|
||||
{
|
||||
clock_gettime(CLOCK_REALTIME, &cur_op->tv_end);
|
||||
}
|
||||
stats.op_stat_count[cur_op->req.hdr.opcode]++;
|
||||
if (!stats.op_stat_count[cur_op->req.hdr.opcode])
|
||||
{
|
||||
|
@ -102,8 +130,8 @@ void osd_messenger_t::measure_exec(osd_op_t *cur_op)
|
|||
stats.op_stat_bytes[cur_op->req.hdr.opcode] = 0;
|
||||
}
|
||||
stats.op_stat_sum[cur_op->req.hdr.opcode] += (
|
||||
(tv_end.tv_sec - cur_op->tv_begin.tv_sec)*1000000 +
|
||||
(tv_end.tv_nsec - cur_op->tv_begin.tv_nsec)/1000
|
||||
(cur_op->tv_end.tv_sec - cur_op->tv_begin.tv_sec)*1000000 +
|
||||
(cur_op->tv_end.tv_nsec - cur_op->tv_begin.tv_nsec)/1000
|
||||
);
|
||||
if (cur_op->req.hdr.opcode == OSD_OP_READ ||
|
||||
cur_op->req.hdr.opcode == OSD_OP_WRITE)
|
||||
|
@ -198,11 +226,8 @@ void osd_messenger_t::handle_send(int result, osd_client_t *cl)
|
|||
{
|
||||
if (cl->outbox[done])
|
||||
{
|
||||
// Operation fully sent
|
||||
if (cl->outbox[done]->op_type == OSD_OP_IN)
|
||||
{
|
||||
delete cl->outbox[done];
|
||||
}
|
||||
// Reply fully sent
|
||||
delete cl->outbox[done];
|
||||
}
|
||||
result -= iov.iov_len;
|
||||
done++;
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
// Similar to qemu-nbd, but sets timeout and uses io_uring
|
||||
|
||||
#include <linux/nbd.h>
|
||||
|
@ -17,6 +17,10 @@
|
|||
#include "epoll_manager.h"
|
||||
#include "cluster_client.h"
|
||||
|
||||
#ifndef MSG_ZEROCOPY
|
||||
#define MSG_ZEROCOPY 0
|
||||
#endif
|
||||
|
||||
const char *exe_name = NULL;
|
||||
|
||||
class nbd_proxy
|
||||
|
@ -107,7 +111,7 @@ public:
|
|||
{
|
||||
printf(
|
||||
"Vitastor NBD proxy\n"
|
||||
"(c) Vitaliy Filippov, 2020 (VNPL-1.0)\n\n"
|
||||
"(c) Vitaliy Filippov, 2020 (VNPL-1.1)\n\n"
|
||||
"USAGE:\n"
|
||||
" %s map --etcd_address <etcd_address> --pool <pool> --inode <inode> --size <size in bytes>\n"
|
||||
" %s unmap /dev/nbd0\n"
|
|
@ -1,17 +1,19 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <stdint.h>
|
||||
#include <functional>
|
||||
|
||||
typedef uint64_t inode_t;
|
||||
|
||||
// 16 bytes per object/stripe id
|
||||
// stripe = (start of the parity stripe + peer role)
|
||||
// i.e. for example (256KB + one of 0,1,2)
|
||||
struct __attribute__((__packed__)) object_id
|
||||
{
|
||||
uint64_t inode;
|
||||
inode_t inode;
|
||||
uint64_t stripe;
|
||||
};
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include <sys/socket.h>
|
||||
#include <sys/poll.h>
|
||||
|
@ -9,15 +9,21 @@
|
|||
|
||||
#include "osd.h"
|
||||
|
||||
osd_t::osd_t(blockstore_config_t & config, blockstore_t *bs, ring_loop_t *ringloop)
|
||||
osd_t::osd_t(blockstore_config_t & config, ring_loop_t *ringloop)
|
||||
{
|
||||
bs_block_size = strtoull(config["block_size"].c_str(), NULL, 10);
|
||||
bs_bitmap_granularity = strtoull(config["bitmap_granularity"].c_str(), NULL, 10);
|
||||
if (!bs_block_size)
|
||||
bs_block_size = DEFAULT_BLOCK_SIZE;
|
||||
if (!bs_bitmap_granularity)
|
||||
bs_bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
|
||||
clean_entry_bitmap_size = bs_block_size / bs_bitmap_granularity / 8;
|
||||
|
||||
this->config = config;
|
||||
this->bs = bs;
|
||||
this->ringloop = ringloop;
|
||||
|
||||
this->bs_block_size = bs->get_block_size();
|
||||
// FIXME: use bitmap granularity instead
|
||||
this->bs_disk_alignment = bs->get_disk_alignment();
|
||||
// FIXME: Create Blockstore from on-disk superblock config and check it against the OSD cluster config
|
||||
this->bs = new blockstore_t(config, ringloop);
|
||||
|
||||
parse_config(config);
|
||||
|
||||
|
@ -37,6 +43,7 @@ osd_t::osd_t(blockstore_config_t & config, blockstore_t *bs, ring_loop_t *ringlo
|
|||
c_cli.ringloop = this->ringloop;
|
||||
c_cli.exec_op = [this](osd_op_t *op) { exec_op(op); };
|
||||
c_cli.repeer_pgs = [this](osd_num_t peer_osd) { repeer_pgs(peer_osd); };
|
||||
c_cli.init();
|
||||
|
||||
init_cluster();
|
||||
|
||||
|
@ -48,6 +55,7 @@ osd_t::~osd_t()
|
|||
{
|
||||
ringloop->unregister_consumer(&consumer);
|
||||
delete epmgr;
|
||||
delete bs;
|
||||
close(listen_fd);
|
||||
}
|
||||
|
||||
|
@ -55,6 +63,7 @@ void osd_t::parse_config(blockstore_config_t & config)
|
|||
{
|
||||
if (config.find("log_level") == config.end())
|
||||
config["log_level"] = "1";
|
||||
log_level = strtoull(config["log_level"].c_str(), NULL, 10);
|
||||
// Initial startup configuration
|
||||
json11::Json json_config = json11::Json(config);
|
||||
st_cli.parse_config(json_config);
|
||||
|
@ -66,6 +75,8 @@ void osd_t::parse_config(blockstore_config_t & config)
|
|||
throw std::runtime_error("osd_num is required in the configuration");
|
||||
c_cli.osd_num = osd_num;
|
||||
run_primary = config["run_primary"] != "false" && config["run_primary"] != "0" && config["run_primary"] != "no";
|
||||
no_rebalance = config["no_rebalance"] == "true" || config["no_rebalance"] == "1" || config["no_rebalance"] == "yes";
|
||||
no_recovery = config["no_recovery"] == "true" || config["no_recovery"] == "1" || config["no_recovery"] == "yes";
|
||||
// Cluster configuration
|
||||
bind_address = config["bind_address"];
|
||||
if (bind_address == "")
|
||||
|
@ -92,6 +103,9 @@ void osd_t::parse_config(blockstore_config_t & config)
|
|||
recovery_queue_depth = strtoull(config["recovery_queue_depth"].c_str(), NULL, 10);
|
||||
if (recovery_queue_depth < 1 || recovery_queue_depth > MAX_RECOVERY_QUEUE)
|
||||
recovery_queue_depth = DEFAULT_RECOVERY_QUEUE;
|
||||
recovery_sync_batch = strtoull(config["recovery_sync_batch"].c_str(), NULL, 10);
|
||||
if (recovery_sync_batch < 1 || recovery_sync_batch > MAX_RECOVERY_QUEUE)
|
||||
recovery_sync_batch = DEFAULT_RECOVERY_BATCH;
|
||||
if (config["readonly"] == "true" || config["readonly"] == "1" || config["readonly"] == "yes")
|
||||
readonly = true;
|
||||
print_stats_interval = strtoull(config["print_stats_interval"].c_str(), NULL, 10);
|
||||
|
@ -100,14 +114,7 @@ void osd_t::parse_config(blockstore_config_t & config)
|
|||
slow_log_interval = strtoull(config["slow_log_interval"].c_str(), NULL, 10);
|
||||
if (!slow_log_interval)
|
||||
slow_log_interval = 10;
|
||||
c_cli.peer_connect_interval = strtoull(config["peer_connect_interval"].c_str(), NULL, 10);
|
||||
if (!c_cli.peer_connect_interval)
|
||||
c_cli.peer_connect_interval = DEFAULT_PEER_CONNECT_INTERVAL;
|
||||
c_cli.peer_connect_timeout = strtoull(config["peer_connect_timeout"].c_str(), NULL, 10);
|
||||
if (!c_cli.peer_connect_timeout)
|
||||
c_cli.peer_connect_timeout = DEFAULT_PEER_CONNECT_TIMEOUT;
|
||||
log_level = strtoull(config["log_level"].c_str(), NULL, 10);
|
||||
c_cli.log_level = log_level;
|
||||
c_cli.parse_config(json_config);
|
||||
}
|
||||
|
||||
void osd_t::bind_socket()
|
||||
|
@ -171,7 +178,7 @@ bool osd_t::shutdown()
|
|||
{
|
||||
return false;
|
||||
}
|
||||
return bs->is_safe_to_stop();
|
||||
return !bs || bs->is_safe_to_stop();
|
||||
}
|
||||
|
||||
void osd_t::loop()
|
||||
|
@ -198,19 +205,25 @@ void osd_t::exec_op(osd_op_t *cur_op)
|
|||
cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE ||
|
||||
cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE) &&
|
||||
(cur_op->req.sec_rw.len > OSD_RW_MAX ||
|
||||
cur_op->req.sec_rw.len % bs_disk_alignment ||
|
||||
cur_op->req.sec_rw.offset % bs_disk_alignment)) ||
|
||||
cur_op->req.sec_rw.len % bs_bitmap_granularity ||
|
||||
cur_op->req.sec_rw.offset % bs_bitmap_granularity)) ||
|
||||
((cur_op->req.hdr.opcode == OSD_OP_READ ||
|
||||
cur_op->req.hdr.opcode == OSD_OP_WRITE ||
|
||||
cur_op->req.hdr.opcode == OSD_OP_DELETE) &&
|
||||
(cur_op->req.rw.len > OSD_RW_MAX ||
|
||||
cur_op->req.rw.len % bs_disk_alignment ||
|
||||
cur_op->req.rw.offset % bs_disk_alignment)))
|
||||
cur_op->req.rw.len % bs_bitmap_granularity ||
|
||||
cur_op->req.rw.offset % bs_bitmap_granularity)))
|
||||
{
|
||||
// Bad command
|
||||
finish_op(cur_op, -EINVAL);
|
||||
return;
|
||||
}
|
||||
if (cur_op->req.hdr.opcode == OSD_OP_PING)
|
||||
{
|
||||
// Pong
|
||||
finish_op(cur_op, 0);
|
||||
return;
|
||||
}
|
||||
if (readonly &&
|
||||
cur_op->req.hdr.opcode != OSD_OP_SEC_READ &&
|
||||
cur_op->req.hdr.opcode != OSD_OP_SEC_LIST &&
|
||||
|
@ -261,9 +274,9 @@ void osd_t::reset_stats()
|
|||
|
||||
void osd_t::print_stats()
|
||||
{
|
||||
for (int i = 0; i <= OSD_OP_MAX; i++)
|
||||
for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
|
||||
{
|
||||
if (c_cli.stats.op_stat_count[i] != prev_stats.op_stat_count[i])
|
||||
if (c_cli.stats.op_stat_count[i] != prev_stats.op_stat_count[i] && i != OSD_OP_PING)
|
||||
{
|
||||
uint64_t avg = (c_cli.stats.op_stat_sum[i] - prev_stats.op_stat_sum[i])/(c_cli.stats.op_stat_count[i] - prev_stats.op_stat_count[i]);
|
||||
uint64_t bw = (c_cli.stats.op_stat_bytes[i] - prev_stats.op_stat_bytes[i]) / print_stats_interval;
|
||||
|
@ -284,7 +297,7 @@ void osd_t::print_stats()
|
|||
prev_stats.op_stat_bytes[i] = c_cli.stats.op_stat_bytes[i];
|
||||
}
|
||||
}
|
||||
for (int i = 0; i <= OSD_OP_MAX; i++)
|
||||
for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
|
||||
{
|
||||
if (c_cli.stats.subop_stat_count[i] != prev_stats.subop_stat_count[i])
|
||||
{
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
||||
|
@ -37,6 +37,7 @@
|
|||
#define DEFAULT_AUTOSYNC_INTERVAL 5
|
||||
#define MAX_RECOVERY_QUEUE 2048
|
||||
#define DEFAULT_RECOVERY_QUEUE 4
|
||||
#define DEFAULT_RECOVERY_BATCH 16
|
||||
|
||||
//#define OSD_STUB
|
||||
|
||||
|
@ -54,6 +55,17 @@ struct osd_recovery_op_t
|
|||
osd_op_t *osd_op = NULL;
|
||||
};
|
||||
|
||||
// Posted as /osd/inodestats/$osd, then accumulated by the monitor
|
||||
#define INODE_STATS_READ 0
|
||||
#define INODE_STATS_WRITE 1
|
||||
#define INODE_STATS_DELETE 2
|
||||
struct inode_stats_t
|
||||
{
|
||||
uint64_t op_sum[3] = { 0 };
|
||||
uint64_t op_count[3] = { 0 };
|
||||
uint64_t op_bytes[3] = { 0 };
|
||||
};
|
||||
|
||||
class osd_t
|
||||
{
|
||||
// config
|
||||
|
@ -64,6 +76,8 @@ class osd_t
|
|||
bool readonly = false;
|
||||
osd_num_t osd_num = 1; // OSD numbers start with 1
|
||||
bool run_primary = false;
|
||||
bool no_rebalance = false;
|
||||
bool no_recovery = false;
|
||||
std::string bind_address;
|
||||
int bind_port, listen_backlog;
|
||||
// FIXME: Implement client queue depth limit
|
||||
|
@ -74,6 +88,7 @@ class osd_t
|
|||
int immediate_commit = IMMEDIATE_NONE;
|
||||
int autosync_interval = DEFAULT_AUTOSYNC_INTERVAL; // sync every 5 seconds
|
||||
int recovery_queue_depth = DEFAULT_RECOVERY_QUEUE;
|
||||
int recovery_sync_batch = DEFAULT_RECOVERY_BATCH;
|
||||
int log_level = 0;
|
||||
|
||||
// cluster state
|
||||
|
@ -95,9 +110,11 @@ class osd_t
|
|||
std::map<pool_pg_num_t, pg_t> pgs;
|
||||
std::set<pool_pg_num_t> dirty_pgs;
|
||||
std::set<osd_num_t> dirty_osds;
|
||||
int copies_to_delete_after_sync_count = 0;
|
||||
uint64_t misplaced_objects = 0, degraded_objects = 0, incomplete_objects = 0;
|
||||
int peering_state = 0;
|
||||
std::map<object_id, osd_recovery_op_t> recovery_ops;
|
||||
int recovery_done = 0;
|
||||
osd_op_t *autosync_op = NULL;
|
||||
|
||||
// Unstable writes
|
||||
|
@ -109,7 +126,7 @@ class osd_t
|
|||
bool stopping = false;
|
||||
int inflight_ops = 0;
|
||||
blockstore_t *bs;
|
||||
uint32_t bs_block_size, bs_disk_alignment;
|
||||
uint32_t bs_block_size, bs_bitmap_granularity, clean_entry_bitmap_size;
|
||||
ring_loop_t *ringloop;
|
||||
timerfd_manager_t *tfd = NULL;
|
||||
epoll_manager_t *epmgr = NULL;
|
||||
|
@ -120,6 +137,7 @@ class osd_t
|
|||
|
||||
// op statistics
|
||||
osd_op_stats_t prev_stats;
|
||||
std::map<uint64_t, inode_stats_t> inode_stats;
|
||||
const char* recovery_stat_names[2] = { "degraded", "misplaced" };
|
||||
uint64_t recovery_stat_count[2][2] = { 0 };
|
||||
uint64_t recovery_stat_bytes[2][2] = { 0 };
|
||||
|
@ -160,6 +178,7 @@ class osd_t
|
|||
void submit_list_subop(osd_num_t role_osd, pg_peering_state_t *ps);
|
||||
void discard_list_subop(osd_op_t *list_op);
|
||||
bool stop_pg(pg_t & pg);
|
||||
void reset_pg(pg_t & pg);
|
||||
void finish_stop_pg(pg_t & pg);
|
||||
|
||||
// flushing, recovery and backfill
|
||||
|
@ -198,6 +217,7 @@ class osd_t
|
|||
void pg_cancel_write_queue(pg_t & pg, osd_op_t *first_op, object_id oid, int retval);
|
||||
void submit_primary_subops(int submit_type, uint64_t op_version, int pg_size, const uint64_t* osd_set, osd_op_t *cur_op);
|
||||
void submit_primary_del_subops(osd_op_t *cur_op, uint64_t *cur_set, uint64_t set_size, pg_osd_set_t & loc_set);
|
||||
void submit_primary_del_batch(osd_op_t *cur_op, obj_ver_osd_t *chunks_to_delete, int chunks_to_delete_count);
|
||||
void submit_primary_sync_subops(osd_op_t *cur_op);
|
||||
void submit_primary_stab_subops(osd_op_t *cur_op);
|
||||
|
||||
|
@ -210,7 +230,7 @@ class osd_t
|
|||
}
|
||||
|
||||
public:
|
||||
osd_t(blockstore_config_t & config, blockstore_t *bs, ring_loop_t *ringloop);
|
||||
osd_t(blockstore_config_t & config, ring_loop_t *ringloop);
|
||||
~osd_t();
|
||||
void force_stop(int exitcode);
|
||||
bool shutdown();
|
|
@ -1,9 +1,10 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "osd.h"
|
||||
#include "base64.h"
|
||||
#include "etcd_state_client.h"
|
||||
#include "osd_rmw.h"
|
||||
|
||||
// Startup sequence:
|
||||
// Start etcd watcher -> Load global OSD configuration -> Bind socket -> Acquire lease -> Report&lock OSD state
|
||||
|
@ -32,12 +33,26 @@ void osd_t::init_cluster()
|
|||
}
|
||||
pgs[{ 1, 1 }] = (pg_t){
|
||||
.state = PG_PEERING,
|
||||
.scheme = POOL_SCHEME_XOR,
|
||||
.pg_cursize = 0,
|
||||
.pg_size = 3,
|
||||
.pg_minsize = 2,
|
||||
.pg_data_size = 2,
|
||||
.pool_id = 1,
|
||||
.pg_num = 1,
|
||||
.target_set = { 1, 2, 3 },
|
||||
.cur_set = { 0, 0, 0 },
|
||||
};
|
||||
st_cli.pool_config[1] = (pool_config_t){
|
||||
.exists = true,
|
||||
.id = 1,
|
||||
.name = "testpool",
|
||||
.scheme = POOL_SCHEME_XOR,
|
||||
.pg_size = 3,
|
||||
.pg_minsize = 2,
|
||||
.pg_count = 1,
|
||||
.real_pg_count = 1,
|
||||
};
|
||||
report_pg_state(pgs[{ 1, 1 }]);
|
||||
pg_counts[1] = 1;
|
||||
}
|
||||
|
@ -127,7 +142,7 @@ json11::Json osd_t::get_statistics()
|
|||
}
|
||||
st["host"] = self_state["host"];
|
||||
json11::Json::object op_stats, subop_stats;
|
||||
for (int i = 0; i <= OSD_OP_MAX; i++)
|
||||
for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
|
||||
{
|
||||
op_stats[osd_op_names[i]] = json11::Json::object {
|
||||
{ "count", c_cli.stats.op_stat_count[i] },
|
||||
|
@ -135,7 +150,7 @@ json11::Json osd_t::get_statistics()
|
|||
{ "bytes", c_cli.stats.op_stat_bytes[i] },
|
||||
};
|
||||
}
|
||||
for (int i = 0; i <= OSD_OP_MAX; i++)
|
||||
for (int i = OSD_OP_MIN; i <= OSD_OP_MAX; i++)
|
||||
{
|
||||
subop_stats[osd_op_names[i]] = json11::Json::object {
|
||||
{ "count", c_cli.stats.subop_stat_count[i] },
|
||||
|
@ -164,11 +179,47 @@ void osd_t::report_statistics()
|
|||
return;
|
||||
}
|
||||
etcd_reporting_stats = true;
|
||||
// Report space usage statistics as a whole
|
||||
// Maybe we'll report it using deltas if we tune for a lot of inodes at some point
|
||||
json11::Json::object inode_space;
|
||||
for (auto kv: bs->get_inode_space_stats())
|
||||
{
|
||||
inode_space[std::to_string(kv.first)] = kv.second;
|
||||
}
|
||||
json11::Json::object inode_ops;
|
||||
for (auto kv: inode_stats)
|
||||
{
|
||||
inode_ops[std::to_string(kv.first)] = json11::Json::object {
|
||||
{ "read", json11::Json::object {
|
||||
{ "count", kv.second.op_count[INODE_STATS_READ] },
|
||||
{ "usec", kv.second.op_sum[INODE_STATS_READ] },
|
||||
{ "bytes", kv.second.op_bytes[INODE_STATS_READ] },
|
||||
} },
|
||||
{ "write", json11::Json::object {
|
||||
{ "count", kv.second.op_count[INODE_STATS_WRITE] },
|
||||
{ "usec", kv.second.op_sum[INODE_STATS_WRITE] },
|
||||
{ "bytes", kv.second.op_bytes[INODE_STATS_WRITE] },
|
||||
} },
|
||||
{ "delete", json11::Json::object {
|
||||
{ "count", kv.second.op_count[INODE_STATS_DELETE] },
|
||||
{ "usec", kv.second.op_sum[INODE_STATS_DELETE] },
|
||||
{ "bytes", kv.second.op_bytes[INODE_STATS_DELETE] },
|
||||
} },
|
||||
};
|
||||
}
|
||||
json11::Json::array txn = { json11::Json::object {
|
||||
{ "request_put", json11::Json::object {
|
||||
{ "key", base64_encode(st_cli.etcd_prefix+"/osd/stats/"+std::to_string(osd_num)) },
|
||||
{ "value", base64_encode(get_statistics().dump()) },
|
||||
} }
|
||||
} },
|
||||
{ "request_put", json11::Json::object {
|
||||
{ "key", base64_encode(st_cli.etcd_prefix+"/osd/space/"+std::to_string(osd_num)) },
|
||||
{ "value", base64_encode(json11::Json(inode_space).dump()) },
|
||||
} },
|
||||
{ "request_put", json11::Json::object {
|
||||
{ "key", base64_encode(st_cli.etcd_prefix+"/osd/inodestats/"+std::to_string(osd_num)) },
|
||||
{ "value", base64_encode(json11::Json(inode_ops).dump()) },
|
||||
} },
|
||||
} };
|
||||
for (auto & p: pgs)
|
||||
{
|
||||
|
@ -369,6 +420,7 @@ void osd_t::create_osd_state()
|
|||
{
|
||||
st_cli.load_pgs();
|
||||
}
|
||||
report_statistics();
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -479,7 +531,11 @@ void osd_t::apply_pg_count()
|
|||
}
|
||||
if (still_active > 0)
|
||||
{
|
||||
printf("[OSD %lu] PG count change detected, but %d PG(s) are still active. This is not allowed. Exiting\n", this->osd_num, still_active);
|
||||
printf(
|
||||
"[OSD %lu] PG count change detected for pool %u (new is %lu, old is %u),"
|
||||
" but %u PG(s) are still active. This is not allowed. Exiting\n",
|
||||
this->osd_num, pool_item.first, pool_item.second.real_pg_count, pg_counts[pool_item.first], still_active
|
||||
);
|
||||
force_stop(1);
|
||||
return;
|
||||
}
|
||||
|
@ -573,7 +629,10 @@ void osd_t::apply_pg_config()
|
|||
}
|
||||
else
|
||||
{
|
||||
throw std::runtime_error("Unexpected PG "+std::to_string(pg_num)+" state: "+std::to_string(pg_it->second.state));
|
||||
throw std::runtime_error(
|
||||
"Unexpected PG "+std::to_string(pool_id)+"/"+std::to_string(pg_num)+
|
||||
" state: "+std::to_string(pg_it->second.state)
|
||||
);
|
||||
}
|
||||
}
|
||||
auto & pg = this->pgs[{ .pool_id = pool_id, .pg_num = pg_num }];
|
||||
|
@ -583,6 +642,8 @@ void osd_t::apply_pg_config()
|
|||
.pg_cursize = 0,
|
||||
.pg_size = pool_item.second.pg_size,
|
||||
.pg_minsize = pool_item.second.pg_minsize,
|
||||
.pg_data_size = pg.scheme == POOL_SCHEME_REPLICATED
|
||||
? 1 : pool_item.second.pg_size - pool_item.second.parity_chunks,
|
||||
.pool_id = pool_id,
|
||||
.pg_num = pg_num,
|
||||
.reported_epoch = pg_cfg.epoch,
|
||||
|
@ -590,6 +651,10 @@ void osd_t::apply_pg_config()
|
|||
.all_peers = std::vector<osd_num_t>(all_peers.begin(), all_peers.end()),
|
||||
.target_set = pg_cfg.target_set,
|
||||
};
|
||||
if (pg.scheme == POOL_SCHEME_JERASURE)
|
||||
{
|
||||
use_jerasure(pg.pg_size, pg.pg_data_size, true);
|
||||
}
|
||||
this->pg_state_dirty.insert({ .pool_id = pool_id, .pg_num = pg_num });
|
||||
pg.print_state();
|
||||
if (pg_cfg.cur_primary == this->osd_num)
|
||||
|
@ -636,7 +701,21 @@ void osd_t::report_pg_states()
|
|||
auto & pg = pg_it->second;
|
||||
reporting_pgs.push_back({ *it, pg.history_changed });
|
||||
std::string state_key_base64 = base64_encode(st_cli.etcd_prefix+"/pg/state/"+std::to_string(pg.pool_id)+"/"+std::to_string(pg.pg_num));
|
||||
if (pg.state == PG_STARTING)
|
||||
bool pg_state_exists = false;
|
||||
if (pg.state != PG_STARTING)
|
||||
{
|
||||
auto pool_it = st_cli.pool_config.find(pg.pool_id);
|
||||
if (pool_it != st_cli.pool_config.end())
|
||||
{
|
||||
auto pg_it = pool_it->second.pg_config.find(pg.pg_num);
|
||||
if (pg_it != pool_it->second.pg_config.end() &&
|
||||
pg_it->second.cur_state != 0)
|
||||
{
|
||||
pg_state_exists = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!pg_state_exists)
|
||||
{
|
||||
// Check that the PG key does not exist
|
||||
// Failed check indicates an unsuccessful PG lock attempt in this case
|
||||
|
@ -648,9 +727,7 @@ void osd_t::report_pg_states()
|
|||
}
|
||||
else
|
||||
{
|
||||
// Check that the key is ours
|
||||
// Failed check indicates success for OFFLINE pgs (PG lock is already deleted)
|
||||
// and an unexpected race condition for started pgs (PG lock is held by someone else)
|
||||
// Check that the key is ours if it already exists
|
||||
checks.push_back(json11::Json::object {
|
||||
{ "target", "LEASE" },
|
||||
{ "lease", etcd_lease_id },
|
||||
|
@ -772,13 +849,16 @@ void osd_t::report_pg_states()
|
|||
for (auto pp: reporting_pgs)
|
||||
{
|
||||
auto pg_it = this->pgs.find(pp.first);
|
||||
if (pg_it != this->pgs.end())
|
||||
if (pg_it != this->pgs.end() &&
|
||||
pg_it->second.state == PG_OFFLINE &&
|
||||
pg_state_dirty.find(pp.first) == pg_state_dirty.end())
|
||||
{
|
||||
if (pg_it->second.state == PG_OFFLINE)
|
||||
// Forget offline PGs after reporting their state
|
||||
if (pg_it->second.scheme == POOL_SCHEME_JERASURE)
|
||||
{
|
||||
// Remove offline PGs after reporting their state
|
||||
this->pgs.erase(pg_it);
|
||||
use_jerasure(pg_it->second.pg_size, pg_it->second.pg_data_size, false);
|
||||
}
|
||||
this->pgs.erase(pg_it);
|
||||
}
|
||||
}
|
||||
// Push other PG state updates, if any
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "osd.h"
|
||||
|
||||
|
@ -95,7 +95,7 @@ void osd_t::handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, p
|
|||
{
|
||||
// This flush batch is done
|
||||
std::vector<osd_op_t*> continue_ops;
|
||||
auto & pg = pgs[pg_id];
|
||||
auto & pg = pgs.at(pg_id);
|
||||
auto it = pg.flush_actions.begin(), prev_it = it;
|
||||
auto erase_start = it;
|
||||
while (1)
|
||||
|
@ -166,7 +166,7 @@ void osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t
|
|||
{
|
||||
// local
|
||||
clock_gettime(CLOCK_REALTIME, &op->tv_begin);
|
||||
op->bs_op = new blockstore_op_t({
|
||||
op->bs_op = new blockstore_op_t((blockstore_op_t){
|
||||
.opcode = (uint64_t)(rollback ? BS_OP_ROLLBACK : BS_OP_STABLE),
|
||||
.callback = [this, op, pool_id, pg_num, fb](blockstore_op_t *bs_op)
|
||||
{
|
||||
|
@ -188,7 +188,7 @@ void osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t
|
|||
op->op_type = OSD_OP_OUT;
|
||||
op->iov.push_back(op->buf, count * sizeof(obj_ver_id));
|
||||
op->peer_fd = peer_fd;
|
||||
op->req = {
|
||||
op->req = (osd_any_op_t){
|
||||
.sec_stab = {
|
||||
.header = {
|
||||
.magic = SECONDARY_OSD_OP_MAGIC,
|
||||
|
@ -209,32 +209,38 @@ void osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t
|
|||
|
||||
bool osd_t::pick_next_recovery(osd_recovery_op_t &op)
|
||||
{
|
||||
for (auto pg_it = pgs.begin(); pg_it != pgs.end(); pg_it++)
|
||||
if (!no_recovery)
|
||||
{
|
||||
if ((pg_it->second.state & (PG_ACTIVE | PG_HAS_DEGRADED)) == (PG_ACTIVE | PG_HAS_DEGRADED))
|
||||
for (auto pg_it = pgs.begin(); pg_it != pgs.end(); pg_it++)
|
||||
{
|
||||
for (auto obj_it = pg_it->second.degraded_objects.begin(); obj_it != pg_it->second.degraded_objects.end(); obj_it++)
|
||||
if ((pg_it->second.state & (PG_ACTIVE | PG_HAS_DEGRADED)) == (PG_ACTIVE | PG_HAS_DEGRADED))
|
||||
{
|
||||
if (recovery_ops.find(obj_it->first) == recovery_ops.end())
|
||||
for (auto obj_it = pg_it->second.degraded_objects.begin(); obj_it != pg_it->second.degraded_objects.end(); obj_it++)
|
||||
{
|
||||
op.degraded = true;
|
||||
op.oid = obj_it->first;
|
||||
return true;
|
||||
if (recovery_ops.find(obj_it->first) == recovery_ops.end())
|
||||
{
|
||||
op.degraded = true;
|
||||
op.oid = obj_it->first;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (auto pg_it = pgs.begin(); pg_it != pgs.end(); pg_it++)
|
||||
if (!no_rebalance)
|
||||
{
|
||||
if ((pg_it->second.state & (PG_ACTIVE | PG_HAS_MISPLACED)) == (PG_ACTIVE | PG_HAS_MISPLACED))
|
||||
for (auto pg_it = pgs.begin(); pg_it != pgs.end(); pg_it++)
|
||||
{
|
||||
for (auto obj_it = pg_it->second.misplaced_objects.begin(); obj_it != pg_it->second.misplaced_objects.end(); obj_it++)
|
||||
if ((pg_it->second.state & (PG_ACTIVE | PG_HAS_MISPLACED)) == (PG_ACTIVE | PG_HAS_MISPLACED))
|
||||
{
|
||||
if (recovery_ops.find(obj_it->first) == recovery_ops.end())
|
||||
for (auto obj_it = pg_it->second.misplaced_objects.begin(); obj_it != pg_it->second.misplaced_objects.end(); obj_it++)
|
||||
{
|
||||
op.degraded = false;
|
||||
op.oid = obj_it->first;
|
||||
return true;
|
||||
if (recovery_ops.find(obj_it->first) == recovery_ops.end())
|
||||
{
|
||||
op.degraded = false;
|
||||
op.oid = obj_it->first;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -246,7 +252,7 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op)
|
|||
{
|
||||
op->osd_op = new osd_op_t();
|
||||
op->osd_op->op_type = OSD_OP_OUT;
|
||||
op->osd_op->req = {
|
||||
op->osd_op->req = (osd_any_op_t){
|
||||
.rw = {
|
||||
.header = {
|
||||
.magic = SECONDARY_OSD_OP_MAGIC,
|
||||
|
@ -264,7 +270,6 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op)
|
|||
}
|
||||
op->osd_op->callback = [this, op](osd_op_t *osd_op)
|
||||
{
|
||||
// Don't sync the write, it will be synced by our regular sync coroutine
|
||||
if (osd_op->reply.hdr.retval < 0)
|
||||
{
|
||||
// Error recovering object
|
||||
|
@ -286,6 +291,17 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op)
|
|||
op->osd_op = NULL;
|
||||
recovery_ops.erase(op->oid);
|
||||
delete osd_op;
|
||||
if (immediate_commit != IMMEDIATE_ALL)
|
||||
{
|
||||
recovery_done++;
|
||||
if (recovery_done >= recovery_sync_batch)
|
||||
{
|
||||
// Force sync every <recovery_sync_batch> operations
|
||||
// This is required not to pile up an excessive amount of delete operations
|
||||
autosync();
|
||||
recovery_done = 0;
|
||||
}
|
||||
}
|
||||
continue_recovery();
|
||||
};
|
||||
exec_op(op->osd_op);
|
|
@ -1,10 +1,11 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
||||
#define POOL_SCHEME_REPLICATED 1
|
||||
#define POOL_SCHEME_XOR 2
|
||||
#define POOL_SCHEME_JERASURE 3
|
||||
#define POOL_ID_MAX 0x10000
|
||||
#define POOL_ID_BITS 16
|
||||
#define INODE_POOL(inode) (pool_id_t)((inode) >> (64 - POOL_ID_BITS))
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 (see README.md for details)
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "osd.h"
|
||||
|
||||
|
@ -41,16 +41,13 @@ int main(int narg, char *args[])
|
|||
signal(SIGINT, handle_sigint);
|
||||
signal(SIGTERM, handle_sigint);
|
||||
ring_loop_t *ringloop = new ring_loop_t(512);
|
||||
// FIXME: Create Blockstore from on-disk superblock config and check it against the OSD cluster config
|
||||
blockstore_t *bs = new blockstore_t(config, ringloop);
|
||||
osd = new osd_t(config, bs, ringloop);
|
||||
osd = new osd_t(config, ringloop);
|
||||
while (1)
|
||||
{
|
||||
ringloop->loop();
|
||||
ringloop->wait();
|
||||
}
|
||||
delete osd;
|
||||
delete bs;
|
||||
delete ringloop;
|
||||
return 0;
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.0 or GNU GPL-2.0+ (see README.md for details)
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#include "osd_ops.h"
|
||||
|
||||
|
@ -19,4 +19,5 @@ const char* osd_op_names[] = {
|
|||
"primary_write",
|
||||
"primary_sync",
|
||||
"primary_delete",
|
||||
"ping",
|
||||
};
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue