Correct testing, change output style
- Implement the correct random write testing algorithm which is not influenced by Bluestore's deferred writes coalescing into a single operation: write 4Kb blocks in random positions of one of 16 4MB objects per OSD, also chosen randomly - Remove MB/sec from the statistical breakdown lines - Use sync writes (basically the same, but code is simpler) - Remove -DYA_PIDOR, always use urandom - Build with tcmalloc - Change curly brace style :-)master
parent
37f0686ff9
commit
63efee1bdf
20
Makefile
20
Makefile
|
|
@ -1,21 +1,17 @@
|
|||
|
||||
ifeq ($(ya_pidor),true)
|
||||
CXXFLAGS += -DYA_PIDOR
|
||||
endif
|
||||
|
||||
|
||||
|
||||
# -fsanitize=undefined
|
||||
CXXFLAGS += -std=c++11 -O3 -Wall -Wextra -I/usr/include/rados -I/usr/include/jsoncpp
|
||||
#-Wa,-adhln -g
|
||||
LDFLAGS += -pthread -lrados -ljsoncpp -lstdc++ -lm
|
||||
CXXFLAGS += -std=c++11 -O3 -g -Wall -Wextra -I/usr/include/rados -I/usr/include/jsoncpp -fno-builtin-malloc -fno-builtin-calloc -fno-builtin-realloc -fno-builtin-free
|
||||
# -Wa,-adhln -g
|
||||
# -lprofiler
|
||||
LDFLAGS += -pthread -lrados -ljsoncpp -lstdc++ -lm -g -ltcmalloc
|
||||
|
||||
#CC=clang-6.0
|
||||
|
||||
|
||||
main: main.o mysignals.o radosutil.o
|
||||
$(CC) $^ -o $@ $(LDFLAGS)
|
||||
|
||||
clean:
|
||||
rm -f main.o mysignals.o radosutil.o ./main
|
||||
|
||||
.cpp.o:
|
||||
$(CC) $(CPPFLAGS) $(CXXFLAGS) -c $< -o $@
|
||||
|
||||
|
|
@ -23,4 +19,4 @@ indent: *.cpp *.h
|
|||
clang-format-6.0 -i $^
|
||||
|
||||
builddep:
|
||||
sudo apt install -y --no-install-recommends libjsoncpp-dev
|
||||
sudo apt install -y --no-install-recommends libjsoncpp-dev libtcmalloc-minimal4
|
||||
|
|
|
|||
318
main.cpp
318
main.cpp
|
|
@ -6,18 +6,11 @@
|
|||
#include <map>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#ifdef YA_PIDOR
|
||||
#include <fstream>
|
||||
#else
|
||||
#include <sys/random.h>
|
||||
#endif
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#include <system_error>
|
||||
|
||||
// TODO: RMOVE IT !
|
||||
#include <json/json.h>
|
||||
|
||||
#include "mysignals.h"
|
||||
#include "radosutil.h"
|
||||
|
||||
|
|
@ -25,22 +18,35 @@ using namespace librados;
|
|||
using namespace std;
|
||||
using namespace chrono;
|
||||
|
||||
template <class T> static double dur2sec(const T &dur) {
|
||||
struct bench_settings
|
||||
{
|
||||
string pool;
|
||||
string mode;
|
||||
string specific_bench_item;
|
||||
int threads;
|
||||
int secs;
|
||||
size_t object_size;
|
||||
size_t block_size;
|
||||
};
|
||||
|
||||
template <class T> static double dur2sec(const T &dur)
|
||||
{
|
||||
return duration_cast<duration<double>>(dur).count();
|
||||
}
|
||||
|
||||
template <class T> static double dur2msec(const T &dur) {
|
||||
template <class T> static double dur2msec(const T &dur)
|
||||
{
|
||||
return duration_cast<duration<double, milli>>(dur).count();
|
||||
}
|
||||
|
||||
template <class T> static uint64_t dur2nsec(const T &dur) {
|
||||
template <class T> static uint64_t dur2nsec(const T &dur)
|
||||
{
|
||||
return duration_cast<duration<uint64_t, nano>>(dur).count();
|
||||
}
|
||||
|
||||
template <class T>
|
||||
static void print_breakdown(const vector<T> &summary, size_t thread_count,
|
||||
size_t block_size) {
|
||||
|
||||
static void print_breakdown(const vector<T> &all_ops, size_t thread_count)
|
||||
{
|
||||
T totaltime(0);
|
||||
|
||||
map<size_t, size_t> dur2count;
|
||||
|
|
@ -49,7 +55,8 @@ static void print_breakdown(const vector<T> &summary, size_t thread_count,
|
|||
T mindur(minutes(42));
|
||||
T maxdur(0);
|
||||
size_t maxcount = 0;
|
||||
for (const auto &res : summary) {
|
||||
for (const auto &res : all_ops)
|
||||
{
|
||||
totaltime += res;
|
||||
if (res > maxdur)
|
||||
maxdur = res;
|
||||
|
|
@ -67,186 +74,148 @@ static void print_breakdown(const vector<T> &summary, size_t thread_count,
|
|||
dur2totaltime[range] += res;
|
||||
}
|
||||
|
||||
cout << "min delay " << dur2msec(mindur) << " msec." << endl;
|
||||
cout << "max delay " << dur2msec(maxdur) << " msec." << endl;
|
||||
cout << "min latency " << dur2msec(mindur) << " ms" << endl;
|
||||
cout << "max latency " << dur2msec(maxdur) << " ms" << endl;
|
||||
|
||||
size_t sum = 0;
|
||||
T sumtime(0);
|
||||
const size_t maxbarsize = 30;
|
||||
|
||||
auto x = [block_size](size_t count, T dur) -> void {
|
||||
cout << " cnt=" << count;
|
||||
cout << ", ";
|
||||
cout << (count / dur2sec(dur)) << " IOPS";
|
||||
cout << ", ";
|
||||
cout << (count * block_size / (dur2sec(dur) * 1000000.0)) << " MB/s";
|
||||
cout << ", ";
|
||||
cout << (count * block_size * 8 / (dur2sec(dur) * 1000000.0)) << " Mb/s";
|
||||
};
|
||||
|
||||
for (const auto &p : dur2count) {
|
||||
for (const auto &p : dur2count)
|
||||
{
|
||||
const auto &nsecgrp = p.first;
|
||||
const auto &count = p.second;
|
||||
const auto barsize = count * maxbarsize / maxcount;
|
||||
|
||||
auto bar = string(barsize, '#') + string(maxbarsize - barsize, ' ');
|
||||
cout << ">=" << setw(5) << nsecgrp / 1000000.0;
|
||||
cout << " ms: " << setw(3) << count * 100 / summary.size() << "% " << bar;
|
||||
x(count, dur2totaltime.at(nsecgrp));
|
||||
cout << endl;
|
||||
if (count > maxcount / 100.0) {
|
||||
sum += count;
|
||||
sumtime += dur2totaltime.at(nsecgrp);
|
||||
}
|
||||
cout << " ms: " << setw(3) << count * 100 / all_ops.size() << "% " << bar;
|
||||
cout << " cnt=" << count << endl;
|
||||
}
|
||||
|
||||
cout << "ops: " << (summary.size() * thread_count) / dur2sec(totaltime)
|
||||
<< endl;
|
||||
cout << "Average iops: " << (all_ops.size() / dur2sec(totaltime)) << endl;
|
||||
|
||||
cout << "ops (count > 0.01 of max): ";
|
||||
x(sum * thread_count, sumtime);
|
||||
cout << endl;
|
||||
cout << "Average latency: " << (dur2msec(totaltime) / all_ops.size()) << " ms" << endl;
|
||||
|
||||
cout << "Total writes: " << all_ops.size() << endl;
|
||||
|
||||
if (thread_count > 1)
|
||||
cout << "ops per thread: " << summary.size() / dur2sec(totaltime) << endl;
|
||||
cout << "iops per thread: " << (all_ops.size() / thread_count / dur2sec(totaltime)) << endl;
|
||||
}
|
||||
|
||||
static void fill_urandom(void *buf_, size_t len) {
|
||||
char *buf = static_cast<char *>(buf_);
|
||||
|
||||
#ifdef YA_PIDOR
|
||||
static void fill_urandom(char *buf, size_t len)
|
||||
{
|
||||
ifstream infile;
|
||||
infile.exceptions(ifstream::failbit | ifstream::badbit);
|
||||
infile.open("/dev/urandom", ios::binary | ios::in);
|
||||
infile.read(buf, len);
|
||||
#else
|
||||
while (len) {
|
||||
ssize_t res;
|
||||
if ((res = getrandom(buf, len, 0)) == -1)
|
||||
throw system_error(errno, system_category(),
|
||||
"Failed to get random bytes");
|
||||
buf += res;
|
||||
len -= res;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// May be called in a thread.
|
||||
static void _do_bench(unsigned int secs, const string &obj_name, IoCtx &ioctx,
|
||||
vector<steady_clock::duration> *ops, size_t block_size) {
|
||||
|
||||
static void _do_bench(
|
||||
const unique_ptr<bench_settings> &settings,
|
||||
const vector<string> &obj_names,
|
||||
IoCtx &ioctx,
|
||||
vector<steady_clock::duration> &ops)
|
||||
{
|
||||
// TODO: pass bufferlist as arguments
|
||||
bufferlist bar1;
|
||||
bufferlist bar2;
|
||||
|
||||
bar1.append(ceph::buffer::create(block_size));
|
||||
fill_urandom(bar1.c_str(), block_size);
|
||||
bar1.append(ceph::buffer::create(settings->block_size));
|
||||
fill_urandom(bar1.c_str(), settings->block_size);
|
||||
|
||||
bar2.append(ceph::buffer::create(block_size));
|
||||
fill_urandom(bar2.c_str(), block_size);
|
||||
bar2.append(ceph::buffer::create(settings->block_size));
|
||||
fill_urandom(bar2.c_str(), settings->block_size);
|
||||
|
||||
if (bar1.contents_equal(bar2))
|
||||
throw "You are looser";
|
||||
throw "Your RNG is not random";
|
||||
|
||||
// utime_t end = ceph_clock_now(); ?!
|
||||
auto b = steady_clock::now();
|
||||
const auto stop = b + seconds(secs);
|
||||
try {
|
||||
while (b <= stop) {
|
||||
abort_if_signalled();
|
||||
const auto stop = b + seconds(settings->secs);
|
||||
|
||||
{
|
||||
unique_ptr<AioCompletion, void (*)(AioCompletion *)> compl2(
|
||||
Rados::aio_create_completion(NULL, NULL, NULL),
|
||||
[](AioCompletion *x) { x->release(); });
|
||||
|
||||
if (ioctx.aio_write_full(obj_name, compl2.get(),
|
||||
(ops->size() % 2) ? bar1 : bar2) < 0)
|
||||
throw "Write error";
|
||||
|
||||
if (compl2->wait_for_safe() < 0)
|
||||
throw "Error waiting to be safe";
|
||||
}
|
||||
const auto b2 = steady_clock::now();
|
||||
ops->push_back(b2 - b);
|
||||
b = b2;
|
||||
}
|
||||
} catch (...) {
|
||||
ioctx.remove(obj_name); // ignore errors.
|
||||
throw;
|
||||
for (const auto &obj_name : obj_names)
|
||||
{
|
||||
ioctx.remove(obj_name);
|
||||
}
|
||||
|
||||
while (b <= stop)
|
||||
{
|
||||
abort_if_signalled();
|
||||
if (ioctx.write(
|
||||
obj_names[rand() % 16],
|
||||
(ops.size() % 2) ? bar1 : bar2,
|
||||
settings->block_size,
|
||||
settings->block_size * (rand() % (settings->object_size / settings->block_size))
|
||||
) < 0)
|
||||
{
|
||||
throw "Write error";
|
||||
}
|
||||
const auto b2 = steady_clock::now();
|
||||
ops.push_back(b2 - b);
|
||||
b = b2;
|
||||
}
|
||||
ioctx.remove(obj_name); // ignore errors.
|
||||
}
|
||||
|
||||
static void do_bench(unsigned int secs, const vector<string> &names,
|
||||
IoCtx &ioctx, size_t block_size) {
|
||||
static void do_bench(const unique_ptr<bench_settings> &settings, const vector<string> &names, IoCtx &ioctx)
|
||||
{
|
||||
vector<steady_clock::duration> all_ops;
|
||||
|
||||
vector<steady_clock::duration> summary;
|
||||
|
||||
if (names.size() > 1) {
|
||||
if (settings->threads > 1)
|
||||
{
|
||||
vector<thread> threads;
|
||||
vector<vector<steady_clock::duration> *> listofopts;
|
||||
vector<vector<steady_clock::duration>> listofops;
|
||||
|
||||
for (const auto &name : names) {
|
||||
|
||||
// TODO: memory leak on exception...
|
||||
auto results = new vector<steady_clock::duration>;
|
||||
listofopts.push_back(results);
|
||||
for (int i = 0; i < settings->threads; i++)
|
||||
{
|
||||
listofops.push_back(vector<steady_clock::duration>());
|
||||
|
||||
sigset_t new_set;
|
||||
sigset_t old_set;
|
||||
sigfillset(&new_set);
|
||||
int err;
|
||||
if ((err = pthread_sigmask(SIG_SETMASK, &new_set, &old_set)))
|
||||
throw std::system_error(err, std::system_category(),
|
||||
"Failed to set thread sigmask");
|
||||
{
|
||||
throw std::system_error(err, std::system_category(), "Failed to set thread sigmask");
|
||||
}
|
||||
|
||||
threads.push_back(
|
||||
thread(_do_bench, secs, name, ref(ioctx), results, block_size));
|
||||
threads.push_back(thread(_do_bench, ref(settings), vector<string>(names.begin()+i*16, names.begin()+i*16+16), ref(ioctx), ref(listofops.back())));
|
||||
|
||||
if ((err = pthread_sigmask(SIG_SETMASK, &old_set, NULL)))
|
||||
throw std::system_error(err, std::system_category(),
|
||||
"Failed to restore thread sigmask");
|
||||
{
|
||||
throw std::system_error(err, std::system_category(), "Failed to restore thread sigmask");
|
||||
}
|
||||
}
|
||||
|
||||
for (auto &th : threads)
|
||||
{
|
||||
th.join();
|
||||
|
||||
// just an optimisation :)
|
||||
size_t qwe = 0;
|
||||
for (const auto &res : listofopts)
|
||||
qwe += res->size();
|
||||
summary.reserve(qwe);
|
||||
|
||||
for (const auto &res : listofopts) {
|
||||
summary.insert(summary.end(), res->begin(), res->end());
|
||||
delete res;
|
||||
}
|
||||
} else {
|
||||
_do_bench(secs, names.at(0), ioctx, &summary, block_size);
|
||||
|
||||
for (const auto &res : listofops)
|
||||
{
|
||||
all_ops.insert(all_ops.end(), res.begin(), res.end());
|
||||
}
|
||||
}
|
||||
print_breakdown(summary, names.size(), block_size);
|
||||
else
|
||||
{
|
||||
_do_bench(settings, names, ioctx, all_ops);
|
||||
}
|
||||
print_breakdown(all_ops, settings->threads);
|
||||
}
|
||||
|
||||
static void _main(int argc, const char *argv[]) {
|
||||
struct {
|
||||
string pool;
|
||||
string mode;
|
||||
string specific_bench_item;
|
||||
unsigned int threads;
|
||||
unsigned int secs;
|
||||
size_t block_size;
|
||||
} settings;
|
||||
static void _main(int argc, const char *argv[])
|
||||
{
|
||||
const unique_ptr<bench_settings> settings(new bench_settings);
|
||||
|
||||
switch (argc) {
|
||||
switch (argc)
|
||||
{
|
||||
case 3:
|
||||
settings.pool = argv[1];
|
||||
settings.mode = argv[2];
|
||||
settings->pool = argv[1];
|
||||
settings->mode = argv[2];
|
||||
break;
|
||||
case 4:
|
||||
settings.pool = argv[1];
|
||||
settings.mode = argv[2];
|
||||
settings.specific_bench_item = argv[3];
|
||||
settings->pool = argv[1];
|
||||
settings->mode = argv[2];
|
||||
settings->specific_bench_item = argv[3];
|
||||
break;
|
||||
default:
|
||||
cerr << "Usage: " << argv[0]
|
||||
|
|
@ -254,28 +223,33 @@ static void _main(int argc, const char *argv[]) {
|
|||
throw "Wrong cmdline";
|
||||
}
|
||||
|
||||
settings.secs = 10;
|
||||
settings.threads = 1;
|
||||
settings.block_size = 4096 * 1024;
|
||||
settings->secs = 10;
|
||||
settings->threads = 1;
|
||||
settings->block_size = 4096;
|
||||
settings->object_size = 4096 * 1024;
|
||||
|
||||
Rados rados;
|
||||
int err;
|
||||
if ((err = rados.init("admin")) < 0) {
|
||||
if ((err = rados.init("admin")) < 0)
|
||||
{
|
||||
cerr << "Failed to init: " << strerror(-err) << endl;
|
||||
throw "Failed to init";
|
||||
}
|
||||
|
||||
if ((err = rados.conf_read_file("/etc/ceph/ceph.conf")) < 0) {
|
||||
if ((err = rados.conf_read_file("/etc/ceph/ceph.conf")) < 0)
|
||||
{
|
||||
cerr << "Failed to read conf file: " << strerror(-err) << endl;
|
||||
throw "Failed to read conf file";
|
||||
}
|
||||
|
||||
if ((err = rados.conf_parse_argv(argc, argv)) < 0) {
|
||||
if ((err = rados.conf_parse_argv(argc, argv)) < 0)
|
||||
{
|
||||
cerr << "Failed to parse argv: " << strerror(-err) << endl;
|
||||
throw "Failed to parse argv";
|
||||
}
|
||||
|
||||
if ((err = rados.connect()) < 0) {
|
||||
if ((err = rados.connect()) < 0)
|
||||
{
|
||||
cerr << "Failed to connect: " << strerror(-err) << endl;
|
||||
throw "Failed to connect";
|
||||
}
|
||||
|
|
@ -283,25 +257,28 @@ static void _main(int argc, const char *argv[]) {
|
|||
// https://tracker.ceph.com/issues/24114
|
||||
this_thread::sleep_for(milliseconds(100));
|
||||
|
||||
try {
|
||||
try
|
||||
{
|
||||
auto rados_utils = RadosUtils(&rados);
|
||||
|
||||
if (rados_utils.get_pool_size(settings.pool) != 1)
|
||||
if (rados_utils.get_pool_size(settings->pool) != 1)
|
||||
throw "It's required to have pool size 1";
|
||||
|
||||
map<unsigned int, map<string, string>> osd2location;
|
||||
|
||||
set<string> bench_items; // node1, node2 ||| osd.1, osd.2, osd.3
|
||||
|
||||
for (const auto &osd : rados_utils.get_osds(settings.pool)) {
|
||||
for (const auto &osd : rados_utils.get_osds(settings->pool))
|
||||
{
|
||||
const auto &location = rados_utils.get_osd_location(osd);
|
||||
|
||||
// TODO: do not fill this map if specific_bench_item specified
|
||||
osd2location[osd] = location;
|
||||
|
||||
const auto &qwe = location.at(settings.mode);
|
||||
if (settings.specific_bench_item.empty() ||
|
||||
qwe == settings.specific_bench_item) {
|
||||
const auto &qwe = location.at(settings->mode);
|
||||
if (settings->specific_bench_item.empty() ||
|
||||
qwe == settings->specific_bench_item)
|
||||
{
|
||||
bench_items.insert(qwe);
|
||||
}
|
||||
}
|
||||
|
|
@ -310,60 +287,69 @@ static void _main(int argc, const char *argv[]) {
|
|||
map<string, vector<string>> name2location;
|
||||
unsigned int cnt = 0;
|
||||
|
||||
// for each bench_item find thread_count names.
|
||||
// for each bench_item find thread_count*16 names
|
||||
// store every name in name2location = [bench_item, names, description]
|
||||
cout << "Finding object names" << endl;
|
||||
const string prefix = "bench_";
|
||||
while (bench_items.size()) {
|
||||
while (bench_items.size())
|
||||
{
|
||||
string name = prefix + to_string(++cnt);
|
||||
|
||||
unsigned int osd =
|
||||
rados_utils.get_obj_acting_primary(name, settings.pool);
|
||||
unsigned int osd = rados_utils.get_obj_acting_primary(name, settings->pool);
|
||||
|
||||
const auto &location = osd2location.at(osd);
|
||||
const auto &bench_item = location.at(settings.mode);
|
||||
const auto &bench_item = location.at(settings->mode);
|
||||
if (!bench_items.count(bench_item))
|
||||
continue;
|
||||
|
||||
auto &names = name2location[bench_item];
|
||||
if (names.size() == settings.threads) {
|
||||
if (names.size() >= (unsigned)settings->threads*16)
|
||||
{
|
||||
bench_items.erase(bench_item);
|
||||
continue;
|
||||
}
|
||||
|
||||
names.push_back(name);
|
||||
|
||||
cout << name << " - " << bench_item << endl;
|
||||
}
|
||||
|
||||
IoCtx ioctx;
|
||||
|
||||
if (rados.ioctx_create(settings.pool.c_str(), ioctx) < 0)
|
||||
if (rados.ioctx_create(settings->pool.c_str(), ioctx) < 0)
|
||||
throw "Failed to create ioctx";
|
||||
|
||||
for (const auto &p : name2location) {
|
||||
for (const auto &p : name2location)
|
||||
{
|
||||
const auto &bench_item = p.first;
|
||||
const auto &obj_names = p.second;
|
||||
cout << "Benching " << settings.mode << " " << bench_item << endl;
|
||||
do_bench(settings.secs, obj_names, ioctx, settings.block_size);
|
||||
cout << "Benching " << settings->mode << " " << bench_item << endl;
|
||||
do_bench(settings, obj_names, ioctx);
|
||||
}
|
||||
} catch (...) {
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
rados.watch_flush();
|
||||
throw;
|
||||
}
|
||||
rados.watch_flush();
|
||||
|
||||
// rados_ioctx_destroy(io);
|
||||
// rados_shutdown(cluster);
|
||||
//rados_ioctx_destroy(io);
|
||||
//rados_shutdown(cluster);
|
||||
}
|
||||
|
||||
int main(int argc, const char *argv[]) {
|
||||
try {
|
||||
int main(int argc, const char *argv[])
|
||||
{
|
||||
try
|
||||
{
|
||||
setup_signal_handlers();
|
||||
_main(argc, argv);
|
||||
} catch (const AbortException &msg) {
|
||||
}
|
||||
catch (const AbortException &msg)
|
||||
{
|
||||
cerr << "Test aborted" << endl;
|
||||
return 1;
|
||||
} catch (const char *msg) {
|
||||
}
|
||||
catch (const char *msg)
|
||||
{
|
||||
cerr << "Unhandled exception: " << msg << endl;
|
||||
return 2;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
#include <memory>
|
||||
#include <set>
|
||||
#include <string>
|
||||
#include <json/json.h>
|
||||
|
||||
namespace Json {
|
||||
class Reader;
|
||||
|
|
|
|||
Loading…
Reference in New Issue