From 1a14ff58c9c40b13df8cda431fd2b35ca079b5d4 Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Mon, 12 Dec 2011 13:46:12 -0800 Subject: [PATCH 01/91] Initial checkin of C++ btree code, has some useless google3-specific bits to be removed and needs a Makefile. --- Makefile | 75 ++ benchmarks.awk | 27 + btree.h | 2420 +++++++++++++++++++++++++++++++++++++++++ btree_bench.cc | 483 ++++++++ btree_container.h | 325 ++++++ btree_map.h | 122 +++ btree_nc.cc | 50 + btree_nc_test.py | 83 ++ btree_printer.py | 125 +++ btree_printer_test.py | 92 ++ btree_set.h | 113 ++ btree_test.cc | 167 +++ btree_test.h | 930 ++++++++++++++++ btree_test_flags.cc | 9 + btree_test_program.cc | 64 ++ safe_btree.h | 379 +++++++ safe_btree_map.h | 70 ++ safe_btree_set.h | 68 ++ safe_btree_test.cc | 67 ++ 19 files changed, 5669 insertions(+) create mode 100644 Makefile create mode 100755 benchmarks.awk create mode 100644 btree.h create mode 100644 btree_bench.cc create mode 100644 btree_container.h create mode 100644 btree_map.h create mode 100644 btree_nc.cc create mode 100755 btree_nc_test.py create mode 100755 btree_printer.py create mode 100755 btree_printer_test.py create mode 100644 btree_set.h create mode 100644 btree_test.cc create mode 100644 btree_test.h create mode 100644 btree_test_flags.cc create mode 100644 btree_test_program.cc create mode 100644 safe_btree.h create mode 100644 safe_btree_map.h create mode 100644 safe_btree_set.h create mode 100644 safe_btree_test.cc diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..c58946f9 --- /dev/null +++ b/Makefile @@ -0,0 +1,75 @@ +# -*- mode: python; -*- + +# This should be a Makefile, but it's not. + +# cc_library(name = "btree", +# srcs = [ "btree.h", +# "btree_container.h", +# "btree_map.h", +# "btree_set.h", +# "safe_btree.h", +# "safe_btree_map.h", +# "safe_btree_set.h" ], +# deps = [ "//strings", +# "//strings:cord" ]) + +# cc_library(name = "btree_test_flags", +# srcs = [ "btree_test_flags.cc" ], +# deps = [ "//base" ]) + +# cc_binary(name = "btree_bench", +# srcs = [ "btree_bench.cc" ], +# deps = [ ":btree", +# ":btree_test_flags", +# "//testing/base", +# "//util/random" ]) + +# cc_test(name = "btree_test", +# srcs = [ "btree_test.cc", ], +# deps = [ ":btree", +# ":btree_test_flags", +# "//base:heapcheck", +# "//testing/base", +# "//util/random", +# ], +# size = "large") + +# cc_test(name = "safe_btree_test", +# srcs = [ "safe_btree_test.cc", ], +# deps = [ ":btree", +# ":btree_test_flags", +# "//base:heapcheck", +# "//testing/base", +# "//util/random", +# ], +# size = "large") + +# cc_fake_binary(name = "btree_nc", +# srcs = [ "btree_nc.cc" ], +# deps = [ ":btree" ], +# legacy = 0) + +# py_test(name = "btree_nc_test", +# srcs = [ "btree_nc_test.py" ], +# deps = [ "//pyglib", +# "//testing/pybase" ], +# data = [ "btree_nc" ], +# size = "large") + +# cc_binary(name = "btree_test_program", +# srcs = [ "btree_test_program.cc" ], +# deps = [ ":btree", +# "//devtools/gdb/component:gdb_test_utils" ], +# testonly = 1) + +# # This test will only actually test the pretty-printing code if it's +# # compiled with debug information (blaze build -c dbg). The default +# # mode, fastbuild, will pass but will not catch any regressions! +# py_test(name = "btree_printer_test", +# size = "large", +# srcs = [ "btree_printer_test.py", +# "btree_printer.py" ], +# deps = [ "//devtools/gdb/component:gdbpy", +# "//testing/pybase", +# "//testing/gdb:gdb_script_test_util", +# ":btree_test_program" ]) diff --git a/benchmarks.awk b/benchmarks.awk new file mode 100755 index 00000000..d5d4aeac --- /dev/null +++ b/benchmarks.awk @@ -0,0 +1,27 @@ +#!/usr/bin/gawk -f + +/^Run on/ { + print $0; + printf "%-25s %5s %-20s\n", + "Benchmark", "STL(ns)", "B-Tree(ns) @ " + printf "--------------------------------------------------------\n"; +} + +/^BM_/ { + split($1, name, "_"); + if (name[2] == "stl") { + stl = $3; + stl_bytes = $5 + printf "%-25s %5d ", name[1] "_" name[3] "_" name[4] "_" name[5], stl; + fflush(); + } else if (name[2] == "btree") { + btree = $3 + btree_size = name[3] + btree_bytes = $5 + printf "%5d %+7.2f%% <%3d>", btree, 100.0 * (stl - btree) / stl, btree_size; + printf " [%4.1f, %4.1f]\n", stl_bytes, btree_bytes; + fflush(); + } else { + printf "ERROR: %s unrecognized\n", $1 + } +} diff --git a/btree.h b/btree.h new file mode 100644 index 00000000..b9fd04e2 --- /dev/null +++ b/btree.h @@ -0,0 +1,2420 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) +// +// A btree implementation of the STL set and map interfaces. A btree is both +// smaller and faster than STL set/map. The red-black tree implementation of +// STL set/map has an overhead of 3 pointers (left, right and parent) plus the +// node color information for each stored value. So a set consumes 20 +// bytes for each value stored. This btree implementation stores multiple +// values on fixed size nodes (usually 256 bytes) and doesn't store child +// pointers for leaf nodes. The result is that a btree_set may use much +// less memory per stored value. For the random insertion benchmark in +// btree_test.cc, a btree_set with node-size of 256 uses 4.9 bytes per +// stored value. +// +// The packing of multiple values on to each node of a btree has another effect +// besides better space utilization: better cache locality due to fewer cache +// lines being accessed. Better cache locality translates into faster +// operations. +// +// CAVEATS +// +// Insertions and deletions on a btree can cause splitting, merging or +// rebalancing of btree nodes. And even without these operations, insertions +// and deletions on a btree will move values around within a node. In both +// cases, the result is that insertions and deletions can invalidate iterators +// pointing to values other than the one being inserted/deleted. This is +// notably different from STL set/map which takes care to not invalidate +// iterators on insert/erase except, of course, for iterators pointing to the +// value being erased. A partial workaround when erasing is available: +// erase() returns an iterator pointing to the item just after the one that was +// erased (or end() if none exists). See also safe_btree. + +// PERFORMANCE +// +// btree_bench --benchmarks=. 2>&1 | ./benchmarks.awk +// +// NOTE(pmattis): My warpstation (pmattis-warp.nyc) often produces slower +// results when running the benchmarks on CPUs 0 and 1 vs CPUs 2 and 3. To get +// consistent benchmark results, I run "taskset 0xc " to run the +// benchmark on CPUs 2 and 3. +// +// Run on pmattis-warp.nyc (4 X 2200 MHz CPUs); 2010/03/04-15:23:06 +// Benchmark STL(ns) B-Tree(ns) @ +// -------------------------------------------------------- +// BM_set_int32_insert 1516 608 +59.89% <256> [40.0, 5.2] +// BM_set_int32_lookup 1160 414 +64.31% <256> [40.0, 5.2] +// BM_set_int32_fulllookup 960 410 +57.29% <256> [40.0, 4.4] +// BM_set_int32_delete 1741 528 +69.67% <256> [40.0, 5.2] +// BM_set_int32_queueaddrem 3078 1046 +66.02% <256> [40.0, 5.5] +// BM_set_int32_mixedaddrem 3600 1384 +61.56% <256> [40.0, 5.3] +// BM_set_int32_fifo 227 113 +50.22% <256> [40.0, 4.4] +// BM_set_int32_fwditer 158 26 +83.54% <256> [40.0, 5.2] +// BM_map_int32_insert 1551 636 +58.99% <256> [48.0, 10.5] +// BM_map_int32_lookup 1200 508 +57.67% <256> [48.0, 10.5] +// BM_map_int32_fulllookup 989 487 +50.76% <256> [48.0, 8.8] +// BM_map_int32_delete 1794 628 +64.99% <256> [48.0, 10.5] +// BM_map_int32_queueaddrem 3189 1266 +60.30% <256> [48.0, 11.6] +// BM_map_int32_mixedaddrem 3822 1623 +57.54% <256> [48.0, 10.9] +// BM_map_int32_fifo 151 134 +11.26% <256> [48.0, 8.8] +// BM_map_int32_fwditer 161 32 +80.12% <256> [48.0, 10.5] +// BM_set_int64_insert 1546 636 +58.86% <256> [40.0, 10.5] +// BM_set_int64_lookup 1200 512 +57.33% <256> [40.0, 10.5] +// BM_set_int64_fulllookup 971 487 +49.85% <256> [40.0, 8.8] +// BM_set_int64_delete 1745 616 +64.70% <256> [40.0, 10.5] +// BM_set_int64_queueaddrem 3163 1195 +62.22% <256> [40.0, 11.6] +// BM_set_int64_mixedaddrem 3760 1564 +58.40% <256> [40.0, 10.9] +// BM_set_int64_fifo 146 103 +29.45% <256> [40.0, 8.8] +// BM_set_int64_fwditer 162 31 +80.86% <256> [40.0, 10.5] +// BM_map_int64_insert 1551 720 +53.58% <256> [48.0, 20.7] +// BM_map_int64_lookup 1214 612 +49.59% <256> [48.0, 20.7] +// BM_map_int64_fulllookup 994 592 +40.44% <256> [48.0, 17.2] +// BM_map_int64_delete 1778 764 +57.03% <256> [48.0, 20.7] +// BM_map_int64_queueaddrem 3189 1547 +51.49% <256> [48.0, 20.9] +// BM_map_int64_mixedaddrem 3779 1887 +50.07% <256> [48.0, 21.6] +// BM_map_int64_fifo 147 145 +1.36% <256> [48.0, 17.2] +// BM_map_int64_fwditer 162 41 +74.69% <256> [48.0, 20.7] +// BM_set_string_insert 1989 1966 +1.16% <256> [64.0, 44.5] +// BM_set_string_lookup 1709 1600 +6.38% <256> [64.0, 44.5] +// BM_set_string_fulllookup 1573 1529 +2.80% <256> [64.0, 35.4] +// BM_set_string_delete 2520 1920 +23.81% <256> [64.0, 44.5] +// BM_set_string_queueaddrem 4706 4309 +8.44% <256> [64.0, 48.3] +// BM_set_string_mixedaddrem 5080 4654 +8.39% <256> [64.0, 46.7] +// BM_set_string_fifo 318 512 -61.01% <256> [64.0, 35.4] +// BM_set_string_fwditer 182 93 +48.90% <256> [64.0, 44.5] +// BM_map_string_insert 2600 2227 +14.35% <256> [72.0, 55.8] +// BM_map_string_lookup 2068 1730 +16.34% <256> [72.0, 55.8] +// BM_map_string_fulllookup 1859 1618 +12.96% <256> [72.0, 44.0] +// BM_map_string_delete 3168 2080 +34.34% <256> [72.0, 55.8] +// BM_map_string_queueaddrem 5840 4701 +19.50% <256> [72.0, 59.4] +// BM_map_string_mixedaddrem 6400 5200 +18.75% <256> [72.0, 57.8] +// BM_map_string_fifo 398 596 -49.75% <256> [72.0, 44.0] +// BM_map_string_fwditer 243 113 +53.50% <256> [72.0, 55.8] +// BM_set_cord_insert 3661 2680 +26.80% <256> [40.0, 10.5] +// BM_set_cord_lookup 2920 2293 +21.47% <256> [40.0, 10.5] +// BM_set_cord_fulllookup 2960 2267 +23.41% <256> [40.0, 8.8] +// BM_set_cord_delete 4679 2535 +45.82% <256> [40.0, 10.5] +// BM_set_cord_queueaddrem 8230 5600 +31.96% <256> [40.0, 11.3] +// BM_set_cord_mixedaddrem 8497 6080 +28.45% <256> [40.0, 10.7] +// BM_set_cord_fifo 358 370 -3.35% <256> [40.0, 8.8] +// BM_set_cord_fwditer 352 193 +45.17% <256> [40.0, 10.5] +// BM_map_cord_insert 3680 2927 +20.46% <256> [48.0, 20.7] +// BM_map_cord_lookup 3018 2466 +18.29% <256> [48.0, 20.7] +// BM_map_cord_fulllookup 2943 2466 +16.21% <256> [48.0, 17.2] +// BM_map_cord_delete 4675 2775 +40.64% <256> [48.0, 20.7] +// BM_map_cord_queueaddrem 8383 6360 +24.13% <256> [48.0, 22.2] +// BM_map_cord_mixedaddrem 8952 6760 +24.49% <256> [48.0, 21.2] +// BM_map_cord_fifo 444 463 -4.28% <256> [48.0, 17.2] +// BM_map_cord_fwditer 391 225 +42.46% <256> [48.0, 20.7] + +#ifndef UTIL_BTREE_BTREE_H__ +#define UTIL_BTREE_BTREE_H__ + +#include +#include +#include +#include +#include +#include +#include // IWYU pragma: export // Clients can rely on this. +#include +#include +#include // IWYU pragma: export // Clients can rely on this. +#include +#include + +#include "base/gdb-scripting.h" +#include "base/integral_types.h" +#include "base/logging.h" +#include "base/macros.h" +#include "base/template_util.h" +#include "base/type_traits.h" +#include "strings/cord.h" +#include "strings/stringpiece.h" + +// iwyu.py found std::swap in , but we include . +// IWYU pragma: no_include +// +// iwyu.py doesn't think we need , so wants . +// IWYU pragma: no_include +// +// iwyu.py sure wants to bring in a lot of private STL headers. +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include + +namespace util { +namespace btree { + +// Inside a btree method, if we just call swap(), it will choose the +// btree::swap method, which we don't want. And we can't say ::swap because +// then MSVC won't pickup any std::swap() implementations. We can't just use +// std::swap() directly because then we don't get the specialization for string +// and Cord (which are not defined in the std namespace!). So the solution is +// to have a special swap helper function whose name doesn't collide with other +// swap functions defined by the btree classes. +template +inline void btree_swap_helper(T &a, T &b) { + using std::swap; + swap(a, b); +} + +// A helper type used to indicate that a key-compare-to functor has been +// provided. A user can specify a key-compare-to functor by doing: +// +// struct MyStringComparer +// : public util::btree::btree_key_compare_to_tag { +// int operator()(const string &a, const string &b) const { +// return a.compare(b); +// } +// }; +// +// Note that the return type is an int and not a bool. There is a +// COMPILE_ASSERT which enforces this return type. +struct btree_key_compare_to_tag { +}; + +// A helper class that indicates if the Compare parameter is derived from +// btree_key_compare_to_tag. +template +struct btree_is_key_compare_to + : public base::is_convertible { +}; + +// A helper class to convert a boolean comparison into a three-way "compare-to" +// comparison that returns a negative value to indicate less-than, zero to +// indicate equality and a positive value to indicate greater-than. This helper +// class is specialized for less, greater, less, +// greater, less and greater. The +// btree_key_compare_to_adapter class is provided so that btree users +// automatically get the more efficient compare-to code when using common +// google string types with common comparison functors. +template +struct btree_key_compare_to_adapter : Compare { + btree_key_compare_to_adapter() { } + btree_key_compare_to_adapter(const Compare &c) : Compare(c) { } + btree_key_compare_to_adapter(const btree_key_compare_to_adapter &c) + : Compare(c) { + } +}; + +template <> +struct btree_key_compare_to_adapter > + : public btree_key_compare_to_tag { + btree_key_compare_to_adapter() {} + btree_key_compare_to_adapter(const less&) {} + btree_key_compare_to_adapter( + const btree_key_compare_to_adapter >&) {} + int operator()(const string &a, const string &b) const { + return a.compare(b); + } +}; + +template <> +struct btree_key_compare_to_adapter > + : public btree_key_compare_to_tag { + btree_key_compare_to_adapter() {} + btree_key_compare_to_adapter(const greater&) {} + btree_key_compare_to_adapter( + const btree_key_compare_to_adapter >&) {} + int operator()(const string &a, const string &b) const { + return b.compare(a); + } +}; + +template <> +struct btree_key_compare_to_adapter > + : public btree_key_compare_to_tag { + btree_key_compare_to_adapter() {} + btree_key_compare_to_adapter(const less&) {} + btree_key_compare_to_adapter( + const btree_key_compare_to_adapter >&) {} + int operator()(const StringPiece &a, const StringPiece &b) const { + return a.compare(b); + } +}; + +template <> +struct btree_key_compare_to_adapter > + : public btree_key_compare_to_tag { + btree_key_compare_to_adapter() {} + btree_key_compare_to_adapter(const greater&) {} + btree_key_compare_to_adapter( + const btree_key_compare_to_adapter >&) {} + int operator()(const StringPiece &a, const StringPiece &b) const { + return b.compare(a); + } +}; + +template <> +struct btree_key_compare_to_adapter > + : public btree_key_compare_to_tag { + btree_key_compare_to_adapter() {} + btree_key_compare_to_adapter(const less&) {} + btree_key_compare_to_adapter( + const btree_key_compare_to_adapter >&) {} + int operator()(const Cord &a, const Cord &b) const { + return a.CompareTo(b); + } +}; + +template <> +struct btree_key_compare_to_adapter > + : public btree_key_compare_to_tag { + btree_key_compare_to_adapter() {} + btree_key_compare_to_adapter(const greater&) {} + btree_key_compare_to_adapter( + const btree_key_compare_to_adapter >&) {} + int operator()(const Cord &a, const Cord &b) const { + return b.CompareTo(a); + } +}; + +// A helper class that allows a compare-to functor to behave like a plain +// compare functor. This specialization is used when we do not have a +// compare-to functor. +template +struct btree_key_comparer { + btree_key_comparer() {} + btree_key_comparer(Compare c) : comp(c) {} + static bool bool_compare(const Compare &comp, const Key &x, const Key &y) { + return comp(x, y); + } + bool operator()(const Key &x, const Key &y) const { + return bool_compare(comp, x, y); + } + Compare comp; +}; + +// A specialization of btree_key_comparer when a compare-to functor is +// present. We need a plain (boolean) comparison in some parts of the btree +// code, such as insert-with-hint. +template +struct btree_key_comparer { + btree_key_comparer() {} + btree_key_comparer(Compare c) : comp(c) {} + static bool bool_compare(const Compare &comp, const Key &x, const Key &y) { + return comp(x, y) < 0; + } + bool operator()(const Key &x, const Key &y) const { + return bool_compare(comp, x, y); + } + Compare comp; +}; + +// A helper function to compare to keys using the specified compare +// functor. This dispatches to the appropriate btree_key_comparer comparison, +// depending on whether we have a compare-to functor or not (which depends on +// whether Compare is derived from btree_key_compare_to_tag). +template +static bool btree_compare_keys( + const Compare &comp, const Key &x, const Key &y) { + typedef btree_key_comparer::value> key_comparer; + return key_comparer::bool_compare(comp, x, y); +} + +template +struct btree_common_params { + // If Compare is derived from btree_key_compare_to_tag then use it as the + // key_compare type. Otherwise, use btree_key_compare_to_adapter<> which will + // fall-back to Compare if we don't have an appropriate specialization. + typedef typename base::if_< + btree_is_key_compare_to::value, + Compare, btree_key_compare_to_adapter >::type key_compare; + // A type which indicates if we have a key-compare-to functor or a plain old + // key-compare functor. + typedef btree_is_key_compare_to is_key_compare_to; + + typedef Alloc allocator_type; + typedef Key key_type; + typedef ssize_t size_type; + typedef ptrdiff_t difference_type; + + enum { + kTargetNodeSize = TargetNodeSize, + }; +}; + +// A parameters structure for holding the type parameters for a btree_map. +template +struct btree_map_params + : public btree_common_params { + typedef Data data_type; + typedef Data mapped_type; + typedef pair value_type; + typedef pair mutable_value_type; + typedef value_type* pointer; + typedef const value_type* const_pointer; + typedef value_type& reference; + typedef const value_type& const_reference; + + enum { + kValueSize = sizeof(Key) + sizeof(data_type), + }; + + static const Key& key(const value_type &x) { return x.first; } + static const Key& key(const mutable_value_type &x) { return x.first; } + static void swap(mutable_value_type *a, mutable_value_type *b) { + btree_swap_helper(a->first, b->first); + btree_swap_helper(a->second, b->second); + } +}; + +// A parameters structure for holding the type parameters for a btree_set. +template +struct btree_set_params + : public btree_common_params { + typedef base::false_type data_type; + typedef base::false_type mapped_type; + typedef Key value_type; + typedef value_type mutable_value_type; + typedef value_type* pointer; + typedef const value_type* const_pointer; + typedef value_type& reference; + typedef const value_type& const_reference; + + enum { + kValueSize = sizeof(Key), + }; + + static const Key& key(const value_type &x) { return x; } + static void swap(mutable_value_type *a, mutable_value_type *b) { + btree_swap_helper(*a, *b); + } +}; + +// An adapter class that converts a lower-bound compare into an upper-bound +// compare. +template +struct btree_upper_bound_adapter : public Compare { + btree_upper_bound_adapter(Compare c) : Compare(c) {} + bool operator()(const Key &a, const Key &b) const { + return !static_cast(*this)(b, a); + } +}; + +template +struct btree_upper_bound_compare_to_adapter : public CompareTo { + btree_upper_bound_compare_to_adapter(CompareTo c) : CompareTo(c) {} + int operator()(const Key &a, const Key &b) const { + return static_cast(*this)(b, a); + } +}; + +// Dispatch helper class for using linear search with plain compare. +template +struct btree_linear_search_plain_compare { + static int lower_bound(const K &k, const N &n, Compare comp) { + return n.linear_search_plain_compare(k, 0, n.count(), comp); + } + static int upper_bound(const K &k, const N &n, Compare comp) { + typedef btree_upper_bound_adapter upper_compare; + return n.linear_search_plain_compare(k, 0, n.count(), upper_compare(comp)); + } +}; + +// Dispatch helper class for using linear search with compare-to +template +struct btree_linear_search_compare_to { + static int lower_bound(const K &k, const N &n, CompareTo comp) { + return n.linear_search_compare_to(k, 0, n.count(), comp); + } + static int upper_bound(const K &k, const N &n, CompareTo comp) { + typedef btree_upper_bound_adapter > upper_compare; + return n.linear_search_plain_compare(k, 0, n.count(), upper_compare(comp)); + } +}; + +// Dispatch helper class for using binary search with plain compare. +template +struct btree_binary_search_plain_compare { + static int lower_bound(const K &k, const N &n, Compare comp) { + return n.binary_search_plain_compare(k, 0, n.count(), comp); + } + static int upper_bound(const K &k, const N &n, Compare comp) { + typedef btree_upper_bound_adapter upper_compare; + return n.binary_search_plain_compare(k, 0, n.count(), upper_compare(comp)); + } +}; + +// Dispatch helper class for using binary search with compare-to. +template +struct btree_binary_search_compare_to { + static int lower_bound(const K &k, const N &n, CompareTo comp) { + return n.binary_search_compare_to(k, 0, n.count(), CompareTo()); + } + static int upper_bound(const K &k, const N &n, CompareTo comp) { + typedef btree_upper_bound_adapter > upper_compare; + return n.linear_search_plain_compare(k, 0, n.count(), upper_compare(comp)); + } +}; + +// A node in the btree holding. The same node type is used for both internal +// and leaf nodes in the btree, though the nodes are allocated in such a way +// that the children array is only valid in internal nodes. +template +class btree_node { + public: + typedef Params params_type; + typedef btree_node self_type; + typedef typename Params::key_type key_type; + typedef typename Params::data_type data_type; + typedef typename Params::value_type value_type; + typedef typename Params::mutable_value_type mutable_value_type; + typedef typename Params::pointer pointer; + typedef typename Params::const_pointer const_pointer; + typedef typename Params::reference reference; + typedef typename Params::const_reference const_reference; + typedef typename Params::key_compare key_compare; + typedef typename Params::size_type size_type; + typedef typename Params::difference_type difference_type; + // Typedefs for the various types of node searches. + typedef btree_linear_search_plain_compare< + key_type, self_type, key_compare> linear_search_plain_compare_type; + typedef btree_linear_search_compare_to< + key_type, self_type, key_compare> linear_search_compare_to_type; + typedef btree_binary_search_plain_compare< + key_type, self_type, key_compare> binary_search_plain_compare_type; + typedef btree_binary_search_compare_to< + key_type, self_type, key_compare> binary_search_compare_to_type; + // If we have a valid key-compare-to type, use linear_search_compare_to, + // otherwise use linear_search_plain_compare. + typedef typename base::if_< + Params::is_key_compare_to::value, + linear_search_compare_to_type, + linear_search_plain_compare_type>::type linear_search_type; + // If we have a valid key-compare-to type, use binary_search_compare_to, + // otherwise use binary_search_plain_compare. + typedef typename base::if_< + Params::is_key_compare_to::value, + binary_search_compare_to_type, + binary_search_plain_compare_type>::type binary_search_type; + // If the key is an integral or floating point type, use linear search which + // is faster than binary search for such types. Might be wise to also + // configure linear search based on node-size. + typedef typename base::if_< + base::is_integral::value || + base::is_floating_point::value, + linear_search_type, binary_search_type>::type search_type; + + struct base_fields { + // A boolean indicating whether the node is a leaf or not. + uint8 leaf; + // The position of the node in the node's parent. + uint8 position; + // The maximum number of values the node can hold. + uint8 max_count; + // The count of the number of values in the node. + uint8 count; + // A pointer to the node's parent. + btree_node *parent; + }; + + enum { + kValueSize = params_type::kValueSize, + kTargetNodeSize = params_type::kTargetNodeSize, + + // Compute how many values we can fit onto a leaf node. + kNodeTargetValues = (kTargetNodeSize - sizeof(base_fields)) / kValueSize, + // We need a minimum of 3 values per internal node in order to perform + // splitting (1 value for the two nodes involved in the split and 1 value + // propagated to the parent as the delimiter for the split). + kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, + + kExactMatch = 1 << 30, + kMatchMask = kExactMatch - 1, + }; + + struct leaf_fields : public base_fields { + // The array of values. Only the first count of these values have been + // constructed and are valid. + mutable_value_type values[kNodeValues]; + }; + + struct internal_fields : public leaf_fields { + // The array of child pointers. The keys in children_[i] are all less than + // key(i). The keys in children_[i + 1] are all greater than key(i). There + // are always count + 1 children. + btree_node *children[kNodeValues + 1]; + }; + + struct root_fields : public internal_fields { + btree_node *rightmost; + size_type size; + }; + + public: + // Getter/setter for whether this is a leaf node or not. This value doesn't + // change after the node is created. + bool leaf() const { return fields_.leaf; } + + // Getter for the position of this node in its parent. + int position() const { return fields_.position; } + void set_position(int v) { fields_.position = v; } + + // Getter/setter for the number of values stored in this node. + int count() const { return fields_.count; } + void set_count(int v) { fields_.count = v; } + int max_count() const { return fields_.max_count; } + + // Getter for the parent of this node. + btree_node* parent() const { return fields_.parent; } + // Getter for whether the node is the root of the tree. The parent of the + // root of the tree is the leftmost node in the tree which is guaranteed to + // be a leaf. + bool is_root() const { return parent()->leaf(); } + void make_root() { + DCHECK(parent()->is_root()); + fields_.parent = fields_.parent->parent(); + } + + // Getter for the rightmost root node field. Only valid on the root node. + btree_node* rightmost() const { return fields_.rightmost; } + btree_node** mutable_rightmost() { return &fields_.rightmost; } + + // Getter for the size root node field. Only valid on the root node. + size_type size() const { return fields_.size; } + size_type* mutable_size() { return &fields_.size; } + + // Getters for the key/value at position i in the node. + const key_type& key(int i) const { + return params_type::key(fields_.values[i]); + } + reference value(int i) { + return reinterpret_cast(fields_.values[i]); + } + const_reference value(int i) const { + return reinterpret_cast(fields_.values[i]); + } + mutable_value_type* mutable_value(int i) { + return &fields_.values[i]; + } + + // Swap value i in this node with value j in node x. + void value_swap(int i, btree_node *x, int j) { + params_type::swap(mutable_value(i), x->mutable_value(j)); + } + + // Getters/setter for the child at position i in the node. + btree_node* child(int i) const { return fields_.children[i]; } + btree_node** mutable_child(int i) { return &fields_.children[i]; } + void set_child(int i, btree_node *c) { + *mutable_child(i) = c; + c->fields_.parent = this; + c->fields_.position = i; + } + + // Returns the position of the first value whose key is not less than k. + template + int lower_bound(const key_type &k, const Compare &comp) const { + return search_type::lower_bound(k, *this, comp); + } + // Returns the position of the first value whose key is greater than k. + template + int upper_bound(const key_type &k, const Compare &comp) const { + return search_type::upper_bound(k, *this, comp); + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using plain compare. + template + int linear_search_plain_compare( + const key_type &k, int s, int e, const Compare &comp) const { + while (s < e) { + if (!btree_compare_keys(comp, key(s), k)) { + break; + } + ++s; + } + return s; + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using compare-to. + template + int linear_search_compare_to( + const key_type &k, int s, int e, const Compare &comp) const { + while (s < e) { + int c = comp(key(s), k); + if (c == 0) { + return s | kExactMatch; + } else if (c > 0) { + break; + } + ++s; + } + return s; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using plain compare. + template + int binary_search_plain_compare( + const key_type &k, int s, int e, const Compare &comp) const { + while (s != e) { + int mid = (s + e) / 2; + if (btree_compare_keys(comp, key(mid), k)) { + s = mid + 1; + } else { + e = mid; + } + } + return s; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using compare-to. + template + int binary_search_compare_to( + const key_type &k, int s, int e, const CompareTo &comp) const { + while (s != e) { + int mid = (s + e) / 2; + int c = comp(key(mid), k); + if (c < 0) { + s = mid + 1; + } else if (c > 0) { + e = mid; + } else { + // Need to return the first value whose key is not less than k, which + // requires continuing the binary search. Note that we are guaranteed + // that the result is an exact match because if "key(mid-1) < k" the + // call to binary_search_compare_to() will return "mid". + s = binary_search_compare_to(k, s, mid, comp); + return s | kExactMatch; + } + } + return s; + } + + // Inserts the value x at position i, shifting all existing values and + // children at positions >= i to the right by 1. + void insert_value(int i, const value_type &x); + + // Removes the value at position i, shifting all existing values and children + // at positions > i to the left by 1. + void remove_value(int i); + + // Rebalances a node with its right sibling. + void rebalance_right_to_left(btree_node *sibling, int to_move); + void rebalance_left_to_right(btree_node *sibling, int to_move); + + // Splits a node, moving a portion of the node's values to its right sibling. + void split(btree_node *sibling, int insert_position); + + // Merges a node with its right sibling, moving all of the values and the + // delimiting key in the parent node onto itself. + void merge(btree_node *sibling); + + // Swap the contents of "this" and "src". + void swap(btree_node *src); + + // Node allocation/deletion routines. + static btree_node* init_leaf( + leaf_fields *f, btree_node *parent, int max_count) { + btree_node *n = reinterpret_cast(f); + f->leaf = 1; + f->position = 0; + f->max_count = max_count; + f->count = 0; + f->parent = parent; + if (DEBUG_MODE) { + memset(&f->values, 0, max_count * sizeof(value_type)); + } + return n; + } + static btree_node* init_internal(internal_fields *f, btree_node *parent) { + btree_node *n = init_leaf(f, parent, kNodeValues); + f->leaf = 0; + if (DEBUG_MODE) { + memset(f->children, 0, sizeof(f->children)); + } + return n; + } + static btree_node* init_root(root_fields *f, btree_node *parent) { + btree_node *n = init_internal(f, parent); + f->rightmost = parent; + f->size = parent->count(); + return n; + } + void destroy() { + for (int i = 0; i < count(); ++i) { + value_destroy(i); + } + } + + private: + void value_init(int i) { + new (&fields_.values[i]) mutable_value_type; + } + void value_init(int i, const value_type &x) { + new (&fields_.values[i]) mutable_value_type(x); + } + void value_destroy(int i) { + fields_.values[i].~mutable_value_type(); + } + + private: + root_fields fields_; + + private: + DISALLOW_EVIL_CONSTRUCTORS(btree_node); +}; + +template +struct btree_iterator { + typedef typename Node::key_type key_type; + typedef typename Node::size_type size_type; + typedef typename Node::difference_type difference_type; + typedef typename Node::params_type params_type; + + typedef Node node_type; + typedef typename base::remove_const::type normal_node; + typedef const Node const_node; + typedef typename params_type::value_type value_type; + typedef typename params_type::pointer normal_pointer; + typedef typename params_type::reference normal_reference; + typedef typename params_type::const_pointer const_pointer; + typedef typename params_type::const_reference const_reference; + + typedef Pointer pointer; + typedef Reference reference; + typedef bidirectional_iterator_tag iterator_category; + + typedef btree_iterator< + normal_node, normal_reference, normal_pointer> iterator; + typedef btree_iterator< + const_node, const_reference, const_pointer> const_iterator; + typedef btree_iterator self_type; + + btree_iterator() + : node(NULL), + position(-1) { + } + btree_iterator(Node *n, int p) + : node(n), + position(p) { + } + btree_iterator(const iterator &x) + : node(x.node), + position(x.position) { + } + + // Increment/decrement the iterator. + void increment() { + if (node->leaf() && ++position < node->count()) { + return; + } + increment_slow(); + } + void increment_by(int count); + void increment_slow(); + + void decrement() { + if (node->leaf() && --position >= 0) { + return; + } + decrement_slow(); + } + void decrement_slow(); + + bool operator==(const const_iterator &x) const { + return node == x.node && position == x.position; + } + bool operator!=(const const_iterator &x) const { + return node != x.node || position != x.position; + } + + // Accessors for the key/value the iterator is pointing at. + const key_type& key() const { + return node->key(position); + } + reference operator*() const { + return node->value(position); + } + pointer operator->() const { + return &node->value(position); + } + + self_type& operator++() { + increment(); + return *this; + } + self_type& operator--() { + decrement(); + return *this; + } + self_type operator++(int) { + self_type tmp = *this; + ++*this; + return tmp; + } + self_type operator--(int) { + self_type tmp = *this; + --*this; + return tmp; + } + + // The node in the tree the iterator is pointing at. + Node *node; + // The position within the node of the tree the iterator is pointing at. + int position; +}; + +// Dispatch helper class for using btree::internal_locate with plain compare. +struct btree_internal_locate_plain_compare { + template + static pair dispatch(const K &k, const T &t, Iter iter) { + return t.internal_locate_plain_compare(k, iter); + } +}; + +// Dispatch helper class for using btree::internal_locate with compare-to. +struct btree_internal_locate_compare_to { + template + static pair dispatch(const K &k, const T &t, Iter iter) { + return t.internal_locate_compare_to(k, iter); + } +}; + +template +class btree : public Params::key_compare { + typedef btree self_type; + typedef btree_node node_type; + typedef typename node_type::base_fields base_fields; + typedef typename node_type::leaf_fields leaf_fields; + typedef typename node_type::internal_fields internal_fields; + typedef typename node_type::root_fields root_fields; + typedef typename Params::is_key_compare_to is_key_compare_to; + + friend class btree_internal_locate_plain_compare; + friend class btree_internal_locate_compare_to; + typedef typename base::if_< + is_key_compare_to::value, + btree_internal_locate_compare_to, + btree_internal_locate_plain_compare>::type internal_locate_type; + + enum { + kNodeValues = node_type::kNodeValues, + kMinNodeValues = kNodeValues / 2, + kValueSize = node_type::kValueSize, + kExactMatch = node_type::kExactMatch, + kMatchMask = node_type::kMatchMask, + }; + + // A helper class to get the empty base class optimization for 0-size + // allocators. Base is internal_allocator_type. + // (e.g. empty_base_handle). If Base is + // 0-size, the compiler doesn't have to reserve any space for it and + // sizeof(empty_base_handle) will simply be sizeof(Data). Google [empty base + // class optimization] for more details. + template + struct empty_base_handle : public Base { + empty_base_handle(const Base &b, const Data &d) + : Base(b), + data(d) { + } + Data data; + }; + + struct node_stats { + node_stats(ssize_t l, ssize_t i) + : leaf_nodes(l), + internal_nodes(i) { + } + + node_stats& operator+=(const node_stats &x) { + leaf_nodes += x.leaf_nodes; + internal_nodes += x.internal_nodes; + return *this; + } + + ssize_t leaf_nodes; + ssize_t internal_nodes; + }; + + public: + typedef Params params_type; + typedef typename Params::key_type key_type; + typedef typename Params::data_type data_type; + typedef typename Params::mapped_type mapped_type; + typedef typename Params::value_type value_type; + typedef typename Params::key_compare key_compare; + typedef typename Params::pointer pointer; + typedef typename Params::const_pointer const_pointer; + typedef typename Params::reference reference; + typedef typename Params::const_reference const_reference; + typedef typename Params::size_type size_type; + typedef typename Params::difference_type difference_type; + typedef btree_iterator iterator; + typedef typename iterator::const_iterator const_iterator; + typedef std::reverse_iterator const_reverse_iterator; + typedef std::reverse_iterator reverse_iterator; + + typedef typename Params::allocator_type allocator_type; + typedef typename allocator_type::template rebind::other + internal_allocator_type; + + public: + // Default constructor. + btree(const key_compare &comp, const allocator_type &alloc); + + // Copy constructor. + btree(const self_type &x); + + // Destructor. + ~btree() { + clear(); + } + + // Iterator routines. + iterator begin() { + return iterator(leftmost(), 0); + } + const_iterator begin() const { + return const_iterator(leftmost(), 0); + } + iterator end() { + return iterator(rightmost(), rightmost() ? rightmost()->count() : 0); + } + const_iterator end() const { + return const_iterator(rightmost(), rightmost() ? rightmost()->count() : 0); + } + reverse_iterator rbegin() { + return reverse_iterator(end()); + } + const_reverse_iterator rbegin() const { + return const_reverse_iterator(end()); + } + reverse_iterator rend() { + return reverse_iterator(begin()); + } + const_reverse_iterator rend() const { + return const_reverse_iterator(begin()); + } + + // Finds the first element whose key is not less than key. + iterator lower_bound(const key_type &key) { + return internal_end( + internal_lower_bound(key, iterator(root(), 0))); + } + const_iterator lower_bound(const key_type &key) const { + return internal_end( + internal_lower_bound(key, const_iterator(root(), 0))); + } + + // Finds the first element whose key is greater than key. + iterator upper_bound(const key_type &key) { + return internal_end( + internal_upper_bound(key, iterator(root(), 0))); + } + const_iterator upper_bound(const key_type &key) const { + return internal_end( + internal_upper_bound(key, const_iterator(root(), 0))); + } + + // Finds the range of values which compare equal to key. The first member of + // the returned pair is equal to lower_bound(key). The second member pair of + // the pair is equal to upper_bound(key). + pair equal_range(const key_type &key) { + return make_pair(lower_bound(key), upper_bound(key)); + } + pair equal_range(const key_type &key) const { + return make_pair(lower_bound(key), upper_bound(key)); + } + + // Inserts a value into the btree only if it does not already exist. The + // boolean return value indicates whether insertion succeeded or failed. The + // ValuePointer type is used to avoid instatiating the value unless the key + // is being inserted. Value is not dereferenced if the key already exists in + // the btree. See btree_map::operator[]. + template + pair insert_unique(const key_type &key, ValuePointer value); + + // Inserts a value into the btree only if it does not already exist. The + // boolean return value indicates whether insertion succeeded or failed. + pair insert_unique(const value_type &v) { + return insert_unique(params_type::key(v), &v); + } + + // Insert with hint. Check to see if the value should be placed immediately + // before position in the tree. If it does, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_unique(v) were made. + iterator insert_unique(iterator position, const value_type &v); + + // Insert a range of values into the btree. + template + void insert_unique(InputIterator b, InputIterator e); + + // Inserts a value into the btree. The ValuePointer type is used to avoid + // instatiating the value unless the key is being inserted. Value is not + // dereferenced if the key already exists in the btree. See + // btree_map::operator[]. + template + iterator insert_multi(const key_type &key, ValuePointer value); + + // Inserts a value into the btree. + iterator insert_multi(const value_type &v) { + return insert_multi(params_type::key(v), &v); + } + + // Insert with hint. Check to see if the value should be placed immediately + // before position in the tree. If it does, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_multi(v) were made. + iterator insert_multi(iterator position, const value_type &v); + + // Insert a range of values into the btree. + template + void insert_multi(InputIterator b, InputIterator e); + + void assign(const self_type &x); + + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + iterator erase(iterator iter); + + // Erases range. Returns the number of keys erased. + int erase(iterator begin, iterator end); + + // Erases the specified key from the btree. Returns 1 if an element was + // erased and 0 otherwise. + int erase_unique(const key_type &key); + + // Erases all of the entries matching the specified key from the + // btree. Returns the number of elements erased. + int erase_multi(const key_type &key); + + // Finds the iterator corresponding to a key or returns end() if the key is + // not present. + iterator find_unique(const key_type &key) { + return internal_end( + internal_find_unique(key, iterator(root(), 0))); + } + const_iterator find_unique(const key_type &key) const { + return internal_end( + internal_find_unique(key, const_iterator(root(), 0))); + } + iterator find_multi(const key_type &key) { + return internal_end( + internal_find_multi(key, iterator(root(), 0))); + } + const_iterator find_multi(const key_type &key) const { + return internal_end( + internal_find_multi(key, const_iterator(root(), 0))); + } + + // Returns a count of the number of times the key appears in the btree. + size_type count_unique(const key_type &key) const { + const_iterator begin = internal_find_unique( + key, const_iterator(root(), 0)); + if (!begin.node) { + // The key doesn't exist in the tree. + return 0; + } + return 1; + } + // Returns a count of the number of times the key appears in the btree. + size_type count_multi(const key_type &key) const { + return distance(lower_bound(key), upper_bound(key)); + } + + // Clear the btree, deleting all of the values it contains. + void clear(); + + // Swap the contents of *this and x. + void swap(self_type &x); + + // Assign the contents of x to *this. + self_type& operator=(const self_type &x) { + if (&x == this) { + // Don't copy onto ourselves. + return *this; + } + assign(x); + return *this; + } + + key_compare* mutable_key_comp() { + return this; + } + const key_compare& key_comp() const { + return *this; + } + bool compare_keys(const key_type &x, const key_type &y) const { + return btree_compare_keys(key_comp(), x, y); + } + + // Dump the btree to the specified ostream. Requires that operator<< is + // defined for Key and Value. + void dump(ostream &os) const { + if (root() != NULL) { + internal_dump(os, root(), 0); + } + } + + // Verifies the structure of the btree. + void verify() const; + + // Size routines. Note that empty() is slightly faster than doing size()==0. + size_type size() const { + if (empty()) return 0; + if (root()->leaf()) return root()->count(); + return root()->size(); + } + size_type max_size() const { return numeric_limits::max(); } + bool empty() const { return root() == NULL; } + + // The height of the btree. An empty tree will have height 0. + size_type height() const { + size_type h = 0; + if (root()) { + // Count the length of the chain from the leftmost node up to the + // root. We actually count from the root back around to the level below + // the root, but the calculation is the same because of the circularity + // of that traversal. + const node_type *n = root(); + do { + ++h; + n = n->parent(); + } while (n != root()); + } + return h; + } + + // The number of internal, leaf and total nodes used by the btree. + size_type leaf_nodes() const { + return internal_stats(root()).leaf_nodes; + } + size_type internal_nodes() const { + return internal_stats(root()).internal_nodes; + } + size_type nodes() const { + node_stats stats = internal_stats(root()); + return stats.leaf_nodes + stats.internal_nodes; + } + + // The total number of bytes used by the btree. + size_type bytes_used() const { + node_stats stats = internal_stats(root()); + if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) { + return sizeof(*this) + + sizeof(base_fields) + root()->max_count() * sizeof(value_type); + } else { + return sizeof(*this) + + sizeof(root_fields) - sizeof(internal_fields) + + stats.leaf_nodes * sizeof(leaf_fields) + + stats.internal_nodes * sizeof(internal_fields); + } + } + + // The average number of bytes used per value stored in the btree. + static double average_bytes_per_value() { + // Returns the number of bytes per value on a leaf node that is 75% + // full. Experimentally, this matches up nicely with the computed number of + // bytes per value in trees that had their values inserted in random order. + return sizeof(leaf_fields) / (kNodeValues * 0.75); + } + + // The fullness of the btree. Computed as the number of elements in the btree + // divided by the maximum number of elements a tree with the current number + // of nodes could hold. A value of 1 indicates perfect space + // utilization. Smaller values indicate space wastage. + double fullness() const { + return double(size()) / (nodes() * kNodeValues); + } + // The overhead of the btree structure in bytes per node. Computed as the + // total number of bytes used by the btree minus the number of bytes used for + // storing elements divided by the number of elements. + double overhead() const { + if (empty()) { + return 0.0; + } + return (bytes_used() - size() * kValueSize) / double(size()); + } + + private: + // Internal accessor routines. + node_type* root() { return root_.data; } + const node_type* root() const { return root_.data; } + node_type** mutable_root() { return &root_.data; } + + // The rightmost node is stored in the root node. + node_type* rightmost() { + return (!root() || root()->leaf()) ? root() : root()->rightmost(); + } + const node_type* rightmost() const { + return (!root() || root()->leaf()) ? root() : root()->rightmost(); + } + node_type** mutable_rightmost() { return root()->mutable_rightmost(); } + + // The leftmost node is stored as the parent of the root node. + node_type* leftmost() { return root() ? root()->parent() : NULL; } + const node_type* leftmost() const { return root() ? root()->parent() : NULL; } + + // The size of the tree is stored in the root node. + size_type* mutable_size() { return root()->mutable_size(); } + + // Allocator routines. + internal_allocator_type* mutable_internal_allocator() { + return static_cast(&root_); + } + const internal_allocator_type& internal_allocator() const { + return *static_cast(&root_); + } + + // Node creation/deletion routines. + node_type* new_internal_node(node_type *parent) { + internal_fields *p = reinterpret_cast( + mutable_internal_allocator()->allocate(sizeof(internal_fields))); + return node_type::init_internal(p, parent); + } + node_type* new_internal_root_node() { + root_fields *p = reinterpret_cast( + mutable_internal_allocator()->allocate(sizeof(root_fields))); + return node_type::init_root(p, root()->parent()); + } + node_type* new_leaf_node(node_type *parent) { + leaf_fields *p = reinterpret_cast( + mutable_internal_allocator()->allocate(sizeof(leaf_fields))); + return node_type::init_leaf(p, parent, kNodeValues); + } + node_type* new_leaf_root_node(int max_count) { + leaf_fields *p = reinterpret_cast( + mutable_internal_allocator()->allocate( + sizeof(base_fields) + max_count * sizeof(value_type))); + return node_type::init_leaf(p, reinterpret_cast(p), max_count); + } + void delete_internal_node(node_type *node) { + node->destroy(); + DCHECK(node != root()); + mutable_internal_allocator()->deallocate( + reinterpret_cast(node), sizeof(internal_fields)); + } + void delete_internal_root_node() { + root()->destroy(); + mutable_internal_allocator()->deallocate( + reinterpret_cast(root()), sizeof(root_fields)); + } + void delete_leaf_node(node_type *node) { + node->destroy(); + mutable_internal_allocator()->deallocate( + reinterpret_cast(node), + sizeof(base_fields) + node->max_count() * sizeof(value_type)); + } + + // Rebalances or splits the node iter points to. + void rebalance_or_split(iterator *iter); + + // Merges the values of left, right and the delimiting key on their parent + // onto left, removing the delimiting key and deleting right. + void merge_nodes(node_type *left, node_type *right); + + // Tries to merge node with its left or right sibling, and failing that, + // rebalance with its left or right sibling. Returns true if a merge + // occurred, at which point it is no longer valid to access node. Returns + // false if no merging took place. + bool try_merge_or_rebalance(iterator *iter); + + // Tries to shrink the height of the tree by 1. + void try_shrink(); + + iterator internal_end(iterator iter) { + return iter.node ? iter : end(); + } + const_iterator internal_end(const_iterator iter) const { + return iter.node ? iter : end(); + } + + // Inserts a value into the btree immediately before iter. Requires that + // key(v) <= iter.key() and (--iter).key() <= key(v). + iterator internal_insert(iterator iter, const value_type &v); + + // Returns an iterator pointing to the first value >= the value "iter" is + // pointing at. Note that "iter" might be pointing to an invalid location as + // iter.position == iter.node->count(). This routine simply moves iter up in + // the tree to a valid location. + template + static IterType internal_last(IterType iter); + + // Returns an iterator pointing to the leaf position at which key would + // reside in the tree. We provide 2 versions of internal_locate. The first + // version (internal_locate_plain_compare) always returns 0 for the second + // field of the pair. The second version (internal_locate_compare_to) is for + // the key-compare-to specialization and returns either kExactMatch (if the + // key was found in the tree) or -kExactMatch (if it wasn't) in the second + // field of the pair. The compare_to specialization allows the caller to + // avoid a subsequent comparison to determine if an exact match was made, + // speeding up string, cord and StringPiece keys. + template + pair internal_locate( + const key_type &key, IterType iter) const; + template + pair internal_locate_plain_compare( + const key_type &key, IterType iter) const; + template + pair internal_locate_compare_to( + const key_type &key, IterType iter) const; + + // Internal routine which implements lower_bound(). + template + IterType internal_lower_bound( + const key_type &key, IterType iter) const; + + // Internal routine which implements upper_bound(). + template + IterType internal_upper_bound( + const key_type &key, IterType iter) const; + + // Internal routine which implements find_unique(). + template + IterType internal_find_unique( + const key_type &key, IterType iter) const; + + // Internal routine which implements find_multi(). + template + IterType internal_find_multi( + const key_type &key, IterType iter) const; + + // Deletes a node and all of its children. + void internal_clear(node_type *node); + + // Dumps a node and all of its children to the specified ostream. + void internal_dump(ostream &os, const node_type *node, int level) const; + + // Verifies the tree structure of node. + int internal_verify(const node_type *node, + const key_type *lo, const key_type *hi) const; + + node_stats internal_stats(const node_type *node) const { + if (!node) { + return node_stats(0, 0); + } + if (node->leaf()) { + return node_stats(1, 0); + } + node_stats res(0, 1); + for (int i = 0; i <= node->count(); ++i) { + res += internal_stats(node->child(i)); + } + return res; + } + + private: + empty_base_handle root_; + + private: + // A never instantiated helper function that returns base::big_ if we have a + // key-compare-to functor or if R is bool and base::small_ otherwise. + template + static typename base::if_< + base::if_, + base::type_equals_ >::type::value, + base::big_, base::small_>::type key_compare_checker(R); + + // A never instantiated helper function that returns the key comparison + // functor. + static key_compare key_compare_helper(); + + // Verify that key_compare returns a bool. This is similar to the way + // is_convertible in base/type_traits.h works. Note that key_compare_checker + // is never actually invoked. The compiler will select which + // key_compare_checker() to instantiate and then figure out the size of the + // return type of key_compare_checker() at compile time which we then check + // against the sizeof of base::big_. + COMPILE_ASSERT( + sizeof(key_compare_checker(key_compare_helper()(key_type(), key_type()))) == + sizeof(base::big_), + key_comparison_function_must_return_bool); +}; + +//// +// btree_node methods +template +inline void btree_node

::insert_value(int i, const value_type &x) { + DCHECK_LE(i, count()); + value_init(count(), x); + for (int j = count(); j > i; --j) { + value_swap(j, this, j - 1); + } + set_count(count() + 1); + + if (!leaf()) { + ++i; + for (int j = count(); j > i; --j) { + *mutable_child(j) = child(j - 1); + child(j)->set_position(j); + } + *mutable_child(i) = NULL; + } +} + +template +inline void btree_node

::remove_value(int i) { + if (!leaf()) { + DCHECK_EQ(child(i + 1)->count(), 0); + for (int j = i + 1; j < count(); ++j) { + *mutable_child(j) = child(j + 1); + child(j)->set_position(j); + } + *mutable_child(count()) = NULL; + } + + set_count(count() - 1); + for (; i < count(); ++i) { + value_swap(i, this, i + 1); + } + value_destroy(i); +} + +template +void btree_node

::rebalance_right_to_left(btree_node *src, int to_move) { + DCHECK_EQ(parent(), src->parent()); + DCHECK_EQ(position() + 1, src->position()); + DCHECK_GE(src->count(), count()); + DCHECK_GE(to_move, 1); + DCHECK_LE(to_move, src->count()); + + // Make room in the left node for the new values. + for (int i = 0; i < to_move; ++i) { + value_init(i + count()); + } + + // Move the delimiting value to the left node and the new delimiting value + // from the right node. + value_swap(count(), parent(), position()); + parent()->value_swap(position(), src, to_move - 1); + + // Move the values from the right to the left node. + for (int i = 1; i < to_move; ++i) { + value_swap(count() + i, src, i - 1); + } + // Shift the values in the right node to their correct position. + for (int i = to_move; i < src->count(); ++i) { + src->value_swap(i - to_move, src, i); + } + for (int i = 1; i <= to_move; ++i) { + src->value_destroy(src->count() - i); + } + + if (!leaf()) { + // Move the child pointers from the right to the left node. + for (int i = 0; i < to_move; ++i) { + set_child(1 + count() + i, src->child(i)); + } + for (int i = 0; i <= src->count() - to_move; ++i) { + DCHECK_LE(i + to_move, src->max_count()); + src->set_child(i, src->child(i + to_move)); + *src->mutable_child(i + to_move) = NULL; + } + } + + // Fixup the counts on the src and dest nodes. + set_count(count() + to_move); + src->set_count(src->count() - to_move); +} + +template +void btree_node

::rebalance_left_to_right(btree_node *dest, int to_move) { + DCHECK_EQ(parent(), dest->parent()); + DCHECK_EQ(position() + 1, dest->position()); + DCHECK_GE(count(), dest->count()); + DCHECK_GE(to_move, 1); + DCHECK_LE(to_move, count()); + + // Make room in the right node for the new values. + for (int i = 0; i < to_move; ++i) { + dest->value_init(i + dest->count()); + } + for (int i = dest->count() - 1; i >= 0; --i) { + dest->value_swap(i, dest, i + to_move); + } + + // Move the delimiting value to the right node and the new delimiting value + // from the left node. + dest->value_swap(to_move - 1, parent(), position()); + parent()->value_swap(position(), this, count() - to_move); + value_destroy(count() - to_move); + + // Move the values from the left to the right node. + for (int i = 1; i < to_move; ++i) { + value_swap(count() - to_move + i, dest, i - 1); + value_destroy(count() - to_move + i); + } + + if (!leaf()) { + // Move the child pointers from the left to the right node. + for (int i = dest->count(); i >= 0; --i) { + dest->set_child(i + to_move, dest->child(i)); + *dest->mutable_child(i) = NULL; + } + for (int i = 1; i <= to_move; ++i) { + dest->set_child(i - 1, child(count() - to_move + i)); + *mutable_child(count() - to_move + i) = NULL; + } + } + + // Fixup the counts on the src and dest nodes. + set_count(count() - to_move); + dest->set_count(dest->count() + to_move); +} + +template +void btree_node

::split(btree_node *dest, int insert_position) { + DCHECK_EQ(dest->count(), 0); + + // We bias the split based on the position being inserted. If we're + // inserting at the beginning of the left node then bias the split to put + // more values on the right node. If we're inserting at the end of the + // right node then bias the split to put more values on the left node. + if (insert_position == 0) { + dest->set_count(count() - 1); + } else if (insert_position == max_count()) { + dest->set_count(0); + } else { + dest->set_count(count() / 2); + } + set_count(count() - dest->count()); + DCHECK_GE(count(), 1); + + // Move values from the left sibling to the right sibling. + for (int i = 0; i < dest->count(); ++i) { + dest->value_init(i); + value_swap(count() + i, dest, i); + value_destroy(count() + i); + } + + // The split key is the largest value in the left sibling. + set_count(count() - 1); + parent()->insert_value(position(), value_type()); + value_swap(count(), parent(), position()); + value_destroy(count()); + parent()->set_child(position() + 1, dest); + + if (!leaf()) { + for (int i = 0; i <= dest->count(); ++i) { + DCHECK(child(count() + i + 1) != NULL); + dest->set_child(i, child(count() + i + 1)); + *mutable_child(count() + i + 1) = NULL; + } + } +} + +template +void btree_node

::merge(btree_node *src) { + DCHECK_EQ(parent(), src->parent()); + DCHECK_EQ(position() + 1, src->position()); + + // Move the delimiting value to the left node. + value_init(count()); + value_swap(count(), parent(), position()); + + // Move the values from the right to the left node. + for (int i = 0; i < src->count(); ++i) { + value_init(1 + count() + i); + value_swap(1 + count() + i, src, i); + src->value_destroy(i); + } + + if (!leaf()) { + // Move the child pointers from the right to the left node. + for (int i = 0; i <= src->count(); ++i) { + set_child(1 + count() + i, src->child(i)); + *src->mutable_child(i) = NULL; + } + } + + // Fixup the counts on the src and dest nodes. + set_count(1 + count() + src->count()); + src->set_count(0); + + // Remove the value on the parent node. + parent()->remove_value(position()); +} + +template +void btree_node

::swap(btree_node *x) { + DCHECK_EQ(leaf(), x->leaf()); + + // Swap the values. + for (int i = count(); i < x->count(); ++i) { + value_init(i); + } + for (int i = x->count(); i < count(); ++i) { + x->value_init(i); + } + int n = max(count(), x->count()); + for (int i = 0; i < n; ++i) { + value_swap(i, x, i); + } + for (int i = count(); i < x->count(); ++i) { + x->value_destroy(i); + } + for (int i = x->count(); i < count(); ++i) { + value_destroy(i); + } + + if (!leaf()) { + // Swap the child pointers. + for (int i = 0; i <= n; ++i) { + btree_swap_helper(*mutable_child(i), *x->mutable_child(i)); + } + for (int i = 0; i <= count(); ++i) { + x->child(i)->fields_.parent = x; + } + for (int i = 0; i <= x->count(); ++i) { + child(i)->fields_.parent = this; + } + } + + // Swap the counts. + btree_swap_helper(fields_.count, x->fields_.count); +} + +//// +// btree_iterator methods +template +void btree_iterator::increment_slow() { + if (node->leaf()) { + DCHECK_GE(position, node->count()); + self_type save(*this); + while (position == node->count() && !node->is_root()) { + DCHECK_EQ(node->parent()->child(node->position()), node); + position = node->position(); + node = node->parent(); + } + if (position == node->count()) { + *this = save; + } + } else { + DCHECK_LT(position, node->count()); + node = node->child(position + 1); + while (!node->leaf()) { + node = node->child(0); + } + position = 0; + } +} + +template +void btree_iterator::increment_by(int count) { + while (count > 0) { + if (node->leaf()) { + int rest = node->count() - position; + position += min(rest, count); + count = count - rest; + if (position < node->count()) { + return; + } + } else { + --count; + } + increment_slow(); + } +} + +template +void btree_iterator::decrement_slow() { + if (node->leaf()) { + DCHECK_LE(position, -1); + self_type save(*this); + while (position < 0 && !node->is_root()) { + DCHECK_EQ(node->parent()->child(node->position()), node); + position = node->position() - 1; + node = node->parent(); + } + if (position < 0) { + *this = save; + } + } else { + DCHECK_GE(position, 0); + node = node->child(position); + while (!node->leaf()) { + node = node->child(node->count()); + } + position = node->count() - 1; + } +} + +//// +// btree methods +template +btree

::btree(const key_compare &comp, const allocator_type &alloc) + : key_compare(comp), + root_(alloc, NULL) { +} + +template +btree

::btree(const self_type &x) + : key_compare(x.key_comp()), + root_(x.internal_allocator(), NULL) { + assign(x); +} + +template template +pair::iterator, bool> +btree

::insert_unique(const key_type &key, ValuePointer value) { + if (empty()) { + *mutable_root() = new_leaf_root_node(1); + } + + pair res = internal_locate(key, iterator(root(), 0)); + iterator &iter = res.first; + if (res.second == kExactMatch) { + // The key already exists in the tree, do nothing. + return make_pair(internal_last(iter), false); + } else if (!res.second) { + iterator last = internal_last(iter); + if (last.node && !compare_keys(key, last.key())) { + // The key already exists in the tree, do nothing. + return make_pair(last, false); + } + } + + return make_pair(internal_insert(iter, *value), true); +} + +template +inline typename btree

::iterator +btree

::insert_unique(iterator position, const value_type &v) { + if (!empty()) { + const key_type &key = params_type::key(v); + if (position == end() || compare_keys(key, position.key())) { + iterator prev = position; + if (position == begin() || compare_keys((--prev).key(), key)) { + // prev.key() < key < position.key() + return internal_insert(position, v); + } + } else if (compare_keys(position.key(), key)) { + iterator next = position; + ++next; + if (next == end() || compare_keys(key, next.key())) { + // position.key() < key < next.key() + return internal_insert(next, v); + } + } else { + // position.key() == key + return position; + } + } + return insert_unique(v).first; +} + +template template +void btree

::insert_unique(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_unique(end(), *b); + } +} + +template template +typename btree

::iterator +btree

::insert_multi(const key_type &key, ValuePointer value) { + if (empty()) { + *mutable_root() = new_leaf_root_node(1); + } + + iterator iter = internal_upper_bound(key, iterator(root(), 0)); + if (!iter.node) { + iter = end(); + } + return internal_insert(iter, *value); +} + +template +typename btree

::iterator +btree

::insert_multi(iterator position, const value_type &v) { + if (!empty()) { + const key_type &key = params_type::key(v); + if (position == end() || !compare_keys(position.key(), key)) { + iterator prev = position; + if (position == begin() || !compare_keys(key, (--prev).key())) { + // prev.key() <= key <= position.key() + return internal_insert(position, v); + } + } else { + iterator next = position; + ++next; + if (next == end() || !compare_keys(next.key(), key)) { + // position.key() < key <= next.key() + return internal_insert(next, v); + } + } + } + return insert_multi(v); +} + +template template +void btree

::insert_multi(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_multi(end(), *b); + } +} + +template +void btree

::assign(const self_type &x) { + clear(); + + *mutable_key_comp() = x.key_comp(); + *mutable_internal_allocator() = x.internal_allocator(); + + // Assignment can avoid key comparisons because we know the order of the + // values is the same order we'll store them in. + for (const_iterator iter = x.begin(); iter != x.end(); ++iter) { + if (empty()) { + insert_multi(*iter); + } else { + // If the btree is not empty, we can just insert the new value at the end + // of the tree! + internal_insert(end(), *iter); + } + } +} + +template +typename btree

::iterator btree

::erase(iterator iter) { + bool internal_delete = false; + if (!iter.node->leaf()) { + // Deletion of a value on an internal node. Swap the key with the largest + // value of our left child. This is easy, we just decrement iter. + iterator tmp_iter(iter--); + DCHECK(iter.node->leaf()); + DCHECK(!compare_keys(tmp_iter.key(), iter.key())); + iter.node->value_swap(iter.position, tmp_iter.node, tmp_iter.position); + internal_delete = true; + --*mutable_size(); + } else if (!root()->leaf()) { + --*mutable_size(); + } + + // Delete the key from the leaf. + iter.node->remove_value(iter.position); + + // We want to return the next value after the one we just erased. If we + // erased from an internal node (internal_delete == true), then the next + // value is ++(++iter). If we erased from a leaf node (internal_delete == + // false) then the next value is ++iter. Note that ++iter may point to an + // internal node and the value in the internal node may move to a leaf node + // (iter.node) when rebalancing is performed at the leaf level. + + // Merge/rebalance as we walk back up the tree. + iterator res(iter); + for (;;) { + if (iter.node == root()) { + try_shrink(); + if (empty()) { + return end(); + } + break; + } + if (iter.node->count() >= kMinNodeValues) { + break; + } + bool merged = try_merge_or_rebalance(&iter); + if (iter.node->leaf()) { + res = iter; + } + if (!merged) { + break; + } + iter.node = iter.node->parent(); + } + + // Adjust our return value. If we're pointing at the end of a node, advance + // the iterator. + if (res.position == res.node->count()) { + res.position = res.node->count() - 1; + ++res; + } + // If we erased from an internal node, advance the iterator. + if (internal_delete) { + ++res; + } + return res; +} + +template +int btree

::erase(iterator begin, iterator end) { + int count = distance(begin, end); + for (int i = 0; i < count; i++) { + begin = erase(begin); + } + return count; +} + +template +int btree

::erase_unique(const key_type &key) { + iterator iter = internal_find_unique(key, iterator(root(), 0)); + if (!iter.node) { + // The key doesn't exist in the tree, return nothing done. + return 0; + } + erase(iter); + return 1; +} + +template +int btree

::erase_multi(const key_type &key) { + iterator begin = internal_lower_bound(key, iterator(root(), 0)); + if (!begin.node) { + // The key doesn't exist in the tree, return nothing done. + return 0; + } + // Delete all of the keys between begin and upper_bound(key). + iterator end = internal_end( + internal_upper_bound(key, iterator(root(), 0))); + return erase(begin, end); +} + +template +void btree

::clear() { + if (root() != NULL) { + internal_clear(root()); + } + *mutable_root() = NULL; +} + +template +void btree

::swap(self_type &x) { + ::swap(static_cast(*this), static_cast(x)); + ::swap(root_, x.root_); +} + +template +void btree

::verify() const { + if (root() != NULL) { + CHECK_EQ(size(), internal_verify(root(), NULL, NULL)); + CHECK_EQ(leftmost(), (++const_iterator(root(), -1)).node); + CHECK_EQ(rightmost(), (--const_iterator(root(), root()->count())).node); + CHECK(leftmost()->leaf()); + CHECK(rightmost()->leaf()); + } else { + CHECK_EQ(size(), 0); + CHECK(leftmost() == NULL); + CHECK(rightmost() == NULL); + } +} + +template +void btree

::rebalance_or_split(iterator *iter) { + node_type *&node = iter->node; + int &insert_position = iter->position; + DCHECK_EQ(node->count(), node->max_count()); + + // First try to make room on the node by rebalancing. + node_type *parent = node->parent(); + if (node != root()) { + if (node->position() > 0) { + // Try rebalancing with our left sibling. + node_type *left = parent->child(node->position() - 1); + if (left->count() < left->max_count()) { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the end of the right node then we bias rebalancing to + // fill up the left node. + int to_move = (left->max_count() - left->count()) / + (1 + (insert_position < left->max_count())); + to_move = max(1, to_move); + + if (((insert_position - to_move) >= 0) || + ((left->count() + to_move) < left->max_count())) { + left->rebalance_right_to_left(node, to_move); + + DCHECK_EQ(node->max_count() - node->count(), to_move); + insert_position = insert_position - to_move; + if (insert_position < 0) { + insert_position = insert_position + left->count() + 1; + node = left; + } + + DCHECK_LT(node->count(), node->max_count()); + return; + } + } + } + + if (node->position() < parent->count()) { + // Try rebalancing with our right sibling. + node_type *right = parent->child(node->position() + 1); + if (right->count() < right->max_count()) { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the beginning of the left node then we bias rebalancing + // to fill up the right node. + int to_move = (right->max_count() - right->count()) / + (1 + (insert_position > 0)); + to_move = max(1, to_move); + + if ((insert_position <= (node->count() - to_move)) || + ((right->count() + to_move) < right->max_count())) { + node->rebalance_left_to_right(right, to_move); + + if (insert_position > node->count()) { + insert_position = insert_position - node->count() - 1; + node = right; + } + + DCHECK_LT(node->count(), node->max_count()); + return; + } + } + } + + // Rebalancing failed, make sure there is room on the parent node for a new + // value. + if (parent->count() == parent->max_count()) { + iterator parent_iter(node->parent(), node->position()); + rebalance_or_split(&parent_iter); + } + } else { + // Rebalancing not possible because this is the root node. + if (root()->leaf()) { + // The root node is currently a leaf node: create a new root node and set + // the current root node as the child of the new root. + parent = new_internal_root_node(); + parent->set_child(0, root()); + *mutable_root() = parent; + DCHECK(*mutable_rightmost() == parent->child(0)); + } else { + // The root node is an internal node. We do not want to create a new root + // node because the root node is special and holds the size of the tree + // and a pointer to the rightmost node. So we create a new internal node + // and move all of the items on the current root into the new node. + parent = new_internal_node(parent); + parent->set_child(0, parent); + parent->swap(root()); + node = parent; + } + } + + // Split the node. + node_type *split_node; + if (node->leaf()) { + split_node = new_leaf_node(parent); + node->split(split_node, insert_position); + if (rightmost() == node) { + *mutable_rightmost() = split_node; + } + } else { + split_node = new_internal_node(parent); + node->split(split_node, insert_position); + } + + if (insert_position > node->count()) { + insert_position = insert_position - node->count() - 1; + node = split_node; + } +} + +template +void btree

::merge_nodes(node_type *left, node_type *right) { + left->merge(right); + if (right->leaf()) { + if (rightmost() == right) { + *mutable_rightmost() = left; + } + delete_leaf_node(right); + } else { + delete_internal_node(right); + } +} + +template +bool btree

::try_merge_or_rebalance(iterator *iter) { + node_type *parent = iter->node->parent(); + if (iter->node->position() > 0) { + // Try merging with our left sibling. + node_type *left = parent->child(iter->node->position() - 1); + if ((1 + left->count() + iter->node->count()) <= left->max_count()) { + iter->position += 1 + left->count(); + merge_nodes(left, iter->node); + iter->node = left; + return true; + } + } + if (iter->node->position() < parent->count()) { + // Try merging with our right sibling. + node_type *right = parent->child(iter->node->position() + 1); + if ((1 + iter->node->count() + right->count()) <= right->max_count()) { + merge_nodes(iter->node, right); + return true; + } + // Try rebalancing with our right sibling. We don't perform rebalancing if + // we deleted the first element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the front of the tree. + if ((right->count() > kMinNodeValues) && + ((iter->node->count() == 0) || + (iter->position > 0))) { + int to_move = (right->count() - iter->node->count()) / 2; + to_move = min(to_move, right->count() - 1); + iter->node->rebalance_right_to_left(right, to_move); + return false; + } + } + if (iter->node->position() > 0) { + // Try rebalancing with our left sibling. We don't perform rebalancing if + // we deleted the last element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the back of the tree. + node_type *left = parent->child(iter->node->position() - 1); + if ((left->count() > kMinNodeValues) && + ((iter->node->count() == 0) || + (iter->position < iter->node->count()))) { + int to_move = (left->count() - iter->node->count()) / 2; + to_move = min(to_move, left->count() - 1); + left->rebalance_left_to_right(iter->node, to_move); + iter->position += to_move; + return false; + } + } + return false; +} + +template +void btree

::try_shrink() { + if (root()->count() > 0) { + return; + } + // Deleted the last item on the root node, shrink the height of the tree. + if (root()->leaf()) { + DCHECK_EQ(size(), 0); + delete_leaf_node(root()); + *mutable_root() = NULL; + } else { + node_type *child = root()->child(0); + if (child->leaf()) { + // The child is a leaf node so simply make it the root node in the tree. + child->make_root(); + delete_internal_root_node(); + *mutable_root() = child; + } else { + // The child is an internal node. We want to keep the existing root node + // so we move all of the values from the child node into the existing + // (empty) root node. + child->swap(root()); + delete_internal_node(child); + } + } +} + +template template +inline IterType btree

::internal_last(IterType iter) { + while (iter.node && iter.position == iter.node->count()) { + iter.position = iter.node->position(); + iter.node = iter.node->parent(); + if (iter.node->leaf()) { + iter.node = NULL; + } + } + return iter; +} + +template +inline typename btree

::iterator +btree

::internal_insert(iterator iter, const value_type &v) { + if (!iter.node->leaf()) { + // We can't insert on an internal node. Instead, we'll insert after the + // previous value which is guaranteed to be on a leaf node. + --iter; + ++iter.position; + } + if (iter.node->count() == iter.node->max_count()) { + // Make room in the leaf for the new item. + if (iter.node->max_count() < kNodeValues) { + // Insertion into the root where the root is smaller that the full node + // size. Simply grow the size of the root node. + DCHECK(iter.node == root()); + iter.node = new_leaf_root_node( + min(kNodeValues, 2 * iter.node->max_count())); + iter.node->swap(root()); + delete_leaf_node(root()); + *mutable_root() = iter.node; + } else { + rebalance_or_split(&iter); + ++*mutable_size(); + } + } else if (!root()->leaf()) { + ++*mutable_size(); + } + iter.node->insert_value(iter.position, v); + return iter; +} + +template template +inline pair btree

::internal_locate( + const key_type &key, IterType iter) const { + return internal_locate_type::dispatch(key, *this, iter); +} + +template template +inline pair btree

::internal_locate_plain_compare( + const key_type &key, IterType iter) const { + for (;;) { + iter.position = iter.node->lower_bound(key, key_comp()); + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return make_pair(iter, 0); +} + +template template +inline pair btree

::internal_locate_compare_to( + const key_type &key, IterType iter) const { + for (;;) { + int res = iter.node->lower_bound(key, key_comp()); + iter.position = res & kMatchMask; + if (res & kExactMatch) { + return make_pair(iter, static_cast(kExactMatch)); + } + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return make_pair(iter, -kExactMatch); +} + +template template +IterType btree

::internal_lower_bound( + const key_type &key, IterType iter) const { + if (iter.node) { + for (;;) { + iter.position = + iter.node->lower_bound(key, key_comp()) & kMatchMask; + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + iter = internal_last(iter); + } + return iter; +} + +template template +IterType btree

::internal_upper_bound( + const key_type &key, IterType iter) const { + if (iter.node) { + for (;;) { + iter.position = iter.node->upper_bound(key, key_comp()); + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + iter = internal_last(iter); + } + return iter; +} + +template template +IterType btree

::internal_find_unique( + const key_type &key, IterType iter) const { + if (iter.node) { + pair res = internal_locate(key, iter); + if (res.second == kExactMatch) { + return res.first; + } + if (!res.second) { + iter = internal_last(res.first); + if (iter.node && !compare_keys(key, iter.key())) { + return iter; + } + } + } + return IterType(NULL, 0); +} + +template template +IterType btree

::internal_find_multi( + const key_type &key, IterType iter) const { + if (iter.node) { + iter = internal_lower_bound(key, iter); + if (iter.node) { + iter = internal_last(iter); + if (iter.node && !compare_keys(key, iter.key())) { + return iter; + } + } + } + return IterType(NULL, 0); +} + +template +void btree

::internal_clear(node_type *node) { + if (!node->leaf()) { + for (int i = 0; i <= node->count(); ++i) { + internal_clear(node->child(i)); + } + if (node == root()) { + delete_internal_root_node(); + } else { + delete_internal_node(node); + } + } else { + delete_leaf_node(node); + } +} + +template +void btree

::internal_dump( + ostream &os, const node_type *node, int level) const { + for (int i = 0; i < node->count(); ++i) { + if (!node->leaf()) { + internal_dump(os, node->child(i), level + 1); + } + for (int j = 0; j < level; ++j) { + os << " "; + } + os << node->key(i) << " [" << level << "]\n"; + } + if (!node->leaf()) { + internal_dump(os, node->child(node->count()), level + 1); + } +} + +template +int btree

::internal_verify( + const node_type *node, const key_type *lo, const key_type *hi) const { + CHECK_GT(node->count(), 0); + CHECK_LE(node->count(), node->max_count()); + if (lo) { + CHECK(!compare_keys(node->key(0), *lo)); + } + if (hi) { + CHECK(!compare_keys(*hi, node->key(node->count() - 1))); + } + for (int i = 1; i < node->count(); ++i) { + CHECK(!compare_keys(node->key(i), node->key(i - 1))); + } + int count = node->count(); + if (!node->leaf()) { + for (int i = 0; i <= node->count(); ++i) { + CHECK(node->child(i) != NULL); + CHECK_EQ(node->child(i)->parent(), node); + CHECK_EQ(node->child(i)->position(), i); + count += internal_verify( + node->child(i), + (i == 0) ? lo : &node->key(i - 1), + (i == node->count()) ? hi : &node->key(i)); + } + } + return count; +} + +} // namespace btree +} // namespace util + +DEFINE_GDB_AUTO_SCRIPT("util/btree/btree_printer.py") + +#endif // UTIL_BTREE_BTREE_H__ diff --git a/btree_bench.cc b/btree_bench.cc new file mode 100644 index 00000000..c67dd664 --- /dev/null +++ b/btree_bench.cc @@ -0,0 +1,483 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) + +#include +#include +#include +#include +#include +#include +#include + +#include "base/commandlineflags.h" +#include "base/init_google.h" +#include "base/integral_types.h" +#include "base/logging.h" +#include "base/stringprintf.h" +#include "base/type_traits.h" +#include "strings/cord.h" +#include "testing/base/public/benchmark.h" +#include "testing/base/public/googletest.h" +#include "util/btree/btree_map.h" +#include "util/btree/btree_set.h" +#include "util/btree/btree_test.h" +#include "util/random/acmrandom.h" + +DECLARE_int32(benchmark_max_iters); + +namespace util { +namespace btree { +namespace { + +// Benchmark insertion of values into a container. +template +void BM_Insert(int n) { + typedef typename base::remove_const::type V; + typename KeyOfValue::type key_of_value; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + + T container; + vector values = GenerateValues(FLAGS_benchmark_values); + for (int i = 0; i < values.size(); i++) { + container.insert(values[i]); + } + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); + + for (int i = 0; i < n; ) { + // Remove and re-insert 10% of the keys + int m = min(n - i, FLAGS_benchmark_values / 10); + + for (int j = i; j < i + m; j++) { + int x = j % FLAGS_benchmark_values; + container.erase(key_of_value(values[x])); + } + + StartBenchmarkTiming(); + + for (int j = i; j < i + m; j++) { + int x = j % FLAGS_benchmark_values; + container.insert(values[x]); + } + + StopBenchmarkTiming(); + + i += m; + } +} + +// Benchmark lookup of values in a container. +template +void BM_Lookup(int n) { + typedef typename base::remove_const::type V; + typename KeyOfValue::type key_of_value; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + + T container; + vector values = GenerateValues(FLAGS_benchmark_values); + + for (int i = 0; i < values.size(); i++) { + container.insert(values[i]); + } + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); + + V r = V(); + + StartBenchmarkTiming(); + + for (int i = 0; i < n; i++) { + int m = i % values.size(); + r = *container.find(key_of_value(values[m])); + } + + StopBenchmarkTiming(); + + VLOG(4) << r; // Keep compiler from optimizing away r. +} + +// Benchmark lookup of values in a full container, meaning that values +// are inserted in-order to take advantage of biased insertion, which +// yields a full tree. +template +void BM_FullLookup(int n) { + typedef typename base::remove_const::type V; + typename KeyOfValue::type key_of_value; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + + T container; + vector values = GenerateValues(FLAGS_benchmark_values); + vector sorted(values); + sort(sorted.begin(), sorted.end()); + + for (int i = 0; i < sorted.size(); i++) { + container.insert(sorted[i]); + } + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); + + V r = V(); + + StartBenchmarkTiming(); + + for (int i = 0; i < n; i++) { + int m = i % values.size(); + r = *container.find(key_of_value(values[m])); + } + + StopBenchmarkTiming(); + + VLOG(4) << r; // Keep compiler from optimizing away r. +} + +// Benchmark deletion of values from a container. +template +void BM_Delete(int n) { + typedef typename base::remove_const::type V; + typename KeyOfValue::type key_of_value; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + + T container; + vector values = GenerateValues(FLAGS_benchmark_values); + for (int i = 0; i < values.size(); i++) { + container.insert(values[i]); + } + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); + + for (int i = 0; i < n; ) { + // Remove and re-insert 10% of the keys + int m = min(n - i, FLAGS_benchmark_values / 10); + + StartBenchmarkTiming(); + + for (int j = i; j < i + m; j++) { + int x = j % FLAGS_benchmark_values; + container.erase(key_of_value(values[x])); + } + + StopBenchmarkTiming(); + + for (int j = i; j < i + m; j++) { + int x = j % FLAGS_benchmark_values; + container.insert(values[x]); + } + + i += m; + } +} + +// Benchmark steady-state insert (into first half of range) and remove +// (from second second half of range), treating the container +// approximately like a queue with log-time access for all elements. +// This benchmark does not test the case where insertion and removal +// happen in the same region of the tree. This benchmark counts two +// value constructors. +template +void BM_QueueAddRem(int n) { + typedef typename base::remove_const::type V; + typename KeyOfValue::type key_of_value; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + CHECK(FLAGS_benchmark_values % 2 == 0); + + T container; + + const int half = FLAGS_benchmark_values / 2; + vector remove_keys(half); + vector add_keys(half); + + for (int i = 0; i < half; i++) { + remove_keys[i] = i; + add_keys[i] = i; + } + + ACMRandom rand(FLAGS_test_random_seed); + + random_shuffle(remove_keys.begin(), remove_keys.end(), rand); + random_shuffle(add_keys.begin(), add_keys.end(), rand); + + Generator g(FLAGS_benchmark_values + FLAGS_benchmark_max_iters); + + for (int i = 0; i < half; i++) { + container.insert(g(add_keys[i])); + container.insert(g(half + remove_keys[i])); + } + + // There are three parts each of size "half": + // 1 is being deleted from [offset - half, offset) + // 2 is standing [offset, offset + half) + // 3 is being inserted into [offset + half, offset + 2 * half) + int offset = 0; + + StartBenchmarkTiming(); + + for (int i = 0; i < n; i++) { + int idx = i % half; + + if (idx == 0) { + StopBenchmarkTiming(); + random_shuffle(remove_keys.begin(), remove_keys.end(), rand); + random_shuffle(add_keys.begin(), add_keys.end(), rand); + offset += half; + StartBenchmarkTiming(); + } + + int e = container.erase(key_of_value(g(offset - half + remove_keys[idx]))); + DCHECK(e == 1); + container.insert(g(offset + half + add_keys[idx])); + } + + StopBenchmarkTiming(); + + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); +} + +// Mixed insertion and deletion in the same range using pre-constructed values. +template +void BM_MixedAddRem(int n) { + typedef typename base::remove_const::type V; + typename KeyOfValue::type key_of_value; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + CHECK(FLAGS_benchmark_values % 2 == 0); + + T container; + ACMRandom rand(FLAGS_test_random_seed); + + vector values = GenerateValues(FLAGS_benchmark_values * 2); + + // Create two random shuffles + vector remove_keys(FLAGS_benchmark_values); + vector add_keys(FLAGS_benchmark_values); + + // Insert the first half of the values (already in random order) + for (int i = 0; i < FLAGS_benchmark_values; i++) { + container.insert(values[i]); + + // remove_keys and add_keys will be swapped before each round, + // therefore fill add_keys here w/ the keys being inserted, so + // they'll be the first to be removed. + remove_keys[i] = i + FLAGS_benchmark_values; + add_keys[i] = i; + } + + StartBenchmarkTiming(); + + for (int i = 0; i < n; i++) { + int idx = i % FLAGS_benchmark_values; + + if (idx == 0) { + StopBenchmarkTiming(); + remove_keys.swap(add_keys); + random_shuffle(remove_keys.begin(), remove_keys.end(), rand); + random_shuffle(add_keys.begin(), add_keys.end(), rand); + StartBenchmarkTiming(); + } + + int e = container.erase(key_of_value(values[remove_keys[idx]])); + DCHECK(e == 1); + container.insert(values[add_keys[idx]]); + } + + StopBenchmarkTiming(); + + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); +} + +// Insertion at end, removal from the beginning. This benchmark +// counts two value constructors. +template +void BM_Fifo(int n) { + typedef typename base::remove_const::type V; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + + T container; + Generator g(FLAGS_benchmark_values + FLAGS_benchmark_max_iters); + + for (int i = 0; i < FLAGS_benchmark_values; i++) { + container.insert(g(i)); + } + + StartBenchmarkTiming(); + + for (int i = 0; i < n; i++) { + container.erase(container.begin()); + container.insert(container.end(), g(i + FLAGS_benchmark_values)); + } + + StopBenchmarkTiming(); + + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); +} + +// Iteration (forward) through the tree +template +void BM_FwdIter(int n) { + typedef typename base::remove_const::type V; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + + T container; + vector values = GenerateValues(FLAGS_benchmark_values); + + for (int i = 0; i < FLAGS_benchmark_values; i++) { + container.insert(values[i]); + } + + typename T::iterator iter; + + V r = V(); + + StartBenchmarkTiming(); + + for (int i = 0; i < n; i++) { + int idx = i % FLAGS_benchmark_values; + + if (idx == 0) { + iter = container.begin(); + } + r = *iter; + ++iter; + } + + StopBenchmarkTiming(); + + VLOG(4) << r; // Keep compiler from optimizing away r. + + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); +} + +typedef set stl_set_int32; +typedef set stl_set_int64; +typedef set stl_set_string; +typedef set stl_set_cord; + +typedef map stl_map_int32; +typedef map stl_map_int64; +typedef map stl_map_string; +typedef map stl_map_cord; + +typedef multiset stl_multiset_int32; +typedef multiset stl_multiset_int64; +typedef multiset stl_multiset_string; +typedef multiset stl_multiset_cord; + +typedef multimap stl_multimap_int32; +typedef multimap stl_multimap_int64; +typedef multimap stl_multimap_string; +typedef multimap stl_multimap_cord; + +#define MY_BENCHMARK_TYPES2(value, name, size) \ + typedef btree ## _set, allocator, size> \ + btree ## _ ## size ## _set_ ## name; \ + typedef btree ## _map, allocator, size> \ + btree ## _ ## size ## _map_ ## name; \ + typedef btree ## _multiset, allocator, size> \ + btree ## _ ## size ## _multiset_ ## name; \ + typedef btree ## _multimap, allocator, size> \ + btree ## _ ## size ## _multimap_ ## name + +#define MY_BENCHMARK_TYPES(value, name) \ + MY_BENCHMARK_TYPES2(value, name, 128); \ + MY_BENCHMARK_TYPES2(value, name, 160); \ + MY_BENCHMARK_TYPES2(value, name, 192); \ + MY_BENCHMARK_TYPES2(value, name, 224); \ + MY_BENCHMARK_TYPES2(value, name, 256); \ + MY_BENCHMARK_TYPES2(value, name, 288); \ + MY_BENCHMARK_TYPES2(value, name, 320); \ + MY_BENCHMARK_TYPES2(value, name, 352); \ + MY_BENCHMARK_TYPES2(value, name, 384); \ + MY_BENCHMARK_TYPES2(value, name, 416); \ + MY_BENCHMARK_TYPES2(value, name, 448); \ + MY_BENCHMARK_TYPES2(value, name, 480); \ + MY_BENCHMARK_TYPES2(value, name, 512) + +MY_BENCHMARK_TYPES(int32, int32); +MY_BENCHMARK_TYPES(int64, int64); +MY_BENCHMARK_TYPES(string, string); +MY_BENCHMARK_TYPES(Cord, cord); + +#define MY_BENCHMARK4(type, name, func) \ + void BM_ ## type ## _ ## name(int n) { BM_ ## func (n); } \ + BENCHMARK(BM_ ## type ## _ ## name) + +// Define NODESIZE_TESTING when running btree_perf.py. You need to do +// a local build or raise the distcc timeout, it takes about 5 minutes +// to build: +// +// blaze build --copts=-DNODESIZE_TESTING --cc_strategy=local +// --compilation_mode=opt util/btree/btree_test + +#ifdef NODESIZE_TESTING +#define MY_BENCHMARK3(tree, type, name, func) \ + MY_BENCHMARK4(tree ## _128_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _160_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _192_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _224_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _256_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _288_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _320_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _352_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _384_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _416_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _448_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _480_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _512_ ## type, name, func) +#else +#define MY_BENCHMARK3(tree, type, name, func) \ + MY_BENCHMARK4(tree ## _256_ ## type, name, func) +#endif + +#define MY_BENCHMARK2(type, name, func) \ + MY_BENCHMARK4(stl_ ## type, name, func); \ + MY_BENCHMARK3(btree, type, name, func) + +#define MY_BENCHMARK(type) \ + MY_BENCHMARK2(type, insert, Insert); \ + MY_BENCHMARK2(type, lookup, Lookup); \ + MY_BENCHMARK2(type, fulllookup, FullLookup); \ + MY_BENCHMARK2(type, delete, Delete); \ + MY_BENCHMARK2(type, queueaddrem, QueueAddRem); \ + MY_BENCHMARK2(type, mixedaddrem, MixedAddRem); \ + MY_BENCHMARK2(type, fifo, Fifo); \ + MY_BENCHMARK2(type, fwditer, FwdIter) + +MY_BENCHMARK(set_int32); +MY_BENCHMARK(map_int32); +MY_BENCHMARK(set_int64); +MY_BENCHMARK(map_int64); +MY_BENCHMARK(set_string); +MY_BENCHMARK(map_string); +MY_BENCHMARK(set_cord); +MY_BENCHMARK(map_cord); + +MY_BENCHMARK(multiset_int32); +MY_BENCHMARK(multimap_int32); +MY_BENCHMARK(multiset_int64); +MY_BENCHMARK(multimap_int64); +MY_BENCHMARK(multiset_string); +MY_BENCHMARK(multimap_string); +MY_BENCHMARK(multiset_cord); +MY_BENCHMARK(multimap_cord); + +} // namespace +} // namespace btree +} // namespace util + +int main(int argc, char **argv) { + FLAGS_logtostderr = true; + InitGoogle(argv[0], &argc, &argv, true); + RunSpecifiedBenchmarks(); + return 0; +} diff --git a/btree_container.h b/btree_container.h new file mode 100644 index 00000000..6703d1bf --- /dev/null +++ b/btree_container.h @@ -0,0 +1,325 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) + +#ifndef UTIL_BTREE_BTREE_CONTAINER_H__ +#define UTIL_BTREE_BTREE_CONTAINER_H__ + +#include +#include + +#include "util/btree/btree.h" // IWYU pragma: export + +namespace util { +namespace btree { + +// A common base class for btree_set, btree_map, btree_multiset and +// btree_multimap. +template +class btree_container { + typedef btree_container self_type; + + public: + typedef typename Tree::params_type params_type; + typedef typename Tree::key_type key_type; + typedef typename Tree::value_type value_type; + typedef typename Tree::key_compare key_compare; + typedef typename Tree::allocator_type allocator_type; + typedef typename Tree::pointer pointer; + typedef typename Tree::const_pointer const_pointer; + typedef typename Tree::reference reference; + typedef typename Tree::const_reference const_reference; + typedef typename Tree::size_type size_type; + typedef typename Tree::difference_type difference_type; + typedef typename Tree::iterator iterator; + typedef typename Tree::const_iterator const_iterator; + typedef typename Tree::reverse_iterator reverse_iterator; + typedef typename Tree::const_reverse_iterator const_reverse_iterator; + + public: + // Default constructor. + btree_container(const key_compare &comp, const allocator_type &alloc) + : tree_(comp, alloc) { + } + + // Copy constructor. + btree_container(const self_type &x) + : tree_(x.tree_) { + } + + // Iterator routines. + iterator begin() { return tree_.begin(); } + const_iterator begin() const { return tree_.begin(); } + iterator end() { return tree_.end(); } + const_iterator end() const { return tree_.end(); } + reverse_iterator rbegin() { return tree_.rbegin(); } + const_reverse_iterator rbegin() const { return tree_.rbegin(); } + reverse_iterator rend() { return tree_.rend(); } + const_reverse_iterator rend() const { return tree_.rend(); } + + // Lookup routines. + iterator lower_bound(const key_type &key) { + return tree_.lower_bound(key); + } + const_iterator lower_bound(const key_type &key) const { + return tree_.lower_bound(key); + } + iterator upper_bound(const key_type &key) { + return tree_.upper_bound(key); + } + const_iterator upper_bound(const key_type &key) const { + return tree_.upper_bound(key); + } + pair equal_range(const key_type &key) { + return tree_.equal_range(key); + } + pair equal_range(const key_type &key) const { + return tree_.equal_range(key); + } + + // Utility routines. + void clear() { + tree_.clear(); + } + void swap(self_type &x) { + tree_.swap(x.tree_); + } + void dump(ostream &os) const { + tree_.dump(os); + } + void verify() const { + tree_.verify(); + } + + // Size routines. + size_type size() const { return tree_.size(); } + size_type max_size() const { return tree_.max_size(); } + bool empty() const { return tree_.empty(); } + size_type height() const { return tree_.height(); } + size_type internal_nodes() const { return tree_.internal_nodes(); } + size_type leaf_nodes() const { return tree_.leaf_nodes(); } + size_type nodes() const { return tree_.nodes(); } + size_type bytes_used() const { return tree_.bytes_used(); } + static double average_bytes_per_value() { + return Tree::average_bytes_per_value(); + } + double fullness() const { return tree_.fullness(); } + double overhead() const { return tree_.overhead(); } + + protected: + Tree tree_; +}; + +template +inline ostream& operator<<(ostream &os, const btree_container &b) { + b.dump(os); + return os; +} + +// A common base class for btree_set and safe_btree_set. +template +class btree_unique_container : public btree_container { + typedef btree_unique_container self_type; + typedef btree_container super_type; + + public: + typedef typename Tree::key_type key_type; + typedef typename Tree::value_type value_type; + typedef typename Tree::size_type size_type; + typedef typename Tree::key_compare key_compare; + typedef typename Tree::allocator_type allocator_type; + typedef typename Tree::iterator iterator; + typedef typename Tree::const_iterator const_iterator; + + public: + // Default constructor. + btree_unique_container(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_unique_container(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_unique_container(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + insert(b, e); + } + + // Lookup routines. + iterator find(const key_type &key) { + return this->tree_.find_unique(key); + } + const_iterator find(const key_type &key) const { + return this->tree_.find_unique(key); + } + size_type count(const key_type &key) const { + return this->tree_.count_unique(key); + } + + // Insertion routines. + pair insert(const value_type &x) { + return this->tree_.insert_unique(x); + } + iterator insert(iterator position, const value_type &x) { + return this->tree_.insert_unique(position, x); + } + template + void insert(InputIterator b, InputIterator e) { + this->tree_.insert_unique(b, e); + } + + // Deletion routines. + int erase(const key_type &key) { + return this->tree_.erase_unique(key); + } + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + iterator erase(const iterator &iter) { + return this->tree_.erase(iter); + } + void erase(const iterator &first, const iterator &last) { + this->tree_.erase(first, last); + } +}; + +// A common base class for btree_map and safe_btree_map. +template +class btree_map_container : public btree_unique_container { + typedef btree_map_container self_type; + typedef btree_unique_container super_type; + + public: + typedef typename Tree::key_type key_type; + typedef typename Tree::data_type data_type; + typedef typename Tree::value_type value_type; + typedef typename Tree::mapped_type mapped_type; + typedef typename Tree::key_compare key_compare; + typedef typename Tree::allocator_type allocator_type; + + private: + // A pointer-like object which only generates its value when + // dereferenced. Used by operator[] to avoid constructing an empty data_type + // if the key already exists in the map. + struct generate_value { + generate_value(const key_type &k) + : key(k) { + } + value_type operator*() const { + return make_pair(key, data_type()); + } + const key_type &key; + }; + + public: + // Default constructor. + btree_map_container(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_map_container(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_map_container(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + insert(b, e); + } + + // Insertion routines. + data_type& operator[](const key_type &key) { + return this->tree_.insert_unique(key, generate_value(key)).first->second; + } +}; + +// A common base class for btree_multiset and btree_multimap. +template +class btree_multi_container : public btree_container { + typedef btree_multi_container self_type; + typedef btree_container super_type; + + public: + typedef typename Tree::key_type key_type; + typedef typename Tree::value_type value_type; + typedef typename Tree::size_type size_type; + typedef typename Tree::key_compare key_compare; + typedef typename Tree::allocator_type allocator_type; + typedef typename Tree::iterator iterator; + typedef typename Tree::const_iterator const_iterator; + + public: + // Default constructor. + btree_multi_container(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_multi_container(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_multi_container(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(b, e, comp, alloc) { + insert(b, e); + } + + // Lookup routines. + iterator find(const key_type &key) { + return this->tree_.find_multi(key); + } + const_iterator find(const key_type &key) const { + return this->tree_.find_multi(key); + } + size_type count(const key_type &key) const { + return this->tree_.count_multi(key); + } + + // Insertion routines. + iterator insert(const value_type &x) { + return this->tree_.insert_multi(x); + } + iterator insert(iterator position, const value_type &x) { + return this->tree_.insert_multi(position, x); + } + template + void insert(InputIterator b, InputIterator e) { + this->tree_.insert_multi(b, e); + } + + // Deletion routines. + int erase(const key_type &key) { + return this->tree_.erase_multi(key); + } + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + iterator erase(const iterator &iter) { + return this->tree_.erase(iter); + } + void erase(const iterator &first, const iterator &last) { + this->tree_.erase(first, last); + } +}; + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_BTREE_CONTAINER_H__ diff --git a/btree_map.h b/btree_map.h new file mode 100644 index 00000000..3c61a7f2 --- /dev/null +++ b/btree_map.h @@ -0,0 +1,122 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) +// +// A btree_map<> implements the STL unique sorted associative container +// interface and the pair associative container interface (a.k.a map<>) using a +// btree. A btree_multimap<> implements the STL multiple sorted associative +// container interface and the pair associtive container interface (a.k.a +// multimap<>) using a btree. See btree.h for details of the btree +// implementation and caveats. + +#ifndef UTIL_BTREE_BTREE_MAP_H__ +#define UTIL_BTREE_BTREE_MAP_H__ + +#include +#include +#include +#include +#include + +#include "util/btree/btree.h" // IWYU pragma: export +#include "util/btree/btree_container.h" // IWYU pragma: export + +namespace util { +namespace btree { + +// The btree_map class is needed mainly for it's constructors. +template , + typename Alloc = std::allocator >, + int TargetNodeSize = 256> +class btree_map : public btree_map_container< + btree > > { + + typedef btree_map self_type; + typedef btree_map_params< + Key, Value, Compare, Alloc, TargetNodeSize> params_type; + typedef btree btree_type; + typedef btree_map_container super_type; + + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + + public: + // Default constructor. + btree_map(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_map(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_map(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } +}; + +template +inline void swap(btree_map &x, + btree_map &y) { + x.swap(y); +} + +// The btree_multimap class is needed mainly for it's constructors. +template , + typename Alloc = std::allocator >, + int TargetNodeSize = 256> +class btree_multimap : public btree_multi_container< + btree > > { + + typedef btree_multimap self_type; + typedef btree_map_params< + Key, Value, Compare, Alloc, TargetNodeSize> params_type; + typedef btree btree_type; + typedef btree_multi_container super_type; + + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + typedef typename btree_type::data_type data_type; + typedef typename btree_type::mapped_type mapped_type; + + public: + // Default constructor. + btree_multimap(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_multimap(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_multimap(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(b, e, comp, alloc) { + } +}; + +template +inline void swap(btree_multimap &x, + btree_multimap &y) { + x.swap(y); +} + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_BTREE_MAP_H__ diff --git a/btree_nc.cc b/btree_nc.cc new file mode 100644 index 00000000..b24c312d --- /dev/null +++ b/btree_nc.cc @@ -0,0 +1,50 @@ +// Copyright 2009 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) + +#include "util/btree/btree_set.h" + +namespace { + +template +struct Compare { + R operator()(int a, int b) const { return reinterpret_cast(a < b); } +}; + +template +struct CompareTo : public util::btree::btree_key_compare_to_tag { + R operator()(int a, int b) const { return reinterpret_cast(a < b); } +}; + +#define TEST_COMPARE(r) \ + void TestCompile() { \ + util::btree::btree_set > s; \ + } + +#define TEST_COMPARE_TO(r) \ + void TestCompile() { \ + util::btree::btree_set > s; \ + } + +#if defined(TEST_bool) +TEST_COMPARE(bool); +#elif defined(TEST_int) +TEST_COMPARE(int); +#elif defined(TEST_float) +TEST_COMPARE(float); +#elif defined(TEST_pointer) +TEST_COMPARE(void*); +#elif defined(TEST_compare_to_bool) +TEST_COMPARE_TO(bool); +#elif defined(TEST_compare_to_int) +TEST_COMPARE_TO(int); +#elif defined(TEST_compare_to_float) +TEST_COMPARE_TO(float); +#elif defined(TEST_compare_to_pointer) +TEST_COMPARE_TO(pointer); +#endif + +} // namespace + +int main() { + return 1; +} diff --git a/btree_nc_test.py b/btree_nc_test.py new file mode 100755 index 00000000..056a75d2 --- /dev/null +++ b/btree_nc_test.py @@ -0,0 +1,83 @@ +#!/usr/bin/python2.4 +# +# Copyright 2006 Google Inc. All Rights Reserved. + +"""Negative compilation unit test for btree.h. +""" + +__author__ = 'pmattis@google.com (Peter Mattis)' + +import os +from google3.testing.pybase import googletest +from google3.testing.pybase import fake_target_util +from google3.pyglib import flags + +_FLAGS = flags.FLAGS + + +class BtreeNegativeUnitTest(googletest.TestCase): + """Negative compilation tests for btree.h""" + + def testCompilerErrors(self): + """Runs a list of tests to verify that erroneous code leads to + expected compiler messages.""" + + # Defines a list of test specs, where each element is a tuple + # (test name, list of regexes for matching the compiler errors). + test_specs = [ + # Test that bool works as a return type for key comparison. + ('bool', None), # None means compilation should succeed. + + # Test that int does not work as a return type for key comparison. + ('int', + [r'error: creating array with negative size', # for gcc + r'', # for icc + ]), + + # Test that float does not work as a return type for key comparison. + ('float', + [r'error: creating array with negative size', # for gcc + r'', # for icc + ]), + + # Test that void* does not work as a return type for key comparison. + ('pointer', + [r'error: creating array with negative size', # for gcc + r'', # for icc + ]), + + # Test that bool does not work as a return type for compare-to + # comparison. + ('compare_to_bool', + [r'error: creating array with negative size', # for gcc + r'', # for icc + ]), + + # Test that int works as a return type for compare-to comparison. + ('compare_to_int', None), # None means compilation should succeed. + + # Test that float does not work as a return type for compare-to + # comparison. + ('compare_to_float', + [r'error: creating array with negative size', # for gcc + r'', # for icc + ]), + + # Test that void* does not work as a return type for compare-to + # comparison. + ('compare_to_pointer', + [r'error: creating array with negative size', # for gcc + r'', # for icc + ]), + ] + + # Runs the list of tests. + fake_target_util.AssertCcCompilerErrors( + self, + 'google3/util/btree/btree_nc', # path to the fake target file. + 'btree_nc.o', # name of the target to build. + test_specs) + + +if __name__ == '__main__': + googletest.main() diff --git a/btree_printer.py b/btree_printer.py new file mode 100755 index 00000000..8f233401 --- /dev/null +++ b/btree_printer.py @@ -0,0 +1,125 @@ +#!/usr/bin/python2.4 +# Copyright 2011 Google Inc. All Rights Reserved. +# GDB support for pretty printing StringPiece. + +"""GDB pretty-printer for btrees.""" + +# This is a module provided by GDB. +# Ref: http://wiki/Main/Google3GDBScripts +import gdb +from google3.devtools.gdb.component import printing + + +class BaseBtreePrinter(object): + """btree pretty-printer, for util/btree.""" + + def __init__(self, typename, val): + self.typename = typename + self.val = val + + def display_hint(self): + return 'array' + + def _my_iter(self, node): + count = node['fields_']['count'] + if node['fields_']['leaf']: + for i in range(count): + key = node['fields_']['values'][i] + yield ('[item]', key) + else: + # recursive generators are annoying: we can't just recurse, we need + # to expand and yield the values. + for i in range(count+1): + child = node['fields_']['children'][i] + for v in self._my_iter(child.dereference()): + yield v + + def children(self): + if self.nelements() != 0: + return self._my_iter(self.val['root_']['data'].dereference()) + else: + return iter([]) + + def nelements(self): + if self.val['root_']['data'] != 0: + root_fields = self.val['root_']['data'].dereference()['fields_'] + if root_fields['leaf']: + return root_fields['count'] + else: + return root_fields['size'] + else: + return 0 + + def to_string(self): # pylint: disable-msg=C6409 + """GDB calls this to compute the pretty-printed form.""" + return '%s with %d elements' % (self.typename, self.nelements()) + + +class BtreePrinter(BaseBtreePrinter): + """btree<> pretty-printer, for util/btree.""" + + def __init__(self, val): + BaseBtreePrinter.__init__(self, 'btree', val) + + +class BtreeSetPrinter(BaseBtreePrinter): + """btree_set<> pretty-printer.""" + + def __init__(self, val): + BaseBtreePrinter.__init__(self, 'btree_set', val['tree_']) + + +class BtreeMultisetPrinter(BaseBtreePrinter): + """btree_multiset<> pretty-printer.""" + + def __init__(self, val): + BaseBtreePrinter.__init__(self, 'btree_multiset', val['tree_']) + + +class BaseBtreeMapPrinter(BaseBtreePrinter): + """btree maps pretty-printer.""" + + def __init__(self, typename, val): + BaseBtreePrinter.__init__(self, typename, val['tree_']) + + def display_hint(self): + return 'map' + + def _my_map_iter(self, g): + for (_, pair) in g: + yield ('[key]', pair['first']) + yield ('[value]', pair['second']) + + def children(self): + # we need to break apart the pairs and yield them separately + if self.nelements() != 0: + return self._my_map_iter(BaseBtreePrinter.children(self)) + else: + return iter([]) + + +class BtreeMapPrinter(BaseBtreeMapPrinter): + """btree_map<> pretty-printer.""" + + def __init__(self, val): + BaseBtreeMapPrinter.__init__(self, 'btree_map', val) + + +class BtreeMultimapPrinter(BaseBtreeMapPrinter): + """btree_multimap<> pretty-printer.""" + + def __init__(self, val): + BaseBtreeMapPrinter.__init__(self, 'btree_multimap', val) + + +if __name__ == '__main__': + printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree<.*>', + BtreePrinter) + printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_set<.*>', + BtreeSetPrinter) + printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_multiset<.*>', + BtreeMultisetPrinter) + printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_map<.*>', + BtreeMapPrinter) + printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_multimap<.*>', + BtreeMultimapPrinter) diff --git a/btree_printer_test.py b/btree_printer_test.py new file mode 100755 index 00000000..95bc351e --- /dev/null +++ b/btree_printer_test.py @@ -0,0 +1,92 @@ +#!/usr/bin/python2.4 +# +# Copyright 2011 Google Inc. All Rights Reserved. + +"""Tests for btree_printer.py gdb pretty printer.""" + +__author__ = "leg@google.com (Lawrence Greenfield)" + +from google3.pyglib import flags +from google3.testing.gdb import gdb_script_test_util +from google3.testing.pybase import googletest + +FLAGS = flags.FLAGS + + +class BtreePrinterTest(gdb_script_test_util.TestCase): + def testBtreeSet(self): + self.InitSession("btree_set", + "util/btree/btree_test_program") + self.RunTo("StopHereForDebugger") + + self.SetOption("print elements", 20) + self.TestPrintOutputMatches("*empty_set", + """btree_set with 0 elements""") + self.TestPrintOutputMatches("*small_set", + """btree_set with 10 elements = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}""") + self.TestPrintOutputMatches("*small_multiset", + """btree_multiset with 10 elements = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4}""") + self.TestPrintOutputMatches("*big_set", + """btree_set with 80 elements = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19...}""") + + self.RunSession() + + def testBtreeMap(self): + self.InitSession("btree_set", + "util/btree/btree_test_program") + self.RunTo("StopHereForDebugger") + + self.SetOption("print elements", 30) + self.TestPrintOutputMatches("*empty_map", + """btree_map with 0 elements""") + self.TestPrintOutputMatches("*small_map", + """btree_map with 10 elements = { + \\[0\\] = 0, + \\[1\\] = 13, + \\[2\\] = 26, + \\[3\\] = 39, + \\[4\\] = 52, + \\[5\\] = 65, + \\[6\\] = 78, + \\[7\\] = 91, + \\[8\\] = 104, + \\[9\\] = 117 +}""") + self.TestPrintOutputMatches("*small_multimap", + """btree_multimap with 10 elements = { + \\[0\\] = 0, + \\[0\\] = 1, + \\[1\\] = 2, + \\[1\\] = 3, + \\[2\\] = 4, + \\[2\\] = 5, + \\[3\\] = 6, + \\[3\\] = 7, + \\[4\\] = 8, + \\[4\\] = 9 +}""") + self.TestPrintOutputMatches("*big_map", + """btree_map with 80 elements = { + \\[0\\] = 0, + \\[1\\] = 7, + \\[2\\] = 14, + \\[3\\] = 21, + \\[4\\] = 28, + \\[5\\] = 35, + \\[6\\] = 42, + \\[7\\] = 49, + \\[8\\] = 56, + \\[9\\] = 63, + \\[10\\] = 70, + \\[11\\] = 77, + \\[12\\] = 84, + \\[13\\] = 91, + \\[14\\] = 98 + ... +}""") + + self.RunSession() + + +if __name__ == "__main__": + googletest.main() diff --git a/btree_set.h b/btree_set.h new file mode 100644 index 00000000..7b5ea88d --- /dev/null +++ b/btree_set.h @@ -0,0 +1,113 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) +// +// A btree_set<> implements the STL unique sorted associative container +// interface (a.k.a set<>) using a btree. A btree_multiset<> implements the STL +// multiple sorted associative container interface (a.k.a multiset<>) using a +// btree. See btree.h for details of the btree implementation and caveats. + +#ifndef UTIL_BTREE_BTREE_SET_H__ +#define UTIL_BTREE_BTREE_SET_H__ + +#include +#include +#include + +#include "util/btree/btree.h" // IWYU pragma: export +#include "util/btree/btree_container.h" // IWYU pragma: export + +namespace util { +namespace btree { + +// The btree_set class is needed mainly for it's constructors. +template , + typename Alloc = std::allocator, + int TargetNodeSize = 256> +class btree_set : public btree_unique_container< + btree > > { + + typedef btree_set self_type; + typedef btree_set_params params_type; + typedef btree btree_type; + typedef btree_unique_container super_type; + + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + + public: + // Default constructor. + btree_set(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_set(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_set(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(b, e, comp, alloc) { + } +}; + +template +inline void swap(btree_set &x, btree_set &y) { + x.swap(y); +} + +// The btree_multiset class is needed mainly for it's constructors. +template , + typename Alloc = std::allocator, + int TargetNodeSize = 256> +class btree_multiset : public btree_multi_container< + btree > > { + + typedef btree_multiset self_type; + typedef btree_set_params params_type; + typedef btree btree_type; + typedef btree_multi_container super_type; + + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + + public: + // Default constructor. + btree_multiset(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_multiset(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_multiset(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(b, e, comp, alloc) { + } +}; + +template +inline void swap(btree_multiset &x, + btree_multiset &y) { + x.swap(y); +} + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_BTREE_SET_H__ diff --git a/btree_test.cc b/btree_test.cc new file mode 100644 index 00000000..ef4e574e --- /dev/null +++ b/btree_test.cc @@ -0,0 +1,167 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) + +#include "base/arena-inl.h" +#include "base/init_google.h" +#include "base/integral_types.h" +#include "base/logging.h" +#include "strings/stringpiece.h" +#include "testing/base/public/gunit.h" +#include "util/btree/btree_map.h" +#include "util/btree/btree_set.h" +#include "util/btree/btree_test.h" + +namespace util { +namespace btree { +namespace { + +template +void SetTest() { + typedef ArenaAllocator ArenaAlloc; + CHECK_EQ(sizeof(btree_set), sizeof(void*)); + BtreeTest, allocator, N>, set >(); + BtreeArenaTest, ArenaAlloc, N> >(); +} + +template +void MapTest() { + typedef ArenaAllocator ArenaAlloc; + CHECK_EQ(sizeof(btree_map), sizeof(void*)); + BtreeTest, allocator, N>, map >(); + BtreeArenaTest, ArenaAlloc, N> >(); + BtreeMapTest, allocator, N> >(); +} + +TEST(Btree, set_int32_32) { SetTest(); } +TEST(Btree, set_int32_64) { SetTest(); } +TEST(Btree, set_int32_128) { SetTest(); } +TEST(Btree, set_int32_256) { SetTest(); } +TEST(Btree, set_int64_256) { SetTest(); } +TEST(Btree, set_string_256) { SetTest(); } +TEST(Btree, set_cord_256) { SetTest(); } +TEST(Btree, set_pair_256) { SetTest, 256>(); } +TEST(Btree, map_int32_256) { MapTest(); } +TEST(Btree, map_int64_256) { MapTest(); } +TEST(Btree, map_string_256) { MapTest(); } +TEST(Btree, map_cord_256) { MapTest(); } +TEST(Btree, map_pair_256) { MapTest, 256>(); } + +template +void MultiSetTest() { + typedef ArenaAllocator ArenaAlloc; + CHECK_EQ(sizeof(btree_multiset), sizeof(void*)); + BtreeMultiTest, allocator, N>, + multiset >(); + BtreeArenaTest, ArenaAlloc, N> >(); +} + +template +void MultiMapTest() { + typedef ArenaAllocator ArenaAlloc; + CHECK_EQ(sizeof(btree_multimap), sizeof(void*)); + BtreeMultiTest, allocator, N>, + multimap >(); + BtreeMultiMapTest, allocator, N> >(); + BtreeArenaTest, ArenaAlloc, N> >(); +} + +TEST(Btree, multiset_int32_256) { MultiSetTest(); } +TEST(Btree, multiset_int64_256) { MultiSetTest(); } +TEST(Btree, multiset_string_256) { MultiSetTest(); } +TEST(Btree, multiset_cord_256) { MultiSetTest(); } +TEST(Btree, multiset_pair_256) { MultiSetTest, 256>(); } +TEST(Btree, multimap_int32_256) { MultiMapTest(); } +TEST(Btree, multimap_int64_256) { MultiMapTest(); } +TEST(Btree, multimap_string_256) { MultiMapTest(); } +TEST(Btree, multimap_cord_256) { MultiMapTest(); } +TEST(Btree, multimap_pair_256) { MultiMapTest, 256>(); } + +// Verify that swapping btrees swaps the key comparision functors. +struct SubstringLess { + SubstringLess() : n(2) {} + SubstringLess(int length) + : n(length) { + } + bool operator()(const string &a, const string &b) const { + return StringPiece(a).substr(0, n) < StringPiece(b).substr(0, n); + } + int n; +}; + +TEST(Btree, SwapKeyCompare) { + typedef btree_set SubstringSet; + SubstringSet s1(SubstringLess(1), SubstringSet::allocator_type()); + SubstringSet s2(SubstringLess(2), SubstringSet::allocator_type()); + + ASSERT_TRUE(s1.insert("a").second); + ASSERT_FALSE(s1.insert("aa").second); + + ASSERT_TRUE(s2.insert("a").second); + ASSERT_TRUE(s2.insert("aa").second); + ASSERT_FALSE(s2.insert("aaa").second); + + swap(s1, s2); + + ASSERT_TRUE(s1.insert("b").second); + ASSERT_TRUE(s1.insert("bb").second); + ASSERT_FALSE(s1.insert("bbb").second); + + ASSERT_TRUE(s2.insert("b").second); + ASSERT_FALSE(s2.insert("bb").second); +} + +TEST(Btree, UpperBoundRegression) { + // Regress a bug where upper_bound would default-construct a new key_compare + // instead of copying the existing one. + typedef btree_set SubstringSet; + SubstringSet my_set(SubstringLess(3)); + my_set.insert("aab"); + my_set.insert("abb"); + // We call upper_bound("aaa"). If this correctly uses the length 3 + // comparator, aaa < aab < abb, so we should get aab as the result. + // If it instead uses the default-constructed length 2 comparator, + // aa == aa < ab, so we'll get abb as our result. + SubstringSet::iterator it = my_set.upper_bound("aaa"); + ASSERT_TRUE(it != my_set.end()); + EXPECT_EQ("aab", *it); +} + + +TEST(Btree, IteratorIncrementBy) { + // Test that increment_by returns the same position as increment. + const int kSetSize = 2341; + btree_set my_set; + for (int i = 0; i < kSetSize; ++i) { + my_set.insert(i); + } + + { + // Simple increment vs. increment by. + btree_set::iterator a = my_set.begin(); + btree_set::iterator b = my_set.begin(); + a.increment(); + b.increment_by(1); + EXPECT_EQ(*a, *b); + } + + btree_set::iterator a = my_set.begin(); + for (int i = 1; i < kSetSize; ++i) { + ++a; + // increment_by + btree_set::iterator b = my_set.begin(); + b.increment_by(i); + EXPECT_EQ(*a, *b) << ": i=" << i; + } +} + + +} // namespace +} // namespace btree +} // namespace util + +int main(int argc, char **argv) { + FLAGS_logtostderr = true; + InitGoogle(argv[0], &argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/btree_test.h b/btree_test.h new file mode 100644 index 00000000..557c1e54 --- /dev/null +++ b/btree_test.h @@ -0,0 +1,930 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) + +#ifndef UTIL_BTREE_BTREE_TEST_H__ +#define UTIL_BTREE_BTREE_TEST_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "base/arena.h" +#include "base/commandlineflags.h" +#include "base/logging.h" +#include "base/type_traits.h" +#include "strings/cord.h" +#include "strings/util.h" +#include "testing/base/public/googletest.h" +#include "util/btree/btree_container.h" +#include "util/random/acmrandom.h" + +DECLARE_int32(test_values); +DECLARE_int32(benchmark_values); + +namespace std { + +// Provide operator<< support for pair. +template +ostream& operator<<(ostream &os, const pair &p) { + os << "(" << p.first << "," << p.second << ")"; + return os; +} + +// Provide pair equality testing that works as long as x.first is comparable to +// y.first and x.second is comparable to y.second. Needed in the test for +// comparing pair to pair. +template +bool operator==(const pair &x, const pair &y) { + return x.first == y.first && x.second == y.second; +} + +} // namespace std + +namespace base { + +// Partial specialization of remove_const that propagates the removal through +// std::pair. +template +struct remove_const > { + typedef std::pair::type, + typename remove_const::type> type; +}; + +} // namespace base + +namespace util { +namespace btree { + +// Utility class to provide an accessor for a key given a value. The default +// behavior is to treat the value as a pair and return the first element. +template +struct KeyOfValue { + typedef select1st type; +}; + +// Partial specialization of KeyOfValue class for when the key and value are +// the same type such as in set<> and btree_set<>. +template +struct KeyOfValue { + typedef identity type; +}; + +// The base class for a sorted associative container checker. TreeType is the +// container type to check and CheckerType is the container type to check +// against. TreeType is expected to be btree_{set,map,multiset,multimap} and +// CheckerType is expected to be {set,map,multiset,multimap}. +template +class base_checker { + typedef base_checker self_type; + + public: + typedef typename TreeType::key_type key_type; + typedef typename TreeType::value_type value_type; + typedef typename TreeType::key_compare key_compare; + typedef typename TreeType::pointer pointer; + typedef typename TreeType::const_pointer const_pointer; + typedef typename TreeType::reference reference; + typedef typename TreeType::const_reference const_reference; + typedef typename TreeType::size_type size_type; + typedef typename TreeType::difference_type difference_type; + typedef typename TreeType::iterator iterator; + typedef typename TreeType::const_iterator const_iterator; + typedef typename TreeType::reverse_iterator reverse_iterator; + typedef typename TreeType::const_reverse_iterator const_reverse_iterator; + + public: + // Default constructor. + base_checker() + : const_tree_(tree_) { + } + // Copy constructor. + base_checker(const self_type &x) + : tree_(x.tree_), + const_tree_(tree_), + checker_(x.checker_) { + } + + // Iterator routines. + iterator begin() { return tree_.begin(); } + const_iterator begin() const { return tree_.begin(); } + iterator end() { return tree_.end(); } + const_iterator end() const { return tree_.end(); } + reverse_iterator rbegin() { return tree_.rbegin(); } + const_reverse_iterator rbegin() const { return tree_.rbegin(); } + reverse_iterator rend() { return tree_.rend(); } + const_reverse_iterator rend() const { return tree_.rend(); } + + // Helper routines. + template + IterType iter_check( + IterType tree_iter, CheckerIterType checker_iter) const { + if (tree_iter == tree_.end()) { + CHECK(checker_iter == checker_.end()); + } else { + CHECK_EQ(*tree_iter, *checker_iter); + } + return tree_iter; + } + template + IterType riter_check( + IterType tree_iter, CheckerIterType checker_iter) const { + if (tree_iter == tree_.rend()) { + CHECK(checker_iter == checker_.rend()); + } else { + CHECK_EQ(*tree_iter, *checker_iter); + } + return tree_iter; + } + void value_check(const value_type &x) { + typename KeyOfValue::type key_of_value; + const key_type &key = key_of_value(x); + CHECK_EQ(*find(key), x); + lower_bound(key); + upper_bound(key); + equal_range(key); + count(key); + } + void erase_check(const key_type &key) { + CHECK(tree_.find(key) == const_tree_.end()); + CHECK(const_tree_.find(key) == tree_.end()); + CHECK(tree_.equal_range(key).first == + const_tree_.equal_range(key).second); + } + + // Lookup routines. + iterator lower_bound(const key_type &key) { + return iter_check(tree_.lower_bound(key), checker_.lower_bound(key)); + } + const_iterator lower_bound(const key_type &key) const { + return iter_check(tree_.lower_bound(key), checker_.lower_bound(key)); + } + iterator upper_bound(const key_type &key) { + return iter_check(tree_.upper_bound(key), checker_.upper_bound(key)); + } + const_iterator upper_bound(const key_type &key) const { + return iter_check(tree_.upper_bound(key), checker_.upper_bound(key)); + } + pair equal_range(const key_type &key) { + pair checker_res = + checker_.equal_range(key); + pair tree_res = tree_.equal_range(key); + iter_check(tree_res.first, checker_res.first); + iter_check(tree_res.second, checker_res.second); + return tree_res; + } + pair equal_range(const key_type &key) const { + pair checker_res = + checker_.equal_range(key); + pair tree_res = tree_.equal_range(key); + iter_check(tree_res.first, checker_res.first); + iter_check(tree_res.second, checker_res.second); + return tree_res; + } + iterator find(const key_type &key) { + return iter_check(tree_.find(key), checker_.find(key)); + } + const_iterator find(const key_type &key) const { + return iter_check(tree_.find(key), checker_.find(key)); + } + size_type count(const key_type &key) const { + size_type res = checker_.count(key); + CHECK_EQ(res, tree_.count(key)); + return res; + } + + // Assignment operator. + self_type& operator=(const self_type &x) { + tree_ = x.tree_; + checker_ = x.checker_; + return *this; + } + + // Deletion routines. + int erase(const key_type &key) { + int size = tree_.size(); + int res = checker_.erase(key); + CHECK_EQ(res, tree_.count(key)); + CHECK_EQ(res, tree_.erase(key)); + CHECK_EQ(tree_.count(key), 0); + CHECK_EQ(tree_.size(), size - res); + erase_check(key); + return res; + } + iterator erase(iterator iter) { + key_type key = iter.key(); + int size = tree_.size(); + int count = tree_.count(key); + typename CheckerType::iterator checker_iter = checker_.find(key); + for (iterator tmp(tree_.find(key)); tmp != iter; ++tmp) { + ++checker_iter; + } + typename CheckerType::iterator checker_next = checker_iter; + ++checker_next; + checker_.erase(checker_iter); + iter = tree_.erase(iter); + CHECK_EQ(tree_.size(), checker_.size()); + CHECK_EQ(tree_.size(), size - 1); + CHECK_EQ(tree_.count(key), count - 1); + if (count == 1) { + erase_check(key); + } + return iter_check(iter, checker_next); + } + + void erase(iterator begin, iterator end) { + int size = tree_.size(); + int count = distance(begin, end); + typename CheckerType::iterator checker_begin = checker_.find(begin.key()); + for (iterator tmp(tree_.find(begin.key())); tmp != begin; ++tmp) { + ++checker_begin; + } + typename CheckerType::iterator checker_end = + end == tree_.end() ? checker_.end() : checker_.find(end.key()); + if (end != tree_.end()) { + for (iterator tmp(tree_.find(end.key())); tmp != end; ++tmp) { + ++checker_end; + } + } + checker_.erase(checker_begin, checker_end); + tree_.erase(begin, end); + CHECK_EQ(tree_.size(), checker_.size()); + CHECK_EQ(tree_.size(), size - count); + } + + // Utility routines. + void clear() { + tree_.clear(); + checker_.clear(); + } + void swap(self_type &x) { + tree_.swap(x.tree_); + checker_.swap(x.checker_); + } + + void verify() const { + tree_.verify(); + CHECK_EQ(tree_.size(), checker_.size()); + + // Move through the forward iterators using increment. + typename CheckerType::const_iterator + checker_iter(checker_.begin()); + const_iterator tree_iter(tree_.begin()); + for (; tree_iter != tree_.end(); + ++tree_iter, ++checker_iter) { + CHECK_EQ(*tree_iter, *checker_iter); + } + + // Move through the forward iterators using decrement. + for (int n = tree_.size() - 1; n >= 0; --n) { + iter_check(tree_iter, checker_iter); + --tree_iter; + --checker_iter; + } + CHECK(tree_iter == tree_.begin()); + CHECK(checker_iter == checker_.begin()); + + // Move through the reverse iterators using increment. + typename CheckerType::const_reverse_iterator + checker_riter(checker_.rbegin()); + const_reverse_iterator tree_riter(tree_.rbegin()); + for (; tree_riter != tree_.rend(); + ++tree_riter, ++checker_riter) { + CHECK_EQ(*tree_riter, *checker_riter); + } + + // Move through the reverse iterators using decrement. + for (int n = tree_.size() - 1; n >= 0; --n) { + riter_check(tree_riter, checker_riter); + --tree_riter; + --checker_riter; + } + CHECK(tree_riter == tree_.rbegin()); + CHECK(checker_riter == checker_.rbegin()); + } + + // Access to the underlying btree. + const TreeType& tree() const { return tree_; } + + // Size routines. + size_type size() const { + CHECK_EQ(tree_.size(), checker_.size()); + return tree_.size(); + } + size_type max_size() const { return tree_.max_size(); } + bool empty() const { + CHECK_EQ(tree_.empty(), checker_.empty()); + return tree_.empty(); + } + size_type height() const { return tree_.height(); } + size_type internal_nodes() const { return tree_.internal_nodes(); } + size_type leaf_nodes() const { return tree_.leaf_nodes(); } + size_type nodes() const { return tree_.nodes(); } + size_type bytes_used() const { return tree_.bytes_used(); } + double fullness() const { return tree_.fullness(); } + double overhead() const { return tree_.overhead(); } + + protected: + TreeType tree_; + const TreeType &const_tree_; + CheckerType checker_; +}; + +// A checker for unique sorted associative containers. TreeType is expected to +// be btree_{set,map} and CheckerType is expected to be {set,map}. +template +class unique_checker : public base_checker { + typedef base_checker super_type; + typedef unique_checker self_type; + + public: + typedef typename super_type::iterator iterator; + typedef typename super_type::value_type value_type; + + public: + // Default constructor. + unique_checker() + : super_type() { + } + // Copy constructor. + unique_checker(const self_type &x) + : super_type(x) { + } + // Range constructor. + template + unique_checker(InputIterator b, InputIterator e) { + insert(b, e); + } + + // Insertion routines. + pair insert(const value_type &x) { + int size = this->tree_.size(); + pair checker_res = + this->checker_.insert(x); + pair tree_res = this->tree_.insert(x); + CHECK_EQ(*tree_res.first, *checker_res.first); + CHECK_EQ(tree_res.second, checker_res.second); + CHECK_EQ(this->tree_.size(), this->checker_.size()); + CHECK_EQ(this->tree_.size(), size + tree_res.second); + return tree_res; + } + iterator insert(iterator position, const value_type &x) { + int size = this->tree_.size(); + pair checker_res = + this->checker_.insert(x); + iterator tree_res = this->tree_.insert(position, x); + CHECK_EQ(*tree_res, *checker_res.first); + CHECK_EQ(this->tree_.size(), this->checker_.size()); + CHECK_EQ(this->tree_.size(), size + checker_res.second); + return tree_res; + } + template + void insert(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert(*b); + } + } +}; + +// A checker for multiple sorted associative containers. TreeType is expected +// to be btree_{multiset,multimap} and CheckerType is expected to be +// {multiset,multimap}. +template +class multi_checker : public base_checker { + typedef base_checker super_type; + typedef multi_checker self_type; + + public: + typedef typename super_type::iterator iterator; + typedef typename super_type::value_type value_type; + + public: + // Default constructor. + multi_checker() + : super_type() { + } + // Copy constructor. + multi_checker(const self_type &x) + : super_type(x) { + } + // Range constructor. + template + multi_checker(InputIterator b, InputIterator e) { + insert(b, e); + } + + // Insertion routines. + iterator insert(const value_type &x) { + int size = this->tree_.size(); + typename CheckerType::iterator checker_res = this->checker_.insert(x); + iterator tree_res = this->tree_.insert(x); + CHECK_EQ(*tree_res, *checker_res); + CHECK_EQ(this->tree_.size(), this->checker_.size()); + CHECK_EQ(this->tree_.size(), size + 1); + return tree_res; + } + iterator insert(iterator position, const value_type &x) { + int size = this->tree_.size(); + typename CheckerType::iterator checker_res = this->checker_.insert(x); + iterator tree_res = this->tree_.insert(position, x); + CHECK_EQ(*tree_res, *checker_res); + CHECK_EQ(this->tree_.size(), this->checker_.size()); + CHECK_EQ(this->tree_.size(), size + 1); + return tree_res; + } + template + void insert(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert(*b); + } + } +}; + +char* GenerateDigits(char buf[16], int val, int maxval) { + DCHECK_LE(val, maxval); + int p = 15; + buf[p--] = 0; + while (maxval > 0) { + buf[p--] = '0' + (val % 10); + val /= 10; + maxval /= 10; + } + return buf + p + 1; +} + +template +struct Generator { + int maxval; + Generator(int m) + : maxval(m) { + } + K operator()(int i) const { + DCHECK_LE(i, maxval); + return i; + } +}; + +template <> +struct Generator { + int maxval; + Generator(int m) + : maxval(m) { + } + string operator()(int i) const { + char buf[16]; + return GenerateDigits(buf, i, maxval); + } +}; + +template <> +struct Generator { + int maxval; + Generator(int m) + : maxval(m) { + } + Cord operator()(int i) const { + char buf[16]; + return Cord(GenerateDigits(buf, i, maxval)); + } +}; + +template +struct Generator > { + Generator::type> tgen; + Generator::type> ugen; + + Generator(int m) + : tgen(m), + ugen(m) { + } + pair operator()(int i) const { + return make_pair(tgen(i), ugen(i)); + } +}; + +// Generate values for our tests and benchmarks. Value range is [0, maxval]. +const vector& GenerateNumbers(int n, int maxval) { + static ACMRandom rand(FLAGS_test_random_seed); + static vector values; + static set unique_values; + + if (values.size() < n) { + + for (int i = values.size(); i < n; i++) { + int value; + do { + value = rand.Next() % (maxval + 1); + } while (unique_values.find(value) != unique_values.end()); + + values.push_back(value); + unique_values.insert(value); + } + } + + return values; +} + +// Generates values in the range +// [0, 4 * min(FLAGS_benchmark_values, FLAGS_test_values)] +template +vector GenerateValues(int n) { + int two_times_max = 2 * max(FLAGS_benchmark_values, FLAGS_test_values); + int four_times_max = 2 * two_times_max; + DCHECK_LE(n, two_times_max); + const vector &nums = GenerateNumbers(n, four_times_max); + Generator gen(four_times_max); + vector vec; + + for (int i = 0; i < n; i++) { + vec.push_back(gen(nums[i])); + } + + return vec; +} + +template +double ContainerInfo(const set &s) { + int sizeof_node = sizeof(std::_Rb_tree_node); + int bytes_used = sizeof(s) + s.size() * sizeof_node; + double bytes_per_value = (double) bytes_used / s.size(); + VLOG(1) << " size=" << s.size() + << " bytes-used=" << bytes_used + << " bytes-per-value=" << bytes_per_value; + return bytes_per_value; +} + +template +double ContainerInfo(const multiset &s) { + int sizeof_node = sizeof(std::_Rb_tree_node); + int bytes_used = sizeof(s) + s.size() * sizeof_node; + double bytes_per_value = (double) bytes_used / s.size(); + VLOG(1) << " size=" << s.size() + << " bytes-used=" << bytes_used + << " bytes-per-value=" << bytes_per_value; + return bytes_per_value; +} + +template +double ContainerInfo(const map &m) { + int sizeof_node = sizeof(std::_Rb_tree_node >); + int bytes_used = sizeof(m) + m.size() * sizeof_node; + double bytes_per_value = (double) bytes_used / m.size(); + VLOG(1) << " size=" << m.size() + << " bytes-used=" << bytes_used + << " bytes-per-value=" << bytes_per_value; + return bytes_per_value; +} + +template +double ContainerInfo(const multimap &m) { + int sizeof_node = sizeof(std::_Rb_tree_node >); + int bytes_used = sizeof(m) + m.size() * sizeof_node; + double bytes_per_value = (double) bytes_used / m.size(); + VLOG(1) << " size=" << m.size() + << " bytes-used=" << bytes_used + << " bytes-per-value=" << bytes_per_value; + return bytes_per_value; +} + +template +double ContainerInfo(const btree_container

&b) { + double bytes_used = sizeof(b) + b.bytes_used(); + double bytes_per_value = (double) bytes_used / b.size(); + VLOG(1) << " size=" << b.size() + << " bytes-used=" << bytes_used + << " bytes-per-value=" << bytes_per_value + << " height=" << b.height() + << " internal-nodes=" << b.internal_nodes() + << " leaf-nodes=" << b.leaf_nodes() + << " fullness=" << b.fullness() + << " overhead=" << b.overhead(); + return bytes_per_value; +} + +template +void DoTest(const char *name, T *b, const vector &values) { + typename KeyOfValue::type key_of_value; + + T &mutable_b = *b; + const T &const_b = *b; + + // Test insert. + for (int i = 0; i < values.size(); ++i) { + mutable_b.insert(values[i]); + mutable_b.value_check(values[i]); + } + + const_b.verify(); + printf(" %s fullness=%0.2f overhead=%0.2f bytes-per-value=%0.2f\n", + name, const_b.fullness(), const_b.overhead(), + double(const_b.bytes_used()) / const_b.size()); + + // Test copy constructor. + T b_copy(const_b); + CHECK_EQ(b_copy.size(), const_b.size()); + CHECK_LE(b_copy.height(), const_b.height()); + CHECK_LE(b_copy.internal_nodes(), const_b.internal_nodes()); + CHECK_LE(b_copy.leaf_nodes(), const_b.leaf_nodes()); + for (int i = 0; i < values.size(); ++i) { + CHECK_EQ(*b_copy.find(key_of_value(values[i])), values[i]); + } + + // Test range constructor. + T b_range(const_b.begin(), const_b.end()); + CHECK_EQ(b_range.size(), const_b.size()); + CHECK_LE(b_range.height(), const_b.height()); + CHECK_LE(b_range.internal_nodes(), const_b.internal_nodes()); + CHECK_LE(b_range.leaf_nodes(), const_b.leaf_nodes()); + for (int i = 0; i < values.size(); ++i) { + CHECK_EQ(*b_range.find(key_of_value(values[i])), values[i]); + } + + // Test range insertion for values that already exist. + b_range.insert(b_copy.begin(), b_copy.end()); + b_range.verify(); + + // Test range insertion for new values. + b_range.clear(); + b_range.insert(b_copy.begin(), b_copy.end()); + CHECK_EQ(b_range.size(), b_copy.size()); + CHECK_EQ(b_range.height(), b_copy.height()); + CHECK_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); + CHECK_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); + for (int i = 0; i < values.size(); ++i) { + CHECK_EQ(*b_range.find(key_of_value(values[i])), values[i]); + } + + // Test assignment to self. Nothing should change. + b_range.operator=(b_range); + CHECK_EQ(b_range.size(), b_copy.size()); + CHECK_EQ(b_range.height(), b_copy.height()); + CHECK_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); + CHECK_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); + + // Test assignment of new values. + b_range.clear(); + b_range = b_copy; + CHECK_EQ(b_range.size(), b_copy.size()); + CHECK_EQ(b_range.height(), b_copy.height()); + CHECK_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); + CHECK_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); + + // Test swap. + b_range.clear(); + b_range.swap(b_copy); + CHECK_EQ(b_copy.size(), 0); + CHECK_EQ(b_range.size(), const_b.size()); + for (int i = 0; i < values.size(); ++i) { + CHECK_EQ(*b_range.find(key_of_value(values[i])), values[i]); + } + b_range.swap(b_copy); + + // Test erase via values. + for (int i = 0; i < values.size(); ++i) { + mutable_b.erase(key_of_value(values[i])); + // Erasing a non-existent key should have no effect. + CHECK_EQ(mutable_b.erase(key_of_value(values[i])), 0); + } + + const_b.verify(); + CHECK_EQ(const_b.internal_nodes(), 0); + CHECK_EQ(const_b.leaf_nodes(), 0); + CHECK_EQ(const_b.size(), 0); + + // Test erase via iterators. + mutable_b = b_copy; + for (int i = 0; i < values.size(); ++i) { + mutable_b.erase(mutable_b.find(key_of_value(values[i]))); + } + + const_b.verify(); + CHECK_EQ(const_b.internal_nodes(), 0); + CHECK_EQ(const_b.leaf_nodes(), 0); + CHECK_EQ(const_b.size(), 0); + + // Test insert with hint. + for (int i = 0; i < values.size(); i++) { + mutable_b.insert(mutable_b.upper_bound(key_of_value(values[i])), values[i]); + } + + const_b.verify(); + + // Test dumping of the btree to an ostream. There should be 1 line for each + // value. + ostringstream strm; + strm << mutable_b.tree(); + CHECK_EQ(mutable_b.size(), strcount(strm.str(), '\n')); + + // Test range erase. + mutable_b.erase(mutable_b.begin(), mutable_b.end()); + CHECK_EQ(mutable_b.size(), 0); + const_b.verify(); + + // First half. + mutable_b = b_copy; + typename T::iterator mutable_iter_end = mutable_b.begin(); + for (int i = 0; i < values.size() / 2; ++i) ++mutable_iter_end; + mutable_b.erase(mutable_b.begin(), mutable_iter_end); + CHECK_EQ(mutable_b.size(), values.size() - values.size() / 2); + const_b.verify(); + + // Second half. + mutable_b = b_copy; + typename T::iterator mutable_iter_begin = mutable_b.begin(); + for (int i = 0; i < values.size() / 2; ++i) ++mutable_iter_begin; + mutable_b.erase(mutable_iter_begin, mutable_b.end()); + CHECK_EQ(mutable_b.size(), values.size() / 2); + const_b.verify(); + + // Second quarter. + mutable_b = b_copy; + mutable_iter_begin = mutable_b.begin(); + for (int i = 0; i < values.size() / 4; ++i) ++mutable_iter_begin; + mutable_iter_end = mutable_iter_begin; + for (int i = 0; i < values.size() / 4; ++i) ++mutable_iter_end; + mutable_b.erase(mutable_iter_begin, mutable_iter_end); + CHECK_EQ(mutable_b.size(), values.size() - values.size() / 4); + const_b.verify(); + + mutable_b.clear(); +} + +template +void ConstTest() { + typedef typename T::value_type value_type; + typename KeyOfValue::type key_of_value; + + T mutable_b; + const T &const_b = mutable_b; + + // Insert a single value into the container and test looking it up. + value_type value = Generator(2)(2); + mutable_b.insert(value); + CHECK(mutable_b.find(key_of_value(value)) != const_b.end()); + CHECK(const_b.find(key_of_value(value)) != mutable_b.end()); + CHECK_EQ(*const_b.lower_bound(key_of_value(value)), value); + CHECK(const_b.upper_bound(key_of_value(value)) == const_b.end()); + CHECK_EQ(*const_b.equal_range(key_of_value(value)).first, value); + + // We can only create a non-const iterator from a non-const container. + typename T::iterator mutable_iter(mutable_b.begin()); + CHECK(mutable_iter == const_b.begin()); + CHECK(mutable_iter != const_b.end()); + CHECK(const_b.begin() == mutable_iter); + CHECK(const_b.end() != mutable_iter); + typename T::reverse_iterator mutable_riter(mutable_b.rbegin()); + CHECK(mutable_riter == const_b.rbegin()); + CHECK(mutable_riter != const_b.rend()); + CHECK(const_b.rbegin() == mutable_riter); + CHECK(const_b.rend() != mutable_riter); + + // We can create a const iterator from a non-const iterator. + typename T::const_iterator const_iter(mutable_iter); + CHECK(const_iter == mutable_b.begin()); + CHECK(const_iter != mutable_b.end()); + CHECK(mutable_b.begin() == const_iter); + CHECK(mutable_b.end() != const_iter); + typename T::const_reverse_iterator const_riter(mutable_riter); + CHECK(const_riter == mutable_b.rbegin()); + CHECK(const_riter != mutable_b.rend()); + CHECK(mutable_b.rbegin() == const_riter); + CHECK(mutable_b.rend() != const_riter); + + // Make sure various methods can be invoked on a const container. + const_b.verify(); + CHECK(!const_b.empty()); + CHECK_EQ(const_b.size(), 1); + CHECK_GT(const_b.max_size(), 0); + CHECK_EQ(const_b.height(), 1); + CHECK_EQ(const_b.count(key_of_value(value)), 1); + CHECK_EQ(const_b.internal_nodes(), 0); + CHECK_EQ(const_b.leaf_nodes(), 1); + CHECK_EQ(const_b.nodes(), 1); + CHECK_GT(const_b.bytes_used(), 0); + CHECK_GT(const_b.fullness(), 0); + CHECK_GT(const_b.overhead(), 0); +} + +template +void BtreeTest() { + ConstTest(); + + typedef typename base::remove_const::type V; + vector random_values = GenerateValues(FLAGS_test_values); + + unique_checker container; + + // Test key insertion/deletion in sorted order. + vector sorted_values(random_values); + sort(sorted_values.begin(), sorted_values.end()); + DoTest("sorted: ", &container, sorted_values); + + // Test key insertion/deletion in reverse sorted order. + reverse(sorted_values.begin(), sorted_values.end()); + DoTest("rsorted: ", &container, sorted_values); + + // Test key insertion/deletion in random order. + DoTest("random: ", &container, random_values); +} + +template +void BtreeMultiTest() { + ConstTest(); + + typedef typename base::remove_const::type V; + const vector& random_values = GenerateValues(FLAGS_test_values); + + multi_checker container; + + // Test keys in sorted order. + vector sorted_values(random_values); + sort(sorted_values.begin(), sorted_values.end()); + DoTest("sorted: ", &container, sorted_values); + + // Test keys in reverse sorted order. + reverse(sorted_values.begin(), sorted_values.end()); + DoTest("rsorted: ", &container, sorted_values); + + // Test keys in random order. + DoTest("random: ", &container, random_values); + + // Test keys in random order w/ duplicates. + vector duplicate_values(random_values); + duplicate_values.insert( + duplicate_values.end(), random_values.begin(), random_values.end()); + DoTest("duplicates:", &container, duplicate_values); + + // Test all identical keys. + vector identical_values(100); + fill(identical_values.begin(), identical_values.end(), Generator(2)(2)); + DoTest("identical: ", &container, identical_values); +} + +template +void BtreeArenaTest() { + typedef typename T::value_type value_type; + + UnsafeArena arena1(1000); + UnsafeArena arena2(1000); + T b1(typename T::key_compare(), &arena1); + T b2(typename T::key_compare(), &arena2); + + // This should swap the allocators! + swap(b1, b2); + + for (int i = 0; i < 1000; i++) { + b1.insert(Generator(1000)(i)); + } + + // We should have allocated out of arena2! + CHECK_LE(b1.bytes_used(), arena2.status().bytes_allocated()); + CHECK_GT(arena2.block_count(), arena1.block_count()); +} + +template +void BtreeMapTest() { + typedef typename T::value_type value_type; + typedef typename T::mapped_type mapped_type; + + mapped_type m = Generator(0)(0); + (void) m; + + T b; + + // Verify we can insert using operator[]. + for (int i = 0; i < 1000; i++) { + value_type v = Generator(1000)(i); + b[v.first] = v.second; + } + CHECK_EQ(b.size(), 1000); + + // Test whether we can use the "->" operator on iterators and + // reverse_iterators. This stresses the btree_map_params::pair_pointer + // mechanism. + CHECK_EQ(b.begin()->first, Generator(1000)(0).first); + CHECK_EQ(b.begin()->second, Generator(1000)(0).second); + CHECK_EQ(b.rbegin()->first, Generator(1000)(999).first); + CHECK_EQ(b.rbegin()->second, Generator(1000)(999).second); +} + +template +void BtreeMultiMapTest() { + typedef typename T::mapped_type mapped_type; + mapped_type m = Generator(0)(0); + (void) m; +} + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_BTREE_TEST_H__ diff --git a/btree_test_flags.cc b/btree_test_flags.cc new file mode 100644 index 00000000..512c24e1 --- /dev/null +++ b/btree_test_flags.cc @@ -0,0 +1,9 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: pmattis@google.com (Peter Mattis) + +#include "base/commandlineflags.h" + +DEFINE_int32(test_values, 10000, + "The number of values to use for tests."); +DEFINE_int32(benchmark_values, 1000000, + "The number of values to use for benchmarks."); diff --git a/btree_test_program.cc b/btree_test_program.cc new file mode 100644 index 00000000..0c2dcf88 --- /dev/null +++ b/btree_test_program.cc @@ -0,0 +1,64 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// Author: leg@google.com (Lawrence Greenfield) + +#include "base/init_google.h" +#include "base/logging.h" +#include "devtools/gdb/component/gdb_test_utils.h" +#include "util/btree/btree_map.h" +#include "util/btree/btree_set.h" + +using util::btree::btree_set; +using util::btree::btree_multiset; +using util::btree::btree_map; +using util::btree::btree_multimap; + +static btree_set* empty_set; +static btree_set* small_set; +static btree_set* big_set; +static btree_multiset* small_multiset; + +static btree_map* empty_map; +static btree_map* small_map; +static btree_map* big_map; +static btree_multimap* small_multimap; + +static void SetupBtreeSets() { + empty_set = new btree_set; + small_set = new btree_set; + small_multiset = new btree_multiset; + big_set = new btree_set; + + for (int i = 0; i < 10; ++i) { + small_set->insert(i); + small_multiset->insert(i / 2); + } + + for (int i = 0; i < 80; ++i) { + big_set->insert(i); + } +} + +static void SetupBtreeMaps() { + empty_map = new btree_map; + small_map = new btree_map; + small_multimap = new btree_multimap; + big_map = new btree_map; + + for (int i = 0; i < 10; ++i) { + small_map->insert(make_pair(i, i * 13)); + small_multimap->insert(make_pair(i / 2, i)); + } + + for (int i = 0; i < 80; ++i) { + big_map->insert(make_pair(i, i * 7)); + } +} + +int main(int argc, char** argv) { + FLAGS_logtostderr = true; + InitGoogle(argv[0], &argc, &argv, true); + SetupBtreeSets(); + SetupBtreeMaps(); + StopHereForDebugger(); + return 0; +} diff --git a/safe_btree.h b/safe_btree.h new file mode 100644 index 00000000..b3db5360 --- /dev/null +++ b/safe_btree.h @@ -0,0 +1,379 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: pmattis@google.com (Peter Mattis) +// +// A safe_btree<> wraps around a btree<> and removes the caveat that insertion +// and deletion invalidate iterators. A safe_btree<> maintains a generation +// number that is incremented on every mutation. A safe_btree<>::iterator keeps +// a pointer to the safe_btree<> it came from, the generation of the tree when +// it was last validated and the key the underlying btree<>::iterator points +// to. If an iterator is accessed and its generation differs from the tree +// generation it is revalidated. + +#ifndef UTIL_BTREE_SAFE_BTREE_H__ +#define UTIL_BTREE_SAFE_BTREE_H__ + +#include +#include +#include + +#include "base/integral_types.h" +#include "base/logging.h" +#include "util/btree/btree.h" + +namespace util { +namespace btree { + +template +class safe_btree_iterator { + public: + typedef typename Iterator::key_type key_type; + typedef typename Iterator::value_type value_type; + typedef typename Iterator::size_type size_type; + typedef typename Iterator::difference_type difference_type; + typedef typename Iterator::pointer pointer; + typedef typename Iterator::reference reference; + typedef typename Iterator::const_pointer const_pointer; + typedef typename Iterator::const_reference const_reference; + typedef typename Iterator::iterator_category iterator_category; + typedef typename Tree::iterator iterator; + typedef typename Tree::const_iterator const_iterator; + typedef safe_btree_iterator self_type; + + void update() const { + if (iter_ != tree_->internal_btree()->end()) { + // A positive generation indicates a valid key. + generation_ = tree_->generation(); + key_ = iter_.key(); + } else { + // Use a negative generation to indicate iter_ points to end(). + generation_ = -tree_->generation(); + } + } + + public: + safe_btree_iterator() + : generation_(0), + key_(), + iter_(), + tree_(NULL) { + } + safe_btree_iterator(const iterator &x) + : generation_(x.generation()), + key_(x.key()), + iter_(x.iter()), + tree_(x.tree()) { + } + safe_btree_iterator(Tree *tree, const Iterator &iter) + : generation_(), + key_(), + iter_(iter), + tree_(tree) { + update(); + } + + Tree* tree() const { return tree_; } + int64 generation() const { return generation_; } + + Iterator* mutable_iter() const { + if (generation_ != tree_->generation()) { + if (generation_ > 0) { + // This does the wrong thing for a multi{set,map}. If my iter was + // pointing to the 2nd of 2 values with the same key, then this will + // reset it to point to the first. This is why we don't provide a + // safe_btree_multi{set,map}. + iter_ = tree_->internal_btree()->lower_bound(key_); + update(); + } else if (-generation_ != tree_->generation()) { + iter_ = tree_->internal_btree()->end(); + generation_ = -tree_->generation(); + } + } + return &iter_; + } + const Iterator& iter() const { + return *mutable_iter(); + } + + // Equality/inequality operators. + bool operator==(const const_iterator &x) const { + return iter() == x.iter(); + } + bool operator!=(const const_iterator &x) const { + return iter() != x.iter(); + } + + // Accessors for the key/value the iterator is pointing at. + const key_type& key() const { + return key_; + } + reference operator*() const { + DCHECK_GT(generation_, 0); + return iter().operator*(); + } + pointer operator->() const { + DCHECK_GT(generation_, 0); + return iter().operator->(); + } + + // Increment/decrement operators. + self_type& operator++() { + ++(*mutable_iter()); + update(); + return *this; + } + self_type& operator--() { + --(*mutable_iter()); + update(); + return *this; + } + self_type operator++(int) { + self_type tmp = *this; + ++*this; + return tmp; + } + self_type operator--(int) { + self_type tmp = *this; + --*this; + return tmp; + } + + private: + // The generation of the tree when "iter" was updated. + mutable int64 generation_; + // The key the iterator points to. + mutable key_type key_; + // The underlying iterator. + mutable Iterator iter_; + // The tree the iterator is associated with. + Tree *tree_; +}; + +template +class safe_btree { + typedef safe_btree self_type; + + typedef btree btree_type; + typedef typename btree_type::iterator tree_iterator; + typedef typename btree_type::const_iterator tree_const_iterator; + + public: + typedef typename btree_type::params_type params_type; + typedef typename btree_type::key_type key_type; + typedef typename btree_type::data_type data_type; + typedef typename btree_type::mapped_type mapped_type; + typedef typename btree_type::value_type value_type; + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + typedef typename btree_type::pointer pointer; + typedef typename btree_type::const_pointer const_pointer; + typedef typename btree_type::reference reference; + typedef typename btree_type::const_reference const_reference; + typedef typename btree_type::size_type size_type; + typedef typename btree_type::difference_type difference_type; + typedef safe_btree_iterator iterator; + typedef safe_btree_iterator< + const self_type, tree_const_iterator> const_iterator; + typedef std::reverse_iterator const_reverse_iterator; + typedef std::reverse_iterator reverse_iterator; + + public: + // Default constructor. + safe_btree(const key_compare &comp, const allocator_type &alloc) + : tree_(comp, alloc), + generation_(1) { + } + + // Copy constructor. + safe_btree(const self_type &x) + : tree_(x.tree_), + generation_(1) { + } + + iterator begin() { + return iterator(this, tree_.begin()); + } + const_iterator begin() const { + return const_iterator(this, tree_.begin()); + } + iterator end() { + return iterator(this, tree_.end()); + } + const_iterator end() const { + return const_iterator(this, tree_.end()); + } + reverse_iterator rbegin() { + return reverse_iterator(end()); + } + const_reverse_iterator rbegin() const { + return const_reverse_iterator(end()); + } + reverse_iterator rend() { + return reverse_iterator(begin()); + } + const_reverse_iterator rend() const { + return const_reverse_iterator(begin()); + } + + // Lookup routines. + iterator lower_bound(const key_type &key) { + return iterator(this, tree_.lower_bound(key)); + } + const_iterator lower_bound(const key_type &key) const { + return const_iterator(this, tree_.lower_bound(key)); + } + iterator upper_bound(const key_type &key) { + return iterator(this, tree_.upper_bound(key)); + } + const_iterator upper_bound(const key_type &key) const { + return const_iterator(this, tree_.upper_bound(key)); + } + pair equal_range(const key_type &key) { + pair p = tree_.equal_range(key); + return make_pair(iterator(this, p.first), + iterator(this, p.second)); + } + pair equal_range(const key_type &key) const { + pair p = tree_.equal_range(key); + return make_pair(const_iterator(this, p.first), + const_iterator(this, p.second)); + } + iterator find_unique(const key_type &key) { + return iterator(this, tree_.find_unique(key)); + } + const_iterator find_unique(const key_type &key) const { + return const_iterator(this, tree_.find_unique(key)); + } + iterator find_multi(const key_type &key) { + return iterator(this, tree_.find_multi(key)); + } + const_iterator find_multi(const key_type &key) const { + return const_iterator(this, tree_.find_multi(key)); + } + size_type count_unique(const key_type &key) const { + return tree_.count_unique(key); + } + size_type count_multi(const key_type &key) const { + return tree_.count_multi(key); + } + + // Insertion routines. + template + pair insert_unique(const key_type &key, ValuePointer value) { + pair p = tree_.insert_unique(key, value); + generation_ += p.second; + return make_pair(iterator(this, p.first), p.second); + } + pair insert_unique(const value_type &v) { + pair p = tree_.insert_unique(v); + generation_ += p.second; + return make_pair(iterator(this, p.first), p.second); + } + iterator insert_unique(iterator position, const value_type &v) { + tree_iterator tree_pos = position.iter(); + ++generation_; + return iterator(this, tree_.insert_unique(tree_pos, v)); + } + template + void insert_unique(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_unique(*b); + } + } + iterator insert_multi(const value_type &v) { + ++generation_; + return iterator(this, tree_.insert_multi(v)); + } + iterator insert_multi(iterator position, const value_type &v) { + tree_iterator tree_pos = position.iter(); + ++generation_; + return iterator(this, tree_.insert_multi(tree_pos, v)); + } + template + void insert_multi(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_multi(*b); + } + } + self_type& operator=(const self_type &x) { + if (&x == this) { + // Don't copy onto ourselves. + return *this; + } + ++generation_; + tree_ = x.tree_; + return *this; + } + + // Deletion routines. + void erase(const iterator &begin, const iterator &end) { + tree_.erase(begin.iter(), end.iter()); + ++generation_; + } + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + iterator erase(iterator iter) { + tree_iterator res = tree_.erase(iter.iter()); + ++generation_; + return iterator(this, res); + } + int erase_unique(const key_type &key) { + int res = tree_.erase_unique(key); + generation_ += res; + return res; + } + int erase_multi(const key_type &key) { + int res = tree_.erase_multi(key); + generation_ += res; + return res; + } + + // Access to the underlying btree. + btree_type* internal_btree() { return &tree_; } + const btree_type* internal_btree() const { return &tree_; } + + // Utility routines. + void clear() { + ++generation_; + tree_.clear(); + } + void swap(self_type &x) { + ++generation_; + ++x.generation_; + tree_.swap(x.tree_); + } + void dump(ostream &os) const { + tree_.dump(os); + } + void verify() const { + tree_.verify(); + } + int64 generation() const { + return generation_; + } + key_compare key_comp() const { return tree_.key_comp(); } + + // Size routines. + size_type size() const { return tree_.size(); } + size_type max_size() const { return tree_.max_size(); } + bool empty() const { return tree_.empty(); } + size_type height() const { return tree_.height(); } + size_type internal_nodes() const { return tree_.internal_nodes(); } + size_type leaf_nodes() const { return tree_.leaf_nodes(); } + size_type nodes() const { return tree_.nodes(); } + size_type bytes_used() const { return tree_.bytes_used(); } + static double average_bytes_per_value() { + return btree_type::average_bytes_per_value(); + } + double fullness() const { return tree_.fullness(); } + double overhead() const { return tree_.overhead(); } + + private: + btree_type tree_; + int64 generation_; +}; + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_SAFE_BTREE_H__ diff --git a/safe_btree_map.h b/safe_btree_map.h new file mode 100644 index 00000000..74c5fa37 --- /dev/null +++ b/safe_btree_map.h @@ -0,0 +1,70 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: pmattis@google.com (Peter Mattis) +// +// The safe_btree_map<> is like btree_map<> except that it removes the caveat +// about insertion and deletion invalidating existing iterators at a small cost +// in making iterators larger and slower. + +#ifndef UTIL_BTREE_SAFE_BTREE_MAP_H__ +#define UTIL_BTREE_SAFE_BTREE_MAP_H__ + +#include +#include +#include + +#include "util/btree/btree_container.h" +#include "util/btree/btree_map.h" +#include "util/btree/safe_btree.h" + +namespace util { +namespace btree { + +// The safe_btree_map class is needed mainly for it's constructors. +template , + typename Alloc = std::allocator >, + int TargetNodeSize = 256> +class safe_btree_map : public btree_map_container< + safe_btree > > { + + typedef safe_btree_map self_type; + typedef btree_map_params< + Key, Value, Compare, Alloc, TargetNodeSize> params_type; + typedef safe_btree btree_type; + typedef btree_map_container super_type; + + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + + public: + // Default constructor. + safe_btree_map(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + safe_btree_map(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + safe_btree_map(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(b, e, comp, alloc) { + } +}; + +template +inline void swap(safe_btree_map &x, + safe_btree_map &y) { + x.swap(y); +} + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_SAFE_BTREE_MAP_H__ diff --git a/safe_btree_set.h b/safe_btree_set.h new file mode 100644 index 00000000..329a46f9 --- /dev/null +++ b/safe_btree_set.h @@ -0,0 +1,68 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: pmattis@google.com (Peter Mattis) +// +// The safe_btree_set<> is like btree_set<> except that it removes the caveat +// about insertion and deletion invalidating existing iterators at a small cost +// in making iterators larger and slower. + +#ifndef UTIL_BTREE_SAFE_BTREE_SET_H__ +#define UTIL_BTREE_SAFE_BTREE_SET_H__ + +#include +#include + +#include "util/btree/btree_container.h" +#include "util/btree/btree_set.h" +#include "util/btree/safe_btree.h" + +namespace util { +namespace btree { + +// The safe_btree_set class is needed mainly for it's constructors. +template , + typename Alloc = std::allocator, + int TargetNodeSize = 256> +class safe_btree_set : public btree_unique_container< + safe_btree > > { + + typedef safe_btree_set self_type; + typedef btree_set_params params_type; + typedef safe_btree btree_type; + typedef btree_unique_container super_type; + + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + + public: + // Default constructor. + safe_btree_set(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + safe_btree_set(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + safe_btree_set(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(b, e, comp, alloc) { + } +}; + +template +inline void swap(safe_btree_set &x, + safe_btree_set &y) { + x.swap(y); +} + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_SAFE_BTREE_SET_H__ diff --git a/safe_btree_test.cc b/safe_btree_test.cc new file mode 100644 index 00000000..d86623f9 --- /dev/null +++ b/safe_btree_test.cc @@ -0,0 +1,67 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) + +// TODO(pmattis): Add some tests that iterators are not invalidated by +// insertion and deletion. + +#include +#include +#include +#include +#include + +#include "base/arena-inl.h" +#include "base/init_google.h" +#include "base/integral_types.h" +#include "base/logging.h" +#include "strings/cord.h" +#include "testing/base/public/gunit.h" +#include "util/btree/btree_test.h" +#include "util/btree/safe_btree_map.h" +#include "util/btree/safe_btree_set.h" + +class UnsafeArena; + +namespace util { +namespace btree { +namespace { + +template +void SetTest() { + typedef ArenaAllocator ArenaAlloc; + BtreeTest, allocator, N>, set >(); + BtreeArenaTest, ArenaAlloc, N> >(); +} + +template +void MapTest() { + typedef ArenaAllocator ArenaAlloc; + BtreeTest, allocator, N>, map >(); + BtreeArenaTest, ArenaAlloc, N> >(); + BtreeMapTest, allocator, N> >(); +} + +TEST(SafeBtree, set_int32_32) { SetTest(); } +TEST(SafeBtree, set_int32_64) { SetTest(); } +TEST(SafeBtree, set_int32_128) { SetTest(); } +TEST(SafeBtree, set_int32_256) { SetTest(); } +TEST(SafeBtree, set_int64_256) { SetTest(); } +TEST(SafeBtree, set_string_256) { SetTest(); } +TEST(SafeBtree, set_cord_256) { SetTest(); } +TEST(SafeBtree, set_pair_256) { SetTest, 256>(); } +TEST(SafeBtree, map_int32_256) { MapTest(); } +TEST(SafeBtree, map_int64_256) { MapTest(); } +TEST(SafeBtree, map_string_256) { MapTest(); } +TEST(SafeBtree, map_cord_256) { MapTest(); } +TEST(SafeBtree, map_pair_256) { MapTest, 256>(); } + +} // namespace +} // namespace btree +} // namespace util + +int main(int argc, char **argv) { + FLAGS_logtostderr = true; + InitGoogle(argv[0], &argc, &argv, true); + return RUN_ALL_TESTS(); +} -- 2.30.2 From dd97f135d03ce44bb54f25e51cda10e71189e1ac Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Tue, 13 Dec 2011 10:48:14 -0800 Subject: [PATCH 02/91] Fix for kTargetNodeSize too large --- btree.h | 5 +++++ btree_nc.cc | 6 +++++- btree_nc_test.py | 28 ++++++++++------------------ 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/btree.h b/btree.h index b9fd04e2..afb508da 100644 --- a/btree.h +++ b/btree.h @@ -1435,6 +1435,11 @@ class btree : public Params::key_compare { sizeof(key_compare_checker(key_compare_helper()(key_type(), key_type()))) == sizeof(base::big_), key_comparison_function_must_return_bool); + + // Note: We insist on kTargetValues, which is computed from + // Params::kTargetNodeSize, falling under 256 because of the uint8 + // fields of base_fields. + COMPILE_ASSERT(kNodeValues < 256, target_node_size_too_large); }; //// diff --git a/btree_nc.cc b/btree_nc.cc index b24c312d..7f1260be 100644 --- a/btree_nc.cc +++ b/btree_nc.cc @@ -40,7 +40,11 @@ TEST_COMPARE_TO(int); #elif defined(TEST_compare_to_float) TEST_COMPARE_TO(float); #elif defined(TEST_compare_to_pointer) -TEST_COMPARE_TO(pointer); +TEST_COMPARE_TO(void*); +#elif defined(TEST_large_nodesize) +void LargeNode() { + util::btree::btree_set, std::allocator, 10000> set; +} #endif } // namespace diff --git a/btree_nc_test.py b/btree_nc_test.py index 056a75d2..b2a83277 100755 --- a/btree_nc_test.py +++ b/btree_nc_test.py @@ -30,28 +30,20 @@ class BtreeNegativeUnitTest(googletest.TestCase): # Test that int does not work as a return type for key comparison. ('int', - [r'error: creating array with negative size', # for gcc - r'', # for icc - ]), + [r'key_comparison_function_must_return_bool']), # Test that float does not work as a return type for key comparison. ('float', - [r'error: creating array with negative size', # for gcc - r'', # for icc - ]), + [r'key_comparison_function_must_return_bool']), # Test that void* does not work as a return type for key comparison. ('pointer', - [r'error: creating array with negative size', # for gcc - r'', # for icc - ]), + [r'key_comparison_function_must_return_bool']), # Test that bool does not work as a return type for compare-to # comparison. ('compare_to_bool', - [r'error: creating array with negative size', # for gcc - r'', # for icc - ]), + [r'key_comparison_function_must_return_bool']), # Test that int works as a return type for compare-to comparison. ('compare_to_int', None), # None means compilation should succeed. @@ -59,16 +51,16 @@ class BtreeNegativeUnitTest(googletest.TestCase): # Test that float does not work as a return type for compare-to # comparison. ('compare_to_float', - [r'error: creating array with negative size', # for gcc - r'', # for icc - ]), + [r'key_comparison_function_must_return_bool']), # Test that void* does not work as a return type for compare-to # comparison. ('compare_to_pointer', - [r'error: creating array with negative size', # for gcc - r'', # for icc - ]), + [r'key_comparison_function_must_return_bool']), + + # Test that large node sizes do not compile. + ('large_nodesize', + [r'target_node_size_too_large']), ] # Runs the list of tests. -- 2.30.2 From 9fe3bb3b33b805572778c3178f01664b25684542 Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Wed, 14 Dec 2011 10:14:35 -0800 Subject: [PATCH 03/91] support for >= 256 values per node --- btree.h | 59 +++++++++++++++++++++++++++++++++++++++++--------- btree_bench.cc | 13 ++++++++--- btree_nc.cc | 5 ++++- btree_test.cc | 34 ++++++++++++++++++++++++++++- 4 files changed, 96 insertions(+), 15 deletions(-) diff --git a/btree.h b/btree.h index afb508da..d911d5bb 100644 --- a/btree.h +++ b/btree.h @@ -316,8 +316,40 @@ static bool btree_compare_keys( return key_comparer::bool_compare(comp, x, y); } +// btree_is_node_big<> is a recursive template to determine whether a +// node of TargetNodeSize bytes needs a larger base_fields type +// (uint16, instead of uint8) to accomodate >= 256 values per node. +template +struct btree_is_node_big : + btree_is_node_big<(TargetNodeSize / 2), (ValueSize / 2)> { +}; +template +struct btree_is_node_big { + enum { + // In the base case, TargetNodeSize corresponds to single-byte + // entries and is the maximum number of values. + is_big = base::integral_constant= 256)>::value, + }; +}; + +// A helper for sizing the btree node's base_fields. The "type" +// typedef in this struct is an integral type large enough to hold as +// many ValueSize-values as will fit a node of TargetNodeSize bytes. +template +struct btree_base_field_type { + enum { + // "value_space" is the maximum leaf node count. leaf nodes have + // a greatest maximum of the node types. + value_space = TargetNodeSize - 2 * sizeof(void*), + }; + typedef typename base::if_< + btree_is_node_big::is_big, + uint16, + uint8>::type type; +}; + template + typename Alloc, int TargetNodeSize, int ValueSize> struct btree_common_params { // If Compare is derived from btree_key_compare_to_tag then use it as the // key_compare type. Otherwise, use btree_key_compare_to_adapter<> which will @@ -333,6 +365,8 @@ struct btree_common_params { typedef Key key_type; typedef ssize_t size_type; typedef ptrdiff_t difference_type; + typedef typename btree_base_field_type::type + base_field_type; enum { kTargetNodeSize = TargetNodeSize, @@ -343,7 +377,8 @@ struct btree_common_params { template struct btree_map_params - : public btree_common_params { + : public btree_common_params { typedef Data data_type; typedef Data mapped_type; typedef pair value_type; @@ -368,7 +403,8 @@ struct btree_map_params // A parameters structure for holding the type parameters for a btree_set. template struct btree_set_params - : public btree_common_params { + : public btree_common_params { typedef base::false_type data_type; typedef base::false_type mapped_type; typedef Key value_type; @@ -505,14 +541,16 @@ class btree_node { linear_search_type, binary_search_type>::type search_type; struct base_fields { + typedef typename Params::base_field_type field_type; + // A boolean indicating whether the node is a leaf or not. - uint8 leaf; + bool leaf; // The position of the node in the node's parent. - uint8 position; + field_type position; // The maximum number of values the node can hold. - uint8 max_count; + field_type max_count; // The count of the number of values in the node. - uint8 count; + field_type count; // A pointer to the node's parent. btree_node *parent; }; @@ -1437,9 +1475,10 @@ class btree : public Params::key_compare { key_comparison_function_must_return_bool); // Note: We insist on kTargetValues, which is computed from - // Params::kTargetNodeSize, falling under 256 because of the uint8 - // fields of base_fields. - COMPILE_ASSERT(kNodeValues < 256, target_node_size_too_large); + // Params::kTargetNodeSize, must fit the base_fields::field_type. + COMPILE_ASSERT(kNodeValues < + (1 << (8 * sizeof(typename base_fields::field_type))), + target_node_size_too_large); }; //// diff --git a/btree_bench.cc b/btree_bench.cc index c67dd664..94a1f272 100644 --- a/btree_bench.cc +++ b/btree_bench.cc @@ -401,7 +401,10 @@ typedef multimap stl_multimap_cord; MY_BENCHMARK_TYPES2(value, name, 416); \ MY_BENCHMARK_TYPES2(value, name, 448); \ MY_BENCHMARK_TYPES2(value, name, 480); \ - MY_BENCHMARK_TYPES2(value, name, 512) + MY_BENCHMARK_TYPES2(value, name, 512); \ + MY_BENCHMARK_TYPES2(value, name, 1024); \ + MY_BENCHMARK_TYPES2(value, name, 1536); \ + MY_BENCHMARK_TYPES2(value, name, 2048) MY_BENCHMARK_TYPES(int32, int32); MY_BENCHMARK_TYPES(int64, int64); @@ -433,10 +436,14 @@ MY_BENCHMARK_TYPES(Cord, cord); MY_BENCHMARK4(tree ## _416_ ## type, name, func); \ MY_BENCHMARK4(tree ## _448_ ## type, name, func); \ MY_BENCHMARK4(tree ## _480_ ## type, name, func); \ - MY_BENCHMARK4(tree ## _512_ ## type, name, func) + MY_BENCHMARK4(tree ## _512_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _1024_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _1536_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _2048_ ## type, name, func) #else #define MY_BENCHMARK3(tree, type, name, func) \ - MY_BENCHMARK4(tree ## _256_ ## type, name, func) + MY_BENCHMARK4(tree ## _256_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _2048_ ## type, name, func) #endif #define MY_BENCHMARK2(type, name, func) \ diff --git a/btree_nc.cc b/btree_nc.cc index 7f1260be..0a6ef512 100644 --- a/btree_nc.cc +++ b/btree_nc.cc @@ -43,7 +43,10 @@ TEST_COMPARE_TO(float); TEST_COMPARE_TO(void*); #elif defined(TEST_large_nodesize) void LargeNode() { - util::btree::btree_set, std::allocator, 10000> set; + // (1 << 20) with 8-byte values is 2^17 values per node, which + // overflows the uint16 of btree::node_type::base_fields. + util::btree::btree_set, std::allocator, 1 << 20> + large_node_set; } #endif diff --git a/btree_test.cc b/btree_test.cc index ef4e574e..d984ec64 100644 --- a/btree_test.cc +++ b/btree_test.cc @@ -47,6 +47,25 @@ TEST(Btree, map_string_256) { MapTest(); } TEST(Btree, map_cord_256) { MapTest(); } TEST(Btree, map_pair_256) { MapTest, 256>(); } +// Large-node tests +TEST(Btree, map_int32_1024) { MapTest(); } +TEST(Btree, map_int32_1032) { MapTest(); } +TEST(Btree, map_int32_1040) { MapTest(); } +TEST(Btree, map_int32_1048) { MapTest(); } +TEST(Btree, map_int32_1056) { MapTest(); } + +TEST(Btree, map_int32_2048) { MapTest(); } +TEST(Btree, map_int32_4096) { MapTest(); } +TEST(Btree, set_int32_1024) { SetTest(); } +TEST(Btree, set_int32_2048) { SetTest(); } +TEST(Btree, set_int32_4096) { SetTest(); } +TEST(Btree, map_string_1024) { MapTest(); } +TEST(Btree, map_string_2048) { MapTest(); } +TEST(Btree, map_string_4096) { MapTest(); } +TEST(Btree, set_string_1024) { SetTest(); } +TEST(Btree, set_string_2048) { SetTest(); } +TEST(Btree, set_string_4096) { SetTest(); } + template void MultiSetTest() { typedef ArenaAllocator ArenaAlloc; @@ -77,6 +96,20 @@ TEST(Btree, multimap_string_256) { MultiMapTest(); } TEST(Btree, multimap_cord_256) { MultiMapTest(); } TEST(Btree, multimap_pair_256) { MultiMapTest, 256>(); } +// Large-node tests +TEST(Btree, multimap_int32_1024) { MultiMapTest(); } +TEST(Btree, multimap_int32_2048) { MultiMapTest(); } +TEST(Btree, multimap_int32_4096) { MultiMapTest(); } +TEST(Btree, multiset_int32_1024) { MultiSetTest(); } +TEST(Btree, multiset_int32_2048) { MultiSetTest(); } +TEST(Btree, multiset_int32_4096) { MultiSetTest(); } +TEST(Btree, multimap_string_1024) { MultiMapTest(); } +TEST(Btree, multimap_string_2048) { MultiMapTest(); } +TEST(Btree, multimap_string_4096) { MultiMapTest(); } +TEST(Btree, multiset_string_1024) { MultiSetTest(); } +TEST(Btree, multiset_string_2048) { MultiSetTest(); } +TEST(Btree, multiset_string_4096) { MultiSetTest(); } + // Verify that swapping btrees swaps the key comparision functors. struct SubstringLess { SubstringLess() : n(2) {} @@ -155,7 +188,6 @@ TEST(Btree, IteratorIncrementBy) { } } - } // namespace } // namespace btree } // namespace util -- 2.30.2 From 89a0af9086bf45328f7ea188aeb3f6918eaf16f6 Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Wed, 14 Dec 2011 13:47:35 -0800 Subject: [PATCH 04/91] Simplify btree_base_field_type<> --- btree.h | 51 ++++++++++++++++----------------------------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/btree.h b/btree.h index d911d5bb..d8e485ef 100644 --- a/btree.h +++ b/btree.h @@ -316,38 +316,6 @@ static bool btree_compare_keys( return key_comparer::bool_compare(comp, x, y); } -// btree_is_node_big<> is a recursive template to determine whether a -// node of TargetNodeSize bytes needs a larger base_fields type -// (uint16, instead of uint8) to accomodate >= 256 values per node. -template -struct btree_is_node_big : - btree_is_node_big<(TargetNodeSize / 2), (ValueSize / 2)> { -}; -template -struct btree_is_node_big { - enum { - // In the base case, TargetNodeSize corresponds to single-byte - // entries and is the maximum number of values. - is_big = base::integral_constant= 256)>::value, - }; -}; - -// A helper for sizing the btree node's base_fields. The "type" -// typedef in this struct is an integral type large enough to hold as -// many ValueSize-values as will fit a node of TargetNodeSize bytes. -template -struct btree_base_field_type { - enum { - // "value_space" is the maximum leaf node count. leaf nodes have - // a greatest maximum of the node types. - value_space = TargetNodeSize - 2 * sizeof(void*), - }; - typedef typename base::if_< - btree_is_node_big::is_big, - uint16, - uint8>::type type; -}; - template struct btree_common_params { @@ -365,12 +333,21 @@ struct btree_common_params { typedef Key key_type; typedef ssize_t size_type; typedef ptrdiff_t difference_type; - typedef typename btree_base_field_type::type - base_field_type; enum { kTargetNodeSize = TargetNodeSize, + + // Available space for values. This is largest for leaf nodes, + // which has overhead no fewer than two pointers. + kNodeValueSpace = TargetNodeSize - 2 * sizeof(void*), }; + + // This is an integral type large enough to hold as many + // ValueSize-values as will fit a node of TargetNodeSize bytes. + typedef typename base::if_< + (kNodeValueSpace / ValueSize) >= 256, + uint16, + uint8>::type node_count_type; }; // A parameters structure for holding the type parameters for a btree_map. @@ -541,7 +518,7 @@ class btree_node { linear_search_type, binary_search_type>::type search_type; struct base_fields { - typedef typename Params::base_field_type field_type; + typedef typename Params::node_count_type field_type; // A boolean indicating whether the node is a leaf or not. bool leaf; @@ -1479,6 +1456,10 @@ class btree : public Params::key_compare { COMPILE_ASSERT(kNodeValues < (1 << (8 * sizeof(typename base_fields::field_type))), target_node_size_too_large); + + // Test the assumption made in setting kNodeValueSpace. + COMPILE_ASSERT(sizeof(base_fields) >= 2 * sizeof(void*), + node_space_assumption_incorrect); }; //// -- 2.30.2 From eaf6eb103dbc14c3d88218fd694f25b8472307bc Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Tue, 17 Jul 2012 17:20:45 -0700 Subject: [PATCH 05/91] Updates from google3; adding safe_btree comments, container (in)equality operators --- btree_container.h | 17 +++++++++++++++ btree_test.cc | 54 ++++++++++++++++++++++++++++++++++++++++++++++ safe_btree.h | 11 +++++++++- safe_btree_map.h | 16 +++++++++++--- safe_btree_set.h | 15 +++++++++++-- safe_btree_test.cc | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 161 insertions(+), 6 deletions(-) diff --git a/btree_container.h b/btree_container.h index 6703d1bf..beebf6ff 100644 --- a/btree_container.h +++ b/btree_container.h @@ -106,6 +106,23 @@ class btree_container { double fullness() const { return tree_.fullness(); } double overhead() const { return tree_.overhead(); } + bool operator==(const self_type& x) const { + if (size() != x.size()) { + return false; + } + for (const_iterator i = begin(), xi = x.begin(); i != end(); ++i, ++xi) { + if (*i != *xi) { + return false; + } + } + return true; + } + + bool operator!=(const self_type& other) const { + return !operator==(other); + } + + protected: Tree tree_; }; diff --git a/btree_test.cc b/btree_test.cc index d984ec64..335318de 100644 --- a/btree_test.cc +++ b/btree_test.cc @@ -188,6 +188,60 @@ TEST(Btree, IteratorIncrementBy) { } } +TEST(Btree, Comparison) { + const int kSetSize = 1201; + btree_set my_set; + for (int i = 0; i < kSetSize; ++i) { + my_set.insert(i); + } + btree_set my_set_copy(my_set); + EXPECT_TRUE(my_set_copy == my_set); + EXPECT_TRUE(my_set == my_set_copy); + EXPECT_FALSE(my_set_copy != my_set); + EXPECT_FALSE(my_set != my_set_copy); + + my_set.insert(kSetSize); + EXPECT_FALSE(my_set_copy == my_set); + EXPECT_FALSE(my_set == my_set_copy); + EXPECT_TRUE(my_set_copy != my_set); + EXPECT_TRUE(my_set != my_set_copy); + + my_set.erase(kSetSize - 1); + EXPECT_FALSE(my_set_copy == my_set); + EXPECT_FALSE(my_set == my_set_copy); + EXPECT_TRUE(my_set_copy != my_set); + EXPECT_TRUE(my_set != my_set_copy); + + btree_map my_map; + for (int i = 0; i < kSetSize; ++i) { + my_map[string(i, 'a')] = i; + } + btree_map my_map_copy(my_map); + EXPECT_TRUE(my_map_copy == my_map); + EXPECT_TRUE(my_map == my_map_copy); + EXPECT_FALSE(my_map_copy != my_map); + EXPECT_FALSE(my_map != my_map_copy); + + ++my_map_copy[string(7, 'a')]; + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); + + my_map_copy = my_map; + my_map["hello"] = kSetSize; + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); + + my_map.erase(string(kSetSize - 1, 'a')); + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); +} + } // namespace } // namespace btree } // namespace util diff --git a/safe_btree.h b/safe_btree.h index b3db5360..2a354232 100644 --- a/safe_btree.h +++ b/safe_btree.h @@ -1,4 +1,4 @@ -// Copyright 2007 Google Inc. All Rights Reserved. +// Copyright 2007, 2012 Google Inc. All Rights Reserved. // Author: pmattis@google.com (Peter Mattis) // // A safe_btree<> wraps around a btree<> and removes the caveat that insertion @@ -8,6 +8,11 @@ // it was last validated and the key the underlying btree<>::iterator points // to. If an iterator is accessed and its generation differs from the tree // generation it is revalidated. +// +// References and pointers returned by safe_btree iterators are not safe. +// +// See the incorrect usage examples mentioned in safe_btree_set.h and +// safe_btree_map.h. #ifndef UTIL_BTREE_SAFE_BTREE_H__ #define UTIL_BTREE_SAFE_BTREE_H__ @@ -106,10 +111,14 @@ class safe_btree_iterator { const key_type& key() const { return key_; } + // This reference value is potentially invalidated by any non-const + // method on the tree; it is NOT safe. reference operator*() const { DCHECK_GT(generation_, 0); return iter().operator*(); } + // This pointer value is potentially invalidated by any non-const + // method on the tree; it is NOT safe. pointer operator->() const { DCHECK_GT(generation_, 0); return iter().operator->(); diff --git a/safe_btree_map.h b/safe_btree_map.h index 74c5fa37..11274930 100644 --- a/safe_btree_map.h +++ b/safe_btree_map.h @@ -1,10 +1,20 @@ -// Copyright 2007 Google Inc. All Rights Reserved. +// Copyright 2007, 2012 Google Inc. All Rights Reserved. // Author: pmattis@google.com (Peter Mattis) // // The safe_btree_map<> is like btree_map<> except that it removes the caveat // about insertion and deletion invalidating existing iterators at a small cost // in making iterators larger and slower. - +// +// Revalidation occurs whenever an iterator is accessed. References +// and pointers returned by safe_btree_map<> iterators are not stable, +// they are potentially invalidated by any non-const method on the map. +// +// BEGIN INCORRECT EXAMPLE +// for (auto i = safe_map->begin(); i != safe_map->end(); ++i) { +// const T *value = &i->second; // DO NOT DO THIS +// [code that modifies safe_map and uses value]; +// } +// END INCORRECT EXAMPLE #ifndef UTIL_BTREE_SAFE_BTREE_MAP_H__ #define UTIL_BTREE_SAFE_BTREE_MAP_H__ @@ -19,7 +29,7 @@ namespace util { namespace btree { -// The safe_btree_map class is needed mainly for it's constructors. +// The safe_btree_map class is needed mainly for its constructors. template , typename Alloc = std::allocator >, diff --git a/safe_btree_set.h b/safe_btree_set.h index 329a46f9..72ed2d91 100644 --- a/safe_btree_set.h +++ b/safe_btree_set.h @@ -1,9 +1,20 @@ -// Copyright 2007 Google Inc. All Rights Reserved. +// Copyright 2007, 2012 Google Inc. All Rights Reserved. // Author: pmattis@google.com (Peter Mattis) // // The safe_btree_set<> is like btree_set<> except that it removes the caveat // about insertion and deletion invalidating existing iterators at a small cost // in making iterators larger and slower. +// +// Revalidation occurs whenever an iterator is accessed. References +// and pointers returned by safe_btree_map<> iterators are not stable, +// they are potentially invalidated by any non-const method on the set. +// +// BEGIN INCORRECT EXAMPLE +// for (auto i = safe_set->begin(); i != safe_set->end(); ++i) { +// const T &value = *i; // DO NOT DO THIS +// [code that modifies safe_set and uses value]; +// } +// END INCORRECT EXAMPLE #ifndef UTIL_BTREE_SAFE_BTREE_SET_H__ #define UTIL_BTREE_SAFE_BTREE_SET_H__ @@ -18,7 +29,7 @@ namespace util { namespace btree { -// The safe_btree_set class is needed mainly for it's constructors. +// The safe_btree_set class is needed mainly for its constructors. template , typename Alloc = std::allocator, diff --git a/safe_btree_test.cc b/safe_btree_test.cc index d86623f9..cf179b28 100644 --- a/safe_btree_test.cc +++ b/safe_btree_test.cc @@ -56,6 +56,60 @@ TEST(SafeBtree, map_string_256) { MapTest(); } TEST(SafeBtree, map_cord_256) { MapTest(); } TEST(SafeBtree, map_pair_256) { MapTest, 256>(); } +TEST(SafeBtree, Comparison) { + const int kSetSize = 1201; + safe_btree_set my_set; + for (int i = 0; i < kSetSize; ++i) { + my_set.insert(i); + } + safe_btree_set my_set_copy(my_set); + EXPECT_TRUE(my_set_copy == my_set); + EXPECT_TRUE(my_set == my_set_copy); + EXPECT_FALSE(my_set_copy != my_set); + EXPECT_FALSE(my_set != my_set_copy); + + my_set.insert(kSetSize); + EXPECT_FALSE(my_set_copy == my_set); + EXPECT_FALSE(my_set == my_set_copy); + EXPECT_TRUE(my_set_copy != my_set); + EXPECT_TRUE(my_set != my_set_copy); + + my_set.erase(kSetSize - 1); + EXPECT_FALSE(my_set_copy == my_set); + EXPECT_FALSE(my_set == my_set_copy); + EXPECT_TRUE(my_set_copy != my_set); + EXPECT_TRUE(my_set != my_set_copy); + + safe_btree_map my_map; + for (int i = 0; i < kSetSize; ++i) { + my_map[string(i, 'a')] = i; + } + safe_btree_map my_map_copy(my_map); + EXPECT_TRUE(my_map_copy == my_map); + EXPECT_TRUE(my_map == my_map_copy); + EXPECT_FALSE(my_map_copy != my_map); + EXPECT_FALSE(my_map != my_map_copy); + + ++my_map_copy[string(7, 'a')]; + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); + + my_map_copy = my_map; + my_map["hello"] = kSetSize; + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); + + my_map.erase(string(kSetSize - 1, 'a')); + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); +} + } // namespace } // namespace btree } // namespace util -- 2.30.2 From 9783752d6226448ad4fdd2e22bd0a77a28ee1012 Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Tue, 17 Jul 2012 17:22:55 -0700 Subject: [PATCH 06/91] Remove files that are very google-specific --- Makefile | 75 ------------------------- btree_nc.cc | 57 ------------------- btree_nc_test.py | 75 ------------------------- btree_printer.py | 125 ------------------------------------------ btree_printer_test.py | 92 ------------------------------- btree_test_program.cc | 64 --------------------- 6 files changed, 488 deletions(-) delete mode 100644 Makefile delete mode 100644 btree_nc.cc delete mode 100755 btree_nc_test.py delete mode 100755 btree_printer.py delete mode 100755 btree_printer_test.py delete mode 100644 btree_test_program.cc diff --git a/Makefile b/Makefile deleted file mode 100644 index c58946f9..00000000 --- a/Makefile +++ /dev/null @@ -1,75 +0,0 @@ -# -*- mode: python; -*- - -# This should be a Makefile, but it's not. - -# cc_library(name = "btree", -# srcs = [ "btree.h", -# "btree_container.h", -# "btree_map.h", -# "btree_set.h", -# "safe_btree.h", -# "safe_btree_map.h", -# "safe_btree_set.h" ], -# deps = [ "//strings", -# "//strings:cord" ]) - -# cc_library(name = "btree_test_flags", -# srcs = [ "btree_test_flags.cc" ], -# deps = [ "//base" ]) - -# cc_binary(name = "btree_bench", -# srcs = [ "btree_bench.cc" ], -# deps = [ ":btree", -# ":btree_test_flags", -# "//testing/base", -# "//util/random" ]) - -# cc_test(name = "btree_test", -# srcs = [ "btree_test.cc", ], -# deps = [ ":btree", -# ":btree_test_flags", -# "//base:heapcheck", -# "//testing/base", -# "//util/random", -# ], -# size = "large") - -# cc_test(name = "safe_btree_test", -# srcs = [ "safe_btree_test.cc", ], -# deps = [ ":btree", -# ":btree_test_flags", -# "//base:heapcheck", -# "//testing/base", -# "//util/random", -# ], -# size = "large") - -# cc_fake_binary(name = "btree_nc", -# srcs = [ "btree_nc.cc" ], -# deps = [ ":btree" ], -# legacy = 0) - -# py_test(name = "btree_nc_test", -# srcs = [ "btree_nc_test.py" ], -# deps = [ "//pyglib", -# "//testing/pybase" ], -# data = [ "btree_nc" ], -# size = "large") - -# cc_binary(name = "btree_test_program", -# srcs = [ "btree_test_program.cc" ], -# deps = [ ":btree", -# "//devtools/gdb/component:gdb_test_utils" ], -# testonly = 1) - -# # This test will only actually test the pretty-printing code if it's -# # compiled with debug information (blaze build -c dbg). The default -# # mode, fastbuild, will pass but will not catch any regressions! -# py_test(name = "btree_printer_test", -# size = "large", -# srcs = [ "btree_printer_test.py", -# "btree_printer.py" ], -# deps = [ "//devtools/gdb/component:gdbpy", -# "//testing/pybase", -# "//testing/gdb:gdb_script_test_util", -# ":btree_test_program" ]) diff --git a/btree_nc.cc b/btree_nc.cc deleted file mode 100644 index 0a6ef512..00000000 --- a/btree_nc.cc +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2009 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) - -#include "util/btree/btree_set.h" - -namespace { - -template -struct Compare { - R operator()(int a, int b) const { return reinterpret_cast(a < b); } -}; - -template -struct CompareTo : public util::btree::btree_key_compare_to_tag { - R operator()(int a, int b) const { return reinterpret_cast(a < b); } -}; - -#define TEST_COMPARE(r) \ - void TestCompile() { \ - util::btree::btree_set > s; \ - } - -#define TEST_COMPARE_TO(r) \ - void TestCompile() { \ - util::btree::btree_set > s; \ - } - -#if defined(TEST_bool) -TEST_COMPARE(bool); -#elif defined(TEST_int) -TEST_COMPARE(int); -#elif defined(TEST_float) -TEST_COMPARE(float); -#elif defined(TEST_pointer) -TEST_COMPARE(void*); -#elif defined(TEST_compare_to_bool) -TEST_COMPARE_TO(bool); -#elif defined(TEST_compare_to_int) -TEST_COMPARE_TO(int); -#elif defined(TEST_compare_to_float) -TEST_COMPARE_TO(float); -#elif defined(TEST_compare_to_pointer) -TEST_COMPARE_TO(void*); -#elif defined(TEST_large_nodesize) -void LargeNode() { - // (1 << 20) with 8-byte values is 2^17 values per node, which - // overflows the uint16 of btree::node_type::base_fields. - util::btree::btree_set, std::allocator, 1 << 20> - large_node_set; -} -#endif - -} // namespace - -int main() { - return 1; -} diff --git a/btree_nc_test.py b/btree_nc_test.py deleted file mode 100755 index b2a83277..00000000 --- a/btree_nc_test.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/python2.4 -# -# Copyright 2006 Google Inc. All Rights Reserved. - -"""Negative compilation unit test for btree.h. -""" - -__author__ = 'pmattis@google.com (Peter Mattis)' - -import os -from google3.testing.pybase import googletest -from google3.testing.pybase import fake_target_util -from google3.pyglib import flags - -_FLAGS = flags.FLAGS - - -class BtreeNegativeUnitTest(googletest.TestCase): - """Negative compilation tests for btree.h""" - - def testCompilerErrors(self): - """Runs a list of tests to verify that erroneous code leads to - expected compiler messages.""" - - # Defines a list of test specs, where each element is a tuple - # (test name, list of regexes for matching the compiler errors). - test_specs = [ - # Test that bool works as a return type for key comparison. - ('bool', None), # None means compilation should succeed. - - # Test that int does not work as a return type for key comparison. - ('int', - [r'key_comparison_function_must_return_bool']), - - # Test that float does not work as a return type for key comparison. - ('float', - [r'key_comparison_function_must_return_bool']), - - # Test that void* does not work as a return type for key comparison. - ('pointer', - [r'key_comparison_function_must_return_bool']), - - # Test that bool does not work as a return type for compare-to - # comparison. - ('compare_to_bool', - [r'key_comparison_function_must_return_bool']), - - # Test that int works as a return type for compare-to comparison. - ('compare_to_int', None), # None means compilation should succeed. - - # Test that float does not work as a return type for compare-to - # comparison. - ('compare_to_float', - [r'key_comparison_function_must_return_bool']), - - # Test that void* does not work as a return type for compare-to - # comparison. - ('compare_to_pointer', - [r'key_comparison_function_must_return_bool']), - - # Test that large node sizes do not compile. - ('large_nodesize', - [r'target_node_size_too_large']), - ] - - # Runs the list of tests. - fake_target_util.AssertCcCompilerErrors( - self, - 'google3/util/btree/btree_nc', # path to the fake target file. - 'btree_nc.o', # name of the target to build. - test_specs) - - -if __name__ == '__main__': - googletest.main() diff --git a/btree_printer.py b/btree_printer.py deleted file mode 100755 index 8f233401..00000000 --- a/btree_printer.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/python2.4 -# Copyright 2011 Google Inc. All Rights Reserved. -# GDB support for pretty printing StringPiece. - -"""GDB pretty-printer for btrees.""" - -# This is a module provided by GDB. -# Ref: http://wiki/Main/Google3GDBScripts -import gdb -from google3.devtools.gdb.component import printing - - -class BaseBtreePrinter(object): - """btree pretty-printer, for util/btree.""" - - def __init__(self, typename, val): - self.typename = typename - self.val = val - - def display_hint(self): - return 'array' - - def _my_iter(self, node): - count = node['fields_']['count'] - if node['fields_']['leaf']: - for i in range(count): - key = node['fields_']['values'][i] - yield ('[item]', key) - else: - # recursive generators are annoying: we can't just recurse, we need - # to expand and yield the values. - for i in range(count+1): - child = node['fields_']['children'][i] - for v in self._my_iter(child.dereference()): - yield v - - def children(self): - if self.nelements() != 0: - return self._my_iter(self.val['root_']['data'].dereference()) - else: - return iter([]) - - def nelements(self): - if self.val['root_']['data'] != 0: - root_fields = self.val['root_']['data'].dereference()['fields_'] - if root_fields['leaf']: - return root_fields['count'] - else: - return root_fields['size'] - else: - return 0 - - def to_string(self): # pylint: disable-msg=C6409 - """GDB calls this to compute the pretty-printed form.""" - return '%s with %d elements' % (self.typename, self.nelements()) - - -class BtreePrinter(BaseBtreePrinter): - """btree<> pretty-printer, for util/btree.""" - - def __init__(self, val): - BaseBtreePrinter.__init__(self, 'btree', val) - - -class BtreeSetPrinter(BaseBtreePrinter): - """btree_set<> pretty-printer.""" - - def __init__(self, val): - BaseBtreePrinter.__init__(self, 'btree_set', val['tree_']) - - -class BtreeMultisetPrinter(BaseBtreePrinter): - """btree_multiset<> pretty-printer.""" - - def __init__(self, val): - BaseBtreePrinter.__init__(self, 'btree_multiset', val['tree_']) - - -class BaseBtreeMapPrinter(BaseBtreePrinter): - """btree maps pretty-printer.""" - - def __init__(self, typename, val): - BaseBtreePrinter.__init__(self, typename, val['tree_']) - - def display_hint(self): - return 'map' - - def _my_map_iter(self, g): - for (_, pair) in g: - yield ('[key]', pair['first']) - yield ('[value]', pair['second']) - - def children(self): - # we need to break apart the pairs and yield them separately - if self.nelements() != 0: - return self._my_map_iter(BaseBtreePrinter.children(self)) - else: - return iter([]) - - -class BtreeMapPrinter(BaseBtreeMapPrinter): - """btree_map<> pretty-printer.""" - - def __init__(self, val): - BaseBtreeMapPrinter.__init__(self, 'btree_map', val) - - -class BtreeMultimapPrinter(BaseBtreeMapPrinter): - """btree_multimap<> pretty-printer.""" - - def __init__(self, val): - BaseBtreeMapPrinter.__init__(self, 'btree_multimap', val) - - -if __name__ == '__main__': - printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree<.*>', - BtreePrinter) - printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_set<.*>', - BtreeSetPrinter) - printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_multiset<.*>', - BtreeMultisetPrinter) - printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_map<.*>', - BtreeMapPrinter) - printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_multimap<.*>', - BtreeMultimapPrinter) diff --git a/btree_printer_test.py b/btree_printer_test.py deleted file mode 100755 index 95bc351e..00000000 --- a/btree_printer_test.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/python2.4 -# -# Copyright 2011 Google Inc. All Rights Reserved. - -"""Tests for btree_printer.py gdb pretty printer.""" - -__author__ = "leg@google.com (Lawrence Greenfield)" - -from google3.pyglib import flags -from google3.testing.gdb import gdb_script_test_util -from google3.testing.pybase import googletest - -FLAGS = flags.FLAGS - - -class BtreePrinterTest(gdb_script_test_util.TestCase): - def testBtreeSet(self): - self.InitSession("btree_set", - "util/btree/btree_test_program") - self.RunTo("StopHereForDebugger") - - self.SetOption("print elements", 20) - self.TestPrintOutputMatches("*empty_set", - """btree_set with 0 elements""") - self.TestPrintOutputMatches("*small_set", - """btree_set with 10 elements = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}""") - self.TestPrintOutputMatches("*small_multiset", - """btree_multiset with 10 elements = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4}""") - self.TestPrintOutputMatches("*big_set", - """btree_set with 80 elements = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19...}""") - - self.RunSession() - - def testBtreeMap(self): - self.InitSession("btree_set", - "util/btree/btree_test_program") - self.RunTo("StopHereForDebugger") - - self.SetOption("print elements", 30) - self.TestPrintOutputMatches("*empty_map", - """btree_map with 0 elements""") - self.TestPrintOutputMatches("*small_map", - """btree_map with 10 elements = { - \\[0\\] = 0, - \\[1\\] = 13, - \\[2\\] = 26, - \\[3\\] = 39, - \\[4\\] = 52, - \\[5\\] = 65, - \\[6\\] = 78, - \\[7\\] = 91, - \\[8\\] = 104, - \\[9\\] = 117 -}""") - self.TestPrintOutputMatches("*small_multimap", - """btree_multimap with 10 elements = { - \\[0\\] = 0, - \\[0\\] = 1, - \\[1\\] = 2, - \\[1\\] = 3, - \\[2\\] = 4, - \\[2\\] = 5, - \\[3\\] = 6, - \\[3\\] = 7, - \\[4\\] = 8, - \\[4\\] = 9 -}""") - self.TestPrintOutputMatches("*big_map", - """btree_map with 80 elements = { - \\[0\\] = 0, - \\[1\\] = 7, - \\[2\\] = 14, - \\[3\\] = 21, - \\[4\\] = 28, - \\[5\\] = 35, - \\[6\\] = 42, - \\[7\\] = 49, - \\[8\\] = 56, - \\[9\\] = 63, - \\[10\\] = 70, - \\[11\\] = 77, - \\[12\\] = 84, - \\[13\\] = 91, - \\[14\\] = 98 - ... -}""") - - self.RunSession() - - -if __name__ == "__main__": - googletest.main() diff --git a/btree_test_program.cc b/btree_test_program.cc deleted file mode 100644 index 0c2dcf88..00000000 --- a/btree_test_program.cc +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2011 Google Inc. All Rights Reserved. -// Author: leg@google.com (Lawrence Greenfield) - -#include "base/init_google.h" -#include "base/logging.h" -#include "devtools/gdb/component/gdb_test_utils.h" -#include "util/btree/btree_map.h" -#include "util/btree/btree_set.h" - -using util::btree::btree_set; -using util::btree::btree_multiset; -using util::btree::btree_map; -using util::btree::btree_multimap; - -static btree_set* empty_set; -static btree_set* small_set; -static btree_set* big_set; -static btree_multiset* small_multiset; - -static btree_map* empty_map; -static btree_map* small_map; -static btree_map* big_map; -static btree_multimap* small_multimap; - -static void SetupBtreeSets() { - empty_set = new btree_set; - small_set = new btree_set; - small_multiset = new btree_multiset; - big_set = new btree_set; - - for (int i = 0; i < 10; ++i) { - small_set->insert(i); - small_multiset->insert(i / 2); - } - - for (int i = 0; i < 80; ++i) { - big_set->insert(i); - } -} - -static void SetupBtreeMaps() { - empty_map = new btree_map; - small_map = new btree_map; - small_multimap = new btree_multimap; - big_map = new btree_map; - - for (int i = 0; i < 10; ++i) { - small_map->insert(make_pair(i, i * 13)); - small_multimap->insert(make_pair(i / 2, i)); - } - - for (int i = 0; i < 80; ++i) { - big_map->insert(make_pair(i, i * 7)); - } -} - -int main(int argc, char** argv) { - FLAGS_logtostderr = true; - InitGoogle(argv[0], &argc, &argv, true); - SetupBtreeSets(); - SetupBtreeMaps(); - StopHereForDebugger(); - return 0; -} -- 2.30.2 From 88853fbc9e377f591bef514f01626ee2732eaea3 Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Tue, 17 Jul 2012 17:25:01 -0700 Subject: [PATCH 07/91] Build outside of Google; add CMakeLists.txt, README --- CMakeLists.txt | 23 +++ README | 15 ++ btree.h | 381 +++++++++++++++-------------------- btree_bench.cc | 2 - btree_container.h | 16 +- btree_map.h | 16 +- btree_set.h | 10 +- btree_test.cc | 197 +++++++++---------- btree_test.h | 469 ++++++++++++++++++++++++-------------------- btree_test_flags.cc | 2 +- safe_btree.h | 44 ++--- safe_btree_map.h | 12 +- safe_btree_set.h | 10 +- safe_btree_test.cc | 73 +++---- 14 files changed, 626 insertions(+), 644 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 README diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..e908f4a4 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,23 @@ +cmake_minimum_required(VERSION 2.6) + +project(cppbtree CXX) + +option(build_tests "Build B-tree tests" OFF) +add_definitions(-std=c++0x) +set(CMAKE_CXX_FLAGS "-g") + +# CMake doesn't have a way to pure template library, +add_library(cppbtree btree.h btree_map.h btree_set.h safe_btree.h safe_btree_map.h safe_btree_set.h) +set_target_properties(cppbtree PROPERTIES LINKER_LANGUAGE CXX) + +if(build_tests) + enable_testing() + include_directories($ENV{GTEST_ROOT}/include) + link_directories($ENV{GTEST_ROOT}) + add_executable(btree_test btree_test.cc btree_test_flags.cc) + add_executable(safe_btree_test safe_btree_test.cc btree_test_flags.cc) + add_executable(btree_bench btree_bench.cc btree_test_flags.cc) + target_link_libraries(btree_test gtest_main gflags) + target_link_libraries(safe_btree_test gtest_main gflags) + target_link_libraries(btree_bench gflags) +endif() diff --git a/README b/README new file mode 100644 index 00000000..152972cf --- /dev/null +++ b/README @@ -0,0 +1,15 @@ +This library is a C++ template library and, as such, there is no +library to build and install. + +---- + +To build and run the provided tests, however, you will need to install +CMake and the Google C++ Test framework. + +Download CMake from http://www.cmake.org + +Download the GoogleTest framework from http://code.google.com/p/googletest + +export GTEST_ROOT=/path/for/gtest-x.y + +cmake . -Dbuild_tests=ON diff --git a/btree.h b/btree.h index d8e485ef..a8d3152d 100644 --- a/btree.h +++ b/btree.h @@ -35,11 +35,6 @@ // // btree_bench --benchmarks=. 2>&1 | ./benchmarks.awk // -// NOTE(pmattis): My warpstation (pmattis-warp.nyc) often produces slower -// results when running the benchmarks on CPUs 0 and 1 vs CPUs 2 and 3. To get -// consistent benchmark results, I run "taskset 0xc " to run the -// benchmark on CPUs 2 and 3. -// // Run on pmattis-warp.nyc (4 X 2200 MHz CPUs); 2010/03/04-15:23:06 // Benchmark STL(ns) B-Tree(ns) @ // -------------------------------------------------------- @@ -91,61 +86,29 @@ // BM_map_string_mixedaddrem 6400 5200 +18.75% <256> [72.0, 57.8] // BM_map_string_fifo 398 596 -49.75% <256> [72.0, 44.0] // BM_map_string_fwditer 243 113 +53.50% <256> [72.0, 55.8] -// BM_set_cord_insert 3661 2680 +26.80% <256> [40.0, 10.5] -// BM_set_cord_lookup 2920 2293 +21.47% <256> [40.0, 10.5] -// BM_set_cord_fulllookup 2960 2267 +23.41% <256> [40.0, 8.8] -// BM_set_cord_delete 4679 2535 +45.82% <256> [40.0, 10.5] -// BM_set_cord_queueaddrem 8230 5600 +31.96% <256> [40.0, 11.3] -// BM_set_cord_mixedaddrem 8497 6080 +28.45% <256> [40.0, 10.7] -// BM_set_cord_fifo 358 370 -3.35% <256> [40.0, 8.8] -// BM_set_cord_fwditer 352 193 +45.17% <256> [40.0, 10.5] -// BM_map_cord_insert 3680 2927 +20.46% <256> [48.0, 20.7] -// BM_map_cord_lookup 3018 2466 +18.29% <256> [48.0, 20.7] -// BM_map_cord_fulllookup 2943 2466 +16.21% <256> [48.0, 17.2] -// BM_map_cord_delete 4675 2775 +40.64% <256> [48.0, 20.7] -// BM_map_cord_queueaddrem 8383 6360 +24.13% <256> [48.0, 22.2] -// BM_map_cord_mixedaddrem 8952 6760 +24.49% <256> [48.0, 21.2] -// BM_map_cord_fifo 444 463 -4.28% <256> [48.0, 17.2] -// BM_map_cord_fwditer 391 225 +42.46% <256> [48.0, 20.7] #ifndef UTIL_BTREE_BTREE_H__ #define UTIL_BTREE_BTREE_H__ +#include #include #include #include #include #include #include -#include // IWYU pragma: export // Clients can rely on this. +#include #include +#include #include -#include // IWYU pragma: export // Clients can rely on this. +#include #include #include -#include "base/gdb-scripting.h" -#include "base/integral_types.h" -#include "base/logging.h" -#include "base/macros.h" -#include "base/template_util.h" -#include "base/type_traits.h" -#include "strings/cord.h" -#include "strings/stringpiece.h" +#ifndef NDEBUG +#define NDEBUG 1 +#endif -// iwyu.py found std::swap in , but we include . -// IWYU pragma: no_include -// -// iwyu.py doesn't think we need , so wants . -// IWYU pragma: no_include -// -// iwyu.py sure wants to bring in a lot of private STL headers. -// IWYU pragma: no_include -// IWYU pragma: no_include -// IWYU pragma: no_include -// IWYU pragma: no_include - -namespace util { namespace btree { // Inside a btree method, if we just call swap(), it will choose the @@ -161,6 +124,32 @@ inline void btree_swap_helper(T &a, T &b) { swap(a, b); } +// A template helper used to select A or B based on a condition. +template +struct if_{ + typedef A type; +}; + +template +struct if_ { + typedef B type; +}; + +// Types small_ and big_ are promise that sizeof(small_) < sizeof(big_) +typedef char small_; + +struct big_ { + char dummy[2]; +}; + +// A compile-time assertion. +template +struct CompileAssert { +}; + +#define COMPILE_ASSERT(expr, msg) \ + typedef CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1] + // A helper type used to indicate that a key-compare-to functor has been // provided. A user can specify a key-compare-to functor by doing: // @@ -180,7 +169,7 @@ struct btree_key_compare_to_tag { // btree_key_compare_to_tag. template struct btree_is_key_compare_to - : public base::is_convertible { + : public std::is_convertible { }; // A helper class to convert a boolean comparison into a three-way "compare-to" @@ -201,77 +190,29 @@ struct btree_key_compare_to_adapter : Compare { }; template <> -struct btree_key_compare_to_adapter > +struct btree_key_compare_to_adapter > : public btree_key_compare_to_tag { btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const less&) {} + btree_key_compare_to_adapter(const std::less&) {} btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const string &a, const string &b) const { + const btree_key_compare_to_adapter >&) {} + int operator()(const std::string &a, const std::string &b) const { return a.compare(b); } }; template <> -struct btree_key_compare_to_adapter > +struct btree_key_compare_to_adapter > : public btree_key_compare_to_tag { btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const greater&) {} + btree_key_compare_to_adapter(const std::greater&) {} btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const string &a, const string &b) const { + const btree_key_compare_to_adapter >&) {} + int operator()(const std::string &a, const std::string &b) const { return b.compare(a); } }; -template <> -struct btree_key_compare_to_adapter > - : public btree_key_compare_to_tag { - btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const less&) {} - btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const StringPiece &a, const StringPiece &b) const { - return a.compare(b); - } -}; - -template <> -struct btree_key_compare_to_adapter > - : public btree_key_compare_to_tag { - btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const greater&) {} - btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const StringPiece &a, const StringPiece &b) const { - return b.compare(a); - } -}; - -template <> -struct btree_key_compare_to_adapter > - : public btree_key_compare_to_tag { - btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const less&) {} - btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const Cord &a, const Cord &b) const { - return a.CompareTo(b); - } -}; - -template <> -struct btree_key_compare_to_adapter > - : public btree_key_compare_to_tag { - btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const greater&) {} - btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const Cord &a, const Cord &b) const { - return b.CompareTo(a); - } -}; - // A helper class that allows a compare-to functor to behave like a plain // compare functor. This specialization is used when we do not have a // compare-to functor. @@ -322,7 +263,7 @@ struct btree_common_params { // If Compare is derived from btree_key_compare_to_tag then use it as the // key_compare type. Otherwise, use btree_key_compare_to_adapter<> which will // fall-back to Compare if we don't have an appropriate specialization. - typedef typename base::if_< + typedef typename if_< btree_is_key_compare_to::value, Compare, btree_key_compare_to_adapter >::type key_compare; // A type which indicates if we have a key-compare-to functor or a plain old @@ -344,10 +285,10 @@ struct btree_common_params { // This is an integral type large enough to hold as many // ValueSize-values as will fit a node of TargetNodeSize bytes. - typedef typename base::if_< + typedef typename if_< (kNodeValueSpace / ValueSize) >= 256, - uint16, - uint8>::type node_count_type; + uint16_t, + uint8_t>::type node_count_type; }; // A parameters structure for holding the type parameters for a btree_map. @@ -358,8 +299,8 @@ struct btree_map_params sizeof(Key) + sizeof(Data)> { typedef Data data_type; typedef Data mapped_type; - typedef pair value_type; - typedef pair mutable_value_type; + typedef std::pair value_type; + typedef std::pair mutable_value_type; typedef value_type* pointer; typedef const value_type* const_pointer; typedef value_type& reference; @@ -382,8 +323,8 @@ template struct btree_set_params : public btree_common_params { - typedef base::false_type data_type; - typedef base::false_type mapped_type; + typedef std::false_type data_type; + typedef std::false_type mapped_type; typedef Key value_type; typedef value_type mutable_value_type; typedef value_type* pointer; @@ -499,22 +440,22 @@ class btree_node { key_type, self_type, key_compare> binary_search_compare_to_type; // If we have a valid key-compare-to type, use linear_search_compare_to, // otherwise use linear_search_plain_compare. - typedef typename base::if_< + typedef typename if_< Params::is_key_compare_to::value, linear_search_compare_to_type, linear_search_plain_compare_type>::type linear_search_type; // If we have a valid key-compare-to type, use binary_search_compare_to, // otherwise use binary_search_plain_compare. - typedef typename base::if_< + typedef typename if_< Params::is_key_compare_to::value, binary_search_compare_to_type, binary_search_plain_compare_type>::type binary_search_type; // If the key is an integral or floating point type, use linear search which // is faster than binary search for such types. Might be wise to also // configure linear search based on node-size. - typedef typename base::if_< - base::is_integral::value || - base::is_floating_point::value, + typedef typename if_< + std::is_integral::value || + std::is_floating_point::value, linear_search_type, binary_search_type>::type search_type; struct base_fields { @@ -586,7 +527,7 @@ class btree_node { // be a leaf. bool is_root() const { return parent()->leaf(); } void make_root() { - DCHECK(parent()->is_root()); + assert(parent()->is_root()); fields_.parent = fields_.parent->parent(); } @@ -739,7 +680,7 @@ class btree_node { f->max_count = max_count; f->count = 0; f->parent = parent; - if (DEBUG_MODE) { + if (!NDEBUG) { memset(&f->values, 0, max_count * sizeof(value_type)); } return n; @@ -747,7 +688,7 @@ class btree_node { static btree_node* init_internal(internal_fields *f, btree_node *parent) { btree_node *n = init_leaf(f, parent, kNodeValues); f->leaf = 0; - if (DEBUG_MODE) { + if (!NDEBUG) { memset(f->children, 0, sizeof(f->children)); } return n; @@ -779,7 +720,8 @@ class btree_node { root_fields fields_; private: - DISALLOW_EVIL_CONSTRUCTORS(btree_node); + btree_node(const btree_node&); + void operator=(const btree_node&); }; template @@ -790,7 +732,7 @@ struct btree_iterator { typedef typename Node::params_type params_type; typedef Node node_type; - typedef typename base::remove_const::type normal_node; + typedef typename std::remove_const::type normal_node; typedef const Node const_node; typedef typename params_type::value_type value_type; typedef typename params_type::pointer normal_pointer; @@ -800,7 +742,7 @@ struct btree_iterator { typedef Pointer pointer; typedef Reference reference; - typedef bidirectional_iterator_tag iterator_category; + typedef std::bidirectional_iterator_tag iterator_category; typedef btree_iterator< normal_node, normal_reference, normal_pointer> iterator; @@ -885,7 +827,7 @@ struct btree_iterator { // Dispatch helper class for using btree::internal_locate with plain compare. struct btree_internal_locate_plain_compare { template - static pair dispatch(const K &k, const T &t, Iter iter) { + static std::pair dispatch(const K &k, const T &t, Iter iter) { return t.internal_locate_plain_compare(k, iter); } }; @@ -893,7 +835,7 @@ struct btree_internal_locate_plain_compare { // Dispatch helper class for using btree::internal_locate with compare-to. struct btree_internal_locate_compare_to { template - static pair dispatch(const K &k, const T &t, Iter iter) { + static std::pair dispatch(const K &k, const T &t, Iter iter) { return t.internal_locate_compare_to(k, iter); } }; @@ -910,7 +852,7 @@ class btree : public Params::key_compare { friend class btree_internal_locate_plain_compare; friend class btree_internal_locate_compare_to; - typedef typename base::if_< + typedef typename if_< is_key_compare_to::value, btree_internal_locate_compare_to, btree_internal_locate_plain_compare>::type internal_locate_type; @@ -1037,11 +979,11 @@ class btree : public Params::key_compare { // Finds the range of values which compare equal to key. The first member of // the returned pair is equal to lower_bound(key). The second member pair of // the pair is equal to upper_bound(key). - pair equal_range(const key_type &key) { - return make_pair(lower_bound(key), upper_bound(key)); + std::pair equal_range(const key_type &key) { + return std::make_pair(lower_bound(key), upper_bound(key)); } - pair equal_range(const key_type &key) const { - return make_pair(lower_bound(key), upper_bound(key)); + std::pair equal_range(const key_type &key) const { + return std::make_pair(lower_bound(key), upper_bound(key)); } // Inserts a value into the btree only if it does not already exist. The @@ -1050,11 +992,11 @@ class btree : public Params::key_compare { // is being inserted. Value is not dereferenced if the key already exists in // the btree. See btree_map::operator[]. template - pair insert_unique(const key_type &key, ValuePointer value); + std::pair insert_unique(const key_type &key, ValuePointer value); // Inserts a value into the btree only if it does not already exist. The // boolean return value indicates whether insertion succeeded or failed. - pair insert_unique(const value_type &v) { + std::pair insert_unique(const value_type &v) { return insert_unique(params_type::key(v), &v); } @@ -1170,7 +1112,7 @@ class btree : public Params::key_compare { // Dump the btree to the specified ostream. Requires that operator<< is // defined for Key and Value. - void dump(ostream &os) const { + void dump(std::ostream &os) const { if (root() != NULL) { internal_dump(os, root(), 0); } @@ -1185,7 +1127,7 @@ class btree : public Params::key_compare { if (root()->leaf()) return root()->count(); return root()->size(); } - size_type max_size() const { return numeric_limits::max(); } + size_type max_size() const { return std::numeric_limits::max(); } bool empty() const { return root() == NULL; } // The height of the btree. An empty tree will have height 0. @@ -1310,7 +1252,7 @@ class btree : public Params::key_compare { } void delete_internal_node(node_type *node) { node->destroy(); - DCHECK(node != root()); + assert(node != root()); mutable_internal_allocator()->deallocate( reinterpret_cast(node), sizeof(internal_fields)); } @@ -1368,15 +1310,15 @@ class btree : public Params::key_compare { // key was found in the tree) or -kExactMatch (if it wasn't) in the second // field of the pair. The compare_to specialization allows the caller to // avoid a subsequent comparison to determine if an exact match was made, - // speeding up string, cord and StringPiece keys. + // speeding up string keys. template - pair internal_locate( + std::pair internal_locate( const key_type &key, IterType iter) const; template - pair internal_locate_plain_compare( + std::pair internal_locate_plain_compare( const key_type &key, IterType iter) const; template - pair internal_locate_compare_to( + std::pair internal_locate_compare_to( const key_type &key, IterType iter) const; // Internal routine which implements lower_bound(). @@ -1403,7 +1345,7 @@ class btree : public Params::key_compare { void internal_clear(node_type *node); // Dumps a node and all of its children to the specified ostream. - void internal_dump(ostream &os, const node_type *node, int level) const; + void internal_dump(std::ostream &os, const node_type *node, int level) const; // Verifies the tree structure of node. int internal_verify(const node_type *node, @@ -1427,14 +1369,14 @@ class btree : public Params::key_compare { empty_base_handle root_; private: - // A never instantiated helper function that returns base::big_ if we have a - // key-compare-to functor or if R is bool and base::small_ otherwise. + // A never instantiated helper function that returns big_ if we have a + // key-compare-to functor or if R is bool and small_ otherwise. template - static typename base::if_< - base::if_, - base::type_equals_ >::type::value, - base::big_, base::small_>::type key_compare_checker(R); + static typename if_< + if_, + std::is_same >::type::value, + big_, small_>::type key_compare_checker(R); // A never instantiated helper function that returns the key comparison // functor. @@ -1445,10 +1387,10 @@ class btree : public Params::key_compare { // is never actually invoked. The compiler will select which // key_compare_checker() to instantiate and then figure out the size of the // return type of key_compare_checker() at compile time which we then check - // against the sizeof of base::big_. + // against the sizeof of big_. COMPILE_ASSERT( sizeof(key_compare_checker(key_compare_helper()(key_type(), key_type()))) == - sizeof(base::big_), + sizeof(big_), key_comparison_function_must_return_bool); // Note: We insist on kTargetValues, which is computed from @@ -1466,7 +1408,7 @@ class btree : public Params::key_compare { // btree_node methods template inline void btree_node

::insert_value(int i, const value_type &x) { - DCHECK_LE(i, count()); + assert(i <= count()); value_init(count(), x); for (int j = count(); j > i; --j) { value_swap(j, this, j - 1); @@ -1486,7 +1428,7 @@ inline void btree_node

::insert_value(int i, const value_type &x) { template inline void btree_node

::remove_value(int i) { if (!leaf()) { - DCHECK_EQ(child(i + 1)->count(), 0); + assert(child(i + 1)->count() == 0); for (int j = i + 1; j < count(); ++j) { *mutable_child(j) = child(j + 1); child(j)->set_position(j); @@ -1503,11 +1445,11 @@ inline void btree_node

::remove_value(int i) { template void btree_node

::rebalance_right_to_left(btree_node *src, int to_move) { - DCHECK_EQ(parent(), src->parent()); - DCHECK_EQ(position() + 1, src->position()); - DCHECK_GE(src->count(), count()); - DCHECK_GE(to_move, 1); - DCHECK_LE(to_move, src->count()); + assert(parent() == src->parent()); + assert(position() + 1 == src->position()); + assert(src->count() >= count()); + assert(to_move >= 1); + assert(to_move <= src->count()); // Make room in the left node for the new values. for (int i = 0; i < to_move; ++i) { @@ -1537,7 +1479,7 @@ void btree_node

::rebalance_right_to_left(btree_node *src, int to_move) { set_child(1 + count() + i, src->child(i)); } for (int i = 0; i <= src->count() - to_move; ++i) { - DCHECK_LE(i + to_move, src->max_count()); + assert(i + to_move <= src->max_count()); src->set_child(i, src->child(i + to_move)); *src->mutable_child(i + to_move) = NULL; } @@ -1550,11 +1492,11 @@ void btree_node

::rebalance_right_to_left(btree_node *src, int to_move) { template void btree_node

::rebalance_left_to_right(btree_node *dest, int to_move) { - DCHECK_EQ(parent(), dest->parent()); - DCHECK_EQ(position() + 1, dest->position()); - DCHECK_GE(count(), dest->count()); - DCHECK_GE(to_move, 1); - DCHECK_LE(to_move, count()); + assert(parent() == dest->parent()); + assert(position() + 1 == dest->position()); + assert(count() >= dest->count()); + assert(to_move >= 1); + assert(to_move <= count()); // Make room in the right node for the new values. for (int i = 0; i < to_move; ++i) { @@ -1595,7 +1537,7 @@ void btree_node

::rebalance_left_to_right(btree_node *dest, int to_move) { template void btree_node

::split(btree_node *dest, int insert_position) { - DCHECK_EQ(dest->count(), 0); + assert(dest->count() == 0); // We bias the split based on the position being inserted. If we're // inserting at the beginning of the left node then bias the split to put @@ -1609,7 +1551,7 @@ void btree_node

::split(btree_node *dest, int insert_position) { dest->set_count(count() / 2); } set_count(count() - dest->count()); - DCHECK_GE(count(), 1); + assert(count() >= 1); // Move values from the left sibling to the right sibling. for (int i = 0; i < dest->count(); ++i) { @@ -1627,7 +1569,7 @@ void btree_node

::split(btree_node *dest, int insert_position) { if (!leaf()) { for (int i = 0; i <= dest->count(); ++i) { - DCHECK(child(count() + i + 1) != NULL); + assert(child(count() + i + 1) != NULL); dest->set_child(i, child(count() + i + 1)); *mutable_child(count() + i + 1) = NULL; } @@ -1636,8 +1578,8 @@ void btree_node

::split(btree_node *dest, int insert_position) { template void btree_node

::merge(btree_node *src) { - DCHECK_EQ(parent(), src->parent()); - DCHECK_EQ(position() + 1, src->position()); + assert(parent() == src->parent()); + assert(position() + 1 == src->position()); // Move the delimiting value to the left node. value_init(count()); @@ -1668,7 +1610,7 @@ void btree_node

::merge(btree_node *src) { template void btree_node

::swap(btree_node *x) { - DCHECK_EQ(leaf(), x->leaf()); + assert(leaf() == x->leaf()); // Swap the values. for (int i = count(); i < x->count(); ++i) { @@ -1677,7 +1619,7 @@ void btree_node

::swap(btree_node *x) { for (int i = x->count(); i < count(); ++i) { x->value_init(i); } - int n = max(count(), x->count()); + int n = std::max(count(), x->count()); for (int i = 0; i < n; ++i) { value_swap(i, x, i); } @@ -1710,10 +1652,10 @@ void btree_node

::swap(btree_node *x) { template void btree_iterator::increment_slow() { if (node->leaf()) { - DCHECK_GE(position, node->count()); + assert(position >= node->count()); self_type save(*this); while (position == node->count() && !node->is_root()) { - DCHECK_EQ(node->parent()->child(node->position()), node); + assert(node->parent()->child(node->position()) == node); position = node->position(); node = node->parent(); } @@ -1721,7 +1663,7 @@ void btree_iterator::increment_slow() { *this = save; } } else { - DCHECK_LT(position, node->count()); + assert(position < node->count()); node = node->child(position + 1); while (!node->leaf()) { node = node->child(0); @@ -1735,7 +1677,7 @@ void btree_iterator::increment_by(int count) { while (count > 0) { if (node->leaf()) { int rest = node->count() - position; - position += min(rest, count); + position += std::min(rest, count); count = count - rest; if (position < node->count()) { return; @@ -1750,10 +1692,10 @@ void btree_iterator::increment_by(int count) { template void btree_iterator::decrement_slow() { if (node->leaf()) { - DCHECK_LE(position, -1); + assert(position <= -1); self_type save(*this); while (position < 0 && !node->is_root()) { - DCHECK_EQ(node->parent()->child(node->position()), node); + assert(node->parent()->child(node->position()) == node); position = node->position() - 1; node = node->parent(); } @@ -1761,7 +1703,7 @@ void btree_iterator::decrement_slow() { *this = save; } } else { - DCHECK_GE(position, 0); + assert(position >= 0); node = node->child(position); while (!node->leaf()) { node = node->child(node->count()); @@ -1786,26 +1728,26 @@ btree

::btree(const self_type &x) } template template -pair::iterator, bool> +std::pair::iterator, bool> btree

::insert_unique(const key_type &key, ValuePointer value) { if (empty()) { *mutable_root() = new_leaf_root_node(1); } - pair res = internal_locate(key, iterator(root(), 0)); + std::pair res = internal_locate(key, iterator(root(), 0)); iterator &iter = res.first; if (res.second == kExactMatch) { // The key already exists in the tree, do nothing. - return make_pair(internal_last(iter), false); + return std::make_pair(internal_last(iter), false); } else if (!res.second) { iterator last = internal_last(iter); if (last.node && !compare_keys(key, last.key())) { // The key already exists in the tree, do nothing. - return make_pair(last, false); + return std::make_pair(last, false); } } - return make_pair(internal_insert(iter, *value), true); + return std::make_pair(internal_insert(iter, *value), true); } template @@ -1912,8 +1854,8 @@ typename btree

::iterator btree

::erase(iterator iter) { // Deletion of a value on an internal node. Swap the key with the largest // value of our left child. This is easy, we just decrement iter. iterator tmp_iter(iter--); - DCHECK(iter.node->leaf()); - DCHECK(!compare_keys(tmp_iter.key(), iter.key())); + assert(iter.node->leaf()); + assert(!compare_keys(tmp_iter.key(), iter.key())); iter.node->value_swap(iter.position, tmp_iter.node, tmp_iter.position); internal_delete = true; --*mutable_size(); @@ -2010,22 +1952,22 @@ void btree

::clear() { template void btree

::swap(self_type &x) { - ::swap(static_cast(*this), static_cast(x)); - ::swap(root_, x.root_); + std::swap(static_cast(*this), static_cast(x)); + std::swap(root_, x.root_); } template void btree

::verify() const { if (root() != NULL) { - CHECK_EQ(size(), internal_verify(root(), NULL, NULL)); - CHECK_EQ(leftmost(), (++const_iterator(root(), -1)).node); - CHECK_EQ(rightmost(), (--const_iterator(root(), root()->count())).node); - CHECK(leftmost()->leaf()); - CHECK(rightmost()->leaf()); + assert(size() == internal_verify(root(), NULL, NULL)); + assert(leftmost() == (++const_iterator(root(), -1)).node); + assert(rightmost() == (--const_iterator(root(), root()->count())).node); + assert(leftmost()->leaf()); + assert(rightmost()->leaf()); } else { - CHECK_EQ(size(), 0); - CHECK(leftmost() == NULL); - CHECK(rightmost() == NULL); + assert(size() == 0); + assert(leftmost() == NULL); + assert(rightmost() == NULL); } } @@ -2033,7 +1975,7 @@ template void btree

::rebalance_or_split(iterator *iter) { node_type *&node = iter->node; int &insert_position = iter->position; - DCHECK_EQ(node->count(), node->max_count()); + assert(node->count() == node->max_count()); // First try to make room on the node by rebalancing. node_type *parent = node->parent(); @@ -2047,20 +1989,20 @@ void btree

::rebalance_or_split(iterator *iter) { // fill up the left node. int to_move = (left->max_count() - left->count()) / (1 + (insert_position < left->max_count())); - to_move = max(1, to_move); + to_move = std::max(1, to_move); if (((insert_position - to_move) >= 0) || ((left->count() + to_move) < left->max_count())) { left->rebalance_right_to_left(node, to_move); - DCHECK_EQ(node->max_count() - node->count(), to_move); + assert(node->max_count() - node->count() == to_move); insert_position = insert_position - to_move; if (insert_position < 0) { insert_position = insert_position + left->count() + 1; node = left; } - DCHECK_LT(node->count(), node->max_count()); + assert(node->count() < node->max_count()); return; } } @@ -2075,7 +2017,7 @@ void btree

::rebalance_or_split(iterator *iter) { // to fill up the right node. int to_move = (right->max_count() - right->count()) / (1 + (insert_position > 0)); - to_move = max(1, to_move); + to_move = std::max(1, to_move); if ((insert_position <= (node->count() - to_move)) || ((right->count() + to_move) < right->max_count())) { @@ -2086,7 +2028,7 @@ void btree

::rebalance_or_split(iterator *iter) { node = right; } - DCHECK_LT(node->count(), node->max_count()); + assert(node->count() < node->max_count()); return; } } @@ -2106,7 +2048,7 @@ void btree

::rebalance_or_split(iterator *iter) { parent = new_internal_root_node(); parent->set_child(0, root()); *mutable_root() = parent; - DCHECK(*mutable_rightmost() == parent->child(0)); + assert(*mutable_rightmost() == parent->child(0)); } else { // The root node is an internal node. We do not want to create a new root // node because the root node is special and holds the size of the tree @@ -2179,7 +2121,7 @@ bool btree

::try_merge_or_rebalance(iterator *iter) { ((iter->node->count() == 0) || (iter->position > 0))) { int to_move = (right->count() - iter->node->count()) / 2; - to_move = min(to_move, right->count() - 1); + to_move = std::min(to_move, right->count() - 1); iter->node->rebalance_right_to_left(right, to_move); return false; } @@ -2194,7 +2136,7 @@ bool btree

::try_merge_or_rebalance(iterator *iter) { ((iter->node->count() == 0) || (iter->position < iter->node->count()))) { int to_move = (left->count() - iter->node->count()) / 2; - to_move = min(to_move, left->count() - 1); + to_move = std::min(to_move, left->count() - 1); left->rebalance_left_to_right(iter->node, to_move); iter->position += to_move; return false; @@ -2210,7 +2152,7 @@ void btree

::try_shrink() { } // Deleted the last item on the root node, shrink the height of the tree. if (root()->leaf()) { - DCHECK_EQ(size(), 0); + assert(size() == 0); delete_leaf_node(root()); *mutable_root() = NULL; } else { @@ -2256,9 +2198,9 @@ btree

::internal_insert(iterator iter, const value_type &v) { if (iter.node->max_count() < kNodeValues) { // Insertion into the root where the root is smaller that the full node // size. Simply grow the size of the root node. - DCHECK(iter.node == root()); + assert(iter.node == root()); iter.node = new_leaf_root_node( - min(kNodeValues, 2 * iter.node->max_count())); + std::min(kNodeValues, 2 * iter.node->max_count())); iter.node->swap(root()); delete_leaf_node(root()); *mutable_root() = iter.node; @@ -2274,13 +2216,13 @@ btree

::internal_insert(iterator iter, const value_type &v) { } template template -inline pair btree

::internal_locate( +inline std::pair btree

::internal_locate( const key_type &key, IterType iter) const { return internal_locate_type::dispatch(key, *this, iter); } template template -inline pair btree

::internal_locate_plain_compare( +inline std::pair btree

::internal_locate_plain_compare( const key_type &key, IterType iter) const { for (;;) { iter.position = iter.node->lower_bound(key, key_comp()); @@ -2289,24 +2231,24 @@ inline pair btree

::internal_locate_plain_compare( } iter.node = iter.node->child(iter.position); } - return make_pair(iter, 0); + return std::make_pair(iter, 0); } template template -inline pair btree

::internal_locate_compare_to( +inline std::pair btree

::internal_locate_compare_to( const key_type &key, IterType iter) const { for (;;) { int res = iter.node->lower_bound(key, key_comp()); iter.position = res & kMatchMask; if (res & kExactMatch) { - return make_pair(iter, static_cast(kExactMatch)); + return std::make_pair(iter, static_cast(kExactMatch)); } if (iter.node->leaf()) { break; } iter.node = iter.node->child(iter.position); } - return make_pair(iter, -kExactMatch); + return std::make_pair(iter, -kExactMatch); } template template @@ -2346,7 +2288,7 @@ template template IterType btree

::internal_find_unique( const key_type &key, IterType iter) const { if (iter.node) { - pair res = internal_locate(key, iter); + std::pair res = internal_locate(key, iter); if (res.second == kExactMatch) { return res.first; } @@ -2393,7 +2335,7 @@ void btree

::internal_clear(node_type *node) { template void btree

::internal_dump( - ostream &os, const node_type *node, int level) const { + std::ostream &os, const node_type *node, int level) const { for (int i = 0; i < node->count(); ++i) { if (!node->leaf()) { internal_dump(os, node->child(i), level + 1); @@ -2411,23 +2353,23 @@ void btree

::internal_dump( template int btree

::internal_verify( const node_type *node, const key_type *lo, const key_type *hi) const { - CHECK_GT(node->count(), 0); - CHECK_LE(node->count(), node->max_count()); + assert(node->count() > 0); + assert(node->count() <= node->max_count()); if (lo) { - CHECK(!compare_keys(node->key(0), *lo)); + assert(!compare_keys(node->key(0), *lo)); } if (hi) { - CHECK(!compare_keys(*hi, node->key(node->count() - 1))); + assert(!compare_keys(*hi, node->key(node->count() - 1))); } for (int i = 1; i < node->count(); ++i) { - CHECK(!compare_keys(node->key(i), node->key(i - 1))); + assert(!compare_keys(node->key(i), node->key(i - 1))); } int count = node->count(); if (!node->leaf()) { for (int i = 0; i <= node->count(); ++i) { - CHECK(node->child(i) != NULL); - CHECK_EQ(node->child(i)->parent(), node); - CHECK_EQ(node->child(i)->position(), i); + assert(node->child(i) != NULL); + assert(node->child(i)->parent() == node); + assert(node->child(i)->position() == i); count += internal_verify( node->child(i), (i == 0) ? lo : &node->key(i - 1), @@ -2438,8 +2380,5 @@ int btree

::internal_verify( } } // namespace btree -} // namespace util - -DEFINE_GDB_AUTO_SCRIPT("util/btree/btree_printer.py") #endif // UTIL_BTREE_BTREE_H__ diff --git a/btree_bench.cc b/btree_bench.cc index 94a1f272..1b45582a 100644 --- a/btree_bench.cc +++ b/btree_bench.cc @@ -26,7 +26,6 @@ DECLARE_int32(benchmark_max_iters); -namespace util { namespace btree { namespace { @@ -480,7 +479,6 @@ MY_BENCHMARK(multimap_cord); } // namespace } // namespace btree -} // namespace util int main(int argc, char **argv) { FLAGS_logtostderr = true; diff --git a/btree_container.h b/btree_container.h index beebf6ff..510c9ae6 100644 --- a/btree_container.h +++ b/btree_container.h @@ -8,9 +8,8 @@ #include #include -#include "util/btree/btree.h" // IWYU pragma: export +#include "btree.h" -namespace util { namespace btree { // A common base class for btree_set, btree_map, btree_multiset and @@ -70,10 +69,10 @@ class btree_container { const_iterator upper_bound(const key_type &key) const { return tree_.upper_bound(key); } - pair equal_range(const key_type &key) { + std::pair equal_range(const key_type &key) { return tree_.equal_range(key); } - pair equal_range(const key_type &key) const { + std::pair equal_range(const key_type &key) const { return tree_.equal_range(key); } @@ -84,7 +83,7 @@ class btree_container { void swap(self_type &x) { tree_.swap(x.tree_); } - void dump(ostream &os) const { + void dump(std::ostream &os) const { tree_.dump(os); } void verify() const { @@ -128,7 +127,7 @@ class btree_container { }; template -inline ostream& operator<<(ostream &os, const btree_container &b) { +inline std::ostream& operator<<(std::ostream &os, const btree_container &b) { b.dump(os); return os; } @@ -181,7 +180,7 @@ class btree_unique_container : public btree_container { } // Insertion routines. - pair insert(const value_type &x) { + std::pair insert(const value_type &x) { return this->tree_.insert_unique(x); } iterator insert(iterator position, const value_type &x) { @@ -230,7 +229,7 @@ class btree_map_container : public btree_unique_container { : key(k) { } value_type operator*() const { - return make_pair(key, data_type()); + return std::make_pair(key, data_type()); } const key_type &key; }; @@ -337,6 +336,5 @@ class btree_multi_container : public btree_container { }; } // namespace btree -} // namespace util #endif // UTIL_BTREE_BTREE_CONTAINER_H__ diff --git a/btree_map.h b/btree_map.h index 3c61a7f2..6e965590 100644 --- a/btree_map.h +++ b/btree_map.h @@ -1,4 +1,4 @@ -// Copyright 2007 Google Inc. All Rights Reserved. +// Copyright 2007, 2012 Google Inc. All Rights Reserved. // Author: jmacd@google.com (Josh MacDonald) // Author: pmattis@google.com (Peter Mattis) // @@ -18,16 +18,15 @@ #include #include -#include "util/btree/btree.h" // IWYU pragma: export -#include "util/btree/btree_container.h" // IWYU pragma: export +#include "btree.h" +#include "btree_container.h" -namespace util { namespace btree { // The btree_map class is needed mainly for it's constructors. template , - typename Alloc = std::allocator >, + typename Compare = std::less, + typename Alloc = std::allocator >, int TargetNodeSize = 256> class btree_map : public btree_map_container< btree > > { @@ -71,8 +70,8 @@ inline void swap(btree_map &x, // The btree_multimap class is needed mainly for it's constructors. template , - typename Alloc = std::allocator >, + typename Compare = std::less, + typename Alloc = std::allocator >, int TargetNodeSize = 256> class btree_multimap : public btree_multi_container< btree > > { @@ -117,6 +116,5 @@ inline void swap(btree_multimap &x, } } // namespace btree -} // namespace util #endif // UTIL_BTREE_BTREE_MAP_H__ diff --git a/btree_set.h b/btree_set.h index 7b5ea88d..0b51e494 100644 --- a/btree_set.h +++ b/btree_set.h @@ -14,15 +14,14 @@ #include #include -#include "util/btree/btree.h" // IWYU pragma: export -#include "util/btree/btree_container.h" // IWYU pragma: export +#include "btree.h" +#include "btree_container.h" -namespace util { namespace btree { // The btree_set class is needed mainly for it's constructors. template , + typename Compare = std::less, typename Alloc = std::allocator, int TargetNodeSize = 256> class btree_set : public btree_unique_container< @@ -65,7 +64,7 @@ inline void swap(btree_set &x, btree_set &y) { // The btree_multiset class is needed mainly for it's constructors. template , + typename Compare = std::less, typename Alloc = std::allocator, int TargetNodeSize = 256> class btree_multiset : public btree_multi_container< @@ -108,6 +107,5 @@ inline void swap(btree_multiset &x, } } // namespace btree -} // namespace util #endif // UTIL_BTREE_BTREE_SET_H__ diff --git a/btree_test.cc b/btree_test.cc index 335318de..7b9cafa5 100644 --- a/btree_test.cc +++ b/btree_test.cc @@ -2,128 +2,120 @@ // Author: jmacd@google.com (Josh MacDonald) // Author: pmattis@google.com (Peter Mattis) -#include "base/arena-inl.h" -#include "base/init_google.h" -#include "base/integral_types.h" -#include "base/logging.h" -#include "strings/stringpiece.h" -#include "testing/base/public/gunit.h" -#include "util/btree/btree_map.h" -#include "util/btree/btree_set.h" -#include "util/btree/btree_test.h" +#include "gtest/gtest.h" +#include "btree_map.h" +#include "btree_set.h" +#include "btree_test.h" -namespace util { namespace btree { namespace { template void SetTest() { - typedef ArenaAllocator ArenaAlloc; - CHECK_EQ(sizeof(btree_set), sizeof(void*)); - BtreeTest, allocator, N>, set >(); - BtreeArenaTest, ArenaAlloc, N> >(); + typedef TestAllocator TestAlloc; + ASSERT_EQ(sizeof(btree_set), sizeof(void*)); + BtreeTest, std::allocator, N>, std::set >(); + BtreeAllocatorTest, TestAlloc, N> >(); } template void MapTest() { - typedef ArenaAllocator ArenaAlloc; - CHECK_EQ(sizeof(btree_map), sizeof(void*)); - BtreeTest, allocator, N>, map >(); - BtreeArenaTest, ArenaAlloc, N> >(); - BtreeMapTest, allocator, N> >(); + typedef TestAllocator TestAlloc; + ASSERT_EQ(sizeof(btree_map), sizeof(void*)); + BtreeTest, std::allocator, N>, std::map >(); + BtreeAllocatorTest, TestAlloc, N> >(); + BtreeMapTest, std::allocator, N> >(); } -TEST(Btree, set_int32_32) { SetTest(); } -TEST(Btree, set_int32_64) { SetTest(); } -TEST(Btree, set_int32_128) { SetTest(); } -TEST(Btree, set_int32_256) { SetTest(); } -TEST(Btree, set_int64_256) { SetTest(); } -TEST(Btree, set_string_256) { SetTest(); } -TEST(Btree, set_cord_256) { SetTest(); } -TEST(Btree, set_pair_256) { SetTest, 256>(); } -TEST(Btree, map_int32_256) { MapTest(); } -TEST(Btree, map_int64_256) { MapTest(); } -TEST(Btree, map_string_256) { MapTest(); } -TEST(Btree, map_cord_256) { MapTest(); } -TEST(Btree, map_pair_256) { MapTest, 256>(); } +TEST(Btree, set_int32_32) { SetTest(); } +TEST(Btree, set_int32_64) { SetTest(); } +TEST(Btree, set_int32_128) { SetTest(); } +TEST(Btree, set_int32_256) { SetTest(); } +TEST(Btree, set_int64_256) { SetTest(); } +TEST(Btree, set_string_256) { SetTest(); } +TEST(Btree, set_pair_256) { SetTest, 256>(); } +TEST(Btree, map_int32_256) { MapTest(); } +TEST(Btree, map_int64_256) { MapTest(); } +TEST(Btree, map_string_256) { MapTest(); } +TEST(Btree, map_pair_256) { MapTest, 256>(); } // Large-node tests -TEST(Btree, map_int32_1024) { MapTest(); } -TEST(Btree, map_int32_1032) { MapTest(); } -TEST(Btree, map_int32_1040) { MapTest(); } -TEST(Btree, map_int32_1048) { MapTest(); } -TEST(Btree, map_int32_1056) { MapTest(); } +TEST(Btree, map_int32_1024) { MapTest(); } +TEST(Btree, map_int32_1032) { MapTest(); } +TEST(Btree, map_int32_1040) { MapTest(); } +TEST(Btree, map_int32_1048) { MapTest(); } +TEST(Btree, map_int32_1056) { MapTest(); } -TEST(Btree, map_int32_2048) { MapTest(); } -TEST(Btree, map_int32_4096) { MapTest(); } -TEST(Btree, set_int32_1024) { SetTest(); } -TEST(Btree, set_int32_2048) { SetTest(); } -TEST(Btree, set_int32_4096) { SetTest(); } -TEST(Btree, map_string_1024) { MapTest(); } -TEST(Btree, map_string_2048) { MapTest(); } -TEST(Btree, map_string_4096) { MapTest(); } -TEST(Btree, set_string_1024) { SetTest(); } -TEST(Btree, set_string_2048) { SetTest(); } -TEST(Btree, set_string_4096) { SetTest(); } +TEST(Btree, map_int32_2048) { MapTest(); } +TEST(Btree, map_int32_4096) { MapTest(); } +TEST(Btree, set_int32_1024) { SetTest(); } +TEST(Btree, set_int32_2048) { SetTest(); } +TEST(Btree, set_int32_4096) { SetTest(); } +TEST(Btree, map_string_1024) { MapTest(); } +TEST(Btree, map_string_2048) { MapTest(); } +TEST(Btree, map_string_4096) { MapTest(); } +TEST(Btree, set_string_1024) { SetTest(); } +TEST(Btree, set_string_2048) { SetTest(); } +TEST(Btree, set_string_4096) { SetTest(); } template void MultiSetTest() { - typedef ArenaAllocator ArenaAlloc; - CHECK_EQ(sizeof(btree_multiset), sizeof(void*)); - BtreeMultiTest, allocator, N>, - multiset >(); - BtreeArenaTest, ArenaAlloc, N> >(); + typedef TestAllocator TestAlloc; + ASSERT_EQ(sizeof(btree_multiset), sizeof(void*)); + BtreeMultiTest, std::allocator, N>, + std::multiset >(); + BtreeAllocatorTest, TestAlloc, N> >(); } template void MultiMapTest() { - typedef ArenaAllocator ArenaAlloc; - CHECK_EQ(sizeof(btree_multimap), sizeof(void*)); - BtreeMultiTest, allocator, N>, - multimap >(); - BtreeMultiMapTest, allocator, N> >(); - BtreeArenaTest, ArenaAlloc, N> >(); + typedef TestAllocator TestAlloc; + ASSERT_EQ(sizeof(btree_multimap), sizeof(void*)); + BtreeMultiTest, std::allocator, N>, + std::multimap >(); + BtreeMultiMapTest, std::allocator, N> >(); + BtreeAllocatorTest, TestAlloc, N> >(); } -TEST(Btree, multiset_int32_256) { MultiSetTest(); } -TEST(Btree, multiset_int64_256) { MultiSetTest(); } -TEST(Btree, multiset_string_256) { MultiSetTest(); } -TEST(Btree, multiset_cord_256) { MultiSetTest(); } -TEST(Btree, multiset_pair_256) { MultiSetTest, 256>(); } -TEST(Btree, multimap_int32_256) { MultiMapTest(); } -TEST(Btree, multimap_int64_256) { MultiMapTest(); } -TEST(Btree, multimap_string_256) { MultiMapTest(); } -TEST(Btree, multimap_cord_256) { MultiMapTest(); } -TEST(Btree, multimap_pair_256) { MultiMapTest, 256>(); } +TEST(Btree, multiset_int32_256) { MultiSetTest(); } +TEST(Btree, multiset_int64_256) { MultiSetTest(); } +TEST(Btree, multiset_string_256) { MultiSetTest(); } +TEST(Btree, multiset_pair_256) { MultiSetTest, 256>(); } +TEST(Btree, multimap_int32_256) { MultiMapTest(); } +TEST(Btree, multimap_int64_256) { MultiMapTest(); } +TEST(Btree, multimap_string_256) { MultiMapTest(); } +TEST(Btree, multimap_pair_256) { MultiMapTest, 256>(); } // Large-node tests -TEST(Btree, multimap_int32_1024) { MultiMapTest(); } -TEST(Btree, multimap_int32_2048) { MultiMapTest(); } -TEST(Btree, multimap_int32_4096) { MultiMapTest(); } -TEST(Btree, multiset_int32_1024) { MultiSetTest(); } -TEST(Btree, multiset_int32_2048) { MultiSetTest(); } -TEST(Btree, multiset_int32_4096) { MultiSetTest(); } -TEST(Btree, multimap_string_1024) { MultiMapTest(); } -TEST(Btree, multimap_string_2048) { MultiMapTest(); } -TEST(Btree, multimap_string_4096) { MultiMapTest(); } -TEST(Btree, multiset_string_1024) { MultiSetTest(); } -TEST(Btree, multiset_string_2048) { MultiSetTest(); } -TEST(Btree, multiset_string_4096) { MultiSetTest(); } +TEST(Btree, multimap_int32_1024) { MultiMapTest(); } +TEST(Btree, multimap_int32_2048) { MultiMapTest(); } +TEST(Btree, multimap_int32_4096) { MultiMapTest(); } +TEST(Btree, multiset_int32_1024) { MultiSetTest(); } +TEST(Btree, multiset_int32_2048) { MultiSetTest(); } +TEST(Btree, multiset_int32_4096) { MultiSetTest(); } +TEST(Btree, multimap_string_1024) { MultiMapTest(); } +TEST(Btree, multimap_string_2048) { MultiMapTest(); } +TEST(Btree, multimap_string_4096) { MultiMapTest(); } +TEST(Btree, multiset_string_1024) { MultiSetTest(); } +TEST(Btree, multiset_string_2048) { MultiSetTest(); } +TEST(Btree, multiset_string_4096) { MultiSetTest(); } // Verify that swapping btrees swaps the key comparision functors. struct SubstringLess { SubstringLess() : n(2) {} - SubstringLess(int length) + SubstringLess(size_t length) : n(length) { } - bool operator()(const string &a, const string &b) const { - return StringPiece(a).substr(0, n) < StringPiece(b).substr(0, n); + bool operator()(const std::string &a, const std::string &b) const { + std::string as(a.data(), std::min(n, a.size())); + std::string bs(b.data(), std::min(n, b.size())); + return as < bs; } - int n; + size_t n; }; TEST(Btree, SwapKeyCompare) { - typedef btree_set SubstringSet; + typedef btree_set SubstringSet; SubstringSet s1(SubstringLess(1), SubstringSet::allocator_type()); SubstringSet s2(SubstringLess(2), SubstringSet::allocator_type()); @@ -147,7 +139,7 @@ TEST(Btree, SwapKeyCompare) { TEST(Btree, UpperBoundRegression) { // Regress a bug where upper_bound would default-construct a new key_compare // instead of copying the existing one. - typedef btree_set SubstringSet; + typedef btree_set SubstringSet; SubstringSet my_set(SubstringLess(3)); my_set.insert("aab"); my_set.insert("abb"); @@ -164,25 +156,25 @@ TEST(Btree, UpperBoundRegression) { TEST(Btree, IteratorIncrementBy) { // Test that increment_by returns the same position as increment. const int kSetSize = 2341; - btree_set my_set; + btree_set my_set; for (int i = 0; i < kSetSize; ++i) { my_set.insert(i); } { // Simple increment vs. increment by. - btree_set::iterator a = my_set.begin(); - btree_set::iterator b = my_set.begin(); + btree_set::iterator a = my_set.begin(); + btree_set::iterator b = my_set.begin(); a.increment(); b.increment_by(1); EXPECT_EQ(*a, *b); } - btree_set::iterator a = my_set.begin(); + btree_set::iterator a = my_set.begin(); for (int i = 1; i < kSetSize; ++i) { ++a; // increment_by - btree_set::iterator b = my_set.begin(); + btree_set::iterator b = my_set.begin(); b.increment_by(i); EXPECT_EQ(*a, *b) << ": i=" << i; } @@ -190,11 +182,11 @@ TEST(Btree, IteratorIncrementBy) { TEST(Btree, Comparison) { const int kSetSize = 1201; - btree_set my_set; + btree_set my_set; for (int i = 0; i < kSetSize; ++i) { my_set.insert(i); } - btree_set my_set_copy(my_set); + btree_set my_set_copy(my_set); EXPECT_TRUE(my_set_copy == my_set); EXPECT_TRUE(my_set == my_set_copy); EXPECT_FALSE(my_set_copy != my_set); @@ -212,17 +204,17 @@ TEST(Btree, Comparison) { EXPECT_TRUE(my_set_copy != my_set); EXPECT_TRUE(my_set != my_set_copy); - btree_map my_map; + btree_map my_map; for (int i = 0; i < kSetSize; ++i) { - my_map[string(i, 'a')] = i; + my_map[std::string(i, 'a')] = i; } - btree_map my_map_copy(my_map); + btree_map my_map_copy(my_map); EXPECT_TRUE(my_map_copy == my_map); EXPECT_TRUE(my_map == my_map_copy); EXPECT_FALSE(my_map_copy != my_map); EXPECT_FALSE(my_map != my_map_copy); - ++my_map_copy[string(7, 'a')]; + ++my_map_copy[std::string(7, 'a')]; EXPECT_FALSE(my_map_copy == my_map); EXPECT_FALSE(my_map == my_map_copy); EXPECT_TRUE(my_map_copy != my_map); @@ -235,7 +227,7 @@ TEST(Btree, Comparison) { EXPECT_TRUE(my_map_copy != my_map); EXPECT_TRUE(my_map != my_map_copy); - my_map.erase(string(kSetSize - 1, 'a')); + my_map.erase(std::string(kSetSize - 1, 'a')); EXPECT_FALSE(my_map_copy == my_map); EXPECT_FALSE(my_map == my_map_copy); EXPECT_TRUE(my_map_copy != my_map); @@ -244,10 +236,3 @@ TEST(Btree, Comparison) { } // namespace } // namespace btree -} // namespace util - -int main(int argc, char **argv) { - FLAGS_logtostderr = true; - InitGoogle(argv[0], &argc, &argv, true); - return RUN_ALL_TESTS(); -} diff --git a/btree_test.h b/btree_test.h index 557c1e54..c368573e 100644 --- a/btree_test.h +++ b/btree_test.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -16,53 +17,50 @@ #include #include -#include "base/arena.h" -#include "base/commandlineflags.h" -#include "base/logging.h" -#include "base/type_traits.h" -#include "strings/cord.h" -#include "strings/util.h" -#include "testing/base/public/googletest.h" -#include "util/btree/btree_container.h" -#include "util/random/acmrandom.h" +#include "gtest/gtest.h" +#include "gflags/gflags.h" +#include "btree_container.h" DECLARE_int32(test_values); DECLARE_int32(benchmark_values); namespace std { -// Provide operator<< support for pair. +// Provide operator<< support for std::pair. template -ostream& operator<<(ostream &os, const pair &p) { +ostream& operator<<(ostream &os, const std::pair &p) { os << "(" << p.first << "," << p.second << ")"; return os; } // Provide pair equality testing that works as long as x.first is comparable to // y.first and x.second is comparable to y.second. Needed in the test for -// comparing pair to pair. +// comparing std::pair to std::pair. template -bool operator==(const pair &x, const pair &y) { +bool operator==(const std::pair &x, const std::pair &y) { return x.first == y.first && x.second == y.second; } -} // namespace std - -namespace base { - // Partial specialization of remove_const that propagates the removal through // std::pair. template -struct remove_const > { - typedef std::pair::type, - typename remove_const::type> type; +struct remove_const > { + typedef pair::type, + typename remove_const::type> type; }; -} // namespace base +} // namespace std -namespace util { namespace btree { +// Select the first member of a pair. +template +struct select1st : public std::unary_function<_Pair, typename _Pair::first_type> { + const typename _Pair::first_type& operator()(const _Pair& __x) const { + return __x.first; + } +}; + // Utility class to provide an accessor for a key given a value. The default // behavior is to treat the value as a pair and return the first element. template @@ -70,6 +68,11 @@ struct KeyOfValue { typedef select1st type; }; +template +struct identity { + inline const T& operator()(const T& t) const { return t; } +}; + // Partial specialization of KeyOfValue class for when the key and value are // the same type such as in set<> and btree_set<>. template @@ -77,6 +80,29 @@ struct KeyOfValue { typedef identity type; }; +// Counts the number of occurances of "c" in a buffer. +inline ptrdiff_t strcount(const char* buf_begin, const char* buf_end, char c) { + if (buf_begin == NULL) + return 0; + if (buf_end <= buf_begin) + return 0; + ptrdiff_t num = 0; + for (const char* bp = buf_begin; bp != buf_end; bp++) { + if (*bp == c) + num++; + } + return num; +} + +// for when the string is not null-terminated. +inline ptrdiff_t strcount(const char* buf, size_t len, char c) { + return strcount(buf, buf + len, c); +} + +inline ptrdiff_t strcount(const std::string& buf, char c) { + return strcount(buf.c_str(), buf.size(), c); +} + // The base class for a sorted associative container checker. TreeType is the // container type to check and CheckerType is the container type to check // against. TreeType is expected to be btree_{set,map,multiset,multimap} and @@ -127,9 +153,9 @@ class base_checker { IterType iter_check( IterType tree_iter, CheckerIterType checker_iter) const { if (tree_iter == tree_.end()) { - CHECK(checker_iter == checker_.end()); + EXPECT_EQ(checker_iter, checker_.end()); } else { - CHECK_EQ(*tree_iter, *checker_iter); + EXPECT_EQ(*tree_iter, *checker_iter); } return tree_iter; } @@ -137,9 +163,9 @@ class base_checker { IterType riter_check( IterType tree_iter, CheckerIterType checker_iter) const { if (tree_iter == tree_.rend()) { - CHECK(checker_iter == checker_.rend()); + EXPECT_EQ(checker_iter, checker_.rend()); } else { - CHECK_EQ(*tree_iter, *checker_iter); + EXPECT_EQ(*tree_iter, *checker_iter); } return tree_iter; } @@ -147,17 +173,17 @@ class base_checker { typename KeyOfValue::type key_of_value; const key_type &key = key_of_value(x); - CHECK_EQ(*find(key), x); + EXPECT_EQ(*find(key), x); lower_bound(key); upper_bound(key); equal_range(key); count(key); } void erase_check(const key_type &key) { - CHECK(tree_.find(key) == const_tree_.end()); - CHECK(const_tree_.find(key) == tree_.end()); - CHECK(tree_.equal_range(key).first == - const_tree_.equal_range(key).second); + EXPECT_TRUE(tree_.find(key) == const_tree_.end()); + EXPECT_TRUE(const_tree_.find(key) == tree_.end()); + EXPECT_TRUE(tree_.equal_range(key).first == + const_tree_.equal_range(key).second); } // Lookup routines. @@ -173,20 +199,20 @@ class base_checker { const_iterator upper_bound(const key_type &key) const { return iter_check(tree_.upper_bound(key), checker_.upper_bound(key)); } - pair equal_range(const key_type &key) { - pair equal_range(const key_type &key) { + std::pair checker_res = checker_.equal_range(key); - pair tree_res = tree_.equal_range(key); + std::pair tree_res = tree_.equal_range(key); iter_check(tree_res.first, checker_res.first); iter_check(tree_res.second, checker_res.second); return tree_res; } - pair equal_range(const key_type &key) const { - pair equal_range(const key_type &key) const { + std::pair checker_res = checker_.equal_range(key); - pair tree_res = tree_.equal_range(key); + std::pair tree_res = tree_.equal_range(key); iter_check(tree_res.first, checker_res.first); iter_check(tree_res.second, checker_res.second); return tree_res; @@ -199,7 +225,7 @@ class base_checker { } size_type count(const key_type &key) const { size_type res = checker_.count(key); - CHECK_EQ(res, tree_.count(key)); + EXPECT_EQ(res, tree_.count(key)); return res; } @@ -214,10 +240,10 @@ class base_checker { int erase(const key_type &key) { int size = tree_.size(); int res = checker_.erase(key); - CHECK_EQ(res, tree_.count(key)); - CHECK_EQ(res, tree_.erase(key)); - CHECK_EQ(tree_.count(key), 0); - CHECK_EQ(tree_.size(), size - res); + EXPECT_EQ(res, tree_.count(key)); + EXPECT_EQ(res, tree_.erase(key)); + EXPECT_EQ(tree_.count(key), 0); + EXPECT_EQ(tree_.size(), size - res); erase_check(key); return res; } @@ -233,9 +259,9 @@ class base_checker { ++checker_next; checker_.erase(checker_iter); iter = tree_.erase(iter); - CHECK_EQ(tree_.size(), checker_.size()); - CHECK_EQ(tree_.size(), size - 1); - CHECK_EQ(tree_.count(key), count - 1); + EXPECT_EQ(tree_.size(), checker_.size()); + EXPECT_EQ(tree_.size(), size - 1); + EXPECT_EQ(tree_.count(key), count - 1); if (count == 1) { erase_check(key); } @@ -258,8 +284,8 @@ class base_checker { } checker_.erase(checker_begin, checker_end); tree_.erase(begin, end); - CHECK_EQ(tree_.size(), checker_.size()); - CHECK_EQ(tree_.size(), size - count); + EXPECT_EQ(tree_.size(), checker_.size()); + EXPECT_EQ(tree_.size(), size - count); } // Utility routines. @@ -274,7 +300,7 @@ class base_checker { void verify() const { tree_.verify(); - CHECK_EQ(tree_.size(), checker_.size()); + EXPECT_EQ(tree_.size(), checker_.size()); // Move through the forward iterators using increment. typename CheckerType::const_iterator @@ -282,7 +308,7 @@ class base_checker { const_iterator tree_iter(tree_.begin()); for (; tree_iter != tree_.end(); ++tree_iter, ++checker_iter) { - CHECK_EQ(*tree_iter, *checker_iter); + EXPECT_EQ(*tree_iter, *checker_iter); } // Move through the forward iterators using decrement. @@ -291,8 +317,8 @@ class base_checker { --tree_iter; --checker_iter; } - CHECK(tree_iter == tree_.begin()); - CHECK(checker_iter == checker_.begin()); + EXPECT_TRUE(tree_iter == tree_.begin()); + EXPECT_TRUE(checker_iter == checker_.begin()); // Move through the reverse iterators using increment. typename CheckerType::const_reverse_iterator @@ -300,7 +326,7 @@ class base_checker { const_reverse_iterator tree_riter(tree_.rbegin()); for (; tree_riter != tree_.rend(); ++tree_riter, ++checker_riter) { - CHECK_EQ(*tree_riter, *checker_riter); + EXPECT_EQ(*tree_riter, *checker_riter); } // Move through the reverse iterators using decrement. @@ -309,8 +335,8 @@ class base_checker { --tree_riter; --checker_riter; } - CHECK(tree_riter == tree_.rbegin()); - CHECK(checker_riter == checker_.rbegin()); + EXPECT_EQ(tree_riter, tree_.rbegin()); + EXPECT_EQ(checker_riter, checker_.rbegin()); } // Access to the underlying btree. @@ -318,12 +344,12 @@ class base_checker { // Size routines. size_type size() const { - CHECK_EQ(tree_.size(), checker_.size()); + EXPECT_EQ(tree_.size(), checker_.size()); return tree_.size(); } size_type max_size() const { return tree_.max_size(); } bool empty() const { - CHECK_EQ(tree_.empty(), checker_.empty()); + EXPECT_EQ(tree_.empty(), checker_.empty()); return tree_.empty(); } size_type height() const { return tree_.height(); } @@ -367,25 +393,25 @@ class unique_checker : public base_checker { } // Insertion routines. - pair insert(const value_type &x) { + std::pair insert(const value_type &x) { int size = this->tree_.size(); - pair checker_res = + std::pair checker_res = this->checker_.insert(x); - pair tree_res = this->tree_.insert(x); - CHECK_EQ(*tree_res.first, *checker_res.first); - CHECK_EQ(tree_res.second, checker_res.second); - CHECK_EQ(this->tree_.size(), this->checker_.size()); - CHECK_EQ(this->tree_.size(), size + tree_res.second); + std::pair tree_res = this->tree_.insert(x); + EXPECT_EQ(*tree_res.first, *checker_res.first); + EXPECT_EQ(tree_res.second, checker_res.second); + EXPECT_EQ(this->tree_.size(), this->checker_.size()); + EXPECT_EQ(this->tree_.size(), size + tree_res.second); return tree_res; } iterator insert(iterator position, const value_type &x) { int size = this->tree_.size(); - pair checker_res = + std::pair checker_res = this->checker_.insert(x); iterator tree_res = this->tree_.insert(position, x); - CHECK_EQ(*tree_res, *checker_res.first); - CHECK_EQ(this->tree_.size(), this->checker_.size()); - CHECK_EQ(this->tree_.size(), size + checker_res.second); + EXPECT_EQ(*tree_res, *checker_res.first); + EXPECT_EQ(this->tree_.size(), this->checker_.size()); + EXPECT_EQ(this->tree_.size(), size + checker_res.second); return tree_res; } template @@ -428,18 +454,18 @@ class multi_checker : public base_checker { int size = this->tree_.size(); typename CheckerType::iterator checker_res = this->checker_.insert(x); iterator tree_res = this->tree_.insert(x); - CHECK_EQ(*tree_res, *checker_res); - CHECK_EQ(this->tree_.size(), this->checker_.size()); - CHECK_EQ(this->tree_.size(), size + 1); + EXPECT_EQ(*tree_res, *checker_res); + EXPECT_EQ(this->tree_.size(), this->checker_.size()); + EXPECT_EQ(this->tree_.size(), size + 1); return tree_res; } iterator insert(iterator position, const value_type &x) { int size = this->tree_.size(); typename CheckerType::iterator checker_res = this->checker_.insert(x); iterator tree_res = this->tree_.insert(position, x); - CHECK_EQ(*tree_res, *checker_res); - CHECK_EQ(this->tree_.size(), this->checker_.size()); - CHECK_EQ(this->tree_.size(), size + 1); + EXPECT_EQ(*tree_res, *checker_res); + EXPECT_EQ(this->tree_.size(), this->checker_.size()); + EXPECT_EQ(this->tree_.size(), size + 1); return tree_res; } template @@ -451,7 +477,7 @@ class multi_checker : public base_checker { }; char* GenerateDigits(char buf[16], int val, int maxval) { - DCHECK_LE(val, maxval); + EXPECT_LE(val, maxval); int p = 15; buf[p--] = 0; while (maxval > 0) { @@ -469,61 +495,48 @@ struct Generator { : maxval(m) { } K operator()(int i) const { - DCHECK_LE(i, maxval); + EXPECT_LE(i, maxval); return i; } }; template <> -struct Generator { +struct Generator { int maxval; Generator(int m) : maxval(m) { } - string operator()(int i) const { + std::string operator()(int i) const { char buf[16]; return GenerateDigits(buf, i, maxval); } }; -template <> -struct Generator { - int maxval; - Generator(int m) - : maxval(m) { - } - Cord operator()(int i) const { - char buf[16]; - return Cord(GenerateDigits(buf, i, maxval)); - } -}; - template -struct Generator > { - Generator::type> tgen; - Generator::type> ugen; +struct Generator > { + Generator::type> tgen; + Generator::type> ugen; Generator(int m) : tgen(m), ugen(m) { } - pair operator()(int i) const { - return make_pair(tgen(i), ugen(i)); + std::pair operator()(int i) const { + return std::make_pair(tgen(i), ugen(i)); } }; // Generate values for our tests and benchmarks. Value range is [0, maxval]. -const vector& GenerateNumbers(int n, int maxval) { - static ACMRandom rand(FLAGS_test_random_seed); - static vector values; - static set unique_values; +const std::vector& GenerateNumbers(int n, int maxval) { + static std::vector values; + static std::set unique_values; if (values.size() < n) { for (int i = values.size(); i < n; i++) { int value; do { - value = rand.Next() % (maxval + 1); + value = rand() % (maxval + 1); } while (unique_values.find(value) != unique_values.end()); values.push_back(value); @@ -537,13 +550,13 @@ const vector& GenerateNumbers(int n, int maxval) { // Generates values in the range // [0, 4 * min(FLAGS_benchmark_values, FLAGS_test_values)] template -vector GenerateValues(int n) { - int two_times_max = 2 * max(FLAGS_benchmark_values, FLAGS_test_values); +std::vector GenerateValues(int n) { + int two_times_max = 2 * std::max(FLAGS_benchmark_values, FLAGS_test_values); int four_times_max = 2 * two_times_max; - DCHECK_LE(n, two_times_max); - const vector &nums = GenerateNumbers(n, four_times_max); + EXPECT_LE(n, two_times_max); + const std::vector &nums = GenerateNumbers(n, four_times_max); Generator gen(four_times_max); - vector vec; + std::vector vec; for (int i = 0; i < n; i++) { vec.push_back(gen(nums[i])); @@ -553,44 +566,44 @@ vector GenerateValues(int n) { } template -double ContainerInfo(const set &s) { +double ContainerInfo(const std::set &s) { int sizeof_node = sizeof(std::_Rb_tree_node); int bytes_used = sizeof(s) + s.size() * sizeof_node; double bytes_per_value = (double) bytes_used / s.size(); - VLOG(1) << " size=" << s.size() + std::cout << " size=" << s.size() << " bytes-used=" << bytes_used << " bytes-per-value=" << bytes_per_value; return bytes_per_value; } template -double ContainerInfo(const multiset &s) { +double ContainerInfo(const std::multiset &s) { int sizeof_node = sizeof(std::_Rb_tree_node); int bytes_used = sizeof(s) + s.size() * sizeof_node; double bytes_per_value = (double) bytes_used / s.size(); - VLOG(1) << " size=" << s.size() + std::cout << " size=" << s.size() << " bytes-used=" << bytes_used << " bytes-per-value=" << bytes_per_value; return bytes_per_value; } template -double ContainerInfo(const map &m) { - int sizeof_node = sizeof(std::_Rb_tree_node >); +double ContainerInfo(const std::map &m) { + int sizeof_node = sizeof(std::_Rb_tree_node >); int bytes_used = sizeof(m) + m.size() * sizeof_node; double bytes_per_value = (double) bytes_used / m.size(); - VLOG(1) << " size=" << m.size() + std::cout << " size=" << m.size() << " bytes-used=" << bytes_used << " bytes-per-value=" << bytes_per_value; return bytes_per_value; } template -double ContainerInfo(const multimap &m) { - int sizeof_node = sizeof(std::_Rb_tree_node >); +double ContainerInfo(const std::multimap &m) { + int sizeof_node = sizeof(std::_Rb_tree_node >); int bytes_used = sizeof(m) + m.size() * sizeof_node; double bytes_per_value = (double) bytes_used / m.size(); - VLOG(1) << " size=" << m.size() + std::cout << " size=" << m.size() << " bytes-used=" << bytes_used << " bytes-per-value=" << bytes_per_value; return bytes_per_value; @@ -600,7 +613,7 @@ template double ContainerInfo(const btree_container

&b) { double bytes_used = sizeof(b) + b.bytes_used(); double bytes_per_value = (double) bytes_used / b.size(); - VLOG(1) << " size=" << b.size() + std::cout << " size=" << b.size() << " bytes-used=" << bytes_used << " bytes-per-value=" << bytes_per_value << " height=" << b.height() @@ -612,7 +625,7 @@ double ContainerInfo(const btree_container

&b) { } template -void DoTest(const char *name, T *b, const vector &values) { +void DoTest(const char *name, T *b, const std::vector &values) { typename KeyOfValue::type key_of_value; T &mutable_b = *b; @@ -631,22 +644,22 @@ void DoTest(const char *name, T *b, const vector &values) { // Test copy constructor. T b_copy(const_b); - CHECK_EQ(b_copy.size(), const_b.size()); - CHECK_LE(b_copy.height(), const_b.height()); - CHECK_LE(b_copy.internal_nodes(), const_b.internal_nodes()); - CHECK_LE(b_copy.leaf_nodes(), const_b.leaf_nodes()); + EXPECT_EQ(b_copy.size(), const_b.size()); + EXPECT_LE(b_copy.height(), const_b.height()); + EXPECT_LE(b_copy.internal_nodes(), const_b.internal_nodes()); + EXPECT_LE(b_copy.leaf_nodes(), const_b.leaf_nodes()); for (int i = 0; i < values.size(); ++i) { - CHECK_EQ(*b_copy.find(key_of_value(values[i])), values[i]); + EXPECT_EQ(*b_copy.find(key_of_value(values[i])), values[i]); } // Test range constructor. T b_range(const_b.begin(), const_b.end()); - CHECK_EQ(b_range.size(), const_b.size()); - CHECK_LE(b_range.height(), const_b.height()); - CHECK_LE(b_range.internal_nodes(), const_b.internal_nodes()); - CHECK_LE(b_range.leaf_nodes(), const_b.leaf_nodes()); + EXPECT_EQ(b_range.size(), const_b.size()); + EXPECT_LE(b_range.height(), const_b.height()); + EXPECT_LE(b_range.internal_nodes(), const_b.internal_nodes()); + EXPECT_LE(b_range.leaf_nodes(), const_b.leaf_nodes()); for (int i = 0; i < values.size(); ++i) { - CHECK_EQ(*b_range.find(key_of_value(values[i])), values[i]); + EXPECT_EQ(*b_range.find(key_of_value(values[i])), values[i]); } // Test range insertion for values that already exist. @@ -656,36 +669,36 @@ void DoTest(const char *name, T *b, const vector &values) { // Test range insertion for new values. b_range.clear(); b_range.insert(b_copy.begin(), b_copy.end()); - CHECK_EQ(b_range.size(), b_copy.size()); - CHECK_EQ(b_range.height(), b_copy.height()); - CHECK_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); - CHECK_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); + EXPECT_EQ(b_range.size(), b_copy.size()); + EXPECT_EQ(b_range.height(), b_copy.height()); + EXPECT_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); + EXPECT_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); for (int i = 0; i < values.size(); ++i) { - CHECK_EQ(*b_range.find(key_of_value(values[i])), values[i]); + EXPECT_EQ(*b_range.find(key_of_value(values[i])), values[i]); } // Test assignment to self. Nothing should change. b_range.operator=(b_range); - CHECK_EQ(b_range.size(), b_copy.size()); - CHECK_EQ(b_range.height(), b_copy.height()); - CHECK_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); - CHECK_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); + EXPECT_EQ(b_range.size(), b_copy.size()); + EXPECT_EQ(b_range.height(), b_copy.height()); + EXPECT_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); + EXPECT_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); // Test assignment of new values. b_range.clear(); b_range = b_copy; - CHECK_EQ(b_range.size(), b_copy.size()); - CHECK_EQ(b_range.height(), b_copy.height()); - CHECK_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); - CHECK_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); + EXPECT_EQ(b_range.size(), b_copy.size()); + EXPECT_EQ(b_range.height(), b_copy.height()); + EXPECT_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); + EXPECT_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); // Test swap. b_range.clear(); b_range.swap(b_copy); - CHECK_EQ(b_copy.size(), 0); - CHECK_EQ(b_range.size(), const_b.size()); + EXPECT_EQ(b_copy.size(), 0); + EXPECT_EQ(b_range.size(), const_b.size()); for (int i = 0; i < values.size(); ++i) { - CHECK_EQ(*b_range.find(key_of_value(values[i])), values[i]); + EXPECT_EQ(*b_range.find(key_of_value(values[i])), values[i]); } b_range.swap(b_copy); @@ -693,13 +706,13 @@ void DoTest(const char *name, T *b, const vector &values) { for (int i = 0; i < values.size(); ++i) { mutable_b.erase(key_of_value(values[i])); // Erasing a non-existent key should have no effect. - CHECK_EQ(mutable_b.erase(key_of_value(values[i])), 0); + EXPECT_EQ(mutable_b.erase(key_of_value(values[i])), 0); } const_b.verify(); - CHECK_EQ(const_b.internal_nodes(), 0); - CHECK_EQ(const_b.leaf_nodes(), 0); - CHECK_EQ(const_b.size(), 0); + EXPECT_EQ(const_b.internal_nodes(), 0); + EXPECT_EQ(const_b.leaf_nodes(), 0); + EXPECT_EQ(const_b.size(), 0); // Test erase via iterators. mutable_b = b_copy; @@ -708,9 +721,9 @@ void DoTest(const char *name, T *b, const vector &values) { } const_b.verify(); - CHECK_EQ(const_b.internal_nodes(), 0); - CHECK_EQ(const_b.leaf_nodes(), 0); - CHECK_EQ(const_b.size(), 0); + EXPECT_EQ(const_b.internal_nodes(), 0); + EXPECT_EQ(const_b.leaf_nodes(), 0); + EXPECT_EQ(const_b.size(), 0); // Test insert with hint. for (int i = 0; i < values.size(); i++) { @@ -721,13 +734,13 @@ void DoTest(const char *name, T *b, const vector &values) { // Test dumping of the btree to an ostream. There should be 1 line for each // value. - ostringstream strm; + std::stringstream strm; strm << mutable_b.tree(); - CHECK_EQ(mutable_b.size(), strcount(strm.str(), '\n')); + EXPECT_EQ(mutable_b.size(), strcount(strm.str(), '\n')); // Test range erase. mutable_b.erase(mutable_b.begin(), mutable_b.end()); - CHECK_EQ(mutable_b.size(), 0); + EXPECT_EQ(mutable_b.size(), 0); const_b.verify(); // First half. @@ -735,7 +748,7 @@ void DoTest(const char *name, T *b, const vector &values) { typename T::iterator mutable_iter_end = mutable_b.begin(); for (int i = 0; i < values.size() / 2; ++i) ++mutable_iter_end; mutable_b.erase(mutable_b.begin(), mutable_iter_end); - CHECK_EQ(mutable_b.size(), values.size() - values.size() / 2); + EXPECT_EQ(mutable_b.size(), values.size() - values.size() / 2); const_b.verify(); // Second half. @@ -743,7 +756,7 @@ void DoTest(const char *name, T *b, const vector &values) { typename T::iterator mutable_iter_begin = mutable_b.begin(); for (int i = 0; i < values.size() / 2; ++i) ++mutable_iter_begin; mutable_b.erase(mutable_iter_begin, mutable_b.end()); - CHECK_EQ(mutable_b.size(), values.size() / 2); + EXPECT_EQ(mutable_b.size(), values.size() / 2); const_b.verify(); // Second quarter. @@ -753,7 +766,7 @@ void DoTest(const char *name, T *b, const vector &values) { mutable_iter_end = mutable_iter_begin; for (int i = 0; i < values.size() / 4; ++i) ++mutable_iter_end; mutable_b.erase(mutable_iter_begin, mutable_iter_end); - CHECK_EQ(mutable_b.size(), values.size() - values.size() / 4); + EXPECT_EQ(mutable_b.size(), values.size() - values.size() / 4); const_b.verify(); mutable_b.clear(); @@ -770,62 +783,62 @@ void ConstTest() { // Insert a single value into the container and test looking it up. value_type value = Generator(2)(2); mutable_b.insert(value); - CHECK(mutable_b.find(key_of_value(value)) != const_b.end()); - CHECK(const_b.find(key_of_value(value)) != mutable_b.end()); - CHECK_EQ(*const_b.lower_bound(key_of_value(value)), value); - CHECK(const_b.upper_bound(key_of_value(value)) == const_b.end()); - CHECK_EQ(*const_b.equal_range(key_of_value(value)).first, value); + EXPECT_TRUE(mutable_b.find(key_of_value(value)) != const_b.end()); + EXPECT_TRUE(const_b.find(key_of_value(value)) != mutable_b.end()); + EXPECT_EQ(*const_b.lower_bound(key_of_value(value)), value); + EXPECT_TRUE(const_b.upper_bound(key_of_value(value)) == const_b.end()); + EXPECT_EQ(*const_b.equal_range(key_of_value(value)).first, value); // We can only create a non-const iterator from a non-const container. typename T::iterator mutable_iter(mutable_b.begin()); - CHECK(mutable_iter == const_b.begin()); - CHECK(mutable_iter != const_b.end()); - CHECK(const_b.begin() == mutable_iter); - CHECK(const_b.end() != mutable_iter); + EXPECT_TRUE(mutable_iter == const_b.begin()); + EXPECT_TRUE(mutable_iter != const_b.end()); + EXPECT_TRUE(const_b.begin() == mutable_iter); + EXPECT_TRUE(const_b.end() != mutable_iter); typename T::reverse_iterator mutable_riter(mutable_b.rbegin()); - CHECK(mutable_riter == const_b.rbegin()); - CHECK(mutable_riter != const_b.rend()); - CHECK(const_b.rbegin() == mutable_riter); - CHECK(const_b.rend() != mutable_riter); + EXPECT_TRUE(mutable_riter == const_b.rbegin()); + EXPECT_TRUE(mutable_riter != const_b.rend()); + EXPECT_TRUE(const_b.rbegin() == mutable_riter); + EXPECT_TRUE(const_b.rend() != mutable_riter); // We can create a const iterator from a non-const iterator. typename T::const_iterator const_iter(mutable_iter); - CHECK(const_iter == mutable_b.begin()); - CHECK(const_iter != mutable_b.end()); - CHECK(mutable_b.begin() == const_iter); - CHECK(mutable_b.end() != const_iter); + EXPECT_TRUE(const_iter == mutable_b.begin()); + EXPECT_TRUE(const_iter != mutable_b.end()); + EXPECT_TRUE(mutable_b.begin() == const_iter); + EXPECT_TRUE(mutable_b.end() != const_iter); typename T::const_reverse_iterator const_riter(mutable_riter); - CHECK(const_riter == mutable_b.rbegin()); - CHECK(const_riter != mutable_b.rend()); - CHECK(mutable_b.rbegin() == const_riter); - CHECK(mutable_b.rend() != const_riter); + EXPECT_EQ(const_riter, mutable_b.rbegin()); + EXPECT_TRUE(const_riter != mutable_b.rend()); + EXPECT_EQ(mutable_b.rbegin(), const_riter); + EXPECT_TRUE(mutable_b.rend() != const_riter); // Make sure various methods can be invoked on a const container. const_b.verify(); - CHECK(!const_b.empty()); - CHECK_EQ(const_b.size(), 1); - CHECK_GT(const_b.max_size(), 0); - CHECK_EQ(const_b.height(), 1); - CHECK_EQ(const_b.count(key_of_value(value)), 1); - CHECK_EQ(const_b.internal_nodes(), 0); - CHECK_EQ(const_b.leaf_nodes(), 1); - CHECK_EQ(const_b.nodes(), 1); - CHECK_GT(const_b.bytes_used(), 0); - CHECK_GT(const_b.fullness(), 0); - CHECK_GT(const_b.overhead(), 0); + EXPECT_FALSE(const_b.empty()); + EXPECT_EQ(const_b.size(), 1); + EXPECT_GT(const_b.max_size(), 0); + EXPECT_EQ(const_b.height(), 1); + EXPECT_EQ(const_b.count(key_of_value(value)), 1); + EXPECT_EQ(const_b.internal_nodes(), 0); + EXPECT_EQ(const_b.leaf_nodes(), 1); + EXPECT_EQ(const_b.nodes(), 1); + EXPECT_GT(const_b.bytes_used(), 0); + EXPECT_GT(const_b.fullness(), 0); + EXPECT_GT(const_b.overhead(), 0); } template void BtreeTest() { ConstTest(); - typedef typename base::remove_const::type V; - vector random_values = GenerateValues(FLAGS_test_values); + typedef typename std::remove_const::type V; + std::vector random_values = GenerateValues(FLAGS_test_values); unique_checker container; // Test key insertion/deletion in sorted order. - vector sorted_values(random_values); + std::vector sorted_values(random_values); sort(sorted_values.begin(), sorted_values.end()); DoTest("sorted: ", &container, sorted_values); @@ -841,13 +854,13 @@ template void BtreeMultiTest() { ConstTest(); - typedef typename base::remove_const::type V; - const vector& random_values = GenerateValues(FLAGS_test_values); + typedef typename std::remove_const::type V; + const std::vector& random_values = GenerateValues(FLAGS_test_values); multi_checker container; // Test keys in sorted order. - vector sorted_values(random_values); + std::vector sorted_values(random_values); sort(sorted_values.begin(), sorted_values.end()); DoTest("sorted: ", &container, sorted_values); @@ -859,25 +872,64 @@ void BtreeMultiTest() { DoTest("random: ", &container, random_values); // Test keys in random order w/ duplicates. - vector duplicate_values(random_values); + std::vector duplicate_values(random_values); duplicate_values.insert( duplicate_values.end(), random_values.begin(), random_values.end()); DoTest("duplicates:", &container, duplicate_values); // Test all identical keys. - vector identical_values(100); + std::vector identical_values(100); fill(identical_values.begin(), identical_values.end(), Generator(2)(2)); DoTest("identical: ", &container, identical_values); } +template > +class TestAllocator : public Alloc { + public: + typedef typename Alloc::pointer pointer; + typedef typename Alloc::size_type size_type; + + TestAllocator() : bytes_used_(NULL) { } + TestAllocator(int64_t *bytes_used) : bytes_used_(bytes_used) { } + + // Constructor used for rebinding + template + TestAllocator(const TestAllocator& x) + : Alloc(x), + bytes_used_(x.bytes_used()) { + } + + pointer allocate(size_type n, std::allocator::const_pointer hint = 0) { + EXPECT_TRUE(bytes_used_ != NULL); + *bytes_used_ += n * sizeof(T); + return Alloc::allocate(n, hint); + } + + void deallocate(pointer p, size_type n) { + Alloc::deallocate(p, n); + EXPECT_TRUE(bytes_used_ != NULL); + *bytes_used_ -= n * sizeof(T); + } + + // Rebind allows an allocator to be used for a different type + template struct rebind { + typedef TestAllocator::other> other; + }; + + int64_t* bytes_used() const { return bytes_used_; } + + private: + int64_t *bytes_used_; +}; + template -void BtreeArenaTest() { +void BtreeAllocatorTest() { typedef typename T::value_type value_type; - UnsafeArena arena1(1000); - UnsafeArena arena2(1000); - T b1(typename T::key_compare(), &arena1); - T b2(typename T::key_compare(), &arena2); + int64_t alloc1 = 0; + int64_t alloc2 = 0; + T b1(typename T::key_compare(), &alloc1); + T b2(typename T::key_compare(), &alloc2); // This should swap the allocators! swap(b1, b2); @@ -886,9 +938,9 @@ void BtreeArenaTest() { b1.insert(Generator(1000)(i)); } - // We should have allocated out of arena2! - CHECK_LE(b1.bytes_used(), arena2.status().bytes_allocated()); - CHECK_GT(arena2.block_count(), arena1.block_count()); + // We should have allocated out of alloc2! + EXPECT_LE(b1.bytes_used(), alloc2 + sizeof(b1)); + EXPECT_GT(alloc2, alloc1); } template @@ -906,15 +958,15 @@ void BtreeMapTest() { value_type v = Generator(1000)(i); b[v.first] = v.second; } - CHECK_EQ(b.size(), 1000); + EXPECT_EQ(b.size(), 1000); // Test whether we can use the "->" operator on iterators and // reverse_iterators. This stresses the btree_map_params::pair_pointer // mechanism. - CHECK_EQ(b.begin()->first, Generator(1000)(0).first); - CHECK_EQ(b.begin()->second, Generator(1000)(0).second); - CHECK_EQ(b.rbegin()->first, Generator(1000)(999).first); - CHECK_EQ(b.rbegin()->second, Generator(1000)(999).second); + EXPECT_EQ(b.begin()->first, Generator(1000)(0).first); + EXPECT_EQ(b.begin()->second, Generator(1000)(0).second); + EXPECT_EQ(b.rbegin()->first, Generator(1000)(999).first); + EXPECT_EQ(b.rbegin()->second, Generator(1000)(999).second); } template @@ -925,6 +977,5 @@ void BtreeMultiMapTest() { } } // namespace btree -} // namespace util #endif // UTIL_BTREE_BTREE_TEST_H__ diff --git a/btree_test_flags.cc b/btree_test_flags.cc index 512c24e1..d43d9250 100644 --- a/btree_test_flags.cc +++ b/btree_test_flags.cc @@ -1,7 +1,7 @@ // Copyright 2007 Google Inc. All Rights Reserved. // Author: pmattis@google.com (Peter Mattis) -#include "base/commandlineflags.h" +#include "gflags/gflags.h" DEFINE_int32(test_values, 10000, "The number of values to use for tests."); diff --git a/safe_btree.h b/safe_btree.h index 2a354232..01cf9faf 100644 --- a/safe_btree.h +++ b/safe_btree.h @@ -21,11 +21,8 @@ #include #include -#include "base/integral_types.h" -#include "base/logging.h" -#include "util/btree/btree.h" +#include "btree.h" -namespace util { namespace btree { template @@ -77,7 +74,7 @@ class safe_btree_iterator { } Tree* tree() const { return tree_; } - int64 generation() const { return generation_; } + int64_t generation() const { return generation_; } Iterator* mutable_iter() const { if (generation_ != tree_->generation()) { @@ -114,13 +111,13 @@ class safe_btree_iterator { // This reference value is potentially invalidated by any non-const // method on the tree; it is NOT safe. reference operator*() const { - DCHECK_GT(generation_, 0); + assert(generation_ > 0); return iter().operator*(); } // This pointer value is potentially invalidated by any non-const // method on the tree; it is NOT safe. pointer operator->() const { - DCHECK_GT(generation_, 0); + assert(generation_ > 0); return iter().operator->(); } @@ -148,7 +145,7 @@ class safe_btree_iterator { private: // The generation of the tree when "iter" was updated. - mutable int64 generation_; + mutable int64_t generation_; // The key the iterator points to. mutable key_type key_; // The underlying iterator. @@ -236,14 +233,14 @@ class safe_btree { const_iterator upper_bound(const key_type &key) const { return const_iterator(this, tree_.upper_bound(key)); } - pair equal_range(const key_type &key) { - pair p = tree_.equal_range(key); - return make_pair(iterator(this, p.first), + std::pair equal_range(const key_type &key) { + std::pair p = tree_.equal_range(key); + return std::make_pair(iterator(this, p.first), iterator(this, p.second)); } - pair equal_range(const key_type &key) const { - pair p = tree_.equal_range(key); - return make_pair(const_iterator(this, p.first), + std::pair equal_range(const key_type &key) const { + std::pair p = tree_.equal_range(key); + return std::make_pair(const_iterator(this, p.first), const_iterator(this, p.second)); } iterator find_unique(const key_type &key) { @@ -267,15 +264,15 @@ class safe_btree { // Insertion routines. template - pair insert_unique(const key_type &key, ValuePointer value) { - pair p = tree_.insert_unique(key, value); + std::pair insert_unique(const key_type &key, ValuePointer value) { + std::pair p = tree_.insert_unique(key, value); generation_ += p.second; - return make_pair(iterator(this, p.first), p.second); + return std::make_pair(iterator(this, p.first), p.second); } - pair insert_unique(const value_type &v) { - pair p = tree_.insert_unique(v); + std::pair insert_unique(const value_type &v) { + std::pair p = tree_.insert_unique(v); generation_ += p.second; - return make_pair(iterator(this, p.first), p.second); + return std::make_pair(iterator(this, p.first), p.second); } iterator insert_unique(iterator position, const value_type &v) { tree_iterator tree_pos = position.iter(); @@ -351,13 +348,13 @@ class safe_btree { ++x.generation_; tree_.swap(x.tree_); } - void dump(ostream &os) const { + void dump(std::ostream &os) const { tree_.dump(os); } void verify() const { tree_.verify(); } - int64 generation() const { + int64_t generation() const { return generation_; } key_compare key_comp() const { return tree_.key_comp(); } @@ -379,10 +376,9 @@ class safe_btree { private: btree_type tree_; - int64 generation_; + int64_t generation_; }; } // namespace btree -} // namespace util #endif // UTIL_BTREE_SAFE_BTREE_H__ diff --git a/safe_btree_map.h b/safe_btree_map.h index 11274930..b8aa787e 100644 --- a/safe_btree_map.h +++ b/safe_btree_map.h @@ -22,17 +22,16 @@ #include #include -#include "util/btree/btree_container.h" -#include "util/btree/btree_map.h" -#include "util/btree/safe_btree.h" +#include "btree_container.h" +#include "btree_map.h" +#include "safe_btree.h" -namespace util { namespace btree { // The safe_btree_map class is needed mainly for its constructors. template , - typename Alloc = std::allocator >, + typename Compare = std::less, + typename Alloc = std::allocator >, int TargetNodeSize = 256> class safe_btree_map : public btree_map_container< safe_btree > > { @@ -75,6 +74,5 @@ inline void swap(safe_btree_map &x, } } // namespace btree -} // namespace util #endif // UTIL_BTREE_SAFE_BTREE_MAP_H__ diff --git a/safe_btree_set.h b/safe_btree_set.h index 72ed2d91..de8f823a 100644 --- a/safe_btree_set.h +++ b/safe_btree_set.h @@ -22,16 +22,15 @@ #include #include -#include "util/btree/btree_container.h" -#include "util/btree/btree_set.h" -#include "util/btree/safe_btree.h" +#include "btree_container.h" +#include "btree_set.h" +#include "safe_btree.h" -namespace util { namespace btree { // The safe_btree_set class is needed mainly for its constructors. template , + typename Compare = std::less, typename Alloc = std::allocator, int TargetNodeSize = 256> class safe_btree_set : public btree_unique_container< @@ -74,6 +73,5 @@ inline void swap(safe_btree_set &x, } } // namespace btree -} // namespace util #endif // UTIL_BTREE_SAFE_BTREE_SET_H__ diff --git a/safe_btree_test.cc b/safe_btree_test.cc index cf179b28..4965916b 100644 --- a/safe_btree_test.cc +++ b/safe_btree_test.cc @@ -11,58 +11,50 @@ #include #include -#include "base/arena-inl.h" -#include "base/init_google.h" -#include "base/integral_types.h" -#include "base/logging.h" -#include "strings/cord.h" -#include "testing/base/public/gunit.h" -#include "util/btree/btree_test.h" -#include "util/btree/safe_btree_map.h" -#include "util/btree/safe_btree_set.h" +#include "gtest/gtest.h" +#include "btree_test.h" +#include "safe_btree_map.h" +#include "safe_btree_set.h" class UnsafeArena; -namespace util { namespace btree { namespace { template void SetTest() { - typedef ArenaAllocator ArenaAlloc; - BtreeTest, allocator, N>, set >(); - BtreeArenaTest, ArenaAlloc, N> >(); + typedef TestAllocator TestAlloc; + BtreeTest, std::allocator, N>, std::set >(); + BtreeAllocatorTest, TestAlloc, N> >(); } template void MapTest() { - typedef ArenaAllocator ArenaAlloc; - BtreeTest, allocator, N>, map >(); - BtreeArenaTest, ArenaAlloc, N> >(); - BtreeMapTest, allocator, N> >(); + typedef TestAllocator TestAlloc; + BtreeTest, std::allocator, N>, std::map >(); + BtreeAllocatorTest, TestAlloc, N> >(); + BtreeMapTest, std::allocator, N> >(); } -TEST(SafeBtree, set_int32_32) { SetTest(); } -TEST(SafeBtree, set_int32_64) { SetTest(); } -TEST(SafeBtree, set_int32_128) { SetTest(); } -TEST(SafeBtree, set_int32_256) { SetTest(); } -TEST(SafeBtree, set_int64_256) { SetTest(); } -TEST(SafeBtree, set_string_256) { SetTest(); } -TEST(SafeBtree, set_cord_256) { SetTest(); } -TEST(SafeBtree, set_pair_256) { SetTest, 256>(); } -TEST(SafeBtree, map_int32_256) { MapTest(); } -TEST(SafeBtree, map_int64_256) { MapTest(); } -TEST(SafeBtree, map_string_256) { MapTest(); } -TEST(SafeBtree, map_cord_256) { MapTest(); } -TEST(SafeBtree, map_pair_256) { MapTest, 256>(); } +TEST(SafeBtree, set_int32_32) { SetTest(); } +TEST(SafeBtree, set_int32_64) { SetTest(); } +TEST(SafeBtree, set_int32_128) { SetTest(); } +TEST(SafeBtree, set_int32_256) { SetTest(); } +TEST(SafeBtree, set_int64_256) { SetTest(); } +TEST(SafeBtree, set_string_256) { SetTest(); } +TEST(SafeBtree, set_pair_256) { SetTest, 256>(); } +TEST(SafeBtree, map_int32_256) { MapTest(); } +TEST(SafeBtree, map_int64_256) { MapTest(); } +TEST(SafeBtree, map_string_256) { MapTest(); } +TEST(SafeBtree, map_pair_256) { MapTest, 256>(); } TEST(SafeBtree, Comparison) { const int kSetSize = 1201; - safe_btree_set my_set; + safe_btree_set my_set; for (int i = 0; i < kSetSize; ++i) { my_set.insert(i); } - safe_btree_set my_set_copy(my_set); + safe_btree_set my_set_copy(my_set); EXPECT_TRUE(my_set_copy == my_set); EXPECT_TRUE(my_set == my_set_copy); EXPECT_FALSE(my_set_copy != my_set); @@ -80,17 +72,17 @@ TEST(SafeBtree, Comparison) { EXPECT_TRUE(my_set_copy != my_set); EXPECT_TRUE(my_set != my_set_copy); - safe_btree_map my_map; + safe_btree_map my_map; for (int i = 0; i < kSetSize; ++i) { - my_map[string(i, 'a')] = i; + my_map[std::string(i, 'a')] = i; } - safe_btree_map my_map_copy(my_map); + safe_btree_map my_map_copy(my_map); EXPECT_TRUE(my_map_copy == my_map); EXPECT_TRUE(my_map == my_map_copy); EXPECT_FALSE(my_map_copy != my_map); EXPECT_FALSE(my_map != my_map_copy); - ++my_map_copy[string(7, 'a')]; + ++my_map_copy[std::string(7, 'a')]; EXPECT_FALSE(my_map_copy == my_map); EXPECT_FALSE(my_map == my_map_copy); EXPECT_TRUE(my_map_copy != my_map); @@ -103,7 +95,7 @@ TEST(SafeBtree, Comparison) { EXPECT_TRUE(my_map_copy != my_map); EXPECT_TRUE(my_map != my_map_copy); - my_map.erase(string(kSetSize - 1, 'a')); + my_map.erase(std::string(kSetSize - 1, 'a')); EXPECT_FALSE(my_map_copy == my_map); EXPECT_FALSE(my_map == my_map_copy); EXPECT_TRUE(my_map_copy != my_map); @@ -112,10 +104,3 @@ TEST(SafeBtree, Comparison) { } // namespace } // namespace btree -} // namespace util - -int main(int argc, char **argv) { - FLAGS_logtostderr = true; - InitGoogle(argv[0], &argc, &argv, true); - return RUN_ALL_TESTS(); -} -- 2.30.2 From 2fd7ed36a9031c794f3247b6952308a35cea8079 Mon Sep 17 00:00:00 2001 From: josh macdonald Date: Sun, 13 Jan 2013 00:16:02 -0800 Subject: [PATCH 08/91] Make btree_bench work; compiled & tested on OS X 10.7.5 using gcc 4.7.2 -std=c++11 --- CMakeLists.txt | 6 +- btree_bench.cc | 240 ++++++++++++++++++++++++++++++++++--------------- btree_test.h | 59 ------------ 3 files changed, 171 insertions(+), 134 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e908f4a4..79c67ac7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,8 +3,8 @@ cmake_minimum_required(VERSION 2.6) project(cppbtree CXX) option(build_tests "Build B-tree tests" OFF) -add_definitions(-std=c++0x) -set(CMAKE_CXX_FLAGS "-g") +add_definitions(-std=c++11) +set(CMAKE_CXX_FLAGS "-g -O2") # CMake doesn't have a way to pure template library, add_library(cppbtree btree.h btree_map.h btree_set.h safe_btree.h safe_btree_map.h safe_btree_set.h) @@ -19,5 +19,5 @@ if(build_tests) add_executable(btree_bench btree_bench.cc btree_test_flags.cc) target_link_libraries(btree_test gtest_main gflags) target_link_libraries(safe_btree_test gtest_main gflags) - target_link_libraries(btree_bench gflags) + target_link_libraries(btree_bench gflags gtest) endif() diff --git a/btree_bench.cc b/btree_bench.cc index 1b45582a..b803ad38 100644 --- a/btree_bench.cc +++ b/btree_bench.cc @@ -3,36 +3,160 @@ // Author: pmattis@google.com (Peter Mattis) #include +#include #include #include #include #include #include +#include +#include #include -#include "base/commandlineflags.h" -#include "base/init_google.h" -#include "base/integral_types.h" -#include "base/logging.h" -#include "base/stringprintf.h" -#include "base/type_traits.h" -#include "strings/cord.h" -#include "testing/base/public/benchmark.h" -#include "testing/base/public/googletest.h" -#include "util/btree/btree_map.h" -#include "util/btree/btree_set.h" -#include "util/btree/btree_test.h" -#include "util/random/acmrandom.h" +#include "gflags/gflags.h" +#include "btree_map.h" +#include "btree_set.h" +#include "btree_test.h" -DECLARE_int32(benchmark_max_iters); +DEFINE_int32(test_random_seed, 123456789, "Seed for srand()"); +DEFINE_int32(benchmark_max_iters, 10000000, "Maximum test iterations"); +DEFINE_int32(benchmark_min_iters, 100, "Minimum test iterations"); +DEFINE_int32(benchmark_target_seconds, 1, + "Attempt to benchmark for this many seconds"); + +using std::allocator; +using std::less; +using std::map; +using std::max; +using std::min; +using std::multimap; +using std::multiset; +using std::set; +using std::string; +using std::vector; namespace btree { namespace { +struct RandGen { + typedef ptrdiff_t result_type; + RandGen(result_type seed) { + srand(seed); + } + result_type operator()(result_type l) { + return rand() % l; + } +}; + +struct BenchmarkRun { + BenchmarkRun(const char *name, void (*func)(int)); + void Run(); + void Stop(); + void Start(); + void Reset(); + + BenchmarkRun *next_benchmark; + const char *benchmark_name; + void (*benchmark_func)(int); + int64_t accum_micros; + int64_t last_started; +}; + +BenchmarkRun *first_benchmark; +BenchmarkRun *current_benchmark; + +int64_t get_micros () { + timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000000 + tv.tv_usec; +} + +BenchmarkRun::BenchmarkRun(const char *name, void (*func)(int)) + : next_benchmark(first_benchmark), + benchmark_name(name), + benchmark_func(func), + accum_micros(0), + last_started(0) { + first_benchmark = this; +} + +#define BTREE_BENCHMARK(name) \ + BTREE_BENCHMARK2(#name, name, __COUNTER__) +#define BTREE_BENCHMARK2(name, func, counter) \ + BTREE_BENCHMARK3(name, func, counter) +#define BTREE_BENCHMARK3(name, func, counter) \ + BenchmarkRun bench ## counter (name, func) + +void StopBenchmarkTiming() { + current_benchmark->Stop(); +} + +void StartBenchmarkTiming() { + current_benchmark->Start(); +} + +void RunBenchmarks() { + for (BenchmarkRun *bench = first_benchmark; bench; + bench = bench->next_benchmark) { + bench->Run(); + } +} + +void BenchmarkRun::Start() { + assert(!last_started); + last_started = get_micros(); +} + +void BenchmarkRun::Stop() { + if (last_started == 0) { + return; + } + accum_micros += get_micros() - last_started; + last_started = 0; +} + +void BenchmarkRun::Reset() { + last_started = 0; + accum_micros = 0; +} + +void BenchmarkRun::Run() { + assert(current_benchmark == NULL); + current_benchmark = this; + int iters = FLAGS_benchmark_min_iters; + for (;;) { + Reset(); + Start(); + benchmark_func(iters); + Stop(); + if (accum_micros > FLAGS_benchmark_target_seconds * 1000000 || + iters >= FLAGS_benchmark_max_iters) { + break; + } else if (accum_micros == 0) { + iters *= 100; + } else { + int64_t target_micros = FLAGS_benchmark_target_seconds * 1000000; + iters = target_micros * iters / accum_micros; + } + iters = min(iters, FLAGS_benchmark_max_iters); + } + fprintf(stdout, "%s\t%qu\t%qu\n", + benchmark_name, + accum_micros * 1000 / iters, + iters); + current_benchmark = NULL; +} + +// Used to avoid compiler optimizations for these benchmarks. +template +void sink(const T& t0) { + volatile T t = t0; +} + // Benchmark insertion of values into a container. template void BM_Insert(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; typename KeyOfValue::type key_of_value; // Disable timing while we perform some initialization. @@ -43,7 +167,6 @@ void BM_Insert(int n) { for (int i = 0; i < values.size(); i++) { container.insert(values[i]); } - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); for (int i = 0; i < n; ) { // Remove and re-insert 10% of the keys @@ -70,7 +193,7 @@ void BM_Insert(int n) { // Benchmark lookup of values in a container. template void BM_Lookup(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; typename KeyOfValue::type key_of_value; // Disable timing while we perform some initialization. @@ -82,7 +205,6 @@ void BM_Lookup(int n) { for (int i = 0; i < values.size(); i++) { container.insert(values[i]); } - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); V r = V(); @@ -95,7 +217,7 @@ void BM_Lookup(int n) { StopBenchmarkTiming(); - VLOG(4) << r; // Keep compiler from optimizing away r. + sink(r); // Keep compiler from optimizing away r. } // Benchmark lookup of values in a full container, meaning that values @@ -103,7 +225,7 @@ void BM_Lookup(int n) { // yields a full tree. template void BM_FullLookup(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; typename KeyOfValue::type key_of_value; // Disable timing while we perform some initialization. @@ -117,7 +239,6 @@ void BM_FullLookup(int n) { for (int i = 0; i < sorted.size(); i++) { container.insert(sorted[i]); } - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); V r = V(); @@ -130,13 +251,13 @@ void BM_FullLookup(int n) { StopBenchmarkTiming(); - VLOG(4) << r; // Keep compiler from optimizing away r. + sink(r); // Keep compiler from optimizing away r. } // Benchmark deletion of values from a container. template void BM_Delete(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; typename KeyOfValue::type key_of_value; // Disable timing while we perform some initialization. @@ -147,7 +268,6 @@ void BM_Delete(int n) { for (int i = 0; i < values.size(); i++) { container.insert(values[i]); } - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); for (int i = 0; i < n; ) { // Remove and re-insert 10% of the keys @@ -179,12 +299,12 @@ void BM_Delete(int n) { // value constructors. template void BM_QueueAddRem(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; typename KeyOfValue::type key_of_value; // Disable timing while we perform some initialization. StopBenchmarkTiming(); - CHECK(FLAGS_benchmark_values % 2 == 0); + assert(FLAGS_benchmark_values % 2 == 0); T container; @@ -197,7 +317,7 @@ void BM_QueueAddRem(int n) { add_keys[i] = i; } - ACMRandom rand(FLAGS_test_random_seed); + RandGen rand(FLAGS_test_random_seed); random_shuffle(remove_keys.begin(), remove_keys.end(), rand); random_shuffle(add_keys.begin(), add_keys.end(), rand); @@ -229,27 +349,25 @@ void BM_QueueAddRem(int n) { } int e = container.erase(key_of_value(g(offset - half + remove_keys[idx]))); - DCHECK(e == 1); + assert(e == 1); container.insert(g(offset + half + add_keys[idx])); } StopBenchmarkTiming(); - - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); } // Mixed insertion and deletion in the same range using pre-constructed values. template void BM_MixedAddRem(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; typename KeyOfValue::type key_of_value; // Disable timing while we perform some initialization. StopBenchmarkTiming(); - CHECK(FLAGS_benchmark_values % 2 == 0); + assert(FLAGS_benchmark_values % 2 == 0); T container; - ACMRandom rand(FLAGS_test_random_seed); + RandGen rand(FLAGS_test_random_seed); vector values = GenerateValues(FLAGS_benchmark_values * 2); @@ -282,20 +400,18 @@ void BM_MixedAddRem(int n) { } int e = container.erase(key_of_value(values[remove_keys[idx]])); - DCHECK(e == 1); + assert(e == 1); container.insert(values[add_keys[idx]]); } StopBenchmarkTiming(); - - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); } // Insertion at end, removal from the beginning. This benchmark // counts two value constructors. template void BM_Fifo(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; // Disable timing while we perform some initialization. StopBenchmarkTiming(); @@ -315,14 +431,12 @@ void BM_Fifo(int n) { } StopBenchmarkTiming(); - - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); } // Iteration (forward) through the tree template void BM_FwdIter(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; // Disable timing while we perform some initialization. StopBenchmarkTiming(); @@ -352,30 +466,24 @@ void BM_FwdIter(int n) { StopBenchmarkTiming(); - VLOG(4) << r; // Keep compiler from optimizing away r. - - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); + sink(r); // Keep compiler from optimizing away r. } -typedef set stl_set_int32; -typedef set stl_set_int64; +typedef set stl_set_int32; +typedef set stl_set_int64; typedef set stl_set_string; -typedef set stl_set_cord; -typedef map stl_map_int32; -typedef map stl_map_int64; +typedef map stl_map_int32; +typedef map stl_map_int64; typedef map stl_map_string; -typedef map stl_map_cord; -typedef multiset stl_multiset_int32; -typedef multiset stl_multiset_int64; +typedef multiset stl_multiset_int32; +typedef multiset stl_multiset_int64; typedef multiset stl_multiset_string; -typedef multiset stl_multiset_cord; -typedef multimap stl_multimap_int32; -typedef multimap stl_multimap_int64; +typedef multimap stl_multimap_int32; +typedef multimap stl_multimap_int64; typedef multimap stl_multimap_string; -typedef multimap stl_multimap_cord; #define MY_BENCHMARK_TYPES2(value, name, size) \ typedef btree ## _set, allocator, size> \ @@ -405,21 +513,15 @@ typedef multimap stl_multimap_cord; MY_BENCHMARK_TYPES2(value, name, 1536); \ MY_BENCHMARK_TYPES2(value, name, 2048) -MY_BENCHMARK_TYPES(int32, int32); -MY_BENCHMARK_TYPES(int64, int64); +MY_BENCHMARK_TYPES(int32_t, int32); +MY_BENCHMARK_TYPES(int64_t, int64); MY_BENCHMARK_TYPES(string, string); -MY_BENCHMARK_TYPES(Cord, cord); #define MY_BENCHMARK4(type, name, func) \ void BM_ ## type ## _ ## name(int n) { BM_ ## func (n); } \ - BENCHMARK(BM_ ## type ## _ ## name) + BTREE_BENCHMARK(BM_ ## type ## _ ## name) -// Define NODESIZE_TESTING when running btree_perf.py. You need to do -// a local build or raise the distcc timeout, it takes about 5 minutes -// to build: -// -// blaze build --copts=-DNODESIZE_TESTING --cc_strategy=local -// --compilation_mode=opt util/btree/btree_test +// Define NODESIZE_TESTING when running btree_perf.py. #ifdef NODESIZE_TESTING #define MY_BENCHMARK3(tree, type, name, func) \ @@ -465,8 +567,6 @@ MY_BENCHMARK(set_int64); MY_BENCHMARK(map_int64); MY_BENCHMARK(set_string); MY_BENCHMARK(map_string); -MY_BENCHMARK(set_cord); -MY_BENCHMARK(map_cord); MY_BENCHMARK(multiset_int32); MY_BENCHMARK(multimap_int32); @@ -474,15 +574,11 @@ MY_BENCHMARK(multiset_int64); MY_BENCHMARK(multimap_int64); MY_BENCHMARK(multiset_string); MY_BENCHMARK(multimap_string); -MY_BENCHMARK(multiset_cord); -MY_BENCHMARK(multimap_cord); } // namespace } // namespace btree int main(int argc, char **argv) { - FLAGS_logtostderr = true; - InitGoogle(argv[0], &argc, &argv, true); - RunSpecifiedBenchmarks(); + btree::RunBenchmarks(); return 0; } diff --git a/btree_test.h b/btree_test.h index c368573e..01ab0fe0 100644 --- a/btree_test.h +++ b/btree_test.h @@ -565,65 +565,6 @@ std::vector GenerateValues(int n) { return vec; } -template -double ContainerInfo(const std::set &s) { - int sizeof_node = sizeof(std::_Rb_tree_node); - int bytes_used = sizeof(s) + s.size() * sizeof_node; - double bytes_per_value = (double) bytes_used / s.size(); - std::cout << " size=" << s.size() - << " bytes-used=" << bytes_used - << " bytes-per-value=" << bytes_per_value; - return bytes_per_value; -} - -template -double ContainerInfo(const std::multiset &s) { - int sizeof_node = sizeof(std::_Rb_tree_node); - int bytes_used = sizeof(s) + s.size() * sizeof_node; - double bytes_per_value = (double) bytes_used / s.size(); - std::cout << " size=" << s.size() - << " bytes-used=" << bytes_used - << " bytes-per-value=" << bytes_per_value; - return bytes_per_value; -} - -template -double ContainerInfo(const std::map &m) { - int sizeof_node = sizeof(std::_Rb_tree_node >); - int bytes_used = sizeof(m) + m.size() * sizeof_node; - double bytes_per_value = (double) bytes_used / m.size(); - std::cout << " size=" << m.size() - << " bytes-used=" << bytes_used - << " bytes-per-value=" << bytes_per_value; - return bytes_per_value; -} - -template -double ContainerInfo(const std::multimap &m) { - int sizeof_node = sizeof(std::_Rb_tree_node >); - int bytes_used = sizeof(m) + m.size() * sizeof_node; - double bytes_per_value = (double) bytes_used / m.size(); - std::cout << " size=" << m.size() - << " bytes-used=" << bytes_used - << " bytes-per-value=" << bytes_per_value; - return bytes_per_value; -} - -template -double ContainerInfo(const btree_container

&b) { - double bytes_used = sizeof(b) + b.bytes_used(); - double bytes_per_value = (double) bytes_used / b.size(); - std::cout << " size=" << b.size() - << " bytes-used=" << bytes_used - << " bytes-per-value=" << bytes_per_value - << " height=" << b.height() - << " internal-nodes=" << b.internal_nodes() - << " leaf-nodes=" << b.leaf_nodes() - << " fullness=" << b.fullness() - << " overhead=" << b.overhead(); - return bytes_per_value; -} - template void DoTest(const char *name, T *b, const std::vector &values) { typename KeyOfValue::type key_of_value; -- 2.30.2 From ce6c9fd34c4d04345e73d7199346db648f00a071 Mon Sep 17 00:00:00 2001 From: josh macdonald Date: Mon, 14 Jan 2013 02:08:38 -0800 Subject: [PATCH 09/91] Build fixes --- CMakeLists.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 79c67ac7..0ef7749b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,8 +7,9 @@ add_definitions(-std=c++11) set(CMAKE_CXX_FLAGS "-g -O2") # CMake doesn't have a way to pure template library, -add_library(cppbtree btree.h btree_map.h btree_set.h safe_btree.h safe_btree_map.h safe_btree_set.h) -set_target_properties(cppbtree PROPERTIES LINKER_LANGUAGE CXX) +# add_library(cppbtree btree.h btree_map.h btree_set.h +# safe_btree.h safe_btree_map.h safe_btree_set.h) +# set_target_properties(cppbtree PROPERTIES LINKER_LANGUAGE CXX) if(build_tests) enable_testing() @@ -17,7 +18,7 @@ if(build_tests) add_executable(btree_test btree_test.cc btree_test_flags.cc) add_executable(safe_btree_test safe_btree_test.cc btree_test_flags.cc) add_executable(btree_bench btree_bench.cc btree_test_flags.cc) - target_link_libraries(btree_test gtest_main gflags) - target_link_libraries(safe_btree_test gtest_main gflags) + target_link_libraries(btree_test gtest_main gtest gflags) + target_link_libraries(safe_btree_test gtest_main gtest gflags) target_link_libraries(btree_bench gflags gtest) endif() -- 2.30.2 From 7aad0f28fdc7325afbea09991d522cfea7754933 Mon Sep 17 00:00:00 2001 From: sername=jmacd Date: Tue, 22 Jan 2013 15:57:34 -0800 Subject: [PATCH 10/91] Update README; remove benchmarks.awk --- README | 6 ++++-- benchmarks.awk | 27 --------------------------- 2 files changed, 4 insertions(+), 29 deletions(-) delete mode 100755 benchmarks.awk diff --git a/README b/README index 152972cf..9ddca6bd 100644 --- a/README +++ b/README @@ -1,15 +1,17 @@ This library is a C++ template library and, as such, there is no -library to build and install. +library to build and install. Copy the .h files and use them! ---- To build and run the provided tests, however, you will need to install -CMake and the Google C++ Test framework. +CMake, the Google C++ Test framework, and the Google flags package. Download CMake from http://www.cmake.org Download the GoogleTest framework from http://code.google.com/p/googletest +Download gflags from https://code.google.com/p/gflags + export GTEST_ROOT=/path/for/gtest-x.y cmake . -Dbuild_tests=ON diff --git a/benchmarks.awk b/benchmarks.awk deleted file mode 100755 index d5d4aeac..00000000 --- a/benchmarks.awk +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/gawk -f - -/^Run on/ { - print $0; - printf "%-25s %5s %-20s\n", - "Benchmark", "STL(ns)", "B-Tree(ns) @ " - printf "--------------------------------------------------------\n"; -} - -/^BM_/ { - split($1, name, "_"); - if (name[2] == "stl") { - stl = $3; - stl_bytes = $5 - printf "%-25s %5d ", name[1] "_" name[3] "_" name[4] "_" name[5], stl; - fflush(); - } else if (name[2] == "btree") { - btree = $3 - btree_size = name[3] - btree_bytes = $5 - printf "%5d %+7.2f%% <%3d>", btree, 100.0 * (stl - btree) / stl, btree_size; - printf " [%4.1f, %4.1f]\n", stl_bytes, btree_bytes; - fflush(); - } else { - printf "ERROR: %s unrecognized\n", $1 - } -} -- 2.30.2 From 5633058db44363552b042d93ea18d2451d6f97ae Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Fri, 25 Jan 2013 00:46:48 -0800 Subject: [PATCH 11/91] Update copyright notices; scrub a few references to Cord, StringPiece --- CMakeLists.txt | 14 +++ COPYING | 202 ++++++++++++++++++++++++++++++++++++++++++++ README | 9 +- btree.h | 44 ++++++---- btree_bench.cc | 16 +++- btree_container.h | 16 +++- btree_map.h | 16 +++- btree_set.h | 16 +++- btree_test.cc | 16 +++- btree_test.h | 16 +++- btree_test_flags.cc | 15 +++- safe_btree.h | 15 +++- safe_btree_map.h | 15 +++- safe_btree_set.h | 15 +++- safe_btree_test.cc | 16 +++- 15 files changed, 392 insertions(+), 49 deletions(-) create mode 100644 COPYING diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ef7749b..3c49ad27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,3 +1,17 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + cmake_minimum_required(VERSION 2.6) project(cppbtree CXX) diff --git a/COPYING b/COPYING new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/COPYING @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README b/README index 9ddca6bd..0e8674cf 100644 --- a/README +++ b/README @@ -1,16 +1,19 @@ This library is a C++ template library and, as such, there is no library to build and install. Copy the .h files and use them! +See http://code.google.com/p/cpp-btree/wiki/UsageInstructions for +details. + ---- To build and run the provided tests, however, you will need to install CMake, the Google C++ Test framework, and the Google flags package. -Download CMake from http://www.cmake.org +Download and install CMake from http://www.cmake.org -Download the GoogleTest framework from http://code.google.com/p/googletest +Download and build the GoogleTest framework from http://code.google.com/p/googletest -Download gflags from https://code.google.com/p/gflags +Download and install gflags from https://code.google.com/p/gflags export GTEST_ROOT=/path/for/gtest-x.y diff --git a/btree.h b/btree.h index a8d3152d..49310a2e 100644 --- a/btree.h +++ b/btree.h @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // // A btree implementation of the STL set and map interfaces. A btree is both // smaller and faster than STL set/map. The red-black tree implementation of @@ -112,12 +122,12 @@ namespace btree { // Inside a btree method, if we just call swap(), it will choose the -// btree::swap method, which we don't want. And we can't say ::swap because -// then MSVC won't pickup any std::swap() implementations. We can't just use -// std::swap() directly because then we don't get the specialization for string -// and Cord (which are not defined in the std namespace!). So the solution is -// to have a special swap helper function whose name doesn't collide with other -// swap functions defined by the btree classes. +// btree::swap method, which we don't want. And we can't say ::swap +// because then MSVC won't pickup any std::swap() implementations. We +// can't just use std::swap() directly because then we don't get the +// specialization for types outside the std namespace. So the solution +// is to have a special swap helper function whose name doesn't +// collide with other swap functions defined by the btree classes. template inline void btree_swap_helper(T &a, T &b) { using std::swap; @@ -172,14 +182,14 @@ struct btree_is_key_compare_to : public std::is_convertible { }; -// A helper class to convert a boolean comparison into a three-way "compare-to" -// comparison that returns a negative value to indicate less-than, zero to -// indicate equality and a positive value to indicate greater-than. This helper -// class is specialized for less, greater, less, -// greater, less and greater. The -// btree_key_compare_to_adapter class is provided so that btree users -// automatically get the more efficient compare-to code when using common -// google string types with common comparison functors. +// A helper class to convert a boolean comparison into a three-way +// "compare-to" comparison that returns a negative value to indicate +// less-than, zero to indicate equality and a positive value to +// indicate greater-than. This helper class is specialized for +// less and greater. The btree_key_compare_to_adapter +// class is provided so that btree users automatically get the more +// efficient compare-to code when using common google string types +// with common comparison functors. template struct btree_key_compare_to_adapter : Compare { btree_key_compare_to_adapter() { } diff --git a/btree_bench.cc b/btree_bench.cc index b803ad38..d5495c5d 100644 --- a/btree_bench.cc +++ b/btree_bench.cc @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include #include diff --git a/btree_container.h b/btree_container.h index 510c9ae6..c9c62c33 100644 --- a/btree_container.h +++ b/btree_container.h @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #ifndef UTIL_BTREE_BTREE_CONTAINER_H__ #define UTIL_BTREE_BTREE_CONTAINER_H__ diff --git a/btree_map.h b/btree_map.h index 6e965590..07b799ea 100644 --- a/btree_map.h +++ b/btree_map.h @@ -1,6 +1,16 @@ -// Copyright 2007, 2012 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // // A btree_map<> implements the STL unique sorted associative container // interface and the pair associative container interface (a.k.a map<>) using a diff --git a/btree_set.h b/btree_set.h index 0b51e494..2bc9e586 100644 --- a/btree_set.h +++ b/btree_set.h @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // // A btree_set<> implements the STL unique sorted associative container // interface (a.k.a set<>) using a btree. A btree_multiset<> implements the STL diff --git a/btree_test.cc b/btree_test.cc index 7b9cafa5..56a787b9 100644 --- a/btree_test.cc +++ b/btree_test.cc @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "gtest/gtest.h" #include "btree_map.h" diff --git a/btree_test.h b/btree_test.h index 01ab0fe0..0ac26eff 100644 --- a/btree_test.h +++ b/btree_test.h @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #ifndef UTIL_BTREE_BTREE_TEST_H__ #define UTIL_BTREE_BTREE_TEST_H__ diff --git a/btree_test_flags.cc b/btree_test_flags.cc index d43d9250..bf608a9b 100644 --- a/btree_test_flags.cc +++ b/btree_test_flags.cc @@ -1,5 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "gflags/gflags.h" diff --git a/safe_btree.h b/safe_btree.h index 01cf9faf..2d85c70b 100644 --- a/safe_btree.h +++ b/safe_btree.h @@ -1,5 +1,16 @@ -// Copyright 2007, 2012 Google Inc. All Rights Reserved. -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // // A safe_btree<> wraps around a btree<> and removes the caveat that insertion // and deletion invalidate iterators. A safe_btree<> maintains a generation diff --git a/safe_btree_map.h b/safe_btree_map.h index b8aa787e..a0668f16 100644 --- a/safe_btree_map.h +++ b/safe_btree_map.h @@ -1,5 +1,16 @@ -// Copyright 2007, 2012 Google Inc. All Rights Reserved. -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // // The safe_btree_map<> is like btree_map<> except that it removes the caveat // about insertion and deletion invalidating existing iterators at a small cost diff --git a/safe_btree_set.h b/safe_btree_set.h index de8f823a..a6cd5418 100644 --- a/safe_btree_set.h +++ b/safe_btree_set.h @@ -1,5 +1,16 @@ -// Copyright 2007, 2012 Google Inc. All Rights Reserved. -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // // The safe_btree_set<> is like btree_set<> except that it removes the caveat // about insertion and deletion invalidating existing iterators at a small cost diff --git a/safe_btree_test.cc b/safe_btree_test.cc index 4965916b..0d77ae06 100644 --- a/safe_btree_test.cc +++ b/safe_btree_test.cc @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // TODO(pmattis): Add some tests that iterators are not invalidated by // insertion and deletion. -- 2.30.2 From c790956d1826bbd0bb11506ae95840a95d96b7a8 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Tue, 5 Feb 2013 22:04:33 -0800 Subject: [PATCH 12/91] Fix several range constructor bugs: (a) btree_multiset's range constructor did not compile (b) btree_map's range constructor ignored the inputs! These bugs were masked by the test, which due to an oversight skipped the use of range constructors as part of the {base,unique,multi}_checker testing types. Verified that the corrected test fails without the fix, added an additional sanity check (which made the problem stand out independent of the problems discovered in btree_test.h) --- btree_container.h | 5 ++--- btree_map.h | 6 +++--- btree_set.h | 4 ++-- btree_test.cc | 19 +++++++++++++++++++ btree_test.h | 16 ++++++++++++---- 5 files changed, 38 insertions(+), 12 deletions(-) diff --git a/btree_container.h b/btree_container.h index c9c62c33..fb617abe 100644 --- a/btree_container.h +++ b/btree_container.h @@ -261,8 +261,7 @@ class btree_map_container : public btree_unique_container { btree_map_container(InputIterator b, InputIterator e, const key_compare &comp = key_compare(), const allocator_type &alloc = allocator_type()) - : super_type(comp, alloc) { - insert(b, e); + : super_type(b, e, comp, alloc) { } // Insertion routines. @@ -303,7 +302,7 @@ class btree_multi_container : public btree_container { btree_multi_container(InputIterator b, InputIterator e, const key_compare &comp = key_compare(), const allocator_type &alloc = allocator_type()) - : super_type(b, e, comp, alloc) { + : super_type(comp, alloc) { insert(b, e); } diff --git a/btree_map.h b/btree_map.h index 07b799ea..b83489f0 100644 --- a/btree_map.h +++ b/btree_map.h @@ -33,7 +33,7 @@ namespace btree { -// The btree_map class is needed mainly for it's constructors. +// The btree_map class is needed mainly for its constructors. template , typename Alloc = std::allocator >, @@ -68,7 +68,7 @@ class btree_map : public btree_map_container< btree_map(InputIterator b, InputIterator e, const key_compare &comp = key_compare(), const allocator_type &alloc = allocator_type()) - : super_type(comp, alloc) { + : super_type(b, e, comp, alloc) { } }; @@ -78,7 +78,7 @@ inline void swap(btree_map &x, x.swap(y); } -// The btree_multimap class is needed mainly for it's constructors. +// The btree_multimap class is needed mainly for its constructors. template , typename Alloc = std::allocator >, diff --git a/btree_set.h b/btree_set.h index 2bc9e586..f9b2e75d 100644 --- a/btree_set.h +++ b/btree_set.h @@ -29,7 +29,7 @@ namespace btree { -// The btree_set class is needed mainly for it's constructors. +// The btree_set class is needed mainly for its constructors. template , typename Alloc = std::allocator, @@ -72,7 +72,7 @@ inline void swap(btree_set &x, btree_set &y) { x.swap(y); } -// The btree_multiset class is needed mainly for it's constructors. +// The btree_multiset class is needed mainly for its constructors. template , typename Alloc = std::allocator, diff --git a/btree_test.cc b/btree_test.cc index 56a787b9..52bd219a 100644 --- a/btree_test.cc +++ b/btree_test.cc @@ -244,5 +244,24 @@ TEST(Btree, Comparison) { EXPECT_TRUE(my_map != my_map_copy); } +TEST(Btree, RangeCtorSanity) { + typedef btree_set, allocator, 256> test_set; + typedef btree_map, allocator, 256> test_map; + typedef btree_multiset, allocator, 256> test_mset; + typedef btree_multimap, allocator, 256> test_mmap; + vector ivec; + ivec.push_back(1); + map imap; + imap.insert(make_pair(1, 2)); + test_mset tmset(ivec.begin(), ivec.end()); + test_mmap tmmap(imap.begin(), imap.end()); + test_set tset(ivec.begin(), ivec.end()); + test_map tmap(imap.begin(), imap.end()); + EXPECT_EQ(1, tmset.size()); + EXPECT_EQ(1, tmmap.size()); + EXPECT_EQ(1, tset.size()); + EXPECT_EQ(1, tmap.size()); +} + } // namespace } // namespace btree diff --git a/btree_test.h b/btree_test.h index 0ac26eff..413dc3c7 100644 --- a/btree_test.h +++ b/btree_test.h @@ -147,6 +147,13 @@ class base_checker { const_tree_(tree_), checker_(x.checker_) { } + // Range constructor. + template + base_checker(InputIterator b, InputIterator e) + : tree_(b, e), + const_tree_(tree_), + checker_(b, e) { + } // Iterator routines. iterator begin() { return tree_.begin(); } @@ -398,8 +405,8 @@ class unique_checker : public base_checker { } // Range constructor. template - unique_checker(InputIterator b, InputIterator e) { - insert(b, e); + unique_checker(InputIterator b, InputIterator e) + : super_type(b, e) { } // Insertion routines. @@ -455,8 +462,8 @@ class multi_checker : public base_checker { } // Range constructor. template - multi_checker(InputIterator b, InputIterator e) { - insert(b, e); + multi_checker(InputIterator b, InputIterator e) + : super_type(b, e) { } // Insertion routines. @@ -587,6 +594,7 @@ void DoTest(const char *name, T *b, const std::vector &values) { mutable_b.insert(values[i]); mutable_b.value_check(values[i]); } + assert(mutable_b.size() == values.size()); const_b.verify(); printf(" %s fullness=%0.2f overhead=%0.2f bytes-per-value=%0.2f\n", -- 2.30.2 From f85129d4d77a447984241dbc1babd31b1eb1dd4e Mon Sep 17 00:00:00 2001 From: josh macdonald Date: Tue, 5 Feb 2013 22:12:13 -0800 Subject: [PATCH 13/91] Fixes for open-source build following upstream bug fix --- btree_test.cc | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/btree_test.cc b/btree_test.cc index 52bd219a..6b1837d3 100644 --- a/btree_test.cc +++ b/btree_test.cc @@ -245,14 +245,17 @@ TEST(Btree, Comparison) { } TEST(Btree, RangeCtorSanity) { - typedef btree_set, allocator, 256> test_set; - typedef btree_map, allocator, 256> test_map; - typedef btree_multiset, allocator, 256> test_mset; - typedef btree_multimap, allocator, 256> test_mmap; - vector ivec; + typedef btree_set, std::allocator, 256> test_set; + typedef btree_map, std::allocator, 256> + test_map; + typedef btree_multiset, std::allocator, 256> + test_mset; + typedef btree_multimap, std::allocator, 256> + test_mmap; + std::vector ivec; ivec.push_back(1); - map imap; - imap.insert(make_pair(1, 2)); + std::map imap; + imap.insert(std::make_pair(1, 2)); test_mset tmset(ivec.begin(), ivec.end()); test_mmap tmmap(imap.begin(), imap.end()); test_set tset(ivec.begin(), ivec.end()); -- 2.30.2 From 54f346f9816f665bf45a96f6eb0fb09b74ccffef Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Fri, 8 Feb 2013 22:56:25 -0800 Subject: [PATCH 14/91] Update README / CMakeLists.txt; fix for -Wformat warning in btree_bench.cc --- CMakeLists.txt | 2 ++ README | 13 ++++++++++++- btree_bench.cc | 4 ++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c49ad27..d005e158 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,8 @@ if(build_tests) enable_testing() include_directories($ENV{GTEST_ROOT}/include) link_directories($ENV{GTEST_ROOT}) + include_directories($ENV{GFLAGS_ROOT}/include) + link_directories($ENV{GFLAGS_ROOT}/lib) add_executable(btree_test btree_test.cc btree_test_flags.cc) add_executable(safe_btree_test safe_btree_test.cc btree_test_flags.cc) add_executable(btree_bench btree_bench.cc btree_test_flags.cc) diff --git a/README b/README index 0e8674cf..319fe9bb 100644 --- a/README +++ b/README @@ -11,10 +11,21 @@ CMake, the Google C++ Test framework, and the Google flags package. Download and install CMake from http://www.cmake.org -Download and build the GoogleTest framework from http://code.google.com/p/googletest +Download and build the GoogleTest framework from +http://code.google.com/p/googletest Download and install gflags from https://code.google.com/p/gflags +Set GTEST_ROOT to the directory where GTEST was built. +Set GFLAGS_ROOT to the directory prefix where GFLAGS is installed. + export GTEST_ROOT=/path/for/gtest-x.y +export GFLAGS_ROOT=/opt cmake . -Dbuild_tests=ON + +For example, to build on a Unix system with the clang++ compiler, + +export GTEST_ROOT=$(HOME)/src/googletest +export GFLAGS_ROOT=/opt +cmake . -G "Unix Makefiles" -Dbuild_tests=ON -DCMAKE_CXX_COMPILER=clang++ diff --git a/btree_bench.cc b/btree_bench.cc index d5495c5d..900c5e63 100644 --- a/btree_bench.cc +++ b/btree_bench.cc @@ -133,7 +133,7 @@ void BenchmarkRun::Reset() { void BenchmarkRun::Run() { assert(current_benchmark == NULL); current_benchmark = this; - int iters = FLAGS_benchmark_min_iters; + int64_t iters = FLAGS_benchmark_min_iters; for (;;) { Reset(); Start(); @@ -152,7 +152,7 @@ void BenchmarkRun::Run() { } fprintf(stdout, "%s\t%qu\t%qu\n", benchmark_name, - accum_micros * 1000 / iters, + accum_micros * 1000ULL / iters, iters); current_benchmark = NULL; } -- 2.30.2 From 92ec61e4b8bf182c5c49ebf6540dac62d569d090 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Fri, 8 Feb 2013 23:14:47 -0800 Subject: [PATCH 15/91] Fix the printf problem -- use std::cout :) --- btree_bench.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/btree_bench.cc b/btree_bench.cc index 900c5e63..6eaed994 100644 --- a/btree_bench.cc +++ b/btree_bench.cc @@ -133,7 +133,7 @@ void BenchmarkRun::Reset() { void BenchmarkRun::Run() { assert(current_benchmark == NULL); current_benchmark = this; - int64_t iters = FLAGS_benchmark_min_iters; + int iters = FLAGS_benchmark_min_iters; for (;;) { Reset(); Start(); @@ -150,10 +150,9 @@ void BenchmarkRun::Run() { } iters = min(iters, FLAGS_benchmark_max_iters); } - fprintf(stdout, "%s\t%qu\t%qu\n", - benchmark_name, - accum_micros * 1000ULL / iters, - iters); + std::cout << benchmark_name << "\t" + << accum_micros * 1000 / iters << "\t" + << iters; current_benchmark = NULL; } -- 2.30.2 From fe5e2167a51ad0f9dba0261843d058ab2e12e369 Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Tue, 3 Sep 2013 14:45:11 -0700 Subject: [PATCH 16/91] Initial commit --- .gitignore | 1 + Makefile | 2 + README.md | 42 ++++ json11.cpp | 674 +++++++++++++++++++++++++++++++++++++++++++++++++++++ json11.hpp | 187 +++++++++++++++ test.cpp | 100 ++++++++ 6 files changed, 1006 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 json11.cpp create mode 100644 json11.hpp create mode 100644 test.cpp diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..9daeafb9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +test diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..f251c1fb --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +test: json11.cpp json11.hpp test.cpp + clang++ -O -std=c++11 -stdlib=libc++ json11.cpp test.cpp -o test -fno-rtti -fno-exceptions diff --git a/README.md b/README.md new file mode 100644 index 00000000..21f4e8b2 --- /dev/null +++ b/README.md @@ -0,0 +1,42 @@ +json11 +------ + +json11 is a tiny JSON library for C++11, providing JSON parsing and serialization. + +The core object provided by the library is json11::Json. A Json object represents any JSON +value: null, bool, number (int or double), string (std::string), array (std::vector), or +object (std::map). + +Json objects act like values. They can be assigned, copied, moved, compared for equality or +order, and so on. There are also helper methods Json::dump, to serialize a Json to a string, and +Json::parse (static) to parse a std::string as a Json object. + +It's easy to make a JSON object with C++11's new initializer syntax: + + Json my_json = Json::object { + { "key1", "value1" }, + { "key2", false }, + { "key3", Json::array { 1, 2, 3 } }, + }; + std::string json_str = my_json.dump(); + +There are also implicit constructors that allow standard and user-defined types to be +automatically converted to JSON. For example: + + class Point { + public: + int x; + int y; + Point (int x, int y) : x(x), y(y) {} + Json to_json() const { return Json::array { x, y }; } + }; + + std::vector points = { { 1, 2 }, { 10, 20 }, { 100, 200 } }; + std::string points_json = Json(points).dump(); + +JSON values can have their values queried and inspected: + + Json json = Json::array { Json::object { { "k", "v" } } }; + std::string str = json[0]["k"].string_value(); + +More documentation is still to come. For now, see json11.hpp. diff --git a/json11.cpp b/json11.cpp new file mode 100644 index 00000000..dc882071 --- /dev/null +++ b/json11.cpp @@ -0,0 +1,674 @@ +/* Copyright (c) 2013 Dropbox, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "json11.hpp" +#include +#include +#include +#include + +namespace json11 { + +using std::string; +using std::vector; +using std::map; +using std::make_shared; +using std::initializer_list; +using std::move; + +/* * * * * * * * * * * * * * * * * * * * + * Serialization + */ + +static void dump(std::nullptr_t, string &out) { + out += "null"; +} + +static void dump(double value, string &out) { + char buf[32]; + snprintf(buf, sizeof buf, "%.17g", value); + out += buf; +} + +static void dump(int value, string &out) { + char buf[32]; + snprintf(buf, sizeof buf, "%d", value); + out += buf; +} + +static void dump(bool value, string &out) { + out += value ? "true" : "false"; +} + +static void dump(const string &value, string &out) { + out += '"'; + for (size_t i = 0; i < value.length(); i++) { + const char ch = value[i]; + if (ch == '\\') { + out += "\\\\"; + } else if (ch == '"') { + out += "\\\""; + } else if (ch == '\b') { + out += "\\b"; + } else if (ch == '\f') { + out += "\\f"; + } else if (ch == '\n') { + out += "\\n"; + } else if (ch == '\r') { + out += "\\r"; + } else if (ch == '\t') { + out += "\\t"; + } else if ((uint8_t)ch <= 0x1f) { + char buf[8]; + snprintf(buf, sizeof buf, "\\u%04x", ch); + out += buf; + } else if ((uint8_t)ch == 0xe2 && (uint8_t)value[i+1] == 0x80 + && (uint8_t)value[i+2] == 0xa8) { + out += "\\u2028"; + i += 2; + } else if ((uint8_t)ch == 0xe2 && (uint8_t)value[i+1] == 0x80 + && (uint8_t)value[i+2] == 0xa9) { + out += "\\u2029"; + i += 2; + } else { + out += ch; + } + } + out += '"'; +} + +static void dump(const Json::array &values, string &out) { + bool first = true; + out += "["; + for (auto &value : values) { + if (!first) + out += ", "; + value.dump(out); + first = false; + } + out += "]"; +} + +static void dump(const Json::object &values, string &out) { + bool first = true; + out += "{"; + for (const std::pair &kv : values) { + if (!first) + out += ", "; + dump(kv.first, out); + out += ": "; + kv.second.dump(out); + first = false; + } + out += "}"; +} + +void Json::dump(string &out) const { + m_ptr->dump(out); +} + +/* * * * * * * * * * * * * * * * * * * * + * Value wrappers + */ + +template +class Value : public JsonValue { +protected: + + // Constructors + Value(const T &value) : m_value(value) {} + Value(T &&value) : m_value(move(value)) {} + + // Get type tag + Json::Type type() const { + return tag; + } + + // Comparisons + bool equals(const JsonValue * other) const { + return m_value == reinterpret_cast *>(other)->m_value; + } + bool less(const JsonValue * other) const { + return m_value < reinterpret_cast *>(other)->m_value; + } + + const T m_value; + void dump(string &out) const { json11::dump(m_value, out); } +}; + +class JsonDouble final : public Value { + double number_value() const { return m_value; } + int int_value() const { return m_value; } + bool equals(const JsonValue * other) const { return m_value == other->number_value(); } + bool less(const JsonValue * other) const { return m_value < other->number_value(); } +public: + JsonDouble(double value) : Value(value) {} +}; + +class JsonInt final : public Value { + double number_value() const { return m_value; } + int int_value() const { return m_value; } + bool equals(const JsonValue * other) const { return m_value == other->number_value(); } + bool less(const JsonValue * other) const { return m_value < other->number_value(); } +public: + JsonInt(double value) : Value(value) {} +}; + +class JsonBoolean final : public Value { + bool bool_value() const { return m_value; } +public: + JsonBoolean(bool value) : Value(value) {} +}; + +class JsonString final : public Value { + const string &string_value() const { return m_value; } +public: + JsonString(const string &value) : Value(value) {} + JsonString(string &&value) : Value(move(value)) {} +}; + +class JsonArray final : public Value { + const Json::array &array_items() const { return m_value; } + const Json & operator[](size_t i) const; +public: + JsonArray(const Json::array &value) : Value(value) {} + JsonArray(Json::array &&value) : Value(move(value)) {} +}; + +class JsonObject final : public Value { + const Json::object &object_items() const { return m_value; } + const Json & operator[](const string &key) const; +public: + JsonObject(const Json::object &value) : Value(value) {} + JsonObject(Json::object &&value) : Value(move(value)) {} +}; + +class JsonNull final : public Value { +public: + JsonNull() : Value(nullptr) {} +}; + +/* * * * * * * * * * * * * * * * * * * * + * Constructors + */ + +static const std::shared_ptr obj_null(make_shared()); +static const std::shared_ptr obj_true(make_shared(true)); +static const std::shared_ptr obj_false(make_shared(false)); + +Json::Json() noexcept : m_ptr(obj_null) {} +Json::Json(std::nullptr_t) noexcept : m_ptr(obj_null) {} +Json::Json(double value) : m_ptr(make_shared(value)) {} +Json::Json(int value) : m_ptr(make_shared(value)) {} +Json::Json(bool value) : m_ptr(value ? obj_true : obj_false) {} +Json::Json(const string &value) : m_ptr(make_shared(value)) {} +Json::Json(string &&value) : m_ptr(make_shared(move(value))) {} +Json::Json(const char * value) : m_ptr(make_shared(value)) {} +Json::Json(const Json::array &values) : m_ptr(make_shared(values)) {} +Json::Json(Json::array &&values) : m_ptr(make_shared(move(values))) {} +Json::Json(const Json::object &values) : m_ptr(make_shared(values)) {} +Json::Json(Json::object &&values) : m_ptr(make_shared(move(values))) {} + +/* * * * * * * * * * * * * * * * * * * * + * Accessors + */ + +static const string empty_string; +static const vector empty_vector; +static const map empty_map; +static const Json json_null; + +Json::Type Json::type() const { return m_ptr->type(); } +double Json::number_value() const { return m_ptr->number_value(); } +int Json::int_value() const { return m_ptr->int_value(); } +bool Json::bool_value() const { return m_ptr->bool_value(); } +const string & Json::string_value() const { return m_ptr->string_value(); } +const vector & Json::array_items() const { return m_ptr->array_items(); } +const map & Json::object_items() const { return m_ptr->object_items(); } +const Json & Json::operator[] (size_t i) const { return (*m_ptr)[i]; } +const Json & Json::operator[] (const string &key) const { return (*m_ptr)[key]; } + +double JsonValue::number_value() const { return 0; } +int JsonValue::int_value() const { return 0; } +bool JsonValue::bool_value() const { return false; } +const string & JsonValue::string_value() const { return empty_string; } +const vector & JsonValue::array_items() const { return empty_vector; } +const map & JsonValue::object_items() const { return empty_map; } +const Json & JsonValue::operator[] (size_t) const { return json_null; } +const Json & JsonValue::operator[] (const string &) const { return json_null; } + +const Json & JsonObject::operator[] (const string &key) const { + auto iter = m_value.find(key); + return (iter == m_value.end()) ? json_null : iter->second; +} +const Json & JsonArray::operator[] (size_t i) const { + if (i >= m_value.size()) return json_null; + else return m_value[i]; +} + +/* * * * * * * * * * * * * * * * * * * * + * Comparison + */ + +bool Json::operator== (const Json &other) const { + if (m_ptr->type() != other.m_ptr->type()) + return false; + + return m_ptr->equals(other.m_ptr.get()); +} + +bool Json::operator< (const Json &other) const { + if (m_ptr->type() != other.m_ptr->type()) + return m_ptr->type() < other.m_ptr->type(); + + return m_ptr->less(other.m_ptr.get()); +} + +/* * * * * * * * * * * * * * * * * * * * + * Parsing + */ + +/* esc(c) + * + * Format char c suitable for printing in an error message. + */ +static inline string esc(char c) { + char buf[12]; + if ((uint8_t)c >= 0x20 && (uint8_t)c <= 0x7f) { + snprintf(buf, sizeof buf, "'%c' (%d)", c, c); + } else { + snprintf(buf, sizeof buf, "(%d)", c); + } + return string(buf); +} + +static inline bool in_range (int x, int lower, int upper) { + return (x >= lower && x <= upper); +} + +/* JsonParser + * + * Object that tracks all state of an in-progress parse. + */ +struct JsonParser { + + /* State + */ + const string &str; + size_t i; + string &err; + bool failed; + + /* fail(msg, err_ret = Json()) + * + * Mark this parse as failed. + */ + Json fail(string &&msg) { + return fail(move(msg), Json()); + } + + template + T fail(string &&msg, const T err_ret) { + if (!failed) + err = std::move(msg); + failed = true; + return err_ret; + } + + /* consume_whitespace() + * + * Advance until the current character is non-whitespace. + */ + void consume_whitespace() { + while (str[i] == ' ' || str[i] == '\r' || str[i] == '\n' || str[i] == '\t') + i++; + } + + /* get_next_token() + * + * Return the next non-whitespace character. If the end of the input is reached, + * flag an error and return 0. + */ + char get_next_token() { + consume_whitespace(); + if (i == str.size()) + return fail("unexpected end of input", 0); + + return str[i++]; + } + + /* encode_utf8(pt, out) + * + * Encode pt as UTF-8 and add it to out. + */ + void encode_utf8(long pt, string & out) { + if (pt < 0) + return; + + if (pt < 0x80) { + out += pt; + } else if (pt < 0x800) { + out += (pt >> 6) | 0xC0; + out += (pt & 0x3F) | 0x80; + } else if (pt < 0x10000) { + out += (pt >> 12) | 0xE0; + out += ((pt >> 6) & 0x3F) | 0x80; + out += (pt & 0x3F) | 0x80; + } else { + out += (pt >> 18) | 0xF0; + out += ((pt >> 12) & 0x3F) | 0x80; + out += ((pt >> 6) & 0x3F) | 0x80; + out += (pt & 0x3F) | 0x80; + } + } + + /* parse_string() + * + * Parse a string, starting at the current position. + */ + string parse_string() { + string out; + long last_escaped_codepoint = -1; + while (true) { + if (i == str.size()) + return fail("unexpected end of input in string", ""); + + char ch = str[i++]; + + if (ch == '"') { + encode_utf8(last_escaped_codepoint, out); + return out; + } + + if (in_range(ch, 0, 0x1f)) + return fail("unescaped " + esc(ch) + " in string", ""); + + // The usual case: non-escaped characters + if (ch != '\\') { + encode_utf8(last_escaped_codepoint, out); + last_escaped_codepoint = -1; + out += ch; + continue; + } + + // Handle escapes + if (i == str.size()) + return fail("unexpected end of input in string", ""); + + ch = str[i++]; + + if (ch == 'u') { + // Extract 4-byte escape sequence + string esc = str.substr(i, 4); + for (int j = 0; j < 4; j++) { + if (!in_range(esc[j], 'a', 'f') && !in_range(esc[j], 'A', 'F') + && !in_range(esc[j], '0', '9')) + return fail("bad \\u escape: " + esc, ""); + } + + long codepoint = strtol(esc.data(), nullptr, 16); + + // JSON specifies that characters outside the BMP shall be encoded as a pair + // of 4-hex-digit \u escapes encoding their surrogate pair components. Check + // whether we're in the middle of such a beast: the previous codepoint was an + // escaped lead (high) surrogate, and this is a trail (low) surrogate. + if (in_range(last_escaped_codepoint, 0xD800, 0xDBFF) + && in_range(codepoint, 0xDC00, 0xDFFF)) { + // Reassemble the two surrogate pairs into one astral-plane character, per + // the UTF-16 algorithm. + encode_utf8((((last_escaped_codepoint - 0xD800) << 10) + | (codepoint - 0xDC00)) + 0x10000, out); + last_escaped_codepoint = -1; + } else { + encode_utf8(last_escaped_codepoint, out); + last_escaped_codepoint = codepoint; + } + + i += 4; + continue; + } + + encode_utf8(last_escaped_codepoint, out); + last_escaped_codepoint = -1; + + if (ch == 'b') { + out += '\b'; + } else if (ch == 'f') { + out += '\f'; + } else if (ch == 'n') { + out += '\n'; + } else if (ch == 'r') { + out += '\r'; + } else if (ch == 't') { + out += '\t'; + } else if (ch == '"' || ch == '\\' || ch == '/') { + out += ch; + } else { + return fail("invalid escape character " + esc(ch), ""); + } + } + } + + /* parse_number() + * + * Parse a double. + */ + Json parse_number() { + size_t start_pos = i; + + if (str[i] == '-') + i++; + + // Integer part + if (str[i] == '0') { + i++; + if (in_range(str[i], '0', '9')) + return fail("leading 0s not permitted in numbers"); + } else if (in_range(str[i], '1', '9')) { + i++; + while (in_range(str[i], '0', '9')) + i++; + } else { + return fail("invalid " + esc(str[i]) + " in number"); + } + + if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' + && (i - start_pos) <= (size_t)std::numeric_limits::digits10) { + return std::atoi(str.c_str() + start_pos); + } + + // Decimal part + if (str[i] == '.') { + i++; + if (!in_range(str[i], '0', '9')) + return fail("at least one digit required in fractional part"); + + while (in_range(str[i], '0', '9')) + i++; + } + + // Exponent part + if (str[i] == 'e' || str[i] == 'E') { + i++; + + if (str[i] == '+' || str[i] == '-') + i++; + + if (!in_range(str[i], '0', '9')) + return fail("at least one digit required in exponent"); + + while (in_range(str[i], '0', '9')) + i++; + } + + return std::atof(str.c_str() + start_pos); + } + + /* expect(str, res) + * + * Expect that 'str' starts at the character that was just read. If it does, advance + * the input and return res. If not, flag an error. + */ + Json expect(const string &expected, Json res) { + assert(i != 0); + i--; + const string found = str.substr(i, expected.length()); + if (expected == found) { + i += expected.length(); + return res; + } else { + return fail("parse error: expected " + expected + ", got " + found); + } + } + + /* parse_json() + * + * Parse a JSON object. + */ + Json parse_json() { + char ch = get_next_token(); + if (failed) + return Json(); + + if (ch == '-' || (ch >= '0' && ch <= '9')) { + i--; + return parse_number(); + } + + if (ch == 't') + return expect("true", true); + + if (ch == 'f') + return expect("false", false); + + if (ch == 'n') + return expect("null", Json()); + + if (ch == '"') + return parse_string(); + + if (ch == '{') { + map data; + ch = get_next_token(); + if (ch == '}') + return data; + + while (1) { + if (ch != '"') + return fail("expected '\"' in object, got " + esc(ch)); + + string key = parse_string(); + if (failed) + return Json(); + + ch = get_next_token(); + if (ch != ':') + return fail("expected ':' in object, got " + esc(ch)); + + data[std::move(key)] = parse_json(); + if (failed) + return Json(); + + ch = get_next_token(); + if (ch == '}') + break; + if (ch != ',') + return fail("expected ',' in object, got " + esc(ch)); + + ch = get_next_token(); + } + return data; + } + + if (ch == '[') { + vector data; + ch = get_next_token(); + if (ch == ']') + return data; + + while (1) { + i--; + data.push_back(parse_json()); + if (failed) + return Json(); + + ch = get_next_token(); + if (ch == ']') + break; + if (ch != ',') + return fail("expected ',' in list, got " + esc(ch)); + + ch = get_next_token(); + (void)ch; + } + return data; + } + + return fail("expected value, got " + esc(ch)); + } +}; + +Json Json::parse(const string &in, string &err) { + JsonParser parser { in, 0, err, false }; + Json result = parser.parse_json(); + + // Check for any trailing garbage + parser.consume_whitespace(); + if (parser.i != in.size()) + return parser.fail("unexpected trailing " + esc(in[parser.i])); + + return result; +} + +// Documented in json11.hpp +vector Json::parse_multi(const string &in, string &err) { + JsonParser parser { in, 0, err, false }; + + vector json_vec; + while (parser.i != in.size() && !parser.failed) { + json_vec.push_back(parser.parse_json()); + // Check for another object + parser.consume_whitespace(); + } + return json_vec; +} + +/* * * * * * * * * * * * * * * * * * * * + * Shape-checking + */ + +bool Json::has_shape(const shape & types, string & err) const { + if (!is_object()) { + err = "expected JSON object, got " + dump(); + return false; + } + + for (auto & item : types) { + if ((*this)[item.first].type() != item.second) { + err = "bad type for " + item.first + " in " + dump(); + return false; + } + } + + return true; +} + +} // namespace json11 diff --git a/json11.hpp b/json11.hpp new file mode 100644 index 00000000..a2fa5c7b --- /dev/null +++ b/json11.hpp @@ -0,0 +1,187 @@ +/* json11 + * + * json11 is a tiny JSON library for C++11, providing JSON parsing and serialization. + * + * The core object provided by the library is json11::Json. A Json object represents any JSON + * value: null, bool, number (int or double), string (std::string), array (std::vector), or + * object (std::map). + * + * Json objects act like values: they can be assigned, copied, moved, compared for equality or + * order, etc. There are also helper methods Json::dump, to serialize a Json to a string, and + * Json::parse (static) to parse a std::string as a Json object. + * + * Internally, the various types of Json object are represented by the JsonValue class + * hierarchy. + */ + +/* Copyright (c) 2013 Dropbox, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace json11 { + +class JsonValue; + +class Json final { +public: + // Types + enum Type { + NUL, NUMBER, BOOL, STRING, ARRAY, OBJECT + }; + + // Array and object typedefs + typedef std::vector array; + typedef std::map object; + + // Constructors for the various types of JSON value. + Json() noexcept; // NUL + Json(std::nullptr_t) noexcept; // NUL + Json(double value); // NUMBER + Json(int value); // NUMBER + Json(bool value); // BOOL + Json(const std::string &value); // STRING + Json(std::string &&value); // STRING + Json(const char * value); // STRING + Json(const array &values); // ARRAY + Json(array &&values); // ARRAY + Json(const object &values); // OBJECT + Json(object &&values); // OBJECT + + // Implicit constructor: anything with a to_json() function. + template + Json(const T & t) : Json(t.to_json()) {} + + // Implicit constructor: map-like objects (std::map, std::unordered_map, etc) + template ().begin()->first)>::value + && std::is_constructible().begin()->second)>::value, + int>::type = 0> + Json(const M & m) : Json(object(m.begin(), m.end())) {} + + // Implicit constructor: vector-like objects (std::list, std::vector, std::set, etc) + template ().begin())>::value, + int>::type = 0> + Json(const V & v) : Json(array(v.begin(), v.end())) {} + + // This prevents Json(some_pointer) from accidentally producing a bool. Use + // Json(bool(some_pointer)) if that behavior is desired. + Json(void *) = delete; + + // Accessors + Type type() const; + + bool is_null() const { return type() == NUL; } + bool is_number() const { return type() == NUMBER; } + bool is_bool() const { return type() == BOOL; } + bool is_string() const { return type() == STRING; } + bool is_array() const { return type() == ARRAY; } + bool is_object() const { return type() == OBJECT; } + + // Return the enclosed value if this is a number, 0 otherwise. Note that json11 does not + // distinguish between integer and non-integer numbers - number_value() and int_value() + // can both be applied to a NUMBER-typed object. + double number_value() const; + int int_value() const; + + // Return the enclosed value if this is a boolean, false otherwise. + bool bool_value() const; + // Return the enclosed string if this is a string, "" otherwise. + const std::string &string_value() const; + // Return the enclosed std::vector if this is an array, or an empty vector otherwise. + const array &array_items() const; + // Return the enclosed std::map if this is an object, or an empty map otherwise. + const object &object_items() const; + + // Return a reference to arr[i] if this is an object, Json() otherwise. + const Json & operator[](size_t i) const; + // Return a reference to obj[key] if this is an object, Json() otherwise. + const Json & operator[](const std::string &key) const; + + // Serialize. + void dump(std::string &out) const; + std::string dump() const { + std::string out; + dump(out); + return out; + } + + // Parse. If parse fails, return Json() and assign an error message to err. + static Json parse(const std::string & in, std::string & err); + static Json parse(const char * in, std::string & err) { + if (in) { + return parse(std::string(in), err); + } else { + err = "null input"; + return nullptr; + } + } + // Parse multiple objects, concatenated or separated by whitespace + static std::vector parse_multi(const std::string & in, std::string & err); + + bool operator== (const Json &rhs) const; + bool operator< (const Json &rhs) const; + bool operator!= (const Json &rhs) { return !(*this == rhs); } + bool operator<= (const Json &rhs) { return !(rhs < *this); } + bool operator> (const Json &rhs) { return (rhs < *this); } + bool operator>= (const Json &rhs) { return !(*this < rhs); } + + /* has_shape(types, err) + * + * Return true if this is a JSON object and, for each item in types, has a field of + * the given type. If not, return false and set err to a descriptive message. + */ + typedef std::initializer_list> shape; + bool has_shape(const shape & types, std::string & err) const; + +private: + std::shared_ptr m_ptr; +}; + +// Internal class hierarchy - JsonValue objects are not exposed to users of this API. +class JsonValue { +protected: + friend class Json; + friend class JsonInt; + friend class JsonDouble; + virtual Json::Type type() const = 0; + virtual bool equals(const JsonValue * other) const = 0; + virtual bool less(const JsonValue * other) const = 0; + virtual void dump(std::string &out) const = 0; + virtual double number_value() const; + virtual int int_value() const; + virtual bool bool_value() const; + virtual const std::string &string_value() const; + virtual const Json::array &array_items() const; + virtual const Json &operator[](size_t i) const; + virtual const Json::object &object_items() const; + virtual const Json &operator[](const std::string &key) const; + virtual ~JsonValue() {} +}; + +} // namespace json11 diff --git a/test.cpp b/test.cpp new file mode 100644 index 00000000..f34c4a00 --- /dev/null +++ b/test.cpp @@ -0,0 +1,100 @@ +#include +#include +#include +#include +#include "json11.hpp" +#include + +using namespace json11; +using std::string; + +// Check that Json has the properties we want. +#include +#define CHECK_TRAIT(x) static_assert(std::x::value, #x) +CHECK_TRAIT(is_nothrow_constructible); +CHECK_TRAIT(is_nothrow_default_constructible); +CHECK_TRAIT(is_copy_constructible); +CHECK_TRAIT(is_nothrow_move_constructible); +CHECK_TRAIT(is_copy_assignable); +CHECK_TRAIT(is_nothrow_move_assignable); +CHECK_TRAIT(is_nothrow_destructible); + +void parse_from_stdin() { + string buf; + while (!std::cin.eof()) buf += std::cin.get(); + + string err; + auto json = Json::parse(buf, err); + if (!err.empty()) { + printf("Failed: %s\n", err.c_str()); + } else { + printf("Result: %s\n", json.dump().c_str()); + } +} + +int main(int argc, char **argv) { + if (argc == 2 && argv[1] == string("--stdin")) { + parse_from_stdin(); + return 0; + } + + const string simple_test = + R"({"k1":"v1", "k2":42, "k3":["a",123,true,false,null]})"; + + string err; + auto json = Json::parse(simple_test, err); + + std::cout << "k1: " << json["k1"].string_value() << "\n"; + std::cout << "k3: " << json["k3"].dump() << "\n"; + + for (auto &k : json["k3"].array_items()) { + std::cout << " - " << k.dump() << "\n"; + } + + // Json literals + Json obj = Json::object({ + { "k1", "v1" }, + { "k2", 42.0 }, + { "k3", Json::array({ "a", 123.0, true, false, nullptr }) }, + }); + + std::cout << "obj: " << obj.dump() << "\n"; + + assert(Json("a").number_value() == 0); + assert(Json("a").string_value() == "a"); + assert(Json().number_value() == 0); + + assert(obj == json); + assert(Json(42) == Json(42.0)); + assert(Json(42) != Json(42.1)); + + const string unicode_escape_test = + R"([ "blah\ud83d\udca9blah\ud83dblah\udca9blah\u0000blah\u1234" ])"; + + const char utf8[] = "blah" "\xf0\x9f\x92\xa9" "blah" "\xed\xa0\xbd" "blah" + "\xed\xb2\xa9" "blah" "\0" "blah" "\xe1\x88\xb4"; + + Json uni = Json::parse(unicode_escape_test, err); + assert(uni[0].string_value().size() == (sizeof utf8) - 1); + assert(memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); + + Json my_json = Json::object { + { "key1", "value1" }, + { "key2", false }, + { "key3", Json::array { 1, 2, 3 } }, + }; + std::string json_str = my_json.dump(); + printf("%s\n", json_str.c_str()); + + class Point { + public: + int x; + int y; + Point (int x, int y) : x(x), y(y) {} + Json to_json() const { return Json::array { x, y }; } + }; + + std::vector points = { { 1, 2 }, { 10, 20 }, { 100, 200 } }; + std::string points_json = Json(points).dump(); + printf("%s\n", points_json.c_str()); +} -- 2.30.2 From 979cb4ca767718d69ed593bbf836dc40c883636e Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Tue, 1 Oct 2013 19:04:32 -0700 Subject: [PATCH 17/91] Add LICENSE.txt; some minor cleanups - Add LICENSE.txt - Add a comment about how we handle numbers - Add more test code - Fix a narrowing-conversion issue - Fix a comment typo --- LICENSE.txt | 19 +++++++++++++++++++ json11.cpp | 2 +- json11.hpp | 15 ++++++++++++++- test.cpp | 13 +++++++++++++ 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..691742e9 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2013 Dropbox, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/json11.cpp b/json11.cpp index dc882071..9666db84 100644 --- a/json11.cpp +++ b/json11.cpp @@ -300,7 +300,7 @@ static inline string esc(char c) { return string(buf); } -static inline bool in_range (int x, int lower, int upper) { +static inline bool in_range(long x, long lower, long upper) { return (x >= lower && x <= upper); } diff --git a/json11.hpp b/json11.hpp index a2fa5c7b..af65cdbc 100644 --- a/json11.hpp +++ b/json11.hpp @@ -12,6 +12,19 @@ * * Internally, the various types of Json object are represented by the JsonValue class * hierarchy. + * + * A note on numbers - JSON specifies the syntax of number formatting but not its semantics, + * so some JSON implementations distinguish between integers and floating-point numbers, while + * some don't. In json11, we choose the latter. Because some JSON implementations (namely + * Javascript itself) treat all numbers as the same type, distinguishing the two leads + * to JSON that will be *silently* changed by a round-trip through those implementations. + * Dangerous! To avoid that risk, json11 stores all numbers as double internally, but also + * provides integer helpers. + * + * Fortunately, double-precision IEEE754 ('double') can precisely store any integer in the + * range +/-2^53, which includes every 'int' on most systems. (Timestamps often use int64 + * or long long to avoid the Y2038K problem; a double storing microseconds since some epoch + * will be exact for +/- 275 years.) */ /* Copyright (c) 2013 Dropbox, Inc. @@ -118,7 +131,7 @@ public: // Return the enclosed std::map if this is an object, or an empty map otherwise. const object &object_items() const; - // Return a reference to arr[i] if this is an object, Json() otherwise. + // Return a reference to arr[i] if this is an array, Json() otherwise. const Json & operator[](size_t i) const; // Return a reference to obj[key] if this is an object, Json() otherwise. const Json & operator[](const std::string &key) const; diff --git a/test.cpp b/test.cpp index f34c4a00..3bfe2e0c 100644 --- a/test.cpp +++ b/test.cpp @@ -4,6 +4,9 @@ #include #include "json11.hpp" #include +#include +#include +#include using namespace json11; using std::string; @@ -51,6 +54,16 @@ int main(int argc, char **argv) { std::cout << " - " << k.dump() << "\n"; } + std::list l1 { 1, 2, 3 }; + std::vector l2 { 1, 2, 3 }; + std::set l3 { 1, 2, 3 }; + assert(Json(l1) == Json(l2)); + assert(Json(l2) == Json(l3)); + + std::map m1 { { "k1", "v1" }, { "k2", "v2" } }; + std::unordered_map m2 { { "k1", "v1" }, { "k2", "v2" } }; + assert(Json(m1) == Json(m2)); + // Json literals Json obj = Json::object({ { "k1", "v1" }, -- 2.30.2 From d2e54859733e27dd94fcb900f075cc3bfb62729b Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Mon, 27 Jan 2014 10:59:17 -0800 Subject: [PATCH 18/91] Spacing fix --- json11.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json11.cpp b/json11.cpp index 9666db84..d2bb65e7 100644 --- a/json11.cpp +++ b/json11.cpp @@ -238,7 +238,7 @@ static const Json json_null; Json::Type Json::type() const { return m_ptr->type(); } double Json::number_value() const { return m_ptr->number_value(); } -int Json::int_value() const { return m_ptr->int_value(); } +int Json::int_value() const { return m_ptr->int_value(); } bool Json::bool_value() const { return m_ptr->bool_value(); } const string & Json::string_value() const { return m_ptr->string_value(); } const vector & Json::array_items() const { return m_ptr->array_items(); } -- 2.30.2 From 277e1b166fd98b4ad98e0d360eda443cf9f248db Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Wed, 29 Jan 2014 13:30:18 -0800 Subject: [PATCH 19/91] Make Json safe to use during static initialization. --- json11.cpp | 52 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/json11.cpp b/json11.cpp index d2bb65e7..dd132980 100644 --- a/json11.cpp +++ b/json11.cpp @@ -206,19 +206,38 @@ public: JsonNull() : Value(nullptr) {} }; +/* * * * * * * * * * * * * * * * * * * * + * Static globals - static-init-safe + */ +struct Statics { + const std::shared_ptr null = make_shared(); + const std::shared_ptr t = make_shared(true); + const std::shared_ptr f = make_shared(false); + const string empty_string; + const vector empty_vector; + const map empty_map; +}; + +const Statics & statics() { + static const Statics s {}; + return s; +} + +const Json & static_null() { + // This has to be separate, not in Statics, because Json() accesses statics().null. + static const Json json_null; + return json_null; +} + /* * * * * * * * * * * * * * * * * * * * * Constructors */ -static const std::shared_ptr obj_null(make_shared()); -static const std::shared_ptr obj_true(make_shared(true)); -static const std::shared_ptr obj_false(make_shared(false)); - -Json::Json() noexcept : m_ptr(obj_null) {} -Json::Json(std::nullptr_t) noexcept : m_ptr(obj_null) {} +Json::Json() noexcept : m_ptr(statics().null) {} +Json::Json(std::nullptr_t) noexcept : m_ptr(statics().null) {} Json::Json(double value) : m_ptr(make_shared(value)) {} Json::Json(int value) : m_ptr(make_shared(value)) {} -Json::Json(bool value) : m_ptr(value ? obj_true : obj_false) {} +Json::Json(bool value) : m_ptr(value ? statics().t : statics().f) {} Json::Json(const string &value) : m_ptr(make_shared(value)) {} Json::Json(string &&value) : m_ptr(make_shared(move(value))) {} Json::Json(const char * value) : m_ptr(make_shared(value)) {} @@ -231,11 +250,6 @@ Json::Json(Json::object &&values) : m_ptr(make_shared(move(valu * Accessors */ -static const string empty_string; -static const vector empty_vector; -static const map empty_map; -static const Json json_null; - Json::Type Json::type() const { return m_ptr->type(); } double Json::number_value() const { return m_ptr->number_value(); } int Json::int_value() const { return m_ptr->int_value(); } @@ -249,18 +263,18 @@ const Json & Json::operator[] (const string &key) const { return (*m_ptr)[key]; double JsonValue::number_value() const { return 0; } int JsonValue::int_value() const { return 0; } bool JsonValue::bool_value() const { return false; } -const string & JsonValue::string_value() const { return empty_string; } -const vector & JsonValue::array_items() const { return empty_vector; } -const map & JsonValue::object_items() const { return empty_map; } -const Json & JsonValue::operator[] (size_t) const { return json_null; } -const Json & JsonValue::operator[] (const string &) const { return json_null; } +const string & JsonValue::string_value() const { return statics().empty_string; } +const vector & JsonValue::array_items() const { return statics().empty_vector; } +const map & JsonValue::object_items() const { return statics().empty_map; } +const Json & JsonValue::operator[] (size_t) const { return static_null(); } +const Json & JsonValue::operator[] (const string &) const { return static_null(); } const Json & JsonObject::operator[] (const string &key) const { auto iter = m_value.find(key); - return (iter == m_value.end()) ? json_null : iter->second; + return (iter == m_value.end()) ? static_null() : iter->second; } const Json & JsonArray::operator[] (size_t i) const { - if (i >= m_value.size()) return json_null; + if (i >= m_value.size()) return static_null(); else return m_value[i]; } -- 2.30.2 From 731322c2b89a5c6d2aa5684d0b809e481e63313b Mon Sep 17 00:00:00 2001 From: Steven Kabbes Date: Thu, 27 Mar 2014 03:55:52 -0700 Subject: [PATCH 20/91] Add const to comparisons --- json11.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/json11.hpp b/json11.hpp index af65cdbc..779445e5 100644 --- a/json11.hpp +++ b/json11.hpp @@ -159,10 +159,10 @@ public: bool operator== (const Json &rhs) const; bool operator< (const Json &rhs) const; - bool operator!= (const Json &rhs) { return !(*this == rhs); } - bool operator<= (const Json &rhs) { return !(rhs < *this); } - bool operator> (const Json &rhs) { return (rhs < *this); } - bool operator>= (const Json &rhs) { return !(*this < rhs); } + bool operator!= (const Json &rhs) const { return !(*this == rhs); } + bool operator<= (const Json &rhs) const { return !(rhs < *this); } + bool operator> (const Json &rhs) const { return (rhs < *this); } + bool operator>= (const Json &rhs) const { return !(*this < rhs); } /* has_shape(types, err) * -- 2.30.2 From d9bdd84c762b3e5ca24d40685ade299d1a836a3b Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Wed, 4 Jun 2014 12:22:06 -0700 Subject: [PATCH 21/91] Limit maximum nesting depth (issue reported by Jeff Larson) --- json11.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/json11.cpp b/json11.cpp index dd132980..ad7f5b17 100644 --- a/json11.cpp +++ b/json11.cpp @@ -27,6 +27,8 @@ namespace json11 { +static const int max_depth = 200; + using std::string; using std::vector; using std::map; @@ -557,7 +559,11 @@ struct JsonParser { * * Parse a JSON object. */ - Json parse_json() { + Json parse_json(int depth) { + if (depth > max_depth) { + return fail("exceeded maximum nesting depth"); + } + char ch = get_next_token(); if (failed) return Json(); @@ -597,7 +603,7 @@ struct JsonParser { if (ch != ':') return fail("expected ':' in object, got " + esc(ch)); - data[std::move(key)] = parse_json(); + data[std::move(key)] = parse_json(depth + 1); if (failed) return Json(); @@ -620,7 +626,7 @@ struct JsonParser { while (1) { i--; - data.push_back(parse_json()); + data.push_back(parse_json(depth + 1)); if (failed) return Json(); @@ -642,7 +648,7 @@ struct JsonParser { Json Json::parse(const string &in, string &err) { JsonParser parser { in, 0, err, false }; - Json result = parser.parse_json(); + Json result = parser.parse_json(0); // Check for any trailing garbage parser.consume_whitespace(); @@ -658,7 +664,7 @@ vector Json::parse_multi(const string &in, string &err) { vector json_vec; while (parser.i != in.size() && !parser.failed) { - json_vec.push_back(parser.parse_json()); + json_vec.push_back(parser.parse_json(0)); // Check for another object parser.consume_whitespace(); } -- 2.30.2 From 710352234426a869b5ebd4f2e5ff8c71ccd6bc07 Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Wed, 13 Aug 2014 14:58:20 -0700 Subject: [PATCH 22/91] Identify map- and vector-like objects in a VS14-compatible way --- json11.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/json11.hpp b/json11.hpp index 779445e5..fe9bba40 100644 --- a/json11.hpp +++ b/json11.hpp @@ -91,14 +91,14 @@ public: // Implicit constructor: map-like objects (std::map, std::unordered_map, etc) template ().begin()->first)>::value - && std::is_constructible().begin()->second)>::value, + std::is_constructible::value + && std::is_constructible::value, int>::type = 0> Json(const M & m) : Json(object(m.begin(), m.end())) {} // Implicit constructor: vector-like objects (std::list, std::vector, std::set, etc) template ().begin())>::value, + std::is_constructible::value, int>::type = 0> Json(const V & v) : Json(array(v.begin(), v.end())) {} -- 2.30.2 From 679e4b83d2bb3595524c26fc97f8e3645ae8ab08 Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Thu, 14 Aug 2014 12:14:21 -0700 Subject: [PATCH 23/91] Another VS14 fix --- json11.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/json11.cpp b/json11.cpp index ad7f5b17..a25d0feb 100644 --- a/json11.cpp +++ b/json11.cpp @@ -218,6 +218,7 @@ struct Statics { const string empty_string; const vector empty_vector; const map empty_map; + Statics() {} }; const Statics & statics() { -- 2.30.2 From 73baf7e6772ca01705fe92b0519e5a936845e9a4 Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Mon, 15 Sep 2014 11:33:11 -0700 Subject: [PATCH 24/91] Style fixes --- json11.cpp | 72 +++++++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/json11.cpp b/json11.cpp index a25d0feb..1ef01db8 100644 --- a/json11.cpp +++ b/json11.cpp @@ -78,16 +78,16 @@ static void dump(const string &value, string &out) { out += "\\r"; } else if (ch == '\t') { out += "\\t"; - } else if ((uint8_t)ch <= 0x1f) { + } else if (static_cast(ch) <= 0x1f) { char buf[8]; snprintf(buf, sizeof buf, "\\u%04x", ch); out += buf; - } else if ((uint8_t)ch == 0xe2 && (uint8_t)value[i+1] == 0x80 - && (uint8_t)value[i+2] == 0xa8) { + } else if (static_cast(ch) == 0xe2 && static_cast(value[i+1]) == 0x80 + && static_cast(value[i+2]) == 0xa8) { out += "\\u2028"; i += 2; - } else if ((uint8_t)ch == 0xe2 && (uint8_t)value[i+1] == 0x80 - && (uint8_t)value[i+2] == 0xa9) { + } else if (static_cast(ch) == 0xe2 && static_cast(value[i+1]) == 0x80 + && static_cast(value[i+2]) == 0xa9) { out += "\\u2029"; i += 2; } else { @@ -136,71 +136,71 @@ class Value : public JsonValue { protected: // Constructors - Value(const T &value) : m_value(value) {} - Value(T &&value) : m_value(move(value)) {} + explicit Value(const T &value) : m_value(value) {} + explicit Value(T &&value) : m_value(move(value)) {} // Get type tag - Json::Type type() const { + Json::Type type() const override { return tag; } // Comparisons - bool equals(const JsonValue * other) const { + bool equals(const JsonValue * other) const override { return m_value == reinterpret_cast *>(other)->m_value; } - bool less(const JsonValue * other) const { + bool less(const JsonValue * other) const override { return m_value < reinterpret_cast *>(other)->m_value; } const T m_value; - void dump(string &out) const { json11::dump(m_value, out); } + void dump(string &out) const override { json11::dump(m_value, out); } }; class JsonDouble final : public Value { - double number_value() const { return m_value; } - int int_value() const { return m_value; } - bool equals(const JsonValue * other) const { return m_value == other->number_value(); } - bool less(const JsonValue * other) const { return m_value < other->number_value(); } + double number_value() const override { return m_value; } + int int_value() const override { return m_value; } + bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } + bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: - JsonDouble(double value) : Value(value) {} + explicit JsonDouble(double value) : Value(value) {} }; class JsonInt final : public Value { - double number_value() const { return m_value; } - int int_value() const { return m_value; } - bool equals(const JsonValue * other) const { return m_value == other->number_value(); } - bool less(const JsonValue * other) const { return m_value < other->number_value(); } + double number_value() const override { return m_value; } + int int_value() const override { return m_value; } + bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } + bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: - JsonInt(double value) : Value(value) {} + explicit JsonInt(double value) : Value(value) {} }; class JsonBoolean final : public Value { - bool bool_value() const { return m_value; } + bool bool_value() const override { return m_value; } public: - JsonBoolean(bool value) : Value(value) {} + explicit JsonBoolean(bool value) : Value(value) {} }; class JsonString final : public Value { - const string &string_value() const { return m_value; } + const string &string_value() const override { return m_value; } public: - JsonString(const string &value) : Value(value) {} - JsonString(string &&value) : Value(move(value)) {} + explicit JsonString(const string &value) : Value(value) {} + explicit JsonString(string &&value) : Value(move(value)) {} }; class JsonArray final : public Value { - const Json::array &array_items() const { return m_value; } - const Json & operator[](size_t i) const; + const Json::array &array_items() const override { return m_value; } + const Json & operator[](size_t i) const override; public: - JsonArray(const Json::array &value) : Value(value) {} - JsonArray(Json::array &&value) : Value(move(value)) {} + explicit JsonArray(const Json::array &value) : Value(value) {} + explicit JsonArray(Json::array &&value) : Value(move(value)) {} }; class JsonObject final : public Value { - const Json::object &object_items() const { return m_value; } - const Json & operator[](const string &key) const; + const Json::object &object_items() const override { return m_value; } + const Json & operator[](const string &key) const override; public: - JsonObject(const Json::object &value) : Value(value) {} - JsonObject(Json::object &&value) : Value(move(value)) {} + explicit JsonObject(const Json::object &value) : Value(value) {} + explicit JsonObject(Json::object &&value) : Value(move(value)) {} }; class JsonNull final : public Value { @@ -309,7 +309,7 @@ bool Json::operator< (const Json &other) const { */ static inline string esc(char c) { char buf[12]; - if ((uint8_t)c >= 0x20 && (uint8_t)c <= 0x7f) { + if (static_cast(c) >= 0x20 && static_cast(c) <= 0x7f) { snprintf(buf, sizeof buf, "'%c' (%d)", c, c); } else { snprintf(buf, sizeof buf, "(%d)", c); @@ -508,7 +508,7 @@ struct JsonParser { } if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' - && (i - start_pos) <= (size_t)std::numeric_limits::digits10) { + && (i - start_pos) <= static_cast(std::numeric_limits::digits10)) { return std::atoi(str.c_str() + start_pos); } -- 2.30.2 From 6de4c29e763613e818d347006d46a638b039b41f Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Mon, 22 Sep 2014 11:59:58 -0700 Subject: [PATCH 25/91] fix int->double->int conversion in JsonInt --- json11.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index 1ef01db8..64245542 100644 --- a/json11.cpp +++ b/json11.cpp @@ -158,7 +158,7 @@ protected: class JsonDouble final : public Value { double number_value() const override { return m_value; } - int int_value() const override { return m_value; } + int int_value() const override { return static_cast(m_value); } bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: @@ -171,7 +171,7 @@ class JsonInt final : public Value { bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: - explicit JsonInt(double value) : Value(value) {} + explicit JsonInt(int value) : Value(value) {} }; class JsonBoolean final : public Value { -- 2.30.2 From c6a873617153984d904eb0c83fd226ae10bfec86 Mon Sep 17 00:00:00 2001 From: Masamitsu MURASE Date: Sun, 30 Nov 2014 03:36:23 +0900 Subject: [PATCH 26/91] Use string::compare to improve performance. --- json11.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/json11.cpp b/json11.cpp index 64245542..88e376be 100644 --- a/json11.cpp +++ b/json11.cpp @@ -547,12 +547,11 @@ struct JsonParser { Json expect(const string &expected, Json res) { assert(i != 0); i--; - const string found = str.substr(i, expected.length()); - if (expected == found) { + if (str.compare(i, expected.length(), expected) == 0) { i += expected.length(); return res; } else { - return fail("parse error: expected " + expected + ", got " + found); + return fail("parse error: expected " + expected + ", got " + str.substr(i, expected.length())); } } -- 2.30.2 From 37ca641d82f9f80dc3ae6dbd6557a479866537fa Mon Sep 17 00:00:00 2001 From: Masamitsu MURASE Date: Sun, 30 Nov 2014 03:49:23 +0900 Subject: [PATCH 27/91] Use static_cast instead of reinterpret_cast for hierarchy navigation. --- json11.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index 64245542..9803846f 100644 --- a/json11.cpp +++ b/json11.cpp @@ -146,10 +146,10 @@ protected: // Comparisons bool equals(const JsonValue * other) const override { - return m_value == reinterpret_cast *>(other)->m_value; + return m_value == static_cast *>(other)->m_value; } bool less(const JsonValue * other) const override { - return m_value < reinterpret_cast *>(other)->m_value; + return m_value < static_cast *>(other)->m_value; } const T m_value; -- 2.30.2 From 7fd738ae42fa03cca84eaba6b460f9e407268943 Mon Sep 17 00:00:00 2001 From: k0zmo Date: Tue, 30 Dec 2014 10:28:51 +0100 Subject: [PATCH 28/91] iterate over pair of const string and Json --- json11.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index c9ac2a55..3d28c400 100644 --- a/json11.cpp +++ b/json11.cpp @@ -100,7 +100,7 @@ static void dump(const string &value, string &out) { static void dump(const Json::array &values, string &out) { bool first = true; out += "["; - for (auto &value : values) { + for (const auto &value : values) { if (!first) out += ", "; value.dump(out); @@ -112,7 +112,7 @@ static void dump(const Json::array &values, string &out) { static void dump(const Json::object &values, string &out) { bool first = true; out += "{"; - for (const std::pair &kv : values) { + for (const auto &kv : values) { if (!first) out += ", "; dump(kv.first, out); -- 2.30.2 From ae9542cc35cedd69e7d74056bf2c0963eff7a470 Mon Sep 17 00:00:00 2001 From: Steve Carroll Date: Tue, 3 Mar 2015 17:47:56 -0800 Subject: [PATCH 29/91] in encode_utf8, there is an implicit truncating cast that VS2015CTP6 is warning about. I've added static_cast to silence the warning and express the intent. --- json11.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/json11.cpp b/json11.cpp index 3d28c400..2dc955ab 100644 --- a/json11.cpp +++ b/json11.cpp @@ -381,19 +381,19 @@ struct JsonParser { return; if (pt < 0x80) { - out += pt; + out += static_cast(pt); } else if (pt < 0x800) { - out += (pt >> 6) | 0xC0; - out += (pt & 0x3F) | 0x80; + out += static_cast((pt >> 6) | 0xC0); + out += static_cast((pt & 0x3F) | 0x80); } else if (pt < 0x10000) { - out += (pt >> 12) | 0xE0; - out += ((pt >> 6) & 0x3F) | 0x80; - out += (pt & 0x3F) | 0x80; + out += static_cast((pt >> 12) | 0xE0); + out += static_cast(((pt >> 6) & 0x3F) | 0x80); + out += static_cast((pt & 0x3F) | 0x80); } else { - out += (pt >> 18) | 0xF0; - out += ((pt >> 12) & 0x3F) | 0x80; - out += ((pt >> 6) & 0x3F) | 0x80; - out += (pt & 0x3F) | 0x80; + out += static_cast((pt >> 18) | 0xF0); + out += static_cast(((pt >> 12) & 0x3F) | 0x80); + out += static_cast(((pt >> 6) & 0x3F) | 0x80); + out += static_cast((pt & 0x3F) | 0x80); } } -- 2.30.2 From cfdd67577c0694f26b7609599a9e087d0ab31643 Mon Sep 17 00:00:00 2001 From: Steven Kabbes Date: Tue, 18 Nov 2014 15:57:04 +0100 Subject: [PATCH 30/91] Use strtod not atof --- json11.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json11.cpp b/json11.cpp index 2dc955ab..f0f85a47 100644 --- a/json11.cpp +++ b/json11.cpp @@ -536,7 +536,7 @@ struct JsonParser { i++; } - return std::atof(str.c_str() + start_pos); + return std::strtod(str.c_str() + start_pos, nullptr); } /* expect(str, res) -- 2.30.2 From e15ff418dd1b824addc0c31c779a041f07003406 Mon Sep 17 00:00:00 2001 From: Andrew Krieger Date: Wed, 22 Apr 2015 14:00:47 -0700 Subject: [PATCH 31/91] For extra safety, add an explicit bounds check in utf8 parsing. --- json11.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/json11.cpp b/json11.cpp index f0f85a47..34f90983 100644 --- a/json11.cpp +++ b/json11.cpp @@ -435,6 +435,12 @@ struct JsonParser { if (ch == 'u') { // Extract 4-byte escape sequence string esc = str.substr(i, 4); + // Explicitly check length of the substring. The following loop + // relies on std::string returning the terminating NUL when + // accessing str[length]. Checking here reduces brittleness. + if (esc.length() < 4) { + return fail("bad \\u escape: " + esc, ""); + } for (int j = 0; j < 4; j++) { if (!in_range(esc[j], 'a', 'f') && !in_range(esc[j], 'A', 'F') && !in_range(esc[j], '0', '9')) -- 2.30.2 From 0e8c5ba68f66b0056cc42373b4cd892b9667171d Mon Sep 17 00:00:00 2001 From: Jean-Claude Monnin Date: Thu, 4 Jun 2015 10:35:12 +0200 Subject: [PATCH 32/91] When dumping non-finite floating point values, output 'null'. In JSON the float point special values NaN and Infinity should serialised to 'null'. Previously, 'snprintf' in 'dump' was giving a string that isn't compliant to the JSON standard. --- json11.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/json11.cpp b/json11.cpp index 34f90983..0aa125b4 100644 --- a/json11.cpp +++ b/json11.cpp @@ -21,6 +21,7 @@ #include "json11.hpp" #include +#include #include #include #include @@ -45,9 +46,13 @@ static void dump(std::nullptr_t, string &out) { } static void dump(double value, string &out) { - char buf[32]; - snprintf(buf, sizeof buf, "%.17g", value); - out += buf; + if (std::isfinite(value)) { + char buf[32]; + snprintf(buf, sizeof buf, "%.17g", value); + out += buf; + } else { + out += "null"; + } } static void dump(int value, string &out) { -- 2.30.2 From 50ed8c90e518338cc5d0b76fea054ca5b2351b64 Mon Sep 17 00:00:00 2001 From: Andrew Twyman Date: Wed, 17 Jun 2015 15:02:22 -0700 Subject: [PATCH 33/91] Test to demonstrate compiler behavior change in Xcode 7 / clang 3.7. --- test.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test.cpp b/test.cpp index 3bfe2e0c..f3c3301a 100644 --- a/test.cpp +++ b/test.cpp @@ -91,6 +91,14 @@ int main(int argc, char **argv) { assert(uni[0].string_value().size() == (sizeof utf8) - 1); assert(memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); + // Demonstrates the behavior change in Xcode 7 / Clang 3.7 described + // here: https://llvm.org/bugs/show_bug.cgi?id=23812 + Json nested_array = Json::array { Json::array { 1, 2, 3 } }; + assert(nested_array.is_array()); + assert(nested_array.array_items().size() == 1); + assert(nested_array.array_items()[0].is_array()); + assert(nested_array.array_items()[0].array_items().size() == 3); + Json my_json = Json::object { { "key1", "value1" }, { "key2", false }, -- 2.30.2 From 61ba0a1dd278534b447a1c4c7cedf2f4e0dbebcf Mon Sep 17 00:00:00 2001 From: Andrew Twyman Date: Wed, 17 Jun 2015 18:02:34 -0700 Subject: [PATCH 34/91] Make test.cpp compilable with GCC. Add clean rule to Makefile. --- Makefile | 7 ++++++- test.cpp | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f251c1fb..5a6adc6e 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,7 @@ test: json11.cpp json11.hpp test.cpp - clang++ -O -std=c++11 -stdlib=libc++ json11.cpp test.cpp -o test -fno-rtti -fno-exceptions + $(CXX) -O -std=c++11 json11.cpp test.cpp -o test -fno-rtti -fno-exceptions + +clean: + if [ -e test ]; then rm test; fi + +.PHONY: clean diff --git a/test.cpp b/test.cpp index f3c3301a..f6fb5473 100644 --- a/test.cpp +++ b/test.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include "json11.hpp" @@ -89,7 +90,7 @@ int main(int argc, char **argv) { Json uni = Json::parse(unicode_escape_test, err); assert(uni[0].string_value().size() == (sizeof utf8) - 1); - assert(memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); + assert(std::memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); // Demonstrates the behavior change in Xcode 7 / Clang 3.7 described // here: https://llvm.org/bugs/show_bug.cgi?id=23812 -- 2.30.2 From 0c6e9d77a41a23d8486bb83d8428f1c5035bd9dd Mon Sep 17 00:00:00 2001 From: Paul C Roberts Date: Wed, 19 Aug 2015 14:53:28 -0700 Subject: [PATCH 35/91] Add cmake support This uses cmake to allow building as a static library and to build the tests To use `cmake` ``` $ mkdir build $ cd build $ cmake .. . . -- Build files have been written to: ~/json11/build $ make Scanning dependencies of target json11 [ 20%] Building CXX object CMakeFiles/json11.dir/json11.cpp.o [ 40%] Linking CXX static library libjson11.a [ 40%] Built target json11 Scanning dependencies of target json11_test [ 60%] Building CXX object CMakeFiles/json11_test.dir/json11.cpp.o [ 80%] Building CXX object CMakeFiles/json11_test.dir/test.cpp.o [100%] Linking CXX executable json11_test [100%] Built target json11_test $ make test running tests... Test project /Users/paul/dev/json11/build Start 1: json11_test 1/1 Test #1: json11_test ...................... Passed 0.00 sec 100% tests passed, 0 tests failed out of 1 Total Test time (real) = 0.01 sec $ ``` --- CMakeLists.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..ba8edd5b --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,21 @@ +project(json11) + +cmake_minimum_required(VERSION 2.8) + +enable_testing() + +add_definitions( + -std=c++11 + -fno-rtti + -fno-exceptions + -Wall + -Wextra + -Werror) + +set(json11_SRCS json11.cpp) + +add_library(json11 STATIC ${json11_SRCS}) + +add_test(json11_test json11_test) + +add_executable(json11_test ${json11_SRCS} test.cpp) -- 2.30.2 From a021c3fbe2542c840346bf75380dcdf1cc6b712f Mon Sep 17 00:00:00 2001 From: Paul C Roberts Date: Wed, 19 Aug 2015 15:00:12 -0700 Subject: [PATCH 36/91] Fixes bug in parse_from_stdin Handles end of file correctly --- test.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test.cpp b/test.cpp index f6fb5473..8b62aeb5 100644 --- a/test.cpp +++ b/test.cpp @@ -25,7 +25,10 @@ CHECK_TRAIT(is_nothrow_destructible); void parse_from_stdin() { string buf; - while (!std::cin.eof()) buf += std::cin.get(); + string line; + while (std::getline(std::cin, line)) { + buf += line + "\n"; + } string err; auto json = Json::parse(buf, err); -- 2.30.2 From 2d1d176a5599fedc4d424cd5a784b002f24ff04d Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Fri, 27 Nov 2015 16:31:43 +0100 Subject: [PATCH 37/91] add routine to detect c-style comments --- json11.cpp | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/json11.cpp b/json11.cpp index 0aa125b4..e2085a1c 100644 --- a/json11.cpp +++ b/json11.cpp @@ -364,6 +364,33 @@ struct JsonParser { i++; } + /* consume_comment() + * + * Advance comments (c-style inline and multiline). + */ + void consume_comment() { + if (str[i] == '/') { + i++; + if (str[i] == '/') { // inline comment + i++; + // advance until next line + while (str[i] != '\n') + i++; + consume_whitespace(); + consume_comment(); + } + else if (str[i] == '*') { // multiline comment + i++; + // advance until closing tokens + while (!(str[i] == '*' && str[i+1] == '/')) + i++; + i += 2; + consume_whitespace(); + consume_comment(); + } + } + } + /* get_next_token() * * Return the next non-whitespace character. If the end of the input is reached, -- 2.30.2 From 08c391f89ad253eafb21fcad6b430a8c4df70c15 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Fri, 27 Nov 2015 16:41:05 +0100 Subject: [PATCH 38/91] introduce consume_garbage() new routine that consumes whitespaces and comments. activated by JSON11_COMMENTS pre-processor flag. --- json11.cpp | 18 +++++++++++++++--- json11.hpp | 2 ++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/json11.cpp b/json11.cpp index e2085a1c..4c473c1e 100644 --- a/json11.cpp +++ b/json11.cpp @@ -391,13 +391,25 @@ struct JsonParser { } } + /* consume_garbage() + * + * Advance until the current character is non-whitespace and non-comment. + */ + void consume_garbage() { + consume_whitespace(); +#ifdef JSON11_COMMENTS + consume_comment(); + consume_whitespace(); +#endif + } + /* get_next_token() * * Return the next non-whitespace character. If the end of the input is reached, * flag an error and return 0. */ char get_next_token() { - consume_whitespace(); + consume_garbage(); if (i == str.size()) return fail("unexpected end of input", 0); @@ -689,7 +701,7 @@ Json Json::parse(const string &in, string &err) { Json result = parser.parse_json(0); // Check for any trailing garbage - parser.consume_whitespace(); + parser.consume_garbage(); if (parser.i != in.size()) return parser.fail("unexpected trailing " + esc(in[parser.i])); @@ -704,7 +716,7 @@ vector Json::parse_multi(const string &in, string &err) { while (parser.i != in.size() && !parser.failed) { json_vec.push_back(parser.parse_json(0)); // Check for another object - parser.consume_whitespace(); + parser.consume_garbage(); } return json_vec; } diff --git a/json11.hpp b/json11.hpp index fe9bba40..ff399d1a 100644 --- a/json11.hpp +++ b/json11.hpp @@ -56,6 +56,8 @@ #include #include +#define JSON11_COMMENTS 1 + namespace json11 { class JsonValue; -- 2.30.2 From de098c4d52e466049f85c2a378ef04b92cd3433a Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Fri, 27 Nov 2015 16:46:18 +0100 Subject: [PATCH 39/91] add testing for comment functionality --- test.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test.cpp b/test.cpp index 8b62aeb5..7a40857b 100644 --- a/test.cpp +++ b/test.cpp @@ -58,6 +58,28 @@ int main(int argc, char **argv) { std::cout << " - " << k.dump() << "\n"; } +#ifdef JSON11_COMMENTS + const string comment_test = R"({ + // comment + "a": 1, + // comment + // continued + "b": "text", + /* multi + line + comment */ + "c": [1, 2, 3] + })"; + + string err_comment; + auto json_comment = Json::parse(comment_test, err_comment); + if (!err_comment.empty()) { + printf("Failed: %s\n", err_comment.c_str()); + } else { + printf("Result: %s\n", json_comment.dump().c_str()); + } +#endif + std::list l1 { 1, 2, 3 }; std::vector l2 { 1, 2, 3 }; std::set l3 { 1, 2, 3 }; -- 2.30.2 From 882feb56ac7e46c703be5a063775ba9da3decbab Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Mon, 30 Nov 2015 12:27:35 +0100 Subject: [PATCH 40/91] add bool to detect comments as run-time option. --- json11.cpp | 19 +++++++++++-------- json11.hpp | 16 ++++++++++------ test.cpp | 5 ++--- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/json11.cpp b/json11.cpp index 4c473c1e..4d239c02 100644 --- a/json11.cpp +++ b/json11.cpp @@ -338,6 +338,7 @@ struct JsonParser { size_t i; string &err; bool failed; + bool detect_comments; /* fail(msg, err_ret = Json()) * @@ -397,10 +398,10 @@ struct JsonParser { */ void consume_garbage() { consume_whitespace(); -#ifdef JSON11_COMMENTS - consume_comment(); - consume_whitespace(); -#endif + if(detect_comments) { + consume_comment(); + consume_whitespace(); + } } /* get_next_token() @@ -696,8 +697,8 @@ struct JsonParser { } }; -Json Json::parse(const string &in, string &err) { - JsonParser parser { in, 0, err, false }; +Json Json::parse(const string &in, string &err, bool detect_comments) { + JsonParser parser { in, 0, err, false, detect_comments }; Json result = parser.parse_json(0); // Check for any trailing garbage @@ -709,8 +710,10 @@ Json Json::parse(const string &in, string &err) { } // Documented in json11.hpp -vector Json::parse_multi(const string &in, string &err) { - JsonParser parser { in, 0, err, false }; +vector Json::parse_multi(const string &in, + string &err, + bool detect_comments) { + JsonParser parser { in, 0, err, false, detect_comments }; vector json_vec; while (parser.i != in.size() && !parser.failed) { diff --git a/json11.hpp b/json11.hpp index ff399d1a..582c43f0 100644 --- a/json11.hpp +++ b/json11.hpp @@ -56,8 +56,6 @@ #include #include -#define JSON11_COMMENTS 1 - namespace json11 { class JsonValue; @@ -147,17 +145,23 @@ public: } // Parse. If parse fails, return Json() and assign an error message to err. - static Json parse(const std::string & in, std::string & err); - static Json parse(const char * in, std::string & err) { + static Json parse(const std::string & in, + std::string & err, + bool detect_comments = false); + static Json parse(const char * in, + std::string & err, + bool detect_comments = false) { if (in) { - return parse(std::string(in), err); + return parse(std::string(in), err, detect_comments); } else { err = "null input"; return nullptr; } } // Parse multiple objects, concatenated or separated by whitespace - static std::vector parse_multi(const std::string & in, std::string & err); + static std::vector parse_multi(const std::string & in, + std::string & err, + bool detect_comments = false); bool operator== (const Json &rhs) const; bool operator< (const Json &rhs) const; diff --git a/test.cpp b/test.cpp index 7a40857b..26fd5b88 100644 --- a/test.cpp +++ b/test.cpp @@ -58,7 +58,6 @@ int main(int argc, char **argv) { std::cout << " - " << k.dump() << "\n"; } -#ifdef JSON11_COMMENTS const string comment_test = R"({ // comment "a": 1, @@ -72,13 +71,13 @@ int main(int argc, char **argv) { })"; string err_comment; - auto json_comment = Json::parse(comment_test, err_comment); + auto json_comment = Json::parse( + comment_test, err_comment, /*detect_comments=*/ true); if (!err_comment.empty()) { printf("Failed: %s\n", err_comment.c_str()); } else { printf("Result: %s\n", json_comment.dump().c_str()); } -#endif std::list l1 { 1, 2, 3 }; std::vector l2 { 1, 2, 3 }; -- 2.30.2 From b05e655c0a37098ca8eaa106cfc78cd424bc4c55 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Mon, 30 Nov 2015 12:28:45 +0100 Subject: [PATCH 41/91] detect multiple comments with a loop instead of using recursion --- json11.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/json11.cpp b/json11.cpp index 4d239c02..a1efd618 100644 --- a/json11.cpp +++ b/json11.cpp @@ -369,7 +369,8 @@ struct JsonParser { * * Advance comments (c-style inline and multiline). */ - void consume_comment() { + bool consume_comment() { + bool comment_found = false; if (str[i] == '/') { i++; if (str[i] == '/') { // inline comment @@ -377,8 +378,7 @@ struct JsonParser { // advance until next line while (str[i] != '\n') i++; - consume_whitespace(); - consume_comment(); + comment_found = true; } else if (str[i] == '*') { // multiline comment i++; @@ -386,10 +386,10 @@ struct JsonParser { while (!(str[i] == '*' && str[i+1] == '/')) i++; i += 2; - consume_whitespace(); - consume_comment(); + comment_found = true; } } + return comment_found; } /* consume_garbage() @@ -399,8 +399,12 @@ struct JsonParser { void consume_garbage() { consume_whitespace(); if(detect_comments) { - consume_comment(); - consume_whitespace(); + bool comment_found = false; + do { + comment_found = consume_comment(); + consume_whitespace(); + } + while(comment_found); } } -- 2.30.2 From 2f5c64225d4bbb9c5189d940cce65189c8d3c3ad Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Mon, 30 Nov 2015 12:40:20 +0100 Subject: [PATCH 42/91] detect malformed comments --- json11.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index a1efd618..f67fda51 100644 --- a/json11.cpp +++ b/json11.cpp @@ -383,11 +383,16 @@ struct JsonParser { else if (str[i] == '*') { // multiline comment i++; // advance until closing tokens - while (!(str[i] == '*' && str[i+1] == '/')) - i++; + while (!(str[i] == '*' && str[i+1] == '/')) { + if (i == str.size()) + return fail( + "unexpected end of input inside multi-line comment", 0); + i++;} i += 2; comment_found = true; } + else + return fail("malformed comment", 0); } return comment_found; } -- 2.30.2 From d292fce9f23f394c381792581dcb22c8adabaef7 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Mon, 30 Nov 2015 12:42:24 +0100 Subject: [PATCH 43/91] improve comment test. test also for nested and mixed comments. whitespaces/newlines are already intermixed between comments. --- test.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test.cpp b/test.cpp index 26fd5b88..87f27ede 100644 --- a/test.cpp +++ b/test.cpp @@ -59,7 +59,7 @@ int main(int argc, char **argv) { } const string comment_test = R"({ - // comment + // comment /* with nested comment */ "a": 1, // comment // continued @@ -67,6 +67,7 @@ int main(int argc, char **argv) { /* multi line comment */ + // and single-line comment "c": [1, 2, 3] })"; -- 2.30.2 From f21b8c360e66527911258828a86c280c842202f4 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Mon, 30 Nov 2015 12:43:40 +0100 Subject: [PATCH 44/91] add malformed comment tests. add 3 testes for: - unended multi-line comment, - malformed single-line comment, - trailing slash --- test.cpp | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test.cpp b/test.cpp index 87f27ede..ba786d83 100644 --- a/test.cpp +++ b/test.cpp @@ -80,6 +80,45 @@ int main(int argc, char **argv) { printf("Result: %s\n", json_comment.dump().c_str()); } + string failing_comment_test = R"({ + /* bad comment + "a": 1, + })"; + + string err_failing_comment; + auto json_failing_comment = Json::parse( + failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + if (!err_failing_comment.empty()) { + printf("Failed: %s\n", err_failing_comment.c_str()); + } else { + printf("Result: %s\n", json_failing_comment.dump().c_str()); + } + + failing_comment_test = R"({ + / / bad comment + "a": 1, + })"; + + json_failing_comment = Json::parse( + failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + if (!err_failing_comment.empty()) { + printf("Failed: %s\n", err_failing_comment.c_str()); + } else { + printf("Result: %s\n", json_failing_comment.dump().c_str()); + } + + failing_comment_test = R"({ + "a": 1, + }/)"; + + json_failing_comment = Json::parse( + failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + if (!err_failing_comment.empty()) { + printf("Failed: %s\n", err_failing_comment.c_str()); + } else { + printf("Result: %s\n", json_failing_comment.dump().c_str()); + } + std::list l1 { 1, 2, 3 }; std::vector l2 { 1, 2, 3 }; std::set l3 { 1, 2, 3 }; -- 2.30.2 From 4b0f5cfd774fd431f15858b15bd1152003f4d524 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Tue, 1 Dec 2015 10:59:22 +0100 Subject: [PATCH 45/91] check for end of input on every increment of the cursor --- json11.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index f67fda51..3ea9a67f 100644 --- a/json11.cpp +++ b/json11.cpp @@ -373,22 +373,35 @@ struct JsonParser { bool comment_found = false; if (str[i] == '/') { i++; + if (i == str.size()) + return fail("unexpected end of input inside comment", 0); if (str[i] == '/') { // inline comment i++; + if (i == str.size()) + return fail("unexpected end of input inside inline comment", 0); // advance until next line - while (str[i] != '\n') + while (str[i] != '\n') { i++; + if (i == str.size()) + return fail("unexpected end of input inside inline comment", 0); + } comment_found = true; } else if (str[i] == '*') { // multiline comment i++; + if (i == str.size()) + return fail("unexpected end of input inside multi-line comment", 0); // advance until closing tokens while (!(str[i] == '*' && str[i+1] == '/')) { + i++; if (i == str.size()) return fail( "unexpected end of input inside multi-line comment", 0); - i++;} + } i += 2; + if (i == str.size()) + return fail( + "unexpected end of input inside multi-line comment", 0); comment_found = true; } else -- 2.30.2 From 982b2d8885741bbef666f07ab874bbe802bb9d0b Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Tue, 1 Dec 2015 11:00:11 +0100 Subject: [PATCH 46/91] improve testing for bad inline comments --- test.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test.cpp b/test.cpp index ba786d83..7d77aa3a 100644 --- a/test.cpp +++ b/test.cpp @@ -95,9 +95,7 @@ int main(int argc, char **argv) { } failing_comment_test = R"({ - / / bad comment - "a": 1, - })"; + / / bad comment })"; json_failing_comment = Json::parse( failing_comment_test, err_failing_comment, /*detect_comments=*/ true); -- 2.30.2 From aa270ad5b7fd62d8ae765d6ff58ca7ad527a00ea Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Tue, 1 Dec 2015 11:00:58 +0100 Subject: [PATCH 47/91] fix test where the trailing / was not reached due to a previous error --- test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.cpp b/test.cpp index 7d77aa3a..7e7d0e32 100644 --- a/test.cpp +++ b/test.cpp @@ -106,7 +106,7 @@ int main(int argc, char **argv) { } failing_comment_test = R"({ - "a": 1, + "a": 1 }/)"; json_failing_comment = Json::parse( -- 2.30.2 From c6c6fcfeff5663cb34e7405bcc4572a2a5424d40 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Tue, 1 Dec 2015 11:01:27 +0100 Subject: [PATCH 48/91] add test for inline comment without trailing newline --- test.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test.cpp b/test.cpp index 7e7d0e32..1e537b99 100644 --- a/test.cpp +++ b/test.cpp @@ -105,6 +105,16 @@ int main(int argc, char **argv) { printf("Result: %s\n", json_failing_comment.dump().c_str()); } + failing_comment_test = R"({// bad comment })"; + + json_failing_comment = Json::parse( + failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + if (!err_failing_comment.empty()) { + printf("Failed: %s\n", err_failing_comment.c_str()); + } else { + printf("Result: %s\n", json_failing_comment.dump().c_str()); + } + failing_comment_test = R"({ "a": 1 }/)"; -- 2.30.2 From f9833b1e7d4b36901c36131c4699d889dae3eaf2 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Tue, 1 Dec 2015 11:01:47 +0100 Subject: [PATCH 49/91] add test for unfinished multi-line comment --- test.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test.cpp b/test.cpp index 1e537b99..ece0e045 100644 --- a/test.cpp +++ b/test.cpp @@ -127,6 +127,17 @@ int main(int argc, char **argv) { printf("Result: %s\n", json_failing_comment.dump().c_str()); } + failing_comment_test = R"({/* bad + comment *})"; + + json_failing_comment = Json::parse( + failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + if (!err_failing_comment.empty()) { + printf("Failed: %s\n", err_failing_comment.c_str()); + } else { + printf("Result: %s\n", json_failing_comment.dump().c_str()); + } + std::list l1 { 1, 2, 3 }; std::vector l2 { 1, 2, 3 }; std::set l3 { 1, 2, 3 }; -- 2.30.2 From 49a6197d08fbdd5a2eb9b0e8ae3aeab88f2a6d29 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Tue, 1 Dec 2015 11:08:37 +0100 Subject: [PATCH 50/91] use an enum to select strategy on comment parsing --- json11.cpp | 12 ++++++------ json11.hpp | 17 +++++++++++------ test.cpp | 12 ++++++------ 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/json11.cpp b/json11.cpp index 3ea9a67f..07e43c0a 100644 --- a/json11.cpp +++ b/json11.cpp @@ -338,7 +338,7 @@ struct JsonParser { size_t i; string &err; bool failed; - bool detect_comments; + JsonParse strategy; /* fail(msg, err_ret = Json()) * @@ -416,7 +416,7 @@ struct JsonParser { */ void consume_garbage() { consume_whitespace(); - if(detect_comments) { + if(strategy == JsonParse::COMMENTS) { bool comment_found = false; do { comment_found = consume_comment(); @@ -719,8 +719,8 @@ struct JsonParser { } }; -Json Json::parse(const string &in, string &err, bool detect_comments) { - JsonParser parser { in, 0, err, false, detect_comments }; +Json Json::parse(const string &in, string &err, JsonParse strategy) { + JsonParser parser { in, 0, err, false, strategy }; Json result = parser.parse_json(0); // Check for any trailing garbage @@ -734,8 +734,8 @@ Json Json::parse(const string &in, string &err, bool detect_comments) { // Documented in json11.hpp vector Json::parse_multi(const string &in, string &err, - bool detect_comments) { - JsonParser parser { in, 0, err, false, detect_comments }; + JsonParse strategy) { + JsonParser parser { in, 0, err, false, strategy }; vector json_vec; while (parser.i != in.size() && !parser.failed) { diff --git a/json11.hpp b/json11.hpp index 582c43f0..a99e241f 100644 --- a/json11.hpp +++ b/json11.hpp @@ -58,6 +58,10 @@ namespace json11 { +enum JsonParse { + STANDARD, COMMENTS +}; + class JsonValue; class Json final { @@ -147,21 +151,22 @@ public: // Parse. If parse fails, return Json() and assign an error message to err. static Json parse(const std::string & in, std::string & err, - bool detect_comments = false); + JsonParse strategy = JsonParse::STANDARD); static Json parse(const char * in, std::string & err, - bool detect_comments = false) { + JsonParse strategy = JsonParse::STANDARD) { if (in) { - return parse(std::string(in), err, detect_comments); + return parse(std::string(in), err, strategy); } else { err = "null input"; return nullptr; } } // Parse multiple objects, concatenated or separated by whitespace - static std::vector parse_multi(const std::string & in, - std::string & err, - bool detect_comments = false); + static std::vector parse_multi( + const std::string & in, + std::string & err, + JsonParse strategy = JsonParse::STANDARD); bool operator== (const Json &rhs) const; bool operator< (const Json &rhs) const; diff --git a/test.cpp b/test.cpp index ece0e045..bd607050 100644 --- a/test.cpp +++ b/test.cpp @@ -73,7 +73,7 @@ int main(int argc, char **argv) { string err_comment; auto json_comment = Json::parse( - comment_test, err_comment, /*detect_comments=*/ true); + comment_test, err_comment, JsonParse::COMMENTS); if (!err_comment.empty()) { printf("Failed: %s\n", err_comment.c_str()); } else { @@ -87,7 +87,7 @@ int main(int argc, char **argv) { string err_failing_comment; auto json_failing_comment = Json::parse( - failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + failing_comment_test, err_failing_comment, JsonParse::COMMENTS); if (!err_failing_comment.empty()) { printf("Failed: %s\n", err_failing_comment.c_str()); } else { @@ -98,7 +98,7 @@ int main(int argc, char **argv) { / / bad comment })"; json_failing_comment = Json::parse( - failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + failing_comment_test, err_failing_comment, JsonParse::COMMENTS); if (!err_failing_comment.empty()) { printf("Failed: %s\n", err_failing_comment.c_str()); } else { @@ -108,7 +108,7 @@ int main(int argc, char **argv) { failing_comment_test = R"({// bad comment })"; json_failing_comment = Json::parse( - failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + failing_comment_test, err_failing_comment, JsonParse::COMMENTS); if (!err_failing_comment.empty()) { printf("Failed: %s\n", err_failing_comment.c_str()); } else { @@ -120,7 +120,7 @@ int main(int argc, char **argv) { }/)"; json_failing_comment = Json::parse( - failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + failing_comment_test, err_failing_comment, JsonParse::COMMENTS); if (!err_failing_comment.empty()) { printf("Failed: %s\n", err_failing_comment.c_str()); } else { @@ -131,7 +131,7 @@ int main(int argc, char **argv) { comment *})"; json_failing_comment = Json::parse( - failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + failing_comment_test, err_failing_comment, JsonParse::COMMENTS); if (!err_failing_comment.empty()) { printf("Failed: %s\n", err_failing_comment.c_str()); } else { -- 2.30.2 From 988a8fc249c49cfd0779eb3e938444b622eaad0a Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Wed, 2 Dec 2015 09:57:25 +0100 Subject: [PATCH 51/91] make JsonParser::strategy const --- json11.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json11.cpp b/json11.cpp index 07e43c0a..3ec83af3 100644 --- a/json11.cpp +++ b/json11.cpp @@ -338,7 +338,7 @@ struct JsonParser { size_t i; string &err; bool failed; - JsonParse strategy; + const JsonParse strategy; /* fail(msg, err_ret = Json()) * -- 2.30.2 From ebc3a6b0387af83c6735f89dbde98ca425741b3b Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Wed, 2 Dec 2015 10:01:29 +0100 Subject: [PATCH 52/91] watch out for i+1 to overflow the buffer --- json11.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/json11.cpp b/json11.cpp index 3ec83af3..ebd7e93d 100644 --- a/json11.cpp +++ b/json11.cpp @@ -389,12 +389,12 @@ struct JsonParser { } else if (str[i] == '*') { // multiline comment i++; - if (i == str.size()) + if (i > str.size()-2) return fail("unexpected end of input inside multi-line comment", 0); - // advance until closing tokens + // advance until closing tokens while (!(str[i] == '*' && str[i+1] == '/')) { i++; - if (i == str.size()) + if (i > str.size()-2) return fail( "unexpected end of input inside multi-line comment", 0); } -- 2.30.2 From e1d5bcc94d48341e4382ee5d0733726f0dde1a15 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 29 Dec 2015 18:55:48 +0100 Subject: [PATCH 53/91] =?UTF-8?q?This=20fixes=20a=20=E2=80=9CNo=20previous?= =?UTF-8?q?=20prototype=20for=20function=E2=80=9D=20warning.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- json11.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index ebd7e93d..38fd4c8f 100644 --- a/json11.cpp +++ b/json11.cpp @@ -226,12 +226,12 @@ struct Statics { Statics() {} }; -const Statics & statics() { +static const Statics & statics() { static const Statics s {}; return s; } -const Json & static_null() { +static const Json & static_null() { // This has to be separate, not in Statics, because Json() accesses statics().null. static const Json json_null; return json_null; -- 2.30.2 From 5e664c99d887133f5e06a902b2b61ea67413e7f4 Mon Sep 17 00:00:00 2001 From: kirisetsz Date: Fri, 8 Apr 2016 01:16:21 +0800 Subject: [PATCH 54/91] Update CMakeLists to 3.x style --- CMakeLists.txt | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ba8edd5b..397104aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,21 +1,14 @@ -project(json11) +project(json11 CXX) -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.2) enable_testing() -add_definitions( - -std=c++11 - -fno-rtti - -fno-exceptions - -Wall - -Wextra - -Werror) +add_library(json11 json11.cpp) +target_include_directories(json11 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_options(json11 + PUBLIC -std=c++11 + PRIVATE -fno-rtti -fno-exceptions -Wall -Wextra -Werror) -set(json11_SRCS json11.cpp) - -add_library(json11 STATIC ${json11_SRCS}) - -add_test(json11_test json11_test) - -add_executable(json11_test ${json11_SRCS} test.cpp) +add_executable(json11_test test.cpp) +target_link_libraries(json11_test json11) -- 2.30.2 From 8076ba74e08907b1e219f748f61f6ae6b69ca145 Mon Sep 17 00:00:00 2001 From: Nick White Date: Sat, 16 Apr 2016 18:41:25 +0100 Subject: [PATCH 55/91] Generate pkg-config File & Add Install ...to the CMake build script. The pkg-config file needs a version number (which it takes from the CMakeLists.txt variable), which I've set to 1.0.0. The cmake project declaration needs to be after the minimum-requirement declaration to avoid errors from the cmake's change in VERSION semantics. --- .gitignore | 11 +++++++++++ CMakeLists.txt | 8 ++++++-- json11.pc.in | 9 +++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 json11.pc.in diff --git a/.gitignore b/.gitignore index 9daeafb9..e959b6fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,12 @@ +# generated files test +libjson11.a +json11.pc + +# Cmake +CMakeCache.txt +CTestTestfile.cmake +CMakeFiles +CMakeScripts +cmake_install.cmake +install_manifest.txt \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 397104aa..9d6adf95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,5 @@ -project(json11 CXX) - cmake_minimum_required(VERSION 3.2) +project(json11 VERSION 1.0.0 LANGUAGES CXX) enable_testing() @@ -9,6 +8,11 @@ target_include_directories(json11 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_compile_options(json11 PUBLIC -std=c++11 PRIVATE -fno-rtti -fno-exceptions -Wall -Wextra -Werror) +configure_file("json11.pc.in" "json11.pc" @ONLY) add_executable(json11_test test.cpp) target_link_libraries(json11_test json11) + +install(TARGETS json11 DESTINATION lib) +install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/json11.hpp" DESTINATION include) +install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/json11.pc" DESTINATION lib/pkgconfig) diff --git a/json11.pc.in b/json11.pc.in new file mode 100644 index 00000000..de4b6dd7 --- /dev/null +++ b/json11.pc.in @@ -0,0 +1,9 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +libdir=${prefix}/lib +includedir=${prefix}/include + +Name: @PROJECT_NAME@ +Description: json11 is a tiny JSON library for C++11, providing JSON parsing and serialization. +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -ljson11 +Cflags: -I${includedir} \ No newline at end of file -- 2.30.2 From 33bca8e178524a6554c2ae9256f90cfaa1f8dd76 Mon Sep 17 00:00:00 2001 From: "Evgeniy A. Dushistov" Date: Wed, 27 Apr 2016 12:47:30 +0300 Subject: [PATCH 56/91] implement parsing of chunked json Update: add more tests + use parser.failed --- json11.cpp | 5 ++++- json11.hpp | 9 +++++++++ test.cpp | 36 +++++++++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index 38fd4c8f..b2fa4cbd 100644 --- a/json11.cpp +++ b/json11.cpp @@ -733,15 +733,18 @@ Json Json::parse(const string &in, string &err, JsonParse strategy) { // Documented in json11.hpp vector Json::parse_multi(const string &in, + std::string::size_type &parser_stop_pos, string &err, JsonParse strategy) { JsonParser parser { in, 0, err, false, strategy }; - + parser_stop_pos = 0; vector json_vec; while (parser.i != in.size() && !parser.failed) { json_vec.push_back(parser.parse_json(0)); // Check for another object parser.consume_garbage(); + if (!parser.failed) + parser_stop_pos = parser.i; } return json_vec; } diff --git a/json11.hpp b/json11.hpp index a99e241f..e9fe251f 100644 --- a/json11.hpp +++ b/json11.hpp @@ -165,9 +165,18 @@ public: // Parse multiple objects, concatenated or separated by whitespace static std::vector parse_multi( const std::string & in, + std::string::size_type & parser_stop_pos, std::string & err, JsonParse strategy = JsonParse::STANDARD); + static inline std::vector parse_multi( + const std::string & in, + std::string & err, + JsonParse strategy = JsonParse::STANDARD) { + std::string::size_type parser_stop_pos; + return parse_multi(in, parser_stop_pos, err, strategy); + } + bool operator== (const Json &rhs) const; bool operator< (const Json &rhs) const; bool operator!= (const Json &rhs) const { return !(*this == rhs); } diff --git a/test.cpp b/test.cpp index bd607050..b1b52fdf 100644 --- a/test.cpp +++ b/test.cpp @@ -1,13 +1,17 @@ +#ifdef NDEBUG +#undef NDEBUG//at now assert will work even in Release build +#endif +#include #include #include #include #include #include #include "json11.hpp" -#include #include #include #include +#include using namespace json11; using std::string; @@ -183,6 +187,36 @@ int main(int argc, char **argv) { assert(nested_array.array_items()[0].is_array()); assert(nested_array.array_items()[0].array_items().size() == 3); + { + const std::string good_json = R"( {"k1" : "v1"})"; + const std::string bad_json1 = good_json + " {"; + const std::string bad_json2 = good_json + R"({"k2":"v2", "k3":[)"; + struct TestMultiParse { + std::string input; + std::string::size_type expect_parser_stop_pos; + size_t expect_not_empty_elms_count; + Json expect_parse_res; + } tests[] = { + {" {", 0, 0, {}}, + {good_json, good_json.size(), 1, Json(std::map{ { "k1", "v1" } })}, + {bad_json1, good_json.size() + 1, 1, Json(std::map{ { "k1", "v1" } })}, + {bad_json2, good_json.size(), 1, Json(std::map{ { "k1", "v1" } })}, + {"{}", 2, 1, Json::object{}}, + }; + for (const auto &tst : tests) { + std::string::size_type parser_stop_pos; + std::string err; + auto res = Json::parse_multi(tst.input, parser_stop_pos, err); + assert(parser_stop_pos == tst.expect_parser_stop_pos); + assert( + std::count_if(res.begin(), res.end(), + [](const Json& j) { return !j.is_null(); }) + == tst.expect_not_empty_elms_count); + if (!res.empty()) { + assert(tst.expect_parse_res == res[0]); + } + } + } Json my_json = Json::object { { "key1", "value1" }, { "key2", false }, -- 2.30.2 From 0e4ace7e40893dee73934431d3a6f7b85f33146a Mon Sep 17 00:00:00 2001 From: "Evgeniy A. Dushistov" Date: Thu, 28 Apr 2016 01:28:42 +0300 Subject: [PATCH 57/91] make parser invisible from outside --- json11.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/json11.cpp b/json11.cpp index 38fd4c8f..07a3a715 100644 --- a/json11.cpp +++ b/json11.cpp @@ -326,11 +326,12 @@ static inline bool in_range(long x, long lower, long upper) { return (x >= lower && x <= upper); } +namespace { /* JsonParser * * Object that tracks all state of an in-progress parse. */ -struct JsonParser { +struct JsonParser final { /* State */ @@ -718,6 +719,7 @@ struct JsonParser { return fail("expected value, got " + esc(ch)); } }; +}//namespace { Json Json::parse(const string &in, string &err, JsonParse strategy) { JsonParser parser { in, 0, err, false, strategy }; -- 2.30.2 From 05b5514400d9bc757a053f57fe08436694294d64 Mon Sep 17 00:00:00 2001 From: Chris Kitching Date: Thu, 5 May 2016 02:43:43 +0100 Subject: [PATCH 58/91] Don't assume in-tree builds when installing json11.pc --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d6adf95..350b4ba9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,4 +15,4 @@ target_link_libraries(json11_test json11) install(TARGETS json11 DESTINATION lib) install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/json11.hpp" DESTINATION include) -install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/json11.pc" DESTINATION lib/pkgconfig) +install(FILES "${CMAKE_BINARY_DIR}/json11.pc" DESTINATION lib/pkgconfig) -- 2.30.2 From 4597f98abe32818c8d8ec1d69654b8af9a68ece2 Mon Sep 17 00:00:00 2001 From: Andrew Twyman Date: Wed, 18 May 2016 18:31:52 -0700 Subject: [PATCH 59/91] Fix sign comparison warnings, and parser confusion --- test.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test.cpp b/test.cpp index b1b52fdf..3425f80e 100644 --- a/test.cpp +++ b/test.cpp @@ -87,6 +87,7 @@ int main(int argc, char **argv) { string failing_comment_test = R"({ /* bad comment "a": 1, + // another comment to make C parsers which don't understand raw strings happy */ })"; string err_failing_comment; @@ -209,8 +210,8 @@ int main(int argc, char **argv) { auto res = Json::parse_multi(tst.input, parser_stop_pos, err); assert(parser_stop_pos == tst.expect_parser_stop_pos); assert( - std::count_if(res.begin(), res.end(), - [](const Json& j) { return !j.is_null(); }) + (size_t)std::count_if(res.begin(), res.end(), + [](const Json& j) { return !j.is_null(); }) == tst.expect_not_empty_elms_count); if (!res.empty()) { assert(tst.expect_parse_res == res[0]); -- 2.30.2 From a20878aaa5bd2546466585b18b6d09808a98233d Mon Sep 17 00:00:00 2001 From: Andrew Twyman Date: Wed, 18 May 2016 18:36:58 -0700 Subject: [PATCH 60/91] Make the DR1467 canary test code optional --- test.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/test.cpp b/test.cpp index 3425f80e..c90d1397 100644 --- a/test.cpp +++ b/test.cpp @@ -180,13 +180,16 @@ int main(int argc, char **argv) { assert(uni[0].string_value().size() == (sizeof utf8) - 1); assert(std::memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); - // Demonstrates the behavior change in Xcode 7 / Clang 3.7 described - // here: https://llvm.org/bugs/show_bug.cgi?id=23812 - Json nested_array = Json::array { Json::array { 1, 2, 3 } }; - assert(nested_array.is_array()); - assert(nested_array.array_items().size() == 1); - assert(nested_array.array_items()[0].is_array()); - assert(nested_array.array_items()[0].array_items().size() == 3); + // Demonstrates the behavior change in Xcode 7 / Clang 3.7, introduced by DR1467 + // and described here: https://llvm.org/bugs/show_bug.cgi?id=23812 + const bool ENABLE_DR1467_CANARY = true; // Allow easy disabling for users who work around it. + if (ENABLE_DR1467_CANARY) { + Json nested_array = Json::array { Json::array { 1, 2, 3 } }; + assert(nested_array.is_array()); + assert(nested_array.array_items().size() == 1); + assert(nested_array.array_items()[0].is_array()); + assert(nested_array.array_items()[0].array_items().size() == 3); + } { const std::string good_json = R"( {"k1" : "v1"})"; -- 2.30.2 From f6c0f687372a3eee8bfb7e5099ba8dfc2c36e3fa Mon Sep 17 00:00:00 2001 From: Shrikant Kelkar Date: Mon, 20 Jun 2016 17:36:17 -0700 Subject: [PATCH 61/91] MSVS 2013 compatibility changes. --- json11.hpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/json11.hpp b/json11.hpp index e9fe251f..5202ef93 100644 --- a/json11.hpp +++ b/json11.hpp @@ -56,6 +56,18 @@ #include #include +#ifdef _MSC_VER + #if _MSC_VER <= 1800 // VS 2013 + #ifndef noexcept + #define noexcept throw() + #endif + + #ifndef snprintf + #define snprintf _snprintf_s + #endif + #endif +#endif + namespace json11 { enum JsonParse { -- 2.30.2 From 370467150873ce8616a859edf9a510fd28fffbe5 Mon Sep 17 00:00:00 2001 From: Bruno Coelho <4brunu@gmail.com> Date: Tue, 5 Jul 2016 15:37:44 +0100 Subject: [PATCH 62/91] Add option to disable unit tests --- CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 350b4ba9..ae8c4123 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,8 @@ project(json11 VERSION 1.0.0 LANGUAGES CXX) enable_testing() +option(JSON11_BUILD_TESTS "Build unit tests" ON) + add_library(json11 json11.cpp) target_include_directories(json11 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_compile_options(json11 @@ -10,8 +12,10 @@ target_compile_options(json11 PRIVATE -fno-rtti -fno-exceptions -Wall -Wextra -Werror) configure_file("json11.pc.in" "json11.pc" @ONLY) -add_executable(json11_test test.cpp) -target_link_libraries(json11_test json11) +if (JSON11_BUILD_TESTS) + add_executable(json11_test test.cpp) + target_link_libraries(json11_test json11) +endif() install(TARGETS json11 DESTINATION lib) install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/json11.hpp" DESTINATION include) -- 2.30.2 From 467dc6ae058e68aaf9d3bbfdf1d2447f4742f64d Mon Sep 17 00:00:00 2001 From: Austin Brunkhorst Date: Sun, 24 Jul 2016 11:34:17 -0700 Subject: [PATCH 63/91] Fixes warning C4800: 'int': forcing value to bool 'true' or 'false' on MSVC 14 --- json11.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/json11.cpp b/json11.cpp index 5f3b7efd..f977e168 100644 --- a/json11.cpp +++ b/json11.cpp @@ -375,38 +375,38 @@ struct JsonParser final { if (str[i] == '/') { i++; if (i == str.size()) - return fail("unexpected end of input inside comment", 0); + return !!fail("unexpected end of input inside comment", 0); if (str[i] == '/') { // inline comment i++; if (i == str.size()) - return fail("unexpected end of input inside inline comment", 0); + return !!fail("unexpected end of input inside inline comment", 0); // advance until next line while (str[i] != '\n') { i++; if (i == str.size()) - return fail("unexpected end of input inside inline comment", 0); + return !!fail("unexpected end of input inside inline comment", 0); } comment_found = true; } else if (str[i] == '*') { // multiline comment i++; if (i > str.size()-2) - return fail("unexpected end of input inside multi-line comment", 0); + return !!fail("unexpected end of input inside multi-line comment", 0); // advance until closing tokens while (!(str[i] == '*' && str[i+1] == '/')) { i++; if (i > str.size()-2) - return fail( + return !!fail( "unexpected end of input inside multi-line comment", 0); } i += 2; if (i == str.size()) - return fail( + return !!fail( "unexpected end of input inside multi-line comment", 0); comment_found = true; } else - return fail("malformed comment", 0); + return !!fail("malformed comment", 0); } return comment_found; } -- 2.30.2 From 40f10bd28d42a560749bb033d33bfecc494b02a5 Mon Sep 17 00:00:00 2001 From: Austin Brunkhorst Date: Mon, 25 Jul 2016 17:28:15 -0700 Subject: [PATCH 64/91] Use `false` instead of `0` + conversion. --- json11.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/json11.cpp b/json11.cpp index f977e168..e3dc1190 100644 --- a/json11.cpp +++ b/json11.cpp @@ -375,38 +375,38 @@ struct JsonParser final { if (str[i] == '/') { i++; if (i == str.size()) - return !!fail("unexpected end of input inside comment", 0); + return fail("unexpected end of input inside comment", false); if (str[i] == '/') { // inline comment i++; if (i == str.size()) - return !!fail("unexpected end of input inside inline comment", 0); + return fail("unexpected end of input inside inline comment", false); // advance until next line while (str[i] != '\n') { i++; if (i == str.size()) - return !!fail("unexpected end of input inside inline comment", 0); + return fail("unexpected end of input inside inline comment", false); } comment_found = true; } else if (str[i] == '*') { // multiline comment i++; if (i > str.size()-2) - return !!fail("unexpected end of input inside multi-line comment", 0); + return fail("unexpected end of input inside multi-line comment", false); // advance until closing tokens while (!(str[i] == '*' && str[i+1] == '/')) { i++; if (i > str.size()-2) - return !!fail( - "unexpected end of input inside multi-line comment", 0); + return fail( + "unexpected end of input inside multi-line comment", false); } i += 2; if (i == str.size()) - return !!fail( - "unexpected end of input inside multi-line comment", 0); + return fail( + "unexpected end of input inside multi-line comment", false); comment_found = true; } else - return !!fail("malformed comment", 0); + return fail("malformed comment", false); } return comment_found; } -- 2.30.2 From 200c98aedef378847f87cb00792423fafc6504c9 Mon Sep 17 00:00:00 2001 From: Andrew Twyman Date: Thu, 28 Jul 2016 22:27:43 -0700 Subject: [PATCH 65/91] Allow json11 tests to be customized with pre-processor defines --- Makefile | 10 ++++- test.cpp | 128 ++++++++++++++++++++++++++++++++++++------------------- 2 files changed, 94 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index 5a6adc6e..f946bdd8 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,13 @@ +# Environment variable to enable or disable code which demonstrates the behavior change +# in Xcode 7 / Clang 3.7, introduced by DR1467 and described here: +# https://llvm.org/bugs/show_bug.cgi?id=23812 +# Defaults to on in order to act as a warning to anyone who's unaware of the issue. +ifneq ($(JSON11_ENABLE_DR1467_CANARY),) +CANARY_ARGS = -DJSON11_ENABLE_DR1467_CANARY=$(JSON11_ENABLE_DR1467_CANARY) +endif + test: json11.cpp json11.hpp test.cpp - $(CXX) -O -std=c++11 json11.cpp test.cpp -o test -fno-rtti -fno-exceptions + $(CXX) $(CANARY_ARGS) -O -std=c++11 json11.cpp test.cpp -o test -fno-rtti -fno-exceptions clean: if [ -e test ]; then rm test; fi diff --git a/test.cpp b/test.cpp index c90d1397..2e6543e3 100644 --- a/test.cpp +++ b/test.cpp @@ -1,6 +1,33 @@ +/* + * Define JSON11_TEST_CUSTOM_CONFIG to 1 if you want to build this tester into + * your own unit-test framework rather than a stand-alone program. By setting + * The values of the variables included below, you can insert your own custom + * code into this file as it builds, in order to make it into a test case for + * your favorite framework. + */ +#if(!JSON11_TEST_CUSTOM_CONFIG) +#define JSON11_TEST_CPP_PREFIX_CODE +#define JSON11_TEST_CPP_SUFFIX_CODE +#define JSON11_TEST_STANDALONE_MAIN 1 +#define JSON11_TEST_CASE(name) static void name() +#define JSON11_TEST_ASSERT(b) assert(b) #ifdef NDEBUG #undef NDEBUG//at now assert will work even in Release build #endif +#endif // JSON11_TEST_CUSTOM_CONFIG + +/* + * Enable or disable code which demonstrates the behavior change in Xcode 7 / Clang 3.7, + * introduced by DR1467 and described here: https://llvm.org/bugs/show_bug.cgi?id=23812 + * Defaults to on in order to act as a warning to anyone who's unaware of the issue. + */ +#ifndef JSON11_ENABLE_DR1467_CANARY +#define JSON11_ENABLE_DR1467_CANARY 1 +#endif + +/* + * Beginning of standard source file, which makes use of the customizations above. + */ #include #include #include @@ -12,12 +39,16 @@ #include #include #include +#include + +// Insert user-defined prefix code (includes, function declarations, etc) +// to set up a custom test suite +JSON11_TEST_CPP_PREFIX_CODE using namespace json11; using std::string; // Check that Json has the properties we want. -#include #define CHECK_TRAIT(x) static_assert(std::x::value, #x) CHECK_TRAIT(is_nothrow_constructible); CHECK_TRAIT(is_nothrow_default_constructible); @@ -27,28 +58,7 @@ CHECK_TRAIT(is_copy_assignable); CHECK_TRAIT(is_nothrow_move_assignable); CHECK_TRAIT(is_nothrow_destructible); -void parse_from_stdin() { - string buf; - string line; - while (std::getline(std::cin, line)) { - buf += line + "\n"; - } - - string err; - auto json = Json::parse(buf, err); - if (!err.empty()) { - printf("Failed: %s\n", err.c_str()); - } else { - printf("Result: %s\n", json.dump().c_str()); - } -} - -int main(int argc, char **argv) { - if (argc == 2 && argv[1] == string("--stdin")) { - parse_from_stdin(); - return 0; - } - +JSON11_TEST_CASE(json11_test) { const string simple_test = R"({"k1":"v1", "k2":42, "k3":["a",123,true,false,null]})"; @@ -146,12 +156,12 @@ int main(int argc, char **argv) { std::list l1 { 1, 2, 3 }; std::vector l2 { 1, 2, 3 }; std::set l3 { 1, 2, 3 }; - assert(Json(l1) == Json(l2)); - assert(Json(l2) == Json(l3)); + JSON11_TEST_ASSERT(Json(l1) == Json(l2)); + JSON11_TEST_ASSERT(Json(l2) == Json(l3)); std::map m1 { { "k1", "v1" }, { "k2", "v2" } }; std::unordered_map m2 { { "k1", "v1" }, { "k2", "v2" } }; - assert(Json(m1) == Json(m2)); + JSON11_TEST_ASSERT(Json(m1) == Json(m2)); // Json literals Json obj = Json::object({ @@ -162,13 +172,13 @@ int main(int argc, char **argv) { std::cout << "obj: " << obj.dump() << "\n"; - assert(Json("a").number_value() == 0); - assert(Json("a").string_value() == "a"); - assert(Json().number_value() == 0); + JSON11_TEST_ASSERT(Json("a").number_value() == 0); + JSON11_TEST_ASSERT(Json("a").string_value() == "a"); + JSON11_TEST_ASSERT(Json().number_value() == 0); - assert(obj == json); - assert(Json(42) == Json(42.0)); - assert(Json(42) != Json(42.1)); + JSON11_TEST_ASSERT(obj == json); + JSON11_TEST_ASSERT(Json(42) == Json(42.0)); + JSON11_TEST_ASSERT(Json(42) != Json(42.1)); const string unicode_escape_test = R"([ "blah\ud83d\udca9blah\ud83dblah\udca9blah\u0000blah\u1234" ])"; @@ -177,18 +187,17 @@ int main(int argc, char **argv) { "\xed\xb2\xa9" "blah" "\0" "blah" "\xe1\x88\xb4"; Json uni = Json::parse(unicode_escape_test, err); - assert(uni[0].string_value().size() == (sizeof utf8) - 1); - assert(std::memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); + JSON11_TEST_ASSERT(uni[0].string_value().size() == (sizeof utf8) - 1); + JSON11_TEST_ASSERT(std::memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); // Demonstrates the behavior change in Xcode 7 / Clang 3.7, introduced by DR1467 // and described here: https://llvm.org/bugs/show_bug.cgi?id=23812 - const bool ENABLE_DR1467_CANARY = true; // Allow easy disabling for users who work around it. - if (ENABLE_DR1467_CANARY) { + if (JSON11_ENABLE_DR1467_CANARY) { Json nested_array = Json::array { Json::array { 1, 2, 3 } }; - assert(nested_array.is_array()); - assert(nested_array.array_items().size() == 1); - assert(nested_array.array_items()[0].is_array()); - assert(nested_array.array_items()[0].array_items().size() == 3); + JSON11_TEST_ASSERT(nested_array.is_array()); + JSON11_TEST_ASSERT(nested_array.array_items().size() == 1); + JSON11_TEST_ASSERT(nested_array.array_items()[0].is_array()); + JSON11_TEST_ASSERT(nested_array.array_items()[0].array_items().size() == 3); } { @@ -211,13 +220,13 @@ int main(int argc, char **argv) { std::string::size_type parser_stop_pos; std::string err; auto res = Json::parse_multi(tst.input, parser_stop_pos, err); - assert(parser_stop_pos == tst.expect_parser_stop_pos); - assert( + JSON11_TEST_ASSERT(parser_stop_pos == tst.expect_parser_stop_pos); + JSON11_TEST_ASSERT( (size_t)std::count_if(res.begin(), res.end(), [](const Json& j) { return !j.is_null(); }) == tst.expect_not_empty_elms_count); if (!res.empty()) { - assert(tst.expect_parse_res == res[0]); + JSON11_TEST_ASSERT(tst.expect_parse_res == res[0]); } } } @@ -241,3 +250,36 @@ int main(int argc, char **argv) { std::string points_json = Json(points).dump(); printf("%s\n", points_json.c_str()); } + +#if(JSON11_TEST_STANDALONE_MAIN) + +static void parse_from_stdin() { + string buf; + string line; + while (std::getline(std::cin, line)) { + buf += line + "\n"; + } + + string err; + auto json = Json::parse(buf, err); + if (!err.empty()) { + printf("Failed: %s\n", err.c_str()); + } else { + printf("Result: %s\n", json.dump().c_str()); + } +} + +int main(int argc, char **argv) { + if (argc == 2 && argv[1] == string("--stdin")) { + parse_from_stdin(); + return 0; + } + + json11_test(); +} + +#endif // JSON11_TEST_STANDALONE_MAIN + +// Insert user-defined suffix code (function definitions, etc) +// to set up a custom test suite +JSON11_TEST_CPP_SUFFIX_CODE -- 2.30.2 From 7a947c1e2f4a7f30860dd7ca11ee883c2cc3f324 Mon Sep 17 00:00:00 2001 From: Andrew Twyman Date: Thu, 28 Jul 2016 22:42:01 -0700 Subject: [PATCH 66/91] Remove extraneous parentheses --- test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.cpp b/test.cpp index 2e6543e3..cf9a46fc 100644 --- a/test.cpp +++ b/test.cpp @@ -5,7 +5,7 @@ * code into this file as it builds, in order to make it into a test case for * your favorite framework. */ -#if(!JSON11_TEST_CUSTOM_CONFIG) +#if !JSON11_TEST_CUSTOM_CONFIG #define JSON11_TEST_CPP_PREFIX_CODE #define JSON11_TEST_CPP_SUFFIX_CODE #define JSON11_TEST_STANDALONE_MAIN 1 @@ -251,7 +251,7 @@ JSON11_TEST_CASE(json11_test) { printf("%s\n", points_json.c_str()); } -#if(JSON11_TEST_STANDALONE_MAIN) +#if JSON11_TEST_STANDALONE_MAIN static void parse_from_stdin() { string buf; -- 2.30.2 From 580fd44b5f83e3cd8976fc1aae27931d131775d2 Mon Sep 17 00:00:00 2001 From: Adam Carlucci Date: Wed, 10 Aug 2016 18:16:59 -0700 Subject: [PATCH 67/91] Use CXX_STANDARD to specify c++11 compile flags --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ae8c4123..02ea263d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,10 +5,12 @@ enable_testing() option(JSON11_BUILD_TESTS "Build unit tests" ON) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + add_library(json11 json11.cpp) target_include_directories(json11 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_compile_options(json11 - PUBLIC -std=c++11 PRIVATE -fno-rtti -fno-exceptions -Wall -Wextra -Werror) configure_file("json11.pc.in" "json11.pc" @ONLY) -- 2.30.2 From f6b4188933d0a68839989044a9bdd80b806ab56d Mon Sep 17 00:00:00 2001 From: Zhijiang TAO Date: Tue, 16 Aug 2016 00:14:02 +0800 Subject: [PATCH 68/91] =?UTF-8?q?=E6=B7=BB=E5=8A=A0int64=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=EF=BC=8C=E4=BD=86=E6=98=AF=E8=BF=98=E6=98=AF=E4=BC=9A=E6=BA=A2?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- json11.cpp | 36 ++++++++++++++++++++++++++++++------ json11.hpp | 11 +++++++++-- test.cpp | 22 ++++++++++++++++++++++ 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/json11.cpp b/json11.cpp index e3dc1190..ac9d727f 100644 --- a/json11.cpp +++ b/json11.cpp @@ -61,6 +61,12 @@ static void dump(int value, string &out) { out += buf; } +static void dump(int64_ value, string &out) { + char buf[64]; + snprintf(buf, sizeof buf, "%ld", value); + out += buf; +} + static void dump(bool value, string &out) { out += value ? "true" : "false"; } @@ -164,6 +170,7 @@ protected: class JsonDouble final : public Value { double number_value() const override { return m_value; } int int_value() const override { return static_cast(m_value); } + int64_ int64_value() const override { return static_cast(m_value); } bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: @@ -173,12 +180,23 @@ public: class JsonInt final : public Value { double number_value() const override { return m_value; } int int_value() const override { return m_value; } + int64_ int64_value() const override { return m_value; } bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: explicit JsonInt(int value) : Value(value) {} }; +class JsonInt64 final : public Value { + double number_value() const override { return m_value; } + int int_value() const override { return m_value; } + int64_ int64_value() const override { return m_value; } + bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } + bool less(const JsonValue * other) const override { return m_value < other->number_value(); } +public: + explicit JsonInt64(int64_ value) : Value(value) {} +}; + class JsonBoolean final : public Value { bool bool_value() const override { return m_value; } public: @@ -244,7 +262,8 @@ static const Json & static_null() { Json::Json() noexcept : m_ptr(statics().null) {} Json::Json(std::nullptr_t) noexcept : m_ptr(statics().null) {} Json::Json(double value) : m_ptr(make_shared(value)) {} -Json::Json(int value) : m_ptr(make_shared(value)) {} +Json::Json(int value) : m_ptr(make_shared(value)) {} +Json::Json(int64_ value) : m_ptr(make_shared(value)) {} Json::Json(bool value) : m_ptr(value ? statics().t : statics().f) {} Json::Json(const string &value) : m_ptr(make_shared(value)) {} Json::Json(string &&value) : m_ptr(make_shared(move(value))) {} @@ -260,7 +279,8 @@ Json::Json(Json::object &&values) : m_ptr(make_shared(move(valu Json::Type Json::type() const { return m_ptr->type(); } double Json::number_value() const { return m_ptr->number_value(); } -int Json::int_value() const { return m_ptr->int_value(); } +int Json::int_value() const { return m_ptr->int_value(); } +int64_ Json::int64_value() const { return m_ptr->int64_value(); } bool Json::bool_value() const { return m_ptr->bool_value(); } const string & Json::string_value() const { return m_ptr->string_value(); } const vector & Json::array_items() const { return m_ptr->array_items(); } @@ -269,7 +289,8 @@ const Json & Json::operator[] (size_t i) const { return (*m_ptr)[i]; const Json & Json::operator[] (const string &key) const { return (*m_ptr)[key]; } double JsonValue::number_value() const { return 0; } -int JsonValue::int_value() const { return 0; } +int JsonValue::int_value() const { return 0; } +int64_ JsonValue::int64_value() const { return 0; } bool JsonValue::bool_value() const { return false; } const string & JsonValue::string_value() const { return statics().empty_string; } const vector & JsonValue::array_items() const { return statics().empty_vector; } @@ -581,9 +602,12 @@ struct JsonParser final { return fail("invalid " + esc(str[i]) + " in number"); } - if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' - && (i - start_pos) <= static_cast(std::numeric_limits::digits10)) { - return std::atoi(str.c_str() + start_pos); + if (str[i] != '.' && str[i] != 'e' && str[i] != 'E') + { + if ((i - start_pos) <= static_cast(std::numeric_limits::digits10)) + return (int)(std::atoll(str.c_str() + start_pos)); + if ((i - start_pos) <= static_cast(std::numeric_limits::digits10)) + return (int64_)(std::atoll(str.c_str() + start_pos)); } // Decimal part diff --git a/json11.hpp b/json11.hpp index 5202ef93..f26d4f2c 100644 --- a/json11.hpp +++ b/json11.hpp @@ -55,6 +55,7 @@ #include #include #include +#include #ifdef _MSC_VER #if _MSC_VER <= 1800 // VS 2013 @@ -76,6 +77,8 @@ enum JsonParse { class JsonValue; +using int64_ = int64_t; + class Json final { public: // Types @@ -91,7 +94,8 @@ public: Json() noexcept; // NUL Json(std::nullptr_t) noexcept; // NUL Json(double value); // NUMBER - Json(int value); // NUMBER + Json(int value); // NUMBER + Json(int64_ value); // NUMBER Json(bool value); // BOOL Json(const std::string &value); // STRING Json(std::string &&value); // STRING @@ -133,10 +137,11 @@ public: bool is_object() const { return type() == OBJECT; } // Return the enclosed value if this is a number, 0 otherwise. Note that json11 does not - // distinguish between integer and non-integer numbers - number_value() and int_value() + // distinguish between integer and non-integer numbers - number_value() and int64_value() // can both be applied to a NUMBER-typed object. double number_value() const; int int_value() const; + int64_ int64_value() const; // Return the enclosed value if this is a boolean, false otherwise. bool bool_value() const; @@ -213,6 +218,7 @@ class JsonValue { protected: friend class Json; friend class JsonInt; + friend class JsonInt64; friend class JsonDouble; virtual Json::Type type() const = 0; virtual bool equals(const JsonValue * other) const = 0; @@ -220,6 +226,7 @@ protected: virtual void dump(std::string &out) const = 0; virtual double number_value() const; virtual int int_value() const; + virtual int64_ int64_value() const; virtual bool bool_value() const; virtual const std::string &string_value() const; virtual const Json::array &array_items() const; diff --git a/test.cpp b/test.cpp index cf9a46fc..fee132cf 100644 --- a/test.cpp +++ b/test.cpp @@ -59,6 +59,28 @@ CHECK_TRAIT(is_nothrow_move_assignable); CHECK_TRAIT(is_nothrow_destructible); JSON11_TEST_CASE(json11_test) { + + std::cout << "int int64 test" << std::endl; + + const string str_it = R"({"message_id":105308320612483198,"msg_type":3,"order":0, + "ques_id":0,"session_id":105308187502051928,"site_id":122062, + "timestamp":1471140271, + "visitor_id":9941658010949867158,"worker_id":133746})"; + string err_it; + auto json_it = Json::parse(str_it, err_it); + + assert(json_it["message_id"].int64_value() == 105308320612483198); + assert(json_it["message_id"].int_value() != 105308320612483198); + assert(json_it["msg_type"].int64_value() == 3); + assert(json_it["msg_type"].int_value() == 3); + assert(json_it["order"].int64_value() == 0); + assert(json_it["session_id"].int64_value() == 105308187502051928); + assert(json_it["site_id"].int_value() == 122062); + assert(json_it["site_id"].int64_value() == 122062); + std::cout << (unsigned long long)json_it["visitor_id"].int64_value() << std::endl; + assert((unsigned long long)(json_it["visitor_id"].int64_value()) == 9941658010949867158); + std::cout << "int int64 test passed!" << std::endl; + const string simple_test = R"({"k1":"v1", "k2":42, "k3":["a",123,true,false,null]})"; -- 2.30.2 From eb4d67a68ee4d4f1ca3fe874602cb4aaa2724d5e Mon Sep 17 00:00:00 2001 From: Zhijiang TAO Date: Tue, 16 Aug 2016 11:00:25 +0800 Subject: [PATCH 69/91] using longlong for internel storage --- json11.cpp | 75 +++++++++++++++++++++++++++++++++--------------------- json11.hpp | 14 +++++++--- 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/json11.cpp b/json11.cpp index ac9d727f..9f593a04 100644 --- a/json11.cpp +++ b/json11.cpp @@ -56,17 +56,18 @@ static void dump(double value, string &out) { } static void dump(int value, string &out) { - char buf[32]; - snprintf(buf, sizeof buf, "%d", value); - out += buf; + out += std::to_string(value); } static void dump(int64_ value, string &out) { - char buf[64]; - snprintf(buf, sizeof buf, "%ld", value); - out += buf; + out += std::to_string(value); } +static void dump(uint64_ value, string &out) { + out += std::to_string(value); +} + + static void dump(bool value, string &out) { out += value ? "true" : "false"; } @@ -168,35 +169,35 @@ protected: }; class JsonDouble final : public Value { - double number_value() const override { return m_value; } - int int_value() const override { return static_cast(m_value); } + double number_value() const override { return static_cast(m_value); } int64_ int64_value() const override { return static_cast(m_value); } + uint64_ uint64_value() const override { return static_cast(m_value); } bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: explicit JsonDouble(double value) : Value(value) {} }; -class JsonInt final : public Value { - double number_value() const override { return m_value; } - int int_value() const override { return m_value; } - int64_ int64_value() const override { return m_value; } - bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } - bool less(const JsonValue * other) const override { return m_value < other->number_value(); } -public: - explicit JsonInt(int value) : Value(value) {} -}; - class JsonInt64 final : public Value { - double number_value() const override { return m_value; } - int int_value() const override { return m_value; } - int64_ int64_value() const override { return m_value; } + double number_value() const override { return static_cast(m_value); } + int64_ int64_value() const override { return static_cast(m_value); } + uint64_ uint64_value() const override { return static_cast(m_value); } bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: explicit JsonInt64(int64_ value) : Value(value) {} }; +class JsonUInt64 final : public Value { + double number_value() const override { return static_cast(m_value); } + int64_ int64_value() const override { return static_cast(m_value); } + uint64_ uint64_value() const override { return static_cast(m_value); } + bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } + bool less(const JsonValue * other) const override { return m_value < other->number_value(); } +public: + explicit JsonUInt64(uint64_ value) : Value(value) {} +}; + class JsonBoolean final : public Value { bool bool_value() const override { return m_value; } public: @@ -262,8 +263,9 @@ static const Json & static_null() { Json::Json() noexcept : m_ptr(statics().null) {} Json::Json(std::nullptr_t) noexcept : m_ptr(statics().null) {} Json::Json(double value) : m_ptr(make_shared(value)) {} -Json::Json(int value) : m_ptr(make_shared(value)) {} +Json::Json(int value) : m_ptr(make_shared(value)) {} Json::Json(int64_ value) : m_ptr(make_shared(value)) {} +Json::Json(uint64_ value) : m_ptr(make_shared(value)) {} Json::Json(bool value) : m_ptr(value ? statics().t : statics().f) {} Json::Json(const string &value) : m_ptr(make_shared(value)) {} Json::Json(string &&value) : m_ptr(make_shared(move(value))) {} @@ -279,8 +281,8 @@ Json::Json(Json::object &&values) : m_ptr(make_shared(move(valu Json::Type Json::type() const { return m_ptr->type(); } double Json::number_value() const { return m_ptr->number_value(); } -int Json::int_value() const { return m_ptr->int_value(); } -int64_ Json::int64_value() const { return m_ptr->int64_value(); } +int64_ Json::int64_value() const { return m_ptr->int64_value(); } +uint64_ Json::uint64_value() const { return m_ptr->uint64_value(); } bool Json::bool_value() const { return m_ptr->bool_value(); } const string & Json::string_value() const { return m_ptr->string_value(); } const vector & Json::array_items() const { return m_ptr->array_items(); } @@ -289,8 +291,8 @@ const Json & Json::operator[] (size_t i) const { return (*m_ptr)[i]; const Json & Json::operator[] (const string &key) const { return (*m_ptr)[key]; } double JsonValue::number_value() const { return 0; } -int JsonValue::int_value() const { return 0; } int64_ JsonValue::int64_value() const { return 0; } +uint64_ JsonValue::uint64_value() const { return 0; } bool JsonValue::bool_value() const { return false; } const string & JsonValue::string_value() const { return statics().empty_string; } const vector & JsonValue::array_items() const { return statics().empty_vector; } @@ -604,10 +606,25 @@ struct JsonParser final { if (str[i] != '.' && str[i] != 'e' && str[i] != 'E') { - if ((i - start_pos) <= static_cast(std::numeric_limits::digits10)) - return (int)(std::atoll(str.c_str() + start_pos)); - if ((i - start_pos) <= static_cast(std::numeric_limits::digits10)) - return (int64_)(std::atoll(str.c_str() + start_pos)); + if (*(str.c_str() + start_pos) == '-') // signed value + { + if ((i - start_pos) <= (ceil(std::numeric_limits::digits * std::log10(2) + 1/*sign*/) )) + { + // On Linux, the global errno variable is thread-specific + int64_t conv_val = std::atoll(str.c_str() + start_pos); + if (errno != ERANGE) + return conv_val; + } + } + else + { + if ((i - start_pos) <= ceil(std::numeric_limits::digits * std::log10(2) )) + { + uint64_ conv_val = std::strtoull(str.c_str() + start_pos, 0, 10); + if (errno != ERANGE) + return conv_val; + } + } } // Decimal part diff --git a/json11.hpp b/json11.hpp index f26d4f2c..37d75e46 100644 --- a/json11.hpp +++ b/json11.hpp @@ -77,7 +77,12 @@ enum JsonParse { class JsonValue; +#if __cplusplus < 201103L + #error This project can only be compiled with a compiler that supports C++11 +#else using int64_ = int64_t; +using uint64_ = uint64_t; +#endif class Json final { public: @@ -94,8 +99,9 @@ public: Json() noexcept; // NUL Json(std::nullptr_t) noexcept; // NUL Json(double value); // NUMBER - Json(int value); // NUMBER + Json(int value); // NUMBER Json(int64_ value); // NUMBER + Json(uint64_ value); // NUMBER Json(bool value); // BOOL Json(const std::string &value); // STRING Json(std::string &&value); // STRING @@ -140,8 +146,8 @@ public: // distinguish between integer and non-integer numbers - number_value() and int64_value() // can both be applied to a NUMBER-typed object. double number_value() const; - int int_value() const; int64_ int64_value() const; + uint64_ uint64_value() const; // Return the enclosed value if this is a boolean, false otherwise. bool bool_value() const; @@ -217,16 +223,16 @@ private: class JsonValue { protected: friend class Json; - friend class JsonInt; friend class JsonInt64; + friend class JsonUInt64; friend class JsonDouble; virtual Json::Type type() const = 0; virtual bool equals(const JsonValue * other) const = 0; virtual bool less(const JsonValue * other) const = 0; virtual void dump(std::string &out) const = 0; virtual double number_value() const; - virtual int int_value() const; virtual int64_ int64_value() const; + virtual uint64_ uint64_value() const; virtual bool bool_value() const; virtual const std::string &string_value() const; virtual const Json::array &array_items() const; -- 2.30.2 From f4ffe052d95dbc3ecb2b26d9d08bc4fc3254ca32 Mon Sep 17 00:00:00 2001 From: Zhijiang TAO Date: Tue, 16 Aug 2016 11:10:14 +0800 Subject: [PATCH 70/91] ADDING INT64 UINT64 test cases --- test.cpp | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/test.cpp b/test.cpp index fee132cf..d25d7481 100644 --- a/test.cpp +++ b/test.cpp @@ -41,6 +41,8 @@ #include #include +#include + // Insert user-defined prefix code (includes, function declarations, etc) // to set up a custom test suite JSON11_TEST_CPP_PREFIX_CODE @@ -70,17 +72,38 @@ JSON11_TEST_CASE(json11_test) { auto json_it = Json::parse(str_it, err_it); assert(json_it["message_id"].int64_value() == 105308320612483198); - assert(json_it["message_id"].int_value() != 105308320612483198); + assert(json_it["message_id"].uint64_value() == 105308320612483198); assert(json_it["msg_type"].int64_value() == 3); - assert(json_it["msg_type"].int_value() == 3); + assert(json_it["msg_type"].uint64_value() == 3); assert(json_it["order"].int64_value() == 0); assert(json_it["session_id"].int64_value() == 105308187502051928); - assert(json_it["site_id"].int_value() == 122062); assert(json_it["site_id"].int64_value() == 122062); - std::cout << (unsigned long long)json_it["visitor_id"].int64_value() << std::endl; + assert(json_it["site_id"].int64_value() == 122062); + std::cout << (signed long long)json_it["visitor_id"].int64_value() << std::endl; assert((unsigned long long)(json_it["visitor_id"].int64_value()) == 9941658010949867158); std::cout << "int int64 test passed!" << std::endl; + std::cout << "uint uint64 test" << std::endl; + const string str_it2 = R"({"int_max":2147483647,"int_min":-2147483648,"uint_max":4294967295, "uint_min":0, + "i64_max":9223372036854775807,"i64_min":-9223372036854775808,"ui64_max":18446744073709551615, "ui_64min":0 })"; + auto json_uit = Json::parse(str_it2, err_it); + std::cout << "ORIGIN:" << str_it2 << std::endl; + string str_it2_ret; + json_uit.dump(str_it2_ret); + std::cout << "DUMP:" << str_it2_ret << std::endl; + + assert(json_uit["int_max"].int64_value() == INT_MAX); + assert(json_uit["int_min"].int64_value() == INT_MIN); + assert(json_uit["uint_max"].uint64_value() == UINT_MAX); + assert(json_uit["uint_min"].uint64_value() == 0); + + assert(json_uit["i64_max"].int64_value() == LLONG_MAX); + assert(json_uit["i64_min"].int64_value() == LLONG_MIN); + assert(json_uit["ui64_max"].uint64_value() == ULLONG_MAX); + assert(json_uit["ui64_min"].uint64_value() == 0); + + std::cout << "int64 uint64 test passed!" << std::endl; + const string simple_test = R"({"k1":"v1", "k2":42, "k3":["a",123,true,false,null]})"; -- 2.30.2 From a4da043b17f02face92c88e24e61b03f7434c205 Mon Sep 17 00:00:00 2001 From: Zhijiang TAO Date: Tue, 16 Aug 2016 11:56:09 +0800 Subject: [PATCH 71/91] update readme --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) mode change 100644 => 100755 README.md diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 21f4e8b2..f545d12b --- a/README.md +++ b/README.md @@ -1,6 +1,29 @@ json11 ------ +#### modified +The original json11 from Dropbox doesnot support integer more than 2^54 for the compatible with javascript cases. But in most development cases, int64_t and uint64_t are very important types. So modified to support these ones. + + std::cout << "uint uint64 test" << std::endl; + const string str_it2 = R"({"int_max":2147483647,"int_min":-2147483648,"uint_max":4294967295, "uint_min":0, "i64_max":9223372036854775807,"i64_min":-9223372036854775808,"ui64_max":18446744073709551615, "ui_64min":0 })"; + auto json_uit = Json::parse(str_it2, err_it); + std::cout << "ORIGIN:" << str_it2 << std::endl; + string str_it2_ret; + json_uit.dump(str_it2_ret); + std::cout << "DUMP:" << str_it2_ret << std::endl; + + assert(json_uit["int_max"].int64_value() == INT_MAX); + assert(json_uit["int_min"].int64_value() == INT_MIN); + assert(json_uit["uint_max"].uint64_value() == UINT_MAX); + assert(json_uit["uint_min"].uint64_value() == 0); + + assert(json_uit["i64_max"].int64_value() == LLONG_MAX); + assert(json_uit["i64_min"].int64_value() == LLONG_MIN); + assert(json_uit["ui64_max"].uint64_value() == ULLONG_MAX); + assert(json_uit["ui64_min"].uint64_value() == 0); + +#### end + json11 is a tiny JSON library for C++11, providing JSON parsing and serialization. The core object provided by the library is json11::Json. A Json object represents any JSON -- 2.30.2 From 88de39206d2431c2aae5ad408a483f388a85eef2 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Mon, 23 Dec 2019 14:14:36 +0300 Subject: [PATCH 72/91] ull --- test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.cpp b/test.cpp index d25d7481..ca5ac889 100644 --- a/test.cpp +++ b/test.cpp @@ -80,7 +80,7 @@ JSON11_TEST_CASE(json11_test) { assert(json_it["site_id"].int64_value() == 122062); assert(json_it["site_id"].int64_value() == 122062); std::cout << (signed long long)json_it["visitor_id"].int64_value() << std::endl; - assert((unsigned long long)(json_it["visitor_id"].int64_value()) == 9941658010949867158); + assert((unsigned long long)(json_it["visitor_id"].int64_value()) == 9941658010949867158ull); std::cout << "int int64 test passed!" << std::endl; std::cout << "uint uint64 test" << std::endl; -- 2.30.2 From 5dc108754ad40d3b1d024f9bd7cca0595ef1a1db Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 14 Mar 2020 02:13:05 +0300 Subject: [PATCH 73/91] Fix "Dereferencing type-punned pointer will break strict aliasing rules..." --- btree.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/btree.h b/btree.h index 49310a2e..f8749bb4 100644 --- a/btree.h +++ b/btree.h @@ -554,10 +554,12 @@ class btree_node { return params_type::key(fields_.values[i]); } reference value(int i) { - return reinterpret_cast(fields_.values[i]); + mutable_value_type *ptr = (mutable_value_type*)&fields_.values[i]; + return reinterpret_cast(*ptr); } const_reference value(int i) const { - return reinterpret_cast(fields_.values[i]); + value_type *ptr = (value_type*)&fields_.values[i]; + return reinterpret_cast(*ptr); } mutable_value_type* mutable_value(int i) { return &fields_.values[i]; -- 2.30.2 From 97f06cb20c1e136fd37d58fb40f57dd8f8a3a4a7 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 24 Apr 2020 02:56:05 +0300 Subject: [PATCH 74/91] Add string->number conversions --- json11.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/json11.cpp b/json11.cpp index 9f593a04..d60c84ff 100644 --- a/json11.cpp +++ b/json11.cpp @@ -206,6 +206,9 @@ public: class JsonString final : public Value { const string &string_value() const override { return m_value; } + double number_value() const override { size_t pos = 0; double v = std::stod(m_value, &pos); if (pos < m_value.length()) { v = 0; } return v; } + int64_ int64_value() const override { size_t pos = 0; int64_ v = std::stoll(m_value, &pos); if (pos < m_value.length()) { v = 0; } return v; } + uint64_ uint64_value() const override { size_t pos = 0; uint64_ v = std::stoull(m_value, &pos); if (pos < m_value.length()) { v = 0; } return v; } public: explicit JsonString(const string &value) : Value(value) {} explicit JsonString(string &&value) : Value(move(value)) {} -- 2.30.2 From b2dbd5fa67ce55ac5f04272aa584ff544525f03a Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Fri, 19 Mar 2021 01:59:16 +0300 Subject: [PATCH 75/91] Update node js modules --- mon/node_modules/sprintf-js/CHANGELOG.md | 17 + mon/node_modules/sprintf-js/CONTRIBUTORS.md | 25 + mon/node_modules/sprintf-js/LICENSE | 24 + mon/node_modules/sprintf-js/README.md | 143 +++ .../sprintf-js/dist/.gitattributes | 4 + .../sprintf-js/dist/angular-sprintf.min.js | 3 + .../dist/angular-sprintf.min.js.map | 1 + .../sprintf-js/dist/sprintf.min.js | 3 + .../sprintf-js/dist/sprintf.min.js.map | 1 + mon/node_modules/sprintf-js/package.json | 63 ++ .../sprintf-js/src/angular-sprintf.js | 24 + mon/node_modules/sprintf-js/src/sprintf.js | 231 +++++ mon/node_modules/ws/LICENSE | 21 + mon/node_modules/ws/README.md | 496 ++++++++++ mon/node_modules/ws/browser.js | 8 + mon/node_modules/ws/index.js | 10 + mon/node_modules/ws/lib/buffer-util.js | 129 +++ mon/node_modules/ws/lib/constants.js | 10 + mon/node_modules/ws/lib/event-target.js | 184 ++++ mon/node_modules/ws/lib/extension.js | 223 +++++ mon/node_modules/ws/lib/limiter.js | 55 ++ mon/node_modules/ws/lib/permessage-deflate.js | 517 ++++++++++ mon/node_modules/ws/lib/receiver.js | 507 ++++++++++ mon/node_modules/ws/lib/sender.js | 405 ++++++++ mon/node_modules/ws/lib/stream.js | 165 ++++ mon/node_modules/ws/lib/validation.js | 30 + mon/node_modules/ws/lib/websocket-server.js | 406 ++++++++ mon/node_modules/ws/lib/websocket.js | 933 ++++++++++++++++++ mon/node_modules/ws/package.json | 90 ++ 29 files changed, 4728 insertions(+) create mode 100644 mon/node_modules/sprintf-js/CHANGELOG.md create mode 100644 mon/node_modules/sprintf-js/CONTRIBUTORS.md create mode 100644 mon/node_modules/sprintf-js/LICENSE create mode 100644 mon/node_modules/sprintf-js/README.md create mode 100644 mon/node_modules/sprintf-js/dist/.gitattributes create mode 100644 mon/node_modules/sprintf-js/dist/angular-sprintf.min.js create mode 100644 mon/node_modules/sprintf-js/dist/angular-sprintf.min.js.map create mode 100644 mon/node_modules/sprintf-js/dist/sprintf.min.js create mode 100644 mon/node_modules/sprintf-js/dist/sprintf.min.js.map create mode 100644 mon/node_modules/sprintf-js/package.json create mode 100644 mon/node_modules/sprintf-js/src/angular-sprintf.js create mode 100644 mon/node_modules/sprintf-js/src/sprintf.js create mode 100644 mon/node_modules/ws/LICENSE create mode 100644 mon/node_modules/ws/README.md create mode 100644 mon/node_modules/ws/browser.js create mode 100644 mon/node_modules/ws/index.js create mode 100644 mon/node_modules/ws/lib/buffer-util.js create mode 100644 mon/node_modules/ws/lib/constants.js create mode 100644 mon/node_modules/ws/lib/event-target.js create mode 100644 mon/node_modules/ws/lib/extension.js create mode 100644 mon/node_modules/ws/lib/limiter.js create mode 100644 mon/node_modules/ws/lib/permessage-deflate.js create mode 100644 mon/node_modules/ws/lib/receiver.js create mode 100644 mon/node_modules/ws/lib/sender.js create mode 100644 mon/node_modules/ws/lib/stream.js create mode 100644 mon/node_modules/ws/lib/validation.js create mode 100644 mon/node_modules/ws/lib/websocket-server.js create mode 100644 mon/node_modules/ws/lib/websocket.js create mode 100644 mon/node_modules/ws/package.json diff --git a/mon/node_modules/sprintf-js/CHANGELOG.md b/mon/node_modules/sprintf-js/CHANGELOG.md new file mode 100644 index 00000000..88c33f79 --- /dev/null +++ b/mon/node_modules/sprintf-js/CHANGELOG.md @@ -0,0 +1,17 @@ +## 1.1.2 (NOT YET RELEASED) + +* Update Travis config to test on all LTE Node versions (i.e. 6+) +* Reorganize README; add instructions re polyfills +* Refactor the code +* Upgrade dependencies +* Fix minifying issue with missing semicolons +* Configure .npmignore to reduce package size + + +## 1.1.1 (2017-05-29) + +* This CHANGELOG +* Various optimizations for modern browsers +* Fix %g, %o, %x and %X specifiers +* Use ESLint instead of JSHint +* Add CONTRIBUTORS file diff --git a/mon/node_modules/sprintf-js/CONTRIBUTORS.md b/mon/node_modules/sprintf-js/CONTRIBUTORS.md new file mode 100644 index 00000000..8172f03d --- /dev/null +++ b/mon/node_modules/sprintf-js/CONTRIBUTORS.md @@ -0,0 +1,25 @@ +Alexander Rose [@arose](https://github.com/arose) +Alexandru Mărășteanu [@alexei](https://github.com/alexei) +Andras [@andrasq](https://github.com/andrasq) +Benoit Giannangeli [@giann](https://github.com/giann) +Branden Visser [@mrvisser](https://github.com/mrvisser) +David Baird +daurnimator [@daurnimator](https://github.com/daurnimator) +Doug Beck [@beck](https://github.com/beck) +Dzmitry Litskalau [@litmit](https://github.com/litmit) +Fred Ludlow [@fredludlow](https://github.com/fredludlow) +Hans Pufal +Henry [@alograg](https://github.com/alograg) +Johnny Shields [@johnnyshields](https://github.com/johnnyshields) +Kamal Abdali +Matt Simerson [@msimerson](https://github.com/msimerson) +Maxime Robert [@marob](https://github.com/marob) +MeriemKhelifi [@MeriemKhelifi](https://github.com/MeriemKhelifi) +Michael Schramm [@wodka](https://github.com/wodka) +Nazar Mokrynskyi [@nazar-pc](https://github.com/nazar-pc) +Oliver Salzburg [@oliversalzburg](https://github.com/oliversalzburg) +Pablo [@ppollono](https://github.com/ppollono) +Rabehaja Stevens [@RABEHAJA-STEVENS](https://github.com/RABEHAJA-STEVENS) +Raphael Pigulla [@pigulla](https://github.com/pigulla) +rebeccapeltz [@rebeccapeltz](https://github.com/rebeccapeltz) +Stefan Tingström [@stingstrom](https://github.com/stingstrom) diff --git a/mon/node_modules/sprintf-js/LICENSE b/mon/node_modules/sprintf-js/LICENSE new file mode 100644 index 00000000..83f832a2 --- /dev/null +++ b/mon/node_modules/sprintf-js/LICENSE @@ -0,0 +1,24 @@ +Copyright (c) 2007-present, Alexandru Mărășteanu +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +* Neither the name of this software nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/mon/node_modules/sprintf-js/README.md b/mon/node_modules/sprintf-js/README.md new file mode 100644 index 00000000..cad1e63f --- /dev/null +++ b/mon/node_modules/sprintf-js/README.md @@ -0,0 +1,143 @@ +# sprintf-js + +[![Build Status][travisci-image]][travisci-url] [![NPM Version][npm-image]][npm-url] [![Dependency Status][dependencies-image]][dependencies-url] [![devDependency Status][dev-dependencies-image]][dev-dependencies-url] + +[travisci-image]: https://travis-ci.org/alexei/sprintf.js.svg?branch=master +[travisci-url]: https://travis-ci.org/alexei/sprintf.js + +[npm-image]: https://badge.fury.io/js/sprintf-js.svg +[npm-url]: https://badge.fury.io/js/sprintf-js + +[dependencies-image]: https://david-dm.org/alexei/sprintf.js.svg +[dependencies-url]: https://david-dm.org/alexei/sprintf.js + +[dev-dependencies-image]: https://david-dm.org/alexei/sprintf.js/dev-status.svg +[dev-dependencies-url]: https://david-dm.org/alexei/sprintf.js#info=devDependencies + +**sprintf-js** is a complete open source JavaScript `sprintf` implementation for the **browser** and **Node.js**. + +**Note: as of v1.1.1 you might need some polyfills for older environments. See [Support](#support) section below.** + +## Usage + + var sprintf = require('sprintf-js').sprintf, + vsprintf = require('sprintf-js').vsprintf + + sprintf('%2$s %3$s a %1$s', 'cracker', 'Polly', 'wants') + vsprintf('The first 4 letters of the english alphabet are: %s, %s, %s and %s', ['a', 'b', 'c', 'd']) + +## Installation + +### NPM + + npm install sprintf-js + +### Bower + + bower install sprintf + +## API + +### `sprintf` + +Returns a formatted string: + + string sprintf(string format, mixed arg1?, mixed arg2?, ...) + +### `vsprintf` + +Same as `sprintf` except it takes an array of arguments, rather than a variable number of arguments: + + string vsprintf(string format, array arguments?) + +## Format specification + +The placeholders in the format string are marked by `%` and are followed by one or more of these elements, in this order: + +* An optional number followed by a `$` sign that selects which argument index to use for the value. If not specified, arguments will be placed in the same order as the placeholders in the input string. +* An optional `+` sign that forces to preceed the result with a plus or minus sign on numeric values. By default, only the `-` sign is used on negative numbers. +* An optional padding specifier that says what character to use for padding (if specified). Possible values are `0` or any other character precedeed by a `'` (single quote). The default is to pad with *spaces*. +* An optional `-` sign, that causes `sprintf` to left-align the result of this placeholder. The default is to right-align the result. +* An optional number, that says how many characters the result should have. If the value to be returned is shorter than this number, the result will be padded. When used with the `j` (JSON) type specifier, the padding length specifies the tab size used for indentation. +* An optional precision modifier, consisting of a `.` (dot) followed by a number, that says how many digits should be displayed for floating point numbers. When used with the `g` type specifier, it specifies the number of significant digits. When used on a string, it causes the result to be truncated. +* A type specifier that can be any of: + * `%` — yields a literal `%` character + * `b` — yields an integer as a binary number + * `c` — yields an integer as the character with that ASCII value + * `d` or `i` — yields an integer as a signed decimal number + * `e` — yields a float using scientific notation + * `u` — yields an integer as an unsigned decimal number + * `f` — yields a float as is; see notes on precision above + * `g` — yields a float as is; see notes on precision above + * `o` — yields an integer as an octal number + * `s` — yields a string as is + * `t` — yields `true` or `false` + * `T` — yields the type of the argument1 + * `v` — yields the primitive value of the specified argument + * `x` — yields an integer as a hexadecimal number (lower-case) + * `X` — yields an integer as a hexadecimal number (upper-case) + * `j` — yields a JavaScript object or array as a JSON encoded string + +## Features + +### Argument swapping + +You can also swap the arguments. That is, the order of the placeholders doesn't have to match the order of the arguments. You can do that by simply indicating in the format string which arguments the placeholders refer to: + + sprintf('%2$s %3$s a %1$s', 'cracker', 'Polly', 'wants') + +And, of course, you can repeat the placeholders without having to increase the number of arguments. + +### Named arguments + +Format strings may contain replacement fields rather than positional placeholders. Instead of referring to a certain argument, you can now refer to a certain key within an object. Replacement fields are surrounded by rounded parentheses - `(` and `)` - and begin with a keyword that refers to a key: + + var user = { + name: 'Dolly', + } + sprintf('Hello %(name)s', user) // Hello Dolly + +Keywords in replacement fields can be optionally followed by any number of keywords or indexes: + + var users = [ + {name: 'Dolly'}, + {name: 'Molly'}, + {name: 'Polly'}, + ] + sprintf('Hello %(users[0].name)s, %(users[1].name)s and %(users[2].name)s', {users: users}) // Hello Dolly, Molly and Polly + +Note: mixing positional and named placeholders is not (yet) supported + +### Computed values + +You can pass in a function as a dynamic value and it will be invoked (with no arguments) in order to compute the value on the fly. + + sprintf('Current date and time: %s', function() { return new Date().toString() }) + +### AngularJS + +You can use `sprintf` and `vsprintf` (also aliased as `fmt` and `vfmt` respectively) in your AngularJS projects. See `demo/`. + +## Support + +### Node.js + +`sprintf-js` runs in all active Node versions (4.x+). + +### Browser + +`sprintf-js` should work in all modern browsers. As of v1.1.1, you might need polyfills for the following: + + - `String.prototype.repeat()` (any IE) + - `Array.isArray()` (IE < 9) + - `Object.create()` (IE < 9) + +YMMV + +## License + +**sprintf-js** is licensed under the terms of the 3-clause BSD license. + +## Notes + +1 `sprintf` doesn't use the `typeof` operator. As such, the value `null` is a `null`, an array is an `array` (not an `object`), a date value is a `date` etc. diff --git a/mon/node_modules/sprintf-js/dist/.gitattributes b/mon/node_modules/sprintf-js/dist/.gitattributes new file mode 100644 index 00000000..d35bca01 --- /dev/null +++ b/mon/node_modules/sprintf-js/dist/.gitattributes @@ -0,0 +1,4 @@ +#ignore all generated files from diff +#also skip line ending check +*.js -diff -text +*.map -diff -text diff --git a/mon/node_modules/sprintf-js/dist/angular-sprintf.min.js b/mon/node_modules/sprintf-js/dist/angular-sprintf.min.js new file mode 100644 index 00000000..dedec81b --- /dev/null +++ b/mon/node_modules/sprintf-js/dist/angular-sprintf.min.js @@ -0,0 +1,3 @@ +/*! sprintf-js v1.1.2 | Copyright (c) 2007-present, Alexandru Mărășteanu | BSD-3-Clause */ +!function(){"use strict";angular.module("sprintf",[]).filter("sprintf",function(){return function(){return sprintf.apply(null,arguments)}}).filter("fmt",["$filter",function(t){return t("sprintf")}]).filter("vsprintf",function(){return function(t,n){return vsprintf(t,n)}}).filter("vfmt",["$filter",function(t){return t("vsprintf")}])}(); +//# sourceMappingURL=angular-sprintf.min.js.map diff --git a/mon/node_modules/sprintf-js/dist/angular-sprintf.min.js.map b/mon/node_modules/sprintf-js/dist/angular-sprintf.min.js.map new file mode 100644 index 00000000..d30f1a38 --- /dev/null +++ b/mon/node_modules/sprintf-js/dist/angular-sprintf.min.js.map @@ -0,0 +1 @@ +{"version":3,"sources":["angular-sprintf.js"],"names":["angular","module","filter","sprintf","apply","arguments","$filter","format","argv","vsprintf"],"mappings":";CAEC,WACG,aAEAA,QACIC,OAAO,UAAW,IAClBC,OAAO,UAAW,WACd,OAAO,WACH,OAAOC,QAAQC,MAAM,KAAMC,cAGnCH,OAAO,MAAO,CAAC,UAAW,SAASI,GAC/B,OAAOA,EAAQ,cAEnBJ,OAAO,WAAY,WACf,OAAO,SAASK,EAAQC,GACpB,OAAOC,SAASF,EAAQC,MAGhCN,OAAO,OAAQ,CAAC,UAAW,SAASI,GAChC,OAAOA,EAAQ,eAnB1B","file":"angular-sprintf.min.js","sourcesContent":["/* global angular, sprintf, vsprintf */\n\n!function() {\n 'use strict'\n\n angular.\n module('sprintf', []).\n filter('sprintf', function() {\n return function() {\n return sprintf.apply(null, arguments)\n }\n }).\n filter('fmt', ['$filter', function($filter) {\n return $filter('sprintf')\n }]).\n filter('vsprintf', function() {\n return function(format, argv) {\n return vsprintf(format, argv)\n }\n }).\n filter('vfmt', ['$filter', function($filter) {\n return $filter('vsprintf')\n }])\n}(); // eslint-disable-line\n"]} \ No newline at end of file diff --git a/mon/node_modules/sprintf-js/dist/sprintf.min.js b/mon/node_modules/sprintf-js/dist/sprintf.min.js new file mode 100644 index 00000000..a3196b88 --- /dev/null +++ b/mon/node_modules/sprintf-js/dist/sprintf.min.js @@ -0,0 +1,3 @@ +/*! sprintf-js v1.1.2 | Copyright (c) 2007-present, Alexandru Mărășteanu | BSD-3-Clause */ +!function(){"use strict";var g={not_string:/[^s]/,not_bool:/[^t]/,not_type:/[^T]/,not_primitive:/[^v]/,number:/[diefg]/,numeric_arg:/[bcdiefguxX]/,json:/[j]/,not_json:/[^j]/,text:/^[^\x25]+/,modulo:/^\x25{2}/,placeholder:/^\x25(?:([1-9]\d*)\$|\(([^)]+)\))?(\+)?(0|'[^$])?(-)?(\d+)?(?:\.(\d+))?([b-gijostTuvxX])/,key:/^([a-z_][a-z_\d]*)/i,key_access:/^\.([a-z_][a-z_\d]*)/i,index_access:/^\[(\d+)\]/,sign:/^[+-]/};function y(e){return function(e,t){var r,n,i,s,a,o,p,c,l,u=1,f=e.length,d="";for(n=0;n>>0).toString(8);break;case"s":r=String(r),r=s.precision?r.substring(0,s.precision):r;break;case"t":r=String(!!r),r=s.precision?r.substring(0,s.precision):r;break;case"T":r=Object.prototype.toString.call(r).slice(8,-1).toLowerCase(),r=s.precision?r.substring(0,s.precision):r;break;case"u":r=parseInt(r,10)>>>0;break;case"v":r=r.valueOf(),r=s.precision?r.substring(0,s.precision):r;break;case"x":r=(parseInt(r,10)>>>0).toString(16);break;case"X":r=(parseInt(r,10)>>>0).toString(16).toUpperCase()}g.json.test(s.type)?d+=r:(!g.number.test(s.type)||c&&!s.sign?l="":(l=c?"+":"-",r=r.toString().replace(g.sign,"")),o=s.pad_char?"0"===s.pad_char?"0":s.pad_char.charAt(1):" ",p=s.width-(l+r).length,a=s.width&&0= 0\n }\n\n switch (ph.type) {\n case 'b':\n arg = parseInt(arg, 10).toString(2)\n break\n case 'c':\n arg = String.fromCharCode(parseInt(arg, 10))\n break\n case 'd':\n case 'i':\n arg = parseInt(arg, 10)\n break\n case 'j':\n arg = JSON.stringify(arg, null, ph.width ? parseInt(ph.width) : 0)\n break\n case 'e':\n arg = ph.precision ? parseFloat(arg).toExponential(ph.precision) : parseFloat(arg).toExponential()\n break\n case 'f':\n arg = ph.precision ? parseFloat(arg).toFixed(ph.precision) : parseFloat(arg)\n break\n case 'g':\n arg = ph.precision ? String(Number(arg.toPrecision(ph.precision))) : parseFloat(arg)\n break\n case 'o':\n arg = (parseInt(arg, 10) >>> 0).toString(8)\n break\n case 's':\n arg = String(arg)\n arg = (ph.precision ? arg.substring(0, ph.precision) : arg)\n break\n case 't':\n arg = String(!!arg)\n arg = (ph.precision ? arg.substring(0, ph.precision) : arg)\n break\n case 'T':\n arg = Object.prototype.toString.call(arg).slice(8, -1).toLowerCase()\n arg = (ph.precision ? arg.substring(0, ph.precision) : arg)\n break\n case 'u':\n arg = parseInt(arg, 10) >>> 0\n break\n case 'v':\n arg = arg.valueOf()\n arg = (ph.precision ? arg.substring(0, ph.precision) : arg)\n break\n case 'x':\n arg = (parseInt(arg, 10) >>> 0).toString(16)\n break\n case 'X':\n arg = (parseInt(arg, 10) >>> 0).toString(16).toUpperCase()\n break\n }\n if (re.json.test(ph.type)) {\n output += arg\n }\n else {\n if (re.number.test(ph.type) && (!is_positive || ph.sign)) {\n sign = is_positive ? '+' : '-'\n arg = arg.toString().replace(re.sign, '')\n }\n else {\n sign = ''\n }\n pad_character = ph.pad_char ? ph.pad_char === '0' ? '0' : ph.pad_char.charAt(1) : ' '\n pad_length = ph.width - (sign + arg).length\n pad = ph.width ? (pad_length > 0 ? pad_character.repeat(pad_length) : '') : ''\n output += ph.align ? sign + arg + pad : (pad_character === '0' ? sign + pad + arg : pad + sign + arg)\n }\n }\n }\n return output\n }\n\n var sprintf_cache = Object.create(null)\n\n function sprintf_parse(fmt) {\n if (sprintf_cache[fmt]) {\n return sprintf_cache[fmt]\n }\n\n var _fmt = fmt, match, parse_tree = [], arg_names = 0\n while (_fmt) {\n if ((match = re.text.exec(_fmt)) !== null) {\n parse_tree.push(match[0])\n }\n else if ((match = re.modulo.exec(_fmt)) !== null) {\n parse_tree.push('%')\n }\n else if ((match = re.placeholder.exec(_fmt)) !== null) {\n if (match[2]) {\n arg_names |= 1\n var field_list = [], replacement_field = match[2], field_match = []\n if ((field_match = re.key.exec(replacement_field)) !== null) {\n field_list.push(field_match[1])\n while ((replacement_field = replacement_field.substring(field_match[0].length)) !== '') {\n if ((field_match = re.key_access.exec(replacement_field)) !== null) {\n field_list.push(field_match[1])\n }\n else if ((field_match = re.index_access.exec(replacement_field)) !== null) {\n field_list.push(field_match[1])\n }\n else {\n throw new SyntaxError('[sprintf] failed to parse named argument key')\n }\n }\n }\n else {\n throw new SyntaxError('[sprintf] failed to parse named argument key')\n }\n match[2] = field_list\n }\n else {\n arg_names |= 2\n }\n if (arg_names === 3) {\n throw new Error('[sprintf] mixing positional and named placeholders is not (yet) supported')\n }\n\n parse_tree.push(\n {\n placeholder: match[0],\n param_no: match[1],\n keys: match[2],\n sign: match[3],\n pad_char: match[4],\n align: match[5],\n width: match[6],\n precision: match[7],\n type: match[8]\n }\n )\n }\n else {\n throw new SyntaxError('[sprintf] unexpected placeholder')\n }\n _fmt = _fmt.substring(match[0].length)\n }\n return sprintf_cache[fmt] = parse_tree\n }\n\n /**\n * export to either browser or node.js\n */\n /* eslint-disable quote-props */\n if (typeof exports !== 'undefined') {\n exports['sprintf'] = sprintf\n exports['vsprintf'] = vsprintf\n }\n if (typeof window !== 'undefined') {\n window['sprintf'] = sprintf\n window['vsprintf'] = vsprintf\n\n if (typeof define === 'function' && define['amd']) {\n define(function() {\n return {\n 'sprintf': sprintf,\n 'vsprintf': vsprintf\n }\n })\n }\n }\n /* eslint-enable quote-props */\n}(); // eslint-disable-line\n"]} \ No newline at end of file diff --git a/mon/node_modules/sprintf-js/package.json b/mon/node_modules/sprintf-js/package.json new file mode 100644 index 00000000..39ec237d --- /dev/null +++ b/mon/node_modules/sprintf-js/package.json @@ -0,0 +1,63 @@ +{ + "_from": "sprintf-js@^1.1.2", + "_id": "sprintf-js@1.1.2", + "_inBundle": false, + "_integrity": "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==", + "_location": "/sprintf-js", + "_phantomChildren": {}, + "_requested": { + "type": "range", + "registry": true, + "raw": "sprintf-js@^1.1.2", + "name": "sprintf-js", + "escapedName": "sprintf-js", + "rawSpec": "^1.1.2", + "saveSpec": null, + "fetchSpec": "^1.1.2" + }, + "_requiredBy": [ + "/" + ], + "_resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz", + "_shasum": "da1765262bf8c0f571749f2ad6c26300207ae673", + "_spec": "sprintf-js@^1.1.2", + "_where": "/home/shaba/RPM/git/vitastor/mon", + "author": { + "name": "Alexandru Mărășteanu", + "email": "hello@alexei.ro" + }, + "bugs": { + "url": "https://github.com/alexei/sprintf.js/issues" + }, + "bundleDependencies": false, + "deprecated": false, + "description": "JavaScript sprintf implementation", + "devDependencies": { + "benchmark": "^2.1.4", + "eslint": "^5.10.0", + "gulp": "^3.9.1", + "gulp-benchmark": "^1.1.1", + "gulp-eslint": "^5.0.0", + "gulp-header": "^2.0.5", + "gulp-mocha": "^6.0.0", + "gulp-rename": "^1.4.0", + "gulp-sourcemaps": "^2.6.4", + "gulp-uglify": "^3.0.1", + "mocha": "^5.2.0" + }, + "homepage": "https://github.com/alexei/sprintf.js#readme", + "license": "BSD-3-Clause", + "main": "src/sprintf.js", + "name": "sprintf-js", + "repository": { + "type": "git", + "url": "git+https://github.com/alexei/sprintf.js.git" + }, + "scripts": { + "lint": "eslint .", + "lint:fix": "eslint --fix .", + "pretest": "npm run lint", + "test": "mocha test/*.js" + }, + "version": "1.1.2" +} diff --git a/mon/node_modules/sprintf-js/src/angular-sprintf.js b/mon/node_modules/sprintf-js/src/angular-sprintf.js new file mode 100644 index 00000000..dbfdd65a --- /dev/null +++ b/mon/node_modules/sprintf-js/src/angular-sprintf.js @@ -0,0 +1,24 @@ +/* global angular, sprintf, vsprintf */ + +!function() { + 'use strict' + + angular. + module('sprintf', []). + filter('sprintf', function() { + return function() { + return sprintf.apply(null, arguments) + } + }). + filter('fmt', ['$filter', function($filter) { + return $filter('sprintf') + }]). + filter('vsprintf', function() { + return function(format, argv) { + return vsprintf(format, argv) + } + }). + filter('vfmt', ['$filter', function($filter) { + return $filter('vsprintf') + }]) +}(); // eslint-disable-line diff --git a/mon/node_modules/sprintf-js/src/sprintf.js b/mon/node_modules/sprintf-js/src/sprintf.js new file mode 100644 index 00000000..65d63246 --- /dev/null +++ b/mon/node_modules/sprintf-js/src/sprintf.js @@ -0,0 +1,231 @@ +/* global window, exports, define */ + +!function() { + 'use strict' + + var re = { + not_string: /[^s]/, + not_bool: /[^t]/, + not_type: /[^T]/, + not_primitive: /[^v]/, + number: /[diefg]/, + numeric_arg: /[bcdiefguxX]/, + json: /[j]/, + not_json: /[^j]/, + text: /^[^\x25]+/, + modulo: /^\x25{2}/, + placeholder: /^\x25(?:([1-9]\d*)\$|\(([^)]+)\))?(\+)?(0|'[^$])?(-)?(\d+)?(?:\.(\d+))?([b-gijostTuvxX])/, + key: /^([a-z_][a-z_\d]*)/i, + key_access: /^\.([a-z_][a-z_\d]*)/i, + index_access: /^\[(\d+)\]/, + sign: /^[+-]/ + } + + function sprintf(key) { + // `arguments` is not an array, but should be fine for this call + return sprintf_format(sprintf_parse(key), arguments) + } + + function vsprintf(fmt, argv) { + return sprintf.apply(null, [fmt].concat(argv || [])) + } + + function sprintf_format(parse_tree, argv) { + var cursor = 1, tree_length = parse_tree.length, arg, output = '', i, k, ph, pad, pad_character, pad_length, is_positive, sign + for (i = 0; i < tree_length; i++) { + if (typeof parse_tree[i] === 'string') { + output += parse_tree[i] + } + else if (typeof parse_tree[i] === 'object') { + ph = parse_tree[i] // convenience purposes only + if (ph.keys) { // keyword argument + arg = argv[cursor] + for (k = 0; k < ph.keys.length; k++) { + if (arg == undefined) { + throw new Error(sprintf('[sprintf] Cannot access property "%s" of undefined value "%s"', ph.keys[k], ph.keys[k-1])) + } + arg = arg[ph.keys[k]] + } + } + else if (ph.param_no) { // positional argument (explicit) + arg = argv[ph.param_no] + } + else { // positional argument (implicit) + arg = argv[cursor++] + } + + if (re.not_type.test(ph.type) && re.not_primitive.test(ph.type) && arg instanceof Function) { + arg = arg() + } + + if (re.numeric_arg.test(ph.type) && (typeof arg !== 'number' && isNaN(arg))) { + throw new TypeError(sprintf('[sprintf] expecting number but found %T', arg)) + } + + if (re.number.test(ph.type)) { + is_positive = arg >= 0 + } + + switch (ph.type) { + case 'b': + arg = parseInt(arg, 10).toString(2) + break + case 'c': + arg = String.fromCharCode(parseInt(arg, 10)) + break + case 'd': + case 'i': + arg = parseInt(arg, 10) + break + case 'j': + arg = JSON.stringify(arg, null, ph.width ? parseInt(ph.width) : 0) + break + case 'e': + arg = ph.precision ? parseFloat(arg).toExponential(ph.precision) : parseFloat(arg).toExponential() + break + case 'f': + arg = ph.precision ? parseFloat(arg).toFixed(ph.precision) : parseFloat(arg) + break + case 'g': + arg = ph.precision ? String(Number(arg.toPrecision(ph.precision))) : parseFloat(arg) + break + case 'o': + arg = (parseInt(arg, 10) >>> 0).toString(8) + break + case 's': + arg = String(arg) + arg = (ph.precision ? arg.substring(0, ph.precision) : arg) + break + case 't': + arg = String(!!arg) + arg = (ph.precision ? arg.substring(0, ph.precision) : arg) + break + case 'T': + arg = Object.prototype.toString.call(arg).slice(8, -1).toLowerCase() + arg = (ph.precision ? arg.substring(0, ph.precision) : arg) + break + case 'u': + arg = parseInt(arg, 10) >>> 0 + break + case 'v': + arg = arg.valueOf() + arg = (ph.precision ? arg.substring(0, ph.precision) : arg) + break + case 'x': + arg = (parseInt(arg, 10) >>> 0).toString(16) + break + case 'X': + arg = (parseInt(arg, 10) >>> 0).toString(16).toUpperCase() + break + } + if (re.json.test(ph.type)) { + output += arg + } + else { + if (re.number.test(ph.type) && (!is_positive || ph.sign)) { + sign = is_positive ? '+' : '-' + arg = arg.toString().replace(re.sign, '') + } + else { + sign = '' + } + pad_character = ph.pad_char ? ph.pad_char === '0' ? '0' : ph.pad_char.charAt(1) : ' ' + pad_length = ph.width - (sign + arg).length + pad = ph.width ? (pad_length > 0 ? pad_character.repeat(pad_length) : '') : '' + output += ph.align ? sign + arg + pad : (pad_character === '0' ? sign + pad + arg : pad + sign + arg) + } + } + } + return output + } + + var sprintf_cache = Object.create(null) + + function sprintf_parse(fmt) { + if (sprintf_cache[fmt]) { + return sprintf_cache[fmt] + } + + var _fmt = fmt, match, parse_tree = [], arg_names = 0 + while (_fmt) { + if ((match = re.text.exec(_fmt)) !== null) { + parse_tree.push(match[0]) + } + else if ((match = re.modulo.exec(_fmt)) !== null) { + parse_tree.push('%') + } + else if ((match = re.placeholder.exec(_fmt)) !== null) { + if (match[2]) { + arg_names |= 1 + var field_list = [], replacement_field = match[2], field_match = [] + if ((field_match = re.key.exec(replacement_field)) !== null) { + field_list.push(field_match[1]) + while ((replacement_field = replacement_field.substring(field_match[0].length)) !== '') { + if ((field_match = re.key_access.exec(replacement_field)) !== null) { + field_list.push(field_match[1]) + } + else if ((field_match = re.index_access.exec(replacement_field)) !== null) { + field_list.push(field_match[1]) + } + else { + throw new SyntaxError('[sprintf] failed to parse named argument key') + } + } + } + else { + throw new SyntaxError('[sprintf] failed to parse named argument key') + } + match[2] = field_list + } + else { + arg_names |= 2 + } + if (arg_names === 3) { + throw new Error('[sprintf] mixing positional and named placeholders is not (yet) supported') + } + + parse_tree.push( + { + placeholder: match[0], + param_no: match[1], + keys: match[2], + sign: match[3], + pad_char: match[4], + align: match[5], + width: match[6], + precision: match[7], + type: match[8] + } + ) + } + else { + throw new SyntaxError('[sprintf] unexpected placeholder') + } + _fmt = _fmt.substring(match[0].length) + } + return sprintf_cache[fmt] = parse_tree + } + + /** + * export to either browser or node.js + */ + /* eslint-disable quote-props */ + if (typeof exports !== 'undefined') { + exports['sprintf'] = sprintf + exports['vsprintf'] = vsprintf + } + if (typeof window !== 'undefined') { + window['sprintf'] = sprintf + window['vsprintf'] = vsprintf + + if (typeof define === 'function' && define['amd']) { + define(function() { + return { + 'sprintf': sprintf, + 'vsprintf': vsprintf + } + }) + } + } + /* eslint-enable quote-props */ +}(); // eslint-disable-line diff --git a/mon/node_modules/ws/LICENSE b/mon/node_modules/ws/LICENSE new file mode 100644 index 00000000..a145cd1d --- /dev/null +++ b/mon/node_modules/ws/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2011 Einar Otto Stangvik + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/mon/node_modules/ws/README.md b/mon/node_modules/ws/README.md new file mode 100644 index 00000000..6aea136b --- /dev/null +++ b/mon/node_modules/ws/README.md @@ -0,0 +1,496 @@ +# ws: a Node.js WebSocket library + +[![Version npm](https://img.shields.io/npm/v/ws.svg?logo=npm)](https://www.npmjs.com/package/ws) +[![Build](https://img.shields.io/github/workflow/status/websockets/ws/CI/master?label=build&logo=github)](https://github.com/websockets/ws/actions?query=workflow%3ACI+branch%3Amaster) +[![Windows x86 Build](https://img.shields.io/appveyor/ci/lpinca/ws/master.svg?logo=appveyor)](https://ci.appveyor.com/project/lpinca/ws) +[![Coverage Status](https://img.shields.io/coveralls/websockets/ws/master.svg)](https://coveralls.io/github/websockets/ws) + +ws is a simple to use, blazing fast, and thoroughly tested WebSocket client and +server implementation. + +Passes the quite extensive Autobahn test suite: [server][server-report], +[client][client-report]. + +**Note**: This module does not work in the browser. The client in the docs is a +reference to a back end with the role of a client in the WebSocket +communication. Browser clients must use the native +[`WebSocket`](https://developer.mozilla.org/en-US/docs/Web/API/WebSocket) +object. To make the same code work seamlessly on Node.js and the browser, you +can use one of the many wrappers available on npm, like +[isomorphic-ws](https://github.com/heineiuo/isomorphic-ws). + +## Table of Contents + +- [Protocol support](#protocol-support) +- [Installing](#installing) + - [Opt-in for performance and spec compliance](#opt-in-for-performance-and-spec-compliance) +- [API docs](#api-docs) +- [WebSocket compression](#websocket-compression) +- [Usage examples](#usage-examples) + - [Sending and receiving text data](#sending-and-receiving-text-data) + - [Sending binary data](#sending-binary-data) + - [Simple server](#simple-server) + - [External HTTP/S server](#external-https-server) + - [Multiple servers sharing a single HTTP/S server](#multiple-servers-sharing-a-single-https-server) + - [Client authentication](#client-authentication) + - [Server broadcast](#server-broadcast) + - [echo.websocket.org demo](#echowebsocketorg-demo) + - [Use the Node.js streams API](#use-the-nodejs-streams-api) + - [Other examples](#other-examples) +- [FAQ](#faq) + - [How to get the IP address of the client?](#how-to-get-the-ip-address-of-the-client) + - [How to detect and close broken connections?](#how-to-detect-and-close-broken-connections) + - [How to connect via a proxy?](#how-to-connect-via-a-proxy) +- [Changelog](#changelog) +- [License](#license) + +## Protocol support + +- **HyBi drafts 07-12** (Use the option `protocolVersion: 8`) +- **HyBi drafts 13-17** (Current default, alternatively option + `protocolVersion: 13`) + +## Installing + +``` +npm install ws +``` + +### Opt-in for performance and spec compliance + +There are 2 optional modules that can be installed along side with the ws +module. These modules are binary addons which improve certain operations. +Prebuilt binaries are available for the most popular platforms so you don't +necessarily need to have a C++ compiler installed on your machine. + +- `npm install --save-optional bufferutil`: Allows to efficiently perform + operations such as masking and unmasking the data payload of the WebSocket + frames. +- `npm install --save-optional utf-8-validate`: Allows to efficiently check if a + message contains valid UTF-8 as required by the spec. + +## API docs + +See [`/doc/ws.md`](./doc/ws.md) for Node.js-like documentation of ws classes and +utility functions. + +## WebSocket compression + +ws supports the [permessage-deflate extension][permessage-deflate] which enables +the client and server to negotiate a compression algorithm and its parameters, +and then selectively apply it to the data payloads of each WebSocket message. + +The extension is disabled by default on the server and enabled by default on the +client. It adds a significant overhead in terms of performance and memory +consumption so we suggest to enable it only if it is really needed. + +Note that Node.js has a variety of issues with high-performance compression, +where increased concurrency, especially on Linux, can lead to [catastrophic +memory fragmentation][node-zlib-bug] and slow performance. If you intend to use +permessage-deflate in production, it is worthwhile to set up a test +representative of your workload and ensure Node.js/zlib will handle it with +acceptable performance and memory usage. + +Tuning of permessage-deflate can be done via the options defined below. You can +also use `zlibDeflateOptions` and `zlibInflateOptions`, which is passed directly +into the creation of [raw deflate/inflate streams][node-zlib-deflaterawdocs]. + +See [the docs][ws-server-options] for more options. + +```js +const WebSocket = require('ws'); + +const wss = new WebSocket.Server({ + port: 8080, + perMessageDeflate: { + zlibDeflateOptions: { + // See zlib defaults. + chunkSize: 1024, + memLevel: 7, + level: 3 + }, + zlibInflateOptions: { + chunkSize: 10 * 1024 + }, + // Other options settable: + clientNoContextTakeover: true, // Defaults to negotiated value. + serverNoContextTakeover: true, // Defaults to negotiated value. + serverMaxWindowBits: 10, // Defaults to negotiated value. + // Below options specified as default values. + concurrencyLimit: 10, // Limits zlib concurrency for perf. + threshold: 1024 // Size (in bytes) below which messages + // should not be compressed. + } +}); +``` + +The client will only use the extension if it is supported and enabled on the +server. To always disable the extension on the client set the +`perMessageDeflate` option to `false`. + +```js +const WebSocket = require('ws'); + +const ws = new WebSocket('ws://www.host.com/path', { + perMessageDeflate: false +}); +``` + +## Usage examples + +### Sending and receiving text data + +```js +const WebSocket = require('ws'); + +const ws = new WebSocket('ws://www.host.com/path'); + +ws.on('open', function open() { + ws.send('something'); +}); + +ws.on('message', function incoming(data) { + console.log(data); +}); +``` + +### Sending binary data + +```js +const WebSocket = require('ws'); + +const ws = new WebSocket('ws://www.host.com/path'); + +ws.on('open', function open() { + const array = new Float32Array(5); + + for (var i = 0; i < array.length; ++i) { + array[i] = i / 2; + } + + ws.send(array); +}); +``` + +### Simple server + +```js +const WebSocket = require('ws'); + +const wss = new WebSocket.Server({ port: 8080 }); + +wss.on('connection', function connection(ws) { + ws.on('message', function incoming(message) { + console.log('received: %s', message); + }); + + ws.send('something'); +}); +``` + +### External HTTP/S server + +```js +const fs = require('fs'); +const https = require('https'); +const WebSocket = require('ws'); + +const server = https.createServer({ + cert: fs.readFileSync('/path/to/cert.pem'), + key: fs.readFileSync('/path/to/key.pem') +}); +const wss = new WebSocket.Server({ server }); + +wss.on('connection', function connection(ws) { + ws.on('message', function incoming(message) { + console.log('received: %s', message); + }); + + ws.send('something'); +}); + +server.listen(8080); +``` + +### Multiple servers sharing a single HTTP/S server + +```js +const http = require('http'); +const WebSocket = require('ws'); +const url = require('url'); + +const server = http.createServer(); +const wss1 = new WebSocket.Server({ noServer: true }); +const wss2 = new WebSocket.Server({ noServer: true }); + +wss1.on('connection', function connection(ws) { + // ... +}); + +wss2.on('connection', function connection(ws) { + // ... +}); + +server.on('upgrade', function upgrade(request, socket, head) { + const pathname = url.parse(request.url).pathname; + + if (pathname === '/foo') { + wss1.handleUpgrade(request, socket, head, function done(ws) { + wss1.emit('connection', ws, request); + }); + } else if (pathname === '/bar') { + wss2.handleUpgrade(request, socket, head, function done(ws) { + wss2.emit('connection', ws, request); + }); + } else { + socket.destroy(); + } +}); + +server.listen(8080); +``` + +### Client authentication + +```js +const http = require('http'); +const WebSocket = require('ws'); + +const server = http.createServer(); +const wss = new WebSocket.Server({ noServer: true }); + +wss.on('connection', function connection(ws, request, client) { + ws.on('message', function message(msg) { + console.log(`Received message ${msg} from user ${client}`); + }); +}); + +server.on('upgrade', function upgrade(request, socket, head) { + // This function is not defined on purpose. Implement it with your own logic. + authenticate(request, (err, client) => { + if (err || !client) { + socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n'); + socket.destroy(); + return; + } + + wss.handleUpgrade(request, socket, head, function done(ws) { + wss.emit('connection', ws, request, client); + }); + }); +}); + +server.listen(8080); +``` + +Also see the provided [example][session-parse-example] using `express-session`. + +### Server broadcast + +A client WebSocket broadcasting to all connected WebSocket clients, including +itself. + +```js +const WebSocket = require('ws'); + +const wss = new WebSocket.Server({ port: 8080 }); + +wss.on('connection', function connection(ws) { + ws.on('message', function incoming(data) { + wss.clients.forEach(function each(client) { + if (client.readyState === WebSocket.OPEN) { + client.send(data); + } + }); + }); +}); +``` + +A client WebSocket broadcasting to every other connected WebSocket clients, +excluding itself. + +```js +const WebSocket = require('ws'); + +const wss = new WebSocket.Server({ port: 8080 }); + +wss.on('connection', function connection(ws) { + ws.on('message', function incoming(data) { + wss.clients.forEach(function each(client) { + if (client !== ws && client.readyState === WebSocket.OPEN) { + client.send(data); + } + }); + }); +}); +``` + +### echo.websocket.org demo + +```js +const WebSocket = require('ws'); + +const ws = new WebSocket('wss://echo.websocket.org/', { + origin: 'https://websocket.org' +}); + +ws.on('open', function open() { + console.log('connected'); + ws.send(Date.now()); +}); + +ws.on('close', function close() { + console.log('disconnected'); +}); + +ws.on('message', function incoming(data) { + console.log(`Roundtrip time: ${Date.now() - data} ms`); + + setTimeout(function timeout() { + ws.send(Date.now()); + }, 500); +}); +``` + +### Use the Node.js streams API + +```js +const WebSocket = require('ws'); + +const ws = new WebSocket('wss://echo.websocket.org/', { + origin: 'https://websocket.org' +}); + +const duplex = WebSocket.createWebSocketStream(ws, { encoding: 'utf8' }); + +duplex.pipe(process.stdout); +process.stdin.pipe(duplex); +``` + +### Other examples + +For a full example with a browser client communicating with a ws server, see the +examples folder. + +Otherwise, see the test cases. + +## FAQ + +### How to get the IP address of the client? + +The remote IP address can be obtained from the raw socket. + +```js +const WebSocket = require('ws'); + +const wss = new WebSocket.Server({ port: 8080 }); + +wss.on('connection', function connection(ws, req) { + const ip = req.socket.remoteAddress; +}); +``` + +When the server runs behind a proxy like NGINX, the de-facto standard is to use +the `X-Forwarded-For` header. + +```js +wss.on('connection', function connection(ws, req) { + const ip = req.headers['x-forwarded-for'].split(/\s*,\s*/)[0]; +}); +``` + +### How to detect and close broken connections? + +Sometimes the link between the server and the client can be interrupted in a way +that keeps both the server and the client unaware of the broken state of the +connection (e.g. when pulling the cord). + +In these cases ping messages can be used as a means to verify that the remote +endpoint is still responsive. + +```js +const WebSocket = require('ws'); + +function noop() {} + +function heartbeat() { + this.isAlive = true; +} + +const wss = new WebSocket.Server({ port: 8080 }); + +wss.on('connection', function connection(ws) { + ws.isAlive = true; + ws.on('pong', heartbeat); +}); + +const interval = setInterval(function ping() { + wss.clients.forEach(function each(ws) { + if (ws.isAlive === false) return ws.terminate(); + + ws.isAlive = false; + ws.ping(noop); + }); +}, 30000); + +wss.on('close', function close() { + clearInterval(interval); +}); +``` + +Pong messages are automatically sent in response to ping messages as required by +the spec. + +Just like the server example above your clients might as well lose connection +without knowing it. You might want to add a ping listener on your clients to +prevent that. A simple implementation would be: + +```js +const WebSocket = require('ws'); + +function heartbeat() { + clearTimeout(this.pingTimeout); + + // Use `WebSocket#terminate()`, which immediately destroys the connection, + // instead of `WebSocket#close()`, which waits for the close timer. + // Delay should be equal to the interval at which your server + // sends out pings plus a conservative assumption of the latency. + this.pingTimeout = setTimeout(() => { + this.terminate(); + }, 30000 + 1000); +} + +const client = new WebSocket('wss://echo.websocket.org/'); + +client.on('open', heartbeat); +client.on('ping', heartbeat); +client.on('close', function clear() { + clearTimeout(this.pingTimeout); +}); +``` + +### How to connect via a proxy? + +Use a custom `http.Agent` implementation like [https-proxy-agent][] or +[socks-proxy-agent][]. + +## Changelog + +We're using the GitHub [releases][changelog] for changelog entries. + +## License + +[MIT](LICENSE) + +[changelog]: https://github.com/websockets/ws/releases +[client-report]: http://websockets.github.io/ws/autobahn/clients/ +[https-proxy-agent]: https://github.com/TooTallNate/node-https-proxy-agent +[node-zlib-bug]: https://github.com/nodejs/node/issues/8871 +[node-zlib-deflaterawdocs]: + https://nodejs.org/api/zlib.html#zlib_zlib_createdeflateraw_options +[permessage-deflate]: https://tools.ietf.org/html/rfc7692 +[server-report]: http://websockets.github.io/ws/autobahn/servers/ +[session-parse-example]: ./examples/express-session-parse +[socks-proxy-agent]: https://github.com/TooTallNate/node-socks-proxy-agent +[ws-server-options]: + https://github.com/websockets/ws/blob/master/doc/ws.md#new-websocketserveroptions-callback diff --git a/mon/node_modules/ws/browser.js b/mon/node_modules/ws/browser.js new file mode 100644 index 00000000..ca4f628a --- /dev/null +++ b/mon/node_modules/ws/browser.js @@ -0,0 +1,8 @@ +'use strict'; + +module.exports = function () { + throw new Error( + 'ws does not work in the browser. Browser clients must use the native ' + + 'WebSocket object' + ); +}; diff --git a/mon/node_modules/ws/index.js b/mon/node_modules/ws/index.js new file mode 100644 index 00000000..722c7867 --- /dev/null +++ b/mon/node_modules/ws/index.js @@ -0,0 +1,10 @@ +'use strict'; + +const WebSocket = require('./lib/websocket'); + +WebSocket.createWebSocketStream = require('./lib/stream'); +WebSocket.Server = require('./lib/websocket-server'); +WebSocket.Receiver = require('./lib/receiver'); +WebSocket.Sender = require('./lib/sender'); + +module.exports = WebSocket; diff --git a/mon/node_modules/ws/lib/buffer-util.js b/mon/node_modules/ws/lib/buffer-util.js new file mode 100644 index 00000000..6fd84c31 --- /dev/null +++ b/mon/node_modules/ws/lib/buffer-util.js @@ -0,0 +1,129 @@ +'use strict'; + +const { EMPTY_BUFFER } = require('./constants'); + +/** + * Merges an array of buffers into a new buffer. + * + * @param {Buffer[]} list The array of buffers to concat + * @param {Number} totalLength The total length of buffers in the list + * @return {Buffer} The resulting buffer + * @public + */ +function concat(list, totalLength) { + if (list.length === 0) return EMPTY_BUFFER; + if (list.length === 1) return list[0]; + + const target = Buffer.allocUnsafe(totalLength); + let offset = 0; + + for (let i = 0; i < list.length; i++) { + const buf = list[i]; + target.set(buf, offset); + offset += buf.length; + } + + if (offset < totalLength) return target.slice(0, offset); + + return target; +} + +/** + * Masks a buffer using the given mask. + * + * @param {Buffer} source The buffer to mask + * @param {Buffer} mask The mask to use + * @param {Buffer} output The buffer where to store the result + * @param {Number} offset The offset at which to start writing + * @param {Number} length The number of bytes to mask. + * @public + */ +function _mask(source, mask, output, offset, length) { + for (let i = 0; i < length; i++) { + output[offset + i] = source[i] ^ mask[i & 3]; + } +} + +/** + * Unmasks a buffer using the given mask. + * + * @param {Buffer} buffer The buffer to unmask + * @param {Buffer} mask The mask to use + * @public + */ +function _unmask(buffer, mask) { + // Required until https://github.com/nodejs/node/issues/9006 is resolved. + const length = buffer.length; + for (let i = 0; i < length; i++) { + buffer[i] ^= mask[i & 3]; + } +} + +/** + * Converts a buffer to an `ArrayBuffer`. + * + * @param {Buffer} buf The buffer to convert + * @return {ArrayBuffer} Converted buffer + * @public + */ +function toArrayBuffer(buf) { + if (buf.byteLength === buf.buffer.byteLength) { + return buf.buffer; + } + + return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); +} + +/** + * Converts `data` to a `Buffer`. + * + * @param {*} data The data to convert + * @return {Buffer} The buffer + * @throws {TypeError} + * @public + */ +function toBuffer(data) { + toBuffer.readOnly = true; + + if (Buffer.isBuffer(data)) return data; + + let buf; + + if (data instanceof ArrayBuffer) { + buf = Buffer.from(data); + } else if (ArrayBuffer.isView(data)) { + buf = Buffer.from(data.buffer, data.byteOffset, data.byteLength); + } else { + buf = Buffer.from(data); + toBuffer.readOnly = false; + } + + return buf; +} + +try { + const bufferUtil = require('bufferutil'); + const bu = bufferUtil.BufferUtil || bufferUtil; + + module.exports = { + concat, + mask(source, mask, output, offset, length) { + if (length < 48) _mask(source, mask, output, offset, length); + else bu.mask(source, mask, output, offset, length); + }, + toArrayBuffer, + toBuffer, + unmask(buffer, mask) { + if (buffer.length < 32) _unmask(buffer, mask); + else bu.unmask(buffer, mask); + } + }; +} catch (e) /* istanbul ignore next */ { + module.exports = { + concat, + mask: _mask, + toArrayBuffer, + toBuffer, + unmask: _unmask + }; +} diff --git a/mon/node_modules/ws/lib/constants.js b/mon/node_modules/ws/lib/constants.js new file mode 100644 index 00000000..4082981f --- /dev/null +++ b/mon/node_modules/ws/lib/constants.js @@ -0,0 +1,10 @@ +'use strict'; + +module.exports = { + BINARY_TYPES: ['nodebuffer', 'arraybuffer', 'fragments'], + GUID: '258EAFA5-E914-47DA-95CA-C5AB0DC85B11', + kStatusCode: Symbol('status-code'), + kWebSocket: Symbol('websocket'), + EMPTY_BUFFER: Buffer.alloc(0), + NOOP: () => {} +}; diff --git a/mon/node_modules/ws/lib/event-target.js b/mon/node_modules/ws/lib/event-target.js new file mode 100644 index 00000000..a6fbe72b --- /dev/null +++ b/mon/node_modules/ws/lib/event-target.js @@ -0,0 +1,184 @@ +'use strict'; + +/** + * Class representing an event. + * + * @private + */ +class Event { + /** + * Create a new `Event`. + * + * @param {String} type The name of the event + * @param {Object} target A reference to the target to which the event was + * dispatched + */ + constructor(type, target) { + this.target = target; + this.type = type; + } +} + +/** + * Class representing a message event. + * + * @extends Event + * @private + */ +class MessageEvent extends Event { + /** + * Create a new `MessageEvent`. + * + * @param {(String|Buffer|ArrayBuffer|Buffer[])} data The received data + * @param {WebSocket} target A reference to the target to which the event was + * dispatched + */ + constructor(data, target) { + super('message', target); + + this.data = data; + } +} + +/** + * Class representing a close event. + * + * @extends Event + * @private + */ +class CloseEvent extends Event { + /** + * Create a new `CloseEvent`. + * + * @param {Number} code The status code explaining why the connection is being + * closed + * @param {String} reason A human-readable string explaining why the + * connection is closing + * @param {WebSocket} target A reference to the target to which the event was + * dispatched + */ + constructor(code, reason, target) { + super('close', target); + + this.wasClean = target._closeFrameReceived && target._closeFrameSent; + this.reason = reason; + this.code = code; + } +} + +/** + * Class representing an open event. + * + * @extends Event + * @private + */ +class OpenEvent extends Event { + /** + * Create a new `OpenEvent`. + * + * @param {WebSocket} target A reference to the target to which the event was + * dispatched + */ + constructor(target) { + super('open', target); + } +} + +/** + * Class representing an error event. + * + * @extends Event + * @private + */ +class ErrorEvent extends Event { + /** + * Create a new `ErrorEvent`. + * + * @param {Object} error The error that generated this event + * @param {WebSocket} target A reference to the target to which the event was + * dispatched + */ + constructor(error, target) { + super('error', target); + + this.message = error.message; + this.error = error; + } +} + +/** + * This provides methods for emulating the `EventTarget` interface. It's not + * meant to be used directly. + * + * @mixin + */ +const EventTarget = { + /** + * Register an event listener. + * + * @param {String} type A string representing the event type to listen for + * @param {Function} listener The listener to add + * @param {Object} [options] An options object specifies characteristics about + * the event listener + * @param {Boolean} [options.once=false] A `Boolean`` indicating that the + * listener should be invoked at most once after being added. If `true`, + * the listener would be automatically removed when invoked. + * @public + */ + addEventListener(type, listener, options) { + if (typeof listener !== 'function') return; + + function onMessage(data) { + listener.call(this, new MessageEvent(data, this)); + } + + function onClose(code, message) { + listener.call(this, new CloseEvent(code, message, this)); + } + + function onError(error) { + listener.call(this, new ErrorEvent(error, this)); + } + + function onOpen() { + listener.call(this, new OpenEvent(this)); + } + + const method = options && options.once ? 'once' : 'on'; + + if (type === 'message') { + onMessage._listener = listener; + this[method](type, onMessage); + } else if (type === 'close') { + onClose._listener = listener; + this[method](type, onClose); + } else if (type === 'error') { + onError._listener = listener; + this[method](type, onError); + } else if (type === 'open') { + onOpen._listener = listener; + this[method](type, onOpen); + } else { + this[method](type, listener); + } + }, + + /** + * Remove an event listener. + * + * @param {String} type A string representing the event type to remove + * @param {Function} listener The listener to remove + * @public + */ + removeEventListener(type, listener) { + const listeners = this.listeners(type); + + for (let i = 0; i < listeners.length; i++) { + if (listeners[i] === listener || listeners[i]._listener === listener) { + this.removeListener(type, listeners[i]); + } + } + } +}; + +module.exports = EventTarget; diff --git a/mon/node_modules/ws/lib/extension.js b/mon/node_modules/ws/lib/extension.js new file mode 100644 index 00000000..87a42132 --- /dev/null +++ b/mon/node_modules/ws/lib/extension.js @@ -0,0 +1,223 @@ +'use strict'; + +// +// Allowed token characters: +// +// '!', '#', '$', '%', '&', ''', '*', '+', '-', +// '.', 0-9, A-Z, '^', '_', '`', a-z, '|', '~' +// +// tokenChars[32] === 0 // ' ' +// tokenChars[33] === 1 // '!' +// tokenChars[34] === 0 // '"' +// ... +// +// prettier-ignore +const tokenChars = [ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0 - 15 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 16 - 31 + 0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, // 32 - 47 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, // 48 - 63 + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 64 - 79 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, // 80 - 95 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 96 - 111 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0 // 112 - 127 +]; + +/** + * Adds an offer to the map of extension offers or a parameter to the map of + * parameters. + * + * @param {Object} dest The map of extension offers or parameters + * @param {String} name The extension or parameter name + * @param {(Object|Boolean|String)} elem The extension parameters or the + * parameter value + * @private + */ +function push(dest, name, elem) { + if (dest[name] === undefined) dest[name] = [elem]; + else dest[name].push(elem); +} + +/** + * Parses the `Sec-WebSocket-Extensions` header into an object. + * + * @param {String} header The field value of the header + * @return {Object} The parsed object + * @public + */ +function parse(header) { + const offers = Object.create(null); + + if (header === undefined || header === '') return offers; + + let params = Object.create(null); + let mustUnescape = false; + let isEscaping = false; + let inQuotes = false; + let extensionName; + let paramName; + let start = -1; + let end = -1; + let i = 0; + + for (; i < header.length; i++) { + const code = header.charCodeAt(i); + + if (extensionName === undefined) { + if (end === -1 && tokenChars[code] === 1) { + if (start === -1) start = i; + } else if (code === 0x20 /* ' ' */ || code === 0x09 /* '\t' */) { + if (end === -1 && start !== -1) end = i; + } else if (code === 0x3b /* ';' */ || code === 0x2c /* ',' */) { + if (start === -1) { + throw new SyntaxError(`Unexpected character at index ${i}`); + } + + if (end === -1) end = i; + const name = header.slice(start, end); + if (code === 0x2c) { + push(offers, name, params); + params = Object.create(null); + } else { + extensionName = name; + } + + start = end = -1; + } else { + throw new SyntaxError(`Unexpected character at index ${i}`); + } + } else if (paramName === undefined) { + if (end === -1 && tokenChars[code] === 1) { + if (start === -1) start = i; + } else if (code === 0x20 || code === 0x09) { + if (end === -1 && start !== -1) end = i; + } else if (code === 0x3b || code === 0x2c) { + if (start === -1) { + throw new SyntaxError(`Unexpected character at index ${i}`); + } + + if (end === -1) end = i; + push(params, header.slice(start, end), true); + if (code === 0x2c) { + push(offers, extensionName, params); + params = Object.create(null); + extensionName = undefined; + } + + start = end = -1; + } else if (code === 0x3d /* '=' */ && start !== -1 && end === -1) { + paramName = header.slice(start, i); + start = end = -1; + } else { + throw new SyntaxError(`Unexpected character at index ${i}`); + } + } else { + // + // The value of a quoted-string after unescaping must conform to the + // token ABNF, so only token characters are valid. + // Ref: https://tools.ietf.org/html/rfc6455#section-9.1 + // + if (isEscaping) { + if (tokenChars[code] !== 1) { + throw new SyntaxError(`Unexpected character at index ${i}`); + } + if (start === -1) start = i; + else if (!mustUnescape) mustUnescape = true; + isEscaping = false; + } else if (inQuotes) { + if (tokenChars[code] === 1) { + if (start === -1) start = i; + } else if (code === 0x22 /* '"' */ && start !== -1) { + inQuotes = false; + end = i; + } else if (code === 0x5c /* '\' */) { + isEscaping = true; + } else { + throw new SyntaxError(`Unexpected character at index ${i}`); + } + } else if (code === 0x22 && header.charCodeAt(i - 1) === 0x3d) { + inQuotes = true; + } else if (end === -1 && tokenChars[code] === 1) { + if (start === -1) start = i; + } else if (start !== -1 && (code === 0x20 || code === 0x09)) { + if (end === -1) end = i; + } else if (code === 0x3b || code === 0x2c) { + if (start === -1) { + throw new SyntaxError(`Unexpected character at index ${i}`); + } + + if (end === -1) end = i; + let value = header.slice(start, end); + if (mustUnescape) { + value = value.replace(/\\/g, ''); + mustUnescape = false; + } + push(params, paramName, value); + if (code === 0x2c) { + push(offers, extensionName, params); + params = Object.create(null); + extensionName = undefined; + } + + paramName = undefined; + start = end = -1; + } else { + throw new SyntaxError(`Unexpected character at index ${i}`); + } + } + } + + if (start === -1 || inQuotes) { + throw new SyntaxError('Unexpected end of input'); + } + + if (end === -1) end = i; + const token = header.slice(start, end); + if (extensionName === undefined) { + push(offers, token, params); + } else { + if (paramName === undefined) { + push(params, token, true); + } else if (mustUnescape) { + push(params, paramName, token.replace(/\\/g, '')); + } else { + push(params, paramName, token); + } + push(offers, extensionName, params); + } + + return offers; +} + +/** + * Builds the `Sec-WebSocket-Extensions` header field value. + * + * @param {Object} extensions The map of extensions and parameters to format + * @return {String} A string representing the given object + * @public + */ +function format(extensions) { + return Object.keys(extensions) + .map((extension) => { + let configurations = extensions[extension]; + if (!Array.isArray(configurations)) configurations = [configurations]; + return configurations + .map((params) => { + return [extension] + .concat( + Object.keys(params).map((k) => { + let values = params[k]; + if (!Array.isArray(values)) values = [values]; + return values + .map((v) => (v === true ? k : `${k}=${v}`)) + .join('; '); + }) + ) + .join('; '); + }) + .join(', '); + }) + .join(', '); +} + +module.exports = { format, parse }; diff --git a/mon/node_modules/ws/lib/limiter.js b/mon/node_modules/ws/lib/limiter.js new file mode 100644 index 00000000..3fd35784 --- /dev/null +++ b/mon/node_modules/ws/lib/limiter.js @@ -0,0 +1,55 @@ +'use strict'; + +const kDone = Symbol('kDone'); +const kRun = Symbol('kRun'); + +/** + * A very simple job queue with adjustable concurrency. Adapted from + * https://github.com/STRML/async-limiter + */ +class Limiter { + /** + * Creates a new `Limiter`. + * + * @param {Number} [concurrency=Infinity] The maximum number of jobs allowed + * to run concurrently + */ + constructor(concurrency) { + this[kDone] = () => { + this.pending--; + this[kRun](); + }; + this.concurrency = concurrency || Infinity; + this.jobs = []; + this.pending = 0; + } + + /** + * Adds a job to the queue. + * + * @param {Function} job The job to run + * @public + */ + add(job) { + this.jobs.push(job); + this[kRun](); + } + + /** + * Removes a job from the queue and runs it if possible. + * + * @private + */ + [kRun]() { + if (this.pending === this.concurrency) return; + + if (this.jobs.length) { + const job = this.jobs.shift(); + + this.pending++; + job(this[kDone]); + } + } +} + +module.exports = Limiter; diff --git a/mon/node_modules/ws/lib/permessage-deflate.js b/mon/node_modules/ws/lib/permessage-deflate.js new file mode 100644 index 00000000..a8974b98 --- /dev/null +++ b/mon/node_modules/ws/lib/permessage-deflate.js @@ -0,0 +1,517 @@ +'use strict'; + +const zlib = require('zlib'); + +const bufferUtil = require('./buffer-util'); +const Limiter = require('./limiter'); +const { kStatusCode, NOOP } = require('./constants'); + +const TRAILER = Buffer.from([0x00, 0x00, 0xff, 0xff]); +const kPerMessageDeflate = Symbol('permessage-deflate'); +const kTotalLength = Symbol('total-length'); +const kCallback = Symbol('callback'); +const kBuffers = Symbol('buffers'); +const kError = Symbol('error'); + +// +// We limit zlib concurrency, which prevents severe memory fragmentation +// as documented in https://github.com/nodejs/node/issues/8871#issuecomment-250915913 +// and https://github.com/websockets/ws/issues/1202 +// +// Intentionally global; it's the global thread pool that's an issue. +// +let zlibLimiter; + +/** + * permessage-deflate implementation. + */ +class PerMessageDeflate { + /** + * Creates a PerMessageDeflate instance. + * + * @param {Object} [options] Configuration options + * @param {Boolean} [options.serverNoContextTakeover=false] Request/accept + * disabling of server context takeover + * @param {Boolean} [options.clientNoContextTakeover=false] Advertise/ + * acknowledge disabling of client context takeover + * @param {(Boolean|Number)} [options.serverMaxWindowBits] Request/confirm the + * use of a custom server window size + * @param {(Boolean|Number)} [options.clientMaxWindowBits] Advertise support + * for, or request, a custom client window size + * @param {Object} [options.zlibDeflateOptions] Options to pass to zlib on + * deflate + * @param {Object} [options.zlibInflateOptions] Options to pass to zlib on + * inflate + * @param {Number} [options.threshold=1024] Size (in bytes) below which + * messages should not be compressed + * @param {Number} [options.concurrencyLimit=10] The number of concurrent + * calls to zlib + * @param {Boolean} [isServer=false] Create the instance in either server or + * client mode + * @param {Number} [maxPayload=0] The maximum allowed message length + */ + constructor(options, isServer, maxPayload) { + this._maxPayload = maxPayload | 0; + this._options = options || {}; + this._threshold = + this._options.threshold !== undefined ? this._options.threshold : 1024; + this._isServer = !!isServer; + this._deflate = null; + this._inflate = null; + + this.params = null; + + if (!zlibLimiter) { + const concurrency = + this._options.concurrencyLimit !== undefined + ? this._options.concurrencyLimit + : 10; + zlibLimiter = new Limiter(concurrency); + } + } + + /** + * @type {String} + */ + static get extensionName() { + return 'permessage-deflate'; + } + + /** + * Create an extension negotiation offer. + * + * @return {Object} Extension parameters + * @public + */ + offer() { + const params = {}; + + if (this._options.serverNoContextTakeover) { + params.server_no_context_takeover = true; + } + if (this._options.clientNoContextTakeover) { + params.client_no_context_takeover = true; + } + if (this._options.serverMaxWindowBits) { + params.server_max_window_bits = this._options.serverMaxWindowBits; + } + if (this._options.clientMaxWindowBits) { + params.client_max_window_bits = this._options.clientMaxWindowBits; + } else if (this._options.clientMaxWindowBits == null) { + params.client_max_window_bits = true; + } + + return params; + } + + /** + * Accept an extension negotiation offer/response. + * + * @param {Array} configurations The extension negotiation offers/reponse + * @return {Object} Accepted configuration + * @public + */ + accept(configurations) { + configurations = this.normalizeParams(configurations); + + this.params = this._isServer + ? this.acceptAsServer(configurations) + : this.acceptAsClient(configurations); + + return this.params; + } + + /** + * Releases all resources used by the extension. + * + * @public + */ + cleanup() { + if (this._inflate) { + this._inflate.close(); + this._inflate = null; + } + + if (this._deflate) { + const callback = this._deflate[kCallback]; + + this._deflate.close(); + this._deflate = null; + + if (callback) { + callback( + new Error( + 'The deflate stream was closed while data was being processed' + ) + ); + } + } + } + + /** + * Accept an extension negotiation offer. + * + * @param {Array} offers The extension negotiation offers + * @return {Object} Accepted configuration + * @private + */ + acceptAsServer(offers) { + const opts = this._options; + const accepted = offers.find((params) => { + if ( + (opts.serverNoContextTakeover === false && + params.server_no_context_takeover) || + (params.server_max_window_bits && + (opts.serverMaxWindowBits === false || + (typeof opts.serverMaxWindowBits === 'number' && + opts.serverMaxWindowBits > params.server_max_window_bits))) || + (typeof opts.clientMaxWindowBits === 'number' && + !params.client_max_window_bits) + ) { + return false; + } + + return true; + }); + + if (!accepted) { + throw new Error('None of the extension offers can be accepted'); + } + + if (opts.serverNoContextTakeover) { + accepted.server_no_context_takeover = true; + } + if (opts.clientNoContextTakeover) { + accepted.client_no_context_takeover = true; + } + if (typeof opts.serverMaxWindowBits === 'number') { + accepted.server_max_window_bits = opts.serverMaxWindowBits; + } + if (typeof opts.clientMaxWindowBits === 'number') { + accepted.client_max_window_bits = opts.clientMaxWindowBits; + } else if ( + accepted.client_max_window_bits === true || + opts.clientMaxWindowBits === false + ) { + delete accepted.client_max_window_bits; + } + + return accepted; + } + + /** + * Accept the extension negotiation response. + * + * @param {Array} response The extension negotiation response + * @return {Object} Accepted configuration + * @private + */ + acceptAsClient(response) { + const params = response[0]; + + if ( + this._options.clientNoContextTakeover === false && + params.client_no_context_takeover + ) { + throw new Error('Unexpected parameter "client_no_context_takeover"'); + } + + if (!params.client_max_window_bits) { + if (typeof this._options.clientMaxWindowBits === 'number') { + params.client_max_window_bits = this._options.clientMaxWindowBits; + } + } else if ( + this._options.clientMaxWindowBits === false || + (typeof this._options.clientMaxWindowBits === 'number' && + params.client_max_window_bits > this._options.clientMaxWindowBits) + ) { + throw new Error( + 'Unexpected or invalid parameter "client_max_window_bits"' + ); + } + + return params; + } + + /** + * Normalize parameters. + * + * @param {Array} configurations The extension negotiation offers/reponse + * @return {Array} The offers/response with normalized parameters + * @private + */ + normalizeParams(configurations) { + configurations.forEach((params) => { + Object.keys(params).forEach((key) => { + let value = params[key]; + + if (value.length > 1) { + throw new Error(`Parameter "${key}" must have only a single value`); + } + + value = value[0]; + + if (key === 'client_max_window_bits') { + if (value !== true) { + const num = +value; + if (!Number.isInteger(num) || num < 8 || num > 15) { + throw new TypeError( + `Invalid value for parameter "${key}": ${value}` + ); + } + value = num; + } else if (!this._isServer) { + throw new TypeError( + `Invalid value for parameter "${key}": ${value}` + ); + } + } else if (key === 'server_max_window_bits') { + const num = +value; + if (!Number.isInteger(num) || num < 8 || num > 15) { + throw new TypeError( + `Invalid value for parameter "${key}": ${value}` + ); + } + value = num; + } else if ( + key === 'client_no_context_takeover' || + key === 'server_no_context_takeover' + ) { + if (value !== true) { + throw new TypeError( + `Invalid value for parameter "${key}": ${value}` + ); + } + } else { + throw new Error(`Unknown parameter "${key}"`); + } + + params[key] = value; + }); + }); + + return configurations; + } + + /** + * Decompress data. Concurrency limited. + * + * @param {Buffer} data Compressed data + * @param {Boolean} fin Specifies whether or not this is the last fragment + * @param {Function} callback Callback + * @public + */ + decompress(data, fin, callback) { + zlibLimiter.add((done) => { + this._decompress(data, fin, (err, result) => { + done(); + callback(err, result); + }); + }); + } + + /** + * Compress data. Concurrency limited. + * + * @param {Buffer} data Data to compress + * @param {Boolean} fin Specifies whether or not this is the last fragment + * @param {Function} callback Callback + * @public + */ + compress(data, fin, callback) { + zlibLimiter.add((done) => { + this._compress(data, fin, (err, result) => { + done(); + callback(err, result); + }); + }); + } + + /** + * Decompress data. + * + * @param {Buffer} data Compressed data + * @param {Boolean} fin Specifies whether or not this is the last fragment + * @param {Function} callback Callback + * @private + */ + _decompress(data, fin, callback) { + const endpoint = this._isServer ? 'client' : 'server'; + + if (!this._inflate) { + const key = `${endpoint}_max_window_bits`; + const windowBits = + typeof this.params[key] !== 'number' + ? zlib.Z_DEFAULT_WINDOWBITS + : this.params[key]; + + this._inflate = zlib.createInflateRaw({ + ...this._options.zlibInflateOptions, + windowBits + }); + this._inflate[kPerMessageDeflate] = this; + this._inflate[kTotalLength] = 0; + this._inflate[kBuffers] = []; + this._inflate.on('error', inflateOnError); + this._inflate.on('data', inflateOnData); + } + + this._inflate[kCallback] = callback; + + this._inflate.write(data); + if (fin) this._inflate.write(TRAILER); + + this._inflate.flush(() => { + const err = this._inflate[kError]; + + if (err) { + this._inflate.close(); + this._inflate = null; + callback(err); + return; + } + + const data = bufferUtil.concat( + this._inflate[kBuffers], + this._inflate[kTotalLength] + ); + + if (this._inflate._readableState.endEmitted) { + this._inflate.close(); + this._inflate = null; + } else { + this._inflate[kTotalLength] = 0; + this._inflate[kBuffers] = []; + + if (fin && this.params[`${endpoint}_no_context_takeover`]) { + this._inflate.reset(); + } + } + + callback(null, data); + }); + } + + /** + * Compress data. + * + * @param {Buffer} data Data to compress + * @param {Boolean} fin Specifies whether or not this is the last fragment + * @param {Function} callback Callback + * @private + */ + _compress(data, fin, callback) { + const endpoint = this._isServer ? 'server' : 'client'; + + if (!this._deflate) { + const key = `${endpoint}_max_window_bits`; + const windowBits = + typeof this.params[key] !== 'number' + ? zlib.Z_DEFAULT_WINDOWBITS + : this.params[key]; + + this._deflate = zlib.createDeflateRaw({ + ...this._options.zlibDeflateOptions, + windowBits + }); + + this._deflate[kTotalLength] = 0; + this._deflate[kBuffers] = []; + + // + // An `'error'` event is emitted, only on Node.js < 10.0.0, if the + // `zlib.DeflateRaw` instance is closed while data is being processed. + // This can happen if `PerMessageDeflate#cleanup()` is called at the wrong + // time due to an abnormal WebSocket closure. + // + this._deflate.on('error', NOOP); + this._deflate.on('data', deflateOnData); + } + + this._deflate[kCallback] = callback; + + this._deflate.write(data); + this._deflate.flush(zlib.Z_SYNC_FLUSH, () => { + if (!this._deflate) { + // + // The deflate stream was closed while data was being processed. + // + return; + } + + let data = bufferUtil.concat( + this._deflate[kBuffers], + this._deflate[kTotalLength] + ); + + if (fin) data = data.slice(0, data.length - 4); + + // + // Ensure that the callback will not be called again in + // `PerMessageDeflate#cleanup()`. + // + this._deflate[kCallback] = null; + + this._deflate[kTotalLength] = 0; + this._deflate[kBuffers] = []; + + if (fin && this.params[`${endpoint}_no_context_takeover`]) { + this._deflate.reset(); + } + + callback(null, data); + }); + } +} + +module.exports = PerMessageDeflate; + +/** + * The listener of the `zlib.DeflateRaw` stream `'data'` event. + * + * @param {Buffer} chunk A chunk of data + * @private + */ +function deflateOnData(chunk) { + this[kBuffers].push(chunk); + this[kTotalLength] += chunk.length; +} + +/** + * The listener of the `zlib.InflateRaw` stream `'data'` event. + * + * @param {Buffer} chunk A chunk of data + * @private + */ +function inflateOnData(chunk) { + this[kTotalLength] += chunk.length; + + if ( + this[kPerMessageDeflate]._maxPayload < 1 || + this[kTotalLength] <= this[kPerMessageDeflate]._maxPayload + ) { + this[kBuffers].push(chunk); + return; + } + + this[kError] = new RangeError('Max payload size exceeded'); + this[kError][kStatusCode] = 1009; + this.removeListener('data', inflateOnData); + this.reset(); +} + +/** + * The listener of the `zlib.InflateRaw` stream `'error'` event. + * + * @param {Error} err The emitted error + * @private + */ +function inflateOnError(err) { + // + // There is no need to call `Zlib#close()` as the handle is automatically + // closed when an error is emitted. + // + this[kPerMessageDeflate]._inflate = null; + err[kStatusCode] = 1007; + this[kCallback](err); +} diff --git a/mon/node_modules/ws/lib/receiver.js b/mon/node_modules/ws/lib/receiver.js new file mode 100644 index 00000000..65a5ab45 --- /dev/null +++ b/mon/node_modules/ws/lib/receiver.js @@ -0,0 +1,507 @@ +'use strict'; + +const { Writable } = require('stream'); + +const PerMessageDeflate = require('./permessage-deflate'); +const { + BINARY_TYPES, + EMPTY_BUFFER, + kStatusCode, + kWebSocket +} = require('./constants'); +const { concat, toArrayBuffer, unmask } = require('./buffer-util'); +const { isValidStatusCode, isValidUTF8 } = require('./validation'); + +const GET_INFO = 0; +const GET_PAYLOAD_LENGTH_16 = 1; +const GET_PAYLOAD_LENGTH_64 = 2; +const GET_MASK = 3; +const GET_DATA = 4; +const INFLATING = 5; + +/** + * HyBi Receiver implementation. + * + * @extends stream.Writable + */ +class Receiver extends Writable { + /** + * Creates a Receiver instance. + * + * @param {String} [binaryType=nodebuffer] The type for binary data + * @param {Object} [extensions] An object containing the negotiated extensions + * @param {Boolean} [isServer=false] Specifies whether to operate in client or + * server mode + * @param {Number} [maxPayload=0] The maximum allowed message length + */ + constructor(binaryType, extensions, isServer, maxPayload) { + super(); + + this._binaryType = binaryType || BINARY_TYPES[0]; + this[kWebSocket] = undefined; + this._extensions = extensions || {}; + this._isServer = !!isServer; + this._maxPayload = maxPayload | 0; + + this._bufferedBytes = 0; + this._buffers = []; + + this._compressed = false; + this._payloadLength = 0; + this._mask = undefined; + this._fragmented = 0; + this._masked = false; + this._fin = false; + this._opcode = 0; + + this._totalPayloadLength = 0; + this._messageLength = 0; + this._fragments = []; + + this._state = GET_INFO; + this._loop = false; + } + + /** + * Implements `Writable.prototype._write()`. + * + * @param {Buffer} chunk The chunk of data to write + * @param {String} encoding The character encoding of `chunk` + * @param {Function} cb Callback + * @private + */ + _write(chunk, encoding, cb) { + if (this._opcode === 0x08 && this._state == GET_INFO) return cb(); + + this._bufferedBytes += chunk.length; + this._buffers.push(chunk); + this.startLoop(cb); + } + + /** + * Consumes `n` bytes from the buffered data. + * + * @param {Number} n The number of bytes to consume + * @return {Buffer} The consumed bytes + * @private + */ + consume(n) { + this._bufferedBytes -= n; + + if (n === this._buffers[0].length) return this._buffers.shift(); + + if (n < this._buffers[0].length) { + const buf = this._buffers[0]; + this._buffers[0] = buf.slice(n); + return buf.slice(0, n); + } + + const dst = Buffer.allocUnsafe(n); + + do { + const buf = this._buffers[0]; + const offset = dst.length - n; + + if (n >= buf.length) { + dst.set(this._buffers.shift(), offset); + } else { + dst.set(new Uint8Array(buf.buffer, buf.byteOffset, n), offset); + this._buffers[0] = buf.slice(n); + } + + n -= buf.length; + } while (n > 0); + + return dst; + } + + /** + * Starts the parsing loop. + * + * @param {Function} cb Callback + * @private + */ + startLoop(cb) { + let err; + this._loop = true; + + do { + switch (this._state) { + case GET_INFO: + err = this.getInfo(); + break; + case GET_PAYLOAD_LENGTH_16: + err = this.getPayloadLength16(); + break; + case GET_PAYLOAD_LENGTH_64: + err = this.getPayloadLength64(); + break; + case GET_MASK: + this.getMask(); + break; + case GET_DATA: + err = this.getData(cb); + break; + default: + // `INFLATING` + this._loop = false; + return; + } + } while (this._loop); + + cb(err); + } + + /** + * Reads the first two bytes of a frame. + * + * @return {(RangeError|undefined)} A possible error + * @private + */ + getInfo() { + if (this._bufferedBytes < 2) { + this._loop = false; + return; + } + + const buf = this.consume(2); + + if ((buf[0] & 0x30) !== 0x00) { + this._loop = false; + return error(RangeError, 'RSV2 and RSV3 must be clear', true, 1002); + } + + const compressed = (buf[0] & 0x40) === 0x40; + + if (compressed && !this._extensions[PerMessageDeflate.extensionName]) { + this._loop = false; + return error(RangeError, 'RSV1 must be clear', true, 1002); + } + + this._fin = (buf[0] & 0x80) === 0x80; + this._opcode = buf[0] & 0x0f; + this._payloadLength = buf[1] & 0x7f; + + if (this._opcode === 0x00) { + if (compressed) { + this._loop = false; + return error(RangeError, 'RSV1 must be clear', true, 1002); + } + + if (!this._fragmented) { + this._loop = false; + return error(RangeError, 'invalid opcode 0', true, 1002); + } + + this._opcode = this._fragmented; + } else if (this._opcode === 0x01 || this._opcode === 0x02) { + if (this._fragmented) { + this._loop = false; + return error(RangeError, `invalid opcode ${this._opcode}`, true, 1002); + } + + this._compressed = compressed; + } else if (this._opcode > 0x07 && this._opcode < 0x0b) { + if (!this._fin) { + this._loop = false; + return error(RangeError, 'FIN must be set', true, 1002); + } + + if (compressed) { + this._loop = false; + return error(RangeError, 'RSV1 must be clear', true, 1002); + } + + if (this._payloadLength > 0x7d) { + this._loop = false; + return error( + RangeError, + `invalid payload length ${this._payloadLength}`, + true, + 1002 + ); + } + } else { + this._loop = false; + return error(RangeError, `invalid opcode ${this._opcode}`, true, 1002); + } + + if (!this._fin && !this._fragmented) this._fragmented = this._opcode; + this._masked = (buf[1] & 0x80) === 0x80; + + if (this._isServer) { + if (!this._masked) { + this._loop = false; + return error(RangeError, 'MASK must be set', true, 1002); + } + } else if (this._masked) { + this._loop = false; + return error(RangeError, 'MASK must be clear', true, 1002); + } + + if (this._payloadLength === 126) this._state = GET_PAYLOAD_LENGTH_16; + else if (this._payloadLength === 127) this._state = GET_PAYLOAD_LENGTH_64; + else return this.haveLength(); + } + + /** + * Gets extended payload length (7+16). + * + * @return {(RangeError|undefined)} A possible error + * @private + */ + getPayloadLength16() { + if (this._bufferedBytes < 2) { + this._loop = false; + return; + } + + this._payloadLength = this.consume(2).readUInt16BE(0); + return this.haveLength(); + } + + /** + * Gets extended payload length (7+64). + * + * @return {(RangeError|undefined)} A possible error + * @private + */ + getPayloadLength64() { + if (this._bufferedBytes < 8) { + this._loop = false; + return; + } + + const buf = this.consume(8); + const num = buf.readUInt32BE(0); + + // + // The maximum safe integer in JavaScript is 2^53 - 1. An error is returned + // if payload length is greater than this number. + // + if (num > Math.pow(2, 53 - 32) - 1) { + this._loop = false; + return error( + RangeError, + 'Unsupported WebSocket frame: payload length > 2^53 - 1', + false, + 1009 + ); + } + + this._payloadLength = num * Math.pow(2, 32) + buf.readUInt32BE(4); + return this.haveLength(); + } + + /** + * Payload length has been read. + * + * @return {(RangeError|undefined)} A possible error + * @private + */ + haveLength() { + if (this._payloadLength && this._opcode < 0x08) { + this._totalPayloadLength += this._payloadLength; + if (this._totalPayloadLength > this._maxPayload && this._maxPayload > 0) { + this._loop = false; + return error(RangeError, 'Max payload size exceeded', false, 1009); + } + } + + if (this._masked) this._state = GET_MASK; + else this._state = GET_DATA; + } + + /** + * Reads mask bytes. + * + * @private + */ + getMask() { + if (this._bufferedBytes < 4) { + this._loop = false; + return; + } + + this._mask = this.consume(4); + this._state = GET_DATA; + } + + /** + * Reads data bytes. + * + * @param {Function} cb Callback + * @return {(Error|RangeError|undefined)} A possible error + * @private + */ + getData(cb) { + let data = EMPTY_BUFFER; + + if (this._payloadLength) { + if (this._bufferedBytes < this._payloadLength) { + this._loop = false; + return; + } + + data = this.consume(this._payloadLength); + if (this._masked) unmask(data, this._mask); + } + + if (this._opcode > 0x07) return this.controlMessage(data); + + if (this._compressed) { + this._state = INFLATING; + this.decompress(data, cb); + return; + } + + if (data.length) { + // + // This message is not compressed so its lenght is the sum of the payload + // length of all fragments. + // + this._messageLength = this._totalPayloadLength; + this._fragments.push(data); + } + + return this.dataMessage(); + } + + /** + * Decompresses data. + * + * @param {Buffer} data Compressed data + * @param {Function} cb Callback + * @private + */ + decompress(data, cb) { + const perMessageDeflate = this._extensions[PerMessageDeflate.extensionName]; + + perMessageDeflate.decompress(data, this._fin, (err, buf) => { + if (err) return cb(err); + + if (buf.length) { + this._messageLength += buf.length; + if (this._messageLength > this._maxPayload && this._maxPayload > 0) { + return cb( + error(RangeError, 'Max payload size exceeded', false, 1009) + ); + } + + this._fragments.push(buf); + } + + const er = this.dataMessage(); + if (er) return cb(er); + + this.startLoop(cb); + }); + } + + /** + * Handles a data message. + * + * @return {(Error|undefined)} A possible error + * @private + */ + dataMessage() { + if (this._fin) { + const messageLength = this._messageLength; + const fragments = this._fragments; + + this._totalPayloadLength = 0; + this._messageLength = 0; + this._fragmented = 0; + this._fragments = []; + + if (this._opcode === 2) { + let data; + + if (this._binaryType === 'nodebuffer') { + data = concat(fragments, messageLength); + } else if (this._binaryType === 'arraybuffer') { + data = toArrayBuffer(concat(fragments, messageLength)); + } else { + data = fragments; + } + + this.emit('message', data); + } else { + const buf = concat(fragments, messageLength); + + if (!isValidUTF8(buf)) { + this._loop = false; + return error(Error, 'invalid UTF-8 sequence', true, 1007); + } + + this.emit('message', buf.toString()); + } + } + + this._state = GET_INFO; + } + + /** + * Handles a control message. + * + * @param {Buffer} data Data to handle + * @return {(Error|RangeError|undefined)} A possible error + * @private + */ + controlMessage(data) { + if (this._opcode === 0x08) { + this._loop = false; + + if (data.length === 0) { + this.emit('conclude', 1005, ''); + this.end(); + } else if (data.length === 1) { + return error(RangeError, 'invalid payload length 1', true, 1002); + } else { + const code = data.readUInt16BE(0); + + if (!isValidStatusCode(code)) { + return error(RangeError, `invalid status code ${code}`, true, 1002); + } + + const buf = data.slice(2); + + if (!isValidUTF8(buf)) { + return error(Error, 'invalid UTF-8 sequence', true, 1007); + } + + this.emit('conclude', code, buf.toString()); + this.end(); + } + } else if (this._opcode === 0x09) { + this.emit('ping', data); + } else { + this.emit('pong', data); + } + + this._state = GET_INFO; + } +} + +module.exports = Receiver; + +/** + * Builds an error object. + * + * @param {(Error|RangeError)} ErrorCtor The error constructor + * @param {String} message The error message + * @param {Boolean} prefix Specifies whether or not to add a default prefix to + * `message` + * @param {Number} statusCode The status code + * @return {(Error|RangeError)} The error + * @private + */ +function error(ErrorCtor, message, prefix, statusCode) { + const err = new ErrorCtor( + prefix ? `Invalid WebSocket frame: ${message}` : message + ); + + Error.captureStackTrace(err, error); + err[kStatusCode] = statusCode; + return err; +} diff --git a/mon/node_modules/ws/lib/sender.js b/mon/node_modules/ws/lib/sender.js new file mode 100644 index 00000000..ad71e195 --- /dev/null +++ b/mon/node_modules/ws/lib/sender.js @@ -0,0 +1,405 @@ +'use strict'; + +const { randomFillSync } = require('crypto'); + +const PerMessageDeflate = require('./permessage-deflate'); +const { EMPTY_BUFFER } = require('./constants'); +const { isValidStatusCode } = require('./validation'); +const { mask: applyMask, toBuffer } = require('./buffer-util'); + +const mask = Buffer.alloc(4); + +/** + * HyBi Sender implementation. + */ +class Sender { + /** + * Creates a Sender instance. + * + * @param {net.Socket} socket The connection socket + * @param {Object} [extensions] An object containing the negotiated extensions + */ + constructor(socket, extensions) { + this._extensions = extensions || {}; + this._socket = socket; + + this._firstFragment = true; + this._compress = false; + + this._bufferedBytes = 0; + this._deflating = false; + this._queue = []; + } + + /** + * Frames a piece of data according to the HyBi WebSocket protocol. + * + * @param {Buffer} data The data to frame + * @param {Object} options Options object + * @param {Number} options.opcode The opcode + * @param {Boolean} [options.readOnly=false] Specifies whether `data` can be + * modified + * @param {Boolean} [options.fin=false] Specifies whether or not to set the + * FIN bit + * @param {Boolean} [options.mask=false] Specifies whether or not to mask + * `data` + * @param {Boolean} [options.rsv1=false] Specifies whether or not to set the + * RSV1 bit + * @return {Buffer[]} The framed data as a list of `Buffer` instances + * @public + */ + static frame(data, options) { + const merge = options.mask && options.readOnly; + let offset = options.mask ? 6 : 2; + let payloadLength = data.length; + + if (data.length >= 65536) { + offset += 8; + payloadLength = 127; + } else if (data.length > 125) { + offset += 2; + payloadLength = 126; + } + + const target = Buffer.allocUnsafe(merge ? data.length + offset : offset); + + target[0] = options.fin ? options.opcode | 0x80 : options.opcode; + if (options.rsv1) target[0] |= 0x40; + + target[1] = payloadLength; + + if (payloadLength === 126) { + target.writeUInt16BE(data.length, 2); + } else if (payloadLength === 127) { + target.writeUInt32BE(0, 2); + target.writeUInt32BE(data.length, 6); + } + + if (!options.mask) return [target, data]; + + randomFillSync(mask, 0, 4); + + target[1] |= 0x80; + target[offset - 4] = mask[0]; + target[offset - 3] = mask[1]; + target[offset - 2] = mask[2]; + target[offset - 1] = mask[3]; + + if (merge) { + applyMask(data, mask, target, offset, data.length); + return [target]; + } + + applyMask(data, mask, data, 0, data.length); + return [target, data]; + } + + /** + * Sends a close message to the other peer. + * + * @param {Number} [code] The status code component of the body + * @param {String} [data] The message component of the body + * @param {Boolean} [mask=false] Specifies whether or not to mask the message + * @param {Function} [cb] Callback + * @public + */ + close(code, data, mask, cb) { + let buf; + + if (code === undefined) { + buf = EMPTY_BUFFER; + } else if (typeof code !== 'number' || !isValidStatusCode(code)) { + throw new TypeError('First argument must be a valid error code number'); + } else if (data === undefined || data === '') { + buf = Buffer.allocUnsafe(2); + buf.writeUInt16BE(code, 0); + } else { + const length = Buffer.byteLength(data); + + if (length > 123) { + throw new RangeError('The message must not be greater than 123 bytes'); + } + + buf = Buffer.allocUnsafe(2 + length); + buf.writeUInt16BE(code, 0); + buf.write(data, 2); + } + + if (this._deflating) { + this.enqueue([this.doClose, buf, mask, cb]); + } else { + this.doClose(buf, mask, cb); + } + } + + /** + * Frames and sends a close message. + * + * @param {Buffer} data The message to send + * @param {Boolean} [mask=false] Specifies whether or not to mask `data` + * @param {Function} [cb] Callback + * @private + */ + doClose(data, mask, cb) { + this.sendFrame( + Sender.frame(data, { + fin: true, + rsv1: false, + opcode: 0x08, + mask, + readOnly: false + }), + cb + ); + } + + /** + * Sends a ping message to the other peer. + * + * @param {*} data The message to send + * @param {Boolean} [mask=false] Specifies whether or not to mask `data` + * @param {Function} [cb] Callback + * @public + */ + ping(data, mask, cb) { + const buf = toBuffer(data); + + if (buf.length > 125) { + throw new RangeError('The data size must not be greater than 125 bytes'); + } + + if (this._deflating) { + this.enqueue([this.doPing, buf, mask, toBuffer.readOnly, cb]); + } else { + this.doPing(buf, mask, toBuffer.readOnly, cb); + } + } + + /** + * Frames and sends a ping message. + * + * @param {Buffer} data The message to send + * @param {Boolean} [mask=false] Specifies whether or not to mask `data` + * @param {Boolean} [readOnly=false] Specifies whether `data` can be modified + * @param {Function} [cb] Callback + * @private + */ + doPing(data, mask, readOnly, cb) { + this.sendFrame( + Sender.frame(data, { + fin: true, + rsv1: false, + opcode: 0x09, + mask, + readOnly + }), + cb + ); + } + + /** + * Sends a pong message to the other peer. + * + * @param {*} data The message to send + * @param {Boolean} [mask=false] Specifies whether or not to mask `data` + * @param {Function} [cb] Callback + * @public + */ + pong(data, mask, cb) { + const buf = toBuffer(data); + + if (buf.length > 125) { + throw new RangeError('The data size must not be greater than 125 bytes'); + } + + if (this._deflating) { + this.enqueue([this.doPong, buf, mask, toBuffer.readOnly, cb]); + } else { + this.doPong(buf, mask, toBuffer.readOnly, cb); + } + } + + /** + * Frames and sends a pong message. + * + * @param {Buffer} data The message to send + * @param {Boolean} [mask=false] Specifies whether or not to mask `data` + * @param {Boolean} [readOnly=false] Specifies whether `data` can be modified + * @param {Function} [cb] Callback + * @private + */ + doPong(data, mask, readOnly, cb) { + this.sendFrame( + Sender.frame(data, { + fin: true, + rsv1: false, + opcode: 0x0a, + mask, + readOnly + }), + cb + ); + } + + /** + * Sends a data message to the other peer. + * + * @param {*} data The message to send + * @param {Object} options Options object + * @param {Boolean} [options.compress=false] Specifies whether or not to + * compress `data` + * @param {Boolean} [options.binary=false] Specifies whether `data` is binary + * or text + * @param {Boolean} [options.fin=false] Specifies whether the fragment is the + * last one + * @param {Boolean} [options.mask=false] Specifies whether or not to mask + * `data` + * @param {Function} [cb] Callback + * @public + */ + send(data, options, cb) { + const buf = toBuffer(data); + const perMessageDeflate = this._extensions[PerMessageDeflate.extensionName]; + let opcode = options.binary ? 2 : 1; + let rsv1 = options.compress; + + if (this._firstFragment) { + this._firstFragment = false; + if (rsv1 && perMessageDeflate) { + rsv1 = buf.length >= perMessageDeflate._threshold; + } + this._compress = rsv1; + } else { + rsv1 = false; + opcode = 0; + } + + if (options.fin) this._firstFragment = true; + + if (perMessageDeflate) { + const opts = { + fin: options.fin, + rsv1, + opcode, + mask: options.mask, + readOnly: toBuffer.readOnly + }; + + if (this._deflating) { + this.enqueue([this.dispatch, buf, this._compress, opts, cb]); + } else { + this.dispatch(buf, this._compress, opts, cb); + } + } else { + this.sendFrame( + Sender.frame(buf, { + fin: options.fin, + rsv1: false, + opcode, + mask: options.mask, + readOnly: toBuffer.readOnly + }), + cb + ); + } + } + + /** + * Dispatches a data message. + * + * @param {Buffer} data The message to send + * @param {Boolean} [compress=false] Specifies whether or not to compress + * `data` + * @param {Object} options Options object + * @param {Number} options.opcode The opcode + * @param {Boolean} [options.readOnly=false] Specifies whether `data` can be + * modified + * @param {Boolean} [options.fin=false] Specifies whether or not to set the + * FIN bit + * @param {Boolean} [options.mask=false] Specifies whether or not to mask + * `data` + * @param {Boolean} [options.rsv1=false] Specifies whether or not to set the + * RSV1 bit + * @param {Function} [cb] Callback + * @private + */ + dispatch(data, compress, options, cb) { + if (!compress) { + this.sendFrame(Sender.frame(data, options), cb); + return; + } + + const perMessageDeflate = this._extensions[PerMessageDeflate.extensionName]; + + this._bufferedBytes += data.length; + this._deflating = true; + perMessageDeflate.compress(data, options.fin, (_, buf) => { + if (this._socket.destroyed) { + const err = new Error( + 'The socket was closed while data was being compressed' + ); + + if (typeof cb === 'function') cb(err); + + for (let i = 0; i < this._queue.length; i++) { + const callback = this._queue[i][4]; + + if (typeof callback === 'function') callback(err); + } + + return; + } + + this._bufferedBytes -= data.length; + this._deflating = false; + options.readOnly = false; + this.sendFrame(Sender.frame(buf, options), cb); + this.dequeue(); + }); + } + + /** + * Executes queued send operations. + * + * @private + */ + dequeue() { + while (!this._deflating && this._queue.length) { + const params = this._queue.shift(); + + this._bufferedBytes -= params[1].length; + Reflect.apply(params[0], this, params.slice(1)); + } + } + + /** + * Enqueues a send operation. + * + * @param {Array} params Send operation parameters. + * @private + */ + enqueue(params) { + this._bufferedBytes += params[1].length; + this._queue.push(params); + } + + /** + * Sends a frame. + * + * @param {Buffer[]} list The frame to send + * @param {Function} [cb] Callback + * @private + */ + sendFrame(list, cb) { + if (list.length === 2) { + this._socket.cork(); + this._socket.write(list[0]); + this._socket.write(list[1], cb); + this._socket.uncork(); + } else { + this._socket.write(list[0], cb); + } + } +} + +module.exports = Sender; diff --git a/mon/node_modules/ws/lib/stream.js b/mon/node_modules/ws/lib/stream.js new file mode 100644 index 00000000..604cf366 --- /dev/null +++ b/mon/node_modules/ws/lib/stream.js @@ -0,0 +1,165 @@ +'use strict'; + +const { Duplex } = require('stream'); + +/** + * Emits the `'close'` event on a stream. + * + * @param {stream.Duplex} The stream. + * @private + */ +function emitClose(stream) { + stream.emit('close'); +} + +/** + * The listener of the `'end'` event. + * + * @private + */ +function duplexOnEnd() { + if (!this.destroyed && this._writableState.finished) { + this.destroy(); + } +} + +/** + * The listener of the `'error'` event. + * + * @param {Error} err The error + * @private + */ +function duplexOnError(err) { + this.removeListener('error', duplexOnError); + this.destroy(); + if (this.listenerCount('error') === 0) { + // Do not suppress the throwing behavior. + this.emit('error', err); + } +} + +/** + * Wraps a `WebSocket` in a duplex stream. + * + * @param {WebSocket} ws The `WebSocket` to wrap + * @param {Object} [options] The options for the `Duplex` constructor + * @return {stream.Duplex} The duplex stream + * @public + */ +function createWebSocketStream(ws, options) { + let resumeOnReceiverDrain = true; + + function receiverOnDrain() { + if (resumeOnReceiverDrain) ws._socket.resume(); + } + + if (ws.readyState === ws.CONNECTING) { + ws.once('open', function open() { + ws._receiver.removeAllListeners('drain'); + ws._receiver.on('drain', receiverOnDrain); + }); + } else { + ws._receiver.removeAllListeners('drain'); + ws._receiver.on('drain', receiverOnDrain); + } + + const duplex = new Duplex({ + ...options, + autoDestroy: false, + emitClose: false, + objectMode: false, + writableObjectMode: false + }); + + ws.on('message', function message(msg) { + if (!duplex.push(msg)) { + resumeOnReceiverDrain = false; + ws._socket.pause(); + } + }); + + ws.once('error', function error(err) { + if (duplex.destroyed) return; + + duplex.destroy(err); + }); + + ws.once('close', function close() { + if (duplex.destroyed) return; + + duplex.push(null); + }); + + duplex._destroy = function (err, callback) { + if (ws.readyState === ws.CLOSED) { + callback(err); + process.nextTick(emitClose, duplex); + return; + } + + let called = false; + + ws.once('error', function error(err) { + called = true; + callback(err); + }); + + ws.once('close', function close() { + if (!called) callback(err); + process.nextTick(emitClose, duplex); + }); + ws.terminate(); + }; + + duplex._final = function (callback) { + if (ws.readyState === ws.CONNECTING) { + ws.once('open', function open() { + duplex._final(callback); + }); + return; + } + + // If the value of the `_socket` property is `null` it means that `ws` is a + // client websocket and the handshake failed. In fact, when this happens, a + // socket is never assigned to the websocket. Wait for the `'error'` event + // that will be emitted by the websocket. + if (ws._socket === null) return; + + if (ws._socket._writableState.finished) { + callback(); + if (duplex._readableState.endEmitted) duplex.destroy(); + } else { + ws._socket.once('finish', function finish() { + // `duplex` is not destroyed here because the `'end'` event will be + // emitted on `duplex` after this `'finish'` event. The EOF signaling + // `null` chunk is, in fact, pushed when the websocket emits `'close'`. + callback(); + }); + ws.close(); + } + }; + + duplex._read = function () { + if (ws.readyState === ws.OPEN && !resumeOnReceiverDrain) { + resumeOnReceiverDrain = true; + if (!ws._receiver._writableState.needDrain) ws._socket.resume(); + } + }; + + duplex._write = function (chunk, encoding, callback) { + if (ws.readyState === ws.CONNECTING) { + ws.once('open', function open() { + duplex._write(chunk, encoding, callback); + }); + return; + } + + ws.send(chunk, callback); + }; + + duplex.on('end', duplexOnEnd); + duplex.on('error', duplexOnError); + return duplex; +} + +module.exports = createWebSocketStream; diff --git a/mon/node_modules/ws/lib/validation.js b/mon/node_modules/ws/lib/validation.js new file mode 100644 index 00000000..32db5a57 --- /dev/null +++ b/mon/node_modules/ws/lib/validation.js @@ -0,0 +1,30 @@ +'use strict'; + +try { + const isValidUTF8 = require('utf-8-validate'); + + exports.isValidUTF8 = + typeof isValidUTF8 === 'object' + ? isValidUTF8.Validation.isValidUTF8 // utf-8-validate@<3.0.0 + : isValidUTF8; +} catch (e) /* istanbul ignore next */ { + exports.isValidUTF8 = () => true; +} + +/** + * Checks if a status code is allowed in a close frame. + * + * @param {Number} code The status code + * @return {Boolean} `true` if the status code is valid, else `false` + * @public + */ +exports.isValidStatusCode = (code) => { + return ( + (code >= 1000 && + code <= 1014 && + code !== 1004 && + code !== 1005 && + code !== 1006) || + (code >= 3000 && code <= 4999) + ); +}; diff --git a/mon/node_modules/ws/lib/websocket-server.js b/mon/node_modules/ws/lib/websocket-server.js new file mode 100644 index 00000000..b99ad050 --- /dev/null +++ b/mon/node_modules/ws/lib/websocket-server.js @@ -0,0 +1,406 @@ +'use strict'; + +const EventEmitter = require('events'); +const { createHash } = require('crypto'); +const { createServer, STATUS_CODES } = require('http'); + +const PerMessageDeflate = require('./permessage-deflate'); +const WebSocket = require('./websocket'); +const { format, parse } = require('./extension'); +const { GUID, kWebSocket } = require('./constants'); + +const keyRegex = /^[+/0-9A-Za-z]{22}==$/; + +/** + * Class representing a WebSocket server. + * + * @extends EventEmitter + */ +class WebSocketServer extends EventEmitter { + /** + * Create a `WebSocketServer` instance. + * + * @param {Object} options Configuration options + * @param {Number} [options.backlog=511] The maximum length of the queue of + * pending connections + * @param {Boolean} [options.clientTracking=true] Specifies whether or not to + * track clients + * @param {Function} [options.handleProtocols] A hook to handle protocols + * @param {String} [options.host] The hostname where to bind the server + * @param {Number} [options.maxPayload=104857600] The maximum allowed message + * size + * @param {Boolean} [options.noServer=false] Enable no server mode + * @param {String} [options.path] Accept only connections matching this path + * @param {(Boolean|Object)} [options.perMessageDeflate=false] Enable/disable + * permessage-deflate + * @param {Number} [options.port] The port where to bind the server + * @param {http.Server} [options.server] A pre-created HTTP/S server to use + * @param {Function} [options.verifyClient] A hook to reject connections + * @param {Function} [callback] A listener for the `listening` event + */ + constructor(options, callback) { + super(); + + options = { + maxPayload: 100 * 1024 * 1024, + perMessageDeflate: false, + handleProtocols: null, + clientTracking: true, + verifyClient: null, + noServer: false, + backlog: null, // use default (511 as implemented in net.js) + server: null, + host: null, + path: null, + port: null, + ...options + }; + + if (options.port == null && !options.server && !options.noServer) { + throw new TypeError( + 'One of the "port", "server", or "noServer" options must be specified' + ); + } + + if (options.port != null) { + this._server = createServer((req, res) => { + const body = STATUS_CODES[426]; + + res.writeHead(426, { + 'Content-Length': body.length, + 'Content-Type': 'text/plain' + }); + res.end(body); + }); + this._server.listen( + options.port, + options.host, + options.backlog, + callback + ); + } else if (options.server) { + this._server = options.server; + } + + if (this._server) { + const emitConnection = this.emit.bind(this, 'connection'); + + this._removeListeners = addListeners(this._server, { + listening: this.emit.bind(this, 'listening'), + error: this.emit.bind(this, 'error'), + upgrade: (req, socket, head) => { + this.handleUpgrade(req, socket, head, emitConnection); + } + }); + } + + if (options.perMessageDeflate === true) options.perMessageDeflate = {}; + if (options.clientTracking) this.clients = new Set(); + this.options = options; + } + + /** + * Returns the bound address, the address family name, and port of the server + * as reported by the operating system if listening on an IP socket. + * If the server is listening on a pipe or UNIX domain socket, the name is + * returned as a string. + * + * @return {(Object|String|null)} The address of the server + * @public + */ + address() { + if (this.options.noServer) { + throw new Error('The server is operating in "noServer" mode'); + } + + if (!this._server) return null; + return this._server.address(); + } + + /** + * Close the server. + * + * @param {Function} [cb] Callback + * @public + */ + close(cb) { + if (cb) this.once('close', cb); + + // + // Terminate all associated clients. + // + if (this.clients) { + for (const client of this.clients) client.terminate(); + } + + const server = this._server; + + if (server) { + this._removeListeners(); + this._removeListeners = this._server = null; + + // + // Close the http server if it was internally created. + // + if (this.options.port != null) { + server.close(() => this.emit('close')); + return; + } + } + + process.nextTick(emitClose, this); + } + + /** + * See if a given request should be handled by this server instance. + * + * @param {http.IncomingMessage} req Request object to inspect + * @return {Boolean} `true` if the request is valid, else `false` + * @public + */ + shouldHandle(req) { + if (this.options.path) { + const index = req.url.indexOf('?'); + const pathname = index !== -1 ? req.url.slice(0, index) : req.url; + + if (pathname !== this.options.path) return false; + } + + return true; + } + + /** + * Handle a HTTP Upgrade request. + * + * @param {http.IncomingMessage} req The request object + * @param {net.Socket} socket The network socket between the server and client + * @param {Buffer} head The first packet of the upgraded stream + * @param {Function} cb Callback + * @public + */ + handleUpgrade(req, socket, head, cb) { + socket.on('error', socketOnError); + + const key = + req.headers['sec-websocket-key'] !== undefined + ? req.headers['sec-websocket-key'].trim() + : false; + const version = +req.headers['sec-websocket-version']; + const extensions = {}; + + if ( + req.method !== 'GET' || + req.headers.upgrade.toLowerCase() !== 'websocket' || + !key || + !keyRegex.test(key) || + (version !== 8 && version !== 13) || + !this.shouldHandle(req) + ) { + return abortHandshake(socket, 400); + } + + if (this.options.perMessageDeflate) { + const perMessageDeflate = new PerMessageDeflate( + this.options.perMessageDeflate, + true, + this.options.maxPayload + ); + + try { + const offers = parse(req.headers['sec-websocket-extensions']); + + if (offers[PerMessageDeflate.extensionName]) { + perMessageDeflate.accept(offers[PerMessageDeflate.extensionName]); + extensions[PerMessageDeflate.extensionName] = perMessageDeflate; + } + } catch (err) { + return abortHandshake(socket, 400); + } + } + + // + // Optionally call external client verification handler. + // + if (this.options.verifyClient) { + const info = { + origin: + req.headers[`${version === 8 ? 'sec-websocket-origin' : 'origin'}`], + secure: !!(req.socket.authorized || req.socket.encrypted), + req + }; + + if (this.options.verifyClient.length === 2) { + this.options.verifyClient(info, (verified, code, message, headers) => { + if (!verified) { + return abortHandshake(socket, code || 401, message, headers); + } + + this.completeUpgrade(key, extensions, req, socket, head, cb); + }); + return; + } + + if (!this.options.verifyClient(info)) return abortHandshake(socket, 401); + } + + this.completeUpgrade(key, extensions, req, socket, head, cb); + } + + /** + * Upgrade the connection to WebSocket. + * + * @param {String} key The value of the `Sec-WebSocket-Key` header + * @param {Object} extensions The accepted extensions + * @param {http.IncomingMessage} req The request object + * @param {net.Socket} socket The network socket between the server and client + * @param {Buffer} head The first packet of the upgraded stream + * @param {Function} cb Callback + * @throws {Error} If called more than once with the same socket + * @private + */ + completeUpgrade(key, extensions, req, socket, head, cb) { + // + // Destroy the socket if the client has already sent a FIN packet. + // + if (!socket.readable || !socket.writable) return socket.destroy(); + + if (socket[kWebSocket]) { + throw new Error( + 'server.handleUpgrade() was called more than once with the same ' + + 'socket, possibly due to a misconfiguration' + ); + } + + const digest = createHash('sha1') + .update(key + GUID) + .digest('base64'); + + const headers = [ + 'HTTP/1.1 101 Switching Protocols', + 'Upgrade: websocket', + 'Connection: Upgrade', + `Sec-WebSocket-Accept: ${digest}` + ]; + + const ws = new WebSocket(null); + let protocol = req.headers['sec-websocket-protocol']; + + if (protocol) { + protocol = protocol.trim().split(/ *, */); + + // + // Optionally call external protocol selection handler. + // + if (this.options.handleProtocols) { + protocol = this.options.handleProtocols(protocol, req); + } else { + protocol = protocol[0]; + } + + if (protocol) { + headers.push(`Sec-WebSocket-Protocol: ${protocol}`); + ws._protocol = protocol; + } + } + + if (extensions[PerMessageDeflate.extensionName]) { + const params = extensions[PerMessageDeflate.extensionName].params; + const value = format({ + [PerMessageDeflate.extensionName]: [params] + }); + headers.push(`Sec-WebSocket-Extensions: ${value}`); + ws._extensions = extensions; + } + + // + // Allow external modification/inspection of handshake headers. + // + this.emit('headers', headers, req); + + socket.write(headers.concat('\r\n').join('\r\n')); + socket.removeListener('error', socketOnError); + + ws.setSocket(socket, head, this.options.maxPayload); + + if (this.clients) { + this.clients.add(ws); + ws.on('close', () => this.clients.delete(ws)); + } + + cb(ws, req); + } +} + +module.exports = WebSocketServer; + +/** + * Add event listeners on an `EventEmitter` using a map of + * pairs. + * + * @param {EventEmitter} server The event emitter + * @param {Object.} map The listeners to add + * @return {Function} A function that will remove the added listeners when + * called + * @private + */ +function addListeners(server, map) { + for (const event of Object.keys(map)) server.on(event, map[event]); + + return function removeListeners() { + for (const event of Object.keys(map)) { + server.removeListener(event, map[event]); + } + }; +} + +/** + * Emit a `'close'` event on an `EventEmitter`. + * + * @param {EventEmitter} server The event emitter + * @private + */ +function emitClose(server) { + server.emit('close'); +} + +/** + * Handle premature socket errors. + * + * @private + */ +function socketOnError() { + this.destroy(); +} + +/** + * Close the connection when preconditions are not fulfilled. + * + * @param {net.Socket} socket The socket of the upgrade request + * @param {Number} code The HTTP response status code + * @param {String} [message] The HTTP response body + * @param {Object} [headers] Additional HTTP response headers + * @private + */ +function abortHandshake(socket, code, message, headers) { + if (socket.writable) { + message = message || STATUS_CODES[code]; + headers = { + Connection: 'close', + 'Content-Type': 'text/html', + 'Content-Length': Buffer.byteLength(message), + ...headers + }; + + socket.write( + `HTTP/1.1 ${code} ${STATUS_CODES[code]}\r\n` + + Object.keys(headers) + .map((h) => `${h}: ${headers[h]}`) + .join('\r\n') + + '\r\n\r\n' + + message + ); + } + + socket.removeListener('error', socketOnError); + socket.destroy(); +} diff --git a/mon/node_modules/ws/lib/websocket.js b/mon/node_modules/ws/lib/websocket.js new file mode 100644 index 00000000..0e2a83d0 --- /dev/null +++ b/mon/node_modules/ws/lib/websocket.js @@ -0,0 +1,933 @@ +'use strict'; + +const EventEmitter = require('events'); +const https = require('https'); +const http = require('http'); +const net = require('net'); +const tls = require('tls'); +const { randomBytes, createHash } = require('crypto'); +const { URL } = require('url'); + +const PerMessageDeflate = require('./permessage-deflate'); +const Receiver = require('./receiver'); +const Sender = require('./sender'); +const { + BINARY_TYPES, + EMPTY_BUFFER, + GUID, + kStatusCode, + kWebSocket, + NOOP +} = require('./constants'); +const { addEventListener, removeEventListener } = require('./event-target'); +const { format, parse } = require('./extension'); +const { toBuffer } = require('./buffer-util'); + +const readyStates = ['CONNECTING', 'OPEN', 'CLOSING', 'CLOSED']; +const protocolVersions = [8, 13]; +const closeTimeout = 30 * 1000; + +/** + * Class representing a WebSocket. + * + * @extends EventEmitter + */ +class WebSocket extends EventEmitter { + /** + * Create a new `WebSocket`. + * + * @param {(String|url.URL)} address The URL to which to connect + * @param {(String|String[])} [protocols] The subprotocols + * @param {Object} [options] Connection options + */ + constructor(address, protocols, options) { + super(); + + this._binaryType = BINARY_TYPES[0]; + this._closeCode = 1006; + this._closeFrameReceived = false; + this._closeFrameSent = false; + this._closeMessage = ''; + this._closeTimer = null; + this._extensions = {}; + this._protocol = ''; + this._readyState = WebSocket.CONNECTING; + this._receiver = null; + this._sender = null; + this._socket = null; + + if (address !== null) { + this._bufferedAmount = 0; + this._isServer = false; + this._redirects = 0; + + if (Array.isArray(protocols)) { + protocols = protocols.join(', '); + } else if (typeof protocols === 'object' && protocols !== null) { + options = protocols; + protocols = undefined; + } + + initAsClient(this, address, protocols, options); + } else { + this._isServer = true; + } + } + + /** + * This deviates from the WHATWG interface since ws doesn't support the + * required default "blob" type (instead we define a custom "nodebuffer" + * type). + * + * @type {String} + */ + get binaryType() { + return this._binaryType; + } + + set binaryType(type) { + if (!BINARY_TYPES.includes(type)) return; + + this._binaryType = type; + + // + // Allow to change `binaryType` on the fly. + // + if (this._receiver) this._receiver._binaryType = type; + } + + /** + * @type {Number} + */ + get bufferedAmount() { + if (!this._socket) return this._bufferedAmount; + + return this._socket._writableState.length + this._sender._bufferedBytes; + } + + /** + * @type {String} + */ + get extensions() { + return Object.keys(this._extensions).join(); + } + + /** + * @type {String} + */ + get protocol() { + return this._protocol; + } + + /** + * @type {Number} + */ + get readyState() { + return this._readyState; + } + + /** + * @type {String} + */ + get url() { + return this._url; + } + + /** + * Set up the socket and the internal resources. + * + * @param {net.Socket} socket The network socket between the server and client + * @param {Buffer} head The first packet of the upgraded stream + * @param {Number} [maxPayload=0] The maximum allowed message size + * @private + */ + setSocket(socket, head, maxPayload) { + const receiver = new Receiver( + this.binaryType, + this._extensions, + this._isServer, + maxPayload + ); + + this._sender = new Sender(socket, this._extensions); + this._receiver = receiver; + this._socket = socket; + + receiver[kWebSocket] = this; + socket[kWebSocket] = this; + + receiver.on('conclude', receiverOnConclude); + receiver.on('drain', receiverOnDrain); + receiver.on('error', receiverOnError); + receiver.on('message', receiverOnMessage); + receiver.on('ping', receiverOnPing); + receiver.on('pong', receiverOnPong); + + socket.setTimeout(0); + socket.setNoDelay(); + + if (head.length > 0) socket.unshift(head); + + socket.on('close', socketOnClose); + socket.on('data', socketOnData); + socket.on('end', socketOnEnd); + socket.on('error', socketOnError); + + this._readyState = WebSocket.OPEN; + this.emit('open'); + } + + /** + * Emit the `'close'` event. + * + * @private + */ + emitClose() { + if (!this._socket) { + this._readyState = WebSocket.CLOSED; + this.emit('close', this._closeCode, this._closeMessage); + return; + } + + if (this._extensions[PerMessageDeflate.extensionName]) { + this._extensions[PerMessageDeflate.extensionName].cleanup(); + } + + this._receiver.removeAllListeners(); + this._readyState = WebSocket.CLOSED; + this.emit('close', this._closeCode, this._closeMessage); + } + + /** + * Start a closing handshake. + * + * +----------+ +-----------+ +----------+ + * - - -|ws.close()|-->|close frame|-->|ws.close()|- - - + * | +----------+ +-----------+ +----------+ | + * +----------+ +-----------+ | + * CLOSING |ws.close()|<--|close frame|<--+-----+ CLOSING + * +----------+ +-----------+ | + * | | | +---+ | + * +------------------------+-->|fin| - - - - + * | +---+ | +---+ + * - - - - -|fin|<---------------------+ + * +---+ + * + * @param {Number} [code] Status code explaining why the connection is closing + * @param {String} [data] A string explaining why the connection is closing + * @public + */ + close(code, data) { + if (this.readyState === WebSocket.CLOSED) return; + if (this.readyState === WebSocket.CONNECTING) { + const msg = 'WebSocket was closed before the connection was established'; + return abortHandshake(this, this._req, msg); + } + + if (this.readyState === WebSocket.CLOSING) { + if (this._closeFrameSent && this._closeFrameReceived) this._socket.end(); + return; + } + + this._readyState = WebSocket.CLOSING; + this._sender.close(code, data, !this._isServer, (err) => { + // + // This error is handled by the `'error'` listener on the socket. We only + // want to know if the close frame has been sent here. + // + if (err) return; + + this._closeFrameSent = true; + if (this._closeFrameReceived) this._socket.end(); + }); + + // + // Specify a timeout for the closing handshake to complete. + // + this._closeTimer = setTimeout( + this._socket.destroy.bind(this._socket), + closeTimeout + ); + } + + /** + * Send a ping. + * + * @param {*} [data] The data to send + * @param {Boolean} [mask] Indicates whether or not to mask `data` + * @param {Function} [cb] Callback which is executed when the ping is sent + * @public + */ + ping(data, mask, cb) { + if (this.readyState === WebSocket.CONNECTING) { + throw new Error('WebSocket is not open: readyState 0 (CONNECTING)'); + } + + if (typeof data === 'function') { + cb = data; + data = mask = undefined; + } else if (typeof mask === 'function') { + cb = mask; + mask = undefined; + } + + if (typeof data === 'number') data = data.toString(); + + if (this.readyState !== WebSocket.OPEN) { + sendAfterClose(this, data, cb); + return; + } + + if (mask === undefined) mask = !this._isServer; + this._sender.ping(data || EMPTY_BUFFER, mask, cb); + } + + /** + * Send a pong. + * + * @param {*} [data] The data to send + * @param {Boolean} [mask] Indicates whether or not to mask `data` + * @param {Function} [cb] Callback which is executed when the pong is sent + * @public + */ + pong(data, mask, cb) { + if (this.readyState === WebSocket.CONNECTING) { + throw new Error('WebSocket is not open: readyState 0 (CONNECTING)'); + } + + if (typeof data === 'function') { + cb = data; + data = mask = undefined; + } else if (typeof mask === 'function') { + cb = mask; + mask = undefined; + } + + if (typeof data === 'number') data = data.toString(); + + if (this.readyState !== WebSocket.OPEN) { + sendAfterClose(this, data, cb); + return; + } + + if (mask === undefined) mask = !this._isServer; + this._sender.pong(data || EMPTY_BUFFER, mask, cb); + } + + /** + * Send a data message. + * + * @param {*} data The message to send + * @param {Object} [options] Options object + * @param {Boolean} [options.compress] Specifies whether or not to compress + * `data` + * @param {Boolean} [options.binary] Specifies whether `data` is binary or + * text + * @param {Boolean} [options.fin=true] Specifies whether the fragment is the + * last one + * @param {Boolean} [options.mask] Specifies whether or not to mask `data` + * @param {Function} [cb] Callback which is executed when data is written out + * @public + */ + send(data, options, cb) { + if (this.readyState === WebSocket.CONNECTING) { + throw new Error('WebSocket is not open: readyState 0 (CONNECTING)'); + } + + if (typeof options === 'function') { + cb = options; + options = {}; + } + + if (typeof data === 'number') data = data.toString(); + + if (this.readyState !== WebSocket.OPEN) { + sendAfterClose(this, data, cb); + return; + } + + const opts = { + binary: typeof data !== 'string', + mask: !this._isServer, + compress: true, + fin: true, + ...options + }; + + if (!this._extensions[PerMessageDeflate.extensionName]) { + opts.compress = false; + } + + this._sender.send(data || EMPTY_BUFFER, opts, cb); + } + + /** + * Forcibly close the connection. + * + * @public + */ + terminate() { + if (this.readyState === WebSocket.CLOSED) return; + if (this.readyState === WebSocket.CONNECTING) { + const msg = 'WebSocket was closed before the connection was established'; + return abortHandshake(this, this._req, msg); + } + + if (this._socket) { + this._readyState = WebSocket.CLOSING; + this._socket.destroy(); + } + } +} + +readyStates.forEach((readyState, i) => { + const descriptor = { enumerable: true, value: i }; + + Object.defineProperty(WebSocket.prototype, readyState, descriptor); + Object.defineProperty(WebSocket, readyState, descriptor); +}); + +[ + 'binaryType', + 'bufferedAmount', + 'extensions', + 'protocol', + 'readyState', + 'url' +].forEach((property) => { + Object.defineProperty(WebSocket.prototype, property, { enumerable: true }); +}); + +// +// Add the `onopen`, `onerror`, `onclose`, and `onmessage` attributes. +// See https://html.spec.whatwg.org/multipage/comms.html#the-websocket-interface +// +['open', 'error', 'close', 'message'].forEach((method) => { + Object.defineProperty(WebSocket.prototype, `on${method}`, { + configurable: true, + enumerable: true, + /** + * Return the listener of the event. + * + * @return {(Function|undefined)} The event listener or `undefined` + * @public + */ + get() { + const listeners = this.listeners(method); + for (let i = 0; i < listeners.length; i++) { + if (listeners[i]._listener) return listeners[i]._listener; + } + + return undefined; + }, + /** + * Add a listener for the event. + * + * @param {Function} listener The listener to add + * @public + */ + set(listener) { + const listeners = this.listeners(method); + for (let i = 0; i < listeners.length; i++) { + // + // Remove only the listeners added via `addEventListener`. + // + if (listeners[i]._listener) this.removeListener(method, listeners[i]); + } + this.addEventListener(method, listener); + } + }); +}); + +WebSocket.prototype.addEventListener = addEventListener; +WebSocket.prototype.removeEventListener = removeEventListener; + +module.exports = WebSocket; + +/** + * Initialize a WebSocket client. + * + * @param {WebSocket} websocket The client to initialize + * @param {(String|url.URL)} address The URL to which to connect + * @param {String} [protocols] The subprotocols + * @param {Object} [options] Connection options + * @param {(Boolean|Object)} [options.perMessageDeflate=true] Enable/disable + * permessage-deflate + * @param {Number} [options.handshakeTimeout] Timeout in milliseconds for the + * handshake request + * @param {Number} [options.protocolVersion=13] Value of the + * `Sec-WebSocket-Version` header + * @param {String} [options.origin] Value of the `Origin` or + * `Sec-WebSocket-Origin` header + * @param {Number} [options.maxPayload=104857600] The maximum allowed message + * size + * @param {Boolean} [options.followRedirects=false] Whether or not to follow + * redirects + * @param {Number} [options.maxRedirects=10] The maximum number of redirects + * allowed + * @private + */ +function initAsClient(websocket, address, protocols, options) { + const opts = { + protocolVersion: protocolVersions[1], + maxPayload: 100 * 1024 * 1024, + perMessageDeflate: true, + followRedirects: false, + maxRedirects: 10, + ...options, + createConnection: undefined, + socketPath: undefined, + hostname: undefined, + protocol: undefined, + timeout: undefined, + method: undefined, + host: undefined, + path: undefined, + port: undefined + }; + + if (!protocolVersions.includes(opts.protocolVersion)) { + throw new RangeError( + `Unsupported protocol version: ${opts.protocolVersion} ` + + `(supported versions: ${protocolVersions.join(', ')})` + ); + } + + let parsedUrl; + + if (address instanceof URL) { + parsedUrl = address; + websocket._url = address.href; + } else { + parsedUrl = new URL(address); + websocket._url = address; + } + + const isUnixSocket = parsedUrl.protocol === 'ws+unix:'; + + if (!parsedUrl.host && (!isUnixSocket || !parsedUrl.pathname)) { + throw new Error(`Invalid URL: ${websocket.url}`); + } + + const isSecure = + parsedUrl.protocol === 'wss:' || parsedUrl.protocol === 'https:'; + const defaultPort = isSecure ? 443 : 80; + const key = randomBytes(16).toString('base64'); + const get = isSecure ? https.get : http.get; + let perMessageDeflate; + + opts.createConnection = isSecure ? tlsConnect : netConnect; + opts.defaultPort = opts.defaultPort || defaultPort; + opts.port = parsedUrl.port || defaultPort; + opts.host = parsedUrl.hostname.startsWith('[') + ? parsedUrl.hostname.slice(1, -1) + : parsedUrl.hostname; + opts.headers = { + 'Sec-WebSocket-Version': opts.protocolVersion, + 'Sec-WebSocket-Key': key, + Connection: 'Upgrade', + Upgrade: 'websocket', + ...opts.headers + }; + opts.path = parsedUrl.pathname + parsedUrl.search; + opts.timeout = opts.handshakeTimeout; + + if (opts.perMessageDeflate) { + perMessageDeflate = new PerMessageDeflate( + opts.perMessageDeflate !== true ? opts.perMessageDeflate : {}, + false, + opts.maxPayload + ); + opts.headers['Sec-WebSocket-Extensions'] = format({ + [PerMessageDeflate.extensionName]: perMessageDeflate.offer() + }); + } + if (protocols) { + opts.headers['Sec-WebSocket-Protocol'] = protocols; + } + if (opts.origin) { + if (opts.protocolVersion < 13) { + opts.headers['Sec-WebSocket-Origin'] = opts.origin; + } else { + opts.headers.Origin = opts.origin; + } + } + if (parsedUrl.username || parsedUrl.password) { + opts.auth = `${parsedUrl.username}:${parsedUrl.password}`; + } + + if (isUnixSocket) { + const parts = opts.path.split(':'); + + opts.socketPath = parts[0]; + opts.path = parts[1]; + } + + let req = (websocket._req = get(opts)); + + if (opts.timeout) { + req.on('timeout', () => { + abortHandshake(websocket, req, 'Opening handshake has timed out'); + }); + } + + req.on('error', (err) => { + if (req === null || req.aborted) return; + + req = websocket._req = null; + websocket._readyState = WebSocket.CLOSING; + websocket.emit('error', err); + websocket.emitClose(); + }); + + req.on('response', (res) => { + const location = res.headers.location; + const statusCode = res.statusCode; + + if ( + location && + opts.followRedirects && + statusCode >= 300 && + statusCode < 400 + ) { + if (++websocket._redirects > opts.maxRedirects) { + abortHandshake(websocket, req, 'Maximum redirects exceeded'); + return; + } + + req.abort(); + + const addr = new URL(location, address); + + initAsClient(websocket, addr, protocols, options); + } else if (!websocket.emit('unexpected-response', req, res)) { + abortHandshake( + websocket, + req, + `Unexpected server response: ${res.statusCode}` + ); + } + }); + + req.on('upgrade', (res, socket, head) => { + websocket.emit('upgrade', res); + + // + // The user may have closed the connection from a listener of the `upgrade` + // event. + // + if (websocket.readyState !== WebSocket.CONNECTING) return; + + req = websocket._req = null; + + const digest = createHash('sha1') + .update(key + GUID) + .digest('base64'); + + if (res.headers['sec-websocket-accept'] !== digest) { + abortHandshake(websocket, socket, 'Invalid Sec-WebSocket-Accept header'); + return; + } + + const serverProt = res.headers['sec-websocket-protocol']; + const protList = (protocols || '').split(/, */); + let protError; + + if (!protocols && serverProt) { + protError = 'Server sent a subprotocol but none was requested'; + } else if (protocols && !serverProt) { + protError = 'Server sent no subprotocol'; + } else if (serverProt && !protList.includes(serverProt)) { + protError = 'Server sent an invalid subprotocol'; + } + + if (protError) { + abortHandshake(websocket, socket, protError); + return; + } + + if (serverProt) websocket._protocol = serverProt; + + if (perMessageDeflate) { + try { + const extensions = parse(res.headers['sec-websocket-extensions']); + + if (extensions[PerMessageDeflate.extensionName]) { + perMessageDeflate.accept(extensions[PerMessageDeflate.extensionName]); + websocket._extensions[ + PerMessageDeflate.extensionName + ] = perMessageDeflate; + } + } catch (err) { + abortHandshake( + websocket, + socket, + 'Invalid Sec-WebSocket-Extensions header' + ); + return; + } + } + + websocket.setSocket(socket, head, opts.maxPayload); + }); +} + +/** + * Create a `net.Socket` and initiate a connection. + * + * @param {Object} options Connection options + * @return {net.Socket} The newly created socket used to start the connection + * @private + */ +function netConnect(options) { + options.path = options.socketPath; + return net.connect(options); +} + +/** + * Create a `tls.TLSSocket` and initiate a connection. + * + * @param {Object} options Connection options + * @return {tls.TLSSocket} The newly created socket used to start the connection + * @private + */ +function tlsConnect(options) { + options.path = undefined; + + if (!options.servername && options.servername !== '') { + options.servername = net.isIP(options.host) ? '' : options.host; + } + + return tls.connect(options); +} + +/** + * Abort the handshake and emit an error. + * + * @param {WebSocket} websocket The WebSocket instance + * @param {(http.ClientRequest|net.Socket)} stream The request to abort or the + * socket to destroy + * @param {String} message The error message + * @private + */ +function abortHandshake(websocket, stream, message) { + websocket._readyState = WebSocket.CLOSING; + + const err = new Error(message); + Error.captureStackTrace(err, abortHandshake); + + if (stream.setHeader) { + stream.abort(); + stream.once('abort', websocket.emitClose.bind(websocket)); + websocket.emit('error', err); + } else { + stream.destroy(err); + stream.once('error', websocket.emit.bind(websocket, 'error')); + stream.once('close', websocket.emitClose.bind(websocket)); + } +} + +/** + * Handle cases where the `ping()`, `pong()`, or `send()` methods are called + * when the `readyState` attribute is `CLOSING` or `CLOSED`. + * + * @param {WebSocket} websocket The WebSocket instance + * @param {*} [data] The data to send + * @param {Function} [cb] Callback + * @private + */ +function sendAfterClose(websocket, data, cb) { + if (data) { + const length = toBuffer(data).length; + + // + // The `_bufferedAmount` property is used only when the peer is a client and + // the opening handshake fails. Under these circumstances, in fact, the + // `setSocket()` method is not called, so the `_socket` and `_sender` + // properties are set to `null`. + // + if (websocket._socket) websocket._sender._bufferedBytes += length; + else websocket._bufferedAmount += length; + } + + if (cb) { + const err = new Error( + `WebSocket is not open: readyState ${websocket.readyState} ` + + `(${readyStates[websocket.readyState]})` + ); + cb(err); + } +} + +/** + * The listener of the `Receiver` `'conclude'` event. + * + * @param {Number} code The status code + * @param {String} reason The reason for closing + * @private + */ +function receiverOnConclude(code, reason) { + const websocket = this[kWebSocket]; + + websocket._socket.removeListener('data', socketOnData); + websocket._socket.resume(); + + websocket._closeFrameReceived = true; + websocket._closeMessage = reason; + websocket._closeCode = code; + + if (code === 1005) websocket.close(); + else websocket.close(code, reason); +} + +/** + * The listener of the `Receiver` `'drain'` event. + * + * @private + */ +function receiverOnDrain() { + this[kWebSocket]._socket.resume(); +} + +/** + * The listener of the `Receiver` `'error'` event. + * + * @param {(RangeError|Error)} err The emitted error + * @private + */ +function receiverOnError(err) { + const websocket = this[kWebSocket]; + + websocket._socket.removeListener('data', socketOnData); + + websocket._readyState = WebSocket.CLOSING; + websocket._closeCode = err[kStatusCode]; + websocket.emit('error', err); + websocket._socket.destroy(); +} + +/** + * The listener of the `Receiver` `'finish'` event. + * + * @private + */ +function receiverOnFinish() { + this[kWebSocket].emitClose(); +} + +/** + * The listener of the `Receiver` `'message'` event. + * + * @param {(String|Buffer|ArrayBuffer|Buffer[])} data The message + * @private + */ +function receiverOnMessage(data) { + this[kWebSocket].emit('message', data); +} + +/** + * The listener of the `Receiver` `'ping'` event. + * + * @param {Buffer} data The data included in the ping frame + * @private + */ +function receiverOnPing(data) { + const websocket = this[kWebSocket]; + + websocket.pong(data, !websocket._isServer, NOOP); + websocket.emit('ping', data); +} + +/** + * The listener of the `Receiver` `'pong'` event. + * + * @param {Buffer} data The data included in the pong frame + * @private + */ +function receiverOnPong(data) { + this[kWebSocket].emit('pong', data); +} + +/** + * The listener of the `net.Socket` `'close'` event. + * + * @private + */ +function socketOnClose() { + const websocket = this[kWebSocket]; + + this.removeListener('close', socketOnClose); + this.removeListener('end', socketOnEnd); + + websocket._readyState = WebSocket.CLOSING; + + // + // The close frame might not have been received or the `'end'` event emitted, + // for example, if the socket was destroyed due to an error. Ensure that the + // `receiver` stream is closed after writing any remaining buffered data to + // it. If the readable side of the socket is in flowing mode then there is no + // buffered data as everything has been already written and `readable.read()` + // will return `null`. If instead, the socket is paused, any possible buffered + // data will be read as a single chunk and emitted synchronously in a single + // `'data'` event. + // + websocket._socket.read(); + websocket._receiver.end(); + + this.removeListener('data', socketOnData); + this[kWebSocket] = undefined; + + clearTimeout(websocket._closeTimer); + + if ( + websocket._receiver._writableState.finished || + websocket._receiver._writableState.errorEmitted + ) { + websocket.emitClose(); + } else { + websocket._receiver.on('error', receiverOnFinish); + websocket._receiver.on('finish', receiverOnFinish); + } +} + +/** + * The listener of the `net.Socket` `'data'` event. + * + * @param {Buffer} chunk A chunk of data + * @private + */ +function socketOnData(chunk) { + if (!this[kWebSocket]._receiver.write(chunk)) { + this.pause(); + } +} + +/** + * The listener of the `net.Socket` `'end'` event. + * + * @private + */ +function socketOnEnd() { + const websocket = this[kWebSocket]; + + websocket._readyState = WebSocket.CLOSING; + websocket._receiver.end(); + this.end(); +} + +/** + * The listener of the `net.Socket` `'error'` event. + * + * @private + */ +function socketOnError() { + const websocket = this[kWebSocket]; + + this.removeListener('error', socketOnError); + this.on('error', NOOP); + + if (websocket) { + websocket._readyState = WebSocket.CLOSING; + this.destroy(); + } +} diff --git a/mon/node_modules/ws/package.json b/mon/node_modules/ws/package.json new file mode 100644 index 00000000..5bb97b43 --- /dev/null +++ b/mon/node_modules/ws/package.json @@ -0,0 +1,90 @@ +{ + "_from": "ws@^7.2.5", + "_id": "ws@7.4.4", + "_inBundle": false, + "_integrity": "sha512-Qm8k8ojNQIMx7S+Zp8u/uHOx7Qazv3Yv4q68MiWWWOJhiwG5W3x7iqmRtJo8xxrciZUY4vRxUTJCKuRnF28ZZw==", + "_location": "/ws", + "_phantomChildren": {}, + "_requested": { + "type": "range", + "registry": true, + "raw": "ws@^7.2.5", + "name": "ws", + "escapedName": "ws", + "rawSpec": "^7.2.5", + "saveSpec": null, + "fetchSpec": "^7.2.5" + }, + "_requiredBy": [ + "/" + ], + "_resolved": "https://registry.npmjs.org/ws/-/ws-7.4.4.tgz", + "_shasum": "383bc9742cb202292c9077ceab6f6047b17f2d59", + "_spec": "ws@^7.2.5", + "_where": "/home/shaba/RPM/git/vitastor/mon", + "author": { + "name": "Einar Otto Stangvik", + "email": "einaros@gmail.com", + "url": "http://2x.io" + }, + "browser": "browser.js", + "bugs": { + "url": "https://github.com/websockets/ws/issues" + }, + "bundleDependencies": false, + "deprecated": false, + "description": "Simple to use, blazing fast and thoroughly tested websocket client and server for Node.js", + "devDependencies": { + "benchmark": "^2.1.4", + "bufferutil": "^4.0.1", + "eslint": "^7.2.0", + "eslint-config-prettier": "^8.1.0", + "eslint-plugin-prettier": "^3.0.1", + "mocha": "^7.0.0", + "nyc": "^15.0.0", + "prettier": "^2.0.5", + "utf-8-validate": "^5.0.2" + }, + "engines": { + "node": ">=8.3.0" + }, + "files": [ + "browser.js", + "index.js", + "lib/*.js" + ], + "homepage": "https://github.com/websockets/ws", + "keywords": [ + "HyBi", + "Push", + "RFC-6455", + "WebSocket", + "WebSockets", + "real-time" + ], + "license": "MIT", + "main": "index.js", + "name": "ws", + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + }, + "repository": { + "type": "git", + "url": "git+https://github.com/websockets/ws.git" + }, + "scripts": { + "integration": "mocha --throw-deprecation test/*.integration.js", + "lint": "eslint --ignore-path .gitignore . && prettier --check --ignore-path .gitignore \"**/*.{json,md,yaml,yml}\"", + "test": "nyc --reporter=lcov --reporter=text mocha --throw-deprecation test/*.test.js" + }, + "version": "7.4.4" +} -- 2.30.2 From 66004b2c8e1facd98d1e30cfdc870f4add34e131 Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Fri, 19 Mar 2021 16:27:06 +0300 Subject: [PATCH 76/91] Drop build qemu driver --- src/CMakeLists.txt | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index aa33e395..99cb527d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,7 +4,6 @@ project(vitastor) include(GNUInstallDirs) -set(QEMU_PLUGINDIR qemu CACHE STRING "QEMU plugin directory suffix (qemu-kvm on RHEL)") set(WITH_ASAN false CACHE BOOL "Build with AddressSanitizer") if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$") if(EXISTS "/etc/debian_version") @@ -36,7 +35,6 @@ string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_RELWITHDEBINFO "${CM find_package(PkgConfig) pkg_check_modules(LIBURING REQUIRED liburing) -pkg_check_modules(GLIB REQUIRED glib-2.0) include_directories( ../ @@ -123,28 +121,6 @@ add_executable(vitastor-dump-journal dump_journal.cpp crc32c.c ) -# qemu_driver.so -add_library(qemu_proxy STATIC qemu_proxy.cpp) -target_compile_options(qemu_proxy PUBLIC -fPIC) -target_include_directories(qemu_proxy PUBLIC - ../qemu/b/qemu - ../qemu/include - ${GLIB_INCLUDE_DIRS} -) -target_link_libraries(qemu_proxy - vitastor_client -) -add_library(qemu_vitastor SHARED - qemu_driver.c -) -target_link_libraries(qemu_vitastor - qemu_proxy -) -set_target_properties(qemu_vitastor PROPERTIES - PREFIX "" - OUTPUT_NAME "block-vitastor" -) - ### Test stubs # stub_osd, stub_bench, osd_test @@ -185,4 +161,3 @@ add_executable(test_allocator test_allocator.cpp allocator.cpp) install(TARGETS vitastor-osd vitastor-dump-journal vitastor-nbd vitastor-rm RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) install(TARGETS fio_vitastor fio_vitastor_blk fio_vitastor_sec vitastor_blk vitastor_client LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) -install(TARGETS qemu_vitastor LIBRARY DESTINATION /usr/${CMAKE_INSTALL_LIBDIR}/${QEMU_PLUGINDIR}) -- 2.30.2 From 278f37c00fdd62f080b67b71f299ed57525582f8 Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Fri, 19 Mar 2021 16:29:26 +0300 Subject: [PATCH 77/91] Drop build fio driver --- src/CMakeLists.txt | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 99cb527d..83bc2461 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -52,15 +52,6 @@ target_link_libraries(vitastor_blk tcmalloc_minimal ) -# libfio_vitastor_blk.so -add_library(fio_vitastor_blk SHARED - fio_engine.cpp - ../json11/json11.cpp -) -target_link_libraries(fio_vitastor_blk - vitastor_blk -) - # vitastor-osd add_executable(vitastor-osd osd_main.cpp osd.cpp osd_secondary.cpp msgr_receive.cpp msgr_send.cpp osd_peering.cpp osd_flush.cpp osd_peering_pg.cpp @@ -72,14 +63,6 @@ target_link_libraries(vitastor-osd Jerasure ) -# libfio_vitastor_sec.so -add_library(fio_vitastor_sec SHARED - fio_sec_osd.cpp - rw_blocking.cpp -) -target_link_libraries(fio_vitastor_sec - tcmalloc_minimal -) # libvitastor_client.so add_library(vitastor_client SHARED @@ -92,14 +75,6 @@ target_link_libraries(vitastor_client ${LIBURING_LIBRARIES} ) -# libfio_vitastor.so -add_library(fio_vitastor SHARED - fio_cluster.cpp -) -target_link_libraries(fio_vitastor - vitastor_client -) - # vitastor-nbd add_executable(vitastor-nbd nbd_proxy.cpp @@ -160,4 +135,4 @@ add_executable(test_allocator test_allocator.cpp allocator.cpp) ### Install install(TARGETS vitastor-osd vitastor-dump-journal vitastor-nbd vitastor-rm RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) -install(TARGETS fio_vitastor fio_vitastor_blk fio_vitastor_sec vitastor_blk vitastor_client LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) +install(TARGETS vitastor_blk vitastor_client LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) -- 2.30.2 From 17c6a562b0e8cb9c055ed80e9cb62572917db772 Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Fri, 19 Mar 2021 17:41:24 +0300 Subject: [PATCH 78/91] Add VERSION to cmake project() --- CMakeLists.txt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 83014777..fef9789c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,15 @@ cmake_minimum_required(VERSION 2.8) -project(vitastor) +set(MAJOR_VERSION "0") +set(MINOR_VERSION "5") +set(PATCH_VERSION "10") +set(VERSION_STRING "${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}") + +if (CMAKE_VERSION VERSION_LESS 3.0) + project(vitastor) +else() + cmake_policy(SET CMP0048 NEW) + project(vitastor VERSION "${VERSION_STRING}") +endif() add_subdirectory(src) -- 2.30.2 From 37d716973d75fe4c297870fd2e8cb0186f2c0272 Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Fri, 19 Mar 2021 17:42:23 +0300 Subject: [PATCH 79/91] Add SOVERSION to libs --- src/CMakeLists.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 83bc2461..53056c2c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,3 @@ -cmake_minimum_required(VERSION 2.8) - -project(vitastor) include(GNUInstallDirs) @@ -51,6 +48,7 @@ target_link_libraries(vitastor_blk ${LIBURING_LIBRARIES} tcmalloc_minimal ) +set_target_properties(vitastor_blk PROPERTIES VERSION ${VERSION_STRING} SOVERSION 0) # vitastor-osd add_executable(vitastor-osd @@ -74,6 +72,7 @@ target_link_libraries(vitastor_client tcmalloc_minimal ${LIBURING_LIBRARIES} ) +set_target_properties(vitastor_client PROPERTIES VERSION ${VERSION_STRING} SOVERSION 0) # vitastor-nbd add_executable(vitastor-nbd -- 2.30.2 From 247597e927758dd5d3312f09bb8e6c72bc0a1e47 Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Fri, 19 Mar 2021 16:17:49 +0300 Subject: [PATCH 80/91] 0.5.10-alt1 - Initial build. --- .gear/rules | 7 ++ .gear/tags/list | 3 + .gear/vitastor.spec | 171 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 .gear/rules create mode 100644 .gear/tags/list create mode 100644 .gear/vitastor.spec diff --git a/.gear/rules b/.gear/rules new file mode 100644 index 00000000..f979673e --- /dev/null +++ b/.gear/rules @@ -0,0 +1,7 @@ +spec: .gear/vitastor.spec +tar: v@version@:. +diff: v@version@:. . name=@name@-@version@.patch +tar: upstream/cpp-btree:. name=cpp-btree base= +tar: upstream/json11:. name=json11 base= +copy?: .gear/*.sysconfig +copy?: .gear/*.service diff --git a/.gear/tags/list b/.gear/tags/list new file mode 100644 index 00000000..bc19faae --- /dev/null +++ b/.gear/tags/list @@ -0,0 +1,3 @@ +7e6e1a5a825496a0b0e75705113cfda237ba3dad v0.5.10 +5dc108754ad40d3b1d024f9bd7cca0595ef1a1db upstream/cpp-btree +97f06cb20c1e136fd37d58fb40f57dd8f8a3a4a7 upstream/json11 diff --git a/.gear/vitastor.spec b/.gear/vitastor.spec new file mode 100644 index 00000000..df1d89f2 --- /dev/null +++ b/.gear/vitastor.spec @@ -0,0 +1,171 @@ + +%global _unpackaged_files_terminate_build 1 + +Name: vitastor +Version: 0.5.10 +Release: alt1 +Summary: Vitastor, a fast software-defined clustered block storage +Group: System/Base + +License: VNPL-1.1 +Url: https://vitastor.io/ +Source0: %name-%version.tar +Source2: cpp-btree.tar +Source3: json11.tar + +Patch: %name-%version.patch + +BuildRequires(pre): rpm-macros-cmake +BuildRequires: cmake gcc-c++ + +BuildRequires: pkgconfig(liburing) +BuildRequires: libgperftools-devel +BuildRequires: node >= 10 +BuildRequires: libjerasure-devel +BuildRequires: libgf-complete-devel + +%description +Vitastor is a small, simple and fast clustered block storage (storage for VM drives), +architecturally similar to Ceph which means strong consistency, primary-replication, +symmetric clustering and automatic data distribution over any number of drives of any +size with configurable redundancy (replication or erasure codes/XOR). + +%package common +Summary: Vitastor SDS Common +Group: System/Base +BuildArch: noarch + +%description common +Common utilities for Vitastor. + +%package mon +Summary: Vitastor SDS monitor service +Group: System/Base +BuildArch: noarch +Requires: node +Requires: lp_solve +Requires: %name-common = %EVR + +%description mon +Vitastor SDS monitor service. +Monitor is a separate daemon that watches cluster state and handles failures. + +%package osd +Summary: Vitastor SDS Object Storage Daemon +Group: System/Base +Requires: %name-common = %EVR + +%description osd +Vitastor SDS Object Storage Daemon is a process that stores data and serves read/write requests. + +%package nbd +Summary: Vitastor SDS NBD proxy +Group: System/Base + +%description nbd +Vitastor SDS NBD proxy for kernel mounts. + +%package -n lib%name-client +Group: System/Libraries +Summary: Vitastor SDS user-space client library +License: VNPL-1.1 OR GPL-2.0+ + +%description -n lib%name-client +Vitastor SDS user-space client library. + +%package -n lib%name-client-devel +Group: Development/C++ +Summary: Vitastor SDS headers of client library +License: VNPL-1.1 OR GPL-2.0+ + +%description -n lib%name-client-devel +This package contains libraries and headers needed to develop programs +that use Vitastor SDS client library. + +%package -n lib%name-blk +Group: System/Libraries +Summary: Vitastor SDS blk library + +%description -n lib%name-blk +Vitastor SDS blk library. + +%package -n lib%name-blk-devel +Group: Development/C++ +Summary: Vitastor SDS blk headers + +%description -n lib%name-blk-devel +This package contains libraries and headers needed to develop programs +that use Vitastor SDS blk library. + +%prep +%setup +%patch -p1 +tar -xf %SOURCE2 -C cpp-btree +tar -xf %SOURCE3 -C json11 + +%build +%cmake +%cmake_build + +%install +%cmakeinstall_std + +mkdir -p %buildroot{%_datadir,%_localstatedir}/%name +cp -r mon %buildroot%_datadir/%name + +%pre common +groupadd -r -f %name 2>/dev/null ||: +useradd -r -g %name -s /sbin/nologin -c "Vitastor daemons" -M -d %_localstatedir/%name %name 2>/dev/null ||: + +#%post mon +#%post_service vitastor-mon + +#%preun mon +#%preun_service vitastor-mon + +#%post osd +#systemctl daemon-reload ||: +#if [ "$1" -eq 1 ]; then +# systemctl -q preset vitastor-osd@\*.service vitastor-osd.target ||: +#else +# systemctl try-restart vitastor-osd.target ||: +#fi + +#%preun osd +#if [ "$1" -eq 0 ]; then +# systemctl --no-reload -q disable vitastor-osd@\*.service vitastor-osd.target ||: +# systemctl stop vitastor-osd@\*.service vitastor-osd.target ||: +#fi + +%files common +%doc README.md VNPL-1.1.txt GPL-2.0.txt +%attr(770,root,%name) %dir %_localstatedir/%name + +%files osd +%_bindir/%name-osd +# ? may be to utils package? +%_bindir/%name-dump-journal +%_bindir/%name-rm + +%files mon +%_datadir/%name + +%files nbd +%_bindir/%name-nbd + +%files -n lib%name-blk +%_libdir/lib%{name}_blk.so.* + +%files -n lib%name-blk-devel +%_libdir/lib%{name}_blk.so + +%files -n lib%name-client +%_libdir/lib%{name}_client.so.* + +%files -n lib%name-client-devel +%_libdir/lib%{name}_client.so + +%changelog +* Fri Mar 19 2021 Alexey Shabalin 0.5.10-alt1 +- Initial build. + -- 2.30.2 From dd8b12d9e42b76b23108763569443085fa5bc2ba Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Mon, 19 Apr 2021 23:08:44 +0300 Subject: [PATCH 81/91] Update VERSION --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fef9789c..1aa69555 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,8 +1,8 @@ cmake_minimum_required(VERSION 2.8) set(MAJOR_VERSION "0") -set(MINOR_VERSION "5") -set(PATCH_VERSION "10") +set(MINOR_VERSION "6") +set(PATCH_VERSION "2") set(VERSION_STRING "${MAJOR_VERSION}.${MINOR_VERSION}.${PATCH_VERSION}") if (CMAKE_VERSION VERSION_LESS 3.0) -- 2.30.2 From 8c411e38dae995333168b970d6bf6140281e9e3c Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Mon, 19 Apr 2021 23:55:12 +0300 Subject: [PATCH 82/91] Fix undefined symbol --- src/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d093b245..a03e266e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -45,6 +45,7 @@ add_library(vitastor_blk SHARED blockstore_write.cpp blockstore_sync.cpp blockstore_stable.cpp blockstore_rollback.cpp blockstore_flush.cpp crc32c.c ringloop.cpp ) target_link_libraries(vitastor_blk + vitastor_common ${LIBURING_LIBRARIES} tcmalloc_minimal ) @@ -57,7 +58,7 @@ add_library(vitastor_common STATIC http_client.cpp osd_ops.cpp pg_states.cpp timerfd_manager.cpp base64.cpp ) target_compile_options(vitastor_common PUBLIC -fPIC) - + # vitastor-osd add_executable(vitastor-osd osd_main.cpp osd.cpp osd_secondary.cpp osd_peering.cpp osd_flush.cpp osd_peering_pg.cpp -- 2.30.2 From 5dfd1cae0c11f4132d89c741b9bb4a691e4dc0af Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Tue, 20 Apr 2021 00:10:50 +0300 Subject: [PATCH 83/91] Make with verbose --- .gear/vitastor.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gear/vitastor.spec b/.gear/vitastor.spec index df1d89f2..fc065e7b 100644 --- a/.gear/vitastor.spec +++ b/.gear/vitastor.spec @@ -104,7 +104,7 @@ tar -xf %SOURCE2 -C cpp-btree tar -xf %SOURCE3 -C json11 %build -%cmake +%cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON %cmake_build %install -- 2.30.2 From 7febcabdb59688764e9a232754aaa3558e5ca085 Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Tue, 20 Apr 2021 00:24:08 +0300 Subject: [PATCH 84/91] Do not redefine FLAGS for RelWithDebInfo --- src/CMakeLists.txt | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a03e266e..7778289b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,19 +16,10 @@ if (${WITH_ASAN}) add_link_options(-fsanitize=address -fno-omit-frame-pointer) endif (${WITH_ASAN}) -set(CMAKE_BUILD_TYPE RelWithDebInfo) -string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}") -string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}") -string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") -string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}") -string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}") -string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") -string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}") -string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}") -string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}") -string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}") -string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}") -string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}") +# If the build type isn't specified, set to Relwithdebinfo as default. +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "RelWithDebInfo") +endif() find_package(PkgConfig) pkg_check_modules(LIBURING REQUIRED liburing) -- 2.30.2 From 04580ce3550699b6af644eac8058c196bc7dee85 Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Mon, 19 Apr 2021 22:07:56 +0300 Subject: [PATCH 85/91] 0.6.2-alt1 - 0.6.2 --- .gear/tags/list | 2 +- .gear/vitastor.spec | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gear/tags/list b/.gear/tags/list index bc19faae..89bc04ac 100644 --- a/.gear/tags/list +++ b/.gear/tags/list @@ -1,3 +1,3 @@ -7e6e1a5a825496a0b0e75705113cfda237ba3dad v0.5.10 5dc108754ad40d3b1d024f9bd7cca0595ef1a1db upstream/cpp-btree 97f06cb20c1e136fd37d58fb40f57dd8f8a3a4a7 upstream/json11 +715bc8d53d724176ea683f6a60d0ce6f1e582030 v0.6.2 diff --git a/.gear/vitastor.spec b/.gear/vitastor.spec index fc065e7b..16d58340 100644 --- a/.gear/vitastor.spec +++ b/.gear/vitastor.spec @@ -2,7 +2,7 @@ %global _unpackaged_files_terminate_build 1 Name: vitastor -Version: 0.5.10 +Version: 0.6.2 Release: alt1 Summary: Vitastor, a fast software-defined clustered block storage Group: System/Base @@ -138,7 +138,7 @@ useradd -r -g %name -s /sbin/nologin -c "Vitastor daemons" -M -d %_localstatedi #fi %files common -%doc README.md VNPL-1.1.txt GPL-2.0.txt +%doc README.md README-ru.md VNPL-1.1.txt GPL-2.0.txt %attr(770,root,%name) %dir %_localstatedir/%name %files osd @@ -166,6 +166,9 @@ useradd -r -g %name -s /sbin/nologin -c "Vitastor daemons" -M -d %_localstatedi %_libdir/lib%{name}_client.so %changelog +* Mon Apr 19 2021 Alexey Shabalin 0.6.2-alt1 +- 0.6.2 + * Fri Mar 19 2021 Alexey Shabalin 0.5.10-alt1 - Initial build. -- 2.30.2 From cc2763d7d855c3b2c9e38a856896169762d9776f Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Thu, 6 May 2021 00:18:39 +0300 Subject: [PATCH 86/91] gear-remotes-save --- .gear/upstream/remotes | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .gear/upstream/remotes diff --git a/.gear/upstream/remotes b/.gear/upstream/remotes new file mode 100644 index 00000000..990fa907 --- /dev/null +++ b/.gear/upstream/remotes @@ -0,0 +1,3 @@ +[remote "upstream"] + url = http://yourcmc.ru/git/vitalif/vitastor.git + fetch = +refs/heads/*:refs/remotes/upstream/* -- 2.30.2 From d37b4603ddcdfdd49518e38be2c575a376e23b8a Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Thu, 6 May 2021 00:29:19 +0300 Subject: [PATCH 87/91] 0.6.3-alt1 - 0.6.3 --- .gear/tags/list | 2 +- .gear/vitastor.spec | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.gear/tags/list b/.gear/tags/list index 89bc04ac..04e7b8bf 100644 --- a/.gear/tags/list +++ b/.gear/tags/list @@ -1,3 +1,3 @@ 5dc108754ad40d3b1d024f9bd7cca0595ef1a1db upstream/cpp-btree 97f06cb20c1e136fd37d58fb40f57dd8f8a3a4a7 upstream/json11 -715bc8d53d724176ea683f6a60d0ce6f1e582030 v0.6.2 +caf2f3c56fef211ba0b32aa136c9e5203d3bd3e9 v0.6.3 diff --git a/.gear/vitastor.spec b/.gear/vitastor.spec index 16d58340..d10692f8 100644 --- a/.gear/vitastor.spec +++ b/.gear/vitastor.spec @@ -2,7 +2,7 @@ %global _unpackaged_files_terminate_build 1 Name: vitastor -Version: 0.6.2 +Version: 0.6.3 Release: alt1 Summary: Vitastor, a fast software-defined clustered block storage Group: System/Base @@ -23,6 +23,7 @@ BuildRequires: libgperftools-devel BuildRequires: node >= 10 BuildRequires: libjerasure-devel BuildRequires: libgf-complete-devel +BuildRequires: rdma-core-devel %description Vitastor is a small, simple and fast clustered block storage (storage for VM drives), @@ -138,7 +139,7 @@ useradd -r -g %name -s /sbin/nologin -c "Vitastor daemons" -M -d %_localstatedi #fi %files common -%doc README.md README-ru.md VNPL-1.1.txt GPL-2.0.txt +%doc README.md README-ru.md VNPL-1.1.txt GPL-2.0.txt %attr(770,root,%name) %dir %_localstatedir/%name %files osd @@ -166,6 +167,9 @@ useradd -r -g %name -s /sbin/nologin -c "Vitastor daemons" -M -d %_localstatedi %_libdir/lib%{name}_client.so %changelog +* Thu May 06 2021 Alexey Shabalin 0.6.3-alt1 +- 0.6.3 + * Mon Apr 19 2021 Alexey Shabalin 0.6.2-alt1 - 0.6.2 -- 2.30.2 From 15e8595b003b479f6a3129d5adb4a96a0c9063c6 Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Wed, 19 May 2021 18:48:09 +0300 Subject: [PATCH 88/91] 0.6.4-alt1 - 0.6.4 --- .gear/tags/list | 2 +- .gear/vitastor.spec | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gear/tags/list b/.gear/tags/list index 04e7b8bf..239be695 100644 --- a/.gear/tags/list +++ b/.gear/tags/list @@ -1,3 +1,3 @@ 5dc108754ad40d3b1d024f9bd7cca0595ef1a1db upstream/cpp-btree 97f06cb20c1e136fd37d58fb40f57dd8f8a3a4a7 upstream/json11 -caf2f3c56fef211ba0b32aa136c9e5203d3bd3e9 v0.6.3 +f9fe72d40acbcba116ec844599f3da5dd806f214 v0.6.4 diff --git a/.gear/vitastor.spec b/.gear/vitastor.spec index d10692f8..58007ef6 100644 --- a/.gear/vitastor.spec +++ b/.gear/vitastor.spec @@ -2,7 +2,7 @@ %global _unpackaged_files_terminate_build 1 Name: vitastor -Version: 0.6.3 +Version: 0.6.4 Release: alt1 Summary: Vitastor, a fast software-defined clustered block storage Group: System/Base @@ -167,6 +167,9 @@ useradd -r -g %name -s /sbin/nologin -c "Vitastor daemons" -M -d %_localstatedi %_libdir/lib%{name}_client.so %changelog +* Wed May 19 2021 Alexey Shabalin 0.6.4-alt1 +- 0.6.4 + * Thu May 06 2021 Alexey Shabalin 0.6.3-alt1 - 0.6.3 -- 2.30.2 From d54138717227783d28bcdaa3824a8f7920974620 Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Fri, 2 Jul 2021 18:18:23 +0300 Subject: [PATCH 89/91] 0.6.4-alt2 - build master snapshot 30bb6026818b66bbe05bde38d70673e4633313e9 - package client header to devel package - merge blk and client devel packages --- .gear/vitastor.spec | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/.gear/vitastor.spec b/.gear/vitastor.spec index 58007ef6..c6f1c537 100644 --- a/.gear/vitastor.spec +++ b/.gear/vitastor.spec @@ -3,7 +3,7 @@ Name: vitastor Version: 0.6.4 -Release: alt1 +Release: alt2 Summary: Vitastor, a fast software-defined clustered block storage Group: System/Base @@ -74,15 +74,6 @@ License: VNPL-1.1 OR GPL-2.0+ %description -n lib%name-client Vitastor SDS user-space client library. -%package -n lib%name-client-devel -Group: Development/C++ -Summary: Vitastor SDS headers of client library -License: VNPL-1.1 OR GPL-2.0+ - -%description -n lib%name-client-devel -This package contains libraries and headers needed to develop programs -that use Vitastor SDS client library. - %package -n lib%name-blk Group: System/Libraries Summary: Vitastor SDS blk library @@ -90,13 +81,15 @@ Summary: Vitastor SDS blk library %description -n lib%name-blk Vitastor SDS blk library. -%package -n lib%name-blk-devel +%package -n lib%name-devel Group: Development/C++ -Summary: Vitastor SDS blk headers +Summary: Vitastor SDS headers of client and blk library +License: VNPL-1.1 OR GPL-2.0+ +Requires: lib%name-blk = %EVR lib%name-client = %EVR -%description -n lib%name-blk-devel +%description -n lib%name-devel This package contains libraries and headers needed to develop programs -that use Vitastor SDS blk library. +that use Vitastor SDS library. %prep %setup @@ -105,7 +98,10 @@ tar -xf %SOURCE2 -C cpp-btree tar -xf %SOURCE3 -C json11 %build -%cmake -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON +%cmake \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DWITH_QEMU=OFF \ + -DWITH_FIO=OFF %cmake_build %install @@ -157,16 +153,19 @@ useradd -r -g %name -s /sbin/nologin -c "Vitastor daemons" -M -d %_localstatedi %files -n lib%name-blk %_libdir/lib%{name}_blk.so.* -%files -n lib%name-blk-devel -%_libdir/lib%{name}_blk.so - %files -n lib%name-client %_libdir/lib%{name}_client.so.* -%files -n lib%name-client-devel -%_libdir/lib%{name}_client.so +%files -n lib%name-devel +%_libdir/*.so +%_includedir/* %changelog +* Fri Jul 02 2021 Alexey Shabalin 0.6.4-alt2 +- build master snapshot 30bb6026818b66bbe05bde38d70673e4633313e9 +- package client header to devel package +- merge blk and client devel packages + * Wed May 19 2021 Alexey Shabalin 0.6.4-alt1 - 0.6.4 -- 2.30.2 From e157e1544e73022cafbd7649d4b6642694eae5d6 Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Sun, 11 Jul 2021 00:56:07 +0300 Subject: [PATCH 90/91] 0.6.5-alt1 - 0.6.5 --- .gear/tags/list | 2 +- .gear/vitastor.spec | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gear/tags/list b/.gear/tags/list index 239be695..40c1866b 100644 --- a/.gear/tags/list +++ b/.gear/tags/list @@ -1,3 +1,3 @@ 5dc108754ad40d3b1d024f9bd7cca0595ef1a1db upstream/cpp-btree 97f06cb20c1e136fd37d58fb40f57dd8f8a3a4a7 upstream/json11 -f9fe72d40acbcba116ec844599f3da5dd806f214 v0.6.4 +cb282d25e07b25ac15fc0ac31ee04f983175ff11 v0.6.5 diff --git a/.gear/vitastor.spec b/.gear/vitastor.spec index c6f1c537..145348ab 100644 --- a/.gear/vitastor.spec +++ b/.gear/vitastor.spec @@ -2,8 +2,8 @@ %global _unpackaged_files_terminate_build 1 Name: vitastor -Version: 0.6.4 -Release: alt2 +Version: 0.6.5 +Release: alt1 Summary: Vitastor, a fast software-defined clustered block storage Group: System/Base @@ -161,6 +161,9 @@ useradd -r -g %name -s /sbin/nologin -c "Vitastor daemons" -M -d %_localstatedi %_includedir/* %changelog +* Sun Jul 11 2021 Alexey Shabalin 0.6.5-alt1 +- 0.6.5 + * Fri Jul 02 2021 Alexey Shabalin 0.6.4-alt2 - build master snapshot 30bb6026818b66bbe05bde38d70673e4633313e9 - package client header to devel package -- 2.30.2 From c6b967ce6b5750882970aa421cbe192576763ba6 Mon Sep 17 00:00:00 2001 From: Alexey Shabalin Date: Sun, 11 Jul 2021 20:34:41 +0300 Subject: [PATCH 91/91] Fix patch to use "git am" --- patches/libvirt-7.5-vitastor.diff | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/patches/libvirt-7.5-vitastor.diff b/patches/libvirt-7.5-vitastor.diff index 620fbed0..393ba88b 100644 --- a/patches/libvirt-7.5-vitastor.diff +++ b/patches/libvirt-7.5-vitastor.diff @@ -1,8 +1,8 @@ -commit c6e1958a1b4974828e8e5852beb252ce6594e670 +From commit c6e1958a1b4974828e8e5852beb252ce6594e670 +From: Vitaliy Filippov Author: Vitaliy Filippov -Date: Mon Jun 28 01:20:19 2021 +0300 - - Add Vitastor support +Date: Mon Jun 28 01:20:19 2021 +0300 +Subject: Add Vitastor support diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng index 5ea14b6..a9df168 100644 -- 2.30.2