From 1a14ff58c9c40b13df8cda431fd2b35ca079b5d4 Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Mon, 12 Dec 2011 13:46:12 -0800 Subject: [PATCH 01/74] Initial checkin of C++ btree code, has some useless google3-specific bits to be removed and needs a Makefile. --- Makefile | 75 ++ benchmarks.awk | 27 + btree.h | 2420 +++++++++++++++++++++++++++++++++++++++++ btree_bench.cc | 483 ++++++++ btree_container.h | 325 ++++++ btree_map.h | 122 +++ btree_nc.cc | 50 + btree_nc_test.py | 83 ++ btree_printer.py | 125 +++ btree_printer_test.py | 92 ++ btree_set.h | 113 ++ btree_test.cc | 167 +++ btree_test.h | 930 ++++++++++++++++ btree_test_flags.cc | 9 + btree_test_program.cc | 64 ++ safe_btree.h | 379 +++++++ safe_btree_map.h | 70 ++ safe_btree_set.h | 68 ++ safe_btree_test.cc | 67 ++ 19 files changed, 5669 insertions(+) create mode 100644 Makefile create mode 100755 benchmarks.awk create mode 100644 btree.h create mode 100644 btree_bench.cc create mode 100644 btree_container.h create mode 100644 btree_map.h create mode 100644 btree_nc.cc create mode 100755 btree_nc_test.py create mode 100755 btree_printer.py create mode 100755 btree_printer_test.py create mode 100644 btree_set.h create mode 100644 btree_test.cc create mode 100644 btree_test.h create mode 100644 btree_test_flags.cc create mode 100644 btree_test_program.cc create mode 100644 safe_btree.h create mode 100644 safe_btree_map.h create mode 100644 safe_btree_set.h create mode 100644 safe_btree_test.cc diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..c58946f9 --- /dev/null +++ b/Makefile @@ -0,0 +1,75 @@ +# -*- mode: python; -*- + +# This should be a Makefile, but it's not. + +# cc_library(name = "btree", +# srcs = [ "btree.h", +# "btree_container.h", +# "btree_map.h", +# "btree_set.h", +# "safe_btree.h", +# "safe_btree_map.h", +# "safe_btree_set.h" ], +# deps = [ "//strings", +# "//strings:cord" ]) + +# cc_library(name = "btree_test_flags", +# srcs = [ "btree_test_flags.cc" ], +# deps = [ "//base" ]) + +# cc_binary(name = "btree_bench", +# srcs = [ "btree_bench.cc" ], +# deps = [ ":btree", +# ":btree_test_flags", +# "//testing/base", +# "//util/random" ]) + +# cc_test(name = "btree_test", +# srcs = [ "btree_test.cc", ], +# deps = [ ":btree", +# ":btree_test_flags", +# "//base:heapcheck", +# "//testing/base", +# "//util/random", +# ], +# size = "large") + +# cc_test(name = "safe_btree_test", +# srcs = [ "safe_btree_test.cc", ], +# deps = [ ":btree", +# ":btree_test_flags", +# "//base:heapcheck", +# "//testing/base", +# "//util/random", +# ], +# size = "large") + +# cc_fake_binary(name = "btree_nc", +# srcs = [ "btree_nc.cc" ], +# deps = [ ":btree" ], +# legacy = 0) + +# py_test(name = "btree_nc_test", +# srcs = [ "btree_nc_test.py" ], +# deps = [ "//pyglib", +# "//testing/pybase" ], +# data = [ "btree_nc" ], +# size = "large") + +# cc_binary(name = "btree_test_program", +# srcs = [ "btree_test_program.cc" ], +# deps = [ ":btree", +# "//devtools/gdb/component:gdb_test_utils" ], +# testonly = 1) + +# # This test will only actually test the pretty-printing code if it's +# # compiled with debug information (blaze build -c dbg). The default +# # mode, fastbuild, will pass but will not catch any regressions! +# py_test(name = "btree_printer_test", +# size = "large", +# srcs = [ "btree_printer_test.py", +# "btree_printer.py" ], +# deps = [ "//devtools/gdb/component:gdbpy", +# "//testing/pybase", +# "//testing/gdb:gdb_script_test_util", +# ":btree_test_program" ]) diff --git a/benchmarks.awk b/benchmarks.awk new file mode 100755 index 00000000..d5d4aeac --- /dev/null +++ b/benchmarks.awk @@ -0,0 +1,27 @@ +#!/usr/bin/gawk -f + +/^Run on/ { + print $0; + printf "%-25s %5s %-20s\n", + "Benchmark", "STL(ns)", "B-Tree(ns) @ " + printf "--------------------------------------------------------\n"; +} + +/^BM_/ { + split($1, name, "_"); + if (name[2] == "stl") { + stl = $3; + stl_bytes = $5 + printf "%-25s %5d ", name[1] "_" name[3] "_" name[4] "_" name[5], stl; + fflush(); + } else if (name[2] == "btree") { + btree = $3 + btree_size = name[3] + btree_bytes = $5 + printf "%5d %+7.2f%% <%3d>", btree, 100.0 * (stl - btree) / stl, btree_size; + printf " [%4.1f, %4.1f]\n", stl_bytes, btree_bytes; + fflush(); + } else { + printf "ERROR: %s unrecognized\n", $1 + } +} diff --git a/btree.h b/btree.h new file mode 100644 index 00000000..b9fd04e2 --- /dev/null +++ b/btree.h @@ -0,0 +1,2420 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) +// +// A btree implementation of the STL set and map interfaces. A btree is both +// smaller and faster than STL set/map. The red-black tree implementation of +// STL set/map has an overhead of 3 pointers (left, right and parent) plus the +// node color information for each stored value. So a set consumes 20 +// bytes for each value stored. This btree implementation stores multiple +// values on fixed size nodes (usually 256 bytes) and doesn't store child +// pointers for leaf nodes. The result is that a btree_set may use much +// less memory per stored value. For the random insertion benchmark in +// btree_test.cc, a btree_set with node-size of 256 uses 4.9 bytes per +// stored value. +// +// The packing of multiple values on to each node of a btree has another effect +// besides better space utilization: better cache locality due to fewer cache +// lines being accessed. Better cache locality translates into faster +// operations. +// +// CAVEATS +// +// Insertions and deletions on a btree can cause splitting, merging or +// rebalancing of btree nodes. And even without these operations, insertions +// and deletions on a btree will move values around within a node. In both +// cases, the result is that insertions and deletions can invalidate iterators +// pointing to values other than the one being inserted/deleted. This is +// notably different from STL set/map which takes care to not invalidate +// iterators on insert/erase except, of course, for iterators pointing to the +// value being erased. A partial workaround when erasing is available: +// erase() returns an iterator pointing to the item just after the one that was +// erased (or end() if none exists). See also safe_btree. + +// PERFORMANCE +// +// btree_bench --benchmarks=. 2>&1 | ./benchmarks.awk +// +// NOTE(pmattis): My warpstation (pmattis-warp.nyc) often produces slower +// results when running the benchmarks on CPUs 0 and 1 vs CPUs 2 and 3. To get +// consistent benchmark results, I run "taskset 0xc " to run the +// benchmark on CPUs 2 and 3. +// +// Run on pmattis-warp.nyc (4 X 2200 MHz CPUs); 2010/03/04-15:23:06 +// Benchmark STL(ns) B-Tree(ns) @ +// -------------------------------------------------------- +// BM_set_int32_insert 1516 608 +59.89% <256> [40.0, 5.2] +// BM_set_int32_lookup 1160 414 +64.31% <256> [40.0, 5.2] +// BM_set_int32_fulllookup 960 410 +57.29% <256> [40.0, 4.4] +// BM_set_int32_delete 1741 528 +69.67% <256> [40.0, 5.2] +// BM_set_int32_queueaddrem 3078 1046 +66.02% <256> [40.0, 5.5] +// BM_set_int32_mixedaddrem 3600 1384 +61.56% <256> [40.0, 5.3] +// BM_set_int32_fifo 227 113 +50.22% <256> [40.0, 4.4] +// BM_set_int32_fwditer 158 26 +83.54% <256> [40.0, 5.2] +// BM_map_int32_insert 1551 636 +58.99% <256> [48.0, 10.5] +// BM_map_int32_lookup 1200 508 +57.67% <256> [48.0, 10.5] +// BM_map_int32_fulllookup 989 487 +50.76% <256> [48.0, 8.8] +// BM_map_int32_delete 1794 628 +64.99% <256> [48.0, 10.5] +// BM_map_int32_queueaddrem 3189 1266 +60.30% <256> [48.0, 11.6] +// BM_map_int32_mixedaddrem 3822 1623 +57.54% <256> [48.0, 10.9] +// BM_map_int32_fifo 151 134 +11.26% <256> [48.0, 8.8] +// BM_map_int32_fwditer 161 32 +80.12% <256> [48.0, 10.5] +// BM_set_int64_insert 1546 636 +58.86% <256> [40.0, 10.5] +// BM_set_int64_lookup 1200 512 +57.33% <256> [40.0, 10.5] +// BM_set_int64_fulllookup 971 487 +49.85% <256> [40.0, 8.8] +// BM_set_int64_delete 1745 616 +64.70% <256> [40.0, 10.5] +// BM_set_int64_queueaddrem 3163 1195 +62.22% <256> [40.0, 11.6] +// BM_set_int64_mixedaddrem 3760 1564 +58.40% <256> [40.0, 10.9] +// BM_set_int64_fifo 146 103 +29.45% <256> [40.0, 8.8] +// BM_set_int64_fwditer 162 31 +80.86% <256> [40.0, 10.5] +// BM_map_int64_insert 1551 720 +53.58% <256> [48.0, 20.7] +// BM_map_int64_lookup 1214 612 +49.59% <256> [48.0, 20.7] +// BM_map_int64_fulllookup 994 592 +40.44% <256> [48.0, 17.2] +// BM_map_int64_delete 1778 764 +57.03% <256> [48.0, 20.7] +// BM_map_int64_queueaddrem 3189 1547 +51.49% <256> [48.0, 20.9] +// BM_map_int64_mixedaddrem 3779 1887 +50.07% <256> [48.0, 21.6] +// BM_map_int64_fifo 147 145 +1.36% <256> [48.0, 17.2] +// BM_map_int64_fwditer 162 41 +74.69% <256> [48.0, 20.7] +// BM_set_string_insert 1989 1966 +1.16% <256> [64.0, 44.5] +// BM_set_string_lookup 1709 1600 +6.38% <256> [64.0, 44.5] +// BM_set_string_fulllookup 1573 1529 +2.80% <256> [64.0, 35.4] +// BM_set_string_delete 2520 1920 +23.81% <256> [64.0, 44.5] +// BM_set_string_queueaddrem 4706 4309 +8.44% <256> [64.0, 48.3] +// BM_set_string_mixedaddrem 5080 4654 +8.39% <256> [64.0, 46.7] +// BM_set_string_fifo 318 512 -61.01% <256> [64.0, 35.4] +// BM_set_string_fwditer 182 93 +48.90% <256> [64.0, 44.5] +// BM_map_string_insert 2600 2227 +14.35% <256> [72.0, 55.8] +// BM_map_string_lookup 2068 1730 +16.34% <256> [72.0, 55.8] +// BM_map_string_fulllookup 1859 1618 +12.96% <256> [72.0, 44.0] +// BM_map_string_delete 3168 2080 +34.34% <256> [72.0, 55.8] +// BM_map_string_queueaddrem 5840 4701 +19.50% <256> [72.0, 59.4] +// BM_map_string_mixedaddrem 6400 5200 +18.75% <256> [72.0, 57.8] +// BM_map_string_fifo 398 596 -49.75% <256> [72.0, 44.0] +// BM_map_string_fwditer 243 113 +53.50% <256> [72.0, 55.8] +// BM_set_cord_insert 3661 2680 +26.80% <256> [40.0, 10.5] +// BM_set_cord_lookup 2920 2293 +21.47% <256> [40.0, 10.5] +// BM_set_cord_fulllookup 2960 2267 +23.41% <256> [40.0, 8.8] +// BM_set_cord_delete 4679 2535 +45.82% <256> [40.0, 10.5] +// BM_set_cord_queueaddrem 8230 5600 +31.96% <256> [40.0, 11.3] +// BM_set_cord_mixedaddrem 8497 6080 +28.45% <256> [40.0, 10.7] +// BM_set_cord_fifo 358 370 -3.35% <256> [40.0, 8.8] +// BM_set_cord_fwditer 352 193 +45.17% <256> [40.0, 10.5] +// BM_map_cord_insert 3680 2927 +20.46% <256> [48.0, 20.7] +// BM_map_cord_lookup 3018 2466 +18.29% <256> [48.0, 20.7] +// BM_map_cord_fulllookup 2943 2466 +16.21% <256> [48.0, 17.2] +// BM_map_cord_delete 4675 2775 +40.64% <256> [48.0, 20.7] +// BM_map_cord_queueaddrem 8383 6360 +24.13% <256> [48.0, 22.2] +// BM_map_cord_mixedaddrem 8952 6760 +24.49% <256> [48.0, 21.2] +// BM_map_cord_fifo 444 463 -4.28% <256> [48.0, 17.2] +// BM_map_cord_fwditer 391 225 +42.46% <256> [48.0, 20.7] + +#ifndef UTIL_BTREE_BTREE_H__ +#define UTIL_BTREE_BTREE_H__ + +#include +#include +#include +#include +#include +#include +#include // IWYU pragma: export // Clients can rely on this. +#include +#include +#include // IWYU pragma: export // Clients can rely on this. +#include +#include + +#include "base/gdb-scripting.h" +#include "base/integral_types.h" +#include "base/logging.h" +#include "base/macros.h" +#include "base/template_util.h" +#include "base/type_traits.h" +#include "strings/cord.h" +#include "strings/stringpiece.h" + +// iwyu.py found std::swap in , but we include . +// IWYU pragma: no_include +// +// iwyu.py doesn't think we need , so wants . +// IWYU pragma: no_include +// +// iwyu.py sure wants to bring in a lot of private STL headers. +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include +// IWYU pragma: no_include + +namespace util { +namespace btree { + +// Inside a btree method, if we just call swap(), it will choose the +// btree::swap method, which we don't want. And we can't say ::swap because +// then MSVC won't pickup any std::swap() implementations. We can't just use +// std::swap() directly because then we don't get the specialization for string +// and Cord (which are not defined in the std namespace!). So the solution is +// to have a special swap helper function whose name doesn't collide with other +// swap functions defined by the btree classes. +template +inline void btree_swap_helper(T &a, T &b) { + using std::swap; + swap(a, b); +} + +// A helper type used to indicate that a key-compare-to functor has been +// provided. A user can specify a key-compare-to functor by doing: +// +// struct MyStringComparer +// : public util::btree::btree_key_compare_to_tag { +// int operator()(const string &a, const string &b) const { +// return a.compare(b); +// } +// }; +// +// Note that the return type is an int and not a bool. There is a +// COMPILE_ASSERT which enforces this return type. +struct btree_key_compare_to_tag { +}; + +// A helper class that indicates if the Compare parameter is derived from +// btree_key_compare_to_tag. +template +struct btree_is_key_compare_to + : public base::is_convertible { +}; + +// A helper class to convert a boolean comparison into a three-way "compare-to" +// comparison that returns a negative value to indicate less-than, zero to +// indicate equality and a positive value to indicate greater-than. This helper +// class is specialized for less, greater, less, +// greater, less and greater. The +// btree_key_compare_to_adapter class is provided so that btree users +// automatically get the more efficient compare-to code when using common +// google string types with common comparison functors. +template +struct btree_key_compare_to_adapter : Compare { + btree_key_compare_to_adapter() { } + btree_key_compare_to_adapter(const Compare &c) : Compare(c) { } + btree_key_compare_to_adapter(const btree_key_compare_to_adapter &c) + : Compare(c) { + } +}; + +template <> +struct btree_key_compare_to_adapter > + : public btree_key_compare_to_tag { + btree_key_compare_to_adapter() {} + btree_key_compare_to_adapter(const less&) {} + btree_key_compare_to_adapter( + const btree_key_compare_to_adapter >&) {} + int operator()(const string &a, const string &b) const { + return a.compare(b); + } +}; + +template <> +struct btree_key_compare_to_adapter > + : public btree_key_compare_to_tag { + btree_key_compare_to_adapter() {} + btree_key_compare_to_adapter(const greater&) {} + btree_key_compare_to_adapter( + const btree_key_compare_to_adapter >&) {} + int operator()(const string &a, const string &b) const { + return b.compare(a); + } +}; + +template <> +struct btree_key_compare_to_adapter > + : public btree_key_compare_to_tag { + btree_key_compare_to_adapter() {} + btree_key_compare_to_adapter(const less&) {} + btree_key_compare_to_adapter( + const btree_key_compare_to_adapter >&) {} + int operator()(const StringPiece &a, const StringPiece &b) const { + return a.compare(b); + } +}; + +template <> +struct btree_key_compare_to_adapter > + : public btree_key_compare_to_tag { + btree_key_compare_to_adapter() {} + btree_key_compare_to_adapter(const greater&) {} + btree_key_compare_to_adapter( + const btree_key_compare_to_adapter >&) {} + int operator()(const StringPiece &a, const StringPiece &b) const { + return b.compare(a); + } +}; + +template <> +struct btree_key_compare_to_adapter > + : public btree_key_compare_to_tag { + btree_key_compare_to_adapter() {} + btree_key_compare_to_adapter(const less&) {} + btree_key_compare_to_adapter( + const btree_key_compare_to_adapter >&) {} + int operator()(const Cord &a, const Cord &b) const { + return a.CompareTo(b); + } +}; + +template <> +struct btree_key_compare_to_adapter > + : public btree_key_compare_to_tag { + btree_key_compare_to_adapter() {} + btree_key_compare_to_adapter(const greater&) {} + btree_key_compare_to_adapter( + const btree_key_compare_to_adapter >&) {} + int operator()(const Cord &a, const Cord &b) const { + return b.CompareTo(a); + } +}; + +// A helper class that allows a compare-to functor to behave like a plain +// compare functor. This specialization is used when we do not have a +// compare-to functor. +template +struct btree_key_comparer { + btree_key_comparer() {} + btree_key_comparer(Compare c) : comp(c) {} + static bool bool_compare(const Compare &comp, const Key &x, const Key &y) { + return comp(x, y); + } + bool operator()(const Key &x, const Key &y) const { + return bool_compare(comp, x, y); + } + Compare comp; +}; + +// A specialization of btree_key_comparer when a compare-to functor is +// present. We need a plain (boolean) comparison in some parts of the btree +// code, such as insert-with-hint. +template +struct btree_key_comparer { + btree_key_comparer() {} + btree_key_comparer(Compare c) : comp(c) {} + static bool bool_compare(const Compare &comp, const Key &x, const Key &y) { + return comp(x, y) < 0; + } + bool operator()(const Key &x, const Key &y) const { + return bool_compare(comp, x, y); + } + Compare comp; +}; + +// A helper function to compare to keys using the specified compare +// functor. This dispatches to the appropriate btree_key_comparer comparison, +// depending on whether we have a compare-to functor or not (which depends on +// whether Compare is derived from btree_key_compare_to_tag). +template +static bool btree_compare_keys( + const Compare &comp, const Key &x, const Key &y) { + typedef btree_key_comparer::value> key_comparer; + return key_comparer::bool_compare(comp, x, y); +} + +template +struct btree_common_params { + // If Compare is derived from btree_key_compare_to_tag then use it as the + // key_compare type. Otherwise, use btree_key_compare_to_adapter<> which will + // fall-back to Compare if we don't have an appropriate specialization. + typedef typename base::if_< + btree_is_key_compare_to::value, + Compare, btree_key_compare_to_adapter >::type key_compare; + // A type which indicates if we have a key-compare-to functor or a plain old + // key-compare functor. + typedef btree_is_key_compare_to is_key_compare_to; + + typedef Alloc allocator_type; + typedef Key key_type; + typedef ssize_t size_type; + typedef ptrdiff_t difference_type; + + enum { + kTargetNodeSize = TargetNodeSize, + }; +}; + +// A parameters structure for holding the type parameters for a btree_map. +template +struct btree_map_params + : public btree_common_params { + typedef Data data_type; + typedef Data mapped_type; + typedef pair value_type; + typedef pair mutable_value_type; + typedef value_type* pointer; + typedef const value_type* const_pointer; + typedef value_type& reference; + typedef const value_type& const_reference; + + enum { + kValueSize = sizeof(Key) + sizeof(data_type), + }; + + static const Key& key(const value_type &x) { return x.first; } + static const Key& key(const mutable_value_type &x) { return x.first; } + static void swap(mutable_value_type *a, mutable_value_type *b) { + btree_swap_helper(a->first, b->first); + btree_swap_helper(a->second, b->second); + } +}; + +// A parameters structure for holding the type parameters for a btree_set. +template +struct btree_set_params + : public btree_common_params { + typedef base::false_type data_type; + typedef base::false_type mapped_type; + typedef Key value_type; + typedef value_type mutable_value_type; + typedef value_type* pointer; + typedef const value_type* const_pointer; + typedef value_type& reference; + typedef const value_type& const_reference; + + enum { + kValueSize = sizeof(Key), + }; + + static const Key& key(const value_type &x) { return x; } + static void swap(mutable_value_type *a, mutable_value_type *b) { + btree_swap_helper(*a, *b); + } +}; + +// An adapter class that converts a lower-bound compare into an upper-bound +// compare. +template +struct btree_upper_bound_adapter : public Compare { + btree_upper_bound_adapter(Compare c) : Compare(c) {} + bool operator()(const Key &a, const Key &b) const { + return !static_cast(*this)(b, a); + } +}; + +template +struct btree_upper_bound_compare_to_adapter : public CompareTo { + btree_upper_bound_compare_to_adapter(CompareTo c) : CompareTo(c) {} + int operator()(const Key &a, const Key &b) const { + return static_cast(*this)(b, a); + } +}; + +// Dispatch helper class for using linear search with plain compare. +template +struct btree_linear_search_plain_compare { + static int lower_bound(const K &k, const N &n, Compare comp) { + return n.linear_search_plain_compare(k, 0, n.count(), comp); + } + static int upper_bound(const K &k, const N &n, Compare comp) { + typedef btree_upper_bound_adapter upper_compare; + return n.linear_search_plain_compare(k, 0, n.count(), upper_compare(comp)); + } +}; + +// Dispatch helper class for using linear search with compare-to +template +struct btree_linear_search_compare_to { + static int lower_bound(const K &k, const N &n, CompareTo comp) { + return n.linear_search_compare_to(k, 0, n.count(), comp); + } + static int upper_bound(const K &k, const N &n, CompareTo comp) { + typedef btree_upper_bound_adapter > upper_compare; + return n.linear_search_plain_compare(k, 0, n.count(), upper_compare(comp)); + } +}; + +// Dispatch helper class for using binary search with plain compare. +template +struct btree_binary_search_plain_compare { + static int lower_bound(const K &k, const N &n, Compare comp) { + return n.binary_search_plain_compare(k, 0, n.count(), comp); + } + static int upper_bound(const K &k, const N &n, Compare comp) { + typedef btree_upper_bound_adapter upper_compare; + return n.binary_search_plain_compare(k, 0, n.count(), upper_compare(comp)); + } +}; + +// Dispatch helper class for using binary search with compare-to. +template +struct btree_binary_search_compare_to { + static int lower_bound(const K &k, const N &n, CompareTo comp) { + return n.binary_search_compare_to(k, 0, n.count(), CompareTo()); + } + static int upper_bound(const K &k, const N &n, CompareTo comp) { + typedef btree_upper_bound_adapter > upper_compare; + return n.linear_search_plain_compare(k, 0, n.count(), upper_compare(comp)); + } +}; + +// A node in the btree holding. The same node type is used for both internal +// and leaf nodes in the btree, though the nodes are allocated in such a way +// that the children array is only valid in internal nodes. +template +class btree_node { + public: + typedef Params params_type; + typedef btree_node self_type; + typedef typename Params::key_type key_type; + typedef typename Params::data_type data_type; + typedef typename Params::value_type value_type; + typedef typename Params::mutable_value_type mutable_value_type; + typedef typename Params::pointer pointer; + typedef typename Params::const_pointer const_pointer; + typedef typename Params::reference reference; + typedef typename Params::const_reference const_reference; + typedef typename Params::key_compare key_compare; + typedef typename Params::size_type size_type; + typedef typename Params::difference_type difference_type; + // Typedefs for the various types of node searches. + typedef btree_linear_search_plain_compare< + key_type, self_type, key_compare> linear_search_plain_compare_type; + typedef btree_linear_search_compare_to< + key_type, self_type, key_compare> linear_search_compare_to_type; + typedef btree_binary_search_plain_compare< + key_type, self_type, key_compare> binary_search_plain_compare_type; + typedef btree_binary_search_compare_to< + key_type, self_type, key_compare> binary_search_compare_to_type; + // If we have a valid key-compare-to type, use linear_search_compare_to, + // otherwise use linear_search_plain_compare. + typedef typename base::if_< + Params::is_key_compare_to::value, + linear_search_compare_to_type, + linear_search_plain_compare_type>::type linear_search_type; + // If we have a valid key-compare-to type, use binary_search_compare_to, + // otherwise use binary_search_plain_compare. + typedef typename base::if_< + Params::is_key_compare_to::value, + binary_search_compare_to_type, + binary_search_plain_compare_type>::type binary_search_type; + // If the key is an integral or floating point type, use linear search which + // is faster than binary search for such types. Might be wise to also + // configure linear search based on node-size. + typedef typename base::if_< + base::is_integral::value || + base::is_floating_point::value, + linear_search_type, binary_search_type>::type search_type; + + struct base_fields { + // A boolean indicating whether the node is a leaf or not. + uint8 leaf; + // The position of the node in the node's parent. + uint8 position; + // The maximum number of values the node can hold. + uint8 max_count; + // The count of the number of values in the node. + uint8 count; + // A pointer to the node's parent. + btree_node *parent; + }; + + enum { + kValueSize = params_type::kValueSize, + kTargetNodeSize = params_type::kTargetNodeSize, + + // Compute how many values we can fit onto a leaf node. + kNodeTargetValues = (kTargetNodeSize - sizeof(base_fields)) / kValueSize, + // We need a minimum of 3 values per internal node in order to perform + // splitting (1 value for the two nodes involved in the split and 1 value + // propagated to the parent as the delimiter for the split). + kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, + + kExactMatch = 1 << 30, + kMatchMask = kExactMatch - 1, + }; + + struct leaf_fields : public base_fields { + // The array of values. Only the first count of these values have been + // constructed and are valid. + mutable_value_type values[kNodeValues]; + }; + + struct internal_fields : public leaf_fields { + // The array of child pointers. The keys in children_[i] are all less than + // key(i). The keys in children_[i + 1] are all greater than key(i). There + // are always count + 1 children. + btree_node *children[kNodeValues + 1]; + }; + + struct root_fields : public internal_fields { + btree_node *rightmost; + size_type size; + }; + + public: + // Getter/setter for whether this is a leaf node or not. This value doesn't + // change after the node is created. + bool leaf() const { return fields_.leaf; } + + // Getter for the position of this node in its parent. + int position() const { return fields_.position; } + void set_position(int v) { fields_.position = v; } + + // Getter/setter for the number of values stored in this node. + int count() const { return fields_.count; } + void set_count(int v) { fields_.count = v; } + int max_count() const { return fields_.max_count; } + + // Getter for the parent of this node. + btree_node* parent() const { return fields_.parent; } + // Getter for whether the node is the root of the tree. The parent of the + // root of the tree is the leftmost node in the tree which is guaranteed to + // be a leaf. + bool is_root() const { return parent()->leaf(); } + void make_root() { + DCHECK(parent()->is_root()); + fields_.parent = fields_.parent->parent(); + } + + // Getter for the rightmost root node field. Only valid on the root node. + btree_node* rightmost() const { return fields_.rightmost; } + btree_node** mutable_rightmost() { return &fields_.rightmost; } + + // Getter for the size root node field. Only valid on the root node. + size_type size() const { return fields_.size; } + size_type* mutable_size() { return &fields_.size; } + + // Getters for the key/value at position i in the node. + const key_type& key(int i) const { + return params_type::key(fields_.values[i]); + } + reference value(int i) { + return reinterpret_cast(fields_.values[i]); + } + const_reference value(int i) const { + return reinterpret_cast(fields_.values[i]); + } + mutable_value_type* mutable_value(int i) { + return &fields_.values[i]; + } + + // Swap value i in this node with value j in node x. + void value_swap(int i, btree_node *x, int j) { + params_type::swap(mutable_value(i), x->mutable_value(j)); + } + + // Getters/setter for the child at position i in the node. + btree_node* child(int i) const { return fields_.children[i]; } + btree_node** mutable_child(int i) { return &fields_.children[i]; } + void set_child(int i, btree_node *c) { + *mutable_child(i) = c; + c->fields_.parent = this; + c->fields_.position = i; + } + + // Returns the position of the first value whose key is not less than k. + template + int lower_bound(const key_type &k, const Compare &comp) const { + return search_type::lower_bound(k, *this, comp); + } + // Returns the position of the first value whose key is greater than k. + template + int upper_bound(const key_type &k, const Compare &comp) const { + return search_type::upper_bound(k, *this, comp); + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using plain compare. + template + int linear_search_plain_compare( + const key_type &k, int s, int e, const Compare &comp) const { + while (s < e) { + if (!btree_compare_keys(comp, key(s), k)) { + break; + } + ++s; + } + return s; + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using compare-to. + template + int linear_search_compare_to( + const key_type &k, int s, int e, const Compare &comp) const { + while (s < e) { + int c = comp(key(s), k); + if (c == 0) { + return s | kExactMatch; + } else if (c > 0) { + break; + } + ++s; + } + return s; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using plain compare. + template + int binary_search_plain_compare( + const key_type &k, int s, int e, const Compare &comp) const { + while (s != e) { + int mid = (s + e) / 2; + if (btree_compare_keys(comp, key(mid), k)) { + s = mid + 1; + } else { + e = mid; + } + } + return s; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using compare-to. + template + int binary_search_compare_to( + const key_type &k, int s, int e, const CompareTo &comp) const { + while (s != e) { + int mid = (s + e) / 2; + int c = comp(key(mid), k); + if (c < 0) { + s = mid + 1; + } else if (c > 0) { + e = mid; + } else { + // Need to return the first value whose key is not less than k, which + // requires continuing the binary search. Note that we are guaranteed + // that the result is an exact match because if "key(mid-1) < k" the + // call to binary_search_compare_to() will return "mid". + s = binary_search_compare_to(k, s, mid, comp); + return s | kExactMatch; + } + } + return s; + } + + // Inserts the value x at position i, shifting all existing values and + // children at positions >= i to the right by 1. + void insert_value(int i, const value_type &x); + + // Removes the value at position i, shifting all existing values and children + // at positions > i to the left by 1. + void remove_value(int i); + + // Rebalances a node with its right sibling. + void rebalance_right_to_left(btree_node *sibling, int to_move); + void rebalance_left_to_right(btree_node *sibling, int to_move); + + // Splits a node, moving a portion of the node's values to its right sibling. + void split(btree_node *sibling, int insert_position); + + // Merges a node with its right sibling, moving all of the values and the + // delimiting key in the parent node onto itself. + void merge(btree_node *sibling); + + // Swap the contents of "this" and "src". + void swap(btree_node *src); + + // Node allocation/deletion routines. + static btree_node* init_leaf( + leaf_fields *f, btree_node *parent, int max_count) { + btree_node *n = reinterpret_cast(f); + f->leaf = 1; + f->position = 0; + f->max_count = max_count; + f->count = 0; + f->parent = parent; + if (DEBUG_MODE) { + memset(&f->values, 0, max_count * sizeof(value_type)); + } + return n; + } + static btree_node* init_internal(internal_fields *f, btree_node *parent) { + btree_node *n = init_leaf(f, parent, kNodeValues); + f->leaf = 0; + if (DEBUG_MODE) { + memset(f->children, 0, sizeof(f->children)); + } + return n; + } + static btree_node* init_root(root_fields *f, btree_node *parent) { + btree_node *n = init_internal(f, parent); + f->rightmost = parent; + f->size = parent->count(); + return n; + } + void destroy() { + for (int i = 0; i < count(); ++i) { + value_destroy(i); + } + } + + private: + void value_init(int i) { + new (&fields_.values[i]) mutable_value_type; + } + void value_init(int i, const value_type &x) { + new (&fields_.values[i]) mutable_value_type(x); + } + void value_destroy(int i) { + fields_.values[i].~mutable_value_type(); + } + + private: + root_fields fields_; + + private: + DISALLOW_EVIL_CONSTRUCTORS(btree_node); +}; + +template +struct btree_iterator { + typedef typename Node::key_type key_type; + typedef typename Node::size_type size_type; + typedef typename Node::difference_type difference_type; + typedef typename Node::params_type params_type; + + typedef Node node_type; + typedef typename base::remove_const::type normal_node; + typedef const Node const_node; + typedef typename params_type::value_type value_type; + typedef typename params_type::pointer normal_pointer; + typedef typename params_type::reference normal_reference; + typedef typename params_type::const_pointer const_pointer; + typedef typename params_type::const_reference const_reference; + + typedef Pointer pointer; + typedef Reference reference; + typedef bidirectional_iterator_tag iterator_category; + + typedef btree_iterator< + normal_node, normal_reference, normal_pointer> iterator; + typedef btree_iterator< + const_node, const_reference, const_pointer> const_iterator; + typedef btree_iterator self_type; + + btree_iterator() + : node(NULL), + position(-1) { + } + btree_iterator(Node *n, int p) + : node(n), + position(p) { + } + btree_iterator(const iterator &x) + : node(x.node), + position(x.position) { + } + + // Increment/decrement the iterator. + void increment() { + if (node->leaf() && ++position < node->count()) { + return; + } + increment_slow(); + } + void increment_by(int count); + void increment_slow(); + + void decrement() { + if (node->leaf() && --position >= 0) { + return; + } + decrement_slow(); + } + void decrement_slow(); + + bool operator==(const const_iterator &x) const { + return node == x.node && position == x.position; + } + bool operator!=(const const_iterator &x) const { + return node != x.node || position != x.position; + } + + // Accessors for the key/value the iterator is pointing at. + const key_type& key() const { + return node->key(position); + } + reference operator*() const { + return node->value(position); + } + pointer operator->() const { + return &node->value(position); + } + + self_type& operator++() { + increment(); + return *this; + } + self_type& operator--() { + decrement(); + return *this; + } + self_type operator++(int) { + self_type tmp = *this; + ++*this; + return tmp; + } + self_type operator--(int) { + self_type tmp = *this; + --*this; + return tmp; + } + + // The node in the tree the iterator is pointing at. + Node *node; + // The position within the node of the tree the iterator is pointing at. + int position; +}; + +// Dispatch helper class for using btree::internal_locate with plain compare. +struct btree_internal_locate_plain_compare { + template + static pair dispatch(const K &k, const T &t, Iter iter) { + return t.internal_locate_plain_compare(k, iter); + } +}; + +// Dispatch helper class for using btree::internal_locate with compare-to. +struct btree_internal_locate_compare_to { + template + static pair dispatch(const K &k, const T &t, Iter iter) { + return t.internal_locate_compare_to(k, iter); + } +}; + +template +class btree : public Params::key_compare { + typedef btree self_type; + typedef btree_node node_type; + typedef typename node_type::base_fields base_fields; + typedef typename node_type::leaf_fields leaf_fields; + typedef typename node_type::internal_fields internal_fields; + typedef typename node_type::root_fields root_fields; + typedef typename Params::is_key_compare_to is_key_compare_to; + + friend class btree_internal_locate_plain_compare; + friend class btree_internal_locate_compare_to; + typedef typename base::if_< + is_key_compare_to::value, + btree_internal_locate_compare_to, + btree_internal_locate_plain_compare>::type internal_locate_type; + + enum { + kNodeValues = node_type::kNodeValues, + kMinNodeValues = kNodeValues / 2, + kValueSize = node_type::kValueSize, + kExactMatch = node_type::kExactMatch, + kMatchMask = node_type::kMatchMask, + }; + + // A helper class to get the empty base class optimization for 0-size + // allocators. Base is internal_allocator_type. + // (e.g. empty_base_handle). If Base is + // 0-size, the compiler doesn't have to reserve any space for it and + // sizeof(empty_base_handle) will simply be sizeof(Data). Google [empty base + // class optimization] for more details. + template + struct empty_base_handle : public Base { + empty_base_handle(const Base &b, const Data &d) + : Base(b), + data(d) { + } + Data data; + }; + + struct node_stats { + node_stats(ssize_t l, ssize_t i) + : leaf_nodes(l), + internal_nodes(i) { + } + + node_stats& operator+=(const node_stats &x) { + leaf_nodes += x.leaf_nodes; + internal_nodes += x.internal_nodes; + return *this; + } + + ssize_t leaf_nodes; + ssize_t internal_nodes; + }; + + public: + typedef Params params_type; + typedef typename Params::key_type key_type; + typedef typename Params::data_type data_type; + typedef typename Params::mapped_type mapped_type; + typedef typename Params::value_type value_type; + typedef typename Params::key_compare key_compare; + typedef typename Params::pointer pointer; + typedef typename Params::const_pointer const_pointer; + typedef typename Params::reference reference; + typedef typename Params::const_reference const_reference; + typedef typename Params::size_type size_type; + typedef typename Params::difference_type difference_type; + typedef btree_iterator iterator; + typedef typename iterator::const_iterator const_iterator; + typedef std::reverse_iterator const_reverse_iterator; + typedef std::reverse_iterator reverse_iterator; + + typedef typename Params::allocator_type allocator_type; + typedef typename allocator_type::template rebind::other + internal_allocator_type; + + public: + // Default constructor. + btree(const key_compare &comp, const allocator_type &alloc); + + // Copy constructor. + btree(const self_type &x); + + // Destructor. + ~btree() { + clear(); + } + + // Iterator routines. + iterator begin() { + return iterator(leftmost(), 0); + } + const_iterator begin() const { + return const_iterator(leftmost(), 0); + } + iterator end() { + return iterator(rightmost(), rightmost() ? rightmost()->count() : 0); + } + const_iterator end() const { + return const_iterator(rightmost(), rightmost() ? rightmost()->count() : 0); + } + reverse_iterator rbegin() { + return reverse_iterator(end()); + } + const_reverse_iterator rbegin() const { + return const_reverse_iterator(end()); + } + reverse_iterator rend() { + return reverse_iterator(begin()); + } + const_reverse_iterator rend() const { + return const_reverse_iterator(begin()); + } + + // Finds the first element whose key is not less than key. + iterator lower_bound(const key_type &key) { + return internal_end( + internal_lower_bound(key, iterator(root(), 0))); + } + const_iterator lower_bound(const key_type &key) const { + return internal_end( + internal_lower_bound(key, const_iterator(root(), 0))); + } + + // Finds the first element whose key is greater than key. + iterator upper_bound(const key_type &key) { + return internal_end( + internal_upper_bound(key, iterator(root(), 0))); + } + const_iterator upper_bound(const key_type &key) const { + return internal_end( + internal_upper_bound(key, const_iterator(root(), 0))); + } + + // Finds the range of values which compare equal to key. The first member of + // the returned pair is equal to lower_bound(key). The second member pair of + // the pair is equal to upper_bound(key). + pair equal_range(const key_type &key) { + return make_pair(lower_bound(key), upper_bound(key)); + } + pair equal_range(const key_type &key) const { + return make_pair(lower_bound(key), upper_bound(key)); + } + + // Inserts a value into the btree only if it does not already exist. The + // boolean return value indicates whether insertion succeeded or failed. The + // ValuePointer type is used to avoid instatiating the value unless the key + // is being inserted. Value is not dereferenced if the key already exists in + // the btree. See btree_map::operator[]. + template + pair insert_unique(const key_type &key, ValuePointer value); + + // Inserts a value into the btree only if it does not already exist. The + // boolean return value indicates whether insertion succeeded or failed. + pair insert_unique(const value_type &v) { + return insert_unique(params_type::key(v), &v); + } + + // Insert with hint. Check to see if the value should be placed immediately + // before position in the tree. If it does, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_unique(v) were made. + iterator insert_unique(iterator position, const value_type &v); + + // Insert a range of values into the btree. + template + void insert_unique(InputIterator b, InputIterator e); + + // Inserts a value into the btree. The ValuePointer type is used to avoid + // instatiating the value unless the key is being inserted. Value is not + // dereferenced if the key already exists in the btree. See + // btree_map::operator[]. + template + iterator insert_multi(const key_type &key, ValuePointer value); + + // Inserts a value into the btree. + iterator insert_multi(const value_type &v) { + return insert_multi(params_type::key(v), &v); + } + + // Insert with hint. Check to see if the value should be placed immediately + // before position in the tree. If it does, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_multi(v) were made. + iterator insert_multi(iterator position, const value_type &v); + + // Insert a range of values into the btree. + template + void insert_multi(InputIterator b, InputIterator e); + + void assign(const self_type &x); + + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + iterator erase(iterator iter); + + // Erases range. Returns the number of keys erased. + int erase(iterator begin, iterator end); + + // Erases the specified key from the btree. Returns 1 if an element was + // erased and 0 otherwise. + int erase_unique(const key_type &key); + + // Erases all of the entries matching the specified key from the + // btree. Returns the number of elements erased. + int erase_multi(const key_type &key); + + // Finds the iterator corresponding to a key or returns end() if the key is + // not present. + iterator find_unique(const key_type &key) { + return internal_end( + internal_find_unique(key, iterator(root(), 0))); + } + const_iterator find_unique(const key_type &key) const { + return internal_end( + internal_find_unique(key, const_iterator(root(), 0))); + } + iterator find_multi(const key_type &key) { + return internal_end( + internal_find_multi(key, iterator(root(), 0))); + } + const_iterator find_multi(const key_type &key) const { + return internal_end( + internal_find_multi(key, const_iterator(root(), 0))); + } + + // Returns a count of the number of times the key appears in the btree. + size_type count_unique(const key_type &key) const { + const_iterator begin = internal_find_unique( + key, const_iterator(root(), 0)); + if (!begin.node) { + // The key doesn't exist in the tree. + return 0; + } + return 1; + } + // Returns a count of the number of times the key appears in the btree. + size_type count_multi(const key_type &key) const { + return distance(lower_bound(key), upper_bound(key)); + } + + // Clear the btree, deleting all of the values it contains. + void clear(); + + // Swap the contents of *this and x. + void swap(self_type &x); + + // Assign the contents of x to *this. + self_type& operator=(const self_type &x) { + if (&x == this) { + // Don't copy onto ourselves. + return *this; + } + assign(x); + return *this; + } + + key_compare* mutable_key_comp() { + return this; + } + const key_compare& key_comp() const { + return *this; + } + bool compare_keys(const key_type &x, const key_type &y) const { + return btree_compare_keys(key_comp(), x, y); + } + + // Dump the btree to the specified ostream. Requires that operator<< is + // defined for Key and Value. + void dump(ostream &os) const { + if (root() != NULL) { + internal_dump(os, root(), 0); + } + } + + // Verifies the structure of the btree. + void verify() const; + + // Size routines. Note that empty() is slightly faster than doing size()==0. + size_type size() const { + if (empty()) return 0; + if (root()->leaf()) return root()->count(); + return root()->size(); + } + size_type max_size() const { return numeric_limits::max(); } + bool empty() const { return root() == NULL; } + + // The height of the btree. An empty tree will have height 0. + size_type height() const { + size_type h = 0; + if (root()) { + // Count the length of the chain from the leftmost node up to the + // root. We actually count from the root back around to the level below + // the root, but the calculation is the same because of the circularity + // of that traversal. + const node_type *n = root(); + do { + ++h; + n = n->parent(); + } while (n != root()); + } + return h; + } + + // The number of internal, leaf and total nodes used by the btree. + size_type leaf_nodes() const { + return internal_stats(root()).leaf_nodes; + } + size_type internal_nodes() const { + return internal_stats(root()).internal_nodes; + } + size_type nodes() const { + node_stats stats = internal_stats(root()); + return stats.leaf_nodes + stats.internal_nodes; + } + + // The total number of bytes used by the btree. + size_type bytes_used() const { + node_stats stats = internal_stats(root()); + if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) { + return sizeof(*this) + + sizeof(base_fields) + root()->max_count() * sizeof(value_type); + } else { + return sizeof(*this) + + sizeof(root_fields) - sizeof(internal_fields) + + stats.leaf_nodes * sizeof(leaf_fields) + + stats.internal_nodes * sizeof(internal_fields); + } + } + + // The average number of bytes used per value stored in the btree. + static double average_bytes_per_value() { + // Returns the number of bytes per value on a leaf node that is 75% + // full. Experimentally, this matches up nicely with the computed number of + // bytes per value in trees that had their values inserted in random order. + return sizeof(leaf_fields) / (kNodeValues * 0.75); + } + + // The fullness of the btree. Computed as the number of elements in the btree + // divided by the maximum number of elements a tree with the current number + // of nodes could hold. A value of 1 indicates perfect space + // utilization. Smaller values indicate space wastage. + double fullness() const { + return double(size()) / (nodes() * kNodeValues); + } + // The overhead of the btree structure in bytes per node. Computed as the + // total number of bytes used by the btree minus the number of bytes used for + // storing elements divided by the number of elements. + double overhead() const { + if (empty()) { + return 0.0; + } + return (bytes_used() - size() * kValueSize) / double(size()); + } + + private: + // Internal accessor routines. + node_type* root() { return root_.data; } + const node_type* root() const { return root_.data; } + node_type** mutable_root() { return &root_.data; } + + // The rightmost node is stored in the root node. + node_type* rightmost() { + return (!root() || root()->leaf()) ? root() : root()->rightmost(); + } + const node_type* rightmost() const { + return (!root() || root()->leaf()) ? root() : root()->rightmost(); + } + node_type** mutable_rightmost() { return root()->mutable_rightmost(); } + + // The leftmost node is stored as the parent of the root node. + node_type* leftmost() { return root() ? root()->parent() : NULL; } + const node_type* leftmost() const { return root() ? root()->parent() : NULL; } + + // The size of the tree is stored in the root node. + size_type* mutable_size() { return root()->mutable_size(); } + + // Allocator routines. + internal_allocator_type* mutable_internal_allocator() { + return static_cast(&root_); + } + const internal_allocator_type& internal_allocator() const { + return *static_cast(&root_); + } + + // Node creation/deletion routines. + node_type* new_internal_node(node_type *parent) { + internal_fields *p = reinterpret_cast( + mutable_internal_allocator()->allocate(sizeof(internal_fields))); + return node_type::init_internal(p, parent); + } + node_type* new_internal_root_node() { + root_fields *p = reinterpret_cast( + mutable_internal_allocator()->allocate(sizeof(root_fields))); + return node_type::init_root(p, root()->parent()); + } + node_type* new_leaf_node(node_type *parent) { + leaf_fields *p = reinterpret_cast( + mutable_internal_allocator()->allocate(sizeof(leaf_fields))); + return node_type::init_leaf(p, parent, kNodeValues); + } + node_type* new_leaf_root_node(int max_count) { + leaf_fields *p = reinterpret_cast( + mutable_internal_allocator()->allocate( + sizeof(base_fields) + max_count * sizeof(value_type))); + return node_type::init_leaf(p, reinterpret_cast(p), max_count); + } + void delete_internal_node(node_type *node) { + node->destroy(); + DCHECK(node != root()); + mutable_internal_allocator()->deallocate( + reinterpret_cast(node), sizeof(internal_fields)); + } + void delete_internal_root_node() { + root()->destroy(); + mutable_internal_allocator()->deallocate( + reinterpret_cast(root()), sizeof(root_fields)); + } + void delete_leaf_node(node_type *node) { + node->destroy(); + mutable_internal_allocator()->deallocate( + reinterpret_cast(node), + sizeof(base_fields) + node->max_count() * sizeof(value_type)); + } + + // Rebalances or splits the node iter points to. + void rebalance_or_split(iterator *iter); + + // Merges the values of left, right and the delimiting key on their parent + // onto left, removing the delimiting key and deleting right. + void merge_nodes(node_type *left, node_type *right); + + // Tries to merge node with its left or right sibling, and failing that, + // rebalance with its left or right sibling. Returns true if a merge + // occurred, at which point it is no longer valid to access node. Returns + // false if no merging took place. + bool try_merge_or_rebalance(iterator *iter); + + // Tries to shrink the height of the tree by 1. + void try_shrink(); + + iterator internal_end(iterator iter) { + return iter.node ? iter : end(); + } + const_iterator internal_end(const_iterator iter) const { + return iter.node ? iter : end(); + } + + // Inserts a value into the btree immediately before iter. Requires that + // key(v) <= iter.key() and (--iter).key() <= key(v). + iterator internal_insert(iterator iter, const value_type &v); + + // Returns an iterator pointing to the first value >= the value "iter" is + // pointing at. Note that "iter" might be pointing to an invalid location as + // iter.position == iter.node->count(). This routine simply moves iter up in + // the tree to a valid location. + template + static IterType internal_last(IterType iter); + + // Returns an iterator pointing to the leaf position at which key would + // reside in the tree. We provide 2 versions of internal_locate. The first + // version (internal_locate_plain_compare) always returns 0 for the second + // field of the pair. The second version (internal_locate_compare_to) is for + // the key-compare-to specialization and returns either kExactMatch (if the + // key was found in the tree) or -kExactMatch (if it wasn't) in the second + // field of the pair. The compare_to specialization allows the caller to + // avoid a subsequent comparison to determine if an exact match was made, + // speeding up string, cord and StringPiece keys. + template + pair internal_locate( + const key_type &key, IterType iter) const; + template + pair internal_locate_plain_compare( + const key_type &key, IterType iter) const; + template + pair internal_locate_compare_to( + const key_type &key, IterType iter) const; + + // Internal routine which implements lower_bound(). + template + IterType internal_lower_bound( + const key_type &key, IterType iter) const; + + // Internal routine which implements upper_bound(). + template + IterType internal_upper_bound( + const key_type &key, IterType iter) const; + + // Internal routine which implements find_unique(). + template + IterType internal_find_unique( + const key_type &key, IterType iter) const; + + // Internal routine which implements find_multi(). + template + IterType internal_find_multi( + const key_type &key, IterType iter) const; + + // Deletes a node and all of its children. + void internal_clear(node_type *node); + + // Dumps a node and all of its children to the specified ostream. + void internal_dump(ostream &os, const node_type *node, int level) const; + + // Verifies the tree structure of node. + int internal_verify(const node_type *node, + const key_type *lo, const key_type *hi) const; + + node_stats internal_stats(const node_type *node) const { + if (!node) { + return node_stats(0, 0); + } + if (node->leaf()) { + return node_stats(1, 0); + } + node_stats res(0, 1); + for (int i = 0; i <= node->count(); ++i) { + res += internal_stats(node->child(i)); + } + return res; + } + + private: + empty_base_handle root_; + + private: + // A never instantiated helper function that returns base::big_ if we have a + // key-compare-to functor or if R is bool and base::small_ otherwise. + template + static typename base::if_< + base::if_, + base::type_equals_ >::type::value, + base::big_, base::small_>::type key_compare_checker(R); + + // A never instantiated helper function that returns the key comparison + // functor. + static key_compare key_compare_helper(); + + // Verify that key_compare returns a bool. This is similar to the way + // is_convertible in base/type_traits.h works. Note that key_compare_checker + // is never actually invoked. The compiler will select which + // key_compare_checker() to instantiate and then figure out the size of the + // return type of key_compare_checker() at compile time which we then check + // against the sizeof of base::big_. + COMPILE_ASSERT( + sizeof(key_compare_checker(key_compare_helper()(key_type(), key_type()))) == + sizeof(base::big_), + key_comparison_function_must_return_bool); +}; + +//// +// btree_node methods +template +inline void btree_node

::insert_value(int i, const value_type &x) { + DCHECK_LE(i, count()); + value_init(count(), x); + for (int j = count(); j > i; --j) { + value_swap(j, this, j - 1); + } + set_count(count() + 1); + + if (!leaf()) { + ++i; + for (int j = count(); j > i; --j) { + *mutable_child(j) = child(j - 1); + child(j)->set_position(j); + } + *mutable_child(i) = NULL; + } +} + +template +inline void btree_node

::remove_value(int i) { + if (!leaf()) { + DCHECK_EQ(child(i + 1)->count(), 0); + for (int j = i + 1; j < count(); ++j) { + *mutable_child(j) = child(j + 1); + child(j)->set_position(j); + } + *mutable_child(count()) = NULL; + } + + set_count(count() - 1); + for (; i < count(); ++i) { + value_swap(i, this, i + 1); + } + value_destroy(i); +} + +template +void btree_node

::rebalance_right_to_left(btree_node *src, int to_move) { + DCHECK_EQ(parent(), src->parent()); + DCHECK_EQ(position() + 1, src->position()); + DCHECK_GE(src->count(), count()); + DCHECK_GE(to_move, 1); + DCHECK_LE(to_move, src->count()); + + // Make room in the left node for the new values. + for (int i = 0; i < to_move; ++i) { + value_init(i + count()); + } + + // Move the delimiting value to the left node and the new delimiting value + // from the right node. + value_swap(count(), parent(), position()); + parent()->value_swap(position(), src, to_move - 1); + + // Move the values from the right to the left node. + for (int i = 1; i < to_move; ++i) { + value_swap(count() + i, src, i - 1); + } + // Shift the values in the right node to their correct position. + for (int i = to_move; i < src->count(); ++i) { + src->value_swap(i - to_move, src, i); + } + for (int i = 1; i <= to_move; ++i) { + src->value_destroy(src->count() - i); + } + + if (!leaf()) { + // Move the child pointers from the right to the left node. + for (int i = 0; i < to_move; ++i) { + set_child(1 + count() + i, src->child(i)); + } + for (int i = 0; i <= src->count() - to_move; ++i) { + DCHECK_LE(i + to_move, src->max_count()); + src->set_child(i, src->child(i + to_move)); + *src->mutable_child(i + to_move) = NULL; + } + } + + // Fixup the counts on the src and dest nodes. + set_count(count() + to_move); + src->set_count(src->count() - to_move); +} + +template +void btree_node

::rebalance_left_to_right(btree_node *dest, int to_move) { + DCHECK_EQ(parent(), dest->parent()); + DCHECK_EQ(position() + 1, dest->position()); + DCHECK_GE(count(), dest->count()); + DCHECK_GE(to_move, 1); + DCHECK_LE(to_move, count()); + + // Make room in the right node for the new values. + for (int i = 0; i < to_move; ++i) { + dest->value_init(i + dest->count()); + } + for (int i = dest->count() - 1; i >= 0; --i) { + dest->value_swap(i, dest, i + to_move); + } + + // Move the delimiting value to the right node and the new delimiting value + // from the left node. + dest->value_swap(to_move - 1, parent(), position()); + parent()->value_swap(position(), this, count() - to_move); + value_destroy(count() - to_move); + + // Move the values from the left to the right node. + for (int i = 1; i < to_move; ++i) { + value_swap(count() - to_move + i, dest, i - 1); + value_destroy(count() - to_move + i); + } + + if (!leaf()) { + // Move the child pointers from the left to the right node. + for (int i = dest->count(); i >= 0; --i) { + dest->set_child(i + to_move, dest->child(i)); + *dest->mutable_child(i) = NULL; + } + for (int i = 1; i <= to_move; ++i) { + dest->set_child(i - 1, child(count() - to_move + i)); + *mutable_child(count() - to_move + i) = NULL; + } + } + + // Fixup the counts on the src and dest nodes. + set_count(count() - to_move); + dest->set_count(dest->count() + to_move); +} + +template +void btree_node

::split(btree_node *dest, int insert_position) { + DCHECK_EQ(dest->count(), 0); + + // We bias the split based on the position being inserted. If we're + // inserting at the beginning of the left node then bias the split to put + // more values on the right node. If we're inserting at the end of the + // right node then bias the split to put more values on the left node. + if (insert_position == 0) { + dest->set_count(count() - 1); + } else if (insert_position == max_count()) { + dest->set_count(0); + } else { + dest->set_count(count() / 2); + } + set_count(count() - dest->count()); + DCHECK_GE(count(), 1); + + // Move values from the left sibling to the right sibling. + for (int i = 0; i < dest->count(); ++i) { + dest->value_init(i); + value_swap(count() + i, dest, i); + value_destroy(count() + i); + } + + // The split key is the largest value in the left sibling. + set_count(count() - 1); + parent()->insert_value(position(), value_type()); + value_swap(count(), parent(), position()); + value_destroy(count()); + parent()->set_child(position() + 1, dest); + + if (!leaf()) { + for (int i = 0; i <= dest->count(); ++i) { + DCHECK(child(count() + i + 1) != NULL); + dest->set_child(i, child(count() + i + 1)); + *mutable_child(count() + i + 1) = NULL; + } + } +} + +template +void btree_node

::merge(btree_node *src) { + DCHECK_EQ(parent(), src->parent()); + DCHECK_EQ(position() + 1, src->position()); + + // Move the delimiting value to the left node. + value_init(count()); + value_swap(count(), parent(), position()); + + // Move the values from the right to the left node. + for (int i = 0; i < src->count(); ++i) { + value_init(1 + count() + i); + value_swap(1 + count() + i, src, i); + src->value_destroy(i); + } + + if (!leaf()) { + // Move the child pointers from the right to the left node. + for (int i = 0; i <= src->count(); ++i) { + set_child(1 + count() + i, src->child(i)); + *src->mutable_child(i) = NULL; + } + } + + // Fixup the counts on the src and dest nodes. + set_count(1 + count() + src->count()); + src->set_count(0); + + // Remove the value on the parent node. + parent()->remove_value(position()); +} + +template +void btree_node

::swap(btree_node *x) { + DCHECK_EQ(leaf(), x->leaf()); + + // Swap the values. + for (int i = count(); i < x->count(); ++i) { + value_init(i); + } + for (int i = x->count(); i < count(); ++i) { + x->value_init(i); + } + int n = max(count(), x->count()); + for (int i = 0; i < n; ++i) { + value_swap(i, x, i); + } + for (int i = count(); i < x->count(); ++i) { + x->value_destroy(i); + } + for (int i = x->count(); i < count(); ++i) { + value_destroy(i); + } + + if (!leaf()) { + // Swap the child pointers. + for (int i = 0; i <= n; ++i) { + btree_swap_helper(*mutable_child(i), *x->mutable_child(i)); + } + for (int i = 0; i <= count(); ++i) { + x->child(i)->fields_.parent = x; + } + for (int i = 0; i <= x->count(); ++i) { + child(i)->fields_.parent = this; + } + } + + // Swap the counts. + btree_swap_helper(fields_.count, x->fields_.count); +} + +//// +// btree_iterator methods +template +void btree_iterator::increment_slow() { + if (node->leaf()) { + DCHECK_GE(position, node->count()); + self_type save(*this); + while (position == node->count() && !node->is_root()) { + DCHECK_EQ(node->parent()->child(node->position()), node); + position = node->position(); + node = node->parent(); + } + if (position == node->count()) { + *this = save; + } + } else { + DCHECK_LT(position, node->count()); + node = node->child(position + 1); + while (!node->leaf()) { + node = node->child(0); + } + position = 0; + } +} + +template +void btree_iterator::increment_by(int count) { + while (count > 0) { + if (node->leaf()) { + int rest = node->count() - position; + position += min(rest, count); + count = count - rest; + if (position < node->count()) { + return; + } + } else { + --count; + } + increment_slow(); + } +} + +template +void btree_iterator::decrement_slow() { + if (node->leaf()) { + DCHECK_LE(position, -1); + self_type save(*this); + while (position < 0 && !node->is_root()) { + DCHECK_EQ(node->parent()->child(node->position()), node); + position = node->position() - 1; + node = node->parent(); + } + if (position < 0) { + *this = save; + } + } else { + DCHECK_GE(position, 0); + node = node->child(position); + while (!node->leaf()) { + node = node->child(node->count()); + } + position = node->count() - 1; + } +} + +//// +// btree methods +template +btree

::btree(const key_compare &comp, const allocator_type &alloc) + : key_compare(comp), + root_(alloc, NULL) { +} + +template +btree

::btree(const self_type &x) + : key_compare(x.key_comp()), + root_(x.internal_allocator(), NULL) { + assign(x); +} + +template template +pair::iterator, bool> +btree

::insert_unique(const key_type &key, ValuePointer value) { + if (empty()) { + *mutable_root() = new_leaf_root_node(1); + } + + pair res = internal_locate(key, iterator(root(), 0)); + iterator &iter = res.first; + if (res.second == kExactMatch) { + // The key already exists in the tree, do nothing. + return make_pair(internal_last(iter), false); + } else if (!res.second) { + iterator last = internal_last(iter); + if (last.node && !compare_keys(key, last.key())) { + // The key already exists in the tree, do nothing. + return make_pair(last, false); + } + } + + return make_pair(internal_insert(iter, *value), true); +} + +template +inline typename btree

::iterator +btree

::insert_unique(iterator position, const value_type &v) { + if (!empty()) { + const key_type &key = params_type::key(v); + if (position == end() || compare_keys(key, position.key())) { + iterator prev = position; + if (position == begin() || compare_keys((--prev).key(), key)) { + // prev.key() < key < position.key() + return internal_insert(position, v); + } + } else if (compare_keys(position.key(), key)) { + iterator next = position; + ++next; + if (next == end() || compare_keys(key, next.key())) { + // position.key() < key < next.key() + return internal_insert(next, v); + } + } else { + // position.key() == key + return position; + } + } + return insert_unique(v).first; +} + +template template +void btree

::insert_unique(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_unique(end(), *b); + } +} + +template template +typename btree

::iterator +btree

::insert_multi(const key_type &key, ValuePointer value) { + if (empty()) { + *mutable_root() = new_leaf_root_node(1); + } + + iterator iter = internal_upper_bound(key, iterator(root(), 0)); + if (!iter.node) { + iter = end(); + } + return internal_insert(iter, *value); +} + +template +typename btree

::iterator +btree

::insert_multi(iterator position, const value_type &v) { + if (!empty()) { + const key_type &key = params_type::key(v); + if (position == end() || !compare_keys(position.key(), key)) { + iterator prev = position; + if (position == begin() || !compare_keys(key, (--prev).key())) { + // prev.key() <= key <= position.key() + return internal_insert(position, v); + } + } else { + iterator next = position; + ++next; + if (next == end() || !compare_keys(next.key(), key)) { + // position.key() < key <= next.key() + return internal_insert(next, v); + } + } + } + return insert_multi(v); +} + +template template +void btree

::insert_multi(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_multi(end(), *b); + } +} + +template +void btree

::assign(const self_type &x) { + clear(); + + *mutable_key_comp() = x.key_comp(); + *mutable_internal_allocator() = x.internal_allocator(); + + // Assignment can avoid key comparisons because we know the order of the + // values is the same order we'll store them in. + for (const_iterator iter = x.begin(); iter != x.end(); ++iter) { + if (empty()) { + insert_multi(*iter); + } else { + // If the btree is not empty, we can just insert the new value at the end + // of the tree! + internal_insert(end(), *iter); + } + } +} + +template +typename btree

::iterator btree

::erase(iterator iter) { + bool internal_delete = false; + if (!iter.node->leaf()) { + // Deletion of a value on an internal node. Swap the key with the largest + // value of our left child. This is easy, we just decrement iter. + iterator tmp_iter(iter--); + DCHECK(iter.node->leaf()); + DCHECK(!compare_keys(tmp_iter.key(), iter.key())); + iter.node->value_swap(iter.position, tmp_iter.node, tmp_iter.position); + internal_delete = true; + --*mutable_size(); + } else if (!root()->leaf()) { + --*mutable_size(); + } + + // Delete the key from the leaf. + iter.node->remove_value(iter.position); + + // We want to return the next value after the one we just erased. If we + // erased from an internal node (internal_delete == true), then the next + // value is ++(++iter). If we erased from a leaf node (internal_delete == + // false) then the next value is ++iter. Note that ++iter may point to an + // internal node and the value in the internal node may move to a leaf node + // (iter.node) when rebalancing is performed at the leaf level. + + // Merge/rebalance as we walk back up the tree. + iterator res(iter); + for (;;) { + if (iter.node == root()) { + try_shrink(); + if (empty()) { + return end(); + } + break; + } + if (iter.node->count() >= kMinNodeValues) { + break; + } + bool merged = try_merge_or_rebalance(&iter); + if (iter.node->leaf()) { + res = iter; + } + if (!merged) { + break; + } + iter.node = iter.node->parent(); + } + + // Adjust our return value. If we're pointing at the end of a node, advance + // the iterator. + if (res.position == res.node->count()) { + res.position = res.node->count() - 1; + ++res; + } + // If we erased from an internal node, advance the iterator. + if (internal_delete) { + ++res; + } + return res; +} + +template +int btree

::erase(iterator begin, iterator end) { + int count = distance(begin, end); + for (int i = 0; i < count; i++) { + begin = erase(begin); + } + return count; +} + +template +int btree

::erase_unique(const key_type &key) { + iterator iter = internal_find_unique(key, iterator(root(), 0)); + if (!iter.node) { + // The key doesn't exist in the tree, return nothing done. + return 0; + } + erase(iter); + return 1; +} + +template +int btree

::erase_multi(const key_type &key) { + iterator begin = internal_lower_bound(key, iterator(root(), 0)); + if (!begin.node) { + // The key doesn't exist in the tree, return nothing done. + return 0; + } + // Delete all of the keys between begin and upper_bound(key). + iterator end = internal_end( + internal_upper_bound(key, iterator(root(), 0))); + return erase(begin, end); +} + +template +void btree

::clear() { + if (root() != NULL) { + internal_clear(root()); + } + *mutable_root() = NULL; +} + +template +void btree

::swap(self_type &x) { + ::swap(static_cast(*this), static_cast(x)); + ::swap(root_, x.root_); +} + +template +void btree

::verify() const { + if (root() != NULL) { + CHECK_EQ(size(), internal_verify(root(), NULL, NULL)); + CHECK_EQ(leftmost(), (++const_iterator(root(), -1)).node); + CHECK_EQ(rightmost(), (--const_iterator(root(), root()->count())).node); + CHECK(leftmost()->leaf()); + CHECK(rightmost()->leaf()); + } else { + CHECK_EQ(size(), 0); + CHECK(leftmost() == NULL); + CHECK(rightmost() == NULL); + } +} + +template +void btree

::rebalance_or_split(iterator *iter) { + node_type *&node = iter->node; + int &insert_position = iter->position; + DCHECK_EQ(node->count(), node->max_count()); + + // First try to make room on the node by rebalancing. + node_type *parent = node->parent(); + if (node != root()) { + if (node->position() > 0) { + // Try rebalancing with our left sibling. + node_type *left = parent->child(node->position() - 1); + if (left->count() < left->max_count()) { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the end of the right node then we bias rebalancing to + // fill up the left node. + int to_move = (left->max_count() - left->count()) / + (1 + (insert_position < left->max_count())); + to_move = max(1, to_move); + + if (((insert_position - to_move) >= 0) || + ((left->count() + to_move) < left->max_count())) { + left->rebalance_right_to_left(node, to_move); + + DCHECK_EQ(node->max_count() - node->count(), to_move); + insert_position = insert_position - to_move; + if (insert_position < 0) { + insert_position = insert_position + left->count() + 1; + node = left; + } + + DCHECK_LT(node->count(), node->max_count()); + return; + } + } + } + + if (node->position() < parent->count()) { + // Try rebalancing with our right sibling. + node_type *right = parent->child(node->position() + 1); + if (right->count() < right->max_count()) { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the beginning of the left node then we bias rebalancing + // to fill up the right node. + int to_move = (right->max_count() - right->count()) / + (1 + (insert_position > 0)); + to_move = max(1, to_move); + + if ((insert_position <= (node->count() - to_move)) || + ((right->count() + to_move) < right->max_count())) { + node->rebalance_left_to_right(right, to_move); + + if (insert_position > node->count()) { + insert_position = insert_position - node->count() - 1; + node = right; + } + + DCHECK_LT(node->count(), node->max_count()); + return; + } + } + } + + // Rebalancing failed, make sure there is room on the parent node for a new + // value. + if (parent->count() == parent->max_count()) { + iterator parent_iter(node->parent(), node->position()); + rebalance_or_split(&parent_iter); + } + } else { + // Rebalancing not possible because this is the root node. + if (root()->leaf()) { + // The root node is currently a leaf node: create a new root node and set + // the current root node as the child of the new root. + parent = new_internal_root_node(); + parent->set_child(0, root()); + *mutable_root() = parent; + DCHECK(*mutable_rightmost() == parent->child(0)); + } else { + // The root node is an internal node. We do not want to create a new root + // node because the root node is special and holds the size of the tree + // and a pointer to the rightmost node. So we create a new internal node + // and move all of the items on the current root into the new node. + parent = new_internal_node(parent); + parent->set_child(0, parent); + parent->swap(root()); + node = parent; + } + } + + // Split the node. + node_type *split_node; + if (node->leaf()) { + split_node = new_leaf_node(parent); + node->split(split_node, insert_position); + if (rightmost() == node) { + *mutable_rightmost() = split_node; + } + } else { + split_node = new_internal_node(parent); + node->split(split_node, insert_position); + } + + if (insert_position > node->count()) { + insert_position = insert_position - node->count() - 1; + node = split_node; + } +} + +template +void btree

::merge_nodes(node_type *left, node_type *right) { + left->merge(right); + if (right->leaf()) { + if (rightmost() == right) { + *mutable_rightmost() = left; + } + delete_leaf_node(right); + } else { + delete_internal_node(right); + } +} + +template +bool btree

::try_merge_or_rebalance(iterator *iter) { + node_type *parent = iter->node->parent(); + if (iter->node->position() > 0) { + // Try merging with our left sibling. + node_type *left = parent->child(iter->node->position() - 1); + if ((1 + left->count() + iter->node->count()) <= left->max_count()) { + iter->position += 1 + left->count(); + merge_nodes(left, iter->node); + iter->node = left; + return true; + } + } + if (iter->node->position() < parent->count()) { + // Try merging with our right sibling. + node_type *right = parent->child(iter->node->position() + 1); + if ((1 + iter->node->count() + right->count()) <= right->max_count()) { + merge_nodes(iter->node, right); + return true; + } + // Try rebalancing with our right sibling. We don't perform rebalancing if + // we deleted the first element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the front of the tree. + if ((right->count() > kMinNodeValues) && + ((iter->node->count() == 0) || + (iter->position > 0))) { + int to_move = (right->count() - iter->node->count()) / 2; + to_move = min(to_move, right->count() - 1); + iter->node->rebalance_right_to_left(right, to_move); + return false; + } + } + if (iter->node->position() > 0) { + // Try rebalancing with our left sibling. We don't perform rebalancing if + // we deleted the last element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the back of the tree. + node_type *left = parent->child(iter->node->position() - 1); + if ((left->count() > kMinNodeValues) && + ((iter->node->count() == 0) || + (iter->position < iter->node->count()))) { + int to_move = (left->count() - iter->node->count()) / 2; + to_move = min(to_move, left->count() - 1); + left->rebalance_left_to_right(iter->node, to_move); + iter->position += to_move; + return false; + } + } + return false; +} + +template +void btree

::try_shrink() { + if (root()->count() > 0) { + return; + } + // Deleted the last item on the root node, shrink the height of the tree. + if (root()->leaf()) { + DCHECK_EQ(size(), 0); + delete_leaf_node(root()); + *mutable_root() = NULL; + } else { + node_type *child = root()->child(0); + if (child->leaf()) { + // The child is a leaf node so simply make it the root node in the tree. + child->make_root(); + delete_internal_root_node(); + *mutable_root() = child; + } else { + // The child is an internal node. We want to keep the existing root node + // so we move all of the values from the child node into the existing + // (empty) root node. + child->swap(root()); + delete_internal_node(child); + } + } +} + +template template +inline IterType btree

::internal_last(IterType iter) { + while (iter.node && iter.position == iter.node->count()) { + iter.position = iter.node->position(); + iter.node = iter.node->parent(); + if (iter.node->leaf()) { + iter.node = NULL; + } + } + return iter; +} + +template +inline typename btree

::iterator +btree

::internal_insert(iterator iter, const value_type &v) { + if (!iter.node->leaf()) { + // We can't insert on an internal node. Instead, we'll insert after the + // previous value which is guaranteed to be on a leaf node. + --iter; + ++iter.position; + } + if (iter.node->count() == iter.node->max_count()) { + // Make room in the leaf for the new item. + if (iter.node->max_count() < kNodeValues) { + // Insertion into the root where the root is smaller that the full node + // size. Simply grow the size of the root node. + DCHECK(iter.node == root()); + iter.node = new_leaf_root_node( + min(kNodeValues, 2 * iter.node->max_count())); + iter.node->swap(root()); + delete_leaf_node(root()); + *mutable_root() = iter.node; + } else { + rebalance_or_split(&iter); + ++*mutable_size(); + } + } else if (!root()->leaf()) { + ++*mutable_size(); + } + iter.node->insert_value(iter.position, v); + return iter; +} + +template template +inline pair btree

::internal_locate( + const key_type &key, IterType iter) const { + return internal_locate_type::dispatch(key, *this, iter); +} + +template template +inline pair btree

::internal_locate_plain_compare( + const key_type &key, IterType iter) const { + for (;;) { + iter.position = iter.node->lower_bound(key, key_comp()); + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return make_pair(iter, 0); +} + +template template +inline pair btree

::internal_locate_compare_to( + const key_type &key, IterType iter) const { + for (;;) { + int res = iter.node->lower_bound(key, key_comp()); + iter.position = res & kMatchMask; + if (res & kExactMatch) { + return make_pair(iter, static_cast(kExactMatch)); + } + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return make_pair(iter, -kExactMatch); +} + +template template +IterType btree

::internal_lower_bound( + const key_type &key, IterType iter) const { + if (iter.node) { + for (;;) { + iter.position = + iter.node->lower_bound(key, key_comp()) & kMatchMask; + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + iter = internal_last(iter); + } + return iter; +} + +template template +IterType btree

::internal_upper_bound( + const key_type &key, IterType iter) const { + if (iter.node) { + for (;;) { + iter.position = iter.node->upper_bound(key, key_comp()); + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + iter = internal_last(iter); + } + return iter; +} + +template template +IterType btree

::internal_find_unique( + const key_type &key, IterType iter) const { + if (iter.node) { + pair res = internal_locate(key, iter); + if (res.second == kExactMatch) { + return res.first; + } + if (!res.second) { + iter = internal_last(res.first); + if (iter.node && !compare_keys(key, iter.key())) { + return iter; + } + } + } + return IterType(NULL, 0); +} + +template template +IterType btree

::internal_find_multi( + const key_type &key, IterType iter) const { + if (iter.node) { + iter = internal_lower_bound(key, iter); + if (iter.node) { + iter = internal_last(iter); + if (iter.node && !compare_keys(key, iter.key())) { + return iter; + } + } + } + return IterType(NULL, 0); +} + +template +void btree

::internal_clear(node_type *node) { + if (!node->leaf()) { + for (int i = 0; i <= node->count(); ++i) { + internal_clear(node->child(i)); + } + if (node == root()) { + delete_internal_root_node(); + } else { + delete_internal_node(node); + } + } else { + delete_leaf_node(node); + } +} + +template +void btree

::internal_dump( + ostream &os, const node_type *node, int level) const { + for (int i = 0; i < node->count(); ++i) { + if (!node->leaf()) { + internal_dump(os, node->child(i), level + 1); + } + for (int j = 0; j < level; ++j) { + os << " "; + } + os << node->key(i) << " [" << level << "]\n"; + } + if (!node->leaf()) { + internal_dump(os, node->child(node->count()), level + 1); + } +} + +template +int btree

::internal_verify( + const node_type *node, const key_type *lo, const key_type *hi) const { + CHECK_GT(node->count(), 0); + CHECK_LE(node->count(), node->max_count()); + if (lo) { + CHECK(!compare_keys(node->key(0), *lo)); + } + if (hi) { + CHECK(!compare_keys(*hi, node->key(node->count() - 1))); + } + for (int i = 1; i < node->count(); ++i) { + CHECK(!compare_keys(node->key(i), node->key(i - 1))); + } + int count = node->count(); + if (!node->leaf()) { + for (int i = 0; i <= node->count(); ++i) { + CHECK(node->child(i) != NULL); + CHECK_EQ(node->child(i)->parent(), node); + CHECK_EQ(node->child(i)->position(), i); + count += internal_verify( + node->child(i), + (i == 0) ? lo : &node->key(i - 1), + (i == node->count()) ? hi : &node->key(i)); + } + } + return count; +} + +} // namespace btree +} // namespace util + +DEFINE_GDB_AUTO_SCRIPT("util/btree/btree_printer.py") + +#endif // UTIL_BTREE_BTREE_H__ diff --git a/btree_bench.cc b/btree_bench.cc new file mode 100644 index 00000000..c67dd664 --- /dev/null +++ b/btree_bench.cc @@ -0,0 +1,483 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) + +#include +#include +#include +#include +#include +#include +#include + +#include "base/commandlineflags.h" +#include "base/init_google.h" +#include "base/integral_types.h" +#include "base/logging.h" +#include "base/stringprintf.h" +#include "base/type_traits.h" +#include "strings/cord.h" +#include "testing/base/public/benchmark.h" +#include "testing/base/public/googletest.h" +#include "util/btree/btree_map.h" +#include "util/btree/btree_set.h" +#include "util/btree/btree_test.h" +#include "util/random/acmrandom.h" + +DECLARE_int32(benchmark_max_iters); + +namespace util { +namespace btree { +namespace { + +// Benchmark insertion of values into a container. +template +void BM_Insert(int n) { + typedef typename base::remove_const::type V; + typename KeyOfValue::type key_of_value; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + + T container; + vector values = GenerateValues(FLAGS_benchmark_values); + for (int i = 0; i < values.size(); i++) { + container.insert(values[i]); + } + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); + + for (int i = 0; i < n; ) { + // Remove and re-insert 10% of the keys + int m = min(n - i, FLAGS_benchmark_values / 10); + + for (int j = i; j < i + m; j++) { + int x = j % FLAGS_benchmark_values; + container.erase(key_of_value(values[x])); + } + + StartBenchmarkTiming(); + + for (int j = i; j < i + m; j++) { + int x = j % FLAGS_benchmark_values; + container.insert(values[x]); + } + + StopBenchmarkTiming(); + + i += m; + } +} + +// Benchmark lookup of values in a container. +template +void BM_Lookup(int n) { + typedef typename base::remove_const::type V; + typename KeyOfValue::type key_of_value; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + + T container; + vector values = GenerateValues(FLAGS_benchmark_values); + + for (int i = 0; i < values.size(); i++) { + container.insert(values[i]); + } + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); + + V r = V(); + + StartBenchmarkTiming(); + + for (int i = 0; i < n; i++) { + int m = i % values.size(); + r = *container.find(key_of_value(values[m])); + } + + StopBenchmarkTiming(); + + VLOG(4) << r; // Keep compiler from optimizing away r. +} + +// Benchmark lookup of values in a full container, meaning that values +// are inserted in-order to take advantage of biased insertion, which +// yields a full tree. +template +void BM_FullLookup(int n) { + typedef typename base::remove_const::type V; + typename KeyOfValue::type key_of_value; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + + T container; + vector values = GenerateValues(FLAGS_benchmark_values); + vector sorted(values); + sort(sorted.begin(), sorted.end()); + + for (int i = 0; i < sorted.size(); i++) { + container.insert(sorted[i]); + } + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); + + V r = V(); + + StartBenchmarkTiming(); + + for (int i = 0; i < n; i++) { + int m = i % values.size(); + r = *container.find(key_of_value(values[m])); + } + + StopBenchmarkTiming(); + + VLOG(4) << r; // Keep compiler from optimizing away r. +} + +// Benchmark deletion of values from a container. +template +void BM_Delete(int n) { + typedef typename base::remove_const::type V; + typename KeyOfValue::type key_of_value; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + + T container; + vector values = GenerateValues(FLAGS_benchmark_values); + for (int i = 0; i < values.size(); i++) { + container.insert(values[i]); + } + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); + + for (int i = 0; i < n; ) { + // Remove and re-insert 10% of the keys + int m = min(n - i, FLAGS_benchmark_values / 10); + + StartBenchmarkTiming(); + + for (int j = i; j < i + m; j++) { + int x = j % FLAGS_benchmark_values; + container.erase(key_of_value(values[x])); + } + + StopBenchmarkTiming(); + + for (int j = i; j < i + m; j++) { + int x = j % FLAGS_benchmark_values; + container.insert(values[x]); + } + + i += m; + } +} + +// Benchmark steady-state insert (into first half of range) and remove +// (from second second half of range), treating the container +// approximately like a queue with log-time access for all elements. +// This benchmark does not test the case where insertion and removal +// happen in the same region of the tree. This benchmark counts two +// value constructors. +template +void BM_QueueAddRem(int n) { + typedef typename base::remove_const::type V; + typename KeyOfValue::type key_of_value; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + CHECK(FLAGS_benchmark_values % 2 == 0); + + T container; + + const int half = FLAGS_benchmark_values / 2; + vector remove_keys(half); + vector add_keys(half); + + for (int i = 0; i < half; i++) { + remove_keys[i] = i; + add_keys[i] = i; + } + + ACMRandom rand(FLAGS_test_random_seed); + + random_shuffle(remove_keys.begin(), remove_keys.end(), rand); + random_shuffle(add_keys.begin(), add_keys.end(), rand); + + Generator g(FLAGS_benchmark_values + FLAGS_benchmark_max_iters); + + for (int i = 0; i < half; i++) { + container.insert(g(add_keys[i])); + container.insert(g(half + remove_keys[i])); + } + + // There are three parts each of size "half": + // 1 is being deleted from [offset - half, offset) + // 2 is standing [offset, offset + half) + // 3 is being inserted into [offset + half, offset + 2 * half) + int offset = 0; + + StartBenchmarkTiming(); + + for (int i = 0; i < n; i++) { + int idx = i % half; + + if (idx == 0) { + StopBenchmarkTiming(); + random_shuffle(remove_keys.begin(), remove_keys.end(), rand); + random_shuffle(add_keys.begin(), add_keys.end(), rand); + offset += half; + StartBenchmarkTiming(); + } + + int e = container.erase(key_of_value(g(offset - half + remove_keys[idx]))); + DCHECK(e == 1); + container.insert(g(offset + half + add_keys[idx])); + } + + StopBenchmarkTiming(); + + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); +} + +// Mixed insertion and deletion in the same range using pre-constructed values. +template +void BM_MixedAddRem(int n) { + typedef typename base::remove_const::type V; + typename KeyOfValue::type key_of_value; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + CHECK(FLAGS_benchmark_values % 2 == 0); + + T container; + ACMRandom rand(FLAGS_test_random_seed); + + vector values = GenerateValues(FLAGS_benchmark_values * 2); + + // Create two random shuffles + vector remove_keys(FLAGS_benchmark_values); + vector add_keys(FLAGS_benchmark_values); + + // Insert the first half of the values (already in random order) + for (int i = 0; i < FLAGS_benchmark_values; i++) { + container.insert(values[i]); + + // remove_keys and add_keys will be swapped before each round, + // therefore fill add_keys here w/ the keys being inserted, so + // they'll be the first to be removed. + remove_keys[i] = i + FLAGS_benchmark_values; + add_keys[i] = i; + } + + StartBenchmarkTiming(); + + for (int i = 0; i < n; i++) { + int idx = i % FLAGS_benchmark_values; + + if (idx == 0) { + StopBenchmarkTiming(); + remove_keys.swap(add_keys); + random_shuffle(remove_keys.begin(), remove_keys.end(), rand); + random_shuffle(add_keys.begin(), add_keys.end(), rand); + StartBenchmarkTiming(); + } + + int e = container.erase(key_of_value(values[remove_keys[idx]])); + DCHECK(e == 1); + container.insert(values[add_keys[idx]]); + } + + StopBenchmarkTiming(); + + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); +} + +// Insertion at end, removal from the beginning. This benchmark +// counts two value constructors. +template +void BM_Fifo(int n) { + typedef typename base::remove_const::type V; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + + T container; + Generator g(FLAGS_benchmark_values + FLAGS_benchmark_max_iters); + + for (int i = 0; i < FLAGS_benchmark_values; i++) { + container.insert(g(i)); + } + + StartBenchmarkTiming(); + + for (int i = 0; i < n; i++) { + container.erase(container.begin()); + container.insert(container.end(), g(i + FLAGS_benchmark_values)); + } + + StopBenchmarkTiming(); + + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); +} + +// Iteration (forward) through the tree +template +void BM_FwdIter(int n) { + typedef typename base::remove_const::type V; + + // Disable timing while we perform some initialization. + StopBenchmarkTiming(); + + T container; + vector values = GenerateValues(FLAGS_benchmark_values); + + for (int i = 0; i < FLAGS_benchmark_values; i++) { + container.insert(values[i]); + } + + typename T::iterator iter; + + V r = V(); + + StartBenchmarkTiming(); + + for (int i = 0; i < n; i++) { + int idx = i % FLAGS_benchmark_values; + + if (idx == 0) { + iter = container.begin(); + } + r = *iter; + ++iter; + } + + StopBenchmarkTiming(); + + VLOG(4) << r; // Keep compiler from optimizing away r. + + SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); +} + +typedef set stl_set_int32; +typedef set stl_set_int64; +typedef set stl_set_string; +typedef set stl_set_cord; + +typedef map stl_map_int32; +typedef map stl_map_int64; +typedef map stl_map_string; +typedef map stl_map_cord; + +typedef multiset stl_multiset_int32; +typedef multiset stl_multiset_int64; +typedef multiset stl_multiset_string; +typedef multiset stl_multiset_cord; + +typedef multimap stl_multimap_int32; +typedef multimap stl_multimap_int64; +typedef multimap stl_multimap_string; +typedef multimap stl_multimap_cord; + +#define MY_BENCHMARK_TYPES2(value, name, size) \ + typedef btree ## _set, allocator, size> \ + btree ## _ ## size ## _set_ ## name; \ + typedef btree ## _map, allocator, size> \ + btree ## _ ## size ## _map_ ## name; \ + typedef btree ## _multiset, allocator, size> \ + btree ## _ ## size ## _multiset_ ## name; \ + typedef btree ## _multimap, allocator, size> \ + btree ## _ ## size ## _multimap_ ## name + +#define MY_BENCHMARK_TYPES(value, name) \ + MY_BENCHMARK_TYPES2(value, name, 128); \ + MY_BENCHMARK_TYPES2(value, name, 160); \ + MY_BENCHMARK_TYPES2(value, name, 192); \ + MY_BENCHMARK_TYPES2(value, name, 224); \ + MY_BENCHMARK_TYPES2(value, name, 256); \ + MY_BENCHMARK_TYPES2(value, name, 288); \ + MY_BENCHMARK_TYPES2(value, name, 320); \ + MY_BENCHMARK_TYPES2(value, name, 352); \ + MY_BENCHMARK_TYPES2(value, name, 384); \ + MY_BENCHMARK_TYPES2(value, name, 416); \ + MY_BENCHMARK_TYPES2(value, name, 448); \ + MY_BENCHMARK_TYPES2(value, name, 480); \ + MY_BENCHMARK_TYPES2(value, name, 512) + +MY_BENCHMARK_TYPES(int32, int32); +MY_BENCHMARK_TYPES(int64, int64); +MY_BENCHMARK_TYPES(string, string); +MY_BENCHMARK_TYPES(Cord, cord); + +#define MY_BENCHMARK4(type, name, func) \ + void BM_ ## type ## _ ## name(int n) { BM_ ## func (n); } \ + BENCHMARK(BM_ ## type ## _ ## name) + +// Define NODESIZE_TESTING when running btree_perf.py. You need to do +// a local build or raise the distcc timeout, it takes about 5 minutes +// to build: +// +// blaze build --copts=-DNODESIZE_TESTING --cc_strategy=local +// --compilation_mode=opt util/btree/btree_test + +#ifdef NODESIZE_TESTING +#define MY_BENCHMARK3(tree, type, name, func) \ + MY_BENCHMARK4(tree ## _128_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _160_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _192_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _224_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _256_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _288_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _320_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _352_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _384_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _416_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _448_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _480_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _512_ ## type, name, func) +#else +#define MY_BENCHMARK3(tree, type, name, func) \ + MY_BENCHMARK4(tree ## _256_ ## type, name, func) +#endif + +#define MY_BENCHMARK2(type, name, func) \ + MY_BENCHMARK4(stl_ ## type, name, func); \ + MY_BENCHMARK3(btree, type, name, func) + +#define MY_BENCHMARK(type) \ + MY_BENCHMARK2(type, insert, Insert); \ + MY_BENCHMARK2(type, lookup, Lookup); \ + MY_BENCHMARK2(type, fulllookup, FullLookup); \ + MY_BENCHMARK2(type, delete, Delete); \ + MY_BENCHMARK2(type, queueaddrem, QueueAddRem); \ + MY_BENCHMARK2(type, mixedaddrem, MixedAddRem); \ + MY_BENCHMARK2(type, fifo, Fifo); \ + MY_BENCHMARK2(type, fwditer, FwdIter) + +MY_BENCHMARK(set_int32); +MY_BENCHMARK(map_int32); +MY_BENCHMARK(set_int64); +MY_BENCHMARK(map_int64); +MY_BENCHMARK(set_string); +MY_BENCHMARK(map_string); +MY_BENCHMARK(set_cord); +MY_BENCHMARK(map_cord); + +MY_BENCHMARK(multiset_int32); +MY_BENCHMARK(multimap_int32); +MY_BENCHMARK(multiset_int64); +MY_BENCHMARK(multimap_int64); +MY_BENCHMARK(multiset_string); +MY_BENCHMARK(multimap_string); +MY_BENCHMARK(multiset_cord); +MY_BENCHMARK(multimap_cord); + +} // namespace +} // namespace btree +} // namespace util + +int main(int argc, char **argv) { + FLAGS_logtostderr = true; + InitGoogle(argv[0], &argc, &argv, true); + RunSpecifiedBenchmarks(); + return 0; +} diff --git a/btree_container.h b/btree_container.h new file mode 100644 index 00000000..6703d1bf --- /dev/null +++ b/btree_container.h @@ -0,0 +1,325 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) + +#ifndef UTIL_BTREE_BTREE_CONTAINER_H__ +#define UTIL_BTREE_BTREE_CONTAINER_H__ + +#include +#include + +#include "util/btree/btree.h" // IWYU pragma: export + +namespace util { +namespace btree { + +// A common base class for btree_set, btree_map, btree_multiset and +// btree_multimap. +template +class btree_container { + typedef btree_container self_type; + + public: + typedef typename Tree::params_type params_type; + typedef typename Tree::key_type key_type; + typedef typename Tree::value_type value_type; + typedef typename Tree::key_compare key_compare; + typedef typename Tree::allocator_type allocator_type; + typedef typename Tree::pointer pointer; + typedef typename Tree::const_pointer const_pointer; + typedef typename Tree::reference reference; + typedef typename Tree::const_reference const_reference; + typedef typename Tree::size_type size_type; + typedef typename Tree::difference_type difference_type; + typedef typename Tree::iterator iterator; + typedef typename Tree::const_iterator const_iterator; + typedef typename Tree::reverse_iterator reverse_iterator; + typedef typename Tree::const_reverse_iterator const_reverse_iterator; + + public: + // Default constructor. + btree_container(const key_compare &comp, const allocator_type &alloc) + : tree_(comp, alloc) { + } + + // Copy constructor. + btree_container(const self_type &x) + : tree_(x.tree_) { + } + + // Iterator routines. + iterator begin() { return tree_.begin(); } + const_iterator begin() const { return tree_.begin(); } + iterator end() { return tree_.end(); } + const_iterator end() const { return tree_.end(); } + reverse_iterator rbegin() { return tree_.rbegin(); } + const_reverse_iterator rbegin() const { return tree_.rbegin(); } + reverse_iterator rend() { return tree_.rend(); } + const_reverse_iterator rend() const { return tree_.rend(); } + + // Lookup routines. + iterator lower_bound(const key_type &key) { + return tree_.lower_bound(key); + } + const_iterator lower_bound(const key_type &key) const { + return tree_.lower_bound(key); + } + iterator upper_bound(const key_type &key) { + return tree_.upper_bound(key); + } + const_iterator upper_bound(const key_type &key) const { + return tree_.upper_bound(key); + } + pair equal_range(const key_type &key) { + return tree_.equal_range(key); + } + pair equal_range(const key_type &key) const { + return tree_.equal_range(key); + } + + // Utility routines. + void clear() { + tree_.clear(); + } + void swap(self_type &x) { + tree_.swap(x.tree_); + } + void dump(ostream &os) const { + tree_.dump(os); + } + void verify() const { + tree_.verify(); + } + + // Size routines. + size_type size() const { return tree_.size(); } + size_type max_size() const { return tree_.max_size(); } + bool empty() const { return tree_.empty(); } + size_type height() const { return tree_.height(); } + size_type internal_nodes() const { return tree_.internal_nodes(); } + size_type leaf_nodes() const { return tree_.leaf_nodes(); } + size_type nodes() const { return tree_.nodes(); } + size_type bytes_used() const { return tree_.bytes_used(); } + static double average_bytes_per_value() { + return Tree::average_bytes_per_value(); + } + double fullness() const { return tree_.fullness(); } + double overhead() const { return tree_.overhead(); } + + protected: + Tree tree_; +}; + +template +inline ostream& operator<<(ostream &os, const btree_container &b) { + b.dump(os); + return os; +} + +// A common base class for btree_set and safe_btree_set. +template +class btree_unique_container : public btree_container { + typedef btree_unique_container self_type; + typedef btree_container super_type; + + public: + typedef typename Tree::key_type key_type; + typedef typename Tree::value_type value_type; + typedef typename Tree::size_type size_type; + typedef typename Tree::key_compare key_compare; + typedef typename Tree::allocator_type allocator_type; + typedef typename Tree::iterator iterator; + typedef typename Tree::const_iterator const_iterator; + + public: + // Default constructor. + btree_unique_container(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_unique_container(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_unique_container(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + insert(b, e); + } + + // Lookup routines. + iterator find(const key_type &key) { + return this->tree_.find_unique(key); + } + const_iterator find(const key_type &key) const { + return this->tree_.find_unique(key); + } + size_type count(const key_type &key) const { + return this->tree_.count_unique(key); + } + + // Insertion routines. + pair insert(const value_type &x) { + return this->tree_.insert_unique(x); + } + iterator insert(iterator position, const value_type &x) { + return this->tree_.insert_unique(position, x); + } + template + void insert(InputIterator b, InputIterator e) { + this->tree_.insert_unique(b, e); + } + + // Deletion routines. + int erase(const key_type &key) { + return this->tree_.erase_unique(key); + } + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + iterator erase(const iterator &iter) { + return this->tree_.erase(iter); + } + void erase(const iterator &first, const iterator &last) { + this->tree_.erase(first, last); + } +}; + +// A common base class for btree_map and safe_btree_map. +template +class btree_map_container : public btree_unique_container { + typedef btree_map_container self_type; + typedef btree_unique_container super_type; + + public: + typedef typename Tree::key_type key_type; + typedef typename Tree::data_type data_type; + typedef typename Tree::value_type value_type; + typedef typename Tree::mapped_type mapped_type; + typedef typename Tree::key_compare key_compare; + typedef typename Tree::allocator_type allocator_type; + + private: + // A pointer-like object which only generates its value when + // dereferenced. Used by operator[] to avoid constructing an empty data_type + // if the key already exists in the map. + struct generate_value { + generate_value(const key_type &k) + : key(k) { + } + value_type operator*() const { + return make_pair(key, data_type()); + } + const key_type &key; + }; + + public: + // Default constructor. + btree_map_container(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_map_container(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_map_container(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + insert(b, e); + } + + // Insertion routines. + data_type& operator[](const key_type &key) { + return this->tree_.insert_unique(key, generate_value(key)).first->second; + } +}; + +// A common base class for btree_multiset and btree_multimap. +template +class btree_multi_container : public btree_container { + typedef btree_multi_container self_type; + typedef btree_container super_type; + + public: + typedef typename Tree::key_type key_type; + typedef typename Tree::value_type value_type; + typedef typename Tree::size_type size_type; + typedef typename Tree::key_compare key_compare; + typedef typename Tree::allocator_type allocator_type; + typedef typename Tree::iterator iterator; + typedef typename Tree::const_iterator const_iterator; + + public: + // Default constructor. + btree_multi_container(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_multi_container(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_multi_container(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(b, e, comp, alloc) { + insert(b, e); + } + + // Lookup routines. + iterator find(const key_type &key) { + return this->tree_.find_multi(key); + } + const_iterator find(const key_type &key) const { + return this->tree_.find_multi(key); + } + size_type count(const key_type &key) const { + return this->tree_.count_multi(key); + } + + // Insertion routines. + iterator insert(const value_type &x) { + return this->tree_.insert_multi(x); + } + iterator insert(iterator position, const value_type &x) { + return this->tree_.insert_multi(position, x); + } + template + void insert(InputIterator b, InputIterator e) { + this->tree_.insert_multi(b, e); + } + + // Deletion routines. + int erase(const key_type &key) { + return this->tree_.erase_multi(key); + } + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + iterator erase(const iterator &iter) { + return this->tree_.erase(iter); + } + void erase(const iterator &first, const iterator &last) { + this->tree_.erase(first, last); + } +}; + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_BTREE_CONTAINER_H__ diff --git a/btree_map.h b/btree_map.h new file mode 100644 index 00000000..3c61a7f2 --- /dev/null +++ b/btree_map.h @@ -0,0 +1,122 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) +// +// A btree_map<> implements the STL unique sorted associative container +// interface and the pair associative container interface (a.k.a map<>) using a +// btree. A btree_multimap<> implements the STL multiple sorted associative +// container interface and the pair associtive container interface (a.k.a +// multimap<>) using a btree. See btree.h for details of the btree +// implementation and caveats. + +#ifndef UTIL_BTREE_BTREE_MAP_H__ +#define UTIL_BTREE_BTREE_MAP_H__ + +#include +#include +#include +#include +#include + +#include "util/btree/btree.h" // IWYU pragma: export +#include "util/btree/btree_container.h" // IWYU pragma: export + +namespace util { +namespace btree { + +// The btree_map class is needed mainly for it's constructors. +template , + typename Alloc = std::allocator >, + int TargetNodeSize = 256> +class btree_map : public btree_map_container< + btree > > { + + typedef btree_map self_type; + typedef btree_map_params< + Key, Value, Compare, Alloc, TargetNodeSize> params_type; + typedef btree btree_type; + typedef btree_map_container super_type; + + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + + public: + // Default constructor. + btree_map(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_map(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_map(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } +}; + +template +inline void swap(btree_map &x, + btree_map &y) { + x.swap(y); +} + +// The btree_multimap class is needed mainly for it's constructors. +template , + typename Alloc = std::allocator >, + int TargetNodeSize = 256> +class btree_multimap : public btree_multi_container< + btree > > { + + typedef btree_multimap self_type; + typedef btree_map_params< + Key, Value, Compare, Alloc, TargetNodeSize> params_type; + typedef btree btree_type; + typedef btree_multi_container super_type; + + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + typedef typename btree_type::data_type data_type; + typedef typename btree_type::mapped_type mapped_type; + + public: + // Default constructor. + btree_multimap(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_multimap(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_multimap(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(b, e, comp, alloc) { + } +}; + +template +inline void swap(btree_multimap &x, + btree_multimap &y) { + x.swap(y); +} + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_BTREE_MAP_H__ diff --git a/btree_nc.cc b/btree_nc.cc new file mode 100644 index 00000000..b24c312d --- /dev/null +++ b/btree_nc.cc @@ -0,0 +1,50 @@ +// Copyright 2009 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) + +#include "util/btree/btree_set.h" + +namespace { + +template +struct Compare { + R operator()(int a, int b) const { return reinterpret_cast(a < b); } +}; + +template +struct CompareTo : public util::btree::btree_key_compare_to_tag { + R operator()(int a, int b) const { return reinterpret_cast(a < b); } +}; + +#define TEST_COMPARE(r) \ + void TestCompile() { \ + util::btree::btree_set > s; \ + } + +#define TEST_COMPARE_TO(r) \ + void TestCompile() { \ + util::btree::btree_set > s; \ + } + +#if defined(TEST_bool) +TEST_COMPARE(bool); +#elif defined(TEST_int) +TEST_COMPARE(int); +#elif defined(TEST_float) +TEST_COMPARE(float); +#elif defined(TEST_pointer) +TEST_COMPARE(void*); +#elif defined(TEST_compare_to_bool) +TEST_COMPARE_TO(bool); +#elif defined(TEST_compare_to_int) +TEST_COMPARE_TO(int); +#elif defined(TEST_compare_to_float) +TEST_COMPARE_TO(float); +#elif defined(TEST_compare_to_pointer) +TEST_COMPARE_TO(pointer); +#endif + +} // namespace + +int main() { + return 1; +} diff --git a/btree_nc_test.py b/btree_nc_test.py new file mode 100755 index 00000000..056a75d2 --- /dev/null +++ b/btree_nc_test.py @@ -0,0 +1,83 @@ +#!/usr/bin/python2.4 +# +# Copyright 2006 Google Inc. All Rights Reserved. + +"""Negative compilation unit test for btree.h. +""" + +__author__ = 'pmattis@google.com (Peter Mattis)' + +import os +from google3.testing.pybase import googletest +from google3.testing.pybase import fake_target_util +from google3.pyglib import flags + +_FLAGS = flags.FLAGS + + +class BtreeNegativeUnitTest(googletest.TestCase): + """Negative compilation tests for btree.h""" + + def testCompilerErrors(self): + """Runs a list of tests to verify that erroneous code leads to + expected compiler messages.""" + + # Defines a list of test specs, where each element is a tuple + # (test name, list of regexes for matching the compiler errors). + test_specs = [ + # Test that bool works as a return type for key comparison. + ('bool', None), # None means compilation should succeed. + + # Test that int does not work as a return type for key comparison. + ('int', + [r'error: creating array with negative size', # for gcc + r'', # for icc + ]), + + # Test that float does not work as a return type for key comparison. + ('float', + [r'error: creating array with negative size', # for gcc + r'', # for icc + ]), + + # Test that void* does not work as a return type for key comparison. + ('pointer', + [r'error: creating array with negative size', # for gcc + r'', # for icc + ]), + + # Test that bool does not work as a return type for compare-to + # comparison. + ('compare_to_bool', + [r'error: creating array with negative size', # for gcc + r'', # for icc + ]), + + # Test that int works as a return type for compare-to comparison. + ('compare_to_int', None), # None means compilation should succeed. + + # Test that float does not work as a return type for compare-to + # comparison. + ('compare_to_float', + [r'error: creating array with negative size', # for gcc + r'', # for icc + ]), + + # Test that void* does not work as a return type for compare-to + # comparison. + ('compare_to_pointer', + [r'error: creating array with negative size', # for gcc + r'', # for icc + ]), + ] + + # Runs the list of tests. + fake_target_util.AssertCcCompilerErrors( + self, + 'google3/util/btree/btree_nc', # path to the fake target file. + 'btree_nc.o', # name of the target to build. + test_specs) + + +if __name__ == '__main__': + googletest.main() diff --git a/btree_printer.py b/btree_printer.py new file mode 100755 index 00000000..8f233401 --- /dev/null +++ b/btree_printer.py @@ -0,0 +1,125 @@ +#!/usr/bin/python2.4 +# Copyright 2011 Google Inc. All Rights Reserved. +# GDB support for pretty printing StringPiece. + +"""GDB pretty-printer for btrees.""" + +# This is a module provided by GDB. +# Ref: http://wiki/Main/Google3GDBScripts +import gdb +from google3.devtools.gdb.component import printing + + +class BaseBtreePrinter(object): + """btree pretty-printer, for util/btree.""" + + def __init__(self, typename, val): + self.typename = typename + self.val = val + + def display_hint(self): + return 'array' + + def _my_iter(self, node): + count = node['fields_']['count'] + if node['fields_']['leaf']: + for i in range(count): + key = node['fields_']['values'][i] + yield ('[item]', key) + else: + # recursive generators are annoying: we can't just recurse, we need + # to expand and yield the values. + for i in range(count+1): + child = node['fields_']['children'][i] + for v in self._my_iter(child.dereference()): + yield v + + def children(self): + if self.nelements() != 0: + return self._my_iter(self.val['root_']['data'].dereference()) + else: + return iter([]) + + def nelements(self): + if self.val['root_']['data'] != 0: + root_fields = self.val['root_']['data'].dereference()['fields_'] + if root_fields['leaf']: + return root_fields['count'] + else: + return root_fields['size'] + else: + return 0 + + def to_string(self): # pylint: disable-msg=C6409 + """GDB calls this to compute the pretty-printed form.""" + return '%s with %d elements' % (self.typename, self.nelements()) + + +class BtreePrinter(BaseBtreePrinter): + """btree<> pretty-printer, for util/btree.""" + + def __init__(self, val): + BaseBtreePrinter.__init__(self, 'btree', val) + + +class BtreeSetPrinter(BaseBtreePrinter): + """btree_set<> pretty-printer.""" + + def __init__(self, val): + BaseBtreePrinter.__init__(self, 'btree_set', val['tree_']) + + +class BtreeMultisetPrinter(BaseBtreePrinter): + """btree_multiset<> pretty-printer.""" + + def __init__(self, val): + BaseBtreePrinter.__init__(self, 'btree_multiset', val['tree_']) + + +class BaseBtreeMapPrinter(BaseBtreePrinter): + """btree maps pretty-printer.""" + + def __init__(self, typename, val): + BaseBtreePrinter.__init__(self, typename, val['tree_']) + + def display_hint(self): + return 'map' + + def _my_map_iter(self, g): + for (_, pair) in g: + yield ('[key]', pair['first']) + yield ('[value]', pair['second']) + + def children(self): + # we need to break apart the pairs and yield them separately + if self.nelements() != 0: + return self._my_map_iter(BaseBtreePrinter.children(self)) + else: + return iter([]) + + +class BtreeMapPrinter(BaseBtreeMapPrinter): + """btree_map<> pretty-printer.""" + + def __init__(self, val): + BaseBtreeMapPrinter.__init__(self, 'btree_map', val) + + +class BtreeMultimapPrinter(BaseBtreeMapPrinter): + """btree_multimap<> pretty-printer.""" + + def __init__(self, val): + BaseBtreeMapPrinter.__init__(self, 'btree_multimap', val) + + +if __name__ == '__main__': + printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree<.*>', + BtreePrinter) + printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_set<.*>', + BtreeSetPrinter) + printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_multiset<.*>', + BtreeMultisetPrinter) + printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_map<.*>', + BtreeMapPrinter) + printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_multimap<.*>', + BtreeMultimapPrinter) diff --git a/btree_printer_test.py b/btree_printer_test.py new file mode 100755 index 00000000..95bc351e --- /dev/null +++ b/btree_printer_test.py @@ -0,0 +1,92 @@ +#!/usr/bin/python2.4 +# +# Copyright 2011 Google Inc. All Rights Reserved. + +"""Tests for btree_printer.py gdb pretty printer.""" + +__author__ = "leg@google.com (Lawrence Greenfield)" + +from google3.pyglib import flags +from google3.testing.gdb import gdb_script_test_util +from google3.testing.pybase import googletest + +FLAGS = flags.FLAGS + + +class BtreePrinterTest(gdb_script_test_util.TestCase): + def testBtreeSet(self): + self.InitSession("btree_set", + "util/btree/btree_test_program") + self.RunTo("StopHereForDebugger") + + self.SetOption("print elements", 20) + self.TestPrintOutputMatches("*empty_set", + """btree_set with 0 elements""") + self.TestPrintOutputMatches("*small_set", + """btree_set with 10 elements = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}""") + self.TestPrintOutputMatches("*small_multiset", + """btree_multiset with 10 elements = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4}""") + self.TestPrintOutputMatches("*big_set", + """btree_set with 80 elements = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19...}""") + + self.RunSession() + + def testBtreeMap(self): + self.InitSession("btree_set", + "util/btree/btree_test_program") + self.RunTo("StopHereForDebugger") + + self.SetOption("print elements", 30) + self.TestPrintOutputMatches("*empty_map", + """btree_map with 0 elements""") + self.TestPrintOutputMatches("*small_map", + """btree_map with 10 elements = { + \\[0\\] = 0, + \\[1\\] = 13, + \\[2\\] = 26, + \\[3\\] = 39, + \\[4\\] = 52, + \\[5\\] = 65, + \\[6\\] = 78, + \\[7\\] = 91, + \\[8\\] = 104, + \\[9\\] = 117 +}""") + self.TestPrintOutputMatches("*small_multimap", + """btree_multimap with 10 elements = { + \\[0\\] = 0, + \\[0\\] = 1, + \\[1\\] = 2, + \\[1\\] = 3, + \\[2\\] = 4, + \\[2\\] = 5, + \\[3\\] = 6, + \\[3\\] = 7, + \\[4\\] = 8, + \\[4\\] = 9 +}""") + self.TestPrintOutputMatches("*big_map", + """btree_map with 80 elements = { + \\[0\\] = 0, + \\[1\\] = 7, + \\[2\\] = 14, + \\[3\\] = 21, + \\[4\\] = 28, + \\[5\\] = 35, + \\[6\\] = 42, + \\[7\\] = 49, + \\[8\\] = 56, + \\[9\\] = 63, + \\[10\\] = 70, + \\[11\\] = 77, + \\[12\\] = 84, + \\[13\\] = 91, + \\[14\\] = 98 + ... +}""") + + self.RunSession() + + +if __name__ == "__main__": + googletest.main() diff --git a/btree_set.h b/btree_set.h new file mode 100644 index 00000000..7b5ea88d --- /dev/null +++ b/btree_set.h @@ -0,0 +1,113 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) +// +// A btree_set<> implements the STL unique sorted associative container +// interface (a.k.a set<>) using a btree. A btree_multiset<> implements the STL +// multiple sorted associative container interface (a.k.a multiset<>) using a +// btree. See btree.h for details of the btree implementation and caveats. + +#ifndef UTIL_BTREE_BTREE_SET_H__ +#define UTIL_BTREE_BTREE_SET_H__ + +#include +#include +#include + +#include "util/btree/btree.h" // IWYU pragma: export +#include "util/btree/btree_container.h" // IWYU pragma: export + +namespace util { +namespace btree { + +// The btree_set class is needed mainly for it's constructors. +template , + typename Alloc = std::allocator, + int TargetNodeSize = 256> +class btree_set : public btree_unique_container< + btree > > { + + typedef btree_set self_type; + typedef btree_set_params params_type; + typedef btree btree_type; + typedef btree_unique_container super_type; + + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + + public: + // Default constructor. + btree_set(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_set(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_set(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(b, e, comp, alloc) { + } +}; + +template +inline void swap(btree_set &x, btree_set &y) { + x.swap(y); +} + +// The btree_multiset class is needed mainly for it's constructors. +template , + typename Alloc = std::allocator, + int TargetNodeSize = 256> +class btree_multiset : public btree_multi_container< + btree > > { + + typedef btree_multiset self_type; + typedef btree_set_params params_type; + typedef btree btree_type; + typedef btree_multi_container super_type; + + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + + public: + // Default constructor. + btree_multiset(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + btree_multiset(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + btree_multiset(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(b, e, comp, alloc) { + } +}; + +template +inline void swap(btree_multiset &x, + btree_multiset &y) { + x.swap(y); +} + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_BTREE_SET_H__ diff --git a/btree_test.cc b/btree_test.cc new file mode 100644 index 00000000..ef4e574e --- /dev/null +++ b/btree_test.cc @@ -0,0 +1,167 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) + +#include "base/arena-inl.h" +#include "base/init_google.h" +#include "base/integral_types.h" +#include "base/logging.h" +#include "strings/stringpiece.h" +#include "testing/base/public/gunit.h" +#include "util/btree/btree_map.h" +#include "util/btree/btree_set.h" +#include "util/btree/btree_test.h" + +namespace util { +namespace btree { +namespace { + +template +void SetTest() { + typedef ArenaAllocator ArenaAlloc; + CHECK_EQ(sizeof(btree_set), sizeof(void*)); + BtreeTest, allocator, N>, set >(); + BtreeArenaTest, ArenaAlloc, N> >(); +} + +template +void MapTest() { + typedef ArenaAllocator ArenaAlloc; + CHECK_EQ(sizeof(btree_map), sizeof(void*)); + BtreeTest, allocator, N>, map >(); + BtreeArenaTest, ArenaAlloc, N> >(); + BtreeMapTest, allocator, N> >(); +} + +TEST(Btree, set_int32_32) { SetTest(); } +TEST(Btree, set_int32_64) { SetTest(); } +TEST(Btree, set_int32_128) { SetTest(); } +TEST(Btree, set_int32_256) { SetTest(); } +TEST(Btree, set_int64_256) { SetTest(); } +TEST(Btree, set_string_256) { SetTest(); } +TEST(Btree, set_cord_256) { SetTest(); } +TEST(Btree, set_pair_256) { SetTest, 256>(); } +TEST(Btree, map_int32_256) { MapTest(); } +TEST(Btree, map_int64_256) { MapTest(); } +TEST(Btree, map_string_256) { MapTest(); } +TEST(Btree, map_cord_256) { MapTest(); } +TEST(Btree, map_pair_256) { MapTest, 256>(); } + +template +void MultiSetTest() { + typedef ArenaAllocator ArenaAlloc; + CHECK_EQ(sizeof(btree_multiset), sizeof(void*)); + BtreeMultiTest, allocator, N>, + multiset >(); + BtreeArenaTest, ArenaAlloc, N> >(); +} + +template +void MultiMapTest() { + typedef ArenaAllocator ArenaAlloc; + CHECK_EQ(sizeof(btree_multimap), sizeof(void*)); + BtreeMultiTest, allocator, N>, + multimap >(); + BtreeMultiMapTest, allocator, N> >(); + BtreeArenaTest, ArenaAlloc, N> >(); +} + +TEST(Btree, multiset_int32_256) { MultiSetTest(); } +TEST(Btree, multiset_int64_256) { MultiSetTest(); } +TEST(Btree, multiset_string_256) { MultiSetTest(); } +TEST(Btree, multiset_cord_256) { MultiSetTest(); } +TEST(Btree, multiset_pair_256) { MultiSetTest, 256>(); } +TEST(Btree, multimap_int32_256) { MultiMapTest(); } +TEST(Btree, multimap_int64_256) { MultiMapTest(); } +TEST(Btree, multimap_string_256) { MultiMapTest(); } +TEST(Btree, multimap_cord_256) { MultiMapTest(); } +TEST(Btree, multimap_pair_256) { MultiMapTest, 256>(); } + +// Verify that swapping btrees swaps the key comparision functors. +struct SubstringLess { + SubstringLess() : n(2) {} + SubstringLess(int length) + : n(length) { + } + bool operator()(const string &a, const string &b) const { + return StringPiece(a).substr(0, n) < StringPiece(b).substr(0, n); + } + int n; +}; + +TEST(Btree, SwapKeyCompare) { + typedef btree_set SubstringSet; + SubstringSet s1(SubstringLess(1), SubstringSet::allocator_type()); + SubstringSet s2(SubstringLess(2), SubstringSet::allocator_type()); + + ASSERT_TRUE(s1.insert("a").second); + ASSERT_FALSE(s1.insert("aa").second); + + ASSERT_TRUE(s2.insert("a").second); + ASSERT_TRUE(s2.insert("aa").second); + ASSERT_FALSE(s2.insert("aaa").second); + + swap(s1, s2); + + ASSERT_TRUE(s1.insert("b").second); + ASSERT_TRUE(s1.insert("bb").second); + ASSERT_FALSE(s1.insert("bbb").second); + + ASSERT_TRUE(s2.insert("b").second); + ASSERT_FALSE(s2.insert("bb").second); +} + +TEST(Btree, UpperBoundRegression) { + // Regress a bug where upper_bound would default-construct a new key_compare + // instead of copying the existing one. + typedef btree_set SubstringSet; + SubstringSet my_set(SubstringLess(3)); + my_set.insert("aab"); + my_set.insert("abb"); + // We call upper_bound("aaa"). If this correctly uses the length 3 + // comparator, aaa < aab < abb, so we should get aab as the result. + // If it instead uses the default-constructed length 2 comparator, + // aa == aa < ab, so we'll get abb as our result. + SubstringSet::iterator it = my_set.upper_bound("aaa"); + ASSERT_TRUE(it != my_set.end()); + EXPECT_EQ("aab", *it); +} + + +TEST(Btree, IteratorIncrementBy) { + // Test that increment_by returns the same position as increment. + const int kSetSize = 2341; + btree_set my_set; + for (int i = 0; i < kSetSize; ++i) { + my_set.insert(i); + } + + { + // Simple increment vs. increment by. + btree_set::iterator a = my_set.begin(); + btree_set::iterator b = my_set.begin(); + a.increment(); + b.increment_by(1); + EXPECT_EQ(*a, *b); + } + + btree_set::iterator a = my_set.begin(); + for (int i = 1; i < kSetSize; ++i) { + ++a; + // increment_by + btree_set::iterator b = my_set.begin(); + b.increment_by(i); + EXPECT_EQ(*a, *b) << ": i=" << i; + } +} + + +} // namespace +} // namespace btree +} // namespace util + +int main(int argc, char **argv) { + FLAGS_logtostderr = true; + InitGoogle(argv[0], &argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/btree_test.h b/btree_test.h new file mode 100644 index 00000000..557c1e54 --- /dev/null +++ b/btree_test.h @@ -0,0 +1,930 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) + +#ifndef UTIL_BTREE_BTREE_TEST_H__ +#define UTIL_BTREE_BTREE_TEST_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "base/arena.h" +#include "base/commandlineflags.h" +#include "base/logging.h" +#include "base/type_traits.h" +#include "strings/cord.h" +#include "strings/util.h" +#include "testing/base/public/googletest.h" +#include "util/btree/btree_container.h" +#include "util/random/acmrandom.h" + +DECLARE_int32(test_values); +DECLARE_int32(benchmark_values); + +namespace std { + +// Provide operator<< support for pair. +template +ostream& operator<<(ostream &os, const pair &p) { + os << "(" << p.first << "," << p.second << ")"; + return os; +} + +// Provide pair equality testing that works as long as x.first is comparable to +// y.first and x.second is comparable to y.second. Needed in the test for +// comparing pair to pair. +template +bool operator==(const pair &x, const pair &y) { + return x.first == y.first && x.second == y.second; +} + +} // namespace std + +namespace base { + +// Partial specialization of remove_const that propagates the removal through +// std::pair. +template +struct remove_const > { + typedef std::pair::type, + typename remove_const::type> type; +}; + +} // namespace base + +namespace util { +namespace btree { + +// Utility class to provide an accessor for a key given a value. The default +// behavior is to treat the value as a pair and return the first element. +template +struct KeyOfValue { + typedef select1st type; +}; + +// Partial specialization of KeyOfValue class for when the key and value are +// the same type such as in set<> and btree_set<>. +template +struct KeyOfValue { + typedef identity type; +}; + +// The base class for a sorted associative container checker. TreeType is the +// container type to check and CheckerType is the container type to check +// against. TreeType is expected to be btree_{set,map,multiset,multimap} and +// CheckerType is expected to be {set,map,multiset,multimap}. +template +class base_checker { + typedef base_checker self_type; + + public: + typedef typename TreeType::key_type key_type; + typedef typename TreeType::value_type value_type; + typedef typename TreeType::key_compare key_compare; + typedef typename TreeType::pointer pointer; + typedef typename TreeType::const_pointer const_pointer; + typedef typename TreeType::reference reference; + typedef typename TreeType::const_reference const_reference; + typedef typename TreeType::size_type size_type; + typedef typename TreeType::difference_type difference_type; + typedef typename TreeType::iterator iterator; + typedef typename TreeType::const_iterator const_iterator; + typedef typename TreeType::reverse_iterator reverse_iterator; + typedef typename TreeType::const_reverse_iterator const_reverse_iterator; + + public: + // Default constructor. + base_checker() + : const_tree_(tree_) { + } + // Copy constructor. + base_checker(const self_type &x) + : tree_(x.tree_), + const_tree_(tree_), + checker_(x.checker_) { + } + + // Iterator routines. + iterator begin() { return tree_.begin(); } + const_iterator begin() const { return tree_.begin(); } + iterator end() { return tree_.end(); } + const_iterator end() const { return tree_.end(); } + reverse_iterator rbegin() { return tree_.rbegin(); } + const_reverse_iterator rbegin() const { return tree_.rbegin(); } + reverse_iterator rend() { return tree_.rend(); } + const_reverse_iterator rend() const { return tree_.rend(); } + + // Helper routines. + template + IterType iter_check( + IterType tree_iter, CheckerIterType checker_iter) const { + if (tree_iter == tree_.end()) { + CHECK(checker_iter == checker_.end()); + } else { + CHECK_EQ(*tree_iter, *checker_iter); + } + return tree_iter; + } + template + IterType riter_check( + IterType tree_iter, CheckerIterType checker_iter) const { + if (tree_iter == tree_.rend()) { + CHECK(checker_iter == checker_.rend()); + } else { + CHECK_EQ(*tree_iter, *checker_iter); + } + return tree_iter; + } + void value_check(const value_type &x) { + typename KeyOfValue::type key_of_value; + const key_type &key = key_of_value(x); + CHECK_EQ(*find(key), x); + lower_bound(key); + upper_bound(key); + equal_range(key); + count(key); + } + void erase_check(const key_type &key) { + CHECK(tree_.find(key) == const_tree_.end()); + CHECK(const_tree_.find(key) == tree_.end()); + CHECK(tree_.equal_range(key).first == + const_tree_.equal_range(key).second); + } + + // Lookup routines. + iterator lower_bound(const key_type &key) { + return iter_check(tree_.lower_bound(key), checker_.lower_bound(key)); + } + const_iterator lower_bound(const key_type &key) const { + return iter_check(tree_.lower_bound(key), checker_.lower_bound(key)); + } + iterator upper_bound(const key_type &key) { + return iter_check(tree_.upper_bound(key), checker_.upper_bound(key)); + } + const_iterator upper_bound(const key_type &key) const { + return iter_check(tree_.upper_bound(key), checker_.upper_bound(key)); + } + pair equal_range(const key_type &key) { + pair checker_res = + checker_.equal_range(key); + pair tree_res = tree_.equal_range(key); + iter_check(tree_res.first, checker_res.first); + iter_check(tree_res.second, checker_res.second); + return tree_res; + } + pair equal_range(const key_type &key) const { + pair checker_res = + checker_.equal_range(key); + pair tree_res = tree_.equal_range(key); + iter_check(tree_res.first, checker_res.first); + iter_check(tree_res.second, checker_res.second); + return tree_res; + } + iterator find(const key_type &key) { + return iter_check(tree_.find(key), checker_.find(key)); + } + const_iterator find(const key_type &key) const { + return iter_check(tree_.find(key), checker_.find(key)); + } + size_type count(const key_type &key) const { + size_type res = checker_.count(key); + CHECK_EQ(res, tree_.count(key)); + return res; + } + + // Assignment operator. + self_type& operator=(const self_type &x) { + tree_ = x.tree_; + checker_ = x.checker_; + return *this; + } + + // Deletion routines. + int erase(const key_type &key) { + int size = tree_.size(); + int res = checker_.erase(key); + CHECK_EQ(res, tree_.count(key)); + CHECK_EQ(res, tree_.erase(key)); + CHECK_EQ(tree_.count(key), 0); + CHECK_EQ(tree_.size(), size - res); + erase_check(key); + return res; + } + iterator erase(iterator iter) { + key_type key = iter.key(); + int size = tree_.size(); + int count = tree_.count(key); + typename CheckerType::iterator checker_iter = checker_.find(key); + for (iterator tmp(tree_.find(key)); tmp != iter; ++tmp) { + ++checker_iter; + } + typename CheckerType::iterator checker_next = checker_iter; + ++checker_next; + checker_.erase(checker_iter); + iter = tree_.erase(iter); + CHECK_EQ(tree_.size(), checker_.size()); + CHECK_EQ(tree_.size(), size - 1); + CHECK_EQ(tree_.count(key), count - 1); + if (count == 1) { + erase_check(key); + } + return iter_check(iter, checker_next); + } + + void erase(iterator begin, iterator end) { + int size = tree_.size(); + int count = distance(begin, end); + typename CheckerType::iterator checker_begin = checker_.find(begin.key()); + for (iterator tmp(tree_.find(begin.key())); tmp != begin; ++tmp) { + ++checker_begin; + } + typename CheckerType::iterator checker_end = + end == tree_.end() ? checker_.end() : checker_.find(end.key()); + if (end != tree_.end()) { + for (iterator tmp(tree_.find(end.key())); tmp != end; ++tmp) { + ++checker_end; + } + } + checker_.erase(checker_begin, checker_end); + tree_.erase(begin, end); + CHECK_EQ(tree_.size(), checker_.size()); + CHECK_EQ(tree_.size(), size - count); + } + + // Utility routines. + void clear() { + tree_.clear(); + checker_.clear(); + } + void swap(self_type &x) { + tree_.swap(x.tree_); + checker_.swap(x.checker_); + } + + void verify() const { + tree_.verify(); + CHECK_EQ(tree_.size(), checker_.size()); + + // Move through the forward iterators using increment. + typename CheckerType::const_iterator + checker_iter(checker_.begin()); + const_iterator tree_iter(tree_.begin()); + for (; tree_iter != tree_.end(); + ++tree_iter, ++checker_iter) { + CHECK_EQ(*tree_iter, *checker_iter); + } + + // Move through the forward iterators using decrement. + for (int n = tree_.size() - 1; n >= 0; --n) { + iter_check(tree_iter, checker_iter); + --tree_iter; + --checker_iter; + } + CHECK(tree_iter == tree_.begin()); + CHECK(checker_iter == checker_.begin()); + + // Move through the reverse iterators using increment. + typename CheckerType::const_reverse_iterator + checker_riter(checker_.rbegin()); + const_reverse_iterator tree_riter(tree_.rbegin()); + for (; tree_riter != tree_.rend(); + ++tree_riter, ++checker_riter) { + CHECK_EQ(*tree_riter, *checker_riter); + } + + // Move through the reverse iterators using decrement. + for (int n = tree_.size() - 1; n >= 0; --n) { + riter_check(tree_riter, checker_riter); + --tree_riter; + --checker_riter; + } + CHECK(tree_riter == tree_.rbegin()); + CHECK(checker_riter == checker_.rbegin()); + } + + // Access to the underlying btree. + const TreeType& tree() const { return tree_; } + + // Size routines. + size_type size() const { + CHECK_EQ(tree_.size(), checker_.size()); + return tree_.size(); + } + size_type max_size() const { return tree_.max_size(); } + bool empty() const { + CHECK_EQ(tree_.empty(), checker_.empty()); + return tree_.empty(); + } + size_type height() const { return tree_.height(); } + size_type internal_nodes() const { return tree_.internal_nodes(); } + size_type leaf_nodes() const { return tree_.leaf_nodes(); } + size_type nodes() const { return tree_.nodes(); } + size_type bytes_used() const { return tree_.bytes_used(); } + double fullness() const { return tree_.fullness(); } + double overhead() const { return tree_.overhead(); } + + protected: + TreeType tree_; + const TreeType &const_tree_; + CheckerType checker_; +}; + +// A checker for unique sorted associative containers. TreeType is expected to +// be btree_{set,map} and CheckerType is expected to be {set,map}. +template +class unique_checker : public base_checker { + typedef base_checker super_type; + typedef unique_checker self_type; + + public: + typedef typename super_type::iterator iterator; + typedef typename super_type::value_type value_type; + + public: + // Default constructor. + unique_checker() + : super_type() { + } + // Copy constructor. + unique_checker(const self_type &x) + : super_type(x) { + } + // Range constructor. + template + unique_checker(InputIterator b, InputIterator e) { + insert(b, e); + } + + // Insertion routines. + pair insert(const value_type &x) { + int size = this->tree_.size(); + pair checker_res = + this->checker_.insert(x); + pair tree_res = this->tree_.insert(x); + CHECK_EQ(*tree_res.first, *checker_res.first); + CHECK_EQ(tree_res.second, checker_res.second); + CHECK_EQ(this->tree_.size(), this->checker_.size()); + CHECK_EQ(this->tree_.size(), size + tree_res.second); + return tree_res; + } + iterator insert(iterator position, const value_type &x) { + int size = this->tree_.size(); + pair checker_res = + this->checker_.insert(x); + iterator tree_res = this->tree_.insert(position, x); + CHECK_EQ(*tree_res, *checker_res.first); + CHECK_EQ(this->tree_.size(), this->checker_.size()); + CHECK_EQ(this->tree_.size(), size + checker_res.second); + return tree_res; + } + template + void insert(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert(*b); + } + } +}; + +// A checker for multiple sorted associative containers. TreeType is expected +// to be btree_{multiset,multimap} and CheckerType is expected to be +// {multiset,multimap}. +template +class multi_checker : public base_checker { + typedef base_checker super_type; + typedef multi_checker self_type; + + public: + typedef typename super_type::iterator iterator; + typedef typename super_type::value_type value_type; + + public: + // Default constructor. + multi_checker() + : super_type() { + } + // Copy constructor. + multi_checker(const self_type &x) + : super_type(x) { + } + // Range constructor. + template + multi_checker(InputIterator b, InputIterator e) { + insert(b, e); + } + + // Insertion routines. + iterator insert(const value_type &x) { + int size = this->tree_.size(); + typename CheckerType::iterator checker_res = this->checker_.insert(x); + iterator tree_res = this->tree_.insert(x); + CHECK_EQ(*tree_res, *checker_res); + CHECK_EQ(this->tree_.size(), this->checker_.size()); + CHECK_EQ(this->tree_.size(), size + 1); + return tree_res; + } + iterator insert(iterator position, const value_type &x) { + int size = this->tree_.size(); + typename CheckerType::iterator checker_res = this->checker_.insert(x); + iterator tree_res = this->tree_.insert(position, x); + CHECK_EQ(*tree_res, *checker_res); + CHECK_EQ(this->tree_.size(), this->checker_.size()); + CHECK_EQ(this->tree_.size(), size + 1); + return tree_res; + } + template + void insert(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert(*b); + } + } +}; + +char* GenerateDigits(char buf[16], int val, int maxval) { + DCHECK_LE(val, maxval); + int p = 15; + buf[p--] = 0; + while (maxval > 0) { + buf[p--] = '0' + (val % 10); + val /= 10; + maxval /= 10; + } + return buf + p + 1; +} + +template +struct Generator { + int maxval; + Generator(int m) + : maxval(m) { + } + K operator()(int i) const { + DCHECK_LE(i, maxval); + return i; + } +}; + +template <> +struct Generator { + int maxval; + Generator(int m) + : maxval(m) { + } + string operator()(int i) const { + char buf[16]; + return GenerateDigits(buf, i, maxval); + } +}; + +template <> +struct Generator { + int maxval; + Generator(int m) + : maxval(m) { + } + Cord operator()(int i) const { + char buf[16]; + return Cord(GenerateDigits(buf, i, maxval)); + } +}; + +template +struct Generator > { + Generator::type> tgen; + Generator::type> ugen; + + Generator(int m) + : tgen(m), + ugen(m) { + } + pair operator()(int i) const { + return make_pair(tgen(i), ugen(i)); + } +}; + +// Generate values for our tests and benchmarks. Value range is [0, maxval]. +const vector& GenerateNumbers(int n, int maxval) { + static ACMRandom rand(FLAGS_test_random_seed); + static vector values; + static set unique_values; + + if (values.size() < n) { + + for (int i = values.size(); i < n; i++) { + int value; + do { + value = rand.Next() % (maxval + 1); + } while (unique_values.find(value) != unique_values.end()); + + values.push_back(value); + unique_values.insert(value); + } + } + + return values; +} + +// Generates values in the range +// [0, 4 * min(FLAGS_benchmark_values, FLAGS_test_values)] +template +vector GenerateValues(int n) { + int two_times_max = 2 * max(FLAGS_benchmark_values, FLAGS_test_values); + int four_times_max = 2 * two_times_max; + DCHECK_LE(n, two_times_max); + const vector &nums = GenerateNumbers(n, four_times_max); + Generator gen(four_times_max); + vector vec; + + for (int i = 0; i < n; i++) { + vec.push_back(gen(nums[i])); + } + + return vec; +} + +template +double ContainerInfo(const set &s) { + int sizeof_node = sizeof(std::_Rb_tree_node); + int bytes_used = sizeof(s) + s.size() * sizeof_node; + double bytes_per_value = (double) bytes_used / s.size(); + VLOG(1) << " size=" << s.size() + << " bytes-used=" << bytes_used + << " bytes-per-value=" << bytes_per_value; + return bytes_per_value; +} + +template +double ContainerInfo(const multiset &s) { + int sizeof_node = sizeof(std::_Rb_tree_node); + int bytes_used = sizeof(s) + s.size() * sizeof_node; + double bytes_per_value = (double) bytes_used / s.size(); + VLOG(1) << " size=" << s.size() + << " bytes-used=" << bytes_used + << " bytes-per-value=" << bytes_per_value; + return bytes_per_value; +} + +template +double ContainerInfo(const map &m) { + int sizeof_node = sizeof(std::_Rb_tree_node >); + int bytes_used = sizeof(m) + m.size() * sizeof_node; + double bytes_per_value = (double) bytes_used / m.size(); + VLOG(1) << " size=" << m.size() + << " bytes-used=" << bytes_used + << " bytes-per-value=" << bytes_per_value; + return bytes_per_value; +} + +template +double ContainerInfo(const multimap &m) { + int sizeof_node = sizeof(std::_Rb_tree_node >); + int bytes_used = sizeof(m) + m.size() * sizeof_node; + double bytes_per_value = (double) bytes_used / m.size(); + VLOG(1) << " size=" << m.size() + << " bytes-used=" << bytes_used + << " bytes-per-value=" << bytes_per_value; + return bytes_per_value; +} + +template +double ContainerInfo(const btree_container

&b) { + double bytes_used = sizeof(b) + b.bytes_used(); + double bytes_per_value = (double) bytes_used / b.size(); + VLOG(1) << " size=" << b.size() + << " bytes-used=" << bytes_used + << " bytes-per-value=" << bytes_per_value + << " height=" << b.height() + << " internal-nodes=" << b.internal_nodes() + << " leaf-nodes=" << b.leaf_nodes() + << " fullness=" << b.fullness() + << " overhead=" << b.overhead(); + return bytes_per_value; +} + +template +void DoTest(const char *name, T *b, const vector &values) { + typename KeyOfValue::type key_of_value; + + T &mutable_b = *b; + const T &const_b = *b; + + // Test insert. + for (int i = 0; i < values.size(); ++i) { + mutable_b.insert(values[i]); + mutable_b.value_check(values[i]); + } + + const_b.verify(); + printf(" %s fullness=%0.2f overhead=%0.2f bytes-per-value=%0.2f\n", + name, const_b.fullness(), const_b.overhead(), + double(const_b.bytes_used()) / const_b.size()); + + // Test copy constructor. + T b_copy(const_b); + CHECK_EQ(b_copy.size(), const_b.size()); + CHECK_LE(b_copy.height(), const_b.height()); + CHECK_LE(b_copy.internal_nodes(), const_b.internal_nodes()); + CHECK_LE(b_copy.leaf_nodes(), const_b.leaf_nodes()); + for (int i = 0; i < values.size(); ++i) { + CHECK_EQ(*b_copy.find(key_of_value(values[i])), values[i]); + } + + // Test range constructor. + T b_range(const_b.begin(), const_b.end()); + CHECK_EQ(b_range.size(), const_b.size()); + CHECK_LE(b_range.height(), const_b.height()); + CHECK_LE(b_range.internal_nodes(), const_b.internal_nodes()); + CHECK_LE(b_range.leaf_nodes(), const_b.leaf_nodes()); + for (int i = 0; i < values.size(); ++i) { + CHECK_EQ(*b_range.find(key_of_value(values[i])), values[i]); + } + + // Test range insertion for values that already exist. + b_range.insert(b_copy.begin(), b_copy.end()); + b_range.verify(); + + // Test range insertion for new values. + b_range.clear(); + b_range.insert(b_copy.begin(), b_copy.end()); + CHECK_EQ(b_range.size(), b_copy.size()); + CHECK_EQ(b_range.height(), b_copy.height()); + CHECK_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); + CHECK_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); + for (int i = 0; i < values.size(); ++i) { + CHECK_EQ(*b_range.find(key_of_value(values[i])), values[i]); + } + + // Test assignment to self. Nothing should change. + b_range.operator=(b_range); + CHECK_EQ(b_range.size(), b_copy.size()); + CHECK_EQ(b_range.height(), b_copy.height()); + CHECK_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); + CHECK_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); + + // Test assignment of new values. + b_range.clear(); + b_range = b_copy; + CHECK_EQ(b_range.size(), b_copy.size()); + CHECK_EQ(b_range.height(), b_copy.height()); + CHECK_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); + CHECK_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); + + // Test swap. + b_range.clear(); + b_range.swap(b_copy); + CHECK_EQ(b_copy.size(), 0); + CHECK_EQ(b_range.size(), const_b.size()); + for (int i = 0; i < values.size(); ++i) { + CHECK_EQ(*b_range.find(key_of_value(values[i])), values[i]); + } + b_range.swap(b_copy); + + // Test erase via values. + for (int i = 0; i < values.size(); ++i) { + mutable_b.erase(key_of_value(values[i])); + // Erasing a non-existent key should have no effect. + CHECK_EQ(mutable_b.erase(key_of_value(values[i])), 0); + } + + const_b.verify(); + CHECK_EQ(const_b.internal_nodes(), 0); + CHECK_EQ(const_b.leaf_nodes(), 0); + CHECK_EQ(const_b.size(), 0); + + // Test erase via iterators. + mutable_b = b_copy; + for (int i = 0; i < values.size(); ++i) { + mutable_b.erase(mutable_b.find(key_of_value(values[i]))); + } + + const_b.verify(); + CHECK_EQ(const_b.internal_nodes(), 0); + CHECK_EQ(const_b.leaf_nodes(), 0); + CHECK_EQ(const_b.size(), 0); + + // Test insert with hint. + for (int i = 0; i < values.size(); i++) { + mutable_b.insert(mutable_b.upper_bound(key_of_value(values[i])), values[i]); + } + + const_b.verify(); + + // Test dumping of the btree to an ostream. There should be 1 line for each + // value. + ostringstream strm; + strm << mutable_b.tree(); + CHECK_EQ(mutable_b.size(), strcount(strm.str(), '\n')); + + // Test range erase. + mutable_b.erase(mutable_b.begin(), mutable_b.end()); + CHECK_EQ(mutable_b.size(), 0); + const_b.verify(); + + // First half. + mutable_b = b_copy; + typename T::iterator mutable_iter_end = mutable_b.begin(); + for (int i = 0; i < values.size() / 2; ++i) ++mutable_iter_end; + mutable_b.erase(mutable_b.begin(), mutable_iter_end); + CHECK_EQ(mutable_b.size(), values.size() - values.size() / 2); + const_b.verify(); + + // Second half. + mutable_b = b_copy; + typename T::iterator mutable_iter_begin = mutable_b.begin(); + for (int i = 0; i < values.size() / 2; ++i) ++mutable_iter_begin; + mutable_b.erase(mutable_iter_begin, mutable_b.end()); + CHECK_EQ(mutable_b.size(), values.size() / 2); + const_b.verify(); + + // Second quarter. + mutable_b = b_copy; + mutable_iter_begin = mutable_b.begin(); + for (int i = 0; i < values.size() / 4; ++i) ++mutable_iter_begin; + mutable_iter_end = mutable_iter_begin; + for (int i = 0; i < values.size() / 4; ++i) ++mutable_iter_end; + mutable_b.erase(mutable_iter_begin, mutable_iter_end); + CHECK_EQ(mutable_b.size(), values.size() - values.size() / 4); + const_b.verify(); + + mutable_b.clear(); +} + +template +void ConstTest() { + typedef typename T::value_type value_type; + typename KeyOfValue::type key_of_value; + + T mutable_b; + const T &const_b = mutable_b; + + // Insert a single value into the container and test looking it up. + value_type value = Generator(2)(2); + mutable_b.insert(value); + CHECK(mutable_b.find(key_of_value(value)) != const_b.end()); + CHECK(const_b.find(key_of_value(value)) != mutable_b.end()); + CHECK_EQ(*const_b.lower_bound(key_of_value(value)), value); + CHECK(const_b.upper_bound(key_of_value(value)) == const_b.end()); + CHECK_EQ(*const_b.equal_range(key_of_value(value)).first, value); + + // We can only create a non-const iterator from a non-const container. + typename T::iterator mutable_iter(mutable_b.begin()); + CHECK(mutable_iter == const_b.begin()); + CHECK(mutable_iter != const_b.end()); + CHECK(const_b.begin() == mutable_iter); + CHECK(const_b.end() != mutable_iter); + typename T::reverse_iterator mutable_riter(mutable_b.rbegin()); + CHECK(mutable_riter == const_b.rbegin()); + CHECK(mutable_riter != const_b.rend()); + CHECK(const_b.rbegin() == mutable_riter); + CHECK(const_b.rend() != mutable_riter); + + // We can create a const iterator from a non-const iterator. + typename T::const_iterator const_iter(mutable_iter); + CHECK(const_iter == mutable_b.begin()); + CHECK(const_iter != mutable_b.end()); + CHECK(mutable_b.begin() == const_iter); + CHECK(mutable_b.end() != const_iter); + typename T::const_reverse_iterator const_riter(mutable_riter); + CHECK(const_riter == mutable_b.rbegin()); + CHECK(const_riter != mutable_b.rend()); + CHECK(mutable_b.rbegin() == const_riter); + CHECK(mutable_b.rend() != const_riter); + + // Make sure various methods can be invoked on a const container. + const_b.verify(); + CHECK(!const_b.empty()); + CHECK_EQ(const_b.size(), 1); + CHECK_GT(const_b.max_size(), 0); + CHECK_EQ(const_b.height(), 1); + CHECK_EQ(const_b.count(key_of_value(value)), 1); + CHECK_EQ(const_b.internal_nodes(), 0); + CHECK_EQ(const_b.leaf_nodes(), 1); + CHECK_EQ(const_b.nodes(), 1); + CHECK_GT(const_b.bytes_used(), 0); + CHECK_GT(const_b.fullness(), 0); + CHECK_GT(const_b.overhead(), 0); +} + +template +void BtreeTest() { + ConstTest(); + + typedef typename base::remove_const::type V; + vector random_values = GenerateValues(FLAGS_test_values); + + unique_checker container; + + // Test key insertion/deletion in sorted order. + vector sorted_values(random_values); + sort(sorted_values.begin(), sorted_values.end()); + DoTest("sorted: ", &container, sorted_values); + + // Test key insertion/deletion in reverse sorted order. + reverse(sorted_values.begin(), sorted_values.end()); + DoTest("rsorted: ", &container, sorted_values); + + // Test key insertion/deletion in random order. + DoTest("random: ", &container, random_values); +} + +template +void BtreeMultiTest() { + ConstTest(); + + typedef typename base::remove_const::type V; + const vector& random_values = GenerateValues(FLAGS_test_values); + + multi_checker container; + + // Test keys in sorted order. + vector sorted_values(random_values); + sort(sorted_values.begin(), sorted_values.end()); + DoTest("sorted: ", &container, sorted_values); + + // Test keys in reverse sorted order. + reverse(sorted_values.begin(), sorted_values.end()); + DoTest("rsorted: ", &container, sorted_values); + + // Test keys in random order. + DoTest("random: ", &container, random_values); + + // Test keys in random order w/ duplicates. + vector duplicate_values(random_values); + duplicate_values.insert( + duplicate_values.end(), random_values.begin(), random_values.end()); + DoTest("duplicates:", &container, duplicate_values); + + // Test all identical keys. + vector identical_values(100); + fill(identical_values.begin(), identical_values.end(), Generator(2)(2)); + DoTest("identical: ", &container, identical_values); +} + +template +void BtreeArenaTest() { + typedef typename T::value_type value_type; + + UnsafeArena arena1(1000); + UnsafeArena arena2(1000); + T b1(typename T::key_compare(), &arena1); + T b2(typename T::key_compare(), &arena2); + + // This should swap the allocators! + swap(b1, b2); + + for (int i = 0; i < 1000; i++) { + b1.insert(Generator(1000)(i)); + } + + // We should have allocated out of arena2! + CHECK_LE(b1.bytes_used(), arena2.status().bytes_allocated()); + CHECK_GT(arena2.block_count(), arena1.block_count()); +} + +template +void BtreeMapTest() { + typedef typename T::value_type value_type; + typedef typename T::mapped_type mapped_type; + + mapped_type m = Generator(0)(0); + (void) m; + + T b; + + // Verify we can insert using operator[]. + for (int i = 0; i < 1000; i++) { + value_type v = Generator(1000)(i); + b[v.first] = v.second; + } + CHECK_EQ(b.size(), 1000); + + // Test whether we can use the "->" operator on iterators and + // reverse_iterators. This stresses the btree_map_params::pair_pointer + // mechanism. + CHECK_EQ(b.begin()->first, Generator(1000)(0).first); + CHECK_EQ(b.begin()->second, Generator(1000)(0).second); + CHECK_EQ(b.rbegin()->first, Generator(1000)(999).first); + CHECK_EQ(b.rbegin()->second, Generator(1000)(999).second); +} + +template +void BtreeMultiMapTest() { + typedef typename T::mapped_type mapped_type; + mapped_type m = Generator(0)(0); + (void) m; +} + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_BTREE_TEST_H__ diff --git a/btree_test_flags.cc b/btree_test_flags.cc new file mode 100644 index 00000000..512c24e1 --- /dev/null +++ b/btree_test_flags.cc @@ -0,0 +1,9 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: pmattis@google.com (Peter Mattis) + +#include "base/commandlineflags.h" + +DEFINE_int32(test_values, 10000, + "The number of values to use for tests."); +DEFINE_int32(benchmark_values, 1000000, + "The number of values to use for benchmarks."); diff --git a/btree_test_program.cc b/btree_test_program.cc new file mode 100644 index 00000000..0c2dcf88 --- /dev/null +++ b/btree_test_program.cc @@ -0,0 +1,64 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// Author: leg@google.com (Lawrence Greenfield) + +#include "base/init_google.h" +#include "base/logging.h" +#include "devtools/gdb/component/gdb_test_utils.h" +#include "util/btree/btree_map.h" +#include "util/btree/btree_set.h" + +using util::btree::btree_set; +using util::btree::btree_multiset; +using util::btree::btree_map; +using util::btree::btree_multimap; + +static btree_set* empty_set; +static btree_set* small_set; +static btree_set* big_set; +static btree_multiset* small_multiset; + +static btree_map* empty_map; +static btree_map* small_map; +static btree_map* big_map; +static btree_multimap* small_multimap; + +static void SetupBtreeSets() { + empty_set = new btree_set; + small_set = new btree_set; + small_multiset = new btree_multiset; + big_set = new btree_set; + + for (int i = 0; i < 10; ++i) { + small_set->insert(i); + small_multiset->insert(i / 2); + } + + for (int i = 0; i < 80; ++i) { + big_set->insert(i); + } +} + +static void SetupBtreeMaps() { + empty_map = new btree_map; + small_map = new btree_map; + small_multimap = new btree_multimap; + big_map = new btree_map; + + for (int i = 0; i < 10; ++i) { + small_map->insert(make_pair(i, i * 13)); + small_multimap->insert(make_pair(i / 2, i)); + } + + for (int i = 0; i < 80; ++i) { + big_map->insert(make_pair(i, i * 7)); + } +} + +int main(int argc, char** argv) { + FLAGS_logtostderr = true; + InitGoogle(argv[0], &argc, &argv, true); + SetupBtreeSets(); + SetupBtreeMaps(); + StopHereForDebugger(); + return 0; +} diff --git a/safe_btree.h b/safe_btree.h new file mode 100644 index 00000000..b3db5360 --- /dev/null +++ b/safe_btree.h @@ -0,0 +1,379 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: pmattis@google.com (Peter Mattis) +// +// A safe_btree<> wraps around a btree<> and removes the caveat that insertion +// and deletion invalidate iterators. A safe_btree<> maintains a generation +// number that is incremented on every mutation. A safe_btree<>::iterator keeps +// a pointer to the safe_btree<> it came from, the generation of the tree when +// it was last validated and the key the underlying btree<>::iterator points +// to. If an iterator is accessed and its generation differs from the tree +// generation it is revalidated. + +#ifndef UTIL_BTREE_SAFE_BTREE_H__ +#define UTIL_BTREE_SAFE_BTREE_H__ + +#include +#include +#include + +#include "base/integral_types.h" +#include "base/logging.h" +#include "util/btree/btree.h" + +namespace util { +namespace btree { + +template +class safe_btree_iterator { + public: + typedef typename Iterator::key_type key_type; + typedef typename Iterator::value_type value_type; + typedef typename Iterator::size_type size_type; + typedef typename Iterator::difference_type difference_type; + typedef typename Iterator::pointer pointer; + typedef typename Iterator::reference reference; + typedef typename Iterator::const_pointer const_pointer; + typedef typename Iterator::const_reference const_reference; + typedef typename Iterator::iterator_category iterator_category; + typedef typename Tree::iterator iterator; + typedef typename Tree::const_iterator const_iterator; + typedef safe_btree_iterator self_type; + + void update() const { + if (iter_ != tree_->internal_btree()->end()) { + // A positive generation indicates a valid key. + generation_ = tree_->generation(); + key_ = iter_.key(); + } else { + // Use a negative generation to indicate iter_ points to end(). + generation_ = -tree_->generation(); + } + } + + public: + safe_btree_iterator() + : generation_(0), + key_(), + iter_(), + tree_(NULL) { + } + safe_btree_iterator(const iterator &x) + : generation_(x.generation()), + key_(x.key()), + iter_(x.iter()), + tree_(x.tree()) { + } + safe_btree_iterator(Tree *tree, const Iterator &iter) + : generation_(), + key_(), + iter_(iter), + tree_(tree) { + update(); + } + + Tree* tree() const { return tree_; } + int64 generation() const { return generation_; } + + Iterator* mutable_iter() const { + if (generation_ != tree_->generation()) { + if (generation_ > 0) { + // This does the wrong thing for a multi{set,map}. If my iter was + // pointing to the 2nd of 2 values with the same key, then this will + // reset it to point to the first. This is why we don't provide a + // safe_btree_multi{set,map}. + iter_ = tree_->internal_btree()->lower_bound(key_); + update(); + } else if (-generation_ != tree_->generation()) { + iter_ = tree_->internal_btree()->end(); + generation_ = -tree_->generation(); + } + } + return &iter_; + } + const Iterator& iter() const { + return *mutable_iter(); + } + + // Equality/inequality operators. + bool operator==(const const_iterator &x) const { + return iter() == x.iter(); + } + bool operator!=(const const_iterator &x) const { + return iter() != x.iter(); + } + + // Accessors for the key/value the iterator is pointing at. + const key_type& key() const { + return key_; + } + reference operator*() const { + DCHECK_GT(generation_, 0); + return iter().operator*(); + } + pointer operator->() const { + DCHECK_GT(generation_, 0); + return iter().operator->(); + } + + // Increment/decrement operators. + self_type& operator++() { + ++(*mutable_iter()); + update(); + return *this; + } + self_type& operator--() { + --(*mutable_iter()); + update(); + return *this; + } + self_type operator++(int) { + self_type tmp = *this; + ++*this; + return tmp; + } + self_type operator--(int) { + self_type tmp = *this; + --*this; + return tmp; + } + + private: + // The generation of the tree when "iter" was updated. + mutable int64 generation_; + // The key the iterator points to. + mutable key_type key_; + // The underlying iterator. + mutable Iterator iter_; + // The tree the iterator is associated with. + Tree *tree_; +}; + +template +class safe_btree { + typedef safe_btree self_type; + + typedef btree btree_type; + typedef typename btree_type::iterator tree_iterator; + typedef typename btree_type::const_iterator tree_const_iterator; + + public: + typedef typename btree_type::params_type params_type; + typedef typename btree_type::key_type key_type; + typedef typename btree_type::data_type data_type; + typedef typename btree_type::mapped_type mapped_type; + typedef typename btree_type::value_type value_type; + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + typedef typename btree_type::pointer pointer; + typedef typename btree_type::const_pointer const_pointer; + typedef typename btree_type::reference reference; + typedef typename btree_type::const_reference const_reference; + typedef typename btree_type::size_type size_type; + typedef typename btree_type::difference_type difference_type; + typedef safe_btree_iterator iterator; + typedef safe_btree_iterator< + const self_type, tree_const_iterator> const_iterator; + typedef std::reverse_iterator const_reverse_iterator; + typedef std::reverse_iterator reverse_iterator; + + public: + // Default constructor. + safe_btree(const key_compare &comp, const allocator_type &alloc) + : tree_(comp, alloc), + generation_(1) { + } + + // Copy constructor. + safe_btree(const self_type &x) + : tree_(x.tree_), + generation_(1) { + } + + iterator begin() { + return iterator(this, tree_.begin()); + } + const_iterator begin() const { + return const_iterator(this, tree_.begin()); + } + iterator end() { + return iterator(this, tree_.end()); + } + const_iterator end() const { + return const_iterator(this, tree_.end()); + } + reverse_iterator rbegin() { + return reverse_iterator(end()); + } + const_reverse_iterator rbegin() const { + return const_reverse_iterator(end()); + } + reverse_iterator rend() { + return reverse_iterator(begin()); + } + const_reverse_iterator rend() const { + return const_reverse_iterator(begin()); + } + + // Lookup routines. + iterator lower_bound(const key_type &key) { + return iterator(this, tree_.lower_bound(key)); + } + const_iterator lower_bound(const key_type &key) const { + return const_iterator(this, tree_.lower_bound(key)); + } + iterator upper_bound(const key_type &key) { + return iterator(this, tree_.upper_bound(key)); + } + const_iterator upper_bound(const key_type &key) const { + return const_iterator(this, tree_.upper_bound(key)); + } + pair equal_range(const key_type &key) { + pair p = tree_.equal_range(key); + return make_pair(iterator(this, p.first), + iterator(this, p.second)); + } + pair equal_range(const key_type &key) const { + pair p = tree_.equal_range(key); + return make_pair(const_iterator(this, p.first), + const_iterator(this, p.second)); + } + iterator find_unique(const key_type &key) { + return iterator(this, tree_.find_unique(key)); + } + const_iterator find_unique(const key_type &key) const { + return const_iterator(this, tree_.find_unique(key)); + } + iterator find_multi(const key_type &key) { + return iterator(this, tree_.find_multi(key)); + } + const_iterator find_multi(const key_type &key) const { + return const_iterator(this, tree_.find_multi(key)); + } + size_type count_unique(const key_type &key) const { + return tree_.count_unique(key); + } + size_type count_multi(const key_type &key) const { + return tree_.count_multi(key); + } + + // Insertion routines. + template + pair insert_unique(const key_type &key, ValuePointer value) { + pair p = tree_.insert_unique(key, value); + generation_ += p.second; + return make_pair(iterator(this, p.first), p.second); + } + pair insert_unique(const value_type &v) { + pair p = tree_.insert_unique(v); + generation_ += p.second; + return make_pair(iterator(this, p.first), p.second); + } + iterator insert_unique(iterator position, const value_type &v) { + tree_iterator tree_pos = position.iter(); + ++generation_; + return iterator(this, tree_.insert_unique(tree_pos, v)); + } + template + void insert_unique(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_unique(*b); + } + } + iterator insert_multi(const value_type &v) { + ++generation_; + return iterator(this, tree_.insert_multi(v)); + } + iterator insert_multi(iterator position, const value_type &v) { + tree_iterator tree_pos = position.iter(); + ++generation_; + return iterator(this, tree_.insert_multi(tree_pos, v)); + } + template + void insert_multi(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_multi(*b); + } + } + self_type& operator=(const self_type &x) { + if (&x == this) { + // Don't copy onto ourselves. + return *this; + } + ++generation_; + tree_ = x.tree_; + return *this; + } + + // Deletion routines. + void erase(const iterator &begin, const iterator &end) { + tree_.erase(begin.iter(), end.iter()); + ++generation_; + } + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + iterator erase(iterator iter) { + tree_iterator res = tree_.erase(iter.iter()); + ++generation_; + return iterator(this, res); + } + int erase_unique(const key_type &key) { + int res = tree_.erase_unique(key); + generation_ += res; + return res; + } + int erase_multi(const key_type &key) { + int res = tree_.erase_multi(key); + generation_ += res; + return res; + } + + // Access to the underlying btree. + btree_type* internal_btree() { return &tree_; } + const btree_type* internal_btree() const { return &tree_; } + + // Utility routines. + void clear() { + ++generation_; + tree_.clear(); + } + void swap(self_type &x) { + ++generation_; + ++x.generation_; + tree_.swap(x.tree_); + } + void dump(ostream &os) const { + tree_.dump(os); + } + void verify() const { + tree_.verify(); + } + int64 generation() const { + return generation_; + } + key_compare key_comp() const { return tree_.key_comp(); } + + // Size routines. + size_type size() const { return tree_.size(); } + size_type max_size() const { return tree_.max_size(); } + bool empty() const { return tree_.empty(); } + size_type height() const { return tree_.height(); } + size_type internal_nodes() const { return tree_.internal_nodes(); } + size_type leaf_nodes() const { return tree_.leaf_nodes(); } + size_type nodes() const { return tree_.nodes(); } + size_type bytes_used() const { return tree_.bytes_used(); } + static double average_bytes_per_value() { + return btree_type::average_bytes_per_value(); + } + double fullness() const { return tree_.fullness(); } + double overhead() const { return tree_.overhead(); } + + private: + btree_type tree_; + int64 generation_; +}; + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_SAFE_BTREE_H__ diff --git a/safe_btree_map.h b/safe_btree_map.h new file mode 100644 index 00000000..74c5fa37 --- /dev/null +++ b/safe_btree_map.h @@ -0,0 +1,70 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: pmattis@google.com (Peter Mattis) +// +// The safe_btree_map<> is like btree_map<> except that it removes the caveat +// about insertion and deletion invalidating existing iterators at a small cost +// in making iterators larger and slower. + +#ifndef UTIL_BTREE_SAFE_BTREE_MAP_H__ +#define UTIL_BTREE_SAFE_BTREE_MAP_H__ + +#include +#include +#include + +#include "util/btree/btree_container.h" +#include "util/btree/btree_map.h" +#include "util/btree/safe_btree.h" + +namespace util { +namespace btree { + +// The safe_btree_map class is needed mainly for it's constructors. +template , + typename Alloc = std::allocator >, + int TargetNodeSize = 256> +class safe_btree_map : public btree_map_container< + safe_btree > > { + + typedef safe_btree_map self_type; + typedef btree_map_params< + Key, Value, Compare, Alloc, TargetNodeSize> params_type; + typedef safe_btree btree_type; + typedef btree_map_container super_type; + + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + + public: + // Default constructor. + safe_btree_map(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + safe_btree_map(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + safe_btree_map(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(b, e, comp, alloc) { + } +}; + +template +inline void swap(safe_btree_map &x, + safe_btree_map &y) { + x.swap(y); +} + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_SAFE_BTREE_MAP_H__ diff --git a/safe_btree_set.h b/safe_btree_set.h new file mode 100644 index 00000000..329a46f9 --- /dev/null +++ b/safe_btree_set.h @@ -0,0 +1,68 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: pmattis@google.com (Peter Mattis) +// +// The safe_btree_set<> is like btree_set<> except that it removes the caveat +// about insertion and deletion invalidating existing iterators at a small cost +// in making iterators larger and slower. + +#ifndef UTIL_BTREE_SAFE_BTREE_SET_H__ +#define UTIL_BTREE_SAFE_BTREE_SET_H__ + +#include +#include + +#include "util/btree/btree_container.h" +#include "util/btree/btree_set.h" +#include "util/btree/safe_btree.h" + +namespace util { +namespace btree { + +// The safe_btree_set class is needed mainly for it's constructors. +template , + typename Alloc = std::allocator, + int TargetNodeSize = 256> +class safe_btree_set : public btree_unique_container< + safe_btree > > { + + typedef safe_btree_set self_type; + typedef btree_set_params params_type; + typedef safe_btree btree_type; + typedef btree_unique_container super_type; + + public: + typedef typename btree_type::key_compare key_compare; + typedef typename btree_type::allocator_type allocator_type; + + public: + // Default constructor. + safe_btree_set(const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + } + + // Copy constructor. + safe_btree_set(const self_type &x) + : super_type(x) { + } + + // Range constructor. + template + safe_btree_set(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(b, e, comp, alloc) { + } +}; + +template +inline void swap(safe_btree_set &x, + safe_btree_set &y) { + x.swap(y); +} + +} // namespace btree +} // namespace util + +#endif // UTIL_BTREE_SAFE_BTREE_SET_H__ diff --git a/safe_btree_test.cc b/safe_btree_test.cc new file mode 100644 index 00000000..d86623f9 --- /dev/null +++ b/safe_btree_test.cc @@ -0,0 +1,67 @@ +// Copyright 2007 Google Inc. All Rights Reserved. +// Author: jmacd@google.com (Josh MacDonald) +// Author: pmattis@google.com (Peter Mattis) + +// TODO(pmattis): Add some tests that iterators are not invalidated by +// insertion and deletion. + +#include +#include +#include +#include +#include + +#include "base/arena-inl.h" +#include "base/init_google.h" +#include "base/integral_types.h" +#include "base/logging.h" +#include "strings/cord.h" +#include "testing/base/public/gunit.h" +#include "util/btree/btree_test.h" +#include "util/btree/safe_btree_map.h" +#include "util/btree/safe_btree_set.h" + +class UnsafeArena; + +namespace util { +namespace btree { +namespace { + +template +void SetTest() { + typedef ArenaAllocator ArenaAlloc; + BtreeTest, allocator, N>, set >(); + BtreeArenaTest, ArenaAlloc, N> >(); +} + +template +void MapTest() { + typedef ArenaAllocator ArenaAlloc; + BtreeTest, allocator, N>, map >(); + BtreeArenaTest, ArenaAlloc, N> >(); + BtreeMapTest, allocator, N> >(); +} + +TEST(SafeBtree, set_int32_32) { SetTest(); } +TEST(SafeBtree, set_int32_64) { SetTest(); } +TEST(SafeBtree, set_int32_128) { SetTest(); } +TEST(SafeBtree, set_int32_256) { SetTest(); } +TEST(SafeBtree, set_int64_256) { SetTest(); } +TEST(SafeBtree, set_string_256) { SetTest(); } +TEST(SafeBtree, set_cord_256) { SetTest(); } +TEST(SafeBtree, set_pair_256) { SetTest, 256>(); } +TEST(SafeBtree, map_int32_256) { MapTest(); } +TEST(SafeBtree, map_int64_256) { MapTest(); } +TEST(SafeBtree, map_string_256) { MapTest(); } +TEST(SafeBtree, map_cord_256) { MapTest(); } +TEST(SafeBtree, map_pair_256) { MapTest, 256>(); } + +} // namespace +} // namespace btree +} // namespace util + +int main(int argc, char **argv) { + FLAGS_logtostderr = true; + InitGoogle(argv[0], &argc, &argv, true); + return RUN_ALL_TESTS(); +} From dd97f135d03ce44bb54f25e51cda10e71189e1ac Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Tue, 13 Dec 2011 10:48:14 -0800 Subject: [PATCH 02/74] Fix for kTargetNodeSize too large --- btree.h | 5 +++++ btree_nc.cc | 6 +++++- btree_nc_test.py | 28 ++++++++++------------------ 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/btree.h b/btree.h index b9fd04e2..afb508da 100644 --- a/btree.h +++ b/btree.h @@ -1435,6 +1435,11 @@ class btree : public Params::key_compare { sizeof(key_compare_checker(key_compare_helper()(key_type(), key_type()))) == sizeof(base::big_), key_comparison_function_must_return_bool); + + // Note: We insist on kTargetValues, which is computed from + // Params::kTargetNodeSize, falling under 256 because of the uint8 + // fields of base_fields. + COMPILE_ASSERT(kNodeValues < 256, target_node_size_too_large); }; //// diff --git a/btree_nc.cc b/btree_nc.cc index b24c312d..7f1260be 100644 --- a/btree_nc.cc +++ b/btree_nc.cc @@ -40,7 +40,11 @@ TEST_COMPARE_TO(int); #elif defined(TEST_compare_to_float) TEST_COMPARE_TO(float); #elif defined(TEST_compare_to_pointer) -TEST_COMPARE_TO(pointer); +TEST_COMPARE_TO(void*); +#elif defined(TEST_large_nodesize) +void LargeNode() { + util::btree::btree_set, std::allocator, 10000> set; +} #endif } // namespace diff --git a/btree_nc_test.py b/btree_nc_test.py index 056a75d2..b2a83277 100755 --- a/btree_nc_test.py +++ b/btree_nc_test.py @@ -30,28 +30,20 @@ class BtreeNegativeUnitTest(googletest.TestCase): # Test that int does not work as a return type for key comparison. ('int', - [r'error: creating array with negative size', # for gcc - r'', # for icc - ]), + [r'key_comparison_function_must_return_bool']), # Test that float does not work as a return type for key comparison. ('float', - [r'error: creating array with negative size', # for gcc - r'', # for icc - ]), + [r'key_comparison_function_must_return_bool']), # Test that void* does not work as a return type for key comparison. ('pointer', - [r'error: creating array with negative size', # for gcc - r'', # for icc - ]), + [r'key_comparison_function_must_return_bool']), # Test that bool does not work as a return type for compare-to # comparison. ('compare_to_bool', - [r'error: creating array with negative size', # for gcc - r'', # for icc - ]), + [r'key_comparison_function_must_return_bool']), # Test that int works as a return type for compare-to comparison. ('compare_to_int', None), # None means compilation should succeed. @@ -59,16 +51,16 @@ class BtreeNegativeUnitTest(googletest.TestCase): # Test that float does not work as a return type for compare-to # comparison. ('compare_to_float', - [r'error: creating array with negative size', # for gcc - r'', # for icc - ]), + [r'key_comparison_function_must_return_bool']), # Test that void* does not work as a return type for compare-to # comparison. ('compare_to_pointer', - [r'error: creating array with negative size', # for gcc - r'', # for icc - ]), + [r'key_comparison_function_must_return_bool']), + + # Test that large node sizes do not compile. + ('large_nodesize', + [r'target_node_size_too_large']), ] # Runs the list of tests. From 9fe3bb3b33b805572778c3178f01664b25684542 Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Wed, 14 Dec 2011 10:14:35 -0800 Subject: [PATCH 03/74] support for >= 256 values per node --- btree.h | 59 +++++++++++++++++++++++++++++++++++++++++--------- btree_bench.cc | 13 ++++++++--- btree_nc.cc | 5 ++++- btree_test.cc | 34 ++++++++++++++++++++++++++++- 4 files changed, 96 insertions(+), 15 deletions(-) diff --git a/btree.h b/btree.h index afb508da..d911d5bb 100644 --- a/btree.h +++ b/btree.h @@ -316,8 +316,40 @@ static bool btree_compare_keys( return key_comparer::bool_compare(comp, x, y); } +// btree_is_node_big<> is a recursive template to determine whether a +// node of TargetNodeSize bytes needs a larger base_fields type +// (uint16, instead of uint8) to accomodate >= 256 values per node. +template +struct btree_is_node_big : + btree_is_node_big<(TargetNodeSize / 2), (ValueSize / 2)> { +}; +template +struct btree_is_node_big { + enum { + // In the base case, TargetNodeSize corresponds to single-byte + // entries and is the maximum number of values. + is_big = base::integral_constant= 256)>::value, + }; +}; + +// A helper for sizing the btree node's base_fields. The "type" +// typedef in this struct is an integral type large enough to hold as +// many ValueSize-values as will fit a node of TargetNodeSize bytes. +template +struct btree_base_field_type { + enum { + // "value_space" is the maximum leaf node count. leaf nodes have + // a greatest maximum of the node types. + value_space = TargetNodeSize - 2 * sizeof(void*), + }; + typedef typename base::if_< + btree_is_node_big::is_big, + uint16, + uint8>::type type; +}; + template + typename Alloc, int TargetNodeSize, int ValueSize> struct btree_common_params { // If Compare is derived from btree_key_compare_to_tag then use it as the // key_compare type. Otherwise, use btree_key_compare_to_adapter<> which will @@ -333,6 +365,8 @@ struct btree_common_params { typedef Key key_type; typedef ssize_t size_type; typedef ptrdiff_t difference_type; + typedef typename btree_base_field_type::type + base_field_type; enum { kTargetNodeSize = TargetNodeSize, @@ -343,7 +377,8 @@ struct btree_common_params { template struct btree_map_params - : public btree_common_params { + : public btree_common_params { typedef Data data_type; typedef Data mapped_type; typedef pair value_type; @@ -368,7 +403,8 @@ struct btree_map_params // A parameters structure for holding the type parameters for a btree_set. template struct btree_set_params - : public btree_common_params { + : public btree_common_params { typedef base::false_type data_type; typedef base::false_type mapped_type; typedef Key value_type; @@ -505,14 +541,16 @@ class btree_node { linear_search_type, binary_search_type>::type search_type; struct base_fields { + typedef typename Params::base_field_type field_type; + // A boolean indicating whether the node is a leaf or not. - uint8 leaf; + bool leaf; // The position of the node in the node's parent. - uint8 position; + field_type position; // The maximum number of values the node can hold. - uint8 max_count; + field_type max_count; // The count of the number of values in the node. - uint8 count; + field_type count; // A pointer to the node's parent. btree_node *parent; }; @@ -1437,9 +1475,10 @@ class btree : public Params::key_compare { key_comparison_function_must_return_bool); // Note: We insist on kTargetValues, which is computed from - // Params::kTargetNodeSize, falling under 256 because of the uint8 - // fields of base_fields. - COMPILE_ASSERT(kNodeValues < 256, target_node_size_too_large); + // Params::kTargetNodeSize, must fit the base_fields::field_type. + COMPILE_ASSERT(kNodeValues < + (1 << (8 * sizeof(typename base_fields::field_type))), + target_node_size_too_large); }; //// diff --git a/btree_bench.cc b/btree_bench.cc index c67dd664..94a1f272 100644 --- a/btree_bench.cc +++ b/btree_bench.cc @@ -401,7 +401,10 @@ typedef multimap stl_multimap_cord; MY_BENCHMARK_TYPES2(value, name, 416); \ MY_BENCHMARK_TYPES2(value, name, 448); \ MY_BENCHMARK_TYPES2(value, name, 480); \ - MY_BENCHMARK_TYPES2(value, name, 512) + MY_BENCHMARK_TYPES2(value, name, 512); \ + MY_BENCHMARK_TYPES2(value, name, 1024); \ + MY_BENCHMARK_TYPES2(value, name, 1536); \ + MY_BENCHMARK_TYPES2(value, name, 2048) MY_BENCHMARK_TYPES(int32, int32); MY_BENCHMARK_TYPES(int64, int64); @@ -433,10 +436,14 @@ MY_BENCHMARK_TYPES(Cord, cord); MY_BENCHMARK4(tree ## _416_ ## type, name, func); \ MY_BENCHMARK4(tree ## _448_ ## type, name, func); \ MY_BENCHMARK4(tree ## _480_ ## type, name, func); \ - MY_BENCHMARK4(tree ## _512_ ## type, name, func) + MY_BENCHMARK4(tree ## _512_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _1024_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _1536_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _2048_ ## type, name, func) #else #define MY_BENCHMARK3(tree, type, name, func) \ - MY_BENCHMARK4(tree ## _256_ ## type, name, func) + MY_BENCHMARK4(tree ## _256_ ## type, name, func); \ + MY_BENCHMARK4(tree ## _2048_ ## type, name, func) #endif #define MY_BENCHMARK2(type, name, func) \ diff --git a/btree_nc.cc b/btree_nc.cc index 7f1260be..0a6ef512 100644 --- a/btree_nc.cc +++ b/btree_nc.cc @@ -43,7 +43,10 @@ TEST_COMPARE_TO(float); TEST_COMPARE_TO(void*); #elif defined(TEST_large_nodesize) void LargeNode() { - util::btree::btree_set, std::allocator, 10000> set; + // (1 << 20) with 8-byte values is 2^17 values per node, which + // overflows the uint16 of btree::node_type::base_fields. + util::btree::btree_set, std::allocator, 1 << 20> + large_node_set; } #endif diff --git a/btree_test.cc b/btree_test.cc index ef4e574e..d984ec64 100644 --- a/btree_test.cc +++ b/btree_test.cc @@ -47,6 +47,25 @@ TEST(Btree, map_string_256) { MapTest(); } TEST(Btree, map_cord_256) { MapTest(); } TEST(Btree, map_pair_256) { MapTest, 256>(); } +// Large-node tests +TEST(Btree, map_int32_1024) { MapTest(); } +TEST(Btree, map_int32_1032) { MapTest(); } +TEST(Btree, map_int32_1040) { MapTest(); } +TEST(Btree, map_int32_1048) { MapTest(); } +TEST(Btree, map_int32_1056) { MapTest(); } + +TEST(Btree, map_int32_2048) { MapTest(); } +TEST(Btree, map_int32_4096) { MapTest(); } +TEST(Btree, set_int32_1024) { SetTest(); } +TEST(Btree, set_int32_2048) { SetTest(); } +TEST(Btree, set_int32_4096) { SetTest(); } +TEST(Btree, map_string_1024) { MapTest(); } +TEST(Btree, map_string_2048) { MapTest(); } +TEST(Btree, map_string_4096) { MapTest(); } +TEST(Btree, set_string_1024) { SetTest(); } +TEST(Btree, set_string_2048) { SetTest(); } +TEST(Btree, set_string_4096) { SetTest(); } + template void MultiSetTest() { typedef ArenaAllocator ArenaAlloc; @@ -77,6 +96,20 @@ TEST(Btree, multimap_string_256) { MultiMapTest(); } TEST(Btree, multimap_cord_256) { MultiMapTest(); } TEST(Btree, multimap_pair_256) { MultiMapTest, 256>(); } +// Large-node tests +TEST(Btree, multimap_int32_1024) { MultiMapTest(); } +TEST(Btree, multimap_int32_2048) { MultiMapTest(); } +TEST(Btree, multimap_int32_4096) { MultiMapTest(); } +TEST(Btree, multiset_int32_1024) { MultiSetTest(); } +TEST(Btree, multiset_int32_2048) { MultiSetTest(); } +TEST(Btree, multiset_int32_4096) { MultiSetTest(); } +TEST(Btree, multimap_string_1024) { MultiMapTest(); } +TEST(Btree, multimap_string_2048) { MultiMapTest(); } +TEST(Btree, multimap_string_4096) { MultiMapTest(); } +TEST(Btree, multiset_string_1024) { MultiSetTest(); } +TEST(Btree, multiset_string_2048) { MultiSetTest(); } +TEST(Btree, multiset_string_4096) { MultiSetTest(); } + // Verify that swapping btrees swaps the key comparision functors. struct SubstringLess { SubstringLess() : n(2) {} @@ -155,7 +188,6 @@ TEST(Btree, IteratorIncrementBy) { } } - } // namespace } // namespace btree } // namespace util From 89a0af9086bf45328f7ea188aeb3f6918eaf16f6 Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Wed, 14 Dec 2011 13:47:35 -0800 Subject: [PATCH 04/74] Simplify btree_base_field_type<> --- btree.h | 51 ++++++++++++++++----------------------------------- 1 file changed, 16 insertions(+), 35 deletions(-) diff --git a/btree.h b/btree.h index d911d5bb..d8e485ef 100644 --- a/btree.h +++ b/btree.h @@ -316,38 +316,6 @@ static bool btree_compare_keys( return key_comparer::bool_compare(comp, x, y); } -// btree_is_node_big<> is a recursive template to determine whether a -// node of TargetNodeSize bytes needs a larger base_fields type -// (uint16, instead of uint8) to accomodate >= 256 values per node. -template -struct btree_is_node_big : - btree_is_node_big<(TargetNodeSize / 2), (ValueSize / 2)> { -}; -template -struct btree_is_node_big { - enum { - // In the base case, TargetNodeSize corresponds to single-byte - // entries and is the maximum number of values. - is_big = base::integral_constant= 256)>::value, - }; -}; - -// A helper for sizing the btree node's base_fields. The "type" -// typedef in this struct is an integral type large enough to hold as -// many ValueSize-values as will fit a node of TargetNodeSize bytes. -template -struct btree_base_field_type { - enum { - // "value_space" is the maximum leaf node count. leaf nodes have - // a greatest maximum of the node types. - value_space = TargetNodeSize - 2 * sizeof(void*), - }; - typedef typename base::if_< - btree_is_node_big::is_big, - uint16, - uint8>::type type; -}; - template struct btree_common_params { @@ -365,12 +333,21 @@ struct btree_common_params { typedef Key key_type; typedef ssize_t size_type; typedef ptrdiff_t difference_type; - typedef typename btree_base_field_type::type - base_field_type; enum { kTargetNodeSize = TargetNodeSize, + + // Available space for values. This is largest for leaf nodes, + // which has overhead no fewer than two pointers. + kNodeValueSpace = TargetNodeSize - 2 * sizeof(void*), }; + + // This is an integral type large enough to hold as many + // ValueSize-values as will fit a node of TargetNodeSize bytes. + typedef typename base::if_< + (kNodeValueSpace / ValueSize) >= 256, + uint16, + uint8>::type node_count_type; }; // A parameters structure for holding the type parameters for a btree_map. @@ -541,7 +518,7 @@ class btree_node { linear_search_type, binary_search_type>::type search_type; struct base_fields { - typedef typename Params::base_field_type field_type; + typedef typename Params::node_count_type field_type; // A boolean indicating whether the node is a leaf or not. bool leaf; @@ -1479,6 +1456,10 @@ class btree : public Params::key_compare { COMPILE_ASSERT(kNodeValues < (1 << (8 * sizeof(typename base_fields::field_type))), target_node_size_too_large); + + // Test the assumption made in setting kNodeValueSpace. + COMPILE_ASSERT(sizeof(base_fields) >= 2 * sizeof(void*), + node_space_assumption_incorrect); }; //// From eaf6eb103dbc14c3d88218fd694f25b8472307bc Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Tue, 17 Jul 2012 17:20:45 -0700 Subject: [PATCH 05/74] Updates from google3; adding safe_btree comments, container (in)equality operators --- btree_container.h | 17 +++++++++++++++ btree_test.cc | 54 ++++++++++++++++++++++++++++++++++++++++++++++ safe_btree.h | 11 +++++++++- safe_btree_map.h | 16 +++++++++++--- safe_btree_set.h | 15 +++++++++++-- safe_btree_test.cc | 54 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 161 insertions(+), 6 deletions(-) diff --git a/btree_container.h b/btree_container.h index 6703d1bf..beebf6ff 100644 --- a/btree_container.h +++ b/btree_container.h @@ -106,6 +106,23 @@ class btree_container { double fullness() const { return tree_.fullness(); } double overhead() const { return tree_.overhead(); } + bool operator==(const self_type& x) const { + if (size() != x.size()) { + return false; + } + for (const_iterator i = begin(), xi = x.begin(); i != end(); ++i, ++xi) { + if (*i != *xi) { + return false; + } + } + return true; + } + + bool operator!=(const self_type& other) const { + return !operator==(other); + } + + protected: Tree tree_; }; diff --git a/btree_test.cc b/btree_test.cc index d984ec64..335318de 100644 --- a/btree_test.cc +++ b/btree_test.cc @@ -188,6 +188,60 @@ TEST(Btree, IteratorIncrementBy) { } } +TEST(Btree, Comparison) { + const int kSetSize = 1201; + btree_set my_set; + for (int i = 0; i < kSetSize; ++i) { + my_set.insert(i); + } + btree_set my_set_copy(my_set); + EXPECT_TRUE(my_set_copy == my_set); + EXPECT_TRUE(my_set == my_set_copy); + EXPECT_FALSE(my_set_copy != my_set); + EXPECT_FALSE(my_set != my_set_copy); + + my_set.insert(kSetSize); + EXPECT_FALSE(my_set_copy == my_set); + EXPECT_FALSE(my_set == my_set_copy); + EXPECT_TRUE(my_set_copy != my_set); + EXPECT_TRUE(my_set != my_set_copy); + + my_set.erase(kSetSize - 1); + EXPECT_FALSE(my_set_copy == my_set); + EXPECT_FALSE(my_set == my_set_copy); + EXPECT_TRUE(my_set_copy != my_set); + EXPECT_TRUE(my_set != my_set_copy); + + btree_map my_map; + for (int i = 0; i < kSetSize; ++i) { + my_map[string(i, 'a')] = i; + } + btree_map my_map_copy(my_map); + EXPECT_TRUE(my_map_copy == my_map); + EXPECT_TRUE(my_map == my_map_copy); + EXPECT_FALSE(my_map_copy != my_map); + EXPECT_FALSE(my_map != my_map_copy); + + ++my_map_copy[string(7, 'a')]; + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); + + my_map_copy = my_map; + my_map["hello"] = kSetSize; + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); + + my_map.erase(string(kSetSize - 1, 'a')); + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); +} + } // namespace } // namespace btree } // namespace util diff --git a/safe_btree.h b/safe_btree.h index b3db5360..2a354232 100644 --- a/safe_btree.h +++ b/safe_btree.h @@ -1,4 +1,4 @@ -// Copyright 2007 Google Inc. All Rights Reserved. +// Copyright 2007, 2012 Google Inc. All Rights Reserved. // Author: pmattis@google.com (Peter Mattis) // // A safe_btree<> wraps around a btree<> and removes the caveat that insertion @@ -8,6 +8,11 @@ // it was last validated and the key the underlying btree<>::iterator points // to. If an iterator is accessed and its generation differs from the tree // generation it is revalidated. +// +// References and pointers returned by safe_btree iterators are not safe. +// +// See the incorrect usage examples mentioned in safe_btree_set.h and +// safe_btree_map.h. #ifndef UTIL_BTREE_SAFE_BTREE_H__ #define UTIL_BTREE_SAFE_BTREE_H__ @@ -106,10 +111,14 @@ class safe_btree_iterator { const key_type& key() const { return key_; } + // This reference value is potentially invalidated by any non-const + // method on the tree; it is NOT safe. reference operator*() const { DCHECK_GT(generation_, 0); return iter().operator*(); } + // This pointer value is potentially invalidated by any non-const + // method on the tree; it is NOT safe. pointer operator->() const { DCHECK_GT(generation_, 0); return iter().operator->(); diff --git a/safe_btree_map.h b/safe_btree_map.h index 74c5fa37..11274930 100644 --- a/safe_btree_map.h +++ b/safe_btree_map.h @@ -1,10 +1,20 @@ -// Copyright 2007 Google Inc. All Rights Reserved. +// Copyright 2007, 2012 Google Inc. All Rights Reserved. // Author: pmattis@google.com (Peter Mattis) // // The safe_btree_map<> is like btree_map<> except that it removes the caveat // about insertion and deletion invalidating existing iterators at a small cost // in making iterators larger and slower. - +// +// Revalidation occurs whenever an iterator is accessed. References +// and pointers returned by safe_btree_map<> iterators are not stable, +// they are potentially invalidated by any non-const method on the map. +// +// BEGIN INCORRECT EXAMPLE +// for (auto i = safe_map->begin(); i != safe_map->end(); ++i) { +// const T *value = &i->second; // DO NOT DO THIS +// [code that modifies safe_map and uses value]; +// } +// END INCORRECT EXAMPLE #ifndef UTIL_BTREE_SAFE_BTREE_MAP_H__ #define UTIL_BTREE_SAFE_BTREE_MAP_H__ @@ -19,7 +29,7 @@ namespace util { namespace btree { -// The safe_btree_map class is needed mainly for it's constructors. +// The safe_btree_map class is needed mainly for its constructors. template , typename Alloc = std::allocator >, diff --git a/safe_btree_set.h b/safe_btree_set.h index 329a46f9..72ed2d91 100644 --- a/safe_btree_set.h +++ b/safe_btree_set.h @@ -1,9 +1,20 @@ -// Copyright 2007 Google Inc. All Rights Reserved. +// Copyright 2007, 2012 Google Inc. All Rights Reserved. // Author: pmattis@google.com (Peter Mattis) // // The safe_btree_set<> is like btree_set<> except that it removes the caveat // about insertion and deletion invalidating existing iterators at a small cost // in making iterators larger and slower. +// +// Revalidation occurs whenever an iterator is accessed. References +// and pointers returned by safe_btree_map<> iterators are not stable, +// they are potentially invalidated by any non-const method on the set. +// +// BEGIN INCORRECT EXAMPLE +// for (auto i = safe_set->begin(); i != safe_set->end(); ++i) { +// const T &value = *i; // DO NOT DO THIS +// [code that modifies safe_set and uses value]; +// } +// END INCORRECT EXAMPLE #ifndef UTIL_BTREE_SAFE_BTREE_SET_H__ #define UTIL_BTREE_SAFE_BTREE_SET_H__ @@ -18,7 +29,7 @@ namespace util { namespace btree { -// The safe_btree_set class is needed mainly for it's constructors. +// The safe_btree_set class is needed mainly for its constructors. template , typename Alloc = std::allocator, diff --git a/safe_btree_test.cc b/safe_btree_test.cc index d86623f9..cf179b28 100644 --- a/safe_btree_test.cc +++ b/safe_btree_test.cc @@ -56,6 +56,60 @@ TEST(SafeBtree, map_string_256) { MapTest(); } TEST(SafeBtree, map_cord_256) { MapTest(); } TEST(SafeBtree, map_pair_256) { MapTest, 256>(); } +TEST(SafeBtree, Comparison) { + const int kSetSize = 1201; + safe_btree_set my_set; + for (int i = 0; i < kSetSize; ++i) { + my_set.insert(i); + } + safe_btree_set my_set_copy(my_set); + EXPECT_TRUE(my_set_copy == my_set); + EXPECT_TRUE(my_set == my_set_copy); + EXPECT_FALSE(my_set_copy != my_set); + EXPECT_FALSE(my_set != my_set_copy); + + my_set.insert(kSetSize); + EXPECT_FALSE(my_set_copy == my_set); + EXPECT_FALSE(my_set == my_set_copy); + EXPECT_TRUE(my_set_copy != my_set); + EXPECT_TRUE(my_set != my_set_copy); + + my_set.erase(kSetSize - 1); + EXPECT_FALSE(my_set_copy == my_set); + EXPECT_FALSE(my_set == my_set_copy); + EXPECT_TRUE(my_set_copy != my_set); + EXPECT_TRUE(my_set != my_set_copy); + + safe_btree_map my_map; + for (int i = 0; i < kSetSize; ++i) { + my_map[string(i, 'a')] = i; + } + safe_btree_map my_map_copy(my_map); + EXPECT_TRUE(my_map_copy == my_map); + EXPECT_TRUE(my_map == my_map_copy); + EXPECT_FALSE(my_map_copy != my_map); + EXPECT_FALSE(my_map != my_map_copy); + + ++my_map_copy[string(7, 'a')]; + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); + + my_map_copy = my_map; + my_map["hello"] = kSetSize; + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); + + my_map.erase(string(kSetSize - 1, 'a')); + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); +} + } // namespace } // namespace btree } // namespace util From 9783752d6226448ad4fdd2e22bd0a77a28ee1012 Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Tue, 17 Jul 2012 17:22:55 -0700 Subject: [PATCH 06/74] Remove files that are very google-specific --- Makefile | 75 ------------------------- btree_nc.cc | 57 ------------------- btree_nc_test.py | 75 ------------------------- btree_printer.py | 125 ------------------------------------------ btree_printer_test.py | 92 ------------------------------- btree_test_program.cc | 64 --------------------- 6 files changed, 488 deletions(-) delete mode 100644 Makefile delete mode 100644 btree_nc.cc delete mode 100755 btree_nc_test.py delete mode 100755 btree_printer.py delete mode 100755 btree_printer_test.py delete mode 100644 btree_test_program.cc diff --git a/Makefile b/Makefile deleted file mode 100644 index c58946f9..00000000 --- a/Makefile +++ /dev/null @@ -1,75 +0,0 @@ -# -*- mode: python; -*- - -# This should be a Makefile, but it's not. - -# cc_library(name = "btree", -# srcs = [ "btree.h", -# "btree_container.h", -# "btree_map.h", -# "btree_set.h", -# "safe_btree.h", -# "safe_btree_map.h", -# "safe_btree_set.h" ], -# deps = [ "//strings", -# "//strings:cord" ]) - -# cc_library(name = "btree_test_flags", -# srcs = [ "btree_test_flags.cc" ], -# deps = [ "//base" ]) - -# cc_binary(name = "btree_bench", -# srcs = [ "btree_bench.cc" ], -# deps = [ ":btree", -# ":btree_test_flags", -# "//testing/base", -# "//util/random" ]) - -# cc_test(name = "btree_test", -# srcs = [ "btree_test.cc", ], -# deps = [ ":btree", -# ":btree_test_flags", -# "//base:heapcheck", -# "//testing/base", -# "//util/random", -# ], -# size = "large") - -# cc_test(name = "safe_btree_test", -# srcs = [ "safe_btree_test.cc", ], -# deps = [ ":btree", -# ":btree_test_flags", -# "//base:heapcheck", -# "//testing/base", -# "//util/random", -# ], -# size = "large") - -# cc_fake_binary(name = "btree_nc", -# srcs = [ "btree_nc.cc" ], -# deps = [ ":btree" ], -# legacy = 0) - -# py_test(name = "btree_nc_test", -# srcs = [ "btree_nc_test.py" ], -# deps = [ "//pyglib", -# "//testing/pybase" ], -# data = [ "btree_nc" ], -# size = "large") - -# cc_binary(name = "btree_test_program", -# srcs = [ "btree_test_program.cc" ], -# deps = [ ":btree", -# "//devtools/gdb/component:gdb_test_utils" ], -# testonly = 1) - -# # This test will only actually test the pretty-printing code if it's -# # compiled with debug information (blaze build -c dbg). The default -# # mode, fastbuild, will pass but will not catch any regressions! -# py_test(name = "btree_printer_test", -# size = "large", -# srcs = [ "btree_printer_test.py", -# "btree_printer.py" ], -# deps = [ "//devtools/gdb/component:gdbpy", -# "//testing/pybase", -# "//testing/gdb:gdb_script_test_util", -# ":btree_test_program" ]) diff --git a/btree_nc.cc b/btree_nc.cc deleted file mode 100644 index 0a6ef512..00000000 --- a/btree_nc.cc +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2009 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) - -#include "util/btree/btree_set.h" - -namespace { - -template -struct Compare { - R operator()(int a, int b) const { return reinterpret_cast(a < b); } -}; - -template -struct CompareTo : public util::btree::btree_key_compare_to_tag { - R operator()(int a, int b) const { return reinterpret_cast(a < b); } -}; - -#define TEST_COMPARE(r) \ - void TestCompile() { \ - util::btree::btree_set > s; \ - } - -#define TEST_COMPARE_TO(r) \ - void TestCompile() { \ - util::btree::btree_set > s; \ - } - -#if defined(TEST_bool) -TEST_COMPARE(bool); -#elif defined(TEST_int) -TEST_COMPARE(int); -#elif defined(TEST_float) -TEST_COMPARE(float); -#elif defined(TEST_pointer) -TEST_COMPARE(void*); -#elif defined(TEST_compare_to_bool) -TEST_COMPARE_TO(bool); -#elif defined(TEST_compare_to_int) -TEST_COMPARE_TO(int); -#elif defined(TEST_compare_to_float) -TEST_COMPARE_TO(float); -#elif defined(TEST_compare_to_pointer) -TEST_COMPARE_TO(void*); -#elif defined(TEST_large_nodesize) -void LargeNode() { - // (1 << 20) with 8-byte values is 2^17 values per node, which - // overflows the uint16 of btree::node_type::base_fields. - util::btree::btree_set, std::allocator, 1 << 20> - large_node_set; -} -#endif - -} // namespace - -int main() { - return 1; -} diff --git a/btree_nc_test.py b/btree_nc_test.py deleted file mode 100755 index b2a83277..00000000 --- a/btree_nc_test.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/python2.4 -# -# Copyright 2006 Google Inc. All Rights Reserved. - -"""Negative compilation unit test for btree.h. -""" - -__author__ = 'pmattis@google.com (Peter Mattis)' - -import os -from google3.testing.pybase import googletest -from google3.testing.pybase import fake_target_util -from google3.pyglib import flags - -_FLAGS = flags.FLAGS - - -class BtreeNegativeUnitTest(googletest.TestCase): - """Negative compilation tests for btree.h""" - - def testCompilerErrors(self): - """Runs a list of tests to verify that erroneous code leads to - expected compiler messages.""" - - # Defines a list of test specs, where each element is a tuple - # (test name, list of regexes for matching the compiler errors). - test_specs = [ - # Test that bool works as a return type for key comparison. - ('bool', None), # None means compilation should succeed. - - # Test that int does not work as a return type for key comparison. - ('int', - [r'key_comparison_function_must_return_bool']), - - # Test that float does not work as a return type for key comparison. - ('float', - [r'key_comparison_function_must_return_bool']), - - # Test that void* does not work as a return type for key comparison. - ('pointer', - [r'key_comparison_function_must_return_bool']), - - # Test that bool does not work as a return type for compare-to - # comparison. - ('compare_to_bool', - [r'key_comparison_function_must_return_bool']), - - # Test that int works as a return type for compare-to comparison. - ('compare_to_int', None), # None means compilation should succeed. - - # Test that float does not work as a return type for compare-to - # comparison. - ('compare_to_float', - [r'key_comparison_function_must_return_bool']), - - # Test that void* does not work as a return type for compare-to - # comparison. - ('compare_to_pointer', - [r'key_comparison_function_must_return_bool']), - - # Test that large node sizes do not compile. - ('large_nodesize', - [r'target_node_size_too_large']), - ] - - # Runs the list of tests. - fake_target_util.AssertCcCompilerErrors( - self, - 'google3/util/btree/btree_nc', # path to the fake target file. - 'btree_nc.o', # name of the target to build. - test_specs) - - -if __name__ == '__main__': - googletest.main() diff --git a/btree_printer.py b/btree_printer.py deleted file mode 100755 index 8f233401..00000000 --- a/btree_printer.py +++ /dev/null @@ -1,125 +0,0 @@ -#!/usr/bin/python2.4 -# Copyright 2011 Google Inc. All Rights Reserved. -# GDB support for pretty printing StringPiece. - -"""GDB pretty-printer for btrees.""" - -# This is a module provided by GDB. -# Ref: http://wiki/Main/Google3GDBScripts -import gdb -from google3.devtools.gdb.component import printing - - -class BaseBtreePrinter(object): - """btree pretty-printer, for util/btree.""" - - def __init__(self, typename, val): - self.typename = typename - self.val = val - - def display_hint(self): - return 'array' - - def _my_iter(self, node): - count = node['fields_']['count'] - if node['fields_']['leaf']: - for i in range(count): - key = node['fields_']['values'][i] - yield ('[item]', key) - else: - # recursive generators are annoying: we can't just recurse, we need - # to expand and yield the values. - for i in range(count+1): - child = node['fields_']['children'][i] - for v in self._my_iter(child.dereference()): - yield v - - def children(self): - if self.nelements() != 0: - return self._my_iter(self.val['root_']['data'].dereference()) - else: - return iter([]) - - def nelements(self): - if self.val['root_']['data'] != 0: - root_fields = self.val['root_']['data'].dereference()['fields_'] - if root_fields['leaf']: - return root_fields['count'] - else: - return root_fields['size'] - else: - return 0 - - def to_string(self): # pylint: disable-msg=C6409 - """GDB calls this to compute the pretty-printed form.""" - return '%s with %d elements' % (self.typename, self.nelements()) - - -class BtreePrinter(BaseBtreePrinter): - """btree<> pretty-printer, for util/btree.""" - - def __init__(self, val): - BaseBtreePrinter.__init__(self, 'btree', val) - - -class BtreeSetPrinter(BaseBtreePrinter): - """btree_set<> pretty-printer.""" - - def __init__(self, val): - BaseBtreePrinter.__init__(self, 'btree_set', val['tree_']) - - -class BtreeMultisetPrinter(BaseBtreePrinter): - """btree_multiset<> pretty-printer.""" - - def __init__(self, val): - BaseBtreePrinter.__init__(self, 'btree_multiset', val['tree_']) - - -class BaseBtreeMapPrinter(BaseBtreePrinter): - """btree maps pretty-printer.""" - - def __init__(self, typename, val): - BaseBtreePrinter.__init__(self, typename, val['tree_']) - - def display_hint(self): - return 'map' - - def _my_map_iter(self, g): - for (_, pair) in g: - yield ('[key]', pair['first']) - yield ('[value]', pair['second']) - - def children(self): - # we need to break apart the pairs and yield them separately - if self.nelements() != 0: - return self._my_map_iter(BaseBtreePrinter.children(self)) - else: - return iter([]) - - -class BtreeMapPrinter(BaseBtreeMapPrinter): - """btree_map<> pretty-printer.""" - - def __init__(self, val): - BaseBtreeMapPrinter.__init__(self, 'btree_map', val) - - -class BtreeMultimapPrinter(BaseBtreeMapPrinter): - """btree_multimap<> pretty-printer.""" - - def __init__(self, val): - BaseBtreeMapPrinter.__init__(self, 'btree_multimap', val) - - -if __name__ == '__main__': - printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree<.*>', - BtreePrinter) - printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_set<.*>', - BtreeSetPrinter) - printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_multiset<.*>', - BtreeMultisetPrinter) - printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_map<.*>', - BtreeMapPrinter) - printing.RegisterGoogle3ClassPrettyPrinter('util::btree::btree_multimap<.*>', - BtreeMultimapPrinter) diff --git a/btree_printer_test.py b/btree_printer_test.py deleted file mode 100755 index 95bc351e..00000000 --- a/btree_printer_test.py +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/python2.4 -# -# Copyright 2011 Google Inc. All Rights Reserved. - -"""Tests for btree_printer.py gdb pretty printer.""" - -__author__ = "leg@google.com (Lawrence Greenfield)" - -from google3.pyglib import flags -from google3.testing.gdb import gdb_script_test_util -from google3.testing.pybase import googletest - -FLAGS = flags.FLAGS - - -class BtreePrinterTest(gdb_script_test_util.TestCase): - def testBtreeSet(self): - self.InitSession("btree_set", - "util/btree/btree_test_program") - self.RunTo("StopHereForDebugger") - - self.SetOption("print elements", 20) - self.TestPrintOutputMatches("*empty_set", - """btree_set with 0 elements""") - self.TestPrintOutputMatches("*small_set", - """btree_set with 10 elements = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}""") - self.TestPrintOutputMatches("*small_multiset", - """btree_multiset with 10 elements = {0, 0, 1, 1, 2, 2, 3, 3, 4, 4}""") - self.TestPrintOutputMatches("*big_set", - """btree_set with 80 elements = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19...}""") - - self.RunSession() - - def testBtreeMap(self): - self.InitSession("btree_set", - "util/btree/btree_test_program") - self.RunTo("StopHereForDebugger") - - self.SetOption("print elements", 30) - self.TestPrintOutputMatches("*empty_map", - """btree_map with 0 elements""") - self.TestPrintOutputMatches("*small_map", - """btree_map with 10 elements = { - \\[0\\] = 0, - \\[1\\] = 13, - \\[2\\] = 26, - \\[3\\] = 39, - \\[4\\] = 52, - \\[5\\] = 65, - \\[6\\] = 78, - \\[7\\] = 91, - \\[8\\] = 104, - \\[9\\] = 117 -}""") - self.TestPrintOutputMatches("*small_multimap", - """btree_multimap with 10 elements = { - \\[0\\] = 0, - \\[0\\] = 1, - \\[1\\] = 2, - \\[1\\] = 3, - \\[2\\] = 4, - \\[2\\] = 5, - \\[3\\] = 6, - \\[3\\] = 7, - \\[4\\] = 8, - \\[4\\] = 9 -}""") - self.TestPrintOutputMatches("*big_map", - """btree_map with 80 elements = { - \\[0\\] = 0, - \\[1\\] = 7, - \\[2\\] = 14, - \\[3\\] = 21, - \\[4\\] = 28, - \\[5\\] = 35, - \\[6\\] = 42, - \\[7\\] = 49, - \\[8\\] = 56, - \\[9\\] = 63, - \\[10\\] = 70, - \\[11\\] = 77, - \\[12\\] = 84, - \\[13\\] = 91, - \\[14\\] = 98 - ... -}""") - - self.RunSession() - - -if __name__ == "__main__": - googletest.main() diff --git a/btree_test_program.cc b/btree_test_program.cc deleted file mode 100644 index 0c2dcf88..00000000 --- a/btree_test_program.cc +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2011 Google Inc. All Rights Reserved. -// Author: leg@google.com (Lawrence Greenfield) - -#include "base/init_google.h" -#include "base/logging.h" -#include "devtools/gdb/component/gdb_test_utils.h" -#include "util/btree/btree_map.h" -#include "util/btree/btree_set.h" - -using util::btree::btree_set; -using util::btree::btree_multiset; -using util::btree::btree_map; -using util::btree::btree_multimap; - -static btree_set* empty_set; -static btree_set* small_set; -static btree_set* big_set; -static btree_multiset* small_multiset; - -static btree_map* empty_map; -static btree_map* small_map; -static btree_map* big_map; -static btree_multimap* small_multimap; - -static void SetupBtreeSets() { - empty_set = new btree_set; - small_set = new btree_set; - small_multiset = new btree_multiset; - big_set = new btree_set; - - for (int i = 0; i < 10; ++i) { - small_set->insert(i); - small_multiset->insert(i / 2); - } - - for (int i = 0; i < 80; ++i) { - big_set->insert(i); - } -} - -static void SetupBtreeMaps() { - empty_map = new btree_map; - small_map = new btree_map; - small_multimap = new btree_multimap; - big_map = new btree_map; - - for (int i = 0; i < 10; ++i) { - small_map->insert(make_pair(i, i * 13)); - small_multimap->insert(make_pair(i / 2, i)); - } - - for (int i = 0; i < 80; ++i) { - big_map->insert(make_pair(i, i * 7)); - } -} - -int main(int argc, char** argv) { - FLAGS_logtostderr = true; - InitGoogle(argv[0], &argc, &argv, true); - SetupBtreeSets(); - SetupBtreeMaps(); - StopHereForDebugger(); - return 0; -} From 88853fbc9e377f591bef514f01626ee2732eaea3 Mon Sep 17 00:00:00 2001 From: Josh MacDonald Date: Tue, 17 Jul 2012 17:25:01 -0700 Subject: [PATCH 07/74] Build outside of Google; add CMakeLists.txt, README --- CMakeLists.txt | 23 +++ README | 15 ++ btree.h | 381 +++++++++++++++-------------------- btree_bench.cc | 2 - btree_container.h | 16 +- btree_map.h | 16 +- btree_set.h | 10 +- btree_test.cc | 197 +++++++++---------- btree_test.h | 469 ++++++++++++++++++++++++-------------------- btree_test_flags.cc | 2 +- safe_btree.h | 44 ++--- safe_btree_map.h | 12 +- safe_btree_set.h | 10 +- safe_btree_test.cc | 73 +++---- 14 files changed, 626 insertions(+), 644 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 README diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..e908f4a4 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,23 @@ +cmake_minimum_required(VERSION 2.6) + +project(cppbtree CXX) + +option(build_tests "Build B-tree tests" OFF) +add_definitions(-std=c++0x) +set(CMAKE_CXX_FLAGS "-g") + +# CMake doesn't have a way to pure template library, +add_library(cppbtree btree.h btree_map.h btree_set.h safe_btree.h safe_btree_map.h safe_btree_set.h) +set_target_properties(cppbtree PROPERTIES LINKER_LANGUAGE CXX) + +if(build_tests) + enable_testing() + include_directories($ENV{GTEST_ROOT}/include) + link_directories($ENV{GTEST_ROOT}) + add_executable(btree_test btree_test.cc btree_test_flags.cc) + add_executable(safe_btree_test safe_btree_test.cc btree_test_flags.cc) + add_executable(btree_bench btree_bench.cc btree_test_flags.cc) + target_link_libraries(btree_test gtest_main gflags) + target_link_libraries(safe_btree_test gtest_main gflags) + target_link_libraries(btree_bench gflags) +endif() diff --git a/README b/README new file mode 100644 index 00000000..152972cf --- /dev/null +++ b/README @@ -0,0 +1,15 @@ +This library is a C++ template library and, as such, there is no +library to build and install. + +---- + +To build and run the provided tests, however, you will need to install +CMake and the Google C++ Test framework. + +Download CMake from http://www.cmake.org + +Download the GoogleTest framework from http://code.google.com/p/googletest + +export GTEST_ROOT=/path/for/gtest-x.y + +cmake . -Dbuild_tests=ON diff --git a/btree.h b/btree.h index d8e485ef..a8d3152d 100644 --- a/btree.h +++ b/btree.h @@ -35,11 +35,6 @@ // // btree_bench --benchmarks=. 2>&1 | ./benchmarks.awk // -// NOTE(pmattis): My warpstation (pmattis-warp.nyc) often produces slower -// results when running the benchmarks on CPUs 0 and 1 vs CPUs 2 and 3. To get -// consistent benchmark results, I run "taskset 0xc " to run the -// benchmark on CPUs 2 and 3. -// // Run on pmattis-warp.nyc (4 X 2200 MHz CPUs); 2010/03/04-15:23:06 // Benchmark STL(ns) B-Tree(ns) @ // -------------------------------------------------------- @@ -91,61 +86,29 @@ // BM_map_string_mixedaddrem 6400 5200 +18.75% <256> [72.0, 57.8] // BM_map_string_fifo 398 596 -49.75% <256> [72.0, 44.0] // BM_map_string_fwditer 243 113 +53.50% <256> [72.0, 55.8] -// BM_set_cord_insert 3661 2680 +26.80% <256> [40.0, 10.5] -// BM_set_cord_lookup 2920 2293 +21.47% <256> [40.0, 10.5] -// BM_set_cord_fulllookup 2960 2267 +23.41% <256> [40.0, 8.8] -// BM_set_cord_delete 4679 2535 +45.82% <256> [40.0, 10.5] -// BM_set_cord_queueaddrem 8230 5600 +31.96% <256> [40.0, 11.3] -// BM_set_cord_mixedaddrem 8497 6080 +28.45% <256> [40.0, 10.7] -// BM_set_cord_fifo 358 370 -3.35% <256> [40.0, 8.8] -// BM_set_cord_fwditer 352 193 +45.17% <256> [40.0, 10.5] -// BM_map_cord_insert 3680 2927 +20.46% <256> [48.0, 20.7] -// BM_map_cord_lookup 3018 2466 +18.29% <256> [48.0, 20.7] -// BM_map_cord_fulllookup 2943 2466 +16.21% <256> [48.0, 17.2] -// BM_map_cord_delete 4675 2775 +40.64% <256> [48.0, 20.7] -// BM_map_cord_queueaddrem 8383 6360 +24.13% <256> [48.0, 22.2] -// BM_map_cord_mixedaddrem 8952 6760 +24.49% <256> [48.0, 21.2] -// BM_map_cord_fifo 444 463 -4.28% <256> [48.0, 17.2] -// BM_map_cord_fwditer 391 225 +42.46% <256> [48.0, 20.7] #ifndef UTIL_BTREE_BTREE_H__ #define UTIL_BTREE_BTREE_H__ +#include #include #include #include #include #include #include -#include // IWYU pragma: export // Clients can rely on this. +#include #include +#include #include -#include // IWYU pragma: export // Clients can rely on this. +#include #include #include -#include "base/gdb-scripting.h" -#include "base/integral_types.h" -#include "base/logging.h" -#include "base/macros.h" -#include "base/template_util.h" -#include "base/type_traits.h" -#include "strings/cord.h" -#include "strings/stringpiece.h" +#ifndef NDEBUG +#define NDEBUG 1 +#endif -// iwyu.py found std::swap in , but we include . -// IWYU pragma: no_include -// -// iwyu.py doesn't think we need , so wants . -// IWYU pragma: no_include -// -// iwyu.py sure wants to bring in a lot of private STL headers. -// IWYU pragma: no_include -// IWYU pragma: no_include -// IWYU pragma: no_include -// IWYU pragma: no_include - -namespace util { namespace btree { // Inside a btree method, if we just call swap(), it will choose the @@ -161,6 +124,32 @@ inline void btree_swap_helper(T &a, T &b) { swap(a, b); } +// A template helper used to select A or B based on a condition. +template +struct if_{ + typedef A type; +}; + +template +struct if_ { + typedef B type; +}; + +// Types small_ and big_ are promise that sizeof(small_) < sizeof(big_) +typedef char small_; + +struct big_ { + char dummy[2]; +}; + +// A compile-time assertion. +template +struct CompileAssert { +}; + +#define COMPILE_ASSERT(expr, msg) \ + typedef CompileAssert<(bool(expr))> msg[bool(expr) ? 1 : -1] + // A helper type used to indicate that a key-compare-to functor has been // provided. A user can specify a key-compare-to functor by doing: // @@ -180,7 +169,7 @@ struct btree_key_compare_to_tag { // btree_key_compare_to_tag. template struct btree_is_key_compare_to - : public base::is_convertible { + : public std::is_convertible { }; // A helper class to convert a boolean comparison into a three-way "compare-to" @@ -201,77 +190,29 @@ struct btree_key_compare_to_adapter : Compare { }; template <> -struct btree_key_compare_to_adapter > +struct btree_key_compare_to_adapter > : public btree_key_compare_to_tag { btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const less&) {} + btree_key_compare_to_adapter(const std::less&) {} btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const string &a, const string &b) const { + const btree_key_compare_to_adapter >&) {} + int operator()(const std::string &a, const std::string &b) const { return a.compare(b); } }; template <> -struct btree_key_compare_to_adapter > +struct btree_key_compare_to_adapter > : public btree_key_compare_to_tag { btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const greater&) {} + btree_key_compare_to_adapter(const std::greater&) {} btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const string &a, const string &b) const { + const btree_key_compare_to_adapter >&) {} + int operator()(const std::string &a, const std::string &b) const { return b.compare(a); } }; -template <> -struct btree_key_compare_to_adapter > - : public btree_key_compare_to_tag { - btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const less&) {} - btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const StringPiece &a, const StringPiece &b) const { - return a.compare(b); - } -}; - -template <> -struct btree_key_compare_to_adapter > - : public btree_key_compare_to_tag { - btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const greater&) {} - btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const StringPiece &a, const StringPiece &b) const { - return b.compare(a); - } -}; - -template <> -struct btree_key_compare_to_adapter > - : public btree_key_compare_to_tag { - btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const less&) {} - btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const Cord &a, const Cord &b) const { - return a.CompareTo(b); - } -}; - -template <> -struct btree_key_compare_to_adapter > - : public btree_key_compare_to_tag { - btree_key_compare_to_adapter() {} - btree_key_compare_to_adapter(const greater&) {} - btree_key_compare_to_adapter( - const btree_key_compare_to_adapter >&) {} - int operator()(const Cord &a, const Cord &b) const { - return b.CompareTo(a); - } -}; - // A helper class that allows a compare-to functor to behave like a plain // compare functor. This specialization is used when we do not have a // compare-to functor. @@ -322,7 +263,7 @@ struct btree_common_params { // If Compare is derived from btree_key_compare_to_tag then use it as the // key_compare type. Otherwise, use btree_key_compare_to_adapter<> which will // fall-back to Compare if we don't have an appropriate specialization. - typedef typename base::if_< + typedef typename if_< btree_is_key_compare_to::value, Compare, btree_key_compare_to_adapter >::type key_compare; // A type which indicates if we have a key-compare-to functor or a plain old @@ -344,10 +285,10 @@ struct btree_common_params { // This is an integral type large enough to hold as many // ValueSize-values as will fit a node of TargetNodeSize bytes. - typedef typename base::if_< + typedef typename if_< (kNodeValueSpace / ValueSize) >= 256, - uint16, - uint8>::type node_count_type; + uint16_t, + uint8_t>::type node_count_type; }; // A parameters structure for holding the type parameters for a btree_map. @@ -358,8 +299,8 @@ struct btree_map_params sizeof(Key) + sizeof(Data)> { typedef Data data_type; typedef Data mapped_type; - typedef pair value_type; - typedef pair mutable_value_type; + typedef std::pair value_type; + typedef std::pair mutable_value_type; typedef value_type* pointer; typedef const value_type* const_pointer; typedef value_type& reference; @@ -382,8 +323,8 @@ template struct btree_set_params : public btree_common_params { - typedef base::false_type data_type; - typedef base::false_type mapped_type; + typedef std::false_type data_type; + typedef std::false_type mapped_type; typedef Key value_type; typedef value_type mutable_value_type; typedef value_type* pointer; @@ -499,22 +440,22 @@ class btree_node { key_type, self_type, key_compare> binary_search_compare_to_type; // If we have a valid key-compare-to type, use linear_search_compare_to, // otherwise use linear_search_plain_compare. - typedef typename base::if_< + typedef typename if_< Params::is_key_compare_to::value, linear_search_compare_to_type, linear_search_plain_compare_type>::type linear_search_type; // If we have a valid key-compare-to type, use binary_search_compare_to, // otherwise use binary_search_plain_compare. - typedef typename base::if_< + typedef typename if_< Params::is_key_compare_to::value, binary_search_compare_to_type, binary_search_plain_compare_type>::type binary_search_type; // If the key is an integral or floating point type, use linear search which // is faster than binary search for such types. Might be wise to also // configure linear search based on node-size. - typedef typename base::if_< - base::is_integral::value || - base::is_floating_point::value, + typedef typename if_< + std::is_integral::value || + std::is_floating_point::value, linear_search_type, binary_search_type>::type search_type; struct base_fields { @@ -586,7 +527,7 @@ class btree_node { // be a leaf. bool is_root() const { return parent()->leaf(); } void make_root() { - DCHECK(parent()->is_root()); + assert(parent()->is_root()); fields_.parent = fields_.parent->parent(); } @@ -739,7 +680,7 @@ class btree_node { f->max_count = max_count; f->count = 0; f->parent = parent; - if (DEBUG_MODE) { + if (!NDEBUG) { memset(&f->values, 0, max_count * sizeof(value_type)); } return n; @@ -747,7 +688,7 @@ class btree_node { static btree_node* init_internal(internal_fields *f, btree_node *parent) { btree_node *n = init_leaf(f, parent, kNodeValues); f->leaf = 0; - if (DEBUG_MODE) { + if (!NDEBUG) { memset(f->children, 0, sizeof(f->children)); } return n; @@ -779,7 +720,8 @@ class btree_node { root_fields fields_; private: - DISALLOW_EVIL_CONSTRUCTORS(btree_node); + btree_node(const btree_node&); + void operator=(const btree_node&); }; template @@ -790,7 +732,7 @@ struct btree_iterator { typedef typename Node::params_type params_type; typedef Node node_type; - typedef typename base::remove_const::type normal_node; + typedef typename std::remove_const::type normal_node; typedef const Node const_node; typedef typename params_type::value_type value_type; typedef typename params_type::pointer normal_pointer; @@ -800,7 +742,7 @@ struct btree_iterator { typedef Pointer pointer; typedef Reference reference; - typedef bidirectional_iterator_tag iterator_category; + typedef std::bidirectional_iterator_tag iterator_category; typedef btree_iterator< normal_node, normal_reference, normal_pointer> iterator; @@ -885,7 +827,7 @@ struct btree_iterator { // Dispatch helper class for using btree::internal_locate with plain compare. struct btree_internal_locate_plain_compare { template - static pair dispatch(const K &k, const T &t, Iter iter) { + static std::pair dispatch(const K &k, const T &t, Iter iter) { return t.internal_locate_plain_compare(k, iter); } }; @@ -893,7 +835,7 @@ struct btree_internal_locate_plain_compare { // Dispatch helper class for using btree::internal_locate with compare-to. struct btree_internal_locate_compare_to { template - static pair dispatch(const K &k, const T &t, Iter iter) { + static std::pair dispatch(const K &k, const T &t, Iter iter) { return t.internal_locate_compare_to(k, iter); } }; @@ -910,7 +852,7 @@ class btree : public Params::key_compare { friend class btree_internal_locate_plain_compare; friend class btree_internal_locate_compare_to; - typedef typename base::if_< + typedef typename if_< is_key_compare_to::value, btree_internal_locate_compare_to, btree_internal_locate_plain_compare>::type internal_locate_type; @@ -1037,11 +979,11 @@ class btree : public Params::key_compare { // Finds the range of values which compare equal to key. The first member of // the returned pair is equal to lower_bound(key). The second member pair of // the pair is equal to upper_bound(key). - pair equal_range(const key_type &key) { - return make_pair(lower_bound(key), upper_bound(key)); + std::pair equal_range(const key_type &key) { + return std::make_pair(lower_bound(key), upper_bound(key)); } - pair equal_range(const key_type &key) const { - return make_pair(lower_bound(key), upper_bound(key)); + std::pair equal_range(const key_type &key) const { + return std::make_pair(lower_bound(key), upper_bound(key)); } // Inserts a value into the btree only if it does not already exist. The @@ -1050,11 +992,11 @@ class btree : public Params::key_compare { // is being inserted. Value is not dereferenced if the key already exists in // the btree. See btree_map::operator[]. template - pair insert_unique(const key_type &key, ValuePointer value); + std::pair insert_unique(const key_type &key, ValuePointer value); // Inserts a value into the btree only if it does not already exist. The // boolean return value indicates whether insertion succeeded or failed. - pair insert_unique(const value_type &v) { + std::pair insert_unique(const value_type &v) { return insert_unique(params_type::key(v), &v); } @@ -1170,7 +1112,7 @@ class btree : public Params::key_compare { // Dump the btree to the specified ostream. Requires that operator<< is // defined for Key and Value. - void dump(ostream &os) const { + void dump(std::ostream &os) const { if (root() != NULL) { internal_dump(os, root(), 0); } @@ -1185,7 +1127,7 @@ class btree : public Params::key_compare { if (root()->leaf()) return root()->count(); return root()->size(); } - size_type max_size() const { return numeric_limits::max(); } + size_type max_size() const { return std::numeric_limits::max(); } bool empty() const { return root() == NULL; } // The height of the btree. An empty tree will have height 0. @@ -1310,7 +1252,7 @@ class btree : public Params::key_compare { } void delete_internal_node(node_type *node) { node->destroy(); - DCHECK(node != root()); + assert(node != root()); mutable_internal_allocator()->deallocate( reinterpret_cast(node), sizeof(internal_fields)); } @@ -1368,15 +1310,15 @@ class btree : public Params::key_compare { // key was found in the tree) or -kExactMatch (if it wasn't) in the second // field of the pair. The compare_to specialization allows the caller to // avoid a subsequent comparison to determine if an exact match was made, - // speeding up string, cord and StringPiece keys. + // speeding up string keys. template - pair internal_locate( + std::pair internal_locate( const key_type &key, IterType iter) const; template - pair internal_locate_plain_compare( + std::pair internal_locate_plain_compare( const key_type &key, IterType iter) const; template - pair internal_locate_compare_to( + std::pair internal_locate_compare_to( const key_type &key, IterType iter) const; // Internal routine which implements lower_bound(). @@ -1403,7 +1345,7 @@ class btree : public Params::key_compare { void internal_clear(node_type *node); // Dumps a node and all of its children to the specified ostream. - void internal_dump(ostream &os, const node_type *node, int level) const; + void internal_dump(std::ostream &os, const node_type *node, int level) const; // Verifies the tree structure of node. int internal_verify(const node_type *node, @@ -1427,14 +1369,14 @@ class btree : public Params::key_compare { empty_base_handle root_; private: - // A never instantiated helper function that returns base::big_ if we have a - // key-compare-to functor or if R is bool and base::small_ otherwise. + // A never instantiated helper function that returns big_ if we have a + // key-compare-to functor or if R is bool and small_ otherwise. template - static typename base::if_< - base::if_, - base::type_equals_ >::type::value, - base::big_, base::small_>::type key_compare_checker(R); + static typename if_< + if_, + std::is_same >::type::value, + big_, small_>::type key_compare_checker(R); // A never instantiated helper function that returns the key comparison // functor. @@ -1445,10 +1387,10 @@ class btree : public Params::key_compare { // is never actually invoked. The compiler will select which // key_compare_checker() to instantiate and then figure out the size of the // return type of key_compare_checker() at compile time which we then check - // against the sizeof of base::big_. + // against the sizeof of big_. COMPILE_ASSERT( sizeof(key_compare_checker(key_compare_helper()(key_type(), key_type()))) == - sizeof(base::big_), + sizeof(big_), key_comparison_function_must_return_bool); // Note: We insist on kTargetValues, which is computed from @@ -1466,7 +1408,7 @@ class btree : public Params::key_compare { // btree_node methods template inline void btree_node

::insert_value(int i, const value_type &x) { - DCHECK_LE(i, count()); + assert(i <= count()); value_init(count(), x); for (int j = count(); j > i; --j) { value_swap(j, this, j - 1); @@ -1486,7 +1428,7 @@ inline void btree_node

::insert_value(int i, const value_type &x) { template inline void btree_node

::remove_value(int i) { if (!leaf()) { - DCHECK_EQ(child(i + 1)->count(), 0); + assert(child(i + 1)->count() == 0); for (int j = i + 1; j < count(); ++j) { *mutable_child(j) = child(j + 1); child(j)->set_position(j); @@ -1503,11 +1445,11 @@ inline void btree_node

::remove_value(int i) { template void btree_node

::rebalance_right_to_left(btree_node *src, int to_move) { - DCHECK_EQ(parent(), src->parent()); - DCHECK_EQ(position() + 1, src->position()); - DCHECK_GE(src->count(), count()); - DCHECK_GE(to_move, 1); - DCHECK_LE(to_move, src->count()); + assert(parent() == src->parent()); + assert(position() + 1 == src->position()); + assert(src->count() >= count()); + assert(to_move >= 1); + assert(to_move <= src->count()); // Make room in the left node for the new values. for (int i = 0; i < to_move; ++i) { @@ -1537,7 +1479,7 @@ void btree_node

::rebalance_right_to_left(btree_node *src, int to_move) { set_child(1 + count() + i, src->child(i)); } for (int i = 0; i <= src->count() - to_move; ++i) { - DCHECK_LE(i + to_move, src->max_count()); + assert(i + to_move <= src->max_count()); src->set_child(i, src->child(i + to_move)); *src->mutable_child(i + to_move) = NULL; } @@ -1550,11 +1492,11 @@ void btree_node

::rebalance_right_to_left(btree_node *src, int to_move) { template void btree_node

::rebalance_left_to_right(btree_node *dest, int to_move) { - DCHECK_EQ(parent(), dest->parent()); - DCHECK_EQ(position() + 1, dest->position()); - DCHECK_GE(count(), dest->count()); - DCHECK_GE(to_move, 1); - DCHECK_LE(to_move, count()); + assert(parent() == dest->parent()); + assert(position() + 1 == dest->position()); + assert(count() >= dest->count()); + assert(to_move >= 1); + assert(to_move <= count()); // Make room in the right node for the new values. for (int i = 0; i < to_move; ++i) { @@ -1595,7 +1537,7 @@ void btree_node

::rebalance_left_to_right(btree_node *dest, int to_move) { template void btree_node

::split(btree_node *dest, int insert_position) { - DCHECK_EQ(dest->count(), 0); + assert(dest->count() == 0); // We bias the split based on the position being inserted. If we're // inserting at the beginning of the left node then bias the split to put @@ -1609,7 +1551,7 @@ void btree_node

::split(btree_node *dest, int insert_position) { dest->set_count(count() / 2); } set_count(count() - dest->count()); - DCHECK_GE(count(), 1); + assert(count() >= 1); // Move values from the left sibling to the right sibling. for (int i = 0; i < dest->count(); ++i) { @@ -1627,7 +1569,7 @@ void btree_node

::split(btree_node *dest, int insert_position) { if (!leaf()) { for (int i = 0; i <= dest->count(); ++i) { - DCHECK(child(count() + i + 1) != NULL); + assert(child(count() + i + 1) != NULL); dest->set_child(i, child(count() + i + 1)); *mutable_child(count() + i + 1) = NULL; } @@ -1636,8 +1578,8 @@ void btree_node

::split(btree_node *dest, int insert_position) { template void btree_node

::merge(btree_node *src) { - DCHECK_EQ(parent(), src->parent()); - DCHECK_EQ(position() + 1, src->position()); + assert(parent() == src->parent()); + assert(position() + 1 == src->position()); // Move the delimiting value to the left node. value_init(count()); @@ -1668,7 +1610,7 @@ void btree_node

::merge(btree_node *src) { template void btree_node

::swap(btree_node *x) { - DCHECK_EQ(leaf(), x->leaf()); + assert(leaf() == x->leaf()); // Swap the values. for (int i = count(); i < x->count(); ++i) { @@ -1677,7 +1619,7 @@ void btree_node

::swap(btree_node *x) { for (int i = x->count(); i < count(); ++i) { x->value_init(i); } - int n = max(count(), x->count()); + int n = std::max(count(), x->count()); for (int i = 0; i < n; ++i) { value_swap(i, x, i); } @@ -1710,10 +1652,10 @@ void btree_node

::swap(btree_node *x) { template void btree_iterator::increment_slow() { if (node->leaf()) { - DCHECK_GE(position, node->count()); + assert(position >= node->count()); self_type save(*this); while (position == node->count() && !node->is_root()) { - DCHECK_EQ(node->parent()->child(node->position()), node); + assert(node->parent()->child(node->position()) == node); position = node->position(); node = node->parent(); } @@ -1721,7 +1663,7 @@ void btree_iterator::increment_slow() { *this = save; } } else { - DCHECK_LT(position, node->count()); + assert(position < node->count()); node = node->child(position + 1); while (!node->leaf()) { node = node->child(0); @@ -1735,7 +1677,7 @@ void btree_iterator::increment_by(int count) { while (count > 0) { if (node->leaf()) { int rest = node->count() - position; - position += min(rest, count); + position += std::min(rest, count); count = count - rest; if (position < node->count()) { return; @@ -1750,10 +1692,10 @@ void btree_iterator::increment_by(int count) { template void btree_iterator::decrement_slow() { if (node->leaf()) { - DCHECK_LE(position, -1); + assert(position <= -1); self_type save(*this); while (position < 0 && !node->is_root()) { - DCHECK_EQ(node->parent()->child(node->position()), node); + assert(node->parent()->child(node->position()) == node); position = node->position() - 1; node = node->parent(); } @@ -1761,7 +1703,7 @@ void btree_iterator::decrement_slow() { *this = save; } } else { - DCHECK_GE(position, 0); + assert(position >= 0); node = node->child(position); while (!node->leaf()) { node = node->child(node->count()); @@ -1786,26 +1728,26 @@ btree

::btree(const self_type &x) } template template -pair::iterator, bool> +std::pair::iterator, bool> btree

::insert_unique(const key_type &key, ValuePointer value) { if (empty()) { *mutable_root() = new_leaf_root_node(1); } - pair res = internal_locate(key, iterator(root(), 0)); + std::pair res = internal_locate(key, iterator(root(), 0)); iterator &iter = res.first; if (res.second == kExactMatch) { // The key already exists in the tree, do nothing. - return make_pair(internal_last(iter), false); + return std::make_pair(internal_last(iter), false); } else if (!res.second) { iterator last = internal_last(iter); if (last.node && !compare_keys(key, last.key())) { // The key already exists in the tree, do nothing. - return make_pair(last, false); + return std::make_pair(last, false); } } - return make_pair(internal_insert(iter, *value), true); + return std::make_pair(internal_insert(iter, *value), true); } template @@ -1912,8 +1854,8 @@ typename btree

::iterator btree

::erase(iterator iter) { // Deletion of a value on an internal node. Swap the key with the largest // value of our left child. This is easy, we just decrement iter. iterator tmp_iter(iter--); - DCHECK(iter.node->leaf()); - DCHECK(!compare_keys(tmp_iter.key(), iter.key())); + assert(iter.node->leaf()); + assert(!compare_keys(tmp_iter.key(), iter.key())); iter.node->value_swap(iter.position, tmp_iter.node, tmp_iter.position); internal_delete = true; --*mutable_size(); @@ -2010,22 +1952,22 @@ void btree

::clear() { template void btree

::swap(self_type &x) { - ::swap(static_cast(*this), static_cast(x)); - ::swap(root_, x.root_); + std::swap(static_cast(*this), static_cast(x)); + std::swap(root_, x.root_); } template void btree

::verify() const { if (root() != NULL) { - CHECK_EQ(size(), internal_verify(root(), NULL, NULL)); - CHECK_EQ(leftmost(), (++const_iterator(root(), -1)).node); - CHECK_EQ(rightmost(), (--const_iterator(root(), root()->count())).node); - CHECK(leftmost()->leaf()); - CHECK(rightmost()->leaf()); + assert(size() == internal_verify(root(), NULL, NULL)); + assert(leftmost() == (++const_iterator(root(), -1)).node); + assert(rightmost() == (--const_iterator(root(), root()->count())).node); + assert(leftmost()->leaf()); + assert(rightmost()->leaf()); } else { - CHECK_EQ(size(), 0); - CHECK(leftmost() == NULL); - CHECK(rightmost() == NULL); + assert(size() == 0); + assert(leftmost() == NULL); + assert(rightmost() == NULL); } } @@ -2033,7 +1975,7 @@ template void btree

::rebalance_or_split(iterator *iter) { node_type *&node = iter->node; int &insert_position = iter->position; - DCHECK_EQ(node->count(), node->max_count()); + assert(node->count() == node->max_count()); // First try to make room on the node by rebalancing. node_type *parent = node->parent(); @@ -2047,20 +1989,20 @@ void btree

::rebalance_or_split(iterator *iter) { // fill up the left node. int to_move = (left->max_count() - left->count()) / (1 + (insert_position < left->max_count())); - to_move = max(1, to_move); + to_move = std::max(1, to_move); if (((insert_position - to_move) >= 0) || ((left->count() + to_move) < left->max_count())) { left->rebalance_right_to_left(node, to_move); - DCHECK_EQ(node->max_count() - node->count(), to_move); + assert(node->max_count() - node->count() == to_move); insert_position = insert_position - to_move; if (insert_position < 0) { insert_position = insert_position + left->count() + 1; node = left; } - DCHECK_LT(node->count(), node->max_count()); + assert(node->count() < node->max_count()); return; } } @@ -2075,7 +2017,7 @@ void btree

::rebalance_or_split(iterator *iter) { // to fill up the right node. int to_move = (right->max_count() - right->count()) / (1 + (insert_position > 0)); - to_move = max(1, to_move); + to_move = std::max(1, to_move); if ((insert_position <= (node->count() - to_move)) || ((right->count() + to_move) < right->max_count())) { @@ -2086,7 +2028,7 @@ void btree

::rebalance_or_split(iterator *iter) { node = right; } - DCHECK_LT(node->count(), node->max_count()); + assert(node->count() < node->max_count()); return; } } @@ -2106,7 +2048,7 @@ void btree

::rebalance_or_split(iterator *iter) { parent = new_internal_root_node(); parent->set_child(0, root()); *mutable_root() = parent; - DCHECK(*mutable_rightmost() == parent->child(0)); + assert(*mutable_rightmost() == parent->child(0)); } else { // The root node is an internal node. We do not want to create a new root // node because the root node is special and holds the size of the tree @@ -2179,7 +2121,7 @@ bool btree

::try_merge_or_rebalance(iterator *iter) { ((iter->node->count() == 0) || (iter->position > 0))) { int to_move = (right->count() - iter->node->count()) / 2; - to_move = min(to_move, right->count() - 1); + to_move = std::min(to_move, right->count() - 1); iter->node->rebalance_right_to_left(right, to_move); return false; } @@ -2194,7 +2136,7 @@ bool btree

::try_merge_or_rebalance(iterator *iter) { ((iter->node->count() == 0) || (iter->position < iter->node->count()))) { int to_move = (left->count() - iter->node->count()) / 2; - to_move = min(to_move, left->count() - 1); + to_move = std::min(to_move, left->count() - 1); left->rebalance_left_to_right(iter->node, to_move); iter->position += to_move; return false; @@ -2210,7 +2152,7 @@ void btree

::try_shrink() { } // Deleted the last item on the root node, shrink the height of the tree. if (root()->leaf()) { - DCHECK_EQ(size(), 0); + assert(size() == 0); delete_leaf_node(root()); *mutable_root() = NULL; } else { @@ -2256,9 +2198,9 @@ btree

::internal_insert(iterator iter, const value_type &v) { if (iter.node->max_count() < kNodeValues) { // Insertion into the root where the root is smaller that the full node // size. Simply grow the size of the root node. - DCHECK(iter.node == root()); + assert(iter.node == root()); iter.node = new_leaf_root_node( - min(kNodeValues, 2 * iter.node->max_count())); + std::min(kNodeValues, 2 * iter.node->max_count())); iter.node->swap(root()); delete_leaf_node(root()); *mutable_root() = iter.node; @@ -2274,13 +2216,13 @@ btree

::internal_insert(iterator iter, const value_type &v) { } template template -inline pair btree

::internal_locate( +inline std::pair btree

::internal_locate( const key_type &key, IterType iter) const { return internal_locate_type::dispatch(key, *this, iter); } template template -inline pair btree

::internal_locate_plain_compare( +inline std::pair btree

::internal_locate_plain_compare( const key_type &key, IterType iter) const { for (;;) { iter.position = iter.node->lower_bound(key, key_comp()); @@ -2289,24 +2231,24 @@ inline pair btree

::internal_locate_plain_compare( } iter.node = iter.node->child(iter.position); } - return make_pair(iter, 0); + return std::make_pair(iter, 0); } template template -inline pair btree

::internal_locate_compare_to( +inline std::pair btree

::internal_locate_compare_to( const key_type &key, IterType iter) const { for (;;) { int res = iter.node->lower_bound(key, key_comp()); iter.position = res & kMatchMask; if (res & kExactMatch) { - return make_pair(iter, static_cast(kExactMatch)); + return std::make_pair(iter, static_cast(kExactMatch)); } if (iter.node->leaf()) { break; } iter.node = iter.node->child(iter.position); } - return make_pair(iter, -kExactMatch); + return std::make_pair(iter, -kExactMatch); } template template @@ -2346,7 +2288,7 @@ template template IterType btree

::internal_find_unique( const key_type &key, IterType iter) const { if (iter.node) { - pair res = internal_locate(key, iter); + std::pair res = internal_locate(key, iter); if (res.second == kExactMatch) { return res.first; } @@ -2393,7 +2335,7 @@ void btree

::internal_clear(node_type *node) { template void btree

::internal_dump( - ostream &os, const node_type *node, int level) const { + std::ostream &os, const node_type *node, int level) const { for (int i = 0; i < node->count(); ++i) { if (!node->leaf()) { internal_dump(os, node->child(i), level + 1); @@ -2411,23 +2353,23 @@ void btree

::internal_dump( template int btree

::internal_verify( const node_type *node, const key_type *lo, const key_type *hi) const { - CHECK_GT(node->count(), 0); - CHECK_LE(node->count(), node->max_count()); + assert(node->count() > 0); + assert(node->count() <= node->max_count()); if (lo) { - CHECK(!compare_keys(node->key(0), *lo)); + assert(!compare_keys(node->key(0), *lo)); } if (hi) { - CHECK(!compare_keys(*hi, node->key(node->count() - 1))); + assert(!compare_keys(*hi, node->key(node->count() - 1))); } for (int i = 1; i < node->count(); ++i) { - CHECK(!compare_keys(node->key(i), node->key(i - 1))); + assert(!compare_keys(node->key(i), node->key(i - 1))); } int count = node->count(); if (!node->leaf()) { for (int i = 0; i <= node->count(); ++i) { - CHECK(node->child(i) != NULL); - CHECK_EQ(node->child(i)->parent(), node); - CHECK_EQ(node->child(i)->position(), i); + assert(node->child(i) != NULL); + assert(node->child(i)->parent() == node); + assert(node->child(i)->position() == i); count += internal_verify( node->child(i), (i == 0) ? lo : &node->key(i - 1), @@ -2438,8 +2380,5 @@ int btree

::internal_verify( } } // namespace btree -} // namespace util - -DEFINE_GDB_AUTO_SCRIPT("util/btree/btree_printer.py") #endif // UTIL_BTREE_BTREE_H__ diff --git a/btree_bench.cc b/btree_bench.cc index 94a1f272..1b45582a 100644 --- a/btree_bench.cc +++ b/btree_bench.cc @@ -26,7 +26,6 @@ DECLARE_int32(benchmark_max_iters); -namespace util { namespace btree { namespace { @@ -480,7 +479,6 @@ MY_BENCHMARK(multimap_cord); } // namespace } // namespace btree -} // namespace util int main(int argc, char **argv) { FLAGS_logtostderr = true; diff --git a/btree_container.h b/btree_container.h index beebf6ff..510c9ae6 100644 --- a/btree_container.h +++ b/btree_container.h @@ -8,9 +8,8 @@ #include #include -#include "util/btree/btree.h" // IWYU pragma: export +#include "btree.h" -namespace util { namespace btree { // A common base class for btree_set, btree_map, btree_multiset and @@ -70,10 +69,10 @@ class btree_container { const_iterator upper_bound(const key_type &key) const { return tree_.upper_bound(key); } - pair equal_range(const key_type &key) { + std::pair equal_range(const key_type &key) { return tree_.equal_range(key); } - pair equal_range(const key_type &key) const { + std::pair equal_range(const key_type &key) const { return tree_.equal_range(key); } @@ -84,7 +83,7 @@ class btree_container { void swap(self_type &x) { tree_.swap(x.tree_); } - void dump(ostream &os) const { + void dump(std::ostream &os) const { tree_.dump(os); } void verify() const { @@ -128,7 +127,7 @@ class btree_container { }; template -inline ostream& operator<<(ostream &os, const btree_container &b) { +inline std::ostream& operator<<(std::ostream &os, const btree_container &b) { b.dump(os); return os; } @@ -181,7 +180,7 @@ class btree_unique_container : public btree_container { } // Insertion routines. - pair insert(const value_type &x) { + std::pair insert(const value_type &x) { return this->tree_.insert_unique(x); } iterator insert(iterator position, const value_type &x) { @@ -230,7 +229,7 @@ class btree_map_container : public btree_unique_container { : key(k) { } value_type operator*() const { - return make_pair(key, data_type()); + return std::make_pair(key, data_type()); } const key_type &key; }; @@ -337,6 +336,5 @@ class btree_multi_container : public btree_container { }; } // namespace btree -} // namespace util #endif // UTIL_BTREE_BTREE_CONTAINER_H__ diff --git a/btree_map.h b/btree_map.h index 3c61a7f2..6e965590 100644 --- a/btree_map.h +++ b/btree_map.h @@ -1,4 +1,4 @@ -// Copyright 2007 Google Inc. All Rights Reserved. +// Copyright 2007, 2012 Google Inc. All Rights Reserved. // Author: jmacd@google.com (Josh MacDonald) // Author: pmattis@google.com (Peter Mattis) // @@ -18,16 +18,15 @@ #include #include -#include "util/btree/btree.h" // IWYU pragma: export -#include "util/btree/btree_container.h" // IWYU pragma: export +#include "btree.h" +#include "btree_container.h" -namespace util { namespace btree { // The btree_map class is needed mainly for it's constructors. template , - typename Alloc = std::allocator >, + typename Compare = std::less, + typename Alloc = std::allocator >, int TargetNodeSize = 256> class btree_map : public btree_map_container< btree > > { @@ -71,8 +70,8 @@ inline void swap(btree_map &x, // The btree_multimap class is needed mainly for it's constructors. template , - typename Alloc = std::allocator >, + typename Compare = std::less, + typename Alloc = std::allocator >, int TargetNodeSize = 256> class btree_multimap : public btree_multi_container< btree > > { @@ -117,6 +116,5 @@ inline void swap(btree_multimap &x, } } // namespace btree -} // namespace util #endif // UTIL_BTREE_BTREE_MAP_H__ diff --git a/btree_set.h b/btree_set.h index 7b5ea88d..0b51e494 100644 --- a/btree_set.h +++ b/btree_set.h @@ -14,15 +14,14 @@ #include #include -#include "util/btree/btree.h" // IWYU pragma: export -#include "util/btree/btree_container.h" // IWYU pragma: export +#include "btree.h" +#include "btree_container.h" -namespace util { namespace btree { // The btree_set class is needed mainly for it's constructors. template , + typename Compare = std::less, typename Alloc = std::allocator, int TargetNodeSize = 256> class btree_set : public btree_unique_container< @@ -65,7 +64,7 @@ inline void swap(btree_set &x, btree_set &y) { // The btree_multiset class is needed mainly for it's constructors. template , + typename Compare = std::less, typename Alloc = std::allocator, int TargetNodeSize = 256> class btree_multiset : public btree_multi_container< @@ -108,6 +107,5 @@ inline void swap(btree_multiset &x, } } // namespace btree -} // namespace util #endif // UTIL_BTREE_BTREE_SET_H__ diff --git a/btree_test.cc b/btree_test.cc index 335318de..7b9cafa5 100644 --- a/btree_test.cc +++ b/btree_test.cc @@ -2,128 +2,120 @@ // Author: jmacd@google.com (Josh MacDonald) // Author: pmattis@google.com (Peter Mattis) -#include "base/arena-inl.h" -#include "base/init_google.h" -#include "base/integral_types.h" -#include "base/logging.h" -#include "strings/stringpiece.h" -#include "testing/base/public/gunit.h" -#include "util/btree/btree_map.h" -#include "util/btree/btree_set.h" -#include "util/btree/btree_test.h" +#include "gtest/gtest.h" +#include "btree_map.h" +#include "btree_set.h" +#include "btree_test.h" -namespace util { namespace btree { namespace { template void SetTest() { - typedef ArenaAllocator ArenaAlloc; - CHECK_EQ(sizeof(btree_set), sizeof(void*)); - BtreeTest, allocator, N>, set >(); - BtreeArenaTest, ArenaAlloc, N> >(); + typedef TestAllocator TestAlloc; + ASSERT_EQ(sizeof(btree_set), sizeof(void*)); + BtreeTest, std::allocator, N>, std::set >(); + BtreeAllocatorTest, TestAlloc, N> >(); } template void MapTest() { - typedef ArenaAllocator ArenaAlloc; - CHECK_EQ(sizeof(btree_map), sizeof(void*)); - BtreeTest, allocator, N>, map >(); - BtreeArenaTest, ArenaAlloc, N> >(); - BtreeMapTest, allocator, N> >(); + typedef TestAllocator TestAlloc; + ASSERT_EQ(sizeof(btree_map), sizeof(void*)); + BtreeTest, std::allocator, N>, std::map >(); + BtreeAllocatorTest, TestAlloc, N> >(); + BtreeMapTest, std::allocator, N> >(); } -TEST(Btree, set_int32_32) { SetTest(); } -TEST(Btree, set_int32_64) { SetTest(); } -TEST(Btree, set_int32_128) { SetTest(); } -TEST(Btree, set_int32_256) { SetTest(); } -TEST(Btree, set_int64_256) { SetTest(); } -TEST(Btree, set_string_256) { SetTest(); } -TEST(Btree, set_cord_256) { SetTest(); } -TEST(Btree, set_pair_256) { SetTest, 256>(); } -TEST(Btree, map_int32_256) { MapTest(); } -TEST(Btree, map_int64_256) { MapTest(); } -TEST(Btree, map_string_256) { MapTest(); } -TEST(Btree, map_cord_256) { MapTest(); } -TEST(Btree, map_pair_256) { MapTest, 256>(); } +TEST(Btree, set_int32_32) { SetTest(); } +TEST(Btree, set_int32_64) { SetTest(); } +TEST(Btree, set_int32_128) { SetTest(); } +TEST(Btree, set_int32_256) { SetTest(); } +TEST(Btree, set_int64_256) { SetTest(); } +TEST(Btree, set_string_256) { SetTest(); } +TEST(Btree, set_pair_256) { SetTest, 256>(); } +TEST(Btree, map_int32_256) { MapTest(); } +TEST(Btree, map_int64_256) { MapTest(); } +TEST(Btree, map_string_256) { MapTest(); } +TEST(Btree, map_pair_256) { MapTest, 256>(); } // Large-node tests -TEST(Btree, map_int32_1024) { MapTest(); } -TEST(Btree, map_int32_1032) { MapTest(); } -TEST(Btree, map_int32_1040) { MapTest(); } -TEST(Btree, map_int32_1048) { MapTest(); } -TEST(Btree, map_int32_1056) { MapTest(); } +TEST(Btree, map_int32_1024) { MapTest(); } +TEST(Btree, map_int32_1032) { MapTest(); } +TEST(Btree, map_int32_1040) { MapTest(); } +TEST(Btree, map_int32_1048) { MapTest(); } +TEST(Btree, map_int32_1056) { MapTest(); } -TEST(Btree, map_int32_2048) { MapTest(); } -TEST(Btree, map_int32_4096) { MapTest(); } -TEST(Btree, set_int32_1024) { SetTest(); } -TEST(Btree, set_int32_2048) { SetTest(); } -TEST(Btree, set_int32_4096) { SetTest(); } -TEST(Btree, map_string_1024) { MapTest(); } -TEST(Btree, map_string_2048) { MapTest(); } -TEST(Btree, map_string_4096) { MapTest(); } -TEST(Btree, set_string_1024) { SetTest(); } -TEST(Btree, set_string_2048) { SetTest(); } -TEST(Btree, set_string_4096) { SetTest(); } +TEST(Btree, map_int32_2048) { MapTest(); } +TEST(Btree, map_int32_4096) { MapTest(); } +TEST(Btree, set_int32_1024) { SetTest(); } +TEST(Btree, set_int32_2048) { SetTest(); } +TEST(Btree, set_int32_4096) { SetTest(); } +TEST(Btree, map_string_1024) { MapTest(); } +TEST(Btree, map_string_2048) { MapTest(); } +TEST(Btree, map_string_4096) { MapTest(); } +TEST(Btree, set_string_1024) { SetTest(); } +TEST(Btree, set_string_2048) { SetTest(); } +TEST(Btree, set_string_4096) { SetTest(); } template void MultiSetTest() { - typedef ArenaAllocator ArenaAlloc; - CHECK_EQ(sizeof(btree_multiset), sizeof(void*)); - BtreeMultiTest, allocator, N>, - multiset >(); - BtreeArenaTest, ArenaAlloc, N> >(); + typedef TestAllocator TestAlloc; + ASSERT_EQ(sizeof(btree_multiset), sizeof(void*)); + BtreeMultiTest, std::allocator, N>, + std::multiset >(); + BtreeAllocatorTest, TestAlloc, N> >(); } template void MultiMapTest() { - typedef ArenaAllocator ArenaAlloc; - CHECK_EQ(sizeof(btree_multimap), sizeof(void*)); - BtreeMultiTest, allocator, N>, - multimap >(); - BtreeMultiMapTest, allocator, N> >(); - BtreeArenaTest, ArenaAlloc, N> >(); + typedef TestAllocator TestAlloc; + ASSERT_EQ(sizeof(btree_multimap), sizeof(void*)); + BtreeMultiTest, std::allocator, N>, + std::multimap >(); + BtreeMultiMapTest, std::allocator, N> >(); + BtreeAllocatorTest, TestAlloc, N> >(); } -TEST(Btree, multiset_int32_256) { MultiSetTest(); } -TEST(Btree, multiset_int64_256) { MultiSetTest(); } -TEST(Btree, multiset_string_256) { MultiSetTest(); } -TEST(Btree, multiset_cord_256) { MultiSetTest(); } -TEST(Btree, multiset_pair_256) { MultiSetTest, 256>(); } -TEST(Btree, multimap_int32_256) { MultiMapTest(); } -TEST(Btree, multimap_int64_256) { MultiMapTest(); } -TEST(Btree, multimap_string_256) { MultiMapTest(); } -TEST(Btree, multimap_cord_256) { MultiMapTest(); } -TEST(Btree, multimap_pair_256) { MultiMapTest, 256>(); } +TEST(Btree, multiset_int32_256) { MultiSetTest(); } +TEST(Btree, multiset_int64_256) { MultiSetTest(); } +TEST(Btree, multiset_string_256) { MultiSetTest(); } +TEST(Btree, multiset_pair_256) { MultiSetTest, 256>(); } +TEST(Btree, multimap_int32_256) { MultiMapTest(); } +TEST(Btree, multimap_int64_256) { MultiMapTest(); } +TEST(Btree, multimap_string_256) { MultiMapTest(); } +TEST(Btree, multimap_pair_256) { MultiMapTest, 256>(); } // Large-node tests -TEST(Btree, multimap_int32_1024) { MultiMapTest(); } -TEST(Btree, multimap_int32_2048) { MultiMapTest(); } -TEST(Btree, multimap_int32_4096) { MultiMapTest(); } -TEST(Btree, multiset_int32_1024) { MultiSetTest(); } -TEST(Btree, multiset_int32_2048) { MultiSetTest(); } -TEST(Btree, multiset_int32_4096) { MultiSetTest(); } -TEST(Btree, multimap_string_1024) { MultiMapTest(); } -TEST(Btree, multimap_string_2048) { MultiMapTest(); } -TEST(Btree, multimap_string_4096) { MultiMapTest(); } -TEST(Btree, multiset_string_1024) { MultiSetTest(); } -TEST(Btree, multiset_string_2048) { MultiSetTest(); } -TEST(Btree, multiset_string_4096) { MultiSetTest(); } +TEST(Btree, multimap_int32_1024) { MultiMapTest(); } +TEST(Btree, multimap_int32_2048) { MultiMapTest(); } +TEST(Btree, multimap_int32_4096) { MultiMapTest(); } +TEST(Btree, multiset_int32_1024) { MultiSetTest(); } +TEST(Btree, multiset_int32_2048) { MultiSetTest(); } +TEST(Btree, multiset_int32_4096) { MultiSetTest(); } +TEST(Btree, multimap_string_1024) { MultiMapTest(); } +TEST(Btree, multimap_string_2048) { MultiMapTest(); } +TEST(Btree, multimap_string_4096) { MultiMapTest(); } +TEST(Btree, multiset_string_1024) { MultiSetTest(); } +TEST(Btree, multiset_string_2048) { MultiSetTest(); } +TEST(Btree, multiset_string_4096) { MultiSetTest(); } // Verify that swapping btrees swaps the key comparision functors. struct SubstringLess { SubstringLess() : n(2) {} - SubstringLess(int length) + SubstringLess(size_t length) : n(length) { } - bool operator()(const string &a, const string &b) const { - return StringPiece(a).substr(0, n) < StringPiece(b).substr(0, n); + bool operator()(const std::string &a, const std::string &b) const { + std::string as(a.data(), std::min(n, a.size())); + std::string bs(b.data(), std::min(n, b.size())); + return as < bs; } - int n; + size_t n; }; TEST(Btree, SwapKeyCompare) { - typedef btree_set SubstringSet; + typedef btree_set SubstringSet; SubstringSet s1(SubstringLess(1), SubstringSet::allocator_type()); SubstringSet s2(SubstringLess(2), SubstringSet::allocator_type()); @@ -147,7 +139,7 @@ TEST(Btree, SwapKeyCompare) { TEST(Btree, UpperBoundRegression) { // Regress a bug where upper_bound would default-construct a new key_compare // instead of copying the existing one. - typedef btree_set SubstringSet; + typedef btree_set SubstringSet; SubstringSet my_set(SubstringLess(3)); my_set.insert("aab"); my_set.insert("abb"); @@ -164,25 +156,25 @@ TEST(Btree, UpperBoundRegression) { TEST(Btree, IteratorIncrementBy) { // Test that increment_by returns the same position as increment. const int kSetSize = 2341; - btree_set my_set; + btree_set my_set; for (int i = 0; i < kSetSize; ++i) { my_set.insert(i); } { // Simple increment vs. increment by. - btree_set::iterator a = my_set.begin(); - btree_set::iterator b = my_set.begin(); + btree_set::iterator a = my_set.begin(); + btree_set::iterator b = my_set.begin(); a.increment(); b.increment_by(1); EXPECT_EQ(*a, *b); } - btree_set::iterator a = my_set.begin(); + btree_set::iterator a = my_set.begin(); for (int i = 1; i < kSetSize; ++i) { ++a; // increment_by - btree_set::iterator b = my_set.begin(); + btree_set::iterator b = my_set.begin(); b.increment_by(i); EXPECT_EQ(*a, *b) << ": i=" << i; } @@ -190,11 +182,11 @@ TEST(Btree, IteratorIncrementBy) { TEST(Btree, Comparison) { const int kSetSize = 1201; - btree_set my_set; + btree_set my_set; for (int i = 0; i < kSetSize; ++i) { my_set.insert(i); } - btree_set my_set_copy(my_set); + btree_set my_set_copy(my_set); EXPECT_TRUE(my_set_copy == my_set); EXPECT_TRUE(my_set == my_set_copy); EXPECT_FALSE(my_set_copy != my_set); @@ -212,17 +204,17 @@ TEST(Btree, Comparison) { EXPECT_TRUE(my_set_copy != my_set); EXPECT_TRUE(my_set != my_set_copy); - btree_map my_map; + btree_map my_map; for (int i = 0; i < kSetSize; ++i) { - my_map[string(i, 'a')] = i; + my_map[std::string(i, 'a')] = i; } - btree_map my_map_copy(my_map); + btree_map my_map_copy(my_map); EXPECT_TRUE(my_map_copy == my_map); EXPECT_TRUE(my_map == my_map_copy); EXPECT_FALSE(my_map_copy != my_map); EXPECT_FALSE(my_map != my_map_copy); - ++my_map_copy[string(7, 'a')]; + ++my_map_copy[std::string(7, 'a')]; EXPECT_FALSE(my_map_copy == my_map); EXPECT_FALSE(my_map == my_map_copy); EXPECT_TRUE(my_map_copy != my_map); @@ -235,7 +227,7 @@ TEST(Btree, Comparison) { EXPECT_TRUE(my_map_copy != my_map); EXPECT_TRUE(my_map != my_map_copy); - my_map.erase(string(kSetSize - 1, 'a')); + my_map.erase(std::string(kSetSize - 1, 'a')); EXPECT_FALSE(my_map_copy == my_map); EXPECT_FALSE(my_map == my_map_copy); EXPECT_TRUE(my_map_copy != my_map); @@ -244,10 +236,3 @@ TEST(Btree, Comparison) { } // namespace } // namespace btree -} // namespace util - -int main(int argc, char **argv) { - FLAGS_logtostderr = true; - InitGoogle(argv[0], &argc, &argv, true); - return RUN_ALL_TESTS(); -} diff --git a/btree_test.h b/btree_test.h index 557c1e54..c368573e 100644 --- a/btree_test.h +++ b/btree_test.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -16,53 +17,50 @@ #include #include -#include "base/arena.h" -#include "base/commandlineflags.h" -#include "base/logging.h" -#include "base/type_traits.h" -#include "strings/cord.h" -#include "strings/util.h" -#include "testing/base/public/googletest.h" -#include "util/btree/btree_container.h" -#include "util/random/acmrandom.h" +#include "gtest/gtest.h" +#include "gflags/gflags.h" +#include "btree_container.h" DECLARE_int32(test_values); DECLARE_int32(benchmark_values); namespace std { -// Provide operator<< support for pair. +// Provide operator<< support for std::pair. template -ostream& operator<<(ostream &os, const pair &p) { +ostream& operator<<(ostream &os, const std::pair &p) { os << "(" << p.first << "," << p.second << ")"; return os; } // Provide pair equality testing that works as long as x.first is comparable to // y.first and x.second is comparable to y.second. Needed in the test for -// comparing pair to pair. +// comparing std::pair to std::pair. template -bool operator==(const pair &x, const pair &y) { +bool operator==(const std::pair &x, const std::pair &y) { return x.first == y.first && x.second == y.second; } -} // namespace std - -namespace base { - // Partial specialization of remove_const that propagates the removal through // std::pair. template -struct remove_const > { - typedef std::pair::type, - typename remove_const::type> type; +struct remove_const > { + typedef pair::type, + typename remove_const::type> type; }; -} // namespace base +} // namespace std -namespace util { namespace btree { +// Select the first member of a pair. +template +struct select1st : public std::unary_function<_Pair, typename _Pair::first_type> { + const typename _Pair::first_type& operator()(const _Pair& __x) const { + return __x.first; + } +}; + // Utility class to provide an accessor for a key given a value. The default // behavior is to treat the value as a pair and return the first element. template @@ -70,6 +68,11 @@ struct KeyOfValue { typedef select1st type; }; +template +struct identity { + inline const T& operator()(const T& t) const { return t; } +}; + // Partial specialization of KeyOfValue class for when the key and value are // the same type such as in set<> and btree_set<>. template @@ -77,6 +80,29 @@ struct KeyOfValue { typedef identity type; }; +// Counts the number of occurances of "c" in a buffer. +inline ptrdiff_t strcount(const char* buf_begin, const char* buf_end, char c) { + if (buf_begin == NULL) + return 0; + if (buf_end <= buf_begin) + return 0; + ptrdiff_t num = 0; + for (const char* bp = buf_begin; bp != buf_end; bp++) { + if (*bp == c) + num++; + } + return num; +} + +// for when the string is not null-terminated. +inline ptrdiff_t strcount(const char* buf, size_t len, char c) { + return strcount(buf, buf + len, c); +} + +inline ptrdiff_t strcount(const std::string& buf, char c) { + return strcount(buf.c_str(), buf.size(), c); +} + // The base class for a sorted associative container checker. TreeType is the // container type to check and CheckerType is the container type to check // against. TreeType is expected to be btree_{set,map,multiset,multimap} and @@ -127,9 +153,9 @@ class base_checker { IterType iter_check( IterType tree_iter, CheckerIterType checker_iter) const { if (tree_iter == tree_.end()) { - CHECK(checker_iter == checker_.end()); + EXPECT_EQ(checker_iter, checker_.end()); } else { - CHECK_EQ(*tree_iter, *checker_iter); + EXPECT_EQ(*tree_iter, *checker_iter); } return tree_iter; } @@ -137,9 +163,9 @@ class base_checker { IterType riter_check( IterType tree_iter, CheckerIterType checker_iter) const { if (tree_iter == tree_.rend()) { - CHECK(checker_iter == checker_.rend()); + EXPECT_EQ(checker_iter, checker_.rend()); } else { - CHECK_EQ(*tree_iter, *checker_iter); + EXPECT_EQ(*tree_iter, *checker_iter); } return tree_iter; } @@ -147,17 +173,17 @@ class base_checker { typename KeyOfValue::type key_of_value; const key_type &key = key_of_value(x); - CHECK_EQ(*find(key), x); + EXPECT_EQ(*find(key), x); lower_bound(key); upper_bound(key); equal_range(key); count(key); } void erase_check(const key_type &key) { - CHECK(tree_.find(key) == const_tree_.end()); - CHECK(const_tree_.find(key) == tree_.end()); - CHECK(tree_.equal_range(key).first == - const_tree_.equal_range(key).second); + EXPECT_TRUE(tree_.find(key) == const_tree_.end()); + EXPECT_TRUE(const_tree_.find(key) == tree_.end()); + EXPECT_TRUE(tree_.equal_range(key).first == + const_tree_.equal_range(key).second); } // Lookup routines. @@ -173,20 +199,20 @@ class base_checker { const_iterator upper_bound(const key_type &key) const { return iter_check(tree_.upper_bound(key), checker_.upper_bound(key)); } - pair equal_range(const key_type &key) { - pair equal_range(const key_type &key) { + std::pair checker_res = checker_.equal_range(key); - pair tree_res = tree_.equal_range(key); + std::pair tree_res = tree_.equal_range(key); iter_check(tree_res.first, checker_res.first); iter_check(tree_res.second, checker_res.second); return tree_res; } - pair equal_range(const key_type &key) const { - pair equal_range(const key_type &key) const { + std::pair checker_res = checker_.equal_range(key); - pair tree_res = tree_.equal_range(key); + std::pair tree_res = tree_.equal_range(key); iter_check(tree_res.first, checker_res.first); iter_check(tree_res.second, checker_res.second); return tree_res; @@ -199,7 +225,7 @@ class base_checker { } size_type count(const key_type &key) const { size_type res = checker_.count(key); - CHECK_EQ(res, tree_.count(key)); + EXPECT_EQ(res, tree_.count(key)); return res; } @@ -214,10 +240,10 @@ class base_checker { int erase(const key_type &key) { int size = tree_.size(); int res = checker_.erase(key); - CHECK_EQ(res, tree_.count(key)); - CHECK_EQ(res, tree_.erase(key)); - CHECK_EQ(tree_.count(key), 0); - CHECK_EQ(tree_.size(), size - res); + EXPECT_EQ(res, tree_.count(key)); + EXPECT_EQ(res, tree_.erase(key)); + EXPECT_EQ(tree_.count(key), 0); + EXPECT_EQ(tree_.size(), size - res); erase_check(key); return res; } @@ -233,9 +259,9 @@ class base_checker { ++checker_next; checker_.erase(checker_iter); iter = tree_.erase(iter); - CHECK_EQ(tree_.size(), checker_.size()); - CHECK_EQ(tree_.size(), size - 1); - CHECK_EQ(tree_.count(key), count - 1); + EXPECT_EQ(tree_.size(), checker_.size()); + EXPECT_EQ(tree_.size(), size - 1); + EXPECT_EQ(tree_.count(key), count - 1); if (count == 1) { erase_check(key); } @@ -258,8 +284,8 @@ class base_checker { } checker_.erase(checker_begin, checker_end); tree_.erase(begin, end); - CHECK_EQ(tree_.size(), checker_.size()); - CHECK_EQ(tree_.size(), size - count); + EXPECT_EQ(tree_.size(), checker_.size()); + EXPECT_EQ(tree_.size(), size - count); } // Utility routines. @@ -274,7 +300,7 @@ class base_checker { void verify() const { tree_.verify(); - CHECK_EQ(tree_.size(), checker_.size()); + EXPECT_EQ(tree_.size(), checker_.size()); // Move through the forward iterators using increment. typename CheckerType::const_iterator @@ -282,7 +308,7 @@ class base_checker { const_iterator tree_iter(tree_.begin()); for (; tree_iter != tree_.end(); ++tree_iter, ++checker_iter) { - CHECK_EQ(*tree_iter, *checker_iter); + EXPECT_EQ(*tree_iter, *checker_iter); } // Move through the forward iterators using decrement. @@ -291,8 +317,8 @@ class base_checker { --tree_iter; --checker_iter; } - CHECK(tree_iter == tree_.begin()); - CHECK(checker_iter == checker_.begin()); + EXPECT_TRUE(tree_iter == tree_.begin()); + EXPECT_TRUE(checker_iter == checker_.begin()); // Move through the reverse iterators using increment. typename CheckerType::const_reverse_iterator @@ -300,7 +326,7 @@ class base_checker { const_reverse_iterator tree_riter(tree_.rbegin()); for (; tree_riter != tree_.rend(); ++tree_riter, ++checker_riter) { - CHECK_EQ(*tree_riter, *checker_riter); + EXPECT_EQ(*tree_riter, *checker_riter); } // Move through the reverse iterators using decrement. @@ -309,8 +335,8 @@ class base_checker { --tree_riter; --checker_riter; } - CHECK(tree_riter == tree_.rbegin()); - CHECK(checker_riter == checker_.rbegin()); + EXPECT_EQ(tree_riter, tree_.rbegin()); + EXPECT_EQ(checker_riter, checker_.rbegin()); } // Access to the underlying btree. @@ -318,12 +344,12 @@ class base_checker { // Size routines. size_type size() const { - CHECK_EQ(tree_.size(), checker_.size()); + EXPECT_EQ(tree_.size(), checker_.size()); return tree_.size(); } size_type max_size() const { return tree_.max_size(); } bool empty() const { - CHECK_EQ(tree_.empty(), checker_.empty()); + EXPECT_EQ(tree_.empty(), checker_.empty()); return tree_.empty(); } size_type height() const { return tree_.height(); } @@ -367,25 +393,25 @@ class unique_checker : public base_checker { } // Insertion routines. - pair insert(const value_type &x) { + std::pair insert(const value_type &x) { int size = this->tree_.size(); - pair checker_res = + std::pair checker_res = this->checker_.insert(x); - pair tree_res = this->tree_.insert(x); - CHECK_EQ(*tree_res.first, *checker_res.first); - CHECK_EQ(tree_res.second, checker_res.second); - CHECK_EQ(this->tree_.size(), this->checker_.size()); - CHECK_EQ(this->tree_.size(), size + tree_res.second); + std::pair tree_res = this->tree_.insert(x); + EXPECT_EQ(*tree_res.first, *checker_res.first); + EXPECT_EQ(tree_res.second, checker_res.second); + EXPECT_EQ(this->tree_.size(), this->checker_.size()); + EXPECT_EQ(this->tree_.size(), size + tree_res.second); return tree_res; } iterator insert(iterator position, const value_type &x) { int size = this->tree_.size(); - pair checker_res = + std::pair checker_res = this->checker_.insert(x); iterator tree_res = this->tree_.insert(position, x); - CHECK_EQ(*tree_res, *checker_res.first); - CHECK_EQ(this->tree_.size(), this->checker_.size()); - CHECK_EQ(this->tree_.size(), size + checker_res.second); + EXPECT_EQ(*tree_res, *checker_res.first); + EXPECT_EQ(this->tree_.size(), this->checker_.size()); + EXPECT_EQ(this->tree_.size(), size + checker_res.second); return tree_res; } template @@ -428,18 +454,18 @@ class multi_checker : public base_checker { int size = this->tree_.size(); typename CheckerType::iterator checker_res = this->checker_.insert(x); iterator tree_res = this->tree_.insert(x); - CHECK_EQ(*tree_res, *checker_res); - CHECK_EQ(this->tree_.size(), this->checker_.size()); - CHECK_EQ(this->tree_.size(), size + 1); + EXPECT_EQ(*tree_res, *checker_res); + EXPECT_EQ(this->tree_.size(), this->checker_.size()); + EXPECT_EQ(this->tree_.size(), size + 1); return tree_res; } iterator insert(iterator position, const value_type &x) { int size = this->tree_.size(); typename CheckerType::iterator checker_res = this->checker_.insert(x); iterator tree_res = this->tree_.insert(position, x); - CHECK_EQ(*tree_res, *checker_res); - CHECK_EQ(this->tree_.size(), this->checker_.size()); - CHECK_EQ(this->tree_.size(), size + 1); + EXPECT_EQ(*tree_res, *checker_res); + EXPECT_EQ(this->tree_.size(), this->checker_.size()); + EXPECT_EQ(this->tree_.size(), size + 1); return tree_res; } template @@ -451,7 +477,7 @@ class multi_checker : public base_checker { }; char* GenerateDigits(char buf[16], int val, int maxval) { - DCHECK_LE(val, maxval); + EXPECT_LE(val, maxval); int p = 15; buf[p--] = 0; while (maxval > 0) { @@ -469,61 +495,48 @@ struct Generator { : maxval(m) { } K operator()(int i) const { - DCHECK_LE(i, maxval); + EXPECT_LE(i, maxval); return i; } }; template <> -struct Generator { +struct Generator { int maxval; Generator(int m) : maxval(m) { } - string operator()(int i) const { + std::string operator()(int i) const { char buf[16]; return GenerateDigits(buf, i, maxval); } }; -template <> -struct Generator { - int maxval; - Generator(int m) - : maxval(m) { - } - Cord operator()(int i) const { - char buf[16]; - return Cord(GenerateDigits(buf, i, maxval)); - } -}; - template -struct Generator > { - Generator::type> tgen; - Generator::type> ugen; +struct Generator > { + Generator::type> tgen; + Generator::type> ugen; Generator(int m) : tgen(m), ugen(m) { } - pair operator()(int i) const { - return make_pair(tgen(i), ugen(i)); + std::pair operator()(int i) const { + return std::make_pair(tgen(i), ugen(i)); } }; // Generate values for our tests and benchmarks. Value range is [0, maxval]. -const vector& GenerateNumbers(int n, int maxval) { - static ACMRandom rand(FLAGS_test_random_seed); - static vector values; - static set unique_values; +const std::vector& GenerateNumbers(int n, int maxval) { + static std::vector values; + static std::set unique_values; if (values.size() < n) { for (int i = values.size(); i < n; i++) { int value; do { - value = rand.Next() % (maxval + 1); + value = rand() % (maxval + 1); } while (unique_values.find(value) != unique_values.end()); values.push_back(value); @@ -537,13 +550,13 @@ const vector& GenerateNumbers(int n, int maxval) { // Generates values in the range // [0, 4 * min(FLAGS_benchmark_values, FLAGS_test_values)] template -vector GenerateValues(int n) { - int two_times_max = 2 * max(FLAGS_benchmark_values, FLAGS_test_values); +std::vector GenerateValues(int n) { + int two_times_max = 2 * std::max(FLAGS_benchmark_values, FLAGS_test_values); int four_times_max = 2 * two_times_max; - DCHECK_LE(n, two_times_max); - const vector &nums = GenerateNumbers(n, four_times_max); + EXPECT_LE(n, two_times_max); + const std::vector &nums = GenerateNumbers(n, four_times_max); Generator gen(four_times_max); - vector vec; + std::vector vec; for (int i = 0; i < n; i++) { vec.push_back(gen(nums[i])); @@ -553,44 +566,44 @@ vector GenerateValues(int n) { } template -double ContainerInfo(const set &s) { +double ContainerInfo(const std::set &s) { int sizeof_node = sizeof(std::_Rb_tree_node); int bytes_used = sizeof(s) + s.size() * sizeof_node; double bytes_per_value = (double) bytes_used / s.size(); - VLOG(1) << " size=" << s.size() + std::cout << " size=" << s.size() << " bytes-used=" << bytes_used << " bytes-per-value=" << bytes_per_value; return bytes_per_value; } template -double ContainerInfo(const multiset &s) { +double ContainerInfo(const std::multiset &s) { int sizeof_node = sizeof(std::_Rb_tree_node); int bytes_used = sizeof(s) + s.size() * sizeof_node; double bytes_per_value = (double) bytes_used / s.size(); - VLOG(1) << " size=" << s.size() + std::cout << " size=" << s.size() << " bytes-used=" << bytes_used << " bytes-per-value=" << bytes_per_value; return bytes_per_value; } template -double ContainerInfo(const map &m) { - int sizeof_node = sizeof(std::_Rb_tree_node >); +double ContainerInfo(const std::map &m) { + int sizeof_node = sizeof(std::_Rb_tree_node >); int bytes_used = sizeof(m) + m.size() * sizeof_node; double bytes_per_value = (double) bytes_used / m.size(); - VLOG(1) << " size=" << m.size() + std::cout << " size=" << m.size() << " bytes-used=" << bytes_used << " bytes-per-value=" << bytes_per_value; return bytes_per_value; } template -double ContainerInfo(const multimap &m) { - int sizeof_node = sizeof(std::_Rb_tree_node >); +double ContainerInfo(const std::multimap &m) { + int sizeof_node = sizeof(std::_Rb_tree_node >); int bytes_used = sizeof(m) + m.size() * sizeof_node; double bytes_per_value = (double) bytes_used / m.size(); - VLOG(1) << " size=" << m.size() + std::cout << " size=" << m.size() << " bytes-used=" << bytes_used << " bytes-per-value=" << bytes_per_value; return bytes_per_value; @@ -600,7 +613,7 @@ template double ContainerInfo(const btree_container

&b) { double bytes_used = sizeof(b) + b.bytes_used(); double bytes_per_value = (double) bytes_used / b.size(); - VLOG(1) << " size=" << b.size() + std::cout << " size=" << b.size() << " bytes-used=" << bytes_used << " bytes-per-value=" << bytes_per_value << " height=" << b.height() @@ -612,7 +625,7 @@ double ContainerInfo(const btree_container

&b) { } template -void DoTest(const char *name, T *b, const vector &values) { +void DoTest(const char *name, T *b, const std::vector &values) { typename KeyOfValue::type key_of_value; T &mutable_b = *b; @@ -631,22 +644,22 @@ void DoTest(const char *name, T *b, const vector &values) { // Test copy constructor. T b_copy(const_b); - CHECK_EQ(b_copy.size(), const_b.size()); - CHECK_LE(b_copy.height(), const_b.height()); - CHECK_LE(b_copy.internal_nodes(), const_b.internal_nodes()); - CHECK_LE(b_copy.leaf_nodes(), const_b.leaf_nodes()); + EXPECT_EQ(b_copy.size(), const_b.size()); + EXPECT_LE(b_copy.height(), const_b.height()); + EXPECT_LE(b_copy.internal_nodes(), const_b.internal_nodes()); + EXPECT_LE(b_copy.leaf_nodes(), const_b.leaf_nodes()); for (int i = 0; i < values.size(); ++i) { - CHECK_EQ(*b_copy.find(key_of_value(values[i])), values[i]); + EXPECT_EQ(*b_copy.find(key_of_value(values[i])), values[i]); } // Test range constructor. T b_range(const_b.begin(), const_b.end()); - CHECK_EQ(b_range.size(), const_b.size()); - CHECK_LE(b_range.height(), const_b.height()); - CHECK_LE(b_range.internal_nodes(), const_b.internal_nodes()); - CHECK_LE(b_range.leaf_nodes(), const_b.leaf_nodes()); + EXPECT_EQ(b_range.size(), const_b.size()); + EXPECT_LE(b_range.height(), const_b.height()); + EXPECT_LE(b_range.internal_nodes(), const_b.internal_nodes()); + EXPECT_LE(b_range.leaf_nodes(), const_b.leaf_nodes()); for (int i = 0; i < values.size(); ++i) { - CHECK_EQ(*b_range.find(key_of_value(values[i])), values[i]); + EXPECT_EQ(*b_range.find(key_of_value(values[i])), values[i]); } // Test range insertion for values that already exist. @@ -656,36 +669,36 @@ void DoTest(const char *name, T *b, const vector &values) { // Test range insertion for new values. b_range.clear(); b_range.insert(b_copy.begin(), b_copy.end()); - CHECK_EQ(b_range.size(), b_copy.size()); - CHECK_EQ(b_range.height(), b_copy.height()); - CHECK_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); - CHECK_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); + EXPECT_EQ(b_range.size(), b_copy.size()); + EXPECT_EQ(b_range.height(), b_copy.height()); + EXPECT_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); + EXPECT_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); for (int i = 0; i < values.size(); ++i) { - CHECK_EQ(*b_range.find(key_of_value(values[i])), values[i]); + EXPECT_EQ(*b_range.find(key_of_value(values[i])), values[i]); } // Test assignment to self. Nothing should change. b_range.operator=(b_range); - CHECK_EQ(b_range.size(), b_copy.size()); - CHECK_EQ(b_range.height(), b_copy.height()); - CHECK_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); - CHECK_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); + EXPECT_EQ(b_range.size(), b_copy.size()); + EXPECT_EQ(b_range.height(), b_copy.height()); + EXPECT_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); + EXPECT_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); // Test assignment of new values. b_range.clear(); b_range = b_copy; - CHECK_EQ(b_range.size(), b_copy.size()); - CHECK_EQ(b_range.height(), b_copy.height()); - CHECK_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); - CHECK_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); + EXPECT_EQ(b_range.size(), b_copy.size()); + EXPECT_EQ(b_range.height(), b_copy.height()); + EXPECT_EQ(b_range.internal_nodes(), b_copy.internal_nodes()); + EXPECT_EQ(b_range.leaf_nodes(), b_copy.leaf_nodes()); // Test swap. b_range.clear(); b_range.swap(b_copy); - CHECK_EQ(b_copy.size(), 0); - CHECK_EQ(b_range.size(), const_b.size()); + EXPECT_EQ(b_copy.size(), 0); + EXPECT_EQ(b_range.size(), const_b.size()); for (int i = 0; i < values.size(); ++i) { - CHECK_EQ(*b_range.find(key_of_value(values[i])), values[i]); + EXPECT_EQ(*b_range.find(key_of_value(values[i])), values[i]); } b_range.swap(b_copy); @@ -693,13 +706,13 @@ void DoTest(const char *name, T *b, const vector &values) { for (int i = 0; i < values.size(); ++i) { mutable_b.erase(key_of_value(values[i])); // Erasing a non-existent key should have no effect. - CHECK_EQ(mutable_b.erase(key_of_value(values[i])), 0); + EXPECT_EQ(mutable_b.erase(key_of_value(values[i])), 0); } const_b.verify(); - CHECK_EQ(const_b.internal_nodes(), 0); - CHECK_EQ(const_b.leaf_nodes(), 0); - CHECK_EQ(const_b.size(), 0); + EXPECT_EQ(const_b.internal_nodes(), 0); + EXPECT_EQ(const_b.leaf_nodes(), 0); + EXPECT_EQ(const_b.size(), 0); // Test erase via iterators. mutable_b = b_copy; @@ -708,9 +721,9 @@ void DoTest(const char *name, T *b, const vector &values) { } const_b.verify(); - CHECK_EQ(const_b.internal_nodes(), 0); - CHECK_EQ(const_b.leaf_nodes(), 0); - CHECK_EQ(const_b.size(), 0); + EXPECT_EQ(const_b.internal_nodes(), 0); + EXPECT_EQ(const_b.leaf_nodes(), 0); + EXPECT_EQ(const_b.size(), 0); // Test insert with hint. for (int i = 0; i < values.size(); i++) { @@ -721,13 +734,13 @@ void DoTest(const char *name, T *b, const vector &values) { // Test dumping of the btree to an ostream. There should be 1 line for each // value. - ostringstream strm; + std::stringstream strm; strm << mutable_b.tree(); - CHECK_EQ(mutable_b.size(), strcount(strm.str(), '\n')); + EXPECT_EQ(mutable_b.size(), strcount(strm.str(), '\n')); // Test range erase. mutable_b.erase(mutable_b.begin(), mutable_b.end()); - CHECK_EQ(mutable_b.size(), 0); + EXPECT_EQ(mutable_b.size(), 0); const_b.verify(); // First half. @@ -735,7 +748,7 @@ void DoTest(const char *name, T *b, const vector &values) { typename T::iterator mutable_iter_end = mutable_b.begin(); for (int i = 0; i < values.size() / 2; ++i) ++mutable_iter_end; mutable_b.erase(mutable_b.begin(), mutable_iter_end); - CHECK_EQ(mutable_b.size(), values.size() - values.size() / 2); + EXPECT_EQ(mutable_b.size(), values.size() - values.size() / 2); const_b.verify(); // Second half. @@ -743,7 +756,7 @@ void DoTest(const char *name, T *b, const vector &values) { typename T::iterator mutable_iter_begin = mutable_b.begin(); for (int i = 0; i < values.size() / 2; ++i) ++mutable_iter_begin; mutable_b.erase(mutable_iter_begin, mutable_b.end()); - CHECK_EQ(mutable_b.size(), values.size() / 2); + EXPECT_EQ(mutable_b.size(), values.size() / 2); const_b.verify(); // Second quarter. @@ -753,7 +766,7 @@ void DoTest(const char *name, T *b, const vector &values) { mutable_iter_end = mutable_iter_begin; for (int i = 0; i < values.size() / 4; ++i) ++mutable_iter_end; mutable_b.erase(mutable_iter_begin, mutable_iter_end); - CHECK_EQ(mutable_b.size(), values.size() - values.size() / 4); + EXPECT_EQ(mutable_b.size(), values.size() - values.size() / 4); const_b.verify(); mutable_b.clear(); @@ -770,62 +783,62 @@ void ConstTest() { // Insert a single value into the container and test looking it up. value_type value = Generator(2)(2); mutable_b.insert(value); - CHECK(mutable_b.find(key_of_value(value)) != const_b.end()); - CHECK(const_b.find(key_of_value(value)) != mutable_b.end()); - CHECK_EQ(*const_b.lower_bound(key_of_value(value)), value); - CHECK(const_b.upper_bound(key_of_value(value)) == const_b.end()); - CHECK_EQ(*const_b.equal_range(key_of_value(value)).first, value); + EXPECT_TRUE(mutable_b.find(key_of_value(value)) != const_b.end()); + EXPECT_TRUE(const_b.find(key_of_value(value)) != mutable_b.end()); + EXPECT_EQ(*const_b.lower_bound(key_of_value(value)), value); + EXPECT_TRUE(const_b.upper_bound(key_of_value(value)) == const_b.end()); + EXPECT_EQ(*const_b.equal_range(key_of_value(value)).first, value); // We can only create a non-const iterator from a non-const container. typename T::iterator mutable_iter(mutable_b.begin()); - CHECK(mutable_iter == const_b.begin()); - CHECK(mutable_iter != const_b.end()); - CHECK(const_b.begin() == mutable_iter); - CHECK(const_b.end() != mutable_iter); + EXPECT_TRUE(mutable_iter == const_b.begin()); + EXPECT_TRUE(mutable_iter != const_b.end()); + EXPECT_TRUE(const_b.begin() == mutable_iter); + EXPECT_TRUE(const_b.end() != mutable_iter); typename T::reverse_iterator mutable_riter(mutable_b.rbegin()); - CHECK(mutable_riter == const_b.rbegin()); - CHECK(mutable_riter != const_b.rend()); - CHECK(const_b.rbegin() == mutable_riter); - CHECK(const_b.rend() != mutable_riter); + EXPECT_TRUE(mutable_riter == const_b.rbegin()); + EXPECT_TRUE(mutable_riter != const_b.rend()); + EXPECT_TRUE(const_b.rbegin() == mutable_riter); + EXPECT_TRUE(const_b.rend() != mutable_riter); // We can create a const iterator from a non-const iterator. typename T::const_iterator const_iter(mutable_iter); - CHECK(const_iter == mutable_b.begin()); - CHECK(const_iter != mutable_b.end()); - CHECK(mutable_b.begin() == const_iter); - CHECK(mutable_b.end() != const_iter); + EXPECT_TRUE(const_iter == mutable_b.begin()); + EXPECT_TRUE(const_iter != mutable_b.end()); + EXPECT_TRUE(mutable_b.begin() == const_iter); + EXPECT_TRUE(mutable_b.end() != const_iter); typename T::const_reverse_iterator const_riter(mutable_riter); - CHECK(const_riter == mutable_b.rbegin()); - CHECK(const_riter != mutable_b.rend()); - CHECK(mutable_b.rbegin() == const_riter); - CHECK(mutable_b.rend() != const_riter); + EXPECT_EQ(const_riter, mutable_b.rbegin()); + EXPECT_TRUE(const_riter != mutable_b.rend()); + EXPECT_EQ(mutable_b.rbegin(), const_riter); + EXPECT_TRUE(mutable_b.rend() != const_riter); // Make sure various methods can be invoked on a const container. const_b.verify(); - CHECK(!const_b.empty()); - CHECK_EQ(const_b.size(), 1); - CHECK_GT(const_b.max_size(), 0); - CHECK_EQ(const_b.height(), 1); - CHECK_EQ(const_b.count(key_of_value(value)), 1); - CHECK_EQ(const_b.internal_nodes(), 0); - CHECK_EQ(const_b.leaf_nodes(), 1); - CHECK_EQ(const_b.nodes(), 1); - CHECK_GT(const_b.bytes_used(), 0); - CHECK_GT(const_b.fullness(), 0); - CHECK_GT(const_b.overhead(), 0); + EXPECT_FALSE(const_b.empty()); + EXPECT_EQ(const_b.size(), 1); + EXPECT_GT(const_b.max_size(), 0); + EXPECT_EQ(const_b.height(), 1); + EXPECT_EQ(const_b.count(key_of_value(value)), 1); + EXPECT_EQ(const_b.internal_nodes(), 0); + EXPECT_EQ(const_b.leaf_nodes(), 1); + EXPECT_EQ(const_b.nodes(), 1); + EXPECT_GT(const_b.bytes_used(), 0); + EXPECT_GT(const_b.fullness(), 0); + EXPECT_GT(const_b.overhead(), 0); } template void BtreeTest() { ConstTest(); - typedef typename base::remove_const::type V; - vector random_values = GenerateValues(FLAGS_test_values); + typedef typename std::remove_const::type V; + std::vector random_values = GenerateValues(FLAGS_test_values); unique_checker container; // Test key insertion/deletion in sorted order. - vector sorted_values(random_values); + std::vector sorted_values(random_values); sort(sorted_values.begin(), sorted_values.end()); DoTest("sorted: ", &container, sorted_values); @@ -841,13 +854,13 @@ template void BtreeMultiTest() { ConstTest(); - typedef typename base::remove_const::type V; - const vector& random_values = GenerateValues(FLAGS_test_values); + typedef typename std::remove_const::type V; + const std::vector& random_values = GenerateValues(FLAGS_test_values); multi_checker container; // Test keys in sorted order. - vector sorted_values(random_values); + std::vector sorted_values(random_values); sort(sorted_values.begin(), sorted_values.end()); DoTest("sorted: ", &container, sorted_values); @@ -859,25 +872,64 @@ void BtreeMultiTest() { DoTest("random: ", &container, random_values); // Test keys in random order w/ duplicates. - vector duplicate_values(random_values); + std::vector duplicate_values(random_values); duplicate_values.insert( duplicate_values.end(), random_values.begin(), random_values.end()); DoTest("duplicates:", &container, duplicate_values); // Test all identical keys. - vector identical_values(100); + std::vector identical_values(100); fill(identical_values.begin(), identical_values.end(), Generator(2)(2)); DoTest("identical: ", &container, identical_values); } +template > +class TestAllocator : public Alloc { + public: + typedef typename Alloc::pointer pointer; + typedef typename Alloc::size_type size_type; + + TestAllocator() : bytes_used_(NULL) { } + TestAllocator(int64_t *bytes_used) : bytes_used_(bytes_used) { } + + // Constructor used for rebinding + template + TestAllocator(const TestAllocator& x) + : Alloc(x), + bytes_used_(x.bytes_used()) { + } + + pointer allocate(size_type n, std::allocator::const_pointer hint = 0) { + EXPECT_TRUE(bytes_used_ != NULL); + *bytes_used_ += n * sizeof(T); + return Alloc::allocate(n, hint); + } + + void deallocate(pointer p, size_type n) { + Alloc::deallocate(p, n); + EXPECT_TRUE(bytes_used_ != NULL); + *bytes_used_ -= n * sizeof(T); + } + + // Rebind allows an allocator to be used for a different type + template struct rebind { + typedef TestAllocator::other> other; + }; + + int64_t* bytes_used() const { return bytes_used_; } + + private: + int64_t *bytes_used_; +}; + template -void BtreeArenaTest() { +void BtreeAllocatorTest() { typedef typename T::value_type value_type; - UnsafeArena arena1(1000); - UnsafeArena arena2(1000); - T b1(typename T::key_compare(), &arena1); - T b2(typename T::key_compare(), &arena2); + int64_t alloc1 = 0; + int64_t alloc2 = 0; + T b1(typename T::key_compare(), &alloc1); + T b2(typename T::key_compare(), &alloc2); // This should swap the allocators! swap(b1, b2); @@ -886,9 +938,9 @@ void BtreeArenaTest() { b1.insert(Generator(1000)(i)); } - // We should have allocated out of arena2! - CHECK_LE(b1.bytes_used(), arena2.status().bytes_allocated()); - CHECK_GT(arena2.block_count(), arena1.block_count()); + // We should have allocated out of alloc2! + EXPECT_LE(b1.bytes_used(), alloc2 + sizeof(b1)); + EXPECT_GT(alloc2, alloc1); } template @@ -906,15 +958,15 @@ void BtreeMapTest() { value_type v = Generator(1000)(i); b[v.first] = v.second; } - CHECK_EQ(b.size(), 1000); + EXPECT_EQ(b.size(), 1000); // Test whether we can use the "->" operator on iterators and // reverse_iterators. This stresses the btree_map_params::pair_pointer // mechanism. - CHECK_EQ(b.begin()->first, Generator(1000)(0).first); - CHECK_EQ(b.begin()->second, Generator(1000)(0).second); - CHECK_EQ(b.rbegin()->first, Generator(1000)(999).first); - CHECK_EQ(b.rbegin()->second, Generator(1000)(999).second); + EXPECT_EQ(b.begin()->first, Generator(1000)(0).first); + EXPECT_EQ(b.begin()->second, Generator(1000)(0).second); + EXPECT_EQ(b.rbegin()->first, Generator(1000)(999).first); + EXPECT_EQ(b.rbegin()->second, Generator(1000)(999).second); } template @@ -925,6 +977,5 @@ void BtreeMultiMapTest() { } } // namespace btree -} // namespace util #endif // UTIL_BTREE_BTREE_TEST_H__ diff --git a/btree_test_flags.cc b/btree_test_flags.cc index 512c24e1..d43d9250 100644 --- a/btree_test_flags.cc +++ b/btree_test_flags.cc @@ -1,7 +1,7 @@ // Copyright 2007 Google Inc. All Rights Reserved. // Author: pmattis@google.com (Peter Mattis) -#include "base/commandlineflags.h" +#include "gflags/gflags.h" DEFINE_int32(test_values, 10000, "The number of values to use for tests."); diff --git a/safe_btree.h b/safe_btree.h index 2a354232..01cf9faf 100644 --- a/safe_btree.h +++ b/safe_btree.h @@ -21,11 +21,8 @@ #include #include -#include "base/integral_types.h" -#include "base/logging.h" -#include "util/btree/btree.h" +#include "btree.h" -namespace util { namespace btree { template @@ -77,7 +74,7 @@ class safe_btree_iterator { } Tree* tree() const { return tree_; } - int64 generation() const { return generation_; } + int64_t generation() const { return generation_; } Iterator* mutable_iter() const { if (generation_ != tree_->generation()) { @@ -114,13 +111,13 @@ class safe_btree_iterator { // This reference value is potentially invalidated by any non-const // method on the tree; it is NOT safe. reference operator*() const { - DCHECK_GT(generation_, 0); + assert(generation_ > 0); return iter().operator*(); } // This pointer value is potentially invalidated by any non-const // method on the tree; it is NOT safe. pointer operator->() const { - DCHECK_GT(generation_, 0); + assert(generation_ > 0); return iter().operator->(); } @@ -148,7 +145,7 @@ class safe_btree_iterator { private: // The generation of the tree when "iter" was updated. - mutable int64 generation_; + mutable int64_t generation_; // The key the iterator points to. mutable key_type key_; // The underlying iterator. @@ -236,14 +233,14 @@ class safe_btree { const_iterator upper_bound(const key_type &key) const { return const_iterator(this, tree_.upper_bound(key)); } - pair equal_range(const key_type &key) { - pair p = tree_.equal_range(key); - return make_pair(iterator(this, p.first), + std::pair equal_range(const key_type &key) { + std::pair p = tree_.equal_range(key); + return std::make_pair(iterator(this, p.first), iterator(this, p.second)); } - pair equal_range(const key_type &key) const { - pair p = tree_.equal_range(key); - return make_pair(const_iterator(this, p.first), + std::pair equal_range(const key_type &key) const { + std::pair p = tree_.equal_range(key); + return std::make_pair(const_iterator(this, p.first), const_iterator(this, p.second)); } iterator find_unique(const key_type &key) { @@ -267,15 +264,15 @@ class safe_btree { // Insertion routines. template - pair insert_unique(const key_type &key, ValuePointer value) { - pair p = tree_.insert_unique(key, value); + std::pair insert_unique(const key_type &key, ValuePointer value) { + std::pair p = tree_.insert_unique(key, value); generation_ += p.second; - return make_pair(iterator(this, p.first), p.second); + return std::make_pair(iterator(this, p.first), p.second); } - pair insert_unique(const value_type &v) { - pair p = tree_.insert_unique(v); + std::pair insert_unique(const value_type &v) { + std::pair p = tree_.insert_unique(v); generation_ += p.second; - return make_pair(iterator(this, p.first), p.second); + return std::make_pair(iterator(this, p.first), p.second); } iterator insert_unique(iterator position, const value_type &v) { tree_iterator tree_pos = position.iter(); @@ -351,13 +348,13 @@ class safe_btree { ++x.generation_; tree_.swap(x.tree_); } - void dump(ostream &os) const { + void dump(std::ostream &os) const { tree_.dump(os); } void verify() const { tree_.verify(); } - int64 generation() const { + int64_t generation() const { return generation_; } key_compare key_comp() const { return tree_.key_comp(); } @@ -379,10 +376,9 @@ class safe_btree { private: btree_type tree_; - int64 generation_; + int64_t generation_; }; } // namespace btree -} // namespace util #endif // UTIL_BTREE_SAFE_BTREE_H__ diff --git a/safe_btree_map.h b/safe_btree_map.h index 11274930..b8aa787e 100644 --- a/safe_btree_map.h +++ b/safe_btree_map.h @@ -22,17 +22,16 @@ #include #include -#include "util/btree/btree_container.h" -#include "util/btree/btree_map.h" -#include "util/btree/safe_btree.h" +#include "btree_container.h" +#include "btree_map.h" +#include "safe_btree.h" -namespace util { namespace btree { // The safe_btree_map class is needed mainly for its constructors. template , - typename Alloc = std::allocator >, + typename Compare = std::less, + typename Alloc = std::allocator >, int TargetNodeSize = 256> class safe_btree_map : public btree_map_container< safe_btree > > { @@ -75,6 +74,5 @@ inline void swap(safe_btree_map &x, } } // namespace btree -} // namespace util #endif // UTIL_BTREE_SAFE_BTREE_MAP_H__ diff --git a/safe_btree_set.h b/safe_btree_set.h index 72ed2d91..de8f823a 100644 --- a/safe_btree_set.h +++ b/safe_btree_set.h @@ -22,16 +22,15 @@ #include #include -#include "util/btree/btree_container.h" -#include "util/btree/btree_set.h" -#include "util/btree/safe_btree.h" +#include "btree_container.h" +#include "btree_set.h" +#include "safe_btree.h" -namespace util { namespace btree { // The safe_btree_set class is needed mainly for its constructors. template , + typename Compare = std::less, typename Alloc = std::allocator, int TargetNodeSize = 256> class safe_btree_set : public btree_unique_container< @@ -74,6 +73,5 @@ inline void swap(safe_btree_set &x, } } // namespace btree -} // namespace util #endif // UTIL_BTREE_SAFE_BTREE_SET_H__ diff --git a/safe_btree_test.cc b/safe_btree_test.cc index cf179b28..4965916b 100644 --- a/safe_btree_test.cc +++ b/safe_btree_test.cc @@ -11,58 +11,50 @@ #include #include -#include "base/arena-inl.h" -#include "base/init_google.h" -#include "base/integral_types.h" -#include "base/logging.h" -#include "strings/cord.h" -#include "testing/base/public/gunit.h" -#include "util/btree/btree_test.h" -#include "util/btree/safe_btree_map.h" -#include "util/btree/safe_btree_set.h" +#include "gtest/gtest.h" +#include "btree_test.h" +#include "safe_btree_map.h" +#include "safe_btree_set.h" class UnsafeArena; -namespace util { namespace btree { namespace { template void SetTest() { - typedef ArenaAllocator ArenaAlloc; - BtreeTest, allocator, N>, set >(); - BtreeArenaTest, ArenaAlloc, N> >(); + typedef TestAllocator TestAlloc; + BtreeTest, std::allocator, N>, std::set >(); + BtreeAllocatorTest, TestAlloc, N> >(); } template void MapTest() { - typedef ArenaAllocator ArenaAlloc; - BtreeTest, allocator, N>, map >(); - BtreeArenaTest, ArenaAlloc, N> >(); - BtreeMapTest, allocator, N> >(); + typedef TestAllocator TestAlloc; + BtreeTest, std::allocator, N>, std::map >(); + BtreeAllocatorTest, TestAlloc, N> >(); + BtreeMapTest, std::allocator, N> >(); } -TEST(SafeBtree, set_int32_32) { SetTest(); } -TEST(SafeBtree, set_int32_64) { SetTest(); } -TEST(SafeBtree, set_int32_128) { SetTest(); } -TEST(SafeBtree, set_int32_256) { SetTest(); } -TEST(SafeBtree, set_int64_256) { SetTest(); } -TEST(SafeBtree, set_string_256) { SetTest(); } -TEST(SafeBtree, set_cord_256) { SetTest(); } -TEST(SafeBtree, set_pair_256) { SetTest, 256>(); } -TEST(SafeBtree, map_int32_256) { MapTest(); } -TEST(SafeBtree, map_int64_256) { MapTest(); } -TEST(SafeBtree, map_string_256) { MapTest(); } -TEST(SafeBtree, map_cord_256) { MapTest(); } -TEST(SafeBtree, map_pair_256) { MapTest, 256>(); } +TEST(SafeBtree, set_int32_32) { SetTest(); } +TEST(SafeBtree, set_int32_64) { SetTest(); } +TEST(SafeBtree, set_int32_128) { SetTest(); } +TEST(SafeBtree, set_int32_256) { SetTest(); } +TEST(SafeBtree, set_int64_256) { SetTest(); } +TEST(SafeBtree, set_string_256) { SetTest(); } +TEST(SafeBtree, set_pair_256) { SetTest, 256>(); } +TEST(SafeBtree, map_int32_256) { MapTest(); } +TEST(SafeBtree, map_int64_256) { MapTest(); } +TEST(SafeBtree, map_string_256) { MapTest(); } +TEST(SafeBtree, map_pair_256) { MapTest, 256>(); } TEST(SafeBtree, Comparison) { const int kSetSize = 1201; - safe_btree_set my_set; + safe_btree_set my_set; for (int i = 0; i < kSetSize; ++i) { my_set.insert(i); } - safe_btree_set my_set_copy(my_set); + safe_btree_set my_set_copy(my_set); EXPECT_TRUE(my_set_copy == my_set); EXPECT_TRUE(my_set == my_set_copy); EXPECT_FALSE(my_set_copy != my_set); @@ -80,17 +72,17 @@ TEST(SafeBtree, Comparison) { EXPECT_TRUE(my_set_copy != my_set); EXPECT_TRUE(my_set != my_set_copy); - safe_btree_map my_map; + safe_btree_map my_map; for (int i = 0; i < kSetSize; ++i) { - my_map[string(i, 'a')] = i; + my_map[std::string(i, 'a')] = i; } - safe_btree_map my_map_copy(my_map); + safe_btree_map my_map_copy(my_map); EXPECT_TRUE(my_map_copy == my_map); EXPECT_TRUE(my_map == my_map_copy); EXPECT_FALSE(my_map_copy != my_map); EXPECT_FALSE(my_map != my_map_copy); - ++my_map_copy[string(7, 'a')]; + ++my_map_copy[std::string(7, 'a')]; EXPECT_FALSE(my_map_copy == my_map); EXPECT_FALSE(my_map == my_map_copy); EXPECT_TRUE(my_map_copy != my_map); @@ -103,7 +95,7 @@ TEST(SafeBtree, Comparison) { EXPECT_TRUE(my_map_copy != my_map); EXPECT_TRUE(my_map != my_map_copy); - my_map.erase(string(kSetSize - 1, 'a')); + my_map.erase(std::string(kSetSize - 1, 'a')); EXPECT_FALSE(my_map_copy == my_map); EXPECT_FALSE(my_map == my_map_copy); EXPECT_TRUE(my_map_copy != my_map); @@ -112,10 +104,3 @@ TEST(SafeBtree, Comparison) { } // namespace } // namespace btree -} // namespace util - -int main(int argc, char **argv) { - FLAGS_logtostderr = true; - InitGoogle(argv[0], &argc, &argv, true); - return RUN_ALL_TESTS(); -} From 2fd7ed36a9031c794f3247b6952308a35cea8079 Mon Sep 17 00:00:00 2001 From: josh macdonald Date: Sun, 13 Jan 2013 00:16:02 -0800 Subject: [PATCH 08/74] Make btree_bench work; compiled & tested on OS X 10.7.5 using gcc 4.7.2 -std=c++11 --- CMakeLists.txt | 6 +- btree_bench.cc | 240 ++++++++++++++++++++++++++++++++++--------------- btree_test.h | 59 ------------ 3 files changed, 171 insertions(+), 134 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e908f4a4..79c67ac7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,8 +3,8 @@ cmake_minimum_required(VERSION 2.6) project(cppbtree CXX) option(build_tests "Build B-tree tests" OFF) -add_definitions(-std=c++0x) -set(CMAKE_CXX_FLAGS "-g") +add_definitions(-std=c++11) +set(CMAKE_CXX_FLAGS "-g -O2") # CMake doesn't have a way to pure template library, add_library(cppbtree btree.h btree_map.h btree_set.h safe_btree.h safe_btree_map.h safe_btree_set.h) @@ -19,5 +19,5 @@ if(build_tests) add_executable(btree_bench btree_bench.cc btree_test_flags.cc) target_link_libraries(btree_test gtest_main gflags) target_link_libraries(safe_btree_test gtest_main gflags) - target_link_libraries(btree_bench gflags) + target_link_libraries(btree_bench gflags gtest) endif() diff --git a/btree_bench.cc b/btree_bench.cc index 1b45582a..b803ad38 100644 --- a/btree_bench.cc +++ b/btree_bench.cc @@ -3,36 +3,160 @@ // Author: pmattis@google.com (Peter Mattis) #include +#include #include #include #include #include #include +#include +#include #include -#include "base/commandlineflags.h" -#include "base/init_google.h" -#include "base/integral_types.h" -#include "base/logging.h" -#include "base/stringprintf.h" -#include "base/type_traits.h" -#include "strings/cord.h" -#include "testing/base/public/benchmark.h" -#include "testing/base/public/googletest.h" -#include "util/btree/btree_map.h" -#include "util/btree/btree_set.h" -#include "util/btree/btree_test.h" -#include "util/random/acmrandom.h" +#include "gflags/gflags.h" +#include "btree_map.h" +#include "btree_set.h" +#include "btree_test.h" -DECLARE_int32(benchmark_max_iters); +DEFINE_int32(test_random_seed, 123456789, "Seed for srand()"); +DEFINE_int32(benchmark_max_iters, 10000000, "Maximum test iterations"); +DEFINE_int32(benchmark_min_iters, 100, "Minimum test iterations"); +DEFINE_int32(benchmark_target_seconds, 1, + "Attempt to benchmark for this many seconds"); + +using std::allocator; +using std::less; +using std::map; +using std::max; +using std::min; +using std::multimap; +using std::multiset; +using std::set; +using std::string; +using std::vector; namespace btree { namespace { +struct RandGen { + typedef ptrdiff_t result_type; + RandGen(result_type seed) { + srand(seed); + } + result_type operator()(result_type l) { + return rand() % l; + } +}; + +struct BenchmarkRun { + BenchmarkRun(const char *name, void (*func)(int)); + void Run(); + void Stop(); + void Start(); + void Reset(); + + BenchmarkRun *next_benchmark; + const char *benchmark_name; + void (*benchmark_func)(int); + int64_t accum_micros; + int64_t last_started; +}; + +BenchmarkRun *first_benchmark; +BenchmarkRun *current_benchmark; + +int64_t get_micros () { + timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000000 + tv.tv_usec; +} + +BenchmarkRun::BenchmarkRun(const char *name, void (*func)(int)) + : next_benchmark(first_benchmark), + benchmark_name(name), + benchmark_func(func), + accum_micros(0), + last_started(0) { + first_benchmark = this; +} + +#define BTREE_BENCHMARK(name) \ + BTREE_BENCHMARK2(#name, name, __COUNTER__) +#define BTREE_BENCHMARK2(name, func, counter) \ + BTREE_BENCHMARK3(name, func, counter) +#define BTREE_BENCHMARK3(name, func, counter) \ + BenchmarkRun bench ## counter (name, func) + +void StopBenchmarkTiming() { + current_benchmark->Stop(); +} + +void StartBenchmarkTiming() { + current_benchmark->Start(); +} + +void RunBenchmarks() { + for (BenchmarkRun *bench = first_benchmark; bench; + bench = bench->next_benchmark) { + bench->Run(); + } +} + +void BenchmarkRun::Start() { + assert(!last_started); + last_started = get_micros(); +} + +void BenchmarkRun::Stop() { + if (last_started == 0) { + return; + } + accum_micros += get_micros() - last_started; + last_started = 0; +} + +void BenchmarkRun::Reset() { + last_started = 0; + accum_micros = 0; +} + +void BenchmarkRun::Run() { + assert(current_benchmark == NULL); + current_benchmark = this; + int iters = FLAGS_benchmark_min_iters; + for (;;) { + Reset(); + Start(); + benchmark_func(iters); + Stop(); + if (accum_micros > FLAGS_benchmark_target_seconds * 1000000 || + iters >= FLAGS_benchmark_max_iters) { + break; + } else if (accum_micros == 0) { + iters *= 100; + } else { + int64_t target_micros = FLAGS_benchmark_target_seconds * 1000000; + iters = target_micros * iters / accum_micros; + } + iters = min(iters, FLAGS_benchmark_max_iters); + } + fprintf(stdout, "%s\t%qu\t%qu\n", + benchmark_name, + accum_micros * 1000 / iters, + iters); + current_benchmark = NULL; +} + +// Used to avoid compiler optimizations for these benchmarks. +template +void sink(const T& t0) { + volatile T t = t0; +} + // Benchmark insertion of values into a container. template void BM_Insert(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; typename KeyOfValue::type key_of_value; // Disable timing while we perform some initialization. @@ -43,7 +167,6 @@ void BM_Insert(int n) { for (int i = 0; i < values.size(); i++) { container.insert(values[i]); } - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); for (int i = 0; i < n; ) { // Remove and re-insert 10% of the keys @@ -70,7 +193,7 @@ void BM_Insert(int n) { // Benchmark lookup of values in a container. template void BM_Lookup(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; typename KeyOfValue::type key_of_value; // Disable timing while we perform some initialization. @@ -82,7 +205,6 @@ void BM_Lookup(int n) { for (int i = 0; i < values.size(); i++) { container.insert(values[i]); } - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); V r = V(); @@ -95,7 +217,7 @@ void BM_Lookup(int n) { StopBenchmarkTiming(); - VLOG(4) << r; // Keep compiler from optimizing away r. + sink(r); // Keep compiler from optimizing away r. } // Benchmark lookup of values in a full container, meaning that values @@ -103,7 +225,7 @@ void BM_Lookup(int n) { // yields a full tree. template void BM_FullLookup(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; typename KeyOfValue::type key_of_value; // Disable timing while we perform some initialization. @@ -117,7 +239,6 @@ void BM_FullLookup(int n) { for (int i = 0; i < sorted.size(); i++) { container.insert(sorted[i]); } - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); V r = V(); @@ -130,13 +251,13 @@ void BM_FullLookup(int n) { StopBenchmarkTiming(); - VLOG(4) << r; // Keep compiler from optimizing away r. + sink(r); // Keep compiler from optimizing away r. } // Benchmark deletion of values from a container. template void BM_Delete(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; typename KeyOfValue::type key_of_value; // Disable timing while we perform some initialization. @@ -147,7 +268,6 @@ void BM_Delete(int n) { for (int i = 0; i < values.size(); i++) { container.insert(values[i]); } - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); for (int i = 0; i < n; ) { // Remove and re-insert 10% of the keys @@ -179,12 +299,12 @@ void BM_Delete(int n) { // value constructors. template void BM_QueueAddRem(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; typename KeyOfValue::type key_of_value; // Disable timing while we perform some initialization. StopBenchmarkTiming(); - CHECK(FLAGS_benchmark_values % 2 == 0); + assert(FLAGS_benchmark_values % 2 == 0); T container; @@ -197,7 +317,7 @@ void BM_QueueAddRem(int n) { add_keys[i] = i; } - ACMRandom rand(FLAGS_test_random_seed); + RandGen rand(FLAGS_test_random_seed); random_shuffle(remove_keys.begin(), remove_keys.end(), rand); random_shuffle(add_keys.begin(), add_keys.end(), rand); @@ -229,27 +349,25 @@ void BM_QueueAddRem(int n) { } int e = container.erase(key_of_value(g(offset - half + remove_keys[idx]))); - DCHECK(e == 1); + assert(e == 1); container.insert(g(offset + half + add_keys[idx])); } StopBenchmarkTiming(); - - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); } // Mixed insertion and deletion in the same range using pre-constructed values. template void BM_MixedAddRem(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; typename KeyOfValue::type key_of_value; // Disable timing while we perform some initialization. StopBenchmarkTiming(); - CHECK(FLAGS_benchmark_values % 2 == 0); + assert(FLAGS_benchmark_values % 2 == 0); T container; - ACMRandom rand(FLAGS_test_random_seed); + RandGen rand(FLAGS_test_random_seed); vector values = GenerateValues(FLAGS_benchmark_values * 2); @@ -282,20 +400,18 @@ void BM_MixedAddRem(int n) { } int e = container.erase(key_of_value(values[remove_keys[idx]])); - DCHECK(e == 1); + assert(e == 1); container.insert(values[add_keys[idx]]); } StopBenchmarkTiming(); - - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); } // Insertion at end, removal from the beginning. This benchmark // counts two value constructors. template void BM_Fifo(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; // Disable timing while we perform some initialization. StopBenchmarkTiming(); @@ -315,14 +431,12 @@ void BM_Fifo(int n) { } StopBenchmarkTiming(); - - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); } // Iteration (forward) through the tree template void BM_FwdIter(int n) { - typedef typename base::remove_const::type V; + typedef typename std::remove_const::type V; // Disable timing while we perform some initialization. StopBenchmarkTiming(); @@ -352,30 +466,24 @@ void BM_FwdIter(int n) { StopBenchmarkTiming(); - VLOG(4) << r; // Keep compiler from optimizing away r. - - SetBenchmarkLabel(StringPrintf(" %0.2f", ContainerInfo(container))); + sink(r); // Keep compiler from optimizing away r. } -typedef set stl_set_int32; -typedef set stl_set_int64; +typedef set stl_set_int32; +typedef set stl_set_int64; typedef set stl_set_string; -typedef set stl_set_cord; -typedef map stl_map_int32; -typedef map stl_map_int64; +typedef map stl_map_int32; +typedef map stl_map_int64; typedef map stl_map_string; -typedef map stl_map_cord; -typedef multiset stl_multiset_int32; -typedef multiset stl_multiset_int64; +typedef multiset stl_multiset_int32; +typedef multiset stl_multiset_int64; typedef multiset stl_multiset_string; -typedef multiset stl_multiset_cord; -typedef multimap stl_multimap_int32; -typedef multimap stl_multimap_int64; +typedef multimap stl_multimap_int32; +typedef multimap stl_multimap_int64; typedef multimap stl_multimap_string; -typedef multimap stl_multimap_cord; #define MY_BENCHMARK_TYPES2(value, name, size) \ typedef btree ## _set, allocator, size> \ @@ -405,21 +513,15 @@ typedef multimap stl_multimap_cord; MY_BENCHMARK_TYPES2(value, name, 1536); \ MY_BENCHMARK_TYPES2(value, name, 2048) -MY_BENCHMARK_TYPES(int32, int32); -MY_BENCHMARK_TYPES(int64, int64); +MY_BENCHMARK_TYPES(int32_t, int32); +MY_BENCHMARK_TYPES(int64_t, int64); MY_BENCHMARK_TYPES(string, string); -MY_BENCHMARK_TYPES(Cord, cord); #define MY_BENCHMARK4(type, name, func) \ void BM_ ## type ## _ ## name(int n) { BM_ ## func (n); } \ - BENCHMARK(BM_ ## type ## _ ## name) + BTREE_BENCHMARK(BM_ ## type ## _ ## name) -// Define NODESIZE_TESTING when running btree_perf.py. You need to do -// a local build or raise the distcc timeout, it takes about 5 minutes -// to build: -// -// blaze build --copts=-DNODESIZE_TESTING --cc_strategy=local -// --compilation_mode=opt util/btree/btree_test +// Define NODESIZE_TESTING when running btree_perf.py. #ifdef NODESIZE_TESTING #define MY_BENCHMARK3(tree, type, name, func) \ @@ -465,8 +567,6 @@ MY_BENCHMARK(set_int64); MY_BENCHMARK(map_int64); MY_BENCHMARK(set_string); MY_BENCHMARK(map_string); -MY_BENCHMARK(set_cord); -MY_BENCHMARK(map_cord); MY_BENCHMARK(multiset_int32); MY_BENCHMARK(multimap_int32); @@ -474,15 +574,11 @@ MY_BENCHMARK(multiset_int64); MY_BENCHMARK(multimap_int64); MY_BENCHMARK(multiset_string); MY_BENCHMARK(multimap_string); -MY_BENCHMARK(multiset_cord); -MY_BENCHMARK(multimap_cord); } // namespace } // namespace btree int main(int argc, char **argv) { - FLAGS_logtostderr = true; - InitGoogle(argv[0], &argc, &argv, true); - RunSpecifiedBenchmarks(); + btree::RunBenchmarks(); return 0; } diff --git a/btree_test.h b/btree_test.h index c368573e..01ab0fe0 100644 --- a/btree_test.h +++ b/btree_test.h @@ -565,65 +565,6 @@ std::vector GenerateValues(int n) { return vec; } -template -double ContainerInfo(const std::set &s) { - int sizeof_node = sizeof(std::_Rb_tree_node); - int bytes_used = sizeof(s) + s.size() * sizeof_node; - double bytes_per_value = (double) bytes_used / s.size(); - std::cout << " size=" << s.size() - << " bytes-used=" << bytes_used - << " bytes-per-value=" << bytes_per_value; - return bytes_per_value; -} - -template -double ContainerInfo(const std::multiset &s) { - int sizeof_node = sizeof(std::_Rb_tree_node); - int bytes_used = sizeof(s) + s.size() * sizeof_node; - double bytes_per_value = (double) bytes_used / s.size(); - std::cout << " size=" << s.size() - << " bytes-used=" << bytes_used - << " bytes-per-value=" << bytes_per_value; - return bytes_per_value; -} - -template -double ContainerInfo(const std::map &m) { - int sizeof_node = sizeof(std::_Rb_tree_node >); - int bytes_used = sizeof(m) + m.size() * sizeof_node; - double bytes_per_value = (double) bytes_used / m.size(); - std::cout << " size=" << m.size() - << " bytes-used=" << bytes_used - << " bytes-per-value=" << bytes_per_value; - return bytes_per_value; -} - -template -double ContainerInfo(const std::multimap &m) { - int sizeof_node = sizeof(std::_Rb_tree_node >); - int bytes_used = sizeof(m) + m.size() * sizeof_node; - double bytes_per_value = (double) bytes_used / m.size(); - std::cout << " size=" << m.size() - << " bytes-used=" << bytes_used - << " bytes-per-value=" << bytes_per_value; - return bytes_per_value; -} - -template -double ContainerInfo(const btree_container

&b) { - double bytes_used = sizeof(b) + b.bytes_used(); - double bytes_per_value = (double) bytes_used / b.size(); - std::cout << " size=" << b.size() - << " bytes-used=" << bytes_used - << " bytes-per-value=" << bytes_per_value - << " height=" << b.height() - << " internal-nodes=" << b.internal_nodes() - << " leaf-nodes=" << b.leaf_nodes() - << " fullness=" << b.fullness() - << " overhead=" << b.overhead(); - return bytes_per_value; -} - template void DoTest(const char *name, T *b, const std::vector &values) { typename KeyOfValue::type key_of_value; From ce6c9fd34c4d04345e73d7199346db648f00a071 Mon Sep 17 00:00:00 2001 From: josh macdonald Date: Mon, 14 Jan 2013 02:08:38 -0800 Subject: [PATCH 09/74] Build fixes --- CMakeLists.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 79c67ac7..0ef7749b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,8 +7,9 @@ add_definitions(-std=c++11) set(CMAKE_CXX_FLAGS "-g -O2") # CMake doesn't have a way to pure template library, -add_library(cppbtree btree.h btree_map.h btree_set.h safe_btree.h safe_btree_map.h safe_btree_set.h) -set_target_properties(cppbtree PROPERTIES LINKER_LANGUAGE CXX) +# add_library(cppbtree btree.h btree_map.h btree_set.h +# safe_btree.h safe_btree_map.h safe_btree_set.h) +# set_target_properties(cppbtree PROPERTIES LINKER_LANGUAGE CXX) if(build_tests) enable_testing() @@ -17,7 +18,7 @@ if(build_tests) add_executable(btree_test btree_test.cc btree_test_flags.cc) add_executable(safe_btree_test safe_btree_test.cc btree_test_flags.cc) add_executable(btree_bench btree_bench.cc btree_test_flags.cc) - target_link_libraries(btree_test gtest_main gflags) - target_link_libraries(safe_btree_test gtest_main gflags) + target_link_libraries(btree_test gtest_main gtest gflags) + target_link_libraries(safe_btree_test gtest_main gtest gflags) target_link_libraries(btree_bench gflags gtest) endif() From 7aad0f28fdc7325afbea09991d522cfea7754933 Mon Sep 17 00:00:00 2001 From: sername=jmacd Date: Tue, 22 Jan 2013 15:57:34 -0800 Subject: [PATCH 10/74] Update README; remove benchmarks.awk --- README | 6 ++++-- benchmarks.awk | 27 --------------------------- 2 files changed, 4 insertions(+), 29 deletions(-) delete mode 100755 benchmarks.awk diff --git a/README b/README index 152972cf..9ddca6bd 100644 --- a/README +++ b/README @@ -1,15 +1,17 @@ This library is a C++ template library and, as such, there is no -library to build and install. +library to build and install. Copy the .h files and use them! ---- To build and run the provided tests, however, you will need to install -CMake and the Google C++ Test framework. +CMake, the Google C++ Test framework, and the Google flags package. Download CMake from http://www.cmake.org Download the GoogleTest framework from http://code.google.com/p/googletest +Download gflags from https://code.google.com/p/gflags + export GTEST_ROOT=/path/for/gtest-x.y cmake . -Dbuild_tests=ON diff --git a/benchmarks.awk b/benchmarks.awk deleted file mode 100755 index d5d4aeac..00000000 --- a/benchmarks.awk +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/gawk -f - -/^Run on/ { - print $0; - printf "%-25s %5s %-20s\n", - "Benchmark", "STL(ns)", "B-Tree(ns) @ " - printf "--------------------------------------------------------\n"; -} - -/^BM_/ { - split($1, name, "_"); - if (name[2] == "stl") { - stl = $3; - stl_bytes = $5 - printf "%-25s %5d ", name[1] "_" name[3] "_" name[4] "_" name[5], stl; - fflush(); - } else if (name[2] == "btree") { - btree = $3 - btree_size = name[3] - btree_bytes = $5 - printf "%5d %+7.2f%% <%3d>", btree, 100.0 * (stl - btree) / stl, btree_size; - printf " [%4.1f, %4.1f]\n", stl_bytes, btree_bytes; - fflush(); - } else { - printf "ERROR: %s unrecognized\n", $1 - } -} From 5633058db44363552b042d93ea18d2451d6f97ae Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Fri, 25 Jan 2013 00:46:48 -0800 Subject: [PATCH 11/74] Update copyright notices; scrub a few references to Cord, StringPiece --- CMakeLists.txt | 14 +++ COPYING | 202 ++++++++++++++++++++++++++++++++++++++++++++ README | 9 +- btree.h | 44 ++++++---- btree_bench.cc | 16 +++- btree_container.h | 16 +++- btree_map.h | 16 +++- btree_set.h | 16 +++- btree_test.cc | 16 +++- btree_test.h | 16 +++- btree_test_flags.cc | 15 +++- safe_btree.h | 15 +++- safe_btree_map.h | 15 +++- safe_btree_set.h | 15 +++- safe_btree_test.cc | 16 +++- 15 files changed, 392 insertions(+), 49 deletions(-) create mode 100644 COPYING diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ef7749b..3c49ad27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,3 +1,17 @@ +# Copyright 2013 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + cmake_minimum_required(VERSION 2.6) project(cppbtree CXX) diff --git a/COPYING b/COPYING new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/COPYING @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README b/README index 9ddca6bd..0e8674cf 100644 --- a/README +++ b/README @@ -1,16 +1,19 @@ This library is a C++ template library and, as such, there is no library to build and install. Copy the .h files and use them! +See http://code.google.com/p/cpp-btree/wiki/UsageInstructions for +details. + ---- To build and run the provided tests, however, you will need to install CMake, the Google C++ Test framework, and the Google flags package. -Download CMake from http://www.cmake.org +Download and install CMake from http://www.cmake.org -Download the GoogleTest framework from http://code.google.com/p/googletest +Download and build the GoogleTest framework from http://code.google.com/p/googletest -Download gflags from https://code.google.com/p/gflags +Download and install gflags from https://code.google.com/p/gflags export GTEST_ROOT=/path/for/gtest-x.y diff --git a/btree.h b/btree.h index a8d3152d..49310a2e 100644 --- a/btree.h +++ b/btree.h @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // // A btree implementation of the STL set and map interfaces. A btree is both // smaller and faster than STL set/map. The red-black tree implementation of @@ -112,12 +122,12 @@ namespace btree { // Inside a btree method, if we just call swap(), it will choose the -// btree::swap method, which we don't want. And we can't say ::swap because -// then MSVC won't pickup any std::swap() implementations. We can't just use -// std::swap() directly because then we don't get the specialization for string -// and Cord (which are not defined in the std namespace!). So the solution is -// to have a special swap helper function whose name doesn't collide with other -// swap functions defined by the btree classes. +// btree::swap method, which we don't want. And we can't say ::swap +// because then MSVC won't pickup any std::swap() implementations. We +// can't just use std::swap() directly because then we don't get the +// specialization for types outside the std namespace. So the solution +// is to have a special swap helper function whose name doesn't +// collide with other swap functions defined by the btree classes. template inline void btree_swap_helper(T &a, T &b) { using std::swap; @@ -172,14 +182,14 @@ struct btree_is_key_compare_to : public std::is_convertible { }; -// A helper class to convert a boolean comparison into a three-way "compare-to" -// comparison that returns a negative value to indicate less-than, zero to -// indicate equality and a positive value to indicate greater-than. This helper -// class is specialized for less, greater, less, -// greater, less and greater. The -// btree_key_compare_to_adapter class is provided so that btree users -// automatically get the more efficient compare-to code when using common -// google string types with common comparison functors. +// A helper class to convert a boolean comparison into a three-way +// "compare-to" comparison that returns a negative value to indicate +// less-than, zero to indicate equality and a positive value to +// indicate greater-than. This helper class is specialized for +// less and greater. The btree_key_compare_to_adapter +// class is provided so that btree users automatically get the more +// efficient compare-to code when using common google string types +// with common comparison functors. template struct btree_key_compare_to_adapter : Compare { btree_key_compare_to_adapter() { } diff --git a/btree_bench.cc b/btree_bench.cc index b803ad38..d5495c5d 100644 --- a/btree_bench.cc +++ b/btree_bench.cc @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include #include diff --git a/btree_container.h b/btree_container.h index 510c9ae6..c9c62c33 100644 --- a/btree_container.h +++ b/btree_container.h @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #ifndef UTIL_BTREE_BTREE_CONTAINER_H__ #define UTIL_BTREE_BTREE_CONTAINER_H__ diff --git a/btree_map.h b/btree_map.h index 6e965590..07b799ea 100644 --- a/btree_map.h +++ b/btree_map.h @@ -1,6 +1,16 @@ -// Copyright 2007, 2012 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // // A btree_map<> implements the STL unique sorted associative container // interface and the pair associative container interface (a.k.a map<>) using a diff --git a/btree_set.h b/btree_set.h index 0b51e494..2bc9e586 100644 --- a/btree_set.h +++ b/btree_set.h @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // // A btree_set<> implements the STL unique sorted associative container // interface (a.k.a set<>) using a btree. A btree_multiset<> implements the STL diff --git a/btree_test.cc b/btree_test.cc index 7b9cafa5..56a787b9 100644 --- a/btree_test.cc +++ b/btree_test.cc @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "gtest/gtest.h" #include "btree_map.h" diff --git a/btree_test.h b/btree_test.h index 01ab0fe0..0ac26eff 100644 --- a/btree_test.h +++ b/btree_test.h @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #ifndef UTIL_BTREE_BTREE_TEST_H__ #define UTIL_BTREE_BTREE_TEST_H__ diff --git a/btree_test_flags.cc b/btree_test_flags.cc index d43d9250..bf608a9b 100644 --- a/btree_test_flags.cc +++ b/btree_test_flags.cc @@ -1,5 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. #include "gflags/gflags.h" diff --git a/safe_btree.h b/safe_btree.h index 01cf9faf..2d85c70b 100644 --- a/safe_btree.h +++ b/safe_btree.h @@ -1,5 +1,16 @@ -// Copyright 2007, 2012 Google Inc. All Rights Reserved. -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // // A safe_btree<> wraps around a btree<> and removes the caveat that insertion // and deletion invalidate iterators. A safe_btree<> maintains a generation diff --git a/safe_btree_map.h b/safe_btree_map.h index b8aa787e..a0668f16 100644 --- a/safe_btree_map.h +++ b/safe_btree_map.h @@ -1,5 +1,16 @@ -// Copyright 2007, 2012 Google Inc. All Rights Reserved. -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // // The safe_btree_map<> is like btree_map<> except that it removes the caveat // about insertion and deletion invalidating existing iterators at a small cost diff --git a/safe_btree_set.h b/safe_btree_set.h index de8f823a..a6cd5418 100644 --- a/safe_btree_set.h +++ b/safe_btree_set.h @@ -1,5 +1,16 @@ -// Copyright 2007, 2012 Google Inc. All Rights Reserved. -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // // The safe_btree_set<> is like btree_set<> except that it removes the caveat // about insertion and deletion invalidating existing iterators at a small cost diff --git a/safe_btree_test.cc b/safe_btree_test.cc index 4965916b..0d77ae06 100644 --- a/safe_btree_test.cc +++ b/safe_btree_test.cc @@ -1,6 +1,16 @@ -// Copyright 2007 Google Inc. All Rights Reserved. -// Author: jmacd@google.com (Josh MacDonald) -// Author: pmattis@google.com (Peter Mattis) +// Copyright 2013 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // TODO(pmattis): Add some tests that iterators are not invalidated by // insertion and deletion. From c790956d1826bbd0bb11506ae95840a95d96b7a8 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Tue, 5 Feb 2013 22:04:33 -0800 Subject: [PATCH 12/74] Fix several range constructor bugs: (a) btree_multiset's range constructor did not compile (b) btree_map's range constructor ignored the inputs! These bugs were masked by the test, which due to an oversight skipped the use of range constructors as part of the {base,unique,multi}_checker testing types. Verified that the corrected test fails without the fix, added an additional sanity check (which made the problem stand out independent of the problems discovered in btree_test.h) --- btree_container.h | 5 ++--- btree_map.h | 6 +++--- btree_set.h | 4 ++-- btree_test.cc | 19 +++++++++++++++++++ btree_test.h | 16 ++++++++++++---- 5 files changed, 38 insertions(+), 12 deletions(-) diff --git a/btree_container.h b/btree_container.h index c9c62c33..fb617abe 100644 --- a/btree_container.h +++ b/btree_container.h @@ -261,8 +261,7 @@ class btree_map_container : public btree_unique_container { btree_map_container(InputIterator b, InputIterator e, const key_compare &comp = key_compare(), const allocator_type &alloc = allocator_type()) - : super_type(comp, alloc) { - insert(b, e); + : super_type(b, e, comp, alloc) { } // Insertion routines. @@ -303,7 +302,7 @@ class btree_multi_container : public btree_container { btree_multi_container(InputIterator b, InputIterator e, const key_compare &comp = key_compare(), const allocator_type &alloc = allocator_type()) - : super_type(b, e, comp, alloc) { + : super_type(comp, alloc) { insert(b, e); } diff --git a/btree_map.h b/btree_map.h index 07b799ea..b83489f0 100644 --- a/btree_map.h +++ b/btree_map.h @@ -33,7 +33,7 @@ namespace btree { -// The btree_map class is needed mainly for it's constructors. +// The btree_map class is needed mainly for its constructors. template , typename Alloc = std::allocator >, @@ -68,7 +68,7 @@ class btree_map : public btree_map_container< btree_map(InputIterator b, InputIterator e, const key_compare &comp = key_compare(), const allocator_type &alloc = allocator_type()) - : super_type(comp, alloc) { + : super_type(b, e, comp, alloc) { } }; @@ -78,7 +78,7 @@ inline void swap(btree_map &x, x.swap(y); } -// The btree_multimap class is needed mainly for it's constructors. +// The btree_multimap class is needed mainly for its constructors. template , typename Alloc = std::allocator >, diff --git a/btree_set.h b/btree_set.h index 2bc9e586..f9b2e75d 100644 --- a/btree_set.h +++ b/btree_set.h @@ -29,7 +29,7 @@ namespace btree { -// The btree_set class is needed mainly for it's constructors. +// The btree_set class is needed mainly for its constructors. template , typename Alloc = std::allocator, @@ -72,7 +72,7 @@ inline void swap(btree_set &x, btree_set &y) { x.swap(y); } -// The btree_multiset class is needed mainly for it's constructors. +// The btree_multiset class is needed mainly for its constructors. template , typename Alloc = std::allocator, diff --git a/btree_test.cc b/btree_test.cc index 56a787b9..52bd219a 100644 --- a/btree_test.cc +++ b/btree_test.cc @@ -244,5 +244,24 @@ TEST(Btree, Comparison) { EXPECT_TRUE(my_map != my_map_copy); } +TEST(Btree, RangeCtorSanity) { + typedef btree_set, allocator, 256> test_set; + typedef btree_map, allocator, 256> test_map; + typedef btree_multiset, allocator, 256> test_mset; + typedef btree_multimap, allocator, 256> test_mmap; + vector ivec; + ivec.push_back(1); + map imap; + imap.insert(make_pair(1, 2)); + test_mset tmset(ivec.begin(), ivec.end()); + test_mmap tmmap(imap.begin(), imap.end()); + test_set tset(ivec.begin(), ivec.end()); + test_map tmap(imap.begin(), imap.end()); + EXPECT_EQ(1, tmset.size()); + EXPECT_EQ(1, tmmap.size()); + EXPECT_EQ(1, tset.size()); + EXPECT_EQ(1, tmap.size()); +} + } // namespace } // namespace btree diff --git a/btree_test.h b/btree_test.h index 0ac26eff..413dc3c7 100644 --- a/btree_test.h +++ b/btree_test.h @@ -147,6 +147,13 @@ class base_checker { const_tree_(tree_), checker_(x.checker_) { } + // Range constructor. + template + base_checker(InputIterator b, InputIterator e) + : tree_(b, e), + const_tree_(tree_), + checker_(b, e) { + } // Iterator routines. iterator begin() { return tree_.begin(); } @@ -398,8 +405,8 @@ class unique_checker : public base_checker { } // Range constructor. template - unique_checker(InputIterator b, InputIterator e) { - insert(b, e); + unique_checker(InputIterator b, InputIterator e) + : super_type(b, e) { } // Insertion routines. @@ -455,8 +462,8 @@ class multi_checker : public base_checker { } // Range constructor. template - multi_checker(InputIterator b, InputIterator e) { - insert(b, e); + multi_checker(InputIterator b, InputIterator e) + : super_type(b, e) { } // Insertion routines. @@ -587,6 +594,7 @@ void DoTest(const char *name, T *b, const std::vector &values) { mutable_b.insert(values[i]); mutable_b.value_check(values[i]); } + assert(mutable_b.size() == values.size()); const_b.verify(); printf(" %s fullness=%0.2f overhead=%0.2f bytes-per-value=%0.2f\n", From f85129d4d77a447984241dbc1babd31b1eb1dd4e Mon Sep 17 00:00:00 2001 From: josh macdonald Date: Tue, 5 Feb 2013 22:12:13 -0800 Subject: [PATCH 13/74] Fixes for open-source build following upstream bug fix --- btree_test.cc | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/btree_test.cc b/btree_test.cc index 52bd219a..6b1837d3 100644 --- a/btree_test.cc +++ b/btree_test.cc @@ -245,14 +245,17 @@ TEST(Btree, Comparison) { } TEST(Btree, RangeCtorSanity) { - typedef btree_set, allocator, 256> test_set; - typedef btree_map, allocator, 256> test_map; - typedef btree_multiset, allocator, 256> test_mset; - typedef btree_multimap, allocator, 256> test_mmap; - vector ivec; + typedef btree_set, std::allocator, 256> test_set; + typedef btree_map, std::allocator, 256> + test_map; + typedef btree_multiset, std::allocator, 256> + test_mset; + typedef btree_multimap, std::allocator, 256> + test_mmap; + std::vector ivec; ivec.push_back(1); - map imap; - imap.insert(make_pair(1, 2)); + std::map imap; + imap.insert(std::make_pair(1, 2)); test_mset tmset(ivec.begin(), ivec.end()); test_mmap tmmap(imap.begin(), imap.end()); test_set tset(ivec.begin(), ivec.end()); From 54f346f9816f665bf45a96f6eb0fb09b74ccffef Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Fri, 8 Feb 2013 22:56:25 -0800 Subject: [PATCH 14/74] Update README / CMakeLists.txt; fix for -Wformat warning in btree_bench.cc --- CMakeLists.txt | 2 ++ README | 13 ++++++++++++- btree_bench.cc | 4 ++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c49ad27..d005e158 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,8 @@ if(build_tests) enable_testing() include_directories($ENV{GTEST_ROOT}/include) link_directories($ENV{GTEST_ROOT}) + include_directories($ENV{GFLAGS_ROOT}/include) + link_directories($ENV{GFLAGS_ROOT}/lib) add_executable(btree_test btree_test.cc btree_test_flags.cc) add_executable(safe_btree_test safe_btree_test.cc btree_test_flags.cc) add_executable(btree_bench btree_bench.cc btree_test_flags.cc) diff --git a/README b/README index 0e8674cf..319fe9bb 100644 --- a/README +++ b/README @@ -11,10 +11,21 @@ CMake, the Google C++ Test framework, and the Google flags package. Download and install CMake from http://www.cmake.org -Download and build the GoogleTest framework from http://code.google.com/p/googletest +Download and build the GoogleTest framework from +http://code.google.com/p/googletest Download and install gflags from https://code.google.com/p/gflags +Set GTEST_ROOT to the directory where GTEST was built. +Set GFLAGS_ROOT to the directory prefix where GFLAGS is installed. + export GTEST_ROOT=/path/for/gtest-x.y +export GFLAGS_ROOT=/opt cmake . -Dbuild_tests=ON + +For example, to build on a Unix system with the clang++ compiler, + +export GTEST_ROOT=$(HOME)/src/googletest +export GFLAGS_ROOT=/opt +cmake . -G "Unix Makefiles" -Dbuild_tests=ON -DCMAKE_CXX_COMPILER=clang++ diff --git a/btree_bench.cc b/btree_bench.cc index d5495c5d..900c5e63 100644 --- a/btree_bench.cc +++ b/btree_bench.cc @@ -133,7 +133,7 @@ void BenchmarkRun::Reset() { void BenchmarkRun::Run() { assert(current_benchmark == NULL); current_benchmark = this; - int iters = FLAGS_benchmark_min_iters; + int64_t iters = FLAGS_benchmark_min_iters; for (;;) { Reset(); Start(); @@ -152,7 +152,7 @@ void BenchmarkRun::Run() { } fprintf(stdout, "%s\t%qu\t%qu\n", benchmark_name, - accum_micros * 1000 / iters, + accum_micros * 1000ULL / iters, iters); current_benchmark = NULL; } From 92ec61e4b8bf182c5c49ebf6540dac62d569d090 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Fri, 8 Feb 2013 23:14:47 -0800 Subject: [PATCH 15/74] Fix the printf problem -- use std::cout :) --- btree_bench.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/btree_bench.cc b/btree_bench.cc index 900c5e63..6eaed994 100644 --- a/btree_bench.cc +++ b/btree_bench.cc @@ -133,7 +133,7 @@ void BenchmarkRun::Reset() { void BenchmarkRun::Run() { assert(current_benchmark == NULL); current_benchmark = this; - int64_t iters = FLAGS_benchmark_min_iters; + int iters = FLAGS_benchmark_min_iters; for (;;) { Reset(); Start(); @@ -150,10 +150,9 @@ void BenchmarkRun::Run() { } iters = min(iters, FLAGS_benchmark_max_iters); } - fprintf(stdout, "%s\t%qu\t%qu\n", - benchmark_name, - accum_micros * 1000ULL / iters, - iters); + std::cout << benchmark_name << "\t" + << accum_micros * 1000 / iters << "\t" + << iters; current_benchmark = NULL; } From fe5e2167a51ad0f9dba0261843d058ab2e12e369 Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Tue, 3 Sep 2013 14:45:11 -0700 Subject: [PATCH 16/74] Initial commit --- .gitignore | 1 + Makefile | 2 + README.md | 42 ++++ json11.cpp | 674 +++++++++++++++++++++++++++++++++++++++++++++++++++++ json11.hpp | 187 +++++++++++++++ test.cpp | 100 ++++++++ 6 files changed, 1006 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 README.md create mode 100644 json11.cpp create mode 100644 json11.hpp create mode 100644 test.cpp diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..9daeafb9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +test diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..f251c1fb --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +test: json11.cpp json11.hpp test.cpp + clang++ -O -std=c++11 -stdlib=libc++ json11.cpp test.cpp -o test -fno-rtti -fno-exceptions diff --git a/README.md b/README.md new file mode 100644 index 00000000..21f4e8b2 --- /dev/null +++ b/README.md @@ -0,0 +1,42 @@ +json11 +------ + +json11 is a tiny JSON library for C++11, providing JSON parsing and serialization. + +The core object provided by the library is json11::Json. A Json object represents any JSON +value: null, bool, number (int or double), string (std::string), array (std::vector), or +object (std::map). + +Json objects act like values. They can be assigned, copied, moved, compared for equality or +order, and so on. There are also helper methods Json::dump, to serialize a Json to a string, and +Json::parse (static) to parse a std::string as a Json object. + +It's easy to make a JSON object with C++11's new initializer syntax: + + Json my_json = Json::object { + { "key1", "value1" }, + { "key2", false }, + { "key3", Json::array { 1, 2, 3 } }, + }; + std::string json_str = my_json.dump(); + +There are also implicit constructors that allow standard and user-defined types to be +automatically converted to JSON. For example: + + class Point { + public: + int x; + int y; + Point (int x, int y) : x(x), y(y) {} + Json to_json() const { return Json::array { x, y }; } + }; + + std::vector points = { { 1, 2 }, { 10, 20 }, { 100, 200 } }; + std::string points_json = Json(points).dump(); + +JSON values can have their values queried and inspected: + + Json json = Json::array { Json::object { { "k", "v" } } }; + std::string str = json[0]["k"].string_value(); + +More documentation is still to come. For now, see json11.hpp. diff --git a/json11.cpp b/json11.cpp new file mode 100644 index 00000000..dc882071 --- /dev/null +++ b/json11.cpp @@ -0,0 +1,674 @@ +/* Copyright (c) 2013 Dropbox, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "json11.hpp" +#include +#include +#include +#include + +namespace json11 { + +using std::string; +using std::vector; +using std::map; +using std::make_shared; +using std::initializer_list; +using std::move; + +/* * * * * * * * * * * * * * * * * * * * + * Serialization + */ + +static void dump(std::nullptr_t, string &out) { + out += "null"; +} + +static void dump(double value, string &out) { + char buf[32]; + snprintf(buf, sizeof buf, "%.17g", value); + out += buf; +} + +static void dump(int value, string &out) { + char buf[32]; + snprintf(buf, sizeof buf, "%d", value); + out += buf; +} + +static void dump(bool value, string &out) { + out += value ? "true" : "false"; +} + +static void dump(const string &value, string &out) { + out += '"'; + for (size_t i = 0; i < value.length(); i++) { + const char ch = value[i]; + if (ch == '\\') { + out += "\\\\"; + } else if (ch == '"') { + out += "\\\""; + } else if (ch == '\b') { + out += "\\b"; + } else if (ch == '\f') { + out += "\\f"; + } else if (ch == '\n') { + out += "\\n"; + } else if (ch == '\r') { + out += "\\r"; + } else if (ch == '\t') { + out += "\\t"; + } else if ((uint8_t)ch <= 0x1f) { + char buf[8]; + snprintf(buf, sizeof buf, "\\u%04x", ch); + out += buf; + } else if ((uint8_t)ch == 0xe2 && (uint8_t)value[i+1] == 0x80 + && (uint8_t)value[i+2] == 0xa8) { + out += "\\u2028"; + i += 2; + } else if ((uint8_t)ch == 0xe2 && (uint8_t)value[i+1] == 0x80 + && (uint8_t)value[i+2] == 0xa9) { + out += "\\u2029"; + i += 2; + } else { + out += ch; + } + } + out += '"'; +} + +static void dump(const Json::array &values, string &out) { + bool first = true; + out += "["; + for (auto &value : values) { + if (!first) + out += ", "; + value.dump(out); + first = false; + } + out += "]"; +} + +static void dump(const Json::object &values, string &out) { + bool first = true; + out += "{"; + for (const std::pair &kv : values) { + if (!first) + out += ", "; + dump(kv.first, out); + out += ": "; + kv.second.dump(out); + first = false; + } + out += "}"; +} + +void Json::dump(string &out) const { + m_ptr->dump(out); +} + +/* * * * * * * * * * * * * * * * * * * * + * Value wrappers + */ + +template +class Value : public JsonValue { +protected: + + // Constructors + Value(const T &value) : m_value(value) {} + Value(T &&value) : m_value(move(value)) {} + + // Get type tag + Json::Type type() const { + return tag; + } + + // Comparisons + bool equals(const JsonValue * other) const { + return m_value == reinterpret_cast *>(other)->m_value; + } + bool less(const JsonValue * other) const { + return m_value < reinterpret_cast *>(other)->m_value; + } + + const T m_value; + void dump(string &out) const { json11::dump(m_value, out); } +}; + +class JsonDouble final : public Value { + double number_value() const { return m_value; } + int int_value() const { return m_value; } + bool equals(const JsonValue * other) const { return m_value == other->number_value(); } + bool less(const JsonValue * other) const { return m_value < other->number_value(); } +public: + JsonDouble(double value) : Value(value) {} +}; + +class JsonInt final : public Value { + double number_value() const { return m_value; } + int int_value() const { return m_value; } + bool equals(const JsonValue * other) const { return m_value == other->number_value(); } + bool less(const JsonValue * other) const { return m_value < other->number_value(); } +public: + JsonInt(double value) : Value(value) {} +}; + +class JsonBoolean final : public Value { + bool bool_value() const { return m_value; } +public: + JsonBoolean(bool value) : Value(value) {} +}; + +class JsonString final : public Value { + const string &string_value() const { return m_value; } +public: + JsonString(const string &value) : Value(value) {} + JsonString(string &&value) : Value(move(value)) {} +}; + +class JsonArray final : public Value { + const Json::array &array_items() const { return m_value; } + const Json & operator[](size_t i) const; +public: + JsonArray(const Json::array &value) : Value(value) {} + JsonArray(Json::array &&value) : Value(move(value)) {} +}; + +class JsonObject final : public Value { + const Json::object &object_items() const { return m_value; } + const Json & operator[](const string &key) const; +public: + JsonObject(const Json::object &value) : Value(value) {} + JsonObject(Json::object &&value) : Value(move(value)) {} +}; + +class JsonNull final : public Value { +public: + JsonNull() : Value(nullptr) {} +}; + +/* * * * * * * * * * * * * * * * * * * * + * Constructors + */ + +static const std::shared_ptr obj_null(make_shared()); +static const std::shared_ptr obj_true(make_shared(true)); +static const std::shared_ptr obj_false(make_shared(false)); + +Json::Json() noexcept : m_ptr(obj_null) {} +Json::Json(std::nullptr_t) noexcept : m_ptr(obj_null) {} +Json::Json(double value) : m_ptr(make_shared(value)) {} +Json::Json(int value) : m_ptr(make_shared(value)) {} +Json::Json(bool value) : m_ptr(value ? obj_true : obj_false) {} +Json::Json(const string &value) : m_ptr(make_shared(value)) {} +Json::Json(string &&value) : m_ptr(make_shared(move(value))) {} +Json::Json(const char * value) : m_ptr(make_shared(value)) {} +Json::Json(const Json::array &values) : m_ptr(make_shared(values)) {} +Json::Json(Json::array &&values) : m_ptr(make_shared(move(values))) {} +Json::Json(const Json::object &values) : m_ptr(make_shared(values)) {} +Json::Json(Json::object &&values) : m_ptr(make_shared(move(values))) {} + +/* * * * * * * * * * * * * * * * * * * * + * Accessors + */ + +static const string empty_string; +static const vector empty_vector; +static const map empty_map; +static const Json json_null; + +Json::Type Json::type() const { return m_ptr->type(); } +double Json::number_value() const { return m_ptr->number_value(); } +int Json::int_value() const { return m_ptr->int_value(); } +bool Json::bool_value() const { return m_ptr->bool_value(); } +const string & Json::string_value() const { return m_ptr->string_value(); } +const vector & Json::array_items() const { return m_ptr->array_items(); } +const map & Json::object_items() const { return m_ptr->object_items(); } +const Json & Json::operator[] (size_t i) const { return (*m_ptr)[i]; } +const Json & Json::operator[] (const string &key) const { return (*m_ptr)[key]; } + +double JsonValue::number_value() const { return 0; } +int JsonValue::int_value() const { return 0; } +bool JsonValue::bool_value() const { return false; } +const string & JsonValue::string_value() const { return empty_string; } +const vector & JsonValue::array_items() const { return empty_vector; } +const map & JsonValue::object_items() const { return empty_map; } +const Json & JsonValue::operator[] (size_t) const { return json_null; } +const Json & JsonValue::operator[] (const string &) const { return json_null; } + +const Json & JsonObject::operator[] (const string &key) const { + auto iter = m_value.find(key); + return (iter == m_value.end()) ? json_null : iter->second; +} +const Json & JsonArray::operator[] (size_t i) const { + if (i >= m_value.size()) return json_null; + else return m_value[i]; +} + +/* * * * * * * * * * * * * * * * * * * * + * Comparison + */ + +bool Json::operator== (const Json &other) const { + if (m_ptr->type() != other.m_ptr->type()) + return false; + + return m_ptr->equals(other.m_ptr.get()); +} + +bool Json::operator< (const Json &other) const { + if (m_ptr->type() != other.m_ptr->type()) + return m_ptr->type() < other.m_ptr->type(); + + return m_ptr->less(other.m_ptr.get()); +} + +/* * * * * * * * * * * * * * * * * * * * + * Parsing + */ + +/* esc(c) + * + * Format char c suitable for printing in an error message. + */ +static inline string esc(char c) { + char buf[12]; + if ((uint8_t)c >= 0x20 && (uint8_t)c <= 0x7f) { + snprintf(buf, sizeof buf, "'%c' (%d)", c, c); + } else { + snprintf(buf, sizeof buf, "(%d)", c); + } + return string(buf); +} + +static inline bool in_range (int x, int lower, int upper) { + return (x >= lower && x <= upper); +} + +/* JsonParser + * + * Object that tracks all state of an in-progress parse. + */ +struct JsonParser { + + /* State + */ + const string &str; + size_t i; + string &err; + bool failed; + + /* fail(msg, err_ret = Json()) + * + * Mark this parse as failed. + */ + Json fail(string &&msg) { + return fail(move(msg), Json()); + } + + template + T fail(string &&msg, const T err_ret) { + if (!failed) + err = std::move(msg); + failed = true; + return err_ret; + } + + /* consume_whitespace() + * + * Advance until the current character is non-whitespace. + */ + void consume_whitespace() { + while (str[i] == ' ' || str[i] == '\r' || str[i] == '\n' || str[i] == '\t') + i++; + } + + /* get_next_token() + * + * Return the next non-whitespace character. If the end of the input is reached, + * flag an error and return 0. + */ + char get_next_token() { + consume_whitespace(); + if (i == str.size()) + return fail("unexpected end of input", 0); + + return str[i++]; + } + + /* encode_utf8(pt, out) + * + * Encode pt as UTF-8 and add it to out. + */ + void encode_utf8(long pt, string & out) { + if (pt < 0) + return; + + if (pt < 0x80) { + out += pt; + } else if (pt < 0x800) { + out += (pt >> 6) | 0xC0; + out += (pt & 0x3F) | 0x80; + } else if (pt < 0x10000) { + out += (pt >> 12) | 0xE0; + out += ((pt >> 6) & 0x3F) | 0x80; + out += (pt & 0x3F) | 0x80; + } else { + out += (pt >> 18) | 0xF0; + out += ((pt >> 12) & 0x3F) | 0x80; + out += ((pt >> 6) & 0x3F) | 0x80; + out += (pt & 0x3F) | 0x80; + } + } + + /* parse_string() + * + * Parse a string, starting at the current position. + */ + string parse_string() { + string out; + long last_escaped_codepoint = -1; + while (true) { + if (i == str.size()) + return fail("unexpected end of input in string", ""); + + char ch = str[i++]; + + if (ch == '"') { + encode_utf8(last_escaped_codepoint, out); + return out; + } + + if (in_range(ch, 0, 0x1f)) + return fail("unescaped " + esc(ch) + " in string", ""); + + // The usual case: non-escaped characters + if (ch != '\\') { + encode_utf8(last_escaped_codepoint, out); + last_escaped_codepoint = -1; + out += ch; + continue; + } + + // Handle escapes + if (i == str.size()) + return fail("unexpected end of input in string", ""); + + ch = str[i++]; + + if (ch == 'u') { + // Extract 4-byte escape sequence + string esc = str.substr(i, 4); + for (int j = 0; j < 4; j++) { + if (!in_range(esc[j], 'a', 'f') && !in_range(esc[j], 'A', 'F') + && !in_range(esc[j], '0', '9')) + return fail("bad \\u escape: " + esc, ""); + } + + long codepoint = strtol(esc.data(), nullptr, 16); + + // JSON specifies that characters outside the BMP shall be encoded as a pair + // of 4-hex-digit \u escapes encoding their surrogate pair components. Check + // whether we're in the middle of such a beast: the previous codepoint was an + // escaped lead (high) surrogate, and this is a trail (low) surrogate. + if (in_range(last_escaped_codepoint, 0xD800, 0xDBFF) + && in_range(codepoint, 0xDC00, 0xDFFF)) { + // Reassemble the two surrogate pairs into one astral-plane character, per + // the UTF-16 algorithm. + encode_utf8((((last_escaped_codepoint - 0xD800) << 10) + | (codepoint - 0xDC00)) + 0x10000, out); + last_escaped_codepoint = -1; + } else { + encode_utf8(last_escaped_codepoint, out); + last_escaped_codepoint = codepoint; + } + + i += 4; + continue; + } + + encode_utf8(last_escaped_codepoint, out); + last_escaped_codepoint = -1; + + if (ch == 'b') { + out += '\b'; + } else if (ch == 'f') { + out += '\f'; + } else if (ch == 'n') { + out += '\n'; + } else if (ch == 'r') { + out += '\r'; + } else if (ch == 't') { + out += '\t'; + } else if (ch == '"' || ch == '\\' || ch == '/') { + out += ch; + } else { + return fail("invalid escape character " + esc(ch), ""); + } + } + } + + /* parse_number() + * + * Parse a double. + */ + Json parse_number() { + size_t start_pos = i; + + if (str[i] == '-') + i++; + + // Integer part + if (str[i] == '0') { + i++; + if (in_range(str[i], '0', '9')) + return fail("leading 0s not permitted in numbers"); + } else if (in_range(str[i], '1', '9')) { + i++; + while (in_range(str[i], '0', '9')) + i++; + } else { + return fail("invalid " + esc(str[i]) + " in number"); + } + + if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' + && (i - start_pos) <= (size_t)std::numeric_limits::digits10) { + return std::atoi(str.c_str() + start_pos); + } + + // Decimal part + if (str[i] == '.') { + i++; + if (!in_range(str[i], '0', '9')) + return fail("at least one digit required in fractional part"); + + while (in_range(str[i], '0', '9')) + i++; + } + + // Exponent part + if (str[i] == 'e' || str[i] == 'E') { + i++; + + if (str[i] == '+' || str[i] == '-') + i++; + + if (!in_range(str[i], '0', '9')) + return fail("at least one digit required in exponent"); + + while (in_range(str[i], '0', '9')) + i++; + } + + return std::atof(str.c_str() + start_pos); + } + + /* expect(str, res) + * + * Expect that 'str' starts at the character that was just read. If it does, advance + * the input and return res. If not, flag an error. + */ + Json expect(const string &expected, Json res) { + assert(i != 0); + i--; + const string found = str.substr(i, expected.length()); + if (expected == found) { + i += expected.length(); + return res; + } else { + return fail("parse error: expected " + expected + ", got " + found); + } + } + + /* parse_json() + * + * Parse a JSON object. + */ + Json parse_json() { + char ch = get_next_token(); + if (failed) + return Json(); + + if (ch == '-' || (ch >= '0' && ch <= '9')) { + i--; + return parse_number(); + } + + if (ch == 't') + return expect("true", true); + + if (ch == 'f') + return expect("false", false); + + if (ch == 'n') + return expect("null", Json()); + + if (ch == '"') + return parse_string(); + + if (ch == '{') { + map data; + ch = get_next_token(); + if (ch == '}') + return data; + + while (1) { + if (ch != '"') + return fail("expected '\"' in object, got " + esc(ch)); + + string key = parse_string(); + if (failed) + return Json(); + + ch = get_next_token(); + if (ch != ':') + return fail("expected ':' in object, got " + esc(ch)); + + data[std::move(key)] = parse_json(); + if (failed) + return Json(); + + ch = get_next_token(); + if (ch == '}') + break; + if (ch != ',') + return fail("expected ',' in object, got " + esc(ch)); + + ch = get_next_token(); + } + return data; + } + + if (ch == '[') { + vector data; + ch = get_next_token(); + if (ch == ']') + return data; + + while (1) { + i--; + data.push_back(parse_json()); + if (failed) + return Json(); + + ch = get_next_token(); + if (ch == ']') + break; + if (ch != ',') + return fail("expected ',' in list, got " + esc(ch)); + + ch = get_next_token(); + (void)ch; + } + return data; + } + + return fail("expected value, got " + esc(ch)); + } +}; + +Json Json::parse(const string &in, string &err) { + JsonParser parser { in, 0, err, false }; + Json result = parser.parse_json(); + + // Check for any trailing garbage + parser.consume_whitespace(); + if (parser.i != in.size()) + return parser.fail("unexpected trailing " + esc(in[parser.i])); + + return result; +} + +// Documented in json11.hpp +vector Json::parse_multi(const string &in, string &err) { + JsonParser parser { in, 0, err, false }; + + vector json_vec; + while (parser.i != in.size() && !parser.failed) { + json_vec.push_back(parser.parse_json()); + // Check for another object + parser.consume_whitespace(); + } + return json_vec; +} + +/* * * * * * * * * * * * * * * * * * * * + * Shape-checking + */ + +bool Json::has_shape(const shape & types, string & err) const { + if (!is_object()) { + err = "expected JSON object, got " + dump(); + return false; + } + + for (auto & item : types) { + if ((*this)[item.first].type() != item.second) { + err = "bad type for " + item.first + " in " + dump(); + return false; + } + } + + return true; +} + +} // namespace json11 diff --git a/json11.hpp b/json11.hpp new file mode 100644 index 00000000..a2fa5c7b --- /dev/null +++ b/json11.hpp @@ -0,0 +1,187 @@ +/* json11 + * + * json11 is a tiny JSON library for C++11, providing JSON parsing and serialization. + * + * The core object provided by the library is json11::Json. A Json object represents any JSON + * value: null, bool, number (int or double), string (std::string), array (std::vector), or + * object (std::map). + * + * Json objects act like values: they can be assigned, copied, moved, compared for equality or + * order, etc. There are also helper methods Json::dump, to serialize a Json to a string, and + * Json::parse (static) to parse a std::string as a Json object. + * + * Internally, the various types of Json object are represented by the JsonValue class + * hierarchy. + */ + +/* Copyright (c) 2013 Dropbox, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#include +#include +#include +#include +#include + +namespace json11 { + +class JsonValue; + +class Json final { +public: + // Types + enum Type { + NUL, NUMBER, BOOL, STRING, ARRAY, OBJECT + }; + + // Array and object typedefs + typedef std::vector array; + typedef std::map object; + + // Constructors for the various types of JSON value. + Json() noexcept; // NUL + Json(std::nullptr_t) noexcept; // NUL + Json(double value); // NUMBER + Json(int value); // NUMBER + Json(bool value); // BOOL + Json(const std::string &value); // STRING + Json(std::string &&value); // STRING + Json(const char * value); // STRING + Json(const array &values); // ARRAY + Json(array &&values); // ARRAY + Json(const object &values); // OBJECT + Json(object &&values); // OBJECT + + // Implicit constructor: anything with a to_json() function. + template + Json(const T & t) : Json(t.to_json()) {} + + // Implicit constructor: map-like objects (std::map, std::unordered_map, etc) + template ().begin()->first)>::value + && std::is_constructible().begin()->second)>::value, + int>::type = 0> + Json(const M & m) : Json(object(m.begin(), m.end())) {} + + // Implicit constructor: vector-like objects (std::list, std::vector, std::set, etc) + template ().begin())>::value, + int>::type = 0> + Json(const V & v) : Json(array(v.begin(), v.end())) {} + + // This prevents Json(some_pointer) from accidentally producing a bool. Use + // Json(bool(some_pointer)) if that behavior is desired. + Json(void *) = delete; + + // Accessors + Type type() const; + + bool is_null() const { return type() == NUL; } + bool is_number() const { return type() == NUMBER; } + bool is_bool() const { return type() == BOOL; } + bool is_string() const { return type() == STRING; } + bool is_array() const { return type() == ARRAY; } + bool is_object() const { return type() == OBJECT; } + + // Return the enclosed value if this is a number, 0 otherwise. Note that json11 does not + // distinguish between integer and non-integer numbers - number_value() and int_value() + // can both be applied to a NUMBER-typed object. + double number_value() const; + int int_value() const; + + // Return the enclosed value if this is a boolean, false otherwise. + bool bool_value() const; + // Return the enclosed string if this is a string, "" otherwise. + const std::string &string_value() const; + // Return the enclosed std::vector if this is an array, or an empty vector otherwise. + const array &array_items() const; + // Return the enclosed std::map if this is an object, or an empty map otherwise. + const object &object_items() const; + + // Return a reference to arr[i] if this is an object, Json() otherwise. + const Json & operator[](size_t i) const; + // Return a reference to obj[key] if this is an object, Json() otherwise. + const Json & operator[](const std::string &key) const; + + // Serialize. + void dump(std::string &out) const; + std::string dump() const { + std::string out; + dump(out); + return out; + } + + // Parse. If parse fails, return Json() and assign an error message to err. + static Json parse(const std::string & in, std::string & err); + static Json parse(const char * in, std::string & err) { + if (in) { + return parse(std::string(in), err); + } else { + err = "null input"; + return nullptr; + } + } + // Parse multiple objects, concatenated or separated by whitespace + static std::vector parse_multi(const std::string & in, std::string & err); + + bool operator== (const Json &rhs) const; + bool operator< (const Json &rhs) const; + bool operator!= (const Json &rhs) { return !(*this == rhs); } + bool operator<= (const Json &rhs) { return !(rhs < *this); } + bool operator> (const Json &rhs) { return (rhs < *this); } + bool operator>= (const Json &rhs) { return !(*this < rhs); } + + /* has_shape(types, err) + * + * Return true if this is a JSON object and, for each item in types, has a field of + * the given type. If not, return false and set err to a descriptive message. + */ + typedef std::initializer_list> shape; + bool has_shape(const shape & types, std::string & err) const; + +private: + std::shared_ptr m_ptr; +}; + +// Internal class hierarchy - JsonValue objects are not exposed to users of this API. +class JsonValue { +protected: + friend class Json; + friend class JsonInt; + friend class JsonDouble; + virtual Json::Type type() const = 0; + virtual bool equals(const JsonValue * other) const = 0; + virtual bool less(const JsonValue * other) const = 0; + virtual void dump(std::string &out) const = 0; + virtual double number_value() const; + virtual int int_value() const; + virtual bool bool_value() const; + virtual const std::string &string_value() const; + virtual const Json::array &array_items() const; + virtual const Json &operator[](size_t i) const; + virtual const Json::object &object_items() const; + virtual const Json &operator[](const std::string &key) const; + virtual ~JsonValue() {} +}; + +} // namespace json11 diff --git a/test.cpp b/test.cpp new file mode 100644 index 00000000..f34c4a00 --- /dev/null +++ b/test.cpp @@ -0,0 +1,100 @@ +#include +#include +#include +#include +#include "json11.hpp" +#include + +using namespace json11; +using std::string; + +// Check that Json has the properties we want. +#include +#define CHECK_TRAIT(x) static_assert(std::x::value, #x) +CHECK_TRAIT(is_nothrow_constructible); +CHECK_TRAIT(is_nothrow_default_constructible); +CHECK_TRAIT(is_copy_constructible); +CHECK_TRAIT(is_nothrow_move_constructible); +CHECK_TRAIT(is_copy_assignable); +CHECK_TRAIT(is_nothrow_move_assignable); +CHECK_TRAIT(is_nothrow_destructible); + +void parse_from_stdin() { + string buf; + while (!std::cin.eof()) buf += std::cin.get(); + + string err; + auto json = Json::parse(buf, err); + if (!err.empty()) { + printf("Failed: %s\n", err.c_str()); + } else { + printf("Result: %s\n", json.dump().c_str()); + } +} + +int main(int argc, char **argv) { + if (argc == 2 && argv[1] == string("--stdin")) { + parse_from_stdin(); + return 0; + } + + const string simple_test = + R"({"k1":"v1", "k2":42, "k3":["a",123,true,false,null]})"; + + string err; + auto json = Json::parse(simple_test, err); + + std::cout << "k1: " << json["k1"].string_value() << "\n"; + std::cout << "k3: " << json["k3"].dump() << "\n"; + + for (auto &k : json["k3"].array_items()) { + std::cout << " - " << k.dump() << "\n"; + } + + // Json literals + Json obj = Json::object({ + { "k1", "v1" }, + { "k2", 42.0 }, + { "k3", Json::array({ "a", 123.0, true, false, nullptr }) }, + }); + + std::cout << "obj: " << obj.dump() << "\n"; + + assert(Json("a").number_value() == 0); + assert(Json("a").string_value() == "a"); + assert(Json().number_value() == 0); + + assert(obj == json); + assert(Json(42) == Json(42.0)); + assert(Json(42) != Json(42.1)); + + const string unicode_escape_test = + R"([ "blah\ud83d\udca9blah\ud83dblah\udca9blah\u0000blah\u1234" ])"; + + const char utf8[] = "blah" "\xf0\x9f\x92\xa9" "blah" "\xed\xa0\xbd" "blah" + "\xed\xb2\xa9" "blah" "\0" "blah" "\xe1\x88\xb4"; + + Json uni = Json::parse(unicode_escape_test, err); + assert(uni[0].string_value().size() == (sizeof utf8) - 1); + assert(memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); + + Json my_json = Json::object { + { "key1", "value1" }, + { "key2", false }, + { "key3", Json::array { 1, 2, 3 } }, + }; + std::string json_str = my_json.dump(); + printf("%s\n", json_str.c_str()); + + class Point { + public: + int x; + int y; + Point (int x, int y) : x(x), y(y) {} + Json to_json() const { return Json::array { x, y }; } + }; + + std::vector points = { { 1, 2 }, { 10, 20 }, { 100, 200 } }; + std::string points_json = Json(points).dump(); + printf("%s\n", points_json.c_str()); +} From 979cb4ca767718d69ed593bbf836dc40c883636e Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Tue, 1 Oct 2013 19:04:32 -0700 Subject: [PATCH 17/74] Add LICENSE.txt; some minor cleanups - Add LICENSE.txt - Add a comment about how we handle numbers - Add more test code - Fix a narrowing-conversion issue - Fix a comment typo --- LICENSE.txt | 19 +++++++++++++++++++ json11.cpp | 2 +- json11.hpp | 15 ++++++++++++++- test.cpp | 13 +++++++++++++ 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 LICENSE.txt diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..691742e9 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2013 Dropbox, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/json11.cpp b/json11.cpp index dc882071..9666db84 100644 --- a/json11.cpp +++ b/json11.cpp @@ -300,7 +300,7 @@ static inline string esc(char c) { return string(buf); } -static inline bool in_range (int x, int lower, int upper) { +static inline bool in_range(long x, long lower, long upper) { return (x >= lower && x <= upper); } diff --git a/json11.hpp b/json11.hpp index a2fa5c7b..af65cdbc 100644 --- a/json11.hpp +++ b/json11.hpp @@ -12,6 +12,19 @@ * * Internally, the various types of Json object are represented by the JsonValue class * hierarchy. + * + * A note on numbers - JSON specifies the syntax of number formatting but not its semantics, + * so some JSON implementations distinguish between integers and floating-point numbers, while + * some don't. In json11, we choose the latter. Because some JSON implementations (namely + * Javascript itself) treat all numbers as the same type, distinguishing the two leads + * to JSON that will be *silently* changed by a round-trip through those implementations. + * Dangerous! To avoid that risk, json11 stores all numbers as double internally, but also + * provides integer helpers. + * + * Fortunately, double-precision IEEE754 ('double') can precisely store any integer in the + * range +/-2^53, which includes every 'int' on most systems. (Timestamps often use int64 + * or long long to avoid the Y2038K problem; a double storing microseconds since some epoch + * will be exact for +/- 275 years.) */ /* Copyright (c) 2013 Dropbox, Inc. @@ -118,7 +131,7 @@ public: // Return the enclosed std::map if this is an object, or an empty map otherwise. const object &object_items() const; - // Return a reference to arr[i] if this is an object, Json() otherwise. + // Return a reference to arr[i] if this is an array, Json() otherwise. const Json & operator[](size_t i) const; // Return a reference to obj[key] if this is an object, Json() otherwise. const Json & operator[](const std::string &key) const; diff --git a/test.cpp b/test.cpp index f34c4a00..3bfe2e0c 100644 --- a/test.cpp +++ b/test.cpp @@ -4,6 +4,9 @@ #include #include "json11.hpp" #include +#include +#include +#include using namespace json11; using std::string; @@ -51,6 +54,16 @@ int main(int argc, char **argv) { std::cout << " - " << k.dump() << "\n"; } + std::list l1 { 1, 2, 3 }; + std::vector l2 { 1, 2, 3 }; + std::set l3 { 1, 2, 3 }; + assert(Json(l1) == Json(l2)); + assert(Json(l2) == Json(l3)); + + std::map m1 { { "k1", "v1" }, { "k2", "v2" } }; + std::unordered_map m2 { { "k1", "v1" }, { "k2", "v2" } }; + assert(Json(m1) == Json(m2)); + // Json literals Json obj = Json::object({ { "k1", "v1" }, From d2e54859733e27dd94fcb900f075cc3bfb62729b Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Mon, 27 Jan 2014 10:59:17 -0800 Subject: [PATCH 18/74] Spacing fix --- json11.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json11.cpp b/json11.cpp index 9666db84..d2bb65e7 100644 --- a/json11.cpp +++ b/json11.cpp @@ -238,7 +238,7 @@ static const Json json_null; Json::Type Json::type() const { return m_ptr->type(); } double Json::number_value() const { return m_ptr->number_value(); } -int Json::int_value() const { return m_ptr->int_value(); } +int Json::int_value() const { return m_ptr->int_value(); } bool Json::bool_value() const { return m_ptr->bool_value(); } const string & Json::string_value() const { return m_ptr->string_value(); } const vector & Json::array_items() const { return m_ptr->array_items(); } From 277e1b166fd98b4ad98e0d360eda443cf9f248db Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Wed, 29 Jan 2014 13:30:18 -0800 Subject: [PATCH 19/74] Make Json safe to use during static initialization. --- json11.cpp | 52 +++++++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/json11.cpp b/json11.cpp index d2bb65e7..dd132980 100644 --- a/json11.cpp +++ b/json11.cpp @@ -206,19 +206,38 @@ public: JsonNull() : Value(nullptr) {} }; +/* * * * * * * * * * * * * * * * * * * * + * Static globals - static-init-safe + */ +struct Statics { + const std::shared_ptr null = make_shared(); + const std::shared_ptr t = make_shared(true); + const std::shared_ptr f = make_shared(false); + const string empty_string; + const vector empty_vector; + const map empty_map; +}; + +const Statics & statics() { + static const Statics s {}; + return s; +} + +const Json & static_null() { + // This has to be separate, not in Statics, because Json() accesses statics().null. + static const Json json_null; + return json_null; +} + /* * * * * * * * * * * * * * * * * * * * * Constructors */ -static const std::shared_ptr obj_null(make_shared()); -static const std::shared_ptr obj_true(make_shared(true)); -static const std::shared_ptr obj_false(make_shared(false)); - -Json::Json() noexcept : m_ptr(obj_null) {} -Json::Json(std::nullptr_t) noexcept : m_ptr(obj_null) {} +Json::Json() noexcept : m_ptr(statics().null) {} +Json::Json(std::nullptr_t) noexcept : m_ptr(statics().null) {} Json::Json(double value) : m_ptr(make_shared(value)) {} Json::Json(int value) : m_ptr(make_shared(value)) {} -Json::Json(bool value) : m_ptr(value ? obj_true : obj_false) {} +Json::Json(bool value) : m_ptr(value ? statics().t : statics().f) {} Json::Json(const string &value) : m_ptr(make_shared(value)) {} Json::Json(string &&value) : m_ptr(make_shared(move(value))) {} Json::Json(const char * value) : m_ptr(make_shared(value)) {} @@ -231,11 +250,6 @@ Json::Json(Json::object &&values) : m_ptr(make_shared(move(valu * Accessors */ -static const string empty_string; -static const vector empty_vector; -static const map empty_map; -static const Json json_null; - Json::Type Json::type() const { return m_ptr->type(); } double Json::number_value() const { return m_ptr->number_value(); } int Json::int_value() const { return m_ptr->int_value(); } @@ -249,18 +263,18 @@ const Json & Json::operator[] (const string &key) const { return (*m_ptr)[key]; double JsonValue::number_value() const { return 0; } int JsonValue::int_value() const { return 0; } bool JsonValue::bool_value() const { return false; } -const string & JsonValue::string_value() const { return empty_string; } -const vector & JsonValue::array_items() const { return empty_vector; } -const map & JsonValue::object_items() const { return empty_map; } -const Json & JsonValue::operator[] (size_t) const { return json_null; } -const Json & JsonValue::operator[] (const string &) const { return json_null; } +const string & JsonValue::string_value() const { return statics().empty_string; } +const vector & JsonValue::array_items() const { return statics().empty_vector; } +const map & JsonValue::object_items() const { return statics().empty_map; } +const Json & JsonValue::operator[] (size_t) const { return static_null(); } +const Json & JsonValue::operator[] (const string &) const { return static_null(); } const Json & JsonObject::operator[] (const string &key) const { auto iter = m_value.find(key); - return (iter == m_value.end()) ? json_null : iter->second; + return (iter == m_value.end()) ? static_null() : iter->second; } const Json & JsonArray::operator[] (size_t i) const { - if (i >= m_value.size()) return json_null; + if (i >= m_value.size()) return static_null(); else return m_value[i]; } From 731322c2b89a5c6d2aa5684d0b809e481e63313b Mon Sep 17 00:00:00 2001 From: Steven Kabbes Date: Thu, 27 Mar 2014 03:55:52 -0700 Subject: [PATCH 20/74] Add const to comparisons --- json11.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/json11.hpp b/json11.hpp index af65cdbc..779445e5 100644 --- a/json11.hpp +++ b/json11.hpp @@ -159,10 +159,10 @@ public: bool operator== (const Json &rhs) const; bool operator< (const Json &rhs) const; - bool operator!= (const Json &rhs) { return !(*this == rhs); } - bool operator<= (const Json &rhs) { return !(rhs < *this); } - bool operator> (const Json &rhs) { return (rhs < *this); } - bool operator>= (const Json &rhs) { return !(*this < rhs); } + bool operator!= (const Json &rhs) const { return !(*this == rhs); } + bool operator<= (const Json &rhs) const { return !(rhs < *this); } + bool operator> (const Json &rhs) const { return (rhs < *this); } + bool operator>= (const Json &rhs) const { return !(*this < rhs); } /* has_shape(types, err) * From d9bdd84c762b3e5ca24d40685ade299d1a836a3b Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Wed, 4 Jun 2014 12:22:06 -0700 Subject: [PATCH 21/74] Limit maximum nesting depth (issue reported by Jeff Larson) --- json11.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/json11.cpp b/json11.cpp index dd132980..ad7f5b17 100644 --- a/json11.cpp +++ b/json11.cpp @@ -27,6 +27,8 @@ namespace json11 { +static const int max_depth = 200; + using std::string; using std::vector; using std::map; @@ -557,7 +559,11 @@ struct JsonParser { * * Parse a JSON object. */ - Json parse_json() { + Json parse_json(int depth) { + if (depth > max_depth) { + return fail("exceeded maximum nesting depth"); + } + char ch = get_next_token(); if (failed) return Json(); @@ -597,7 +603,7 @@ struct JsonParser { if (ch != ':') return fail("expected ':' in object, got " + esc(ch)); - data[std::move(key)] = parse_json(); + data[std::move(key)] = parse_json(depth + 1); if (failed) return Json(); @@ -620,7 +626,7 @@ struct JsonParser { while (1) { i--; - data.push_back(parse_json()); + data.push_back(parse_json(depth + 1)); if (failed) return Json(); @@ -642,7 +648,7 @@ struct JsonParser { Json Json::parse(const string &in, string &err) { JsonParser parser { in, 0, err, false }; - Json result = parser.parse_json(); + Json result = parser.parse_json(0); // Check for any trailing garbage parser.consume_whitespace(); @@ -658,7 +664,7 @@ vector Json::parse_multi(const string &in, string &err) { vector json_vec; while (parser.i != in.size() && !parser.failed) { - json_vec.push_back(parser.parse_json()); + json_vec.push_back(parser.parse_json(0)); // Check for another object parser.consume_whitespace(); } From 710352234426a869b5ebd4f2e5ff8c71ccd6bc07 Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Wed, 13 Aug 2014 14:58:20 -0700 Subject: [PATCH 22/74] Identify map- and vector-like objects in a VS14-compatible way --- json11.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/json11.hpp b/json11.hpp index 779445e5..fe9bba40 100644 --- a/json11.hpp +++ b/json11.hpp @@ -91,14 +91,14 @@ public: // Implicit constructor: map-like objects (std::map, std::unordered_map, etc) template ().begin()->first)>::value - && std::is_constructible().begin()->second)>::value, + std::is_constructible::value + && std::is_constructible::value, int>::type = 0> Json(const M & m) : Json(object(m.begin(), m.end())) {} // Implicit constructor: vector-like objects (std::list, std::vector, std::set, etc) template ().begin())>::value, + std::is_constructible::value, int>::type = 0> Json(const V & v) : Json(array(v.begin(), v.end())) {} From 679e4b83d2bb3595524c26fc97f8e3645ae8ab08 Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Thu, 14 Aug 2014 12:14:21 -0700 Subject: [PATCH 23/74] Another VS14 fix --- json11.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/json11.cpp b/json11.cpp index ad7f5b17..a25d0feb 100644 --- a/json11.cpp +++ b/json11.cpp @@ -218,6 +218,7 @@ struct Statics { const string empty_string; const vector empty_vector; const map empty_map; + Statics() {} }; const Statics & statics() { From 73baf7e6772ca01705fe92b0519e5a936845e9a4 Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Mon, 15 Sep 2014 11:33:11 -0700 Subject: [PATCH 24/74] Style fixes --- json11.cpp | 72 +++++++++++++++++++++++++++--------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/json11.cpp b/json11.cpp index a25d0feb..1ef01db8 100644 --- a/json11.cpp +++ b/json11.cpp @@ -78,16 +78,16 @@ static void dump(const string &value, string &out) { out += "\\r"; } else if (ch == '\t') { out += "\\t"; - } else if ((uint8_t)ch <= 0x1f) { + } else if (static_cast(ch) <= 0x1f) { char buf[8]; snprintf(buf, sizeof buf, "\\u%04x", ch); out += buf; - } else if ((uint8_t)ch == 0xe2 && (uint8_t)value[i+1] == 0x80 - && (uint8_t)value[i+2] == 0xa8) { + } else if (static_cast(ch) == 0xe2 && static_cast(value[i+1]) == 0x80 + && static_cast(value[i+2]) == 0xa8) { out += "\\u2028"; i += 2; - } else if ((uint8_t)ch == 0xe2 && (uint8_t)value[i+1] == 0x80 - && (uint8_t)value[i+2] == 0xa9) { + } else if (static_cast(ch) == 0xe2 && static_cast(value[i+1]) == 0x80 + && static_cast(value[i+2]) == 0xa9) { out += "\\u2029"; i += 2; } else { @@ -136,71 +136,71 @@ class Value : public JsonValue { protected: // Constructors - Value(const T &value) : m_value(value) {} - Value(T &&value) : m_value(move(value)) {} + explicit Value(const T &value) : m_value(value) {} + explicit Value(T &&value) : m_value(move(value)) {} // Get type tag - Json::Type type() const { + Json::Type type() const override { return tag; } // Comparisons - bool equals(const JsonValue * other) const { + bool equals(const JsonValue * other) const override { return m_value == reinterpret_cast *>(other)->m_value; } - bool less(const JsonValue * other) const { + bool less(const JsonValue * other) const override { return m_value < reinterpret_cast *>(other)->m_value; } const T m_value; - void dump(string &out) const { json11::dump(m_value, out); } + void dump(string &out) const override { json11::dump(m_value, out); } }; class JsonDouble final : public Value { - double number_value() const { return m_value; } - int int_value() const { return m_value; } - bool equals(const JsonValue * other) const { return m_value == other->number_value(); } - bool less(const JsonValue * other) const { return m_value < other->number_value(); } + double number_value() const override { return m_value; } + int int_value() const override { return m_value; } + bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } + bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: - JsonDouble(double value) : Value(value) {} + explicit JsonDouble(double value) : Value(value) {} }; class JsonInt final : public Value { - double number_value() const { return m_value; } - int int_value() const { return m_value; } - bool equals(const JsonValue * other) const { return m_value == other->number_value(); } - bool less(const JsonValue * other) const { return m_value < other->number_value(); } + double number_value() const override { return m_value; } + int int_value() const override { return m_value; } + bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } + bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: - JsonInt(double value) : Value(value) {} + explicit JsonInt(double value) : Value(value) {} }; class JsonBoolean final : public Value { - bool bool_value() const { return m_value; } + bool bool_value() const override { return m_value; } public: - JsonBoolean(bool value) : Value(value) {} + explicit JsonBoolean(bool value) : Value(value) {} }; class JsonString final : public Value { - const string &string_value() const { return m_value; } + const string &string_value() const override { return m_value; } public: - JsonString(const string &value) : Value(value) {} - JsonString(string &&value) : Value(move(value)) {} + explicit JsonString(const string &value) : Value(value) {} + explicit JsonString(string &&value) : Value(move(value)) {} }; class JsonArray final : public Value { - const Json::array &array_items() const { return m_value; } - const Json & operator[](size_t i) const; + const Json::array &array_items() const override { return m_value; } + const Json & operator[](size_t i) const override; public: - JsonArray(const Json::array &value) : Value(value) {} - JsonArray(Json::array &&value) : Value(move(value)) {} + explicit JsonArray(const Json::array &value) : Value(value) {} + explicit JsonArray(Json::array &&value) : Value(move(value)) {} }; class JsonObject final : public Value { - const Json::object &object_items() const { return m_value; } - const Json & operator[](const string &key) const; + const Json::object &object_items() const override { return m_value; } + const Json & operator[](const string &key) const override; public: - JsonObject(const Json::object &value) : Value(value) {} - JsonObject(Json::object &&value) : Value(move(value)) {} + explicit JsonObject(const Json::object &value) : Value(value) {} + explicit JsonObject(Json::object &&value) : Value(move(value)) {} }; class JsonNull final : public Value { @@ -309,7 +309,7 @@ bool Json::operator< (const Json &other) const { */ static inline string esc(char c) { char buf[12]; - if ((uint8_t)c >= 0x20 && (uint8_t)c <= 0x7f) { + if (static_cast(c) >= 0x20 && static_cast(c) <= 0x7f) { snprintf(buf, sizeof buf, "'%c' (%d)", c, c); } else { snprintf(buf, sizeof buf, "(%d)", c); @@ -508,7 +508,7 @@ struct JsonParser { } if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' - && (i - start_pos) <= (size_t)std::numeric_limits::digits10) { + && (i - start_pos) <= static_cast(std::numeric_limits::digits10)) { return std::atoi(str.c_str() + start_pos); } From 6de4c29e763613e818d347006d46a638b039b41f Mon Sep 17 00:00:00 2001 From: Jacob Potter Date: Mon, 22 Sep 2014 11:59:58 -0700 Subject: [PATCH 25/74] fix int->double->int conversion in JsonInt --- json11.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index 1ef01db8..64245542 100644 --- a/json11.cpp +++ b/json11.cpp @@ -158,7 +158,7 @@ protected: class JsonDouble final : public Value { double number_value() const override { return m_value; } - int int_value() const override { return m_value; } + int int_value() const override { return static_cast(m_value); } bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: @@ -171,7 +171,7 @@ class JsonInt final : public Value { bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: - explicit JsonInt(double value) : Value(value) {} + explicit JsonInt(int value) : Value(value) {} }; class JsonBoolean final : public Value { From c6a873617153984d904eb0c83fd226ae10bfec86 Mon Sep 17 00:00:00 2001 From: Masamitsu MURASE Date: Sun, 30 Nov 2014 03:36:23 +0900 Subject: [PATCH 26/74] Use string::compare to improve performance. --- json11.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/json11.cpp b/json11.cpp index 64245542..88e376be 100644 --- a/json11.cpp +++ b/json11.cpp @@ -547,12 +547,11 @@ struct JsonParser { Json expect(const string &expected, Json res) { assert(i != 0); i--; - const string found = str.substr(i, expected.length()); - if (expected == found) { + if (str.compare(i, expected.length(), expected) == 0) { i += expected.length(); return res; } else { - return fail("parse error: expected " + expected + ", got " + found); + return fail("parse error: expected " + expected + ", got " + str.substr(i, expected.length())); } } From 37ca641d82f9f80dc3ae6dbd6557a479866537fa Mon Sep 17 00:00:00 2001 From: Masamitsu MURASE Date: Sun, 30 Nov 2014 03:49:23 +0900 Subject: [PATCH 27/74] Use static_cast instead of reinterpret_cast for hierarchy navigation. --- json11.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index 64245542..9803846f 100644 --- a/json11.cpp +++ b/json11.cpp @@ -146,10 +146,10 @@ protected: // Comparisons bool equals(const JsonValue * other) const override { - return m_value == reinterpret_cast *>(other)->m_value; + return m_value == static_cast *>(other)->m_value; } bool less(const JsonValue * other) const override { - return m_value < reinterpret_cast *>(other)->m_value; + return m_value < static_cast *>(other)->m_value; } const T m_value; From 7fd738ae42fa03cca84eaba6b460f9e407268943 Mon Sep 17 00:00:00 2001 From: k0zmo Date: Tue, 30 Dec 2014 10:28:51 +0100 Subject: [PATCH 28/74] iterate over pair of const string and Json --- json11.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index c9ac2a55..3d28c400 100644 --- a/json11.cpp +++ b/json11.cpp @@ -100,7 +100,7 @@ static void dump(const string &value, string &out) { static void dump(const Json::array &values, string &out) { bool first = true; out += "["; - for (auto &value : values) { + for (const auto &value : values) { if (!first) out += ", "; value.dump(out); @@ -112,7 +112,7 @@ static void dump(const Json::array &values, string &out) { static void dump(const Json::object &values, string &out) { bool first = true; out += "{"; - for (const std::pair &kv : values) { + for (const auto &kv : values) { if (!first) out += ", "; dump(kv.first, out); From ae9542cc35cedd69e7d74056bf2c0963eff7a470 Mon Sep 17 00:00:00 2001 From: Steve Carroll Date: Tue, 3 Mar 2015 17:47:56 -0800 Subject: [PATCH 29/74] in encode_utf8, there is an implicit truncating cast that VS2015CTP6 is warning about. I've added static_cast to silence the warning and express the intent. --- json11.cpp | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/json11.cpp b/json11.cpp index 3d28c400..2dc955ab 100644 --- a/json11.cpp +++ b/json11.cpp @@ -381,19 +381,19 @@ struct JsonParser { return; if (pt < 0x80) { - out += pt; + out += static_cast(pt); } else if (pt < 0x800) { - out += (pt >> 6) | 0xC0; - out += (pt & 0x3F) | 0x80; + out += static_cast((pt >> 6) | 0xC0); + out += static_cast((pt & 0x3F) | 0x80); } else if (pt < 0x10000) { - out += (pt >> 12) | 0xE0; - out += ((pt >> 6) & 0x3F) | 0x80; - out += (pt & 0x3F) | 0x80; + out += static_cast((pt >> 12) | 0xE0); + out += static_cast(((pt >> 6) & 0x3F) | 0x80); + out += static_cast((pt & 0x3F) | 0x80); } else { - out += (pt >> 18) | 0xF0; - out += ((pt >> 12) & 0x3F) | 0x80; - out += ((pt >> 6) & 0x3F) | 0x80; - out += (pt & 0x3F) | 0x80; + out += static_cast((pt >> 18) | 0xF0); + out += static_cast(((pt >> 12) & 0x3F) | 0x80); + out += static_cast(((pt >> 6) & 0x3F) | 0x80); + out += static_cast((pt & 0x3F) | 0x80); } } From cfdd67577c0694f26b7609599a9e087d0ab31643 Mon Sep 17 00:00:00 2001 From: Steven Kabbes Date: Tue, 18 Nov 2014 15:57:04 +0100 Subject: [PATCH 30/74] Use strtod not atof --- json11.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json11.cpp b/json11.cpp index 2dc955ab..f0f85a47 100644 --- a/json11.cpp +++ b/json11.cpp @@ -536,7 +536,7 @@ struct JsonParser { i++; } - return std::atof(str.c_str() + start_pos); + return std::strtod(str.c_str() + start_pos, nullptr); } /* expect(str, res) From e15ff418dd1b824addc0c31c779a041f07003406 Mon Sep 17 00:00:00 2001 From: Andrew Krieger Date: Wed, 22 Apr 2015 14:00:47 -0700 Subject: [PATCH 31/74] For extra safety, add an explicit bounds check in utf8 parsing. --- json11.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/json11.cpp b/json11.cpp index f0f85a47..34f90983 100644 --- a/json11.cpp +++ b/json11.cpp @@ -435,6 +435,12 @@ struct JsonParser { if (ch == 'u') { // Extract 4-byte escape sequence string esc = str.substr(i, 4); + // Explicitly check length of the substring. The following loop + // relies on std::string returning the terminating NUL when + // accessing str[length]. Checking here reduces brittleness. + if (esc.length() < 4) { + return fail("bad \\u escape: " + esc, ""); + } for (int j = 0; j < 4; j++) { if (!in_range(esc[j], 'a', 'f') && !in_range(esc[j], 'A', 'F') && !in_range(esc[j], '0', '9')) From 0e8c5ba68f66b0056cc42373b4cd892b9667171d Mon Sep 17 00:00:00 2001 From: Jean-Claude Monnin Date: Thu, 4 Jun 2015 10:35:12 +0200 Subject: [PATCH 32/74] When dumping non-finite floating point values, output 'null'. In JSON the float point special values NaN and Infinity should serialised to 'null'. Previously, 'snprintf' in 'dump' was giving a string that isn't compliant to the JSON standard. --- json11.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/json11.cpp b/json11.cpp index 34f90983..0aa125b4 100644 --- a/json11.cpp +++ b/json11.cpp @@ -21,6 +21,7 @@ #include "json11.hpp" #include +#include #include #include #include @@ -45,9 +46,13 @@ static void dump(std::nullptr_t, string &out) { } static void dump(double value, string &out) { - char buf[32]; - snprintf(buf, sizeof buf, "%.17g", value); - out += buf; + if (std::isfinite(value)) { + char buf[32]; + snprintf(buf, sizeof buf, "%.17g", value); + out += buf; + } else { + out += "null"; + } } static void dump(int value, string &out) { From 50ed8c90e518338cc5d0b76fea054ca5b2351b64 Mon Sep 17 00:00:00 2001 From: Andrew Twyman Date: Wed, 17 Jun 2015 15:02:22 -0700 Subject: [PATCH 33/74] Test to demonstrate compiler behavior change in Xcode 7 / clang 3.7. --- test.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test.cpp b/test.cpp index 3bfe2e0c..f3c3301a 100644 --- a/test.cpp +++ b/test.cpp @@ -91,6 +91,14 @@ int main(int argc, char **argv) { assert(uni[0].string_value().size() == (sizeof utf8) - 1); assert(memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); + // Demonstrates the behavior change in Xcode 7 / Clang 3.7 described + // here: https://llvm.org/bugs/show_bug.cgi?id=23812 + Json nested_array = Json::array { Json::array { 1, 2, 3 } }; + assert(nested_array.is_array()); + assert(nested_array.array_items().size() == 1); + assert(nested_array.array_items()[0].is_array()); + assert(nested_array.array_items()[0].array_items().size() == 3); + Json my_json = Json::object { { "key1", "value1" }, { "key2", false }, From 61ba0a1dd278534b447a1c4c7cedf2f4e0dbebcf Mon Sep 17 00:00:00 2001 From: Andrew Twyman Date: Wed, 17 Jun 2015 18:02:34 -0700 Subject: [PATCH 34/74] Make test.cpp compilable with GCC. Add clean rule to Makefile. --- Makefile | 7 ++++++- test.cpp | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f251c1fb..5a6adc6e 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,7 @@ test: json11.cpp json11.hpp test.cpp - clang++ -O -std=c++11 -stdlib=libc++ json11.cpp test.cpp -o test -fno-rtti -fno-exceptions + $(CXX) -O -std=c++11 json11.cpp test.cpp -o test -fno-rtti -fno-exceptions + +clean: + if [ -e test ]; then rm test; fi + +.PHONY: clean diff --git a/test.cpp b/test.cpp index f3c3301a..f6fb5473 100644 --- a/test.cpp +++ b/test.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include "json11.hpp" @@ -89,7 +90,7 @@ int main(int argc, char **argv) { Json uni = Json::parse(unicode_escape_test, err); assert(uni[0].string_value().size() == (sizeof utf8) - 1); - assert(memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); + assert(std::memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); // Demonstrates the behavior change in Xcode 7 / Clang 3.7 described // here: https://llvm.org/bugs/show_bug.cgi?id=23812 From 0c6e9d77a41a23d8486bb83d8428f1c5035bd9dd Mon Sep 17 00:00:00 2001 From: Paul C Roberts Date: Wed, 19 Aug 2015 14:53:28 -0700 Subject: [PATCH 35/74] Add cmake support This uses cmake to allow building as a static library and to build the tests To use `cmake` ``` $ mkdir build $ cd build $ cmake .. . . -- Build files have been written to: ~/json11/build $ make Scanning dependencies of target json11 [ 20%] Building CXX object CMakeFiles/json11.dir/json11.cpp.o [ 40%] Linking CXX static library libjson11.a [ 40%] Built target json11 Scanning dependencies of target json11_test [ 60%] Building CXX object CMakeFiles/json11_test.dir/json11.cpp.o [ 80%] Building CXX object CMakeFiles/json11_test.dir/test.cpp.o [100%] Linking CXX executable json11_test [100%] Built target json11_test $ make test running tests... Test project /Users/paul/dev/json11/build Start 1: json11_test 1/1 Test #1: json11_test ...................... Passed 0.00 sec 100% tests passed, 0 tests failed out of 1 Total Test time (real) = 0.01 sec $ ``` --- CMakeLists.txt | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..ba8edd5b --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,21 @@ +project(json11) + +cmake_minimum_required(VERSION 2.8) + +enable_testing() + +add_definitions( + -std=c++11 + -fno-rtti + -fno-exceptions + -Wall + -Wextra + -Werror) + +set(json11_SRCS json11.cpp) + +add_library(json11 STATIC ${json11_SRCS}) + +add_test(json11_test json11_test) + +add_executable(json11_test ${json11_SRCS} test.cpp) From a021c3fbe2542c840346bf75380dcdf1cc6b712f Mon Sep 17 00:00:00 2001 From: Paul C Roberts Date: Wed, 19 Aug 2015 15:00:12 -0700 Subject: [PATCH 36/74] Fixes bug in parse_from_stdin Handles end of file correctly --- test.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test.cpp b/test.cpp index f6fb5473..8b62aeb5 100644 --- a/test.cpp +++ b/test.cpp @@ -25,7 +25,10 @@ CHECK_TRAIT(is_nothrow_destructible); void parse_from_stdin() { string buf; - while (!std::cin.eof()) buf += std::cin.get(); + string line; + while (std::getline(std::cin, line)) { + buf += line + "\n"; + } string err; auto json = Json::parse(buf, err); From 2d1d176a5599fedc4d424cd5a784b002f24ff04d Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Fri, 27 Nov 2015 16:31:43 +0100 Subject: [PATCH 37/74] add routine to detect c-style comments --- json11.cpp | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/json11.cpp b/json11.cpp index 0aa125b4..e2085a1c 100644 --- a/json11.cpp +++ b/json11.cpp @@ -364,6 +364,33 @@ struct JsonParser { i++; } + /* consume_comment() + * + * Advance comments (c-style inline and multiline). + */ + void consume_comment() { + if (str[i] == '/') { + i++; + if (str[i] == '/') { // inline comment + i++; + // advance until next line + while (str[i] != '\n') + i++; + consume_whitespace(); + consume_comment(); + } + else if (str[i] == '*') { // multiline comment + i++; + // advance until closing tokens + while (!(str[i] == '*' && str[i+1] == '/')) + i++; + i += 2; + consume_whitespace(); + consume_comment(); + } + } + } + /* get_next_token() * * Return the next non-whitespace character. If the end of the input is reached, From 08c391f89ad253eafb21fcad6b430a8c4df70c15 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Fri, 27 Nov 2015 16:41:05 +0100 Subject: [PATCH 38/74] introduce consume_garbage() new routine that consumes whitespaces and comments. activated by JSON11_COMMENTS pre-processor flag. --- json11.cpp | 18 +++++++++++++++--- json11.hpp | 2 ++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/json11.cpp b/json11.cpp index e2085a1c..4c473c1e 100644 --- a/json11.cpp +++ b/json11.cpp @@ -391,13 +391,25 @@ struct JsonParser { } } + /* consume_garbage() + * + * Advance until the current character is non-whitespace and non-comment. + */ + void consume_garbage() { + consume_whitespace(); +#ifdef JSON11_COMMENTS + consume_comment(); + consume_whitespace(); +#endif + } + /* get_next_token() * * Return the next non-whitespace character. If the end of the input is reached, * flag an error and return 0. */ char get_next_token() { - consume_whitespace(); + consume_garbage(); if (i == str.size()) return fail("unexpected end of input", 0); @@ -689,7 +701,7 @@ Json Json::parse(const string &in, string &err) { Json result = parser.parse_json(0); // Check for any trailing garbage - parser.consume_whitespace(); + parser.consume_garbage(); if (parser.i != in.size()) return parser.fail("unexpected trailing " + esc(in[parser.i])); @@ -704,7 +716,7 @@ vector Json::parse_multi(const string &in, string &err) { while (parser.i != in.size() && !parser.failed) { json_vec.push_back(parser.parse_json(0)); // Check for another object - parser.consume_whitespace(); + parser.consume_garbage(); } return json_vec; } diff --git a/json11.hpp b/json11.hpp index fe9bba40..ff399d1a 100644 --- a/json11.hpp +++ b/json11.hpp @@ -56,6 +56,8 @@ #include #include +#define JSON11_COMMENTS 1 + namespace json11 { class JsonValue; From de098c4d52e466049f85c2a378ef04b92cd3433a Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Fri, 27 Nov 2015 16:46:18 +0100 Subject: [PATCH 39/74] add testing for comment functionality --- test.cpp | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test.cpp b/test.cpp index 8b62aeb5..7a40857b 100644 --- a/test.cpp +++ b/test.cpp @@ -58,6 +58,28 @@ int main(int argc, char **argv) { std::cout << " - " << k.dump() << "\n"; } +#ifdef JSON11_COMMENTS + const string comment_test = R"({ + // comment + "a": 1, + // comment + // continued + "b": "text", + /* multi + line + comment */ + "c": [1, 2, 3] + })"; + + string err_comment; + auto json_comment = Json::parse(comment_test, err_comment); + if (!err_comment.empty()) { + printf("Failed: %s\n", err_comment.c_str()); + } else { + printf("Result: %s\n", json_comment.dump().c_str()); + } +#endif + std::list l1 { 1, 2, 3 }; std::vector l2 { 1, 2, 3 }; std::set l3 { 1, 2, 3 }; From 882feb56ac7e46c703be5a063775ba9da3decbab Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Mon, 30 Nov 2015 12:27:35 +0100 Subject: [PATCH 40/74] add bool to detect comments as run-time option. --- json11.cpp | 19 +++++++++++-------- json11.hpp | 16 ++++++++++------ test.cpp | 5 ++--- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/json11.cpp b/json11.cpp index 4c473c1e..4d239c02 100644 --- a/json11.cpp +++ b/json11.cpp @@ -338,6 +338,7 @@ struct JsonParser { size_t i; string &err; bool failed; + bool detect_comments; /* fail(msg, err_ret = Json()) * @@ -397,10 +398,10 @@ struct JsonParser { */ void consume_garbage() { consume_whitespace(); -#ifdef JSON11_COMMENTS - consume_comment(); - consume_whitespace(); -#endif + if(detect_comments) { + consume_comment(); + consume_whitespace(); + } } /* get_next_token() @@ -696,8 +697,8 @@ struct JsonParser { } }; -Json Json::parse(const string &in, string &err) { - JsonParser parser { in, 0, err, false }; +Json Json::parse(const string &in, string &err, bool detect_comments) { + JsonParser parser { in, 0, err, false, detect_comments }; Json result = parser.parse_json(0); // Check for any trailing garbage @@ -709,8 +710,10 @@ Json Json::parse(const string &in, string &err) { } // Documented in json11.hpp -vector Json::parse_multi(const string &in, string &err) { - JsonParser parser { in, 0, err, false }; +vector Json::parse_multi(const string &in, + string &err, + bool detect_comments) { + JsonParser parser { in, 0, err, false, detect_comments }; vector json_vec; while (parser.i != in.size() && !parser.failed) { diff --git a/json11.hpp b/json11.hpp index ff399d1a..582c43f0 100644 --- a/json11.hpp +++ b/json11.hpp @@ -56,8 +56,6 @@ #include #include -#define JSON11_COMMENTS 1 - namespace json11 { class JsonValue; @@ -147,17 +145,23 @@ public: } // Parse. If parse fails, return Json() and assign an error message to err. - static Json parse(const std::string & in, std::string & err); - static Json parse(const char * in, std::string & err) { + static Json parse(const std::string & in, + std::string & err, + bool detect_comments = false); + static Json parse(const char * in, + std::string & err, + bool detect_comments = false) { if (in) { - return parse(std::string(in), err); + return parse(std::string(in), err, detect_comments); } else { err = "null input"; return nullptr; } } // Parse multiple objects, concatenated or separated by whitespace - static std::vector parse_multi(const std::string & in, std::string & err); + static std::vector parse_multi(const std::string & in, + std::string & err, + bool detect_comments = false); bool operator== (const Json &rhs) const; bool operator< (const Json &rhs) const; diff --git a/test.cpp b/test.cpp index 7a40857b..26fd5b88 100644 --- a/test.cpp +++ b/test.cpp @@ -58,7 +58,6 @@ int main(int argc, char **argv) { std::cout << " - " << k.dump() << "\n"; } -#ifdef JSON11_COMMENTS const string comment_test = R"({ // comment "a": 1, @@ -72,13 +71,13 @@ int main(int argc, char **argv) { })"; string err_comment; - auto json_comment = Json::parse(comment_test, err_comment); + auto json_comment = Json::parse( + comment_test, err_comment, /*detect_comments=*/ true); if (!err_comment.empty()) { printf("Failed: %s\n", err_comment.c_str()); } else { printf("Result: %s\n", json_comment.dump().c_str()); } -#endif std::list l1 { 1, 2, 3 }; std::vector l2 { 1, 2, 3 }; From b05e655c0a37098ca8eaa106cfc78cd424bc4c55 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Mon, 30 Nov 2015 12:28:45 +0100 Subject: [PATCH 41/74] detect multiple comments with a loop instead of using recursion --- json11.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/json11.cpp b/json11.cpp index 4d239c02..a1efd618 100644 --- a/json11.cpp +++ b/json11.cpp @@ -369,7 +369,8 @@ struct JsonParser { * * Advance comments (c-style inline and multiline). */ - void consume_comment() { + bool consume_comment() { + bool comment_found = false; if (str[i] == '/') { i++; if (str[i] == '/') { // inline comment @@ -377,8 +378,7 @@ struct JsonParser { // advance until next line while (str[i] != '\n') i++; - consume_whitespace(); - consume_comment(); + comment_found = true; } else if (str[i] == '*') { // multiline comment i++; @@ -386,10 +386,10 @@ struct JsonParser { while (!(str[i] == '*' && str[i+1] == '/')) i++; i += 2; - consume_whitespace(); - consume_comment(); + comment_found = true; } } + return comment_found; } /* consume_garbage() @@ -399,8 +399,12 @@ struct JsonParser { void consume_garbage() { consume_whitespace(); if(detect_comments) { - consume_comment(); - consume_whitespace(); + bool comment_found = false; + do { + comment_found = consume_comment(); + consume_whitespace(); + } + while(comment_found); } } From 2f5c64225d4bbb9c5189d940cce65189c8d3c3ad Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Mon, 30 Nov 2015 12:40:20 +0100 Subject: [PATCH 42/74] detect malformed comments --- json11.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index a1efd618..f67fda51 100644 --- a/json11.cpp +++ b/json11.cpp @@ -383,11 +383,16 @@ struct JsonParser { else if (str[i] == '*') { // multiline comment i++; // advance until closing tokens - while (!(str[i] == '*' && str[i+1] == '/')) - i++; + while (!(str[i] == '*' && str[i+1] == '/')) { + if (i == str.size()) + return fail( + "unexpected end of input inside multi-line comment", 0); + i++;} i += 2; comment_found = true; } + else + return fail("malformed comment", 0); } return comment_found; } From d292fce9f23f394c381792581dcb22c8adabaef7 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Mon, 30 Nov 2015 12:42:24 +0100 Subject: [PATCH 43/74] improve comment test. test also for nested and mixed comments. whitespaces/newlines are already intermixed between comments. --- test.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test.cpp b/test.cpp index 26fd5b88..87f27ede 100644 --- a/test.cpp +++ b/test.cpp @@ -59,7 +59,7 @@ int main(int argc, char **argv) { } const string comment_test = R"({ - // comment + // comment /* with nested comment */ "a": 1, // comment // continued @@ -67,6 +67,7 @@ int main(int argc, char **argv) { /* multi line comment */ + // and single-line comment "c": [1, 2, 3] })"; From f21b8c360e66527911258828a86c280c842202f4 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Mon, 30 Nov 2015 12:43:40 +0100 Subject: [PATCH 44/74] add malformed comment tests. add 3 testes for: - unended multi-line comment, - malformed single-line comment, - trailing slash --- test.cpp | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test.cpp b/test.cpp index 87f27ede..ba786d83 100644 --- a/test.cpp +++ b/test.cpp @@ -80,6 +80,45 @@ int main(int argc, char **argv) { printf("Result: %s\n", json_comment.dump().c_str()); } + string failing_comment_test = R"({ + /* bad comment + "a": 1, + })"; + + string err_failing_comment; + auto json_failing_comment = Json::parse( + failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + if (!err_failing_comment.empty()) { + printf("Failed: %s\n", err_failing_comment.c_str()); + } else { + printf("Result: %s\n", json_failing_comment.dump().c_str()); + } + + failing_comment_test = R"({ + / / bad comment + "a": 1, + })"; + + json_failing_comment = Json::parse( + failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + if (!err_failing_comment.empty()) { + printf("Failed: %s\n", err_failing_comment.c_str()); + } else { + printf("Result: %s\n", json_failing_comment.dump().c_str()); + } + + failing_comment_test = R"({ + "a": 1, + }/)"; + + json_failing_comment = Json::parse( + failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + if (!err_failing_comment.empty()) { + printf("Failed: %s\n", err_failing_comment.c_str()); + } else { + printf("Result: %s\n", json_failing_comment.dump().c_str()); + } + std::list l1 { 1, 2, 3 }; std::vector l2 { 1, 2, 3 }; std::set l3 { 1, 2, 3 }; From 4b0f5cfd774fd431f15858b15bd1152003f4d524 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Tue, 1 Dec 2015 10:59:22 +0100 Subject: [PATCH 45/74] check for end of input on every increment of the cursor --- json11.cpp | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index f67fda51..3ea9a67f 100644 --- a/json11.cpp +++ b/json11.cpp @@ -373,22 +373,35 @@ struct JsonParser { bool comment_found = false; if (str[i] == '/') { i++; + if (i == str.size()) + return fail("unexpected end of input inside comment", 0); if (str[i] == '/') { // inline comment i++; + if (i == str.size()) + return fail("unexpected end of input inside inline comment", 0); // advance until next line - while (str[i] != '\n') + while (str[i] != '\n') { i++; + if (i == str.size()) + return fail("unexpected end of input inside inline comment", 0); + } comment_found = true; } else if (str[i] == '*') { // multiline comment i++; + if (i == str.size()) + return fail("unexpected end of input inside multi-line comment", 0); // advance until closing tokens while (!(str[i] == '*' && str[i+1] == '/')) { + i++; if (i == str.size()) return fail( "unexpected end of input inside multi-line comment", 0); - i++;} + } i += 2; + if (i == str.size()) + return fail( + "unexpected end of input inside multi-line comment", 0); comment_found = true; } else From 982b2d8885741bbef666f07ab874bbe802bb9d0b Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Tue, 1 Dec 2015 11:00:11 +0100 Subject: [PATCH 46/74] improve testing for bad inline comments --- test.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test.cpp b/test.cpp index ba786d83..7d77aa3a 100644 --- a/test.cpp +++ b/test.cpp @@ -95,9 +95,7 @@ int main(int argc, char **argv) { } failing_comment_test = R"({ - / / bad comment - "a": 1, - })"; + / / bad comment })"; json_failing_comment = Json::parse( failing_comment_test, err_failing_comment, /*detect_comments=*/ true); From aa270ad5b7fd62d8ae765d6ff58ca7ad527a00ea Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Tue, 1 Dec 2015 11:00:58 +0100 Subject: [PATCH 47/74] fix test where the trailing / was not reached due to a previous error --- test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.cpp b/test.cpp index 7d77aa3a..7e7d0e32 100644 --- a/test.cpp +++ b/test.cpp @@ -106,7 +106,7 @@ int main(int argc, char **argv) { } failing_comment_test = R"({ - "a": 1, + "a": 1 }/)"; json_failing_comment = Json::parse( From c6c6fcfeff5663cb34e7405bcc4572a2a5424d40 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Tue, 1 Dec 2015 11:01:27 +0100 Subject: [PATCH 48/74] add test for inline comment without trailing newline --- test.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test.cpp b/test.cpp index 7e7d0e32..1e537b99 100644 --- a/test.cpp +++ b/test.cpp @@ -105,6 +105,16 @@ int main(int argc, char **argv) { printf("Result: %s\n", json_failing_comment.dump().c_str()); } + failing_comment_test = R"({// bad comment })"; + + json_failing_comment = Json::parse( + failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + if (!err_failing_comment.empty()) { + printf("Failed: %s\n", err_failing_comment.c_str()); + } else { + printf("Result: %s\n", json_failing_comment.dump().c_str()); + } + failing_comment_test = R"({ "a": 1 }/)"; From f9833b1e7d4b36901c36131c4699d889dae3eaf2 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Tue, 1 Dec 2015 11:01:47 +0100 Subject: [PATCH 49/74] add test for unfinished multi-line comment --- test.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test.cpp b/test.cpp index 1e537b99..ece0e045 100644 --- a/test.cpp +++ b/test.cpp @@ -127,6 +127,17 @@ int main(int argc, char **argv) { printf("Result: %s\n", json_failing_comment.dump().c_str()); } + failing_comment_test = R"({/* bad + comment *})"; + + json_failing_comment = Json::parse( + failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + if (!err_failing_comment.empty()) { + printf("Failed: %s\n", err_failing_comment.c_str()); + } else { + printf("Result: %s\n", json_failing_comment.dump().c_str()); + } + std::list l1 { 1, 2, 3 }; std::vector l2 { 1, 2, 3 }; std::set l3 { 1, 2, 3 }; From 49a6197d08fbdd5a2eb9b0e8ae3aeab88f2a6d29 Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Tue, 1 Dec 2015 11:08:37 +0100 Subject: [PATCH 50/74] use an enum to select strategy on comment parsing --- json11.cpp | 12 ++++++------ json11.hpp | 17 +++++++++++------ test.cpp | 12 ++++++------ 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/json11.cpp b/json11.cpp index 3ea9a67f..07e43c0a 100644 --- a/json11.cpp +++ b/json11.cpp @@ -338,7 +338,7 @@ struct JsonParser { size_t i; string &err; bool failed; - bool detect_comments; + JsonParse strategy; /* fail(msg, err_ret = Json()) * @@ -416,7 +416,7 @@ struct JsonParser { */ void consume_garbage() { consume_whitespace(); - if(detect_comments) { + if(strategy == JsonParse::COMMENTS) { bool comment_found = false; do { comment_found = consume_comment(); @@ -719,8 +719,8 @@ struct JsonParser { } }; -Json Json::parse(const string &in, string &err, bool detect_comments) { - JsonParser parser { in, 0, err, false, detect_comments }; +Json Json::parse(const string &in, string &err, JsonParse strategy) { + JsonParser parser { in, 0, err, false, strategy }; Json result = parser.parse_json(0); // Check for any trailing garbage @@ -734,8 +734,8 @@ Json Json::parse(const string &in, string &err, bool detect_comments) { // Documented in json11.hpp vector Json::parse_multi(const string &in, string &err, - bool detect_comments) { - JsonParser parser { in, 0, err, false, detect_comments }; + JsonParse strategy) { + JsonParser parser { in, 0, err, false, strategy }; vector json_vec; while (parser.i != in.size() && !parser.failed) { diff --git a/json11.hpp b/json11.hpp index 582c43f0..a99e241f 100644 --- a/json11.hpp +++ b/json11.hpp @@ -58,6 +58,10 @@ namespace json11 { +enum JsonParse { + STANDARD, COMMENTS +}; + class JsonValue; class Json final { @@ -147,21 +151,22 @@ public: // Parse. If parse fails, return Json() and assign an error message to err. static Json parse(const std::string & in, std::string & err, - bool detect_comments = false); + JsonParse strategy = JsonParse::STANDARD); static Json parse(const char * in, std::string & err, - bool detect_comments = false) { + JsonParse strategy = JsonParse::STANDARD) { if (in) { - return parse(std::string(in), err, detect_comments); + return parse(std::string(in), err, strategy); } else { err = "null input"; return nullptr; } } // Parse multiple objects, concatenated or separated by whitespace - static std::vector parse_multi(const std::string & in, - std::string & err, - bool detect_comments = false); + static std::vector parse_multi( + const std::string & in, + std::string & err, + JsonParse strategy = JsonParse::STANDARD); bool operator== (const Json &rhs) const; bool operator< (const Json &rhs) const; diff --git a/test.cpp b/test.cpp index ece0e045..bd607050 100644 --- a/test.cpp +++ b/test.cpp @@ -73,7 +73,7 @@ int main(int argc, char **argv) { string err_comment; auto json_comment = Json::parse( - comment_test, err_comment, /*detect_comments=*/ true); + comment_test, err_comment, JsonParse::COMMENTS); if (!err_comment.empty()) { printf("Failed: %s\n", err_comment.c_str()); } else { @@ -87,7 +87,7 @@ int main(int argc, char **argv) { string err_failing_comment; auto json_failing_comment = Json::parse( - failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + failing_comment_test, err_failing_comment, JsonParse::COMMENTS); if (!err_failing_comment.empty()) { printf("Failed: %s\n", err_failing_comment.c_str()); } else { @@ -98,7 +98,7 @@ int main(int argc, char **argv) { / / bad comment })"; json_failing_comment = Json::parse( - failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + failing_comment_test, err_failing_comment, JsonParse::COMMENTS); if (!err_failing_comment.empty()) { printf("Failed: %s\n", err_failing_comment.c_str()); } else { @@ -108,7 +108,7 @@ int main(int argc, char **argv) { failing_comment_test = R"({// bad comment })"; json_failing_comment = Json::parse( - failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + failing_comment_test, err_failing_comment, JsonParse::COMMENTS); if (!err_failing_comment.empty()) { printf("Failed: %s\n", err_failing_comment.c_str()); } else { @@ -120,7 +120,7 @@ int main(int argc, char **argv) { }/)"; json_failing_comment = Json::parse( - failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + failing_comment_test, err_failing_comment, JsonParse::COMMENTS); if (!err_failing_comment.empty()) { printf("Failed: %s\n", err_failing_comment.c_str()); } else { @@ -131,7 +131,7 @@ int main(int argc, char **argv) { comment *})"; json_failing_comment = Json::parse( - failing_comment_test, err_failing_comment, /*detect_comments=*/ true); + failing_comment_test, err_failing_comment, JsonParse::COMMENTS); if (!err_failing_comment.empty()) { printf("Failed: %s\n", err_failing_comment.c_str()); } else { From 988a8fc249c49cfd0779eb3e938444b622eaad0a Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Wed, 2 Dec 2015 09:57:25 +0100 Subject: [PATCH 51/74] make JsonParser::strategy const --- json11.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/json11.cpp b/json11.cpp index 07e43c0a..3ec83af3 100644 --- a/json11.cpp +++ b/json11.cpp @@ -338,7 +338,7 @@ struct JsonParser { size_t i; string &err; bool failed; - JsonParse strategy; + const JsonParse strategy; /* fail(msg, err_ret = Json()) * From ebc3a6b0387af83c6735f89dbde98ca425741b3b Mon Sep 17 00:00:00 2001 From: Antonio Cervone Date: Wed, 2 Dec 2015 10:01:29 +0100 Subject: [PATCH 52/74] watch out for i+1 to overflow the buffer --- json11.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/json11.cpp b/json11.cpp index 3ec83af3..ebd7e93d 100644 --- a/json11.cpp +++ b/json11.cpp @@ -389,12 +389,12 @@ struct JsonParser { } else if (str[i] == '*') { // multiline comment i++; - if (i == str.size()) + if (i > str.size()-2) return fail("unexpected end of input inside multi-line comment", 0); - // advance until closing tokens + // advance until closing tokens while (!(str[i] == '*' && str[i+1] == '/')) { i++; - if (i == str.size()) + if (i > str.size()-2) return fail( "unexpected end of input inside multi-line comment", 0); } From e1d5bcc94d48341e4382ee5d0733726f0dde1a15 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 29 Dec 2015 18:55:48 +0100 Subject: [PATCH 53/74] =?UTF-8?q?This=20fixes=20a=20=E2=80=9CNo=20previous?= =?UTF-8?q?=20prototype=20for=20function=E2=80=9D=20warning.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- json11.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index ebd7e93d..38fd4c8f 100644 --- a/json11.cpp +++ b/json11.cpp @@ -226,12 +226,12 @@ struct Statics { Statics() {} }; -const Statics & statics() { +static const Statics & statics() { static const Statics s {}; return s; } -const Json & static_null() { +static const Json & static_null() { // This has to be separate, not in Statics, because Json() accesses statics().null. static const Json json_null; return json_null; From 5e664c99d887133f5e06a902b2b61ea67413e7f4 Mon Sep 17 00:00:00 2001 From: kirisetsz Date: Fri, 8 Apr 2016 01:16:21 +0800 Subject: [PATCH 54/74] Update CMakeLists to 3.x style --- CMakeLists.txt | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ba8edd5b..397104aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,21 +1,14 @@ -project(json11) +project(json11 CXX) -cmake_minimum_required(VERSION 2.8) +cmake_minimum_required(VERSION 3.2) enable_testing() -add_definitions( - -std=c++11 - -fno-rtti - -fno-exceptions - -Wall - -Wextra - -Werror) +add_library(json11 json11.cpp) +target_include_directories(json11 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_options(json11 + PUBLIC -std=c++11 + PRIVATE -fno-rtti -fno-exceptions -Wall -Wextra -Werror) -set(json11_SRCS json11.cpp) - -add_library(json11 STATIC ${json11_SRCS}) - -add_test(json11_test json11_test) - -add_executable(json11_test ${json11_SRCS} test.cpp) +add_executable(json11_test test.cpp) +target_link_libraries(json11_test json11) From 8076ba74e08907b1e219f748f61f6ae6b69ca145 Mon Sep 17 00:00:00 2001 From: Nick White Date: Sat, 16 Apr 2016 18:41:25 +0100 Subject: [PATCH 55/74] Generate pkg-config File & Add Install ...to the CMake build script. The pkg-config file needs a version number (which it takes from the CMakeLists.txt variable), which I've set to 1.0.0. The cmake project declaration needs to be after the minimum-requirement declaration to avoid errors from the cmake's change in VERSION semantics. --- .gitignore | 11 +++++++++++ CMakeLists.txt | 8 ++++++-- json11.pc.in | 9 +++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 json11.pc.in diff --git a/.gitignore b/.gitignore index 9daeafb9..e959b6fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,12 @@ +# generated files test +libjson11.a +json11.pc + +# Cmake +CMakeCache.txt +CTestTestfile.cmake +CMakeFiles +CMakeScripts +cmake_install.cmake +install_manifest.txt \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 397104aa..9d6adf95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,5 @@ -project(json11 CXX) - cmake_minimum_required(VERSION 3.2) +project(json11 VERSION 1.0.0 LANGUAGES CXX) enable_testing() @@ -9,6 +8,11 @@ target_include_directories(json11 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_compile_options(json11 PUBLIC -std=c++11 PRIVATE -fno-rtti -fno-exceptions -Wall -Wextra -Werror) +configure_file("json11.pc.in" "json11.pc" @ONLY) add_executable(json11_test test.cpp) target_link_libraries(json11_test json11) + +install(TARGETS json11 DESTINATION lib) +install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/json11.hpp" DESTINATION include) +install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/json11.pc" DESTINATION lib/pkgconfig) diff --git a/json11.pc.in b/json11.pc.in new file mode 100644 index 00000000..de4b6dd7 --- /dev/null +++ b/json11.pc.in @@ -0,0 +1,9 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +libdir=${prefix}/lib +includedir=${prefix}/include + +Name: @PROJECT_NAME@ +Description: json11 is a tiny JSON library for C++11, providing JSON parsing and serialization. +Version: @PROJECT_VERSION@ +Libs: -L${libdir} -ljson11 +Cflags: -I${includedir} \ No newline at end of file From 33bca8e178524a6554c2ae9256f90cfaa1f8dd76 Mon Sep 17 00:00:00 2001 From: "Evgeniy A. Dushistov" Date: Wed, 27 Apr 2016 12:47:30 +0300 Subject: [PATCH 56/74] implement parsing of chunked json Update: add more tests + use parser.failed --- json11.cpp | 5 ++++- json11.hpp | 9 +++++++++ test.cpp | 36 +++++++++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/json11.cpp b/json11.cpp index 38fd4c8f..b2fa4cbd 100644 --- a/json11.cpp +++ b/json11.cpp @@ -733,15 +733,18 @@ Json Json::parse(const string &in, string &err, JsonParse strategy) { // Documented in json11.hpp vector Json::parse_multi(const string &in, + std::string::size_type &parser_stop_pos, string &err, JsonParse strategy) { JsonParser parser { in, 0, err, false, strategy }; - + parser_stop_pos = 0; vector json_vec; while (parser.i != in.size() && !parser.failed) { json_vec.push_back(parser.parse_json(0)); // Check for another object parser.consume_garbage(); + if (!parser.failed) + parser_stop_pos = parser.i; } return json_vec; } diff --git a/json11.hpp b/json11.hpp index a99e241f..e9fe251f 100644 --- a/json11.hpp +++ b/json11.hpp @@ -165,9 +165,18 @@ public: // Parse multiple objects, concatenated or separated by whitespace static std::vector parse_multi( const std::string & in, + std::string::size_type & parser_stop_pos, std::string & err, JsonParse strategy = JsonParse::STANDARD); + static inline std::vector parse_multi( + const std::string & in, + std::string & err, + JsonParse strategy = JsonParse::STANDARD) { + std::string::size_type parser_stop_pos; + return parse_multi(in, parser_stop_pos, err, strategy); + } + bool operator== (const Json &rhs) const; bool operator< (const Json &rhs) const; bool operator!= (const Json &rhs) const { return !(*this == rhs); } diff --git a/test.cpp b/test.cpp index bd607050..b1b52fdf 100644 --- a/test.cpp +++ b/test.cpp @@ -1,13 +1,17 @@ +#ifdef NDEBUG +#undef NDEBUG//at now assert will work even in Release build +#endif +#include #include #include #include #include #include #include "json11.hpp" -#include #include #include #include +#include using namespace json11; using std::string; @@ -183,6 +187,36 @@ int main(int argc, char **argv) { assert(nested_array.array_items()[0].is_array()); assert(nested_array.array_items()[0].array_items().size() == 3); + { + const std::string good_json = R"( {"k1" : "v1"})"; + const std::string bad_json1 = good_json + " {"; + const std::string bad_json2 = good_json + R"({"k2":"v2", "k3":[)"; + struct TestMultiParse { + std::string input; + std::string::size_type expect_parser_stop_pos; + size_t expect_not_empty_elms_count; + Json expect_parse_res; + } tests[] = { + {" {", 0, 0, {}}, + {good_json, good_json.size(), 1, Json(std::map{ { "k1", "v1" } })}, + {bad_json1, good_json.size() + 1, 1, Json(std::map{ { "k1", "v1" } })}, + {bad_json2, good_json.size(), 1, Json(std::map{ { "k1", "v1" } })}, + {"{}", 2, 1, Json::object{}}, + }; + for (const auto &tst : tests) { + std::string::size_type parser_stop_pos; + std::string err; + auto res = Json::parse_multi(tst.input, parser_stop_pos, err); + assert(parser_stop_pos == tst.expect_parser_stop_pos); + assert( + std::count_if(res.begin(), res.end(), + [](const Json& j) { return !j.is_null(); }) + == tst.expect_not_empty_elms_count); + if (!res.empty()) { + assert(tst.expect_parse_res == res[0]); + } + } + } Json my_json = Json::object { { "key1", "value1" }, { "key2", false }, From 0e4ace7e40893dee73934431d3a6f7b85f33146a Mon Sep 17 00:00:00 2001 From: "Evgeniy A. Dushistov" Date: Thu, 28 Apr 2016 01:28:42 +0300 Subject: [PATCH 57/74] make parser invisible from outside --- json11.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/json11.cpp b/json11.cpp index 38fd4c8f..07a3a715 100644 --- a/json11.cpp +++ b/json11.cpp @@ -326,11 +326,12 @@ static inline bool in_range(long x, long lower, long upper) { return (x >= lower && x <= upper); } +namespace { /* JsonParser * * Object that tracks all state of an in-progress parse. */ -struct JsonParser { +struct JsonParser final { /* State */ @@ -718,6 +719,7 @@ struct JsonParser { return fail("expected value, got " + esc(ch)); } }; +}//namespace { Json Json::parse(const string &in, string &err, JsonParse strategy) { JsonParser parser { in, 0, err, false, strategy }; From 05b5514400d9bc757a053f57fe08436694294d64 Mon Sep 17 00:00:00 2001 From: Chris Kitching Date: Thu, 5 May 2016 02:43:43 +0100 Subject: [PATCH 58/74] Don't assume in-tree builds when installing json11.pc --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d6adf95..350b4ba9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,4 +15,4 @@ target_link_libraries(json11_test json11) install(TARGETS json11 DESTINATION lib) install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/json11.hpp" DESTINATION include) -install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/json11.pc" DESTINATION lib/pkgconfig) +install(FILES "${CMAKE_BINARY_DIR}/json11.pc" DESTINATION lib/pkgconfig) From 4597f98abe32818c8d8ec1d69654b8af9a68ece2 Mon Sep 17 00:00:00 2001 From: Andrew Twyman Date: Wed, 18 May 2016 18:31:52 -0700 Subject: [PATCH 59/74] Fix sign comparison warnings, and parser confusion --- test.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test.cpp b/test.cpp index b1b52fdf..3425f80e 100644 --- a/test.cpp +++ b/test.cpp @@ -87,6 +87,7 @@ int main(int argc, char **argv) { string failing_comment_test = R"({ /* bad comment "a": 1, + // another comment to make C parsers which don't understand raw strings happy */ })"; string err_failing_comment; @@ -209,8 +210,8 @@ int main(int argc, char **argv) { auto res = Json::parse_multi(tst.input, parser_stop_pos, err); assert(parser_stop_pos == tst.expect_parser_stop_pos); assert( - std::count_if(res.begin(), res.end(), - [](const Json& j) { return !j.is_null(); }) + (size_t)std::count_if(res.begin(), res.end(), + [](const Json& j) { return !j.is_null(); }) == tst.expect_not_empty_elms_count); if (!res.empty()) { assert(tst.expect_parse_res == res[0]); From a20878aaa5bd2546466585b18b6d09808a98233d Mon Sep 17 00:00:00 2001 From: Andrew Twyman Date: Wed, 18 May 2016 18:36:58 -0700 Subject: [PATCH 60/74] Make the DR1467 canary test code optional --- test.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/test.cpp b/test.cpp index 3425f80e..c90d1397 100644 --- a/test.cpp +++ b/test.cpp @@ -180,13 +180,16 @@ int main(int argc, char **argv) { assert(uni[0].string_value().size() == (sizeof utf8) - 1); assert(std::memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); - // Demonstrates the behavior change in Xcode 7 / Clang 3.7 described - // here: https://llvm.org/bugs/show_bug.cgi?id=23812 - Json nested_array = Json::array { Json::array { 1, 2, 3 } }; - assert(nested_array.is_array()); - assert(nested_array.array_items().size() == 1); - assert(nested_array.array_items()[0].is_array()); - assert(nested_array.array_items()[0].array_items().size() == 3); + // Demonstrates the behavior change in Xcode 7 / Clang 3.7, introduced by DR1467 + // and described here: https://llvm.org/bugs/show_bug.cgi?id=23812 + const bool ENABLE_DR1467_CANARY = true; // Allow easy disabling for users who work around it. + if (ENABLE_DR1467_CANARY) { + Json nested_array = Json::array { Json::array { 1, 2, 3 } }; + assert(nested_array.is_array()); + assert(nested_array.array_items().size() == 1); + assert(nested_array.array_items()[0].is_array()); + assert(nested_array.array_items()[0].array_items().size() == 3); + } { const std::string good_json = R"( {"k1" : "v1"})"; From f6c0f687372a3eee8bfb7e5099ba8dfc2c36e3fa Mon Sep 17 00:00:00 2001 From: Shrikant Kelkar Date: Mon, 20 Jun 2016 17:36:17 -0700 Subject: [PATCH 61/74] MSVS 2013 compatibility changes. --- json11.hpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/json11.hpp b/json11.hpp index e9fe251f..5202ef93 100644 --- a/json11.hpp +++ b/json11.hpp @@ -56,6 +56,18 @@ #include #include +#ifdef _MSC_VER + #if _MSC_VER <= 1800 // VS 2013 + #ifndef noexcept + #define noexcept throw() + #endif + + #ifndef snprintf + #define snprintf _snprintf_s + #endif + #endif +#endif + namespace json11 { enum JsonParse { From 370467150873ce8616a859edf9a510fd28fffbe5 Mon Sep 17 00:00:00 2001 From: Bruno Coelho <4brunu@gmail.com> Date: Tue, 5 Jul 2016 15:37:44 +0100 Subject: [PATCH 62/74] Add option to disable unit tests --- CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 350b4ba9..ae8c4123 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,6 +3,8 @@ project(json11 VERSION 1.0.0 LANGUAGES CXX) enable_testing() +option(JSON11_BUILD_TESTS "Build unit tests" ON) + add_library(json11 json11.cpp) target_include_directories(json11 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_compile_options(json11 @@ -10,8 +12,10 @@ target_compile_options(json11 PRIVATE -fno-rtti -fno-exceptions -Wall -Wextra -Werror) configure_file("json11.pc.in" "json11.pc" @ONLY) -add_executable(json11_test test.cpp) -target_link_libraries(json11_test json11) +if (JSON11_BUILD_TESTS) + add_executable(json11_test test.cpp) + target_link_libraries(json11_test json11) +endif() install(TARGETS json11 DESTINATION lib) install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/json11.hpp" DESTINATION include) From 467dc6ae058e68aaf9d3bbfdf1d2447f4742f64d Mon Sep 17 00:00:00 2001 From: Austin Brunkhorst Date: Sun, 24 Jul 2016 11:34:17 -0700 Subject: [PATCH 63/74] Fixes warning C4800: 'int': forcing value to bool 'true' or 'false' on MSVC 14 --- json11.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/json11.cpp b/json11.cpp index 5f3b7efd..f977e168 100644 --- a/json11.cpp +++ b/json11.cpp @@ -375,38 +375,38 @@ struct JsonParser final { if (str[i] == '/') { i++; if (i == str.size()) - return fail("unexpected end of input inside comment", 0); + return !!fail("unexpected end of input inside comment", 0); if (str[i] == '/') { // inline comment i++; if (i == str.size()) - return fail("unexpected end of input inside inline comment", 0); + return !!fail("unexpected end of input inside inline comment", 0); // advance until next line while (str[i] != '\n') { i++; if (i == str.size()) - return fail("unexpected end of input inside inline comment", 0); + return !!fail("unexpected end of input inside inline comment", 0); } comment_found = true; } else if (str[i] == '*') { // multiline comment i++; if (i > str.size()-2) - return fail("unexpected end of input inside multi-line comment", 0); + return !!fail("unexpected end of input inside multi-line comment", 0); // advance until closing tokens while (!(str[i] == '*' && str[i+1] == '/')) { i++; if (i > str.size()-2) - return fail( + return !!fail( "unexpected end of input inside multi-line comment", 0); } i += 2; if (i == str.size()) - return fail( + return !!fail( "unexpected end of input inside multi-line comment", 0); comment_found = true; } else - return fail("malformed comment", 0); + return !!fail("malformed comment", 0); } return comment_found; } From 40f10bd28d42a560749bb033d33bfecc494b02a5 Mon Sep 17 00:00:00 2001 From: Austin Brunkhorst Date: Mon, 25 Jul 2016 17:28:15 -0700 Subject: [PATCH 64/74] Use `false` instead of `0` + conversion. --- json11.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/json11.cpp b/json11.cpp index f977e168..e3dc1190 100644 --- a/json11.cpp +++ b/json11.cpp @@ -375,38 +375,38 @@ struct JsonParser final { if (str[i] == '/') { i++; if (i == str.size()) - return !!fail("unexpected end of input inside comment", 0); + return fail("unexpected end of input inside comment", false); if (str[i] == '/') { // inline comment i++; if (i == str.size()) - return !!fail("unexpected end of input inside inline comment", 0); + return fail("unexpected end of input inside inline comment", false); // advance until next line while (str[i] != '\n') { i++; if (i == str.size()) - return !!fail("unexpected end of input inside inline comment", 0); + return fail("unexpected end of input inside inline comment", false); } comment_found = true; } else if (str[i] == '*') { // multiline comment i++; if (i > str.size()-2) - return !!fail("unexpected end of input inside multi-line comment", 0); + return fail("unexpected end of input inside multi-line comment", false); // advance until closing tokens while (!(str[i] == '*' && str[i+1] == '/')) { i++; if (i > str.size()-2) - return !!fail( - "unexpected end of input inside multi-line comment", 0); + return fail( + "unexpected end of input inside multi-line comment", false); } i += 2; if (i == str.size()) - return !!fail( - "unexpected end of input inside multi-line comment", 0); + return fail( + "unexpected end of input inside multi-line comment", false); comment_found = true; } else - return !!fail("malformed comment", 0); + return fail("malformed comment", false); } return comment_found; } From 200c98aedef378847f87cb00792423fafc6504c9 Mon Sep 17 00:00:00 2001 From: Andrew Twyman Date: Thu, 28 Jul 2016 22:27:43 -0700 Subject: [PATCH 65/74] Allow json11 tests to be customized with pre-processor defines --- Makefile | 10 ++++- test.cpp | 128 ++++++++++++++++++++++++++++++++++++------------------- 2 files changed, 94 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index 5a6adc6e..f946bdd8 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,13 @@ +# Environment variable to enable or disable code which demonstrates the behavior change +# in Xcode 7 / Clang 3.7, introduced by DR1467 and described here: +# https://llvm.org/bugs/show_bug.cgi?id=23812 +# Defaults to on in order to act as a warning to anyone who's unaware of the issue. +ifneq ($(JSON11_ENABLE_DR1467_CANARY),) +CANARY_ARGS = -DJSON11_ENABLE_DR1467_CANARY=$(JSON11_ENABLE_DR1467_CANARY) +endif + test: json11.cpp json11.hpp test.cpp - $(CXX) -O -std=c++11 json11.cpp test.cpp -o test -fno-rtti -fno-exceptions + $(CXX) $(CANARY_ARGS) -O -std=c++11 json11.cpp test.cpp -o test -fno-rtti -fno-exceptions clean: if [ -e test ]; then rm test; fi diff --git a/test.cpp b/test.cpp index c90d1397..2e6543e3 100644 --- a/test.cpp +++ b/test.cpp @@ -1,6 +1,33 @@ +/* + * Define JSON11_TEST_CUSTOM_CONFIG to 1 if you want to build this tester into + * your own unit-test framework rather than a stand-alone program. By setting + * The values of the variables included below, you can insert your own custom + * code into this file as it builds, in order to make it into a test case for + * your favorite framework. + */ +#if(!JSON11_TEST_CUSTOM_CONFIG) +#define JSON11_TEST_CPP_PREFIX_CODE +#define JSON11_TEST_CPP_SUFFIX_CODE +#define JSON11_TEST_STANDALONE_MAIN 1 +#define JSON11_TEST_CASE(name) static void name() +#define JSON11_TEST_ASSERT(b) assert(b) #ifdef NDEBUG #undef NDEBUG//at now assert will work even in Release build #endif +#endif // JSON11_TEST_CUSTOM_CONFIG + +/* + * Enable or disable code which demonstrates the behavior change in Xcode 7 / Clang 3.7, + * introduced by DR1467 and described here: https://llvm.org/bugs/show_bug.cgi?id=23812 + * Defaults to on in order to act as a warning to anyone who's unaware of the issue. + */ +#ifndef JSON11_ENABLE_DR1467_CANARY +#define JSON11_ENABLE_DR1467_CANARY 1 +#endif + +/* + * Beginning of standard source file, which makes use of the customizations above. + */ #include #include #include @@ -12,12 +39,16 @@ #include #include #include +#include + +// Insert user-defined prefix code (includes, function declarations, etc) +// to set up a custom test suite +JSON11_TEST_CPP_PREFIX_CODE using namespace json11; using std::string; // Check that Json has the properties we want. -#include #define CHECK_TRAIT(x) static_assert(std::x::value, #x) CHECK_TRAIT(is_nothrow_constructible); CHECK_TRAIT(is_nothrow_default_constructible); @@ -27,28 +58,7 @@ CHECK_TRAIT(is_copy_assignable); CHECK_TRAIT(is_nothrow_move_assignable); CHECK_TRAIT(is_nothrow_destructible); -void parse_from_stdin() { - string buf; - string line; - while (std::getline(std::cin, line)) { - buf += line + "\n"; - } - - string err; - auto json = Json::parse(buf, err); - if (!err.empty()) { - printf("Failed: %s\n", err.c_str()); - } else { - printf("Result: %s\n", json.dump().c_str()); - } -} - -int main(int argc, char **argv) { - if (argc == 2 && argv[1] == string("--stdin")) { - parse_from_stdin(); - return 0; - } - +JSON11_TEST_CASE(json11_test) { const string simple_test = R"({"k1":"v1", "k2":42, "k3":["a",123,true,false,null]})"; @@ -146,12 +156,12 @@ int main(int argc, char **argv) { std::list l1 { 1, 2, 3 }; std::vector l2 { 1, 2, 3 }; std::set l3 { 1, 2, 3 }; - assert(Json(l1) == Json(l2)); - assert(Json(l2) == Json(l3)); + JSON11_TEST_ASSERT(Json(l1) == Json(l2)); + JSON11_TEST_ASSERT(Json(l2) == Json(l3)); std::map m1 { { "k1", "v1" }, { "k2", "v2" } }; std::unordered_map m2 { { "k1", "v1" }, { "k2", "v2" } }; - assert(Json(m1) == Json(m2)); + JSON11_TEST_ASSERT(Json(m1) == Json(m2)); // Json literals Json obj = Json::object({ @@ -162,13 +172,13 @@ int main(int argc, char **argv) { std::cout << "obj: " << obj.dump() << "\n"; - assert(Json("a").number_value() == 0); - assert(Json("a").string_value() == "a"); - assert(Json().number_value() == 0); + JSON11_TEST_ASSERT(Json("a").number_value() == 0); + JSON11_TEST_ASSERT(Json("a").string_value() == "a"); + JSON11_TEST_ASSERT(Json().number_value() == 0); - assert(obj == json); - assert(Json(42) == Json(42.0)); - assert(Json(42) != Json(42.1)); + JSON11_TEST_ASSERT(obj == json); + JSON11_TEST_ASSERT(Json(42) == Json(42.0)); + JSON11_TEST_ASSERT(Json(42) != Json(42.1)); const string unicode_escape_test = R"([ "blah\ud83d\udca9blah\ud83dblah\udca9blah\u0000blah\u1234" ])"; @@ -177,18 +187,17 @@ int main(int argc, char **argv) { "\xed\xb2\xa9" "blah" "\0" "blah" "\xe1\x88\xb4"; Json uni = Json::parse(unicode_escape_test, err); - assert(uni[0].string_value().size() == (sizeof utf8) - 1); - assert(std::memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); + JSON11_TEST_ASSERT(uni[0].string_value().size() == (sizeof utf8) - 1); + JSON11_TEST_ASSERT(std::memcmp(uni[0].string_value().data(), utf8, sizeof utf8) == 0); // Demonstrates the behavior change in Xcode 7 / Clang 3.7, introduced by DR1467 // and described here: https://llvm.org/bugs/show_bug.cgi?id=23812 - const bool ENABLE_DR1467_CANARY = true; // Allow easy disabling for users who work around it. - if (ENABLE_DR1467_CANARY) { + if (JSON11_ENABLE_DR1467_CANARY) { Json nested_array = Json::array { Json::array { 1, 2, 3 } }; - assert(nested_array.is_array()); - assert(nested_array.array_items().size() == 1); - assert(nested_array.array_items()[0].is_array()); - assert(nested_array.array_items()[0].array_items().size() == 3); + JSON11_TEST_ASSERT(nested_array.is_array()); + JSON11_TEST_ASSERT(nested_array.array_items().size() == 1); + JSON11_TEST_ASSERT(nested_array.array_items()[0].is_array()); + JSON11_TEST_ASSERT(nested_array.array_items()[0].array_items().size() == 3); } { @@ -211,13 +220,13 @@ int main(int argc, char **argv) { std::string::size_type parser_stop_pos; std::string err; auto res = Json::parse_multi(tst.input, parser_stop_pos, err); - assert(parser_stop_pos == tst.expect_parser_stop_pos); - assert( + JSON11_TEST_ASSERT(parser_stop_pos == tst.expect_parser_stop_pos); + JSON11_TEST_ASSERT( (size_t)std::count_if(res.begin(), res.end(), [](const Json& j) { return !j.is_null(); }) == tst.expect_not_empty_elms_count); if (!res.empty()) { - assert(tst.expect_parse_res == res[0]); + JSON11_TEST_ASSERT(tst.expect_parse_res == res[0]); } } } @@ -241,3 +250,36 @@ int main(int argc, char **argv) { std::string points_json = Json(points).dump(); printf("%s\n", points_json.c_str()); } + +#if(JSON11_TEST_STANDALONE_MAIN) + +static void parse_from_stdin() { + string buf; + string line; + while (std::getline(std::cin, line)) { + buf += line + "\n"; + } + + string err; + auto json = Json::parse(buf, err); + if (!err.empty()) { + printf("Failed: %s\n", err.c_str()); + } else { + printf("Result: %s\n", json.dump().c_str()); + } +} + +int main(int argc, char **argv) { + if (argc == 2 && argv[1] == string("--stdin")) { + parse_from_stdin(); + return 0; + } + + json11_test(); +} + +#endif // JSON11_TEST_STANDALONE_MAIN + +// Insert user-defined suffix code (function definitions, etc) +// to set up a custom test suite +JSON11_TEST_CPP_SUFFIX_CODE From 7a947c1e2f4a7f30860dd7ca11ee883c2cc3f324 Mon Sep 17 00:00:00 2001 From: Andrew Twyman Date: Thu, 28 Jul 2016 22:42:01 -0700 Subject: [PATCH 66/74] Remove extraneous parentheses --- test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.cpp b/test.cpp index 2e6543e3..cf9a46fc 100644 --- a/test.cpp +++ b/test.cpp @@ -5,7 +5,7 @@ * code into this file as it builds, in order to make it into a test case for * your favorite framework. */ -#if(!JSON11_TEST_CUSTOM_CONFIG) +#if !JSON11_TEST_CUSTOM_CONFIG #define JSON11_TEST_CPP_PREFIX_CODE #define JSON11_TEST_CPP_SUFFIX_CODE #define JSON11_TEST_STANDALONE_MAIN 1 @@ -251,7 +251,7 @@ JSON11_TEST_CASE(json11_test) { printf("%s\n", points_json.c_str()); } -#if(JSON11_TEST_STANDALONE_MAIN) +#if JSON11_TEST_STANDALONE_MAIN static void parse_from_stdin() { string buf; From 580fd44b5f83e3cd8976fc1aae27931d131775d2 Mon Sep 17 00:00:00 2001 From: Adam Carlucci Date: Wed, 10 Aug 2016 18:16:59 -0700 Subject: [PATCH 67/74] Use CXX_STANDARD to specify c++11 compile flags --- CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ae8c4123..02ea263d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,10 +5,12 @@ enable_testing() option(JSON11_BUILD_TESTS "Build unit tests" ON) +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + add_library(json11 json11.cpp) target_include_directories(json11 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) target_compile_options(json11 - PUBLIC -std=c++11 PRIVATE -fno-rtti -fno-exceptions -Wall -Wextra -Werror) configure_file("json11.pc.in" "json11.pc" @ONLY) From f6b4188933d0a68839989044a9bdd80b806ab56d Mon Sep 17 00:00:00 2001 From: Zhijiang TAO Date: Tue, 16 Aug 2016 00:14:02 +0800 Subject: [PATCH 68/74] =?UTF-8?q?=E6=B7=BB=E5=8A=A0int64=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=EF=BC=8C=E4=BD=86=E6=98=AF=E8=BF=98=E6=98=AF=E4=BC=9A=E6=BA=A2?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- json11.cpp | 36 ++++++++++++++++++++++++++++++------ json11.hpp | 11 +++++++++-- test.cpp | 22 ++++++++++++++++++++++ 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/json11.cpp b/json11.cpp index e3dc1190..ac9d727f 100644 --- a/json11.cpp +++ b/json11.cpp @@ -61,6 +61,12 @@ static void dump(int value, string &out) { out += buf; } +static void dump(int64_ value, string &out) { + char buf[64]; + snprintf(buf, sizeof buf, "%ld", value); + out += buf; +} + static void dump(bool value, string &out) { out += value ? "true" : "false"; } @@ -164,6 +170,7 @@ protected: class JsonDouble final : public Value { double number_value() const override { return m_value; } int int_value() const override { return static_cast(m_value); } + int64_ int64_value() const override { return static_cast(m_value); } bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: @@ -173,12 +180,23 @@ public: class JsonInt final : public Value { double number_value() const override { return m_value; } int int_value() const override { return m_value; } + int64_ int64_value() const override { return m_value; } bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: explicit JsonInt(int value) : Value(value) {} }; +class JsonInt64 final : public Value { + double number_value() const override { return m_value; } + int int_value() const override { return m_value; } + int64_ int64_value() const override { return m_value; } + bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } + bool less(const JsonValue * other) const override { return m_value < other->number_value(); } +public: + explicit JsonInt64(int64_ value) : Value(value) {} +}; + class JsonBoolean final : public Value { bool bool_value() const override { return m_value; } public: @@ -244,7 +262,8 @@ static const Json & static_null() { Json::Json() noexcept : m_ptr(statics().null) {} Json::Json(std::nullptr_t) noexcept : m_ptr(statics().null) {} Json::Json(double value) : m_ptr(make_shared(value)) {} -Json::Json(int value) : m_ptr(make_shared(value)) {} +Json::Json(int value) : m_ptr(make_shared(value)) {} +Json::Json(int64_ value) : m_ptr(make_shared(value)) {} Json::Json(bool value) : m_ptr(value ? statics().t : statics().f) {} Json::Json(const string &value) : m_ptr(make_shared(value)) {} Json::Json(string &&value) : m_ptr(make_shared(move(value))) {} @@ -260,7 +279,8 @@ Json::Json(Json::object &&values) : m_ptr(make_shared(move(valu Json::Type Json::type() const { return m_ptr->type(); } double Json::number_value() const { return m_ptr->number_value(); } -int Json::int_value() const { return m_ptr->int_value(); } +int Json::int_value() const { return m_ptr->int_value(); } +int64_ Json::int64_value() const { return m_ptr->int64_value(); } bool Json::bool_value() const { return m_ptr->bool_value(); } const string & Json::string_value() const { return m_ptr->string_value(); } const vector & Json::array_items() const { return m_ptr->array_items(); } @@ -269,7 +289,8 @@ const Json & Json::operator[] (size_t i) const { return (*m_ptr)[i]; const Json & Json::operator[] (const string &key) const { return (*m_ptr)[key]; } double JsonValue::number_value() const { return 0; } -int JsonValue::int_value() const { return 0; } +int JsonValue::int_value() const { return 0; } +int64_ JsonValue::int64_value() const { return 0; } bool JsonValue::bool_value() const { return false; } const string & JsonValue::string_value() const { return statics().empty_string; } const vector & JsonValue::array_items() const { return statics().empty_vector; } @@ -581,9 +602,12 @@ struct JsonParser final { return fail("invalid " + esc(str[i]) + " in number"); } - if (str[i] != '.' && str[i] != 'e' && str[i] != 'E' - && (i - start_pos) <= static_cast(std::numeric_limits::digits10)) { - return std::atoi(str.c_str() + start_pos); + if (str[i] != '.' && str[i] != 'e' && str[i] != 'E') + { + if ((i - start_pos) <= static_cast(std::numeric_limits::digits10)) + return (int)(std::atoll(str.c_str() + start_pos)); + if ((i - start_pos) <= static_cast(std::numeric_limits::digits10)) + return (int64_)(std::atoll(str.c_str() + start_pos)); } // Decimal part diff --git a/json11.hpp b/json11.hpp index 5202ef93..f26d4f2c 100644 --- a/json11.hpp +++ b/json11.hpp @@ -55,6 +55,7 @@ #include #include #include +#include #ifdef _MSC_VER #if _MSC_VER <= 1800 // VS 2013 @@ -76,6 +77,8 @@ enum JsonParse { class JsonValue; +using int64_ = int64_t; + class Json final { public: // Types @@ -91,7 +94,8 @@ public: Json() noexcept; // NUL Json(std::nullptr_t) noexcept; // NUL Json(double value); // NUMBER - Json(int value); // NUMBER + Json(int value); // NUMBER + Json(int64_ value); // NUMBER Json(bool value); // BOOL Json(const std::string &value); // STRING Json(std::string &&value); // STRING @@ -133,10 +137,11 @@ public: bool is_object() const { return type() == OBJECT; } // Return the enclosed value if this is a number, 0 otherwise. Note that json11 does not - // distinguish between integer and non-integer numbers - number_value() and int_value() + // distinguish between integer and non-integer numbers - number_value() and int64_value() // can both be applied to a NUMBER-typed object. double number_value() const; int int_value() const; + int64_ int64_value() const; // Return the enclosed value if this is a boolean, false otherwise. bool bool_value() const; @@ -213,6 +218,7 @@ class JsonValue { protected: friend class Json; friend class JsonInt; + friend class JsonInt64; friend class JsonDouble; virtual Json::Type type() const = 0; virtual bool equals(const JsonValue * other) const = 0; @@ -220,6 +226,7 @@ protected: virtual void dump(std::string &out) const = 0; virtual double number_value() const; virtual int int_value() const; + virtual int64_ int64_value() const; virtual bool bool_value() const; virtual const std::string &string_value() const; virtual const Json::array &array_items() const; diff --git a/test.cpp b/test.cpp index cf9a46fc..fee132cf 100644 --- a/test.cpp +++ b/test.cpp @@ -59,6 +59,28 @@ CHECK_TRAIT(is_nothrow_move_assignable); CHECK_TRAIT(is_nothrow_destructible); JSON11_TEST_CASE(json11_test) { + + std::cout << "int int64 test" << std::endl; + + const string str_it = R"({"message_id":105308320612483198,"msg_type":3,"order":0, + "ques_id":0,"session_id":105308187502051928,"site_id":122062, + "timestamp":1471140271, + "visitor_id":9941658010949867158,"worker_id":133746})"; + string err_it; + auto json_it = Json::parse(str_it, err_it); + + assert(json_it["message_id"].int64_value() == 105308320612483198); + assert(json_it["message_id"].int_value() != 105308320612483198); + assert(json_it["msg_type"].int64_value() == 3); + assert(json_it["msg_type"].int_value() == 3); + assert(json_it["order"].int64_value() == 0); + assert(json_it["session_id"].int64_value() == 105308187502051928); + assert(json_it["site_id"].int_value() == 122062); + assert(json_it["site_id"].int64_value() == 122062); + std::cout << (unsigned long long)json_it["visitor_id"].int64_value() << std::endl; + assert((unsigned long long)(json_it["visitor_id"].int64_value()) == 9941658010949867158); + std::cout << "int int64 test passed!" << std::endl; + const string simple_test = R"({"k1":"v1", "k2":42, "k3":["a",123,true,false,null]})"; From eb4d67a68ee4d4f1ca3fe874602cb4aaa2724d5e Mon Sep 17 00:00:00 2001 From: Zhijiang TAO Date: Tue, 16 Aug 2016 11:00:25 +0800 Subject: [PATCH 69/74] using longlong for internel storage --- json11.cpp | 75 +++++++++++++++++++++++++++++++++--------------------- json11.hpp | 14 +++++++--- 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/json11.cpp b/json11.cpp index ac9d727f..9f593a04 100644 --- a/json11.cpp +++ b/json11.cpp @@ -56,17 +56,18 @@ static void dump(double value, string &out) { } static void dump(int value, string &out) { - char buf[32]; - snprintf(buf, sizeof buf, "%d", value); - out += buf; + out += std::to_string(value); } static void dump(int64_ value, string &out) { - char buf[64]; - snprintf(buf, sizeof buf, "%ld", value); - out += buf; + out += std::to_string(value); } +static void dump(uint64_ value, string &out) { + out += std::to_string(value); +} + + static void dump(bool value, string &out) { out += value ? "true" : "false"; } @@ -168,35 +169,35 @@ protected: }; class JsonDouble final : public Value { - double number_value() const override { return m_value; } - int int_value() const override { return static_cast(m_value); } + double number_value() const override { return static_cast(m_value); } int64_ int64_value() const override { return static_cast(m_value); } + uint64_ uint64_value() const override { return static_cast(m_value); } bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: explicit JsonDouble(double value) : Value(value) {} }; -class JsonInt final : public Value { - double number_value() const override { return m_value; } - int int_value() const override { return m_value; } - int64_ int64_value() const override { return m_value; } - bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } - bool less(const JsonValue * other) const override { return m_value < other->number_value(); } -public: - explicit JsonInt(int value) : Value(value) {} -}; - class JsonInt64 final : public Value { - double number_value() const override { return m_value; } - int int_value() const override { return m_value; } - int64_ int64_value() const override { return m_value; } + double number_value() const override { return static_cast(m_value); } + int64_ int64_value() const override { return static_cast(m_value); } + uint64_ uint64_value() const override { return static_cast(m_value); } bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } bool less(const JsonValue * other) const override { return m_value < other->number_value(); } public: explicit JsonInt64(int64_ value) : Value(value) {} }; +class JsonUInt64 final : public Value { + double number_value() const override { return static_cast(m_value); } + int64_ int64_value() const override { return static_cast(m_value); } + uint64_ uint64_value() const override { return static_cast(m_value); } + bool equals(const JsonValue * other) const override { return m_value == other->number_value(); } + bool less(const JsonValue * other) const override { return m_value < other->number_value(); } +public: + explicit JsonUInt64(uint64_ value) : Value(value) {} +}; + class JsonBoolean final : public Value { bool bool_value() const override { return m_value; } public: @@ -262,8 +263,9 @@ static const Json & static_null() { Json::Json() noexcept : m_ptr(statics().null) {} Json::Json(std::nullptr_t) noexcept : m_ptr(statics().null) {} Json::Json(double value) : m_ptr(make_shared(value)) {} -Json::Json(int value) : m_ptr(make_shared(value)) {} +Json::Json(int value) : m_ptr(make_shared(value)) {} Json::Json(int64_ value) : m_ptr(make_shared(value)) {} +Json::Json(uint64_ value) : m_ptr(make_shared(value)) {} Json::Json(bool value) : m_ptr(value ? statics().t : statics().f) {} Json::Json(const string &value) : m_ptr(make_shared(value)) {} Json::Json(string &&value) : m_ptr(make_shared(move(value))) {} @@ -279,8 +281,8 @@ Json::Json(Json::object &&values) : m_ptr(make_shared(move(valu Json::Type Json::type() const { return m_ptr->type(); } double Json::number_value() const { return m_ptr->number_value(); } -int Json::int_value() const { return m_ptr->int_value(); } -int64_ Json::int64_value() const { return m_ptr->int64_value(); } +int64_ Json::int64_value() const { return m_ptr->int64_value(); } +uint64_ Json::uint64_value() const { return m_ptr->uint64_value(); } bool Json::bool_value() const { return m_ptr->bool_value(); } const string & Json::string_value() const { return m_ptr->string_value(); } const vector & Json::array_items() const { return m_ptr->array_items(); } @@ -289,8 +291,8 @@ const Json & Json::operator[] (size_t i) const { return (*m_ptr)[i]; const Json & Json::operator[] (const string &key) const { return (*m_ptr)[key]; } double JsonValue::number_value() const { return 0; } -int JsonValue::int_value() const { return 0; } int64_ JsonValue::int64_value() const { return 0; } +uint64_ JsonValue::uint64_value() const { return 0; } bool JsonValue::bool_value() const { return false; } const string & JsonValue::string_value() const { return statics().empty_string; } const vector & JsonValue::array_items() const { return statics().empty_vector; } @@ -604,10 +606,25 @@ struct JsonParser final { if (str[i] != '.' && str[i] != 'e' && str[i] != 'E') { - if ((i - start_pos) <= static_cast(std::numeric_limits::digits10)) - return (int)(std::atoll(str.c_str() + start_pos)); - if ((i - start_pos) <= static_cast(std::numeric_limits::digits10)) - return (int64_)(std::atoll(str.c_str() + start_pos)); + if (*(str.c_str() + start_pos) == '-') // signed value + { + if ((i - start_pos) <= (ceil(std::numeric_limits::digits * std::log10(2) + 1/*sign*/) )) + { + // On Linux, the global errno variable is thread-specific + int64_t conv_val = std::atoll(str.c_str() + start_pos); + if (errno != ERANGE) + return conv_val; + } + } + else + { + if ((i - start_pos) <= ceil(std::numeric_limits::digits * std::log10(2) )) + { + uint64_ conv_val = std::strtoull(str.c_str() + start_pos, 0, 10); + if (errno != ERANGE) + return conv_val; + } + } } // Decimal part diff --git a/json11.hpp b/json11.hpp index f26d4f2c..37d75e46 100644 --- a/json11.hpp +++ b/json11.hpp @@ -77,7 +77,12 @@ enum JsonParse { class JsonValue; +#if __cplusplus < 201103L + #error This project can only be compiled with a compiler that supports C++11 +#else using int64_ = int64_t; +using uint64_ = uint64_t; +#endif class Json final { public: @@ -94,8 +99,9 @@ public: Json() noexcept; // NUL Json(std::nullptr_t) noexcept; // NUL Json(double value); // NUMBER - Json(int value); // NUMBER + Json(int value); // NUMBER Json(int64_ value); // NUMBER + Json(uint64_ value); // NUMBER Json(bool value); // BOOL Json(const std::string &value); // STRING Json(std::string &&value); // STRING @@ -140,8 +146,8 @@ public: // distinguish between integer and non-integer numbers - number_value() and int64_value() // can both be applied to a NUMBER-typed object. double number_value() const; - int int_value() const; int64_ int64_value() const; + uint64_ uint64_value() const; // Return the enclosed value if this is a boolean, false otherwise. bool bool_value() const; @@ -217,16 +223,16 @@ private: class JsonValue { protected: friend class Json; - friend class JsonInt; friend class JsonInt64; + friend class JsonUInt64; friend class JsonDouble; virtual Json::Type type() const = 0; virtual bool equals(const JsonValue * other) const = 0; virtual bool less(const JsonValue * other) const = 0; virtual void dump(std::string &out) const = 0; virtual double number_value() const; - virtual int int_value() const; virtual int64_ int64_value() const; + virtual uint64_ uint64_value() const; virtual bool bool_value() const; virtual const std::string &string_value() const; virtual const Json::array &array_items() const; From f4ffe052d95dbc3ecb2b26d9d08bc4fc3254ca32 Mon Sep 17 00:00:00 2001 From: Zhijiang TAO Date: Tue, 16 Aug 2016 11:10:14 +0800 Subject: [PATCH 70/74] ADDING INT64 UINT64 test cases --- test.cpp | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/test.cpp b/test.cpp index fee132cf..d25d7481 100644 --- a/test.cpp +++ b/test.cpp @@ -41,6 +41,8 @@ #include #include +#include + // Insert user-defined prefix code (includes, function declarations, etc) // to set up a custom test suite JSON11_TEST_CPP_PREFIX_CODE @@ -70,17 +72,38 @@ JSON11_TEST_CASE(json11_test) { auto json_it = Json::parse(str_it, err_it); assert(json_it["message_id"].int64_value() == 105308320612483198); - assert(json_it["message_id"].int_value() != 105308320612483198); + assert(json_it["message_id"].uint64_value() == 105308320612483198); assert(json_it["msg_type"].int64_value() == 3); - assert(json_it["msg_type"].int_value() == 3); + assert(json_it["msg_type"].uint64_value() == 3); assert(json_it["order"].int64_value() == 0); assert(json_it["session_id"].int64_value() == 105308187502051928); - assert(json_it["site_id"].int_value() == 122062); assert(json_it["site_id"].int64_value() == 122062); - std::cout << (unsigned long long)json_it["visitor_id"].int64_value() << std::endl; + assert(json_it["site_id"].int64_value() == 122062); + std::cout << (signed long long)json_it["visitor_id"].int64_value() << std::endl; assert((unsigned long long)(json_it["visitor_id"].int64_value()) == 9941658010949867158); std::cout << "int int64 test passed!" << std::endl; + std::cout << "uint uint64 test" << std::endl; + const string str_it2 = R"({"int_max":2147483647,"int_min":-2147483648,"uint_max":4294967295, "uint_min":0, + "i64_max":9223372036854775807,"i64_min":-9223372036854775808,"ui64_max":18446744073709551615, "ui_64min":0 })"; + auto json_uit = Json::parse(str_it2, err_it); + std::cout << "ORIGIN:" << str_it2 << std::endl; + string str_it2_ret; + json_uit.dump(str_it2_ret); + std::cout << "DUMP:" << str_it2_ret << std::endl; + + assert(json_uit["int_max"].int64_value() == INT_MAX); + assert(json_uit["int_min"].int64_value() == INT_MIN); + assert(json_uit["uint_max"].uint64_value() == UINT_MAX); + assert(json_uit["uint_min"].uint64_value() == 0); + + assert(json_uit["i64_max"].int64_value() == LLONG_MAX); + assert(json_uit["i64_min"].int64_value() == LLONG_MIN); + assert(json_uit["ui64_max"].uint64_value() == ULLONG_MAX); + assert(json_uit["ui64_min"].uint64_value() == 0); + + std::cout << "int64 uint64 test passed!" << std::endl; + const string simple_test = R"({"k1":"v1", "k2":42, "k3":["a",123,true,false,null]})"; From a4da043b17f02face92c88e24e61b03f7434c205 Mon Sep 17 00:00:00 2001 From: Zhijiang TAO Date: Tue, 16 Aug 2016 11:56:09 +0800 Subject: [PATCH 71/74] update readme --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) mode change 100644 => 100755 README.md diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 21f4e8b2..f545d12b --- a/README.md +++ b/README.md @@ -1,6 +1,29 @@ json11 ------ +#### modified +The original json11 from Dropbox doesnot support integer more than 2^54 for the compatible with javascript cases. But in most development cases, int64_t and uint64_t are very important types. So modified to support these ones. + + std::cout << "uint uint64 test" << std::endl; + const string str_it2 = R"({"int_max":2147483647,"int_min":-2147483648,"uint_max":4294967295, "uint_min":0, "i64_max":9223372036854775807,"i64_min":-9223372036854775808,"ui64_max":18446744073709551615, "ui_64min":0 })"; + auto json_uit = Json::parse(str_it2, err_it); + std::cout << "ORIGIN:" << str_it2 << std::endl; + string str_it2_ret; + json_uit.dump(str_it2_ret); + std::cout << "DUMP:" << str_it2_ret << std::endl; + + assert(json_uit["int_max"].int64_value() == INT_MAX); + assert(json_uit["int_min"].int64_value() == INT_MIN); + assert(json_uit["uint_max"].uint64_value() == UINT_MAX); + assert(json_uit["uint_min"].uint64_value() == 0); + + assert(json_uit["i64_max"].int64_value() == LLONG_MAX); + assert(json_uit["i64_min"].int64_value() == LLONG_MIN); + assert(json_uit["ui64_max"].uint64_value() == ULLONG_MAX); + assert(json_uit["ui64_min"].uint64_value() == 0); + +#### end + json11 is a tiny JSON library for C++11, providing JSON parsing and serialization. The core object provided by the library is json11::Json. A Json object represents any JSON From 88de39206d2431c2aae5ad408a483f388a85eef2 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Mon, 23 Dec 2019 14:14:36 +0300 Subject: [PATCH 72/74] ull --- test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.cpp b/test.cpp index d25d7481..ca5ac889 100644 --- a/test.cpp +++ b/test.cpp @@ -80,7 +80,7 @@ JSON11_TEST_CASE(json11_test) { assert(json_it["site_id"].int64_value() == 122062); assert(json_it["site_id"].int64_value() == 122062); std::cout << (signed long long)json_it["visitor_id"].int64_value() << std::endl; - assert((unsigned long long)(json_it["visitor_id"].int64_value()) == 9941658010949867158); + assert((unsigned long long)(json_it["visitor_id"].int64_value()) == 9941658010949867158ull); std::cout << "int int64 test passed!" << std::endl; std::cout << "uint uint64 test" << std::endl; From 5dc108754ad40d3b1d024f9bd7cca0595ef1a1db Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Sat, 14 Mar 2020 02:13:05 +0300 Subject: [PATCH 73/74] Fix "Dereferencing type-punned pointer will break strict aliasing rules..." --- btree.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/btree.h b/btree.h index 49310a2e..f8749bb4 100644 --- a/btree.h +++ b/btree.h @@ -554,10 +554,12 @@ class btree_node { return params_type::key(fields_.values[i]); } reference value(int i) { - return reinterpret_cast(fields_.values[i]); + mutable_value_type *ptr = (mutable_value_type*)&fields_.values[i]; + return reinterpret_cast(*ptr); } const_reference value(int i) const { - return reinterpret_cast(fields_.values[i]); + value_type *ptr = (value_type*)&fields_.values[i]; + return reinterpret_cast(*ptr); } mutable_value_type* mutable_value(int i) { return &fields_.values[i]; From 97f06cb20c1e136fd37d58fb40f57dd8f8a3a4a7 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 24 Apr 2020 02:56:05 +0300 Subject: [PATCH 74/74] Add string->number conversions --- json11.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/json11.cpp b/json11.cpp index 9f593a04..d60c84ff 100644 --- a/json11.cpp +++ b/json11.cpp @@ -206,6 +206,9 @@ public: class JsonString final : public Value { const string &string_value() const override { return m_value; } + double number_value() const override { size_t pos = 0; double v = std::stod(m_value, &pos); if (pos < m_value.length()) { v = 0; } return v; } + int64_ int64_value() const override { size_t pos = 0; int64_ v = std::stoll(m_value, &pos); if (pos < m_value.length()) { v = 0; } return v; } + uint64_ uint64_value() const override { size_t pos = 0; uint64_ v = std::stoull(m_value, &pos); if (pos < m_value.length()) { v = 0; } return v; } public: explicit JsonString(const string &value) : Value(value) {} explicit JsonString(string &&value) : Value(move(value)) {}