Optimize all_of / any_of / none_of for vector<bool> with some predicates (#5802)

Co-authored-by: Stephan T. Lavavej <stl@nuwen.net>

Author: AlexGuteniev

benchmarks/CMakeLists.txt

@@ -144,5 +144,6 @@ add_benchmark(unique src/unique.cpp)
 add_benchmark(vector_bool_copy src/vector_bool_copy.cpp)
 add_benchmark(vector_bool_copy_n src/vector_bool_copy_n.cpp)
 add_benchmark(vector_bool_count src/vector_bool_count.cpp)
+add_benchmark(vector_bool_meow_of src/vector_bool_meow_of.cpp)
 add_benchmark(vector_bool_move src/vector_bool_move.cpp)
 add_benchmark(vector_bool_transform src/vector_bool_transform.cpp)

benchmarks/src/vector_bool_meow_of.cpp

@@ -0,0 +1,54 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <benchmark/benchmark.h>
+//
+#include <algorithm>
+#include <cstddef>
+#include <functional>
+#include <vector>
+
+#include "skewed_allocator.hpp"
+
+using namespace std;
+
+enum class alg { all_, any_, none_ };
+enum class content { ones_then_zeros, zeros_then_ones };
+
+template <alg Alg, content Content, class Pred = identity>
+void meow_of(benchmark::State& state) {
+    const auto n = static_cast<size_t>(state.range(0));
+    vector<bool, not_highly_aligned_allocator<bool>> source(n);
+
+    if constexpr (Content == content::ones_then_zeros) {
+        fill(source.begin(), source.begin() + source.size() / 2, true);
+    } else {
+        fill(source.begin() + source.size() / 2, source.end(), true);
+    }
+
+    for (auto _ : state) {
+        benchmark::DoNotOptimize(source);
+        bool result;
+        if constexpr (Alg == alg::all_) {
+            result = all_of(source.begin(), source.end(), Pred{});
+        } else if constexpr (Alg == alg::any_) {
+            result = any_of(source.begin(), source.end(), Pred{});
+        } else {
+            result = none_of(source.begin(), source.end(), Pred{});
+        }
+        benchmark::DoNotOptimize(result);
+    }
+}
+
+void common_args(benchmark::Benchmark* bm) {
+    bm->RangeMultiplier(64)->Range(64, 64 << 10);
+}
+
+using not_ = logical_not<>;
+
+BENCHMARK(meow_of<alg::all_, content::ones_then_zeros>)->Apply(common_args);
+BENCHMARK(meow_of<alg::any_, content::zeros_then_ones>)->Apply(common_args);
+BENCHMARK(meow_of<alg::any_, content::ones_then_zeros, not_>)->Apply(common_args);
+BENCHMARK(meow_of<alg::none_, content::zeros_then_ones>)->Apply(common_args);
+
+BENCHMARK_MAIN();

stl/inc/algorithm

@@ -1619,18 +1619,30 @@ namespace ranges {
 } // namespace ranges
 #endif // _HAS_CXX20
 
+struct _All_of_vbool_traits;
+struct _Any_of_vbool_traits;
+struct _None_of_vbool_traits;
+
+template <class _Traits, class _VbIt, class _Mapped_fn>
+_NODISCARD _CONSTEXPR20 bool _Meow_of_vbool(_VbIt _First, _VbIt _Last, _Mapped_fn _Mapped_func);
+
 _EXPORT_STD template <class _InIt, class _Pr>
 _NODISCARD _CONSTEXPR20 bool all_of(_InIt _First, _InIt _Last, _Pr _Pred) { // test if all elements satisfy _Pred
     _STD _Adl_verify_range(_First, _Last);
     auto _UFirst      = _STD _Get_unwrapped(_First);
     const auto _ULast = _STD _Get_unwrapped(_Last);
-    for (; _UFirst != _ULast; ++_UFirst) {
-        if (!_Pred(*_UFirst)) {
-            return false;
+
+    if constexpr (_Is_vb_iterator<decltype(_UFirst)> && !is_void_v<_Map_vb_functor_t<_Pr>>) {
+        return _Meow_of_vbool<_All_of_vbool_traits>(_UFirst, _ULast, _Map_vb_functor_t<_Pr>{});
+    } else {
+        for (; _UFirst != _ULast; ++_UFirst) {
+            if (!_Pred(*_UFirst)) {
+                return false;
+            }
         }
-    }
 
-    return true;
+        return true;
+    }
 }
 
 #if _HAS_CXX17
@@ -1686,13 +1698,18 @@ _NODISCARD _CONSTEXPR20 bool any_of(const _InIt _First, const _InIt _Last, _Pr _
     _STD _Adl_verify_range(_First, _Last);
     auto _UFirst      = _STD _Get_unwrapped(_First);
     const auto _ULast = _STD _Get_unwrapped(_Last);
-    for (; _UFirst != _ULast; ++_UFirst) {
-        if (_Pred(*_UFirst)) {
-            return true;
+
+    if constexpr (_Is_vb_iterator<decltype(_UFirst)> && !is_void_v<_Map_vb_functor_t<_Pr>>) {
+        return _Meow_of_vbool<_Any_of_vbool_traits>(_UFirst, _ULast, _Map_vb_functor_t<_Pr>{});
+    } else {
+        for (; _UFirst != _ULast; ++_UFirst) {
+            if (_Pred(*_UFirst)) {
+                return true;
+            }
         }
-    }
 
-    return false;
+        return false;
+    }
 }
 
 #if _HAS_CXX17
@@ -1748,13 +1765,17 @@ _NODISCARD _CONSTEXPR20 bool none_of(const _InIt _First, const _InIt _Last, _Pr
     _STD _Adl_verify_range(_First, _Last);
     auto _UFirst      = _STD _Get_unwrapped(_First);
     const auto _ULast = _STD _Get_unwrapped(_Last);
-    for (; _UFirst != _ULast; ++_UFirst) {
-        if (_Pred(*_UFirst)) {
-            return false;
+    if constexpr (_Is_vb_iterator<decltype(_UFirst)> && !is_void_v<_Map_vb_functor_t<_Pr>>) {
+        return _Meow_of_vbool<_None_of_vbool_traits>(_UFirst, _ULast, _Map_vb_functor_t<_Pr>{});
+    } else {
+        for (; _UFirst != _ULast; ++_UFirst) {
+            if (_Pred(*_UFirst)) {
+                return false;
+            }
         }
-    }
 
-    return true;
+        return true;
+    }
 }
 
 #if _HAS_CXX17

stl/inc/vector

@@ -4051,6 +4051,79 @@ _CONSTEXPR20 _OutIt _Transform_vbool_aligned(
     return _Dest;
 }
 
+struct _All_of_vbool_traits {
+    static constexpr bool _Default_result = true;
+
+    static _CONSTEXPR20 bool _Check(const _Vbase _Value) {
+        return _Value != ~_Vbase{0};
+    }
+
+    static _CONSTEXPR20 bool _Check(const _Vbase _Value, const _Vbase _Mask) {
+        return (_Value & _Mask) != _Mask;
+    }
+};
+
+struct _Any_of_vbool_traits_base {
+    static _CONSTEXPR20 bool _Check(const _Vbase _Value) {
+        return _Value != 0;
+    }
+
+    static _CONSTEXPR20 bool _Check(const _Vbase _Value, const _Vbase _Mask) {
+        return (_Value & _Mask) != 0;
+    }
+};
+
+struct _Any_of_vbool_traits : _Any_of_vbool_traits_base {
+    static constexpr bool _Default_result = false;
+};
+
+struct _None_of_vbool_traits : _Any_of_vbool_traits_base {
+    static constexpr bool _Default_result = true;
+};
+
+template <class _Traits, class _VbIt, class _Mapped_fn>
+_NODISCARD _CONSTEXPR20 bool _Meow_of_vbool(const _VbIt _First, const _VbIt _Last, const _Mapped_fn _Mapped_func) {
+    constexpr bool _Early_result = !_Traits::_Default_result;
+    auto _First_ptr              = _First._Myptr;
+    const auto _Last_ptr         = _Last._Myptr;
+
+    if (_First_ptr == _Last_ptr) {
+        if (_First._Myoff == _Last._Myoff) { // empty, can't read the word
+            return _Traits::_Default_result;
+        }
+
+        const _Vbase _Mask = (_Vbase{1} << _Last._Myoff) - (_Vbase{1} << _First._Myoff); // handle partial single word
+        if (_Traits::_Check(_Mapped_func(*_First_ptr), _Mask)) {
+            return _Early_result;
+        }
+        return _Traits::_Default_result;
+    }
+
+    if (_First._Myoff != 0) { // if we have a partial first word, handle it
+        const _Vbase _Mask = static_cast<_Vbase>(-1) << _First._Myoff;
+        if (_Traits::_Check(_Mapped_func(*_First_ptr), _Mask)) {
+            return _Early_result;
+        }
+
+        ++_First_ptr;
+    }
+
+    for (; _First_ptr != _Last_ptr; ++_First_ptr) { // handle full words
+        if (_Traits::_Check(_Mapped_func(*_First_ptr))) {
+            return _Early_result;
+        }
+    }
+
+    if (_Last._Myoff != 0) { // if we have a partial last word, handle it
+        const _Vbase _Mask = (_Vbase{1} << _Last._Myoff) - 1;
+        if (_Traits::_Check(_Mapped_func(*_First_ptr), _Mask)) {
+            return _Early_result;
+        }
+    }
+
+    return _Traits::_Default_result;
+}
+
 #undef _ASAN_VECTOR_MODIFY
 #undef _ASAN_VECTOR_REMOVE
 #undef _ASAN_VECTOR_CREATE

stl/inc/xutility

@@ -5028,6 +5028,13 @@ struct _Map_vb_functor {
     using type = void;
 };
 
+#if _HAS_CXX20
+template <>
+struct _Map_vb_functor<identity> {
+    using type = identity;
+};
+#endif // _HAS_CXX20
+
 template <class _Fn>
 using _Map_vb_functor_t = typename _Map_vb_functor<_Fn>::type;
 

tests/std/tests/GH_000625_vector_bool_optimization/test.cpp

@@ -233,6 +233,121 @@ CONSTEXPR20 bool test_transform() {
     return true;
 }
 
+CONSTEXPR20 bool test_meow_of_helper(const size_t length_before, const size_t length, const size_t length_after) {
+    const size_t total_length = length_before + length + length_after;
+
+    vector<bool> zeros(total_length);
+    vector<bool> ones(total_length);
+    vector<bool> mix(total_length);
+
+    const auto first_0 = zeros.begin() + static_cast<ptrdiff_t>(length_before);
+    const auto last_0  = zeros.end() - static_cast<ptrdiff_t>(length_after);
+    const auto first_1 = ones.begin() + static_cast<ptrdiff_t>(length_before);
+    const auto last_1  = ones.end() - static_cast<ptrdiff_t>(length_after);
+    const auto first_m = mix.begin() + static_cast<ptrdiff_t>(length_before);
+    const auto last_m  = mix.end() - static_cast<ptrdiff_t>(length_after);
+
+    fill(zeros.begin(), first_0, true);
+    fill(last_0, zeros.end(), true);
+    fill(first_1, last_1, true);
+    fill(mix.begin(), first_m, true);
+    fill(first_m + static_cast<ptrdiff_t>(length / 2), last_m, true);
+
+    if (length == 0) {
+#if _HAS_CXX20
+        assert(all_of(first_0, last_0, identity{}) == true);
+        assert(all_of(first_1, last_1, identity{}) == true);
+        assert(all_of(first_m, last_m, identity{}) == true);
+
+        assert(any_of(first_0, last_0, identity{}) == false);
+        assert(any_of(first_1, last_1, identity{}) == false);
+        assert(any_of(first_m, last_m, identity{}) == false);
+
+        assert(none_of(first_0, last_0, identity{}) == true);
+        assert(none_of(first_1, last_1, identity{}) == true);
+        assert(none_of(first_m, last_m, identity{}) == true);
+#endif // _HAS_CXX20
+
+        assert(all_of(first_0, last_0, logical_not<>{}) == true);
+        assert(all_of(first_1, last_1, logical_not<>{}) == true);
+        assert(all_of(first_m, last_m, logical_not<>{}) == true);
+
+        assert(any_of(first_0, last_0, logical_not<>{}) == false);
+        assert(any_of(first_1, last_1, logical_not<>{}) == false);
+        assert(any_of(first_m, last_m, logical_not<>{}) == false);
+
+        assert(none_of(first_0, last_0, logical_not<>{}) == true);
+        assert(none_of(first_1, last_1, logical_not<>{}) == true);
+        assert(none_of(first_m, last_m, logical_not<>{}) == true);
+    } else {
+        assert(length != 1); // [first_m, last_m) needs to contain both true and false
+
+#if _HAS_CXX20
+        assert(all_of(first_0, last_0, identity{}) == false);
+        assert(all_of(first_1, last_1, identity{}) == true);
+        assert(all_of(first_m, last_m, identity{}) == false);
+
+        assert(any_of(first_0, last_0, identity{}) == false);
+        assert(any_of(first_1, last_1, identity{}) == true);
+        assert(any_of(first_m, last_m, identity{}) == true);
+
+        assert(none_of(first_0, last_0, identity{}) == true);
+        assert(none_of(first_1, last_1, identity{}) == false);
+        assert(none_of(first_m, last_m, identity{}) == false);
+#endif // _HAS_CXX20
+
+        assert(all_of(first_0, last_0, logical_not<>{}) == true);
+        assert(all_of(first_1, last_1, logical_not<>{}) == false);
+        assert(all_of(first_m, last_m, logical_not<>{}) == false);
+
+        assert(any_of(first_0, last_0, logical_not<>{}) == true);
+        assert(any_of(first_1, last_1, logical_not<>{}) == false);
+        assert(any_of(first_m, last_m, logical_not<>{}) == true);
+
+        assert(none_of(first_0, last_0, logical_not<>{}) == false);
+        assert(none_of(first_1, last_1, logical_not<>{}) == true);
+        assert(none_of(first_m, last_m, logical_not<>{}) == false);
+    }
+
+    return true;
+}
+
+CONSTEXPR20 bool test_meow_of() {
+    { // Super empty range
+        const vector<bool>::const_iterator it{}; // value-initialized, compares equal to itself
+
+#if _HAS_CXX20
+        assert(all_of(it, it, identity{}) == true);
+        assert(any_of(it, it, identity{}) == false);
+        assert(none_of(it, it, identity{}) == true);
+#endif // _HAS_CXX20
+
+        assert(all_of(it, it, logical_not<>{}) == true);
+        assert(any_of(it, it, logical_not<>{}) == false);
+        assert(none_of(it, it, logical_not<>{}) == true);
+    }
+
+    // Empty range
+    test_meow_of_helper(0, 0, 3);
+    test_meow_of_helper(3, 0, 3);
+
+    // One block, ends within block
+    test_meow_of_helper(0, 10, 3);
+    test_meow_of_helper(3, 10, 3);
+
+    // One block, exactly
+    test_meow_of_helper(0, blockSize, 0);
+
+    // Multiple blocks, spanning
+    test_meow_of_helper(3, blockSize - 2, 3);
+    test_meow_of_helper(3, blockSize + 2, 3);
+
+    // Many blocks, exactly
+    test_meow_of_helper(blockSize, 4 * blockSize, blockSize);
+
+    return true;
+}
+
 CONSTEXPR20 void test_fill_helper(const size_t length) {
     // No offset
     {
@@ -1590,6 +1705,7 @@ static_assert(test_fill());
 static_assert(test_find());
 static_assert(test_count());
 static_assert(test_transform());
+static_assert(test_meow_of());
 
 #if defined(__clang__) || defined(__EDG__) // TRANSITION, VSO-2574489
 static_assert(test_copy_part_1());
@@ -1602,6 +1718,7 @@ int main() {
     test_find();
     test_count();
     test_transform();
+    test_meow_of();
     test_copy_part_1();
     test_copy_part_2();