154concept SegmentFn = std::invocable<Fn, IndexMaskSegment>;
161concept RangeFn = std::invocable<Fn, IndexRange>;
300 template<
typename Fn>
320 template<
typename T,
typename Fn>
323 Fn &&get_group_index,
356 std::optional<RawMaskIterator>
find(
int64_t query_index)
const;
449 template<ForeachIndexFn Fn, exec_mode::Tag Mode = exec_mode::Serial>
457 template<
typename IndexT, ForeachIndexFn Fn, exec_mode::Tag Mode = exec_mode::Serial>
471 template<ForeachSegmentFn Fn, exec_mode::Tag Mode = exec_mode::Serial>
483 template<ForeachSegmentOrRangePosFn Fn, exec_mode::Tag Mode = exec_mode::Serial>
518 std::optional<IndexRange>
to_range()
const;
533 template<
int64_t N = 4>
553 const int16_t *segment_indices_;
554 std::array<int64_t, 2> cumulative_segment_sizes_;
565 cumulative_segment_sizes_[0] = 0;
566 data.segments_num_ = 1;
567 data.indices_by_segment_ = &segment_indices_;
568 data.segment_offsets_ = &segment_offset_;
569 data.cumulative_segment_sizes_ = cumulative_segment_sizes_.data();
570 data.begin_index_in_segment_ = 0;
583 segment_offset_ = segment.offset();
584 segment_indices_ = indices.
data();
585 cumulative_segment_sizes_[1] =
int16_t(indices_num);
586 data.indices_num_ = indices_num;
587 data.end_index_in_segment_ = indices_num;
602 alignas(64)
static const std::array<int16_t, max_segment_size>
data =
695 static constexpr int64_t cumulative_sizes_for_empty_mask[1] = {0};
697 data.indices_num_ = 0;
698 data.segments_num_ = 0;
699 data.cumulative_segment_sizes_ = cumulative_sizes_for_empty_mask;
787 return this->
last() + 1;
799 [&](
const int64_t cumulative_size) {
return cumulative_size <= full_index; });
840 const int64_t segment_size = end_index - begin_index;
855template<ForeachIndexFn Fn, exec_mode::Tag Mode>
858 if constexpr (mode.is_parallel) {
861 const IndexMask sub_mask = this->slice(range);
862 sub_mask.foreach_index([&](const int64_t i, [[maybe_unused]] const int64_t index_pos) {
863 if constexpr (std::is_invocable_r_v<void, Fn, int64_t, int64_t>) {
864 fn(i, index_pos + range.start());
873 this->foreach_segment(
877 fn(indices[i], start_segment_pos + i);
889template<
typename T,
typename Fn>
890#if (defined(__GNUC__) && !defined(__clang__))
891[[gnu::optimize(
"O3")]]
895 BLI_assert(segment.last() < std::numeric_limits<T>::max());
897 const T start =
T(segment[0]);
898 const T last =
T(segment.last());
899 for (
T i = start; i <= last; i++) {
904 for (
const int64_t i : segment) {
910template<
typename T,
typename Fn>
911#if (defined(__GNUC__) && !defined(__clang__))
912[[gnu::optimize(
"O3")]]
918 BLI_assert(segment.last() < std::numeric_limits<T>::max());
919 BLI_assert(segment.size() + segment_pos < std::numeric_limits<T>::max());
921 const T start =
T(segment[0]);
922 const T last =
T(segment.last());
923 for (
T i = start, pos =
T(segment_pos); i <= last; i++, pos++) {
928 T pos =
T(segment_pos);
929 for (
const int64_t i : segment.index_range()) {
937template<
typename IndexT, ForeachIndexFn Fn, exec_mode::Tag Mode>
940 if constexpr (mode.is_parallel) {
943 const IndexMask sub_mask = this->slice(range);
944 sub_mask.foreach_segment([&](const IndexMaskSegment segment,
945 [[maybe_unused]] const int64_t segment_pos) {
946 if constexpr (IndexPosFn<Fn>) {
947 optimized_foreach_index_with_pos<IndexT>(segment, segment_pos + range.start(), fn);
950 optimized_foreach_index<IndexT>(segment, fn);
956 this->foreach_segment(
968template<ForeachSegmentOrRangePosFn Fn, exec_mode::Tag Mode>
971 if constexpr (mode.is_parallel) {
974 const IndexMask sub_mask = this->slice(range);
975 sub_mask.foreach_segment_optimized(
976 [&fn, range_start = range.start()](
977 const auto segment, [[maybe_unused]] const int64_t start_segment_pos) {
978 if constexpr (SegmentOrRangePosFn<Fn>) {
979 fn(segment, start_segment_pos + range_start);
988 this->foreach_segment(
991 const IndexRange range(segment[0], segment.size());
993 fn(range, start_segment_pos);
1001 fn(segment, start_segment_pos);
1012template<ForeachSegmentFn Fn>
1015 if constexpr (mode.is_parallel) {
1018 const IndexMask sub_mask = this->slice(range);
1019 sub_mask.foreach_segment(
1020 [&fn, range_start = range.start()](const IndexMaskSegment mask_segment,
1021 [[maybe_unused]] const int64_t segment_pos) {
1022 if constexpr (SegmentPosFn<Fn>) {
1023 fn(mask_segment, segment_pos + range_start);
1032 [[maybe_unused]]
int64_t segment_pos = 0;
1036 fn(segment, segment_pos);
1037 segment_pos += segment.size();
1054 fn(range, segment_pos);
1059 segment_pos += next_range_size;
1060 base_indices = base_indices.
drop_front(next_range_size);
1073template<
typename Fn>
1083 int16_t *r_current = r_true_indices;
1086 for (
const int16_t *in_current = indices.
base_span().data(); in_current < in_end;
1089 const int16_t local_index = *in_current;
1091 const bool condition = predicate(global_index);
1092 *r_current = local_index;
1099 const int16_t true_indices_num =
int16_t(r_current - r_true_indices);
1100 return true_indices_num;
1105template<
typename T,
typename Fn>
1108 Fn &&get_group_index,
1111 if (r_masks.
size() == 1) {
1115 r_masks[0] = universe;
1121 const int group_index = get_group_index(i);
1122 indices_by_group[group_index].
append(
T(i));
1146 return std::nullopt;
1178template<
typename T,
typename SegmentT>
1179#if (defined(__GNUC__) && !defined(__clang__))
1180[[gnu::optimize(
"-funroll-loops")]] [[gnu::optimize(
"O3")]]
1183 const SegmentT segment,
1186 for (
int16_t i = 0; i < segment.size(); i++) {
1187 dst[i] = src[segment[i]];
1191template<
typename T,
typename SegmentT>
1192#if (defined(__GNUC__) && !defined(__clang__))
1193[[gnu::optimize(
"-funroll-loops")]] [[gnu::optimize(
"O3")]]
1197 for (
const int64_t i : segment) {
1202template<
typename T,
typename SegmentT>
1203#if (defined(__GNUC__) && !defined(__clang__))
1204[[gnu::optimize(
"-funroll-loops")]] [[gnu::optimize(
"O3")]]
1208 if constexpr (std::is_same_v<SegmentT, IndexRange>) {
1209 if constexpr (std::is_trivially_copy_assignable_v<T>) {
1214#if defined(__GNUC__) && !defined(__clang__)
1215# pragma GCC diagnostic push
1216# pragma GCC diagnostic ignored "-Wclass-memaccess"
1219#if defined(__GNUC__) && !defined(__clang__)
1220# pragma GCC diagnostic pop
1226 for (
const int64_t i : segment) {
1234 mask.foreach_segment_optimized(
1245 [src, dst](
const auto segment,
const int64_t segment_pos) {
1254 mask.foreach_segment_optimized(
1298 const float probability,
1303 const float probability,
1310using index_mask::IndexMask;
1311using index_mask::IndexMaskFromSegment;
1312using index_mask::IndexMaskMemory;
1313using index_mask::IndexMaskSegment;
const IndexMask & update(IndexMaskSegment segment)
IndexMaskSegment slice(const IndexRange &range) const
IndexMaskSegment shift(const int64_t shift) const
IndexMaskSegment(const OffsetSpan< int64_t, int16_t > span)
void foreach_range(ForeachRangeFn auto &&fn) const
void foreach_index_optimized(Fn &&fn, Mode mode=exec_mode::serial) const
IndexMaskSegment segment(int64_t segment_i) const
static IndexMask from_predicate(const IndexMask &universe, LinearAllocator<> &memory, Fn &&predicate, exec_mode::Mode mode=exec_mode::parallel)
std::optional< IndexRange > to_range() const
IndexMask slice(IndexRange range) const
void foreach_segment(Fn &&fn, Mode mode=exec_mode::serial) const
IndexRange bounds() const
Vector< T > to_indices() const
int64_t operator[](int64_t i) const
int64_t iterator_to_index(const RawMaskIterator &it) const
void foreach_segment_optimized(Fn &&fn, Mode mode=exec_mode::serial) const
int64_t min_array_size() const
IndexRange index_range() const
Vector< std::variant< IndexRange, IndexMaskSegment >, N > to_spans_and_ranges() const
IndexMaskData & data_for_inplace_construction()
static void from_groups(const IndexMask &universe, LinearAllocator<> &memory, Fn &&get_group_index, MutableSpan< IndexMask > r_masks)
void foreach_index(Fn &&fn, Mode mode=exec_mode::serial) const
static IndexMask from_indices(Span< T > indices, LinearAllocator<> &memory)
RawMaskIterator index_to_iterator(int64_t index) const
int64_t segments_num() const
constexpr int64_t first() const
constexpr int64_t one_after_last() const
constexpr int64_t last(const int64_t n=0) const
constexpr int64_t size() const
constexpr bool is_empty() const
constexpr IndexRange()=default
constexpr int64_t start() const
static constexpr IndexRange from_begin_end_inclusive(const int64_t begin, const int64_t last)
void provide_buffer(void *buffer, const int64_t size)
constexpr int64_t size() const
constexpr T * data() const
constexpr IndexRange index_range() const
IndexRange index_range() const
Span< int16_t > base_span() const
constexpr Span drop_front(int64_t n) const
constexpr const T * data() const
constexpr int64_t size() const
constexpr const T & last(const int64_t n=0) const
constexpr const T * end() const
constexpr const T * begin() const
constexpr bool is_empty() const
void append(const T &value)
NonCopyable(const NonCopyable &other)=delete
NonMovable(NonMovable &&other)=delete
const IndexMask & update(IndexMaskSegment segment)
IndexMaskSegment slice(const IndexRange &range) const
IndexMaskSegment shift(const int64_t shift) const
IndexMaskSegment(const OffsetSpan< int64_t, int16_t > span)
static IndexMask from_segments(Span< IndexMaskSegment > segments, LinearAllocator<> &memory)
void foreach_range(ForeachRangeFn auto &&fn) const
void foreach_index_optimized(Fn &&fn, Mode mode=exec_mode::serial) const
IndexMaskSegment segment(int64_t segment_i) const
static IndexMask from_predicate(const IndexMask &universe, LinearAllocator<> &memory, Fn &&predicate, exec_mode::Mode mode=exec_mode::parallel)
IndexMask slice_content(IndexRange range) const
Vector< IndexRange > to_ranges_invert(IndexRange universe) const
Vector< IndexRange > to_ranges() const
std::optional< RawMaskIterator > find(int64_t query_index) const
static IndexMask from_bools(Span< bool > bools, LinearAllocator<> &memory)
std::optional< IndexRange > to_range() const
IndexMask shift(const int64_t offset, LinearAllocator<> &memory) const
static IndexMask from_difference(const IndexMask &mask_a, const IndexMask &mask_b, LinearAllocator<> &memory)
static IndexMask from_ranges(OffsetIndices< T > offsets, const IndexMask &mask, LinearAllocator<> &memory)
std::optional< RawMaskIterator > find_larger_equal(int64_t query_index) const
IndexMask slice(IndexRange range) const
void foreach_segment(Fn &&fn, Mode mode=exec_mode::serial) const
IndexRange bounds() const
int64_t operator[](int64_t i) const
int64_t iterator_to_index(const RawMaskIterator &it) const
static IndexMask from_union(const IndexMask &mask_a, const IndexMask &mask_b, LinearAllocator<> &memory)
void to_indices(MutableSpan< T > r_indices) const
static IndexMask from_repeating(const IndexMask &mask_to_repeat, int64_t repetitions, int64_t stride, int64_t initial_offset, LinearAllocator<> &memory)
void foreach_segment_optimized(Fn &&fn, Mode mode=exec_mode::serial) const
int64_t min_array_size() const
IndexRange index_range() const
void set_bits(MutableBitSpan r_bits, int64_t offset=0) const
static IndexMask from_bits(BitSpan bits, LinearAllocator<> &memory)
Vector< std::variant< IndexRange, IndexMaskSegment >, N > to_spans_and_ranges() const
std::optional< RawMaskIterator > find_smaller_equal(int64_t query_index) const
friend bool operator!=(const IndexMask &a, const IndexMask &b)
IndexMaskData & data_for_inplace_construction()
bool contains(int64_t query_index) const
static void from_groups(const IndexMask &universe, LinearAllocator<> &memory, Fn &&get_group_index, MutableSpan< IndexMask > r_masks)
void foreach_index(Fn &&fn, Mode mode=exec_mode::serial) const
static IndexMask from_bools_inverse(Span< bool > bools, LinearAllocator<> &memory)
void to_bits(MutableBitSpan r_bits, int64_t offset=0) const
IndexMask complement(const IndexMask &universe, LinearAllocator<> &memory) const
static IndexMask from_indices(Span< T > indices, LinearAllocator<> &memory)
static IndexMask from_batch_predicate(const IndexMask &universe, LinearAllocator<> &memory, FunctionRef< int64_t(const IndexMaskSegment &universe_segment, IndexRangesBuilder< int16_t > &builder)> batch_predicate, exec_mode::Mode mode=exec_mode::parallel)
friend bool operator==(const IndexMask &a, const IndexMask &b)
static IndexMask from_every_nth(int64_t n, int64_t indices_num, const int64_t initial_offset, LinearAllocator<> &memory)
void to_bools(MutableSpan< bool > r_bools) const
static IndexMask from_initializers(const Span< Initializer > initializers, LinearAllocator<> &memory)
RawMaskIterator index_to_iterator(int64_t index) const
static void foreach_segment_zipped(Span< IndexMask > masks, FunctionRef< bool(Span< IndexMaskSegment > segments)> fn)
int64_t segments_num() const
std::variant< IndexRange, Span< int64_t >, Span< int >, int64_t > Initializer
IndexMask slice_and_shift(IndexRange range, int64_t offset, LinearAllocator<> &memory) const
static IndexMask from_intersection(const IndexMask &mask_a, const IndexMask &mask_b, LinearAllocator<> &memory)
static Vector< IndexMask, 4 > from_group_ids(const VArray< int > &group_ids, LinearAllocator<> &memory)
int64_t last_if(const Range &range, Predicate &&predicate)
constexpr Parallel parallel
void gather_assign_segment(const T *__restrict src, const SegmentT segment, T *__restrict dst)
void copy_assign_segment(const T *__restrict src, const SegmentT segment, T *__restrict dst)
void fill_segment(T *__restrict data, const T &value, const SegmentT segment)
BLI_NOINLINE void fill(T *__restrict data, const T &value, const IndexMask &mask)
BLI_NOINLINE void copy_assign(const T *__restrict src, const IndexMask &mask, T *__restrict dst)
BLI_NOINLINE void gather_assign(const T *__restrict src, const IndexMask &indices, T *__restrict dst)
IndexMask from_predicate_impl(const IndexMask &universe, LinearAllocator<> &memory, FunctionRef< int64_t(IndexMaskSegment indices, int16_t *r_true_indices)> filter_indices, exec_mode::Mode mode)
void copy_assign(const Span< T > src, const IndexMask &mask, const MutableSpan< T > dst)
IndexMask random_mask(const IndexMask &mask, const int64_t universe_size, const uint32_t random_seed, const float probability, LinearAllocator<> &memory)
void index_range_to_mask_segments(const IndexRange range, Vector< IndexMaskSegment, N > &r_segments)
static constexpr int64_t max_segment_size
static constexpr int64_t max_segment_size_mask_high
void gather_assign(const Span< T > src, const IndexMask &mask, const MutableSpan< T > dst)
void build_reverse_map(const IndexMask &mask, MutableSpan< T > r_map)
static constexpr int64_t max_segment_size_mask_low
std::ostream & operator<<(std::ostream &stream, const IndexMask &mask)
void optimized_foreach_index(const IndexMaskSegment segment, const Fn fn)
bool operator==(const RawMaskIterator &a, const RawMaskIterator &b)
std::array< int16_t, max_segment_size > build_static_indices_array()
void optimized_foreach_index_with_pos(const IndexMaskSegment segment, const int64_t segment_pos, const Fn fn)
const IndexMask & get_static_index_mask_for_min_size(const int64_t min_size)
bool operator!=(const RawMaskIterator &a, const RawMaskIterator &b)
int64_t consolidate_index_mask_segments(MutableSpan< IndexMaskSegment > segments, LinearAllocator<> &memory)
const std::array< int16_t, max_segment_size > & get_static_indices_array()
void masked_fill(MutableSpan< T > data, const T &value, const IndexMask &mask)
void init_empty_mask(IndexMaskData &data)
static constexpr int64_t max_segment_size_shift
void parallel_for(const IndexRange range, const int64_t grain_size, const Function &function, const TaskSizeHints &size_hints=detail::TaskSizeHints_Static(1))
int64_t find_size_of_next_range(const Span< T > indices)
bool non_empty_is_range(const Span< T > indices)
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
constexpr bool value_is_zero_memory(const VecBase< T, Size > &v)
BMesh const char void * data
const int64_t * cumulative_segment_sizes_
const int16_t ** indices_by_segment_
int64_t end_index_in_segment_
int64_t begin_index_in_segment_
const int64_t * segment_offsets_