64 template<
typename... ParamTags,
typename... LoadedParams,
size_t...
I>
66 std::index_sequence<I...> ,
67 const std::tuple<LoadedParams...> &loaded_params)
const
69 return std::make_tuple([&]() {
73 const GVArrayImpl &varray_impl = *std::get<I>(loaded_params);
80 T *ptr = std::get<I>(loaded_params);
96 template<
typename... ParamTags,
typename... LoadedParams,
size_t...
I>
98 std::index_sequence<I...> ,
99 const std::tuple<LoadedParams...> &loaded_params)
const
101 return std::make_tuple([&]() {
106 constexpr bool UseSpan =
ValueSequence<size_t, Indices...>::template contains<I>();
107 const GVArrayImpl &varray_impl = *std::get<I>(loaded_params);
114 T *ptr = std::get<I>(loaded_params);
130template<
typename MaskT,
typename... Args,
typename ElementFn>
133#if (defined(__GNUC__) && !defined(__clang__))
134[[gnu::optimize(
"-funroll-loops")]] [[gnu::optimize(
"O3")]]
140 Args &&__restrict... args)
142 if constexpr (std::is_same_v<std::decay_t<MaskT>,
IndexRange>) {
146 for (
int64_t i = start; i < end; i++) {
147 element_fn(args[i]...);
152 element_fn(args[i]...);
172template<
typename... ParamTags,
typename ElementFn,
typename... Chunks>
173#if (defined(__GNUC__) && !defined(__clang__))
174[[gnu::optimize(
"-funroll-loops")]] [[gnu::optimize(
"O3")]]
177 const ElementFn &element_fn,
179 Chunks &&__restrict... chunks)
181 for (
int64_t i = 0; i < size; i++) {
182 element_fn(chunks[i]...);
191template<
typename... ParamTags,
size_t...
I,
typename ElementFn,
typename... LoadedParams>
193 std::index_sequence<I...> ,
194 const ElementFn &element_fn,
196 const std::tuple<LoadedParams...> &loaded_params)
203 static constexpr int64_t MaxChunkSize = 64;
205 const int64_t tmp_buffer_size = std::min(mask_size, MaxChunkSize);
208 std::tuple<TypedBuffer<typename ParamTags::base_type, MaxChunkSize>...> temporary_buffers;
211 std::tuple<MaterializeArgInfo<ParamTags>...> args_info;
221 const GVArrayImpl &varray_impl = *std::get<I>(loaded_params);
226 const T &in_single = *
static_cast<const T *
>(common_info.
data);
227 T *tmp_buffer = std::get<I>(temporary_buffers).ptr();
246 for (
int64_t chunk_start = 0; chunk_start < mask_size; chunk_start += MaxChunkSize) {
247 const int64_t chunk_end = std::min<int64_t>(chunk_start + MaxChunkSize, mask_size);
248 const int64_t chunk_size = chunk_end - chunk_start;
250 const int64_t mask_start = sliced_mask[0];
255 if (!sliced_mask_is_range) {
262 T *tmp_buffer = std::get<I>(temporary_buffers).ptr();
263 T *param_buffer = std::get<I>(loaded_params);
264 for (
int64_t i = 0; i < chunk_size; i++) {
265 new (tmp_buffer + i)
T(std::move(param_buffer[sliced_mask[i]]));
272 const IndexMask *current_segment_mask =
nullptr;
283 T *tmp_buffer = std::get<I>(temporary_buffers);
287 return const_cast<const T *
>(tmp_buffer);
295 const GVArrayImpl &varray_impl = *std::get<I>(loaded_params);
296 if (current_segment_mask ==
nullptr) {
297 current_segment_mask = &index_mask_from_segment.
update(
298 {segment_offset, sliced_mask.
base_span()});
306 return const_cast<const T *
>(tmp_buffer);
313 if (sliced_mask_is_range) {
315 T *param_buffer = std::get<I>(loaded_params);
316 return param_buffer + mask_start;
325 if (!sliced_mask_is_range) {
335 T *tmp_buffer = std::get<I>(temporary_buffers).ptr();
336 T *param_buffer = std::get<I>(loaded_params);
337 for (
int64_t i = 0; i < chunk_size; i++) {
338 new (param_buffer + sliced_mask[i])
T(std::move(tmp_buffer[i]));
339 std::destroy_at(tmp_buffer + i);
355 T *tmp_buffer = std::get<I>(temporary_buffers).ptr();
372 T *tmp_buffer = std::get<I>(temporary_buffers).ptr();
380template<
typename ElementFn,
typename ExecPreset,
typename... ParamTags,
size_t...
I>
382 const ExecPreset exec_preset,
386 std::index_sequence<I...> )
391 const auto loaded_params = std::make_tuple([&]() {
408 bool executed_devirtualized =
false;
409 if constexpr (ExecPreset::use_devirtualization) {
412 mask.to_spans_and_ranges<16>();
414 const auto devirtualizers = exec_preset.create_devirtualizers(
417 devirtualizers, [&](
auto &&...args) {
418 for (
const std::variant<IndexRange, IndexMaskSegment> &segment : mask_segments) {
419 if (std::holds_alternative<IndexRange>(segment)) {
420 const auto segment_range = std::get<IndexRange>(segment);
421 execute_array(element_fn, segment_range, std::forward<
decltype(args)>(args)...);
424 const auto segment_indices = std::get<IndexMaskSegment>(segment);
425 execute_array(element_fn, segment_indices, std::forward<
decltype(args)>(args)...);
436 if (!executed_devirtualized) {
442 std::index_sequence<I...>(),
456 const GVArrayImpl &varray_impl = *std::get<I>(loaded_params);
457 return GVArray(&varray_impl).typed<
T>();
463 T *ptr = std::get<I>(loaded_params);
478template<
typename ElementFn,
typename ExecPreset,
typename... ParamTags>
480 const ExecPreset exec_preset,
483 return [element_fn = std::forward<ElementFn>(element_fn), exec_preset](
const IndexMask &
mask,
490 std::make_index_sequence<
sizeof...(ParamTags)>());
504 : call_fn_(std::move(call_fn))
508 ([&] { builder.
add(ParamTags(),
""); }(), ...);
514 call_fn_(
mask, params);
518template<
typename Out,
typename... In,
typename ElementFn,
typename ExecPreset>
520 ElementFn &&element_fn,
521 const ExecPreset exec_preset,
527 [element_fn = std::forward<ElementFn>(element_fn)](
const In &...
in, Out &
out) {
528 new (&
out) Out(element_fn(
in...));
532 return CustomMF(name, std::move(call_fn), param_tags);
535template<
typename Out1,
typename Out2,
typename... In,
typename ElementFn,
typename ExecPreset>
537 ElementFn &&element_fn,
538 const ExecPreset exec_preset,
545 std::forward<ElementFn>(element_fn), exec_preset, param_tags);
546 return CustomMF(name, call_fn, param_tags);
552template<
typename In1,
557 ElementFn &&element_fn,
565template<
typename In1,
571 ElementFn &&element_fn,
579template<
typename In1,
586 ElementFn &&element_fn,
594template<
typename In1,
602 ElementFn &&element_fn,
610template<
typename In1,
619 ElementFn &&element_fn,
624 std::forward<ElementFn>(element_fn),
630template<
typename In1,
640 ElementFn &&element_fn,
645 std::forward<ElementFn>(element_fn),
651template<
typename In1,
663 ElementFn &&element_fn,
668 std::forward<ElementFn>(element_fn),
674template<
typename Mut1,
typename ElementFn,
typename ExecPreset = exec_presets::AllSpanOrSingle>
675inline auto SM(
const char *name,
676 ElementFn &&element_fn,
681 std::forward<ElementFn>(element_fn), exec_preset, param_tags);
686template<
typename In1,
692 ElementFn &&element_fn,
700template<
typename In1,
707 ElementFn &&element_fn,
715template<
typename In1,
723 ElementFn &&element_fn,
731template<
typename In1,
740 ElementFn &&element_fn,
748template<
typename In1,
758 ElementFn &&element_fn,
763 std::forward<ElementFn>(element_fn),
769template<
typename In1,
776 ElementFn &&element_fn,
784 std::forward<ElementFn>(element_fn), exec_preset, param_tags);
789template<
typename In1,
797 ElementFn &&element_fn,
806 std::forward<ElementFn>(element_fn), exec_preset, param_tags);
886 if (other1 !=
nullptr) {
887 return value_ == other1->value_;
891 if (other2 !=
nullptr) {
893 if (type == other2->type_) {
894 return type.
is_equal_or_false(
static_cast<const void *
>(&value_), other2->value_);
static const CPPType & get()
bool is_equal_or_false(const void *a, const void *b) const
virtual void materialize_compressed(const IndexMask &mask, void *dst, bool dst_is_uninitialized) const
virtual CommonVArrayInfo common_info() const
Span< BaseT > base_span() const
CustomMF_Constant(U &&value)
bool equals(const MultiFunction &other) const override
void hash_unique(UniqueHashBytes &hash) const override
void call(const IndexMask &mask, Params params, Context) const override
CustomMF_DefaultOutput(Span< DataType > input_types, Span< DataType > output_types)
void call(const IndexMask &mask, Params params, Context context) const override
void call(const IndexMask &mask, Params params, Context context) const override
CustomMF_GenericConstantArray(GSpan array)
CustomMF_GenericConstant(const CPPType &type, const void *value, bool make_value_copy)
bool equals(const MultiFunction &other) const override
static constexpr int8_t HASH_ID
friend class CustomMF_Constant
void hash_unique(UniqueHashBytes &hash) const override
void call(const IndexMask &mask, Params params, Context context) const override
CustomMF_GenericCopy(DataType data_type)
void call(const IndexMask &mask, Params params, Context context) const override
StringRefNull name() const
void set_signature(const Signature *signature)
MutableSpan< T > uninitialized_single_output(int param_index, StringRef name="")
MutableSpan< T > single_mutable(int param_index, StringRef name="")
VArray< T > readonly_single_input(int param_index, StringRef name="")
void add(ParamTag< Category, T >, const char *name)
void single_output(const char *name, const ParamFlag flag=ParamFlag::None)
CustomMF(const char *name, CallFn call_fn, TypeSequence< ParamTags... >)
void call(const IndexMask &mask, Params params, Context) const override
const IndexMask & update(IndexMaskSegment segment)
auto build_multi_function_with_n_inputs_one_output(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset, TypeSequence< In... >)
auto build_multi_function_with_n_inputs_two_outputs(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset, TypeSequence< In... >)
auto build_multi_function_call_from_element_fn(ElementFn &&element_fn, const ExecPreset exec_preset, TypeSequence< ParamTags... >)
void execute_element_fn_as_multi_function(const ElementFn &element_fn, const ExecPreset exec_preset, const IndexMask &mask, Params params, TypeSequence< ParamTags... >, std::index_sequence< I... >)
void execute_array(const ElementFn &element_fn, MaskT mask, Args &&__restrict... args)
void execute_materialized_impl(TypeSequence< ParamTags... >, const ElementFn &element_fn, const int64_t size, Chunks &&__restrict... chunks)
void execute_materialized(TypeSequence< ParamTags... >, std::index_sequence< I... >, const ElementFn &element_fn, const IndexMaskSegment mask, const std::tuple< LoadedParams... > &loaded_params)
auto SI1_SO2(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SI4_SO2(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SI4_SO(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SI2_SO(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SI5_SO(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SI8_SO(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SI1_SO3(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SM(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::AllSpanOrSingle())
auto SI1_SO4(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SI2_SO2(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SI5_SO2(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SI3_SO(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SI1_SO(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SI3_SO2(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
auto SI6_SO(const char *name, ElementFn &&element_fn, const ExecPreset exec_preset=exec_presets::Materialized())
bool non_empty_is_range(const Span< T > indices)
bool call_with_devirtualized_parameters(const std::tuple< Devirtualizers... > &devis, const Fn &fn)
BMesh const char void * data
void uninitialized_fill_n(T *dst, int64_t n, const T &value)
void hash_unique_default(const T &value, UniqueHashBytes &hash)
void destruct_n(T *ptr, int64_t n)
static constexpr ParamCategory category
const ParamTag::base_type * internal_span_data
static constexpr FallbackMode fallback_mode
auto create_devirtualizers(TypeSequence< ParamTags... >, std::index_sequence< I... >, const std::tuple< LoadedParams... > &loaded_params) const
static constexpr bool use_devirtualization
static constexpr FallbackMode fallback_mode
static constexpr bool use_devirtualization
static constexpr bool use_devirtualization
static constexpr FallbackMode fallback_mode
auto create_devirtualizers(TypeSequence< ParamTags... >, std::index_sequence< I... >, const std::tuple< LoadedParams... > &loaded_params) const
static constexpr FallbackMode fallback_mode
static constexpr bool use_devirtualization