Blender V5.3
multi_function_procedure_executor.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2023 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
10
11#include "BLI_stack.hh"
12
14
15ProcedureExecutor::ProcedureExecutor(const Procedure &procedure) : procedure_(procedure)
16{
17 SignatureBuilder builder("Procedure Executor", signature_);
18
19 for (const ConstParameter &param : procedure.params()) {
20 builder.add("Parameter", ParamType(param.type, param.variable->data_type()));
21 }
22
23 this->set_signature(&signature_);
24}
25
26using IndicesSplitVectors = std::array<Vector<int64_t>, 2>;
27
28namespace {
29enum class ValueType {
30 GVArray = 0,
31 Span = 1,
32 GVVectorArray = 2,
33 GVectorArray = 3,
34 OneSingle = 4,
35 OneVector = 5,
36};
37constexpr int tot_variable_value_types = 6;
38} // namespace
39
45 ValueType type;
46
47 VariableValue(ValueType type) : type(type) {}
48};
49
50/* This variable is the unmodified virtual array from the caller. */
52 static constexpr ValueType static_type = ValueType::GVArray;
53 const GVArray &data;
54
59};
60
61/* This variable has a different value for every index. Some values may be uninitialized. The span
62 * may be owned by the caller. */
64 static constexpr ValueType static_type = ValueType::Span;
65 void *data;
66 bool owned;
67
71};
72
73/* This variable is the unmodified virtual vector array from the caller. */
75 static constexpr ValueType static_type = ValueType::GVVectorArray;
77
81};
82
83/* This variable has a different vector for every index. */
85 static constexpr ValueType static_type = ValueType::GVectorArray;
87 bool owned;
88
93};
94
95/* This variable has the same value for every index. */
97 static constexpr ValueType static_type = ValueType::OneSingle;
98 void *data;
99 bool is_initialized = false;
100
102};
103
104/* This variable has the same vector for every index. */
106 static constexpr ValueType static_type = ValueType::OneVector;
108
110};
111
112static_assert(std::is_trivially_destructible_v<VariableValue_GVArray>);
113static_assert(std::is_trivially_destructible_v<VariableValue_Span>);
114static_assert(std::is_trivially_destructible_v<VariableValue_GVVectorArray>);
115static_assert(std::is_trivially_destructible_v<VariableValue_GVectorArray>);
116static_assert(std::is_trivially_destructible_v<VariableValue_OneSingle>);
117static_assert(std::is_trivially_destructible_v<VariableValue_OneVector>);
118
119class VariableState;
120
126 private:
131 static constexpr int min_alignment = 64;
132
134 LinearAllocator<> &linear_allocator_;
135
140 std::array<Stack<VariableValue *>, tot_variable_value_types> variable_value_free_lists_;
141
146 Stack<void *> small_span_buffers_free_list_;
147 Map<int, Stack<void *>> span_buffers_free_lists_;
148
150 static constexpr int small_value_max_size = 16;
151 static constexpr int small_value_max_alignment = 8;
152 Stack<void *> small_single_value_free_list_;
153 Map<const CPPType *, Stack<void *>> single_value_free_lists_;
154
155 public:
157
159 {
160 return this->obtain<VariableValue_GVArray>(varray);
161 }
162
164 {
165 return this->obtain<VariableValue_GVVectorArray>(varray);
166 }
167
169 {
170 return this->obtain<VariableValue_Span>(buffer, false);
171 }
172
173 VariableValue_Span *obtain_Span(const CPPType &type, int size)
174 {
175 void *buffer = nullptr;
176
177 const int64_t element_size = type.size;
178 const int64_t alignment = type.alignment;
179
180 if (alignment > min_alignment) {
181 /* In this rare case we fall back to not reusing existing buffers. */
182 buffer = linear_allocator_.allocate(element_size * size, alignment);
183 }
184 else {
185 Stack<void *> *stack = type.can_exist_in_buffer(small_value_max_size,
186 small_value_max_alignment) ?
187 &small_span_buffers_free_list_ :
188 span_buffers_free_lists_.lookup_ptr(element_size);
189 if (stack == nullptr || stack->is_empty()) {
190 buffer = linear_allocator_.allocate(
191 std::max<int64_t>(element_size, small_value_max_size) * size, min_alignment);
192 }
193 else {
194 /* Reuse existing buffer. */
195 buffer = stack->pop();
196 }
197 }
198
199 return this->obtain<VariableValue_Span>(buffer, true);
200 }
201
203 {
204 return this->obtain<VariableValue_GVectorArray>(data, false);
205 }
206
208 {
209 GVectorArray *vector_array = new GVectorArray(type, size);
210 return this->obtain<VariableValue_GVectorArray>(*vector_array, true);
211 }
212
214 {
215 const bool is_small = type.can_exist_in_buffer(small_value_max_size,
216 small_value_max_alignment);
217 Stack<void *> &stack = is_small ? small_single_value_free_list_ :
218 single_value_free_lists_.lookup_or_add_default(&type);
219 void *buffer;
220 if (stack.is_empty()) {
221 buffer = linear_allocator_.allocate(
222 std::max<int>(small_value_max_size, type.size),
223 std::max<int>(small_value_max_alignment, type.alignment));
224 }
225 else {
226 buffer = stack.pop();
227 }
228 return this->obtain<VariableValue_OneSingle>(buffer);
229 }
230
232 {
233 GVectorArray *vector_array = new GVectorArray(type, 1);
234 return this->obtain<VariableValue_OneVector>(*vector_array);
235 }
236
237 void release_value(VariableValue *value, const DataType &data_type)
238 {
239 switch (value->type) {
240 case ValueType::GVArray: {
241 break;
242 }
243 case ValueType::Span: {
244 auto *value_typed = static_cast<VariableValue_Span *>(value);
245 if (value_typed->owned) {
246 const CPPType &type = data_type.single_type();
247 /* Assumes all values in the buffer are uninitialized already. */
248 Stack<void *> &buffers = type.can_exist_in_buffer(small_value_max_size,
249 small_value_max_alignment) ?
250 small_span_buffers_free_list_ :
251 span_buffers_free_lists_.lookup_or_add_default(type.size);
252 buffers.push(value_typed->data);
253 }
254 break;
255 }
256 case ValueType::GVVectorArray: {
257 break;
258 }
259 case ValueType::GVectorArray: {
260 auto *value_typed = static_cast<VariableValue_GVectorArray *>(value);
261 if (value_typed->owned) {
262 delete &value_typed->data;
263 }
264 break;
265 }
266 case ValueType::OneSingle: {
267 auto *value_typed = static_cast<VariableValue_OneSingle *>(value);
268 const CPPType &type = data_type.single_type();
269 if (value_typed->is_initialized) {
270 type.destruct(value_typed->data);
271 }
272 const bool is_small = type.can_exist_in_buffer(small_value_max_size,
273 small_value_max_alignment);
274 if (is_small) {
275 small_single_value_free_list_.push(value_typed->data);
276 }
277 else {
278 single_value_free_lists_.lookup_or_add_default(&type).push(value_typed->data);
279 }
280 break;
281 }
282 case ValueType::OneVector: {
283 auto *value_typed = static_cast<VariableValue_OneVector *>(value);
284 delete &value_typed->data;
285 break;
286 }
287 }
288
289 Stack<VariableValue *> &stack = variable_value_free_lists_[int(value->type)];
290 stack.push(value);
291 }
292
293 private:
294 template<typename T, typename... Args> T *obtain(Args &&...args)
295 {
296 static_assert(std::is_base_of_v<VariableValue, T>);
297 Stack<VariableValue *> &stack = variable_value_free_lists_[int(T::static_type)];
298 if (stack.is_empty()) {
299 void *buffer = linear_allocator_.allocate(sizeof(T), alignof(T));
300 return new (buffer) T(std::forward<Args>(args)...);
301 }
302 return new (stack.pop()) T(std::forward<Args>(args)...);
303 }
304};
305
310 public:
315 /* This a non-owning pointer to either span buffer or #GVectorArray or null. */
317
318 void destruct_value(ValueAllocator &value_allocator, const DataType &data_type)
319 {
320 value_allocator.release_value(value_, data_type);
321 value_ = nullptr;
322 }
323
324 /* True if this contains only one value for all indices, i.e. the value for all indices is
325 * the same. */
326 bool is_one() const
327 {
328 if (value_ == nullptr) {
329 return true;
330 }
331 switch (value_->type) {
332 case ValueType::GVArray:
333 return this->value_as<VariableValue_GVArray>()->data.is_single();
334 case ValueType::Span:
335 return tot_initialized_ == 0;
336 case ValueType::GVVectorArray:
337 return this->value_as<VariableValue_GVVectorArray>()->data.is_single_vector();
338 case ValueType::GVectorArray:
339 return tot_initialized_ == 0;
340 case ValueType::OneSingle:
341 return true;
342 case ValueType::OneVector:
343 return true;
344 }
346 return false;
347 }
348
349 bool is_fully_initialized(const IndexMask &full_mask)
350 {
351 return tot_initialized_ == full_mask.size();
352 }
353
354 bool is_fully_uninitialized(const IndexMask &full_mask)
355 {
356 UNUSED_VARS(full_mask);
357 return tot_initialized_ == 0;
358 }
359
360 void add_as_input(ParamsBuilder &params, const IndexMask &mask, const DataType &data_type) const
361 {
362 /* Sanity check to make sure that enough values are initialized. */
364 BLI_assert(value_ != nullptr);
365
366 switch (value_->type) {
367 case ValueType::GVArray: {
369 break;
370 }
371 case ValueType::Span: {
372 const void *data = this->value_as<VariableValue_Span>()->data;
373 const GSpan span{data_type.single_type(), data, mask.min_array_size()};
374 params.add_readonly_single_input(span);
375 break;
376 }
377 case ValueType::GVVectorArray: {
379 break;
380 }
381 case ValueType::GVectorArray: {
383 break;
384 }
385 case ValueType::OneSingle: {
386 const auto *value_typed = this->value_as<VariableValue_OneSingle>();
387 BLI_assert(value_typed->is_initialized);
388 const GPointer gpointer{data_type.single_type(), value_typed->data};
389 params.add_readonly_single_input(gpointer);
390 break;
391 }
392 case ValueType::OneVector: {
394 break;
395 }
396 }
397 }
398
399 void ensure_is_mutable(const IndexMask &full_mask,
400 const DataType &data_type,
401 ValueAllocator &value_allocator)
402 {
403 if (value_ != nullptr && ELEM(value_->type, ValueType::Span, ValueType::GVectorArray)) {
404 return;
405 }
406
407 const int array_size = full_mask.min_array_size();
408
409 switch (data_type.category()) {
410 case DataType::Single: {
411 const CPPType &type = data_type.single_type();
412 VariableValue_Span *new_value = nullptr;
413 if (caller_provided_storage_ == nullptr) {
414 new_value = value_allocator.obtain_Span(type, array_size);
415 }
416 else {
417 /* Reuse the storage provided caller when possible. */
418 new_value = value_allocator.obtain_Span_not_owned(caller_provided_storage_);
419 }
420 if (value_ != nullptr) {
421 if (value_->type == ValueType::GVArray) {
422 /* Fill new buffer with data from virtual array. */
423 this->value_as<VariableValue_GVArray>()->data.materialize_to_uninitialized(
424 full_mask, new_value->data);
425 }
426 else if (value_->type == ValueType::OneSingle) {
427 auto *old_value_typed_ = this->value_as<VariableValue_OneSingle>();
428 if (old_value_typed_->is_initialized) {
429 /* Fill the buffer with a single value. */
430 type.fill_construct_indices(old_value_typed_->data, new_value->data, full_mask);
431 }
432 }
433 else {
435 }
436 value_allocator.release_value(value_, data_type);
437 }
438 value_ = new_value;
439 break;
440 }
441 case DataType::Vector: {
442 const CPPType &type = data_type.vector_base_type();
443 VariableValue_GVectorArray *new_value = nullptr;
444 if (caller_provided_storage_ == nullptr) {
445 new_value = value_allocator.obtain_GVectorArray(type, array_size);
446 }
447 else {
448 new_value = value_allocator.obtain_GVectorArray_not_owned(
449 *static_cast<GVectorArray *>(caller_provided_storage_));
450 }
451 if (value_ != nullptr) {
452 if (value_->type == ValueType::GVVectorArray) {
453 /* Fill new vector array with data from virtual vector array. */
454 new_value->data.extend(full_mask, this->value_as<VariableValue_GVVectorArray>()->data);
455 }
456 else if (value_->type == ValueType::OneVector) {
457 /* Fill all indices with the same value. */
458 const GSpan vector = this->value_as<VariableValue_OneVector>()->data[0];
459 new_value->data.extend(full_mask, GVVectorArray_For_SingleGSpan{vector, array_size});
460 }
461 else {
463 }
464 value_allocator.release_value(value_, data_type);
465 }
466 value_ = new_value;
467 break;
468 }
469 }
470 }
471
473 const IndexMask &mask,
474 const IndexMask &full_mask,
475 const DataType &data_type,
476 ValueAllocator &value_allocator)
477 {
478 /* Sanity check to make sure that enough values are initialized. */
480
481 this->ensure_is_mutable(full_mask, data_type, value_allocator);
482 BLI_assert(value_ != nullptr);
483
484 switch (value_->type) {
485 case ValueType::Span: {
486 void *data = this->value_as<VariableValue_Span>()->data;
487 const GMutableSpan span{data_type.single_type(), data, mask.min_array_size()};
488 params.add_single_mutable(span);
489 break;
490 }
491 case ValueType::GVectorArray: {
493 break;
494 }
495 case ValueType::GVArray:
496 case ValueType::GVVectorArray:
497 case ValueType::OneSingle:
498 case ValueType::OneVector: {
500 break;
501 }
502 }
503 }
504
506 const IndexMask &mask,
507 const IndexMask &full_mask,
508 const DataType &data_type,
509 ValueAllocator &value_allocator)
510 {
511 /* Sanity check to make sure that enough values are not initialized. */
512 BLI_assert(mask.size() <= full_mask.size() - tot_initialized_);
513 this->ensure_is_mutable(full_mask, data_type, value_allocator);
514 BLI_assert(value_ != nullptr);
515
516 switch (value_->type) {
517 case ValueType::Span: {
518 void *data = this->value_as<VariableValue_Span>()->data;
519 const GMutableSpan span{data_type.single_type(), data, mask.min_array_size()};
521 break;
522 }
523 case ValueType::GVectorArray: {
525 break;
526 }
527 case ValueType::GVArray:
528 case ValueType::GVVectorArray:
529 case ValueType::OneSingle:
530 case ValueType::OneVector: {
532 break;
533 }
534 }
535
536 tot_initialized_ += mask.size();
537 }
538
539 void add_as_input__one(ParamsBuilder &params, const DataType &data_type) const
540 {
541 BLI_assert(this->is_one());
542 BLI_assert(value_ != nullptr);
543
544 switch (value_->type) {
545 case ValueType::GVArray: {
547 break;
548 }
549 case ValueType::GVVectorArray: {
551 break;
552 }
553 case ValueType::OneSingle: {
554 const auto *value_typed = this->value_as<VariableValue_OneSingle>();
555 BLI_assert(value_typed->is_initialized);
556 GPointer ptr{data_type.single_type(), value_typed->data};
557 params.add_readonly_single_input(ptr);
558 break;
559 }
560 case ValueType::OneVector: {
562 break;
563 }
564 case ValueType::Span:
565 case ValueType::GVectorArray: {
567 break;
568 }
569 }
570 }
571
572 void ensure_is_mutable__one(const DataType &data_type, ValueAllocator &value_allocator)
573 {
574 BLI_assert(this->is_one());
575 if (value_ != nullptr && ELEM(value_->type, ValueType::OneSingle, ValueType::OneVector)) {
576 return;
577 }
578
579 switch (data_type.category()) {
580 case DataType::Single: {
581 const CPPType &type = data_type.single_type();
582 VariableValue_OneSingle *new_value = value_allocator.obtain_OneSingle(type);
583 if (value_ != nullptr) {
584 if (value_->type == ValueType::GVArray) {
585 this->value_as<VariableValue_GVArray>()->data.get_internal_single_to_uninitialized(
586 new_value->data);
587 new_value->is_initialized = true;
588 }
589 else if (value_->type == ValueType::Span) {
591 /* Nothing to do, the single value is uninitialized already. */
592 }
593 else {
595 }
596 value_allocator.release_value(value_, data_type);
597 }
598 value_ = new_value;
599 break;
600 }
601 case DataType::Vector: {
602 const CPPType &type = data_type.vector_base_type();
603 VariableValue_OneVector *new_value = value_allocator.obtain_OneVector(type);
604 if (value_ != nullptr) {
605 if (value_->type == ValueType::GVVectorArray) {
606 const GVVectorArray &old_vector_array =
608 new_value->data.extend(IndexRange(1), old_vector_array);
609 }
610 else if (value_->type == ValueType::GVectorArray) {
612 /* Nothing to do. */
613 }
614 else {
616 }
617 value_allocator.release_value(value_, data_type);
618 }
619 value_ = new_value;
620 break;
621 }
622 }
623 }
624
626 const DataType &data_type,
627 ValueAllocator &value_allocator)
628 {
629 BLI_assert(this->is_one());
630 this->ensure_is_mutable__one(data_type, value_allocator);
631 BLI_assert(value_ != nullptr);
632
633 switch (value_->type) {
634 case ValueType::OneSingle: {
635 auto *value_typed = this->value_as<VariableValue_OneSingle>();
636 BLI_assert(value_typed->is_initialized);
637 params.add_single_mutable(GMutableSpan{data_type.single_type(), value_typed->data, 1});
638 break;
639 }
640 case ValueType::OneVector: {
642 break;
643 }
644 case ValueType::GVArray:
645 case ValueType::Span:
646 case ValueType::GVVectorArray:
647 case ValueType::GVectorArray: {
649 break;
650 }
651 }
652 }
653
655 const IndexMask &mask,
656 const DataType &data_type,
657 ValueAllocator &value_allocator)
658 {
659 BLI_assert(this->is_one());
660 this->ensure_is_mutable__one(data_type, value_allocator);
661 BLI_assert(value_ != nullptr);
662
663 switch (value_->type) {
664 case ValueType::OneSingle: {
665 auto *value_typed = this->value_as<VariableValue_OneSingle>();
666 BLI_assert(!value_typed->is_initialized);
668 GMutableSpan{data_type.single_type(), value_typed->data, 1});
669 /* It becomes initialized when the multi-function is called. */
670 value_typed->is_initialized = true;
671 break;
672 }
673 case ValueType::OneVector: {
674 auto *value_typed = this->value_as<VariableValue_OneVector>();
675 BLI_assert(value_typed->data[0].is_empty());
676 params.add_vector_output(value_typed->data);
677 break;
678 }
679 case ValueType::GVArray:
680 case ValueType::Span:
681 case ValueType::GVVectorArray:
682 case ValueType::GVectorArray: {
684 break;
685 }
686 }
687
688 tot_initialized_ += mask.size();
689 }
690
697 const IndexMask &full_mask,
698 const DataType &data_type,
699 ValueAllocator &value_allocator)
700 {
701 BLI_assert(value_ != nullptr);
702 int new_tot_initialized = tot_initialized_ - mask.size();
703
704 /* Sanity check to make sure that enough indices can be destructed. */
705 BLI_assert(new_tot_initialized >= 0);
706
707 switch (value_->type) {
708 case ValueType::GVArray: {
709 if (mask.size() < full_mask.size()) {
710 /* Not all elements are destructed. Since we can't work on the original array, we have to
711 * create a copy first. */
712 this->ensure_is_mutable(full_mask, data_type, value_allocator);
713 BLI_assert(value_->type == ValueType::Span);
714 const CPPType &type = data_type.single_type();
716 }
717 break;
718 }
719 case ValueType::Span: {
720 const CPPType &type = data_type.single_type();
722 break;
723 }
724 case ValueType::GVVectorArray: {
725 if (mask.size() < full_mask.size()) {
726 /* Not all elements are cleared. Since we can't work on the original vector array, we
727 * have to create a copy first. A possible future optimization is to create the partial
728 * copy directly. */
729 this->ensure_is_mutable(full_mask, data_type, value_allocator);
730 BLI_assert(value_->type == ValueType::GVectorArray);
731 this->value_as<VariableValue_GVectorArray>()->data.clear(mask);
732 }
733 break;
734 }
735 case ValueType::GVectorArray: {
736 this->value_as<VariableValue_GVectorArray>()->data.clear(mask);
737 break;
738 }
739 case ValueType::OneSingle: {
740 auto *value_typed = this->value_as<VariableValue_OneSingle>();
741 BLI_assert(value_typed->is_initialized);
742 UNUSED_VARS_NDEBUG(value_typed);
743 if (mask.size() == tot_initialized_) {
744 const CPPType &type = data_type.single_type();
745 type.destruct(value_typed->data);
746 value_typed->is_initialized = false;
747 }
748 break;
749 }
750 case ValueType::OneVector: {
751 auto *value_typed = this->value_as<VariableValue_OneVector>();
752 if (mask.size() == tot_initialized_) {
753 value_typed->data.clear(IndexRange(1));
754 }
755 break;
756 }
757 }
758
759 tot_initialized_ = new_tot_initialized;
760
761 const bool should_self_destruct = new_tot_initialized == 0 &&
762 caller_provided_storage_ == nullptr;
763 return should_self_destruct;
764 }
765
767 {
769 BLI_assert(value_ != nullptr);
770
771 switch (value_->type) {
772 case ValueType::GVArray: {
773 const VArray<bool> varray = this->value_as<VariableValue_GVArray>()->data.typed<bool>();
774 mask.foreach_index([&](const int64_t i) { r_indices[varray[i]].append(i); });
775 break;
776 }
777 case ValueType::Span: {
778 const Span<bool> span(
779 static_cast<const bool *>(this->value_as<VariableValue_Span>()->data),
780 mask.min_array_size());
781 mask.foreach_index([&](const int64_t i) { r_indices[span[i]].append(i); });
782 break;
783 }
784 case ValueType::OneSingle: {
785 auto *value_typed = this->value_as<VariableValue_OneSingle>();
786 BLI_assert(value_typed->is_initialized);
787 const bool condition = *static_cast<const bool *>(value_typed->data);
788 Vector<int64_t> &indices = r_indices[condition];
789 indices.reserve(indices.size() + mask.size());
790 mask.foreach_index_optimized<int64_t>([&](const int64_t i) { indices.append(i); });
791 break;
792 }
793 case ValueType::GVVectorArray:
794 case ValueType::GVectorArray:
795 case ValueType::OneVector: {
797 break;
798 }
799 }
800 }
801
802 template<typename T> T *value_as()
803 {
804 BLI_assert(value_ != nullptr);
805 BLI_assert(value_->type == T::static_type);
806 return static_cast<T *>(value_);
807 }
808
809 template<typename T> const T *value_as() const
810 {
811 BLI_assert(value_ != nullptr);
812 BLI_assert(value_->type == T::static_type);
813 return static_cast<T *>(value_);
814 }
815};
816
819 private:
820 ValueAllocator value_allocator_;
821 const Procedure &procedure_;
823 Array<VariableState> variable_states_;
824 const IndexMask &full_mask_;
825
826 public:
828 const Procedure &procedure,
829 const IndexMask &full_mask)
830 : value_allocator_(linear_allocator),
831 procedure_(procedure),
832 variable_states_(procedure.variables().size()),
833 full_mask_(full_mask)
834 {
835 }
836
838 {
839 for (const int variable_i : procedure_.variables().index_range()) {
840 VariableState &state = variable_states_[variable_i];
841 if (state.value_ != nullptr) {
842 const Variable *variable = procedure_.variables()[variable_i];
843 state.destruct_value(value_allocator_, variable->data_type());
844 }
845 }
846 }
847
849 {
850 return value_allocator_;
851 }
852
853 const IndexMask &full_mask() const
854 {
855 return full_mask_;
856 }
857
859 const Procedure &procedure,
860 Params &params)
861 {
862 for (const int param_index : fn.param_indices()) {
863 ParamType param_type = fn.param_type(param_index);
864 const Variable *variable = procedure.params()[param_index].variable;
865
866 auto add_state = [&](VariableValue *value,
867 bool input_is_initialized,
868 void *caller_provided_storage = nullptr) {
869 const int tot_initialized = input_is_initialized ? full_mask_.size() : 0;
870 const int variable_i = variable->index_in_procedure();
871 VariableState &variable_state = variable_states_[variable_i];
872 BLI_assert(variable_state.value_ == nullptr);
873 variable_state.value_ = value;
874 variable_state.tot_initialized_ = tot_initialized;
875 variable_state.caller_provided_storage_ = caller_provided_storage;
876 };
877
878 switch (param_type.category()) {
880 const GVArray &data = params.readonly_single_input(param_index);
881 add_state(value_allocator_.obtain_GVArray(data), true);
882 break;
883 }
885 const GVVectorArray &data = params.readonly_vector_input(param_index);
886 add_state(value_allocator_.obtain_GVVectorArray(data), true);
887 break;
888 }
890 GMutableSpan data = params.uninitialized_single_output(param_index);
891 add_state(value_allocator_.obtain_Span_not_owned(data.data()), false, data.data());
892 break;
893 }
895 GVectorArray &data = params.vector_output(param_index);
896 add_state(value_allocator_.obtain_GVectorArray_not_owned(data), false, &data);
897 break;
898 }
900 GMutableSpan data = params.single_mutable(param_index);
901 add_state(value_allocator_.obtain_Span_not_owned(data.data()), true, data.data());
902 break;
903 }
905 GVectorArray &data = params.vector_mutable(param_index);
906 add_state(value_allocator_.obtain_GVectorArray_not_owned(data), true, &data);
907 break;
908 }
909 }
910 }
911 }
912
913 void add_as_param(VariableState &variable_state,
914 ParamsBuilder &params,
915 const ParamType &param_type,
916 const IndexMask &mask)
917 {
918 const DataType data_type = param_type.data_type();
919 switch (param_type.interface_type()) {
920 case ParamType::Input: {
921 variable_state.add_as_input(params, mask, data_type);
922 break;
923 }
924 case ParamType::Mutable: {
925 variable_state.add_as_mutable(params, mask, full_mask_, data_type, value_allocator_);
926 break;
927 }
928 case ParamType::Output: {
929 variable_state.add_as_output(params, mask, full_mask_, data_type, value_allocator_);
930 break;
931 }
932 }
933 }
934
935 void add_as_param__one(VariableState &variable_state,
936 ParamsBuilder &params,
937 const ParamType &param_type,
938 const IndexMask &mask)
939 {
940 const DataType data_type = param_type.data_type();
941 switch (param_type.interface_type()) {
942 case ParamType::Input: {
943 variable_state.add_as_input__one(params, data_type);
944 break;
945 }
946 case ParamType::Mutable: {
947 variable_state.add_as_mutable__one(params, data_type, value_allocator_);
948 break;
949 }
950 case ParamType::Output: {
951 variable_state.add_as_output__one(params, mask, data_type, value_allocator_);
952 break;
953 }
954 }
955 }
956
957 void destruct(const Variable &variable, const IndexMask &mask)
958 {
959 VariableState &variable_state = this->get_variable_state(variable);
960 if (variable_state.destruct(mask, full_mask_, variable.data_type(), value_allocator_)) {
961 variable_state.destruct_value(value_allocator_, variable.data_type());
962 }
963 }
964
966 {
967 const int variable_i = variable.index_in_procedure();
968 VariableState &variable_state = variable_states_[variable_i];
969 return variable_state;
970 }
971};
972
973static bool evaluate_as_one(Span<VariableState *> param_variable_states,
974 const IndexMask &mask,
975 const IndexMask &full_mask)
976{
977 if (mask.size() < full_mask.size()) {
978 return false;
979 }
980 for (VariableState *state : param_variable_states) {
981 if (state != nullptr && state->value_ != nullptr && !state->is_one()) {
982 return false;
983 }
984 }
985 return true;
986}
987
989 const CallInstruction &instruction,
990 VariableStates &variable_states,
991 MutableSpan<VariableState *> r_param_variable_states)
992{
993 for (const int param_index : fn.param_indices()) {
994 const Variable *variable = instruction.params()[param_index];
995 if (variable == nullptr) {
996 r_param_variable_states[param_index] = nullptr;
997 }
998 else {
999 VariableState &variable_state = variable_states.get_variable_state(*variable);
1000 r_param_variable_states[param_index] = &variable_state;
1001 }
1002 }
1003}
1004
1006 const IndexMask &mask,
1007 ParamsBuilder &params,
1008 VariableStates &variable_states,
1009 const Span<VariableState *> param_variable_states)
1010{
1011 for (const int param_index : fn.param_indices()) {
1012 const ParamType param_type = fn.param_type(param_index);
1013 VariableState *variable_state = param_variable_states[param_index];
1014 if (variable_state == nullptr) {
1016 }
1017 else {
1018 variable_states.add_as_param__one(*variable_state, params, param_type, mask);
1019 }
1020 }
1021}
1022
1023static void fill_params(const MultiFunction &fn,
1024 const IndexMask &mask,
1025 ParamsBuilder &params,
1026 VariableStates &variable_states,
1027 const Span<VariableState *> param_variable_states)
1028{
1029 for (const int param_index : fn.param_indices()) {
1030 const ParamType param_type = fn.param_type(param_index);
1031 VariableState *variable_state = param_variable_states[param_index];
1032 if (variable_state == nullptr) {
1034 }
1035 else {
1036 variable_states.add_as_param(*variable_state, params, param_type, mask);
1037 }
1038 }
1039}
1040
1041static void execute_call_instruction(const CallInstruction &instruction,
1042 const IndexMask &mask,
1043 VariableStates &variable_states,
1044 const Context &context)
1045{
1046 const MultiFunction &fn = instruction.fn();
1047
1048 Vector<VariableState *> param_variable_states;
1049 param_variable_states.resize(fn.param_amount());
1050 gather_parameter_variable_states(fn, instruction, variable_states, param_variable_states);
1051
1052 /* If all inputs to the function are constant, it's enough to call the function only once instead
1053 * of for every index. */
1054 if (evaluate_as_one(param_variable_states, mask, variable_states.full_mask())) {
1055 static const IndexMask one_mask(1);
1056 ParamsBuilder params(fn, &one_mask);
1057 fill_params__one(fn, mask, params, variable_states, param_variable_states);
1058
1059 try {
1060 fn.call(one_mask, params, context);
1061 }
1062 catch (...) {
1063 /* Multi-functions must not throw exceptions. */
1065 }
1066 }
1067 else {
1068 ParamsBuilder params(fn, &mask);
1069 fill_params(fn, mask, params, variable_states, param_variable_states);
1070
1071 try {
1072 fn.call_auto(mask, params, context);
1073 }
1074 catch (...) {
1075 /* Multi-functions must not throw exceptions. */
1077 }
1078 }
1079}
1080
1083 std::unique_ptr<IndexMaskMemory> memory;
1085
1086 const IndexMask &mask() const
1087 {
1088 return this->referenced_indices;
1089 }
1090};
1091
1094 const Instruction *instruction = nullptr;
1096
1097 const IndexMask &mask() const
1098 {
1099 return this->indices.mask();
1100 }
1101
1102 operator bool() const
1103 {
1104 return this->instruction != nullptr;
1105 }
1106};
1107
1113 private:
1114 Stack<NextInstructionInfo> next_instructions_;
1115
1116 public:
1118
1119 void add_referenced_indices(const Instruction &instruction, const IndexMask &mask)
1120 {
1121 if (mask.is_empty()) {
1122 return;
1123 }
1124 InstructionIndices new_indices;
1125 new_indices.referenced_indices = mask;
1126 next_instructions_.push({&instruction, std::move(new_indices)});
1127 }
1128
1129 void add_owned_indices(const Instruction &instruction, Vector<int64_t> indices)
1130 {
1131 if (indices.is_empty()) {
1132 return;
1133 }
1134
1135 InstructionIndices new_indices;
1136 new_indices.memory = std::make_unique<IndexMaskMemory>();
1138 *new_indices.memory);
1139 next_instructions_.push({&instruction, std::move(new_indices)});
1140 }
1141
1142 bool is_done() const
1143 {
1144 return next_instructions_.is_empty();
1145 }
1146
1148 {
1149 BLI_assert(!this->is_done());
1150 return next_instructions_.peek();
1151 }
1152
1154 {
1155 next_instructions_.peek().instruction = &instruction;
1156 }
1157
1159 {
1160 return next_instructions_.pop();
1161 }
1162};
1163
1164void ProcedureExecutor::call(const IndexMask &full_mask, Params params, Context context) const
1165{
1166 BLI_assert(procedure_.validate());
1167
1168 AlignedBuffer<512, 64> local_buffer;
1170 linear_allocator.provide_buffer(local_buffer);
1171
1172 VariableStates variable_states{linear_allocator, procedure_, full_mask};
1173 variable_states.add_initial_variable_states(*this, procedure_, params);
1174
1175 InstructionScheduler scheduler;
1176 scheduler.add_referenced_indices(*procedure_.entry(), full_mask);
1177
1178 /* Loop until all indices got to a return instruction. */
1179 while (!scheduler.is_done()) {
1180 const NextInstructionInfo &instr_info = scheduler.peek();
1181 const Instruction &instruction = *instr_info.instruction;
1182 switch (instruction.type()) {
1183 case InstructionType::Call: {
1184 const CallInstruction &call_instruction = static_cast<const CallInstruction &>(
1185 instruction);
1186 execute_call_instruction(call_instruction, instr_info.mask(), variable_states, context);
1187 scheduler.update_instruction_pointer(*call_instruction.next());
1188 break;
1189 }
1191 const BranchInstruction &branch_instruction = static_cast<const BranchInstruction &>(
1192 instruction);
1193 const Variable *condition_var = branch_instruction.condition();
1194 VariableState &variable_state = variable_states.get_variable_state(*condition_var);
1195
1196 IndicesSplitVectors new_indices;
1197 variable_state.indices_split(instr_info.mask(), new_indices);
1198 scheduler.pop();
1199 scheduler.add_owned_indices(*branch_instruction.branch_false(), new_indices[false]);
1200 scheduler.add_owned_indices(*branch_instruction.branch_true(), new_indices[true]);
1201 break;
1202 }
1204 const DestructInstruction &destruct_instruction = static_cast<const DestructInstruction &>(
1205 instruction);
1206 const Variable *variable = destruct_instruction.variable();
1207 variable_states.destruct(*variable, instr_info.mask());
1208 scheduler.update_instruction_pointer(*destruct_instruction.next());
1209 break;
1210 }
1212 const DummyInstruction &dummy_instruction = static_cast<const DummyInstruction &>(
1213 instruction);
1214 scheduler.update_instruction_pointer(*dummy_instruction.next());
1215 break;
1216 }
1218 /* Don't insert the indices back into the scheduler. */
1219 scheduler.pop();
1220 break;
1221 }
1222 }
1223 }
1224
1225 for (const int param_index : this->param_indices()) {
1226 const ParamType param_type = this->param_type(param_index);
1227 const Variable *variable = procedure_.params()[param_index].variable;
1228 VariableState &variable_state = variable_states.get_variable_state(*variable);
1229 switch (param_type.interface_type()) {
1230 case ParamType::Input: {
1231 /* Input variables must be destructed in the end. */
1232 BLI_assert(variable_state.is_fully_uninitialized(full_mask));
1233 break;
1234 }
1235 case ParamType::Mutable:
1236 case ParamType::Output: {
1237 /* Mutable and output variables must be initialized in the end. */
1238 BLI_assert(variable_state.is_fully_initialized(full_mask));
1239 /* Make sure that the data is in the memory provided by the caller. */
1240 variable_state.ensure_is_mutable(
1241 full_mask, param_type.data_type(), variable_states.value_allocator());
1242 break;
1243 }
1244 }
1245 }
1246}
1247
1249{
1250 ExecutionHints hints;
1251 hints.allocates_array = true;
1252 hints.min_grain_size = 10000;
1253 return hints;
1254}
1255
1256} // namespace blender::fn::multi_function
#define BLI_assert_unreachable()
#define UNUSED_VARS(...)
#define UNUSED_VARS_NDEBUG(...)
#define ELEM(...)
void fill_construct_indices(const void *value, void *dst, const IndexMask &mask) const
void destruct_indices(void *ptr, const IndexMask &mask) const
void destruct(void *ptr) const
bool can_exist_in_buffer(const int64_t buffer_size, const int64_t buffer_alignment) const
const void * data() const
void extend(int64_t index, const GVArray &values)
static IndexMask from_indices(Span< T > indices, LinearAllocator<> &memory)
constexpr IndexRange()=default
void * allocate(const int64_t size, const int64_t alignment)
bool is_empty() const
Definition BLI_stack.hh:310
void push(const T &value)
Definition BLI_stack.hh:215
int64_t size() const
void append(const T &value)
bool is_empty() const
void resize(const int64_t new_size)
void reserve(const int64_t min_capacity)
NonCopyable(const NonCopyable &other)=delete
NonMovable(NonMovable &&other)=delete
void add_referenced_indices(const Instruction &instruction, const IndexMask &mask)
void add_owned_indices(const Instruction &instruction, Vector< int64_t > indices)
void set_signature(const Signature *signature)
ParamType param_type(int param_index) const
virtual ExecutionHints get_execution_hints() const
ParamType(InterfaceType interface_type, DataType data_type)
void add_single_mutable(GMutableSpan ref, StringRef expected_name="")
void add_ignored_single_output(StringRef expected_name="")
void add_uninitialized_single_output(T *value, StringRef expected_name="")
void add_vector_output(GVectorArray &vector_array, StringRef expected_name="")
void add_vector_mutable(GVectorArray &vector_array, StringRef expected_name="")
void add_readonly_single_input(const T *value, StringRef expected_name="")
void add_readonly_vector_input(const GVectorArray &vector_array, StringRef expected_name="")
const VVectorArray< T > & readonly_vector_input(int param_index, StringRef name="")
MutableSpan< T > uninitialized_single_output(int param_index, StringRef name="")
GVectorArray_TypedMutableRef< T > vector_mutable(int param_index, StringRef name="")
GVectorArray_TypedMutableRef< T > vector_output(int param_index, StringRef name="")
MutableSpan< T > single_mutable(int param_index, StringRef name="")
VArray< T > readonly_single_input(int param_index, StringRef name="")
void call(const IndexMask &mask, Params params, Context context) const override
void add(ParamTag< Category, T >, const char *name)
VariableValue_GVectorArray * obtain_GVectorArray(const CPPType &type, int size)
VariableValue_GVectorArray * obtain_GVectorArray_not_owned(GVectorArray &data)
void release_value(VariableValue *value, const DataType &data_type)
VariableValue_OneSingle * obtain_OneSingle(const CPPType &type)
VariableValue_GVArray * obtain_GVArray(const GVArray &varray)
VariableValue_Span * obtain_Span(const CPPType &type, int size)
VariableValue_GVVectorArray * obtain_GVVectorArray(const GVVectorArray &varray)
VariableValue_OneVector * obtain_OneVector(const CPPType &type)
void indices_split(const IndexMask &mask, IndicesSplitVectors &r_indices)
void destruct_value(ValueAllocator &value_allocator, const DataType &data_type)
void ensure_is_mutable(const IndexMask &full_mask, const DataType &data_type, ValueAllocator &value_allocator)
void add_as_mutable(ParamsBuilder &params, const IndexMask &mask, const IndexMask &full_mask, const DataType &data_type, ValueAllocator &value_allocator)
void add_as_output(ParamsBuilder &params, const IndexMask &mask, const IndexMask &full_mask, const DataType &data_type, ValueAllocator &value_allocator)
void add_as_input__one(ParamsBuilder &params, const DataType &data_type) const
void ensure_is_mutable__one(const DataType &data_type, ValueAllocator &value_allocator)
void add_as_input(ParamsBuilder &params, const IndexMask &mask, const DataType &data_type) const
void add_as_output__one(ParamsBuilder &params, const IndexMask &mask, const DataType &data_type, ValueAllocator &value_allocator)
bool destruct(const IndexMask &mask, const IndexMask &full_mask, const DataType &data_type, ValueAllocator &value_allocator)
void add_as_mutable__one(ParamsBuilder &params, const DataType &data_type, ValueAllocator &value_allocator)
VariableState & get_variable_state(const Variable &variable)
void destruct(const Variable &variable, const IndexMask &mask)
void add_initial_variable_states(const ProcedureExecutor &fn, const Procedure &procedure, Params &params)
void add_as_param__one(VariableState &variable_state, ParamsBuilder &params, const ParamType &param_type, const IndexMask &mask)
void add_as_param(VariableState &variable_state, ParamsBuilder &params, const ParamType &param_type, const IndexMask &mask)
VariableStates(LinearAllocator<> &linear_allocator, const Procedure &procedure, const IndexMask &full_mask)
#define condition(cond)
const int state
#define T
MultiFunction::ExecutionHints ExecutionHints
static bool evaluate_as_one(Span< VariableState * > param_variable_states, const IndexMask &mask, const IndexMask &full_mask)
static void gather_parameter_variable_states(const MultiFunction &fn, const CallInstruction &instruction, VariableStates &variable_states, MutableSpan< VariableState * > r_param_variable_states)
static void fill_params(const MultiFunction &fn, const IndexMask &mask, ParamsBuilder &params, VariableStates &variable_states, const Span< VariableState * > param_variable_states)
static void execute_call_instruction(const CallInstruction &instruction, const IndexMask &mask, VariableStates &variable_states, const Context &context)
static void fill_params__one(const MultiFunction &fn, const IndexMask &mask, ParamsBuilder &params, VariableStates &variable_states, const Span< VariableState * > param_variable_states)
std::array< Vector< int64_t >, 2 > IndicesSplitVectors
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
__int64 int64_t
Definition BKE_text.h:90
BMesh const char void * data