Blender V5.3
node_composite_plane_track_deform.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2013 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
5#include "BLI_array.hh"
8#include "BLI_string_utf8.hh"
9
10#include "DNA_movieclip_types.h"
11#include "DNA_tracking_types.h"
12
13#include "BKE_context.hh"
14#include "BKE_lib_id.hh"
15#include "BKE_movieclip.hh"
16#include "BKE_tracking.hh"
17
18#include "RNA_access.hh"
19#include "RNA_prototypes.hh"
20
21#include "UI_interface.hh"
23#include "UI_resources.hh"
24
25#include "GPU_shader.hh"
26#include "GPU_texture.hh"
27#include "GPU_uniform_buffer.hh"
28
29#include "COM_algorithm_smaa.hh"
30#include "COM_node_operation.hh"
31#include "COM_utilities.hh"
32
34
36
38
40{
41 b.use_custom_socket_order();
42 b.allow_any_socket_order();
43
44 b.add_input<decl::Color>("Image"_ustr)
45 .hide_value()
46 .compositor_realization_mode(CompositorInputRealizationMode::Transforms)
47 .structure_type(StructureType::Dynamic);
48 b.add_output<decl::Color>("Image"_ustr)
49 .structure_type(StructureType::Dynamic)
50 .align_with_previous();
51 b.add_output<decl::Float>("Plane"_ustr).structure_type(StructureType::Dynamic);
52
53 b.add_layout([](ui::Layout &layout, bContext *C, PointerRNA *ptr) {
54 bNode *node = ptr->data_as<bNode>();
55
56 template_id(&layout, C, ptr, "clip", nullptr, "CLIP_OT_open", nullptr);
57
58 if (node->id) {
59 MovieClip *clip = reinterpret_cast<MovieClip *>(node->id);
60 MovieTracking *tracking = &clip->tracking;
61 MovieTrackingObject *tracking_object;
63 &clip->id, RNA_MovieTracking, tracking);
64
65 ui::Layout &col = layout.column(false);
66 col.prop_search(ptr, "tracking_object", &tracking_ptr, "objects", "", ICON_OBJECT_DATA);
67
68 tracking_object = BKE_tracking_object_get_named(tracking,
69 node_storage(*node).tracking_object);
70 if (tracking_object) {
72 &clip->id, RNA_MovieTrackingObject, tracking_object);
73
74 col.prop_search(ptr, "plane_track_name", &object_ptr, "plane_tracks", "", ICON_ANIM_DATA);
75 }
76 else {
77 layout.prop(ptr, "plane_track_name", UI_ITEM_NONE, "", ICON_ANIM_DATA);
78 }
79 }
80 });
81
82 PanelDeclarationBuilder &motion_blur_panel =
83 b.add_panel("Motion Blur"_ustr).default_closed(true);
84 motion_blur_panel.add_input<decl::Bool>("Motion Blur"_ustr)
85 .default_value(false)
86 .panel_toggle()
87 .description("Use multi-sampled motion blur of the plane");
88 motion_blur_panel.add_input<decl::Int>("Samples"_ustr, "Motion Blur Samples"_ustr)
89 .default_value(16)
90 .min(1)
91 .max(64)
92 .description("Number of motion blur samples");
93 motion_blur_panel.add_input<decl::Float>("Shutter"_ustr, "Motion Blur Shutter"_ustr)
94 .default_value(0.5f)
95 .subtype(PROP_FACTOR)
96 .min(0.0f)
97 .max(1.0f)
98 .description("Exposure for motion blur as a factor of FPS");
99}
100
101static void node_init(bNodeTree * /*node_tree*/, bNode *node)
102{
103 NodePlaneTrackDeformData *data = MEM_new<NodePlaneTrackDeformData>(__func__);
104 node->storage = data;
105}
106
107static void node_init_api(const bContext *C, PointerRNA *node_ptr)
108{
109 bNode *node = node_ptr->data_as<bNode>();
110 NodePlaneTrackDeformData &data = node_storage(*node);
111
112 const Scene *scene = CTX_data_scene(C);
113 MovieClip *movie_clip = scene->clip;
114 if (!movie_clip) {
115 return;
116 }
117
118 node->id = &movie_clip->id;
119 id_us_plus(&movie_clip->id);
120
121 MovieTracking *tracking = &movie_clip->tracking;
122 const MovieTrackingObject *tracking_object = BKE_tracking_object_get_active(tracking);
123 STRNCPY_UTF8(data.tracking_object, tracking_object->name);
124
125 if (tracking_object->active_plane_track) {
126 STRNCPY_UTF8(data.plane_track_name, tracking_object->active_plane_track->name);
127 }
128}
129
130using namespace blender::compositor;
131
133 public:
135
136 void execute() override
137 {
139
140 const Result &input_image = get_input("Image");
141 if (input_image.is_single_value() || !plane_track) {
142 Result &output_image = get_result("Image");
143 if (output_image.should_compute()) {
144 output_image.share_data(input_image);
145 }
146
147 Result &output_mask = get_result("Plane");
148 if (output_mask.should_compute()) {
149 output_mask.allocate_single_value();
150 output_mask.set_single_value(1.0f);
151 }
152 return;
153 }
154
155 const Array<float4x4> homography_matrices = compute_homography_matrices(plane_track);
156
157 if (this->context().use_gpu()) {
158 this->execute_gpu(homography_matrices);
159 }
160 else {
161 this->execute_cpu(homography_matrices);
162 }
163 }
164
165 void execute_gpu(const Array<float4x4> homography_matrices)
166 {
167 gpu::UniformBuf *homography_matrices_buffer = GPU_uniformbuf_create_ex(
168 homography_matrices.size() * sizeof(float4x4),
169 homography_matrices.data(),
170 "Plane Track Deform Homography Matrices");
171
172 Result plane_mask = this->compute_plane_mask_gpu(homography_matrices,
173 homography_matrices_buffer);
174 Result anti_aliased_plane_mask = this->context().create_result(ResultType::Float);
175 smaa(context(), plane_mask, anti_aliased_plane_mask);
176 plane_mask.release();
177
178 Result &output_image = get_result("Image");
179 if (output_image.should_compute()) {
180 this->compute_plane_gpu(
181 homography_matrices, homography_matrices_buffer, anti_aliased_plane_mask);
182 }
183
184 GPU_uniformbuf_free(homography_matrices_buffer);
185
186 Result &output_mask = get_result("Plane");
187 if (output_mask.should_compute()) {
188 output_mask.share_data(anti_aliased_plane_mask);
189 }
190
191 anti_aliased_plane_mask.release();
192 }
193
194 void compute_plane_gpu(const Array<float4x4> &homography_matrices,
195 gpu::UniformBuf *homography_matrices_buffer,
196 Result &plane_mask)
197 {
198 gpu::Shader *shader = context().get_shader("compositor_plane_deform_motion_blur");
200
201 GPU_shader_uniform_1i(shader, "number_of_motion_blur_samples", homography_matrices.size());
202
203 const int ubo_location = GPU_shader_get_ubo_binding(shader, "homography_matrices");
204 GPU_uniformbuf_bind(homography_matrices_buffer, ubo_location);
205
206 Result &input_image = get_input("Image");
207 GPU_texture_mipmap_mode(input_image, true, true);
208 GPU_texture_anisotropic_filter(input_image, true);
209 /* We actually need zero boundary conditions, but we sampled using extended boundaries then
210 * multiply by the anti-aliased plane mask to get better quality anti-aliased planes. */
212 input_image.bind_as_texture(shader, "input_tx");
213
214 plane_mask.bind_as_texture(shader, "mask_tx");
215
216 const Domain domain = compute_domain();
217 Result &output_image = get_result("Image");
218 output_image.allocate_texture(domain);
219 output_image.bind_as_image(shader, "output_img");
220
222
223 input_image.unbind_as_texture();
224 plane_mask.unbind_as_texture();
225 output_image.unbind_as_image();
226 GPU_uniformbuf_unbind(homography_matrices_buffer);
228 }
229
231 gpu::UniformBuf *homography_matrices_buffer)
232 {
233 gpu::Shader *shader = context().get_shader("compositor_plane_deform_motion_blur_mask");
235
236 GPU_shader_uniform_1i(shader, "number_of_motion_blur_samples", homography_matrices.size());
237
238 const int ubo_location = GPU_shader_get_ubo_binding(shader, "homography_matrices");
239 GPU_uniformbuf_bind(homography_matrices_buffer, ubo_location);
240
241 const Domain domain = compute_domain();
243 plane_mask.allocate_texture(domain);
244 plane_mask.bind_as_image(shader, "mask_img");
245
247
248 plane_mask.unbind_as_image();
249 GPU_uniformbuf_unbind(homography_matrices_buffer);
251
252 return plane_mask;
253 }
254
255 void execute_cpu(const Array<float4x4> homography_matrices)
256 {
257 Result plane_mask = this->compute_plane_mask_cpu(homography_matrices);
258 Result anti_aliased_plane_mask = this->context().create_result(ResultType::Float);
259 smaa(context(), plane_mask, anti_aliased_plane_mask);
260 plane_mask.release();
261
262 Result &output_image = get_result("Image");
263 if (output_image.should_compute()) {
264 this->compute_plane_cpu(homography_matrices, anti_aliased_plane_mask);
265 }
266
267 Result &output_mask = get_result("Plane");
268 if (output_mask.should_compute()) {
269 output_mask.share_data(anti_aliased_plane_mask);
270 }
271
272 anti_aliased_plane_mask.release();
273 }
274
275 void compute_plane_cpu(const Array<float4x4> &homography_matrices, Result &plane_mask)
276 {
277 Result &input = get_input("Image");
278
279 const Domain domain = compute_domain();
280 Result &output = get_result("Image");
281 output.allocate_texture(domain);
282
283 const int2 size = domain.data_size;
284 parallel_for(size, [&](const int2 texel) {
285 float2 coordinates = (float2(texel) + float2(0.5f)) / float2(size);
286
287 float4 accumulated_color = float4(0.0f);
288 for (const float4x4 &homography_matrix : homography_matrices) {
289 float3 transformed_coordinates = float3x3(homography_matrix) * float3(coordinates, 1.0f);
290 /* Point is at infinity and will be zero when sampled, so early exit. */
291 if (transformed_coordinates.z == 0.0f) {
292 continue;
293 }
294 float2 projected_coordinates = transformed_coordinates.xy() / transformed_coordinates.z;
295 /* Derivative of transformed_coordinates.xy / transformed_coordinates.z vs texels. */
296 float2 x_gradient = (homography_matrix[0].xy() * transformed_coordinates.z -
297 transformed_coordinates.xy() * homography_matrix[0].z) /
298 (math::square(transformed_coordinates.z) * size.x);
299 float2 y_gradient = (homography_matrix[1].xy() * transformed_coordinates.z -
300 transformed_coordinates.xy() * homography_matrix[1].z) /
301 (math::square(transformed_coordinates.z) * size.y);
302 const float2x2 jacobian = float2x2(x_gradient, y_gradient);
303 float4 sampled_color = float4(input.sample<Color>(projected_coordinates,
307 jacobian));
308 accumulated_color += sampled_color;
309 }
310
311 accumulated_color /= homography_matrices.size();
312
313 /* Premultiply the mask value as an alpha. */
314 float4 plane_color = accumulated_color * plane_mask.load_pixel<float>(texel);
315
316 output.store_pixel(texel, Color(plane_color));
317 });
318 }
319
321 {
322 const Domain domain = compute_domain();
324 plane_mask.allocate_texture(domain);
325
326 const int2 size = domain.data_size;
327 parallel_for(size, [&](const int2 texel) {
328 float2 coordinates = (float2(texel) + float2(0.5f)) / float2(size);
329
330 float accumulated_mask = 0.0f;
331 for (const float4x4 &homography_matrix : homography_matrices) {
332 float3 transformed_coordinates = float3x3(homography_matrix) * float3(coordinates, 1.0f);
333 /* Point is at infinity and will be zero when sampled, so early exit. */
334 if (transformed_coordinates.z == 0.0f) {
335 continue;
336 }
337 float2 projected_coordinates = transformed_coordinates.xy() / transformed_coordinates.z;
338
339 bool is_inside_plane = projected_coordinates.x >= 0.0f &&
340 projected_coordinates.y >= 0.0f &&
341 projected_coordinates.x <= 1.0f && projected_coordinates.y <= 1.0f;
342 accumulated_mask += is_inside_plane ? 1.0f : 0.0f;
343 }
344
345 accumulated_mask /= homography_matrices.size();
346
347 plane_mask.store_pixel(texel, accumulated_mask);
348 });
349
350 return plane_mask;
351 }
352
354 {
356
357 Result &input_image = get_input("Image");
358 if (input_image.is_single_value() || !plane_track) {
359 return input_image.domain();
360 }
361
362 return Domain(get_movie_clip_size());
363 }
364
366 {
367 /* We evaluate at the frames in the range [frame - shutter, frame + shutter], if no motion blur
368 * is enabled or the motion blur samples is set to 1, we just evaluate at the current frame. */
369 const int samples = this->get_motion_blur_samples();
370 const float shutter = samples != 1 ? this->get_motion_blur_shutter() : 0.0f;
371 const float start_frame = context().get_frame_number() - shutter;
372 const float frame_step = (shutter * 2.0f) / samples;
373
374 Array<float4x4> matrices(samples);
375 for (int i = 0; i < samples; i++) {
376 const float frame = start_frame + frame_step * i;
377 const float clip_frame = BKE_movieclip_remap_scene_to_clip_frame(get_movie_clip(), frame);
378
379 float corners[4][2];
380 BKE_tracking_plane_marker_get_subframe_corners(plane_track, clip_frame, corners);
381
382 /* Compute a 2D projection matrix that projects from the corners of the image in normalized
383 * coordinates into the corners of the tracking plane. */
384 float3x3 homography_matrix;
385 float identity_corners[4][2] = {{0.0f, 0.0f}, {1.0f, 0.0f}, {1.0f, 1.0f}, {0.0f, 1.0f}};
387 corners, identity_corners, homography_matrix.ptr());
388
389 /* Store in a 4x4 matrix due to the alignment requirements of GPU uniform buffers. */
390 matrices[i] = float4x4(homography_matrix);
391 }
392
393 return matrices;
394 }
395
397 {
398 MovieClip *movie_clip = get_movie_clip();
399
400 if (!movie_clip) {
401 return nullptr;
402 }
403
405 &movie_clip->tracking, node_storage(node()).tracking_object);
406
407 if (!tracking_object) {
408 return nullptr;
409 }
410
412 node_storage(node()).plane_track_name);
413 }
414
416 {
417 MovieClipUser user = {};
418 BKE_movieclip_user_set_frame(&user, context().get_frame_number());
419
420 int2 size;
421 BKE_movieclip_get_size(get_movie_clip(), &user, &size.x, &size.y);
422 return size;
423 }
424
426 {
427 const int samples = math::clamp(
428 this->get_input("Motion Blur Samples").get_single_value_default<int>(), 1, 64);
429 return this->use_motion_blur() ? samples : 1;
430 }
431
433 {
434 return math::clamp(
435 this->get_input("Motion Blur Shutter").get_single_value_default<float>(), 0.0f, 1.0f);
436 }
437
439 {
440 return this->get_input("Motion Blur").get_single_value_default<bool>();
441 }
442
444 {
445 return reinterpret_cast<MovieClip *>(node().id);
446 }
447};
448
450{
451 return new PlaneTrackDeformOperation(context, node);
452}
453
454static void node_register()
455{
456 static bke::bNodeType ntype;
457
458 cmp_node_type_base(&ntype, "CompositorNodePlaneTrackDeform"_ustr, CMP_NODE_PLANETRACKDEFORM);
459 ntype.ui_name = "Plane Track Deform";
460 ntype.ui_description =
461 "Replace flat planes in footage by another image, detected by plane tracks from motion "
462 "tracking";
463 ntype.enum_name_legacy = "PLANETRACKDEFORM";
465 ntype.declare = node_declare;
466 ntype.initfunc = node_init;
469 ntype, "NodePlaneTrackDeformData", node_free_standard_storage, node_copy_standard_storage);
471
473}
475
476} // namespace blender::nodes::node_composite_plane_track_deform_cc
#define NODE_STORAGE_FUNCS(StorageT)
Definition BKE_node.hh:1353
#define NODE_CLASS_DISTORT
Definition BKE_node.hh:486
#define CMP_NODE_PLANETRACKDEFORM
#define STRNCPY_UTF8(dst, src)
#define NOD_REGISTER_NODE(REGISTER_FUNC)
#define C
Definition RandGen.cpp:29
#define UI_ITEM_NONE
btScalar max
#define output
int64_t size() const
Definition BLI_array.hh:267
const T * data() const
Definition BLI_array.hh:323
Result create_result(ResultType type, ResultPrecision precision)
gpu::Shader * get_shader(const char *info_name, ResultPrecision precision)
Domain(const int2 &size)
Definition domain.cc:20
NodeOperation(Context &context, const bNode &node)
Result & get_result(StringRef identifier)
Definition operation.cc:40
Context & context() const
Definition operation.cc:57
Result & get_input(StringRef identifier) const
Definition operation.cc:35
void share_data(const Result &source)
Definition result.cc:996
T sample(const float2 &coordinates, const Interpolation &interpolation, const Extension &extension_mode_x, const Extension &extension_mode_y, std::optional< float2x2 > jacobian=std::nullopt) const
void store_pixel(const int2 &texel, const T &pixel_value)
bool should_compute() const
Definition result.cc:1124
void allocate_texture(const Domain domain, const bool from_pool=true, const std::optional< ResultStorageType > storage_type=std::nullopt)
Definition result.cc:554
void unbind_as_texture() const
Definition result.cc:964
void set_single_value(const T &value)
void bind_as_texture(gpu::Shader *shader, const char *texture_name) const
Definition result.cc:854
T load_pixel(const int2 &texel) const
void unbind_as_image() const
Definition result.cc:990
void bind_as_image(gpu::Shader *shader, const char *image_name, bool read=false) const
Definition result.cc:951
DeclType::Builder & add_input(UString name, UString identifier=""_ustr)
void compute_plane_gpu(const Array< float4x4 > &homography_matrices, gpu::UniformBuf *homography_matrices_buffer, Result &plane_mask)
void compute_plane_cpu(const Array< float4x4 > &homography_matrices, Result &plane_mask)
Result compute_plane_mask_gpu(const Array< float4x4 > &homography_matrices, gpu::UniformBuf *homography_matrices_buffer)
void node_register_type(bNodeType &ntype)
Definition node.cc:2920
void node_type_storage(bNodeType &ntype, std::optional< StringRefNull > storagename, void(*freefunc)(bNode *node), void(*copyfunc)(bNodeTree *dest_ntree, bNode *dest_node, const bNode *src_node))
Definition node.cc:6166
void compute_dispatch_threads_at_least(gpu::Shader *shader, int2 threads_range, int2 local_size=int2(16))
Definition utilities.cc:235
ColorSceneLinear4f< eAlpha::Premultiplied > Color
Definition COM_result.hh:96
void smaa(Context &context, const Result &input, Result &output, const float threshold=0.1f, const float local_contrast_adaptation_factor=2.0f, const int corner_rounding=25)
Definition smaa.cc:1693
void parallel_for(const int2 range, const Function &function)
T clamp(const T &a, const T &min, const T &max)
T square(const T &a)
static NodeOperation * get_compositor_operation(Context &context, const bNode &node)
static void node_init_api(const bContext *C, PointerRNA *node_ptr)
void BKE_movieclip_get_size(MovieClip *clip, const MovieClipUser *user, int *r_width, int *r_height)
void GPU_uniformbuf_free(gpu::UniformBuf *ubo)
void id_us_plus(ID *id)
Definition lib_id.cc:433
MovieTrackingObject * BKE_tracking_object_get_active(const MovieTracking *tracking)
Definition tracking.cc:1982
MatBase< float, 2, 2 > float2x2
@ GPU_SAMPLER_EXTEND_MODE_EXTEND
void GPU_texture_anisotropic_filter(gpu::Texture *texture, bool use_aniso)
void BKE_tracking_plane_marker_get_subframe_corners(MovieTrackingPlaneTrack *plane_track, float framenr, float corners[4][2])
Definition tracking.cc:1871
void GPU_texture_mipmap_mode(gpu::Texture *texture, bool use_mipmap, bool use_filter)
MovieTrackingPlaneTrack * BKE_tracking_object_find_plane_track_with_name(MovieTrackingObject *tracking_object, const char *name)
Definition tracking.cc:2010
BMesh const char void * data
void GPU_shader_unbind()
PointerRNA RNA_pointer_create_discrete(ID *id, StructRNA *type, void *data)
void GPU_shader_bind(gpu::Shader *shader, const gpu::shader::SpecializationConstants *constants_state=nullptr)
void GPU_shader_uniform_1i(gpu::Shader *sh, const char *name, int value)
void BKE_tracking_homography_between_two_quads(float reference_corners[4][2], float corners[4][2], float H[3][3])
void GPU_uniformbuf_bind(gpu::UniformBuf *ubo, int slot)
gpu::UniformBuf * GPU_uniformbuf_create_ex(size_t size, const void *data, const char *name)
void GPU_texture_extend_mode(gpu::Texture *texture, GPUSamplerExtendMode extend_mode)
void BKE_movieclip_user_set_frame(MovieClipUser *user, int framenr)
float BKE_movieclip_remap_scene_to_clip_frame(const MovieClip *clip, float framenr)
int GPU_shader_get_ubo_binding(gpu::Shader *shader, const char *name)
MovieTrackingObject * BKE_tracking_object_get_named(MovieTracking *tracking, const char *name)
Definition tracking.cc:1971
void node_copy_standard_storage(bNodeTree *, bNode *dest_node, const bNode *src_node)
Definition node_util.cc:57
Scene * CTX_data_scene(const bContext *C)
void node_free_standard_storage(bNode *node)
Definition node_util.cc:45
@ PROP_FACTOR
Definition RNA_types.hh:308
void GPU_uniformbuf_unbind(gpu::UniformBuf *ubo)
void cmp_node_type_base(bke::bNodeType *ntype, UString idname, const std::optional< int16_t > legacy_type)
#define min(a, b)
Definition sort.cc:38
const c_style_mat & ptr() const
struct MovieTracking tracking
MovieTrackingPlaneTrack * active_plane_track
T * data_as() const
Definition RNA_types.hh:139
struct MovieClip * clip
Defines a node type.
Definition BKE_node.hh:265
std::string ui_description
Definition BKE_node.hh:271
void(*) initfunc_api(const bContext *C, PointerRNA *ptr)
Definition BKE_node.hh:333
NodeGetCompositorOperationFunction get_compositor_operation
Definition BKE_node.hh:379
void(*) initfunc(bNodeTree *ntree, bNode *node)
Definition BKE_node.hh:320
const char * enum_name_legacy
Definition BKE_node.hh:274
NodeDeclareFunction declare
Definition BKE_node.hh:393
void prop_search(PointerRNA *ptr, PropertyRNA *prop, PointerRNA *searchptr, PropertyRNA *searchprop, PropertyRNA *item_searchpropname, std::optional< StringRefNull > name, int icon, bool results_are_suggestions)
void prop(PointerRNA *ptr, PropertyRNA *prop, int index, int value, eUI_Item_Flag flag, std::optional< StringRef > name_opt, int icon, std::optional< StringRef > placeholder=std::nullopt)
Layout & column(bool align)