Blender V5.3
keying_screen.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2023 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
5#include <cstdint>
6#include <memory>
7#include <string>
8
9#include "BLI_hash.hh"
10#include "BLI_listbase.hh"
11#include "BLI_math_base.hh"
12#include "BLI_math_color_c.hh"
13#include "BLI_math_vector.hh"
15#include "BLI_vector.hh"
16
17#include "IMB_imbuf.hh"
18
19#include "DNA_movieclip_types.h"
20#include "DNA_tracking_types.h"
21
22#include "GPU_shader.hh"
23#include "GPU_storage_buffer.hh"
24
25#include "BKE_movieclip.hh"
26#include "BKE_tracking.hh"
27
28#include "COM_context.hh"
29#include "COM_keying_screen.hh"
30#include "COM_result.hh"
31#include "COM_utilities.hh"
32
33namespace blender::compositor {
34
35/* --------------------------------------------------------------------
36 * Keying Screen Key.
37 */
38
43
48
50{
51 return a.frame == b.frame && a.smoothness == b.smoothness;
52}
53
54/* --------------------------------------------------------------------
55 * Keying Screen.
56 */
57
58/* Computes the color and normalized positions of the keying screen markers in the given movie
59 * tracking object. The color is computed as the mean color of the search pattern of the marker. */
60static void compute_marker_points(MovieClip *movie_clip,
61 MovieClipUser &movie_clip_user,
62 MovieTrackingObject *movie_tracking_object,
63 Vector<float2> &marker_positions,
64 Vector<float4> &marker_colors)
65{
66 BLI_assert(marker_positions.is_empty());
67 BLI_assert(marker_colors.is_empty());
68
69 ImBuf *image_buffer = BKE_movieclip_get_ibuf(movie_clip, &movie_clip_user);
70 if (!image_buffer) {
71 return;
72 }
73
74 for (MovieTrackingTrack &track : movie_tracking_object->tracks) {
75 const MovieTrackingMarker *marker = BKE_tracking_marker_get(&track, movie_clip_user.framenr);
76 if (marker->flag & MARKER_DISABLED) {
77 continue;
78 }
79
80 /* Skip out of bound markers since they have no corresponding color. */
81 const float2 position = float2(marker->pos) + float2(track.offset);
82 if (math::clamp(position, float2(0.0f), float2(1.0f)) != position) {
83 continue;
84 }
85
86 ImBuf *pattern_image_buffer = BKE_tracking_get_pattern_imbuf(
87 image_buffer, &track, marker, true, false);
88 if (!pattern_image_buffer) {
89 continue;
90 }
91
92 /* Find the mean color of the rectangular search pattern of the marker. */
93 float4 mean_color = float4(0.0f);
94 const uchar *byte_data = pattern_image_buffer->byte_data();
95 const float *float_data = pattern_image_buffer->float_data();
96 for (int i = 0; i < pattern_image_buffer->x * pattern_image_buffer->y; i++) {
97 if (float_data) {
98 mean_color += float4(&float_data[i * 4]);
99 }
100 else {
101 float4 linear_color;
102 uchar4 srgb_color = uchar4(&byte_data[i * 4]);
103 srgb_to_linearrgb_uchar4(linear_color, srgb_color);
104 mean_color += linear_color;
105 }
106 }
107 mean_color /= pattern_image_buffer->x * pattern_image_buffer->y;
108
109 marker_colors.append(mean_color);
110 marker_positions.append(position);
111
112 IMB_freeImBuf(pattern_image_buffer);
113 }
114
115 IMB_freeImBuf(image_buffer);
116}
117
118/* Get a MovieClipUser with an initialized clip frame number. */
120{
121 MovieClipUser movie_clip_user = {};
122 const int scene_frame = context.get_frame_number();
123 const int clip_frame = BKE_movieclip_remap_scene_to_clip_frame(movie_clip, scene_frame);
124 BKE_movieclip_user_set_frame(&movie_clip_user, clip_frame);
125 return movie_clip_user;
126}
127
129 MovieClip *movie_clip,
130 MovieTrackingObject *movie_tracking_object,
131 const float smoothness)
132 : result(context.create_result(ResultType::Color))
133{
134 int2 size;
135 MovieClipUser movie_clip_user = get_movie_clip_user(context, movie_clip);
136 BKE_movieclip_get_size(movie_clip, &movie_clip_user, &size.x, &size.y);
137
138 Vector<float2> marker_positions;
139 Vector<float4> marker_colors;
141 movie_clip, movie_clip_user, movie_tracking_object, marker_positions, marker_colors);
142
143 if (marker_positions.is_empty()) {
144 return;
145 }
146
147 this->result.allocate_texture(Domain(size), false);
148 if (context.use_gpu()) {
149 this->compute_gpu(context, smoothness, marker_positions, marker_colors);
150 }
151 else {
152 this->compute_cpu(smoothness, marker_positions, marker_colors);
153 }
154}
155
156void KeyingScreen::compute_gpu(Context &context,
157 const float smoothness,
158 Vector<float2> &marker_positions,
159 const Vector<float4> &marker_colors)
160{
161 gpu::Shader *shader = context.get_shader("compositor_keying_screen");
163
164 GPU_shader_uniform_1f(shader, "smoothness", smoothness);
165 GPU_shader_uniform_1i(shader, "number_of_markers", marker_positions.size());
166
167 /* SSBO needs to be aligned to 16 bytes, and since sizeof(float2) is only 8 bytes, we need to add
168 * a dummy element at the end for odd sizes to satisfy the alignment requirement. Notice that the
169 * number_of_markers uniform was already assigned above to the original size, so the dummy
170 * element has no effect in the shader. Also notice that the marker colors are always 16 byte
171 * aligned since sizeof(float4) is 16 bytes, so not need to add anything there. */
172 if (marker_positions.size() % 2 == 1) {
173 marker_positions.append(float2(0.0f));
174 }
175
176 gpu::StorageBuf *positions_ssbo = GPU_storagebuf_create_ex(marker_positions.size() *
177 sizeof(float2),
178 marker_positions.data(),
180 "Marker Positions");
181 const int positions_ssbo_location = GPU_shader_get_ssbo_binding(shader, "marker_positions");
182 GPU_storagebuf_bind(positions_ssbo, positions_ssbo_location);
183
184 gpu::StorageBuf *colors_ssbo = GPU_storagebuf_create_ex(marker_colors.size() * sizeof(float4),
185 marker_colors.data(),
187 "Marker Colors");
188 const int colors_ssbo_location = GPU_shader_get_ssbo_binding(shader, "marker_colors");
189 GPU_storagebuf_bind(colors_ssbo, colors_ssbo_location);
190
191 this->result.bind_as_image(shader, "output_img");
192
194
195 this->result.unbind_as_image();
196 GPU_storagebuf_unbind(positions_ssbo);
197 GPU_storagebuf_unbind(colors_ssbo);
199
200 GPU_storagebuf_free(positions_ssbo);
201 GPU_storagebuf_free(colors_ssbo);
202}
203
204void KeyingScreen::compute_cpu(const float smoothness,
205 const Vector<float2> &marker_positions,
206 const Vector<float4> &marker_colors)
207{
208 float squared_shape_parameter = math::square(1.0f / smoothness);
209 const int2 size = this->result.domain().data_size;
210 parallel_for(size, [&](const int2 texel) {
211 float2 normalized_pixel_location = (float2(texel) + float2(0.5f)) / float2(size);
212
213 /* Interpolate the markers using a Gaussian Radial Basis Function Interpolation with the
214 * reciprocal of the smoothness as the shaping parameter. Equal weights are assigned to all
215 * markers, so no RBF fitting is required. */
216 float sum_of_weights = 0.0f;
217 float4 weighted_sum = float4(0.0f);
218 for (const int64_t i : marker_positions.index_range()) {
219 float2 marker_position = marker_positions[i];
220 float2 difference = normalized_pixel_location - marker_position;
221 float squared_distance = math::dot(difference, difference);
222 float gaussian = math::exp(-squared_distance * squared_shape_parameter);
223
224 float4 marker_color = marker_colors[i];
225 weighted_sum += marker_color * gaussian;
226 sum_of_weights += gaussian;
227 }
228 weighted_sum /= sum_of_weights;
229
230 this->result.store_pixel(texel, Color(weighted_sum));
231 });
232}
233
238
239/* --------------------------------------------------------------------
240 * Keying Screen Container.
241 */
242
244{
245 /* First, delete all cached keying screens that are no longer needed. */
246 for (auto &cached_keying_screens_for_id : map_.values()) {
247 cached_keying_screens_for_id.remove_if([](auto item) { return !item.value->needed; });
248 }
249 map_.remove_if([](auto item) { return item.value.is_empty(); });
250
251 /* Second, reset the needed status of the remaining cached keying screens to false to ready them
252 * to track their needed status for the next evaluation. */
253 for (auto &cached_keying_screens_for_id : map_.values()) {
254 for (auto &value : cached_keying_screens_for_id.values()) {
255 value->needed = false;
256 }
257 }
258}
259
261 MovieClip *movie_clip,
262 MovieTrackingObject *movie_tracking_object,
263 float smoothness)
264{
265 const KeyingScreenKey key(context.get_frame_number(), smoothness);
266
267 /* We concatenate the movie clip ID name with the tracking object name to cache multiple tracking
268 * objects per movie clip. */
269 const std::string library_key = movie_clip->id.lib ? movie_clip->id.lib->id.name : "";
270 const std::string id_key = std::string(movie_clip->id.name) + library_key;
271 const std::string object_key = id_key + movie_tracking_object->name;
272 auto &cached_keying_screens_for_id = map_.lookup_or_add_default(object_key);
273
274 /* Invalidate the cache for that movie clip if it was changed since it was cached. */
275 if (!cached_keying_screens_for_id.is_empty() &&
276 movie_clip->runtime.last_update != update_counts_.lookup(id_key))
277 {
278 cached_keying_screens_for_id.clear();
279 }
280
281 auto &keying_screen = *cached_keying_screens_for_id.lookup_or_add_cb(key, [&]() {
282 return std::make_unique<KeyingScreen>(context, movie_clip, movie_tracking_object, smoothness);
283 });
284
285 /* Store the current update count to later compare to and check if the movie clip changed. */
286 update_counts_.add_overwrite(id_key, movie_clip->runtime.last_update);
287
288 keying_screen.needed = true;
289 return keying_screen.result;
290}
291
292} // namespace blender::compositor
long long int int64_t
static DBVT_INLINE btScalar size(const btDbvtVolume &a)
Definition btDbvt.cpp:52
Domain(const int2 &size)
Definition domain.cc:20
Result & get(Context &context, MovieClip *movie_clip, MovieTrackingObject *movie_tracking_object, float smoothness)
KeyingScreenKey(int frame, float smoothness)
KeyingScreen(Context &context, MovieClip *movie_clip, MovieTrackingObject *movie_tracking_object, const float smoothness)
void store_pixel(const int2 &texel, const T &pixel_value)
void allocate_texture(const Domain domain, const bool from_pool=true, const std::optional< ResultStorageType > storage_type=std::nullopt)
Definition result.cc:554
void unbind_as_image() const
Definition result.cc:990
void bind_as_image(gpu::Shader *shader, const char *image_name, bool read=false) const
Definition result.cc:951
#define position
#define weighted_sum(val0, val1, val2, val3, weights)
void compute_dispatch_threads_at_least(gpu::Shader *shader, int2 threads_range, int2 local_size=int2(16))
Definition utilities.cc:235
static MovieClipUser get_movie_clip_user(Context &context, MovieClip *movie_clip)
ColorSceneLinear4f< eAlpha::Premultiplied > Color
Definition COM_result.hh:96
bool operator==(const BokehKernelKey &a, const BokehKernelKey &b)
static void compute_marker_points(MovieClip *movie_clip, MovieClipUser &movie_clip_user, MovieTrackingObject *movie_tracking_object, Vector< float2 > &marker_positions, Vector< float4 > &marker_colors)
void parallel_for(const int2 range, const Function &function)
T clamp(const T &a, const T &min, const T &max)
T exp(const T &x)
T dot(const QuaternionBase< T > &a, const QuaternionBase< T > &b)
T square(const T &a)
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
VecBase< uint8_t, 4 > uchar4
unsigned __int64 uint64_t
Definition BKE_text.h:91
void BKE_movieclip_get_size(MovieClip *clip, const MovieClipUser *user, int *r_width, int *r_height)
gpu::StorageBuf * GPU_storagebuf_create_ex(size_t size, const void *data, GPUUsageType usage, const char *name)
MINLINE void srgb_to_linearrgb_uchar4(float linear[4], const unsigned char srgb[4])
int GPU_shader_get_ssbo_binding(gpu::Shader *shader, const char *name)
VecBase< float, 4 > float4
void IMB_freeImBuf(ImBuf *ibuf)
VecBase< float, 2 > float2
unsigned char uchar
Definition BKE_text.h:58
void GPU_shader_unbind()
ImBuf * BKE_movieclip_get_ibuf(MovieClip *clip, const MovieClipUser *user)
void GPU_shader_bind(gpu::Shader *shader, const gpu::shader::SpecializationConstants *constants_state=nullptr)
void GPU_shader_uniform_1i(gpu::Shader *sh, const char *name, int value)
void GPU_storagebuf_bind(gpu::StorageBuf *ssbo, int slot)
void GPU_storagebuf_unbind(gpu::StorageBuf *ssbo)
void BKE_movieclip_user_set_frame(MovieClipUser *user, int framenr)
float BKE_movieclip_remap_scene_to_clip_frame(const MovieClip *clip, float framenr)
void GPU_shader_uniform_1f(gpu::Shader *sh, const char *name, float value)
void GPU_storagebuf_free(gpu::StorageBuf *ssbo)
ImBuf * BKE_tracking_get_pattern_imbuf(const ImBuf *ibuf, const MovieTrackingTrack *track, const MovieTrackingMarker *marker, bool anchored, bool disable_channels)
Definition tracking.cc:2627
constexpr uint64_t get_default_hash(const T &v, const Args &...args)
Definition BLI_hash.hh:252
MovieTrackingMarker * BKE_tracking_marker_get(MovieTrackingTrack *track, int framenr)
Definition tracking.cc:1363
char name[258]
Definition DNA_ID.h:433
struct Library * lib
Definition DNA_ID.h:421
const uint8_t * byte_data() const
const float * float_data() const
MovieClip_Runtime runtime
ListBaseT< MovieTrackingTrack > tracks
int size() const
Definition vector.hh:130