Blender V5.3
tracking_solver.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2011 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
10
11#include <climits>
12
13#include "MEM_guardedalloc.h"
14
15#include "DNA_movieclip_types.h"
16
17#include "BLI_listbase.hh"
18#include "BLI_math_matrix_c.hh"
19#include "BLI_math_vector_c.hh"
20#include "BLI_string_utf8.hh"
21
22#include "BLT_translation.hh"
23
24#include "BKE_fcurve.hh"
25#include "BKE_movieclip.hh"
26#include "BKE_tracking.hh"
27
28#include "RNA_prototypes.hh"
29
30#include "libmv-capi.h"
31#include "tracking_private.hh"
32
33namespace blender {
34
57
65
66/* Create new libmv Tracks structure from blender's tracks list. */
68 ListBaseT<MovieTrackingTrack> *tracksbase,
69 int width,
70 int height)
71{
72 int tracknr = 0;
73 MovieTrackingTrack *track;
74 libmv_Tracks *tracks = libmv_tracksNew();
75
76 track = tracksbase->first();
77 while (track) {
78 const FCurve *weight_fcurve = id_data_find_fcurve(
79 &clip->id, track, RNA_MovieTrackingTrack, "weight", 0, nullptr);
80
81 for (int a = 0; a < track->markersnr; a++) {
82 MovieTrackingMarker *marker = &track->markers[a];
83
84 if ((marker->flag & MARKER_DISABLED) == 0) {
85 float weight = track->weight;
86
87 if (weight_fcurve) {
88 int scene_framenr = BKE_movieclip_remap_clip_to_scene_frame(clip, marker->framenr);
89 weight = evaluate_fcurve(weight_fcurve, scene_framenr);
90 }
91
92 libmv_tracksInsert(tracks,
93 marker->framenr,
94 tracknr,
95 (marker->pos[0] + track->offset[0]) * width,
96 (marker->pos[1] + track->offset[1]) * height,
97 weight);
98 }
99 }
100
101 track = track->next;
102 tracknr++;
103 }
104
105 return tracks;
106}
107
108/* Retrieve refined camera intrinsics from libmv to blender. */
110 MovieTracking *tracking)
111{
112 libmv_Reconstruction *libmv_reconstruction = context->reconstruction;
114 libmv_reconstruction);
115
116 libmv_CameraIntrinsicsOptions camera_intrinsics_options;
117 libmv_cameraIntrinsicsExtractOptions(libmv_intrinsics, &camera_intrinsics_options);
118
119 tracking_trackingCameraFromIntrinscisOptions(tracking, &camera_intrinsics_options);
120}
121
122/* Retrieve reconstructed tracks from libmv to blender.
123 * Actually, this also copies reconstructed cameras
124 * from libmv to movie clip datablock.
125 */
127 MovieTracking *tracking)
128{
129 libmv_Reconstruction *libmv_reconstruction = context->reconstruction;
130 bool ok = true;
131 bool origin_set = false;
132 int sfra = context->sfra, efra = context->efra;
133 float imat[4][4];
134
135 MovieTrackingObject *tracking_object = BKE_tracking_object_get_named(tracking,
136 context->object_name);
137 MovieTrackingReconstruction *reconstruction = &tracking_object->reconstruction;
138
139 unit_m4(imat);
140
141 {
142
143 for (const auto [track_index, track] : tracking_object->tracks.enumerate()) {
144 double pos[3];
145
146 if (libmv_reprojectionPointForTrack(libmv_reconstruction, track_index, pos)) {
147 track.bundle_pos[0] = pos[0];
148 track.bundle_pos[1] = pos[1];
149 track.bundle_pos[2] = pos[2];
150
151 track.flag |= TRACK_HAS_BUNDLE;
152 track.error = libmv_reprojectionErrorForTrack(libmv_reconstruction, track_index);
153 }
154 else {
155 track.flag &= ~TRACK_HAS_BUNDLE;
156 ok = false;
157
158 printf("Unable to reconstruct position for track #%d '%s'\n", track_index, track.name);
159 }
160 }
161 }
162
163 if (reconstruction->cameras) {
164 MEM_delete(reconstruction->cameras);
165 }
166
167 reconstruction->camnr = 0;
168 reconstruction->cameras = nullptr;
169
170 MovieReconstructedCamera *reconstructed_cameras = MEM_new_array<MovieReconstructedCamera>(
171 (efra - sfra + 1), "temp reconstructed camera");
172
173 for (int a = sfra; a <= efra; a++) {
174 double matd[4][4];
175
176 if (libmv_reprojectionCameraForImage(libmv_reconstruction, a, matd)) {
177 float mat[4][4];
178 float error = libmv_reprojectionErrorForImage(libmv_reconstruction, a);
179
180 /* TODO(sergey): Use transpose utility. */
181 for (int i = 0; i < 4; i++) {
182 for (int j = 0; j < 4; j++) {
183 mat[i][j] = matd[i][j];
184 }
185 }
186
187 /* Ensure first camera has got zero rotation and transform.
188 * This is essential for object tracking to work -- this way
189 * we'll always know object and environment are properly
190 * oriented.
191 *
192 * There's one weak part tho, which is requirement object
193 * motion starts at the same frame as camera motion does,
194 * otherwise that;' be a Russian roulette whether object is
195 * aligned correct or not.
196 */
197 if (!origin_set) {
198 invert_m4_m4(imat, mat);
199 unit_m4(mat);
200 origin_set = true;
201 }
202 else {
203 mul_m4_m4m4(mat, imat, mat);
204 }
205
206 MovieReconstructedCamera *reconstructed_camera =
207 &reconstructed_cameras[reconstruction->camnr];
208 copy_m4_m4(reconstructed_camera->mat, mat);
209 reconstructed_camera->framenr = a;
210 reconstructed_camera->error = error;
211 reconstruction->camnr++;
212 }
213 else {
214 ok = false;
215 printf("No camera for frame %d\n", a);
216 }
217 }
218
219 if (reconstruction->camnr) {
220 const size_t size = reconstruction->camnr * sizeof(MovieReconstructedCamera);
221 reconstruction->cameras = MEM_new_array<MovieReconstructedCamera>(reconstruction->camnr,
222 "reconstructed camera");
223 memcpy(reconstruction->cameras, reconstructed_cameras, size);
224 }
225
226 if (origin_set) {
227 for (MovieTrackingTrack &track : tracking_object->tracks) {
228 if (track.flag & TRACK_HAS_BUNDLE) {
229 mul_v3_m4v3(track.bundle_pos, imat, track.bundle_pos);
230 }
231 }
232 }
233
234 MEM_delete(reconstructed_cameras);
235
236 return ok;
237}
238
239/* Retrieve all the libmv data from context to blender's side data blocks. */
241{
242 /* take the intrinsics back from libmv */
243 reconstruct_retrieve_libmv_intrinsics(context, tracking);
244
245 return reconstruct_retrieve_libmv_tracks(context, tracking);
246}
247
248/* Convert blender's refinement flags to libmv's. */
250 MovieTrackingObject *tracking_object)
251{
254 int flags = 0;
255
256 if ((tracking_object->flag & TRACKING_OBJECT_CAMERA) == 0) {
257 return 0;
258 }
259
260 if (refine & REFINE_FOCAL_LENGTH) {
262 }
263
264 if (refine & REFINE_PRINCIPAL_POINT) {
266 }
267
268 if (refine & REFINE_RADIAL_DISTORTION) {
270 }
271
272 if (refine & REFINE_TANGENTIAL_DISTORTION) {
274 }
275
276 return flags;
277}
278
279/* Count tracks which has markers at both of keyframes. */
281{
282 const int frame1 = tracking_object->keyframe1, frame2 = tracking_object->keyframe2;
283
284 int tot = 0;
285 for (MovieTrackingTrack &track : tracking_object->tracks) {
288 tot++;
289 }
290 }
291 }
292
293 return tot;
294}
295
297 MovieTrackingObject *tracking_object,
298 char *error_msg,
299 int error_msg_maxncpy)
300{
302 /* TODO: check for number of tracks? */
303 return true;
304 }
306 /* automatic keyframe selection does not require any pre-process checks */
307 if (reconstruct_count_tracks_on_both_keyframes(tracking_object) < 8) {
309 error_msg,
310 N_("At least 8 common tracks on both keyframes are needed for reconstruction"),
311 error_msg_maxncpy);
312
313 return false;
314 }
315 }
316
317#ifndef WITH_LIBMV
319 error_msg, N_("Blender is compiled without motion tracking library"), error_msg_maxncpy);
320 return false;
321#endif
322
323 return true;
324}
325
327 MovieClip *clip,
328 MovieTrackingObject *tracking_object,
329 int keyframe1,
330 int keyframe2,
331 int width,
332 int height)
333{
334 MovieTracking *tracking = &clip->tracking;
336 "MovieReconstructContext data");
337 const float aspy = 1.0f / tracking->camera.pixel_aspect;
338 const int num_tracks = tracking_object->tracks.count();
339 int sfra = INT_MAX, efra = INT_MIN;
340
341 STRNCPY_UTF8(context->object_name, tracking_object->name);
342 context->motion_flag = TrackingMotionFlag(tracking->settings.motion_flag);
343
344 context->select_keyframes = (tracking->settings.reconstruction_flag &
346
348 tracking, width, height, &context->camera_intrinsics_options);
349
350 context->tracks_map = tracks_map_new(context->object_name, num_tracks);
351
352 for (MovieTrackingTrack &track : tracking_object->tracks) {
353 int first = 0, last = track.markersnr - 1;
354 MovieTrackingMarker *first_marker = &track.markers[0];
355 MovieTrackingMarker *last_marker = &track.markers[track.markersnr - 1];
356
357 /* find first not-disabled marker */
358 while (first <= track.markersnr - 1 && first_marker->flag & MARKER_DISABLED) {
359 first++;
360 first_marker++;
361 }
362
363 /* find last not-disabled marker */
364 while (last >= 0 && last_marker->flag & MARKER_DISABLED) {
365 last--;
366 last_marker--;
367 }
368
369 if (first <= track.markersnr - 1) {
370 sfra = min_ii(sfra, first_marker->framenr);
371 }
372
373 if (last >= 0) {
374 efra = max_ii(efra, last_marker->framenr);
375 }
376
377 tracks_map_insert(context->tracks_map, &track);
378 }
379
380 context->sfra = sfra;
381 context->efra = efra;
382
383 context->tracks = libmv_tracks_new(clip, &tracking_object->tracks, width, height * aspy);
384 context->keyframe1 = keyframe1;
385 context->keyframe2 = keyframe2;
386 context->refine_flags = reconstruct_refine_intrinsics_get_flags(tracking, tracking_object);
387
388 context->error_message[0] = '\0';
389
390 return context;
391}
392
394 const char *error_message)
395{
396 if (context->error_message[0]) {
397 /* Only keep initial error message, the rest are inducted ones. */
398 return;
399 }
400 STRNCPY_UTF8(context->error_message, error_message);
401}
402
404{
405 return context->error_message;
406}
407
409{
410 if (context->reconstruction) {
411 libmv_reconstructionDestroy(context->reconstruction);
412 }
413
414 libmv_tracksDestroy(context->tracks);
415
416 tracks_map_free(context->tracks_map);
417
418 MEM_delete(context);
419}
420
421/* Callback which is called from libmv side to update progress in the interface. */
422static void reconstruct_update_solve_cb(void *customdata, double progress, const char *message)
423{
424 ReconstructProgressData *progressdata = static_cast<ReconstructProgressData *>(customdata);
425
426 if (progressdata->progress) {
427 *progressdata->progress = progress;
428 *progressdata->do_update = true;
429 }
430
432 progressdata->stats_message, progressdata->message_size, "Solving camera | %s", message);
433}
434
435/* Fill in reconstruction options structure from reconstruction context. */
438{
439 reconstruction_options->select_keyframes = context->select_keyframes;
440
441 reconstruction_options->keyframe1 = context->keyframe1;
442 reconstruction_options->keyframe2 = context->keyframe2;
443
444 reconstruction_options->refine_intrinsics = context->refine_flags;
445}
446
448 bool *stop,
449 bool *do_update,
450 float *progress,
451 char *stats_message,
452 int message_size)
453{
454 float error;
455
456 ReconstructProgressData progressdata;
457
458 libmv_ReconstructionOptions reconstruction_options;
459
460 progressdata.stop = stop;
461 progressdata.do_update = do_update;
462 progressdata.progress = progress;
463 progressdata.stats_message = stats_message;
464 progressdata.message_size = message_size;
465
466 reconstructionOptionsFromContext(&reconstruction_options, context);
467
468 if (context->motion_flag & TRACKING_MOTION_MODAL) {
469 context->reconstruction = libmv_solveModal(context->tracks,
470 &context->camera_intrinsics_options,
471 &reconstruction_options,
473 &progressdata);
474 }
475 else {
476 context->reconstruction = libmv_solveReconstruction(context->tracks,
477 &context->camera_intrinsics_options,
478 &reconstruction_options,
480 &progressdata);
481
482 if (context->select_keyframes) {
483 /* store actual keyframes used for reconstruction to update them in the interface later */
484 context->keyframe1 = reconstruction_options.keyframe1;
485 context->keyframe2 = reconstruction_options.keyframe2;
486 }
487 }
488
489 error = libmv_reprojectionError(context->reconstruction);
490
491 context->reprojection_error = error;
492}
493
495{
496 if (!libmv_reconstructionIsValid(context->reconstruction)) {
498 context, "Failed to solve the motion: most likely there are no good keyframes");
499 return false;
500 }
501
502 tracks_map_merge(context->tracks_map, tracking);
504
505 MovieTrackingObject *tracking_object = BKE_tracking_object_get_named(tracking,
506 context->object_name);
507 MovieTrackingReconstruction *reconstruction = &tracking_object->reconstruction;
508
509 /* update keyframe in the interface */
510 if (context->select_keyframes) {
511 tracking_object->keyframe1 = context->keyframe1;
512 tracking_object->keyframe2 = context->keyframe2;
513 }
514
515 reconstruction->error = context->reprojection_error;
516 reconstruction->flag |= TRACKING_RECONSTRUCTED;
517
518 if (!reconstruct_retrieve_libmv(context, tracking)) {
519 return false;
520 }
521
522 return true;
523}
524
525static void tracking_scale_reconstruction(ListBaseT<MovieTrackingTrack> *tracksbase,
526 MovieTrackingReconstruction *reconstruction,
527 const float scale[3])
528{
529 float first_camera_delta[3] = {0.0f, 0.0f, 0.0f};
530
531 if (reconstruction->camnr > 0) {
532 mul_v3_v3v3(first_camera_delta, reconstruction->cameras[0].mat[3], scale);
533 }
534
535 for (int i = 0; i < reconstruction->camnr; i++) {
536 MovieReconstructedCamera *camera = &reconstruction->cameras[i];
537 mul_v3_v3(camera->mat[3], scale);
538 sub_v3_v3(camera->mat[3], first_camera_delta);
539 }
540
541 for (MovieTrackingTrack &track : *tracksbase) {
542 if (track.flag & TRACK_HAS_BUNDLE) {
543 mul_v3_v3(track.bundle_pos, scale);
544 sub_v3_v3(track.bundle_pos, first_camera_delta);
545 }
546 }
547}
548
550{
551 for (MovieTrackingObject &object : tracking->objects) {
552 tracking_scale_reconstruction(&object.tracks, &object.reconstruction, scale);
553 }
554}
555
556} // namespace blender
#define STRNCPY_UTF8(dst, src)
#define MAX_NAME
Definition DNA_defs.h:63
Read Blender memory allocation and freeing.
#define INT_MAX
void camera(KernelGlobals &kg, const ShadingData &sd, float3 &outview, float &outdepth, float &outdist)
float3x3 scale(float3x3 mat, float2 scale)
void libmv_cameraIntrinsicsExtractOptions(const libmv_CameraIntrinsics *libmv_intrinsics, libmv_CameraIntrinsicsOptions *camera_intrinsics_options)
struct libmv_CameraIntrinsics libmv_CameraIntrinsics
libmv_Reconstruction * libmv_solveModal(const libmv_Tracks *libmv_tracks, const libmv_CameraIntrinsicsOptions *libmv_camera_intrinsics_options, const libmv_ReconstructionOptions *libmv_reconstruction_options, reconstruct_progress_update_cb progress_update_callback, void *callback_customdata)
double libmv_reprojectionErrorForImage(const libmv_Reconstruction *libmv_reconstruction, int image)
double libmv_reprojectionError(const libmv_Reconstruction *libmv_reconstruction)
void libmv_reconstructionDestroy(libmv_Reconstruction *libmv_reconstruction)
int libmv_reconstructionIsValid(libmv_Reconstruction *libmv_reconstruction)
int libmv_reprojectionCameraForImage(const libmv_Reconstruction *libmv_reconstruction, int image, double mat[4][4])
libmv_CameraIntrinsics * libmv_reconstructionExtractIntrinsics(libmv_Reconstruction *libmv_reconstruction)
int libmv_reprojectionPointForTrack(const libmv_Reconstruction *libmv_reconstruction, int track, double pos[3])
libmv_Reconstruction * libmv_solveReconstruction(const libmv_Tracks *libmv_tracks, const libmv_CameraIntrinsicsOptions *libmv_camera_intrinsics_options, libmv_ReconstructionOptions *libmv_reconstruction_options, reconstruct_progress_update_cb progress_update_callback, void *callback_customdata)
double libmv_reprojectionErrorForTrack(const libmv_Reconstruction *libmv_reconstruction, int track)
@ LIBMV_REFINE_TANGENTIAL_DISTORTION
@ LIBMV_REFINE_FOCAL_LENGTH
@ LIBMV_REFINE_RADIAL_DISTORTION
@ LIBMV_REFINE_PRINCIPAL_POINT
struct libmv_Tracks libmv_Tracks
BMesh Iterators.
void BKE_tracking_reconstruction_report_error_message(MovieReconstructContext *context, const char *error_message)
size_t size_t size_t BLI_snprintf_utf8(char *__restrict dst, size_t dst_maxncpy, const char *__restrict format,...) ATTR_NONNULL(1
bool BKE_tracking_reconstruction_finish(MovieReconstructContext *context, MovieTracking *tracking)
bool BKE_tracking_reconstruction_check(MovieTracking *tracking, MovieTrackingObject *tracking_object, char *error_msg, int error_msg_maxncpy)
void mul_v3_m4v3(float r[3], const float mat[4][4], const float vec[3])
bool invert_m4_m4(float inverse[4][4], const float mat[4][4])
char * BLI_strncpy_utf8(char *__restrict dst, const char *__restrict src, size_t dst_maxncpy) ATTR_NONNULL(1
void tracking_trackingCameraFromIntrinscisOptions(MovieTracking *tracking, const libmv_CameraIntrinsicsOptions *camera_intrinsics_options)
void mul_m4_m4m4(float R[4][4], const float A[4][4], const float B[4][4])
MINLINE int max_ii(int a, int b)
static void tracking_scale_reconstruction(ListBaseT< MovieTrackingTrack > *tracksbase, MovieTrackingReconstruction *reconstruction, const float scale[3])
void tracks_map_insert(TracksMap *map, MovieTrackingTrack *track)
void BKE_tracking_reconstruction_solve(MovieReconstructContext *context, bool *stop, bool *do_update, float *progress, char *stats_message, int message_size)
@ TRACKING_USE_KEYFRAME_SELECTION
bool BKE_tracking_track_has_enabled_marker_at_frame(MovieTrackingTrack *track, int framenr)
Definition tracking.cc:720
void BKE_tracking_dopesheet_tag_update(MovieTracking *tracking)
Definition tracking.cc:3355
MINLINE void sub_v3_v3(float r[3], const float a[3])
static int reconstruct_count_tracks_on_both_keyframes(MovieTrackingObject *tracking_object)
void BKE_tracking_reconstruction_scale(MovieTracking *tracking, float scale[3])
void tracking_cameraIntrinscisOptionsFromTracking(MovieTracking *tracking, int calibration_width, int calibration_height, libmv_CameraIntrinsicsOptions *camera_intrinsics_options)
static void reconstruct_update_solve_cb(void *customdata, double progress, const char *message)
MINLINE void mul_v3_v3(float r[3], const float a[3])
void * MEM_new_zeroed(size_t len, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2)
MovieReconstructContext * BKE_tracking_reconstruction_context_new(MovieClip *clip, MovieTrackingObject *tracking_object, int keyframe1, int keyframe2, int width, int height)
static bool reconstruct_retrieve_libmv_tracks(MovieReconstructContext *context, MovieTracking *tracking)
void unit_m4(float m[4][4])
@ REFINE_TANGENTIAL_DISTORTION
void tracks_map_merge(TracksMap *map, MovieTracking *tracking)
void tracks_map_free(TracksMap *map)
static int reconstruct_refine_intrinsics_get_flags(MovieTracking *tracking, MovieTrackingObject *tracking_object)
const char * BKE_tracking_reconstruction_error_message_get(const MovieReconstructContext *context)
TracksMap * tracks_map_new(const char *object_name, int num_tracks)
FCurve * id_data_find_fcurve(ID *id, void *data, StructRNA *type, const char *prop_name, int index, bool *r_driven)
float BKE_movieclip_remap_clip_to_scene_frame(const MovieClip *clip, float framenr)
MovieTrackingObject * BKE_tracking_object_get_named(MovieTracking *tracking, const char *name)
Definition tracking.cc:1971
static void reconstructionOptionsFromContext(libmv_ReconstructionOptions *reconstruction_options, MovieReconstructContext *context)
void copy_m4_m4(float m1[4][4], const float m2[4][4])
MINLINE void mul_v3_v3v3(float r[3], const float v1[3], const float v2[3])
void BKE_tracking_reconstruction_context_free(MovieReconstructContext *context)
MINLINE int min_ii(int a, int b)
static void reconstruct_retrieve_libmv_intrinsics(MovieReconstructContext *context, MovieTracking *tracking)
static libmv_Tracks * libmv_tracks_new(MovieClip *clip, ListBaseT< MovieTrackingTrack > *tracksbase, int width, int height)
float evaluate_fcurve(const FCurve *fcu, float evaltime)
static int reconstruct_retrieve_libmv(MovieReconstructContext *context, MovieTracking *tracking)
struct MovieTracking tracking
libmv_Reconstruction * reconstruction
libmv_CameraIntrinsicsOptions camera_intrinsics_options
MovieTrackingReconstruction reconstruction
ListBaseT< MovieTrackingTrack > tracks
struct MovieReconstructedCamera * cameras
TrackingRefineCameraFlag refine_camera_intrinsics
TrackingSettingsReconstructionFlag reconstruction_flag
struct MovieTrackingTrack * next
MovieTrackingMarker * markers
MovieTrackingCamera camera
MovieTrackingSettings settings
ListBaseT< MovieTrackingObject > objects
libmv_Tracks * libmv_tracksNew(void)
Definition stub.cc:84
void libmv_tracksDestroy(libmv_Tracks *)
Definition stub.cc:96
void libmv_tracksInsert(libmv_Tracks *, int, int, double, double, double)
Definition stub.cc:88
#define N_(msgid)