Blender V5.3
movie_read.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2001-2002 NaN Holding BV. All rights reserved.
2 * SPDX-FileCopyrightText: 2024-2026 Blender Authors
3 *
4 * SPDX-License-Identifier: GPL-2.0-or-later */
5
9
10#include <algorithm>
11#include <cctype>
12#include <climits>
13#include <cmath>
14#include <cstdio>
15#include <sys/types.h>
16
17#include "BLI_build_config.hh"
18#include "BLI_path_utils.hh"
19#include "BLI_string.hh"
20#include "BLI_string_utf8.hh"
21#include "BLI_task.hh"
22#include "BLI_utildefines.hh"
23
24#include "DNA_scene_types.h"
25#include "DNA_userdef_types.h"
26
27#include "MEM_guardedalloc.h"
28
30#include "IMB_imbuf.hh"
31#include "IMB_imbuf_types.hh"
32
33#include "CLG_log.h"
34
35#include "MOV_read.hh"
36
37#include "IMB_metadata.hh"
39#include "movie_read.hh"
40
41#ifdef WITH_FFMPEG
42# include "ffmpeg_swscale.hh"
43# include "movie_util.hh"
44
45extern "C" {
46# include <libavcodec/avcodec.h>
47# include <libavformat/avformat.h>
48# include <libavutil/hwcontext.h>
49# include <libavutil/imgutils.h>
50# include <libavutil/rational.h>
51# include <libswscale/swscale.h>
52
53# include "ffmpeg_compat.h"
54}
55
56#endif /* WITH_FFMPEG */
57
58namespace blender {
59
60#ifdef WITH_FFMPEG
61static CLG_LogRef LOG = {"video.read"};
62#endif
63
64#ifdef WITH_FFMPEG
65static void free_anim_ffmpeg(MovieReader *anim);
66#endif
67
68static bool anim_getnew(MovieReader *anim);
69
71{
72 if (anim == nullptr) {
73 return;
74 }
75
76#ifdef WITH_FFMPEG
77 free_anim_ffmpeg(anim);
78#endif
81
82 MEM_delete(anim);
83}
84
85void MOV_get_filename(const MovieReader *anim, char *filename, int filename_maxncpy)
86{
87 BLI_path_split_file_part(anim->filepath, filename, filename_maxncpy);
88}
89
91{
92 if (anim->state == MovieReader::State::Valid) {
93#ifdef WITH_FFMPEG
94 BLI_assert(anim->format_ctx != nullptr);
95 av_log(anim->format_ctx, AV_LOG_DEBUG, "METADATA FETCH\n");
96
97 AVDictionaryEntry *entry = nullptr;
98 while (true) {
99 entry = av_dict_get(anim->format_ctx->metadata, "", entry, AV_DICT_IGNORE_SUFFIX);
100 if (entry == nullptr) {
101 break;
102 }
103
104 /* Delay creation of the property group until there is actual metadata to put in there. */
106 IMB_metadata_set_field(anim->metadata, entry->key, entry->value);
107 }
108#endif
109 }
110 return anim->metadata;
111}
112
113static void probe_video_colorspace(MovieReader *anim, char r_colorspace_name[IM_MAX_SPACE])
114{
115 /* Use default role as fallback (i.e. it is an unknown combination of colorspace and primaries)
116 */
117 BLI_strncpy_utf8(r_colorspace_name,
120
122 if (!anim_getnew(anim)) {
123 return;
124 }
125 }
126
127#ifdef WITH_FFMPEG
128 /* Note that the ffmpeg enums are documented to match CICP codes. */
129 const int cicp[4] = {anim->codec_ctx->color_primaries,
130 anim->codec_ctx->color_trc,
131 anim->codec_ctx->colorspace,
132 anim->codec_ctx->color_range};
135
136 if (colorspace == nullptr) {
137 return;
138 }
139
142#endif /* WITH_FFMPEG */
143}
144
145MovieReader *MOV_open_file(const char *filepath,
146 const ImBufFlags ib_flags,
147 const int streamindex,
148 const bool keep_original_colorspace,
149 ColorManagedColorspaceSettings *colorspace_settings)
150{
151 MovieReader *anim;
152
153 BLI_assert(!BLI_path_is_rel(filepath));
154
155 anim = MEM_new<MovieReader>("anim struct");
156 if (anim != nullptr) {
157
158 STRNCPY(anim->filepath, filepath);
159 anim->ib_flags = ib_flags;
160 anim->streamindex = streamindex;
161 anim->keep_original_colorspace = keep_original_colorspace;
162
163 if (colorspace_settings && colorspace_settings->name[0] != '\0') {
164 /* Use colorspace from argument, if provided. */
165 STRNCPY_UTF8(anim->colorspace, colorspace_settings->name);
166 }
167 else {
168 /* Try to initialize colorspace from the FFmpeg stream by interpreting color information from
169 * it. */
170 char file_colorspace[IM_MAX_SPACE];
171 probe_video_colorspace(anim, file_colorspace);
172 STRNCPY_UTF8(anim->colorspace, file_colorspace);
173 if (colorspace_settings) {
174 /* Copy the used colorspace into output argument. */
175 IMB_colormanagement_colorspace_settings_set(colorspace_settings, file_colorspace);
176 }
177 }
178 }
179 return anim;
180}
181
183{
184#if !defined(WITH_FFMPEG)
185 UNUSED_VARS(anim);
186#endif
187
188#ifdef WITH_FFMPEG
189 if (anim->codec_ctx != nullptr) {
190 return true;
191 }
192#endif
193 return false;
194}
195
196void MOV_set_multiview_suffix(MovieReader *anim, const char *suffix)
197{
198 STRNCPY(anim->suffix, suffix);
199}
200
201#ifdef WITH_FFMPEG
202
203static double ffmpeg_stream_start_time_get(const AVStream *stream)
204{
205 if (stream->start_time == AV_NOPTS_VALUE) {
206 return 0.0;
207 }
208
209 return stream->start_time * av_q2d(stream->time_base);
210}
211
212static int ffmpeg_container_frame_count_get(const AVFormatContext *format_ctx,
213 const AVStream *video_stream,
214 const double frame_rate)
215{
216 /* Find audio stream to guess the duration of the video.
217 * Sometimes the audio AND the video stream have a start offset.
218 * The difference between these is the offset we want to use to
219 * calculate the video duration.
220 */
221 const double video_start = ffmpeg_stream_start_time_get(video_stream);
222 double audio_start = 0;
223
224 for (int i = 0; i < format_ctx->nb_streams; i++) {
225 if (format_ctx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_AUDIO) {
226 AVStream *audio_stream = format_ctx->streams[i];
227 audio_start = ffmpeg_stream_start_time_get(audio_stream);
228 break;
229 }
230 }
231
232 double stream_dur;
233
234 if (video_start > audio_start) {
235 stream_dur = double(format_ctx->duration) / AV_TIME_BASE - (video_start - audio_start);
236 }
237 else {
238 /* The video stream starts before or at the same time as the audio stream!
239 * We have to assume that the video stream is as long as the full format_ctx->duration.
240 */
241 stream_dur = double(format_ctx->duration) / AV_TIME_BASE;
242 }
243
244 return lround(stream_dur * frame_rate);
245}
246
247static int ffmpeg_frame_count_get(const AVFormatContext *format_ctx,
248 const AVStream *video_stream,
249 const double frame_rate)
250{
251 /* Use stream duration to determine frame count. */
252 if (video_stream->duration != AV_NOPTS_VALUE) {
253 const double stream_dur = video_stream->duration * av_q2d(video_stream->time_base);
254 return lround(stream_dur * frame_rate);
255 }
256
257 /* Fall back to manually estimating the video stream duration.
258 * This is because the video stream duration can be shorter than the `format_ctx->duration`.
259 */
260 if (format_ctx->duration != AV_NOPTS_VALUE) {
261 return ffmpeg_container_frame_count_get(format_ctx, video_stream, frame_rate);
262 }
263
264 /* Read frame count from the stream if we can. Note, that this value can not be trusted. */
265 if (video_stream->nb_frames != 0) {
266 return video_stream->nb_frames;
267 }
268
269 /* The duration has not been set, happens for single JPEG2000 images.
270 * NOTE: Leave the duration zeroed, although it could set to 1 so the file is recognized
271 * as a movie with 1 frame, leave as-is since image loading code-paths are preferred
272 * in this case. The following assertion should be valid in this case. */
273 BLI_assert(format_ctx->duration == AV_NOPTS_VALUE);
274 return 0;
275}
276
277static int calc_pix_fmt_max_component_bits(AVPixelFormat fmt)
278{
279 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt);
280 if (desc == nullptr) {
281 return 0;
282 }
283 int bits = 0;
284 for (int i = 0; i < desc->nb_components; i++) {
285 bits = max_ii(bits, desc->comp[i].depth);
286 }
287 return bits;
288}
289
290static AVFormatContext *init_format_context(const char *filepath,
291 int video_stream_index,
292 int &r_stream_index,
293 const AVCodec *forced_video_decoder)
294{
295 AVFormatContext *format_ctx = nullptr;
296 if (forced_video_decoder != nullptr) {
297 format_ctx = avformat_alloc_context();
298 format_ctx->video_codec_id = forced_video_decoder->id;
299 format_ctx->video_codec = forced_video_decoder;
300 }
301
302 if (avformat_open_input(&format_ctx, filepath, nullptr, nullptr) != 0) {
303 return nullptr;
304 }
305
306 if (avformat_find_stream_info(format_ctx, nullptr) < 0) {
307 avformat_close_input(&format_ctx);
308 return nullptr;
309 }
310
311 av_dump_format(format_ctx, 0, filepath, 0);
312
313 /* Find the video stream. */
314 r_stream_index = -1;
315 for (int i = 0; i < format_ctx->nb_streams; i++) {
316 if (ffmpeg_stream_counts_as_video(format_ctx, format_ctx->streams[i])) {
317 if (video_stream_index > 0) {
318 video_stream_index--;
319 continue;
320 }
321 r_stream_index = i;
322 break;
323 }
324 }
325
326 if (r_stream_index == -1) {
327 avformat_close_input(&format_ctx);
328 return nullptr;
329 }
330
331 return format_ctx;
332}
333
334static AVFormatContext *init_format_context_vpx_workarounds(const char *filepath,
335 int video_stream_index,
336 int &r_stream_index,
337 const AVCodec *&r_codec)
338{
339 AVFormatContext *format_ctx = init_format_context(
340 filepath, video_stream_index, r_stream_index, nullptr);
341 if (format_ctx == nullptr) {
342 return nullptr;
343 }
344
345 /* By default FFMPEG uses built-in VP8/VP9 decoders, however those do not detect
346 * alpha channel (see FFMPEG issue #8344 https://trac.ffmpeg.org/ticket/8344).
347 * The trick for VP8/VP9 is to explicitly force use of LIBVPX decoder.
348 * Only do this where alpha_mode=1 metadata is set. Note that in order to work,
349 * the previously initialized format context must be closed and a fresh one
350 * with explicitly requested codec must be created. */
351 r_codec = nullptr;
352 const AVStream *video_stream = format_ctx->streams[r_stream_index];
353 if (ELEM(video_stream->codecpar->codec_id, AV_CODEC_ID_VP8, AV_CODEC_ID_VP9)) {
354 AVDictionaryEntry *tag = nullptr;
355 tag = av_dict_get(video_stream->metadata, "alpha_mode", tag, AV_DICT_IGNORE_SUFFIX);
356 if (tag && STREQ(tag->value, "1")) {
357 r_codec = avcodec_find_decoder_by_name(
358 video_stream->codecpar->codec_id == AV_CODEC_ID_VP8 ? "libvpx" : "libvpx-vp9");
359 if (r_codec != nullptr) {
360 avformat_close_input(&format_ctx);
361 format_ctx = avformat_alloc_context();
362 format_ctx = init_format_context(filepath, video_stream_index, r_stream_index, r_codec);
363 if (format_ctx == nullptr) {
364 return nullptr;
365 }
366 }
367 }
368 }
369
370 if (r_codec == nullptr) {
371 /* Use default decoder. */
372 r_codec = avcodec_find_decoder(video_stream->codecpar->codec_id);
373 }
374
375 return format_ctx;
376}
377
378/* Device types are listed in order of priority. */
379# if OS_MAC
380static constexpr AVHWDeviceType ffmpeg_hw_device_types[] = {AV_HWDEVICE_TYPE_VIDEOTOOLBOX};
381# elif OS_WINDOWS
382static constexpr AVHWDeviceType ffmpeg_hw_device_types[] = {
383 AV_HWDEVICE_TYPE_D3D11VA, AV_HWDEVICE_TYPE_CUDA, AV_HWDEVICE_TYPE_VULKAN};
384# else
385static constexpr AVHWDeviceType ffmpeg_hw_device_types[] = {AV_HWDEVICE_TYPE_CUDA,
386 AV_HWDEVICE_TYPE_VULKAN};
387# endif
388
389static bool ffmpeg_hw_codec_supports_device(const AVCodec *codec, const AVHWDeviceType device_type)
390{
391 for (int i = 0; const AVCodecHWConfig *config = avcodec_get_hw_config(codec, i); i++) {
392 if ((config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) &&
393 config->device_type == device_type)
394 {
395 return true;
396 }
397 }
398 return false;
399}
400
401static AVHWDeviceType ffmpeg_hw_device_type_forced()
402{
403 if (!USER_DEVELOPER_TOOL_TEST(&U, use_video_decoding_debug)) {
404 return AV_HWDEVICE_TYPE_NONE;
405 }
406 switch (U.video_decoding_device) {
408 return AV_HWDEVICE_TYPE_NONE;
410 return AV_HWDEVICE_TYPE_VIDEOTOOLBOX;
412 return AV_HWDEVICE_TYPE_D3D11VA;
414 return AV_HWDEVICE_TYPE_CUDA;
416 return AV_HWDEVICE_TYPE_VULKAN;
417 }
418 return AV_HWDEVICE_TYPE_NONE;
419}
420
421static void ffmpeg_hw_setup_decode(const MovieReader *anim,
422 AVCodecContext *codec_ctx,
423 const AVCodec *codec)
424{
425 if ((U.gpu_flag & USER_GPU_FLAG_VIDEO_DECODING) == 0 ||
426 flag_is_set(anim->ib_flags, ImBufFlags::Deinterlace))
427 {
428 return;
429 }
430 const char *pix_fmt_name = av_get_pix_fmt_name(codec_ctx->pix_fmt);
431 if (pix_fmt_name == nullptr) {
432 pix_fmt_name = "unknown";
433 }
434 const AVHWDeviceType forced_device_type = ffmpeg_hw_device_type_forced();
435 const Span<AVHWDeviceType> device_types = forced_device_type != AV_HWDEVICE_TYPE_NONE ?
436 Span(&forced_device_type, 1) :
437 Span(ffmpeg_hw_device_types,
438 ARRAY_SIZE(ffmpeg_hw_device_types));
439 for (const AVHWDeviceType device_type : device_types) {
440 if (!ffmpeg_hw_codec_supports_device(codec, device_type)) {
441 continue;
442 }
443 if (AVBufferRef *hw_device_ctx = ffmpeg_hw_device_get(device_type)) {
444 codec_ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx);
445 CLOG_INFO(&LOG,
446 "Using %s hardware decoding for %s %s %dx%d (%s)",
447 av_hwdevice_get_type_name(device_type),
448 codec->name,
449 pix_fmt_name,
450 codec_ctx->width,
451 codec_ctx->height,
452 anim->filepath);
453 return;
454 }
455 }
456 CLOG_INFO(&LOG,
457 "Using software decoding for %s %s %dx%d (%s)",
458 codec->name,
459 pix_fmt_name,
460 codec_ctx->width,
461 codec_ctx->height,
462 anim->filepath);
463}
464
465static AVPixelFormat ffmpeg_codec_pix_fmt_get(const AVCodecContext *codec_ctx)
466{
467 /* If using hardware-accelerated, #pix_fmt does not have the nominal codec format and only refers
468 * to the hardware frame format. However, we can't just always choose #sw_pix_fmt, as it's not
469 * set until the first frame is decoded. */
470 const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(codec_ctx->pix_fmt);
471 return (desc->flags & AV_PIX_FMT_FLAG_HWACCEL) ? codec_ctx->sw_pix_fmt : codec_ctx->pix_fmt;
472}
473
474static void ffmpeg_free_decoder(MovieReader *anim)
475{
476 avcodec_free_context(&anim->codec_ctx);
477 avformat_close_input(&anim->format_ctx);
478 av_packet_free(&anim->cur_packet);
479
480 av_frame_free(&anim->frame);
481 av_frame_free(&anim->frame_sw);
482 av_frame_free(&anim->frame_backup);
483 av_frame_free(&anim->frame_rgb);
484 if (anim->frame_deinterlaced->data[0] != nullptr) {
485 MEM_delete(anim->frame_deinterlaced->data[0]);
486 }
487 av_frame_free(&anim->frame_deinterlaced);
488 if (anim->sws_ctx != nullptr) {
489 ffmpeg_sws_release_context(anim->sws_ctx);
490 anim->sws_ctx = nullptr;
491 }
492}
493
494static int startffmpeg(MovieReader *anim)
495{
496 if (anim == nullptr) {
497 return -1;
498 }
499
500 int video_stream_index;
501 const AVCodec *codec = nullptr;
502 AVFormatContext *format_ctx = init_format_context_vpx_workarounds(
503 anim->filepath, anim->streamindex, video_stream_index, codec);
504 if (format_ctx == nullptr || codec == nullptr) {
505 avformat_close_input(&format_ctx);
506 return -1;
507 }
508
509 AVCodecContext *codec_ctx = avcodec_alloc_context3(nullptr);
510 AVStream *video_stream = format_ctx->streams[video_stream_index];
511 avcodec_parameters_to_context(codec_ctx, video_stream->codecpar);
512 codec_ctx->workaround_bugs = FF_BUG_AUTODETECT;
513
514 if (codec->capabilities & AV_CODEC_CAP_OTHER_THREADS) {
515 codec_ctx->thread_count = 0;
516 }
517 else {
518 codec_ctx->thread_count = MOV_thread_count();
519 }
520
521 if (codec->capabilities & AV_CODEC_CAP_FRAME_THREADS) {
522 codec_ctx->thread_type = FF_THREAD_FRAME;
523 }
524 else if (codec->capabilities & AV_CODEC_CAP_SLICE_THREADS) {
525 codec_ctx->thread_type = FF_THREAD_SLICE;
526 }
527
528 ffmpeg_hw_setup_decode(anim, codec_ctx, codec);
529
530 if (avcodec_open2(codec_ctx, codec, nullptr) < 0) {
531 avcodec_free_context(&codec_ctx);
532 avformat_close_input(&format_ctx);
533 return -1;
534 }
535 if (codec_ctx->pix_fmt == AV_PIX_FMT_NONE) {
536 avcodec_free_context(&codec_ctx);
537 avformat_close_input(&format_ctx);
538 return -1;
539 }
540
541 /* Check if we need the "never seek, only decode one frame" ffmpeg bug workaround. */
542 const bool is_ogg_container = STREQ(format_ctx->iformat->name, "ogg");
543 const bool is_non_ogg_video = video_stream->codecpar->codec_id != AV_CODEC_ID_THEORA;
544 const bool is_video_thumbnail = (video_stream->disposition & AV_DISPOSITION_ATTACHED_PIC) != 0;
545 anim->never_seek_decode_one_frame = is_ogg_container && is_non_ogg_video && is_video_thumbnail;
546
547 anim->frame_rate = av_guess_frame_rate(format_ctx, video_stream, nullptr);
548 if (anim->never_seek_decode_one_frame) {
549 /* Files that need this workaround have nonsensical frame rates too, resulting
550 * in "millions of frames" if done through regular math. Treat frame-rate as 24/1 instead. */
551 anim->frame_rate = {24, 1};
552 }
553 int frs_num = anim->frame_rate.num;
554 double frs_den = anim->frame_rate.den;
555
556 frs_den *= AV_TIME_BASE;
557
558 while (frs_num % 10 == 0 && frs_den >= 2.0 && frs_num > 10) {
559 frs_num /= 10;
560 frs_den /= 10;
561 }
562
563 anim->frs_sec = frs_num;
564 anim->frs_sec_base = frs_den / AV_TIME_BASE;
565 /* Save the relative start time for the video. IE the start time in relation to where playback
566 * starts. */
567 anim->start_offset = ffmpeg_stream_start_time_get(video_stream);
568 anim->duration_in_frames = ffmpeg_frame_count_get(
569 format_ctx, video_stream, av_q2d(anim->frame_rate));
570
571 anim->x = codec_ctx->width;
572 anim->y = codec_ctx->height;
573 anim->video_rotation = ffmpeg_get_video_rotation(video_stream);
574
575 /* Decode >8bit videos into floating point image. */
576 anim->is_float = calc_pix_fmt_max_component_bits(ffmpeg_codec_pix_fmt_get(codec_ctx)) > 8;
577
578 anim->format_ctx = format_ctx;
579 anim->codec_ctx = codec_ctx;
580 anim->codec = codec;
581 anim->video_stream_index = video_stream_index;
582
583 anim->cur_frame_index = 0;
584 anim->cur_pts = -1;
585 anim->cur_key_frame_pts = -1;
586 anim->cur_packet = av_packet_alloc();
587 anim->cur_packet->stream_index = -1;
588
589 anim->frame = av_frame_alloc();
590 anim->frame_sw = av_frame_alloc();
591 anim->frame_backup = av_frame_alloc();
592 anim->frame_backup_complete = false;
593 anim->frame_complete = false;
594 anim->frame_deinterlaced = av_frame_alloc();
595 anim->frame_rgb = av_frame_alloc();
596 /* Ideally we'd use AV_PIX_FMT_RGBAF32LE for floats, but currently (ffmpeg 6.1)
597 * swscale does not support that as destination. So using AV_PIX_FMT_GBRAPF32LE
598 * with manual interleaving to RGBA floats. */
599 anim->frame_rgb->format = anim->is_float ? AV_PIX_FMT_GBRAPF32LE : AV_PIX_FMT_RGBA;
600 anim->frame_rgb->width = anim->x;
601 anim->frame_rgb->height = anim->y;
602
603 const size_t align = ffmpeg_get_buffer_alignment();
604 if (av_frame_get_buffer(anim->frame_rgb, align) < 0) {
605 CLOG_ERROR(&LOG, "Could not allocate frame data.");
606 ffmpeg_free_decoder(anim);
607 return -1;
608 }
609
610 if (flag_is_set(anim->ib_flags, ImBufFlags::Deinterlace)) {
611 anim->frame_deinterlaced->format = anim->codec_ctx->pix_fmt;
612 anim->frame_deinterlaced->width = anim->codec_ctx->width;
613 anim->frame_deinterlaced->height = anim->codec_ctx->height;
614 av_image_fill_arrays(
615 anim->frame_deinterlaced->data,
616 anim->frame_deinterlaced->linesize,
618 av_image_get_buffer_size(
619 anim->codec_ctx->pix_fmt, anim->codec_ctx->width, anim->codec_ctx->height, 1),
620 "ffmpeg deinterlace"),
621 anim->codec_ctx->pix_fmt,
622 anim->codec_ctx->width,
623 anim->codec_ctx->height,
624 1);
625 }
626
627 anim->src_pix_fmt = ffmpeg_codec_pix_fmt_get(anim->codec_ctx);
628
629 /* Use full_chroma_int + accurate_rnd YUV->RGB conversion flags. Otherwise
630 * the conversion is not fully accurate and introduces some banding and color
631 * shifts, particularly in dark regions. See issue #111703 or upstream
632 * ffmpeg ticket https://trac.ffmpeg.org/ticket/1582 */
633 anim->sws_ctx = ffmpeg_sws_get_context(anim->x,
634 anim->y,
635 anim->src_pix_fmt,
636 anim->codec_ctx->color_range == AVCOL_RANGE_JPEG,
637 anim->codec_ctx->colorspace,
638 anim->x,
639 anim->y,
640 anim->frame_rgb->format,
641 false,
642 -1,
643 SWS_POINT | SWS_FULL_CHR_H_INT | SWS_ACCURATE_RND);
644
645 if (!anim->sws_ctx) {
647 "ffmpeg: swscale can't transform from pixel format %s to %s (%s)",
648 av_get_pix_fmt_name(anim->src_pix_fmt),
649 av_get_pix_fmt_name((AVPixelFormat)anim->frame_rgb->format),
650 anim->filepath);
651 ffmpeg_free_decoder(anim);
652 return -1;
653 }
654
655 return 0;
656}
657
658static double ffmpeg_pts_per_frame_get(const MovieReader *anim)
659{
660 const AVStream *v_st = anim->format_ctx->streams[anim->video_stream_index];
661 const AVRational time_base = v_st->time_base;
662 return av_q2d(av_inv_q(av_mul_q(anim->frame_rate, time_base)));
663}
664
665/* Store backup frame.
666 * With VFR movies, if PTS is not matched perfectly, scanning continues to look for next PTS.
667 * It is likely to overshoot and scanning stops. Having previous frame backed up, it is possible
668 * to use it when overshoot happens.
669 */
670static void ffmpeg_double_buffer_backup_frame_store(MovieReader *anim, int64_t target_pts)
671{
672 /* `anim->frame` is beyond `target_pts`. Don't store it. */
673 if (anim->frame_backup_complete && anim->cur_pts >= target_pts) {
674 return;
675 }
676 if (!anim->frame_complete) {
677 return;
678 }
679
680 if (anim->frame_backup_complete) {
681 av_frame_unref(anim->frame_backup);
682 }
683
684 av_frame_move_ref(anim->frame_backup, anim->frame);
685 anim->frame_backup_complete = true;
686}
687
688/* Free stored backup frame. */
689static void ffmpeg_double_buffer_backup_frame_clear(MovieReader *anim)
690{
691 if (anim->frame_backup_complete) {
692 av_frame_unref(anim->frame_backup);
693 }
694 anim->frame_backup_complete = false;
695}
696
697/* Return recently decoded frame. If it does not exist, return frame from backup buffer. */
698static AVFrame *ffmpeg_double_buffer_frame_fallback_get(MovieReader *anim)
699{
700 av_log(anim->format_ctx, AV_LOG_ERROR, "DECODE UNHAPPY: PTS not matched!\n");
701
702 if (anim->frame_complete) {
703 return anim->frame;
704 }
705 if (anim->frame_backup_complete) {
706 return anim->frame_backup;
707 }
708 return nullptr;
709}
710
711/* Convert from ffmpeg planar GBRA layout to ImBuf interleaved RGBA, applying
712 * video rotation in the same go if needed. */
713static void float_planar_to_interleaved(const AVFrame *frame, const int rotation, ImBuf *ibuf)
714{
715 const size_t src_linesize = frame->linesize[0];
716 BLI_assert_msg(frame->linesize[1] == src_linesize && frame->linesize[2] == src_linesize &&
717 frame->linesize[3] == src_linesize,
718 "ffmpeg frame should be 4 same size planes for a floating point image case");
719 float *float_data = ibuf->float_data_for_write();
720 threading::parallel_for(IndexRange(ibuf->y), 256, [&](const IndexRange y_range) {
721 const int size_x = ibuf->x;
722 const int size_y = ibuf->y;
723 if (rotation == 90) {
724 /* 90 degree rotation. */
725 for (const int64_t y : y_range) {
726 int64_t src_offset = src_linesize * (size_y - y - 1);
727 const float *src_g = reinterpret_cast<const float *>(frame->data[0] + src_offset);
728 const float *src_b = reinterpret_cast<const float *>(frame->data[1] + src_offset);
729 const float *src_r = reinterpret_cast<const float *>(frame->data[2] + src_offset);
730 const float *src_a = reinterpret_cast<const float *>(frame->data[3] + src_offset);
731 float *dst = float_data + (y + (size_x - 1) * size_y) * 4;
732 for (int x = 0; x < size_x; x++) {
733 dst[0] = *src_r++;
734 dst[1] = *src_g++;
735 dst[2] = *src_b++;
736 dst[3] = *src_a++;
737 dst -= size_y * 4;
738 }
739 }
740 }
741 else if (rotation == 180) {
742 /* 180 degree rotation. */
743 for (const int64_t y : y_range) {
744 int64_t src_offset = src_linesize * (size_y - y - 1);
745 const float *src_g = reinterpret_cast<const float *>(frame->data[0] + src_offset);
746 const float *src_b = reinterpret_cast<const float *>(frame->data[1] + src_offset);
747 const float *src_r = reinterpret_cast<const float *>(frame->data[2] + src_offset);
748 const float *src_a = reinterpret_cast<const float *>(frame->data[3] + src_offset);
749 float *dst = float_data + ((size_y - y - 1) * size_x + size_x - 1) * 4;
750 for (int x = 0; x < size_x; x++) {
751 dst[0] = *src_r++;
752 dst[1] = *src_g++;
753 dst[2] = *src_b++;
754 dst[3] = *src_a++;
755 dst -= 4;
756 }
757 }
758 }
759 else if (rotation == 270) {
760 /* 270 degree rotation. */
761 for (const int64_t y : y_range) {
762 int64_t src_offset = src_linesize * (size_y - y - 1);
763 const float *src_g = reinterpret_cast<const float *>(frame->data[0] + src_offset);
764 const float *src_b = reinterpret_cast<const float *>(frame->data[1] + src_offset);
765 const float *src_r = reinterpret_cast<const float *>(frame->data[2] + src_offset);
766 const float *src_a = reinterpret_cast<const float *>(frame->data[3] + src_offset);
767 float *dst = float_data + (size_y - y - 1) * 4;
768 for (int x = 0; x < size_x; x++) {
769 dst[0] = *src_r++;
770 dst[1] = *src_g++;
771 dst[2] = *src_b++;
772 dst[3] = *src_a++;
773 dst += size_y * 4;
774 }
775 }
776 }
777 else if (rotation == 0) {
778 /* No rotation. */
779 for (const int64_t y : y_range) {
780 int64_t src_offset = src_linesize * (size_y - y - 1);
781 const float *src_g = reinterpret_cast<const float *>(frame->data[0] + src_offset);
782 const float *src_b = reinterpret_cast<const float *>(frame->data[1] + src_offset);
783 const float *src_r = reinterpret_cast<const float *>(frame->data[2] + src_offset);
784 const float *src_a = reinterpret_cast<const float *>(frame->data[3] + src_offset);
785 float *dst = float_data + size_x * y * 4;
786 for (int x = 0; x < size_x; x++) {
787 *dst++ = *src_r++;
788 *dst++ = *src_g++;
789 *dst++ = *src_b++;
790 *dst++ = *src_a++;
791 }
792 }
793 }
794 });
795
796 if (ELEM(rotation, 90, 270)) {
797 std::swap(ibuf->x, ibuf->y);
798 }
799}
800
806static void ffmpeg_postprocess(MovieReader *anim, AVFrame *input, ImBuf *ibuf)
807{
808 int filter_y = 0;
809
810 /* This means the data wasn't read properly,
811 * this check stops crashing */
812 if (input->data[0] == nullptr && input->data[1] == nullptr && input->data[2] == nullptr &&
813 input->data[3] == nullptr)
814 {
816 "ffmpeg_fetchibuf: "
817 "data not read properly...");
818 return;
819 }
820
821 av_log(anim->format_ctx,
822 AV_LOG_DEBUG,
823 " POSTPROC: AVFrame planes: %p %p %p %p\n",
824 input->data[0],
825 input->data[1],
826 input->data[2],
827 input->data[3]);
828
829 if (flag_is_set(anim->ib_flags, ImBufFlags::Deinterlace)) {
830 if (ffmpeg_deinterlace(anim->frame_deinterlaced,
831 anim->frame,
832 anim->codec_ctx->pix_fmt,
833 anim->codec_ctx->width,
834 anim->codec_ctx->height) < 0)
835 {
836 filter_y = true;
837 }
838 else {
839 input = anim->frame_deinterlaced;
840 }
841 }
842
843 /* Hardware-decoded frames from the GPU might be in a different format than the video's original
844 * format, e.g. we might get NV12 from a YUV420P video. Even with software decoding, the format
845 * can change mid-stream, e.g. 8-bit to 10-bit, so try to resync the swscale context. */
846 if (input->format != anim->src_pix_fmt && anim->sws_ctx != nullptr) {
847 ffmpeg_sws_release_context(anim->sws_ctx);
848 anim->src_pix_fmt = AVPixelFormat(input->format);
849 anim->sws_ctx = ffmpeg_sws_get_context(anim->x,
850 anim->y,
851 anim->src_pix_fmt,
852 anim->codec_ctx->color_range == AVCOL_RANGE_JPEG,
853 anim->codec_ctx->colorspace,
854 anim->x,
855 anim->y,
856 anim->frame_rgb->format,
857 false,
858 -1,
859 SWS_POINT | SWS_FULL_CHR_H_INT | SWS_ACCURATE_RND);
860 }
861
862 if (anim->sws_ctx == nullptr) {
864 "ffmpeg: swscale can't transform from pixel format %s to %s (%s)",
865 av_get_pix_fmt_name(anim->src_pix_fmt),
866 av_get_pix_fmt_name(AVPixelFormat(anim->frame_rgb->format)),
867 anim->filepath);
868 return;
869 }
870
871 bool already_rotated = false;
872 if (anim->is_float) {
873 /* Float images are converted into planar GBRA layout by swscale (since
874 * it does not support direct YUV->RGBA float interleaved conversion).
875 * Do vertical flip and interleave into RGBA manually. */
876 /* Decode, then do vertical flip into destination. */
877 ffmpeg_sws_scale_frame(anim->sws_ctx, anim->frame_rgb, input);
878
879 float_planar_to_interleaved(anim->frame_rgb, anim->video_rotation, ibuf);
880 already_rotated = true;
881 }
882 else {
883 /* If final destination image layout matches that of decoded RGB frame (including
884 * any line padding done by ffmpeg for SIMD alignment), we can directly
885 * decode into that, doing the vertical flip in the same step. Otherwise have
886 * to do a separate flip. */
887 const int ibuf_linesize = ibuf->x * 4;
888 const int rgb_linesize = anim->frame_rgb->linesize[0];
889 bool scale_to_ibuf = (rgb_linesize == ibuf_linesize);
890 /* swscale on arm64 before ffmpeg 6.0 (libswscale major version 7)
891 * could not handle negative line sizes. That has been fixed in all major
892 * ffmpeg releases in early 2023, but easier to just check for "below 7". */
893# if (defined(__aarch64__) || defined(_M_ARM64)) && (LIBSWSCALE_VERSION_MAJOR < 7)
894 scale_to_ibuf = false;
895# endif
896 uint8_t *rgb_data = anim->frame_rgb->data[0];
897
898 if (scale_to_ibuf) {
899 /* Decode RGB and do vertical flip directly into destination image, by using negative
900 * line size. */
901 anim->frame_rgb->linesize[0] = -ibuf_linesize;
902 anim->frame_rgb->data[0] = ibuf->byte_data_for_write() + (ibuf->y - 1) * ibuf_linesize;
903
904 ffmpeg_sws_scale_frame(anim->sws_ctx, anim->frame_rgb, input);
905
906 anim->frame_rgb->linesize[0] = rgb_linesize;
907 anim->frame_rgb->data[0] = rgb_data;
908 }
909 else {
910 /* Decode, then do vertical flip into destination. */
911 ffmpeg_sws_scale_frame(anim->sws_ctx, anim->frame_rgb, input);
912
913 /* Use negative line size to do vertical image flip. */
914 const int src_linesize[4] = {-rgb_linesize, 0, 0, 0};
915 const uint8_t *const src[4] = {
916 rgb_data + (anim->y - 1) * rgb_linesize, nullptr, nullptr, nullptr};
917 int dst_size = av_image_get_buffer_size(AVPixelFormat(anim->frame_rgb->format),
918 anim->frame_rgb->width,
919 anim->frame_rgb->height,
920 1);
921 av_image_copy_to_buffer(ibuf->byte_data_for_write(),
922 dst_size,
923 src,
924 src_linesize,
925 AVPixelFormat(anim->frame_rgb->format),
926 anim->x,
927 anim->y,
928 1);
929 }
930 }
931
932 if (filter_y) {
933 IMB_filtery(ibuf);
934 }
935
936 /* Rotate video if display matrix is multiple of 90 degrees. */
937 if (!already_rotated && ELEM(anim->video_rotation, 90, 180, 270)) {
938 IMB_rotate_orthogonal(ibuf, anim->video_rotation);
939 }
940}
941
942static void final_frame_log(MovieReader *anim,
943 int64_t frame_pts_start,
944 int64_t frame_pts_end,
945 const char *str)
946{
947 av_log(anim->format_ctx,
948 AV_LOG_INFO,
949 "DECODE HAPPY: %s frame PTS range %" PRId64 " - %" PRId64 ".\n",
950 str,
951 frame_pts_start,
952 frame_pts_end);
953}
954
955static bool ffmpeg_pts_isect(int64_t pts_start, int64_t pts_end, int64_t target_pts)
956{
957 return pts_start <= target_pts && target_pts < pts_end;
958}
959
960/* Return frame that matches `target_pts`, nullptr if matching frame does not exist. */
961static AVFrame *ffmpeg_frame_by_pts_get(MovieReader *anim, int64_t target_pts)
962{
963 /* NOTE: `frame->pts + frame->pkt_duration` does not always match pts of next frame.
964 * See footage from #86361. Here it is OK to use, because PTS must match current or backup frame.
965 * If there is no current frame, return nullptr.
966 */
967 if (!anim->frame_complete) {
968 return nullptr;
969 }
970
971 if (anim->never_seek_decode_one_frame) {
972 /* If we only decode one frame, return it. */
973 return anim->frame;
974 }
975
976 const bool backup_frame_ready = anim->frame_backup_complete;
977 const int64_t recent_start = av_get_pts_from_frame(anim->frame);
978 const int64_t recent_end = recent_start + av_get_frame_duration_in_pts_units(anim->frame);
979 const int64_t backup_start = backup_frame_ready ? av_get_pts_from_frame(anim->frame_backup) : 0;
980
981 AVFrame *best_frame = nullptr;
982 if (ffmpeg_pts_isect(recent_start, recent_end, target_pts)) {
983 final_frame_log(anim, recent_start, recent_end, "Recent");
984 best_frame = anim->frame;
985 }
986 else if (backup_frame_ready && ffmpeg_pts_isect(backup_start, recent_start, target_pts)) {
987 final_frame_log(anim, backup_start, recent_start, "Backup");
988 best_frame = anim->frame_backup;
989 }
990 return best_frame;
991}
992
993static void ffmpeg_decode_store_frame_pts(MovieReader *anim)
994{
995 anim->cur_pts = av_get_pts_from_frame(anim->frame);
996
997# ifdef FFMPEG_OLD_KEY_FRAME_QUERY_METHOD
998 if (anim->frame->key_frame)
999# else
1000 if (anim->frame->flags & AV_FRAME_FLAG_KEY)
1001# endif
1002 {
1003 anim->cur_key_frame_pts = anim->cur_pts;
1004 }
1005
1006 av_log(anim->format_ctx,
1007 AV_LOG_DEBUG,
1008 " FRAME DONE: cur_pts=%" PRId64 ", guessed_pts=%" PRId64 "\n",
1009 av_get_pts_from_frame(anim->frame),
1010 int64_t(anim->cur_pts));
1011}
1012
1013static int ffmpeg_read_video_frame(MovieReader *anim, AVPacket *packet)
1014{
1015 int ret = 0;
1016 while ((ret = av_read_frame(anim->format_ctx, packet)) >= 0) {
1017 if (packet->stream_index == anim->video_stream_index) {
1018 break;
1019 }
1020 av_packet_unref(packet);
1021 packet->stream_index = -1;
1022 }
1023
1024 return ret;
1025}
1026
1027static void ffmpeg_cur_packet_clear(MovieReader *anim)
1028{
1029 av_packet_unref(anim->cur_packet);
1030 anim->cur_packet->stream_index = -1;
1031}
1032
1033/* decode one video frame also considering the packet read into cur_packet */
1034static bool ffmpeg_decode_video_frame(MovieReader *anim)
1035{
1036 av_log(anim->format_ctx, AV_LOG_DEBUG, " DECODE VIDEO FRAME\n");
1037
1038 /* Sometimes, decoder returns more than one frame per sent packet. Check if frames are available.
1039 * This frames must be read, otherwise decoding will fail. See #91405. */
1040 anim->frame_complete = avcodec_receive_frame(anim->codec_ctx, anim->frame) == 0;
1041 if (anim->frame_complete) {
1042 av_log(anim->format_ctx, AV_LOG_DEBUG, " DECODE FROM CODEC BUFFER\n");
1043 ffmpeg_decode_store_frame_pts(anim);
1044 return true;
1045 }
1046
1047 ffmpeg_cur_packet_clear(anim);
1048
1049 int ret;
1050 while ((ret = ffmpeg_read_video_frame(anim, anim->cur_packet)) >= 0) {
1051 av_log(anim->format_ctx,
1052 AV_LOG_DEBUG,
1053 "READ: strID=%d dts=%" PRId64 " pts=%" PRId64 " %s\n",
1054 anim->cur_packet->stream_index,
1055 (anim->cur_packet->dts == AV_NOPTS_VALUE) ? -1 : int64_t(anim->cur_packet->dts),
1056 (anim->cur_packet->pts == AV_NOPTS_VALUE) ? -1 : int64_t(anim->cur_packet->pts),
1057 (anim->cur_packet->flags & AV_PKT_FLAG_KEY) ? " KEY" : "");
1058
1059 avcodec_send_packet(anim->codec_ctx, anim->cur_packet);
1060 anim->frame_complete = avcodec_receive_frame(anim->codec_ctx, anim->frame) == 0;
1061 if (anim->frame_complete) {
1062 ffmpeg_decode_store_frame_pts(anim);
1063 return true;
1064 }
1065 ffmpeg_cur_packet_clear(anim);
1066 }
1067
1068 if (ret == AVERROR_EOF) {
1069 /* Flush any remaining frames out of the decoder. */
1070 avcodec_send_packet(anim->codec_ctx, nullptr);
1071 anim->frame_complete = avcodec_receive_frame(anim->codec_ctx, anim->frame) == 0;
1072 if (anim->frame_complete) {
1073 ffmpeg_decode_store_frame_pts(anim);
1074 return true;
1075 }
1076 }
1077
1078 ffmpeg_cur_packet_clear(anim);
1079
1080 char error_str[AV_ERROR_MAX_STRING_SIZE];
1081 av_make_error_string(error_str, AV_ERROR_MAX_STRING_SIZE, ret);
1082 av_log(anim->format_ctx,
1083 AV_LOG_ERROR,
1084 " DECODE READ FAILED: av_read_frame() "
1085 "returned error: %s\n",
1086 error_str);
1087 return false;
1088}
1089
1090static int64_t ffmpeg_seek_pts_get(const MovieReader *anim, int64_t target_pts)
1091{
1092 /* FFMPEG seeks internally using DTS values instead of PTS. In some files DTS and PTS values are
1093 * offset and sometimes FFMPEG fails to take this into account when seeking.
1094 * Therefore we need to seek backwards a certain offset to make sure the frame we want is in
1095 * front of us. It is not possible to determine the exact needed offset,
1096 * this value is determined experimentally.
1097 * NOTE: Too big offset can impact performance. Current 3 frame offset has no measurable impact.
1098 */
1099 int64_t seek_pts = target_pts - (ffmpeg_pts_per_frame_get(anim) * 3);
1100
1101 seek_pts = std::max<int64_t>(seek_pts, 0);
1102 return seek_pts;
1103}
1104
1105/* This gives us an estimate of which pts our requested frame will have.
1106 * Note that this might be off a bit in certain video files, but it should still be close enough.
1107 */
1108static int64_t ffmpeg_frame_index_to_pts(MovieReader *anim, int frame_index)
1109{
1110 AVStream *v_st = anim->format_ctx->streams[anim->video_stream_index];
1111 int64_t start_pts = v_st->start_time;
1112
1113 int64_t target_pts = round(frame_index * ffmpeg_pts_per_frame_get(anim));
1114
1115 if (start_pts != AV_NOPTS_VALUE) {
1116 target_pts += start_pts;
1117 }
1118 return target_pts;
1119}
1120
1121static void ffmpeg_scan_log(MovieReader *anim, int64_t target_pts)
1122{
1123 int64_t frame_pts_start = av_get_pts_from_frame(anim->frame);
1124 int64_t frame_pts_end = frame_pts_start + av_get_frame_duration_in_pts_units(anim->frame);
1125 av_log(anim->format_ctx,
1126 AV_LOG_DEBUG,
1127 " SCAN WHILE: PTS range %" PRId64 " - %" PRId64 " in search of %" PRId64 "\n",
1128 frame_pts_start,
1129 frame_pts_end,
1130 target_pts);
1131}
1132
1133/* Decode frames one by one until its PTS matches target_pts. */
1134static void ffmpeg_decode_until_pts(MovieReader *anim, int64_t target_pts, const bool did_seek)
1135{
1136 const int64_t start_key_frame_pts = anim->cur_key_frame_pts;
1137 bool decode_error = false;
1138
1139 while (!decode_error && anim->cur_pts < target_pts) {
1140 ffmpeg_scan_log(anim, target_pts);
1141 ffmpeg_double_buffer_backup_frame_store(anim, target_pts);
1142 decode_error = !ffmpeg_decode_video_frame(anim);
1143
1144 /* We should not get a new GOP keyframe while scanning if seeking is working as intended.
1145 * If this condition triggers, there may be and error in our seeking code.
1146 * NOTE: This seems to happen if DTS value is used for seeking in ffmpeg internally. There
1147 * seems to be no good way to handle such case. */
1148 if (did_seek && start_key_frame_pts != anim->cur_key_frame_pts) {
1149 av_log(anim->format_ctx, AV_LOG_ERROR, "SCAN: Frame belongs to an unexpected GOP!\n");
1150 }
1151 }
1152}
1153
1154/* Wrapper over av_seek_frame(), for formats that doesn't have its own read_seek() or
1155 * read_seek2() functions defined. When seeking in these formats, rule to seek to last
1156 * necessary I-frame is not honored. It is not even guaranteed that I-frame, that must be
1157 * decoded will be read. See https://trac.ffmpeg.org/ticket/1607 & #86944. */
1158static int ffmpeg_generic_seek_workaround(MovieReader *anim, int64_t *seek_pts, int64_t target_pts)
1159{
1160 AVStream *v_st = anim->format_ctx->streams[anim->video_stream_index];
1161 int64_t start_pts = v_st->start_time;
1162 int64_t current_pts = *seek_pts;
1163 int64_t offset = 0;
1164 AVPacket *packet = av_packet_alloc();
1165
1166 /* Step backward frame by frame until we find the key frame we are looking for. */
1167 while (current_pts != 0) {
1168 current_pts = *seek_pts - int64_t(round(offset * ffmpeg_pts_per_frame_get(anim)));
1169 current_pts = std::max(current_pts, int64_t(0));
1170
1171 /* Seek to timestamp. */
1172 if (av_seek_frame(
1173 anim->format_ctx, anim->video_stream_index, current_pts, AVSEEK_FLAG_BACKWARD) < 0)
1174 {
1175 break;
1176 }
1177
1178 /* Read first video stream packet. */
1179 ffmpeg_read_video_frame(anim, packet);
1180
1181 /* If this packet contains an I-frame, this could be the frame that we need. */
1182 const bool is_key_frame = packet->flags & AV_PKT_FLAG_KEY;
1183 /* We need to check the packet timestamp as the key frame could be for a GOP forward in the
1184 * video stream. So if it has a larger timestamp than the frame we want, ignore it.
1185 */
1186 const int64_t cur_pts = timestamp_from_pts_or_dts(packet->pts, packet->dts);
1187 av_packet_unref(packet);
1188
1189 if (is_key_frame) {
1190 if (cur_pts <= target_pts) {
1191 /* We found the I-frame we were looking for! */
1192 break;
1193 }
1194 }
1195
1196 /* We have hit the beginning of the stream. */
1197 if (cur_pts <= start_pts) {
1198 break;
1199 }
1200
1201 offset++;
1202 }
1203 av_packet_free(&packet);
1204
1205 *seek_pts = current_pts;
1206
1207 /* Re-seek to timestamp that gave I-frame, so it can be read by decode function. */
1208 return av_seek_frame(
1209 anim->format_ctx, anim->video_stream_index, current_pts, AVSEEK_FLAG_BACKWARD);
1210}
1211
1212/* Read packet until timestamp matches `anim->cur_packet`, thus recovering internal `anim` stream
1213 * position state. */
1214static void ffmpeg_seek_recover_stream_position(MovieReader *anim)
1215{
1216 AVPacket *temp_packet = av_packet_alloc();
1217 while (ffmpeg_read_video_frame(anim, temp_packet) >= 0) {
1218 int64_t current_pts = timestamp_from_pts_or_dts(anim->cur_packet->pts, anim->cur_packet->dts);
1219 int64_t temp_pts = timestamp_from_pts_or_dts(temp_packet->pts, temp_packet->dts);
1220 av_packet_unref(temp_packet);
1221
1222 if (current_pts == temp_pts) {
1223 break;
1224 }
1225 }
1226 av_packet_free(&temp_packet);
1227}
1228
1229/* Check if seeking and mainly flushing codec buffers is needed. */
1230static bool ffmpeg_seek_needs_flush(MovieReader *anim, int frame_index, int64_t seek_pts)
1231{
1232 /* Get timestamp of packet read after seeking. */
1233 AVPacket *packet = av_packet_alloc();
1234 ffmpeg_read_video_frame(anim, packet);
1235 const int64_t seek_key_frame_pts = timestamp_from_pts_or_dts(packet->pts, packet->dts);
1236 av_packet_free(&packet);
1237
1238 /* Seeking gives packet, that is currently read. No seeking was necessary, so buffers don't have
1239 * to be flushed. */
1240 if (seek_key_frame_pts ==
1241 timestamp_from_pts_or_dts(anim->cur_packet->pts, anim->cur_packet->dts))
1242 {
1243 return false;
1244 }
1245
1246 /* Packet after seeking is same key frame as current, and further in time. No seeking was
1247 * necessary, so buffers don't have to be flushed. But stream position has to be recovered. */
1248 if (seek_key_frame_pts == anim->cur_key_frame_pts && frame_index > anim->cur_frame_index) {
1249 ffmpeg_seek_recover_stream_position(anim);
1250 return false;
1251 }
1252
1253 /* Seeking was necessary, but we have read packets. Therefore we must seek again. */
1254 av_seek_frame(anim->format_ctx, anim->video_stream_index, seek_pts, AVSEEK_FLAG_BACKWARD);
1255 anim->cur_key_frame_pts = seek_key_frame_pts;
1256 return true;
1257}
1258
1259/* Seek to last necessary key frame. */
1260static void ffmpeg_seek_to_key_frame(MovieReader *anim, int frame_index, int64_t target_pts)
1261{
1262 int64_t seek_pts = ffmpeg_seek_pts_get(anim, target_pts);
1263 av_log(anim->format_ctx, AV_LOG_DEBUG, "Final seek seek_pts = %" PRId64 "\n", seek_pts);
1264
1265 int ret;
1266 /* This used to check if the codec implemented "read_seek" or "read_seek2". However this is
1267 * now hidden from us in FFMPEG 7.0. While not as accurate, usually the AVFMT_TS_DISCONT is
1268 * set for formats where we need to apply the seek workaround to (like in MPEGTS). */
1269 if (!(anim->format_ctx->iformat->flags & AVFMT_TS_DISCONT)) {
1270 ret = av_seek_frame(
1271 anim->format_ctx, anim->video_stream_index, seek_pts, AVSEEK_FLAG_BACKWARD);
1272 }
1273 else {
1274 ret = ffmpeg_generic_seek_workaround(anim, &seek_pts, target_pts);
1275 av_log(
1276 anim->format_ctx, AV_LOG_DEBUG, "Adjusted final seek seek_pts = %" PRId64 "\n", seek_pts);
1277 }
1278
1279 if (ret <= 0 && !ffmpeg_seek_needs_flush(anim, frame_index, seek_pts)) {
1280 return;
1281 }
1282
1283 if (ret < 0) {
1284 av_log(anim->format_ctx,
1285 AV_LOG_ERROR,
1286 "FETCH: "
1287 "error while seeking to DTS = %" PRId64 " (frameno = %d, PTS = %" PRId64
1288 "): errcode = %d\n",
1289 seek_pts,
1290 frame_index,
1291 target_pts,
1292 ret);
1293 }
1294 /* Flush the internal buffers of ffmpeg. This needs to be done after seeking to avoid decoding
1295 * errors. */
1296 avcodec_flush_buffers(anim->codec_ctx);
1297 ffmpeg_double_buffer_backup_frame_clear(anim);
1298 anim->cur_pts = -1;
1299 ffmpeg_cur_packet_clear(anim);
1300}
1301
1302static bool ffmpeg_must_seek(const MovieReader *anim, int frame_index, int64_t target_pts)
1303{
1304 if (!anim->frame_complete || frame_index < anim->cur_frame_index) {
1305 return true;
1306 }
1307 if (frame_index == anim->cur_frame_index + 1) {
1308 return false;
1309 }
1310
1311 AVStream *stream = anim->format_ctx->streams[anim->video_stream_index];
1312 const int64_t seek_pts = ffmpeg_seek_pts_get(anim, target_pts);
1313
1314 /* Get the index (not timestamp) of the closest keyframe before the #seek_pts to seek to. */
1315 const int index = av_index_search_timestamp(stream, seek_pts, AVSEEK_FLAG_BACKWARD);
1316 if (index < 0 || index + 1 >= avformat_index_get_entries_count(stream)) {
1317 return true;
1318 }
1319 const AVIndexEntry *key_frame = avformat_index_get_entry(stream, index);
1320 if (key_frame == nullptr) {
1321 return true;
1322 }
1323 /* If the keyframe we would seek to (closest before #seek_pts) is at or before #cur_pts,
1324 * #seek_pts is in the current GOP and decoding forward is faster.
1325 *
1326 * Otherwise it is in a new GOP, and if we seek, we can skip decoding up to that keyframe, but we
1327 * will have to flush the frames the decoder threads are working on (roughly N frames for N
1328 * threads) to avoid receiving stale reference frames. If the key frame is within those N frames,
1329 * it is already being decoded, so only seek if the key frame is more than N frames ahead.
1330 */
1331 const double flush_cost_pts = ffmpeg_pts_per_frame_get(anim) * MOV_thread_count();
1332 return key_frame->timestamp > anim->cur_pts + flush_cost_pts;
1333}
1334
1335static ImBuf *ffmpeg_fetchibuf(MovieReader *anim, int frame_index)
1336{
1337 av_log(anim->format_ctx, AV_LOG_DEBUG, "FETCH: frame_index=%d\n", frame_index);
1338
1339 const int64_t target_pts = ffmpeg_frame_index_to_pts(anim, frame_index);
1340
1341 if (anim->never_seek_decode_one_frame) {
1342 /* If we must only ever decode one frame, and never seek, do so here. */
1343 if (!anim->frame_complete) {
1344 ffmpeg_decode_video_frame(anim);
1345 }
1346 }
1347 else {
1348 /* For all regular video files, do the seek/decode as needed. */
1349 const AVStream *v_st = anim->format_ctx->streams[anim->video_stream_index];
1350 av_log(anim->format_ctx,
1351 AV_LOG_DEBUG,
1352 "FETCH: looking for PTS=%" PRId64 " (pts_timebase=%g, frame_rate=%g, start_pts=%" PRId64
1353 ")\n",
1354 target_pts,
1355 av_q2d(v_st->time_base),
1356 av_q2d(v_st->r_frame_rate),
1357 v_st->start_time);
1358
1359 if (!anim->frame_complete || anim->cur_frame_index != frame_index) {
1360 const bool must_seek = ffmpeg_must_seek(anim, frame_index, target_pts);
1361 if (must_seek) {
1362 ffmpeg_seek_to_key_frame(anim, frame_index, target_pts);
1363 }
1364 ffmpeg_decode_until_pts(anim, target_pts, must_seek);
1365 }
1366 }
1367
1368 /* Update resolution as it can change per-frame with WebM. See #100741 & #100081. */
1369 anim->x = anim->codec_ctx->width;
1370 anim->y = anim->codec_ctx->height;
1371
1372 const AVPixFmtDescriptor *pix_fmt_descriptor = av_pix_fmt_desc_get(
1373 ffmpeg_codec_pix_fmt_get(anim->codec_ctx));
1374
1375 ImColorMode color_mode = ImColorMode::RGBA;
1376 if ((pix_fmt_descriptor->flags & AV_PIX_FMT_FLAG_ALPHA) == 0) {
1377 color_mode = ImColorMode::RGB;
1378 }
1379
1380 ImBuf *cur_frame_final = IMB_allocImBuf(anim->x, anim->y, ImBufFlags::Zero);
1381 cur_frame_final->color_mode = color_mode;
1382
1383 /* Allocate the storage explicitly to ensure the memory is aligned. */
1384 const size_t align = ffmpeg_get_buffer_alignment();
1385 const size_t pixel_size = anim->is_float ? 16 : 4;
1386 uint8_t *buffer_data = static_cast<uint8_t *>(
1387 MEM_new_uninitialized_aligned(pixel_size * anim->x * anim->y, align, "ffmpeg ibuf"));
1388 if (anim->is_float) {
1389 cur_frame_final->assign_float_data((float *)buffer_data);
1390 }
1391 else {
1392 cur_frame_final->assign_byte_data(buffer_data);
1393 }
1394
1395 AVFrame *final_frame = ffmpeg_frame_by_pts_get(anim, target_pts);
1396 if (final_frame == nullptr) {
1397 /* No valid frame was decoded for requested PTS, fall back on most recent decoded frame, even
1398 * if it is incorrect. */
1399 final_frame = ffmpeg_double_buffer_frame_fallback_get(anim);
1400 }
1401
1402 /* If the final frame is a GPU surface, transfer only this frame to system RAM */
1403 if (final_frame != nullptr && final_frame->hw_frames_ctx != nullptr) {
1404 av_frame_unref(anim->frame_sw);
1405 const int ret = av_hwframe_transfer_data(anim->frame_sw, final_frame, 0);
1406 if (ret < 0) {
1407 char error_str[AV_ERROR_MAX_STRING_SIZE];
1408 av_make_error_string(error_str, AV_ERROR_MAX_STRING_SIZE, ret);
1409 CLOG_ERROR(&LOG, "ffmpeg: couldn't transfer frame from the decoding device: %s", error_str);
1410 final_frame = nullptr;
1411 }
1412 else {
1413 av_frame_copy_props(anim->frame_sw, final_frame);
1414 final_frame = anim->frame_sw;
1415 }
1416 }
1417
1418 /* Even with the fallback from above it is possible that the current decode frame is nullptr. In
1419 * this case skip post-processing and return current image buffer. */
1420 if (final_frame != nullptr) {
1421 ffmpeg_postprocess(anim, final_frame, cur_frame_final);
1422 }
1423 else {
1424 memset(buffer_data, 0, pixel_size * anim->x * anim->y);
1425 }
1426
1427 if (anim->is_float) {
1428 if (anim->keep_original_colorspace) {
1429 /* Movie has been explicitly requested to keep original colorspace, regardless of the nature
1430 * of the buffer. */
1431 cur_frame_final->float_buffer.colorspace = colormanage_colorspace_get_named(
1432 anim->colorspace);
1433 }
1434 else {
1435 /* Float buffers are expected to be in the scene linear color space.
1436 * Linearize the buffer if it is in a different space.
1437 *
1438 * It might not be the most optimal thing to do from the playback performance in the
1439 * sequencer perspective, but it ensures that other areas in Blender do not run into obscure
1440 * color space mismatches. */
1442 cur_frame_final, anim->colorspace, ColorManagedFileOutput::Video);
1443 }
1444 }
1445 else {
1446 /* Color-space conversion is lossy for byte buffers, so only assign the color-space.
1447 * It is up to artists to ensure operations on byte buffers do not involve mixing different
1448 * color-spaces. */
1449 cur_frame_final->byte_buffer.colorspace = colormanage_colorspace_get_named(anim->colorspace);
1450 }
1451
1452 anim->cur_frame_index = frame_index;
1453
1454 return cur_frame_final;
1455}
1456
1457static void free_anim_ffmpeg(MovieReader *anim)
1458{
1459 if (anim == nullptr) {
1460 return;
1461 }
1462
1463 if (anim->codec_ctx) {
1464 ffmpeg_free_decoder(anim);
1465 }
1466 anim->duration_in_frames = 0;
1467}
1468
1469#endif
1470
1475static bool anim_getnew(MovieReader *anim)
1476{
1477 if (anim == nullptr) {
1478 /* Nothing to initialize. */
1479 return false;
1480 }
1481
1483
1484#ifdef WITH_FFMPEG
1485 free_anim_ffmpeg(anim);
1486 if (startffmpeg(anim)) {
1488 return false;
1489 }
1490#endif
1492 return true;
1493}
1494
1496{
1497 ImBuf *ibuf = nullptr;
1498 int frame_index = 0;
1499
1500 ibuf = MOV_decode_frame(anim, 0, IMB_PROXY_NONE);
1501 if (ibuf) {
1502 IMB_freeImBuf(ibuf);
1503 frame_index = anim->duration_in_frames / 2;
1504 ibuf = MOV_decode_frame(anim, frame_index, IMB_PROXY_NONE);
1505
1506 char value[128];
1508 SNPRINTF_UTF8(value, "%i", anim->x);
1509 IMB_metadata_set_field(metadata, "Thumb::Video::Width", value);
1510 SNPRINTF_UTF8(value, "%i", anim->y);
1511 IMB_metadata_set_field(metadata, "Thumb::Video::Height", value);
1512 SNPRINTF_UTF8(value, "%i", anim->duration_in_frames);
1513 IMB_metadata_set_field(metadata, "Thumb::Video::Frames", value);
1514
1515#ifdef WITH_FFMPEG
1516 if (anim->format_ctx) {
1517 AVStream *v_st = anim->format_ctx->streams[anim->video_stream_index];
1518 AVRational frame_rate = av_guess_frame_rate(anim->format_ctx, v_st, nullptr);
1519 if (frame_rate.num != 0) {
1520 double duration = anim->duration_in_frames / av_q2d(frame_rate);
1521 SNPRINTF_UTF8(value, "%g", av_q2d(frame_rate));
1522 IMB_metadata_set_field(metadata, "Thumb::Video::FPS", value);
1523 SNPRINTF_UTF8(value, "%g", duration);
1524 IMB_metadata_set_field(metadata, "Thumb::Video::Duration", value);
1525 IMB_metadata_set_field(metadata, "Thumb::Video::Codec", anim->codec->long_name);
1526 }
1527 }
1528#endif
1529 }
1530 return ibuf;
1531}
1532
1533ImBuf *MOV_decode_frame(MovieReader *anim, int frame_index, IMB_Proxy_Size preview_size)
1534{
1535 ImBuf *ibuf = nullptr;
1536 if (anim == nullptr) {
1537 return nullptr;
1538 }
1539
1540 if (preview_size == IMB_PROXY_NONE) {
1542 if (!anim_getnew(anim)) {
1543 return nullptr;
1544 }
1545 }
1546
1547 if (frame_index < 0) {
1548 return nullptr;
1549 }
1550 if (frame_index >= anim->duration_in_frames) {
1551 return nullptr;
1552 }
1553 }
1554 else {
1555 MovieReader *proxy = movie_open_proxy(anim, preview_size);
1556 if (proxy) {
1557 return MOV_decode_frame(proxy, frame_index, IMB_PROXY_NONE);
1558 }
1559 }
1560
1561#ifdef WITH_FFMPEG
1562 if (anim->state == MovieReader::State::Valid) {
1563 ibuf = ffmpeg_fetchibuf(anim, frame_index);
1564 }
1565#endif
1566
1567 if (ibuf) {
1568 ibuf->filepath = anim->filepath;
1569 ibuf->fileframe = anim->cur_frame_index + 1;
1570 }
1571 return ibuf;
1572}
1573
1575{
1576#ifdef WITH_FFMPEG
1577 if (anim == nullptr) {
1578 return 0;
1579 }
1580 if (anim->state == MovieReader::State::Uninitialized && !anim_getnew(anim)) {
1581 return 0;
1582 }
1583 if (anim->format_ctx == nullptr) {
1584 return 0;
1585 }
1586 int count = 0;
1587 for (int i = 0; i < anim->format_ctx->nb_streams; i++) {
1588 if (ffmpeg_stream_counts_as_video(anim->format_ctx, anim->format_ctx->streams[i])) {
1589 count++;
1590 }
1591 }
1592 return count;
1593#else
1594 UNUSED_VARS(anim);
1595 return 0;
1596#endif
1597}
1598
1600{
1601 return anim->duration_in_frames;
1602}
1603
1605{
1606 return anim->start_offset;
1607}
1608
1609float MOV_get_fps(const MovieReader *anim)
1610{
1611 if (anim->frs_sec > 0 && anim->frs_sec_base > 0) {
1612 return float(double(anim->frs_sec) / anim->frs_sec_base);
1613 }
1614 return 0.0f;
1615}
1616
1617bool MOV_get_fps_num_denom(const MovieReader *anim, short &r_fps_num, float &r_fps_denom)
1618{
1619 if (anim->frs_sec > 0 && anim->frs_sec_base > 0) {
1620 if (anim->frs_sec > SHRT_MAX) {
1621 /* If numerator is larger than the max short, we need to approximate. */
1622 r_fps_num = SHRT_MAX;
1623 r_fps_denom = float(anim->frs_sec_base * double(SHRT_MAX) / double(anim->frs_sec));
1624 }
1625 else {
1626 r_fps_num = anim->frs_sec;
1627 r_fps_denom = float(anim->frs_sec_base);
1628 }
1629 return true;
1630 }
1631 return false;
1632}
1633
1635{
1636 return ELEM(anim->video_rotation, 90, 270) ? anim->y : anim->x;
1637}
1638
1640{
1641 return ELEM(anim->video_rotation, 90, 270) ? anim->x : anim->y;
1642}
1643
1644} // namespace blender
#define SNPRINTF_UTF8(dst, format,...)
#define STRNCPY_UTF8(dst, src)
#define CLOG_ERROR(clg_ref,...)
Definition CLG_log.h:188
#define CLOG_INFO(clg_ref,...)
Definition CLG_log.h:190
#define USER_DEVELOPER_TOOL_TEST(userdef, member)
#define IM_MAX_SPACE
Read Blender memory allocation and freeing.
#define UNUSED_VARS(...)
#define ELEM(...)
#define BLI_assert_msg(a, msg)
#define STREQ(a, b)
long long int int64_t
constexpr IndexRange()=default
constexpr Span()=default
#define str(s)
FFMPEG_INLINE int64_t av_get_frame_duration_in_pts_units(const AVFrame *picture)
FFMPEG_INLINE size_t ffmpeg_get_buffer_alignment()
FFMPEG_INLINE int64_t timestamp_from_pts_or_dts(int64_t pts, int64_t dts)
FFMPEG_INLINE int64_t av_get_pts_from_frame(AVFrame *picture)
FFMPEG_INLINE int ffmpeg_get_video_rotation(const AVStream *stream)
#define SHRT_MAX
#define index(i)
#define round
#define PRId64
Definition inttypes.h:78
#define LOG(level)
Definition log.h:97
void * MEM_new_uninitialized_aligned(size_t len, size_t alignment, const char *str)
Definition mallocn.cc:134
std::string filename(const std::string_view &str, size_t pos)
void parallel_for(const IndexRange range, const int64_t grain_size, const Function &function, const TaskSizeHints &size_hints=detail::TaskSizeHints_Static(1))
Definition BLI_task.hh:69
BMesh Iterators.
bool IMB_rotate_orthogonal(ImBuf *ibuf, int degrees)
UserDef U
Definition blender.cc:58
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
bool MOV_is_initialized_and_valid(const MovieReader *anim)
static bool anim_getnew(MovieReader *anim)
__int64 int64_t
Definition BKE_text.h:90
char * STRNCPY(char(&dst)[N], const char *src)
const ColorSpace * IMB_colormanagement_space_from_cicp(const int cicp[4], ColorManagedFileOutput output)
int MOV_get_image_height(const MovieReader *anim)
int MOV_get_image_width(const MovieReader *anim)
void MOV_close(MovieReader *anim)
Definition movie_read.cc:70
bool MOV_get_fps_num_denom(const MovieReader *anim, short &r_fps_num, float &r_fps_denom)
char * BLI_strncpy_utf8(char *__restrict dst, const char *__restrict src, size_t dst_maxncpy) ATTR_NONNULL(1
ImBuf * MOV_decode_frame(MovieReader *anim, int frame_index, IMB_Proxy_Size preview_size)
void colormanage_imbuf_make_linear(ImBuf *ibuf, const char *from_colorspace, const ColorManagedFileOutput output)
MINLINE int max_ii(int a, int b)
void MOV_set_multiview_suffix(MovieReader *anim, const char *suffix)
void IMB_metadata_ensure(IDProperty **metadata)
MovieReader * MOV_open_file(const char *filepath, const ImBufFlags ib_flags, const int streamindex, const bool keep_original_colorspace, ColorManagedColorspaceSettings *colorspace_settings)
void IMB_freeImBuf(ImBuf *ibuf)
const char * IMB_colormanagement_role_colorspace_name_get(int role)
void IMB_colormanagement_colorspace_settings_set(ColorManagedColorspaceSettings *settings, const char *name)
unsigned char uint8_t
Definition BKE_text.h:79
@ USER_GPU_FLAG_VIDEO_DECODING
const char * IMB_colormanagement_colorspace_get_name(const ColorSpace *colorspace)
MovieReader * movie_open_proxy(MovieReader *anim, IMB_Proxy_Size preview_size)
constexpr size_t ARRAY_SIZE(const T(&arg)[N]) noexcept
double MOV_get_start_offset_seconds(const MovieReader *anim)
ImBuf * IMB_allocImBuf(unsigned int x, unsigned int y, ImBufFlags flags)
void void void BLI_path_split_file_part(const char *filepath, char *file, size_t file_maxncpy) ATTR_NONNULL(1
IDProperty * MOV_load_metadata(MovieReader *anim)
Definition movie_read.cc:90
const ColorSpace * colormanage_colorspace_get_named(const char *name)
int MOV_get_video_stream_count(MovieReader *anim)
bool BLI_path_is_rel(const char *path) ATTR_NONNULL(1) ATTR_WARN_UNUSED_RESULT
float MOV_get_fps(const MovieReader *anim)
@ USER_VIDEO_DECODING_DEVICE_AUTOMATIC
@ USER_VIDEO_DECODING_DEVICE_CUDA
@ USER_VIDEO_DECODING_DEVICE_VULKAN
@ USER_VIDEO_DECODING_DEVICE_D3D11VA
@ USER_VIDEO_DECODING_DEVICE_VIDEOTOOLBOX
ImBuf * MOV_decode_preview_frame(MovieReader *anim)
void MOV_close_proxies(MovieReader *anim)
int MOV_get_duration_frames(const MovieReader *anim)
void IMB_filtery(ImBuf *ibuf)
Definition filter.cc:65
static void probe_video_colorspace(MovieReader *anim, char r_colorspace_name[IM_MAX_SPACE])
void IMB_metadata_set_field(IDProperty *metadata, const char *key, const char *value)
void MOV_get_filename(const MovieReader *anim, char *filename, int filename_maxncpy)
Definition movie_read.cc:85
ocio::ColorSpace ColorSpace
Definition BLF_api.hh:42
void IMB_metadata_free(IDProperty *metadata)
void * MEM_new_array_zeroed(size_t len, size_t size, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float
IDProperty * metadata_for_write()
std::string filepath
IDProperty * metadata
Definition movie_read.hh:99