Blender V5.3
sequencer/intern/sound.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2001-2002 NaN Holding BV. All rights reserved.
2 * SPDX-FileCopyrightText: 2003-2009 Blender Authors
3 * SPDX-FileCopyrightText: 2005-2006 Peter Schlaile <peter [at] schlaile [dot] de>
4 *
5 * SPDX-License-Identifier: GPL-2.0-or-later */
6
10
11#include <algorithm>
12#include <cmath>
13#include <cstring>
14#include <iostream>
15#include <xxhash.h>
16
17#include "MEM_guardedalloc.h"
18
19#include "DNA_curve_types.h"
20#include "DNA_scene_types.h"
21#include "DNA_sequence_types.h"
22#include "DNA_sound_types.h"
23
24#include "BLI_listbase.hh"
25#include "BLI_utildefines.hh"
26
27#include "BKE_colortools.hh"
28#include "BKE_sound.hh"
29
30#include "SEQ_modifier.hh"
31#include "SEQ_sequencer.hh"
32#include "SEQ_sound.hh"
33
34#include "strip_time.hh"
35
36#ifdef WITH_AUDASPACE
37# include <fx/Echo.h>
38# ifdef WITH_CONVOLUTION
39# include <fx/Equalizer.h>
40# endif
41# include <fx/TimeStretchPitchScale.h>
42# include <util/Buffer.h>
43#endif
44
45namespace blender::seq {
46
47/* Unlike _update_sound_ functions,
48 * these ones take info from audaspace to update sequence length! */
54
55#ifdef WITH_CONVOLUTION
56static bool sequencer_refresh_sound_length_recursive(Main *bmain,
57 Scene *scene,
58 ListBaseT<Strip> *seqbase)
59{
60 bool changed = false;
61
62 for (Strip &strip : *seqbase) {
63 if (strip.type == STRIP_TYPE_META) {
64 if (sequencer_refresh_sound_length_recursive(bmain, scene, &strip.seqbase)) {
65 changed = true;
66 }
67 }
68 else if (strip.type == STRIP_TYPE_SOUND && strip.sound) {
69 SoundInfo info;
70 if (!BKE_sound_info_get(bmain, strip.sound, &info)) {
71 continue;
72 }
73
74 int old = strip.content_length();
75 float fac;
76
77 strip.content_length_set(std::max(
78 1, int(round((info.length - strip.sound->offset_time) * scene->frames_per_second()))));
79 fac = float(strip.content_length()) / float(old);
80 old = strip.startofs;
81 strip.startofs *= fac;
82 strip.endofs = strip.end_offset() * fac;
83 strip.start += (old -
84 strip.startofs); /* So that visual/"real" start frame does not change! */
85
86 changed = true;
87 }
88 }
89 return changed;
90}
91#endif
92
93void sound_update_length(Main *bmain, Scene *scene)
94{
95#ifdef WITH_CONVOLUTION
96 if (scene->ed) {
97 sequencer_refresh_sound_length_recursive(bmain, scene, &scene->ed->seqbase);
98 }
99#else
100 UNUSED_VARS(bmain, scene);
101#endif
102}
103
105{
106 Editing *ed = scene->ed;
107
108 if (ed) {
109 for (Strip &strip : ed->seqbase) {
110 if (strip.type == STRIP_TYPE_META) {
112 }
113 else if (ELEM(strip.type, STRIP_TYPE_SOUND, STRIP_TYPE_SCENE)) {
114 sound_update_bounds(scene, &strip);
115 }
116 }
117 }
118}
119
120void sound_update_bounds(Scene *scene, Strip *strip)
121{
122 if (strip->type == STRIP_TYPE_SCENE) {
123 if (strip->scene && strip->runtime->scene_sound) {
124 /* We have to take into account start frame of the sequence's scene! */
125 int startofs = strip->startofs + strip->anim_startofs + strip->scene->r.sfra;
126
128 strip->runtime->scene_sound,
129 strip->left_handle(),
130 strip->right_handle(scene),
131 startofs,
132 0.0);
133 }
134 }
135 else {
137 }
138 /* mute is set in strip_update_muting_recursive */
139}
140
141static void strip_update_sound_recursive(Scene *scene, ListBaseT<Strip> *seqbasep, bSound *sound)
142{
143 for (Strip &strip : *seqbasep) {
144 if (strip.type == STRIP_TYPE_META) {
145 strip_update_sound_recursive(scene, &strip.seqbase, sound);
146 }
147 else if (strip.type == STRIP_TYPE_SOUND) {
148 if (strip.runtime->scene_sound && sound == strip.sound) {
149 BKE_sound_update_scene_sound(strip.runtime->scene_sound, sound);
150 }
151 }
152 }
153}
154
155void sound_update(Scene *scene, bSound *sound)
156{
157 if (scene->ed) {
158 strip_update_sound_recursive(scene, &scene->ed->seqbase, sound);
159 }
160}
161
162float sound_pitch_get(const Scene *scene, const Strip *strip)
163{
164 const Strip *meta_parent = lookup_meta_by_strip(scene->ed, strip);
165 if (meta_parent != nullptr) {
166 return strip->speed_factor * sound_pitch_get(scene, meta_parent);
167 }
168 return strip->speed_factor;
169}
170
172{
173 EQCurveMappingData *eqcmd;
174
175 if (maxX < 0) {
177 }
178 if (minX < 0) {
179 minX = 0.0;
180 }
181 /* It's the same as #BKE_curvemapping_add, but changing the name. */
182 eqcmd = MEM_new<EQCurveMappingData>("Equalizer");
184 1, /* Total. */
185 minX,
186 -SOUND_EQUALIZER_DEFAULT_MAX_DB, /* Min x, y */
187 maxX,
188 SOUND_EQUALIZER_DEFAULT_MAX_DB, /* Max x, y */
190
192
193 rctf clipr;
194 clipr.xmin = minX;
195 clipr.xmax = maxX;
196 clipr.ymin = 0.0;
197 clipr.ymax = 0.0;
198
200 &clipr,
203
204 BLI_addtail(&semd->graphics, eqcmd);
205
206 return eqcmd;
207}
208
210{
211 sound_equalizermodifier_free(reinterpret_cast<StripModifierData *>(semd));
212 if (number == 1) {
214 }
215 else if (number == 2) {
216 sound_equalizer_add(semd, 30.0, 2000.0);
217 sound_equalizer_add(semd, 2000.1, 20000.0);
218 }
219 else if (number == 3) {
220 sound_equalizer_add(semd, 30.0, 1000.0);
221 sound_equalizer_add(semd, 1000.1, 5000.0);
222 sound_equalizer_add(semd, 5000.1, 20000.0);
223 }
224}
225
227 float min_freq,
228 float max_freq)
229{
230 if (min_freq < 0.0) {
231 return nullptr;
232 }
233 if (max_freq < 0.0) {
234 return nullptr;
235 }
236 if (max_freq <= min_freq) {
237 return nullptr;
238 }
239 return sound_equalizer_add(semd, min_freq, max_freq);
240}
241
243 EQCurveMappingData *eqcmd)
244{
245 BLI_remlink_safe(&semd->graphics, eqcmd);
246 MEM_delete(eqcmd);
247}
248
255
257{
258 SoundEqualizerModifierData *semd = reinterpret_cast<SoundEqualizerModifierData *>(smd);
259 for (EQCurveMappingData &eqcmd : semd->graphics.items_mutable()) {
260 BKE_curvemapping_free_data(&eqcmd.curve_mapping);
261 MEM_delete(&eqcmd);
262 }
264}
265
267{
268 SoundEqualizerModifierData *semd = reinterpret_cast<SoundEqualizerModifierData *>(smd);
269 SoundEqualizerModifierData *semd_target = reinterpret_cast<SoundEqualizerModifierData *>(target);
270 EQCurveMappingData *eqcmd_n;
271
272 BLI_listbase_clear(&semd_target->graphics);
273
274 for (EQCurveMappingData &eqcmd : semd->graphics) {
275 eqcmd_n = MEM_dupalloc(&eqcmd);
277
278 eqcmd_n->next = eqcmd_n->prev = nullptr;
279 BLI_addtail(&semd_target->graphics, eqcmd_n);
280 }
281}
282
283#ifdef WITH_CONVOLUTION
284static uint64_t sound_equalizermodifier_get_params_hash(float *buf)
285{
286 return XXH3_64bits(buf, sizeof(float) * SOUND_EQUALIZER_SIZE_DEFINITION);
287}
288#endif
289
292 AUD_Sound sound_in,
293 bool &needs_update)
294{
295#ifdef WITH_CONVOLUTION
296 UNUSED_VARS(strip);
297
299
300 /* No equalizer definition. */
301 if (semd->graphics.is_empty()) {
302 return sound_in;
303 }
304
306
307 CurveMapping *eq_mapping;
308 CurveMap *cm;
309 float minX;
310 float maxX;
312
313 /* Visit all equalizer definitions. */
314 for (EQCurveMappingData &mapping : semd->graphics) {
315 eq_mapping = &mapping.curve_mapping;
316 BKE_curvemapping_init(eq_mapping);
317 cm = eq_mapping->cm;
318 minX = eq_mapping->curr.xmin;
319 maxX = eq_mapping->curr.xmax;
320 int idx = int(ceil(minX / interval));
321 int i = idx;
322 for (; i * interval <= maxX && i < SOUND_EQUALIZER_SIZE_DEFINITION; i++) {
323 float freq = i * interval;
324 float val = BKE_curvemap_evaluateF(eq_mapping, cm, freq);
326 val = (val / fabs(val)) * SOUND_EQUALIZER_DEFAULT_MAX_DB;
327 }
328 buf[i] = val;
329 /* To soften lower limit, but not the first position which is the constant value */
330 if (i == idx && i > 2) {
331 buf[i - 1] = 0.5 * (buf[i] + buf[i - 1]);
332 }
333 }
334 /* To soften higher limit */
336 buf[i] = 0.5 * (buf[i] + buf[i - 1]);
337 }
338 }
339
340 const uint64_t curr_params_hash = sound_equalizermodifier_get_params_hash(buf);
341 /* Only make new sound when necessary. It is faster and it prevents audio glitches. */
342 if (!needs_update && smd->runtime->last_sound_in == sound_in &&
343 curr_params_hash == smd->runtime->params_hash)
344 {
345 MEM_delete(buf);
346 return smd->runtime->last_sound_out;
347 }
348
349 std::shared_ptr<aud::Buffer> aud_buf = std::make_shared<aud::Buffer>(
350 sizeof(float) * SOUND_EQUALIZER_SIZE_DEFINITION);
351 std::memcpy(aud_buf->getBuffer(), buf, sizeof(float) * SOUND_EQUALIZER_SIZE_DEFINITION);
352 AUD_Sound sound_out = AUD_Sound(new aud::Equalizer(sound_in,
353 aud_buf,
357
358 needs_update = true;
359 smd->runtime->last_sound_in = sound_in;
360 smd->runtime->last_sound_out = sound_out;
361 smd->runtime->params_hash = curr_params_hash;
362 MEM_delete(buf);
363
364 return sound_out;
365#else
366 UNUSED_VARS(strip, smd, sound_in, needs_update);
367 return nullptr;
368#endif
369}
370
372{
373 XXH3_state_t *state = XXH3_createState();
374 XXH3_64bits_reset(state);
375
376 XXH3_64bits_update(state, &pmd->mode, sizeof(pmd->mode));
377 XXH3_64bits_update(state, &pmd->quality, sizeof(pmd->quality));
378 XXH3_64bits_update(state, &pmd->semitones, sizeof(pmd->semitones));
379 XXH3_64bits_update(state, &pmd->cents, sizeof(pmd->cents));
380 XXH3_64bits_update(state, &pmd->ratio, sizeof(pmd->ratio));
381 XXH3_64bits_update(state, &pmd->preserve_formant, sizeof(pmd->preserve_formant));
382
383 uint64_t hash = XXH3_64bits_digest(state);
384 XXH3_freeState(state);
385 return hash;
386}
387
390 AUD_Sound sound_in,
391 bool &needs_update)
392{
393 const uint64_t curr_params_hash = pitchmodifier_get_params_hash((PitchModifierData *)smd);
394 if (!needs_update && smd->runtime->last_sound_in == sound_in &&
395 curr_params_hash == smd->runtime->params_hash)
396 {
397 return smd->runtime->last_sound_out;
398 }
399
400#if defined(WITH_AUDASPACE) && defined(WITH_RUBBERBAND)
402
403 aud::StretcherQuality quality;
404 switch (pmd->quality) {
406 quality = aud::StretcherQuality::HIGH;
407 break;
409 quality = aud::StretcherQuality::FAST;
410 break;
412 quality = aud::StretcherQuality::CONSISTENT;
413 break;
414 default:
415 quality = aud::StretcherQuality::HIGH;
416 }
417
418 double pitch_scale = 0;
419 int mode = pmd->mode;
420 if (mode == PITCH_MODE_SEMITONES) {
421 pitch_scale = pow(2.0, (pmd->semitones + (pmd->cents / 100.0)) / 12.0);
422 }
423 else if (mode == PITCH_MODE_RATIO) {
424 pitch_scale = pmd->ratio;
425
426 if (pitch_scale <= 0.0) {
427 pitch_scale = 1.0;
428 pmd->ratio = 1.0;
429 }
430 }
431
432 if (pitch_scale == 0) {
433 if (smd->runtime->last_sound_in == sound_in) {
434 return smd->runtime->last_sound_out;
435 }
436 return sound_in;
437 }
438
439 AUD_Sound sound_out = AUD_Sound(
440 new aud::TimeStretchPitchScale(sound_in, 1, pitch_scale, quality, pmd->preserve_formant));
441 needs_update = true;
442 smd->runtime->last_sound_in = sound_in;
443 smd->runtime->last_sound_out = sound_out;
444 smd->runtime->params_hash = curr_params_hash;
445
446 return sound_out;
447#else
448 if (smd->runtime->last_sound_in == sound_in) {
449 return smd->runtime->last_sound_out;
450 }
451 else {
452 return sound_in;
453 }
454#endif
455}
456
457#ifdef WITH_AUDASPACE
458static uint64_t echomodifier_get_params_hash(EchoModifierData *emd)
459{
460 XXH3_state_t *state = XXH3_createState();
461 XXH3_64bits_reset(state);
462
463 XXH3_64bits_update(state, &emd->delay, sizeof(emd->delay));
464 XXH3_64bits_update(state, &emd->feedback, sizeof(emd->feedback));
465 XXH3_64bits_update(state, &emd->mix, sizeof(emd->mix));
466
467 uint64_t hash = XXH3_64bits_digest(state);
468 XXH3_freeState(state);
469 return hash;
470}
471#endif
472
475 AUD_Sound sound_in,
476 bool &needs_update)
477{
478#if defined(WITH_AUDASPACE)
479 const uint64_t curr_params_hash = echomodifier_get_params_hash((EchoModifierData *)smd);
480 if (!needs_update && smd->runtime->last_sound_in == sound_in &&
481 curr_params_hash == smd->runtime->params_hash)
482 {
483 return smd->runtime->last_sound_out;
484 }
486
487 AUD_Sound sound_out = AUD_Sound(
488 new aud::Echo(sound_in, emd->delay, emd->feedback, emd->mix, true));
489 needs_update = true;
490 smd->runtime->last_sound_in = sound_in;
491 smd->runtime->last_sound_out = sound_out;
492 smd->runtime->params_hash = curr_params_hash;
493 return sound_out;
494#else
495 UNUSED_VARS(smd, sound_in, needs_update);
496 return nullptr;
497#endif
498}
499
501{
502 for (int i = 0; workersSoundModifiers[i].type > 0; i++) {
503 if (workersSoundModifiers[i].type == type) {
504 return &workersSoundModifiers[i];
505 }
506 }
507 return nullptr;
508}
509
512 AUD_Sound sound,
513 bool &needs_update)
514{
515
516 /* Check if the modifier mute flag has changed. */
518 eStripModifierFlag runtime_flag = smd->runtime->flag;
519 /* Update the runtime mute flag and flag the sound handle for update. */
520 runtime_flag &= ~(STRIP_MODIFIER_FLAG_MUTE); /* Clear the bit. */
521 runtime_flag |= (smd->flag & STRIP_MODIFIER_FLAG_MUTE); /* Set the bit. */
522 smd->runtime->flag = runtime_flag;
523 needs_update = true;
524 }
525
526 if (!(smd->flag & STRIP_MODIFIER_FLAG_MUTE)) {
528 return smwi->recreator(strip, smd, sound, needs_update);
529 }
530 return sound;
531}
532
533} // namespace blender::seq
std::shared_ptr< void > AUD_Sound
Read Blender memory allocation and freeing.
#define SOUND_EQUALIZER_DEFAULT_MAX_FREQ
Definition SEQ_sound.hh:38
#define SOUND_EQUALIZER_SIZE_CONVERSION
Definition SEQ_sound.hh:40
#define SOUND_EQUALIZER_SIZE_DEFINITION
Definition SEQ_sound.hh:41
#define SOUND_EQUALIZER_DEFAULT_MIN_FREQ
Definition SEQ_sound.hh:37
#define SOUND_EQUALIZER_DEFAULT_MAX_DB
Definition SEQ_sound.hh:39
#define UNUSED_VARS(...)
#define ELEM(...)
#define round
#define pow
#define ceil
const int state
ccl_device_inline dual3 fabs(const ccl_private dual3 &a)
Definition math_dual.h:241
Strip * lookup_meta_by_strip(Editing *ed, const Strip *key)
void sound_update_bounds_all(Scene *scene)
static uint64_t pitchmodifier_get_params_hash(PitchModifierData *pmd)
void sound_equalizermodifier_init_data(StripModifierData *smd)
EQCurveMappingData * sound_equalizer_add(SoundEqualizerModifierData *semd, float minX, float maxX)
static void strip_update_sound_recursive(Scene *scene, ListBaseT< Strip > *seqbasep, bSound *sound)
void sound_update_length(Main *bmain, Scene *scene)
const SoundModifierWorkerInfo workersSoundModifiers[]
AUD_Sound echomodifier_recreator(Strip *, StripModifierData *smd, AUD_Sound sound_in, bool &needs_update)
void sound_equalizermodifier_remove_graph(SoundEqualizerModifierData *semd, EQCurveMappingData *eqcmd)
AUD_Sound sound_modifier_recreator(Strip *strip, StripModifierData *smd, AUD_Sound sound, bool &needs_update)
void sound_equalizermodifier_free(StripModifierData *smd)
EQCurveMappingData * sound_equalizermodifier_add_graph(SoundEqualizerModifierData *semd, float min_freq, float max_freq)
void sound_update_bounds(Scene *scene, Strip *strip)
void strip_update_sound_bounds_recursive(const Scene *scene, Strip *strip_meta)
float sound_pitch_get(const Scene *scene, const Strip *strip)
void sound_equalizermodifier_copy_data(StripModifierData *target, StripModifierData *smd)
void sound_equalizermodifier_set_graphs(SoundEqualizerModifierData *semd, int number)
void sound_update(Scene *scene, bSound *sound)
AUD_Sound pitchmodifier_recreator(Strip *, StripModifierData *smd, AUD_Sound sound_in, bool &needs_update)
AUD_Sound sound_equalizermodifier_recreator(Strip *strip, StripModifierData *smd, AUD_Sound sound_in, bool &needs_update)
const SoundModifierWorkerInfo * sound_modifier_worker_info_get(int type)
unsigned __int64 uint64_t
Definition BKE_text.h:91
void BKE_sound_move_scene_sound_defaults(Scene *scene, Strip *strip)
BLI_INLINE void BLI_listbase_clear(ListBase *lb)
void BKE_sound_move_scene_sound(const Scene *scene, AUD_SequenceEntry handle, int startframe, int endframe, int frameskip, double audio_offset)
void BKE_curvemapping_copy_data(CurveMapping *target, const CurveMapping *cumap)
float BKE_curvemap_evaluateF(const CurveMapping *cumap, const CurveMap *cuma, float value)
@ CURVE_PRESET_CONSTANT_MEDIAN
bool BKE_sound_info_get(Main *main, bSound *sound, SoundInfo *sound_info)
void BKE_curvemapping_init(CurveMapping *cumap)
void BKE_curvemap_reset(CurveMap *cuma, const rctf *clipr, int preset, CurveMapSlopeType slope)
void BKE_curvemapping_set_defaults(CurveMapping *cumap, int tot, float minx, float miny, float maxx, float maxy, eBezTriple_Handle default_handle_type)
Definition colortools.cc:47
void BKE_sound_update_scene_sound(AUD_SequenceEntry handle, bSound *sound)
void BKE_curvemapping_free_data(CurveMapping *cumap)
void BLI_addtail(ListBase *listbase, void *vlink) ATTR_NONNULL(1)
Definition listbase.cc:118
bool BLI_remlink_safe(ListBase *listbase, void *vlink) ATTR_NONNULL(1)
Definition listbase.cc:161
@ eSeqModifierType_SoundEqualizer
void * MEM_new_array_zeroed(size_t len, size_t size, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1
#define hash
Definition noise_c.cc:156
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float
eCurveMappingPreset preset
struct EQCurveMappingData * prev
struct EQCurveMappingData * next
ListBaseT< Strip > seqbase
struct Editing * ed
ListBaseT< EQCurveMappingData > graphics
blender::seq::StripModifierDataRuntime * runtime
AUD_Sound(*) recreator(Strip *strip, StripModifierData *smd, AUD_Sound sound, bool &needs_update)
Definition SEQ_sound.hh:31