32 const bool normalize_kernel)
34#if defined(WITH_FFTW3)
45 const int2 needed_padding_amount =
math::max(kernel_size, image_size);
46 const int2 needed_spatial_size = image_size + needed_padding_amount - 1;
53 const int2 frequency_size =
int2(spatial_size.x / 2 + 1, spatial_size.y);
55 constexpr int input_channels_count = 4;
56 const int64_t spatial_pixels_count =
int64_t(spatial_size.x) * spatial_size.y;
57 const int64_t frequency_pixels_count =
int64_t(frequency_size.x) * frequency_size.y;
61 struct ForwardTransformTask {
63 std::complex<float> *
output;
69 Array<float *> image_spatial_domain_channels(input_channels_count);
71 for (
const int channel : image_spatial_domain_channels.
index_range()) {
72 image_spatial_domain_channels[channel] = fftwf_alloc_real(spatial_pixels_count);
73 image_frequency_domain_channels[channel] =
reinterpret_cast<std::complex<float> *
>(
74 fftwf_alloc_complex(frequency_pixels_count));
75 forward_transform_tasks.append(ForwardTransformTask{image_spatial_domain_channels[channel],
76 image_frequency_domain_channels[channel]});
80 for (
const int channel : image_spatial_domain_channels.
index_range()) {
81 fftwf_free(image_spatial_domain_channels[channel]);
82 fftwf_free(image_frequency_domain_channels[channel]);
87 const bool is_color_kernel = kernel_channels_count == 4;
91 Array<float *> kernel_spatial_domain_channels(kernel_channels_count);
93 for (
const int channel : kernel_spatial_domain_channels.
index_range()) {
94 kernel_spatial_domain_channels[channel] = fftwf_alloc_real(spatial_pixels_count);
95 kernel_frequency_domain_channels[channel] =
reinterpret_cast<std::complex<float> *
>(
96 fftwf_alloc_complex(frequency_pixels_count));
97 forward_transform_tasks.append(ForwardTransformTask{
98 kernel_spatial_domain_channels[channel], kernel_frequency_domain_channels[channel]});
102 for (
const int channel : kernel_spatial_domain_channels.
index_range()) {
103 fftwf_free(kernel_spatial_domain_channels[channel]);
104 fftwf_free(kernel_frequency_domain_channels[channel]);
119 fftwf_plan forward_plan = fftwf_plan_dft_r2c_2d(
122 image_spatial_domain_channels[0],
123 reinterpret_cast<fftwf_complex *
>(image_frequency_domain_channels[0]),
125 fftwf_plan backward_plan = fftwf_plan_dft_c2r_2d(
128 reinterpret_cast<fftwf_complex *
>(image_frequency_domain_channels[0]),
129 image_spatial_domain_channels[0],
133 fftwf_destroy_plan(forward_plan);
134 fftwf_destroy_plan(backward_plan);
137 Result convolve_input = context.create_result(input.
type());
138 Result convolve_kernel = context.create_result(kernel.
type());
140 if (context.use_gpu()) {
150 convolve_input.share_data(input);
158 const Color pixel_color = convolve_input.load_pixel_zero<
Color>(texel);
159 for (
const int channel :
IndexRange(input_channels_count)) {
160 float *buffer = image_spatial_domain_channels[channel];
162 buffer[
index] = pixel_color[channel];
167 convolve_input.release();
175 const int2 kernel_center = kernel_size / 2;
180 const int2 centered_texel = kernel_center - texel;
181 const int2 wrapped_texel =
int2(
mod_i(centered_texel.x, spatial_size.x),
182 mod_i(centered_texel.y, spatial_size.y));
184 const float4 kernel_value = is_color_kernel ?
187 for (
const int channel :
IndexRange(kernel_channels_count)) {
188 float *buffer = kernel_spatial_domain_channels[channel];
189 buffer[texel.x + texel.y *
int64_t(spatial_size.x)] = kernel_value[channel];
191 sum_by_thread.local() +=
double4(kernel_value);
201 std::accumulate(sum_by_thread.begin(), sum_by_thread.end(),
double4(0.0)));
202 const float4 sanitized_sum =
float4(sum[0] == 0.0f ? 1.0f : sum[0],
203 sum[1] == 0.0f ? 1.0f : sum[1],
204 sum[2] == 0.0f ? 1.0f : sum[2],
205 sum[3] == 0.0f ? 1.0f : sum[3]);
206 const float4 normalization_factor = normalize_kernel ? sanitized_sum :
float4(1.0f);
210 forward_transform_tasks.index_range(), 1, [&](
const IndexRange sub_range) {
211 for (const int64_t i : sub_range) {
212 fftwf_execute_dft_r2c(
214 forward_transform_tasks[i].input,
215 reinterpret_cast<fftwf_complex *>(forward_transform_tasks[i].output));
224 const float4 normalization_scale =
float(spatial_size.x) * spatial_size.y * normalization_factor;
226 for (const int64_t channel : IndexRange(input_channels_count)) {
227 const int kernel_channel = is_color_kernel ? channel : 0;
228 std::complex<float> *image_buffer = image_frequency_domain_channels[channel];
229 const std::complex<float> *kernel_buffer = kernel_frequency_domain_channels[kernel_channel];
230 for (const int64_t y : sub_y_range) {
231 for (const int64_t x : IndexRange(frequency_size.x)) {
232 const int64_t index = x + y * int64_t(frequency_size.x);
233 image_buffer[index] *= kernel_buffer[index] / normalization_scale[kernel_channel];
240 threading::parallel_for(IndexRange(input_channels_count), 1, [&](
const IndexRange sub_range) {
241 for (
const int64_t channel : sub_range) {
242 fftwf_execute_dft_c2r(
244 reinterpret_cast<fftwf_complex *
>(image_frequency_domain_channels[channel]),
245 image_spatial_domain_channels[channel]);
249 Result output_cpu =
context.create_result(input.type());
250 output_cpu.allocate_texture(input.domain(),
true, ResultStorageType::CPUImage);
253 threading::memory_bandwidth_bound_task(input.size_in_bytes(), [&]() {
254 parallel_for(image_size, [&](const int2 texel) {
255 float4 color = float4(0.0f);
256 for (const int channel : IndexRange(input_channels_count)) {
257 const int64_t index = texel.x + texel.y * int64_t(spatial_size.x);
258 color[channel] = image_spatial_domain_channels[channel][index];
260 output_cpu.store_pixel(texel, Color(color));
265 Result output_gpu = output_cpu.upload_to_gpu(
true);
266 output.share_data(output_gpu);
267 output_gpu.release();
270 output.share_data(output_cpu);
272 output_cpu.release();
275 output.allocate_texture(input.domain());
281 output.store_pixel(texel, input.load_pixel<float4>(texel));