Blender V5.3
spz_read_ngsp.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2026 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#include "spz_read.hh"
10
11#include <zstd.h>
12
13#include <cstdint>
14#include <utility>
15
16#include "BLI_array.hh"
17#include "BLI_fileops.hh"
18#include "BLI_math_base.hh"
19
20#include "BKE_lib_id.hh"
21#include "BKE_pointcloud.hh"
22#include "BKE_report.hh"
23
24#include "IO_gsplat.hh"
25
26#include "CLG_log.h"
27
28#include "spz_header.hh"
29#include "spz_read_common.hh"
30
31namespace blender::io::spz {
32
33static CLG_LogRef LOG = {"io.spz"};
34
35namespace {
36
37struct StreamInfo {
38 /* Offset of the stream from the beginning of the file. */
39 uint64_t compressed_offset;
40
41 uint64_t compressed_size;
42 uint64_t uncompressed_size;
43};
44
45/* Helper class that takes care of reading data from SPZ stream from file in chunks. */
46class BufferedStreamReader {
47 static constexpr int BUFFER_SIZE = 65536;
48
49 FILE *file_ = nullptr;
50 StreamInfo stream_info_;
51 Array<uint8_t> buffer_;
52
53 uint64_t num_read_bytes_ = 0;
54
55 public:
56 BufferedStreamReader(FILE *file, const StreamInfo &stream_info)
57 : file_(file), stream_info_(stream_info), buffer_(BUFFER_SIZE)
58 {
59 }
60
61 bool initialize()
62 {
63 return BLI_fseek(file_, int64_t(stream_info_.compressed_offset), SEEK_SET) != -1;
64 }
65
66 std::optional<Span<uint8_t>> read()
67 {
68 const uint64_t num_bytes_to_read = math::min(stream_info_.compressed_size - num_read_bytes_,
69 uint64_t(buffer_.size()));
70 const size_t num_read_bytes = fread(buffer_.data(), 1, num_bytes_to_read, file_);
71 if (ferror(file_) || num_bytes_to_read != num_read_bytes) {
72 return std::nullopt;
73 }
74 num_read_bytes_ += num_bytes_to_read;
75 return buffer_.as_span().slice(0, num_read_bytes);
76 }
77};
78
79/* Helper class that allows reading a Gzip-compressed file from disk with the minimum amount of
80 * extra memory usage, but allowing to easily access data of specific size. */
81class StreamedZstdReader {
82 BufferedStreamReader buffered_reader_;
83 ZSTD_DCtx *dctx_ = nullptr;
84 ZSTD_inBuffer input_buffer_;
85
86 public:
87 StreamedZstdReader(FILE *file, const StreamInfo &stream_info)
88 : buffered_reader_(file, stream_info)
89 {
90 }
91
92 ~StreamedZstdReader()
93 {
94 if (dctx_) {
95 ZSTD_freeDCtx(dctx_);
96 }
97 }
98
99 bool initialize()
100 {
101 if (!buffered_reader_.initialize()) {
102 return false;
103 }
104
105 dctx_ = ZSTD_createDCtx();
106
107 input_buffer_.src = nullptr;
108 input_buffer_.size = 0;
109 input_buffer_.pos = 0;
110
111 return true;
112 }
113
114 bool read_buffer(const MutableSpan<uint8_t> buffer)
115 {
116 ZSTD_outBuffer output_buffer = {.dst = buffer.data(), .size = size_t(buffer.size()), .pos = 0};
117
118 do {
119 /* There might be left-over data in the input since previous read.
120 * If there is, process that data first. Otherwise, load the next chunk from the input file.
121 */
122 if (input_buffer_.pos == input_buffer_.size) {
123 std::optional<Span<uint8_t>> read_result = buffered_reader_.read();
124 if (!read_result.has_value() || read_result->size() == 0) {
125 return false;
126 }
127 input_buffer_.src = read_result->data();
128 input_buffer_.size = read_result->size();
129 input_buffer_.pos = 0;
130 }
131
132 const size_t ret = ZSTD_decompressStream(dctx_, &output_buffer, &input_buffer_);
133 if (ZSTD_isError(ret)) {
134 // TODO(sergey): Report error using ZSTD_getErrorName(lastResult) ?
135 return false;
136 }
137 } while (output_buffer.pos < output_buffer.size);
138
139 return true;
140 }
141
142 template<class T> bool read(T &data)
143 {
144 return read_buffer(MutableSpan<uint8_t>(reinterpret_cast<uint8_t *>(&data), sizeof(T)));
145 }
146};
147
148} // namespace
149
150template<class T> static bool read_spz_struct(FILE *file, T &data)
151{
152 if (fread(&data, sizeof(T), 1, file) != 1) {
153 return false;
154 }
155 return true;
156}
157
158static bool read_toc(FILE *file, const NgspFileHeader &header, Array<StreamInfo> &stream_infos)
159{
160 struct PackedStreamInfo {
161 uint64_t compressed_size;
162 uint64_t uncompressed_size;
163 };
164 static_assert(sizeof(PackedStreamInfo) == 16);
165
166 if (header.num_streams == 0) {
167 return true;
168 }
169
170 if (BLI_fseek(file, header.toc_byte_offset, SEEK_SET) == -1) {
171 return false;
172 }
173
174 Vector<PackedStreamInfo> packed_stream_infos(header.num_streams);
175 if (fread(packed_stream_infos.data(), sizeof(PackedStreamInfo), header.num_streams, file) !=
176 header.num_streams)
177 {
178 return false;
179 }
180
181 stream_infos.reinitialize(header.num_streams);
182 stream_infos[0].compressed_offset = header.toc_byte_offset +
183 packed_stream_infos.size() * sizeof(PackedStreamInfo);
184 stream_infos[0].compressed_size = packed_stream_infos[0].compressed_size;
185 stream_infos[0].uncompressed_size = packed_stream_infos[0].uncompressed_size;
186
187 for (int i = 1; i < header.num_streams; ++i) {
188 stream_infos[i].compressed_offset = stream_infos[i - 1].compressed_offset +
189 stream_infos[i - 1].compressed_size;
190 stream_infos[i].compressed_size = packed_stream_infos[i].compressed_size;
191 stream_infos[i].uncompressed_size = packed_stream_infos[i].uncompressed_size;
192 }
193
194 return true;
195}
196
197static bool read_spz(FILE *file,
198 const NgspFileHeader &header,
199 const Span<StreamInfo> stream_infos,
200 ReportList *reports,
202{
203 if (stream_infos.size() < 6) {
204 return false;
205 }
206
207 const MutableSpan<float3> positions = accessor.positions_for_write();
208 const MutableSpan<math::Quaternion> rotations = accessor.rotations_for_write();
209 const Span<MutableSpan<float3>> sh_attrs = accessor.sh_for_write();
210
211 /* Positions. */
212 {
213 StreamedZstdReader reader(file, stream_infos[0]);
214 if (!reader.initialize()) {
215 BKE_report(reports, RPT_ERROR, "SPZ Read: Error initializing positions reader");
216 return false;
217 }
218 if (!read_positions(reader, header.fractional_bits, positions)) {
219 BKE_report(reports, RPT_ERROR, "SPZ Read: Error reading positions");
220 return false;
221 }
222 }
223
224 /* Alphas and colors. */
225 {
226 StreamedZstdReader reader(file, stream_infos[1]);
227 if (!reader.initialize()) {
228 BKE_report(reports, RPT_ERROR, "SPZ Read: Error initializing alphas reader");
229 return false;
230 }
231 if (!read_alphas(reader, accessor.radiance_base_for_write())) {
232 BKE_report(reports, RPT_ERROR, "SPZ Read: Error reading alphas");
233 return false;
234 }
235 }
236 {
237 StreamedZstdReader reader(file, stream_infos[2]);
238 if (!reader.initialize()) {
239 BKE_report(reports, RPT_ERROR, "SPZ Read: Error initializing colors reader");
240 return false;
241 }
242 if (!read_colors(reader, accessor.radiance_base_for_write())) {
243 BKE_report(reports, RPT_ERROR, "SPZ Read: Error reading colors");
244 return false;
245 }
246 }
247
248 /* Scales. */
249 {
250 StreamedZstdReader reader(file, stream_infos[3]);
251 if (!reader.initialize()) {
252 BKE_report(reports, RPT_ERROR, "SPZ Read: Error initializing scales reader");
253 return false;
254 }
255 if (!read_scales(reader, accessor.scales_for_write())) {
256 BKE_report(reports, RPT_ERROR, "SPZ Read: Error reading scales");
257 return false;
258 }
259 }
260
261 /* Rotations. */
262 {
263 StreamedZstdReader reader(file, stream_infos[4]);
264 if (!reader.initialize()) {
265 BKE_report(reports, RPT_ERROR, "SPZ Read: Error initializing rotations reader");
266 return false;
267 }
268 if (!read_rotations(reader, header.version, rotations)) {
269 BKE_report(reports, RPT_ERROR, "SPZ Read: Error reading rotations");
270 return false;
271 }
272 }
273
274 /* SH. */
275 {
276 StreamedZstdReader reader(file, stream_infos[5]);
277 if (!reader.initialize()) {
278 BKE_report(reports, RPT_ERROR, "SPZ Read: Error initializing SH reader");
279 return false;
280 }
281 if (!read_sh(reader, sh_attrs)) {
282 BKE_report(reports, RPT_ERROR, "SPZ Read: Error reading spherical harmonics");
283 return false;
284 }
285 }
286
287 /* TODO(sergey): Handle extensions. */
288
289 convert_axis_to_blender(positions, rotations, sh_attrs);
290
291 return true;
292}
293
295{
296 NgspFileHeader header;
297 if (!read_spz_struct(file, header)) {
298 BKE_report(reports, RPT_ERROR, "SPZ Read: Error reading NGSP header");
299 return nullptr;
300 }
301
302 CLOG_DEBUG(&LOG, "SPZ header magic: 0x%X", header.magic);
303 CLOG_DEBUG(&LOG, "SPZ header version: %u", header.version);
304
305 if (const std::optional<std::string> error = check_header_for_errors(header)) {
306 BKE_reportf(reports, RPT_ERROR, "SPZ Read: %s", error->c_str());
307 return nullptr;
308 }
309
310 CLOG_DEBUG(&LOG, "SPZ header num_points: %u", header.num_points);
311 CLOG_DEBUG(&LOG, "SPZ header sh_degree: %d", int(header.sh_degree));
312 CLOG_DEBUG(&LOG, "SPZ header fractional_bits: %d", int(header.fractional_bits));
313 CLOG_DEBUG(&LOG, "SPZ header flags: %d", int(header.flags));
314 CLOG_DEBUG(&LOG, "SPZ header num_streams: %d", int(header.num_streams));
315 CLOG_DEBUG(&LOG, "SPZ header toc_byte_offset: %u", header.toc_byte_offset);
316
317 if (header.flags & SPZ_HEADER_ANTIALIASED) {
318 /* TODO(sergey): Support anti-aliased data. */
319 CLOG_WARN(&LOG, "SPZ data was trained with anti-aliasing which is not fully supported");
320 }
321 if (header.flags & SPZ_HEADER_HAS_EXTENSIONS) {
322 /* TODO(sergey): Support extensions. */
323 CLOG_WARN(&LOG, "SPZ file contains extensions that are not yet supported");
324 }
325
326 Array<StreamInfo> stream_infos;
327 if (!read_toc(file, header, stream_infos)) {
328 return nullptr;
329 }
330
332 gsplat::GsplatMutableAttributeAccessor accessor(*pointcloud, header.sh_degree);
333
334 if (!read_spz(file, header, stream_infos, reports, accessor)) {
335 accessor.finish();
336 BKE_id_free(nullptr, &pointcloud->id);
337 return nullptr;
338 }
339
340 accessor.finish();
341
342 return pointcloud;
343}
344
345} // namespace blender::io::spz
General operations for point clouds.
File and directory operations.
#define BUFFER_SIZE
#define CLOG_DEBUG(clg_ref,...)
Definition CLG_log.h:191
#define CLOG_WARN(clg_ref,...)
Definition CLG_log.h:189
long long int int64_t
void initialize()
void reinitialize(const int64_t new_size)
Definition BLI_array.hh:430
constexpr int64_t size() const
Definition BLI_span.hh:256
int64_t size() const
MutableSpan< math::Quaternion > rotations_for_write()
Definition gsplat.cc:48
Span< MutableSpan< float3 > > sh_for_write()
Definition gsplat.cc:53
#define T
static bool read_toc(FILE *file, const NgspFileHeader &header, Array< StreamInfo > &stream_infos)
static bool read_spz(FILE *file, const NgspFileHeader &header, const Span< StreamInfo > stream_infos, ReportList *reports, gsplat::GsplatMutableAttributeAccessor &accessor)
bool read_sh(ReaderType &reader, const Span< MutableSpan< float3 > > sh)
bool read_positions(ReaderType &reader, const int fractional_bits, const MutableSpan< float3 > positions)
void convert_axis_to_blender(const MutableSpan< float3 > positions, const MutableSpan< math::Quaternion > rotations, const Span< MutableSpan< float3 > > sh_attrs)
std::optional< std::string > check_header_for_errors(const PackedGaussiansHeader &header)
Definition spz_header.cc:41
bool read_colors(ReaderType &reader, const MutableSpan< float4 > radiance_base)
bool read_alphas(ReaderType &reader, const MutableSpan< float4 > radiance_base)
static CLG_LogRef LOG
Definition spz_read.cc:23
bool read_scales(ReaderType &reader, const MutableSpan< float3 > scales)
static bool read_spz_struct(FILE *file, T &data)
bool read_rotations(ReaderType &reader, const int version, const MutableSpan< math::Quaternion > rotations)
PointCloud * read_spz_ngsp_file(FILE *file, ReportList *reports)
T min(const T &a, const T &b)
unsigned __int64 uint64_t
Definition BKE_text.h:91
int BLI_fseek(FILE *stream, int64_t offset, int whence)
Definition storage.cc:201
void BKE_report(ReportList *reports, eReportType type, const char *message)
Definition report.cc:162
BMesh const char void * data
unsigned char uint8_t
Definition BKE_text.h:79
void BKE_id_free(Main *bmain, void *idv)
void BKE_reportf(ReportList *reports, eReportType type, const char *format,...) ATTR_PRINTF_FORMAT(3
Definition report.cc:191
PointCloud * BKE_pointcloud_new_nomain(PointCloudType type, int totpoint)
T read(T depth_buffer_value)