Blender V5.3
filereader_zstd.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2021 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#include <algorithm>
10#include <cstring>
11#include <zstd.h>
12
13#include "BLI_fileops.hh"
14#include "BLI_filereader.hh"
15
16#include "MEM_guardedalloc.h"
17
18namespace blender {
19
20struct ZstdReader {
22
24
25 ZSTD_DCtx *ctx;
26 ZSTD_inBuffer in_buf;
28
29 struct {
30 int frames_num;
31 size_t *compressed_ofs;
32 size_t *uncompressed_ofs;
33
34 char *cached_content;
35 int cached_frame;
37};
38
39static bool zstd_read_u32(FileReader *base, uint32_t *val)
40{
41 if (base->read(base, val, sizeof(uint32_t)) != sizeof(uint32_t)) {
42 return false;
43 }
44 /* NOTE: this is endianness-sensitive.
45 * `val` would need to be switched on a big endian system. */
46 return true;
47}
48
50{
51 FileReader *base = zstd->base;
52
53 /* The seek table frame is at the end of the file, so seek there
54 * and verify that there is enough data. */
55 if (base->seek(base, -4, SEEK_END) < 13) {
56 return false;
57 }
58 uint32_t magic;
59 if (!zstd_read_u32(base, &magic) || magic != 0x8F92EAB1) {
60 return false;
61 }
62
63 uint8_t flags;
64 if (base->seek(base, -5, SEEK_END) < 0 || base->read(base, &flags, 1) != 1) {
65 return false;
66 }
67 /* Bit 7 indicates check-sums. Bits 5 and 6 must be zero. */
68 bool has_checksums = (flags & 0x80);
69 if (flags & 0x60) {
70 return false;
71 }
72
73 uint32_t frames_num;
74 if (base->seek(base, -9, SEEK_END) < 0 || !zstd_read_u32(base, &frames_num)) {
75 return false;
76 }
77
78 /* Each frame has either 2 or 3 uint32_t, and after that we have
79 * frames_num, flags and magic for another 9 bytes. */
80 uint32_t expected_frame_length = frames_num * (has_checksums ? 12 : 8) + 9;
81 /* The frame starts with another magic number and its length, but these
82 * two fields are not included when counting length. */
83 off64_t frame_start_ofs = 8 + expected_frame_length;
84 /* Sanity check: Before the start of the seek table frame,
85 * there must be frames_num frames, each of which at least 8 bytes long. */
86 off64_t seek_frame_start = base->seek(base, -frame_start_ofs, SEEK_END);
87 if (seek_frame_start < frames_num * 8) {
88 return false;
89 }
90
91 if (!zstd_read_u32(base, &magic) || magic != 0x184D2A5E) {
92 return false;
93 }
94
95 uint32_t frame_length;
96 if (!zstd_read_u32(base, &frame_length) || frame_length != expected_frame_length) {
97 return false;
98 }
99
100 zstd->seek.frames_num = frames_num;
101 zstd->seek.compressed_ofs = MEM_new_array_uninitialized<size_t>(frames_num + 1, __func__);
102 zstd->seek.uncompressed_ofs = MEM_new_array_uninitialized<size_t>(frames_num + 1, __func__);
103
104 size_t compressed_ofs = 0;
105 size_t uncompressed_ofs = 0;
106 for (int i = 0; i < frames_num; i++) {
107 uint32_t compressed_size, uncompressed_size;
108 if (!zstd_read_u32(base, &compressed_size) || !zstd_read_u32(base, &uncompressed_size)) {
109 break;
110 }
111 if (has_checksums && base->seek(base, 4, SEEK_CUR) < 0) {
112 break;
113 }
114 zstd->seek.compressed_ofs[i] = compressed_ofs;
115 zstd->seek.uncompressed_ofs[i] = uncompressed_ofs;
116 compressed_ofs += compressed_size;
117 uncompressed_ofs += uncompressed_size;
118 }
119 zstd->seek.compressed_ofs[frames_num] = compressed_ofs;
120 zstd->seek.uncompressed_ofs[frames_num] = uncompressed_ofs;
121
122 /* Seek to the end of the previous frame for the following #BHead frame detection. */
123 if (seek_frame_start != compressed_ofs || base->seek(base, seek_frame_start, SEEK_SET) < 0) {
124 MEM_delete(zstd->seek.compressed_ofs);
125 MEM_delete(zstd->seek.uncompressed_ofs);
126 memset(&zstd->seek, 0, sizeof(zstd->seek));
127 return false;
128 }
129
130 zstd->seek.cached_frame = -1;
131
132 return true;
133}
134
135/* Find out which frame contains the given position in the uncompressed stream.
136 * Basically just bisection. */
137static int zstd_frame_from_pos(ZstdReader *zstd, size_t pos)
138{
139 int low = 0, high = zstd->seek.frames_num;
140
141 if (pos >= zstd->seek.uncompressed_ofs[zstd->seek.frames_num]) {
142 return -1;
143 }
144
145 while (low + 1 < high) {
146 int mid = low + ((high - low) >> 1);
147 if (zstd->seek.uncompressed_ofs[mid] <= pos) {
148 low = mid;
149 }
150 else {
151 high = mid;
152 }
153 }
154
155 return low;
156}
157
158/* Ensure that the currently loaded frame is the correct one. */
159static const char *zstd_ensure_cache(ZstdReader *zstd, int frame)
160{
161 if (zstd->seek.cached_frame == frame) {
162 /* Cached frame matches, so just return it. */
163 return zstd->seek.cached_content;
164 }
165
166 /* Cached frame doesn't match, so discard it and cache the wanted one instead. */
167 MEM_SAFE_DELETE(zstd->seek.cached_content);
168
169 size_t compressed_size = zstd->seek.compressed_ofs[frame + 1] - zstd->seek.compressed_ofs[frame];
170 size_t uncompressed_size = zstd->seek.uncompressed_ofs[frame + 1] -
171 zstd->seek.uncompressed_ofs[frame];
172
173 char *uncompressed_data = MEM_new_array_uninitialized<char>(uncompressed_size, __func__);
174 char *compressed_data = MEM_new_array_uninitialized<char>(compressed_size, __func__);
175 if (zstd->base->seek(zstd->base, zstd->seek.compressed_ofs[frame], SEEK_SET) < 0 ||
176 zstd->base->read(zstd->base, compressed_data, compressed_size) < compressed_size)
177 {
178 MEM_delete(compressed_data);
179 MEM_delete(uncompressed_data);
180 return nullptr;
181 }
182
183 size_t res = ZSTD_decompressDCtx(
184 zstd->ctx, uncompressed_data, uncompressed_size, compressed_data, compressed_size);
185 MEM_delete(compressed_data);
186 if (ZSTD_isError(res) || res < uncompressed_size) {
187 MEM_delete(uncompressed_data);
188 return nullptr;
189 }
190
191 zstd->seek.cached_frame = frame;
192 zstd->seek.cached_content = uncompressed_data;
193 return uncompressed_data;
194}
195
196static int64_t zstd_read_seekable(FileReader *reader, void *buffer, size_t size)
197{
198 ZstdReader *zstd = reinterpret_cast<ZstdReader *>(reader);
199
200 size_t end_offset = zstd->reader.offset + size, read_len = 0;
201 while (zstd->reader.offset < end_offset) {
202 int frame = zstd_frame_from_pos(zstd, zstd->reader.offset);
203 if (frame < 0) {
204 /* EOF is reached, so return as much as we can. */
205 break;
206 }
207
208 const char *framedata = zstd_ensure_cache(zstd, frame);
209 if (framedata == nullptr) {
210 /* Error while reading the frame, so return as much as we can. */
211 break;
212 }
213
214 size_t frame_end_offset = std::min(zstd->seek.uncompressed_ofs[frame + 1], end_offset);
215 size_t frame_read_len = frame_end_offset - zstd->reader.offset;
216
217 size_t offset_in_frame = zstd->reader.offset - zstd->seek.uncompressed_ofs[frame];
218 memcpy(static_cast<char *>(buffer) + read_len, framedata + offset_in_frame, frame_read_len);
219 read_len += frame_read_len;
220 zstd->reader.offset = frame_end_offset;
221 }
222
223 return read_len;
224}
225
226static off64_t zstd_seek(FileReader *reader, off64_t offset, int whence)
227{
228 ZstdReader *zstd = reinterpret_cast<ZstdReader *>(reader);
229 off64_t new_pos;
230 if (whence == SEEK_SET) {
231 new_pos = offset;
232 }
233 else if (whence == SEEK_END) {
234 new_pos = zstd->seek.uncompressed_ofs[zstd->seek.frames_num] + offset;
235 }
236 else {
237 new_pos = zstd->reader.offset + offset;
238 }
239
240 if (new_pos < 0 || new_pos > zstd->seek.uncompressed_ofs[zstd->seek.frames_num]) {
241 return -1;
242 }
243 zstd->reader.offset = new_pos;
244 return zstd->reader.offset;
245}
246
247static int64_t zstd_read(FileReader *reader, void *buffer, size_t size)
248{
249 ZstdReader *zstd = reinterpret_cast<ZstdReader *>(reader);
250 ZSTD_outBuffer output = {buffer, size, 0};
251
252 while (output.pos < output.size) {
253 if (zstd->in_buf.pos == zstd->in_buf.size) {
254 /* Ran out of buffered input data, read some more. */
255 zstd->in_buf.pos = 0;
256 int64_t readsize = zstd->base->read(
257 zstd->base, const_cast<void *>(zstd->in_buf.src), zstd->in_buf_max_size);
258
259 if (readsize > 0) {
260 /* We got some data, so mark the buffer as refilled. */
261 zstd->in_buf.size = readsize;
262 }
263 else {
264 /* The underlying file is EOF, so return as much as we can. */
265 break;
266 }
267 }
268
269 if (ZSTD_isError(ZSTD_decompressStream(zstd->ctx, &output, &zstd->in_buf))) {
270 break;
271 }
272 }
273
274 zstd->reader.offset += output.pos;
275 return output.pos;
276}
277
278static void zstd_close(FileReader *reader)
279{
280 ZstdReader *zstd = reinterpret_cast<ZstdReader *>(reader);
281
282 ZSTD_freeDCtx(zstd->ctx);
283 if (zstd->reader.seek) {
284 MEM_delete(zstd->seek.uncompressed_ofs);
285 MEM_delete(zstd->seek.compressed_ofs);
286 /* When an error has occurred this may be nullptr, see: #99744. */
287 if (zstd->seek.cached_content) {
288 MEM_delete(zstd->seek.cached_content);
289 }
290 }
291 else {
292 MEM_delete(static_cast<const std::byte *>(zstd->in_buf.src));
293 }
294
295 zstd->base->close(zstd->base);
296 MEM_delete(zstd);
297}
298
300{
301 ZstdReader *zstd = MEM_new_zeroed<ZstdReader>(__func__);
302
303 zstd->ctx = ZSTD_createDCtx();
304 zstd->base = base;
305
306 if (zstd_read_seek_table(zstd)) {
308 zstd->reader.seek = zstd_seek;
309 }
310 else {
311 zstd->reader.read = zstd_read;
312 zstd->reader.seek = nullptr;
313
314 zstd->in_buf_max_size = ZSTD_DStreamInSize();
316 "zstd in buf");
317 zstd->in_buf.size = zstd->in_buf_max_size;
318 /* This signals that the buffer has run out,
319 * which will make the read function refill it on the first call. */
320 zstd->in_buf.pos = zstd->in_buf_max_size;
321 }
322 zstd->reader.close = zstd_close;
323
324 /* Rewind after the seek table check so that zstd_read starts at the file's start. */
325 zstd->base->seek(zstd->base, 0, SEEK_SET);
326
327 return reinterpret_cast<FileReader *>(zstd);
328}
329
330} // namespace blender
File and directory operations.
Wrapper for reading from various sources (e.g. raw files, compressed files, memory....
Read Blender memory allocation and freeing.
#define output
BMesh Iterators.
__int64 int64_t
Definition BKE_text.h:90
void * MEM_new_array_uninitialized(size_t len, size_t size, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1
static void zstd_close(FileReader *reader)
unsigned int uint32_t
Definition BKE_text.h:81
void * MEM_new_zeroed(size_t len, const char *str) ATTR_WARN_UNUSED_RESULT ATTR_ALLOC_SIZE(1) ATTR_NONNULL(2)
static const char * zstd_ensure_cache(ZstdReader *zstd, int frame)
static off64_t zstd_seek(FileReader *reader, off64_t offset, int whence)
static int zstd_frame_from_pos(ZstdReader *zstd, size_t pos)
unsigned char uint8_t
Definition BKE_text.h:79
static int64_t zstd_read_seekable(FileReader *reader, void *buffer, size_t size)
static bool zstd_read_seek_table(ZstdReader *zstd)
static bool zstd_read_u32(FileReader *base, uint32_t *val)
FileReader * BLI_filereader_new_zstd(FileReader *base) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL()
static int64_t zstd_read(FileReader *reader, void *buffer, size_t size)
FileReaderCloseFn close
FileReaderSeekFn seek
FileReaderReadFn read
struct blender::ZstdReader::@143246372204067102016176054074122254013224125366 seek