Blender V5.3
id_hash.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2025 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#include <fcntl.h>
10#ifndef WIN32
11# include <unistd.h>
12#else
13# include <io.h>
14#endif
15#include <fmt/format.h>
16#include <mutex>
17#include <xxhash.h>
18
19#include "BKE_id_hash.hh"
20#include "BKE_lib_id.hh"
21#include "BKE_lib_query.hh"
22#include "BKE_library.hh"
23#include "BKE_main.hh"
24
25#include "BLI_fileops.hh"
26#include "BLI_mmap.hh"
27#include "BLI_mutex.hh"
28#include "BLI_set.hh"
29
30namespace blender::bke::id_hash {
31
32static std::optional<Vector<char>> read_file(const StringRefNull path)
33{
34 fstream stream{path.c_str(), std::ios_base::in | std::ios_base::binary};
35 stream.seekg(0, std::ios_base::end);
36 const int64_t size = stream.tellg();
37 stream.seekg(0, std::ios_base::beg);
38
39 Vector<char> buffer(size);
40 stream.read(buffer.data(), size);
41 if (stream.bad()) {
42 return std::nullopt;
43 }
44
45 return buffer;
46}
47
48static std::optional<XXH128_hash_t> compute_file_hash_with_file_read(const StringRefNull path)
49{
50 const std::optional<Vector<char>> buffer = read_file(path);
51 if (!buffer) {
52 return std::nullopt;
53 }
54 return XXH3_128bits(buffer->data(), buffer->size());
55}
56
57static std::optional<XXH128_hash_t> compute_file_hash_with_memory_map(const StringRefNull path)
58{
59 const int file = BLI_open(path.c_str(), O_BINARY | O_RDONLY, 0);
60 if (file == -1) {
61 return std::nullopt;
62 }
63 BLI_SCOPED_DEFER([&]() { close(file); });
64
65 BLI_mmap_file *mmap_file = BLI_mmap_open(file);
66 if (!mmap_file) {
67 return std::nullopt;
68 }
69 BLI_SCOPED_DEFER([&]() { BLI_mmap_free(mmap_file); });
70 const size_t size = BLI_mmap_get_length(mmap_file);
71 const void *data = BLI_mmap_get_pointer(mmap_file);
72 const XXH128_hash_t hash = XXH3_128bits(data, size);
73 if (BLI_mmap_any_io_error(mmap_file)) {
74 return std::nullopt;
75 }
76 return hash;
77}
78
79static std::optional<XXH128_hash_t> compute_file_hash(const StringRefNull path)
80{
81 /* First try the memory map the file, because it avoids an extra copy. */
82 if (const std::optional<XXH128_hash_t> hash = compute_file_hash_with_memory_map(path)) {
83 /* Make sure both code paths are tested even if memory mapping should almost always work. */
84 BLI_assert(hash->low64 == compute_file_hash_with_file_read(path)->low64);
85 return hash;
86 }
87 if (const std::optional<XXH128_hash_t> hash = compute_file_hash_with_file_read(path)) {
88 return hash;
89 }
90 return std::nullopt;
91}
92
95 XXH128_hash_t hash;
96};
97
98static std::optional<XXH128_hash_t> get_source_file_hash(const ID &id, DeepHashErrors &r_errors)
99{
101 static Mutex mutex;
102
103 const StringRefNull path = id.lib->runtime->filepath_abs;
104
105 BLI_stat_t stat;
106 if (BLI_stat(path.c_str(), &stat) == -1) {
107 r_errors.missing_files.add_as(path);
108 return std::nullopt;
109 }
110 else if ((id.tag & ID_TAG_MISSING) != 0) {
111 r_errors.missing_from_files.add_as(path);
112 return std::nullopt;
113 }
114
115 std::lock_guard lock(mutex);
116 if (const CachedFileHash *cached_hash = cache.lookup_ptr_as(path)) {
117 if (cached_hash->last_modified == stat.st_mtime) {
118 return cached_hash->hash;
119 }
120 }
121
122 /* The modification time may not be set if the data-block is added as linked data as part of
123 * versioning (e.g. in #do_versions_after_setup). */
124 if (id.runtime->src_blend_modifification_time != 0) {
125 if (stat.st_mtime != id.runtime->src_blend_modifification_time) {
126 r_errors.updated_files.add_as(path);
127 return std::nullopt;
128 }
129 }
130
131 if (const std::optional<XXH128_hash_t> hash = compute_file_hash(path)) {
132 cache.add_overwrite(path, CachedFileHash{stat.st_mtime, *hash});
133 return hash;
134 }
135 r_errors.missing_files.add_as(path);
136 return std::nullopt;
137}
138
139static std::optional<XXH128_hash_t> get_id_shallow_hash(const ID &id, DeepHashErrors &r_errors)
140{
142 const StringRefNull id_name = id.name;
143 const std::optional<XXH128_hash_t> file_hash = get_source_file_hash(id, r_errors);
144 if (!file_hash) {
145 return std::nullopt;
146 }
147
148 XXH3_state_t *hash_state = XXH3_createState();
149 XXH3_128bits_reset(hash_state);
150 XXH3_128bits_update(hash_state, id_name.data(), id_name.size());
151 XXH3_128bits_update(hash_state, &*file_hash, sizeof(XXH128_hash_t));
152 XXH128_hash_t shallow_hash = XXH3_128bits_digest(hash_state);
153 XXH3_freeState(hash_state);
154 return shallow_hash;
155}
156
157static void compute_deep_hash_recursive(const Main &bmain,
158 const ID &id,
159 Set<const ID *> &current_stack,
160 Map<const ID *, IDHash> &r_hashes,
161 DeepHashErrors &r_errors)
162{
163 if (r_hashes.contains(&id)) {
164 return;
165 }
166 if (!id.deep_hash.is_null()) {
167 r_hashes.add(&id, id.deep_hash);
168 return;
169 }
170 current_stack.add(&id);
171 BLI_SCOPED_DEFER([&]() -> void { current_stack.remove(&id); });
172 const std::optional<XXH128_hash_t> id_shallow_hash = get_id_shallow_hash(id, r_errors);
173 if (!id_shallow_hash) {
174 return;
175 }
176
177 XXH3_state_t *hash_state = XXH3_createState();
178 BLI_SCOPED_DEFER([&hash_state]() -> void { XXH3_freeState(hash_state); })
179 XXH3_128bits_reset(hash_state);
180 XXH3_128bits_update(hash_state, &*id_shallow_hash, sizeof(XXH128_hash_t));
181
182 bool success = true;
184 const_cast<Main *>(&bmain),
185 const_cast<ID *>(&id),
186 [&](LibraryIDLinkCallbackData *cb_data) {
187 if (cb_data->cb_flag & IDWALK_CB_LOOPBACK) {
188 /* Loop-back pointer (e.g. from a shape-key to its owner geometry ID, or from a
189 * collection to its parents) should always be ignored, as they do not represent an
190 * actual dependency. The dependency relationship should already have been
191 * processed from the owner to its dependency anyway (if applicable). */
192 return IDWALK_RET_NOP;
193 }
195 /* Embedded data are part of their owner's internal data, and as such already computed as
196 * part of the owner's shallow hash. */
197 return IDWALK_RET_NOP;
198 }
199 if (cb_data->cb_flag & IDWALK_CB_HASH_IGNORE) {
200 /* This pointer is explicitly ignored for the hash computation. */
201 return IDWALK_RET_NOP;
202 }
203 ID *referenced_id = *cb_data->id_pointer;
204 if (!referenced_id) {
205 /* Need to update the hash even if there is no id. There is a difference between the case
206 * where there is no id and the case where this callback is not called at all.*/
207 const int random_data = 452942579;
208 XXH3_128bits_update(hash_state, &random_data, sizeof(int));
209 return IDWALK_RET_NOP;
210 }
211 /* All embedded ID usages should already have been excluded above. */
212 BLI_assert((referenced_id->flag & ID_FLAG_EMBEDDED_DATA) == 0);
213 if (current_stack.contains(referenced_id)) {
214 /* Somehow encode that we had a circular reference here. */
215 const int random_data = 234632342;
216 XXH3_128bits_update(hash_state, &random_data, sizeof(int));
217 return IDWALK_RET_NOP;
218 }
219 compute_deep_hash_recursive(bmain, *referenced_id, current_stack, r_hashes, r_errors);
220 const IDHash *referenced_id_hash = r_hashes.lookup_ptr(referenced_id);
221 if (!referenced_id_hash) {
222 success = false;
224 }
225 XXH3_128bits_update(hash_state, referenced_id_hash->data, sizeof(IDHash));
226 return IDWALK_RET_NOP;
227 },
228 nullptr,
230
231 if (!success) {
232 return;
233 }
234 IDHash new_deep_hash;
235 const XXH128_hash_t new_deep_hash_xxh128 = XXH3_128bits_digest(hash_state);
236 static_assert(sizeof(IDHash) == sizeof(XXH128_hash_t));
237 memcpy(new_deep_hash.data, &new_deep_hash_xxh128, sizeof(IDHash));
238 r_hashes.add(&id, new_deep_hash);
239}
240
242{
243#ifndef NDEBUG
244 for (const ID *id : ids) {
246 }
247#endif
248
249 if (ids.is_empty()) {
250 return ValidDeepHashes{};
251 }
252
254 Set<const ID *> current_stack;
255 DeepHashErrors errors;
256 for (const ID *id : ids) {
257 compute_deep_hash_recursive(bmain, *id, current_stack, hashes, errors);
258 }
259 if (!errors.missing_files.is_empty() || !errors.missing_from_files.is_empty() ||
260 !errors.updated_files.is_empty())
261 {
262 return errors;
263 }
264 return ValidDeepHashes{hashes};
265}
266
267std::string id_hash_to_hex(const IDHash &hash)
268{
269 std::string hex_str;
270 for (const uint8_t byte : hash.data) {
271 hex_str += fmt::format("{:02x}", byte);
272 }
273 return hex_str;
274}
275
276} // namespace blender::bke::id_hash
File and directory operations.
#define O_BINARY
#define BLI_SCOPED_DEFER(function_to_defer)
#define ID_IS_LINKED(_id)
Definition DNA_ID.h:697
volatile int lock
const Value * lookup_ptr(const Key &key) const
Definition BLI_map.hh:508
bool add_overwrite(const Key &key, const Value &value)
Definition BLI_map.hh:325
bool add(const Key &key, const Value &value)
Definition BLI_map.hh:295
const Value * lookup_ptr_as(const ForwardKey &key) const
Definition BLI_map.hh:516
bool contains(const Key &key) const
Definition BLI_map.hh:353
bool contains(const Key &key) const
Definition BLI_set.hh:310
bool add(const Key &key)
Definition BLI_set.hh:248
bool remove(const Key &key)
Definition BLI_set.hh:404
constexpr bool is_empty() const
Definition BLI_span.hh:264
constexpr const char * c_str() const
bool add_as(ForwardKey &&key)
constexpr int64_t size() const
constexpr const char * data() const
static std::optional< XXH128_hash_t > get_source_file_hash(const ID &id, DeepHashErrors &r_errors)
Definition id_hash.cc:98
static std::optional< XXH128_hash_t > compute_file_hash_with_file_read(const StringRefNull path)
Definition id_hash.cc:48
static void compute_deep_hash_recursive(const Main &bmain, const ID &id, Set< const ID * > &current_stack, Map< const ID *, IDHash > &r_hashes, DeepHashErrors &r_errors)
Definition id_hash.cc:157
static std::optional< XXH128_hash_t > compute_file_hash(const StringRefNull path)
Definition id_hash.cc:79
static std::optional< XXH128_hash_t > get_id_shallow_hash(const ID &id, DeepHashErrors &r_errors)
Definition id_hash.cc:139
static std::optional< XXH128_hash_t > compute_file_hash_with_memory_map(const StringRefNull path)
Definition id_hash.cc:57
std::variant< ValidDeepHashes, DeepHashErrors > IDHashResult
static std::optional< Vector< char > > read_file(const StringRefNull path)
Definition id_hash.cc:32
IDHashResult compute_linked_id_deep_hashes(const Main &bmain, Span< const ID * > root_ids)
Definition id_hash.cc:241
std::string id_hash_to_hex(const IDHash &hash)
Definition id_hash.cc:267
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
__int64 int64_t
Definition BKE_text.h:90
size_t BLI_mmap_get_length(const BLI_mmap_file *file) ATTR_WARN_UNUSED_RESULT
Definition BLI_mmap.cc:484
BLI_mmap_file * BLI_mmap_open(int fd) ATTR_MALLOC ATTR_WARN_UNUSED_RESULT
Definition BLI_mmap.cc:369
int BLI_open(const char *filepath, int oflag, int pmode) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL()
void BLI_mmap_free(BLI_mmap_file *file) ATTR_NONNULL(1)
Definition BLI_mmap.cc:494
BMesh const char void * data
int BLI_stat(const char *path, BLI_stat_t *buffer) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL()
unsigned char uint8_t
Definition BKE_text.h:79
bool BLI_mmap_any_io_error(const BLI_mmap_file *file) ATTR_WARN_UNUSED_RESULT
Definition BLI_mmap.cc:489
@ IDWALK_RET_STOP_ITER
std::mutex Mutex
Definition BLI_mutex.hh:51
void * BLI_mmap_get_pointer(BLI_mmap_file *file) ATTR_WARN_UNUSED_RESULT
Definition BLI_mmap.cc:479
@ ID_TAG_MISSING
Definition DNA_ID.h:870
@ ID_FLAG_EMBEDDED_DATA
Definition DNA_ID.h:775
struct stat BLI_stat_t
@ IDWALK_CB_LOOPBACK
@ IDWALK_CB_EMBEDDED_NOT_OWNING
@ IDWALK_CB_HASH_IGNORE
@ IDWALK_CB_EMBEDDED
void BKE_library_foreach_ID_link(Main *bmain, ID *id, FunctionRef< LibraryIDLinkCallback > callback, void *user_data, LibraryForeachIDFlag flag)
Definition lib_query.cc:449
#define hash
Definition noise_c.cc:156
char data[16]
Definition DNA_ID.h:379
short flag
Definition DNA_ID.h:439
LibraryForeachIDCallbackFlag cb_flag
VectorSet< std::string > updated_files
VectorSet< std::string > missing_from_files
VectorSet< std::string > missing_files