Blender V5.3
asset_indexer_remote_listing_v1.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2024 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#include <fmt/format.h>
10
11#include "BLI_assert.hh"
12#include "BLI_fileops.hh"
13#include "BLI_path_utils.hh"
14#include "BLI_serialize.hh"
15#include "BLI_set.hh"
16#include "BLI_string_ref.hh"
17#include "BLI_vector.hh"
18
19#include "BKE_asset.hh"
20#include "BKE_blender_version.h"
21#include "BKE_idtype.hh"
22
23#include "BLT_translation.hh"
24
25#include "CLG_log.h"
26
27#include "ED_asset_indexer.hh"
28
29#include "AS_remote_library.hh"
30
31#include "asset_index.hh"
34
35static CLG_LogRef LOG = {"asset.remote_listing"};
36
38
39using namespace blender::io::serialize;
40
41/* -------------------------------------------------------------------- */
44
46 static ReadingResult<> read_asset_entries(const StringRefNull listing_root_dirpath,
47 const StringRefNull filepath,
49};
50
55static std::optional<int> blender_version_from_string(const blender::StringRef str)
56{
57 const int64_t dot = str.find('.');
59 return {};
60 }
61 int major, minor;
62 const blender::StringRef major_str = str.substr(0, dot);
63 const blender::StringRef minor_str = str.substr(dot + 1);
64 if (std::from_chars(major_str.begin(), major_str.end(), major).ec != std::errc() ||
65 std::from_chars(minor_str.begin(), minor_str.end(), minor).ec != std::errc())
66 {
67 return {};
68 }
69 if (major < 0 || minor < 0 || minor >= 100) {
70 return {};
71 }
72 return major * 100 + minor;
73}
74
76{
77 const DictionaryValue *bl_versions_dict = asset_dictionary.lookup_dict("bl_versions");
78 if (!bl_versions_dict) {
80 N_("could not read asset Blender versions, 'bl_versions' field not set"));
81 }
82
83 /* Check the 'min' field. */
84 const std::optional<StringRef> min_opt = bl_versions_dict->lookup_str("min");
85 if (!min_opt) {
87 N_("could not read asset Blender versions, 'bl_versions.min' field not set"));
88 }
89
90 const std::optional<int> bl_version_min = blender_version_from_string(*min_opt);
91 if (!bl_version_min) {
93 N_("could not read asset Blender versions, 'bl_versions.min' field not in X.Y notation"));
94 }
95 if (BLENDER_VERSION < *bl_version_min) {
96 /* This Blender version is older than what the asset needs, so skip it. */
97 return ReadingResult<bool>::Success(false);
98 }
99
100 /* Check the 'until' field. */
101 const std::optional<StringRef> until_opt = bl_versions_dict->lookup_str("until");
102 if (!until_opt) {
103 /* Fine to be missing, this field is optional. If it is not there, the asset has no maximum
104 * version. */
105 return ReadingResult<bool>::Success(true);
106 }
107
108 const std::optional<int> bl_version_until = blender_version_from_string(*until_opt);
109 if (!bl_version_until) {
111 N_("could not read asset Blender versions, 'bl_versions.min' field not in X.Y notation"));
112 }
113 return ReadingResult<bool>::Success(BLENDER_VERSION < *bl_version_until);
114}
115
117 const DictionaryValue &dictionary,
118 const Map<std::string, RemoteListingFileEntry> &file_path_to_entry_map)
119{
120 RemoteListingAssetEntry listing_entry{};
121
122 /* Check the min/until Blender versions first. If the current Blender doesn't match, the entire
123 * asset can be ignored. */
124 ReadingResult<bool> version_check = blender_version_matches(dictionary);
125 if (!version_check.is_success()) {
127 std::move(version_check.failure_reason));
128 }
129 if (!*version_check) {
130 /* Return an empty entry, to indicate to the caller a successfully parsed entry that didn't
131 * yield an asset. */
133 }
134
135 /* 'id': name of the asset. Required string. */
136 const std::optional<StringRefNull> asset_name_opt = dictionary.lookup_str("name");
137 if (!asset_name_opt) {
139 N_("could not read asset name, 'name' field not set"));
140 }
141 const StringRefNull asset_name = *asset_name_opt;
142 asset_name.copy_utf8_truncated(listing_entry.datablock_info.name);
143
144 /* 'type': data-block type, must match the #IDTypeInfo.name of the given type. required string.
145 */
146 if (const std::optional<StringRefNull> idtype_name = dictionary.lookup_str("id_type")) {
147 const char *normalized_name = BKE_idtype_name_normalize(idtype_name->c_str());
148 if (!normalized_name) {
149 /* This could actually be a new asset type that's not supported by this Blender. Just
150 * silently ignore it and continue. */
152 N_("could not read type of asset '%s': 'id_type' field is not a valid type (%s)"),
153 asset_name.c_str(),
154 idtype_name->c_str());
156 }
157 listing_entry.idcode = BKE_idtype_idcode_from_name(normalized_name);
158 }
159 else {
161 fmt::format(N_("could not read type of asset '{:s}', 'type' field not set"), asset_name));
162 }
163
164 /* 'files': required list of strings. */
165 if (const ArrayValue *file_paths = dictionary.lookup_array("files")) {
166 if (file_paths->elements().is_empty()) {
168 fmt::format(N_("asset '{:s}' has no files"), asset_name));
169 }
170 for (const std::shared_ptr<Value> &file_path_element : file_paths->elements()) {
172
173 const io::serialize::StringValue *file_path_string = file_path_element->as_string_value();
174 if (!file_path_string) {
176 N_("asset '{:s}' has a non-string entry in its 'files' list"), asset_name));
177 }
178 file.path = file_path_string->value();
179 if (file.path.empty()) {
180 /* TODO: use CLOG to have _some_ logging of this dubious empty file
181 * entry. But keep going, maybe there's another, non-empty entry. */
182 continue;
183 }
184
185 /* Look up the file URL and hash from the <files> section of the JSON. */
186 if (const RemoteListingFileEntry *file_entry = file_path_to_entry_map.lookup_ptr(file.path))
187 {
188 file.url = file_entry->download_url;
189 file.size_in_bytes = file_entry->size_in_bytes;
190 }
191 else {
193 fmt::format(N_("asset '{:s}' references unknown file '{:s}'"), asset_name, file.path));
194 }
195
196 listing_entry.online_info.files.append(file);
197 }
198 }
199 else {
201 fmt::format(N_("asset '{:s}' has no 'files' field"), asset_name));
202 }
203
204 /* 'thumbnail': URL and hash of the preview image. */
206 dictionary.lookup_dict("thumbnail"));
207
208 /* 'metadata': optional dictionary. If all the metadata fields are empty, this can be left out of
209 * the listing. Default metadata will then be allocated, with all fields empty/0. */
210 const DictionaryValue *metadata_dict = dictionary.lookup_dict("meta");
211 listing_entry.datablock_info.asset_data = metadata_dict ?
212 asset_metadata_from_dictionary(*metadata_dict) :
214 listing_entry.datablock_info.free_asset_data = true;
215
216 return ReadingResult<RemoteListingAssetEntry>::Success(std::move(listing_entry));
217}
218
220 const DictionaryValue &dictionary)
221{
222 RemoteListingFileEntry file_entry{};
223
224 /* Path is mandatory. */
225 if (const std::optional<StringRefNull> path = dictionary.lookup_str("path")) {
226 file_entry.local_path = *path;
227 }
228 else {
230 N_("Error reading asset listing file entry, skipping. Reason: found a file without 'path' "
231 "field"));
232 }
233
234 /* Hash is mandatory. */
235 if (const std::optional<StringRefNull> hash = dictionary.lookup_str("hash")) {
236 file_entry.download_url.hash = *hash;
237 }
238 else {
240 N_("Error reading asset listing file entry, skipping. Reason: found a file ({:s}) without "
241 "'hash' field"),
242 file_entry.local_path.c_str()));
243 }
244
245 /* Size is mandatory. */
246 if (const std::optional<int64_t> size_in_bytes = dictionary.lookup_int("size_in_bytes")) {
247 file_entry.size_in_bytes = *size_in_bytes;
248 }
249 else {
251 N_("Error reading asset listing file entry, skipping. Reason: found a file ({:s}) without "
252 "'size_in_bytes' field"),
253 file_entry.local_path.c_str()));
254 }
255
256 /* URL is optional, and defaults to the local path. That's handled in Python
257 * (see `download_asset()` in `asset_downloader.py`) so here we can just use
258 * an empty string to indicate "no URL". */
259 file_entry.download_url.url = dictionary.lookup_str("url").value_or("");
260
261 return ReadingResult<RemoteListingFileEntry>::Success(std::move(file_entry));
262}
263
265 const DictionaryValue &value,
266 const RemoteListingEntryProcessFn process_fn)
267{
268 const ArrayValue *assets = value.lookup_array("assets");
269 BLI_assert(assets != nullptr);
270 if (assets == nullptr) {
271 return ReadingResult<>::Failure(N_("no assets listed"));
272 }
273
274 /* Build a mapping from local file path to its file info. */
275 const ArrayValue *files = value.lookup_array("files");
276 BLI_assert(files != nullptr);
277 if (files == nullptr) {
278 /* The 'files' section is mandatory in the OpenAPI schema. */
280 N_("error reading asset listing, page file has no files section"));
281 }
282
283 Vector<std::string> warnings;
285 for (const std::shared_ptr<Value> &file_element : files->elements()) {
287 *file_element->as_dictionary_value());
288 if (file_result.is_failure()) {
289 warnings.append(std::move(file_result.failure_reason));
290 continue;
291 }
292 if (file_result.is_cancelled()) {
294 }
295 BLI_assert(file_result.is_success());
296 RemoteListingFileEntry &file_entry = *file_result;
297 if (file_entry.local_path.empty()) {
298 continue;
299 }
300 std::string local_path = file_entry.local_path; /* Make a copy before std::moving. */
301 path_to_file_info.add_overwrite(local_path, std::move(file_entry));
302 }
303
304 /* Store whether asset files match their listing's hash or not. */
305 FileStatusChecker file_status_checker(listing_root_dirpath);
306
307 /* Convert the assets into RemoteListingAssetEntry objects. */
308 for (const std::shared_ptr<Value> &asset_element : assets->elements()) {
310 *asset_element->as_dictionary_value(), path_to_file_info);
311 if (result.is_failure()) {
312 if (!result.failure_reason.empty()) {
313 warnings.append(std::move(result.failure_reason));
314 }
315 continue;
316 }
317
319 if (entry.is_empty()) {
320 continue;
321 }
322
323 /* Check the up-to-dateness of the asset's files. */
324 /* TODO: this has to change when Blender starts supporting multi-file assets. */
325 {
326 const StringRef asset_file = entry.online_info.asset_file();
327 RemoteListingFileEntry *file_entry = path_to_file_info.lookup_ptr(asset_file);
328 BLI_assert_msg(file_entry, "Assets without file info should have been filtered out by now");
329 entry.remote_file_status = file_status_checker.remote_file_status(*file_entry);
330 }
331
332 if (!process_fn(entry)) {
334 }
335 }
336
337 ReadingResult<> overall_result = ReadingResult<>::Success();
338 overall_result.warnings.extend(std::move(warnings));
339 return overall_result;
340}
341
343 const StringRefNull listing_root_dirpath,
344 const StringRefNull filepath,
345 const RemoteListingEntryProcessFn process_fn)
346{
347 if (!BLI_exists(filepath.c_str())) {
348 return ReadingResult<>::Failure(fmt::format(N_("file does not exist: {:s}"), filepath));
349 }
350
351 const std::unique_ptr<Value> contents = read_contents(filepath);
352 if (!contents) {
353 return ReadingResult<>::Failure(fmt::format(N_("file is empty: {:s}"), filepath));
354 }
355
356 const DictionaryValue *root = contents->as_dictionary_value();
357 if (!root) {
359 fmt::format(N_("file is not a JSON dictionary: {:s}"), filepath));
360 }
361
362 return listing_entries_from_root(listing_root_dirpath, *root, process_fn);
363}
364
366
367/* -------------------------------------------------------------------- */
374
379
380 static std::optional<AssetLibraryListingV1> read(const StringRefNull listing_filepath);
381};
382
383std::optional<AssetLibraryListingV1> AssetLibraryListingV1::read(
384 const StringRefNull listing_filepath)
385{
386 if (!BLI_exists(listing_filepath.c_str())) {
388 return {};
389 }
390
391 const std::unique_ptr<Value> contents = read_contents(listing_filepath);
392 if (!contents) {
394 return {};
395 }
396
397 const DictionaryValue *root = contents->as_dictionary_value();
398 if (!root) {
400 return {};
401 }
402
403 const ArrayValue *entries = root->lookup_array("pages");
404 BLI_assert(entries != nullptr);
405 if (entries == nullptr) {
406 return {};
407 }
408
409 AssetLibraryListingV1 listing;
410
411 int i = 0;
412 for (const std::shared_ptr<Value> &element : entries->elements()) {
413 const std::optional<asset_system::URLWithHash> page_info = parse_url_with_hash_dict(
414 element->as_dictionary_value());
415 if (!page_info) {
416 printf("Error reading asset listing page path at index %i in %s - ignoring\n",
417 i,
418 listing_filepath.c_str());
419 i++;
420 continue;
421 }
422 listing.page_rel_paths.append(std::move(page_info->url));
423 i++;
424 }
425
426 return listing;
427}
428
430
432 const RemoteListingEntryProcessFn process_fn,
433 const RemoteListingWaitForPagesFn wait_fn,
434 const std::optional<Timestamp> ignore_before_timestamp)
435{
436 /* Version 1 asset indices are always stored in this path by RemoteAssetListingDownloader. */
437 constexpr const char *asset_index_relpath = "_v1/asset-index.processed.json";
438
439 char asset_index_abspath[FILE_MAX];
440 BLI_path_join(asset_index_abspath,
441 sizeof(asset_index_abspath),
442 listing_root_dirpath.c_str(),
443 asset_index_relpath);
444
445 if (ignore_before_timestamp) {
446 std::optional<bool> is_older = file_older_than_timestamp(asset_index_abspath,
447 *ignore_before_timestamp);
448 if (!is_older) {
450 fmt::format(N_("Couldn't find index file {:s}"), asset_index_abspath));
451 }
452 /* TODO the .processed.json file doesn't get touched by the downloader to indicate it's up to
453 * date. Should this be done, or should we just note compare the timestamps for meta-files? The
454 * downloader notifies about them being in place already anyway. */
455 // if (*is_older) {
456 // CLOG_ERROR(&LOG, "Index file too old %s\n", asset_index_abspath);
457 // return {};
458 // }
459 }
460
461 const std::optional<AssetLibraryListingV1> listing = AssetLibraryListingV1::read(
462 asset_index_abspath);
463 if (!listing) {
465 fmt::format(N_("Couldn't read V1 listing from {:s}"), asset_index_abspath));
466 }
467
468 Set<StringRef> done_pages;
469 char filepath[FILE_MAX];
470
471 // TODO should we have some timeout here too? Like timeout after 30 seconds without a new page?
472
473 Vector<std::string> warnings;
474 while (true) {
475 for (const std::string &page_path : listing->page_rel_paths) {
476 if (done_pages.contains(page_path)) {
477 continue;
478 }
479
480 BLI_path_join(filepath, sizeof(filepath), listing_root_dirpath.c_str(), page_path.c_str());
481 if (wait_fn) {
482 if (!BLI_exists(filepath)) {
483 continue;
484 }
485 if (ignore_before_timestamp &&
486 file_older_than_timestamp(filepath, *ignore_before_timestamp).value_or(true))
487 {
488 CLOG_DEBUG(&LOG, "Ignoring old listing file %s - waiting for a new version\n", filepath);
489 continue;
490 }
491 }
492
494 listing_root_dirpath, filepath, process_fn);
495 done_pages.add(page_path);
496 if (page_result.is_cancelled()) {
497 return page_result;
498 }
499 if (page_result.is_failure()) {
500 printf("Couldn't read V1 listing from %s%c%s: %s\n",
501 listing_root_dirpath.c_str(),
502 SEP,
503 page_path.c_str(),
504 page_result.failure_reason.c_str());
505 return page_result;
506 }
507 BLI_assert(page_result.is_success());
508
509 /* Gather per-page warnings into the overall result. */
510 if (page_result.has_warnings()) {
511 warnings.extend(std::move(page_result.warnings));
512 }
513 }
514
515 BLI_assert(done_pages.size() <= listing->page_rel_paths.size());
516 if (done_pages.size() >= listing->page_rel_paths.size()) {
517 break;
518 }
519 if (!wait_fn) {
520 break;
521 }
522 if (!wait_fn()) {
524 }
525 }
526
527 /* Return a success, with all the warnings. */
529 result.warnings.extend(std::move(warnings));
530 return result;
531}
532
533} // namespace blender::ed::asset::index
#define BLENDER_VERSION
File and directory operations.
#define FILE_MAX
#define BLI_path_join(...)
#define CLOG_DEBUG(clg_ref,...)
Definition CLG_log.h:191
#define BLI_assert_msg(a, msg)
const Value * lookup_ptr(const Key &key) const
Definition BLI_map.hh:508
bool add_overwrite(const Key &key, const Value &value)
Definition BLI_map.hh:325
int64_t size() const
Definition BLI_set.hh:606
bool contains(const Key &key) const
Definition BLI_set.hh:310
bool add(const Key &key)
Definition BLI_set.hh:248
constexpr const char * c_str() const
void append(const T &value)
void extend(Span< T > array)
constexpr const char * end() const
constexpr const char * begin() const
static constexpr int64_t not_found
void copy_utf8_truncated(char *dst, int64_t dst_size) const
Definition string_ref.cc:28
asset_system::RemoteAssetFileStatus remote_file_status(RemoteListingFileEntry &file_to_check)
Span< std::shared_ptr< Value > > elements() const
std::optional< StringRefNull > lookup_str(StringRef key) const
Definition serialize.cc:275
std::optional< int64_t > lookup_int(StringRef key) const
Definition serialize.cc:285
const DictionaryValue * lookup_dict(StringRef key) const
Definition serialize.cc:315
const ArrayValue * lookup_array(StringRef key) const
Definition serialize.cc:323
const std::string & value() const
const StringValue * as_string_value() const
Definition serialize.cc:16
#define str(s)
float dot(VecOp< float, D >, VecOp< float, D >) RET
#define LOG(level)
Definition log.h:97
void hash(const IDProperty &base_prop, XXH3_state_t *hash_state)
Definition idprop.cc:2251
ReadingResult read_remote_listing_v1(StringRefNull listing_root_dirpath, RemoteListingEntryProcessFn process_fn, RemoteListingWaitForPagesFn wait_fn=nullptr, const std::optional< Timestamp > ignore_before_timestamp=std::nullopt)
static ReadingResult< bool > blender_version_matches(const DictionaryValue &asset_dictionary)
FunctionRef< bool(RemoteListingAssetEntry &)> RemoteListingEntryProcessFn
FunctionRef< bool()> RemoteListingWaitForPagesFn
std::optional< bool > file_older_than_timestamp(const char *filepath, Timestamp timestamp)
static ReadingResult listing_entries_from_root(const StringRefNull listing_root_dirpath, const DictionaryValue &value, const RemoteListingEntryProcessFn process_fn)
AssetMetaData * asset_metadata_from_dictionary(const io::serialize::DictionaryValue &entry)
static ReadingResult< RemoteListingFileEntry > listing_file_from_asset_dictionary(const DictionaryValue &dictionary)
static std::optional< int > blender_version_from_string(const blender::StringRef str)
static ReadingResult< RemoteListingAssetEntry > listing_entry_from_asset_dictionary(const DictionaryValue &dictionary, const Map< std::string, RemoteListingFileEntry > &file_path_to_entry_map)
std::optional< asset_system::URLWithHash > parse_url_with_hash_dict(const DictionaryValue *url_with_hash_dict)
std::unique_ptr< io::serialize::Value > read_contents(StringRefNull filepath)
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
__int64 int64_t
Definition BKE_text.h:90
bool BLI_exists(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL()
Definition storage.cc:404
const char * BKE_idtype_name_normalize(const char *idtype_name)
Definition idtype.cc:296
AssetMetaData * BKE_asset_metadata_create()
Definition asset.cc:31
static CLG_LogRef LOG
short BKE_idtype_idcode_from_name(const char *idtype_name)
Definition idtype.cc:282
ATTR_WARN_UNUSED_RESULT const void * element
constexpr char SEP
std::optional< URLWithHash > preview_url
static ReadingResult read_asset_entries(const StringRefNull listing_root_dirpath, const StringRefNull filepath, RemoteListingEntryProcessFn process_fn)
static std::optional< AssetLibraryListingV1 > read(const StringRefNull listing_filepath)
asset_system::RemoteAssetFileStatus remote_file_status
#define N_(msgid)