Blender V5.3
asset_indexer_remote_listing.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2024 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#include <chrono>
10#include <filesystem>
11#include <fmt/format.h>
12#include <fstream>
13#include <optional>
14#include <string>
15
16#include "BLI_fileops.hh"
17#include "BLI_path_utils.hh"
18#include "BLI_serialize.hh"
19
20#include "BKE_report.hh"
21
22#include "BLT_translation.hh"
23
24#include "CLG_log.h"
25
26#include "ED_asset_indexer.hh"
27#include "asset_index.hh"
29
30static CLG_LogRef LOG = {"asset.remote_listing"};
31
33
34using namespace blender::io::serialize;
35
36/* -------------------------------------------------------------------- */
39
41{
42 this->datablock_info = other.datablock_info;
43 other.datablock_info = {};
44 this->idcode = other.idcode;
45
46 this->online_info = std::move(other.online_info);
47}
48
50{
51 if (this == &other) {
52 return *this;
53 }
54 std::destroy_at(this);
55 new (this) RemoteListingAssetEntry(std::move(other));
56 return *this;
57}
58
63
65
66/* -------------------------------------------------------------------- */
69
70std::unique_ptr<Value> read_contents(const StringRefNull filepath)
71{
72 JsonFormatter formatter;
73 std::ifstream is;
74 is.open(filepath.c_str());
75 BLI_SCOPED_DEFER([&]() { is.close(); });
76
77 return formatter.deserialize(is);
78}
79
80std::optional<asset_system::URLWithHash> parse_url_with_hash_dict(
81 const DictionaryValue *url_with_hash_dict)
82{
83 if (!url_with_hash_dict) {
84 return {};
85 }
86
87 const std::optional<StringRefNull> url = url_with_hash_dict->lookup_str("url");
88 const std::optional<StringRefNull> hash = url_with_hash_dict->lookup_str("hash");
89
90 /* A URL without hash is not up to spec, but we can work with it. But without
91 * a URL it's hopeless. */
92 if (!url) {
93 return {};
94 }
95
96 return asset_system::URLWithHash{*url, hash.value_or("")};
97}
98
100
101/* -------------------------------------------------------------------- */
107
111
112 static ReadingResult<AssetLibraryMeta> read(const StringRefNull root_dirpath,
113 std::optional<Timestamp> ignore_before_timestamp);
114};
115
120std::optional<bool> file_older_than_timestamp(const char *filepath, Timestamp timestamp)
121{
122 std::error_code error;
123 Timestamp file_timestamp = std::filesystem::last_write_time(filepath, error);
125 if (error) {
126 printf("Can't find file at path %s: %s\n", filepath, error.message().c_str());
127 return {};
128 }
129
130 return file_timestamp < timestamp;
131}
132
137 const StringRefNull root_dirpath, const std::optional<Timestamp> ignore_before_timestamp)
138{
139 char filepath[FILE_MAX];
140 BLI_path_join(filepath,
141 sizeof(filepath),
142 root_dirpath.c_str(),
144
145 if (!BLI_exists(filepath)) {
147 fmt::format(N_("file does not exist: {:s}"), filepath));
148 }
149
150 if (ignore_before_timestamp) {
151 std::optional<bool> is_older = file_older_than_timestamp(filepath, *ignore_before_timestamp);
152 if (!is_older) {
154 fmt::format(N_("file does not exist: {:s}"), filepath));
155 }
156 if (*is_older) {
158 fmt::format(N_("file is too old: {:s}"), filepath));
159 }
160 }
161
162 const std::unique_ptr<Value> contents = read_contents(filepath);
163 if (!contents) {
165 fmt::format(N_("file does not contain JSON: {:s}"), filepath));
166 }
167
168 const DictionaryValue *root = contents->as_dictionary_value();
169 if (!root) {
171 fmt::format(N_("file is not a JSON dictionary: {:s}"), filepath));
172 }
173
174 const DictionaryValue *entries = root->lookup_dict("api_versions");
175 BLI_assert(entries != nullptr);
176 if (entries == nullptr) {
178 fmt::format(N_("no API versions defined: {:s}"), filepath));
179 }
180
181 AssetLibraryMeta library_meta;
182
183 for (const DictionaryValue::Item &version : entries->elements()) {
184 /* Relative path to the listing meta-file (e.g. `_v1/asset-index.json`). */
185 const DictionaryValue *index_path_info = version.second->as_dictionary_value();
186 if (!index_path_info) {
187 CLOG_WARN(&LOG,
188 "Error reading asset listing API version '%s' in %s - ignoring",
189 version.first.c_str(),
190 filepath);
191 continue;
192 }
193
194 std::optional<asset_system::URLWithHash> url_with_hash = parse_url_with_hash_dict(
195 index_path_info);
196 if (!url_with_hash) {
197 CLOG_WARN(&LOG,
198 "Error reading asset listing API version '%s' in %s, no URL+hash found - ignoring",
199 version.first.c_str(),
200 filepath);
201 continue;
202 }
203 library_meta.api_versions.add(version.first, std::move(*url_with_hash));
204 }
205
206 return ReadingResult<AssetLibraryMeta>::Success(std::move(library_meta));
207}
208
210
218
220{
221 /* API versions this version of Blender can handle, in descending order (most preferred to least
222 * preferred order). */
223 const Vector<std::pair<uint, StringRefNull>> readable_versions = {
224 {1, "v1"},
225 };
226
227 for (const auto &[version_nr, version_str] : readable_versions) {
228 if (const asset_system::URLWithHash *url_with_hash = library_meta.api_versions.lookup_ptr(
229 version_str))
230 {
231 ApiVersionInfo version_info{version_nr, url_with_hash->url, url_with_hash->hash};
232 return ReadingResult<ApiVersionInfo>::Success(std::move(version_info));
233 }
234 }
235
237 N_("remote does not offer an API version supported by this version of Blender"));
238}
239
240bool read_remote_listing(const StringRefNull root_dirpath,
241 const StringRefNull asset_library_name,
242 ReportList &reports,
243 const RemoteListingEntryProcessFn process_fn,
244 const RemoteListingWaitForPagesFn wait_fn,
245 const std::optional<Timestamp> ignore_before_timestamp)
246{
247 /* This actually does the work, and returns a ReadingResult. It's implemented as a lambda
248 * function, to be able to use early returns on error. */
249 auto get_result = [&]() {
251 ignore_before_timestamp);
252 if (!meta.is_success()) {
253 return meta.without_success_value();
254 }
255
256 const ReadingResult<ApiVersionInfo> api_version_info = choose_api_version(*meta);
257 if (!api_version_info.is_success()) {
258 return api_version_info.without_success_value();
259 }
260
261 /* Path to the listing meta-file is version-dependent. */
262 switch (api_version_info->version_nr) {
263 case 1: {
264 return read_remote_listing_v1(root_dirpath, process_fn, wait_fn, ignore_before_timestamp);
265 }
266 default:
267 /* choose_api_version() should not have chosen this version. */
269 return ReadingResult<>::Failure(N_("internal error, please report a bug"));
270 }
271 };
272
273 const ReadingResult<> result = get_result();
274
275 /* Get these messages up-stream. The last call to BKE_report(f) will be the one shown in the
276 * status bar. The rest are just printed to the terminal and gathered at the Info editor. */
277 if (result.is_failure()) {
278 BKE_reportf(&reports,
279 RPT_ERROR,
280 "Asset Library '%s': %s",
281 asset_library_name.c_str(),
282 RPT_(result.failure_reason.c_str()));
283 BKE_reportf(&reports,
284 RPT_ERROR,
285 "Could not read asset listing '%s', see Info Editor for details",
286 asset_library_name.c_str());
287 return false;
288 }
289 if (result.is_cancelled()) {
290 return false;
291 }
292 if (result.has_warnings()) {
293 for (const std::string &warning : result.warnings) {
294 BKE_reportf(&reports,
296 "Asset Library '%s': %s",
297 asset_library_name.c_str(),
298 RPT_(warning.c_str()));
299 }
300 BKE_reportf(&reports,
302 "Could not read asset listing for '%s', see Info Editor for details",
303 asset_library_name.c_str());
304 }
305 return true;
306}
307
308} // namespace blender::ed::asset::index
File and directory operations.
#define BLI_SCOPED_DEFER(function_to_defer)
#define FILE_MAX
#define BLI_path_join(...)
#define RPT_(msgid)
#define CLOG_WARN(clg_ref,...)
Definition CLG_log.h:189
#define BLI_assert_unreachable()
constexpr const char * c_str() const
ReadingResult without_success_value() const
std::optional< StringRefNull > lookup_str(StringRef key) const
Definition serialize.cc:275
std::pair< std::string, std::shared_ptr< Value > > Item
const DictionaryValue * lookup_dict(StringRef key) const
Definition serialize.cc:315
std::unique_ptr< Value > deserialize(std::istream &is) override
Definition serialize.cc:382
#define LOG(level)
Definition log.h:97
constexpr StringRefNull REMOTE_LIBRARY_TOP_META_FILE_NAME
void hash(const IDProperty &base_prop, XXH3_state_t *hash_state)
Definition idprop.cc:2251
static ReadingResult< ApiVersionInfo > choose_api_version(const AssetLibraryMeta &library_meta)
ReadingResult read_remote_listing_v1(StringRefNull listing_root_dirpath, RemoteListingEntryProcessFn process_fn, RemoteListingWaitForPagesFn wait_fn=nullptr, const std::optional< Timestamp > ignore_before_timestamp=std::nullopt)
std::filesystem::file_time_type Timestamp
FunctionRef< bool(RemoteListingAssetEntry &)> RemoteListingEntryProcessFn
FunctionRef< bool()> RemoteListingWaitForPagesFn
std::optional< bool > file_older_than_timestamp(const char *filepath, Timestamp timestamp)
bool read_remote_listing(StringRefNull root_dirpath, StringRefNull asset_library_name, ReportList &reports, RemoteListingEntryProcessFn process_fn, RemoteListingWaitForPagesFn wait_fn=nullptr, std::optional< Timestamp > ignore_before_timestamp=std::nullopt)
std::optional< asset_system::URLWithHash > parse_url_with_hash_dict(const DictionaryValue *url_with_hash_dict)
std::unique_ptr< io::serialize::Value > read_contents(StringRefNull filepath)
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
@ RPT_WARNING
Definition BKE_report.hh:40
unsigned int uint
Definition BKE_text.h:55
bool BLI_exists(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL()
Definition storage.cc:404
void BLO_datablock_info_free(BLODataBlockInfo *datablock_info)
void BKE_reportf(ReportList *reports, eReportType type, const char *format,...) ATTR_PRINTF_FORMAT(3
Definition report.cc:191
static ReadingResult< AssetLibraryMeta > read(const StringRefNull root_dirpath, std::optional< Timestamp > ignore_before_timestamp)
Map< std::string, asset_system::URLWithHash > api_versions
RemoteListingAssetEntry & operator=(RemoteListingAssetEntry &&)
#define N_(msgid)