Blender V5.3
filelist_readjob_asset_library_remote.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2007 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#include <thread>
10
11#include "AS_asset_library.hh"
13#include "AS_remote_library.hh"
14
15#include "BLI_assert.hh"
16#include "BLI_listbase.hh"
17#include "BLI_map.hh"
18#include "BLI_path_utils.hh"
19#include "BLI_string.hh"
20#include "BLI_string_utils.hh"
21
22#include "BKE_global.hh"
23#include "BKE_idtype.hh"
24#include "BKE_preferences.h"
25
26#include "ED_asset_indexer.hh"
27
28#include "filelist_intern.hh"
29#include "filelist_readjob.hh"
30
31namespace blender {
32
34
35/* TODO handle \a progress. */
42 FileListReadJob *job_params,
43 RemoteLibraryRequest &request,
44 const bool *stop,
45 bool *do_update,
46 float * /*progress*/,
48{
49 using namespace ed::asset;
50
51 FileList *filelist = job_params->tmp_filelist; /* Use the thread-safe filelist queue. */
52
53 char dirpath[FILE_MAX];
54 StringRef(request.dirpath).copy_utf8_truncated(dirpath);
55
56 BLI_path_normalize_dir(dirpath, sizeof(dirpath));
57 if (!BLI_is_dir(dirpath)) {
58 return;
59 }
60
61 /* #index::read_remote_listing() below calls this for every asset entry it finished reading from
62 * the asset listing pages. */
63 const auto process_asset_fn = [&](index::RemoteListingAssetEntry &entry) {
64 if (*stop || request.cancel) {
65 /* Cancel reading when requested. */
66 return false;
67 }
68
69 const char *group_name = BKE_idtype_idcode_to_name(entry.idcode);
70
71 /* For assets already on disk, stamp the listing's file status onto the existing
72 * AssetRepresentation so the UI can reflect whether it is up to date. */
73 std::string asset_file = entry.online_info.asset_file();
74 /* Relative identifiers use native slashes, but #asset_system::OnlineAssetInfo::asset_file()
75 * has forward slashes instead, so they need to be converted to native. */
76 BLI_path_slash_native(asset_file.data());
77 {
78 BLI_assert(asset_file.ends_with(".blend"));
79
80 /* Matches #asset_system::AssetRepresentation.library_relative_identifier(). */
81 char asset_identifier[FILE_MAX_LIBEXTRA];
82 BLI_string_join(asset_identifier,
83 sizeof(asset_identifier),
84 asset_file.c_str(),
85 SEP_STR,
86 group_name,
87 SEP_STR,
88 entry.datablock_info.name);
89 if (asset_system::AssetRepresentation **existing = already_downloaded_assets.lookup_ptr(
90 asset_identifier))
91 {
92 if (entry.remote_file_status == asset_system::RemoteAssetFileStatus::NO_MATCH) {
93 /* Store the full online info so the asset can be re-downloaded. */
94 (*existing)->online_info_set(std::move(entry.online_info));
95 }
96 (*existing)->remote_file_status_set(entry.remote_file_status);
97 *do_update = true;
98 return true;
99 }
100 }
101
102 ListBaseT<FileListInternEntry> entries = {nullptr};
103
104 STRNCPY(job_params->cur_relbase, asset_file.c_str());
106 &entries,
107 &entry.datablock_info,
108 true,
109 entry.idcode,
110 group_name,
111 entry.online_info);
112
113 int entries_num = 0;
114 for (FileListInternEntry &entry : entries) {
115 entry.uid = filelist_uid_generate(filelist);
116 char dir[FILE_MAX_LIBEXTRA];
117 entry.name = fileentry_uiname(dirpath, &entry, dir);
118 entry.free_name = true;
119 entries_num++;
120 }
121
122 if (filelist_readjob_append_entries(job_params, &entries, entries_num)) {
123 *do_update = true;
124 }
125 return true;
126 };
127 /* A busy wait function for while asset listing pages are being downloaded.
128 * #index::read_remote_listing() calls this every time it's done looking for new pages, until all
129 * pages are there (or until this returns false). */
130 const auto wait_for_pages_fn = [&]() {
131 while (true) {
132 if (*stop || request.cancel) {
133 return false;
134 }
135
136 /* Atomically test and reset the new pages flag. */
137 if (request.new_pages_available.exchange(false)) {
138 /* New pages available or loading ended. Done waiting. */
139 return true;
140 }
141
142 if (!request.is_downloading) {
143 /* Nothing is downloading any more, so it doesn't make sense for the caller to continue
144 * looping. */
145 return false;
146 }
147
148 /* Busy waiting for new files, with some sleeping to avoid wasting a lot of CPU
149 * cycles. */
150 std::this_thread::sleep_for(std::chrono::milliseconds(10));
151 }
152 };
153
154 if (!index::read_remote_listing(dirpath,
155 job_params->load_asset_library->name(),
156 job_params->reports,
157 process_asset_fn,
158 wait_for_pages_fn,
159 request.request_time))
160 {
161 return;
162 }
163}
164
165/* Used by the remote library loading job and the "All" library. */
167 RemoteLibraryRequest &request,
168 bool *stop,
169 bool *do_update,
170 float *progress)
171{
172 FileList *filelist = job_params->tmp_filelist; /* Use the thread-safe filelist queue. */
173
174 /* Mapping from asset library-relative identifier to its AssetRepresentation. */
176
177 /* Get assets that were downloaded already. */
178 {
179 job_params->on_asset_added =
180 [&already_downloaded_assets](asset_system::AssetRepresentation &asset) {
181 already_downloaded_assets.add(asset.library_relative_identifier(), &asset);
182 };
183
184 float progress_on_disk = 0.0;
185
186 filelist_readjob_recursive_dir_add_items(true, job_params, stop, do_update, &progress_on_disk);
187 job_params->on_asset_added = std::nullopt;
188
189 /* A bit arbitrary: Let on-disk reading only take up to 10% of the total progress. We don't
190 * have enough data here to make a more informed choice. But practically the downloading is
191 * probably the bigger bottleneck than the listing of already downloaded assets directly from
192 * disk. For assets on disk there's the local asset index anyway, so listing them should be
193 * fast. Plus, giving 90% to the remaining work can make it feel like there's more steady
194 * progress towards the end, which is nicer for users. */
195 *progress = progress_on_disk * 0.1f;
196 }
197
198 BLI_assert(job_params->load_asset_library &&
200
201 while (request.is_downloading && !request.metafiles_in_place) {
202 /* Busy waiting for the metafiles, with some sleeping to avoid wasting a lot of CPU
203 * cycles. */
204 std::this_thread::sleep_for(std::chrono::milliseconds(20));
205
206 if (*stop || request.cancel) {
207 return;
208 }
209 }
210
211 if ((filelist->flags & FL_ASSETS_INCLUDE_ONLINE) == 0) {
212 return;
213 }
214
215 /* Enforce latest catalogs from the downloader to be used. */
217
218 if (*stop || request.cancel) {
219 return;
220 }
221
223 job_params, request, stop, do_update, progress, already_downloaded_assets);
224}
225
227 StringRef remote_url)
228{
229 /* On timeout the loading status will be set to cancelled. */
231 request.cancel = true;
232 }
233
234 const auto last_new_pages_time = RemoteLibraryLoadingStatus::last_new_pages_time(remote_url);
235 if (last_new_pages_time && *last_new_pages_time != request.last_new_pages_time) {
236 request.new_pages_available = true;
237 request.last_new_pages_time = *last_new_pages_time;
238 }
241 request.metafiles_in_place =
242 RemoteLibraryLoadingStatus::metafiles_in_place(remote_url).value_or(false);
243}
244
245/* Called when starting the job (from the main thread). */
248{
249 if (!USER_EXPERIMENTAL_TEST(&U, use_remote_asset_libraries)) {
250 return;
251 }
252 if ((G.f & G_FLAG_INTERNET_ALLOW) == 0) {
253 return;
254 }
255 if ((job_params->filelist->flags & FL_ASSETS_INCLUDE_ONLINE) == 0) {
256 return;
257 }
258
259 /* Check if the library's cache directory exists, otherwise, request download. */
260 const std::string top_meta_file_path = remote_library_top_meta_file_path(library);
261 if (!BLI_is_file(top_meta_file_path.c_str())) {
263 }
264
265 std::unique_ptr<RemoteLibraryRequest> request = std::make_unique<RemoteLibraryRequest>();
266 request->dirpath = library.cache_dirpath;
267 request->request_time = RemoteLibraryLoadingStatus::loading_start_time(library.remote_url);
268
270
271 job_params->remote_library_requests.add(library.remote_url, std::move(request));
272}
273
275{
278 if (!library || !(library->flag & ASSET_LIBRARY_USE_REMOTE_URL)) {
279 return nullptr;
280 }
281
282 return library;
283}
284
286{
289 }
290}
291
293 bool *stop,
294 bool *do_update,
295 float *progress)
296{
297 FileList *filelist = job_params->tmp_filelist; /* Use the thread-safe filelist queue. */
298
299 BLI_assert(filelist->filelist.entries.is_empty() &&
301
302 /* A valid, but empty file-list from now. */
303 filelist->filelist.entries_num = 0;
304
305 filelist_readjob_load_asset_library_data(job_params, do_update);
306
307 /* There are no online requests when internet access is disabled or the online assets filter is
308 * disabled. */
309 if (job_params->remote_library_requests.is_empty()) {
310 /* Only get assets that were downloaded already. */
311 filelist_readjob_recursive_dir_add_items(true, job_params, stop, do_update, progress);
312 return;
313 }
314
315 BLI_assert_msg(job_params->remote_library_requests.size() == 1,
316 "reading callback for a single remote library should only have a single remote "
317 "library request registered (check what the starting callback is requesting)");
318 for (auto [url, request] : job_params->remote_library_requests.items()) {
319 remote_asset_library_load(job_params, *request, stop, do_update, progress);
320 break;
321 }
322}
323
324/* This may also be called for the "All" asset library. */
326{
327 for (auto [url, request] : job_params->remote_library_requests.items()) {
329 }
330}
331
343
344} // namespace blender
Main runtime representation of an asset.
#define FILE_MAX
#define BLI_string_join(...)
#define FILE_MAX_LIBEXTRA
#define USER_EXPERIMENTAL_TEST(userdef, member)
#define BLI_assert_msg(a, msg)
const Value * lookup_ptr(const Key &key) const
Definition BLI_map.hh:508
bool add(const Key &key, const Value &value)
Definition BLI_map.hh:295
constexpr StringRef()
eAssetLibraryType library_type() const
static std::optional< bool > metafiles_in_place(StringRef url)
static std::optional< RemoteLibraryLoadingStatus::Status > status(StringRef url)
static std::optional< TimePoint > last_new_pages_time(StringRef url)
static std::optional< FileSystemTimePoint > loading_start_time(const StringRef url)
#define FILEDIR_NBR_ENTRIES_UNSET
void remote_library_request_download(const RemoteLibraryDefinitionRef &library_definition)
std::string remote_library_top_meta_file_path(const RemoteLibraryDefinitionRef &library)
bool read_remote_listing(StringRefNull root_dirpath, StringRefNull asset_library_name, ReportList &reports, RemoteListingEntryProcessFn process_fn, RemoteListingWaitForPagesFn wait_fn=nullptr, std::optional< Timestamp > ignore_before_timestamp=std::nullopt)
BMesh Iterators.
const char * BKE_idtype_idcode_to_name(short idcode)
Definition idtype.cc:261
UserDef U
Definition blender.cc:58
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
static void filelist_start_job_remote_asset_library(FileListReadJob *job_params)
static void filelist_remote_asset_library_update_loading_flags(RemoteLibraryRequest &request, StringRef remote_url)
char * STRNCPY(char(&dst)[N], const char *src)
static void filelist_readjob_remote_asset_library(FileListReadJob *job_params, bool *stop, bool *do_update, float *progress)
@ FL_ASSETS_INCLUDE_ONLINE
@ ASSET_LIBRARY_USE_REMOTE_URL
int BLI_path_normalize_dir(char *dir, size_t dir_maxncpy) ATTR_NONNULL(1)
void remote_asset_library_request(FileListReadJob *job_params, const asset_system::RemoteLibraryDefinitionRef &library)
struct bUserAssetLibrary * BKE_preferences_asset_library_find_index(const struct UserDef *userdef, int index) ATTR_NONNULL() ATTR_WARN_UNUSED_RESULT
bool is_filtered_asset_library(FileListInternEntry *file, const char *root, FileListFilter *filter)
void prepare_filter_asset_library(const FileList *filelist, FileListFilter *filter)
void BLI_path_slash_native(char *path) ATTR_NONNULL(1)
void remote_asset_library_load(FileListReadJob *job_params, RemoteLibraryRequest &request, bool *stop, bool *do_update, float *progress)
Global G
Definition blender.cc:57
bool filelist_checkdir_return_always_valid(const FileList *, char[1024+258], const bool)
void filelist_set_readjob_remote_asset_library(FileList *filelist)
void filelist_readjob_load_asset_library_data(FileListReadJob *job_params, bool *do_update)
static void filelist_readjob_remote_asset_library_index_read(FileListReadJob *job_params, RemoteLibraryRequest &request, const bool *stop, bool *do_update, float *, Map< StringRef, asset_system::AssetRepresentation * > &already_downloaded_assets)
const char * fileentry_uiname(const char *root, FileListInternEntry *entry, char *buff)
Definition filelist.cc:1073
void filelist_readjob_list_lib_add_datablock(FileListReadJob *job_params, ListBaseT< FileListInternEntry > *entries, BLODataBlockInfo *datablock_info, const bool prefix_relpath_with_group_name, const int idcode, const char *group_name, const std::optional< asset_system::OnlineAssetInfo > online_asset_info=std::nullopt)
@ FILELIST_TAGS_APPLY_FUZZY_SEARCH
bool BLI_is_dir(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL()
Definition storage.cc:457
void filelist_timer_step_remote_asset_library(FileListReadJob *job_params)
FileUID filelist_uid_generate(FileList *filelist)
Definition filelist.cc:1422
@ G_FLAG_INTERNET_ALLOW
void filelist_readjob_recursive_dir_add_items(const bool do_lib, FileListReadJob *job_params, const bool *stop, bool *do_update, float *progress)
static bUserAssetLibrary * lookup_remote_library(const FileListReadJob *job_params)
bool BLI_is_file(const char *path) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL()
Definition storage.cc:469
bool filelist_readjob_append_entries(FileListReadJob *job_params, ListBaseT< FileListInternEntry > *from_entries, int from_entries_num)
ListBaseT< struct FileListInternEntry > entries
asset_system::AssetLibrary * load_asset_library
std::optional< std::function< void(asset_system::AssetRepresentation &)> > on_asset_added
Map< std::string, std::unique_ptr< RemoteLibraryRequest > > remote_library_requests
void(*) prepare_filter_fn(const FileList *filelist, FileListFilter *filter)
void(*) timer_step_fn(FileListReadJob *job_params)
bool(*) check_dir_fn(const FileList *filelist, char dirpath[FILE_MAX_LIBEXTRA], const bool do_change)
void(*) start_job_fn(FileListReadJob *job_params)
AssetLibraryReference * asset_library_ref
void(*) read_job_fn(FileListReadJob *job_params, bool *stop, bool *do_update, float *progress)
FileDirEntryArr filelist
bool(*) filter_fn(FileListInternEntry *file, const char *root, FileListFilter *filter)
std::atomic< bool > metafiles_in_place
std::atomic< bool > is_downloading
asset_system::RemoteLibraryLoadingStatus::TimePoint last_new_pages_time
std::optional< asset_system::RemoteLibraryLoadingStatus::FileSystemTimePoint > request_time
std::atomic< bool > new_pages_available
#define SEP_STR
Definition unit.cc:41