Blender V5.3
remote_library.cc
Go to the documentation of this file.
1/* SPDX-FileCopyrightText: 2023 Blender Authors
2 *
3 * SPDX-License-Identifier: GPL-2.0-or-later */
4
8
9#include <fmt/format.h>
10
11#include "BLI_assert.hh"
12#include "BLI_fileops.hh"
13#include "BLI_hash_md5.hh"
14#include "BLI_listbase.hh"
15#include "BLI_path_utils.hh"
16#include "BLI_string.hh"
17#include "BLI_string_ref.hh"
18#include "BLI_threads.hh"
19
20#include "BLT_translation.hh"
21
22#include "BKE_appdir.hh"
23#include "BKE_context.hh"
24#include "BKE_global.hh"
25#include "BKE_idprop.hh"
26#include "BKE_preferences.h"
27#include "BKE_report.hh"
28
29#ifdef WITH_PYTHON
30# include "BPY_extern_run.hh"
31#endif
32
33#include "CLG_log.h"
34
35#include "DNA_asset_types.h"
36#include "DNA_space_enums.h"
37#include "DNA_userdef_types.h"
38
39#include "ED_asset.hh"
40#include "ED_fileselect.hh"
41#include "ED_render.hh"
42
43#include "RNA_access.hh"
44
45#include "WM_api.hh"
46#include "WM_message.hh"
47
50#include "AS_remote_library.hh"
51
52#include "remote_library.hh"
53
54static CLG_LogRef LOG = {"asset.remote_library"};
55
63#define AUTH_TOKEN_MAYBE_OVERRIDE_FROM_EXTENSIONS \
64 "from _bpy_internal.extensions.asset_library_query import token_from_asset_url\n" \
65 "if (auth_token_override := token_from_asset_url(library_url)) is not None:\n" \
66 " auth_token = auth_token_override\n"
67
68namespace blender::asset_system {
69
71 : remote_url(library_definition.remote_url),
72 cache_dirpath(library_definition.dirpath),
73 auth_token(library_definition.auth_token ?
74 std::optional<StringRefNull>(library_definition.auth_token) :
75 std::nullopt)
76{
77 BLI_assert((library_definition.flag & ASSET_LIBRARY_USE_REMOTE_URL) != 0);
78}
79
80/* -------------------------------------------------------------------- */
85
95
101
102std::optional<eAssetImportMethod> RemoteAssetLibrary::import_method() const
103{
104 if (U.experimental.no_data_block_packing) {
106 }
107 return ASSET_IMPORT_PACK;
108}
109
110std::optional<StringRefNull> RemoteAssetLibrary::remote_url() const
111{
112 return remote_url_;
113}
114
119
121
122/* -------------------------------------------------------------------- */
125
127 const bUserAssetLibrary &custom_library)
129 /*is_read_only=*/true,
130 /*remote_url=*/custom_library.remote_url,
131 /*name=*/custom_library.name,
132 /*root_path=*/custom_library.dirpath),
133 user_library_(custom_library)
134{
137}
138
139std::optional<AssetLibraryReference> PreferencesRemoteAssetLibrary::library_reference() const
140{
141 const bUserAssetLibrary *library_definition = user_library_.user_asset_library();
142 if (library_definition == nullptr) {
143 return {};
144 }
145
146 const int index = BLI_findindex(&U.asset_libraries, library_definition);
147 if (index == -1) {
148 /* Should have been caught by the #user_asset_library() call above already. */
150 return {};
151 }
152
153 BLI_assert(library_definition->flag & ASSET_LIBRARY_USE_REMOTE_URL);
154 AssetLibraryReference library_ref{};
155 library_ref.type = ASSET_LIBRARY_CUSTOM;
156 library_ref.custom_library_index = index;
157 return library_ref;
158}
159
161{
162 return user_library_.user_asset_library();
163}
164
166{
167 const bUserAssetLibrary *library_definition = user_library_.user_asset_library();
168 if (!library_definition) {
169 return false;
170 }
171
172 return BKE_preferences_asset_library_is_available(&U, library_definition);
173}
174
175std::optional<StringRefNull> PreferencesRemoteAssetLibrary::auth_token() const
176{
177 const bUserAssetLibrary *library_definition = user_library_.user_asset_library();
178 if (library_definition == nullptr) {
179 return std::nullopt;
180 }
181
182 if (library_definition->auth_token == nullptr) {
183 return std::nullopt;
184 }
185
186 StringRefNull token(library_definition->auth_token);
187 if (token.is_empty()) {
188 return std::nullopt;
189 }
190
191 BLI_assert(library_definition->flag & ASSET_LIBRARY_USE_AUTH_TOKEN);
192 return token;
193}
194
195std::optional<eAssetImportMethod> PreferencesRemoteAssetLibrary::import_method() const
196{
197 const bUserAssetLibrary *library_definition = user_library_.user_asset_library();
198 if (!library_definition) {
199 return {};
200 }
201 if (!ELEM(library_definition->import_method,
205 {
207 return ASSET_IMPORT_PACK;
208 }
209
210 return eAssetImportMethod(library_definition->import_method);
211}
212
214
215/* -------------------------------------------------------------------- */
218
221 std::string library_url;
222 std::string file_request_url;
223
225 {
227 }
228 bool operator==(const RequestIdentifier &other) const
229 {
230 return library_url == other.library_url && file_request_url == other.file_request_url;
231 }
232};
233
238
267
272
275
277
279 static void file_requested(wmWindowManager &wm,
280 RequestIdentifier &&request,
281 std::string &&abs_url,
282 int64_t size_in_bytes);
283 static void file_report_progress(StringRef absolute_file_url, int64_t size_in_bytes);
285 static void file_finished(StringRef absolute_file_url, DownloadOutcome outcome);
286
288 static void on_all_finished(wmWindowManager &wm);
289
295 static bool is_any_loading();
296};
297
300
302 RequestIdentifier &&request,
303 std::string &&abs_url,
304 const int64_t size_in_bytes)
305{
307 abs_url, FileProgress{.expected_size_in_bytes = size_in_bytes});
308 /* Make the absolute URL known to the progress reporting, so we can query it later using the
309 * potentially relative URL that is known to the asset system. */
310 ProgressTracker::current.req_to_full_urls.add(std::move(request), std::move(abs_url));
311
313
316 &wm, nullptr, NC_WM | ND_JOB, 0.1);
317 }
318}
319
321 const int64_t size_in_bytes)
322{
323 if (FileProgress *progress = ProgressTracker::current.requested_files.lookup_ptr_as(
324 absolute_file_url))
325 {
326 progress->current_size_in_bytes = size_in_bytes;
327 }
328}
329
330void ProgressTracker::file_finished(const StringRef absolute_file_url,
331 const DownloadOutcome outcome)
332{
333 if (FileProgress *progress = ProgressTracker::current.requested_files.lookup_ptr_as(
334 absolute_file_url))
335 {
336 switch (outcome) {
338 /* The file is 'done', but shouldn't count towards any download progress any more. */
339 break;
341 ProgressTracker::current.done_files.add(absolute_file_url, *progress);
342 break;
343 }
344
345 /* Regardless of the outcome, the file is no longer downloading. */
346 ProgressTracker::current.requested_files.remove_contained_as(absolute_file_url);
347 }
348}
349
351{
352 /* Clear all progress data. */
357 }
358 /* Add notifier so job UIs redraw, and the progress/cancel buttons disappear. */
359 WM_event_add_notifier_ex(&wm, nullptr, NC_WM | ND_JOB, nullptr);
360}
361
366
368{
369 int64_t expected_bytes = 0;
370 int64_t current_bytes = 0;
371 for (const FileProgress &progress : ProgressTracker::current.requested_files.values()) {
372 expected_bytes += progress.expected_size_in_bytes;
373 current_bytes += progress.current_size_in_bytes;
374 }
375
376 for (const FileProgress &finished : ProgressTracker::current.done_files.values()) {
377 expected_bytes += finished.expected_size_in_bytes;
378 current_bytes += finished.expected_size_in_bytes;
379 }
380
381 if (!expected_bytes) {
382 return 0.0f;
383 }
384
385 const float progress = float(current_bytes) / expected_bytes;
386 BLI_assert(progress >= -0.0001f && progress <= 1.0001f);
387 return std::clamp(progress, 0.0f, 1.0f);
388}
389
394
396
397/* -------------------------------------------------------------------- */
400
401/*
402 * NOTE: Some of the status setters here only modify status if the current status is
403 * #RemoteLibraryLoadingStatus::Loading. That is done to avoid status changes after loading ended,
404 * mostly after a timeout. E.g. after a timeout of the C++ status because Python didn't send status
405 * updates, Python might eventually resume sending updates. These shouldn't affect the C++ status
406 * anymore, since the earlier failure aborted the C++ side asset library loading. Plus, the UI
407 * might show an error, and would suddenly switch to showing an incompletely loaded asset library.
408 */
409
411
413{
417 "Remote library status isn't synchronized and should only be accessed from the main thread");
418 return map;
419}
420
421void RemoteLibraryLoadingStatus::reset_timeout()
422{
423 this->last_updated_time_point_ = std::chrono::steady_clock::now();
424}
425
426void RemoteLibraryLoadingStatus::begin_loading(const StringRef url, const float timeout)
427{
428 BLI_assert(timeout > 0.0f);
429
430 RemoteLibraryLoadingStatus new_status{};
431 new_status.timeout_ = timeout;
432 new_status.status_ = RemoteLibraryLoadingStatus::Loading;
433 new_status.reset_timeout();
434 new_status.loading_start_time_point_ = FileSystemTimePoint::clock::now();
435 new_status.last_new_pages_time_point_ = std::chrono::steady_clock::now();
436 library_to_status_map().add_overwrite(url, new_status);
437}
438
440{
442 if (!this_) {
443 return;
444 }
445
446 if (this_->status_ == RemoteLibraryLoadingStatus::Loading) {
447 this_->reset_timeout();
448 }
449}
450
452{
454 if (!this_) {
455 return;
456 }
457
458 if (this_->status_ == RemoteLibraryLoadingStatus::Loading) {
459 this_->reset_timeout();
460 this_->last_new_pages_time_point_ = std::chrono::steady_clock::now();
461 }
462}
463
465 const StringRef preview_full_filepath)
466{
467 ED_preview_online_download_finished(CTX_wm_manager(&C), preview_full_filepath);
468}
469
471 const int64_t size_in_bytes)
472{
473 ProgressTracker::file_report_progress(absolute_file_url, size_in_bytes);
474}
475
477 const StringRef library_url,
478 const StringRef absolute_file_url,
479 const StringRef local_file_abspath,
480 const DownloadOutcome outcome)
481{
483
484 ed::asset::list::on_remote_assets_downloaded(*wm, library_url, local_file_abspath);
485 ProgressTracker::file_finished(absolute_file_url, outcome);
486
487 /* In case this download was triggered by a `Download` overlay button drawn over an asset button,
488 * add a mousemove event so the asset button becomes active and can be dragged right away. */
490}
491
493 const bContext &C,
494 const StringRef library_url,
495 const StringRef absolute_file_url,
496 const StringRef local_file_abspath)
497{
499 C, library_url, absolute_file_url, local_file_abspath, DownloadOutcome::Succeeded);
500}
501
503 const bContext &C,
504 const StringRef library_url,
505 const StringRef absolute_file_url,
506 const StringRef local_file_abspath)
507{
509 C, library_url, absolute_file_url, local_file_abspath, DownloadOutcome::Failed);
510}
511
516
518{
520 if (!this_) {
521 return;
522 }
523
524 this_->metafiles_in_place_ = true;
525}
526
527std::optional<RemoteLibraryLoadingStatus::Status> RemoteLibraryLoadingStatus::status(
528 const StringRef url)
529{
531 if (!this_) {
532 return {};
533 }
534
535 return this_->status_;
536}
537
539{
541 if (!this_) {
542 return {};
543 }
544
545 return this_->metafiles_in_place_;
546}
547
548std::optional<RemoteLibraryLoadingStatus::FileSystemTimePoint> RemoteLibraryLoadingStatus::
550{
552 if (!this_) {
553 return {};
554 }
555 if (this_->status_ != Loading) {
556 return {};
557 }
558
559 return this_->loading_start_time_point_;
560}
561
562std::optional<RemoteLibraryLoadingStatus::TimePoint> RemoteLibraryLoadingStatus::
564{
566 if (!this_) {
567 return {};
568 }
569
570 return this_->last_new_pages_time_point_;
571}
572
574{
576 if (!this_) {
577 return;
578 }
579
580 if (this_->status_ == RemoteLibraryLoadingStatus::Loading) {
582 this_->reset_timeout();
583 }
584}
585
587{
589 if (!this_) {
590 return;
591 }
592
593 if (this_->status_ == RemoteLibraryLoadingStatus::Loading) {
595 this_->reset_timeout();
596 }
597}
598
600 const std::optional<StringRefNull> failure_message)
601{
603 if (!this_) {
604 return;
605 }
606
607 if (this_->status_ == RemoteLibraryLoadingStatus::Loading) {
609 this_->failure_message_ = failure_message;
610 this_->reset_timeout();
611 }
612}
613
614std::optional<StringRefNull> RemoteLibraryLoadingStatus::failure_message(const StringRef url)
615{
617 if (!this_) {
618 return {};
619 }
620
621 if (this_->status_ == RemoteLibraryLoadingStatus::Failure) {
622 return this_->failure_message_;
623 }
624 return {};
625}
626
628{
630 if (!this_ || this_->status_ != RemoteLibraryLoadingStatus::Loading) {
631 /* Only handle timeouts while loading. */
632 return false;
633 }
634
635 const TimePoint now = std::chrono::steady_clock::now();
636 /* Keep track of how long ago the timeout was checked last. This is to avoid blocking processes
637 * from interfering with the timeout handling. If the timeout wasn't checked for a longer period
638 * of time (more than `0.9 * timeout_`), we skip timeout handling. */
639 if ((now - this_->last_timeout_handled_time_point_).count() > (0.9f * this_->timeout_)) {
640 return false;
641 }
642 this_->last_timeout_handled_time_point_ = now;
643
644 const std::chrono::duration<float> elapsed = now - this_->last_updated_time_point_;
645 if (elapsed.count() < this_->timeout_) {
646 return false;
647 }
648
650 this_->failure_message_ = RPT_("Asset system lost connection to downloader (timed out).");
651 return true;
652}
653
655
656/* -------------------------------------------------------------------- */
659
661{
662 BLI_assert_msg(BLI_thread_is_main(), "Calling into Python from a thread is not safe");
663 /* Ensure we don't attempt to download anything when online access is disabled. */
664 if ((G.f & G_FLAG_INTERNET_ALLOW) == 0) {
665 return;
666 }
667
668 if (!library_definition.remote_url[0]) {
669 return;
670 }
671
674 "The online essentials library must be downloaded to "
675 "online_essentials_cache_directory_path()");
676
677#ifdef WITH_PYTHON
678 /* Remote library is already downloading. */
679 if (RemoteLibraryLoadingStatus::status(library_definition.remote_url) ==
681 {
682 return;
683 }
684
685 /* Returns true if the directory exists, also if it pre-existed. */
686 if (!BLI_dir_create_recursive(library_definition.cache_dirpath.c_str())) {
687 return;
688 }
689
690 {
691 std::string script =
692 "import bl_pkg\n"
693 "from pathlib import Path\n"
695 "\n"
696 "bl_pkg.remote_asset_library_sync(\n"
697 " library_url, auth_token, Path(library_path)\n"
698 ")\n";
699
700 std::unique_ptr locals = bke::idprop::create_group("locals");
701 IDP_AddToGroup(locals.get(), IDP_NewString(library_definition.remote_url, "library_url"));
702 IDP_AddToGroup(locals.get(), IDP_NewString(library_definition.cache_dirpath, "library_path"));
703 IDP_AddToGroup(locals.get(),
704 IDP_NewString(library_definition.auth_token.value_or(""), "auth_token"));
705
706 /* TODO: report errors in the UI somehow. */
707 BPY_run_string_exec_with_locals(nullptr, script, *locals);
708 }
709#endif
710}
711
713{
714#ifdef WITH_PYTHON
715 constexpr const char *SCRIPT = R"(
716import bl_pkg
717
718bl_pkg.remote_asset_library_sync_cancel()
719 )";
720
721 std::unique_ptr locals = bke::idprop::create_group("locals");
722 BPY_run_string_exec_with_locals(const_cast<bContext *>(&C), SCRIPT, *locals);
723
724 for (StringRef remote_url : library_to_status_map().keys()) {
726 }
727#else
728 UNUSED_VARS(C);
729#endif
730}
731
732#ifdef WITH_PYTHON
738static std::optional<std::string> remote_library_request_asset_download_file(
739 const bContext &C,
740 ReportList *reports,
741 const StringRefNull asset_name,
743 const StringRefNull dst_filepath,
744 const URLWithHash &asset_url)
745{
746 BLI_assert(library.remote_url());
747
748 if (dst_filepath.is_empty()) {
750 reports,
752 "Asset listing does not indicate where the file should be downloaded to, for asset '%s'",
753 asset_name.c_str());
754 return std::nullopt;
755 }
756
757 /* Protect against maliciously constructed file paths. This code can just check & reject, as the
758 * actual sanitisation happens when the listing is downloaded (see `listing_downloader.py`). */
759 if (BLI_path_is_abs_from_cwd(dst_filepath.c_str())) {
760 /* Absolute file paths. */
761 BKE_reportf(reports,
762 RPT_ERROR,
763 "Asset '%s' references a file with an absolute path, which is not allowed",
764 asset_name.c_str());
765 return std::nullopt;
766 }
767
768 /* Check '..' entries, which can be "../" at the start of the path, or "/../" in the middle of
769 * the path. */
770 std::string path_native(dst_filepath);
771 BLI_path_slash_native(path_native.data());
772
773 static constexpr char slash_dot_dot_slash[]{SEP, '.', '.', SEP, '\0'};
774 static constexpr char const *dot_dot_slash = slash_dot_dot_slash + 1;
775
776 if (path_native.starts_with(dot_dot_slash) ||
777 path_native.find(slash_dot_dot_slash) != std::string::npos)
778 {
779 BKE_reportf(reports,
780 RPT_ERROR,
781 "Asset '%s' references a file with '..' in its path, which is not allowed",
782 asset_name.c_str());
783 return std::nullopt;
784 }
785
786 /* No need to check the URL. If it's empty, the Python code uses
787 * `dst_filepath` as URL, relative to the asset library URL. */
788
789 std::string script =
790 "import _bpy_internal.assets.remote_library.asset_downloader as asset_dl\n"
791 "from pathlib import Path\n"
793 "\n"
794 "_result = asset_dl.download_asset_file(\n"
795 " library_url, auth_token, Path(library_path),\n"
796 " asset_url, asset_hash, Path(dst_filepath),\n"
797 ")\n";
798
799 std::unique_ptr locals = bke::idprop::create_group("locals");
800 IDP_AddToGroup(locals.get(), IDP_NewString(*library.remote_url(), "library_url"));
801 IDP_AddToGroup(locals.get(), IDP_NewString(library.auth_token().value_or(""), "auth_token"));
802 IDP_AddToGroup(locals.get(), IDP_NewString(library.root_path(), "library_path"));
803 IDP_AddToGroup(locals.get(), IDP_NewString(dst_filepath, "dst_filepath"));
804 IDP_AddToGroup(locals.get(), IDP_NewString(asset_url.url, "asset_url"));
805 IDP_AddToGroup(locals.get(), IDP_NewString(asset_url.hash, "asset_hash"));
806
807 std::optional<IDProperty *> abs_url_idptr =
808 /* TODO Casting away const is annoying. Could pass a context copy instead, but `BPY_run_`
809 * functions don't handle that well yet. */
811 const_cast<bContext *>(&C), script, *locals, "_result");
812
813 if (!abs_url_idptr) {
814 /* This can happen when there is a malicious asset listing, for example when asset file entries
815 * are constructed such that they overwrite Blender's local asset listing files. */
816 /* TODO: improve the error reporting. */
818 reports,
819 RPT_ERROR,
820 "Could not queue the download of asset '%s', check the system console for more info",
821 asset_name.c_str());
822 return std::nullopt;
823 }
824 BLI_SCOPED_DEFER([&] { IDP_FreeProperty(*abs_url_idptr); });
825 if (!*abs_url_idptr) {
826 CLOG_ERROR(&LOG, "Failed to retrieve URL from downloader");
827 return std::nullopt;
828 }
829 if ((*abs_url_idptr)->type != IDP_STRING) {
830 CLOG_ERROR(&LOG, "Failed to retrieve URL from downloader - expected string return value");
831 return std::nullopt;
832 }
833
834 const std::string abs_url(IDP_string_get(*abs_url_idptr));
835 return abs_url;
836}
837
838#endif
839
841 const AssetRepresentation &asset,
842 ReportList *reports)
843{
844 /* Ensure we don't attempt to download anything when online access is disabled. */
845 if ((G.f & G_FLAG_INTERNET_ALLOW) == 0) {
846 BKE_report(reports, RPT_ERROR, "Internet access is disabled");
847 return;
848 }
849
850 if (!asset.needs_download()) {
851 BKE_report(reports, RPT_ERROR, "This asset does not require downloading");
852 return;
853 }
854
855#ifdef WITH_PYTHON
857 const std::optional<StringRefNull> library_url = library.remote_url();
858 if (!library_url || library_url->is_empty()) {
859 BKE_reportf(reports,
861 "Could not find asset library URL for asset '%s'",
862 asset.get_name().c_str());
863 return;
864 }
865
867
868 /* The main file is listed first, and has to be downloaded last. By reversing the list of files,
869 * first the dependencies are downloaded, followed by the asset itself. That way, when the main
870 * asset file appears on disk, it is ready for use.
871 *
872 * NOTE: if in the future the downloader supports parallel downloads, this will break. In that
873 * case, we'll have to move to something more "atomic", where all files that make up this asset
874 * retain their temporary-because-I'm-being-downloaded name until all downloads are complete. */
875 const Span<OnlineAssetFile> asset_files = asset.online_asset_files();
876 const StringRefNull asset_name = asset.get_name();
877 for (int i = asset_files.size() - 1; i >= 0; i--) {
878 const OnlineAssetFile &asset_file = asset_files[i];
879 std::optional<std::string> abs_url = remote_library_request_asset_download_file(
880 C, reports, asset_name, library, asset_file.path, asset_file.url);
881 if (!abs_url) {
882 /* remote_library_request_asset_download_file() will have reported the error.
883 *
884 * Better to stop here, because if a dependency download couldn't be triggered, the main file
885 * should not be downloaded either. Because, if that would work, we have a half-downloaded
886 * asset that Blender's asset browser doesn't know is broken. */
887 break;
888 }
889
891 RequestIdentifier{*library_url, asset_file.url.url},
892 std::move(*abs_url),
893 asset_file.size_in_bytes);
894 }
895#else
896 UNUSED_VARS(C, asset);
898 reports, RPT_ERROR, "Downloading assets requires Python, and this Blender is built without");
899#endif
900}
901
903 const AssetRepresentation &asset,
904 const StringRef dst_filepath,
905 ReportList *reports)
906{
907 /* Ensure we don't attempt to download anything when online access is disabled. */
908 if ((G.f & G_FLAG_INTERNET_ALLOW) == 0) {
909 BKE_report(reports, RPT_ERROR, "Internet access is disabled");
910 return;
911 }
912
913#ifdef WITH_PYTHON
914 const std::optional<StringRefNull> preview_url = asset.online_asset_preview_url();
915 if (!preview_url) {
916 return;
917 }
918 const std::optional<StringRefNull> preview_hash = asset.online_asset_preview_hash();
919 if (!preview_hash) {
920 return;
921 }
923 const std::optional<StringRef> library_url = library.remote_url();
924 if (!library_url) {
925 BKE_reportf(reports,
927 "Could not find asset library URL for asset '%s'",
928 asset.get_name().c_str());
929 return;
930 }
931
932 /* Notify the preview loading UI that a download for this preview is pending. */
934
935 {
936 std::string script =
937 "import _bpy_internal.assets.remote_library.asset_downloader as asset_dl\n"
938 "from pathlib import Path\n"
940 "\n"
941 "asset_dl.download_preview(\n"
942 " library_url, auth_token, Path(library_path),\n"
943 " preview_url, preview_hash, Path(dst_filepath),\n"
944 ")\n";
945
946 std::unique_ptr locals = bke::idprop::create_group("locals");
947 IDP_AddToGroup(locals.get(), IDP_NewString(*library_url, "library_url"));
948 IDP_AddToGroup(locals.get(), IDP_NewString(library.auth_token().value_or(""), "auth_token"));
949 IDP_AddToGroup(locals.get(), IDP_NewString(library.root_path(), "library_path"));
950 IDP_AddToGroup(locals.get(), IDP_NewString(*preview_url, "preview_url"));
951 IDP_AddToGroup(locals.get(), IDP_NewString(*preview_hash, "preview_hash"));
952 IDP_AddToGroup(locals.get(), IDP_NewString(dst_filepath, "dst_filepath"));
953
954 /* TODO Casting away const is annoying. Could pass a context copy instead, but `BPY_run_`
955 * functions don't handle that well yet. */
956 /* TODO: report errors in the UI somehow. */
957 BPY_run_string_exec_with_locals(const_cast<bContext *>(&C), script, *locals);
958 }
959
960#else
961 UNUSED_VARS(C, asset, dst_filepath);
962 /* TODO should we use CLOG here? Otherwise every preview will trigger a report. */
963 BKE_report(reports,
964 RPT_ERROR,
965 "Downloading asset previews requires Python, and this Blender is built without");
966#endif
967}
968
970
971/* -------------------------------------------------------------------- */
974
976{
977#ifdef WITH_PYTHON
978 constexpr const char *SCRIPT = R"(
979import _bpy_internal.assets.remote_library.asset_downloader as asset_dl
980
981asset_dl.cancel_download_all_assets()
982 )";
983
984 std::unique_ptr locals = bke::idprop::create_group("locals");
985 BPY_run_string_exec_with_locals(&C, SCRIPT, *locals);
986#else
987 UNUSED_VARS(C);
988#endif
989}
990
992
994{
995 if (this->files.is_empty()) {
996 return {};
997 }
998 return this->files[0].path;
999}
1000
1001/* -------------------------------------------------------------------- */
1004
1012
1014{
1016 remote_url != online_essentials_url(),
1017 "Online Essentials URL should use asset_system::online_essentials_cache_directory_path()");
1018
1019 char library_identifier[REMOTE_LIBRARY_DIRNAME_LEN + 1];
1020 {
1021 /* MD5 hash part. */
1022 uchar digest[16];
1023 BLI_hash_md5_buffer(remote_url.data(), remote_url.size(), digest);
1024 char hex_digest[33];
1025 BLI_hash_md5_to_hexdigest(digest, hex_digest);
1026 /* This adds a null terminator. */
1027 BLI_strncpy(library_identifier, hex_digest, REMOTE_LIBRARY_DIRNAME_LEN + 1);
1028 }
1029
1030 return remote_library_cache_directory_path(library_identifier);
1031}
1032
1033std::string remote_library_cache_directory_path(const StringRefNull library_dirname)
1034{
1035 char cache_path[FILE_MAXDIR];
1036 BKE_appdir_folder_caches(cache_path, sizeof(cache_path));
1037
1038 char library_cache_path[FILE_MAXDIR];
1039 BLI_path_join(library_cache_path,
1040 sizeof(library_cache_path),
1041 cache_path,
1042 "remote-assets",
1043 library_dirname.c_str());
1044
1045 return library_cache_path;
1046}
1047
1049{
1050 const StringRefNull library_cache_dir = asset.owner_asset_library().root_path();
1051
1052 char thumbs_dir_path[FILE_MAXDIR];
1054 thumbs_dir_path, sizeof(thumbs_dir_path), library_cache_dir.c_str(), "_thumbs", "large");
1055
1056 const std::optional<StringRefNull> preview_url = asset.online_asset_preview_url();
1057 if (!preview_url) {
1059 return "";
1060 }
1061
1062 const std::string asset_path = asset.full_path();
1063
1064 char thumb_name[40];
1065 {
1066 char hexdigest[33];
1067 uchar digest[16];
1068 BLI_hash_md5_buffer(asset_path.data(), asset_path.size(), digest);
1069 BLI_hash_md5_to_hexdigest(digest, hexdigest);
1070
1071 /* If the download URL has an extension, preserve that for the downloaded file (will be
1072 * either the period before the last extension, or the null character at the end of the file
1073 * name). */
1074 const char *ext = BLI_path_extension_or_end(preview_url->c_str());
1075 SNPRINTF(thumb_name, "%s%s", hexdigest, ext);
1076 }
1077
1078 /* First two letters of the thumbnail name (MD5 hash of the URI) as sub-directory name. */
1079 char thumb_prefix[3];
1080 thumb_prefix[0] = thumb_name[0];
1081 thumb_prefix[1] = thumb_name[1];
1082 thumb_prefix[2] = '\0';
1083
1084 /* Finally, the path of the thumbnail itself. */
1085 char thumb_path[FILE_MAX];
1086 BLI_path_join(thumb_path, sizeof(thumb_path), thumbs_dir_path, thumb_prefix, thumb_name + 2);
1087
1088 return thumb_path;
1089}
1090
1092
1093/* -------------------------------------------------------------------- */
1096
1098{
1099 char filepath[FILE_MAX];
1100 BLI_path_join(filepath,
1101 sizeof(filepath),
1102 library.cache_dirpath.c_str(),
1104 return filepath;
1105}
1106
1114
1116{
1117 for (bUserAssetLibrary &library : U.asset_libraries) {
1118 if ((library.flag & ASSET_LIBRARY_USE_REMOTE_URL) && library.remote_url[0]) {
1119 fn(library);
1120 }
1121 }
1122}
1123
1125
1126} // namespace blender::asset_system
Main runtime representation of an asset.
#define IDP_string_get(prop)
File and directory operations.
#define BLI_SCOPED_DEFER(function_to_defer)
#define FILE_MAX
#define BLI_path_join(...)
#define FILE_MAXDIR
#define SNPRINTF(dst, format,...)
#define RPT_(msgid)
#define CLOG_ERROR(clg_ref,...)
Definition CLG_log.h:188
#define C
Definition RandGen.cpp:29
#define ND_JOB
Definition WM_types.hh:432
#define NC_WM
Definition WM_types.hh:387
#define BLI_assert_unreachable()
#define UNUSED_VARS(...)
#define ELEM(...)
#define BLI_assert_msg(a, msg)
const Value * lookup_ptr(const Key &key) const
Definition BLI_map.hh:508
bool add_overwrite(const Key &key, const Value &value)
Definition BLI_map.hh:325
constexpr int64_t size() const
Definition BLI_span.hh:256
constexpr const char * c_str() const
eAssetLibraryType library_type() const
virtual std::optional< StringRefNull > remote_url() const
AssetLibrary(eAssetLibraryType library_type, bool is_read_only, StringRef name="", StringRef root_path="")
AssetCatalogService & catalog_service() const
virtual std::optional< StringRefNull > auth_token() const
std::optional< StringRefNull > online_asset_preview_url() const
Span< OnlineAssetFile > online_asset_files() const
std::optional< StringRefNull > online_asset_preview_hash() const
std::optional< AssetLibraryReference > library_reference() const override
PreferencesRemoteAssetLibrary(const bUserAssetLibrary &custom_library)
std::optional< StringRefNull > auth_token() const override
std::optional< eAssetImportMethod > import_method() const override
const bUserAssetLibrary * user_asset_library() const override
RemoteAssetLibrary(eAssetLibraryType library_type, bool is_read_only, StringRef remote_url, StringRef name, StringRef root_path)
std::optional< StringRefNull > remote_url() const override
std::optional< eAssetImportMethod > import_method() const override
void force_remote_listing_download() const override
static void ping_asset_file_download_succeeded(const bContext &C, StringRef library_url, StringRef absolute_file_url, StringRef local_file_abspath)
static void set_failure(StringRef url, std::optional< StringRefNull > failure_message)
static std::optional< bool > metafiles_in_place(StringRef url)
std::chrono::time_point< std::chrono::steady_clock > TimePoint
static std::optional< StringRefNull > failure_message(StringRef url)
static void begin_loading(StringRef url, float timeout)
static std::optional< RemoteLibraryLoadingStatus::Status > status(StringRef url)
static std::optional< TimePoint > last_new_pages_time(StringRef url)
static void ping_download_queue_done(const bContext &C)
static void ping_asset_file_download_failed(const bContext &C, StringRef library_url, StringRef absolute_file_url, StringRef local_file_abspath)
static void ping_asset_file_progress(StringRef absolute_file_url, int64_t size_in_bytes)
static void set_cancelled(const StringRef url)
static std::optional< FileSystemTimePoint > loading_start_time(const StringRef url)
static void ping_new_preview(const bContext &C, StringRef preview_full_filepath)
constexpr bool is_empty() const
constexpr int64_t size() const
constexpr const char * data() const
constexpr bool endswith(StringRef suffix) const
#define LOG(level)
Definition log.h:97
static void ping_asset_file_done_impl(const bContext &C, const StringRef library_url, const StringRef absolute_file_url, const StringRef local_file_abspath, const DownloadOutcome outcome)
bool remote_library_has_unfinished_asset_downloads()
void foreach_registered_user_remote_library(FunctionRef< void(bUserAssetLibrary &)> fn)
bool is_online_essentials_dirpath(StringRef dirpath)
float remote_library_total_asset_downloads_progress()
void remote_library_cancel_all_listing_downloads(const bContext &C)
void remote_library_request_preview_download(const bContext &C, const AssetRepresentation &asset, const StringRef dst_filepath, ReportList *reports)
std::string remote_library_asset_preview_path(const AssetRepresentation &asset)
Map< std::string, asset_system::RemoteLibraryLoadingStatus > UrlToLibraryStatusMap
constexpr StringRefNull REMOTE_LIBRARY_TOP_META_FILE_NAME
StringRefNull online_essentials_url()
void remote_library_cancel_all_asset_downloads(bContext &C)
std::string remote_library_cache_directory_path(StringRefNull library_dirname)
void remote_library_request_asset_download(const bContext &C, const AssetRepresentation &asset, ReportList *reports)
constexpr StringRefNull REMOTE_LIBRARY_TOP_META_FILE_NAME_LEADING_SLASH
void remote_library_request_download(const RemoteLibraryDefinitionRef &library_definition)
static UrlToLibraryStatusMap & library_to_status_map()
const int8_t REMOTE_LIBRARY_DIRNAME_LEN
bool is_online_essentials_url(StringRef url)
std::string remote_library_top_meta_file_path(const RemoteLibraryDefinitionRef &library)
bool remote_library_url_ends_with_top_meta_file_name(const StringRef url)
std::string remote_library_cache_directory_path_from_url(StringRef remote_url)
std::unique_ptr< IDProperty, IDPropertyDeleter > create_group(StringRef prop_name, eIDPropertyFlag flags={})
Allocate a new IDProperty of type IDP_GROUP.
void on_remote_assets_downloaded(wmWindowManager &wm, StringRef library_url, StringRef downloaded_file_abspath)
UserDef U
Definition blender.cc:58
BLI_assert(slot->slot_subtype.map==BMO_OP_SLOT_SUBTYPE_MAP_FLT)
unsigned __int64 uint64_t
Definition BKE_text.h:91
const char * BLI_path_extension_or_end(const char *filepath) ATTR_NONNULL(1) ATTR_WARN_UNUSED_RESULT ATTR_RETURNS_NONNULL
__int64 int64_t
Definition BKE_text.h:90
bool BPY_run_string_exec_with_locals(bContext *C, StringRefNull script, IDProperty &locals)
void * BLI_hash_md5_buffer(const char *buffer, size_t len, void *resblock)
Definition hash_md5.cc:345
void WM_event_add_notifier_ex(wmWindowManager *wm, const wmWindow *win, uint type, void *reference)
@ RPT_WARNING
Definition BKE_report.hh:40
@ ASSET_LIBRARY_USE_AUTH_TOKEN
@ ASSET_LIBRARY_USE_REMOTE_URL
bool BLI_dir_create_recursive(const char *dirname) ATTR_NONNULL()
Definition fileops_c.cc:416
bool IDP_AddToGroup(IDProperty *group, IDProperty *prop) ATTR_NONNULL()
Definition idprop.cc:788
wmWindow * CTX_wm_window(const bContext *C)
IDProperty * IDP_NewString(const char *st, StringRef name, eIDPropertyFlag flags={}) ATTR_WARN_UNUSED_RESULT
Definition idprop.cc:443
void BLI_path_slash_native(char *path) ATTR_NONNULL(1)
@ ASSET_LIBRARY_CUSTOM
wmTimer * WM_event_timer_add_notifier(wmWindowManager *wm, wmWindow *win, const uint type, const double time_step)
Global G
Definition blender.cc:57
int BLI_findindex(const ListBase *listbase, const void *vlink) ATTR_WARN_UNUSED_RESULT ATTR_NONNULL(1)
Definition listbase.cc:593
char * BLI_strncpy(char *__restrict dst, const char *__restrict src, size_t dst_maxncpy) ATTR_NONNULL(1
@ ASSET_IMPORT_APPEND
@ ASSET_IMPORT_APPEND_REUSE
unsigned char uchar
Definition BKE_text.h:58
void BKE_report(ReportList *reports, eReportType type, const char *message)
Definition report.cc:162
wmWindowManager * CTX_wm_manager(const bContext *C)
bool BLI_path_is_abs_from_cwd(const char *path) ATTR_NONNULL(1) ATTR_WARN_UNUSED_RESULT
int BLI_thread_is_main()
Definition threads.cc:184
void ED_preview_online_download_finished(wmWindowManager *wm, StringRef preview_full_filepath)
bool BKE_preferences_asset_library_is_available(const struct UserDef *userdef, const struct bUserAssetLibrary *library) ATTR_NONNULL() ATTR_WARN_UNUSED_RESULT
void BKE_reportf(ReportList *reports, eReportType type, const char *format,...) ATTR_PRINTF_FORMAT(3
Definition report.cc:191
@ G_FLAG_INTERNET_ALLOW
void BKE_appdir_folder_caches(char *path, size_t path_maxncpy) ATTR_NONNULL(1)
Definition appdir.cc:206
char * BLI_hash_md5_to_hexdigest(const void *resblock, char r_hex_digest[33])
Definition hash_md5.cc:381
void IDP_FreeProperty(IDProperty *prop)
Definition idprop.cc:1403
std::optional< IDProperty * > BPY_run_string_exec_with_locals_return_idprop(bContext *C, StringRefNull script, IDProperty &locals, StringRefNull result_var_name)
void WM_event_timer_remove(wmWindowManager *wm, wmWindow *, wmTimer *timer)
constexpr char SEP
void ED_preview_online_download_requested(StringRef preview_full_filepath)
void WM_event_add_mousemove(wmWindow *win)
constexpr uint64_t get_default_hash(const T &v, const Args &...args)
Definition BLI_hash.hh:252
signed char int8_t
Definition BKE_text.h:76
#define AUTH_TOKEN_MAYBE_OVERRIDE_FROM_EXTENSIONS
KERNEL_FEATURE_LIGHT_PASSES KERNEL_FEATURE_PATH_GUIDING float
Map< std::string, FileProgress > done_files
Map< std::string, FileProgress > requested_files
Map< RequestIdentifier, std::string > req_to_full_urls
static void file_requested(wmWindowManager &wm, RequestIdentifier &&request, std::string &&abs_url, int64_t size_in_bytes)
static void on_all_finished(wmWindowManager &wm)
static void file_finished(StringRef absolute_file_url, DownloadOutcome outcome)
static void file_report_progress(StringRef absolute_file_url, int64_t size_in_bytes)
RemoteLibraryDefinitionRef(const bUserAssetLibrary &library_definition)
bool operator==(const RequestIdentifier &other) const