Blender V5.3
blender::ed::asset::index Namespace Reference

Classes

class  AbstractFile
struct  ApiVersionInfo
struct  AssetIndex
class  AssetIndexFile
struct  AssetLibraryIndex
 References the asset library directory. More...
struct  AssetLibraryListingPageV1
struct  AssetLibraryListingV1
struct  AssetLibraryMeta
class  BlendFile
 Reference to a blend file that can be indexed. More...
class  FileStatusChecker
class  ReadingResult
struct  RemoteListingAssetEntry
struct  RemoteListingFileEntry

Typedefs

using RemoteListingEntryProcessFn = FunctionRef<bool(RemoteListingAssetEntry &)>
using RemoteListingWaitForPagesFn = FunctionRef<bool()>
using Timestamp = std::filesystem::file_time_type

Functions

bool read_remote_listing (StringRefNull root_dirpath, StringRefNull asset_library_name, ReportList &reports, RemoteListingEntryProcessFn process_fn, RemoteListingWaitForPagesFn wait_fn=nullptr, std::optional< Timestamp > ignore_before_timestamp=std::nullopt)
AssetMetaData * asset_metadata_from_dictionary (const io::serialize::DictionaryValue &entry)
ReadingResult read_remote_listing_v1 (StringRefNull listing_root_dirpath, RemoteListingEntryProcessFn process_fn, RemoteListingWaitForPagesFn wait_fn=nullptr, const std::optional< Timestamp > ignore_before_timestamp=std::nullopt)
constexpr StringRef ATTRIBUTE_VERSION ("version")
 Indexer for asset libraries.
constexpr StringRef ATTRIBUTE_ENTRIES ("entries")
constexpr StringRef ATTRIBUTE_ENTRIES_NAME ("name")
constexpr StringRef ATTRIBUTE_ENTRIES_CATALOG_ID ("catalog_id")
constexpr StringRef ATTRIBUTE_ENTRIES_CATALOG_NAME ("catalog_name")
constexpr StringRef ATTRIBUTE_ENTRIES_DESCRIPTION ("description")
constexpr StringRef ATTRIBUTE_ENTRIES_AUTHOR ("author")
constexpr StringRef ATTRIBUTE_ENTRIES_COPYRIGHT ("copyright")
constexpr StringRef ATTRIBUTE_ENTRIES_LICENSE ("license")
constexpr StringRef ATTRIBUTE_ENTRIES_TAGS ("tags")
constexpr StringRef ATTRIBUTE_ENTRIES_PROPERTIES ("properties")
constexpr StringRef ATTRIBUTE_ENTRIES_PREFERRED_IMPORT_METHOD ("preferred_import_method")
static void add_id_name (DictionaryValue &result, const short idcode, const StringRefNull name)
 add id + name to the attributes.
static void init_value_from_file_indexer_entry (DictionaryValue &result, const FileIndexerEntry *indexer_entry)
static void init_value_from_file_indexer_entries (DictionaryValue &result, const FileIndexerEntries &indexer_entries)
static void init_indexer_entry_from_value (FileIndexerEntry &indexer_entry, const DictionaryValue &entry)
static int init_indexer_entries_from_value (FileIndexerEntries &indexer_entries, const DictionaryValue &value)
static eFileIndexerResult read_index (const char *filename, FileIndexerEntries *entries, int *r_read_entries_len, void *user_data)
static void update_index (const char *filename, FileIndexerEntries *entries, void *user_data)
static void * init_user_data (const char *root_directory, size_t root_directory_maxncpy)
static void free_user_data (void *user_data)
static void filelist_finished (void *user_data)
constexpr FileIndexerType asset_indexer ()
static ReadingResult< ApiVersionInfo > choose_api_version (const AssetLibraryMeta &library_meta)
General functions for reading.
std::unique_ptr< io::serialize::Value > read_contents (StringRefNull filepath)
std::optional< asset_system::URLWithHash > parse_url_with_hash_dict (const DictionaryValue *url_with_hash_dict)
Meta file

Containing info like the author and contact information (all of which is ignored here), as well as the API version.

std::optional< bool > file_older_than_timestamp (const char *filepath, Timestamp timestamp)
Remote asset listing page
static std::optional< int > blender_version_from_string (const blender::StringRef str)
static ReadingResult< bool > blender_version_matches (const DictionaryValue &asset_dictionary)
static ReadingResult< RemoteListingAssetEntry > listing_entry_from_asset_dictionary (const DictionaryValue &dictionary, const Map< std::string, RemoteListingFileEntry > &file_path_to_entry_map)
static ReadingResult< RemoteListingFileEntry > listing_file_from_asset_dictionary (const DictionaryValue &dictionary)
static ReadingResult listing_entries_from_root (const StringRefNull listing_root_dirpath, const DictionaryValue &value, const RemoteListingEntryProcessFn process_fn)

Variables

const FileIndexerType file_indexer_asset = asset_indexer()
constexpr const char * hash_service_filename_prefix = "_file_hashes"

Typedef Documentation

◆ RemoteListingEntryProcessFn

◆ RemoteListingWaitForPagesFn

◆ Timestamp

using blender::ed::asset::index::Timestamp = std::filesystem::file_time_type

Definition at line 96 of file ED_asset_indexer.hh.

Function Documentation

◆ add_id_name()

void blender::ed::asset::index::add_id_name ( DictionaryValue & result,
const short idcode,
const StringRefNull name )
static

add id + name to the attributes.

NOTE: id and name are encoded like #ID.name

Definition at line 154 of file asset_indexer.cc.

References add_id_name(), ATTRIBUTE_ENTRIES_NAME(), and blender::result.

Referenced by add_id_name(), and init_value_from_file_indexer_entry().

◆ asset_indexer()

◆ asset_metadata_from_dictionary()

AssetMetaData * blender::ed::asset::index::asset_metadata_from_dictionary ( const io::serialize::DictionaryValue & entry)

◆ ATTRIBUTE_ENTRIES()

StringRef blender::ed::asset::index::ATTRIBUTE_ENTRIES ( "entries" )
constexpr

◆ ATTRIBUTE_ENTRIES_AUTHOR()

StringRef blender::ed::asset::index::ATTRIBUTE_ENTRIES_AUTHOR ( "author" )
constexpr

◆ ATTRIBUTE_ENTRIES_CATALOG_ID()

StringRef blender::ed::asset::index::ATTRIBUTE_ENTRIES_CATALOG_ID ( "catalog_id" )
constexpr

◆ ATTRIBUTE_ENTRIES_CATALOG_NAME()

StringRef blender::ed::asset::index::ATTRIBUTE_ENTRIES_CATALOG_NAME ( "catalog_name" )
constexpr

◆ ATTRIBUTE_ENTRIES_COPYRIGHT()

StringRef blender::ed::asset::index::ATTRIBUTE_ENTRIES_COPYRIGHT ( "copyright" )
constexpr

◆ ATTRIBUTE_ENTRIES_DESCRIPTION()

StringRef blender::ed::asset::index::ATTRIBUTE_ENTRIES_DESCRIPTION ( "description" )
constexpr

◆ ATTRIBUTE_ENTRIES_LICENSE()

StringRef blender::ed::asset::index::ATTRIBUTE_ENTRIES_LICENSE ( "license" )
constexpr

◆ ATTRIBUTE_ENTRIES_NAME()

StringRef blender::ed::asset::index::ATTRIBUTE_ENTRIES_NAME ( "name" )
constexpr

◆ ATTRIBUTE_ENTRIES_PREFERRED_IMPORT_METHOD()

StringRef blender::ed::asset::index::ATTRIBUTE_ENTRIES_PREFERRED_IMPORT_METHOD ( "preferred_import_method" )
constexpr

◆ ATTRIBUTE_ENTRIES_PROPERTIES()

StringRef blender::ed::asset::index::ATTRIBUTE_ENTRIES_PROPERTIES ( "properties" )
constexpr

◆ ATTRIBUTE_ENTRIES_TAGS()

StringRef blender::ed::asset::index::ATTRIBUTE_ENTRIES_TAGS ( "tags" )
constexpr

◆ ATTRIBUTE_VERSION()

StringRef blender::ed::asset::index::ATTRIBUTE_VERSION ( "version" )
constexpr

Indexer for asset libraries.

Indexes are stored per input file. Each index can contain zero to multiple asset entries. The indexes are grouped together per asset library. They are stored in BKE_appdir_folder_caches + /asset-library-indices/<asset-library-hash>/<asset-index-hash>_<asset_file>.index.json.

The structure of an index file is

{
"version": <file version number>,
"entries": [{
"name": "<asset name>",
"catalog_id": "<catalog_id>",
"catalog_name": "<catalog_name>",
"description": "<description>",
"author": "<author>",
"copyright": "<copyright>",
"license": "<license>",
"tags": ["<tag>"],
"preferred_import_method": eAssetImportMethod,
"properties": [..]
}]
}

NOTE: entries, author, description, copyright, license, tags and properties are optional attributes. If preferred_import_method is set, the ASSETDATA_USE_OWN_IMPORT_METHOD flag will be set on the asset metadata. Otherwise, it will not be set.

NOTE: File browser uses name and idcode separate. Inside the index they are joined together like #ID.name. NOTE: File browser group name isn't stored in the index as it is a translatable name.

References ATTRIBUTE_VERSION().

Referenced by blender::ed::asset::index::AssetIndex::AssetIndex(), ATTRIBUTE_VERSION(), and blender::ed::asset::index::AssetIndex::get_version().

◆ blender_version_from_string()

std::optional< int > blender::ed::asset::index::blender_version_from_string ( const blender::StringRef str)
static

Parse the string as "major.minor" version, returning (major*100 + minor). This can then be compared to BLENDER_VERSION from BKE_blender_version.h.

Definition at line 55 of file asset_indexer_remote_listing_v1.cc.

References blender::blender::StringRefBase::begin(), blender_version_from_string(), dot(), blender::blender::StringRefBase::end(), blender::blender::StringRefBase::not_found, and str.

Referenced by blender_version_from_string(), and blender_version_matches().

◆ blender_version_matches()

◆ choose_api_version()

◆ file_older_than_timestamp()

std::optional< bool > blender::ed::asset::index::file_older_than_timestamp ( const char * filepath,
Timestamp timestamp )

Note that this uses std::filesystem::file_time_type to get

Returns
true if the file is older than the timestamp, or no value if the file was not found.

TODO better report error message?

Definition at line 120 of file asset_indexer_remote_listing.cc.

References file_older_than_timestamp(), and blender::gpu::shader::metadata::printf.

Referenced by file_older_than_timestamp(), blender::ed::asset::index::AssetLibraryMeta::read(), and read_remote_listing_v1().

◆ filelist_finished()

void blender::ed::asset::index::filelist_finished ( void * user_data)
static

◆ free_user_data()

void blender::ed::asset::index::free_user_data ( void * user_data)
static

Definition at line 790 of file asset_indexer.cc.

References free_user_data().

Referenced by asset_indexer(), and free_user_data().

◆ init_indexer_entries_from_value()

◆ init_indexer_entry_from_value()

◆ init_user_data()

void * blender::ed::asset::index::init_user_data ( const char * root_directory,
size_t root_directory_maxncpy )
static

◆ init_value_from_file_indexer_entries()

◆ init_value_from_file_indexer_entry()

◆ listing_entries_from_root()

ReadingResult blender::ed::asset::index::listing_entries_from_root ( const StringRefNull listing_root_dirpath,
const DictionaryValue & value,
const RemoteListingEntryProcessFn process_fn )
static

Definition at line 264 of file asset_indexer_remote_listing_v1.cc.

References blender::Map< Key, Value, InlineBufferCapacity, ProbingStrategy, Hash, IsEqual, Slot, Allocator >::add_overwrite(), blender::Vector< T, InlineBufferCapacity, Allocator >::append(), blender::asset_system::OnlineAssetInfo::asset_file(), blender::BLI_assert(), BLI_assert_msg, blender::ed::asset::index::ReadingResult< T >::Cancelled(), blender::io::serialize::ArrayValue::elements(), blender::ed::asset::index::ReadingResult< T >::Failure, blender::ed::asset::index::ReadingResult< T >::failure_reason, blender::ed::asset::index::ReadingResult< T >::is_cancelled(), blender::ed::asset::index::RemoteListingAssetEntry::is_empty(), blender::ed::asset::index::ReadingResult< T >::is_failure(), blender::ed::asset::index::ReadingResult< T >::is_success(), listing_entries_from_root(), listing_entry_from_asset_dictionary(), listing_file_from_asset_dictionary(), blender::ed::asset::index::RemoteListingFileEntry::local_path, blender::io::serialize::DictionaryValue::lookup_array(), blender::Map< Key, Value, InlineBufferCapacity, ProbingStrategy, Hash, IsEqual, Slot, Allocator >::lookup_ptr(), N_, blender::ed::asset::index::RemoteListingAssetEntry::online_info, blender::ed::asset::index::FileStatusChecker::remote_file_status(), blender::ed::asset::index::RemoteListingAssetEntry::remote_file_status, blender::result, blender::ed::asset::index::ReadingResult< T >::Success(), and blender::ed::asset::index::ReadingResult< T >::warnings.

Referenced by listing_entries_from_root(), and blender::ed::asset::index::AssetLibraryListingPageV1::read_asset_entries().

◆ listing_entry_from_asset_dictionary()

ReadingResult< RemoteListingAssetEntry > blender::ed::asset::index::listing_entry_from_asset_dictionary ( const DictionaryValue & dictionary,
const Map< std::string, RemoteListingFileEntry > & file_path_to_entry_map )
static

Definition at line 116 of file asset_indexer_remote_listing_v1.cc.

References blender::io::serialize::Value::as_string_value(), blender::BLODataBlockInfo::asset_data, asset_metadata_from_dictionary(), blender::BKE_asset_metadata_create(), blender::BKE_idtype_idcode_from_name(), blender::BKE_idtype_name_normalize(), blender_version_matches(), blender::StringRefNull::c_str(), CLOG_DEBUG, blender::blender::StringRefBase::copy_utf8_truncated(), blender::ed::asset::index::RemoteListingAssetEntry::datablock_info, blender::ed::asset::index::ReadingResult< T >::Failure, blender::ed::asset::index::ReadingResult< T >::failure_reason, blender::asset_system::OnlineAssetInfo::files, blender::BLODataBlockInfo::free_asset_data, blender::ed::asset::index::RemoteListingAssetEntry::idcode, blender::ed::asset::index::ReadingResult< T >::is_success(), listing_entry_from_asset_dictionary(), blender::LOG, blender::io::serialize::DictionaryValue::lookup_array(), blender::io::serialize::DictionaryValue::lookup_dict(), blender::Map< Key, Value, InlineBufferCapacity, ProbingStrategy, Hash, IsEqual, Slot, Allocator >::lookup_ptr(), blender::io::serialize::DictionaryValue::lookup_str(), N_, blender::BLODataBlockInfo::name, blender::ed::asset::index::RemoteListingAssetEntry::online_info, parse_url_with_hash_dict(), blender::asset_system::OnlineAssetInfo::preview_url, blender::ed::asset::index::ReadingResult< T >::Success(), and blender::io::serialize::StringValue::value().

Referenced by listing_entries_from_root(), and listing_entry_from_asset_dictionary().

◆ listing_file_from_asset_dictionary()

◆ parse_url_with_hash_dict()

std::optional< asset_system::URLWithHash > blender::ed::asset::index::parse_url_with_hash_dict ( const io::serialize::DictionaryValue * url_with_hash_dict)

Parse a dictionary {url: "https://some.url/", hash: "sha256:abcd"} into a URLWithHash object.

If url_with_hash_dict is nullptr, or has no "url" field, std::nullopt is returned.

If the "hash" field is missing, it will simply be set to an empty string on the returned URLWithHash.

Definition at line 80 of file asset_indexer_remote_listing.cc.

References blender::bke::idprop::hash(), blender::io::serialize::DictionaryValue::lookup_str(), and parse_url_with_hash_dict().

Referenced by listing_entry_from_asset_dictionary(), parse_url_with_hash_dict(), blender::ed::asset::index::AssetLibraryListingV1::read(), and blender::ed::asset::index::AssetLibraryMeta::read().

◆ read_contents()

◆ read_index()

◆ read_remote_listing()

bool blender::ed::asset::index::read_remote_listing ( StringRefNull root_dirpath,
StringRefNull asset_library_name,
ReportList & reports,
RemoteListingEntryProcessFn process_fn,
RemoteListingWaitForPagesFn wait_fn = nullptr,
std::optional< Timestamp > ignore_before_timestamp = std::nullopt )
Parameters
process_fnCalled for each asset entry read from the listing. It's fine to move out the passed RemoteListingAssetEntry. Returning false will cancel the whole reading process and not read any further entries.
wait_fnIf this is set, reading will keep retrying to load unavailable pages, and call this wait function for each try. The wait function can block until it thinks new pages might be available. If this returns false the whole reading process will be cancelled.

Definition at line 240 of file asset_indexer_remote_listing.cc.

References blender::BKE_reportf(), BLI_assert_unreachable, blender::StringRefNull::c_str(), choose_api_version(), blender::ed::asset::index::ReadingResult< T >::Failure, blender::ed::asset::index::ReadingResult< T >::is_success(), N_, blender::ed::asset::index::AssetLibraryMeta::read(), read_remote_listing(), read_remote_listing_v1(), blender::result, RPT_, blender::RPT_ERROR, blender::RPT_WARNING, and blender::ed::asset::index::ReadingResult< T >::without_success_value().

Referenced by blender::filelist_readjob_remote_asset_library_index_read(), and read_remote_listing().

◆ read_remote_listing_v1()

ReadingResult blender::ed::asset::index::read_remote_listing_v1 ( StringRefNull listing_root_dirpath,
RemoteListingEntryProcessFn process_fn,
RemoteListingWaitForPagesFn wait_fn = nullptr,
const std::optional< Timestamp > ignore_before_timestamp = std::nullopt )

Reading of API schema version 1. See read_remote_listing() on process_fn.

Parameters
listing_root_dirpathAbsolute path to the remote listing root directory.

Definition at line 431 of file asset_indexer_remote_listing_v1.cc.

References blender::Set< Key, InlineBufferCapacity, ProbingStrategy, Hash, IsEqual, Slot, Allocator >::add(), blender::BLI_assert(), blender::BLI_exists(), BLI_path_join, blender::StringRefNull::c_str(), blender::ed::asset::index::ReadingResult< T >::Cancelled(), CLOG_DEBUG, blender::Set< Key, InlineBufferCapacity, ProbingStrategy, Hash, IsEqual, Slot, Allocator >::contains(), blender::Vector< T, InlineBufferCapacity, Allocator >::extend(), blender::ed::asset::index::ReadingResult< T >::Failure, blender::ed::asset::index::ReadingResult< T >::failure_reason, FILE_MAX, file_older_than_timestamp(), blender::ed::asset::index::ReadingResult< T >::has_warnings(), blender::ed::asset::index::ReadingResult< T >::is_cancelled(), blender::ed::asset::index::ReadingResult< T >::is_failure(), blender::ed::asset::index::ReadingResult< T >::is_success(), blender::LOG, N_, blender::gpu::shader::metadata::printf, blender::ed::asset::index::AssetLibraryListingV1::read(), blender::ed::asset::index::AssetLibraryListingPageV1::read_asset_entries(), read_remote_listing_v1(), blender::result, blender::SEP, blender::Set< Key, InlineBufferCapacity, ProbingStrategy, Hash, IsEqual, Slot, Allocator >::size(), blender::ed::asset::index::ReadingResult< T >::Success(), and blender::ed::asset::index::ReadingResult< T >::warnings.

Referenced by read_remote_listing(), and read_remote_listing_v1().

◆ update_index()

Variable Documentation

◆ file_indexer_asset

const FileIndexerType blender::ed::asset::index::file_indexer_asset = asset_indexer()

File Indexer Service for indexing asset files.

Opening and parsing a large collection of asset files inside a library can take a lot of time. To reduce the time it takes the files are indexed.

  • Index files are created for each blend file in the asset library, even when the blend file doesn't contain any assets.
  • Indexes are stored in an persistent cache folder (BKE_appdir_folder_caches + asset_library_indexes/{asset_library_dir}/{asset_index_file.json}).
  • The content of the indexes are used when:
    • Index exists and can be opened
    • Last modification date is earlier than the file it represents.
    • The index file version is the latest.
  • Blend files without any assets can be determined by the size of the index file for some additional performance.

Definition at line 815 of file asset_indexer.cc.

Referenced by blender::ed::asset::list::AssetList::setup().

◆ hash_service_filename_prefix

const char* blender::ed::asset::index::hash_service_filename_prefix = "_file_hashes"
constexpr

Filename prefix for the Disk File Hash Service used by FileStatusChecker.

The Disk File Hash Service itself will complete the filename depending on the back-end used. At the moment of writing that's SQLite, which will append _v{schema version}.sqlite.

NOTE: if this changes, also update the RemoteAssetListingLocator class in listing_downloader.py.

Definition at line 33 of file asset_indexer_remote_file_status.cc.

Referenced by blender::ed::asset::index::FileStatusChecker::FileStatusChecker().