diff --git a/.github/workflows/all_builds.yml b/.github/workflows/all_builds.yml index d2668fb0c..2712f54c4 100644 --- a/.github/workflows/all_builds.yml +++ b/.github/workflows/all_builds.yml @@ -28,7 +28,7 @@ env: # TODO: change this back to godotengine/godot and target master when #109685 and #109475 are merged GODOT_REPOSITORY: nikitalita/godot # Change the README too - GODOT_MAIN_SYNC_REF: gdre-wb-df6235838b6 + GODOT_MAIN_SYNC_REF: gdre-wb-f964fa714f5 SCONSFLAGS: verbose=yes warnings=all werror=no module_text_server_fb_enabled=yes minizip=yes deprecated=yes angle=yes accesskit=no SCONSFLAGS_TEMPLATE: disable_path_overrides=no no_editor_splash=yes module_camera_enabled=no module_mobile_vr_enabled=no module_upnp_enabled=no module_websocket_enabled=no module_csg_enabled=yes module_gridmap_enabled=yes use_static_cpp=yes builtin_freetype=yes builtin_libpng=yes builtin_zlib=yes builtin_libwebp=yes builtin_libvorbis=yes builtin_libogg=yes disable_3d=no SCONS_CACHE_MSVC_CONFIG: true @@ -117,7 +117,7 @@ jobs: bin: ./bin/godot.linuxbsd.editor.x86_64 - name: "🍎 macOS Editor" - os: "macos-latest" + os: "macos-26" id: macos-editor platform: macos target: editor @@ -127,7 +127,7 @@ jobs: bin: ./bin/godot.macos.editor.arm64 - name: "🍎 macOS Template Release" - os: "macos-latest" + os: "macos-26" id: macos-template platform: macos target: template_release @@ -159,10 +159,6 @@ jobs: run: | cp -R modules/gdsdecomp/.github/actions/* .github/actions/ - - name: Select Xcode 16 - if: matrix.platform == 'macos' - run: sudo xcode-select -s /Applications/Xcode_16.2.app - - name: Restore Godot build cache uses: ./.github/actions/godot-cache-restore id: restore-cache diff --git a/.scripts/export.ps1 b/.scripts/export.ps1 index 62e73fb80..1c5ff0a8a 100644 --- a/.scripts/export.ps1 +++ b/.scripts/export.ps1 @@ -47,7 +47,7 @@ function Get-VersionInfo { # Try to find git command $git = Get-Command git -ErrorAction SilentlyContinue $versionInfo = "unknown" - + if ($null -eq $git) { Write-Host "GDRE WARNING: cannot find git on path, unknown version will be saved in gdre_version.gen.h" } @@ -61,7 +61,7 @@ function Get-VersionInfo { } else { $versionInfo = $versionInfo.Trim() - + # git describe --exact-match --tags HEAD $res = & git describe --exact-match --tags HEAD 2>$null if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrEmpty($res)) { @@ -70,9 +70,9 @@ function Get-VersionInfo { $buildNum = $splits[-2] # everything but the last two elements $newVersionInfo = ($splits[0..($splits.Length - 3)] -join '-') - + $semverRegex = '^[vV]?(?0|[1-9]\d*)\.(?0|[1-9]\d*)\.(?0|[1-9]\d*)(?:-(?(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$' - + if ($newVersionInfo -match $semverRegex) { $major = $Matches.major $minor = $Matches.minor @@ -97,7 +97,7 @@ function Get-VersionInfo { $prereleaseTag = "" $buildMetadata = "" } - + $devStuff = "dev.$buildNum+$buildInfo" if ($prereleaseTag) { $prereleaseNameParts = $prereleaseTag -split '\.' @@ -138,7 +138,7 @@ function Get-VersionInfo { if ($versionInfo.StartsWith("v")) { $versionInfo = $versionInfo.Substring(1) } - + return $versionInfo } @@ -226,7 +226,7 @@ $platform = "linuxbsd" if ($env:OS -eq "Windows_NT") { $platform = "windows" } -elseif ($PSVersionTable.Os.StartsWith("Darwin")) { +elseif ($PSVersionTable.Os.StartsWith("Darwin") -or $PSVersionTable.Os.StartsWith("macOS")) { $platform = "macos" } @@ -335,7 +335,7 @@ if ($export_preset -eq "Android") { $user_settings = Get-Content $user_settings_path $user_settings = $user_settings -replace 'export/android/java_sdk_path = ".*"', "export/android/java_sdk_path = ""$java_home""" $user_settings = $user_settings -replace 'export/android/android_sdk_path = ".*"', "export/android/android_sdk_path = ""$android_home""" - } else { + } else { New-Item -ItemType File -Path $user_settings_path $user_settings = "[gd_resource type=""EditorSettings"" format=3] [resource] diff --git a/.scripts/rebase_godot.sh b/.scripts/rebase_godot.sh index b6bb178b2..4038336df 100755 --- a/.scripts/rebase_godot.sh +++ b/.scripts/rebase_godot.sh @@ -37,10 +37,10 @@ BRANCHES_TO_MERGE=( gltf-fix-skeleton-bone gltf-fix-double-precision gltf-fix-vertex-colors - ensure-bptc-textures fix-v3-meshes fix-clearcoat-gloss - fix-blend-export + gltf-mutex-all-document-extensions + gltf-node-weights ) # set fail on error @@ -61,7 +61,7 @@ else sed_in_place() { sed -i "$@"; } fi -# git push nikitalita $NEW_BRANCH_NAME --set-upstream +git push nikitalita $NEW_BRANCH_NAME --set-upstream # change the branch name in .github/workflows/all_builds.yml and the README.md sed_in_place "s/GODOT_MAIN_SYNC_REF: .*/GODOT_MAIN_SYNC_REF: $NEW_BRANCH_NAME/" "$GDRE_PATH/.github/workflows/all_builds.yml" diff --git a/README.md b/README.md index 028063ddb..856a6b340 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ During SCons configure, the module auto-applies the patches under `modules/gdsde ### Requirements -[Our fork of godot](https://github.com/nikitalita/godot) @ branch `gdre-wb-df6235838b6` +[Our fork of godot](https://github.com/nikitalita/godot) @ branch `gdre-wb-f964fa714f5` - Support for building on 3.x has been dropped and no new features are being pushed - Godot RE Tools still retains the ability to decompile 3.x and 2.x projects, however. diff --git a/bytecode/bytecode_base.cpp b/bytecode/bytecode_base.cpp index c70cc8786..f5ea37f47 100644 --- a/bytecode/bytecode_base.cpp +++ b/bytecode/bytecode_base.cpp @@ -6,8 +6,6 @@ #include "bytecode/bytecode_versions.h" #include "bytecode/gdscript_tokenizer_compat.h" -#include "bytecode/gdscript_v1_tokenizer_compat.h" -#include "bytecode/gdscript_v2_tokenizer_buffer.h" #include "compat/file_access_encrypted_v3.h" #include "compat/variant_decoder_compat.h" #include "compat/variant_writer_compat.h" @@ -22,7 +20,6 @@ #include "core/io/file_access_encrypted.h" #include "core/io/marshalls.h" #include "core/object/class_db.h" -#include "core/variant/binder_common.h" #define GDSDECOMP_FAIL_V_MSG(m_retval, m_msg) \ error_message = RTR(m_msg); \ @@ -671,15 +668,12 @@ Error GDScriptDecomp::decompile_buffer(Vector p_buffer) { if (curr_line <= prev_line) { curr_line = prev_line + 1; // force new line } - bool was_escaped = false; while (curr_line > prev_line) { if (curr_token != G_TK_NEWLINE && bytecode_version < GDSCRIPT_2_0_VERSION) { script_text += "\\"; // line continuation - was_escaped = true; } else if (bytecode_version >= GDSCRIPT_2_0_VERSION && tokens[i].start_line != tokens[i].end_line) { if (!first_line || (!gdre::remove_whitespace(line).is_empty())) { script_text += "\\"; - was_escaped = true; } } script_text += "\n"; diff --git a/compat/config_file_compat.cpp b/compat/config_file_compat.cpp index e3b746ffa..98dfaf5e9 100644 --- a/compat/config_file_compat.cpp +++ b/compat/config_file_compat.cpp @@ -107,6 +107,22 @@ Vector ConfigFileCompat::get_section_keys(const String &p_section) const return keys; } +Vector> ConfigFileCompat::get_section_keys_with_values_beginning_with(const String &p_section, const String &p_prefix) const { + Vector> keys; + ERR_FAIL_COND_V_MSG(!values.has(p_section), keys, vformat("Cannot get keys from nonexistent section \"%s\".", p_section)); + + const HashMap &keys_map = values[p_section]; + keys.reserve(keys_map.size()); + + for (const KeyValue &E : keys_map) { + if (E.key.begins_with(p_prefix)) { + keys.push_back({ E.key, E.value }); + } + } + + return keys; +} + void ConfigFileCompat::erase_section(const String &p_section) { ERR_FAIL_COND_MSG(!values.has(p_section), vformat("Cannot erase nonexistent section \"%s\".", p_section)); values.erase(p_section); diff --git a/compat/config_file_compat.h b/compat/config_file_compat.h index fa834491a..c4aefd053 100644 --- a/compat/config_file_compat.h +++ b/compat/config_file_compat.h @@ -60,6 +60,7 @@ class ConfigFileCompat : public RefCounted { Vector get_sections() const; Vector get_section_keys(const String &p_section) const; + Vector> get_section_keys_with_values_beginning_with(const String &p_section, const String &p_key_prefix) const; void erase_section(const String &p_section); void erase_section_key(const String &p_section, const String &p_key); diff --git a/compat/image_enum_compat.cpp b/compat/image_enum_compat.cpp index 4a9f2b572..c64876ec9 100644 --- a/compat/image_enum_compat.cpp +++ b/compat/image_enum_compat.cpp @@ -90,7 +90,8 @@ const char *v3_format_identifiers[V3Image::FORMAT_MAX] = { "ETC2_RG11S", "ETC2_RGB8", "ETC2_RGBA8", - "ETC2_RGB8A1" + "ETC2_RGB8A1", + "ASTC_8x8", }; const char *v3_format_names[V3Image::FORMAT_MAX] = { @@ -131,6 +132,7 @@ const char *v3_format_names[V3Image::FORMAT_MAX] = { "ETC2_RGB8", "ETC2_RGBA8", "ETC2_RGB8A1", + "ASTC_8x8", }; const char *v4_format_identifiers[Image::FORMAT_MAX] = { diff --git a/compat/resource_compat_binary.cpp b/compat/resource_compat_binary.cpp index 10cf8009e..303d6dafd 100644 --- a/compat/resource_compat_binary.cpp +++ b/compat/resource_compat_binary.cpp @@ -2117,21 +2117,20 @@ void ResourceFormatSaverCompatBinaryInstance::write_variant(Ref f, c } break; case Variant::OBJECT: { // This will only be triggered on godot 2.x, where the image variant is loaded as an image object vs. a resource - if (ver_format <= 1) { - Ref resp = p_property; - if (resp->is_class("Image")) { + Ref res = p_property; + if (ver_format <= 1 && res.is_valid()) { + if (res->is_class("Image")) { f->store_32(VARIANT_IMAGE); // storing lossless compressed by default ImageParserV2::write_image_v2_to_bin(f, p_property, true); break; - } else if (resp->is_class("InputEvent")) { + } else if (res->is_class("InputEvent")) { // these should never be saved to binary; just store the type f->store_32(VARIANT_INPUT_EVENT); break; } } f->store_32(VARIANT_OBJECT); - Ref res = p_property; if (res.is_null() || res->get_meta(SNAME("_skip_save_"), false)) { f->store_32(OBJECT_EMPTY); return; // Don't save it. @@ -2437,15 +2436,11 @@ static String _resource_get_class(Ref p_resource) { /* this is really only appropriate for saving fake-loaded resources right now; don't use it to save anything else*/ Error ResourceFormatSaverCompatBinaryInstance::save(const String &p_path, const Ref &p_resource, uint32_t p_flags) { - // Resource::seed_scene_unique_id(p_path.hash()); - // get metadata from the resource - Error err; path = get_local_path(p_path, p_resource); set_save_settings(p_resource, p_flags); - ResourceUID::ID uid = res_uid; Ref f; using_compression = using_compression || p_flags & ResourceSaver::FLAG_COMPRESS; @@ -2464,6 +2459,21 @@ Error ResourceFormatSaverCompatBinaryInstance::save(const String &p_path, const } ERR_FAIL_COND_V_MSG(err != OK, err, "Cannot create file '" + p_path + "'."); + return save_to_file(f, p_path, p_resource, p_flags); +} + +Error ResourceFormatSaverCompatBinaryInstance::save_to_file(const Ref &p_f, const String &p_path, const Ref &p_resource, uint32_t p_flags) { + path = get_local_path(p_path, p_resource); + + set_save_settings(p_resource, p_flags); + return _save_to_file(p_f, p_path, p_resource, p_flags); +} + +Error ResourceFormatSaverCompatBinaryInstance::_save_to_file(const Ref &p_f, const String &p_path, const Ref &p_resource, uint32_t p_flags) { + // Resource::seed_scene_unique_id(p_path.hash()); + + Ref f = p_f; + ResourceUID::ID uid = res_uid; if (using_real_t_double) { f->real_is_double = true; @@ -3525,9 +3535,6 @@ Error ResourceFormatSaverCompatBinaryInstance::set_save_settings(const Refver_major; ver_minor = compat->ver_minor; } - if (ver_major == 0) { - WARN_PRINT("Resource has a major version of 0, this is not supported."); - } String format = compat->resource_format; script_class = compat->script_class; using_script_class = compat->using_script_class(); @@ -3643,7 +3650,7 @@ int ResourceFormatSaverCompatBinary::get_ver_minor_from_format_version(int ver_f } Error ResourceFormatSaverCompatBinary::save_custom(const Ref &p_resource, const String &p_path, int ver_format, int ver_major, int ver_minor, uint32_t p_flags) { - ERR_FAIL_COND_V_MSG(ver_format <= 0 || ver_major <= 0, ERR_INVALID_PARAMETER, "Invalid version info"); + ERR_FAIL_COND_V_MSG(ver_format < 0 || ver_major < 0, ERR_INVALID_PARAMETER, "Invalid version info"); p_flags = CompatFormatLoader::set_version_info_in_flags(p_flags, ver_format, ver_major, ver_minor); return ResourceFormatSaverCompatBinary::save(p_resource, p_path, p_flags); diff --git a/compat/resource_compat_binary.h b/compat/resource_compat_binary.h index 7672eea05..060a78200 100644 --- a/compat/resource_compat_binary.h +++ b/compat/resource_compat_binary.h @@ -241,6 +241,8 @@ class ResourceFormatSaverCompatBinaryInstance { static String get_local_path(const String &p_path, const Ref &p_resource); + Error _save_to_file(const Ref &p_f, const String &p_path, const Ref &p_resource, uint32_t p_flags = 0); + public: enum { FORMAT_FLAG_NAMED_SCENE_IDS = 1, @@ -253,6 +255,7 @@ class ResourceFormatSaverCompatBinaryInstance { }; Error write_v2_import_metadata(Ref f, Ref imd, HashMap, int> &p_resource_map); Error save(const String &p_path, const Ref &p_resource, uint32_t p_flags = 0); + Error save_to_file(const Ref &p_f, const String &p_path, const Ref &p_resource, uint32_t p_flags = 0); Error set_uid(const String &p_path, ResourceUID::ID p_uid); void write_variant(Ref f, const Variant &p_property, HashMap, int> &resource_map, HashMap, int> &external_resources, HashMap &string_map, const PropertyInfo &p_hint = PropertyInfo()); }; diff --git a/compat/resource_compat_obdb.cpp b/compat/resource_compat_obdb.cpp index 8a7e6bd90..bc84f5ec8 100644 --- a/compat/resource_compat_obdb.cpp +++ b/compat/resource_compat_obdb.cpp @@ -228,6 +228,33 @@ Error ResourceLoaderCompatOBDB::parse_property(int &r_name_idx, Variant &r_v) { return parse_variant(r_v); } +bool ResourceLoaderCompatOBDB::should_threaded_load() const { + return is_real_load() && ResourceCompatLoader::is_globally_available() && load_type == ResourceCompatLoader::get_default_load_type(); +} + +Ref ResourceLoaderCompatOBDB::do_ext_load(const String &p_path, const String &p_type_hint) { + if (resource_cache.has(p_path)) { + return resource_cache[p_path]; + } + Error err = OK; + ERR_FAIL_COND_V_MSG(is_real_load() && p_path == local_path, Ref(), "Circular dependency detected: " + p_path); + Ref res; + if (!should_threaded_load()) { + if (is_real_load()) { + res = ResourceCompatLoader::custom_load(p_path, p_type_hint, load_type, &err, use_sub_threads, cache_mode_for_external); + } else { + res = CompatFormatLoader::create_missing_external_resource(p_path, p_type_hint, ResourceUID::INVALID_ID); + } + } else { // real load + Ref load_token = ResourceLoader::_load_start(p_path, p_type_hint, use_sub_threads ? ResourceLoader::LOAD_THREAD_DISTRIBUTE : ResourceLoader::LOAD_THREAD_FROM_CURRENT, cache_mode_for_external); + if (load_token.is_valid()) { + res = ResourceLoader::_load_complete(*load_token.ptr(), &err); + } + } + resource_cache[p_path] = res; + return res; +} + Error ResourceLoaderCompatOBDB::parse_variant(Variant &r_v) { uint32_t type = f->get_32(); @@ -382,16 +409,7 @@ Error ResourceLoaderCompatOBDB::parse_variant(Variant &r_v) { if (!path.contains("://") && path.is_relative_path()) { path = GDRESettings::get_singleton()->localize_path(res_path.get_base_dir().path_join(path)); } - if (remaps.find(path)) { - path = remaps[path]; - } - Error err = OK; - Ref res; - if (!is_real_load()) { - res = CompatFormatLoader::create_missing_external_resource(path, exttype, ResourceUID::INVALID_ID); - } else { - res = ResourceCompatLoader::custom_load(path, exttype, load_type, &err, use_sub_threads, cache_mode_for_external); - } + Ref res = do_ext_load(path, exttype); if (res.is_null()) { WARN_PRINT(vformat("Couldn't load resource: %s.", path)); } @@ -625,7 +643,12 @@ bool ResourceLoaderCompatOBDB::_index_sections() { } ir.body_offset = f->get_position(); ir.end_offset = section_end; - internal_resources.push_back(ir); + // dedupe external resources with the same path + if (ir.is_local_id || !internal_resources.has(ir)) { + internal_resources.push_back(ir); + } else { + print_line(vformat("Deduplicated external resource: %s", ir.path)); + } } else if (section == SECTION_OBJECT) { SceneNodeEntry e; e.kind = SECTION_OBJECT; @@ -878,22 +901,6 @@ Error ResourceLoaderCompatOBDB::_load_internal_resources_phase() { res = Ref(r); } - if (r) { - if (!path.is_empty()) { - if (cache_mode != ResourceFormatLoader::CACHE_MODE_IGNORE) { - r->set_path(path, cache_mode == ResourceFormatLoader::CACHE_MODE_REPLACE); - } else { - r->set_path_cache(path); - } - } - r->set_scene_unique_id(id); - } else if (converter.is_null() && !is_real_load()) { - if (!path.is_empty()) { - res->set_path_cache(path); - } - res->set_scene_unique_id(id); - } - internal_index_cache[path] = res; Object *target = r ? (Object *)r : (Object *)missing_resource.ptr(); @@ -920,19 +927,30 @@ Error ResourceLoaderCompatOBDB::_load_internal_resources_phase() { if (!ResourceInfo::resource_has_info(res)) { compat->set_on_resource(res); } - if (!path.is_empty()) { - if (cache_mode != ResourceFormatLoader::CACHE_MODE_IGNORE && is_real_load()) { - res->set_path(path, cache_mode == ResourceFormatLoader::CACHE_MODE_REPLACE); - } else { - res->set_path_cache(path); - } - } - res->set_scene_unique_id(id); internal_index_cache[path] = res; } } } } + if (!path.is_empty()) { + if (cache_mode != ResourceFormatLoader::CACHE_MODE_IGNORE && is_real_load()) { + if (cache_mode == ResourceFormatLoader::CACHE_MODE_REUSE) { + // someone managed to load it before us; let's use the cached version + Ref cached = ResourceCache::get_ref(path); + if (cached.is_valid()) { + res = cached; + r = res.ptr(); + } else { + res->set_path(path, false); + } + } else { + res->set_path(path, cache_mode == ResourceFormatLoader::CACHE_MODE_REPLACE || cache_mode == ResourceFormatLoader::CACHE_MODE_REPLACE_DEEP); + } + } else { + res->set_path_cache(path); + } + } + res->set_scene_unique_id(id); if (!missing_resource_properties.is_empty()) { res->set_meta(META_MISSING_RESOURCES, missing_resource_properties); @@ -942,7 +960,7 @@ Error ResourceLoaderCompatOBDB::_load_internal_resources_phase() { res->set_edited(false); #endif set_internal_resource_compat_meta(path, id, t, res); - resource_cache.push_back(res); + resource_cache[path] = res; } return OK; @@ -1181,13 +1199,7 @@ Error ResourceLoaderCompatOBDB::_build_packed_scene() { int instance_field; if (p.kind == SECTION_META_OBJECT && !p.instance_path.is_empty()) { // Instance node (no real Object data, only meta + overrides). - Ref ext; - Error eerr = OK; - if (!is_real_load()) { - ext = CompatFormatLoader::create_missing_external_resource(p.instance_path, "PackedScene", ResourceUID::INVALID_ID); - } else { - ext = ResourceCompatLoader::custom_load(p.instance_path, "PackedScene", load_type, &eerr, use_sub_threads, cache_mode_for_external); - } + Ref ext = do_ext_load(p.instance_path, "PackedScene"); type_idx = (int)SceneState::TYPE_INSTANTIATED; instance_field = intern_variant(ext); } else { diff --git a/compat/resource_compat_obdb.h b/compat/resource_compat_obdb.h index 2c0d67f43..7c0c81fa2 100644 --- a/compat/resource_compat_obdb.h +++ b/compat/resource_compat_obdb.h @@ -28,6 +28,12 @@ class ResourceLoaderCompatOBDB { bool is_local_id = true; uint64_t body_offset = 0; // Where properties begin (after type/path). uint64_t end_offset = 0; // First byte after the section. + bool operator==(const InternalResource &p_other) const { + return type == p_other.type && path == p_other.path && is_local_id == p_other.is_local_id && body_offset == p_other.body_offset && end_offset == p_other.end_offset; + } + bool operator!=(const InternalResource &p_other) const { + return !(*this == p_other); + } }; struct SceneNodeEntry { @@ -66,11 +72,9 @@ class ResourceLoaderCompatOBDB { Vector internal_resources; HashMap> internal_index_cache; - List> resource_cache; + HashMap> resource_cache; Vector scene_nodes; - HashMap remaps; - Ref resource; int packed_scene_version = -1; @@ -94,6 +98,8 @@ class ResourceLoaderCompatOBDB { bool is_real_load() const { return load_type == ResourceInfo::REAL_LOAD || load_type == ResourceInfo::GLTF_LOAD; } + bool should_threaded_load() const; + Ref do_ext_load(const String &p_path, const String &p_type_hint); // Recognize-only header read; populates `type` and `is_scene_mode`. bool _read_header(Ref p_f, bool p_load_strings); @@ -106,7 +112,6 @@ class ResourceLoaderCompatOBDB { String recognize(Ref p_f); void get_dependencies(Ref p_f, List *p_dependencies, bool p_add_types); - void set_remaps(const HashMap &p_remaps) { remaps = p_remaps; } void set_translation_remapped(bool p_remapped) { translation_remapped = p_remapped; } ResourceLoaderCompatOBDB() {} diff --git a/compat/resource_compat_text.cpp b/compat/resource_compat_text.cpp index 8f1cc69b7..4e7c66f33 100644 --- a/compat/resource_compat_text.cpp +++ b/compat/resource_compat_text.cpp @@ -3116,7 +3116,7 @@ int ResourceFormatSaverCompatText::get_ver_minor_from_format_version(int ver_for } Error ResourceFormatSaverCompatText::save_custom(const Ref &p_resource, const String &p_path, int ver_format, int ver_major, int ver_minor, uint32_t p_flags) { - ERR_FAIL_COND_V_MSG(ver_format <= 0 || ver_major <= 0, ERR_INVALID_PARAMETER, "Invalid version info"); + ERR_FAIL_COND_V_MSG(ver_format <= 0 || ver_major < 0, ERR_INVALID_PARAMETER, "Invalid version info"); p_flags = CompatFormatLoader::set_version_info_in_flags(p_flags, ver_format, ver_major, ver_minor); return ResourceFormatSaverCompatText::save(p_resource, p_path, p_flags); diff --git a/compat/resource_format_xml.cpp b/compat/resource_format_xml.cpp index 6b17f0317..63e9b59b5 100644 --- a/compat/resource_format_xml.cpp +++ b/compat/resource_format_xml.cpp @@ -2967,6 +2967,12 @@ Error ResourceFormatSaverXML::save(const Ref &p_resource, const String return saver.save(p_path, p_resource, p_flags); } +Error ResourceFormatSaverXML::save_custom(const Ref &p_resource, const String &p_path, int ver_format, int ver_major, int ver_minor, uint32_t p_flags) { + ERR_FAIL_COND_V_MSG(ver_format < 0 || ver_major < 0, ERR_INVALID_PARAMETER, "Invalid version info"); + p_flags = CompatFormatLoader::set_version_info_in_flags(p_flags, 0, ver_major, ver_minor); + return save(p_resource, p_path, p_flags); +} + bool ResourceFormatSaverXML::recognize(const Ref &p_resource) const { return true; // all recognized! } diff --git a/compat/resource_format_xml.h b/compat/resource_format_xml.h index fdaa1fbea..b8d2e6701 100644 --- a/compat/resource_format_xml.h +++ b/compat/resource_format_xml.h @@ -248,5 +248,6 @@ class ResourceFormatSaverXML : public ResourceFormatSaver { virtual bool recognize(const Ref &p_resource) const override; virtual void get_recognized_extensions(const Ref &p_resource, List *p_extensions) const override; + Error save_custom(const Ref &p_resource, const String &p_path, int ver_format, int ver_major, int ver_minor, uint32_t p_flags = 0); ResourceFormatSaverXML(); }; diff --git a/compat/resource_import_metadatav2.cpp b/compat/resource_import_metadatav2.cpp index 2f662659d..bb8aee18c 100644 --- a/compat/resource_import_metadatav2.cpp +++ b/compat/resource_import_metadatav2.cpp @@ -63,9 +63,10 @@ bool ResourceImportMetadatav2::has_option(const String &p_key) const { return options.has(p_key); } -Variant ResourceImportMetadatav2::get_option(const String &p_key) const { - ERR_FAIL_COND_V(!options.has(p_key), Variant()); - +Variant ResourceImportMetadatav2::get_option(const String &p_key, const Variant &p_default) const { + if (!options.has(p_key)) { + return p_default; + } return options[p_key]; } @@ -121,7 +122,7 @@ void ResourceImportMetadatav2::_bind_methods() { ClassDB::bind_method(D_METHOD("remove_source", "idx"), &ResourceImportMetadatav2::remove_source); ClassDB::bind_method(D_METHOD("get_source_count"), &ResourceImportMetadatav2::get_source_count); ClassDB::bind_method(D_METHOD("set_option", "key", "value"), &ResourceImportMetadatav2::set_option); - ClassDB::bind_method(D_METHOD("get_option", "key"), &ResourceImportMetadatav2::get_option); + ClassDB::bind_method(D_METHOD("get_option", "key", "default"), &ResourceImportMetadatav2::get_option, DEFVAL(Variant())); ClassDB::bind_method(D_METHOD("get_options"), &ResourceImportMetadatav2::_get_options); ClassDB::bind_method(D_METHOD("to_json"), &ResourceImportMetadatav2::to_json); ClassDB::bind_static_method(get_class_static(), D_METHOD("from_json", "dict"), &ResourceImportMetadatav2::from_json); diff --git a/compat/resource_import_metadatav2.h b/compat/resource_import_metadatav2.h index 2004da8e8..bf2e4cd79 100644 --- a/compat/resource_import_metadatav2.h +++ b/compat/resource_import_metadatav2.h @@ -35,7 +35,7 @@ class ResourceImportMetadatav2 : public RefCounted { void remove_source(int p_idx); int get_source_count() const; void set_option(const String &p_key, const Variant &p_value); - Variant get_option(const String &p_key) const; + Variant get_option(const String &p_key, const Variant &p_default = Variant()) const; bool has_option(const String &p_key) const; void get_options(List *r_options) const; Dictionary get_options_as_dictionary() const; diff --git a/compat/resource_loader_compat.cpp b/compat/resource_loader_compat.cpp index 88f9b0ff5..89fbeab73 100644 --- a/compat/resource_loader_compat.cpp +++ b/compat/resource_loader_compat.cpp @@ -218,6 +218,47 @@ void ResourceCompatLoader::get_base_extensions_for_type(const String &p_type, Li } } +Vector ResourceCompatLoader::get_base_extension_set_for_type(const String &p_type, int ver_major) { + Vector extensions; + bool has_res = false; + + if (ver_major > 0) { + switch (ver_major) { + case 1: + case 2: + case 3: + case 4: { + auto &ext_map = ver_major == 4 ? ext_to_v4_types : (ver_major == 3 ? ext_to_v3_types : ext_to_v2_types); + for (const auto &[ext, types] : ext_map) { + if (p_type.is_empty() || types.has(p_type)) { + if (ext == "res") { + has_res = true; // ensure it comes last + } else { + extensions.push_back(ext); + } + } + } + } break; + default: + ERR_FAIL_V_MSG(extensions, "Invalid version."); + } + } else { + List base_extensions; + get_base_extensions_for_type(p_type, &base_extensions); + for (const String &ext : extensions) { + if (ext == "res") { + has_res = true; + } else { + extensions.push_back(ext); + } + } + } + if (has_res) { + extensions.push_back("res"); + } + return extensions; +} + static inline void add_types_from_ext_to_types(const HashMap> &ext_to_types, const String &p_extension, HashSet &unique_types) { if (ext_to_types.has(p_extension)) { const HashSet &types = ext_to_types.get(p_extension); @@ -545,7 +586,12 @@ String ResourceCompatLoader::resource_to_string(const String &p_path, bool p_ski String save_path; String base_ext = path.get_basename().get_basename().get_extension(); - if (path.contains(".converted.") && (base_ext == "xml" || base_ext == "x" + path.get_extension().to_lower())) { + bool optimized = path.contains(".optimized."); + bool optimized_or_converted = optimized || path.contains(".converted."); + if (optimized && path.has_extension("scn")) { + base_ext = "xml"; + } + if (optimized_or_converted && (base_ext == "xml" || base_ext == "x" + path.get_extension().to_lower())) { ResourceFormatSaverXMLInstance saver; save_path = path.get_basename() + ".xml"; err = saver.save_to_file(f, save_path, res, 0); @@ -682,11 +728,16 @@ bool ResourceCompatLoader::is_globally_available() { return globally_available; } -Error ResourceCompatLoader::save_custom(const Ref &p_resource, const String &p_path, int ver_major, int ver_minor) { +Error ResourceCompatLoader::save_custom(const Ref &p_resource, const String &p_path, int ver_major, int ver_minor, uint32_t p_flags) { String path = GDRESettings::get_singleton()->globalize_path(p_path); - ERR_FAIL_COND_V_MSG(ver_major <= 0, ERR_INVALID_PARAMETER, "Invalid version info"); + ERR_FAIL_COND_V_MSG(ver_major < 0, ERR_INVALID_PARAMETER, "Invalid version info"); Error err = gdre::ensure_dir(path.get_base_dir()); ERR_FAIL_COND_V_MSG(err != OK, err, "Could not ensure directory for " + path); + String ext = path.get_extension().to_lower(); + if (ext == "xml" || ext == "xscn" || ext == "xres" || (ver_major <= 2 && ext.begins_with("x") && ext_to_v2_types.has(ext.substr(1)))) { + ResourceFormatSaverXML saver; + return saver.save_custom(p_resource, path, 0, ver_major, ver_minor, p_flags); + } if (path.get_extension() == "tres" || path.get_extension() == "tscn") { int ver_format = ResourceFormatSaverCompatText::get_default_format_version(ver_major, ver_minor); ResourceFormatSaverCompatText saver; @@ -715,6 +766,29 @@ String ResourceCompatConverter::get_resource_name(const Ref &re return name; } +Error ResourceCompatLoader::save_custom_to_file(const Ref &p_resource, const String &p_path, Ref &p_f, int ver_major, int ver_minor, uint32_t p_flags) { + String path = GDRESettings::get_singleton()->globalize_path(p_path); + ERR_FAIL_COND_V_MSG(ver_major < 0, ERR_INVALID_PARAMETER, "Invalid version info"); + Error err = gdre::ensure_dir(path.get_base_dir()); + ERR_FAIL_COND_V_MSG(err != OK, err, "Could not ensure directory for " + path); + String ext = path.get_extension().to_lower(); + if (ext == "xml" || ext == "xscn" || ext == "xres" || (ver_major <= 2 && ext.begins_with("x") && ext_to_v2_types.has(ext.substr(1)))) { + uint32_t flags = CompatFormatLoader::set_version_info_in_flags(p_flags, 0, ver_major, ver_minor); + ResourceFormatSaverXMLInstance saver; + return saver.save_to_file(p_f, path, p_resource, flags); + } + if (path.get_extension() == "tres" || path.get_extension() == "tscn") { + int ver_format = ResourceFormatSaverCompatText::get_default_format_version(ver_major, ver_minor); + uint32_t flags = CompatFormatLoader::set_version_info_in_flags(p_flags, ver_format, ver_major, ver_minor); + ResourceFormatSaverCompatTextInstance saver; + return saver.save_to_file(p_f, path, p_resource, flags); + } + int ver_format = ResourceFormatSaverCompatBinary::get_default_format_version(ver_major, ver_minor); + uint32_t flags = CompatFormatLoader::set_version_info_in_flags(p_flags, ver_format, ver_major, ver_minor); + ResourceFormatSaverCompatBinaryInstance saver; + return saver.save_to_file(p_f, path, p_resource, flags); +} + void ResourceCompatLoader::_init() { if (ResourceCompatLoader::initialized) { return; diff --git a/compat/resource_loader_compat.h b/compat/resource_loader_compat.h index f4ccad1f8..ead7103bc 100644 --- a/compat/resource_loader_compat.h +++ b/compat/resource_loader_compat.h @@ -61,6 +61,7 @@ class ResourceCompatLoader { static bool is_globally_available(); static void get_base_extensions_for_type(const String &p_type, List *p_extensions); + static Vector get_base_extension_set_for_type(const String &p_type, int ver_major = 0); static void get_base_extensions(List *p_extensions, int ver_major = 0); static void get_type_for_extension(const String &p_extension, List *p_types, int ver_major = 0); @@ -72,7 +73,8 @@ class ResourceCompatLoader { #endif // only supports resource text and binary formats, not texture formats - static Error save_custom(const Ref &p_resource, const String &p_path, int ver_major, int ver_minor); + static Error save_custom(const Ref &p_resource, const String &p_path, int ver_major, int ver_minor, uint32_t p_flags = 0); + static Error save_custom_to_file(const Ref &p_resource, const String &p_path, Ref &p_f, int ver_major, int ver_minor, uint32_t p_flags = 0); }; class CompatFormatLoader : public ResourceFormatLoader { diff --git a/compat/texture_loader_compat.cpp b/compat/texture_loader_compat.cpp index a20f45c1b..d5f48cf74 100644 --- a/compat/texture_loader_compat.cpp +++ b/compat/texture_loader_compat.cpp @@ -7,6 +7,7 @@ #include "utility/common.h" #include "utility/gdre_settings.h" #include "utility/image_saver.h" +#include "utility/macros.h" #include "utility/resource_info.h" #include "core/error/error_list.h" @@ -773,6 +774,10 @@ bool ResourceConverterTexture2D::handles_type(const String &p_type, int ver_majo Ref ResourceConverterTexture2D::convert(const Ref &res, ResourceInfo::LoadType p_type, int ver_major, Error *r_error) { String name; Ref texture; + if (const Ref img = res->get("image"); img.is_valid()) { + ImageTextureConverterCompat converter; + return converter.convert(res, p_type, ver_major, r_error); + } if (p_type == ResourceInfo::LoadType::NON_GLOBAL_LOAD) { return res; @@ -1175,7 +1180,7 @@ Ref ImageTextureConverterCompat::convert(const Ref &r } return img; }; - ERR_FAIL_COND_V_MSG(type != "ImageTexture", res, "Unsupported type: " + type); + ERR_FAIL_COND_V_MSG(type != "ImageTexture" && type != "Texture", res, "Unsupported type: " + type); name = get_resource_name(res, ver_major); image = convert_image(res->get("image")); if (image.is_null()) { diff --git a/docs/gdre_custom_pack_source.gd b/docs/gdre_custom_pack_source.gd index 78158be3d..1930a669e 100644 --- a/docs/gdre_custom_pack_source.gd +++ b/docs/gdre_custom_pack_source.gd @@ -1,9 +1,12 @@ -class_name GDRECustomPackSource +class_name GDREStandardPackSource extends PackSourceCustom +# This script re-implements the standard pack source logic for Godot PCK files. +# Use this as a base for your own custom pack source logic. -# Godot's packed file magic header ("GDPC" in ASCII, little-endian). +# Godot's packed file magic header ("GDPC" in ASCII, stored as little-endian in the file). const PACK_HEADER_MAGIC: int = 0x43504447 +# 'G', 'D', 'P', 'C' in ASCII. var PACK_HEADER_MAGIC_BYTES: PackedByteArray = PackedByteArray([0x47, 0x44, 0x50, 0x43]) const PACK_FORMAT_VERSION_V2: int = 2 const PACK_FORMAT_VERSION_V3: int = 3 @@ -26,7 +29,7 @@ func open_encrypted_file(base: FileAccess, key: PackedByteArray) -> FileAccess: return FileAccessEncryptedCustom.create_and_parse_non_custom(base, key, FileAccessEncryptedCustom.MODE_READ, false) func _try_open_pack(pck_path: String, p_replace_files: bool, p_offset: int, p_decryption_key: PackedByteArray) -> bool: - var ext: String = p_path.get_extension().to_lower() + var ext: String = pck_path.get_extension().to_lower() if ext == "apk" or ext == "zip": return false diff --git a/exporters/scene_exporter.cpp b/exporters/scene_exporter.cpp index a586cdf36..30a07335f 100644 --- a/exporters/scene_exporter.cpp +++ b/exporters/scene_exporter.cpp @@ -2849,7 +2849,7 @@ Node *GLBExporterInstance::_instantiate_scene(Ref scene) { return root; } -Error GLBExporterInstance::_load_scene_and_deps(Ref &r_scene) { +Error GLBExporterInstance::_load_scene_and_deps(Ref &r_scene) { MeshInstance3D::upgrading_skeleton_compat = true; err = _load_deps(); if (err != OK) { @@ -2858,7 +2858,7 @@ Error GLBExporterInstance::_load_scene_and_deps(Ref &r_scene) { return _load_scene(r_scene); } -Error GLBExporterInstance::_load_scene(Ref &r_scene) { +Error GLBExporterInstance::_load_scene(Ref &r_scene) { auto mode_type = ResourceCompatLoader::get_default_load_type(); // loading older scenes will spam warnings about deprecated features #ifndef DEBUG_ENABLED @@ -2866,15 +2866,15 @@ Error GLBExporterInstance::_load_scene(Ref &r_scene) { _silence_errors(true); } #endif - std::optional> result; + std::optional> result; // For some reason, scenes with meshes fail to load without the load done by ResourceLoader::load, possibly due to notification shenanigans. if (ResourceCompatLoader::is_globally_available()) { - result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function()>)[&]() -> Ref { - return ResourceLoader::load(source_path, "PackedScene", ResourceFormatLoader::CACHE_MODE_REUSE, &err); + result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function()>)[&]() -> Ref { + return ResourceLoader::load(source_path, "", ResourceFormatLoader::CACHE_MODE_REUSE, &err); }); } else { - result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function()>)[&]() -> Ref { - return ResourceCompatLoader::custom_load(source_path, "PackedScene", mode_type, &err, using_threaded_load(), ResourceFormatLoader::CACHE_MODE_REUSE); + result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function()>)[&]() -> Ref { + return ResourceCompatLoader::custom_load(source_path, "", mode_type, &err, using_threaded_load(), ResourceFormatLoader::CACHE_MODE_REUSE); }); } if (!result.has_value()) { @@ -3294,6 +3294,18 @@ struct BatchExportToken : public TaskRunnerStruct { return _check_unsupported(ver_major, is_text_output()); } + Error create_packed_scene_from_mesh(const Ref &mesh, Ref &scene) { + ERR_FAIL_COND_V_MSG(mesh.is_null(), ERR_INVALID_PARAMETER, "Mesh is null"); + return TaskManager::get_singleton()->dispatch_to_main_thread((std::function)[&]() -> Error { + MeshInstance3D *root = memnew(MeshInstance3D); + root->set_mesh(mesh); + scene = Ref(memnew(PackedScene)); + scene->pack(root); + return OK; + }) + .value_or(ERR_SKIP); + } + // scene loading and scene instancing has to be done on the main thread to avoid deadlocks and crashes bool batch_preload() { GDRELogger::clear_error_queues(); @@ -3318,13 +3330,31 @@ struct BatchExportToken : public TaskRunnerStruct { err = gdre::ensure_dir(p_dest_path.get_base_dir()); report->set_error(err); - ERR_FAIL_COND_V_MSG(err, false, "Failed to ensure directory " + p_dest_path.get_base_dir()); + if (err) { + after_preload(); + ERR_FAIL_V_MSG(false, "Failed to ensure directory " + p_dest_path.get_base_dir()); + } { - Ref scene; - err = instance._load_scene_and_deps(scene); - if (scene.is_null() && err == OK) { + String resource_type = report->get_import_info()->get_type(); + bool is_mesh = false; + if (resource_type != "PackedScene") { + if (resource_type != "Mesh" && !ClassDB::is_parent_class(resource_type, "Mesh")) { + after_preload(); + ERR_FAIL_V_MSG(false, "Unsupported resource type: " + resource_type); + } + is_mesh = true; + } + Ref resource; + err = instance._load_scene_and_deps(resource); + if (resource.is_null() && err == OK) { err = ERR_CANT_ACQUIRE_RESOURCE; } + + Ref scene = resource; + if (err == OK && scene.is_null() && is_mesh) { + err = create_packed_scene_from_mesh(resource, scene); + } + if (err != OK) { report->set_error(err); after_preload(); diff --git a/exporters/scene_exporter.h b/exporters/scene_exporter.h index 811861c29..51cebb4fc 100644 --- a/exporters/scene_exporter.h +++ b/exporters/scene_exporter.h @@ -174,8 +174,8 @@ class GLBExporterInstance { Dictionary _get_default_subresource_options(); Error _check_model_can_load(const String &p_dest_path); Error _load_deps(); - Error _load_scene_and_deps(Ref &r_scene); - Error _load_scene(Ref &r_scene); + Error _load_scene_and_deps(Ref &r_scene); + Error _load_scene(Ref &r_scene); void recompute_animation_tracks_for_library(AnimationPlayer *p_player, const Ref &p_anim_lib, const LocalVector &p_anim_names); void convert_animation_tracks_to_v4_for_player(AnimationPlayer *p_player); diff --git a/exporters/texture_exporter.cpp b/exporters/texture_exporter.cpp index 252c61db0..e0095ad10 100644 --- a/exporters/texture_exporter.cpp +++ b/exporters/texture_exporter.cpp @@ -1022,27 +1022,18 @@ Error TextureExporter::_convert_svg(const String &p_path, const String &dest_pat Error get_extant_texture_path(Ref iinfo, String &path) { path = iinfo->get_path(); - int ver_major = iinfo->get_ver_major(); - int ver_minor = iinfo->get_ver_minor(); - // Prefer s3tc textures over other formats for v3 (the etc2 compressor was vastly inferior to s3tc in v3) - if (ver_major <= 3) { - String format_type = path.get_basename().get_extension(); - if (format_type != "s3tc") { - PackedStringArray dest_files = iinfo->get_dest_files(); - String new_path = path.get_basename() + ".s3tc" + path.get_extension(); - if (dest_files.has(new_path) && FileAccess::exists(new_path)) { + if (iinfo->get_dest_files().size() > 1 && iinfo->get_ver_major() >= 3) { + // Prefer s3tc textures over other formats for v3 (the etc2 compressor was vastly inferior to s3tc in v3, and v3 astc is SCU-only and lower-quality than s3tc) + static const Vector preferred_formats_v3 = { "s3tc", "etc2", "atsc", "s3tc-low", "nx-low", "atsc-low" }; + static const Vector preferred_formats_v4 = { "bptc", "astc", "s3tc", "etc2" }; + const Vector &preferred_formats = iinfo->get_ver_major() <= 3 ? preferred_formats_v3 : preferred_formats_v4; + for (int i = 0; i < preferred_formats_v3.size(); i++) { + String new_path = iinfo->get_iinfo_val("remap", "path." + preferred_formats[i], String()); + if (!new_path.is_empty() && FileAccess::exists(new_path)) { path = new_path; return OK; } - Vector preferred_formats = { "s3tc", "etc2", "atsc", "s3tc-low", "nx-low", "atsc-low" }; - for (int i = 0; i < preferred_formats.size(); i++) { - Variant new_path = iinfo->get_iinfo_val("remap", "path." + preferred_formats[i]); - if (new_path.get_type() == Variant::STRING && !new_path.operator String().is_empty() && FileAccess::exists(new_path.operator String())) { - path = new_path; - return OK; - } - } } } @@ -1380,7 +1371,7 @@ Error TextureExporter::recreate_missing_variants(const String &output_dir, Refget_importer(); - if (importer != "texture" && importer != "texture_2d" && import_infos->get_ver_major() != 3) { + if ((importer != "texture" && importer != "texture_2d") || import_infos->get_ver_major() != 3) { return ERR_UNAVAILABLE; } diff --git a/exporters/translation_exporter.cpp b/exporters/translation_exporter.cpp index 9f65fdbb5..458354650 100644 --- a/exporters/translation_exporter.cpp +++ b/exporters/translation_exporter.cpp @@ -2333,6 +2333,12 @@ Ref TranslationExporter::export_resource(const String &output_dir, report->set_error(get_translations(iinfo, default_locale, default_translation, translations, keys)); ERR_FAIL_COND_V_MSG(report->get_error() != OK, report, "Could not get translations"); + if (iinfo->get_export_dest().has_extension("po") && translations.size() > 1) { + report->set_error(ERR_INVALID_DATA); + report->set_message("PO files must have only one associated translation file"); + return report; + } + Vector default_messages = default_translation->get_translated_message_list(); Vector> translation_messages; for (auto &translation : translations) { @@ -2404,29 +2410,45 @@ Ref TranslationExporter::export_resource(const String &output_dir, Ref f = FileAccess::open(output_path, FileAccess::WRITE, &export_err); ERR_FAIL_COND_V_MSG(export_err != OK, report, "Could not open file " + output_path); ERR_FAIL_COND_V_MSG(f.is_null(), report, "Could not open file " + output_path); - // Set UTF-8 BOM (required for opening with Excel in UTF-8 format, works with all Godot versions) - f->store_8(0xef); - f->store_8(0xbb); - f->store_8(0xbf); - f->store_string(header); - const String missing_key_prefix = MISSING_KEY_PREFIX; - for (int i = 0; i < keys.size(); i++) { - Vector line_values; - line_values.push_back(keys[i]); - for (int j = 0; j < translation_messages.size(); j++) { - auto &messages = translation_messages[j]; - if (out_of_sync_translations.has(j) && !keys[i].begins_with(missing_key_prefix)) { - line_values.push_back(translations[j]->get_message(keys[i])); - } else if (i >= messages.size()) { - line_values.push_back(""); - } else { - line_values.push_back(messages[i]); + if (output_path.has_extension("po")) { + f->store_string("msgid \"\"\n"); + f->store_string("msgstr \"\"\n"); + f->store_string(vformat("\"Language: %s\\n\"\n\n", default_translation->get_locale())); + for (int i = 0; i < keys.size(); i++) { + f->store_string(vformat("msgid \"%s\"\n", keys[i])); + const String &message = i < translation_messages[0].size() ? translation_messages[0][i] : ""; + Vector lines = message.split("\n"); + f->store_string(vformat("msgstr \"%s\"\n", lines[0])); + for (int j = 1; j < lines.size(); j++) { + f->store_string(vformat("\"%s\"\n", lines[j])); + } + f->store_string("\n"); + } + } else { // csv + // Set UTF-8 BOM (required for opening with Excel in UTF-8 format, works with all Godot versions) + f->store_8(0xef); + f->store_8(0xbb); + f->store_8(0xbf); + f->store_string(header); + const String missing_key_prefix = MISSING_KEY_PREFIX; + for (int i = 0; i < keys.size(); i++) { + Vector line_values; + line_values.push_back(keys[i]); + for (int j = 0; j < translation_messages.size(); j++) { + auto &messages = translation_messages[j]; + if (out_of_sync_translations.has(j) && !keys[i].begins_with(missing_key_prefix)) { + line_values.push_back(translations[j]->get_message(keys[i])); + } else if (i >= messages.size()) { + line_values.push_back(""); + } else { + line_values.push_back(messages[i]); + } } + f->store_csv_line(line_values, ","); } - f->store_csv_line(line_values, ","); + f->flush(); + f->close(); } - f->flush(); - f->close(); report->set_error(OK); Dictionary extra_info; extra_info["missing_keys"] = missing_keys; diff --git a/gui/gdre_audio_stream_preview.h b/gui/gdre_audio_stream_preview.h index f8657f861..367fed19d 100644 --- a/gui/gdre_audio_stream_preview.h +++ b/gui/gdre_audio_stream_preview.h @@ -30,7 +30,6 @@ #pragma once -#include "core/os/thread.h" #include "core/templates/safe_refcount.h" #include "scene/main/node.h" #include "servers/audio/audio_stream.h" diff --git a/gui/gui_icons.cpp b/gui/gui_icons.cpp index 570ddd7e0..1ea646241 100644 --- a/gui/gui_icons.cpp +++ b/gui/gui_icons.cpp @@ -1,13 +1,10 @@ #include "gui_icons.h" -#include "gui/gdre_icons.gen.h" + #include "scene/gui/control.h" #include "scene/resources/dpi_texture.h" -#include "scene/resources/image_texture.h" #include "scene/theme/theme_db.h" -#ifdef MODULE_SVG_ENABLED -#include "modules/svg/image_loader_svg.h" -#endif +#include "gui/gdre_icons.gen.h" // HashMap>> GDREGuiIcons::icons; // bool GDREGuiIcons::initialized = false; diff --git a/gui/gui_icons.h b/gui/gui_icons.h index 0f2ef1c67..8ad763c54 100644 --- a/gui/gui_icons.h +++ b/gui/gui_icons.h @@ -1,4 +1,5 @@ -#include "core/object/object.h" +#pragma once + #include "scene/resources/dpi_texture.h" class Control; diff --git a/gui/texture_previewer.cpp b/gui/texture_previewer.cpp index 9f55d51a8..93e093188 100644 --- a/gui/texture_previewer.cpp +++ b/gui/texture_previewer.cpp @@ -31,6 +31,7 @@ #include "texture_previewer.h" #include "core/object/callable_mp.h" +#include "core/object/class_db.h" #include "scene/gui/aspect_ratio_container.h" #include "scene/gui/color_rect.h" #include "scene/gui/label.h" diff --git a/main/gdre_main_loop.h b/main/gdre_main_loop.h index 3dd55fe1d..225c4052b 100644 --- a/main/gdre_main_loop.h +++ b/main/gdre_main_loop.h @@ -1,7 +1,6 @@ #pragma once #include "scene/main/node.h" #include "scene/main/scene_tree.h" -#include "utility/gd_parallel_queue.h" // We have a SceneTree and a regular node class that call into our singleton because of the editor; normally we would be able to // just use the SceneTree class, but when running in editor mode, Main::start() always sets the main loop to the base SceneTree class and @@ -12,10 +11,10 @@ class GDREMainLoop : public Object { static GDREMainLoop *singleton; static bool testing; - double last_physics_process_time; - double last_process_time; - bool processing; - bool running_process_calls; + double last_physics_process_time = 0.0; + double last_process_time = 0.0; + bool processing = false; + bool running_process_calls = false; Mutex next_process_calls_mutex; Vector next_process_calls; diff --git a/register_types.cpp b/register_types.cpp index ff35871f8..85bf2d3a2 100644 --- a/register_types.cpp +++ b/register_types.cpp @@ -16,6 +16,7 @@ #include "gui/texture_previewer.h" #include "modules/regex/regex.h" #include "modules/register_module_types.h" +#include "servers/rendering/rendering_server.h" #include "utility/app_version_getter.h" #include "utility/file_access_gdre.h" #include "utility/file_access_patched_gdre.h" @@ -618,16 +619,20 @@ void initialize_gdsdecomp_module(ModuleInitializationLevel p_level) { // Register ICO image loader ico_loader.instantiate(); ImageLoader::add_image_format_loader(ico_loader); - TextureLayeredPreviewer::init_shaders(); - TexturePreviewer::init_shaders(); + if (RenderingServer::get_singleton()) { + TextureLayeredPreviewer::init_shaders(); + TexturePreviewer::init_shaders(); + } } void uninitialize_gdsdecomp_module(ModuleInitializationLevel p_level) { if (p_level != MODULE_INITIALIZATION_LEVEL_SCENE) { return; } - TextureLayeredPreviewer::finish_shaders(); - TexturePreviewer::finish_shaders(); + if (RenderingServer::get_singleton()) { + TextureLayeredPreviewer::finish_shaders(); + TexturePreviewer::finish_shaders(); + } if (ico_loader.is_valid()) { ImageLoader::remove_image_format_loader(ico_loader); ico_loader.unref(); diff --git a/standalone/gdre_config_dialog.gd b/standalone/gdre_config_dialog.gd index 8aa0141cb..6849dce8b 100644 --- a/standalone/gdre_config_dialog.gd +++ b/standalone/gdre_config_dialog.gd @@ -94,6 +94,10 @@ func make_button_label(text: String) -> Label: func set_setting_value(setting: GDREConfigSetting, value: Variant): + if setting.get_type() == TYPE_INT: + value = int(value) + elif setting.get_type() == TYPE_FLOAT: + value = float(value) setting_value_map[setting] = value func setting_callback(setting: GDREConfigSetting, value: Variant, control: Control): diff --git a/standalone/gdre_main.gd b/standalone/gdre_main.gd index 0aff55d91..e66ce468d 100644 --- a/standalone/gdre_main.gd +++ b/standalone/gdre_main.gd @@ -951,18 +951,10 @@ func recovery( input_files:PackedStringArray, print("Error: failed to clear output directory except for git") return 1 - if is_dir: - if extract_only: - print("Why did you open a folder to extract it??? What's wrong with you?!!?") - return 1 - if output_dir.simplify_path() != input_file.simplify_path() and GDRECommon.copy_dir(input_file, output_dir) != OK: - print("Error: failed to copy " + input_file + " to " + output_dir) - return 1 - else: - err = dump_files(output_dir, files, ignore_checksum_errors, skip_md5) - if (err != OK): - print("Error: failed to extract PAK file, not exporting assets") - return 1 + err = dump_files(output_dir, files, ignore_checksum_errors, skip_md5) + if (err != OK): + print("Error: failed to extract PAK file, not exporting assets") + return 1 var end_time; var secs_taken; if (extract_only): diff --git a/standalone/gdre_new_pck.gd b/standalone/gdre_new_pck.gd index c9e8333da..7afc5cb46 100644 --- a/standalone/gdre_new_pck.gd +++ b/standalone/gdre_new_pck.gd @@ -32,7 +32,7 @@ var DESKTOP_DIR = OS.get_system_dir(OS.SystemDir.SYSTEM_DIR_DESKTOP) var ver_info = Engine.get_version_info() # TODO: make this use something dynamic (GDRESettings?) -const PCK_VERSION_DEFAULT = 2 +const PCK_VERSION_DEFAULT = 4 var DEFAULT_VER_MAJOR = ver_info["major"] var DEFAULT_VER_MINOR = ver_info["minor"] var DEFAULT_VER_PATCH = ver_info["patch"] diff --git a/standalone/gdre_new_pck.tscn b/standalone/gdre_new_pck.tscn index d6f534eae..79498bb12 100644 --- a/standalone/gdre_new_pck.tscn +++ b/standalone/gdre_new_pck.tscn @@ -87,7 +87,7 @@ offset_left = 15.0 offset_top = 117.0 offset_right = 174.0 offset_bottom = 148.0 -item_count = 4 +item_count = 5 popup/item_0/text = "0 (Godot 1.x, 2.x)" popup/item_0/id = 0 popup/item_1/text = "1 (Godot 3.x)" @@ -96,6 +96,8 @@ popup/item_2/text = "2 (Godot 4.0-4.4)" popup/item_2/id = 2 popup/item_3/text = "3 (Godot 4.5-4.6)" popup/item_3/id = 3 +popup/item_4/text = "4 (Godot 4.7+)" +popup/item_4/id = 4 [node name="EncryptionLabel" type="Label" parent="Control" unique_id=1571452124] layout_mode = 0 diff --git a/standalone/gdre_recover.gd b/standalone/gdre_recover.gd index 08f6b280c..a45b5afff 100644 --- a/standalone/gdre_recover.gd +++ b/standalone/gdre_recover.gd @@ -82,6 +82,7 @@ func _get_all_files(files: PackedStringArray) -> PackedStringArray: const DIR_STRUCTURE_OPTION_NAME = "Directory Structure" const EXPORT_SCENE_OPTION_NAME = "Export Scenes as" +const EXPORT_MESH_OPTION_NAME = "Export Meshes as" enum DirStructure { FLAT, @@ -96,6 +97,14 @@ enum ExportSceneType { GLTF } +enum ExportMeshType { + AUTO, + TRES, + OBJ, + GLB, + GLTF +} + const DIR_STRUCTURE_NAMES: PackedStringArray = [ "Flat", "Relative Hierarchical", @@ -109,6 +118,14 @@ const EXPORT_SCENE_TYPE_NAMES: PackedStringArray = [ "GLTF", ] +const EXPORT_MESH_TYPE_NAMES: PackedStringArray = [ + "Auto", + "tres", + "OBJ", + "GLB", + "GLTF", +] + func get_output_file_name(src: String, output_folder: String, dir_structure_option: DirStructure, new_ext: String = "", rel_base: String = "") -> String: var new_name = "" if dir_structure_option == DirStructure.FLAT: @@ -150,6 +167,44 @@ func _export_scene(file: String, output_dir: String, dir_structure: DirStructure report.error = OK return report +# TODO: A more generic way to export resources, stop copying all this code around +func _export_mesh(file: String, output_dir: String, dir_structure: DirStructure, rel_base: String, export_type: ExportMeshType) -> ExportReport: + var source_file = file + var iinfo = GDRESettings.get_import_info_by_dest(file) + if iinfo: + source_file = iinfo.source_file + + var ext = source_file.get_extension().to_lower() + + if export_type == ExportMeshType.GLB: + ext = "glb" + elif export_type == ExportMeshType.GLTF: + ext = "gltf" + elif export_type == ExportMeshType.OBJ: + ext = "obj" + elif export_type == ExportMeshType.TRES: + ext = "tres" + else: # AUTO + if not is_instance_valid(iinfo): + ext = "tres" + + var report: ExportReport = ExportReport.new() + var export_dest = get_output_file_name(source_file, output_dir, dir_structure, ext, rel_base) + if ext == "tres": + # just use bin to text + report.error = ResourceCompatLoader.to_text(file, export_dest) + return report + + if export_type == ExportMeshType.OBJ: + report.error = ObjExporter.export_file_with_options(export_dest, file, {}) + else: + report = SceneExporter.export_file_with_options(export_dest, file, { + "Exporter/Scene/GLTF/replace_shader_materials": true, + }) + if (report.error == ERR_BUG or report.error == ERR_PRINTER_ON_FIRE or report.error == ERR_DATABASE_CANT_READ): + report.error = OK + return report + func get_log_error_string(errs: PackedStringArray) -> String: return "\n".join(GDRECommon.filter_error_backtraces(errs)) @@ -167,7 +222,7 @@ func convert_pcfg_to_text(path: String, output_dir: String) -> Array: return [err, text_file] return [loader.save_cfb(output_dir, ver_major, ver_minor), text_file] -func _export_files(files: PackedStringArray, output_dir: String, dir_structure: DirStructure, rel_base: String, export_glb: ExportSceneType) -> PackedStringArray: +func _export_files(files: PackedStringArray, output_dir: String, dir_structure: DirStructure, rel_base: String, export_glb: ExportSceneType, export_mesh: ExportMeshType) -> PackedStringArray: var errs: PackedStringArray = [] files = _get_all_files(files) @@ -201,6 +256,12 @@ func _export_files(files: PackedStringArray, output_dir: String, dir_structure: errs.append("Exporting cancelled: " + file + "\n" + report.message + "\n" + get_log_error_string(report.get_error_messages())) break errs.append("Failed to export resource: " + file + "\n" + report.message + "\n" + get_log_error_string(report.get_error_messages())) + if file_ext == "mesh" or (_ret and _ret.get_compat_type().contains("Mesh")) and export_mesh != ExportMeshType.AUTO: + var report: ExportReport = _export_mesh(file, output_dir, dir_structure, rel_base, export_mesh) + if not report: + errs.append("Failed to export resource: " + file + get_log_error_string(GDRESettings.get_errors())) + elif report.error != OK and report.error != ERR_PRINTER_ON_FIRE: + errs.append("Failed to export resource: " + file + "\n" + report.message + "\n" + get_log_error_string(report.get_error_messages())) elif _ret: var iinfo: ImportInfo = ImportInfo.copy(_ret) iinfo.export_dest = get_output_file_name(iinfo.source_file, "res://", dir_structure, iinfo.source_file.get_extension().to_lower(), rel_base) @@ -263,8 +324,9 @@ func _do_export(output_dir: String, export_preview_visible: bool): var options = %ExportResDirDialog.get_selected_options() var dir_structure = options.get(DIR_STRUCTURE_OPTION_NAME, DirStructure.RELATIVE_HIERARCHICAL) var export_glb: ExportSceneType = options.get(EXPORT_SCENE_OPTION_NAME, int(ExportSceneType.AUTO)) + var export_mesh: ExportMeshType = options.get(EXPORT_MESH_OPTION_NAME, int(ExportMeshType.AUTO)) - errs = _export_files(files, output_dir, dir_structure, rel_base, export_glb) + errs = _export_files(files, output_dir, dir_structure, rel_base, export_glb, export_mesh) if export_preview_visible: %GdreResourcePreview.set_main_view_visible(true) @@ -371,6 +433,12 @@ func _set_file_dialog_options(file_dialog: FileDialog, default_dir_structure: Di if not include_scene: return var include_glb = GDRESettings.get_ver_major() >= SceneExporter.get_minimum_godot_ver_supported() + var mesh_default = options.get(EXPORT_MESH_OPTION_NAME, int(ExportMeshType.AUTO)) + var mesh_opts = EXPORT_MESH_TYPE_NAMES.duplicate() + if not include_glb: + mesh_opts.remove_at(int(ExportMeshType.GLTF)) + mesh_opts.remove_at(int(ExportMeshType.GLB)) + file_dialog.add_option(EXPORT_MESH_OPTION_NAME, mesh_opts, mesh_default) var scene_default = options.get(EXPORT_SCENE_OPTION_NAME, int(ExportSceneType.AUTO)) #file_dialog.set_option_default(0, int(default_dir_structure)) var glb_opts = EXPORT_SCENE_TYPE_NAMES.duplicate() diff --git a/utility/common.cpp b/utility/common.cpp index 7b76103a7..b70aaf266 100644 --- a/utility/common.cpp +++ b/utility/common.cpp @@ -21,9 +21,8 @@ struct RecursiveListDirTaskData { const bool absolute; const Vector excludes; const Vector banned_files; - const bool include_hidden; - const bool files_first; const bool exclude_dot_prefix_and_gdignore; + const bool files_first; const bool show_progress; struct Token { @@ -80,7 +79,7 @@ struct RecursiveListDirTaskData { Vector files; String base = absolute ? dir : ""; - da->set_include_hidden(include_hidden); + da->set_include_hidden(true); da->list_dir_begin(); String f = da->get_next(); while (!f.is_empty()) { @@ -177,30 +176,19 @@ Vector gdre::get_recursive_dir_list_multithread( const String &dir, const Vector &wildcards, bool absolute, - bool include_hidden, + bool exclude_dot_prefix_and_gdignore, const Vector &p_exclude_filters, + const Vector &p_banned_files, bool files_first, - bool exclude_dot_prefix_and_gdignore, bool show_progress) { - Vector banned_files; - Vector exclude_filters; - for (auto &wc : p_exclude_filters) { - // if the wildcard starts with '*/' and has no '/' following it, then it is a banned file - if (wc.begins_with("*/") && wc.substr(2).find("/") == -1) { - banned_files.push_back(wc.trim_prefix("*/")); - } else { - exclude_filters.push_back(wc); - } - } RecursiveListDirTaskData task_data{ dir, wildcards, absolute, - exclude_filters, - banned_files, - include_hidden, - files_first, + p_exclude_filters, + p_banned_files, exclude_dot_prefix_and_gdignore, + files_first, show_progress }; return task_data.run(); diff --git a/utility/common.h b/utility/common.h index 5757c3dcd..7f37d465d 100644 --- a/utility/common.h +++ b/utility/common.h @@ -1,7 +1,6 @@ #pragma once #include "core/templates/hash_set.h" #include "core/variant/variant.h" -#include "macros.h" #include #include @@ -16,10 +15,10 @@ Vector get_recursive_dir_list_multithread( const String &dir, const Vector &wildcards = {}, bool absolute = true, - bool include_hidden = true, + bool exclude_dot_prefix_and_gdignore = false, const Vector &exclude_filters = {}, + const Vector &banned_files = {}, bool files_first = false, - bool exclude_dot_prefix_and_gdignore = false, bool show_progress = false); Vector get_recursive_dir_list(const String &dir, const Vector &wildcards = {}, bool absolute = true, bool include_hidden = true); diff --git a/utility/diff_result.h b/utility/diff_result.h index 0625e8399..041816059 100644 --- a/utility/diff_result.h +++ b/utility/diff_result.h @@ -5,7 +5,6 @@ #include "core/variant/dictionary.h" #include "core/variant/variant.h" #include "scene/main/node.h" -#include "scene/resources/packed_scene.h" class Resource; class FileDiffResult; diff --git a/utility/file_access_gdre.cpp b/utility/file_access_gdre.cpp index cd05cf5eb..31ae8a437 100644 --- a/utility/file_access_gdre.cpp +++ b/utility/file_access_gdre.cpp @@ -4,6 +4,7 @@ #include "file_access_gdre.h" #include "core/io/file_access.h" +#include "core/io/file_access_memory.h" #include "core/object/class_db.h" #include "core/os/os.h" #include "crypto/custom_decryptor.h" @@ -13,7 +14,7 @@ #include "gdre_settings.h" #include "packed_file_info.h" #include "utility/common.h" -#include "utility/file_access_buffer.h" +#include "utility/macros.h" namespace CoreBind { PackedFile::PackedFile(const PackedData::PackedFile &p_pf) : pf(p_pf) {} @@ -222,17 +223,22 @@ bool DummySource::try_open_pack(const String &p_path, bool p_replace_files, uint } Ref DummySource::get_file(const String &p_path, PackedData::PackedFile *p_file, const Vector &p_decryption_key) { - (void)p_path; (void)p_file; (void)p_decryption_key; - Ref buffer = FileAccessBuffer::create(); - Vector data = { '\0' }; - Error err = buffer->open_custom(data); - if (err != OK) { - return nullptr; + String simplified_path = p_path.simplify_path().trim_prefix("res://"); + PathMD5 pmd5(simplified_path.md5_buffer()); + if (file_contents.has(pmd5)) { + Ref file_access_memory = Ref(memnew(FileAccessMemory)); + file_access_memory->open_custom(file_contents[pmd5].ptr(), file_contents[pmd5].size()); + return file_access_memory; } - buffer->seek(0); - return buffer; + ERR_FAIL_V_MSG(nullptr, "File not found"); +} + +void DummySource::add_file_content(const String &p_path, const Vector &p_file_content) { + String simplified_path = p_path.simplify_path().trim_prefix("res://"); + PathMD5 pmd5(simplified_path.md5_buffer()); + file_contents[pmd5] = p_file_content; } Error GDREPackedData::add_pack(const String &p_path, bool p_replace_files, uint64_t p_offset) { @@ -354,9 +360,11 @@ void GDREPackedData::add_pack_source(PackSource *p_source) { } } -void GDREPackedData::add_dummy_path(const String &p_pkg_path, const String &p_path) { +void GDREPackedData::add_dummy_path(const String &p_pkg_path, const String &p_path, const Vector &p_data) { // Dummy files must have non-zero offset to avoid being treated as erased. - add_path(p_pkg_path, p_path, 1, 1, MD5_EMPTY, &dummy_source, false, false, false, false); + const Vector &data = p_data.is_empty() ? Vector({ '\0' }) : p_data; + dummy_source.add_file_content(p_path, data); + add_path(p_pkg_path, p_path, 1, data.size(), MD5_EMPTY, &dummy_source, false, false, false, false); } uint8_t *GDREPackedData::get_file_hash(const String &p_path) { diff --git a/utility/file_access_gdre.h b/utility/file_access_gdre.h index d0803b9c3..0b58091e2 100644 --- a/utility/file_access_gdre.h +++ b/utility/file_access_gdre.h @@ -102,8 +102,34 @@ class DirSource : public PackSource { ~DirSource(); }; +struct PathMD5 { + uint64_t a = 0; + uint64_t b = 0; + + bool operator==(const PathMD5 &p_val) const { + return (a == p_val.a) && (b == p_val.b); + } + static uint32_t hash(const PathMD5 &p_val) { + uint32_t h = hash_murmur3_one_32(p_val.a); + return hash_fmix32(hash_murmur3_one_32(p_val.b, h)); + } + + PathMD5() {} + + explicit PathMD5(const Vector &p_buf) { + a = *((uint64_t *)&p_buf[0]); + b = *((uint64_t *)&p_buf[8]); + } +}; + +class GDREPackedData; class DummySource : public PackSource { static DummySource *singleton; + HashMap, PathMD5> file_contents; + + friend class GDREPackedData; + bool has_file_content(const String &p_path) const; + void add_file_content(const String &p_path, const Vector &p_file_content); public: static DummySource *get_singleton(); @@ -127,26 +153,6 @@ class GDREPackedData : public Object { HashSet files; }; - struct PathMD5 { - uint64_t a = 0; - uint64_t b = 0; - - bool operator==(const PathMD5 &p_val) const { - return (a == p_val.a) && (b == p_val.b); - } - static uint32_t hash(const PathMD5 &p_val) { - uint32_t h = hash_murmur3_one_32(p_val.a); - return hash_fmix32(hash_murmur3_one_32(p_val.b, h)); - } - - PathMD5() {} - - explicit PathMD5(const Vector &p_buf) { - a = *((uint64_t *)&p_buf[0]); - b = *((uint64_t *)&p_buf[8]); - } - }; - private: HashMap files; HashMap, PathMD5> delta_patches; @@ -180,7 +186,7 @@ class GDREPackedData : public Object { void set_default_file_access(); void reset_default_file_access(); void add_pack_source(PackSource *p_source); - void add_dummy_path(const String &p_pkg_path, const String &p_path); + void add_dummy_path(const String &p_pkg_path, const String &p_path, const Vector &p_data = {}); void add_path(const String &p_pkg_path, const String &p_path, uint64_t p_ofs, uint64_t p_size, const uint8_t *p_md5, PackSource *p_src, bool p_replace_files, bool p_encrypted = false, bool p_bundle = false, bool p_delta = false, const String &p_salt = String()); // for PackSource void remove_path(const String &p_path); uint8_t *get_file_hash(const String &p_path); diff --git a/utility/gdre_config.cpp b/utility/gdre_config.cpp index 50003a65c..1233a0779 100644 --- a/utility/gdre_config.cpp +++ b/utility/gdre_config.cpp @@ -7,6 +7,7 @@ #include "core/object/class_db.h" #include "core/object/worker_thread_pool.h" #include "core/os/os.h" +#include "core/variant/variant_utility.h" #include "gdre_logger.h" #include "gdre_settings.h" #include "gdre_version.gen.h" @@ -486,8 +487,12 @@ String GDREConfig::get_config_path() { void GDREConfig::save_config() { auto cfg_path = get_config_path(); Ref config = memnew(ConfigFile); - for (const auto &[key, value] : settings) { + for (const auto &[key, p_value] : settings) { String name = get_name_from_key(key); + Variant value = p_value; + if (default_settings.has(key) && default_settings[key]->get_type() != Variant::Type::NIL) { + value = VariantUtilityFunctions::type_convert(p_value, default_settings[key]->get_type()); + } if (!default_settings.has(key) || get_default_value(key) != value) { config->set_value(get_section_from_key(key), name, value); } @@ -508,11 +513,16 @@ String get_full_name(const String &p_setting) { } void GDREConfig::set_setting(const String &p_setting, const Variant &p_value, bool p_force_ephemeral) { - if (p_force_ephemeral || ephemeral_settings.contains(get_full_name(p_setting))) { - ephemeral_settings.try_emplace_l(get_full_name(p_setting), [=](auto &v) { v.second = p_value; }, p_value); + String full_name = get_full_name(p_setting); + Variant value = p_value; + if (default_settings.has(full_name) && default_settings[full_name]->get_type() != Variant::Type::NIL) { + value = VariantUtilityFunctions::type_convert(p_value, default_settings[full_name]->get_type()); + } + if (p_force_ephemeral || ephemeral_settings.contains(full_name)) { + ephemeral_settings.try_emplace_l(full_name, [=](auto &v) { v.second = value; }, value); return; } - settings.try_emplace_l(get_full_name(p_setting), [=](auto &v) { v.second = p_value; }, p_value); + settings.try_emplace_l(full_name, [=](auto &v) { v.second = value; }, value); } bool GDREConfig::has_setting(const String &p_setting) const { diff --git a/utility/gdre_packed_source.cpp b/utility/gdre_packed_source.cpp index 35d419d0e..77ea9d66d 100644 --- a/utility/gdre_packed_source.cpp +++ b/utility/gdre_packed_source.cpp @@ -242,7 +242,7 @@ bool GDREPackedSource::_get_exe_embedded_pck_info(Ref f, const Strin bool GDREPackedSource::seek_offset_from_exe(Ref f, const String &p_path, uint64_t &r_pck_size, const PackedByteArray &custom_magic) { EXEPCKInfo info; - auto ret = _get_exe_embedded_pck_info(f, p_path, info); + auto ret = _get_exe_embedded_pck_info(f, p_path, info, custom_magic); #ifdef DEBUG_ENABLED if (ret) { if (info.pck_section_header_pos == 0) { @@ -472,7 +472,12 @@ bool GDREPackedSource::try_open_pack(const String &p_path, bool p_replace_files, String path; path.append_utf8(cs.ptr()); String p_file = path.get_file(); - ERR_FAIL_COND_V_MSG(p_file.begins_with("gdre_") && p_file != "gdre_export.log", false, "Don't try to extract the GDRE pack files, just download the source from github."); + + if (unlikely(p_file.begins_with("gdre_"))) { + if (p_file != "gdre_export.log" && p_file != "gdre_export.json") { + WARN_PRINT_ONCE(vformat("GDRE prefixed file found, please report this on the GitHub issue tracker: %s", p_file)); + } + } // TODO: Ask bruvzg about whether or not p_offset is needed here. uint64_t ofs = file_base + f->get_64() + (version >= PACK_FORMAT_VERSION_V3 ? 0 : p_offset); diff --git a/utility/gdre_settings.cpp b/utility/gdre_settings.cpp index 19f05b270..5680bf1ad 100644 --- a/utility/gdre_settings.cpp +++ b/utility/gdre_settings.cpp @@ -10,7 +10,6 @@ #include "core/io/dir_access.h" #include "core/io/file_access.h" #include "core/object/class_db.h" -#include "core/object/message_queue.h" #include "core/string/print_string.h" #include "crypto/custom_decryptor.h" #include "exporters/translation_exporter.h" @@ -20,6 +19,7 @@ #include "plugin_manager/plugin_manager.h" #include "utility/app_version_getter.h" #include "utility/common.h" +#include "utility/file_access_buffer.h" #include "utility/file_access_gdre.h" #include "utility/gdre_logger.h" #include "utility/gdre_packed_source.h" @@ -807,6 +807,14 @@ Error GDRESettings::_project_post_load(bool initial_load, const String &csharp_a } } + // Pre 1.x Godot exported `OBDB` format resources which packed in all referenced external resources; we need to load and extract them so that we can use them in the project. + if (get_ver_major() == 0) { + err = _load_obdb_resources(); + if (err) { + ERR_FAIL_V_MSG(err, "FATAL ERROR: Could not load OBDB resources!"); + } + } + // Load the import files err = load_import_files(); ERR_FAIL_COND_V_MSG(err, ERR_FILE_CANT_READ, "FATAL ERROR: Could not load imported binary files!"); @@ -1656,8 +1664,7 @@ Dictionary GDRESettings::get_remaps(bool include_imports) const { } } if (include_imports) { - for (int i = 0; i < import_files.size(); i++) { - Ref iinfo = import_files[i]; + for (auto &[path, iinfo] : import_files) { ret[iinfo->get_source_file()] = iinfo->get_path(); } } @@ -1665,6 +1672,7 @@ Dictionary GDRESettings::get_remaps(bool include_imports) const { return ret; } +namespace { bool has_old_remap(const Vector &remaps, const String &src, const String &dst) { int idx = remaps.find(src); if (idx != -1 && idx % 2 == 0) { @@ -1676,6 +1684,35 @@ bool has_old_remap(const Vector &remaps, const String &src, const String return false; } +String get_mapped_path_unloaded(const String &p_path) { + String src = p_path; + if (src.begins_with("uid://")) { + auto id = ResourceUID::get_singleton()->text_to_id(src); + if (ResourceUID::get_singleton()->has_id(id)) { + src = ResourceUID::get_singleton()->get_id_path(id); + } else { + return ""; + } + } + + String iinfo_path = src + ".remap"; + if (FileAccess::exists(iinfo_path)) { + String new_path = ImportInfoRemap::get_remap_path_from_file(iinfo_path); + if (!new_path.is_empty()) { + return new_path; + } + } + iinfo_path = src + ".import"; + if (FileAccess::exists(iinfo_path)) { + String new_path = ImportInfoModern::get_remap_path_from_file(iinfo_path); + if (!new_path.is_empty()) { + return new_path; + } + } + return src; +} +} //namespace + String GDRESettings::get_remapped_source_path(const String &p_dst) const { if (is_pack_loaded()) { if (get_ver_major() >= 3) { @@ -1698,6 +1735,9 @@ String GDRESettings::get_remapped_source_path(const String &p_dst) const { } String GDRESettings::get_mapped_path(const String &p_src) const { + if (!is_pack_loaded()) { + return get_mapped_path_unloaded(p_src); + } String src = p_src; if (src.begins_with("uid://")) { auto id = ResourceUID::get_singleton()->text_to_id(src); @@ -1707,43 +1747,19 @@ String GDRESettings::get_mapped_path(const String &p_src) const { return ""; } } - if (is_pack_loaded()) { - String local_src = localize_path(src); - String remapped_path = get_remap(local_src); - if (!remapped_path.is_empty()) { - return remapped_path; - } + String local_src = localize_path(src); + String remapped_path = get_remap(local_src); + if (!remapped_path.is_empty()) { + return remapped_path; + } + String import_path = local_src + ".import"; + if (import_files.has(import_path)) { + return import_files[import_path]->get_path(); + } - for (int i = 0; i < import_files.size(); i++) { - Ref iinfo = import_files[i]; - if (iinfo->get_source_file().nocasecmp_to(local_src) == 0) { - return iinfo->get_path(); - } - } - } else { - Ref iinfo; - String iinfo_path = src + ".import"; - String dep_path; - if (FileAccess::exists(iinfo_path)) { - iinfo = ImportInfo::load_from_file(iinfo_path, 0, 0); - if (iinfo.is_valid()) { - if (FileAccess::exists(iinfo->get_path())) { - return iinfo->get_path(); - } - auto dests = iinfo->get_dest_files(); - for (int i = 0; i < dests.size(); i++) { - if (FileAccess::exists(dests[i])) { - return dests[i]; - } - } - } - } - iinfo_path = src + ".remap"; - if (FileAccess::exists(iinfo_path)) { - iinfo = ImportInfo::load_from_file(iinfo_path, 0, 0); - if (iinfo.is_valid() && FileAccess::exists(iinfo->get_path())) { - return iinfo->get_path(); - } + for (auto &[path, iinfo] : import_files) { + if (iinfo->get_source_file().nocasecmp_to(local_src) == 0) { + return iinfo->get_path(); } } return src; @@ -1984,12 +2000,9 @@ Error GDRESettings::close_log_file() { } Array GDRESettings::get_import_files(bool copy) { - if (!copy) { - return import_files; - } Array ifiles; - for (int i = 0; i < import_files.size(); i++) { - ifiles.push_back(ImportInfo::copy(import_files[i])); + for (auto &[path, iinfo] : import_files) { + ifiles.push_back(copy ? ImportInfo::copy(iinfo) : iinfo); } return ifiles; } @@ -2063,13 +2076,13 @@ Error GDRESettings::load_pack_uid_cache(bool p_reset) { if (old_path.simplify_path() == new_path.simplify_path()) { // Sometimes uid caches have duplicate paths when paths were not simplified before saving; this is a workaround new_path = new_path.simplify_path(); - } else if (has_path_loaded(old_path)) { - if (!has_path_loaded(new_path)) { // had old path, but not new path + } else if (has_path_loaded(get_mapped_path_unloaded(old_path))) { + if (!has_path_loaded(get_mapped_path_unloaded(new_path))) { // had old path, but not new path continue; // skip } // has both dupes.push_back(ResourceUID::get_singleton()->id_to_text(E.second) + " -> " + old_path + "\n Replacing with: " + new_path); - } else if (!has_path_loaded(new_path)) { // has neither + } else if (!has_path_loaded(get_mapped_path_unloaded(new_path))) { // has neither dupes.push_back(ResourceUID::get_singleton()->id_to_text(E.second) + " -> " + old_path + "\n Replacing with: " + new_path); } // else we have the new_path but not the old path } @@ -2428,7 +2441,7 @@ Error GDRESettings::load_import_files() { remap_iinfo.insert(tokens[i].path, tokens[i].info); } } - import_files.push_back(tokens[i].info); + import_files.insert(tokens[i].path, tokens[i].info); } return OK; } @@ -2437,7 +2450,7 @@ Error GDRESettings::load_import_file(const String &p_path) { Ref i_info = ImportInfo::load_from_file(p_path, get_ver_major(), get_ver_minor()); ERR_FAIL_COND_V_MSG(i_info.is_null(), ERR_FILE_CANT_OPEN, "Failed to load import file " + p_path); - import_files.push_back(i_info); + import_files.insert(p_path, i_info); if (i_info->get_iitype() == ImportInfo::REMAP) { if (!FileAccess::exists(i_info->get_path())) { print_line(vformat("Remapped path does not exist: %s -> %s", i_info->get_source_file(), i_info->get_path())); @@ -2450,25 +2463,17 @@ Error GDRESettings::load_import_file(const String &p_path) { Ref GDRESettings::get_import_info_by_source(const String &p_path) { Ref iinfo; - for (int i = 0; i < import_files.size(); i++) { - iinfo = import_files[i]; + for (const auto &[path, iinfo] : import_files) { if (iinfo->get_source_file() == p_path) { return iinfo; } } - // not found return Ref(); } Ref GDRESettings::get_import_info_by_dest(const String &p_path) const { Ref iinfo; - for (int i = 0; i < import_files.size(); i++) { - iinfo = import_files[i]; - // for (auto &dest : iinfo->get_dest_files()) { - // if (dest.to_lower() == p_path.to_lower()) { - // return iinfo; - // } - // } + for (auto &[path, iinfo] : import_files) { if (iinfo->get_dest_files().has(p_path)) { return iinfo; } @@ -2721,6 +2726,13 @@ void GDRESettings::_do_string_load(uint32_t i, StringLoadToken *tokens) { } // append the whole file just in case we missed something tokens[i].strings.append(text); + } else if (src_ext == "nut") { + // find all `"[^"]+"` strings + Ref re = RegEx::create_from_string("\"([^\"]+)\""); + TypedArray matches = re->search_all(text); + for (Ref match : matches) { + tokens[i].strings.append(match->get_string(1)); + } } else { // append the whole file; the "Partial resource strings" stage in the translation exporter will handle splitting it up tokens[i].strings.append(text); @@ -2771,6 +2783,7 @@ void GDRESettings::load_all_resource_strings() { wildcards.push_back("*.xml"); wildcards.push_back("*.cfg"); wildcards.push_back("*.esc"); + wildcards.push_back("*.nut"); Vector r_files = get_file_list(wildcards); if (has_loaded_dotnet_assembly()) { @@ -3379,3 +3392,193 @@ Variant GDRESettings::get_shader_global(const String &p_name) const { } return shader_globals.get(p_name); } + +struct ODBDLoadTask { + Vector paths; + Mutex global_lock; + HashSet base_extensions_set; + ParallelFlatHashSet resource_paths; + struct ResToSet { + Vector data; + String path; + String new_path; + }; + Vector res_to_set; + int ver_major = 0; + int ver_minor = 0; + + void _find_resources(const String &path, const Variant &p_variant, bool p_main, HashSet> &external_resources, HashSet> &resource_set) { + switch (p_variant.get_type()) { + case Variant::OBJECT: { + Ref res = p_variant; + if (res.is_null() || !CompatFormatLoader::resource_is_resource(res, ver_major) || external_resources.has(res) || res->get_meta(SNAME("_skip_save_"), false)) { + return; + } + if (!p_main && !res->is_built_in()) { + if (res->get_path() == path) { + ERR_PRINT(vformat("Circular reference to resource being saved found: '%s' will be null next time it's loaded.", path)); + return; + } + external_resources.insert(res); + return; + } + if (resource_set.has(res)) { + return; + } + resource_set.insert(res); + List property_list; + res->get_property_list(&property_list); + for (const PropertyInfo &E : property_list) { + if (E.usage & PROPERTY_USAGE_STORAGE) { + Variant value = res->get(E.name); + _find_resources(path, value, false, external_resources, resource_set); + } + } + + // COMPAT: get the missing resources too + Dictionary missing_resources = res->get_meta(META_MISSING_RESOURCES, Dictionary()); + if (missing_resources.size()) { + LocalVector keys = missing_resources.get_key_list(); + for (Variant key : keys) { + _find_resources(path, missing_resources[key], false, external_resources, resource_set); + } + } + + } break; + + case Variant::ARRAY: { + Array varray = p_variant; + _find_resources(path, varray.get_typed_script(), false, external_resources, resource_set); + for (const Variant &v : varray) { + _find_resources(path, v, false, external_resources, resource_set); + } + } break; + case Variant::DICTIONARY: { + Dictionary d = p_variant; + _find_resources(path, d.get_typed_key_script(), false, external_resources, resource_set); + _find_resources(path, d.get_typed_value_script(), false, external_resources, resource_set); + for (const KeyValue &kv : d) { + _find_resources(path, kv.key, false, external_resources, resource_set); + _find_resources(path, kv.value, false, external_resources, resource_set); + } + } break; + default: { + } + } + } + void _do_obdb_load(uint32_t i, void *p_userdata) { + String packed_path = paths[i]; + Ref packed_resource = ResourceCompatLoader::fake_load(packed_path); + if (packed_resource.is_valid()) { + HashSet> external_resources; + HashSet> resource_set; + _find_resources(packed_path, packed_resource, true, external_resources, resource_set); + for (const Ref &resource : external_resources) { + if (resource_paths.emplace(resource->get_path()).second) { + String path = resource->get_path(); + if (FileAccess::exists(path)) { + continue; + } + Ref missing_resource = resource; + if (missing_resource.is_valid()) { + if (missing_resource->get_original_class() == "Texture") { + missing_resource->set_original_class("ImageTexture"); + } + } + String ext = path.get_extension().to_lower(); + String base_ext = ext; + if (ext.begins_with("x") && ext != "xml") { + base_ext = ext.substr(1); + } + String new_path = path; + Vector data; + if (base_ext == "nut") { + String source = resource->get("script/source"); + if (source.is_empty()) { + ERR_CONTINUE_MSG(source.is_empty(), vformat("Resource %s has no script source", path)); + } + data = source.to_utf8_buffer(); + } else if (base_ext == "spx") { + Dictionary bundled = resource->get("_bundled"); + data = bundled.get("data", Vector()); + if (data.is_empty()) { + ERR_CONTINUE_MSG(data.is_empty(), vformat("Resource %s has no bundled data", path)); + } + } else { + Ref file_access = FileAccessBuffer::create(); + + if (!base_extensions_set.has(base_ext)) { + Vector base_extensions = ResourceCompatLoader::get_base_extension_set_for_type(resource->get_save_class(), 1); + if (base_extensions.is_empty()) { + ERR_CONTINUE_MSG(base_extensions.is_empty(), vformat("No base extensions found for resource %s of type: %s", path, resource->get_class())); + } + new_path = path + ".converted." + base_extensions[0]; + } + Ref fa = file_access; + Error err = ResourceCompatLoader::save_custom_to_file(resource, new_path, fa, ver_major, ver_minor, 0); + if (err != OK) { + ERR_CONTINUE_MSG(err, vformat("Failed to save resource %s to file %s", path, new_path)); + } + data = file_access->get_data(); + } + { + MutexLock lock(global_lock); + res_to_set.push_back({ data, path, new_path }); + } + } + } + } + } + + String get_description(uint32_t i, void *p_userdata) { + return vformat("Loading resource: %s", paths[i]); + } +}; + +Error GDRESettings::_load_obdb_resources() { + // gather all the files that have `.optimized.scn` or `.optimized.res` extensions + ODBDLoadTask task; + static const Vector optimized_extensions = { "*.optimized.scn", "*.optimized.res" }; + task.paths = get_file_list(optimized_extensions); + if (task.paths.is_empty()) { + return OK; + } + + task.ver_major = get_ver_major(); + task.ver_minor = get_ver_minor(); + + List base_extensions; + ResourceCompatLoader::get_base_extensions(&base_extensions, 1); + task.base_extensions_set = { "xml", "xres", "xscn" }; + for (const String &ext : base_extensions) { + task.base_extensions_set.insert(ext); + } + Error err = TaskManager::get_singleton()->run_multithreaded_group_task( + &task, + &ODBDLoadTask::_do_obdb_load, + task.paths.ptrw(), + task.paths.size(), + &ODBDLoadTask::get_description, + "GDRESettings::_load_obdb_resources", + RTR("Loading packed OBDB resources..."), + false); + if (err != OK) { + return err; + } + + for (const auto &res_to_set : task.res_to_set) { + if (res_to_set.path != res_to_set.new_path) { + GDRESettings::get_singleton()->add_remap(res_to_set.path, res_to_set.new_path); + } + GDREPackedData::get_singleton()->add_dummy_path("OBDBPackedResources", res_to_set.new_path, res_to_set.data); + } + HashSet remaps = gdre::vector_to_hashset((Vector)get_project_setting(v2_remap_setting, Vector())); + for (const String &path : task.paths) { + if (!remaps.has(path)) { + String new_path = path.get_basename().get_basename() + ".xml"; + add_remap(new_path, path); + } + } + + return err; +} diff --git a/utility/gdre_settings.h b/utility/gdre_settings.h index 612a1e011..c70f5323f 100644 --- a/utility/gdre_settings.h +++ b/utility/gdre_settings.h @@ -1,6 +1,4 @@ #pragma once -#include "core/object/class_db.h" -#include "core/variant/binder_common.h" #include "gd_parallel_hashmap.h" #include "import_info.h" @@ -53,7 +51,7 @@ class GDRESettings : public Object { Ref current_project; Ref version_override; GDRELogger *logger; - Array import_files; + HashMap> import_files; HashMap> remap_iinfo; String gdre_resource_path = ""; String v2_remap_setting = "remap/all"; @@ -176,6 +174,8 @@ class GDRESettings : public Object { void _get_app_version(); + Error _load_obdb_resources(); + protected: static void _bind_methods(); Error _add_pack(const String &p_path); diff --git a/utility/import_exporter.cpp b/utility/import_exporter.cpp index 47332f440..d3f9e9e99 100644 --- a/utility/import_exporter.cpp +++ b/utility/import_exporter.cpp @@ -1580,9 +1580,9 @@ Error ImportExporter::export_imports(const String &p_out_dir, const Vector ImportInfoModern::get_dest_files() const { return cf->get_value("deps", "dest_files", Vector()); } namespace { -Vector get_remap_paths(const Ref &cf) { +struct RemapPathSorter { + bool operator()(const Pair &a, const Pair &b) const { + String feature = a.first.get_slicec('.', 1); + return a.first < b.first; + } +}; + +void _insert_remap_paths(const String &key, const String &value, int &decomp_paths_found, Vector &remap_paths, Vector &candidates) { + if (key.begins_with("path.")) { + String feature = key.get_slicec('.', 1); + if (OS::get_singleton()->has_feature(feature)) { + remap_paths.push_back(value); + } else if (Image::can_decompress(feature)) { // When loading, check for decompressable formats and use first one found if nothing else is supported. + candidates.insert(decomp_paths_found, value); + decomp_paths_found++; + } else { + candidates.push_back(value); + } + } else if (key == "path") { + remap_paths.push_back(value); + } +} + +Pair> get_remap_paths_from_cf(const Ref &cf) { Vector remap_paths; - Vector remap_keys = cf->get_section_keys("remap"); + Vector candidates; + Vector ret; + int decomp_paths_found = 0; + Vector> remap_keys = cf->get_section_keys_with_values_beginning_with("remap", "path"); + if (remap_keys.is_empty()) { + return {}; + } else if (remap_keys.size() == 1) { + return { remap_keys[0].second.operator String(), { remap_keys[0].second.operator String() } }; + } // iterate over keys in remap section - for (int64_t i = 0; i < remap_keys.size(); i++) { - // if we find a path key, we have a match - if (remap_keys[i].begins_with("path.") || remap_keys[i] == "path") { - String try_path = cf->get_value("remap", remap_keys[i], ""); - remap_paths.push_back(try_path); + for (auto &E : remap_keys) { + ret.push_back(E.second.operator String()); + _insert_remap_paths(E.first, E.second.operator String(), decomp_paths_found, remap_paths, candidates); + } + remap_paths.append_array(candidates); + for (auto &E : remap_paths) { + if (FileAccess::exists(E)) { + return { E, ret }; } } + return { remap_keys[0].second.operator String(), ret }; +} + +Vector _parse_remap_paths_from_import_file(const Ref &p_f) { + Vector remap_paths; + Error err; + + VariantParser::StreamFile stream; + stream.f = p_f; + String assign, error_text; + Variant value; + VariantParser::Tag next_tag; + int lines = 0; + int decomp_paths_found = 0; + Vector candidates; + while (true) { + assign = Variant(); + next_tag.fields.clear(); + next_tag.name = String(); + + err = VariantParser::parse_tag_assign_eof(&stream, lines, error_text, next_tag, assign, value, nullptr, true); + if (err != OK) { + break; + } + if (!assign.is_empty()) { + _insert_remap_paths(assign, value, decomp_paths_found, remap_paths, candidates); + } else if (next_tag.name != "remap") { + break; + } + } + remap_paths.append_array(candidates); return remap_paths; } + Array vec_to_array(const Vector &vec) { Array arr; for (int64_t i = 0; i < vec.size(); i++) { @@ -479,11 +545,11 @@ void ImportInfoModern::set_metadata_prop(Dictionary r_dict) { dirty = true; } -Variant ImportInfoModern::get_param(const String &p_key) const { +Variant ImportInfoModern::get_param(const String &p_key, const Variant &p_default) const { if (!cf->has_section_key("params", p_key)) { - return Variant(); + return p_default; } - return cf->get_value("params", p_key); + return cf->get_value("params", p_key, p_default); } void ImportInfoModern::set_param(const String &p_key, const Variant &p_val) { @@ -496,11 +562,11 @@ bool ImportInfoModern::has_param(const String &p_key) const { return cf->has_section_key("params", p_key); } -Variant ImportInfoModern::get_iinfo_val(const String &p_section, const String &p_prop) const { +Variant ImportInfoModern::get_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_default) const { if (!cf->has_section_key(p_section, p_prop)) { - return Variant(); + return p_default; } - return cf->get_value(p_section, p_prop); + return cf->get_value(p_section, p_prop, p_default); } void ImportInfoModern::set_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_val) { @@ -553,7 +619,10 @@ Error ImportInfoModern::_load(const String &p_path) { cf->set_value("deps", "dest_files", vec_to_array({ preferred_import_path })); } else { // this is a multi-path import, get all the "path.*" key values - dest_files = get_remap_paths(cf); + auto [p, d] = get_remap_paths_from_cf(cf); + preferred_import_path = p; + dest_files = d; + // No path values at all; may be a translation file if (dest_files.is_empty()) { String importer = cf->get_value("remap", "importer", ""); @@ -608,20 +677,20 @@ Error ImportInfoModern::_load(const String &p_path) { if (preferred_import_path.is_empty()) { //check destination files if (dest_files.size() == 0) { - dest_files = get_remap_paths(cf); - // Reverse the order; we want to get the s3tc textures first if they exist. - dest_files.reverse(); + auto [p, d] = get_remap_paths_from_cf(cf); + preferred_import_path = p; + dest_files = d; } if (dest_files.size() == 0) { dest_files = get_dest_files(); - } - ERR_FAIL_COND_V_MSG(dest_files.size() == 0, ERR_FILE_CORRUPT, p_path + ": no destination files found in import data"); - for (int64_t i = 0; i < dest_files.size(); i++) { - if (FileAccess::exists(dest_files[i])) { - preferred_import_path = dest_files[i]; - break; + for (int64_t i = 0; i < dest_files.size(); i++) { + if (FileAccess::exists(dest_files[i])) { + preferred_import_path = dest_files[i]; + break; + } } } + ERR_FAIL_COND_V_MSG(dest_files.size() == 0, ERR_FILE_CORRUPT, p_path + ": no destination files found in import data"); if (preferred_import_path.is_empty()) { // just set it to the first one preferred_import_path = dest_files[0]; @@ -644,6 +713,28 @@ Error ImportInfoModern::_load(const String &p_path) { return OK; } +String ImportInfoModern::get_remap_path_from_file(const String &p_path) { + if (FileAccess::exists(p_path)) { + Ref f = FileAccess::open(p_path + ".import", FileAccess::READ); + if (f.is_null()) { + return ""; + } + Vector remap_paths = _parse_remap_paths_from_import_file(f); + if (remap_paths.is_empty()) { + return ""; + } else if (remap_paths.size() == 1) { + return remap_paths[0]; + } + for (int i = 0; i < remap_paths.size(); i++) { + if (FileAccess::exists(remap_paths[i])) { + return remap_paths[i]; + } + } + return remap_paths[0]; + } + return ""; +} + Error ImportInfoDummy::_load(const String &p_path) { Error err; Ref res_info; @@ -699,6 +790,35 @@ Ref ImportInfoDummy::create_dummy(const String &p_path) { return iinfo; } +String ImportInfoRemap::get_remap_path_from_file(const String &p_path) { + Error err; + Ref f = FileAccess::open(p_path, FileAccess::READ, &err); + if (f.is_valid()) { + VariantParser::StreamFile stream; + stream.f = f; + String assign, error_text; + Variant value; + VariantParser::Tag next_tag; + int lines = 0; + while (true) { + assign.clear(); + next_tag.fields.clear(); + next_tag.name.clear(); + err = VariantParserCompat::parse_tag_assign_eof(&stream, lines, error_text, next_tag, assign, value, nullptr, true); + if (err) { + break; + } + + if (assign == "path") { + return value; + } else if (next_tag.name != "remap") { + break; + } + } + } + return ""; +} + Error ImportInfoRemap::_load(const String &p_path) { Ref cf; cf.instantiate(); @@ -808,7 +928,7 @@ Error ImportInfov2::_load(const String &p_path) { auto_converted_export = true; source_file = source_file.is_empty() ? p_path.get_basename().trim_suffix(".converted") + ".gd" : source_file; importer = "script_bytecode"; - } else if (source_file.is_empty() && !p_path.contains(".converted.")) { + } else if (source_file.is_empty() && !p_path.contains(".converted.") && !p_path.contains(".optimized.")) { String base_dir = "res://.assets"; String new_ext = get_new_ext(old_ext); if (type == "AtlasTexture") { @@ -823,9 +943,23 @@ Error ImportInfov2::_load(const String &p_path) { // if this doesn't match "filename.ext.converted.newext" String base = spl[0]; String ext = spl.size() != 4 ? get_new_ext(old_ext) : spl[1]; + if (p_path.contains(".optimized.") && spl.size() != 4 && old_ext == "scn") { + ext = "xml"; + } source_file = p_path.get_base_dir().path_join(base + "." + ext); } - if (!res_info->get_type().to_lower().contains("texture") && !res_info->get_type().to_lower().contains("sample")) { + String new_ext = source_file.get_extension().to_lower(); + String type = res_info->get_type().to_lower(); + static const HashSet autoconverted_extensions = { + "xml", + "xres", + "xscn", + "tscn", + "tres", + }; + if (new_ext == "po") { + importer = "translation"; + } else if (autoconverted_extensions.has(new_ext) || (new_ext.begins_with("x") && (new_ext.length() == 4 || new_ext == "xxl" || new_ext == "xgt"))) { importer = "autoconverted"; } } @@ -943,8 +1077,8 @@ void ImportInfov2::set_additional_sources(const Vector &p_add_sources) { dirty = true; } -Variant ImportInfov2::get_param(const String &p_key) const { - return v2metadata->get_option(p_key); +Variant ImportInfov2::get_param(const String &p_key, const Variant &p_default) const { + return v2metadata->get_option(p_key, p_default); } void ImportInfov2::set_param(const String &p_key, const Variant &p_val) { @@ -956,12 +1090,12 @@ bool ImportInfov2::has_param(const String &p_key) const { return v2metadata->has_option(p_key); } -Variant ImportInfov2::get_iinfo_val(const String &p_section, const String &p_prop) const { +Variant ImportInfov2::get_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_default) const { if (p_section == "params" || p_section == "options") { - return v2metadata->get_option(p_prop); + return v2metadata->get_option(p_prop, p_default); } //TODO: others? - return Variant(); + return p_default; } void ImportInfov2::set_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_val) { @@ -1194,11 +1328,11 @@ void ImportInfo::_bind_methods() { ClassDB::bind_method(D_METHOD("set_metadata_prop", "metadata_prop"), &ImportInfo::set_metadata_prop); ADD_PROPERTY(PropertyInfo(Variant::DICTIONARY, "metadata_prop"), "set_metadata_prop", "get_metadata_prop"); - ClassDB::bind_method(D_METHOD("get_param", "key"), &ImportInfo::get_param); + ClassDB::bind_method(D_METHOD("get_param", "key", "default"), &ImportInfo::get_param, DEFVAL(Variant())); ClassDB::bind_method(D_METHOD("set_param", "key", "value"), &ImportInfo::set_param); ClassDB::bind_method(D_METHOD("has_param", "key"), &ImportInfo::has_param); - ClassDB::bind_method(D_METHOD("get_iinfo_val", "p_section", "p_prop"), &ImportInfo::get_iinfo_val); + ClassDB::bind_method(D_METHOD("get_iinfo_val", "p_section", "p_prop", "p_default"), &ImportInfo::get_iinfo_val, DEFVAL(Variant())); ClassDB::bind_method(D_METHOD("set_iinfo_val", "p_section", "p_prop", "p_val"), &ImportInfo::set_iinfo_val); ClassDB::bind_method(D_METHOD("get_params"), &ImportInfo::get_params); @@ -1473,13 +1607,13 @@ String ImportInfoGDExt::get_compatibility_maximum() const { // virtual Variant get_iinfo_val(const String &p_section, const String &p_prop) const override; // virtual void set_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_val) override; -Variant ImportInfoGDExt::get_iinfo_val(const String &p_section, const String &p_prop) const { +Variant ImportInfoGDExt::get_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_default) const { if (cf->has_section(p_section)) { if (cf->has_section_key(p_section, p_prop)) { - return cf->get_value(p_section, p_prop, ""); + return cf->get_value(p_section, p_prop, p_default); } } - return Variant(); + return p_default; } void ImportInfoGDExt::set_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_val) { diff --git a/utility/import_info.h b/utility/import_info.h index e840f4a45..04f4d4e7d 100644 --- a/utility/import_info.h +++ b/utility/import_info.h @@ -5,7 +5,6 @@ #include "core/object/object.h" #include "core/object/ref_counted.h" #include "core/os/shared_object.h" -#include "core/variant/binder_common.h" #include "utility/resource_info.h" namespace V2ImportEnums { @@ -157,11 +156,11 @@ class ImportInfo : public RefCounted { virtual Dictionary get_metadata_prop() const { return Dictionary(); } virtual void set_metadata_prop(Dictionary r_dict) { return; } - virtual Variant get_param(const String &p_key) const = 0; + virtual Variant get_param(const String &p_key, const Variant &p_default = Variant()) const = 0; virtual void set_param(const String &p_key, const Variant &p_val) = 0; virtual bool has_param(const String &p_key) const = 0; - virtual Variant get_iinfo_val(const String &p_section, const String &p_prop) const = 0; + virtual Variant get_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_default = Variant()) const = 0; virtual void set_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_val) = 0; // gets the parameters used to import the resource. @@ -206,6 +205,8 @@ class ImportInfoModern : public ImportInfo { virtual void _set_from_json(const Dictionary &p_json) override; public: + static String get_remap_path_from_file(const String &p_import_file_path); + // Gets the Godot resource type (e.g. "StreamTexture") virtual String get_type() const override; virtual void set_type(const String &p_type) override; @@ -232,11 +233,11 @@ class ImportInfoModern : public ImportInfo { virtual Dictionary get_metadata_prop() const override; virtual void set_metadata_prop(Dictionary r_dict) override; - virtual Variant get_param(const String &p_key) const override; + virtual Variant get_param(const String &p_key, const Variant &p_default = Variant()) const override; virtual void set_param(const String &p_key, const Variant &p_val) override; virtual bool has_param(const String &p_key) const override; - virtual Variant get_iinfo_val(const String &p_section, const String &p_prop) const override; + virtual Variant get_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_default = Variant()) const override; virtual void set_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_val) override; // gets the parameters used to import the resource. @@ -290,11 +291,11 @@ class ImportInfov2 : public ImportInfo { virtual Vector get_dest_files() const override; virtual void set_dest_files(const Vector p_dest_files) override; - virtual Variant get_param(const String &p_key) const override; + virtual Variant get_param(const String &p_key, const Variant &p_default = Variant()) const override; virtual void set_param(const String &p_key, const Variant &p_val) override; virtual bool has_param(const String &p_key) const override; - virtual Variant get_iinfo_val(const String &p_section, const String &p_prop) const override; + virtual Variant get_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_default = Variant()) const override; virtual void set_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_val) override; // gets the parameters used to import the resource. @@ -347,11 +348,11 @@ class ImportInfoDummy : public ImportInfo { virtual Vector get_dest_files() const override { return dest_files; } virtual void set_dest_files(const Vector p_dest_files) override { dest_files = p_dest_files; } - virtual Variant get_param(const String &p_key) const override { return Variant(); } + virtual Variant get_param(const String &p_key, const Variant &p_default = Variant()) const override { return p_default; } virtual void set_param(const String &p_key, const Variant &p_val) override { return; } virtual bool has_param(const String &p_key) const override { return false; } - virtual Variant get_iinfo_val(const String &p_section, const String &p_prop) const override { return Variant(); } + virtual Variant get_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_default = Variant()) const override { return p_default; } virtual void set_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_val) override { return; } virtual Dictionary get_params() const override { return Dictionary(); } @@ -373,6 +374,7 @@ class ImportInfoRemap : public ImportInfoDummy { virtual Error _load(const String &p_path) override; public: + static String get_remap_path_from_file(const String &p_remap_file_path); ImportInfoRemap(); }; @@ -392,7 +394,7 @@ class ImportInfoGDExt : public ImportInfoDummy { public: Error load_from_string(const String &p_fakepath, const String &p_string); - virtual Variant get_iinfo_val(const String &p_section, const String &p_prop) const override; + virtual Variant get_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_default = Variant()) const override; virtual void set_iinfo_val(const String &p_section, const String &p_prop, const Variant &p_val) override; Vector get_libaries(bool fix_rel_paths = true) const; diff --git a/utility/macros.h b/utility/macros.h index dda443b4c..f052d18c1 100644 --- a/utility/macros.h +++ b/utility/macros.h @@ -1,4 +1,5 @@ #pragma once +#include #include namespace internal { diff --git a/utility/pck_creator.cpp b/utility/pck_creator.cpp index 26a6108c3..a49d0e31a 100644 --- a/utility/pck_creator.cpp +++ b/utility/pck_creator.cpp @@ -25,137 +25,8 @@ void PckCreator::reset() { static const Vector banned_files = { "thumbs.db", ".DS_Store" }; -struct PckCreateListDirTaskData { - const String dir; - const Vector wildcards; - const bool absolute; - const String start_rel; - const Vector excludes; - - struct Token { - String subdir; - Vector ret; - }; - - bool should_filter_file(const String &file) { - for (int64_t j = 0; j < banned_files.size(); j++) { - if (file.ends_with(banned_files[j])) { - return true; - } - } - // we have to check the exclude filters now - for (int64_t j = 0; j < excludes.size(); j++) { - if (file.matchn(excludes[j])) { - return true; - } - } - if (wildcards.size() > 0) { - for (int64_t j = 0; j < wildcards.size(); j++) { - if (file.matchn(wildcards[j])) { - return false; - } - } - return true; - } - return false; - } - - void do_subdir_task(int i, Token *p_subdir) { - Token &token = p_subdir[i]; - token.ret = list_dir(token.subdir, false); - } - - String get_step_description(int i, Token *p_subdir) { - return "Reading folder " + p_subdir[i].subdir + "..."; - } - - Vector run() { - return list_dir(start_rel, true); - } - - Vector list_dir(const String &rel, bool is_main = false) { - Vector ret; - Error err; - Ref da = DirAccess::open(dir.path_join(rel), &err); - ERR_FAIL_COND_V_MSG(da.is_null(), ret, "Failed to open directory " + dir); - - if (da.is_null()) { - return ret; - } - Vector dirs; - Vector files; - - String base = absolute ? dir : ""; - da->list_dir_begin(); - String f = da->get_next(); - while (!f.is_empty()) { - if (f == "." || f == "..") { - f = da->get_next(); - continue; - } else if (da->current_is_dir()) { - dirs.push_back(f); - } else { - files.push_back(f); - } - f = da->get_next(); - } - da->list_dir_end(); - - dirs.sort_custom(); - if (is_main) { - Vector tokens; - for (auto &d : dirs) { - tokens.push_back(PckCreateListDirTaskData::Token{ rel.path_join(d), {} }); - } - - Ref ep; - TaskManager::TaskManagerID group_id = -1; - if (tokens.size() > 0) { - String desc = "Reading folder " + dir + " structure..."; - String task = "ListDirTaskData(" + dir + +")_" + String::num_int64(OS::get_singleton()->get_ticks_usec()); - ep = EditorProgressGDDC::create(nullptr, task, desc, -1, true); - group_id = TaskManager::get_singleton()->add_group_task( - this, &PckCreateListDirTaskData::do_subdir_task, - tokens.ptrw(), tokens.size(), - &PckCreateListDirTaskData::get_step_description, - task, desc, - true, -1, true, ep, 0); - } - // while we wait for the subdirs to be read, we can filter the files - files.sort_custom(); - for (int64_t i = files.size() - 1; i >= 0; i--) { - TaskManager::get_singleton()->update_progress_bg(); - files.write[i] = base.path_join(rel).path_join(files[i]); - if (should_filter_file(files[i])) { - files.remove_at(i); - continue; - } - } - if (group_id != -1) { - TaskManager::get_singleton()->wait_for_task_completion(group_id); - } - for (auto &t : tokens) { - ret.append_array(std::move(t.ret)); - } - ret.append_array(std::move(files)); - } else { - for (auto &d : dirs) { - ret.append_array(list_dir(rel.path_join(d), false)); - } - for (auto &file : files) { - String full_path = base.path_join(rel).path_join(file); - if (!should_filter_file(full_path)) { - ret.append(full_path); - } - } - } - - return ret; - } -}; - Vector PckCreator::get_files_to_pack(const String &p_dir, const Vector &include_filters, const Vector &p_exclude_filters) { - return PckCreateListDirTaskData{ p_dir, include_filters, false, "", p_exclude_filters }.run(); + return gdre::get_recursive_dir_list_multithread(p_dir, include_filters, false, false, p_exclude_filters, banned_files, false, true); } #ifdef DEBUG_ENABLED diff --git a/utility/pck_dumper.cpp b/utility/pck_dumper.cpp index ae1563204..370c2d707 100644 --- a/utility/pck_dumper.cpp +++ b/utility/pck_dumper.cpp @@ -197,10 +197,11 @@ Error PckDumper::_pck_dump_to_dir( gdre::CaselessHashSet seen_paths; HashSet files_to_extract_set = gdre::vector_to_hashset(files_to_extract); for (int i = 0; i < files.size(); i++) { - if (files.get(i)->is_dummy()) { + const auto &file = files.get(i); + if (file->is_dummy() && file->get_size() <= 1) { + // empty dummy file continue; } - const auto &file = files.get(i); String path = file->get_path(); if (!files_to_extract_set.is_empty() && !files_to_extract_set.has(path)) { continue;