From 4fc1f0491e652767c9662332e4c1641e23685182 Mon Sep 17 00:00:00 2001 From: WerWolv Date: Sun, 17 Aug 2025 14:33:34 +0200 Subject: [PATCH] fix: Various font loading issues and Weblate fuck ups --- .../include/hex/api/localization_manager.hpp | 17 ++-- .../source/api/localization_manager.cpp | 97 +++++++++++++++---- plugins/fonts/romfs/lang/languages.json | 4 + .../fonts/romfs/lang/{ru.json => ru_RU.json} | 0 .../remote/romfs/lang/{de.json => de_DE.json} | 0 .../remote/romfs/lang/{es.json => es_ES.json} | 0 .../remote/romfs/lang/{fr.json => fr_FR.json} | 0 .../remote/romfs/lang/{hu.json => hu_HU.json} | 0 .../remote/romfs/lang/{it.json => it_IT.json} | 0 .../remote/romfs/lang/{ja.json => ja_JP.json} | 0 .../remote/romfs/lang/{ko.json => ko_KR.json} | 0 plugins/remote/romfs/lang/languages.json | 48 +++++++++ .../remote/romfs/lang/{pl.json => pl_PL.json} | 0 .../remote/romfs/lang/{ru.json => ru_RU.json} | 0 .../romfs/lang/{zh_Hans.json => zh_CN.json} | 0 .../romfs/lang/{zh_Hant.json => zh_TW.json} | 0 .../romfs/lang/{es.json => es_ES.json} | 0 .../romfs/lang/{it.json => it_IT.json} | 0 .../romfs/lang/{ja.json => ja_JP.json} | 0 .../romfs/lang/{ko.json => ko_KR.json} | 0 .../script_loader/romfs/lang/languages.json | 20 ++++ .../romfs/lang/{es.json => es_ES.json} | 0 .../romfs/lang/{it.json => it_IT.json} | 0 .../romfs/lang/{ja.json => ja_JP.json} | 0 plugins/windows/romfs/lang/languages.json | 12 +++ 25 files changed, 170 insertions(+), 28 deletions(-) rename plugins/fonts/romfs/lang/{ru.json => ru_RU.json} (100%) rename plugins/remote/romfs/lang/{de.json => de_DE.json} (100%) rename plugins/remote/romfs/lang/{es.json => es_ES.json} (100%) rename plugins/remote/romfs/lang/{fr.json => fr_FR.json} (100%) rename plugins/remote/romfs/lang/{hu.json => hu_HU.json} (100%) rename plugins/remote/romfs/lang/{it.json => it_IT.json} (100%) rename plugins/remote/romfs/lang/{ja.json => ja_JP.json} (100%) rename plugins/remote/romfs/lang/{ko.json => ko_KR.json} (100%) rename plugins/remote/romfs/lang/{pl.json => pl_PL.json} (100%) rename plugins/remote/romfs/lang/{ru.json => ru_RU.json} (100%) rename plugins/remote/romfs/lang/{zh_Hans.json => zh_CN.json} (100%) rename plugins/remote/romfs/lang/{zh_Hant.json => zh_TW.json} (100%) rename plugins/script_loader/romfs/lang/{es.json => es_ES.json} (100%) rename plugins/script_loader/romfs/lang/{it.json => it_IT.json} (100%) rename plugins/script_loader/romfs/lang/{ja.json => ja_JP.json} (100%) rename plugins/script_loader/romfs/lang/{ko.json => ko_KR.json} (100%) rename plugins/windows/romfs/lang/{es.json => es_ES.json} (100%) rename plugins/windows/romfs/lang/{it.json => it_IT.json} (100%) rename plugins/windows/romfs/lang/{ja.json => ja_JP.json} (100%) diff --git a/lib/libimhex/include/hex/api/localization_manager.hpp b/lib/libimhex/include/hex/api/localization_manager.hpp index 77e8ddc19..a9f5fcab0 100644 --- a/lib/libimhex/include/hex/api/localization_manager.hpp +++ b/lib/libimhex/include/hex/api/localization_manager.hpp @@ -13,6 +13,9 @@ EXPORT_MODULE namespace hex { + struct UnlocalizedString; + using LanguageId = std::string; + namespace LocalizationManager { struct PathEntry { @@ -21,25 +24,23 @@ EXPORT_MODULE namespace hex { }; struct LanguageDefinition { - std::string id; + LanguageId id; std::string name, nativeName; std::string flag; std::string filePath; - std::string fallbackLanguageId; + LanguageId fallbackLanguageId; std::vector languageFilePaths; }; void addLanguages(const std::string_view &languageList, std::function callback); - void setLanguage(const std::string &languageId); - [[nodiscard]] const std::string& getSelectedLanguageId(); - [[nodiscard]] const std::string& get(const std::string &unlocalizedString); - [[nodiscard]] const std::map& getLanguageDefinitions(); + void setLanguage(const LanguageId &languageId); + [[nodiscard]] const LanguageId& getSelectedLanguageId(); + [[nodiscard]] const std::string& get(const LanguageId& languageId, const UnlocalizedString &unlocalizedString); + [[nodiscard]] const std::map& getLanguageDefinitions(); } - struct UnlocalizedString; - class LangConst; class Lang { diff --git a/lib/libimhex/source/api/localization_manager.cpp b/lib/libimhex/source/api/localization_manager.cpp index 0eea641e9..399b233be 100644 --- a/lib/libimhex/source/api/localization_manager.cpp +++ b/lib/libimhex/source/api/localization_manager.cpp @@ -2,6 +2,7 @@ #include #include +#include #include @@ -11,9 +12,9 @@ namespace hex { namespace { - AutoReset> s_languageDefinitions; + AutoReset> s_languageDefinitions; AutoReset> s_localizations; - AutoReset s_selectedLanguageId; + AutoReset s_selectedLanguageId; } @@ -40,44 +41,100 @@ namespace hex { definition.fallbackLanguageId = item["fallback"].get(); } - definition.languageFilePaths.emplace_back(PathEntry{ item["path"].get(), std::move(callback) }); + const auto path = item["path"].get(); + + definition.languageFilePaths.emplace_back(PathEntry{ path, callback }); } } - static void populateLocalization(const std::string &languageId) { - if (languageId.empty()) - return; + static LanguageId findBestLanguageMatch(LanguageId languageId) { + if (s_languageDefinitions->contains(languageId)) + return languageId; - log::debug("Populating localization for language: {}", languageId); + if (const auto pos = languageId.find('_'); pos != std::string::npos) { + // Turn language Ids like "en_US" into "en-US" + languageId[pos] = '-'; + } - const auto &definition = (*s_languageDefinitions)[languageId]; - for (const auto &path : definition.languageFilePaths) { - const auto translation = path.callback(path.path); - const auto json = nlohmann::json::parse(translation); + if (const auto pos = languageId.find('-'); pos != std::string::npos) { + // Try to find a match with the language code without region + languageId = languageId.substr(0, pos); - for (const auto &entry : json.items()) { - auto value = entry.value().get(); - if (value.empty()) - continue; - - s_localizations->try_emplace(LangConst::hash(entry.key()), std::move(value)); + for (const auto &definition : *s_languageDefinitions) { + if (definition.first.starts_with(languageId) || definition.first.starts_with(toLower(languageId))) { + return definition.first; + } } } - populateLocalization(definition.fallbackLanguageId); + // Fall back to English if no better match was found + return "en-US"; } - void setLanguage(const std::string &languageId) { + static void populateLocalization(LanguageId languageId, std::unordered_map &localizations) { + if (languageId.empty()) + return; + + languageId = findBestLanguageMatch(languageId); + + if (const auto it = s_languageDefinitions->find(languageId); it == s_languageDefinitions->end()) { + log::error("No language definition found for language: {}", languageId); + + constexpr static auto FallbackLanguageId = "en-US"; + if (languageId != FallbackLanguageId) + populateLocalization(FallbackLanguageId, localizations); + } else { + const auto &definition = it->second; + for (const auto &path : definition.languageFilePaths) { + try { + const auto translation = path.callback(path.path); + const auto json = nlohmann::json::parse(translation); + + for (const auto &entry : json.items()) { + auto value = entry.value().get(); + if (value.empty()) + continue; + + localizations.try_emplace(LangConst::hash(entry.key()), std::move(value)); + } + } catch (std::exception &e) { + log::error("Failed to load localization file '{}': {}", path.path, e.what()); + } + } + + populateLocalization(definition.fallbackLanguageId, localizations); + } + } + + void setLanguage(const LanguageId &languageId) { + if (*s_selectedLanguageId == languageId) + return; + s_localizations->clear(); s_selectedLanguageId = languageId; - populateLocalization(languageId); + populateLocalization(languageId, s_localizations); } [[nodiscard]] const std::string& getSelectedLanguageId() { return *s_selectedLanguageId; } + [[nodiscard]] const std::string& get(const LanguageId &languageId, const UnlocalizedString &unlocalizedString) { + static AutoReset currentLanguageId; + static AutoReset> loadedLocalization; + static std::mutex mutex; + + std::lock_guard lock(mutex); + if (*currentLanguageId != languageId) { + currentLanguageId = languageId; + loadedLocalization->clear(); + populateLocalization(languageId, *loadedLocalization); + } + + return (*loadedLocalization)[LangConst::hash(unlocalizedString.get())]; + } + const std::map& getLanguageDefinitions() { return *s_languageDefinitions; } diff --git a/plugins/fonts/romfs/lang/languages.json b/plugins/fonts/romfs/lang/languages.json index 43941db43..24b7558ef 100644 --- a/plugins/fonts/romfs/lang/languages.json +++ b/plugins/fonts/romfs/lang/languages.json @@ -27,6 +27,10 @@ "code": "pt-BR", "path": "lang/pt_BR.json" }, + { + "code": "ru-RU", + "path": "lang/ru_RU.json" + }, { "code": "pl-PL", "path": "lang/pl_PL.json" diff --git a/plugins/fonts/romfs/lang/ru.json b/plugins/fonts/romfs/lang/ru_RU.json similarity index 100% rename from plugins/fonts/romfs/lang/ru.json rename to plugins/fonts/romfs/lang/ru_RU.json diff --git a/plugins/remote/romfs/lang/de.json b/plugins/remote/romfs/lang/de_DE.json similarity index 100% rename from plugins/remote/romfs/lang/de.json rename to plugins/remote/romfs/lang/de_DE.json diff --git a/plugins/remote/romfs/lang/es.json b/plugins/remote/romfs/lang/es_ES.json similarity index 100% rename from plugins/remote/romfs/lang/es.json rename to plugins/remote/romfs/lang/es_ES.json diff --git a/plugins/remote/romfs/lang/fr.json b/plugins/remote/romfs/lang/fr_FR.json similarity index 100% rename from plugins/remote/romfs/lang/fr.json rename to plugins/remote/romfs/lang/fr_FR.json diff --git a/plugins/remote/romfs/lang/hu.json b/plugins/remote/romfs/lang/hu_HU.json similarity index 100% rename from plugins/remote/romfs/lang/hu.json rename to plugins/remote/romfs/lang/hu_HU.json diff --git a/plugins/remote/romfs/lang/it.json b/plugins/remote/romfs/lang/it_IT.json similarity index 100% rename from plugins/remote/romfs/lang/it.json rename to plugins/remote/romfs/lang/it_IT.json diff --git a/plugins/remote/romfs/lang/ja.json b/plugins/remote/romfs/lang/ja_JP.json similarity index 100% rename from plugins/remote/romfs/lang/ja.json rename to plugins/remote/romfs/lang/ja_JP.json diff --git a/plugins/remote/romfs/lang/ko.json b/plugins/remote/romfs/lang/ko_KR.json similarity index 100% rename from plugins/remote/romfs/lang/ko.json rename to plugins/remote/romfs/lang/ko_KR.json diff --git a/plugins/remote/romfs/lang/languages.json b/plugins/remote/romfs/lang/languages.json index fcb78112b..24b7558ef 100644 --- a/plugins/remote/romfs/lang/languages.json +++ b/plugins/remote/romfs/lang/languages.json @@ -2,5 +2,53 @@ { "code": "en-US", "path": "lang/en_US.json" + }, + { + "code": "es-ES", + "path": "lang/es_ES.json" + }, + { + "code": "fr-FR", + "path": "lang/fr_FR.json" + }, + { + "code": "de-DE", + "path": "lang/de_DE.json" + }, + { + "code": "it-IT", + "path": "lang/it_IT.json" + }, + { + "code": "hu-HU", + "path": "lang/hu_HU.json" + }, + { + "code": "pt-BR", + "path": "lang/pt_BR.json" + }, + { + "code": "ru-RU", + "path": "lang/ru_RU.json" + }, + { + "code": "pl-PL", + "path": "lang/pl_PL.json" + }, + { + "code": "zh-CN", + "path": "lang/zh_CN.json" + }, + { + "code": "zh-TW", + "path": "lang/zh_TW.json" + }, + { + "code": "ja-JP", + "path": "lang/ja_JP.json" + }, + { + "code": "ko-KR", + "path": "lang/ko_KR.json" } ] \ No newline at end of file diff --git a/plugins/remote/romfs/lang/pl.json b/plugins/remote/romfs/lang/pl_PL.json similarity index 100% rename from plugins/remote/romfs/lang/pl.json rename to plugins/remote/romfs/lang/pl_PL.json diff --git a/plugins/remote/romfs/lang/ru.json b/plugins/remote/romfs/lang/ru_RU.json similarity index 100% rename from plugins/remote/romfs/lang/ru.json rename to plugins/remote/romfs/lang/ru_RU.json diff --git a/plugins/remote/romfs/lang/zh_Hans.json b/plugins/remote/romfs/lang/zh_CN.json similarity index 100% rename from plugins/remote/romfs/lang/zh_Hans.json rename to plugins/remote/romfs/lang/zh_CN.json diff --git a/plugins/remote/romfs/lang/zh_Hant.json b/plugins/remote/romfs/lang/zh_TW.json similarity index 100% rename from plugins/remote/romfs/lang/zh_Hant.json rename to plugins/remote/romfs/lang/zh_TW.json diff --git a/plugins/script_loader/romfs/lang/es.json b/plugins/script_loader/romfs/lang/es_ES.json similarity index 100% rename from plugins/script_loader/romfs/lang/es.json rename to plugins/script_loader/romfs/lang/es_ES.json diff --git a/plugins/script_loader/romfs/lang/it.json b/plugins/script_loader/romfs/lang/it_IT.json similarity index 100% rename from plugins/script_loader/romfs/lang/it.json rename to plugins/script_loader/romfs/lang/it_IT.json diff --git a/plugins/script_loader/romfs/lang/ja.json b/plugins/script_loader/romfs/lang/ja_JP.json similarity index 100% rename from plugins/script_loader/romfs/lang/ja.json rename to plugins/script_loader/romfs/lang/ja_JP.json diff --git a/plugins/script_loader/romfs/lang/ko.json b/plugins/script_loader/romfs/lang/ko_KR.json similarity index 100% rename from plugins/script_loader/romfs/lang/ko.json rename to plugins/script_loader/romfs/lang/ko_KR.json diff --git a/plugins/script_loader/romfs/lang/languages.json b/plugins/script_loader/romfs/lang/languages.json index ed8fa3a5d..24b7558ef 100644 --- a/plugins/script_loader/romfs/lang/languages.json +++ b/plugins/script_loader/romfs/lang/languages.json @@ -3,6 +3,10 @@ "code": "en-US", "path": "lang/en_US.json" }, + { + "code": "es-ES", + "path": "lang/es_ES.json" + }, { "code": "fr-FR", "path": "lang/fr_FR.json" @@ -11,10 +15,18 @@ "code": "de-DE", "path": "lang/de_DE.json" }, + { + "code": "it-IT", + "path": "lang/it_IT.json" + }, { "code": "hu-HU", "path": "lang/hu_HU.json" }, + { + "code": "pt-BR", + "path": "lang/pt_BR.json" + }, { "code": "ru-RU", "path": "lang/ru_RU.json" @@ -30,5 +42,13 @@ { "code": "zh-TW", "path": "lang/zh_TW.json" + }, + { + "code": "ja-JP", + "path": "lang/ja_JP.json" + }, + { + "code": "ko-KR", + "path": "lang/ko_KR.json" } ] \ No newline at end of file diff --git a/plugins/windows/romfs/lang/es.json b/plugins/windows/romfs/lang/es_ES.json similarity index 100% rename from plugins/windows/romfs/lang/es.json rename to plugins/windows/romfs/lang/es_ES.json diff --git a/plugins/windows/romfs/lang/it.json b/plugins/windows/romfs/lang/it_IT.json similarity index 100% rename from plugins/windows/romfs/lang/it.json rename to plugins/windows/romfs/lang/it_IT.json diff --git a/plugins/windows/romfs/lang/ja.json b/plugins/windows/romfs/lang/ja_JP.json similarity index 100% rename from plugins/windows/romfs/lang/ja.json rename to plugins/windows/romfs/lang/ja_JP.json diff --git a/plugins/windows/romfs/lang/languages.json b/plugins/windows/romfs/lang/languages.json index 78b2f2f59..24b7558ef 100644 --- a/plugins/windows/romfs/lang/languages.json +++ b/plugins/windows/romfs/lang/languages.json @@ -15,6 +15,10 @@ "code": "de-DE", "path": "lang/de_DE.json" }, + { + "code": "it-IT", + "path": "lang/it_IT.json" + }, { "code": "hu-HU", "path": "lang/hu_HU.json" @@ -27,6 +31,10 @@ "code": "ru-RU", "path": "lang/ru_RU.json" }, + { + "code": "pl-PL", + "path": "lang/pl_PL.json" + }, { "code": "zh-CN", "path": "lang/zh_CN.json" @@ -35,6 +43,10 @@ "code": "zh-TW", "path": "lang/zh_TW.json" }, + { + "code": "ja-JP", + "path": "lang/ja_JP.json" + }, { "code": "ko-KR", "path": "lang/ko_KR.json"