imports/imhex
1
0
Fork 0
mirror of https://github.com/WerWolv/ImHex.git synced 2026-03-28 07:47:03 -05:00
Code Issues Packages Projects Releases Wiki Activity

Make the syntax highlighter more thread safety aware (#2585)

Browse Source 
By creating copies of the required inputs on the main thread just before
the task is spawned. A;so if task is still running when new data can be
copied then the task is interrupted thus avoiding concurrency without
mutexes. Atomics are used to signal state information used to determine
what and when to spawn.

Also includes update to pattern editor library and some fixes to syntax
highlighting error when custom types defined inside namespaces were used
inside the namespaces without the full qualified name and other small
changes mostly to improve the current style.
This commit is contained in:
paxcut
2025-12-26 20:21:19 -07:00
committed by GitHub
parent ba7e789a80
commit 64cbd5fc8d
8 changed files with 876 additions and 816 deletions
Download Patch File Download Diff File Expand all files Collapse all files

308
plugins/builtin/include/content/text_highlighting/pattern_language.hpp
View File

@@ -1,13 +1,12 @@
#pragma once
#include <pl/core/token.hpp>
#include <pl/core/preprocessor.hpp>
#include <pl/pattern_language.hpp>
#include <pl/helpers/safe_iterator.hpp>
#include <ui/text_editor.hpp>
#include <hex/helpers/types.hpp>
#include <hex/helpers/logger.hpp>
namespace pl {
class PatternLanguage;
}
namespace hex::plugin::builtin {
class ViewPatternEditor;
@@ -16,7 +15,6 @@ namespace hex::plugin::builtin {
class Interval;
using Token = pl::core::Token;
using ASTNode = pl::core::ast::ASTNode;
using ExcludedLocation = pl::core::Preprocessor::ExcludedLocation;
using CompileError = pl::core::err::CompileError;
using Identifier = Token::Identifier;
using IdentifierType = Identifier::IdentifierType;
@@ -24,12 +22,20 @@ namespace hex::plugin::builtin {
using OrderedBlocks = std::map<Interval,std::string>;
using Scopes = std::set<Interval>;
using Location = pl::core::Location;
using VectorString = std::vector<std::string>;
using TokenIter = pl::hlp::SafeIterator<std::vector<Token>::const_iterator>;
using VariableScopes = std::map<std::string,Scopes>;
using Inheritances = std::map<std::string,std::vector<std::string>>;
using Inheritances = std::map<std::string,VectorString>;
using IdentifierTypeColor = std::map<Identifier::IdentifierType,ui::TextEditor::PaletteIndex>;
using TokenTypeColor = std::map<Token::Type,ui::TextEditor::PaletteIndex>;
using TokenColor = std::map<Token *,ui::TextEditor::PaletteIndex>;
using TokenColor = std::map<i32, ui::TextEditor::PaletteIndex>;
using Types = std::map<std::string, pl::hlp::safe_shared_ptr<pl::core::ast::ASTNodeTypeDecl>>;
using ParsedImports = std::map<std::string,std::vector<Token>>;
using Str2StrMap = std::map<std::string,std::string>;
using CompileErrors = std::vector<CompileError>;
using TokenSequence = std::vector<Token>;
using TokenIdVector = std::vector<i32>;
using Instances = std::map<std::string,std::vector<i32>>;
struct ParentDefinition;
struct Definition {
@@ -48,6 +54,29 @@ namespace hex::plugin::builtin {
i32 tokenIndex;
Location location;
};
struct RequiredInputs {
friend class TextHighlighter;
private:
TextHighlighter *m_textHighlighter;
Types definedTypes;
VectorString usedNamespaces;
ParsedImports parsedImports;
Str2StrMap importedHeaders;
TokenSequence fullTokens;
std::string editedText;
CompileErrors compileErrors;
VectorString linesOfColors;
public:
RequiredInputs() : m_textHighlighter(nullptr) {};
explicit RequiredInputs(TextHighlighter *textHighlighter) : m_textHighlighter(textHighlighter) {}
void setRequiredInputs();
void setTypes();
void setNamespaces();
void setImports();
void setText();
void setCompileErrors();
};
/// to define functions and types
using Definitions = std::map<std::string,ParentDefinition>;
/// to define global variables
@@ -55,16 +84,15 @@ namespace hex::plugin::builtin {
/// to define UDT and function variables
using VariableMap = std::map<std::string,Variables>;
private:
std::string m_text;
std::vector<std::string> m_lines;
std::vector<i32> m_firstTokenIdOfLine;
VectorString m_lines;
TokenIdVector m_firstTokenIdOfLine;
ViewPatternEditor *m_viewPatternEditor;
std::vector<ExcludedLocation> m_excludedLocations;
std::vector<Token> m_tokens;
TokenColor m_tokenColors;
std::unique_ptr<pl::PatternLanguage> *patternLanguage;
std::vector<CompileError> m_compileErrors;
std::map<std::string,std::vector<i32>> m_instances;
Instances m_instances;
Definitions m_UDTDefinitions;
Definitions m_functionDefinitions;
@@ -78,15 +106,16 @@ namespace hex::plugin::builtin {
VariableMap m_functionVariables;
Variables m_globalVariables;
std::map<std::string, std::vector<Token>> m_parsedImports;
std::map<std::string,std::string> m_attributeFunctionArgumentType;
std::map<std::string,std::string> m_typeDefMap;
std::map<std::string,std::string> m_typeDefInvMap;
std::vector<std::string> m_nameSpaces;
std::vector<std::string> m_UDTs;
Str2StrMap m_attributeFunctionArgumentType;
Str2StrMap m_typeDefMap;
Str2StrMap m_typeDefInvMap;
VectorString m_UDTs;
std::set<i32> m_taggedIdentifiers;
std::set<i32> m_memberChains;
std::set<i32> m_scopeChains;
RequiredInputs m_requiredInputs;
TokenIter m_curr;
TokenIter m_startToken, m_originalPosition, m_partOriginalPosition;
@@ -98,7 +127,6 @@ namespace hex::plugin::builtin {
const static IdentifierTypeColor m_identifierTypeColor;
const static TokenTypeColor m_tokenTypeColor;
i32 m_runningColorizers=0;
public:
/// Intervals are the sets finite contiguous non-negative integer that
@@ -118,49 +146,29 @@ namespace hex::plugin::builtin {
i32 start;
i32 end;
Interval() : start(0), end(0) {}
Interval(i32 start, i32 end) : start(start), end(end) {
if (start > end)
throw std::invalid_argument("Interval start must be less than or equal to end");
}
bool operator<(const Interval &other) const {
return other.end > end;
}
bool operator>(const Interval &other) const {
return end > other.end;
}
bool operator==(const Interval &other) const {
return start == other.start && end == other.end;
}
bool operator!=(const Interval &other) const {
return start != other.start || end != other.end;
}
bool operator<=(const Interval &other) const {
return other.end >= end;
}
bool operator>=(const Interval &other) const {
return end >= other.end;
}
bool contains(const Interval &other) const {
return other.start >= start && other.end <= end;
}
bool contains(i32 value) const {
return value >= start && value <= end;
}
bool contiguous(const Interval &other) const {
return ((start - other.end) == 1 || (other.start - end) == 1);
}
Interval(i32 start, i32 end);
bool operator<(const Interval &other) const;
bool operator>(const Interval &other) const;
bool operator==(const Interval &other) const;
bool operator!=(const Interval &other) const;
bool operator<=(const Interval &other) const;
bool operator>=(const Interval &other) const;
bool contains(const Interval &other) const;
bool contains(i32 value) const ;
bool contiguous(const Interval &other) const;
};
std::atomic<bool> m_needsToUpdateColors = true;
std::atomic<bool> m_wasInterrupted = false;
std::atomic<bool> m_interrupt = false;
constexpr static u32 Normal = 0;
constexpr static u32 Not = 1;
pl::PatternLanguage *getPatternLanguage();
void updateRequiredInputs();
RequiredInputs& getRequiredInputs();
ViewPatternEditor* getViewPatternEditor();
void setViewPatternEditor(ViewPatternEditor *viewPatternEditor);
void interrupt() {
m_interrupt = true;
}
TextHighlighter(ViewPatternEditor *viewPatternEditor, std::unique_ptr<pl::PatternLanguage> *patternLanguage ) :
m_viewPatternEditor(viewPatternEditor), patternLanguage(patternLanguage), m_needsToUpdateColors(true) {}
TextHighlighter();
~TextHighlighter();
explicit TextHighlighter(ViewPatternEditor *viewPatternEditor) : m_viewPatternEditor(viewPatternEditor) {}
/**
* @brief Entry point to syntax highlighting
*/
@@ -184,14 +192,16 @@ namespace hex::plugin::builtin {
/**
* @brief Only identifiers not in chains should remain
*/
void colorRemainingIdentifierTokens();
/**
* @brief Renders compile errors in real time
*/
void renderErrors();
/// A token range is the set of token indices of a definition. The namespace token
/// ranges are obtained first because they are needed to obtain unique identifiers.
void getAllTokenRanges(IdentifierType identifierTypeToSearch);
void getTokenRanges(IdentifierType identifierTypeToSearch);
/// The global scope is the complement of the union of all the function and UDT token ranges
void getGlobalTokenRanges();
/// If the current token is a function or UDT, creates a map entry from the name to the token range. These are ordered alphabetically by name.
@@ -217,14 +227,14 @@ namespace hex::plugin::builtin {
/// Chains are sequences of identifiers separated by scope resolution or dot operators.
void fixChains();
bool colorSeparatorScopeChain();
bool colorRealSeparatorScopeChain();
bool colorImplicitSeparatorScopeChain();
bool colorOperatorDotChain();
/// Returns the next/previous valid source code line
u32 nextLine(u32 line);
u32 previousLine(u32 line);
/// Loads the source code and calculates the first token index of each line
void loadText();
/// Used to obtain the color to be applied.
ui::TextEditor::PaletteIndex getPaletteIndex(Token::Literal *literal);
/// The complement of a set is also known as its inverse
void invertGlobalTokenRange();
/// Starting at the identifier, it tracks all the scope resolution and dot operators and returns the full chain without arrays, templates, pointers,...
@@ -251,11 +261,11 @@ namespace hex::plugin::builtin {
void skipDelimiters(i32 maxSkipCount, Token delimiter[2], i8 increment);
void skipToken(Token token, i8 step=1);
/// from given or current names find the corresponding definition
bool findIdentifierDefinition(Definition &result, const std::string &optionalIdentifierName = "", std::string optionalName = "", bool setInstances = false);
bool findIdentifierDefinition(Definition &result, const std::string &optionalIdentifierName = "", std::string optionalName = "", bool optional = false);
/// To deal with the Parent keyword
std::optional<Definition> setChildrenTypes();
bool findParentTypes(std::vector<std::string> &parentTypes, const std::string &optionalUDTName="");
bool findAllParentTypes(std::vector<std::string> &parentTypes, std::vector<Identifier *> &identifiers, std::string &optionalFullName);
bool findParentTypes(VectorString &parentTypes, const std::string &optionalName="");
bool findAllParentTypes(VectorString &parentTypes, std::vector<Identifier *> &identifiers, std::string &optionalFullName);
bool tryParentType(const std::string &parentType, std::string &variableName, std::optional<Definition> &result, std::vector<Identifier *> &identifiers);
/// Convenience function
bool isTokenIdValid(i32 tokenId);
@@ -274,160 +284,22 @@ namespace hex::plugin::builtin {
void getTokenIdForArgument(i32 start, i32 argNumber, Token delimiter);
///Creates a map from function name to argument type
void linkAttribute();
/// Comment and strings usethese function to determine their coordinates
template<typename T> ui::TextEditor::Coordinates commentCoordinates(Token *token);
ui::TextEditor::Coordinates stringCoordinates();
/// Returns the number of tasks highlighting code. Shouldn't be > 1
i32 getRunningColorizers() {
return m_runningColorizers;
}
enum class HighlightStage {
Starting,
NamespaceTokenRanges,
UDTTokenRanges,
FunctionTokenRanges,
GlobalTokenRanges,
FixGlobalVariables,
SetInitialColors,
LoadInstances,
AttributeTokenRanges,
Definitions,
FixAutos,
FixChains,
ExcludedLocations,
ColorRemainingTokens,
SetRequestedIdentifierColors,
Stage1,
Stage2,
Stage3,
Stage4,
Stage5,
Stage6,
Stage7,
Stage8,
Stage9,
Stage10,
Stage11,
};
HighlightStage m_highlightStage = HighlightStage::Starting;
/// The following functions were copied from the parser and some were modified
template<typename T>
T *getValue(const i32 index) {
return const_cast<T*>(std::get_if<T>(&m_curr[index].value));
}
void next(i32 count = 1) {
if (m_interrupt) {
m_interrupt = false;
throw std::out_of_range("Highlights were deliberately interrupted");
}
if (count == 0)
return;
i32 id = getTokenId(m_curr->location);
i32 maxChange;
if (count > 0)
maxChange = std::min(count,static_cast<i32>(m_tokens.size() - id));
else
maxChange = -std::min(-count,id);
m_curr += maxChange;
}
constexpr static u32 Normal = 0;
constexpr static u32 Not = 1;
bool begin() {
m_originalPosition = m_curr;
return true;
}
void partBegin() {
m_partOriginalPosition = m_curr;
}
void reset() {
m_curr = m_originalPosition;
}
void partReset() {
m_curr = m_partOriginalPosition;
}
bool resetIfFailed(const bool value) {
if (!value) reset();
return value;
}
template<auto S = Normal>
bool sequenceImpl() {
if constexpr (S == Normal)
return true;
else if constexpr (S == Not)
return false;
else
std::unreachable();
}
template<auto S = Normal>
bool matchOne(const Token &token) {
if constexpr (S == Normal) {
if (!peek(token)) {
partReset();
return false;
}
next();
return true;
} else if constexpr (S == Not) {
if (!peek(token))
return true;
next();
partReset();
return false;
} else
std::unreachable();
}
template<auto S = Normal>
bool sequenceImpl(const auto &... args) {
return (matchOne<S>(args) && ...);
}
template<auto S = Normal>
bool sequence(const Token &token, const auto &... args) {
partBegin();
return sequenceImpl<S>(token, args...);
}
bool isValid() {
Token token;
try {
token = m_curr[0];
}
catch (const std::out_of_range &e) {
auto t = e.what();
if (t == nullptr)
return false;
return false;
}
if (!isLocationValid(token.location))
return false;
return true;
}
bool peek(const Token &token, const i32 index = 0) {
if (!isValid())
return false;
i32 id = getTokenId(m_curr->location);
if (id+index < 0 || id+index >= (i32)m_tokens.size())
return false;
return m_curr[index].type == token.type && m_curr[index] == token.value;
}
template<typename T> T *getValue(const i32 index);
void next(i32 count = 1);
bool begin();
void partBegin();
void reset();
void partReset();
bool resetIfFailed(const bool value) ;
template<auto S = Normal> bool sequenceImpl();
template<auto S = Normal> bool matchOne(const Token &token);
template<auto S = Normal> bool sequenceImpl(const auto &... args);
template<auto S = Normal> bool sequence(const Token &token, const auto &... args);
bool isValid();
bool peek(const Token &token, const i32 index = 0);
};
}

72
plugins/builtin/include/content/views/view_pattern_editor.hpp
View File

@@ -9,7 +9,6 @@
#include <ui/text_editor.hpp>
#include <content/text_highlighting/pattern_language.hpp>
#include <hex/api/content_registry/settings.hpp>
#include <hex/helpers/magic.hpp>
#include <ui/pattern_drawer.hpp>
@@ -32,52 +31,34 @@ namespace hex::plugin::builtin {
std::string m_sharedSource;
};
using TextHighlighter = hex::plugin::builtin::TextHighlighter;
class ViewPatternEditor : public View::Window {
public:
ViewPatternEditor();
~ViewPatternEditor() override;
void drawAlwaysVisibleContent() override;
std::unique_ptr<pl::PatternLanguage> *getPatternLanguage() {
return &m_editorRuntime;
}
ui::TextEditor *getTextEditor() {
auto provider = ImHexApi::Provider::get();
if (provider == nullptr)
return nullptr;
return &m_textEditor.get(provider);
}
bool getChangesWereParsed() const {
return m_changesWereParsed;
}
u32 getRunningParsers () const {
return m_runningParsers;
}
u32 getRunningEvaluators () const {
return m_runningEvaluators;
}
void setChangesWereParsed(bool changesWereParsed) {
m_changesWereParsed = changesWereParsed;
}
std::unique_ptr<pl::PatternLanguage> *getPatternLanguage() { return &m_editorRuntime; }
ui::TextEditor *getTextEditor();
u32 getRunningParsers () const { return m_runningParsers;}
u32 getRunningEvaluators () const { return m_runningEvaluators;}
u32 getRunningHighlighters () const { return m_runningHighlighters;}
void incrementRunningParsers( i32 amount) { m_runningParsers += amount; }
void incrementRunningEvaluators( i32 amount) { m_runningEvaluators += amount; }
void incrementRunningHighlighters( i32 amount) { m_runningHighlighters += amount; }
bool hasUnevaluatedChanges(prv::Provider *provider) const;
void setChangesWereParsed(bool changesWereParsed) { m_changesWereParsed = changesWereParsed;}
void setChangesWereColored(bool changesWereColored) { m_changesWereColored = changesWereColored; }
void drawContent() override;
void setPopupWindowHeight(u32 height) { m_popupWindowHeight = height; }
u32 getPopupWindowHeight() const { return m_popupWindowHeight; }
enum class DangerousFunctionPerms : u8 {
Ask,
Allow,
Deny
};
enum class DangerousFunctionPerms : u8 { Ask, Allow, Deny };
void drawHelpText() override;
void setWasInterrupted(bool wasInterrupted) { m_wasInterrupted = wasInterrupted; }
bool wasInterrupted() const { return m_wasInterrupted;}
void resetInterrupt() { m_wasInterrupted = false; m_interrupt = false;}
void interrupt() { m_interrupt = true; }
private:
class PopupAcceptPattern;
@@ -127,19 +108,26 @@ namespace hex::plugin::builtin {
std::atomic<u32> m_runningEvaluators = 0;
std::atomic<u32> m_runningParsers = 0;
std::atomic<u32> m_runningHighlighters = 0;
std::atomic<bool> m_changesWereParsed = false;
PerProvider<bool> m_hasUnevaluatedChanges;
std::atomic<bool> m_changesWereParsed;
PerProvider<bool> m_hasUncoloredChanges;
std::atomic<bool> m_changesWereColored;
std::atomic<bool> m_wasInterrupted;
std::atomic<bool> m_interrupt;
std::chrono::time_point<std::chrono::steady_clock> m_lastEditorChangeTime;
PerProvider<ui::TextEditor> m_textEditor, m_consoleEditor;
PerProvider<TextHighlighter> m_textHighlighter;
std::atomic<bool> m_consoleNeedsUpdate = false;
std::atomic<bool> m_dangerousFunctionCalled = false;
std::atomic<DangerousFunctionPerms> m_dangerousFunctionsAllowed = DangerousFunctionPerms::Ask;
ContentRegistry::Settings::SettingsVariable<bool, "hex.builtin.setting.general", "hex.builtin.setting.general.suggest_patterns"> m_suggestSupportedPatterns = true;
ContentRegistry::Settings::SettingsVariable<bool, "hex.builtin.setting.general", "hex.builtin.setting.general.auto_apply_patterns"> m_autoApplyPatterns = false;
bool m_suggestSupportedPatterns = true;
bool m_autoApplyPatterns = false;
PerProvider<ui::VisualizerDrawer> m_visualizerDrawer;
bool m_tooltipJustOpened = false;
@@ -178,7 +166,7 @@ namespace hex::plugin::builtin {
std::array<AccessData, 512> m_accessHistory = {};
u32 m_accessHistoryIndex = 0;
ContentRegistry::Settings::SettingsVariable<bool, "hex.builtin.setting.hex_editor", "hex.builtin.setting.hex_editor.pattern_parent_highlighting"> m_parentHighlightingEnabled = false;
bool m_parentHighlightingEnabled = true;
bool m_replaceMode = false;
bool m_openFindReplacePopUp = false;
bool m_openGotoLinePopUp = false;
@@ -188,13 +176,14 @@ namespace hex::plugin::builtin {
PerProvider<bool> m_ignoreNextChangeEvent;
PerProvider<bool> m_changeEventAcknowledgementPending;
PerProvider<bool> m_patternFileDirty;
PerProvider<bool> m_patternFileInitialized;
ImRect m_textEditorHoverBox;
ImRect m_consoleHoverBox;
std::string m_focusedSubWindowName;
float m_popupWindowHeight = 0;
float m_popupWindowHeightChange = 0;
bool m_frPopupIsClosed = true;
bool m_findReplacePopupIsClosed = true;
bool m_gotoPopupIsClosed = true;
static inline std::array<std::string,256> m_findHistory;
@@ -204,7 +193,6 @@ namespace hex::plugin::builtin {
static inline u32 m_replaceHistorySize = 0;
static inline u32 m_replaceHistoryIndex = 0;
TextHighlighter m_textHighlighter = TextHighlighter(this,&this->m_editorRuntime);
private:
void drawConsole(ImVec2 size);
void drawDebugger(ImVec2 size);

1026
plugins/builtin/source/content/text_highlighting/pattern_language.cpp
View File

File diff suppressed because it is too large Load Diff

187
plugins/builtin/source/content/views/view_pattern_editor.cpp
View File

@@ -22,18 +22,13 @@
#include <hex/helpers/fs.hpp>
#include <hex/helpers/utils.hpp>
#include <hex/helpers/magic.hpp>
#include <hex/helpers/binary_pattern.hpp>
#include <hex/helpers/default_paths.hpp>
#include <banners/banner_button.hpp>
#include <hex/providers/memory_provider.hpp>
#include <hex/helpers/fmt.hpp>
#include <fmt/chrono.h>
#include <popups/popup_question.hpp>
#include <popups/popup_file_chooser.hpp>
#include <toasts/toast_notification.hpp>
#include <chrono>
@@ -43,11 +38,25 @@
#include <wolv/utils/lock.hpp>
#include <fonts/fonts.hpp>
#include <hex/api/content_registry/communication_interface.hpp>
#include <hex/api/events/requests_gui.hpp>
#include <hex/helpers/menu_items.hpp>
#include <hex/helpers/logger.hpp>
#include <romfs/romfs.hpp>
#include <content/text_highlighting/pattern_language.hpp>
#include <fmt/chrono.h>
// Specialization for std::chrono::duration<double>
template <>
struct fmt::formatter<std::chrono::duration<double>> {
constexpr auto parse(fmt::format_parse_context& ctx) -> decltype(ctx.begin()) {
return ctx.end();
}
template <typename FormatContext>
auto format(const std::chrono::duration<double>& duration, FormatContext& ctx) const {
return fmt::format_to(ctx.out(), "{} seconds", duration.count());
}
};
namespace hex::plugin::builtin {
@@ -73,7 +82,7 @@ namespace hex::plugin::builtin {
return;
}
ui::TextEditor *editor = m_view->getTextEditor();
ui::TextEditor const *editor = m_view->getTextEditor();
if (editor != nullptr) {
if (m_view->m_sourceCode.hasProviderSpecificSource(provider)) {
this->close();
@@ -306,16 +315,12 @@ namespace hex::plugin::builtin {
m_editorRuntime = std::make_unique<pl::PatternLanguage>();
ContentRegistry::PatternLanguage::configureRuntime(*m_editorRuntime, nullptr);
this->registerEvents();
this->registerMenuItems();
this->registerHandlers();
registerEvents();
registerMenuItems();
registerHandlers();
// Initialize the text editor with some basic help text
m_textEditor.setOnCreateCallback([this](prv::Provider *provider, ui::TextEditor &editor) {
if (m_sourceCode.isSynced() && !m_sourceCode.get(provider).empty())
return;
m_textEditor.setOnCreateCallback([](auto, ui::TextEditor &editor) {
std::string text = "hex.builtin.view.pattern_editor.default_help_text"_lang;
text = "// " + wolv::util::replaceStrings(text, "\n", "\n// ");
@@ -999,9 +1004,9 @@ namespace hex::plugin::builtin {
}
}
ImGui::EndPopup();
m_frPopupIsClosed = false;
} else if (!m_frPopupIsClosed) {
m_frPopupIsClosed = true;
m_findReplacePopupIsClosed = false;
} else if (!m_findReplacePopupIsClosed) {
m_findReplacePopupIsClosed = true;
m_popupWindowHeight = 0;
m_textEditor.get(provider).setTopMarginChanged(0);
}
@@ -1436,10 +1441,7 @@ namespace hex::plugin::builtin {
if (m_textEditor.get(provider).isTextChanged()) {
m_textEditor.get(provider).setTextChanged(false);
if (!m_hasUnevaluatedChanges.get(provider) ) {
m_hasUnevaluatedChanges.get(provider) = true;
m_changesWereParsed = false;
}
m_hasUnevaluatedChanges.get(provider) = true;
m_lastEditorChangeTime = std::chrono::steady_clock::now();
ImHexApi::Provider::markDirty();
markPatternFileDirty(provider);
@@ -1450,7 +1452,6 @@ namespace hex::plugin::builtin {
auto code = m_textEditor.get(provider).getText();
EventPatternEditorChanged::post(code);
TaskManager::createBackgroundTask("hex.builtin.task.parsing_pattern", [this, code = std::move(code), provider](auto &){
this->parsePattern(code, provider);
@@ -1464,14 +1465,35 @@ namespace hex::plugin::builtin {
this->evaluatePattern(m_textEditor.get(provider).getText(), provider);
}
if (m_textHighlighter.m_needsToUpdateColors && m_changesWereParsed && (m_runningParsers + m_runningEvaluators == 0)) {
if (m_textHighlighter.getRunningColorizers() == 0) {
m_textHighlighter.m_needsToUpdateColors = false;
m_changesWereParsed = false;
TaskManager::createBackgroundTask("HighlightSourceCode", [this](auto &) { m_textHighlighter.highlightSourceCode(); });
} else {
m_textHighlighter.interrupt();
TaskHolder coloringTaskHolder;
if (m_changesWereParsed || m_wasInterrupted || m_hasUnevaluatedChanges.get(provider)) {
if (coloringTaskHolder.isRunning())
interrupt();
if (m_wasInterrupted)
resetInterrupt();
m_changesWereParsed = false;
if(!m_hasUnevaluatedChanges.get(provider)) {
m_hasUncoloredChanges.get(provider) = true;
m_changesWereColored = false;
} else
m_hasUncoloredChanges.get(provider) = false;
}
if (m_hasUncoloredChanges.get(provider) && (m_runningHighlighters + m_runningEvaluators == 0)) {
try {
m_textHighlighter.get(provider).setViewPatternEditor(this);
m_textHighlighter.get(provider).updateRequiredInputs();
coloringTaskHolder = TaskManager::createBackgroundTask("HighlightSourceCode", [this,provider](auto &) { m_textHighlighter.get(provider).highlightSourceCode(); });
m_hasUncoloredChanges.get(provider) = false;
} catch (const std::out_of_range&) {
interrupt();
}
} else if (m_changesWereColored) {
m_textHighlighter.get(provider).setRequestedIdentifierColors();
m_changesWereColored = false;
}
if (m_dangerousFunctionCalled && !ImGui::IsPopupOpen(ImGuiID(0), ImGuiPopupFlags_AnyPopup)) {
@@ -1641,7 +1663,6 @@ namespace hex::plugin::builtin {
m_changeTracker.get(provider) = wolv::io::ChangeTracker(file);
m_changeTracker.get(provider).startTracking([this, provider]{ this->handleFileChange(provider); });
}
m_textHighlighter.m_needsToUpdateColors = false;
TaskManager::createBackgroundTask("hex.builtin.task.parsing_pattern", [this, code, provider](auto&) { this->parsePattern(code, provider); });
}
}
@@ -1651,7 +1672,7 @@ namespace hex::plugin::builtin {
ContentRegistry::PatternLanguage::configureRuntime(*m_editorRuntime, nullptr);
const auto &ast = m_editorRuntime->parseString(code, pl::api::Source::DefaultSource);
m_textEditor.get(provider).setLongestLineLength(m_editorRuntime->getInternals().preprocessor->getLongestLineLength());
m_textEditor.get(provider).setLongestLineLength(m_editorRuntime->getInternals().preprocessor.get()->getLongestLineLength());
auto &patternVariables = m_patternVariables.get(provider);
auto oldPatternVariables = std::move(patternVariables);
@@ -1683,7 +1704,6 @@ namespace hex::plugin::builtin {
patternVariables = std::move(oldPatternVariables);
}
m_textHighlighter.m_needsToUpdateColors = true;
m_changesWereParsed = true;
m_runningParsers -= 1;
}
@@ -1707,7 +1727,6 @@ namespace hex::plugin::builtin {
m_accessHistory = {};
m_accessHistoryIndex = 0;
m_patternEvaluating = true;
EventHighlightingChanged::post();
@@ -1835,13 +1854,20 @@ namespace hex::plugin::builtin {
m_textEditor.get(provider).setText(wolv::util::preprocessText(code));
m_sourceCode.get(provider) = code;
m_hasUnevaluatedChanges.get(provider) = true;
m_textHighlighter.m_needsToUpdateColors = false;
});
ContentRegistry::Settings::onChange("hex.builtin.setting.general", "hex.builtin.setting.general.sync_pattern_source", [this](const ContentRegistry::Settings::SettingsValue &value) {
m_sourceCode.enableSync(value.get<bool>(false));
});
ContentRegistry::Settings::onChange("hex.builtin.setting.general", "hex.builtin.setting.general.suggest_patterns", [this](const ContentRegistry::Settings::SettingsValue &value) {
m_suggestSupportedPatterns = value.get<bool>(true);
});
ContentRegistry::Settings::onChange("hex.builtin.setting.general", "hex.builtin.setting.general.auto_apply_patterns", [this](const ContentRegistry::Settings::SettingsValue &value) {
m_autoApplyPatterns = value.get<bool>(true);
});
EventProviderOpened::subscribe(this, [this](prv::Provider *provider) {
m_textEditor.get(provider).setLanguageDefinition(PatternLanguage());
m_textEditor.get(provider).setShowWhitespaces(false);
@@ -1893,7 +1919,6 @@ namespace hex::plugin::builtin {
m_consoleEditor.get(newProvider).setScroll(m_consoleScroll.get(newProvider));
}
m_textHighlighter.m_needsToUpdateColors = false;
});
@@ -2238,6 +2263,10 @@ namespace hex::plugin::builtin {
}
});
ContentRegistry::Settings::onChange("hex.builtin.setting.hex_editor", "hex.builtin.setting.hex_editor.pattern_parent_highlighting", [this](const ContentRegistry::Settings::SettingsValue &value) {
m_parentHighlightingEnabled = bool(value.get<int>(false));
});
ImHexApi::HexEditor::addBackgroundHighlightingProvider([this](u64 address, const u8 *data, size_t size, bool) -> std::optional<color_t> {
std::ignore = data;
std::ignore = size;
@@ -2337,7 +2366,6 @@ namespace hex::plugin::builtin {
m_textEditor.get(provider).setText(sourceCode);
m_hasUnevaluatedChanges.get(provider) = true;
m_textHighlighter.m_needsToUpdateColors = false;
return true;
},
.store = [this](prv::Provider *provider, const std::fs::path &basePath, const Tar &tar) {
@@ -2553,81 +2581,6 @@ namespace hex::plugin::builtin {
return result;
});
ContentRegistry::MCP::registerTool(romfs::get("mcp/tools/execute_pattern_code.json").string(), [this](const nlohmann::json &data) -> nlohmann::json {
auto provider = ImHexApi::Provider::get();
auto sourceCode = data.at("source_code").get<std::string>();
this->evaluatePattern(sourceCode, provider);
// Wait until evaluation has finished
while (m_runningEvaluators > 0) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
auto lock = std::scoped_lock(ContentRegistry::PatternLanguage::getRuntimeLock());
auto evaluationResult = m_lastEvaluationResult.load();
nlohmann::json result = {
{ "handle", provider->getID() },
{ "result_code", evaluationResult }
};
return mcp::StructuredContent {
.text = result.dump(),
.data = result
};
});
ContentRegistry::MCP::registerTool(romfs::get("mcp/tools/get_pattern_console_content.json").string(), [this](const nlohmann::json &data) -> nlohmann::json {
std::ignore = data;
auto provider = ImHexApi::Provider::get();
// Wait until evaluation has finished
while (m_runningEvaluators > 0) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
auto lock = std::scoped_lock(ContentRegistry::PatternLanguage::getRuntimeLock());
auto consoleOutput = m_console.get(provider);
nlohmann::json result = {
{ "handle", provider->getID() },
{ "content", wolv::util::combineStrings(consoleOutput, "\n") }
};
return mcp::StructuredContent {
.text = result.dump(),
.data = result
};
});
ContentRegistry::MCP::registerTool(romfs::get("mcp/tools/get_patterns.json").string(), [this](const nlohmann::json &data) -> nlohmann::json {
std::ignore = data;
auto provider = ImHexApi::Provider::get();
// Wait until evaluation has finished
while (m_runningEvaluators > 0) {
std::this_thread::sleep_for(std::chrono::milliseconds(100));
}
auto lock = std::scoped_lock(ContentRegistry::PatternLanguage::getRuntimeLock());
pl::gen::fmt::FormatterJson formatter;
auto formattedPatterns = formatter.format(ContentRegistry::PatternLanguage::getRuntime());
nlohmann::json result = {
{ "handle", provider->getID() },
{ "patterns", nlohmann::json::parse(std::string(formattedPatterns.begin(), formattedPatterns.end())) }
};
return mcp::StructuredContent {
.text = result.dump(),
.data = result
};
});
}
void ViewPatternEditor::handleFileChange(prv::Provider *provider) {
@@ -2745,4 +2698,12 @@ namespace hex::plugin::builtin {
ImGuiExt::TextFormattedWrapped("This will execute your code, output any log messages to the console window below and create a pattern tree that gets displayed in the Pattern Data view and highlights matching regions in the Hex Editor view.");
}
ui::TextEditor *ViewPatternEditor::getTextEditor() {
auto provider = ImHexApi::Provider::get();
if (provider == nullptr)
return nullptr;
return &m_textEditor.get(provider);
}
}

29
plugins/ui/include/ui/text_editor.hpp
View File

@@ -1,14 +1,15 @@
#pragma once
#include "imgui_internal.h"
#include <string>
#include <vector>
#include <array>
#include <memory>
#include <functional>
#include <unordered_set>
#include <unordered_map>
#include <map>
#include <regex>
#include "imgui.h"
#include "imgui_internal.h"
#include <chrono>
#include <hex/helpers/utils.hpp>
#include <pl/core/location.hpp>
@@ -103,7 +104,7 @@ namespace hex::ui {
public:
friend class TextEditor;
bool operator==(const EditorState &o) const;
EditorState() = default;
EditorState() : m_selection(), m_cursorPosition() {}
EditorState(const Range &selection, const Coordinates &cursorPosition) : m_selection(selection), m_cursorPosition(cursorPosition) {}
private:
Range m_selection;
@@ -151,11 +152,11 @@ namespace hex::ui {
bool m_matchCase;
bool m_wholeWord;
bool m_findRegEx;
bool m_optionsChanged = false;
bool m_optionsChanged;
Matches m_matches;
};
enum class PaletteIndex: u8 {
enum class PaletteIndex {
Default, Identifier, Directive, Operator, Separator, BuiltInType, Keyword, NumericLiteral, StringLiteral, CharLiteral, Cursor, Background, LineNumber, Selection, Breakpoint, ErrorMarker, PreprocessorDeactivated,
CurrentLineFill, CurrentLineFillInactive, CurrentLineEdge, ErrorText, WarningText, DebugText, DefaultText, Attribute, PatternVariable, LocalVariable, CalculatedPointer, TemplateArgument, Function, View,
FunctionVariable, FunctionParameter, UserDefinedType, PlacedVariable, GlobalVariable, NameSpace, TypeDef, UnkIdentifier, DocComment, DocBlockComment, BlockComment, GlobalDocComment, Comment, PreprocIdentifier, Max
@@ -228,12 +229,12 @@ namespace hex::ui {
class LineIterator {
public:
friend class hex::ui::TextEditor;
LineIterator(const LineIterator &other) = default;
LineIterator(const LineIterator &other) : m_charsIter(other.m_charsIter), m_colorsIter(other.m_colorsIter), m_flagsIter(other.m_flagsIter) {}
LineIterator() = default;
char operator*();
LineIterator operator++();
LineIterator& operator=(const LineIterator &other);
LineIterator operator=(const LineIterator &other);
bool operator!=(const LineIterator &other) const;
bool operator==(const LineIterator &other) const;
LineIterator operator+(i32 n);
@@ -275,10 +276,10 @@ namespace hex::ui {
Line() : m_lineMaxColumn(-1) {}
explicit Line(const char *line) : Line(std::string(line)) {}
explicit Line(const std::string &line) : m_chars(line), m_colors(std::string(line.size(), 0x00)), m_flags(std::string(line.size(), 0x00)), m_lineMaxColumn(maxColumn()) {}
explicit Line(const std::string &line) : m_chars(line), m_colors(std::string(line.size(), 0x00)), m_flags(std::string(line.size(), 0x00)), m_colorized(false), m_lineMaxColumn(maxColumn()) {}
Line(const Line &line) : m_chars(std::string(line.m_chars)), m_colors(std::string(line.m_colors)), m_flags(std::string(line.m_flags)), m_colorized(line.m_colorized), m_lineMaxColumn(line.m_lineMaxColumn) {}
Line(Line &&line) noexcept : m_chars(std::move(line.m_chars)), m_colors(std::move(line.m_colors)), m_flags(std::move(line.m_flags)), m_colorized(line.m_colorized), m_lineMaxColumn(line.m_lineMaxColumn) {}
Line(std::string chars, std::string colors, std::string flags) : m_chars(std::move(chars)), m_colors(std::move(colors)), m_flags(std::move(flags)), m_lineMaxColumn(maxColumn()) {}
Line(std::string chars, std::string colors, std::string flags) : m_chars(std::move(chars)), m_colors(std::move(colors)), m_flags(std::move(flags)), m_colorized(false), m_lineMaxColumn(maxColumn()) {}
bool operator==(const Line &line) const;
bool operator!=(const Line &line) const;
@@ -311,12 +312,12 @@ namespace hex::ui {
std::string operator[](i64 column) const;
void setNeedsUpdate(bool needsUpdate);
void append(const char *text);
void append(char text);
void append(const char text);
void append(const std::string &text);
void append(const Line &line);
void append(LineIterator begin, LineIterator end);
void insert(LineIterator iter, const std::string &text);
void insert(LineIterator iter, char text);
void insert(LineIterator iter, const char text);
void insert(LineIterator iter, strConstIter beginString, strConstIter endString);
void insert(LineIterator iter, const Line &line);
void insert(LineIterator iter, LineIterator beginLine, LineIterator endLine);
@@ -372,7 +373,7 @@ namespace hex::ui {
class UndoRecord {
public:
friend class TextEditor;
UndoRecord() = default;
UndoRecord() {}
~UndoRecord() {}
UndoRecord( std::string added,
Range addedRange,
@@ -457,7 +458,7 @@ namespace hex::ui {
void setLongestLineLength(u64 line) { m_longestLineLength = line; }
u64 getLongestLineLength() const { return m_longestLineLength; }
void setTopMarginChanged(i32 newMargin);
void setFocusAtCoords(const Coordinates &coords, bool scrollToCursor = false);
void setFocusAtCoords(const Coordinates &coords, bool ensureVisible = false);
void clearErrorMarkers();
void clearActionables();
private:

7
plugins/ui/source/ui/text_editor/highlighter.cpp
View File

@@ -346,7 +346,12 @@ namespace hex::ui {
if (isGlobalDocComment || isBlockDocComment || isBlockComment) {
commentStartLine = currentLine;
commentStartIndex = currentIndex;
if (isGlobalDocComment) {
if (currentIndex < line.size() - 4 && isBlockComment &&
line.m_chars[currentIndex + 2] == '*' &&
line.m_chars[currentIndex + 3] == '/') {
withinBlockComment = true;
commentLength = 2;
} else if (isGlobalDocComment) {
withinGlobalDocComment = true;
commentLength = 3;
} else if (isBlockDocComment) {

61
plugins/ui/source/ui/text_editor/support.cpp
View File

@@ -37,11 +37,11 @@ namespace hex::ui {
}
Coordinates Coordinates::operator+(const Coordinates &o) const {
return {m_line + o.m_line, m_column + o.m_column};
return Coordinates(m_line + o.m_line, m_column + o.m_column);
}
Coordinates Coordinates::operator-(const Coordinates &o) const {
return {m_line - o.m_line, m_column - o.m_column};
return Coordinates(m_line - o.m_line, m_column - o.m_column);
}
bool Range::operator==(const Range &o) const {
@@ -52,13 +52,13 @@ namespace hex::ui {
}
Coordinates Range::getSelectedLines() {
return {m_start.m_line, m_end.m_line};
return Coordinates(m_start.m_line, m_end.m_line);
}
Coordinates Range::getSelectedColumns() {
if (isSingleLine())
return {m_start.m_column, m_end.m_column - m_start.m_column};
return {m_start.m_column, m_end.m_column};
return Coordinates(m_start.m_column, m_end.m_column - m_start.m_column);
return Coordinates(m_start.m_column, m_end.m_column);
}
bool Range::isSingleLine() {
@@ -148,7 +148,12 @@ namespace hex::ui {
return iter;
}
LineIterator& LineIterator::operator=(const LineIterator &other) = default;
LineIterator LineIterator::operator=(const LineIterator &other) {
m_charsIter = other.m_charsIter;
m_colorsIter = other.m_colorsIter;
m_flagsIter = other.m_flagsIter;
return *this;
}
bool LineIterator::operator!=(const LineIterator &other) const {
return m_charsIter != other.m_charsIter || m_colorsIter != other.m_colorsIter ||
@@ -204,7 +209,14 @@ namespace hex::ui {
return iter;
}
Line &Line::operator=(const Line &line) = default;
Line &Line::operator=(const Line &line) {
m_chars = line.m_chars;
m_colors = line.m_colors;
m_flags = line.m_flags;
m_colorized = line.m_colorized;
m_lineMaxColumn = line.m_lineMaxColumn;
return *this;
}
Line &Line::operator=(Line &&line) noexcept {
m_chars = std::move(line.m_chars);
@@ -310,21 +322,26 @@ namespace hex::ui {
}
char Line::operator[](u64 index) const {
i64 signedIndex = std::clamp((i64) index,0 - (i64) m_chars.size(), (i64) (m_chars.size() - 1));
auto charsSize = (i64) m_chars.size();
if (charsSize == 0)
return '\0';
i64 signedIndex = std::clamp((i64) index,0 - charsSize,charsSize - 1);
if (signedIndex < 0)
return m_chars[m_chars.size() + signedIndex];
return m_chars[charsSize + signedIndex];
return m_chars[signedIndex];
}
// C++ can't overload functions based on return type, so use any type other
// than u64 to avoid ambiguity.
std::string Line::operator[](i64 column) const {
std::string Line::operator[](i64 index) const {
i64 utf8Length = TextEditor::stringCharacterCount(m_chars);
column = std::clamp(column, (-utf8Length), utf8Length - 1);
if (column < 0)
column = utf8Length + column;
if (utf8Length == 0)
return "";
index = std::clamp(index, -utf8Length, utf8Length - 1);
if (index < 0)
index = utf8Length + index;
i64 utf8Start = 0;
for (i64 utf8Index = 0; utf8Index < column; ++utf8Index) {
for (i64 utf8Index = 0; utf8Index < index; ++utf8Index) {
utf8Start += TextEditor::utf8CharLength(m_chars[utf8Start]);
}
i64 utf8CharLen = TextEditor::utf8CharLength(m_chars[utf8Start]);
@@ -462,8 +479,10 @@ namespace hex::ui {
bool TextEditor::ActionableBox::trigger() {
auto mousePos = ImGui::GetMousePos();
return mousePos.x > m_box.Min.x && mousePos.x < m_box.Max.x &&
mousePos.y >= m_box.Min.y && mousePos.y <= m_box.Max.y;
if (mousePos.x <= m_box.Min.x || mousePos.x >= m_box.Max.x ||
mousePos.y < m_box.Min.y || mousePos.y > m_box.Max.y)
return false;
return true;
}
void TextEditor::ActionableBox::shiftBoxVertically(float lineCount, float lineHeight) {
@@ -527,7 +546,7 @@ namespace hex::ui {
if (m_readOnly)
return;
m_undoBuffer.resize(m_undoIndex + 1);
m_undoBuffer.resize((u64) (m_undoIndex + 1));
m_undoBuffer.back() = UndoAction(value);
m_undoIndex++;
}
@@ -885,11 +904,11 @@ namespace hex::ui {
std::string wordLower = m_findWord;
if (!getMatchCase())
std::ranges::transform(wordLower, wordLower.begin(), ::tolower);
std::transform(wordLower.begin(), wordLower.end(), wordLower.begin(), ::tolower);
std::string textSrc = editor->getText();
if (!getMatchCase())
std::ranges::transform(textSrc, textSrc.begin(), ::tolower);
std::transform(textSrc.begin(), textSrc.end(), textSrc.begin(), ::tolower);
u64 textLoc;
// TODO: use regexp find iterator in all cases
@@ -928,8 +947,7 @@ namespace hex::ui {
while (iter != end) {
iter++;
pos = iter->position();
if (pos > byteIndex)
if (((pos = iter->position()) > byteIndex))
break;
}
}
@@ -997,6 +1015,7 @@ namespace hex::ui {
editor->m_state = saveState;
editor->ensureCursorVisible();
return;
}