imports/imhex
1
0
Fork 0
mirror of https://github.com/WerWolv/ImHex.git synced 2026-03-30 05:05:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

Make the syntax highlighter more thread safety aware (#2585)

Browse Source 
By creating copies of the required inputs on the main thread just before
the task is spawned. A;so if task is still running when new data can be
copied then the task is interrupted thus avoiding concurrency without
mutexes. Atomics are used to signal state information used to determine
what and when to spawn.

Also includes update to pattern editor library and some fixes to syntax
highlighting error when custom types defined inside namespaces were used
inside the namespaces without the full qualified name and other small
changes mostly to improve the current style.
This commit is contained in:
paxcut
2025-12-26 20:21:19 -07:00
committed by GitHub
parent ba7e789a80
commit 64cbd5fc8d
8 changed files with 876 additions and 816 deletions
Download Patch File Download Diff File Expand all files Collapse all files

308
plugins/builtin/include/content/text_highlighting/pattern_language.hpp
View File

@@ -1,13 +1,12 @@
#pragma once
#include <pl/core/token.hpp>
#include <pl/core/preprocessor.hpp>
#include <pl/pattern_language.hpp>
#include <pl/helpers/safe_iterator.hpp>
#include <ui/text_editor.hpp>
#include <hex/helpers/types.hpp>
#include <hex/helpers/logger.hpp>
namespace pl {
class PatternLanguage;
}
namespace hex::plugin::builtin {
class ViewPatternEditor;
@@ -16,7 +15,6 @@ namespace hex::plugin::builtin {
class Interval;
using Token = pl::core::Token;
using ASTNode = pl::core::ast::ASTNode;
using ExcludedLocation = pl::core::Preprocessor::ExcludedLocation;
using CompileError = pl::core::err::CompileError;
using Identifier = Token::Identifier;
using IdentifierType = Identifier::IdentifierType;
@@ -24,12 +22,20 @@ namespace hex::plugin::builtin {
using OrderedBlocks = std::map<Interval,std::string>;
using Scopes = std::set<Interval>;
using Location = pl::core::Location;
using VectorString = std::vector<std::string>;
using TokenIter = pl::hlp::SafeIterator<std::vector<Token>::const_iterator>;
using VariableScopes = std::map<std::string,Scopes>;
using Inheritances = std::map<std::string,std::vector<std::string>>;
using Inheritances = std::map<std::string,VectorString>;
using IdentifierTypeColor = std::map<Identifier::IdentifierType,ui::TextEditor::PaletteIndex>;
using TokenTypeColor = std::map<Token::Type,ui::TextEditor::PaletteIndex>;
using TokenColor = std::map<Token *,ui::TextEditor::PaletteIndex>;
using TokenColor = std::map<i32, ui::TextEditor::PaletteIndex>;
using Types = std::map<std::string, pl::hlp::safe_shared_ptr<pl::core::ast::ASTNodeTypeDecl>>;
using ParsedImports = std::map<std::string,std::vector<Token>>;
using Str2StrMap = std::map<std::string,std::string>;
using CompileErrors = std::vector<CompileError>;
using TokenSequence = std::vector<Token>;
using TokenIdVector = std::vector<i32>;
using Instances = std::map<std::string,std::vector<i32>>;
struct ParentDefinition;
struct Definition {
@@ -48,6 +54,29 @@ namespace hex::plugin::builtin {
i32 tokenIndex;
Location location;
};
struct RequiredInputs {
friend class TextHighlighter;
private:
TextHighlighter *m_textHighlighter;
Types definedTypes;
VectorString usedNamespaces;
ParsedImports parsedImports;
Str2StrMap importedHeaders;
TokenSequence fullTokens;
std::string editedText;
CompileErrors compileErrors;
VectorString linesOfColors;
public:
RequiredInputs() : m_textHighlighter(nullptr) {};
explicit RequiredInputs(TextHighlighter *textHighlighter) : m_textHighlighter(textHighlighter) {}
void setRequiredInputs();
void setTypes();
void setNamespaces();
void setImports();
void setText();
void setCompileErrors();
};
/// to define functions and types
using Definitions = std::map<std::string,ParentDefinition>;
/// to define global variables
@@ -55,16 +84,15 @@ namespace hex::plugin::builtin {
/// to define UDT and function variables
using VariableMap = std::map<std::string,Variables>;
private:
std::string m_text;
std::vector<std::string> m_lines;
std::vector<i32> m_firstTokenIdOfLine;
VectorString m_lines;
TokenIdVector m_firstTokenIdOfLine;
ViewPatternEditor *m_viewPatternEditor;
std::vector<ExcludedLocation> m_excludedLocations;
std::vector<Token> m_tokens;
TokenColor m_tokenColors;
std::unique_ptr<pl::PatternLanguage> *patternLanguage;
std::vector<CompileError> m_compileErrors;
std::map<std::string,std::vector<i32>> m_instances;
Instances m_instances;
Definitions m_UDTDefinitions;
Definitions m_functionDefinitions;
@@ -78,15 +106,16 @@ namespace hex::plugin::builtin {
VariableMap m_functionVariables;
Variables m_globalVariables;
std::map<std::string, std::vector<Token>> m_parsedImports;
std::map<std::string,std::string> m_attributeFunctionArgumentType;
std::map<std::string,std::string> m_typeDefMap;
std::map<std::string,std::string> m_typeDefInvMap;
std::vector<std::string> m_nameSpaces;
std::vector<std::string> m_UDTs;
Str2StrMap m_attributeFunctionArgumentType;
Str2StrMap m_typeDefMap;
Str2StrMap m_typeDefInvMap;
VectorString m_UDTs;
std::set<i32> m_taggedIdentifiers;
std::set<i32> m_memberChains;
std::set<i32> m_scopeChains;
RequiredInputs m_requiredInputs;
TokenIter m_curr;
TokenIter m_startToken, m_originalPosition, m_partOriginalPosition;
@@ -98,7 +127,6 @@ namespace hex::plugin::builtin {
const static IdentifierTypeColor m_identifierTypeColor;
const static TokenTypeColor m_tokenTypeColor;
i32 m_runningColorizers=0;
public:
/// Intervals are the sets finite contiguous non-negative integer that
@@ -118,49 +146,29 @@ namespace hex::plugin::builtin {
i32 start;
i32 end;
Interval() : start(0), end(0) {}
Interval(i32 start, i32 end) : start(start), end(end) {
if (start > end)
throw std::invalid_argument("Interval start must be less than or equal to end");
}
bool operator<(const Interval &other) const {
return other.end > end;
}
bool operator>(const Interval &other) const {
return end > other.end;
}
bool operator==(const Interval &other) const {
return start == other.start && end == other.end;
}
bool operator!=(const Interval &other) const {
return start != other.start || end != other.end;
}
bool operator<=(const Interval &other) const {
return other.end >= end;
}
bool operator>=(const Interval &other) const {
return end >= other.end;
}
bool contains(const Interval &other) const {
return other.start >= start && other.end <= end;
}
bool contains(i32 value) const {
return value >= start && value <= end;
}
bool contiguous(const Interval &other) const {
return ((start - other.end) == 1 || (other.start - end) == 1);
}
Interval(i32 start, i32 end);
bool operator<(const Interval &other) const;
bool operator>(const Interval &other) const;
bool operator==(const Interval &other) const;
bool operator!=(const Interval &other) const;
bool operator<=(const Interval &other) const;
bool operator>=(const Interval &other) const;
bool contains(const Interval &other) const;
bool contains(i32 value) const ;
bool contiguous(const Interval &other) const;
};
std::atomic<bool> m_needsToUpdateColors = true;
std::atomic<bool> m_wasInterrupted = false;
std::atomic<bool> m_interrupt = false;
constexpr static u32 Normal = 0;
constexpr static u32 Not = 1;
pl::PatternLanguage *getPatternLanguage();
void updateRequiredInputs();
RequiredInputs& getRequiredInputs();
ViewPatternEditor* getViewPatternEditor();
void setViewPatternEditor(ViewPatternEditor *viewPatternEditor);
void interrupt() {
m_interrupt = true;
}
TextHighlighter(ViewPatternEditor *viewPatternEditor, std::unique_ptr<pl::PatternLanguage> *patternLanguage ) :
m_viewPatternEditor(viewPatternEditor), patternLanguage(patternLanguage), m_needsToUpdateColors(true) {}
TextHighlighter();
~TextHighlighter();
explicit TextHighlighter(ViewPatternEditor *viewPatternEditor) : m_viewPatternEditor(viewPatternEditor) {}
/**
* @brief Entry point to syntax highlighting
*/
@@ -184,14 +192,16 @@ namespace hex::plugin::builtin {
/**
* @brief Only identifiers not in chains should remain
*/
void colorRemainingIdentifierTokens();
/**
* @brief Renders compile errors in real time
*/
void renderErrors();
/// A token range is the set of token indices of a definition. The namespace token
/// ranges are obtained first because they are needed to obtain unique identifiers.
void getAllTokenRanges(IdentifierType identifierTypeToSearch);
void getTokenRanges(IdentifierType identifierTypeToSearch);
/// The global scope is the complement of the union of all the function and UDT token ranges
void getGlobalTokenRanges();
/// If the current token is a function or UDT, creates a map entry from the name to the token range. These are ordered alphabetically by name.
@@ -217,14 +227,14 @@ namespace hex::plugin::builtin {
/// Chains are sequences of identifiers separated by scope resolution or dot operators.
void fixChains();
bool colorSeparatorScopeChain();
bool colorRealSeparatorScopeChain();
bool colorImplicitSeparatorScopeChain();
bool colorOperatorDotChain();
/// Returns the next/previous valid source code line
u32 nextLine(u32 line);
u32 previousLine(u32 line);
/// Loads the source code and calculates the first token index of each line
void loadText();
/// Used to obtain the color to be applied.
ui::TextEditor::PaletteIndex getPaletteIndex(Token::Literal *literal);
/// The complement of a set is also known as its inverse
void invertGlobalTokenRange();
/// Starting at the identifier, it tracks all the scope resolution and dot operators and returns the full chain without arrays, templates, pointers,...
@@ -251,11 +261,11 @@ namespace hex::plugin::builtin {
void skipDelimiters(i32 maxSkipCount, Token delimiter[2], i8 increment);
void skipToken(Token token, i8 step=1);
/// from given or current names find the corresponding definition
bool findIdentifierDefinition(Definition &result, const std::string &optionalIdentifierName = "", std::string optionalName = "", bool setInstances = false);
bool findIdentifierDefinition(Definition &result, const std::string &optionalIdentifierName = "", std::string optionalName = "", bool optional = false);
/// To deal with the Parent keyword
std::optional<Definition> setChildrenTypes();
bool findParentTypes(std::vector<std::string> &parentTypes, const std::string &optionalUDTName="");
bool findAllParentTypes(std::vector<std::string> &parentTypes, std::vector<Identifier *> &identifiers, std::string &optionalFullName);
bool findParentTypes(VectorString &parentTypes, const std::string &optionalName="");
bool findAllParentTypes(VectorString &parentTypes, std::vector<Identifier *> &identifiers, std::string &optionalFullName);
bool tryParentType(const std::string &parentType, std::string &variableName, std::optional<Definition> &result, std::vector<Identifier *> &identifiers);
/// Convenience function
bool isTokenIdValid(i32 tokenId);
@@ -274,160 +284,22 @@ namespace hex::plugin::builtin {
void getTokenIdForArgument(i32 start, i32 argNumber, Token delimiter);
///Creates a map from function name to argument type
void linkAttribute();
/// Comment and strings usethese function to determine their coordinates
template<typename T> ui::TextEditor::Coordinates commentCoordinates(Token *token);
ui::TextEditor::Coordinates stringCoordinates();
/// Returns the number of tasks highlighting code. Shouldn't be > 1
i32 getRunningColorizers() {
return m_runningColorizers;
}
enum class HighlightStage {
Starting,
NamespaceTokenRanges,
UDTTokenRanges,
FunctionTokenRanges,
GlobalTokenRanges,
FixGlobalVariables,
SetInitialColors,
LoadInstances,
AttributeTokenRanges,
Definitions,
FixAutos,
FixChains,
ExcludedLocations,
ColorRemainingTokens,
SetRequestedIdentifierColors,
Stage1,
Stage2,
Stage3,
Stage4,
Stage5,
Stage6,
Stage7,
Stage8,
Stage9,
Stage10,
Stage11,
};
HighlightStage m_highlightStage = HighlightStage::Starting;
/// The following functions were copied from the parser and some were modified
template<typename T>
T *getValue(const i32 index) {
return const_cast<T*>(std::get_if<T>(&m_curr[index].value));
}
void next(i32 count = 1) {
if (m_interrupt) {
m_interrupt = false;
throw std::out_of_range("Highlights were deliberately interrupted");
}
if (count == 0)
return;
i32 id = getTokenId(m_curr->location);
i32 maxChange;
if (count > 0)
maxChange = std::min(count,static_cast<i32>(m_tokens.size() - id));
else
maxChange = -std::min(-count,id);
m_curr += maxChange;
}
constexpr static u32 Normal = 0;
constexpr static u32 Not = 1;
bool begin() {
m_originalPosition = m_curr;
return true;
}
void partBegin() {
m_partOriginalPosition = m_curr;
}
void reset() {
m_curr = m_originalPosition;
}
void partReset() {
m_curr = m_partOriginalPosition;
}
bool resetIfFailed(const bool value) {
if (!value) reset();
return value;
}
template<auto S = Normal>
bool sequenceImpl() {
if constexpr (S == Normal)
return true;
else if constexpr (S == Not)
return false;
else
std::unreachable();
}
template<auto S = Normal>
bool matchOne(const Token &token) {
if constexpr (S == Normal) {
if (!peek(token)) {
partReset();
return false;
}
next();
return true;
} else if constexpr (S == Not) {
if (!peek(token))
return true;
next();
partReset();
return false;
} else
std::unreachable();
}
template<auto S = Normal>
bool sequenceImpl(const auto &... args) {
return (matchOne<S>(args) && ...);
}
template<auto S = Normal>
bool sequence(const Token &token, const auto &... args) {
partBegin();
return sequenceImpl<S>(token, args...);
}
bool isValid() {
Token token;
try {
token = m_curr[0];
}
catch (const std::out_of_range &e) {
auto t = e.what();
if (t == nullptr)
return false;
return false;
}
if (!isLocationValid(token.location))
return false;
return true;
}
bool peek(const Token &token, const i32 index = 0) {
if (!isValid())
return false;
i32 id = getTokenId(m_curr->location);
if (id+index < 0 || id+index >= (i32)m_tokens.size())
return false;
return m_curr[index].type == token.type && m_curr[index] == token.value;
}
template<typename T> T *getValue(const i32 index);
void next(i32 count = 1);
bool begin();
void partBegin();
void reset();
void partReset();
bool resetIfFailed(const bool value) ;
template<auto S = Normal> bool sequenceImpl();
template<auto S = Normal> bool matchOne(const Token &token);
template<auto S = Normal> bool sequenceImpl(const auto &... args);
template<auto S = Normal> bool sequence(const Token &token, const auto &... args);
bool isValid();
bool peek(const Token &token, const i32 index = 0);
};
}

72
plugins/builtin/include/content/views/view_pattern_editor.hpp
View File

@@ -9,7 +9,6 @@
#include <ui/text_editor.hpp>
#include <content/text_highlighting/pattern_language.hpp>
#include <hex/api/content_registry/settings.hpp>
#include <hex/helpers/magic.hpp>
#include <ui/pattern_drawer.hpp>
@@ -32,52 +31,34 @@ namespace hex::plugin::builtin {
std::string m_sharedSource;
};
using TextHighlighter = hex::plugin::builtin::TextHighlighter;
class ViewPatternEditor : public View::Window {
public:
ViewPatternEditor();
~ViewPatternEditor() override;
void drawAlwaysVisibleContent() override;
std::unique_ptr<pl::PatternLanguage> *getPatternLanguage() {
return &m_editorRuntime;
}
ui::TextEditor *getTextEditor() {
auto provider = ImHexApi::Provider::get();
if (provider == nullptr)
return nullptr;
return &m_textEditor.get(provider);
}
bool getChangesWereParsed() const {
return m_changesWereParsed;
}
u32 getRunningParsers () const {
return m_runningParsers;
}
u32 getRunningEvaluators () const {
return m_runningEvaluators;
}
void setChangesWereParsed(bool changesWereParsed) {
m_changesWereParsed = changesWereParsed;
}
std::unique_ptr<pl::PatternLanguage> *getPatternLanguage() { return &m_editorRuntime; }
ui::TextEditor *getTextEditor();
u32 getRunningParsers () const { return m_runningParsers;}
u32 getRunningEvaluators () const { return m_runningEvaluators;}
u32 getRunningHighlighters () const { return m_runningHighlighters;}
void incrementRunningParsers( i32 amount) { m_runningParsers += amount; }
void incrementRunningEvaluators( i32 amount) { m_runningEvaluators += amount; }
void incrementRunningHighlighters( i32 amount) { m_runningHighlighters += amount; }
bool hasUnevaluatedChanges(prv::Provider *provider) const;
void setChangesWereParsed(bool changesWereParsed) { m_changesWereParsed = changesWereParsed;}
void setChangesWereColored(bool changesWereColored) { m_changesWereColored = changesWereColored; }
void drawContent() override;
void setPopupWindowHeight(u32 height) { m_popupWindowHeight = height; }
u32 getPopupWindowHeight() const { return m_popupWindowHeight; }
enum class DangerousFunctionPerms : u8 {
Ask,
Allow,
Deny
};
enum class DangerousFunctionPerms : u8 { Ask, Allow, Deny };
void drawHelpText() override;
void setWasInterrupted(bool wasInterrupted) { m_wasInterrupted = wasInterrupted; }
bool wasInterrupted() const { return m_wasInterrupted;}
void resetInterrupt() { m_wasInterrupted = false; m_interrupt = false;}
void interrupt() { m_interrupt = true; }
private:
class PopupAcceptPattern;
@@ -127,19 +108,26 @@ namespace hex::plugin::builtin {
std::atomic<u32> m_runningEvaluators = 0;
std::atomic<u32> m_runningParsers = 0;
std::atomic<u32> m_runningHighlighters = 0;
std::atomic<bool> m_changesWereParsed = false;
PerProvider<bool> m_hasUnevaluatedChanges;
std::atomic<bool> m_changesWereParsed;
PerProvider<bool> m_hasUncoloredChanges;
std::atomic<bool> m_changesWereColored;
std::atomic<bool> m_wasInterrupted;
std::atomic<bool> m_interrupt;
std::chrono::time_point<std::chrono::steady_clock> m_lastEditorChangeTime;
PerProvider<ui::TextEditor> m_textEditor, m_consoleEditor;
PerProvider<TextHighlighter> m_textHighlighter;
std::atomic<bool> m_consoleNeedsUpdate = false;
std::atomic<bool> m_dangerousFunctionCalled = false;
std::atomic<DangerousFunctionPerms> m_dangerousFunctionsAllowed = DangerousFunctionPerms::Ask;
ContentRegistry::Settings::SettingsVariable<bool, "hex.builtin.setting.general", "hex.builtin.setting.general.suggest_patterns"> m_suggestSupportedPatterns = true;
ContentRegistry::Settings::SettingsVariable<bool, "hex.builtin.setting.general", "hex.builtin.setting.general.auto_apply_patterns"> m_autoApplyPatterns = false;
bool m_suggestSupportedPatterns = true;
bool m_autoApplyPatterns = false;
PerProvider<ui::VisualizerDrawer> m_visualizerDrawer;
bool m_tooltipJustOpened = false;
@@ -178,7 +166,7 @@ namespace hex::plugin::builtin {
std::array<AccessData, 512> m_accessHistory = {};
u32 m_accessHistoryIndex = 0;
ContentRegistry::Settings::SettingsVariable<bool, "hex.builtin.setting.hex_editor", "hex.builtin.setting.hex_editor.pattern_parent_highlighting"> m_parentHighlightingEnabled = false;
bool m_parentHighlightingEnabled = true;
bool m_replaceMode = false;
bool m_openFindReplacePopUp = false;
bool m_openGotoLinePopUp = false;
@@ -188,13 +176,14 @@ namespace hex::plugin::builtin {
PerProvider<bool> m_ignoreNextChangeEvent;
PerProvider<bool> m_changeEventAcknowledgementPending;
PerProvider<bool> m_patternFileDirty;
PerProvider<bool> m_patternFileInitialized;
ImRect m_textEditorHoverBox;
ImRect m_consoleHoverBox;
std::string m_focusedSubWindowName;
float m_popupWindowHeight = 0;
float m_popupWindowHeightChange = 0;
bool m_frPopupIsClosed = true;
bool m_findReplacePopupIsClosed = true;
bool m_gotoPopupIsClosed = true;
static inline std::array<std::string,256> m_findHistory;
@@ -204,7 +193,6 @@ namespace hex::plugin::builtin {
static inline u32 m_replaceHistorySize = 0;
static inline u32 m_replaceHistoryIndex = 0;
TextHighlighter m_textHighlighter = TextHighlighter(this,&this->m_editorRuntime);
private:
void drawConsole(ImVec2 size);
void drawDebugger(ImVec2 size);