From 64cbd5fc8d142ac8fc166a79bc5ac9bbac7fb60b Mon Sep 17 00:00:00 2001 From: paxcut <53811119+paxcut@users.noreply.github.com> Date: Fri, 26 Dec 2025 20:21:19 -0700 Subject: [PATCH] Make the syntax highlighter more thread safety aware (#2585) By creating copies of the required inputs on the main thread just before the task is spawned. A;so if task is still running when new data can be copied then the task is interrupted thus avoiding concurrency without mutexes. Atomics are used to signal state information used to determine what and when to spawn. Also includes update to pattern editor library and some fixes to syntax highlighting error when custom types defined inside namespaces were used inside the namespaces without the full qualified name and other small changes mostly to improve the current style. --- lib/external/pattern_language | 2 +- .../text_highlighting/pattern_language.hpp | 308 ++--- .../content/views/view_pattern_editor.hpp | 72 +- .../text_highlighting/pattern_language.cpp | 1026 ++++++++++------- .../content/views/view_pattern_editor.cpp | 187 ++- plugins/ui/include/ui/text_editor.hpp | 29 +- .../ui/source/ui/text_editor/highlighter.cpp | 7 +- plugins/ui/source/ui/text_editor/support.cpp | 61 +- 8 files changed, 876 insertions(+), 816 deletions(-) diff --git a/lib/external/pattern_language b/lib/external/pattern_language index a0973fa8a..2eee33914 160000 --- a/lib/external/pattern_language +++ b/lib/external/pattern_language @@ -1 +1 @@ -Subproject commit a0973fa8ab0832b4ecf150dd0e500b66764178d5 +Subproject commit 2eee339144cc09f8bc4d7d0c7f0ccf0e39dfd46b diff --git a/plugins/builtin/include/content/text_highlighting/pattern_language.hpp b/plugins/builtin/include/content/text_highlighting/pattern_language.hpp index 9f3917599..b764170f3 100644 --- a/plugins/builtin/include/content/text_highlighting/pattern_language.hpp +++ b/plugins/builtin/include/content/text_highlighting/pattern_language.hpp @@ -1,13 +1,12 @@ #pragma once #include #include +#include #include #include #include +#include -namespace pl { - class PatternLanguage; -} namespace hex::plugin::builtin { class ViewPatternEditor; @@ -16,7 +15,6 @@ namespace hex::plugin::builtin { class Interval; using Token = pl::core::Token; using ASTNode = pl::core::ast::ASTNode; - using ExcludedLocation = pl::core::Preprocessor::ExcludedLocation; using CompileError = pl::core::err::CompileError; using Identifier = Token::Identifier; using IdentifierType = Identifier::IdentifierType; @@ -24,12 +22,20 @@ namespace hex::plugin::builtin { using OrderedBlocks = std::map; using Scopes = std::set; using Location = pl::core::Location; + using VectorString = std::vector; using TokenIter = pl::hlp::SafeIterator::const_iterator>; using VariableScopes = std::map; - using Inheritances = std::map>; + using Inheritances = std::map; using IdentifierTypeColor = std::map; using TokenTypeColor = std::map; - using TokenColor = std::map; + using TokenColor = std::map; + using Types = std::map>; + using ParsedImports = std::map>; + using Str2StrMap = std::map; + using CompileErrors = std::vector; + using TokenSequence = std::vector; + using TokenIdVector = std::vector; + using Instances = std::map>; struct ParentDefinition; struct Definition { @@ -48,6 +54,29 @@ namespace hex::plugin::builtin { i32 tokenIndex; Location location; }; + + struct RequiredInputs { + friend class TextHighlighter; + private: + TextHighlighter *m_textHighlighter; + Types definedTypes; + VectorString usedNamespaces; + ParsedImports parsedImports; + Str2StrMap importedHeaders; + TokenSequence fullTokens; + std::string editedText; + CompileErrors compileErrors; + VectorString linesOfColors; + public: + RequiredInputs() : m_textHighlighter(nullptr) {}; + explicit RequiredInputs(TextHighlighter *textHighlighter) : m_textHighlighter(textHighlighter) {} + void setRequiredInputs(); + void setTypes(); + void setNamespaces(); + void setImports(); + void setText(); + void setCompileErrors(); + }; /// to define functions and types using Definitions = std::map; /// to define global variables @@ -55,16 +84,15 @@ namespace hex::plugin::builtin { /// to define UDT and function variables using VariableMap = std::map; private: - std::string m_text; - std::vector m_lines; - std::vector m_firstTokenIdOfLine; + + VectorString m_lines; + TokenIdVector m_firstTokenIdOfLine; ViewPatternEditor *m_viewPatternEditor; - std::vector m_excludedLocations; - std::vector m_tokens; + + TokenColor m_tokenColors; - std::unique_ptr *patternLanguage; - std::vector m_compileErrors; - std::map> m_instances; + + Instances m_instances; Definitions m_UDTDefinitions; Definitions m_functionDefinitions; @@ -78,15 +106,16 @@ namespace hex::plugin::builtin { VariableMap m_functionVariables; Variables m_globalVariables; - std::map> m_parsedImports; - std::map m_attributeFunctionArgumentType; - std::map m_typeDefMap; - std::map m_typeDefInvMap; - std::vector m_nameSpaces; - std::vector m_UDTs; + + Str2StrMap m_attributeFunctionArgumentType; + Str2StrMap m_typeDefMap; + Str2StrMap m_typeDefInvMap; + + VectorString m_UDTs; std::set m_taggedIdentifiers; std::set m_memberChains; std::set m_scopeChains; + RequiredInputs m_requiredInputs; TokenIter m_curr; TokenIter m_startToken, m_originalPosition, m_partOriginalPosition; @@ -98,7 +127,6 @@ namespace hex::plugin::builtin { const static IdentifierTypeColor m_identifierTypeColor; const static TokenTypeColor m_tokenTypeColor; - i32 m_runningColorizers=0; public: /// Intervals are the sets finite contiguous non-negative integer that @@ -118,49 +146,29 @@ namespace hex::plugin::builtin { i32 start; i32 end; Interval() : start(0), end(0) {} - Interval(i32 start, i32 end) : start(start), end(end) { - if (start > end) - throw std::invalid_argument("Interval start must be less than or equal to end"); - } - bool operator<(const Interval &other) const { - return other.end > end; - } - bool operator>(const Interval &other) const { - return end > other.end; - } - bool operator==(const Interval &other) const { - return start == other.start && end == other.end; - } - bool operator!=(const Interval &other) const { - return start != other.start || end != other.end; - } - bool operator<=(const Interval &other) const { - return other.end >= end; - } - bool operator>=(const Interval &other) const { - return end >= other.end; - } - bool contains(const Interval &other) const { - return other.start >= start && other.end <= end; - } - bool contains(i32 value) const { - return value >= start && value <= end; - } - bool contiguous(const Interval &other) const { - return ((start - other.end) == 1 || (other.start - end) == 1); - } + Interval(i32 start, i32 end); + bool operator<(const Interval &other) const; + bool operator>(const Interval &other) const; + bool operator==(const Interval &other) const; + bool operator!=(const Interval &other) const; + bool operator<=(const Interval &other) const; + bool operator>=(const Interval &other) const; + bool contains(const Interval &other) const; + bool contains(i32 value) const ; + bool contiguous(const Interval &other) const; }; - std::atomic m_needsToUpdateColors = true; - std::atomic m_wasInterrupted = false; - std::atomic m_interrupt = false; + constexpr static u32 Normal = 0; + constexpr static u32 Not = 1; + pl::PatternLanguage *getPatternLanguage(); + void updateRequiredInputs(); + RequiredInputs& getRequiredInputs(); + ViewPatternEditor* getViewPatternEditor(); + void setViewPatternEditor(ViewPatternEditor *viewPatternEditor); - void interrupt() { - m_interrupt = true; - } - - TextHighlighter(ViewPatternEditor *viewPatternEditor, std::unique_ptr *patternLanguage ) : - m_viewPatternEditor(viewPatternEditor), patternLanguage(patternLanguage), m_needsToUpdateColors(true) {} + TextHighlighter(); + ~TextHighlighter(); + explicit TextHighlighter(ViewPatternEditor *viewPatternEditor) : m_viewPatternEditor(viewPatternEditor) {} /** * @brief Entry point to syntax highlighting */ @@ -184,14 +192,16 @@ namespace hex::plugin::builtin { /** * @brief Only identifiers not in chains should remain */ + void colorRemainingIdentifierTokens(); + /** * @brief Renders compile errors in real time */ void renderErrors(); /// A token range is the set of token indices of a definition. The namespace token /// ranges are obtained first because they are needed to obtain unique identifiers. - void getAllTokenRanges(IdentifierType identifierTypeToSearch); + void getTokenRanges(IdentifierType identifierTypeToSearch); /// The global scope is the complement of the union of all the function and UDT token ranges void getGlobalTokenRanges(); /// If the current token is a function or UDT, creates a map entry from the name to the token range. These are ordered alphabetically by name. @@ -217,14 +227,14 @@ namespace hex::plugin::builtin { /// Chains are sequences of identifiers separated by scope resolution or dot operators. void fixChains(); bool colorSeparatorScopeChain(); + bool colorRealSeparatorScopeChain(); + bool colorImplicitSeparatorScopeChain(); bool colorOperatorDotChain(); /// Returns the next/previous valid source code line u32 nextLine(u32 line); u32 previousLine(u32 line); /// Loads the source code and calculates the first token index of each line void loadText(); - /// Used to obtain the color to be applied. - ui::TextEditor::PaletteIndex getPaletteIndex(Token::Literal *literal); /// The complement of a set is also known as its inverse void invertGlobalTokenRange(); /// Starting at the identifier, it tracks all the scope resolution and dot operators and returns the full chain without arrays, templates, pointers,... @@ -251,11 +261,11 @@ namespace hex::plugin::builtin { void skipDelimiters(i32 maxSkipCount, Token delimiter[2], i8 increment); void skipToken(Token token, i8 step=1); /// from given or current names find the corresponding definition - bool findIdentifierDefinition(Definition &result, const std::string &optionalIdentifierName = "", std::string optionalName = "", bool setInstances = false); + bool findIdentifierDefinition(Definition &result, const std::string &optionalIdentifierName = "", std::string optionalName = "", bool optional = false); /// To deal with the Parent keyword std::optional setChildrenTypes(); - bool findParentTypes(std::vector &parentTypes, const std::string &optionalUDTName=""); - bool findAllParentTypes(std::vector &parentTypes, std::vector &identifiers, std::string &optionalFullName); + bool findParentTypes(VectorString &parentTypes, const std::string &optionalName=""); + bool findAllParentTypes(VectorString &parentTypes, std::vector &identifiers, std::string &optionalFullName); bool tryParentType(const std::string &parentType, std::string &variableName, std::optional &result, std::vector &identifiers); /// Convenience function bool isTokenIdValid(i32 tokenId); @@ -274,160 +284,22 @@ namespace hex::plugin::builtin { void getTokenIdForArgument(i32 start, i32 argNumber, Token delimiter); ///Creates a map from function name to argument type void linkAttribute(); - /// Comment and strings usethese function to determine their coordinates - template ui::TextEditor::Coordinates commentCoordinates(Token *token); - ui::TextEditor::Coordinates stringCoordinates(); - /// Returns the number of tasks highlighting code. Shouldn't be > 1 - i32 getRunningColorizers() { - return m_runningColorizers; - } - enum class HighlightStage { - Starting, - NamespaceTokenRanges, - UDTTokenRanges, - FunctionTokenRanges, - GlobalTokenRanges, - FixGlobalVariables, - SetInitialColors, - LoadInstances, - AttributeTokenRanges, - Definitions, - FixAutos, - FixChains, - ExcludedLocations, - ColorRemainingTokens, - SetRequestedIdentifierColors, - Stage1, - Stage2, - Stage3, - Stage4, - Stage5, - Stage6, - Stage7, - Stage8, - Stage9, - Stage10, - Stage11, - }; - - HighlightStage m_highlightStage = HighlightStage::Starting; /// The following functions were copied from the parser and some were modified - template - T *getValue(const i32 index) { - return const_cast(std::get_if(&m_curr[index].value)); - } - - void next(i32 count = 1) { - if (m_interrupt) { - m_interrupt = false; - throw std::out_of_range("Highlights were deliberately interrupted"); - } - if (count == 0) - return; - i32 id = getTokenId(m_curr->location); - i32 maxChange; - if (count > 0) - maxChange = std::min(count,static_cast(m_tokens.size() - id)); - else - maxChange = -std::min(-count,id); - m_curr += maxChange; - } - constexpr static u32 Normal = 0; - constexpr static u32 Not = 1; - - bool begin() { - m_originalPosition = m_curr; - - return true; - } - - void partBegin() { - m_partOriginalPosition = m_curr; - } - - void reset() { - m_curr = m_originalPosition; - } - - void partReset() { - m_curr = m_partOriginalPosition; - } - - bool resetIfFailed(const bool value) { - if (!value) reset(); - - return value; - } - - template - bool sequenceImpl() { - if constexpr (S == Normal) - return true; - else if constexpr (S == Not) - return false; - else - std::unreachable(); - } - - template - bool matchOne(const Token &token) { - if constexpr (S == Normal) { - if (!peek(token)) { - partReset(); - return false; - } - - next(); - return true; - } else if constexpr (S == Not) { - if (!peek(token)) - return true; - - next(); - partReset(); - return false; - } else - std::unreachable(); - } - - template - bool sequenceImpl(const auto &... args) { - return (matchOne(args) && ...); - } - - template - bool sequence(const Token &token, const auto &... args) { - partBegin(); - return sequenceImpl(token, args...); - } - - bool isValid() { - Token token; - try { - token = m_curr[0]; - } - catch (const std::out_of_range &e) { - auto t = e.what(); - if (t == nullptr) - return false; - return false; - } - if (!isLocationValid(token.location)) - return false; - return true; - } - - bool peek(const Token &token, const i32 index = 0) { - if (!isValid()) - return false; - i32 id = getTokenId(m_curr->location); - if (id+index < 0 || id+index >= (i32)m_tokens.size()) - return false; - return m_curr[index].type == token.type && m_curr[index] == token.value; - } - + template T *getValue(const i32 index); + void next(i32 count = 1); + bool begin(); + void partBegin(); + void reset(); + void partReset(); + bool resetIfFailed(const bool value) ; + template bool sequenceImpl(); + template bool matchOne(const Token &token); + template bool sequenceImpl(const auto &... args); + template bool sequence(const Token &token, const auto &... args); + bool isValid(); + bool peek(const Token &token, const i32 index = 0); }; } \ No newline at end of file diff --git a/plugins/builtin/include/content/views/view_pattern_editor.hpp b/plugins/builtin/include/content/views/view_pattern_editor.hpp index 9e7058a64..9627d593d 100644 --- a/plugins/builtin/include/content/views/view_pattern_editor.hpp +++ b/plugins/builtin/include/content/views/view_pattern_editor.hpp @@ -9,7 +9,6 @@ #include #include -#include #include #include @@ -32,52 +31,34 @@ namespace hex::plugin::builtin { std::string m_sharedSource; }; + using TextHighlighter = hex::plugin::builtin::TextHighlighter; class ViewPatternEditor : public View::Window { public: ViewPatternEditor(); ~ViewPatternEditor() override; void drawAlwaysVisibleContent() override; - std::unique_ptr *getPatternLanguage() { - return &m_editorRuntime; - } - - ui::TextEditor *getTextEditor() { - auto provider = ImHexApi::Provider::get(); - if (provider == nullptr) - return nullptr; - - return &m_textEditor.get(provider); - } - - bool getChangesWereParsed() const { - return m_changesWereParsed; - } - - u32 getRunningParsers () const { - return m_runningParsers; - } - - u32 getRunningEvaluators () const { - return m_runningEvaluators; - } - - void setChangesWereParsed(bool changesWereParsed) { - m_changesWereParsed = changesWereParsed; - } - + std::unique_ptr *getPatternLanguage() { return &m_editorRuntime; } + ui::TextEditor *getTextEditor(); + u32 getRunningParsers () const { return m_runningParsers;} + u32 getRunningEvaluators () const { return m_runningEvaluators;} + u32 getRunningHighlighters () const { return m_runningHighlighters;} + void incrementRunningParsers( i32 amount) { m_runningParsers += amount; } + void incrementRunningEvaluators( i32 amount) { m_runningEvaluators += amount; } + void incrementRunningHighlighters( i32 amount) { m_runningHighlighters += amount; } + bool hasUnevaluatedChanges(prv::Provider *provider) const; + void setChangesWereParsed(bool changesWereParsed) { m_changesWereParsed = changesWereParsed;} + void setChangesWereColored(bool changesWereColored) { m_changesWereColored = changesWereColored; } void drawContent() override; - void setPopupWindowHeight(u32 height) { m_popupWindowHeight = height; } u32 getPopupWindowHeight() const { return m_popupWindowHeight; } - - enum class DangerousFunctionPerms : u8 { - Ask, - Allow, - Deny - }; + enum class DangerousFunctionPerms : u8 { Ask, Allow, Deny }; void drawHelpText() override; + void setWasInterrupted(bool wasInterrupted) { m_wasInterrupted = wasInterrupted; } + bool wasInterrupted() const { return m_wasInterrupted;} + void resetInterrupt() { m_wasInterrupted = false; m_interrupt = false;} + void interrupt() { m_interrupt = true; } private: class PopupAcceptPattern; @@ -127,19 +108,26 @@ namespace hex::plugin::builtin { std::atomic m_runningEvaluators = 0; std::atomic m_runningParsers = 0; + std::atomic m_runningHighlighters = 0; - std::atomic m_changesWereParsed = false; PerProvider m_hasUnevaluatedChanges; + std::atomic m_changesWereParsed; + PerProvider m_hasUncoloredChanges; + std::atomic m_changesWereColored; + std::atomic m_wasInterrupted; + std::atomic m_interrupt; + std::chrono::time_point m_lastEditorChangeTime; PerProvider m_textEditor, m_consoleEditor; + PerProvider m_textHighlighter; std::atomic m_consoleNeedsUpdate = false; std::atomic m_dangerousFunctionCalled = false; std::atomic m_dangerousFunctionsAllowed = DangerousFunctionPerms::Ask; - ContentRegistry::Settings::SettingsVariable m_suggestSupportedPatterns = true; - ContentRegistry::Settings::SettingsVariable m_autoApplyPatterns = false; + bool m_suggestSupportedPatterns = true; + bool m_autoApplyPatterns = false; PerProvider m_visualizerDrawer; bool m_tooltipJustOpened = false; @@ -178,7 +166,7 @@ namespace hex::plugin::builtin { std::array m_accessHistory = {}; u32 m_accessHistoryIndex = 0; - ContentRegistry::Settings::SettingsVariable m_parentHighlightingEnabled = false; + bool m_parentHighlightingEnabled = true; bool m_replaceMode = false; bool m_openFindReplacePopUp = false; bool m_openGotoLinePopUp = false; @@ -188,13 +176,14 @@ namespace hex::plugin::builtin { PerProvider m_ignoreNextChangeEvent; PerProvider m_changeEventAcknowledgementPending; PerProvider m_patternFileDirty; + PerProvider m_patternFileInitialized; ImRect m_textEditorHoverBox; ImRect m_consoleHoverBox; std::string m_focusedSubWindowName; float m_popupWindowHeight = 0; float m_popupWindowHeightChange = 0; - bool m_frPopupIsClosed = true; + bool m_findReplacePopupIsClosed = true; bool m_gotoPopupIsClosed = true; static inline std::array m_findHistory; @@ -204,7 +193,6 @@ namespace hex::plugin::builtin { static inline u32 m_replaceHistorySize = 0; static inline u32 m_replaceHistoryIndex = 0; - TextHighlighter m_textHighlighter = TextHighlighter(this,&this->m_editorRuntime); private: void drawConsole(ImVec2 size); void drawDebugger(ImVec2 size); diff --git a/plugins/builtin/source/content/text_highlighting/pattern_language.cpp b/plugins/builtin/source/content/text_highlighting/pattern_language.cpp index a40dd3b37..a96de0dda 100644 --- a/plugins/builtin/source/content/text_highlighting/pattern_language.cpp +++ b/plugins/builtin/source/content/text_highlighting/pattern_language.cpp @@ -15,14 +15,246 @@ namespace hex::plugin::builtin { using namespace pl::core; - using Identifier = Token::Identifier; - using Keyword = Token::Keyword; - using Separator = Token::Separator; - using Operator = Token::Operator; - using Comment = Token::Comment; - using DocComment = Token::DocComment; - using Literal = Token::Literal; - using ValueType = Token::ValueType; + using Identifier = Token::Identifier; + using Keyword = Token::Keyword; + using Separator = Token::Separator; + using Operator = Token::Operator; + using Comment = Token::Comment; + using DocComment = Token::DocComment; + using Literal = Token::Literal; + using ValueType = Token::ValueType; + using RequiredInputs = TextHighlighter::RequiredInputs; + using Interval = TextHighlighter::Interval; + + Interval::Interval(i32 start, i32 end) : start(start), end(end) { + if (start > end) + throw std::invalid_argument("Interval start must be less than or equal to end"); + } + + bool Interval::operator<(const Interval &other) const { + return other.end > end; + } + + bool Interval::operator>(const Interval &other) const { + return end > other.end; + } + + bool Interval::operator==(const Interval &other) const { + return start == other.start && end == other.end; + } + + bool Interval::operator!=(const Interval &other) const { + return start != other.start || end != other.end; + } + + bool Interval::operator<=(const Interval &other) const { + return other.end >= end; + } + + bool Interval::operator>=(const Interval &other) const { + return end >= other.end; + } + + bool Interval::contains(const Interval &other) const { + return other.start >= start && other.end <= end; + } + + bool Interval::contains(i32 value) const { + return value >= start && value <= end; + } + + bool Interval::contiguous(const Interval &other) const { + return ((start - other.end) == 1 || (other.start - end) == 1); + } + + void TextHighlighter::next(i32 count) { + if (m_viewPatternEditor->wasInterrupted()) { + m_viewPatternEditor->resetInterrupt(); + throw std::out_of_range("Highlights were deliberately interrupted"); + } + if (count == 0) + return; + i32 id = getTokenId(m_curr->location); + i32 maxChange; + if (count > 0) + maxChange = std::min(count,static_cast(m_requiredInputs.fullTokens.size() - id)); + else + maxChange = -std::min(-count,id); + m_curr += maxChange; + } + + pl::PatternLanguage *TextHighlighter::getPatternLanguage() { + return m_viewPatternEditor->getPatternLanguage()->get(); + } + + void TextHighlighter::RequiredInputs::setTypes() { + auto &types = m_textHighlighter->getPatternLanguage()->getInternals().parser.get()->getTypes(); + std::ranges::copy(types.begin(), types.end(), std::inserter(definedTypes, definedTypes.begin())); + } + + void TextHighlighter::RequiredInputs::setNamespaces() { + auto &namespaces = m_textHighlighter->getPatternLanguage()->getInternals().preprocessor.get()->getNamespaces(); + usedNamespaces.resize(namespaces.size()); + std::ranges::copy(namespaces, usedNamespaces.begin()); + } + + void TextHighlighter::RequiredInputs::setImports() { + auto *preprocessor = m_textHighlighter->getPatternLanguage()->getInternals().preprocessor.get(); + std::ranges::copy(preprocessor->getParsedImports().begin(), preprocessor->getParsedImports().end(), std::inserter(parsedImports, parsedImports.begin())); + m_textHighlighter->clearVariables(); + for (auto &[name, tokens]: parsedImports) { + importedHeaders[name] = tokens[0].location.source->content; + if (importedHeaders[name].empty() || importedHeaders[name] == "\n") + continue; + fullTokens = tokens; + editedText = importedHeaders[name]; + m_textHighlighter->loadText(); + m_textHighlighter->processSource(); + } + } + + void TextHighlighter::RequiredInputs::setText() { + ui::TextEditor *editor = m_textHighlighter->m_viewPatternEditor->getTextEditor(); + if (editor == nullptr) + return; + fullTokens.clear(); + auto &result = m_textHighlighter->getPatternLanguage()->getInternals().preprocessor.get()->getResult(); + std::ranges::copy(result.begin(),result.end(),std::back_inserter(fullTokens)); + editedText = editor->getText(); + m_textHighlighter->loadText();; + + linesOfColors.clear(); + for (auto &line : m_textHighlighter->m_lines) + linesOfColors.push_back(std::string(line.size(), ' ')); + } + + void TextHighlighter::RequiredInputs::setCompileErrors() { + auto errors = m_textHighlighter->getPatternLanguage()->getCompileErrors(); + compileErrors.resize(errors.size()); + std::ranges::copy(errors, compileErrors.begin()); + } + + void TextHighlighter::RequiredInputs::setRequiredInputs() { + setTypes(); + setNamespaces(); + setImports(); + setText(); + setCompileErrors(); + } + + void TextHighlighter::updateRequiredInputs() { + m_requiredInputs.m_textHighlighter = this; + m_requiredInputs.setRequiredInputs(); + } + + RequiredInputs& TextHighlighter::getRequiredInputs() { + return m_requiredInputs; + } + + ViewPatternEditor* TextHighlighter::getViewPatternEditor() { + return m_viewPatternEditor; + } + + template + T *TextHighlighter::getValue(const i32 index) { + return const_cast(std::get_if(&m_curr[index].value)); + } + + + void TextHighlighter::setViewPatternEditor(ViewPatternEditor *viewPatternEditor) { + m_viewPatternEditor = viewPatternEditor; + } + + bool TextHighlighter::begin() { + m_originalPosition = m_curr; + + return true; + } + + void TextHighlighter::partBegin() { + m_partOriginalPosition = m_curr; + } + + void TextHighlighter::reset() { + m_curr = m_originalPosition; + } + + void TextHighlighter::partReset() { + m_curr = m_partOriginalPosition; + } + + bool TextHighlighter::resetIfFailed(const bool value) { + if (!value) reset(); + + return value; + } + + template + bool TextHighlighter::sequenceImpl() { + if constexpr (S == Normal) + return true; + else if constexpr (S == Not) + return false; + else + std::unreachable(); + } + + template + bool TextHighlighter::matchOne(const Token &token) { + if constexpr (S == Normal) { + if (!peek(token)) { + partReset(); + return false; + } + + next(); + return true; + } else if constexpr (S == Not) { + if (!peek(token)) + return true; + + next(); + partReset(); + return false; + } else + std::unreachable(); + } + + template + bool TextHighlighter::sequenceImpl(const auto &... args) { + return (matchOne(args) && ...); + } + + template + bool TextHighlighter::sequence(const Token &token, const auto &... args) { + partBegin(); + return sequenceImpl(token, args...); + } + + bool TextHighlighter::isValid() { + Token token; + try { + token = m_curr[0]; + } + catch (const std::out_of_range &e) { + auto t = e.what(); + if (t == nullptr) + return false; + return false; + } + if (!isLocationValid(token.location)) + return false; + return true; + } + + bool TextHighlighter::peek(const Token &token, const i32 index) { + if (!isValid()) + return false; + i32 id = getTokenId(m_curr->location); + if (id+index < 0 || id+index >= (i32)m_requiredInputs.fullTokens.size()) + return false; + return m_curr[index].type == token.type && m_curr[index] == token.value; + } bool TextHighlighter::getIdentifierName(std::string &identifierName, Identifier *identifier) { auto keyword = getValue(0); @@ -47,6 +279,9 @@ namespace hex::plugin::builtin { return false; } + TextHighlighter::TextHighlighter() {} + TextHighlighter::~TextHighlighter() {} + // Returns a chain of identifiers like a.b.c or a::b::c bool TextHighlighter::getFullName(std::string &identifierName, std::vector &identifiers, bool preserveCurr) { Identifier *identifier = nullptr; @@ -127,7 +362,7 @@ namespace hex::plugin::builtin { } bool found = true; for (const auto &name : vectorString) { - found = found || std::ranges::find(m_nameSpaces, name) != m_nameSpaces.end(); + found = found || std::ranges::find(m_requiredInputs.usedNamespaces, name) != m_requiredInputs.usedNamespaces.end(); } if (found) { if (!shortName.empty()) @@ -171,10 +406,8 @@ namespace hex::plugin::builtin { } // Finds the token range of a function, namespace or UDT - bool TextHighlighter::getTokenRange(std::vector keywords, - TextHighlighter::UnorderedBlocks &tokenRange, - TextHighlighter::OrderedBlocks &tokenRangeInv, - bool fullName, VariableScopes *blocks) { + bool TextHighlighter::getTokenRange(std::vector keywords, UnorderedBlocks &tokenRange, + OrderedBlocks &tokenRangeInv, bool fullName, VariableScopes *blocks) { bool addArgumentBlock = !fullName; std::vector tokenStack; @@ -195,7 +428,7 @@ namespace hex::plugin::builtin { name = fmt::format("{}::{}", nameSpace, name); } - i32 tokenCount = m_tokens.size(); + i32 tokenCount = m_requiredInputs.fullTokens.size(); auto saveCurr = m_curr - 1; skipTemplate(200); next(); @@ -223,7 +456,7 @@ namespace hex::plugin::builtin { return false; u32 nestedLevel = 0; next(); - auto endToken = TokenIter(m_tokens.begin() + tokenCount, m_tokens.end()); + auto endToken = TokenIter(m_requiredInputs.fullTokens.begin() + tokenCount, m_requiredInputs.fullTokens.end()); while (endToken > m_curr) { if (sequence(tkn::Separator::LeftBrace)) { @@ -252,6 +485,8 @@ namespace hex::plugin::builtin { else next(); } + if (m_curr > endToken || endToken == m_curr) + return false; i32 index2 = getTokenId(m_curr->location); if (index2 > index1 && index2 < tokenCount) { @@ -273,15 +508,15 @@ namespace hex::plugin::builtin { } // Searches through tokens and loads all the ranges of one kind. First namespaces are searched. - void TextHighlighter::getAllTokenRanges(IdentifierType identifierTypeToSearch) { + void TextHighlighter::getTokenRanges(IdentifierType identifierTypeToSearch) { - if (m_tokens.empty()) + if (m_requiredInputs.fullTokens.size() == 1) return; Identifier *identifier; IdentifierType identifierType; - m_startToken = TokenIter(m_tokens.begin(), m_tokens.end()); - auto endToken = TokenIter(m_tokens.end(), m_tokens.end()); + m_startToken = TokenIter(m_requiredInputs.fullTokens.begin(), m_requiredInputs.fullTokens.end()); + auto endToken = TokenIter(m_requiredInputs.fullTokens.end(), m_requiredInputs.fullTokens.end()); for (m_curr = m_startToken; endToken > m_curr; next()) { auto curr = m_curr; @@ -297,8 +532,8 @@ namespace hex::plugin::builtin { getTokenRange({tkn::Keyword::Function}, m_functionTokenRange, m_namespaceTokenRange, false, &m_functionBlocks); break; case IdentifierType::NameSpace: - if (std::ranges::find(m_nameSpaces, name) == m_nameSpaces.end()) - m_nameSpaces.push_back(name); + if (std::ranges::find(m_requiredInputs.usedNamespaces, name) == m_requiredInputs.usedNamespaces.end()) + m_requiredInputs.usedNamespaces.push_back(name); getTokenRange({tkn::Keyword::Namespace}, m_functionTokenRange, m_namespaceTokenRange, true, nullptr); break; case IdentifierType::UDT: @@ -327,7 +562,7 @@ namespace hex::plugin::builtin { if (!isValid()) return; i32 tokenId = getTokenId(m_curr->location); - auto tokenCount = m_tokens.size(); + auto tokenCount = m_requiredInputs.fullTokens.size(); if (tokenId == -1 || tokenId >= (i32) tokenCount-1) return; @@ -458,7 +693,7 @@ namespace hex::plugin::builtin { } else if (separator == "::") { next(); - if (std::ranges::find(m_nameSpaces, currentName) != m_nameSpaces.end()) { + if (std::ranges::find(m_requiredInputs.usedNamespaces, currentName) != m_requiredInputs.usedNamespaces.end()) { nameSpace += currentName + "::"; variableParentType = vectorString[index]; @@ -507,7 +742,7 @@ namespace hex::plugin::builtin { for (auto instance: m_instances[name]) { if (block.contains(instance)) { - if (auto identifier = std::get_if(&m_tokens[instance].value); + if (auto identifier = std::get_if(&m_requiredInputs.fullTokens.at(instance).value); identifier != nullptr && identifier->getType() == IdentifierType::Unknown) setIdentifierColor(instance, definition.idType); } @@ -545,10 +780,9 @@ namespace hex::plugin::builtin { blocksIterBegin = m_functionBlocks[name].begin(); blocksIterEnd = m_functionBlocks[name].end(); --blocksIterEnd; - } else if (m_globalVariables.contains(identifierName)) { definitions = m_globalVariables[identifierName]; - tokenRange = Interval(0, m_tokens.size()); + tokenRange = Interval(0, m_requiredInputs.fullTokens.size()); blocks.insert(tokenRange); blocksIterBegin = blocks.begin(); blocksIterEnd = blocks.end(); @@ -622,7 +856,7 @@ namespace hex::plugin::builtin { bool TextHighlighter::colorOperatorDotChain() { std::vector identifiers; std::string variableName; - auto tokenCount = m_tokens.size(); + auto tokenCount = m_requiredInputs.fullTokens.size(); if (!getQualifiedName(variableName, identifiers, true)) return false; @@ -632,12 +866,12 @@ namespace hex::plugin::builtin { u32 currentLine = m_curr->location.line - 1; - u32 startingLineTokenIndex = m_firstTokenIdOfLine[currentLine]; + u32 startingLineTokenIndex = m_firstTokenIdOfLine.at(currentLine); if (startingLineTokenIndex == 0xFFFFFFFFu || startingLineTokenIndex > tokenCount) return false; - if (auto *keyword = std::get_if(&m_tokens[startingLineTokenIndex].value); + if (auto *keyword = std::get_if(&m_requiredInputs.fullTokens.at(startingLineTokenIndex).value); keyword != nullptr && *keyword == Keyword::Import) { while (index < vectorString.size()) { auto tokenIndex = getTokenId(m_curr->location); @@ -684,9 +918,9 @@ namespace hex::plugin::builtin { std::string templateName; auto instances = m_instances[variableParentType]; for (auto instance : instances) { - if (auto *identifier = std::get_if(&m_tokens[instance].value); identifier != nullptr && identifier->getType() == IdentifierType::TemplateArgument) { + if (auto *identifier = std::get_if(&m_requiredInputs.fullTokens.at(instance).value); identifier != nullptr && identifier->getType() == IdentifierType::TemplateArgument) { auto tokenRange = m_UDTTokenRange[result.typeStr]; - auto tokenIndex = m_firstTokenIdOfLine[getLocation(parentDefinition.tokenIndex).line - 1]; + auto tokenIndex = m_firstTokenIdOfLine.at(getLocation(parentDefinition.tokenIndex).line - 1); i32 argNumber = getArgumentNumber(tokenRange.start, instance); getTokenIdForArgument(tokenIndex, argNumber, tkn::Operator::BoolLessThan); if (auto *identifier2 = std::get_if(&m_curr->value); identifier2 != nullptr) { @@ -707,9 +941,9 @@ namespace hex::plugin::builtin { std::string typeName; instances = m_instances[variableParentType]; for (auto instance: instances) { - if (auto *identifier = std::get_if(&m_tokens[instance].value); + if (auto *identifier = std::get_if(&m_requiredInputs.fullTokens.at(instance).value); identifier != nullptr && identifier->getType() == IdentifierType::Typedef) { - if (auto *identifier2 = std::get_if(&m_tokens[instance + 2].value); + if (auto *identifier2 = std::get_if(&m_requiredInputs.fullTokens.at(instance + 2).value); identifier2 != nullptr) { typeName = identifier2->get(); break; @@ -740,12 +974,40 @@ namespace hex::plugin::builtin { } bool TextHighlighter::colorSeparatorScopeChain() { + if (!colorRealSeparatorScopeChain()) + return colorImplicitSeparatorScopeChain(); + return true; + } + + bool TextHighlighter::colorImplicitSeparatorScopeChain() { + auto identifier = getValue(0); + if (identifier == nullptr) + return false; + std::string identifierName = identifier->get(); + std::string nameSpace; + if (!findNamespace(nameSpace)) + return false; + std::string qualifiedName = fmt::format("{}::{}", nameSpace, identifierName); + if (m_UDTDefinitions.contains(qualifiedName)) { + setIdentifierColor(-1, IdentifierType::UDT); + return true; + } else if (m_functionDefinitions.contains(qualifiedName)) { + setIdentifierColor(-1, IdentifierType::Function); + return true; + } else + return false; + + } + + bool TextHighlighter::colorRealSeparatorScopeChain() { std::vector identifiers; std::string identifierName; - if (!getQualifiedName(identifierName, identifiers, true)) + if (!getQualifiedName(identifierName, identifiers, true)) { return false; - auto tokenCount = m_tokens.size(); + } + + auto tokenCount = m_requiredInputs.fullTokens.size(); auto vectorString = wolv::util::splitString(identifierName, "::"); auto vectorStringCount = vectorString.size(); if (identifiers.size() != vectorStringCount) @@ -757,14 +1019,14 @@ namespace hex::plugin::builtin { auto name = vectorString[i]; auto identifier = identifiers[i]; - if (std::ranges::find(m_nameSpaces, name) != m_nameSpaces.end()) { + if (std::ranges::find(m_requiredInputs.usedNamespaces, name) != m_requiredInputs.usedNamespaces.end()) { setIdentifierColor(-1, IdentifierType::NameSpace); nameSpace += name + "::"; } else if (m_UDTDefinitions.contains(nameSpace+name)) { name.insert(0, nameSpace); auto udtDefinition = m_UDTDefinitions[name]; auto definitionIndex = udtDefinition.tokenIndex-1; - if (auto *keyword = std::get_if(&m_tokens[definitionIndex].value); keyword != nullptr) { + if (auto *keyword = std::get_if(&m_requiredInputs.fullTokens.at(definitionIndex).value); keyword != nullptr) { setIdentifierColor(-1, IdentifierType::UDT); if (*keyword == Keyword::Enum) { next(); @@ -800,7 +1062,7 @@ namespace hex::plugin::builtin { } m_curr = curr; - if (std::ranges::find(m_nameSpaces, identifierName) != m_nameSpaces.end()) { + if (std::ranges::find(m_requiredInputs.usedNamespaces, identifierName) != m_requiredInputs.usedNamespaces.end()) { setIdentifierColor(-1, IdentifierType::NameSpace); return true; } @@ -868,14 +1130,14 @@ namespace hex::plugin::builtin { return !nameSpace.empty(); } -//The context is the name of the function or UDT that the variable is in. +//The context is the name of the function or UDT that the variable is in aka its parent. std::string TextHighlighter::findIdentifierTypeStr(const std::string &identifierName, std::string context) { Definition result; findIdentifierDefinition(result, identifierName, context); return result.typeStr; } - //The context is the name of the function or UDT that the variable is in. + TextHighlighter::IdentifierType TextHighlighter::findIdentifierType(const std::string &identifierName, std::string context) { Definition result; findIdentifierDefinition(result, identifierName, context); @@ -886,9 +1148,6 @@ namespace hex::plugin::builtin { void TextHighlighter::linkAttribute() { auto curr = m_curr; bool qualifiedAttribute = false; - using Types = std::map>; - auto parser = patternLanguage->get()->getInternals().parser.get(); - Types types = parser->getTypes(); while (sequence(tkn::Literal::Identifier, tkn::Operator::ScopeResolution)) qualifiedAttribute = true; @@ -911,25 +1170,30 @@ namespace hex::plugin::builtin { if (sequence(tkn::Separator::LeftParenthesis, tkn::Literal::String)) { functionName = getValue(-1)->toString(false); - - if (!functionName.contains("::")) { - std::string namespaceName; - - if (findNamespace(namespaceName)) - functionName = fmt::format("{}::{}", namespaceName, functionName); - } else { - - auto vectorString = wolv::util::splitString(functionName, "::"); - vectorString.pop_back(); - for (const auto &nameSpace: vectorString) { - - if (std::ranges::find(m_nameSpaces, nameSpace) == m_nameSpaces.end()) - m_nameSpaces.push_back(nameSpace); - } - } + } else if (sequence(tkn::Separator::LeftParenthesis, tkn::Literal::Identifier)) { + next(-1); + std::vector identifiers; + if (!getFullName(functionName, identifiers, false)) + functionName = getValue(0)->get(); } else return; + if (!functionName.contains("::")) { + std::string namespaceName; + + if (findNamespace(namespaceName)) + functionName = fmt::format("{}::{}", namespaceName, functionName); + } else { + + auto vectorString = wolv::util::splitString(functionName, "::"); + vectorString.pop_back(); + for (const auto &nameSpace: vectorString) { + + if (std::ranges::find(m_requiredInputs.usedNamespaces, nameSpace) == m_requiredInputs.usedNamespaces.end()) + m_requiredInputs.usedNamespaces.push_back(nameSpace); + } + } + u32 line = m_curr->location.line; i32 tokenIndex; @@ -938,7 +1202,7 @@ namespace hex::plugin::builtin { if (line = previousLine(line); line > m_firstTokenIdOfLine.size()-1) return; - if (tokenIndex = m_firstTokenIdOfLine[line]; !isTokenIdValid(tokenIndex)) + if (tokenIndex = m_firstTokenIdOfLine.at(line); !isTokenIdValid(tokenIndex)) return; m_curr = m_startToken; @@ -967,7 +1231,7 @@ namespace hex::plugin::builtin { if (findNamespace(namespaceName)) UDTName = fmt::format("{}::{}", namespaceName, UDTName); } - if (types.contains(UDTName)) + if (m_requiredInputs.definedTypes.contains(UDTName)) m_attributeFunctionArgumentType[functionName] = UDTName; } else if (sequence(tkn::ValueType::Any)) { @@ -1055,8 +1319,8 @@ namespace hex::plugin::builtin { } else { auto curr = m_curr; for (const auto &[name, range] : m_UDTTokenRange) { - auto startToken = TokenIter(m_tokens.begin()+range.start,m_tokens.begin()+range.end); - auto endToken = TokenIter(m_tokens.begin()+range.end,m_tokens.end()); + auto startToken = TokenIter(m_requiredInputs.fullTokens.begin()+range.start,m_requiredInputs.fullTokens.begin()+range.end); + auto endToken = TokenIter(m_requiredInputs.fullTokens.begin()+range.end,m_requiredInputs.fullTokens.end()); for ( m_curr = startToken; endToken > m_curr; next()) { if (auto *identifier = std::get_if(&m_curr->value); identifier != nullptr) { @@ -1173,20 +1437,6 @@ namespace hex::plugin::builtin { return result; } - const TextHighlighter::TokenTypeColor TextHighlighter::m_tokenTypeColor = { - {Token::Type::Keyword, ui::TextEditor::PaletteIndex::Keyword}, - {Token::Type::ValueType, ui::TextEditor::PaletteIndex::BuiltInType}, - {Token::Type::Operator, ui::TextEditor::PaletteIndex::Operator}, - {Token::Type::Separator, ui::TextEditor::PaletteIndex::Separator}, - {Token::Type::String, ui::TextEditor::PaletteIndex::StringLiteral}, - {Token::Type::Directive, ui::TextEditor::PaletteIndex::Directive}, - {Token::Type::Comment, ui::TextEditor::PaletteIndex::Comment}, - {Token::Type::Integer, ui::TextEditor::PaletteIndex::NumericLiteral}, - {Token::Type::Identifier, ui::TextEditor::PaletteIndex::Identifier}, - {Token::Type::DocComment, ui::TextEditor::PaletteIndex::DocComment} - - }; - const TextHighlighter::IdentifierTypeColor TextHighlighter::m_identifierTypeColor = { {Identifier::IdentifierType::Macro, ui::TextEditor::PaletteIndex::PreprocIdentifier}, {Identifier::IdentifierType::UDT, ui::TextEditor::PaletteIndex::UserDefinedType}, @@ -1209,19 +1459,6 @@ namespace hex::plugin::builtin { {Identifier::IdentifierType::GlobalVariable, ui::TextEditor::PaletteIndex::GlobalVariable}, }; -// Second paletteIndex called from processLineTokens to process literals - ui::TextEditor::PaletteIndex TextHighlighter::getPaletteIndex(Literal *literal) { - - if (literal->isFloatingPoint() || literal->isSigned() || literal->isUnsigned()) - return ui::TextEditor::PaletteIndex::NumericLiteral; - - else if (literal->isCharacter() || literal->isBoolean()) return ui::TextEditor::PaletteIndex::CharLiteral; - - else if (literal->isString()) return ui::TextEditor::PaletteIndex::StringLiteral; - - else return ui::TextEditor::PaletteIndex::Default; - } - // Render the compilation errors using squiggly lines void TextHighlighter::renderErrors() { const auto processMessage = [](const auto &message) { @@ -1239,8 +1476,8 @@ namespace hex::plugin::builtin { }; ui::TextEditor::ErrorMarkers errorMarkers; - if (!m_compileErrors.empty()) { - for (const auto &error: m_compileErrors) { + if (!m_requiredInputs.compileErrors.empty()) { + for (const auto &error: m_requiredInputs.compileErrors) { if (isLocationValid(error.getLocation())) { auto key = ui::TextEditor::Coordinates(error.getLocation().line, error.getLocation().column); @@ -1261,101 +1498,130 @@ namespace hex::plugin::builtin { // od every instance of the variable name in the code. void TextHighlighter::setInitialColors() { - m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); - auto endToken = TokenIter(m_tokens.end(), m_tokens.end()); + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_requiredInputs.fullTokens.begin(), m_requiredInputs.fullTokens.end()); + auto endToken = TokenIter(m_requiredInputs.fullTokens.end(), m_requiredInputs.fullTokens.end()); for (m_curr = m_startToken; endToken > m_curr; next()) { - if (peek(tkn::Literal::Identifier)) { + if (peek(tkn::Separator::EndOfProgram)) + return; - if (auto identifier = getValue(0); identifier != nullptr) { - if (auto identifierType = identifier->getType(); identifierType != IdentifierType::Unknown && identifierType != IdentifierType::MemberUnknown && - identifierType != IdentifierType::FunctionUnknown && identifierType != IdentifierType::ScopeResolutionUnknown) { + if (auto identifier = getValue(0); identifier != nullptr) { - setIdentifierColor(-1, identifierType); - } - } - } else if (peek(tkn::Separator::EndOfProgram)) - return; + if (auto identifierType = identifier->getType(); identifierType != IdentifierType::Unknown && identifierType != IdentifierType::MemberUnknown &&identifierType != IdentifierType::FunctionUnknown && identifierType != IdentifierType::ScopeResolutionUnknown) + setIdentifierColor(-1, identifierType); + } } } void TextHighlighter::loadInstances() { std::map> instances; - m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); - auto endToken = TokenIter(m_tokens.end(), m_tokens.end()); + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_requiredInputs.fullTokens.begin(), m_requiredInputs.fullTokens.end()); + auto endToken = TokenIter(m_requiredInputs.fullTokens.end(), m_requiredInputs.fullTokens.end()); for (m_curr = m_startToken; endToken > m_curr; next()) { - if (peek(tkn::Literal::Identifier)) { - std::string name; + if (peek(tkn::Literal::Identifier)) { + std::string name; - if (auto identifier = getValue(0); identifier != nullptr) { + if (auto identifier = getValue(0); identifier != nullptr) { - if (auto identifierType = identifier->getType(); identifierType != IdentifierType::Unknown && identifierType != IdentifierType::MemberUnknown && + if (auto identifierType = identifier->getType(); identifierType != IdentifierType::Unknown && identifierType != IdentifierType::MemberUnknown && identifierType != IdentifierType::FunctionUnknown && identifierType != IdentifierType::ScopeResolutionUnknown) { - name = identifier->get(); + name = identifier->get(); - if (identifierType == IdentifierType::Typedef) { - auto curr = m_curr; - next(); + if (identifierType == IdentifierType::Typedef) { + auto curr = m_curr; + //std::string typeName = ""; + skipTemplate(200, true); + next(); + if (sequence(tkn::Operator::Assign, tkn::Literal::Identifier)) { + next(-1); + i32 fullNameTokenId = getTokenId(m_curr->location); + std::vector identifiers; std::string typeName; - if (sequence(tkn::Operator::Assign, tkn::Literal::Identifier)) { - auto identifier2 = getValue(-1); + if (!getFullName(typeName, identifiers, false)) { + auto identifier2 = getValue(0); if (identifier2 != nullptr) { typeName = identifier2->get(); - if (!m_typeDefMap.contains(name) && !typeName.empty()) { - m_typeDefMap[name] = typeName; - m_typeDefInvMap[typeName] = name; - } + } + } else { + auto count = std::count(typeName.begin(), typeName.end(), ':') >> 1; + for (auto i = 0; i < count; i++) { + setIdentifierColor(fullNameTokenId, IdentifierType::NameSpace); + fullNameTokenId += 2; } } - - m_curr = curr; - } - } else { - name = identifier->get(); - auto curr = m_curr; - auto tokenIndex = getTokenId(m_curr->location); - skipArray(200, true); - next(); - bool chainStarted = false; - while (sequence(tkn::Operator::ScopeResolution, tkn::Literal::Identifier)) { - - if (identifier = getValue(-1); identifier != nullptr) - name += "::" + identifier->get(); - - if (!chainStarted) { - chainStarted = true; - m_scopeChains.insert(tokenIndex); + setIdentifierColor(fullNameTokenId, IdentifierType::UDT); + if (!m_typeDefMap.contains(name) && !typeName.empty()) { + m_typeDefMap[name] = typeName; + m_typeDefInvMap[typeName] = name; } - curr = m_curr; - } - while (sequence(tkn::Separator::Dot, tkn::Literal::Identifier)) { - - if (identifier = getValue(-1); identifier != nullptr) - name += "." + identifier->get(); - - if (!chainStarted) { - chainStarted = true; - m_memberChains.insert(tokenIndex); + auto start = m_curr; + skipTemplate(200, true); + auto end = m_curr; + for (m_curr = start; end > m_curr; next()) { + if (auto identifier2 = getValue(0); identifier2 != nullptr) { + auto tokenId = getTokenId(m_curr->location); + setIdentifierColor(tokenId, IdentifierType::TemplateArgument); + } } - skipArray(200, true); - curr = m_curr; } m_curr = curr; } + } else { + name = identifier->get(); + auto curr = m_curr; + auto tokenIndex = getTokenId(m_curr->location); + skipArray(200, true); + next(); + bool chainStarted = false; + while (sequence(tkn::Operator::ScopeResolution, tkn::Literal::Identifier)) { + + if (identifier = getValue(-1); identifier != nullptr) + name += "::" + identifier->get(); + std::string nameSpace; + if (!chainStarted) { + chainStarted = true; + m_scopeChains.insert(tokenIndex); + } else if (findNamespace(nameSpace) && !nameSpace.empty()) { + m_scopeChains.insert(tokenIndex); + } + + + curr = m_curr; + } + while (sequence(tkn::Separator::Dot, tkn::Literal::Identifier)) { + + if (identifier = getValue(-1); identifier != nullptr) + name += "." + identifier->get(); + + if (!chainStarted) { + chainStarted = true; + m_memberChains.insert(tokenIndex); + } + skipArray(200, true); + curr = m_curr; + } + if (peek(tkn::Literal::Identifier)) { + std::string nameScape; + if (findNamespace(nameScape) && !nameScape.empty() && std::ranges::find(m_UDTs,(nameScape + "::" + name)) != m_UDTs.end()) { + m_scopeChains.insert(tokenIndex); + } + } + m_curr = curr; } - auto id = getTokenId(m_curr->location); + } + auto id = getTokenId(m_curr->location); - if (instances.contains(name)) { - auto &nameInstances = instances[name]; + if (instances.contains(name)) { + auto &nameInstances = instances[name]; - if (std::ranges::find(nameInstances, id) == nameInstances.end()) - nameInstances.push_back(id); - } else - instances[name].push_back(id); - } else if (peek(tkn::Separator::EndOfProgram)) - break; + if (std::ranges::find(nameInstances, id) == nameInstances.end()) + nameInstances.push_back(id); + } else + instances[name].push_back(id); + } else if (peek(tkn::Separator::EndOfProgram)) + break; } m_instances = std::move(instances); } @@ -1363,21 +1629,30 @@ namespace hex::plugin::builtin { // Get the location of a given token index pl::core::Location TextHighlighter::getLocation(i32 tokenId) { - if (tokenId >= (i32) m_tokens.size()) + if (tokenId >= (i32) m_requiredInputs.fullTokens.size()) return Location::Empty(); - return m_tokens[tokenId].location; + return m_requiredInputs.fullTokens.at(tokenId).location; } // Get the token index for a given location. i32 TextHighlighter::getTokenId(pl::core::Location location) { + if (location == m_requiredInputs.fullTokens.at(0).location) + return 0; + if (location == m_requiredInputs.fullTokens.back().location) + return (i32) m_requiredInputs.fullTokens.size() - 1; if (!isLocationValid(location)) return -1; auto line1 = location.line - 1; - auto line2 = nextLine(line1); - auto tokenCount = m_tokens.size(); - i32 tokenStart = m_firstTokenIdOfLine[line1]; - i32 tokenEnd = m_firstTokenIdOfLine[line2] - 1; + auto tokenCount = m_requiredInputs.fullTokens.size(); + i32 tokenStart = m_firstTokenIdOfLine.at(line1); + i32 tokenEnd = -1; + if (line1 == m_firstTokenIdOfLine.size() -1) { + tokenEnd = tokenCount - 1; + } else { + auto line2 = nextLine(line1); + tokenEnd = m_firstTokenIdOfLine.at(line2) - 1; + } if (tokenEnd >= (i32) tokenCount) tokenEnd = tokenCount - 1; @@ -1387,76 +1662,44 @@ namespace hex::plugin::builtin { for (i32 i = tokenStart; i <= tokenEnd; i++) { - if (m_tokens[i].location.column >= location.column) + if (m_requiredInputs.fullTokens.at(i).location.column >= location.column) return i; } return -1; } void TextHighlighter::setIdentifierColor(i32 tokenId, const IdentifierType &type) { - Token *token; + const Token *constToken; - if (tokenId == -1) - token = const_cast(&m_curr[0]); - else - token = const_cast(&m_tokens[tokenId]); + if (tokenId == -1) { + constToken = &m_curr[0]; + tokenId = getTokenId(m_curr->location); + } else + constToken = &m_requiredInputs.fullTokens.at(tokenId); + + auto token = const_cast(constToken); + + if (token->type == Token::Type::Identifier && (!m_tokenColors.contains(tokenId) || m_tokenColors.at(tokenId) == ui::TextEditor::PaletteIndex::Default || m_tokenColors.at(tokenId) == ui::TextEditor::PaletteIndex::UnkIdentifier)) + m_tokenColors[tokenId] = m_identifierTypeColor.at(type); - if (token->type == Token::Type::Identifier && (!m_tokenColors.contains(token) || m_tokenColors.at(token) == ui::TextEditor::PaletteIndex::Default || m_tokenColors.at(token) == ui::TextEditor::PaletteIndex::UnkIdentifier)) - m_tokenColors[token] = m_identifierTypeColor.at(type); - else if (!m_tokenColors.contains(token)) - m_tokenColors[token] = m_tokenTypeColor.at(token->type); } void TextHighlighter::setColor(i32 tokenId, const IdentifierType &type) { - Token *token; if (tokenId == -1) - token = const_cast(&m_curr[0]); - else - token = const_cast(&m_tokens[tokenId]); + tokenId = getTokenId(m_curr->location); - if (token->type == Token::Type::Integer) { - auto literal = getValue(0); - - if (literal != nullptr && !m_tokenColors.contains(token)) - m_tokenColors[token] = getPaletteIndex(literal); - - - } else if (token->type == Token::Type::DocComment) { - auto docComment = getValue(0); - - if (docComment != nullptr && !m_tokenColors.contains(token)) { - - if (docComment->singleLine) - m_tokenColors[token] = ui::TextEditor::PaletteIndex::DocComment; - else if (docComment->global) - m_tokenColors[token] = ui::TextEditor::PaletteIndex::GlobalDocComment; - else - m_tokenColors[token] = ui::TextEditor::PaletteIndex::DocBlockComment; - } - } else if (token->type == Token::Type::Comment) { - auto comment = getValue(0); - - if (comment != nullptr && !m_tokenColors.contains(token)) { - - if (comment->singleLine) - m_tokenColors[token] = ui::TextEditor::PaletteIndex::Comment; - else - m_tokenColors[token] = ui::TextEditor::PaletteIndex::BlockComment; - } - } else - setIdentifierColor(tokenId, type); + setIdentifierColor(tokenId, type); } void TextHighlighter::colorRemainingIdentifierTokens() { std::vector taggedIdentifiers; - taggedIdentifiers.reserve(m_taggedIdentifiers.size()); for (auto index: m_taggedIdentifiers) { taggedIdentifiers.push_back(index); } m_taggedIdentifiers.clear(); - m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); - auto endToken = TokenIter(m_tokens.end(), m_tokens.end()); + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_requiredInputs.fullTokens.begin(), m_requiredInputs.fullTokens.end()); + auto endToken = TokenIter(m_requiredInputs.fullTokens.end(), m_requiredInputs.fullTokens.end()); m_curr = m_startToken; while (endToken > m_curr) { @@ -1469,15 +1712,13 @@ namespace hex::plugin::builtin { taggedIdentifiers.pop_back(); } - auto *token = const_cast(&m_curr[0]); - if (sequence(tkn::Keyword::Import, tkn::Literal::Identifier)) { next(-1); do { if (auto identifier = const_cast(getValue(0)); identifier != nullptr) { setIdentifierColor(-1, IdentifierType::NameSpace); - if (std::ranges::find(m_nameSpaces, identifier->get()) == m_nameSpaces.end()) { - m_nameSpaces.push_back(identifier->get()); + if (std::ranges::find(m_requiredInputs.usedNamespaces, identifier->get()) == m_requiredInputs.usedNamespaces.end()) { + m_requiredInputs.usedNamespaces.push_back(identifier->get()); } } } while (sequence(tkn::Literal::Identifier,tkn::Separator::Dot)); @@ -1486,8 +1727,8 @@ namespace hex::plugin::builtin { next(-1); if (auto identifier = const_cast(getValue(0)); identifier != nullptr) { setIdentifierColor(-1, IdentifierType::NameSpace); - if (std::ranges::find(m_nameSpaces, identifier->get()) == m_nameSpaces.end()) { - m_nameSpaces.push_back(identifier->get()); + if (std::ranges::find(m_requiredInputs.usedNamespaces, identifier->get()) == m_requiredInputs.usedNamespaces.end()) { + m_requiredInputs.usedNamespaces.push_back(identifier->get()); } } } @@ -1502,12 +1743,12 @@ namespace hex::plugin::builtin { identifierType = identifier->getType(); std::string variableName = identifier->get(); - if (m_tokenColors.contains(token) && (m_tokenColors.at(token) != ui::TextEditor::PaletteIndex::Default && identifierType != IdentifierType::Unknown)) { + if (m_tokenColors.contains(tokenId) && (m_tokenColors.at(tokenId) != ui::TextEditor::PaletteIndex::Default && identifierType != IdentifierType::Unknown)) { next(); continue; } Definition definition; - + std::string nameSpace; if (peek(tkn::Keyword::Parent, -2)) { auto save = m_curr; next(-2); @@ -1526,13 +1767,15 @@ namespace hex::plugin::builtin { m_curr = save; next(); continue; - } else if (peek(tkn::Operator::ScopeResolution, 1)) { - if (std::ranges::find(m_nameSpaces, variableName) != m_nameSpaces.end()) { + } + if (peek(tkn::Operator::ScopeResolution, 1)) { + if (std::ranges::find(m_requiredInputs.usedNamespaces, variableName) != m_requiredInputs.usedNamespaces.end()) { setIdentifierColor(-1, IdentifierType::NameSpace); next(); continue; } - } else if (peek(tkn::Operator::ScopeResolution, -1)) { + } + if (peek(tkn::Operator::ScopeResolution, -1)) { auto save = m_curr; next(-2); if (auto parentIdentifier = const_cast(getValue(0)); parentIdentifier != nullptr) { @@ -1546,43 +1789,57 @@ namespace hex::plugin::builtin { m_curr = save; next(); continue; - } else if (findIdentifierDefinition(definition)) { + } + if (findIdentifierDefinition(definition)) { identifierType = definition.idType; setIdentifierColor(-1, identifierType); next(); continue; - } else if (std::ranges::find(m_UDTs, variableName) != m_UDTs.end()) { + } + if (findNamespace(nameSpace,tokenId)) { + auto fullName = nameSpace + "::" + variableName; + auto typeName = findIdentifierType(fullName, ""); + if (typeName != IdentifierType::Unknown) { + setIdentifierColor(-1, typeName); + next(); + continue; + } + } + if (std::ranges::find(m_UDTs, variableName) != m_UDTs.end()) { if (m_typeDefMap.contains(variableName)) setIdentifierColor(-1, IdentifierType::Typedef); else setIdentifierColor(-1, IdentifierType::UDT); next(); continue; - } else if (peek(tkn::Keyword::From, -1)) { + } + if (peek(tkn::Keyword::From, -1)) { setIdentifierColor(-1, IdentifierType::GlobalVariable); next(); continue; - } else { - setIdentifierColor(-1, IdentifierType::Unknown); - next(); - continue; } + setIdentifierColor(-1, IdentifierType::Unknown); + next(); } next(); } } void TextHighlighter::setRequestedIdentifierColors() { + if (m_tokenColors.empty() || m_firstTokenIdOfLine.empty() || m_requiredInputs.fullTokens.size() < 2) + return; auto topLine = 0; + while (m_firstTokenIdOfLine.at(topLine) == -1) + topLine++; auto bottomLine = m_lines.size(); for (u32 line = topLine; line < bottomLine; line = nextLine(line)) { if (m_lines[line].empty()) continue; - std::string lineOfColors = std::string(m_lines[line].size(), 0); - for (auto tokenIndex = m_firstTokenIdOfLine[line]; tokenIndex < m_firstTokenIdOfLine[nextLine(line)]; tokenIndex++) { - auto *token = const_cast(&m_tokens[tokenIndex]); - if (m_tokenColors.contains(token) && token->type == Token::Type::Identifier) { - u8 color = (u8) m_tokenColors.at(token); + std::string &lineOfColors = m_requiredInputs.linesOfColors[line];//std::string(m_lines[line].size(), 0); + for (auto tokenIndex = m_firstTokenIdOfLine.at(line); tokenIndex < m_firstTokenIdOfLine.at(nextLine(line)); tokenIndex++) { + Token *token = const_cast(&m_requiredInputs.fullTokens.at(tokenIndex)); + if (m_tokenColors.contains(tokenIndex) && token->type == Token::Type::Identifier) { + u8 color = (u8) m_tokenColors.at(tokenIndex); u32 tokenLength = token->location.length; u32 tokenOffset = token->location.column - 1; if (token->location.line != line + 1) @@ -1613,7 +1870,7 @@ namespace hex::plugin::builtin { u32 tokenIndex = tokenRange.start; for (auto token = tokenRange.start; token < tokenRange.end; token++) { - if (auto operatorTkn = std::get_if(&m_tokens.at(token).value); + if (auto operatorTkn = std::get_if(&m_requiredInputs.fullTokens.at(token).value); operatorTkn != nullptr && *operatorTkn == Token::Operator::Colon) tokenIndex = token + 1; } @@ -1658,7 +1915,7 @@ namespace hex::plugin::builtin { } bool TextHighlighter::isTokenIdValid(i32 tokenId) { - return tokenId >= 0 && tokenId < (i32) m_tokens.size(); + return tokenId >= 0 && tokenId < (i32) m_requiredInputs.fullTokens.size(); } bool TextHighlighter::isLocationValid(hex::plugin::builtin::TextHighlighter::Location location) { @@ -1723,11 +1980,7 @@ namespace hex::plugin::builtin { next(-1); } typeStr = nameSpace + typeStr; - using Types = std::map>; - auto parser = patternLanguage->get()->getInternals().parser.get(); - Types types = parser->getTypes(); - - if (types.contains(typeStr)) { + if (m_requiredInputs.definedTypes.contains(typeStr)) { m_curr = curr; return typeStr; } @@ -1750,11 +2003,11 @@ namespace hex::plugin::builtin { // Definitions of global variables and placed variables. void TextHighlighter::loadGlobalDefinitions(Scopes tokenRangeSet, std::vector identifierTypes, Variables &variables) { - m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_requiredInputs.fullTokens.begin(), m_requiredInputs.fullTokens.end()); for (auto range: tokenRangeSet) { - auto startToken = TokenIter(m_tokens.begin()+range.start,m_tokens.begin()+range.end); - auto endToken = TokenIter(m_tokens.begin()+range.end,m_tokens.end()); + auto startToken = TokenIter(m_requiredInputs.fullTokens.begin()+range.start,m_requiredInputs.fullTokens.begin()+range.end); + auto endToken = TokenIter(m_requiredInputs.fullTokens.begin()+range.end,m_requiredInputs.fullTokens.end()); for ( m_curr = startToken; endToken > m_curr; next()) { @@ -1814,7 +2067,7 @@ namespace hex::plugin::builtin { endToken = endToken + range.end; Keyword *keyword; - for (keyword = std::get_if(&m_tokens[range.start].value); endToken > m_curr; next()) { + for (keyword = std::get_if(&m_requiredInputs.fullTokens.at(range.start).value); endToken > m_curr; next()) { if (peek(tkn::Literal::Identifier)) { auto identifier = getValue(0); @@ -1887,29 +2140,22 @@ namespace hex::plugin::builtin { using IdentifierType::UDT; using IdentifierType::Function; - if (!m_UDTDefinitions.empty()) - m_UDTDefinitions.clear(); + m_UDTDefinitions.clear(); loadTypeDefinitions(m_UDTTokenRange, {UDT}, m_UDTDefinitions); - if (!m_globalVariables.empty()) - m_globalVariables.clear(); + m_globalVariables.clear(); loadGlobalDefinitions(m_globalTokenRange, {GlobalVariable, PlacedVariable}, m_globalVariables); - if (!m_UDTVariables.empty()) - m_UDTVariables.clear(); - loadVariableDefinitions(m_UDTTokenRange, tkn::Operator::BoolLessThan, tkn::Operator::BoolGreaterThan, - {TemplateArgument}, true, m_UDTVariables); + m_UDTVariables.clear(); + loadVariableDefinitions(m_UDTTokenRange, tkn::Operator::BoolLessThan, tkn::Operator::BoolGreaterThan, {TemplateArgument}, true, m_UDTVariables); - loadVariableDefinitions(m_UDTTokenRange, tkn::Operator::BoolLessThan, tkn::Operator::BoolGreaterThan, - {LocalVariable, PatternVariable, CalculatedPointer}, false, m_UDTVariables); + loadVariableDefinitions(m_UDTTokenRange, tkn::Operator::BoolLessThan, tkn::Operator::BoolGreaterThan, {LocalVariable, PatternVariable, CalculatedPointer}, false, m_UDTVariables); appendInheritances(); - if (!m_functionDefinitions.empty()) - m_functionDefinitions.clear(); + m_functionDefinitions.clear(); loadTypeDefinitions(m_functionTokenRange, {Function}, m_functionDefinitions); - if (!m_functionVariables.empty()) - m_functionVariables.clear(); + m_functionVariables.clear(); loadVariableDefinitions(m_functionTokenRange, tkn::Separator::LeftParenthesis, tkn::Separator::RightParenthesis, {FunctionParameter}, true, m_functionVariables); @@ -1926,58 +2172,92 @@ namespace hex::plugin::builtin { if (!m_lines.empty()) m_lines.clear(); - if (m_text.empty()) { + if (m_requiredInputs.editedText.empty()) { ui::TextEditor *editor = m_viewPatternEditor->getTextEditor(); if (editor != nullptr) - m_text = editor->getText(); + m_requiredInputs.editedText = editor->getText(); else log::warn("Text editor not found, provider is null"); } - m_lines = wolv::util::splitString(m_text, "\n"); - m_lines.emplace_back(""); + m_lines = wolv::util::splitString(m_requiredInputs.editedText, "\n"); + m_lines.push_back(""); m_firstTokenIdOfLine.clear(); - m_firstTokenIdOfLine.resize(m_lines.size(), -1); - i32 tokenId = 0; - i32 tokenCount = m_tokens.size(); - i32 index; - if (tokenCount > 0) { - index = m_tokens[0].location.line - 1; - m_firstTokenIdOfLine[index] = 0; - } - i32 count = m_lines.size(); - for (i32 currentLine = 0; currentLine < count; currentLine++) { - for (index = m_tokens[tokenId].location.line - 1; index <= currentLine && tokenId + 1 < tokenCount; tokenId++) { - index = m_tokens[tokenId + 1].location.line - 1; + u32 tokenId = 0; + u32 tokenCount = m_requiredInputs.fullTokens.size(); + u32 lineIndex; + u32 count; + + if (tokenCount == 0) + return; + lineIndex = m_requiredInputs.fullTokens.at(tokenId).location.line - 1; + count = m_requiredInputs.fullTokens.at(tokenCount - 1).location.line + 1; + m_firstTokenIdOfLine.resize(count, -1); + m_firstTokenIdOfLine.at(lineIndex) = 0; + tokenId++; + u32 currentLine = lineIndex; + while ( currentLine < count) { + while (lineIndex <= currentLine && tokenId <= tokenCount - 1) { + lineIndex = m_requiredInputs.fullTokens.at(tokenId).location.line - 1; + tokenId++; } - - if (index > currentLine) { - m_firstTokenIdOfLine[index] = tokenId; + if (tokenId > tokenCount - 1) + break; + if (tokenId > 0) + tokenId--; + if ((m_requiredInputs.fullTokens.at(tokenId).type == pl::core::Token::Type::DocComment || m_requiredInputs.fullTokens.at(tokenId).type == pl::core::Token::Type::Comment)) { + auto *comment = std::get_if(&(m_requiredInputs.fullTokens.at(tokenId).value)); + auto *docComment = std::get_if(&(m_requiredInputs.fullTokens.at(tokenId).value)); + if ((comment != nullptr && !comment->singleLine) || (docComment != nullptr && !docComment->singleLine)) { + auto commentTokenId = tokenId; + auto commentStartLine = m_requiredInputs.fullTokens.at(tokenId).location.line - 1; + std::string value = m_requiredInputs.fullTokens.at(tokenId).getFormattedValue(); + auto commentEndLine = commentStartLine + std::count(value.begin(), value.end(), '\n'); + m_firstTokenIdOfLine.at(commentStartLine) = commentTokenId; + for (u32 i = commentStartLine + 1; i <= commentEndLine; i++) { + m_firstTokenIdOfLine.at(i) = -1; + } + lineIndex = commentEndLine; + tokenId++; + } else { + m_firstTokenIdOfLine.at(lineIndex) = tokenId; + tokenId++; + } + } else { + m_firstTokenIdOfLine.at(lineIndex) = tokenId; + tokenId++; } + currentLine = lineIndex; } - if (m_firstTokenIdOfLine.back() != tokenCount) - m_firstTokenIdOfLine.push_back(tokenCount); + if (m_firstTokenIdOfLine.back() != (i32) tokenCount - 1) + m_firstTokenIdOfLine.push_back(tokenCount - 1); } // Some tokens span many lines and some lines have no tokens. This -// function helps to find the next line number in the inner loop. +// function helps to find the next line number in loops over tokens. u32 TextHighlighter::nextLine(u32 line) { - auto currentTokenId = m_firstTokenIdOfLine[line]; + if (line >= m_firstTokenIdOfLine.size()) + return line; + auto currentTokenId = m_firstTokenIdOfLine.at(line); u32 i = 1; - while (line + i < m_lines.size() && - (m_firstTokenIdOfLine[line + i] == currentTokenId || m_firstTokenIdOfLine[line + i] == (i32) 0xFFFFFFFF)) + while (line + i < m_firstTokenIdOfLine.size() && + (m_firstTokenIdOfLine.at(line + i) == currentTokenId || m_firstTokenIdOfLine.at(line + i) == (i32) 0xFFFFFFFF)) i++; return i + line; } u32 TextHighlighter::previousLine(u32 line) { - auto currentTokenId = m_firstTokenIdOfLine[line]; + if (line == 0) + return 0; + if (line >= m_firstTokenIdOfLine.size()) + return line - 1; + auto currentTokenId = m_firstTokenIdOfLine.at(line); u32 i = 1; - while (line - i < m_lines.size() && - (m_firstTokenIdOfLine[line - i] == currentTokenId || m_firstTokenIdOfLine[line - i] == (i32) 0xFFFFFFFF)) + while (i <= line && line < m_firstTokenIdOfLine.size() && + (m_firstTokenIdOfLine.at(line - i) == currentTokenId || m_firstTokenIdOfLine.at(line - i) == (i32) 0xFFFFFFFF)) i++; return line - i; } @@ -1987,7 +2267,7 @@ namespace hex::plugin::builtin { void TextHighlighter::invertGlobalTokenRange() { std::set ranges; auto size = m_globalTokenRange.size(); - auto tokenCount = m_tokens.size(); + auto tokenCount = m_requiredInputs.fullTokens.size(); if (size == 0) { ranges.insert(Interval(0, tokenCount)); @@ -2156,7 +2436,7 @@ namespace hex::plugin::builtin { invertGlobalTokenRange(); for (auto tokenRange: m_globalTokenRange) { - if ((u32) tokenRange.end == m_tokens.size()) { + if ((u32) tokenRange.end == m_requiredInputs.fullTokens.size()) { tokenRange.end -= 1; m_globalBlocks.insert(tokenRange); } @@ -2166,10 +2446,10 @@ namespace hex::plugin::builtin { // Parser labels global variables that are not placed as // function variables. void TextHighlighter::fixGlobalVariables() { - m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_requiredInputs.fullTokens.begin(), m_requiredInputs.fullTokens.end()); for (auto range: m_globalTokenRange) { - auto startToken = TokenIter(m_tokens.begin() + range.start, m_tokens.begin() + range.end); - auto endToken = TokenIter(m_tokens.begin() + range.end, m_tokens.end()); + auto startToken = TokenIter(m_requiredInputs.fullTokens.begin() + range.start, m_requiredInputs.fullTokens.begin() + range.end); + auto endToken = TokenIter(m_requiredInputs.fullTokens.begin() + range.end, m_requiredInputs.fullTokens.end()); for (m_curr = startToken; endToken > m_curr; next()) { @@ -2195,143 +2475,75 @@ namespace hex::plugin::builtin { } void TextHighlighter::clearVariables() { - if (!m_inheritances.empty()) - m_inheritances.clear(); - - if (!m_globalTokenRange.empty()) - m_globalTokenRange.clear(); - - if (!m_namespaceTokenRange.empty()) - m_namespaceTokenRange.clear(); - - if (!m_UDTDefinitions.empty()) - m_UDTDefinitions.clear(); - - if (!m_UDTBlocks.empty()) - m_UDTBlocks.clear(); - - if (!m_UDTTokenRange.empty()) - m_UDTTokenRange.clear(); - - if (!m_functionDefinitions.empty()) - m_functionDefinitions.clear(); - - if (!m_functionBlocks.empty()) - m_functionBlocks.clear(); - - if (!m_functionTokenRange.empty()) - m_functionTokenRange.clear(); - - if (!m_functionVariables.empty()) - m_functionVariables.clear(); - - if (!m_attributeFunctionArgumentType.empty()) - m_attributeFunctionArgumentType.clear(); - - if (!m_memberChains.empty()) - m_memberChains.clear(); - - if (!m_scopeChains.empty()) - m_scopeChains.clear(); + m_inheritances.clear(); + m_globalTokenRange.clear(); + m_namespaceTokenRange.clear(); + m_UDTDefinitions.clear(); + m_UDTBlocks.clear(); + m_UDTTokenRange.clear(); + m_functionDefinitions.clear(); + m_functionBlocks.clear(); + m_functionTokenRange.clear(); + m_functionVariables.clear(); + m_attributeFunctionArgumentType.clear(); + m_memberChains.clear(); + m_scopeChains.clear(); + m_tokenColors.clear(); } void TextHighlighter::processSource() { + m_UDTVariables.clear(); + m_UDTTokenRange.clear(); + getTokenRanges(IdentifierType::UDT); + loadVariableDefinitions(m_UDTTokenRange, tkn::Operator::BoolLessThan, tkn::Operator::BoolGreaterThan, + {IdentifierType::TemplateArgument}, true, m_UDTVariables); - getAllTokenRanges(IdentifierType::NameSpace); - getAllTokenRanges(IdentifierType::UDT); - getDefinitions(); + loadVariableDefinitions(m_UDTTokenRange, tkn::Operator::BoolLessThan, tkn::Operator::BoolGreaterThan, + {IdentifierType::LocalVariable, IdentifierType::PatternVariable, IdentifierType::CalculatedPointer}, false, m_UDTVariables); m_ImportedUDTVariables.insert(m_UDTVariables.begin(), m_UDTVariables.end()); - - clearVariables(); } // Only update if needed. Must wait for the parser to finish first. void TextHighlighter::highlightSourceCode() { - m_wasInterrupted = false; + m_viewPatternEditor->resetInterrupt(); ON_SCOPE_EXIT { - if (!m_tokenColors.empty()) - m_tokenColors.clear(); - m_runningColorizers--; - if (m_wasInterrupted) { - m_needsToUpdateColors = true; - m_viewPatternEditor->setChangesWereParsed(true); - } else { - m_needsToUpdateColors = false; - m_viewPatternEditor->setChangesWereParsed(false); - } + m_viewPatternEditor->incrementRunningHighlighters(-1); + m_viewPatternEditor->setChangesWereColored(!m_viewPatternEditor->wasInterrupted()); }; try { - m_runningColorizers++; - auto preprocessor = patternLanguage->get()->getInternals().preprocessor.get(); - auto parser = patternLanguage->get()->getInternals().parser.get(); - using Types = std::map>; - Types types = parser->getTypes(); - + m_viewPatternEditor->incrementRunningHighlighters(1); + clearVariables(); if (!m_UDTs.empty()) m_UDTs.clear(); - for (auto &[name, type]: types) + for (auto &[name, type]: m_requiredInputs.definedTypes) m_UDTs.push_back(name); - // Namespaces from included files. - m_nameSpaces.clear(); - m_nameSpaces = preprocessor->getNamespaces(); - clearVariables(); - - m_parsedImports = preprocessor->getParsedImports(); - for (auto &[name, tokens]: m_parsedImports) { - m_tokens = tokens; - m_text = tokens[0].location.source->content; - if (m_text.empty() || m_text == "\n") - return; - loadText(); - processSource(); - if (!m_tokenColors.empty()) - m_tokenColors.clear(); - } - - m_tokens = preprocessor->getResult(); - if (m_tokens.empty()) - return; - if (!m_globalTokenRange.empty()) m_globalTokenRange.clear(); - m_globalTokenRange.insert(Interval(0, m_tokens.size()-1)); + if (m_requiredInputs.fullTokens.size() > 1) { + m_globalTokenRange.insert(Interval(0, m_requiredInputs.fullTokens.size() - 1)); + getTokenRanges(IdentifierType::NameSpace); + getTokenRanges(IdentifierType::UDT); + getTokenRanges(IdentifierType::Function); + getGlobalTokenRanges(); + fixGlobalVariables(); + setInitialColors(); + loadInstances(); + getTokenRanges(IdentifierType::Attribute); + getDefinitions(); + fixAutos(); + fixChains(); + colorRemainingIdentifierTokens(); + } ui::TextEditor *editor = m_viewPatternEditor->getTextEditor(); - if (editor != nullptr) - m_text = editor->getText(); - else - log::warn("Text editor not found, provider is null"); - - if (m_text.empty() || m_text == "\n") - return; - loadText(); - - getAllTokenRanges(IdentifierType::NameSpace); - getAllTokenRanges(IdentifierType::UDT); - getAllTokenRanges(IdentifierType::Function); - getGlobalTokenRanges(); - fixGlobalVariables(); - setInitialColors(); - loadInstances(); - getAllTokenRanges(IdentifierType::Attribute); - getDefinitions(); - fixAutos(); - fixChains(); - - m_excludedLocations = preprocessor->getExcludedLocations(); - - colorRemainingIdentifierTokens(); - setRequestedIdentifierColors(); - - editor = m_viewPatternEditor->getTextEditor(); if (editor != nullptr) editor->clearErrorMarkers(); else log::warn("Text editor not found, provider is null"); - m_compileErrors = patternLanguage->get()->getCompileErrors(); + m_requiredInputs.compileErrors = getPatternLanguage()->getCompileErrors(); - if (!m_compileErrors.empty()) + if (!m_requiredInputs.compileErrors.empty()) renderErrors(); else { editor = m_viewPatternEditor->getTextEditor(); @@ -2342,8 +2554,10 @@ namespace hex::plugin::builtin { } } catch (const std::out_of_range &e) { log::debug("TextHighlighter::highlightSourceCode: Out of range error: {}", e.what()); - m_wasInterrupted = true; + m_viewPatternEditor->setWasInterrupted(true); return; } + + return; } } diff --git a/plugins/builtin/source/content/views/view_pattern_editor.cpp b/plugins/builtin/source/content/views/view_pattern_editor.cpp index 6d672d986..d721df01f 100644 --- a/plugins/builtin/source/content/views/view_pattern_editor.cpp +++ b/plugins/builtin/source/content/views/view_pattern_editor.cpp @@ -22,18 +22,13 @@ #include #include #include -#include #include #include #include -#include -#include - #include #include -#include #include @@ -43,11 +38,25 @@ #include #include -#include #include #include #include -#include +#include + +#include + +// Specialization for std::chrono::duration +template <> +struct fmt::formatter> { + constexpr auto parse(fmt::format_parse_context& ctx) -> decltype(ctx.begin()) { + return ctx.end(); + } + + template + auto format(const std::chrono::duration& duration, FormatContext& ctx) const { + return fmt::format_to(ctx.out(), "{} seconds", duration.count()); + } +}; namespace hex::plugin::builtin { @@ -73,7 +82,7 @@ namespace hex::plugin::builtin { return; } - ui::TextEditor *editor = m_view->getTextEditor(); + ui::TextEditor const *editor = m_view->getTextEditor(); if (editor != nullptr) { if (m_view->m_sourceCode.hasProviderSpecificSource(provider)) { this->close(); @@ -306,16 +315,12 @@ namespace hex::plugin::builtin { m_editorRuntime = std::make_unique(); ContentRegistry::PatternLanguage::configureRuntime(*m_editorRuntime, nullptr); - - this->registerEvents(); - this->registerMenuItems(); - this->registerHandlers(); + registerEvents(); + registerMenuItems(); + registerHandlers(); // Initialize the text editor with some basic help text - m_textEditor.setOnCreateCallback([this](prv::Provider *provider, ui::TextEditor &editor) { - if (m_sourceCode.isSynced() && !m_sourceCode.get(provider).empty()) - return; - + m_textEditor.setOnCreateCallback([](auto, ui::TextEditor &editor) { std::string text = "hex.builtin.view.pattern_editor.default_help_text"_lang; text = "// " + wolv::util::replaceStrings(text, "\n", "\n// "); @@ -999,9 +1004,9 @@ namespace hex::plugin::builtin { } } ImGui::EndPopup(); - m_frPopupIsClosed = false; - } else if (!m_frPopupIsClosed) { - m_frPopupIsClosed = true; + m_findReplacePopupIsClosed = false; + } else if (!m_findReplacePopupIsClosed) { + m_findReplacePopupIsClosed = true; m_popupWindowHeight = 0; m_textEditor.get(provider).setTopMarginChanged(0); } @@ -1436,10 +1441,7 @@ namespace hex::plugin::builtin { if (m_textEditor.get(provider).isTextChanged()) { m_textEditor.get(provider).setTextChanged(false); - if (!m_hasUnevaluatedChanges.get(provider) ) { - m_hasUnevaluatedChanges.get(provider) = true; - m_changesWereParsed = false; - } + m_hasUnevaluatedChanges.get(provider) = true; m_lastEditorChangeTime = std::chrono::steady_clock::now(); ImHexApi::Provider::markDirty(); markPatternFileDirty(provider); @@ -1450,7 +1452,6 @@ namespace hex::plugin::builtin { auto code = m_textEditor.get(provider).getText(); EventPatternEditorChanged::post(code); - TaskManager::createBackgroundTask("hex.builtin.task.parsing_pattern", [this, code = std::move(code), provider](auto &){ this->parsePattern(code, provider); @@ -1464,14 +1465,35 @@ namespace hex::plugin::builtin { this->evaluatePattern(m_textEditor.get(provider).getText(), provider); } - if (m_textHighlighter.m_needsToUpdateColors && m_changesWereParsed && (m_runningParsers + m_runningEvaluators == 0)) { - if (m_textHighlighter.getRunningColorizers() == 0) { - m_textHighlighter.m_needsToUpdateColors = false; - m_changesWereParsed = false; - TaskManager::createBackgroundTask("HighlightSourceCode", [this](auto &) { m_textHighlighter.highlightSourceCode(); }); - } else { - m_textHighlighter.interrupt(); + TaskHolder coloringTaskHolder; + if (m_changesWereParsed || m_wasInterrupted || m_hasUnevaluatedChanges.get(provider)) { + if (coloringTaskHolder.isRunning()) + interrupt(); + if (m_wasInterrupted) + resetInterrupt(); + + m_changesWereParsed = false; + if(!m_hasUnevaluatedChanges.get(provider)) { + m_hasUncoloredChanges.get(provider) = true; + m_changesWereColored = false; + } else + m_hasUncoloredChanges.get(provider) = false; + } + + + if (m_hasUncoloredChanges.get(provider) && (m_runningHighlighters + m_runningEvaluators == 0)) { + + try { + m_textHighlighter.get(provider).setViewPatternEditor(this); + m_textHighlighter.get(provider).updateRequiredInputs(); + coloringTaskHolder = TaskManager::createBackgroundTask("HighlightSourceCode", [this,provider](auto &) { m_textHighlighter.get(provider).highlightSourceCode(); }); + m_hasUncoloredChanges.get(provider) = false; + } catch (const std::out_of_range&) { + interrupt(); } + } else if (m_changesWereColored) { + m_textHighlighter.get(provider).setRequestedIdentifierColors(); + m_changesWereColored = false; } if (m_dangerousFunctionCalled && !ImGui::IsPopupOpen(ImGuiID(0), ImGuiPopupFlags_AnyPopup)) { @@ -1641,7 +1663,6 @@ namespace hex::plugin::builtin { m_changeTracker.get(provider) = wolv::io::ChangeTracker(file); m_changeTracker.get(provider).startTracking([this, provider]{ this->handleFileChange(provider); }); } - m_textHighlighter.m_needsToUpdateColors = false; TaskManager::createBackgroundTask("hex.builtin.task.parsing_pattern", [this, code, provider](auto&) { this->parsePattern(code, provider); }); } } @@ -1651,7 +1672,7 @@ namespace hex::plugin::builtin { ContentRegistry::PatternLanguage::configureRuntime(*m_editorRuntime, nullptr); const auto &ast = m_editorRuntime->parseString(code, pl::api::Source::DefaultSource); - m_textEditor.get(provider).setLongestLineLength(m_editorRuntime->getInternals().preprocessor->getLongestLineLength()); + m_textEditor.get(provider).setLongestLineLength(m_editorRuntime->getInternals().preprocessor.get()->getLongestLineLength()); auto &patternVariables = m_patternVariables.get(provider); auto oldPatternVariables = std::move(patternVariables); @@ -1683,7 +1704,6 @@ namespace hex::plugin::builtin { patternVariables = std::move(oldPatternVariables); } - m_textHighlighter.m_needsToUpdateColors = true; m_changesWereParsed = true; m_runningParsers -= 1; } @@ -1707,7 +1727,6 @@ namespace hex::plugin::builtin { m_accessHistory = {}; m_accessHistoryIndex = 0; - m_patternEvaluating = true; EventHighlightingChanged::post(); @@ -1835,13 +1854,20 @@ namespace hex::plugin::builtin { m_textEditor.get(provider).setText(wolv::util::preprocessText(code)); m_sourceCode.get(provider) = code; m_hasUnevaluatedChanges.get(provider) = true; - m_textHighlighter.m_needsToUpdateColors = false; }); ContentRegistry::Settings::onChange("hex.builtin.setting.general", "hex.builtin.setting.general.sync_pattern_source", [this](const ContentRegistry::Settings::SettingsValue &value) { m_sourceCode.enableSync(value.get(false)); }); + ContentRegistry::Settings::onChange("hex.builtin.setting.general", "hex.builtin.setting.general.suggest_patterns", [this](const ContentRegistry::Settings::SettingsValue &value) { + m_suggestSupportedPatterns = value.get(true); + }); + + ContentRegistry::Settings::onChange("hex.builtin.setting.general", "hex.builtin.setting.general.auto_apply_patterns", [this](const ContentRegistry::Settings::SettingsValue &value) { + m_autoApplyPatterns = value.get(true); + }); + EventProviderOpened::subscribe(this, [this](prv::Provider *provider) { m_textEditor.get(provider).setLanguageDefinition(PatternLanguage()); m_textEditor.get(provider).setShowWhitespaces(false); @@ -1893,7 +1919,6 @@ namespace hex::plugin::builtin { m_consoleEditor.get(newProvider).setScroll(m_consoleScroll.get(newProvider)); } - m_textHighlighter.m_needsToUpdateColors = false; }); @@ -2238,6 +2263,10 @@ namespace hex::plugin::builtin { } }); + ContentRegistry::Settings::onChange("hex.builtin.setting.hex_editor", "hex.builtin.setting.hex_editor.pattern_parent_highlighting", [this](const ContentRegistry::Settings::SettingsValue &value) { + m_parentHighlightingEnabled = bool(value.get(false)); + }); + ImHexApi::HexEditor::addBackgroundHighlightingProvider([this](u64 address, const u8 *data, size_t size, bool) -> std::optional { std::ignore = data; std::ignore = size; @@ -2337,7 +2366,6 @@ namespace hex::plugin::builtin { m_textEditor.get(provider).setText(sourceCode); m_hasUnevaluatedChanges.get(provider) = true; - m_textHighlighter.m_needsToUpdateColors = false; return true; }, .store = [this](prv::Provider *provider, const std::fs::path &basePath, const Tar &tar) { @@ -2553,81 +2581,6 @@ namespace hex::plugin::builtin { return result; }); - - ContentRegistry::MCP::registerTool(romfs::get("mcp/tools/execute_pattern_code.json").string(), [this](const nlohmann::json &data) -> nlohmann::json { - auto provider = ImHexApi::Provider::get(); - - auto sourceCode = data.at("source_code").get(); - - this->evaluatePattern(sourceCode, provider); - - // Wait until evaluation has finished - while (m_runningEvaluators > 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - auto lock = std::scoped_lock(ContentRegistry::PatternLanguage::getRuntimeLock()); - - auto evaluationResult = m_lastEvaluationResult.load(); - - nlohmann::json result = { - { "handle", provider->getID() }, - { "result_code", evaluationResult } - }; - return mcp::StructuredContent { - .text = result.dump(), - .data = result - }; - }); - - ContentRegistry::MCP::registerTool(romfs::get("mcp/tools/get_pattern_console_content.json").string(), [this](const nlohmann::json &data) -> nlohmann::json { - std::ignore = data; - - auto provider = ImHexApi::Provider::get(); - - // Wait until evaluation has finished - while (m_runningEvaluators > 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - auto lock = std::scoped_lock(ContentRegistry::PatternLanguage::getRuntimeLock()); - - auto consoleOutput = m_console.get(provider); - - nlohmann::json result = { - { "handle", provider->getID() }, - { "content", wolv::util::combineStrings(consoleOutput, "\n") } - }; - return mcp::StructuredContent { - .text = result.dump(), - .data = result - }; - }); - - ContentRegistry::MCP::registerTool(romfs::get("mcp/tools/get_patterns.json").string(), [this](const nlohmann::json &data) -> nlohmann::json { - std::ignore = data; - - auto provider = ImHexApi::Provider::get(); - - // Wait until evaluation has finished - while (m_runningEvaluators > 0) { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - auto lock = std::scoped_lock(ContentRegistry::PatternLanguage::getRuntimeLock()); - - pl::gen::fmt::FormatterJson formatter; - auto formattedPatterns = formatter.format(ContentRegistry::PatternLanguage::getRuntime()); - - nlohmann::json result = { - { "handle", provider->getID() }, - { "patterns", nlohmann::json::parse(std::string(formattedPatterns.begin(), formattedPatterns.end())) } - }; - return mcp::StructuredContent { - .text = result.dump(), - .data = result - }; - }); } void ViewPatternEditor::handleFileChange(prv::Provider *provider) { @@ -2745,4 +2698,12 @@ namespace hex::plugin::builtin { ImGuiExt::TextFormattedWrapped("This will execute your code, output any log messages to the console window below and create a pattern tree that gets displayed in the Pattern Data view and highlights matching regions in the Hex Editor view."); } + ui::TextEditor *ViewPatternEditor::getTextEditor() { + auto provider = ImHexApi::Provider::get(); + if (provider == nullptr) + return nullptr; + + return &m_textEditor.get(provider); + } + } diff --git a/plugins/ui/include/ui/text_editor.hpp b/plugins/ui/include/ui/text_editor.hpp index e4a4dab98..a13d82223 100644 --- a/plugins/ui/include/ui/text_editor.hpp +++ b/plugins/ui/include/ui/text_editor.hpp @@ -1,14 +1,15 @@ #pragma once - +#include "imgui_internal.h" #include #include #include +#include +#include #include #include #include #include -#include "imgui.h" -#include "imgui_internal.h" +#include #include #include @@ -103,7 +104,7 @@ namespace hex::ui { public: friend class TextEditor; bool operator==(const EditorState &o) const; - EditorState() = default; + EditorState() : m_selection(), m_cursorPosition() {} EditorState(const Range &selection, const Coordinates &cursorPosition) : m_selection(selection), m_cursorPosition(cursorPosition) {} private: Range m_selection; @@ -151,11 +152,11 @@ namespace hex::ui { bool m_matchCase; bool m_wholeWord; bool m_findRegEx; - bool m_optionsChanged = false; + bool m_optionsChanged; Matches m_matches; }; - enum class PaletteIndex: u8 { + enum class PaletteIndex { Default, Identifier, Directive, Operator, Separator, BuiltInType, Keyword, NumericLiteral, StringLiteral, CharLiteral, Cursor, Background, LineNumber, Selection, Breakpoint, ErrorMarker, PreprocessorDeactivated, CurrentLineFill, CurrentLineFillInactive, CurrentLineEdge, ErrorText, WarningText, DebugText, DefaultText, Attribute, PatternVariable, LocalVariable, CalculatedPointer, TemplateArgument, Function, View, FunctionVariable, FunctionParameter, UserDefinedType, PlacedVariable, GlobalVariable, NameSpace, TypeDef, UnkIdentifier, DocComment, DocBlockComment, BlockComment, GlobalDocComment, Comment, PreprocIdentifier, Max @@ -228,12 +229,12 @@ namespace hex::ui { class LineIterator { public: friend class hex::ui::TextEditor; - LineIterator(const LineIterator &other) = default; + LineIterator(const LineIterator &other) : m_charsIter(other.m_charsIter), m_colorsIter(other.m_colorsIter), m_flagsIter(other.m_flagsIter) {} LineIterator() = default; char operator*(); LineIterator operator++(); - LineIterator& operator=(const LineIterator &other); + LineIterator operator=(const LineIterator &other); bool operator!=(const LineIterator &other) const; bool operator==(const LineIterator &other) const; LineIterator operator+(i32 n); @@ -275,10 +276,10 @@ namespace hex::ui { Line() : m_lineMaxColumn(-1) {} explicit Line(const char *line) : Line(std::string(line)) {} - explicit Line(const std::string &line) : m_chars(line), m_colors(std::string(line.size(), 0x00)), m_flags(std::string(line.size(), 0x00)), m_lineMaxColumn(maxColumn()) {} + explicit Line(const std::string &line) : m_chars(line), m_colors(std::string(line.size(), 0x00)), m_flags(std::string(line.size(), 0x00)), m_colorized(false), m_lineMaxColumn(maxColumn()) {} Line(const Line &line) : m_chars(std::string(line.m_chars)), m_colors(std::string(line.m_colors)), m_flags(std::string(line.m_flags)), m_colorized(line.m_colorized), m_lineMaxColumn(line.m_lineMaxColumn) {} Line(Line &&line) noexcept : m_chars(std::move(line.m_chars)), m_colors(std::move(line.m_colors)), m_flags(std::move(line.m_flags)), m_colorized(line.m_colorized), m_lineMaxColumn(line.m_lineMaxColumn) {} - Line(std::string chars, std::string colors, std::string flags) : m_chars(std::move(chars)), m_colors(std::move(colors)), m_flags(std::move(flags)), m_lineMaxColumn(maxColumn()) {} + Line(std::string chars, std::string colors, std::string flags) : m_chars(std::move(chars)), m_colors(std::move(colors)), m_flags(std::move(flags)), m_colorized(false), m_lineMaxColumn(maxColumn()) {} bool operator==(const Line &line) const; bool operator!=(const Line &line) const; @@ -311,12 +312,12 @@ namespace hex::ui { std::string operator[](i64 column) const; void setNeedsUpdate(bool needsUpdate); void append(const char *text); - void append(char text); + void append(const char text); void append(const std::string &text); void append(const Line &line); void append(LineIterator begin, LineIterator end); void insert(LineIterator iter, const std::string &text); - void insert(LineIterator iter, char text); + void insert(LineIterator iter, const char text); void insert(LineIterator iter, strConstIter beginString, strConstIter endString); void insert(LineIterator iter, const Line &line); void insert(LineIterator iter, LineIterator beginLine, LineIterator endLine); @@ -372,7 +373,7 @@ namespace hex::ui { class UndoRecord { public: friend class TextEditor; - UndoRecord() = default; + UndoRecord() {} ~UndoRecord() {} UndoRecord( std::string added, Range addedRange, @@ -457,7 +458,7 @@ namespace hex::ui { void setLongestLineLength(u64 line) { m_longestLineLength = line; } u64 getLongestLineLength() const { return m_longestLineLength; } void setTopMarginChanged(i32 newMargin); - void setFocusAtCoords(const Coordinates &coords, bool scrollToCursor = false); + void setFocusAtCoords(const Coordinates &coords, bool ensureVisible = false); void clearErrorMarkers(); void clearActionables(); private: diff --git a/plugins/ui/source/ui/text_editor/highlighter.cpp b/plugins/ui/source/ui/text_editor/highlighter.cpp index 1118ec048..a2ead5aa2 100644 --- a/plugins/ui/source/ui/text_editor/highlighter.cpp +++ b/plugins/ui/source/ui/text_editor/highlighter.cpp @@ -346,7 +346,12 @@ namespace hex::ui { if (isGlobalDocComment || isBlockDocComment || isBlockComment) { commentStartLine = currentLine; commentStartIndex = currentIndex; - if (isGlobalDocComment) { + if (currentIndex < line.size() - 4 && isBlockComment && + line.m_chars[currentIndex + 2] == '*' && + line.m_chars[currentIndex + 3] == '/') { + withinBlockComment = true; + commentLength = 2; + } else if (isGlobalDocComment) { withinGlobalDocComment = true; commentLength = 3; } else if (isBlockDocComment) { diff --git a/plugins/ui/source/ui/text_editor/support.cpp b/plugins/ui/source/ui/text_editor/support.cpp index 928f3ee3a..e3c28ef52 100644 --- a/plugins/ui/source/ui/text_editor/support.cpp +++ b/plugins/ui/source/ui/text_editor/support.cpp @@ -37,11 +37,11 @@ namespace hex::ui { } Coordinates Coordinates::operator+(const Coordinates &o) const { - return {m_line + o.m_line, m_column + o.m_column}; + return Coordinates(m_line + o.m_line, m_column + o.m_column); } Coordinates Coordinates::operator-(const Coordinates &o) const { - return {m_line - o.m_line, m_column - o.m_column}; + return Coordinates(m_line - o.m_line, m_column - o.m_column); } bool Range::operator==(const Range &o) const { @@ -52,13 +52,13 @@ namespace hex::ui { } Coordinates Range::getSelectedLines() { - return {m_start.m_line, m_end.m_line}; + return Coordinates(m_start.m_line, m_end.m_line); } Coordinates Range::getSelectedColumns() { if (isSingleLine()) - return {m_start.m_column, m_end.m_column - m_start.m_column}; - return {m_start.m_column, m_end.m_column}; + return Coordinates(m_start.m_column, m_end.m_column - m_start.m_column); + return Coordinates(m_start.m_column, m_end.m_column); } bool Range::isSingleLine() { @@ -148,7 +148,12 @@ namespace hex::ui { return iter; } - LineIterator& LineIterator::operator=(const LineIterator &other) = default; + LineIterator LineIterator::operator=(const LineIterator &other) { + m_charsIter = other.m_charsIter; + m_colorsIter = other.m_colorsIter; + m_flagsIter = other.m_flagsIter; + return *this; + } bool LineIterator::operator!=(const LineIterator &other) const { return m_charsIter != other.m_charsIter || m_colorsIter != other.m_colorsIter || @@ -204,7 +209,14 @@ namespace hex::ui { return iter; } - Line &Line::operator=(const Line &line) = default; + Line &Line::operator=(const Line &line) { + m_chars = line.m_chars; + m_colors = line.m_colors; + m_flags = line.m_flags; + m_colorized = line.m_colorized; + m_lineMaxColumn = line.m_lineMaxColumn; + return *this; + } Line &Line::operator=(Line &&line) noexcept { m_chars = std::move(line.m_chars); @@ -310,21 +322,26 @@ namespace hex::ui { } char Line::operator[](u64 index) const { - i64 signedIndex = std::clamp((i64) index,0 - (i64) m_chars.size(), (i64) (m_chars.size() - 1)); + auto charsSize = (i64) m_chars.size(); + if (charsSize == 0) + return '\0'; + i64 signedIndex = std::clamp((i64) index,0 - charsSize,charsSize - 1); if (signedIndex < 0) - return m_chars[m_chars.size() + signedIndex]; + return m_chars[charsSize + signedIndex]; return m_chars[signedIndex]; } // C++ can't overload functions based on return type, so use any type other // than u64 to avoid ambiguity. - std::string Line::operator[](i64 column) const { + std::string Line::operator[](i64 index) const { i64 utf8Length = TextEditor::stringCharacterCount(m_chars); - column = std::clamp(column, (-utf8Length), utf8Length - 1); - if (column < 0) - column = utf8Length + column; + if (utf8Length == 0) + return ""; + index = std::clamp(index, -utf8Length, utf8Length - 1); + if (index < 0) + index = utf8Length + index; i64 utf8Start = 0; - for (i64 utf8Index = 0; utf8Index < column; ++utf8Index) { + for (i64 utf8Index = 0; utf8Index < index; ++utf8Index) { utf8Start += TextEditor::utf8CharLength(m_chars[utf8Start]); } i64 utf8CharLen = TextEditor::utf8CharLength(m_chars[utf8Start]); @@ -462,8 +479,10 @@ namespace hex::ui { bool TextEditor::ActionableBox::trigger() { auto mousePos = ImGui::GetMousePos(); - return mousePos.x > m_box.Min.x && mousePos.x < m_box.Max.x && - mousePos.y >= m_box.Min.y && mousePos.y <= m_box.Max.y; + if (mousePos.x <= m_box.Min.x || mousePos.x >= m_box.Max.x || + mousePos.y < m_box.Min.y || mousePos.y > m_box.Max.y) + return false; + return true; } void TextEditor::ActionableBox::shiftBoxVertically(float lineCount, float lineHeight) { @@ -527,7 +546,7 @@ namespace hex::ui { if (m_readOnly) return; - m_undoBuffer.resize(m_undoIndex + 1); + m_undoBuffer.resize((u64) (m_undoIndex + 1)); m_undoBuffer.back() = UndoAction(value); m_undoIndex++; } @@ -885,11 +904,11 @@ namespace hex::ui { std::string wordLower = m_findWord; if (!getMatchCase()) - std::ranges::transform(wordLower, wordLower.begin(), ::tolower); + std::transform(wordLower.begin(), wordLower.end(), wordLower.begin(), ::tolower); std::string textSrc = editor->getText(); if (!getMatchCase()) - std::ranges::transform(textSrc, textSrc.begin(), ::tolower); + std::transform(textSrc.begin(), textSrc.end(), textSrc.begin(), ::tolower); u64 textLoc; // TODO: use regexp find iterator in all cases @@ -928,8 +947,7 @@ namespace hex::ui { while (iter != end) { iter++; - pos = iter->position(); - if (pos > byteIndex) + if (((pos = iter->position()) > byteIndex)) break; } } @@ -997,6 +1015,7 @@ namespace hex::ui { editor->m_state = saveState; editor->ensureCursorVisible(); + return; }