From e7a2729d36f2d7319980f5801f904e83be263205 Mon Sep 17 00:00:00 2001 From: paxcut <53811119+paxcut@users.noreply.github.com> Date: Fri, 11 Jul 2025 22:47:56 -0700 Subject: [PATCH] feat: semantic syntax highlights for pattern editor. (#2214) allows the ability to assign colors to global placed and non-placed variables, pattern, local and calculated pointer variables, template arguments, function variables and arguments, etc etc etc. It accomplishes this using the parser and the token sequence generated by the lexer. It still uses the original colorizing code but the underlying data holding the pattern has been updated to be easier to use and to debug. The changes are too numerous to cite here.It is a big but necessary step to bring the pattern editor to a somewhat useful state. There may be one commit in the pattern language repo needed to be able to run this code --- .../ColorTextEditor/include/TextEditor.h | 598 ++++- .../ColorTextEditor/source/TextEditor.cpp | 1776 ++++++------- main/gui/source/window/window.cpp | 2 - plugins/builtin/CMakeLists.txt | 2 + .../text_highlighting/pattern_language.hpp | 419 +++ .../content/views/view_pattern_editor.hpp | 27 + plugins/builtin/romfs/themes/classic.json | 36 +- plugins/builtin/romfs/themes/dark.json | 36 +- plugins/builtin/romfs/themes/light.json | 37 +- .../text_highlighting/pattern_language.cpp | 2332 +++++++++++++++++ plugins/builtin/source/content/themes.cpp | 70 +- .../content/views/view_pattern_editor.cpp | 87 +- plugins/ui/source/ui/pattern_drawer.cpp | 20 +- 13 files changed, 4384 insertions(+), 1058 deletions(-) create mode 100644 plugins/builtin/include/content/text_highlighting/pattern_language.hpp create mode 100644 plugins/builtin/source/content/text_highlighting/pattern_language.cpp diff --git a/lib/third_party/imgui/ColorTextEditor/include/TextEditor.h b/lib/third_party/imgui/ColorTextEditor/include/TextEditor.h index faf810498..d0c16c544 100644 --- a/lib/third_party/imgui/ColorTextEditor/include/TextEditor.h +++ b/lib/third_party/imgui/ColorTextEditor/include/TextEditor.h @@ -9,71 +9,89 @@ #include #include #include +#include +#include #include "imgui.h" #include "imgui_internal.h" - +using strConstIter = std::string::const_iterator; +// https://en.wikipedia.org/wiki/UTF-8 +// We assume that the char is a standalone character (<128) or a leading byte of an UTF-8 code sequence (non-10xxxxxx code) +static int UTF8CharLength(uint8_t c) { + if ((c & 0xFE) == 0xFC) + return 6; + if ((c & 0xFC) == 0xF8) + return 5; + if ((c & 0xF8) == 0xF0) + return 4; + if ((c & 0xF0) == 0xE0) + return 3; + if ((c & 0xE0) == 0xC0) + return 2; + return 1; +} class TextEditor { public: enum class PaletteIndex { Default, - Keyword, - Number, - String, - CharLiteral, - Punctuation, - Preprocessor, Identifier, - KnownIdentifier, - PreprocIdentifier, - GlobalDocComment, - DocComment, - Comment, - MultiLineComment, - PreprocessorDeactivated, - Background, + Directive, + Operator, + Separator, + BuiltInType, + Keyword, + NumericLiteral, + StringLiteral, + CharLiteral, Cursor, - Selection, - ErrorMarker, - Breakpoint, + Background, LineNumber, + Selection, + Breakpoint, + ErrorMarker, + PreprocessorDeactivated, CurrentLineFill, CurrentLineFillInactive, CurrentLineEdge, + ErrorText, + WarningText, + DebugText, + DefaultText, + Attribute, + PatternVariable, + LocalVariable, + CalculatedPointer, + TemplateArgument, + Function, + View, + FunctionVariable, + FunctionParameter, + UserDefinedType, + PlacedVariable, + GlobalVariable, + NameSpace, + TypeDef, + UnkIdentifier, + DocComment, + DocBlockComment, + BlockComment, + GlobalDocComment, + Comment, + PreprocIdentifier, Max - }; + }; - - struct Breakpoint - { - int mLine; - bool mEnabled; - std::string mCondition; - - Breakpoint() - : mLine(-1) - , mEnabled(false) - {} - }; - - // Represents a character coordinate from the user's point of view, - // i. e. consider an uniform grid (assuming fixed-width font) on the - // screen as it is rendered, and each cell has its own coordinate, starting from 0. - // Tabs are counted as [1..mTabSize] count empty spaces, depending on - // how many space is necessary to reach the next tab stop. - // For example, coordinate (1, 5) represents the character 'B' in a line "\tABC", when mTabSize = 4, - // because it is rendered as " ABC" on the screen. + // indices of the arrays that contain the lines (vector) and the columns (a string) of the + // text editor. Negative values indicate the distance to the last element of the array. + // When comparing coordinates ensure they have the same sign because coordinates don't have + // information about the size of the array. Currently positive coordinates are always bigger + // than negatives even if that gives a wrong result. struct Coordinates { int mLine, mColumn; Coordinates() : mLine(0), mColumn(0) {} - Coordinates(int aLine, int aColumn) : mLine(aLine), mColumn(aColumn) - { - IM_ASSERT(aLine >= 0); - IM_ASSERT(aColumn >= 0); - } - static Coordinates Invalid() { static Coordinates invalid(-1, -1); return invalid; } + Coordinates(int aLine, int aColumn) : mLine(aLine), mColumn(aColumn) {} bool operator ==(const Coordinates& o) const { @@ -129,8 +147,8 @@ public: using Keywords = std::unordered_set ; using ErrorMarkers = std::map>; using Breakpoints = std::unordered_set; - using Palette = std::array; - using Char = uint8_t ; + using Palette = std::array; + using Glyph = uint8_t ; class ActionableBox { @@ -203,35 +221,381 @@ public: }; using ErrorHoverBoxes = std::map; - struct Glyph - { - Char mChar; - PaletteIndex mColorIndex = PaletteIndex::Default; - bool mComment : 1; - bool mMultiLineComment : 1; - bool mPreprocessor : 1; - bool mDocComment : 1; - bool mGlobalDocComment : 1; - bool mDeactivated : 1; + // A line of text in the pattern editor consists of three strings; the character encoding, the color encoding and the flags. + // The char encoding is utf-8, the color encoding are indices to the color palette and the flags are used to override the colors + // depending on priorities; e.g. comments, strings, etc. + + class Line { + public: + struct FlagBits { + bool mComment : 1; + bool mBlockComment : 1; + bool mDocComment : 1; + bool mBlockDocComment : 1; + bool mGlobalDocComment : 1; + bool mDeactivated : 1; + bool mPreprocessor : 1; + }; + union Flags { + Flags(char value) : mValue(value) {} + Flags(FlagBits bits) : mBits(bits) {} + FlagBits mBits; + char mValue; + }; + constexpr static char InComment = 31; - Glyph(Char aChar, PaletteIndex aColorIndex) : mChar(aChar), mColorIndex(aColorIndex), mComment(false), - mMultiLineComment(false), mPreprocessor(false), mDocComment(false), mGlobalDocComment(false), mDeactivated(false) {} - }; + class LineIterator { + public: + strConstIter mCharsIter; + strConstIter mColorsIter; + strConstIter mFlagsIter; - typedef std::vector Line; - typedef std::vector Lines; + LineIterator(const LineIterator &other) : mCharsIter(other.mCharsIter), mColorsIter(other.mColorsIter), mFlagsIter(other.mFlagsIter) {} + + LineIterator() = default; + + char operator*() { + return *mCharsIter; + } + + LineIterator operator++() { + LineIterator iter = *this; + ++iter.mCharsIter; + ++iter.mColorsIter; + ++iter.mFlagsIter; + return iter; + } + + LineIterator operator=(const LineIterator &other) { + mCharsIter = other.mCharsIter; + mColorsIter = other.mColorsIter; + mFlagsIter = other.mFlagsIter; + return *this; + } + + bool operator!=(const LineIterator &other) const { + return mCharsIter != other.mCharsIter || mColorsIter != other.mColorsIter || mFlagsIter != other.mFlagsIter; + } + + bool operator==(const LineIterator &other) const { + return mCharsIter == other.mCharsIter && mColorsIter == other.mColorsIter && mFlagsIter == other.mFlagsIter; + } + + LineIterator operator+(int n) { + LineIterator iter = *this; + iter.mCharsIter += n; + iter.mColorsIter += n; + iter.mFlagsIter += n; + return iter; + } + + int operator-(LineIterator l) { + return mCharsIter - l.mCharsIter; + } + }; + + LineIterator begin() const { + LineIterator iter; + iter.mCharsIter = mChars.begin(); + iter.mColorsIter = mColors.begin(); + iter.mFlagsIter = mFlags.begin(); + return iter; + } + + LineIterator end() const { + LineIterator iter; + iter.mCharsIter = mChars.end(); + iter.mColorsIter = mColors.end(); + iter.mFlagsIter = mFlags.end(); + return iter; + } + + std::string mChars; + std::string mColors; + std::string mFlags; + bool mColorized = false; + Line() : mChars(), mColors(), mFlags(), mColorized(false) {} + + explicit Line(const char *line) { + Line(std::string(line)); + } + + explicit Line(const std::string &line) : mChars(line), mColors(std::string(line.size(), 0x00)), mFlags(std::string(line.size(), 0x00)), mColorized(false) {} + Line(const Line &line) : mChars(line.mChars), mColors(line.mColors), mFlags(line.mFlags), mColorized(line.mColorized) {} + + LineIterator begin() { + LineIterator iter; + iter.mCharsIter = mChars.begin(); + iter.mColorsIter = mColors.begin(); + iter.mFlagsIter = mFlags.begin(); + return iter; + } + + LineIterator end() { + LineIterator iter; + iter.mCharsIter = mChars.end(); + iter.mColorsIter = mColors.end(); + iter.mFlagsIter = mFlags.end(); + return iter; + } + + Line &operator=(const Line &line) { + mChars = line.mChars; + mColors = line.mColors; + mFlags = line.mFlags; + mColorized = line.mColorized; + return *this; + } + + Line &operator=(Line &&line) noexcept { + mChars = std::move(line.mChars); + mColors = std::move(line.mColors); + mFlags = std::move(line.mFlags); + mColorized = line.mColorized; + return *this; + } + + size_t size() const { + return mChars.size(); + } + enum class LinePart { + Chars, + Colors, + Flags + }; + + char front(LinePart part = LinePart::Chars) const { + if (part == LinePart::Chars && !mChars.empty()) + return mChars.front(); + if (part == LinePart::Colors && !mColors.empty()) + return mColors.front(); + if (part == LinePart::Flags && !mFlags.empty()) + return mFlags.front(); + return 0x00; + } + + void push_back(char c) { + mChars.push_back(c); + mColors.push_back(0x00); + mFlags.push_back(0x00); + mColorized = false; + } + + bool empty() const { + return mChars.empty(); + } + + std::string substrUtf8(size_t start, size_t length = (size_t)-1, LinePart part = LinePart::Chars ) const { + if (start >= mChars.size()) + return ""; + if (length == (size_t)-1 || length >= mChars.size() - start) + length = mChars.size() - start; + size_t utf8Start= 0; + while (utf8Start= mChars.size()) + return ""; + if (start + length >= mChars.size()) + length = mChars.size() - start; + if (part == LinePart::Chars && length > 0) + return mChars.substr(start, length); + if (part == LinePart::Colors && length > 0) + return mColors.substr(start, length); + if (part == LinePart::Flags && length > 0) + return mFlags.substr(start, length); + return ""; + } + + template + char operator[](size_t index) const { + if (part == LinePart::Chars) + return mChars[index]; + if (part == LinePart::Colors) + return mColors[index]; + if (part == LinePart::Flags) + return mFlags[index]; + return mChars[index]; + } + + template + const char &operator[](size_t index) { + if (part == LinePart::Chars) + return mChars[index]; + if (part == LinePart::Colors) + return mColors[index]; + if (part == LinePart::Flags) + return mFlags[index]; + return mChars[index]; + } + + void SetNeedsUpdate(bool needsUpdate) { + mColorized = mColorized && !needsUpdate; + } + + void append(const char *text) { + append(std::string(text)); + } + + void append(const char text) { + append(std::string(1, text)); + } + + void append(const std::string &text) { + mChars.append(text); + mColors.append(text.size(), 0x00); + mFlags.append(text.size(), 0x00); + mColorized = false; + } + + void append(const Line &line) { + append(line.begin(), line.end()); + } + + void append(LineIterator begin, LineIterator end) { + if (begin.mCharsIter < end.mCharsIter) + mChars.append(begin.mCharsIter, end.mCharsIter); + if (begin.mColorsIter < end.mColorsIter) + mColors.append(begin.mColorsIter, end.mColorsIter); + if (begin.mFlagsIter < end.mFlagsIter) + mFlags.append(begin.mFlagsIter, end.mFlagsIter); + mColorized = false; + } + + void insert(LineIterator iter, const std::string &text) { + if (iter == end()) + append(text); + else + insert(iter, text.begin(), text.end()); + } + + void insert(LineIterator iter, const char text) { + if (iter == end()) + append(text); + else + insert(iter,std::string(1, text)); + } + + void insert(LineIterator iter, strConstIter beginString, strConstIter endString) { + if (iter == end()) + append(std::string(beginString, endString)); + else { + std::string charsString(beginString, endString); + mChars.insert(iter.mCharsIter, beginString, endString); + std::string colorString(charsString.size(), 0x00); + try { + mColors.insert(iter.mColorsIter, colorString.begin(), colorString.end()); + } catch (const std::exception &e) { + std::cerr << "Exception: " << e.what() << std::endl; + mColorized = false; + return; + } + std::string flagsString(charsString.size(), 0x00); + try { + mFlags.insert(iter.mFlagsIter, flagsString.begin(), flagsString.end()); + } catch (const std::exception &e) { + std::cerr << "Exception: " << e.what() << std::endl; + mColorized = false; + return; + } + mColorized = false; + } + } + + void insert(LineIterator iter,const Line &line) { + if (iter == end()) + append(line.begin(), line.end()); + else + insert(iter, line.begin(), line.end()); + } + + void insert(LineIterator iter,LineIterator beginLine, LineIterator endLine) { + if (iter == end()) + append(beginLine, endLine); + else { + mChars.insert(iter.mCharsIter, beginLine.mCharsIter, endLine.mCharsIter); + mColors.insert(iter.mColorsIter, beginLine.mColorsIter, endLine.mColorsIter); + mFlags.insert(iter.mFlagsIter, beginLine.mFlagsIter, endLine.mFlagsIter); + mColorized = false; + } + } + + void erase(LineIterator begin) { + mChars.erase(begin.mCharsIter); + mColors.erase(begin.mColorsIter); + mFlags.erase(begin.mFlagsIter); + mColorized = false; + } + + void erase(LineIterator begin, size_t count) { + if (count == (size_t) -1) + count = mChars.size() - (begin.mCharsIter - mChars.begin()); + mChars.erase(begin.mCharsIter, begin.mCharsIter + count); + mColors.erase(begin.mColorsIter, begin.mColorsIter + count); + mFlags.erase(begin.mFlagsIter, begin.mFlagsIter + count); + mColorized = false; + } + + void clear() { + mChars.clear(); + mColors.clear(); + mFlags.clear(); + mColorized = false; + } + + void SetLine(const std::string &text) { + mChars = text; + mColors = std::string(text.size(), 0x00); + mFlags = std::string(text.size(), 0x00); + mColorized = false; + } + + void SetLine(const Line &text) { + mChars = text.mChars; + mColors = text.mColors; + mFlags = text.mFlags; + mColorized = text.mColorized; + } + + + bool NeedsUpdate() const { + return !mColorized; + } + + }; + + using Lines = std::vector; struct LanguageDefinition { typedef std::pair TokenRegexString; typedef std::vector TokenRegexStrings; - typedef bool(*TokenizeCallback)(const char * in_begin, const char * in_end, const char *& out_begin, const char *& out_end, PaletteIndex & paletteIndex); + typedef bool(*TokenizeCallback)(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end, PaletteIndex &paletteIndex); std::string mName; Keywords mKeywords; Identifiers mIdentifiers; Identifiers mPreprocIdentifiers; - std::string mCommentStart, mCommentEnd, mSingleLineComment, mGlobalDocComment, mDocComment; + std::string mSingleLineComment, mCommentEnd, mCommentStart, mGlobalDocComment, mDocComment, mBlockDocComment; char mPreprocChar; bool mAutoIndentation; @@ -241,10 +605,8 @@ public: bool mCaseSensitive; - LanguageDefinition() - : mPreprocChar('#'), mAutoIndentation(true), mTokenize(nullptr), mCaseSensitive(true) - { - } + LanguageDefinition() : mName(""), mKeywords({}), mIdentifiers({}), mPreprocIdentifiers({}), mSingleLineComment(""), mCommentEnd(""), + mCommentStart(""), mGlobalDocComment(""), mDocComment(""), mBlockDocComment(""), mPreprocChar('#'), mAutoIndentation(true), mTokenize(nullptr), mTokenRegexStrings({}), mCaseSensitive(true) {} static const LanguageDefinition& CPlusPlus(); static const LanguageDefinition& HLSL(); @@ -303,10 +665,59 @@ public: } std::string GetText() const; bool isEmpty() const { - auto text = GetText(); - return text.empty() || text == "\n"; + if (mLines.empty()) + return true; + if (mLines.size() == 1) { + if (mLines[0].empty()) + return true; + if (mLines[0].size() == 1 && mLines[0].front() == '\n') + return true; + } + return false; } + void SetTopLine(); + uint32_t GetTopLine() const { + return static_cast(std::floor(mTopLine)); + } + uint32_t GetBottomLine() const { + return static_cast(std::ceil(mTopLine+mNumberOfLinesDisplayed)); + } + void SetNeedsUpdate (uint32_t line, bool needsUpdate) { + if (line < mLines.size()) + mLines[line].SetNeedsUpdate(needsUpdate); + } + + void SetColorizedLineSize(size_t line) { + if (line < mLines.size()) { + const auto &size = mLines[line].mChars.size(); + if (mLines[line].mColors.size() != size) { + mLines[line].mColors.resize(size); + std::fill(mLines[line].mColors.begin(), mLines[line].mColors.end(), 0x00); + } + } + } + + void SetColorizedLine(size_t line, const std::string &tokens) { + if (line < mLines.size()) { + auto &lineTokens = mLines[line].mColors; + if (lineTokens.size() != tokens.size()) { + lineTokens.resize(tokens.size()); + std::fill(lineTokens.begin(), lineTokens.end(), 0x00); + } + bool needsUpdate = false; + for (size_t i = 0; i < tokens.size(); ++i) { + if (tokens[i] != 0x00) { + if (tokens[i] != lineTokens[i]) { + lineTokens[i] = tokens[i]; + needsUpdate = true; + } + } + } + SetNeedsUpdate(line, needsUpdate); + } + } + void SetScrollY(); void SetTextLines(const std::vector& aLines); std::vector GetTextLines() const; @@ -345,14 +756,23 @@ public: void SetOverwrite(bool aValue) { mOverwrite = aValue; } std::string ReplaceStrings(std::string string, const std::string &search, const std::string &replace); - std::vector SplitString(const std::string &string, const std::string &delimiter, bool removeEmpty); + static std::vector SplitString(const std::string &string, const std::string &delimiter, bool removeEmpty); std::string ReplaceTabsWithSpaces(const std::string& string, uint32_t tabSize); std::string PreprocessText(const std::string &code); void SetReadOnly(bool aValue); + bool IsEndOfLine(const Coordinates &aCoordinates) const; + bool IsEndOfFile(const Coordinates &aCoordinates) const; bool IsReadOnly() const { return mReadOnly; } bool IsTextChanged() const { return mTextChanged; } - void SetTextChanged(bool aValue=false) { mTextChanged = aValue; } + void SetTextChanged(bool aValue) { mTextChanged = aValue; } + void SetTimeStamp(int code) { + auto now = std::chrono::high_resolution_clock::now(); + if (code == 0) + mLinesTimestamp = std::chrono::duration_cast(now.time_since_epoch()).count(); + else + mColorizedLinesTimestamp = std::chrono::duration_cast(now.time_since_epoch()).count(); + } bool IsCursorPositionChanged() const { return mCursorPositionChanged; } bool IsBreakpointsChanged() const { return mBreakPointsChanged; } void ClearBreakpointsChanged() { mBreakPointsChanged = false; } @@ -362,6 +782,7 @@ public: bool IsColorizerEnabled() const { return mColorizerEnabled; } void SetColorizerEnable(bool aValue); + void Colorize(); Coordinates GetCursorPosition() const { return GetActualCursorCoordinates(); } void SetCursorPosition(const Coordinates& aPosition); @@ -541,8 +962,7 @@ private: typedef std::vector UndoBuffer; void ProcessInputs(); - void Colorize(int aFromLine = 0, int aCount = -1); - void ColorizeRange(int aFromLine = 0, int aToLine = 0); + void ColorizeRange(); void ColorizeInternal(); float TextDistanceToLineStart(const Coordinates& aFrom) const; void EnsureCursorVisible(); @@ -553,35 +973,40 @@ private: void DeleteRange(const Coordinates& aStart, const Coordinates& aEnd); int InsertTextAt(Coordinates& aWhere, const std::string &aValue); void AddUndo(UndoRecord& aValue); - Coordinates ScreenPosToCoordinates(const ImVec2& aPosition) const; + Coordinates ScreenPosToCoordinates(const ImVec2& aPosition) const; Coordinates FindWordStart(const Coordinates& aFrom) const; Coordinates FindWordEnd(const Coordinates& aFrom) const; + Coordinates FindPreviousWord(const Coordinates& aFrom) const; Coordinates FindNextWord(const Coordinates& aFrom) const; Coordinates StringIndexToCoordinates(int aIndex, const std::string &str) const; int GetCharacterIndex(const Coordinates& aCoordinates) const; int GetCharacterColumn(int aLine, int aIndex) const; int GetLineCharacterCount(int aLine) const; - int Utf8CharsToBytes(const Coordinates &aCoordinates) const; - unsigned long long GetLineByteCount(int aLine) const; + int Utf8CharsToBytes(const Coordinates &aCoordinates) const; + static int Utf8CharsToBytes(std::string line, uint32_t start, uint32_t numChars); + unsigned long long GetLineByteCount(int aLine) const; int GetStringCharacterCount(std::string str) const; int GetLineMaxColumn(int aLine) const; bool IsOnWordBoundary(const Coordinates& aAt) const; void RemoveLine(int aStart, int aEnd); void RemoveLine(int aIndex); Line& InsertLine(int aIndex); - void EnterCharacter(ImWchar aChar, bool aShift); + void InsertLine(int aIndex, const std::string &aText); + void EnterCharacter(ImWchar aChar, bool aShift); void DeleteSelection(); std::string GetWordUnderCursor() const; std::string GetWordAt(const Coordinates& aCoords) const; - ImU32 GetGlyphColor(const Glyph& aGlyph) const; + TextEditor::PaletteIndex GetColorIndexFromFlags(Line::Flags flags); void ResetCursorBlinkTime(); - + uint32_t SkipSpaces(const Coordinates &aFrom); void HandleKeyboardInputs(); void HandleMouseInputs(); void RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPos, const ImVec2 &textEditorSize); void SetFocus(); float mLineSpacing = 1.0F; Lines mLines; + uint64_t mLinesTimestamp = 0; + uint64_t mColorizedLinesTimestamp = 0; EditorState mState = {}; UndoBuffer mUndoBuffer; int mUndoIndex = 0; @@ -616,7 +1041,7 @@ private: Palette mPalette = {}; LanguageDefinition mLanguageDefinition = {}; RegexList mRegexList; - bool mCheckComments = true; + bool mUpdateFlags = true; Breakpoints mBreakpoints = {}; ErrorMarkers mErrorMarkers = {}; ErrorHoverBoxes mErrorHoverBoxes = {}; @@ -646,8 +1071,9 @@ private: static const int sCursorBlinkOnTime; }; -bool TokenizeCStyleString(const char * in_begin, const char * in_end, const char *& out_begin, const char *& out_end); -bool TokenizeCStyleCharacterLiteral(const char * in_begin, const char * in_end, const char *& out_begin, const char *& out_end); -bool TokenizeCStyleIdentifier(const char * in_begin, const char * in_end, const char *& out_begin, const char *& out_end); -bool TokenizeCStyleNumber(const char * in_begin, const char * in_end, const char *& out_begin, const char *& out_end); -bool TokenizeCStylePunctuation(const char * in_begin, const char * in_end, const char *& out_begin, const char *& out_end); \ No newline at end of file +bool TokenizeCStyleString(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end); +bool TokenizeCStyleCharacterLiteral(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end); +bool TokenizeCStyleIdentifier(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end); +bool TokenizeCStyleNumber(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end); +bool TokenizeCStyleOperator(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end); +bool TokenizeCStyleSeparator(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end); \ No newline at end of file diff --git a/lib/third_party/imgui/ColorTextEditor/source/TextEditor.cpp b/lib/third_party/imgui/ColorTextEditor/source/TextEditor.cpp index ca543c954..6b1fa5028 100644 --- a/lib/third_party/imgui/ColorTextEditor/source/TextEditor.cpp +++ b/lib/third_party/imgui/ColorTextEditor/source/TextEditor.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "TextEditor.h" @@ -85,33 +86,40 @@ void TextEditor::SetPalette(const Palette &aValue) { sPaletteBase = aValue; } +bool TextEditor::IsEndOfLine(const Coordinates &aCoordinates) const { + if (aCoordinates.mLine < (int)mLines.size()) { + auto &line = mLines[aCoordinates.mLine]; + auto cindex = GetCharacterIndex(aCoordinates); + return cindex >= (int)line.size(); + } + return true; +} + +bool TextEditor::IsEndOfFile(const Coordinates &aCoordinates) const { + if (aCoordinates.mLine < (int)mLines.size()) { + auto &line = mLines[aCoordinates.mLine]; + auto cindex = GetCharacterIndex(aCoordinates); + return cindex >= (int)line.size() && aCoordinates.mLine == (int)mLines.size() - 1; + } + return true; +} + std::string TextEditor::GetText(const Coordinates &aStart, const Coordinates &aEnd) const { - std::string result; - - auto lstart = aStart.mLine; - auto lend = aEnd.mLine; - auto istart = GetCharacterIndex(aStart); - auto iend = GetCharacterIndex(aEnd); - size_t s = 0; - - for (size_t i = lstart; i < lend; i++) - s += mLines[i].size(); - - result.reserve(s + s / 8); - - while (istart < iend || lstart < lend) { - if (lstart >= (int)mLines.size()) - break; - - auto &line = mLines[lstart]; - if (istart < (int)line.size()) { - result += line[istart].mChar; - istart++; - } else { - istart = 0; - ++lstart; - result += '\n'; + std::string result=""; + auto start = SanitizeCoordinates(aStart); + auto end = SanitizeCoordinates(aEnd); + auto lineStart = start.mLine; + auto lineEnd = end.mLine; + auto startIndex = GetCharacterIndex(start); + auto endIndex = GetCharacterIndex(end); + if (lineStart == lineEnd) + result = mLines[lineStart].substr(startIndex,endIndex-startIndex); + else { + result = mLines[lineStart].substr(startIndex) + '\n'; + for (size_t i = lineStart+1; i < lineEnd; i++) { + result += mLines[i].mChars + '\n'; } + result += mLines[lineEnd].substr(0, endIndex); } return result; @@ -122,191 +130,145 @@ TextEditor::Coordinates TextEditor::GetActualCursorCoordinates() const { } TextEditor::Coordinates TextEditor::SanitizeCoordinates(const Coordinates &aValue) const { - auto line = aValue.mLine; - auto column = aValue.mColumn; - if (line >= (int)mLines.size()) { - if (isEmpty()) { - line = 0; - column = 0; - } else { - line = (int)mLines.size() - 1; - column = GetLineMaxColumn(line); - } - return Coordinates(line, column); - } else { - column = isEmpty() ? 0 : std::min(column, GetLineMaxColumn(line)); - return Coordinates(line, column); - } -} + Coordinates result = aValue; + if (aValue.mLine < 0) + result.mLine = mLines.size() + aValue.mLine; + if (aValue.mColumn < 0) + result.mColumn = GetLineMaxColumn(result.mLine) + aValue.mColumn + 1; -// https://en.wikipedia.org/wiki/UTF-8 -// We assume that the char is a standalone character (<128) or a leading byte of an UTF-8 code sequence (non-10xxxxxx code) -static int UTF8CharLength(TextEditor::Char c) { - if ((c & 0xFE) == 0xFC) - return 6; - if ((c & 0xFC) == 0xF8) - return 5; - if ((c & 0xF8) == 0xF0) - return 4; - else if ((c & 0xF0) == 0xE0) - return 3; - else if ((c & 0xE0) == 0xC0) - return 2; - return 1; + result.mLine = std::clamp(result.mLine, 0, (int)mLines.size()-1); + result.mColumn = std::clamp(result.mColumn, 0, GetLineMaxColumn(result.mLine)); + return result; } // "Borrowed" from ImGui source -static inline int ImTextCharToUtf8(char *buf, int buf_size, unsigned int c) { +static inline void ImTextCharToUtf8(std::string &buffer, unsigned int c) { if (c < 0x80) { - buf[0] = (char)c; - return 1; + buffer += (char) c; + return; } if (c < 0x800) { - if (buf_size < 2) return 0; - buf[0] = (char)(0xc0 + (c >> 6)); - buf[1] = (char)(0x80 + (c & 0x3f)); - return 2; - } - if (c >= 0xdc00 && c < 0xe000) { - return 0; + buffer += (char)(0xc0 + (c >> 6)); + buffer += (char)(0x80 + (c & 0x3f)); + return; } + if (c >= 0xdc00 && c < 0xe000) + return; if (c >= 0xd800 && c < 0xdc00) { - if (buf_size < 4) return 0; - buf[0] = (char)(0xf0 + (c >> 18)); - buf[1] = (char)(0x80 + ((c >> 12) & 0x3f)); - buf[2] = (char)(0x80 + ((c >> 6) & 0x3f)); - buf[3] = (char)(0x80 + ((c)&0x3f)); - return 4; + buffer += (char)(0xf0 + (c >> 18)); + buffer += (char)(0x80 + ((c >> 12) & 0x3f)); + buffer += (char)(0x80 + ((c >> 6) & 0x3f)); + buffer += (char)(0x80 + ((c)&0x3f)); + return; } // else if (c < 0x10000) { - if (buf_size < 3) return 0; - buf[0] = (char)(0xe0 + (c >> 12)); - buf[1] = (char)(0x80 + ((c >> 6) & 0x3f)); - buf[2] = (char)(0x80 + ((c)&0x3f)); - return 3; + buffer += (char)(0xe0 + (c >> 12)); + buffer += (char)(0x80 + ((c >> 6) & 0x3f)); + buffer += (char)(0x80 + ((c)&0x3f)); + return; } } +static inline int ImTextCharToUtf8(char *buffer, int buf_size, unsigned int c) { + std::string input; + ImTextCharToUtf8(input,c); + auto size = input.size(); + int i=0; + for (;i= aStart); - IM_ASSERT(!mReadOnly); - - // printf("D(%d.%d)-(%d.%d)\n", aStart.mLine, aStart.mColumn, aEnd.mLine, aEnd.mColumn); - - if (aEnd == aStart) + if (aEnd <= aStart || mReadOnly) return; - - auto start = GetCharacterIndex(aStart); - auto end = GetCharacterIndex(aEnd); - if (start == -1 || end == -1) - return; - - if (aStart.mLine == aEnd.mLine) { - auto &line = mLines[aStart.mLine]; - auto n = GetLineMaxColumn(aStart.mLine); - if (aEnd.mColumn >= n) - line.erase(line.begin() + start, line.end()); + auto start = SanitizeCoordinates(aStart); + auto end = SanitizeCoordinates(aEnd); + auto startIndex = GetCharacterIndex(start); + auto endIndex = GetCharacterIndex(end); + if (start.mLine == end.mLine) { + auto &line = mLines[start.mLine]; + if (endIndex >= GetLineMaxColumn(start.mLine)) + line.erase(line.begin() + startIndex, -1); else - line.erase(line.begin() + start, line.begin() + end); + line.erase(line.begin() + startIndex, endIndex-startIndex); + line.mColorized = false; } else { - auto &firstLine = mLines[aStart.mLine]; - auto &lastLine = mLines[aEnd.mLine]; + auto &firstLine = mLines[start.mLine]; + auto &lastLine = mLines[end.mLine]; - firstLine.erase(firstLine.begin() + start, firstLine.end()); - lastLine.erase(lastLine.begin(), lastLine.begin() + end); + firstLine.erase(firstLine.begin() + startIndex, -1); + lastLine.erase(lastLine.begin(), endIndex); - if (aStart.mLine < aEnd.mLine) + if (aStart.mLine < aEnd.mLine) { firstLine.insert(firstLine.end(), lastLine.begin(), lastLine.end()); - - if (aStart.mLine < aEnd.mLine) + firstLine.mColorized = false; RemoveLine(aStart.mLine + 1, aEnd.mLine); + } } mTextChanged = true; } void TextEditor::AppendLine(const std::string &aValue) { - if (mLines.size() != 1 || !mLines[0].empty()) - mLines.push_back(Line()); - Coordinates lastLine = {( int )mLines.size() - 1, 0}; - InsertTextAt(lastLine, aValue); - SetCursorPosition({( int )mLines.size() - 1, 0}); + auto text = PreprocessText(aValue); + if (text.empty()) + return; + if (isEmpty()) { + mLines[0].mChars = text; + mLines[0].mColors = std::string(text.size(),0); + mLines[0].mFlags = std::string(text.size(),0); + } else + mLines.push_back(Line(text)); + SetCursorPosition(Coordinates((int)mLines.size() - 1, 0)); + mLines.back().mColorized = false; EnsureCursorVisible(); mTextChanged = true; } -int TextEditor::InsertTextAt(Coordinates & /* inout */ aWhere, const std::string &aValueString) { - int cindex = GetCharacterIndex(aWhere); - int totalLines = 0; - auto aValue = aValueString.begin(); - while (aValue != aValueString.end()) { - if (mLines.empty()) { - mLines.push_back(Line()); - mTextChanged = true; - } - if (*aValue == '\r') { - // skip - ++aValue; - } else if (*aValue == '\t') { - auto &line = mLines[aWhere.mLine]; - auto c = GetCharacterColumn(aWhere.mLine, cindex); - auto r = c % mTabSize; - auto d = mTabSize - r; - auto i = d; - while (i-- > 0) - line.insert(line.begin() + cindex++, Glyph(' ', PaletteIndex::Default)); +int TextEditor::InsertTextAt(Coordinates & /* inout */ aWhere, const std::string &aValue) { + auto text = PreprocessText(aValue); + if (text.empty()) + return 0; + auto start = SanitizeCoordinates(aWhere); + auto &line = mLines[start.mLine]; + auto stringVector = SplitString(text, "\n", false); + auto lineCount = (int)stringVector.size(); + aWhere.mLine += lineCount - 1; + aWhere.mColumn += stringVector[lineCount-1].size(); + stringVector[lineCount - 1].append(line.substr(GetCharacterIndex(start))); + if (GetCharacterIndex(start) < (int)line.size()) + line.erase(line.begin() + GetCharacterIndex(start),-1); - aWhere.mColumn += d; - aValue++; - } else if (*aValue == '\n') { - if (cindex < (int)mLines[aWhere.mLine].size()) { - auto &newLine = InsertLine(aWhere.mLine + 1); - auto &line = mLines[aWhere.mLine]; - newLine.insert(newLine.begin(), line.begin() + cindex, line.end()); - line.erase(line.begin() + cindex, line.end()); - } else { - InsertLine(aWhere.mLine + 1); - } - ++aWhere.mLine; - aWhere.mColumn = 0; - cindex = 0; - ++totalLines; - ++aValue; - } else if (*aValue == 0) { - auto &line = mLines[aWhere.mLine]; - line.insert(line.begin() + cindex++, Glyph('.', PaletteIndex::Default)); - ++aWhere.mColumn; - ++aValue; - } else { - auto &line = mLines[aWhere.mLine]; - auto d = UTF8CharLength(*aValue); - while (d-- > 0 && aValue != aValueString.end()) - line.insert(line.begin() + cindex++, Glyph(*aValue++, PaletteIndex::Default)); - ++aWhere.mColumn; - } - - mTextChanged = true; + line.append(stringVector[0]); + line.mColorized = false; + for (int i = 1; i < lineCount; i++) { + InsertLine(start.mLine + i, stringVector[i]); + mLines[start.mLine + i].mColorized = false; } - - return totalLines; + mTextChanged = true; + return lineCount; } void TextEditor::AddUndo(UndoRecord &aValue) { @@ -324,66 +286,31 @@ void TextEditor::AddUndo(UndoRecord &aValue) { } TextEditor::Coordinates TextEditor::ScreenPosToCoordinates(const ImVec2 &aPosition) const { - ImVec2 origin = ImGui::GetCursorScreenPos(); - ImVec2 local(aPosition.x - origin.x, aPosition.y - origin.y); + ImVec2 local = aPosition - ImGui::GetCursorScreenPos(); int lineNo = std::max(0, (int)floor(local.y / mCharAdvance.y)); - if (local.x < mCharAdvance.x) - return Coordinates(lineNo, 0); - local.x -= mCharAdvance.x; + if (local.x < (mLeftMargin - 2) || lineNo >= (int)mLines.size() || mLines[lineNo].empty()) + return Coordinates(std::min(lineNo,(int)mLines.size()-1), 0); - int columnCoord = 0; + local.x -= (mLeftMargin - 2); + auto column = (int)floor(local.x / mCharAdvance.x); + auto rem = static_cast(local.x) % static_cast(mCharAdvance.x); + if (rem > mCharAdvance.x - 2) + column++; - if (lineNo >= 0 && lineNo < (int)mLines.size()) { - auto &line = mLines.at(lineNo); - - int columnIndex = 0; - float columnX = 0.0f; - - while ((size_t)columnIndex < line.size()) { - float columnWidth = 0.0f; - - if (line[columnIndex].mChar == '\t') { - float spaceSize = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, " ").x; - float oldX = columnX; - float newColumnX = (1.0f + std::floor((1.0f + columnX) / (float(mTabSize) * spaceSize))) * (float(mTabSize) * spaceSize); - columnWidth = newColumnX - oldX; - if (columnX + columnWidth > local.x) - break; - columnX = newColumnX; - columnCoord = (columnCoord / mTabSize) * mTabSize + mTabSize; - columnIndex++; - } else { - char buf[7]; - auto d = UTF8CharLength(line[columnIndex].mChar); - int i = 0; - while (i < 6 && d-- > 0) - buf[i++] = line[columnIndex++].mChar; - buf[i] = '\0'; - columnWidth = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, buf).x; - if (columnX + columnWidth > local.x) - break; - columnX += columnWidth; - columnCoord++; - } - } - } - - return SanitizeCoordinates(Coordinates(lineNo, columnCoord)); + return SanitizeCoordinates(Coordinates(lineNo, column)); } void TextEditor::DeleteWordLeft() { const auto wordEnd = GetCursorPosition(); - MoveLeft(); - const auto wordStart = FindWordStart(GetCursorPosition()); + const auto wordStart = FindPreviousWord(GetCursorPosition()); SetSelection(wordStart, wordEnd); Backspace(); } void TextEditor::DeleteWordRight() { const auto wordStart = GetCursorPosition(); - MoveRight(); - const auto wordEnd = FindWordEnd(GetCursorPosition()); + const auto wordEnd = FindNextWord(GetCursorPosition()); SetSelection(wordStart, wordEnd); Backspace(); } @@ -401,17 +328,16 @@ TextEditor::Coordinates TextEditor::FindWordStart(const Coordinates &aFrom) cons auto &line = mLines[at.mLine]; auto cindex = GetCharacterIndex(at); - if (cindex >= (int)line.size()) - return at; - - while (cindex > 0 && !isWordChar(line[cindex-1].mChar)) - --cindex; - - while (cindex > 0 && isWordChar(line[cindex - 1].mChar)) - --cindex; - - if (cindex==0 && line[cindex].mChar == '\"') - ++cindex; + if (isWordChar(line.mChars[cindex])) { + while (cindex > 0 && isWordChar(line.mChars[cindex-1])) + --cindex; + } else if (ispunct(line.mChars[cindex])) { + while (cindex > 0 && ispunct(line.mChars[cindex-1])) + --cindex; + } else if (isspace(line.mChars[cindex])) { + while (cindex > 0 && isspace(line.mChars[cindex-1])) + --cindex; + } return Coordinates(at.mLine, GetCharacterColumn(at.mLine, cindex)); } @@ -423,17 +349,16 @@ TextEditor::Coordinates TextEditor::FindWordEnd(const Coordinates &aFrom) const auto &line = mLines[at.mLine]; auto cindex = GetCharacterIndex(at); - if (cindex >= (int)line.size()) - return at; - - while (cindex < (line.size()) && !isWordChar(line[cindex].mChar)) - ++cindex; - while (cindex < (line.size()) && isWordChar(line[cindex].mChar)) - ++cindex; - - if (line[cindex-1].mChar == '\"') - --cindex; - + if (isWordChar(line.mChars[cindex])) { + while (cindex < (int)line.mChars.size() && isWordChar(line.mChars[cindex])) + ++cindex; + } else if (ispunct(line.mChars[cindex])) { + while (cindex < (int)line.mChars.size() && ispunct(line.mChars[cindex])) + ++cindex; + } else if (isspace(line.mChars[cindex])) { + while (cindex < (int)line.mChars.size() && isspace(line.mChars[cindex])) + ++cindex; + } return Coordinates(aFrom.mLine, GetCharacterColumn(aFrom.mLine, cindex)); } @@ -442,42 +367,57 @@ TextEditor::Coordinates TextEditor::FindNextWord(const Coordinates &aFrom) const if (at.mLine >= (int)mLines.size()) return at; - // skip to the next non-word character - auto cindex = GetCharacterIndex(aFrom); - bool isword = false; - bool skip = false; - if (cindex < (int)mLines[at.mLine].size()) { - const auto &line = mLines[at.mLine]; - isword = isalnum(line[cindex].mChar); - skip = isword; + auto &line = mLines[at.mLine]; + auto cindex = GetCharacterIndex(at); + + if (isspace(line.mChars[cindex])) { + while (cindex < (int)line.mChars.size() && isspace(line.mChars[cindex])) + ++cindex; } - - while (!isword || skip) { - if (at.mLine >= mLines.size()) { - auto l = std::max(0, (int)mLines.size() - 1); - return Coordinates(l, GetLineMaxColumn(l)); - } - - auto &line = mLines[at.mLine]; - if (cindex < (int)line.size()) { - isword = isalnum(line[cindex].mChar); - - if (isword && !skip) - return Coordinates(at.mLine, GetCharacterColumn(at.mLine, cindex)); - - if (!isword) - skip = false; - - cindex++; - } else { - cindex = 0; - ++at.mLine; - skip = false; - isword = false; - } + if (isWordChar(line.mChars[cindex])) { + while (cindex < (int)line.mChars.size() && (isWordChar(line.mChars[cindex]))) + ++cindex; + } else if (ispunct(line.mChars[cindex])) { + while (cindex < (int)line.mChars.size() && (ispunct(line.mChars[cindex]))) + ++cindex; } + return Coordinates(aFrom.mLine, GetCharacterColumn(aFrom.mLine, cindex)); +} - return at; +TextEditor::Coordinates TextEditor::FindPreviousWord(const Coordinates &aFrom) const { + Coordinates at = aFrom; + if (at.mLine >= (int)mLines.size()) + return at; + + auto &line = mLines[at.mLine]; + auto cindex = GetCharacterIndex(at); + + if (isspace(line.mChars[cindex-1])) { + while (cindex > 0 && isspace(line.mChars[cindex-1])) + --cindex; + } + if (isWordChar(line.mChars[cindex-1])) { + while (cindex > 0 && isWordChar(line.mChars[cindex-1])) + --cindex; + } else if (ispunct(line.mChars[cindex-1])) { + while (cindex > 0 && ispunct(line.mChars[cindex-1])) + --cindex; + } + return Coordinates(at.mLine, GetCharacterColumn(at.mLine, cindex)); +} + + + +int TextEditor::Utf8CharsToBytes(std::string line, uint32_t start, uint32_t numChars) { + if (line.empty()) + return 0; + int column = 0; + int index = 0; + while (start + index < line.size() && column < numChars) { + index += UTF8CharLength(line[start + index]); + ++column; + } + return index; } int TextEditor::Utf8CharsToBytes(const Coordinates &aCoordinates) const { @@ -489,8 +429,8 @@ int TextEditor::Utf8CharsToBytes(const Coordinates &aCoordinates) const { int c = 0; int i = 0; while (i < line.size() && c < aCoordinates.mColumn) { - i += UTF8CharLength(line[i].mChar); - if (line[i].mChar == '\t') + i += UTF8CharLength(line[i]); + if (line[i] == '\t') c = (c / mTabSize) * mTabSize + mTabSize; else ++c; @@ -518,7 +458,7 @@ int TextEditor::GetCharacterIndex(const Coordinates &aCoordinates) const { int column = 0; int index = 0; while (index < line.size() && column < aCoordinates.mColumn) { - const auto character = line[index].mChar; + const auto character = line[index]; index += UTF8CharLength(character); if (character == '\t') column = (column / mTabSize) * mTabSize + mTabSize; @@ -536,7 +476,7 @@ int TextEditor::GetCharacterColumn(int aLine, int aIndex) const { int col = 0; int i = 0; while (i < aIndex && i < (int)line.size()) { - auto c = line[i].mChar; + auto c = line[i]; i += UTF8CharLength(c); if (c == '\t') col = (col / mTabSize) * mTabSize + mTabSize; @@ -561,7 +501,7 @@ int TextEditor::GetLineCharacterCount(int aLine) const { auto &line = mLines[aLine]; int c = 0; for (unsigned i = 0; i < line.size(); c++) - i += UTF8CharLength(line[i].mChar); + i += UTF8CharLength(line[i]); return c; } @@ -578,7 +518,7 @@ int TextEditor::GetLineMaxColumn(int aLine) const { auto &line = mLines[aLine]; int col = 0; for (unsigned i = 0; i < line.size();) { - auto c = line[i].mChar; + auto c = line[i]; if (c == '\t') col = (col / mTabSize) * mTabSize + mTabSize; else @@ -597,15 +537,23 @@ bool TextEditor::IsOnWordBoundary(const Coordinates &aAt) const { if (cindex >= (int)line.size()) return true; - if (mColorizerEnabled) - return line[cindex].mColorIndex != line[size_t(cindex - 1)].mColorIndex; - - return isspace(line[cindex].mChar) != isspace(line[cindex - 1].mChar); + if (mColorizerEnabled) { + int wordStart = 0; + int newWordStart = 0; + int wordEnd = 0; + while (wordEnd < cindex) { + wordStart = newWordStart; + auto color = line.mColors[wordStart]; + int tokenLength = line.mColors.find_first_not_of(color, wordStart)-wordStart; + wordEnd = wordStart + tokenLength; + newWordStart = wordEnd + 1; + } + return cindex == wordStart || cindex == wordEnd; + } + return isspace(line[cindex]) != isspace(line[cindex - 1]); } void TextEditor::RemoveLine(int aStart, int aEnd) { - IM_ASSERT(!mReadOnly); - IM_ASSERT(aEnd >= aStart); ErrorMarkers etmp; for (auto &i : mErrorMarkers) { @@ -631,38 +579,24 @@ void TextEditor::RemoveLine(int aStart, int aEnd) { // use clamp to ensure valid results instead of assert. auto start = std::clamp(aStart, 0, (int)mLines.size()-1); auto end = std::clamp(aEnd, 0, (int)mLines.size()); + if (start > end) + std::swap(start, end); + mLines.erase(mLines.begin() + aStart, mLines.begin() + aEnd + 1); mTextChanged = true; } void TextEditor::RemoveLine(int aIndex) { - IM_ASSERT(!mReadOnly); - IM_ASSERT(mLines.size() > 1); - - ErrorMarkers etmp; - for (auto &i : mErrorMarkers) { - ErrorMarkers::value_type e(i.first.mLine > aIndex ? Coordinates(i.first.mLine - 1 ,i.first.mColumn) : i.first, i.second); - if (e.first.mLine - 1 == aIndex) - continue; - etmp.insert(e); - } - mErrorMarkers = std::move(etmp); - - Breakpoints btmp; - for (auto breakpoint : mBreakpoints) { - if (breakpoint > aIndex) { - btmp.insert(breakpoint - 1); - mBreakPointsChanged = true; - }else - btmp.insert(breakpoint); - } - if (mBreakPointsChanged) - mBreakpoints = std::move(btmp); - - mLines.erase(mLines.begin() + aIndex); - IM_ASSERT(!mLines.empty()); + RemoveLine(aIndex, aIndex); +} +void TextEditor::InsertLine(int aIndex, const std::string &aText) { + if (aIndex < 0 || aIndex > (int)mLines.size()) + return; + auto &line = InsertLine(aIndex); + line.append(aText); + line.mColorized = false; mTextChanged = true; } @@ -677,6 +611,7 @@ TextEditor::Line &TextEditor::InsertLine(int aIndex) { auto newLine = Line(); TextEditor::Line &result = *mLines.insert(mLines.begin() + aIndex, newLine); + result.mColorized = false; ErrorMarkers etmp; for (auto &i : mErrorMarkers) @@ -712,34 +647,39 @@ std::string TextEditor::GetWordAt(const Coordinates &aCoords) const { auto iend = GetCharacterIndex(end); for (auto it = istart; it < iend; ++it) - r.push_back(mLines[aCoords.mLine][it].mChar); + r.push_back(mLines[aCoords.mLine][it]); return r; } -ImU32 TextEditor::GetGlyphColor(const Glyph &aGlyph) const { - if (!mColorizerEnabled) - return mPalette[(int)PaletteIndex::Default]; - if (aGlyph.mGlobalDocComment) - return mPalette[(int)PaletteIndex::GlobalDocComment]; - if (aGlyph.mDocComment) - return mPalette[(int)PaletteIndex::DocComment]; - if (aGlyph.mComment) - return mPalette[(int)PaletteIndex::Comment]; - if (aGlyph.mMultiLineComment) - return mPalette[(int)PaletteIndex::MultiLineComment]; - if (aGlyph.mDeactivated) - return mPalette[(int)PaletteIndex::PreprocessorDeactivated]; - const auto color = mPalette[(int)aGlyph.mColorIndex]; - if (aGlyph.mPreprocessor) { - const auto ppcolor = mPalette[(int)PaletteIndex::Preprocessor]; - const int c0 = ((ppcolor & 0xff) + (color & 0xff)) / 2; - const int c1 = (((ppcolor >> 8) & 0xff) + ((color >> 8) & 0xff)) / 2; - const int c2 = (((ppcolor >> 16) & 0xff) + ((color >> 16) & 0xff)) / 2; - const int c3 = (((ppcolor >> 24) & 0xff) + ((color >> 24) & 0xff)) / 2; - return ImU32(c0 | (c1 << 8) | (c2 << 16) | (c3 << 24)); - } - return color; +uint8_t ByteCount(uint32_t number, std::string &bytes) { + bytes[0] = static_cast(number); + if (!(number & 0xFFFFFF00)) + return 1; + bytes[1] = static_cast(number >> 8); + if (!(number & 0xFFFF0000)) + return 2; + bytes[2] = static_cast(number >> 16); + bytes[3] = static_cast(number >> 24); + return 4; +} + +TextEditor::PaletteIndex TextEditor::GetColorIndexFromFlags(Line::Flags flags) { + if (flags.mBits.mGlobalDocComment) + return PaletteIndex::GlobalDocComment; + if (flags.mBits.mBlockDocComment ) + return PaletteIndex::DocBlockComment; + if (flags.mBits.mDocComment) + return PaletteIndex::DocComment; + if (flags.mBits.mBlockComment) + return PaletteIndex::BlockComment; + if (flags.mBits.mComment) + return PaletteIndex::Comment; + if (flags.mBits.mDeactivated) + return PaletteIndex::PreprocessorDeactivated; + if (flags.mBits.mPreprocessor) + return PaletteIndex::Directive; + return PaletteIndex::Default; } void TextEditor::HandleKeyboardInputs() { @@ -869,6 +809,22 @@ inline void TextUnformattedColoredAt(const ImVec2 &pos, const ImU32 &color, cons ImGui::TextUnformatted(text); ImGui::PopStyleColor(); } +uint32_t TextEditor::SkipSpaces(const Coordinates &aFrom) { + auto line = aFrom.mLine; + if (line >= mLines.size()) + return 0; + auto &lines = mLines[line].mChars; + auto &colors = mLines[line].mColors; + auto cindex = GetCharacterIndex(aFrom); + uint32_t s = 0; + while (cindex < (int)lines.size() && lines[cindex] == ' ' && colors[cindex] == 0x00) { + ++s; + ++cindex; + } + if (mUpdateFocus) + SetFocus(); + return s; +} void TextEditor::SetFocus() { mState.mCursorPosition = mFocusAtCoords; @@ -922,14 +878,8 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo auto lineMax = std::clamp(lineNo + mNumberOfLinesDisplayed, 0.0F, globalLineMax-1.0F); int totalDigitCount = std::floor(std::log10(globalLineMax)) + 1; - // Deduce mTextStart by evaluating mLines size (global lineMax) plus two spaces as text width char buf[16]; - if (mShowLineNumbers) - snprintf(buf, 16, " %d ", int(globalLineMax)); - else - buf[0] = '\0'; - mTextStart = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, buf, nullptr, nullptr).x + mLeftMargin; if (!mLines.empty()) { float spaceSize = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, " ", nullptr, nullptr).x; @@ -939,7 +889,7 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo ImVec2 textScreenPos = lineStartScreenPos; auto &line = mLines[lineNo]; - auto columnNo = 0; + auto colors = mLines[lineNo].mColors; Coordinates lineStartCoord(lineNo, 0); Coordinates lineEndCoord(lineNo, GetLineMaxColumn(lineNo)); @@ -985,28 +935,31 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo else mBreakpoints.insert(lineNo + 1); mBreakPointsChanged = true; + mState.mCursorPosition = Coordinates(lineNo, 0); JumpToCoords(mState.mCursorPosition); } - TextUnformattedColoredAt(ImVec2(mLeftMargin + lineNoStartScreenPos.x, lineStartScreenPos.y), mPalette[(int) PaletteIndex::LineNumber], lineNoStr.c_str()); - } - // Draw breakpoints - if (mBreakpoints.count(lineNo + 1) != 0) { - auto end = ImVec2(lineNoStartScreenPos.x + contentSize.x + mLineNumberFieldWidth, lineStartScreenPos.y + mCharAdvance.y); - drawList->AddRectFilled(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y), end, mPalette[(int)PaletteIndex::Breakpoint]); - - drawList->AddCircleFilled(start + ImVec2(0, mCharAdvance.y) / 2, mCharAdvance.y / 3, mPalette[(int)PaletteIndex::Breakpoint]); - drawList->AddCircle(start + ImVec2(0, mCharAdvance.y) / 2, mCharAdvance.y / 3, mPalette[(int)PaletteIndex::Default]); - } - - if (mState.mCursorPosition.mLine == lineNo && mShowCursor) { - - // Highlight the current line (where the cursor is) - if (!HasSelection()) { + // Draw breakpoints + if (mBreakpoints.count(lineNo + 1) != 0) { auto end = ImVec2(lineNoStartScreenPos.x + contentSize.x + mLineNumberFieldWidth, lineStartScreenPos.y + mCharAdvance.y); - drawList->AddRectFilled(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y), end, mPalette[(int)(focused ? PaletteIndex::CurrentLineFill : PaletteIndex::CurrentLineFillInactive)]); - drawList->AddRect(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y), end, mPalette[(int)PaletteIndex::CurrentLineEdge], 1.0f); + drawList->AddRectFilled(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y), end, mPalette[(int)PaletteIndex::Breakpoint]); + + drawList->AddCircleFilled(start + ImVec2(0, mCharAdvance.y) / 2, mCharAdvance.y / 3, mPalette[(int)PaletteIndex::Breakpoint]); + drawList->AddCircle(start + ImVec2(0, mCharAdvance.y) / 2, mCharAdvance.y / 3, mPalette[(int)PaletteIndex::Default]); + drawList->AddText(ImVec2(lineNoStartScreenPos.x + mLeftMargin, lineStartScreenPos.y),mPalette[(int) PaletteIndex::LineNumber], lineNoStr.c_str()); } + + if (mState.mCursorPosition.mLine == lineNo && mShowCursor) { + + // Highlight the current line (where the cursor is) + if (!HasSelection()) { + auto end = ImVec2(lineNoStartScreenPos.x + contentSize.x + mLineNumberFieldWidth, lineStartScreenPos.y + mCharAdvance.y); + drawList->AddRectFilled(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y), end, mPalette[(int)(focused ? PaletteIndex::CurrentLineFill : PaletteIndex::CurrentLineFillInactive)]); + drawList->AddRect(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y), end, mPalette[(int)PaletteIndex::CurrentLineEdge], 1.0f); + } + } + + TextUnformattedColoredAt(ImVec2(mLeftMargin + lineNoStartScreenPos.x, lineStartScreenPos.y), mPalette[(int) PaletteIndex::LineNumber], lineNoStr.c_str()); } if (mShowLineNumbers && !mIgnoreImGuiChild) ImGui::EndChild(); @@ -1024,13 +977,13 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo float cx = TextDistanceToLineStart(mState.mCursorPosition); if (mOverwrite && cindex < (int)line.size()) { - auto c = line[cindex].mChar; + auto c = line[cindex]; if (c == '\t') { auto x = (1.0f + std::floor((1.0f + cx) / (float(mTabSize) * spaceSize))) * (float(mTabSize) * spaceSize); width = x - cx; } else { char buf2[2]; - buf2[0] = line[cindex].mChar; + buf2[0] = line[cindex]; buf2[1] = '\0'; width = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, buf2).x; } @@ -1087,16 +1040,23 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo } // Render colorized text - auto prevColor = line.empty() ? mPalette[(int)PaletteIndex::Default] : GetGlyphColor(line[0]); - ImVec2 bufferOffset; - - if (mUpdateFocus) { - SetFocus(); + if (line.empty()) { + ImGui::Dummy(mCharAdvance); + lineNo = std::floor(lineNo + 1.0F); + if (mUpdateFocus) + SetFocus(); + continue; } - - for (int i = 0; i < line.size();) { - auto &glyph = line[i]; - auto color = GetGlyphColor(glyph); + int i = 0; + auto colorsSize = static_cast(colors.size()); + i += SkipSpaces(Coordinates(lineNo, i)); + while (i < colorsSize) { + char color = colors[i]; + uint32_t tokenLength = colors.find_first_not_of(color, i) - i; + if (mUpdateFocus) + SetFocus(); + color = std::clamp(color, (char)PaletteIndex::Default, (char)((uint8_t)PaletteIndex::Max-1)); + tokenLength = std::clamp(tokenLength, 1u, colorsSize - i); bool underwaved = false; ErrorMarkers::iterator errorIt; @@ -1104,13 +1064,14 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo underwaved = true; } - if ((color != prevColor || glyph.mChar == '\t' || glyph.mChar == ' ') && !mLineBuffer.empty()) { - const ImVec2 newOffset(textScreenPos.x + bufferOffset.x, textScreenPos.y + bufferOffset.y); - TextUnformattedColoredAt(newOffset, prevColor, mLineBuffer.c_str()); - auto textSize = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, mLineBuffer.c_str(), nullptr, nullptr); - bufferOffset.x += textSize.x; - mLineBuffer.clear(); - } + mLineBuffer = line.substr(i, tokenLength); + ImGui::PushStyleColor(ImGuiCol_Text, mPalette[(uint64_t) color]); + auto charsBefore = ImGui::CalcTextSize(line.mChars.substr(0, i).c_str()).x; + const ImVec2 textScreenPosition(lineStartScreenPos.x + charsBefore, lineStartScreenPos.y); + ImGui::SetCursorScreenPos(textScreenPosition); + ImGui::TextUnformatted(mLineBuffer.c_str()); + ImGui::PopStyleColor(); + mLineBuffer.clear(); if (underwaved) { auto textStart = TextDistanceToLineStart(Coordinates(lineNo, i)); auto begin = ImVec2(lineStartScreenPos.x + textStart, lineStartScreenPos.y); @@ -1129,67 +1090,33 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo if (box.trigger()) box.callback(); } - prevColor = color; - if (glyph.mChar == '\t') { - auto oldX = bufferOffset.x; - bufferOffset.x = (1.0f + std::floor((1.0f + bufferOffset.x) / (float(mTabSize) * spaceSize))) * (float(mTabSize) * spaceSize); - ++i; + i += tokenLength; + i += SkipSpaces(Coordinates(lineNo, i)); - if (mShowWhitespaces) { - const auto s = ImGui::GetFontSize(); - const auto x1 = textScreenPos.x + oldX + 1.0f; - const auto x2 = textScreenPos.x + bufferOffset.x - 1.0f; - const auto y = textScreenPos.y + bufferOffset.y + s * 0.5f; - const ImVec2 p1(x1, y); - const ImVec2 p2(x2, y); - const ImVec2 p3(x2 - s * 0.2f, y - s * 0.2f); - const ImVec2 p4(x2 - s * 0.2f, y + s * 0.2f); - drawList->AddLine(p1, p2, 0x90909090); - drawList->AddLine(p2, p3, 0x90909090); - drawList->AddLine(p2, p4, 0x90909090); - } - } else if (glyph.mChar == ' ') { - if (mShowWhitespaces) { - const auto s = ImGui::GetFontSize(); - const auto x = textScreenPos.x + bufferOffset.x + spaceSize * 0.5f; - const auto y = textScreenPos.y + bufferOffset.y + s * 0.5f; - drawList->AddCircleFilled(ImVec2(x, y), 1.5f, 0x80808080, 4); - } - bufferOffset.x += spaceSize; - i++; - } else { - auto l = UTF8CharLength(glyph.mChar); - while (l-- > 0) - mLineBuffer.push_back(line[i++].mChar); - } - ++columnNo; - } - - if (!mLineBuffer.empty()) { - const ImVec2 newOffset(textScreenPos.x + bufferOffset.x, textScreenPos.y + bufferOffset.y); - TextUnformattedColoredAt(newOffset, prevColor, mLineBuffer.c_str()); - mLineBuffer.clear(); } lineNo = std::floor(lineNo + 1.0F); } } + ImVec2 lineStartScreenPos = ImVec2(cursorScreenPos.x + mLeftMargin, mTopMargin + cursorScreenPos.y + std::floor(lineNo) * mCharAdvance.y); if (!mIgnoreImGuiChild) ImGui::EndChild(); if (mShowLineNumbers && !mIgnoreImGuiChild) { ImGui::BeginChild("##lineNumbers"); + ImGui::SetCursorScreenPos(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y)); ImGui::Dummy(ImVec2(mLineNumberFieldWidth, (globalLineMax - lineMax - 1) * mCharAdvance.y + ImGui::GetCurrentWindow()->InnerClipRect.GetHeight() - mCharAdvance.y)); ImGui::EndChild(); } if (!mIgnoreImGuiChild) ImGui::BeginChild(aTitle); + ImGui::SetCursorScreenPos(lineStartScreenPos); if (mShowLineNumbers) - ImGui::Dummy(ImVec2(mLongestLineLength * mCharAdvance.x, (globalLineMax - lineMax - 2.0F) * mCharAdvance.y + ImGui::GetCurrentWindow()->InnerClipRect.GetHeight())); + ImGui::Dummy(ImVec2(mLongestLineLength * mCharAdvance.x + mCharAdvance.x, (globalLineMax - lineMax - 2.0F) * mCharAdvance.y + ImGui::GetCurrentWindow()->InnerClipRect.GetHeight())); else - ImGui::Dummy(ImVec2(mLongestLineLength * mCharAdvance.x, (globalLineMax - 1.0f - lineMax + GetPageSize() - 1.0f ) * mCharAdvance.y - 2 * ImGuiStyle().WindowPadding.y)); + ImGui::Dummy(ImVec2(mLongestLineLength * mCharAdvance.x + mCharAdvance.x, (globalLineMax - lineMax - 3.0F) * mCharAdvance.y + ImGui::GetCurrentWindow()->InnerClipRect.GetHeight() - 1.0f)); if (mScrollToCursor) EnsureCursorVisible(); @@ -1224,6 +1151,9 @@ void TextEditor::Render(const char *aTitle, const ImVec2 &aSize, bool aBorder) { mWithinRender = true; mCursorPositionChanged = false; + if (mLines.capacity() < 2*mLines.size()) + mLines.reserve(2*mLines.size()); + auto scrollBg = ImGui::GetStyleColorVec4(ImGuiCol_ScrollbarBg); scrollBg.w = 0.0f; auto scrollBarSize = ImGui::GetStyle().ScrollbarSize; @@ -1250,9 +1180,11 @@ void TextEditor::Render(const char *aTitle, const ImVec2 &aSize, bool aBorder) { ImVec2 textEditorSize = aSize; textEditorSize.x -= mLineNumberFieldWidth; - bool scroll_x = mLongestLineLength * mCharAdvance.x > textEditorSize.x; + + bool scroll_x = mLongestLineLength * mCharAdvance.x >= textEditorSize.x; + bool scroll_y = mLines.size() > 1; - if (!aBorder && scroll_y) + if (!aBorder) textEditorSize.x -= scrollBarSize; ImGui::SetCursorScreenPos(ImVec2(position.x + mLineNumberFieldWidth, position.y)); ImGuiChildFlags childFlags = aBorder ? ImGuiChildFlags_Borders : ImGuiChildFlags_None; @@ -1280,6 +1212,7 @@ void TextEditor::Render(const char *aTitle, const ImVec2 &aSize, bool aBorder) { if (mHandleMouseInputs) HandleMouseInputs(); + ColorizeInternal(); RenderText(aTitle, position, textEditorSize); @@ -1303,13 +1236,15 @@ void TextEditor::SetText(const std::string &aText, bool aUndo) { u.mRemovedEnd = Coordinates((int) mLines.size() - 1, GetLineMaxColumn((int) mLines.size() - 1)); } mLines.resize(1); - mLines[0].clear(); + mLines[0] = Line(); + mLines[0].mColorized = false; std::string text = PreprocessText(aText); for (auto chr : text) { - if (chr == '\n') + if (chr == '\n') { mLines.push_back(Line()); - else - mLines.back().push_back(Glyph(chr, PaletteIndex::Default)); + mLines.back().mColorized = false; + } else + mLines.back().push_back(Glyph(chr)); } if (!mReadOnly && aUndo) { u.mAdded = text; @@ -1330,23 +1265,21 @@ void TextEditor::SetText(const std::string &aText, bool aUndo) { void TextEditor::SetTextLines(const std::vector &aLines) { mLines.clear(); - if (isEmpty()) { + if (aLines.empty()) { mLines.emplace_back(Line()); + mLines[0].mColorized = false; } else { - mLines.resize(aLines.size()); - - for (size_t i = 0; i < aLines.size(); ++i) { - const std::string &aLine = PreprocessText(aLines[i]); - - mLines[i].reserve(aLine.size()); - for (size_t j = 0; j < aLine.size(); ++j) - mLines[i].emplace_back(Glyph(aLine[j], PaletteIndex::Default)); - } + auto lineCount = aLines.size(); + mLines.resize(lineCount); + for (size_t i = 0; i < lineCount; i++) { + mLines[i].SetLine(PreprocessText(aLines[i])); + mLines[i].mColorized = false; + } } mTextChanged = true; mScrollToTop = true; - + SetTimeStamp(0); mUndoBuffer.clear(); mUndoIndex = 0; @@ -1355,7 +1288,6 @@ void TextEditor::SetTextLines(const std::vector &aLines) { void TextEditor::EnterCharacter(ImWchar aChar, bool aShift) { IM_ASSERT(!mReadOnly); - UndoRecord u; u.mBefore = mState; @@ -1389,19 +1321,21 @@ void TextEditor::EnterCharacter(ImWchar aChar, bool aShift) { auto &line = mLines[i]; if (aShift) { if (!line.empty()) { - if (line.front().mChar == '\t') { + if (line.front() == '\t') { line.erase(line.begin()); modified = true; } else { - for (int j = 0; j < mTabSize && !line.empty() && line.front().mChar == ' '; j++) { + for (int j = 0; j < mTabSize && !line.empty() && line.front() == ' '; j++) { line.erase(line.begin()); modified = true; } } } } else { - for (int j = start.mColumn % mTabSize; j < mTabSize; j++) - line.insert(line.begin(), Glyph(' ', PaletteIndex::Background)); + auto spacesToInsert = mTabSize - (start.mColumn % mTabSize); + std::string spaces(spacesToInsert, ' '); + line.insert(line.begin(), spaces.begin(), spaces.end()); + line.mColorized = false; modified = true; } } @@ -1455,7 +1389,7 @@ void TextEditor::EnterCharacter(ImWchar aChar, bool aShift) { auto &newLine = mLines[coord.mLine + 1]; if (mLanguageDefinition.mAutoIndentation) - for (size_t it = 0; it < line.size() && isascii(line[it].mChar) && isblank(line[it].mChar); ++it) + for (size_t it = 0; it < line.size() && isascii(line[it]) && isblank(line[it]); ++it) newLine.push_back(line[it]); const size_t whitespaceSize = newLine.size(); @@ -1470,7 +1404,8 @@ void TextEditor::EnterCharacter(ImWchar aChar, bool aShift) { cpos = (int) whitespaceSize; } newLine.insert(newLine.end(), line.begin() + cstart, line.end()); - line.erase(line.begin() + cstart, line.begin() + line.size()); + line.erase(line.begin() + cstart,-1); + line.mColorized = false; SetCursorPosition(Coordinates(coord.mLine + 1, GetCharacterColumn(coord.mLine + 1, cpos))); u.mAdded = (char)aChar; } else if (aChar == '\t') { @@ -1479,60 +1414,57 @@ void TextEditor::EnterCharacter(ImWchar aChar, bool aShift) { if (!aShift) { auto spacesToInsert = mTabSize - (cindex % mTabSize); - for (int j = 0; j < spacesToInsert; j++) - line.insert(line.begin() + cindex, Glyph(' ', PaletteIndex::Background)); + std::string spaces(spacesToInsert, ' '); + line.insert(line.begin() + cindex, spaces.begin(), spaces.end()); + line.mColorized = false; SetCursorPosition(Coordinates(coord.mLine, GetCharacterColumn(coord.mLine, cindex + spacesToInsert))); } else { auto spacesToRemove = (cindex % mTabSize); if (spacesToRemove == 0) spacesToRemove = mTabSize; spacesToRemove = std::min(spacesToRemove, (int32_t) line.size()); for (int j = 0; j < spacesToRemove; j++) { - if ((line.begin() + cindex - 1)->mChar == ' ') { - line.erase(line.begin() + cindex - 1); + if (*(line.begin() + (cindex - 1)) == ' ') { + line.erase(line.begin() + (cindex - 1)); cindex -= 1; } } - + line.mColorized = false; SetCursorPosition(Coordinates(coord.mLine, GetCharacterColumn(coord.mLine, std::max(0, cindex)))); } } else { - char buf[7]; - int e = ImTextCharToUtf8(buf, 7, aChar); - if (e > 0) { - buf[e] = '\0'; + std::string buf = ""; + ImTextCharToUtf8(buf, aChar); + if (buf.size() > 0) { auto &line = mLines[coord.mLine]; auto cindex = GetCharacterIndex(coord); if (mOverwrite && cindex < (int)line.size()) { - auto d = UTF8CharLength(line[cindex].mChar); + auto d = UTF8CharLength(line[cindex]); u.mRemovedStart = mState.mCursorPosition; - u.mRemovedEnd = Coordinates(coord.mLine, GetCharacterColumn(coord.mLine, cindex + d)); - - while (d-- > 0 && cindex < (int)line.size()) { - u.mRemoved += line[cindex].mChar; - line.erase(line.begin() + cindex); - } + u.mRemovedEnd = Coordinates(coord.mLine, GetCharacterColumn(coord.mLine, cindex + d)); + u.mRemoved = std::string(line.mChars.begin() + cindex, line.mChars.begin() + cindex + d); + line.erase(line.begin() + cindex, d); + line.mColorized = false; } - - for (auto p = buf; *p != '\0'; p++, ++cindex) - line.insert(line.begin() + cindex, Glyph(*p, PaletteIndex::Default)); + line.insert(line.begin() + cindex, buf.begin(), buf.end()); + line.mColorized = false; u.mAdded = buf; - - SetCursorPosition(Coordinates(coord.mLine, GetCharacterColumn(coord.mLine, cindex))); + auto charCount = GetStringCharacterCount(buf); + SetCursorPosition(Coordinates(coord.mLine, GetCharacterColumn(coord.mLine, cindex + charCount))); } else return; } - mTextChanged = true; - u.mAddedEnd = GetActualCursorCoordinates(); u.mAfter = mState; + mTextChanged = true; + AddUndo(u); - Colorize(coord.mLine - 1, 3); + Colorize(); std::string findWord = mFindReplaceHandler.GetFindWord(); if (!findWord.empty()) { @@ -1603,10 +1535,10 @@ void TextEditor::InsertText(const char *aValue) { auto pos = GetActualCursorCoordinates(); auto start = std::min(pos, mState.mSelectionStart); - int totalLines = pos.mLine - start.mLine; auto text = PreprocessText(aValue); - totalLines += InsertTextAt(pos, text); + InsertTextAt(pos, text); + mLines[pos.mLine].mColorized = false; SetSelection(pos, pos); SetCursorPosition(pos); @@ -1616,7 +1548,7 @@ void TextEditor::InsertText(const char *aValue) { mFindReplaceHandler.resetMatches(); mFindReplaceHandler.FindAllMatches(this, findWord); } - Colorize(start.mLine - 1, totalLines + 2); + Colorize(); } void TextEditor::DeleteSelection() { @@ -1634,7 +1566,7 @@ void TextEditor::DeleteSelection() { mFindReplaceHandler.resetMatches(); mFindReplaceHandler.FindAllMatches(this, findWord); } - Colorize(mState.mSelectionStart.mLine, 1); + Colorize(); } void TextEditor::JumpToLine(int line) { @@ -1742,21 +1674,18 @@ void TextEditor::MoveLeft(int aAmount, bool aSelect, bool aWordMode) { else cindex = 0; } + } else if (aWordMode) { + mState.mCursorPosition = FindPreviousWord(mState.mCursorPosition); + cindex = GetCharacterIndex(mState.mCursorPosition); } else { --cindex; if (cindex > 0) { if ((int)mLines.size() > lindex) { - while (cindex > 0 && IsUTFSequence(line[cindex].mChar)) + while (cindex > 0 && IsUTFSequence(line[cindex])) --cindex; } } } - - mState.mCursorPosition = Coordinates(lindex, GetCharacterColumn(lindex, cindex)); - if (aWordMode) { - mState.mCursorPosition = FindWordStart(mState.mCursorPosition); - cindex = GetCharacterIndex(mState.mCursorPosition); - } } mState.mCursorPosition = Coordinates(lindex, GetCharacterColumn(lindex, cindex)); @@ -1773,6 +1702,7 @@ void TextEditor::MoveLeft(int aAmount, bool aSelect, bool aWordMode) { } } else mInteractiveStart = mInteractiveEnd = mState.mCursorPosition; + SetSelection(mInteractiveStart, mInteractiveEnd); EnsureCursorVisible(); @@ -1797,22 +1727,18 @@ void TextEditor::MoveRight(int aAmount, bool aSelect, bool aWordMode) { ++lindex; cindex = 0; } + } else if (aWordMode) { + mState.mCursorPosition = FindNextWord(mState.mCursorPosition); + cindex = GetCharacterIndex(mState.mCursorPosition); } else { ++cindex; if (cindex < (int)line.size()) { if ((int)mLines.size() > lindex) { - while (cindex < (int)line.size() && IsUTFSequence(line[cindex].mChar)) + while (cindex < (int)line.size() && IsUTFSequence(line[cindex])) ++cindex; } } } - - mState.mCursorPosition = Coordinates(lindex, GetCharacterColumn(lindex, cindex)); - - if (aWordMode) { - mState.mCursorPosition = FindWordEnd(mState.mCursorPosition); - cindex = GetCharacterIndex(mState.mCursorPosition); - } } mState.mCursorPosition = Coordinates(lindex, GetCharacterColumn(lindex, cindex)); @@ -1829,6 +1755,7 @@ void TextEditor::MoveRight(int aAmount, bool aSelect, bool aWordMode) { } } else mInteractiveStart = mInteractiveEnd = mState.mCursorPosition; + SetSelection(mInteractiveStart, mInteractiveEnd); EnsureCursorVisible(); @@ -1865,20 +1792,49 @@ void TextEditor::TextEditor::MoveBottom(bool aSelect) { void TextEditor::MoveHome(bool aSelect) { ResetCursorBlinkTime(); auto oldPos = mState.mCursorPosition; - auto &line = mLines[mState.mCursorPosition.mLine]; - if (line.size() == 0) + + auto &line = mLines[oldPos.mLine]; + auto prefix = line.substr(0, oldPos.mColumn); + auto postfix = line.substr(oldPos.mColumn); + if (prefix.empty() && postfix.empty()) return; - - auto limit = oldPos.mColumn != 0 ? oldPos.mColumn : line.size(); - auto home=0; - while (home < limit && isspace(line[home].mChar)) - home++; - - if (home == oldPos.mColumn) { - home = 0; + if (!prefix.empty()) { + auto idx = prefix.find_first_not_of(" "); + if (idx == std::string::npos) { + auto postIdx = postfix.find_first_of(" "); + if (postIdx == std::string::npos || postIdx == 0) + home=0; + else { + postIdx = postfix.find_first_not_of(" "); + if (postIdx == std::string::npos) + home = GetLineMaxColumn(oldPos.mLine); + else if (postIdx == 0) + home = 0; + else + home = oldPos.mColumn + postIdx; + } + } else + home = idx; + } else { + auto postIdx = postfix.find_first_of(" "); + if (postIdx == std::string::npos) + home = 0; + else { + postIdx = postfix.find_first_not_of(" "); + if (postIdx == std::string::npos) + home = GetLineMaxColumn(oldPos.mLine); + else + home = oldPos.mColumn + postIdx; + } } + + + //while (home < line.size() && isspace(line[home])) + // home++; + //if (home == oldPos.mColumn) + // home = 0; SetCursorPosition(Coordinates(mState.mCursorPosition.mLine, home)); if (mState.mCursorPosition != oldPos) { if (aSelect) { @@ -1931,7 +1887,6 @@ void TextEditor::Delete() { u.mRemoved = GetSelectedText(); u.mRemovedStart = mState.mSelectionStart; u.mRemovedEnd = mState.mSelectionEnd; - DeleteSelection(); } else { auto pos = GetActualCursorCoordinates(); @@ -1948,21 +1903,23 @@ void TextEditor::Delete() { auto &nextLine = mLines[pos.mLine + 1]; line.insert(line.end(), nextLine.begin(), nextLine.end()); + line.mColorized = false; RemoveLine(pos.mLine + 1); + } else { auto cindex = GetCharacterIndex(pos); u.mRemovedStart = u.mRemovedEnd = GetActualCursorCoordinates(); u.mRemovedEnd.mColumn++; u.mRemoved = GetText(u.mRemovedStart, u.mRemovedEnd); - auto d = UTF8CharLength(line[cindex].mChar); - while (d-- > 0 && cindex < (int)line.size()) - line.erase(line.begin() + cindex); + auto d = UTF8CharLength(line[cindex]); + line.erase(line.begin() + cindex, d); + line.mColorized = false; } mTextChanged = true; - Colorize(pos.mLine, 1); + Colorize(); } u.mAfter = mState; @@ -1976,9 +1933,7 @@ void TextEditor::Delete() { void TextEditor::Backspace() { ResetCursorBlinkTime(); - IM_ASSERT(!mReadOnly); - - if (isEmpty()) + if (isEmpty() || mReadOnly) return; UndoRecord u; @@ -1988,54 +1943,57 @@ void TextEditor::Backspace() { u.mRemoved = GetSelectedText(); u.mRemovedStart = mState.mSelectionStart; u.mRemovedEnd = mState.mSelectionEnd; - DeleteSelection(); } else { auto pos = GetActualCursorCoordinates(); SetCursorPosition(pos); + auto &line = mLines[pos.mLine]; - if (mState.mCursorPosition.mColumn == 0) { - if (mState.mCursorPosition.mLine == 0) + if (pos.mColumn == 0) { + if (pos.mLine == 0) return; u.mRemoved = '\n'; u.mRemovedStart = u.mRemovedEnd = Coordinates(pos.mLine - 1, GetLineMaxColumn(pos.mLine - 1)); Advance(u.mRemovedEnd); - auto &line = mLines[mState.mCursorPosition.mLine]; - auto &prevLine = mLines[mState.mCursorPosition.mLine - 1]; - auto prevSize = GetLineMaxColumn(mState.mCursorPosition.mLine - 1); - prevLine.insert(prevLine.end(), line.begin(), line.end()); + auto &prevLine = mLines[pos.mLine - 1]; + auto prevSize = GetLineMaxColumn(pos.mLine - 1); + if (prevSize == 0) + prevLine = line; + else + prevLine.insert(prevLine.end(), line.begin(), line.end()); + prevLine.mColorized = false; + ErrorMarkers etmp; for (auto &i : mErrorMarkers) etmp.insert(ErrorMarkers::value_type(i.first.mLine - 1 == mState.mCursorPosition.mLine ? Coordinates(i.first.mLine - 1,i.first.mColumn) : i.first, i.second)); mErrorMarkers = std::move(etmp); - RemoveLine(mState.mCursorPosition.mLine); --mState.mCursorPosition.mLine; mState.mCursorPosition.mColumn = prevSize; } else { - auto &line = mLines[mState.mCursorPosition.mLine]; auto cindex = GetCharacterIndex(pos) - 1; auto cend = cindex + 1; - while (cindex > 0 && IsUTFSequence(line[cindex].mChar)) + while (cindex > 0 && IsUTFSequence(line[cindex])) --cindex; u.mRemovedStart = u.mRemovedEnd = GetActualCursorCoordinates(); --u.mRemovedStart.mColumn; mState.mCursorPosition.mColumn = GetCharacterColumn(mState.mCursorPosition.mLine, cindex); - - while (cindex < line.size() && cend-- > cindex) { - u.mRemoved += line[cindex].mChar; - line.erase(line.begin() + cindex); - } + u.mRemoved = GetText(u.mRemovedStart, u.mRemovedEnd); + if (cend > cindex && cend < (int) line.size()) + line.erase(line.begin() + cindex, cend-cindex); + else + line.erase(line.begin() + cindex, -1); + line.mColorized = false; } mTextChanged = true; EnsureCursorVisible(); - Colorize(mState.mCursorPosition.mLine, 1); + Colorize(); } u.mAfter = mState; @@ -2066,9 +2024,8 @@ void TextEditor::Copy() { } else { if (!isEmpty()) { std::string str; - auto &line = mLines[GetActualCursorCoordinates().mLine]; - for (auto &g : line) - str.push_back(g.mChar); + const auto &line = mLines[GetActualCursorCoordinates().mLine]; + std::copy(line.mChars.begin(), line.mChars.end(), std::back_inserter(str)); ImGui::SetClipboardText(str.c_str()); } } @@ -2078,19 +2035,23 @@ void TextEditor::Cut() { if (IsReadOnly()) { Copy(); } else { - if (HasSelection()) { - UndoRecord u; - u.mBefore = mState; - u.mRemoved = GetSelectedText(); - u.mRemovedStart = mState.mSelectionStart; - u.mRemovedEnd = mState.mSelectionEnd; - - Copy(); - DeleteSelection(); - - u.mAfter = mState; - AddUndo(u); + if (!HasSelection()) { + auto lineIndex = GetActualCursorCoordinates().mLine; + if (lineIndex < 0 || lineIndex >= (int)mLines.size()) + return; + SetSelection(Coordinates(lineIndex, 0), Coordinates(lineIndex+1, 0)); } + UndoRecord u; + u.mBefore = mState; + u.mRemoved = GetSelectedText(); + u.mRemovedStart = mState.mSelectionStart; + u.mRemovedEnd = mState.mSelectionEnd; + + Copy(); + DeleteSelection(); + + u.mAfter = mState; + AddUndo(u); } std::string findWord = mFindReplaceHandler.GetFindWord(); if (!findWord.empty()) { @@ -2100,16 +2061,27 @@ void TextEditor::Cut() { } std::string TextEditor::ReplaceStrings(std::string string, const std::string &search, const std::string &replace) { - if (search.empty()) + std::string result; + if (string.empty()) return string; - - std::size_t pos = 0; - while ((pos = string.find(search, pos)) != std::string::npos) { - string.replace(pos, search.size(), replace); - pos += replace.size(); + if (search.empty()) { + for (auto c: string) { + if (c == '\0') { + result.append(replace); + } else { + result.push_back(c); + } + } + } else { + result = string; + std::size_t pos = 0; + while ((pos = result.find(search, pos)) != std::string::npos) { + result.replace(pos, search.size(), replace); + pos += replace.size(); + } } - return string; + return result; } std::vector TextEditor::SplitString(const std::string &string, const std::string &delimiter, bool removeEmpty) { @@ -2152,7 +2124,7 @@ std::string TextEditor::ReplaceTabsWithSpaces(const std::string& string, uint32_ while ((pos = line.find('\t', pos)) != std::string::npos) { auto spaces = tabSize - (pos % tabSize); line.replace(pos, 1, std::string(spaces, ' ')); - pos += tabSize - 1; + pos += spaces - 1; } result += line; if (i < size - 1) @@ -2165,6 +2137,7 @@ std::string TextEditor::ReplaceTabsWithSpaces(const std::string& string, uint32_ std::string TextEditor::PreprocessText(const std::string &code) { std::string result = ReplaceStrings(code, "\r\n", "\n"); result = ReplaceStrings(result, "\r", "\n"); + result = ReplaceStrings(result, "\000", "."); result = ReplaceTabsWithSpaces(result, 4); return result; @@ -2175,27 +2148,28 @@ void TextEditor::Paste() { return; auto clipText = ImGui::GetClipboardText(); - if (clipText != nullptr && strlen(clipText) > 0) { - std::string text = PreprocessText(clipText); + if (clipText != nullptr ) { + auto len = strlen(clipText); + if (len > 0 ) { + std::string text = PreprocessText(clipText); + UndoRecord u; + u.mBefore = mState; - UndoRecord u; - u.mBefore = mState; + if (HasSelection()) { + u.mRemoved = GetSelectedText(); + u.mRemovedStart = mState.mSelectionStart; + u.mRemovedEnd = mState.mSelectionEnd; + DeleteSelection(); + } - if (HasSelection()) { - u.mRemoved = GetSelectedText(); - u.mRemovedStart = mState.mSelectionStart; - u.mRemovedEnd = mState.mSelectionEnd; - DeleteSelection(); + u.mAdded = text; + u.mAddedStart = GetActualCursorCoordinates(); + InsertText(text); + + u.mAddedEnd = GetActualCursorCoordinates(); + u.mAfter = mState; + AddUndo(u); } - - u.mAdded = text; - u.mAddedStart = GetActualCursorCoordinates(); - - InsertText(text); - - u.mAddedEnd = GetActualCursorCoordinates(); - u.mAfter = mState; - AddUndo(u); } std::string findWord = mFindReplaceHandler.GetFindWord(); if (!findWord.empty()) { @@ -2239,7 +2213,7 @@ void TextEditor::FindReplaceHandler::SelectFound(TextEditor *editor, int index) auto selectionEnd = mMatches[index].mSelectionEnd; editor->SetSelection(selectionStart, selectionEnd); editor->SetCursorPosition(selectionEnd); - editor->mScrollToCursor = true; + editor->EnsureCursorVisible(); } // The returned index is shown in the form @@ -2366,8 +2340,7 @@ std::string make_wholeWord(const std::string &s) { bool TextEditor::FindReplaceHandler::FindNext(TextEditor *editor) { Coordinates curPos; curPos.mLine = mMatches.empty() ? editor->mState.mCursorPosition.mLine : mMatches.back().mCursorPosition.mLine; - curPos.mColumn = mMatches.empty() ? editor->mState.mCursorPosition.mColumn : editor->Utf8CharsToBytes( - mMatches.back().mCursorPosition); + curPos.mColumn = mMatches.empty() ? editor->mState.mCursorPosition.mColumn : editor->Utf8CharsToBytes(mMatches.back().mCursorPosition); unsigned long matchLength = editor->GetStringCharacterCount(mFindWord); size_t byteIndex = 0; @@ -2493,7 +2466,6 @@ void TextEditor::FindReplaceHandler::FindAllMatches(TextEditor *editor,std::stri bool TextEditor::FindReplaceHandler::Replace(TextEditor *editor, bool next) { - if (mMatches.empty() || mFindWord == mReplaceWord || mFindWord.empty()) return false; @@ -2519,7 +2491,6 @@ bool TextEditor::FindReplaceHandler::Replace(TextEditor *editor, bool next) { u.mRemoved = editor->GetSelectedText(); u.mRemovedStart = editor->mState.mSelectionStart; u.mRemovedEnd = editor->mState.mSelectionEnd; - editor->DeleteSelection(); if (GetFindRegEx()) { std::string replacedText = std::regex_replace(editor->GetText(), std::regex(mFindWord), mReplaceWord, std::regex_constants::format_first_only | std::regex_constants::format_no_copy); @@ -2528,8 +2499,8 @@ bool TextEditor::FindReplaceHandler::Replace(TextEditor *editor, bool next) { u.mAdded = mReplaceWord; u.mAddedStart = editor->GetActualCursorCoordinates(); - editor->InsertText(u.mAdded); + editor->SetCursorPosition(editor->mState.mSelectionEnd); u.mAddedEnd = editor->GetActualCursorCoordinates(); @@ -2654,7 +2625,7 @@ const TextEditor::Palette &TextEditor::GetRetroBluePalette() { std::string TextEditor::GetText() const { - return GetText(Coordinates(), Coordinates((int)mLines.size(), 0)); + return GetText(Coordinates(), Coordinates(-1, -1)); } std::vector TextEditor::GetTextLines() const { @@ -2663,13 +2634,7 @@ std::vector TextEditor::GetTextLines() const { result.reserve(mLines.size()); for (auto &line : mLines) { - std::string text; - - text.resize(line.size()); - - for (size_t i = 0; i < line.size(); ++i) - text[i] = line[i].mChar; - + std::string text = line.mChars; result.emplace_back(std::move(text)); } @@ -2692,102 +2657,136 @@ std::string TextEditor::GetLineText(int line) const { void TextEditor::ProcessInputs() { } -void TextEditor::Colorize(int aFromLine, int aLines) { - int toLine = aLines == -1 ? (int)mLines.size() : std::min((int)mLines.size(), aFromLine + aLines); - mColorRangeMin = std::min(mColorRangeMin, aFromLine); - mColorRangeMax = std::max(mColorRangeMax, toLine); - mColorRangeMin = std::max(0, mColorRangeMin); - mColorRangeMax = std::max(mColorRangeMin, mColorRangeMax); - mCheckComments = true; +void TextEditor::Colorize() { + mUpdateFlags = true; } -void TextEditor::ColorizeRange(int aFromLine, int aToLine) { - if (isEmpty() || aFromLine >= aToLine) +void TextEditor::ColorizeRange() { + + if (isEmpty()) return; - std::string buffer; - std::cmatch results; + std::smatch results; std::string id; - - int endLine = std::max(0, std::min((int)mLines.size(), aToLine)); - for (int i = aFromLine; i < endLine; ++i) { + if (mLanguageDefinition.mTokenize == nullptr) + mLanguageDefinition.mTokenize = []( strConstIter, strConstIter, strConstIter &, strConstIter &, PaletteIndex &) { return false; }; + auto linesSize = mLines.size(); + for (int i = 0; i < linesSize; ++i) { auto &line = mLines[i]; + auto size = line.size(); - if (line.empty()) - continue; - - buffer.resize(line.size()); - for (size_t j = 0; j < line.size(); ++j) { - auto &col = line[j]; - buffer[j] = col.mChar; - col.mColorIndex = PaletteIndex::Default; + if (line.mColors.size() != size) { + line.mColors.resize(size, 0); + line.mColorized = false; } - const char *bufferBegin = &buffer.front(); - const char *bufferEnd = bufferBegin + buffer.size(); + if (line.mColorized || line.empty()) + continue; - auto last = bufferEnd; + auto last = line.end(); - for (auto first = bufferBegin; first != last;) { - const char *token_begin = nullptr; - const char *token_end = nullptr; + auto first = line.begin(); + for (auto current = first;(current-first) < size;) { + strConstIter token_begin; + strConstIter token_end; PaletteIndex token_color = PaletteIndex::Default; - bool hasTokenizeResult = false; - - if (mLanguageDefinition.mTokenize != nullptr) { - if (mLanguageDefinition.mTokenize(first, last, token_begin, token_end, token_color)) - hasTokenizeResult = true; - } + bool hasTokenizeResult = mLanguageDefinition.mTokenize(current.mCharsIter, last.mCharsIter, token_begin, token_end, token_color); + auto token_offset = token_begin - first.mCharsIter; if (!hasTokenizeResult) { // todo : remove // printf("using regex for %.*s\n", first + 10 < last ? 10 : int(last - first), first); for (auto &p : mRegexList) { - if (std::regex_search(first, last, results, p.first, std::regex_constants::match_continuous)) { + if (std::regex_search(first.mCharsIter, last.mCharsIter, results, p.first, std::regex_constants::match_continuous)) { hasTokenizeResult = true; - auto &v = *results.begin(); - token_begin = v.first; - token_end = v.second; + const auto &v = results.begin(); + token_begin = v->first; + token_end = v->second; token_color = p.second; break; } } } - if (hasTokenizeResult == false) { - first++; - } else { - const size_t token_length = token_end - token_begin; + if (!hasTokenizeResult) + current=current + 1; + else { + current = first + token_offset; + size_t token_length=0; + Line::Flags flags(0); + flags.mValue = line.mFlags[token_offset]; + if (flags.mValue == 0) { + token_length = token_end - token_begin; + if (token_color == PaletteIndex::Identifier) { + id.assign(token_begin, token_end); - if (token_color == PaletteIndex::Identifier) { - id.assign(token_begin, token_end); - - // todo : allmost all language definitions use lower case to specify keywords, so shouldn't this use ::tolower ? - if (!mLanguageDefinition.mCaseSensitive) - std::transform(id.begin(), id.end(), id.begin(), ::toupper); - - if (!line[first - bufferBegin].mPreprocessor) { - if (mLanguageDefinition.mKeywords.count(id) != 0) + // todo : almost all language definitions use lower case to specify keywords, so shouldn't this use ::tolower ? + if (!mLanguageDefinition.mCaseSensitive) + std::transform(id.begin(), id.end(), id.begin(), ::toupper); + else if (mLanguageDefinition.mKeywords.count(id) != 0) token_color = PaletteIndex::Keyword; else if (mLanguageDefinition.mIdentifiers.count(id) != 0) - token_color = PaletteIndex::KnownIdentifier; - else if (mLanguageDefinition.mPreprocIdentifiers.count(id) != 0) - token_color = PaletteIndex::PreprocIdentifier; - } else { - if (mLanguageDefinition.mPreprocIdentifiers.count(id) != 0) + token_color = PaletteIndex::BuiltInType; + else if (id == "$") + token_color = PaletteIndex::GlobalVariable; + } + } else { + if ((token_color == PaletteIndex::Identifier || flags.mBits.mPreprocessor) && !flags.mBits.mDeactivated && !(flags.mValue & Line::InComment)) { + id.assign(token_begin, token_end); + if (mLanguageDefinition.mPreprocIdentifiers.count(id) != 0) { + token_color = PaletteIndex::Directive; + token_begin -= 1; + token_length = token_end - token_begin; + token_offset -= 1; + } else if (flags.mBits.mPreprocessor) { token_color = PaletteIndex::PreprocIdentifier; + token_length = token_end - token_begin; + } + } + if ((token_color != PaletteIndex::Directive && token_color != PaletteIndex::PreprocIdentifier) || flags.mBits.mDeactivated) { + if (flags.mBits.mDeactivated && flags.mBits.mPreprocessor) { + token_color = PaletteIndex::PreprocessorDeactivated; + token_begin -= 1; + token_offset -= 1; + } else if (id.assign(token_begin, token_end);flags.mValue & Line::InComment && mLanguageDefinition.mPreprocIdentifiers.count(id) != 0) { + token_color = GetColorIndexFromFlags(flags); + token_begin -= 1; + token_offset -= 1; + } + + auto flag = line.mFlags[token_offset]; + token_length = line.mFlags.find_first_not_of(flag, token_offset + 1); + if (token_length == std::string::npos) + token_length = line.size() - token_offset; + else + token_length -= token_offset; + + token_end = token_begin + token_length; + if (!flags.mBits.mPreprocessor || flags.mBits.mDeactivated) + token_color = GetColorIndexFromFlags(flags); } } - for (size_t j = 0; j < token_length; ++j) - line[(token_begin - bufferBegin) + j].mColorIndex = token_color; - - first = token_end; + if (token_color != PaletteIndex::Identifier || *current.mColorsIter == static_cast(PaletteIndex::Identifier)) { + if (token_offset + token_length >= (int)line.mColors.size()) { + auto colors = line.mColors; + line.mColors.resize(token_offset + token_length, static_cast(PaletteIndex::Default)); + std::copy(colors.begin(), colors.end(), line.mColors.begin()); + } + try { + line.mColors.replace(token_offset, token_length, token_length, static_cast(token_color)); + } catch (const std::exception &e) { + std::cerr << "Error replacing color: " << e.what() << std::endl; + return; + } + } + current = current + token_length; } } + line.mColorized = true; } } @@ -2795,218 +2794,197 @@ void TextEditor::ColorizeInternal() { if (isEmpty() || !mColorizerEnabled) return; - if (mCheckComments) { - auto endLine = mLines.size(); - auto endIndex = 0; - auto commentStartLine = endLine; - auto commentStartIndex = endIndex; - auto withinGlobalDocComment = false; - auto withinDocComment = false; - auto withinComment = false; - auto withinString = false; - auto withinSingleLineComment = false; - auto withinPreproc = false; - auto withinNotDef = false; - auto firstChar = true; // there is no other non-whitespace characters in the line before - auto currentLine = 0; - auto currentIndex = 0; - auto commentLength = 0; - auto &startStr = mLanguageDefinition.mCommentStart; - auto &singleStartStr = mLanguageDefinition.mSingleLineComment; - auto &docStartStr = mLanguageDefinition.mDocComment; - auto &globalStartStr = mLanguageDefinition.mGlobalDocComment; + if (mUpdateFlags) { + auto endLine = mLines.size(); + auto commentStartLine = endLine; + auto commentStartIndex = 0; + auto withinGlobalDocComment = false; + auto withinBlockDocComment = false; + auto withinString = false; + auto withinBlockComment = false; + auto withinNotDef = false; + auto currentLine = 0; + auto commentLength = 0; std::vector ifDefs; ifDefs.push_back(true); - - while (currentLine < endLine || currentIndex < endIndex) { + mDefines.push_back("__IMHEX__"); + for (currentLine = 0; currentLine < endLine; currentLine++) { auto &line = mLines[currentLine]; + auto lineLength = line.size(); - auto setGlyphFlags = [&](int index) { - line[index].mMultiLineComment = withinComment; - line[index].mComment = withinSingleLineComment; - line[index].mDocComment = withinDocComment; - line[index].mGlobalDocComment = withinGlobalDocComment; - line[index].mDeactivated = withinNotDef; - }; - - if (currentIndex == 0) { - withinSingleLineComment = false; - withinPreproc = false; - firstChar = true; + if (line.mFlags.size() != lineLength) { + line.mFlags.resize(lineLength, 0); + line.mColorized = false; } + //if (!line.mColorized) { - if (!line.empty()) { - auto &g = line[currentIndex]; - auto c = g.mChar; + auto withinComment = false; + auto withinDocComment = false; + auto withinPreproc = false; + auto firstChar = true; // there is no other non-whitespace characters in the line before - if (c != mLanguageDefinition.mPreprocChar && !isspace(c)) - firstChar = false; + auto setGlyphFlags = [&](int index) { + Line::Flags flags(0); + flags.mBits.mComment = withinComment; + flags.mBits.mBlockComment = withinBlockComment; + flags.mBits.mDocComment = withinDocComment; + flags.mBits.mGlobalDocComment = withinGlobalDocComment; + flags.mBits.mBlockDocComment = withinBlockDocComment; + flags.mBits.mDeactivated = withinNotDef; + if (mLines[currentLine].mFlags[index] != flags.mValue) { + mLines[currentLine].mColorized = false; + mLines[currentLine].mFlags[index] = flags.mValue; + } + }; - bool inComment = (commentStartLine < currentLine || (commentStartLine == currentLine && commentStartIndex <= currentIndex)); + auto currentIndex = 0; + if (line.empty()) + continue; + while (currentIndex < lineLength) { - if (withinString) { - setGlyphFlags(currentIndex); - if (c == '\\') { - currentIndex++; + auto &g = line[currentIndex]; + auto c = g; + + if (c != mLanguageDefinition.mPreprocChar && !isspace(c)) + firstChar = false; + + bool inComment = (commentStartLine < currentLine || (commentStartLine == currentLine && commentStartIndex <= currentIndex)); + + if (withinString) { setGlyphFlags(currentIndex); - } else if (c == '\"') - withinString = false; - } else { - if (firstChar && c == mLanguageDefinition.mPreprocChar) { - withinPreproc = true; - std::string directive; - auto start = currentIndex + 1; - while (start < (int) line.size() && !isspace(line[start].mChar)) { - directive += line[start].mChar; - start++; - } + if (c == '\\') { + currentIndex++; + setGlyphFlags(currentIndex); + } else if (c == '\"') + withinString = false; + } else { + if (firstChar && c == mLanguageDefinition.mPreprocChar && !inComment && !withinComment && !withinDocComment && !withinString) { + withinPreproc = true; + std::string directive; + auto start = currentIndex + 1; + while (start < (int) line.size() && !isspace(line[start])) { + directive += line[start]; + start++; + } - if (start < (int) line.size()) { + while (start < (int) line.size() && isspace(line[start])) + start++; - if (isspace(line[start].mChar)) { - start += 1; - if (directive == "define") { - while (start < (int) line.size() && isspace(line[start].mChar)) - start++; - std::string identifier; - while (start < (int) line.size() && !isspace(line[start].mChar)) { - identifier += line[start].mChar; - start++; - } - if (identifier.size() > 0 && !withinNotDef && std::find(mDefines.begin(),mDefines.end(),identifier) == mDefines.end()) - mDefines.push_back(identifier); - } else if (directive == "undef") { - while (start < (int) line.size() && isspace(line[start].mChar)) - start++; - std::string identifier; - while (start < (int) line.size() && !isspace(line[start].mChar)) { - identifier += line[start].mChar; - start++; - } - if (identifier.size() > 0 && !withinNotDef) - mDefines.erase(std::remove(mDefines.begin(), mDefines.end(), identifier), mDefines.end()); + if (directive == "endif" && !ifDefs.empty()) { + ifDefs.pop_back(); + withinNotDef = !ifDefs.back(); + } else { + std::string identifier; + while (start < (int) line.size() && !isspace(line[start])) { + identifier += line[start]; + start++; + } + if (directive == "define") { + if (identifier.size() > 0 && !withinNotDef && std::find(mDefines.begin(), mDefines.end(), identifier) == mDefines.end()) + mDefines.push_back(identifier); + } else if (directive == "undef") { + if (identifier.size() > 0 && !withinNotDef) + mDefines.erase(std::remove(mDefines.begin(), mDefines.end(), identifier), mDefines.end()); } else if (directive == "ifdef") { - while (start < (int) line.size() && isspace(line[start].mChar)) - start++; - std::string identifier; - while (start < (int) line.size() && !isspace(line[start].mChar)) { - identifier += line[start].mChar; - start++; - } if (!withinNotDef) { - bool isConditionMet = std::find(mDefines.begin(),mDefines.end(),identifier) != mDefines.end(); + bool isConditionMet = std::find(mDefines.begin(), mDefines.end(), identifier) != mDefines.end(); ifDefs.push_back(isConditionMet); } else ifDefs.push_back(false); } else if (directive == "ifndef") { - while (start < (int) line.size() && isspace(line[start].mChar)) - start++; - std::string identifier; - while (start < (int) line.size() && !isspace(line[start].mChar)) { - identifier += line[start].mChar; - start++; - } if (!withinNotDef) { - bool isConditionMet = std::find(mDefines.begin(),mDefines.end(),identifier) == mDefines.end(); + bool isConditionMet = std::find(mDefines.begin(), mDefines.end(), identifier) == mDefines.end(); ifDefs.push_back(isConditionMet); } else ifDefs.push_back(false); } } - } else { - if (directive == "endif") { - if (ifDefs.size() > 1) { - ifDefs.pop_back(); - withinNotDef = !ifDefs.back(); - } - } } - } - if (c == '\"') { - withinString = true; - setGlyphFlags(currentIndex); - } else { - auto pred = [](const char &a, const Glyph &b) { return a == b.mChar; }; + if (c == '\"' && !withinPreproc && !inComment && !withinComment && !withinDocComment) { + withinString = true; + setGlyphFlags(currentIndex); + } else { + auto pred = [](const char &a, const char &b) { return a == b; }; - auto compareForth = [&](const std::string &a, const std::vector &b) { - return !a.empty() && currentIndex + a.size() <= b.size() && equals(a.begin(), a.end(), - b.begin() + currentIndex, b.begin() + currentIndex + a.size(), pred); - }; + auto compareForth = [&](const std::string &a, const std::string &b) { + return !a.empty() && (currentIndex + a.size() <= b.size()) && equals(a.begin(), a.end(), b.begin() + currentIndex, b.begin() + (currentIndex + a.size()), pred); + }; - auto compareBack = [&](const std::string &a, const std::vector &b) { - return !a.empty() && currentIndex + 1 >= (int)a.size() && equals(a.begin(), a.end(), - b.begin() + currentIndex + 1 - a.size(), b.begin() + currentIndex + 1, pred); - }; + auto compareBack = [&](const std::string &a, const std::string &b) { + return !a.empty() && currentIndex + 1 >= (int) a.size() && equals(a.begin(), a.end(), b.begin() + (currentIndex + 1 - a.size()), b.begin() + (currentIndex + 1), pred); + }; - if (!inComment && !withinSingleLineComment && !withinPreproc) { - if (compareForth(singleStartStr, line)) { - withinSingleLineComment = !inComment; - } else { - bool isGlobalDocComment = compareForth(globalStartStr, line); - bool isDocComment = compareForth(docStartStr, line); - bool isComment = compareForth(startStr, line); - if (isGlobalDocComment || isDocComment || isComment) { - commentStartLine = currentLine; - commentStartIndex = currentIndex; - if (isGlobalDocComment) { - withinGlobalDocComment = true; - commentLength = 3; - } else if (isDocComment) { - withinDocComment = true; - commentLength = 3; - } else { - withinComment = true; - commentLength = 2; + if (!inComment && !withinComment && !withinDocComment && !withinPreproc && !withinString) { + if (compareForth(mLanguageDefinition.mDocComment, line.mChars)) { + withinDocComment = !inComment; + commentLength = 3; + } else if (compareForth(mLanguageDefinition.mSingleLineComment, line.mChars)) { + withinComment = !inComment; + commentLength = 2; + } else { + bool isGlobalDocComment = compareForth(mLanguageDefinition.mGlobalDocComment, line.mChars); + bool isBlockDocComment = compareForth(mLanguageDefinition.mBlockDocComment, line.mChars); + bool isBlockComment = compareForth(mLanguageDefinition.mCommentStart, line.mChars); + if (isGlobalDocComment || isBlockDocComment || isBlockComment) { + commentStartLine = currentLine; + commentStartIndex = currentIndex; + if (currentIndex < line.size() - 4 && isBlockComment && + line.mChars[currentIndex + 2] == '*' && + line.mChars[currentIndex + 3] == '/') { + withinBlockComment = true; + commentLength = 2; + } else if (isGlobalDocComment) { + withinGlobalDocComment = true; + commentLength = 3; + } else if (isBlockDocComment) { + withinBlockDocComment = true; + commentLength = 3; + } else { + withinBlockComment = true; + commentLength = 2; + } } } + inComment = (commentStartLine < currentLine || (commentStartLine == currentLine && commentStartIndex <= currentIndex)); } - inComment = (commentStartLine < currentLine || (commentStartLine == currentLine && commentStartIndex <= currentIndex)); - } - setGlyphFlags(currentIndex); + setGlyphFlags(currentIndex); - auto &endStr = mLanguageDefinition.mCommentEnd; - if (compareBack(endStr, line) && ((commentStartLine != currentLine) || (commentStartIndex + commentLength < currentIndex))) { - withinComment = false; - withinDocComment = false; - withinGlobalDocComment = false; - commentStartLine = endLine; - commentStartIndex = endIndex; - commentLength = 0; + if (compareBack(mLanguageDefinition.mCommentEnd, line.mChars) && ((commentStartLine != currentLine) || (commentStartIndex + commentLength < currentIndex))) { + withinBlockComment = false; + withinBlockDocComment = false; + withinGlobalDocComment = false; + commentStartLine = endLine; + commentStartIndex = 0; + commentLength = 0; + } } } + if (currentIndex < line.size()) { + Line::Flags flags(0); + flags.mValue = mLines[currentLine].mFlags[currentIndex]; + flags.mBits.mPreprocessor = withinPreproc; + mLines[currentLine].mFlags[currentIndex] = flags.mValue; + } + auto utf8CharLength = UTF8CharLength(c); + if (utf8CharLength > 1) { + Line::Flags flags(0); + flags.mValue = mLines[currentLine].mFlags[currentIndex]; + for (int j = 1; j < utf8CharLength; j++) { + currentIndex++; + mLines[currentLine].mFlags[currentIndex] = flags.mValue; + } + } + currentIndex++; } - if (currentIndex < line.size()) - line[currentIndex].mPreprocessor = withinPreproc; - - currentIndex += UTF8CharLength(c); - if (currentIndex >= (int)line.size()) { - withinNotDef = !ifDefs.back(); - currentIndex = 0; - ++currentLine; - } - } else { - currentIndex = 0; - ++currentLine; - } + withinNotDef = !ifDefs.back(); + // } + // mUpdateFlags = false; } mDefines.clear(); - mCheckComments = false; - } - - if (mColorRangeMin < mColorRangeMax) { - const int increment = (mLanguageDefinition.mTokenize == nullptr) ? 10 : 10000; - const int to = std::min(mColorRangeMin + increment, mColorRangeMax); - ColorizeRange(mColorRangeMin, to); - mColorRangeMin = to; - - if (mColorRangeMax == mColorRangeMin) { - mColorRangeMin = std::numeric_limits::max(); - mColorRangeMax = 0; - } - return; } + ColorizeRange(); } float TextEditor::TextDistanceToLineStart(const Coordinates &aFrom) const { @@ -3015,15 +2993,15 @@ float TextEditor::TextDistanceToLineStart(const Coordinates &aFrom) const { float spaceSize = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, " ", nullptr, nullptr).x; int colIndex = GetCharacterIndex(aFrom); for (size_t it = 0u; it < line.size() && it < colIndex;) { - if (line[it].mChar == '\t') { + if (line[it] == '\t') { distance = (1.0f + std::floor((1.0f + distance) / (float(mTabSize) * spaceSize))) * (float(mTabSize) * spaceSize); ++it; } else { - auto d = UTF8CharLength(line[it].mChar); + auto d = UTF8CharLength(line[it]); char tempCString[7]; int i = 0; for (; i < 6 && d-- > 0 && it < (int)line.size(); i++, it++) - tempCString[i] = line[it].mChar; + tempCString[i] = line[it]; tempCString[i] = '\0'; distance += ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, tempCString, nullptr, nullptr).x; @@ -3132,13 +3110,13 @@ TextEditor::UndoRecord::UndoRecord( void TextEditor::UndoRecord::Undo(TextEditor *aEditor) { if (!mAdded.empty()) { aEditor->DeleteRange(mAddedStart, mAddedEnd); - aEditor->Colorize(mAddedStart.mLine - 1, mAddedEnd.mLine - mAddedStart.mLine + 2); + aEditor->Colorize(); } if (!mRemoved.empty()) { auto start = mRemovedStart; - aEditor->InsertTextAt(start, mRemoved); - aEditor->Colorize(mRemovedStart.mLine - 1, mRemovedEnd.mLine - mRemovedStart.mLine + 2); + aEditor->InsertTextAt(start, mRemoved.c_str()); + aEditor->Colorize(); } aEditor->mState = mBefore; @@ -3148,13 +3126,13 @@ void TextEditor::UndoRecord::Undo(TextEditor *aEditor) { void TextEditor::UndoRecord::Redo(TextEditor *aEditor) { if (!mRemoved.empty()) { aEditor->DeleteRange(mRemovedStart, mRemovedEnd); - aEditor->Colorize(mRemovedStart.mLine - 1, mRemovedEnd.mLine - mRemovedStart.mLine + 1); + aEditor->Colorize(); } if (!mAdded.empty()) { auto start = mAddedStart; - aEditor->InsertTextAt(start, mAdded); - aEditor->Colorize(mAddedStart.mLine - 1, mAddedEnd.mLine - mAddedStart.mLine + 1); + aEditor->InsertTextAt(start, mAdded.c_str()); + aEditor->Colorize(); } aEditor->mState = mAfter; @@ -3162,8 +3140,8 @@ void TextEditor::UndoRecord::Redo(TextEditor *aEditor) { } -bool TokenizeCStyleString(const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end) { - const char *p = in_begin; +bool TokenizeCStyleString(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end) { + auto p = in_begin; if (*p == '"') { p++; @@ -3189,8 +3167,8 @@ bool TokenizeCStyleString(const char *in_begin, const char *in_end, const char * return false; } -bool TokenizeCStyleCharacterLiteral(const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end) { - const char *p = in_begin; +bool TokenizeCStyleCharacterLiteral(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end) { + auto p = in_begin; if (*p == '\'') { p++; @@ -3213,10 +3191,10 @@ bool TokenizeCStyleCharacterLiteral(const char *in_begin, const char *in_end, co return false; } -bool TokenizeCStyleIdentifier(const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end) { - const char *p = in_begin; +bool TokenizeCStyleIdentifier(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end) { + auto p = in_begin; - if ((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || *p == '_') { + if ((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || *p == '_' || *p == '$') { p++; while ((p < in_end) && ((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || (*p >= '0' && *p <= '9') || *p == '_')) @@ -3230,12 +3208,12 @@ bool TokenizeCStyleIdentifier(const char *in_begin, const char *in_end, const ch return false; } -bool TokenizeCStyleNumber(const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end) { - const char *p = in_begin; +bool TokenizeCStyleNumber(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end) { + auto p = in_begin; const bool startsWithNumber = *p >= '0' && *p <= '9'; - if (*p != '+' && *p != '-' && !startsWithNumber) + if (!startsWithNumber) return false; p++; @@ -3306,8 +3284,8 @@ bool TokenizeCStyleNumber(const char *in_begin, const char *in_end, const char * return false; } - // single precision floating point type - if (p < in_end && *p == 'f') + // single and double precision floating point type + if (p < in_end && (*p == 'f' || *p == 'F' || *p == 'd' || *p == 'D')) p++; } @@ -3322,21 +3300,15 @@ bool TokenizeCStyleNumber(const char *in_begin, const char *in_end, const char * return true; } -bool TokenizeCStylePunctuation(const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end) { +bool TokenizeCStyleOperator(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end) { (void)in_end; switch (*in_begin) { - case '[': - case ']': - case '{': - case '}': case '!': case '%': case '^': case '&': case '*': - case '(': - case ')': case '-': case '+': case '=': @@ -3347,6 +3319,25 @@ bool TokenizeCStylePunctuation(const char *in_begin, const char *in_end, const c case '?': case ':': case '/': + case '@': + out_begin = in_begin; + out_end = in_begin + 1; + return true; + } + + return false; +} + +bool TokenizeCStyleSeparator(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end) { + (void)in_end; + + switch (*in_begin) { + case '[': + case ']': + case '{': + case '}': + case '(': + case ')': case ';': case ',': case '.': @@ -3377,7 +3368,7 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::CPlusPlus( langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenize = [](const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end, PaletteIndex &paletteIndex) -> bool { + langDef.mTokenize = [](strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end, PaletteIndex &paletteIndex) -> bool { paletteIndex = PaletteIndex::Max; while (in_begin < in_end && isascii(*in_begin) && isblank(*in_begin)) @@ -3388,22 +3379,24 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::CPlusPlus( out_end = in_end; paletteIndex = PaletteIndex::Default; } else if (TokenizeCStyleString(in_begin, in_end, out_begin, out_end)) - paletteIndex = PaletteIndex::String; + paletteIndex = PaletteIndex::StringLiteral; else if (TokenizeCStyleCharacterLiteral(in_begin, in_end, out_begin, out_end)) paletteIndex = PaletteIndex::CharLiteral; else if (TokenizeCStyleIdentifier(in_begin, in_end, out_begin, out_end)) paletteIndex = PaletteIndex::Identifier; else if (TokenizeCStyleNumber(in_begin, in_end, out_begin, out_end)) - paletteIndex = PaletteIndex::Number; - else if (TokenizeCStylePunctuation(in_begin, in_end, out_begin, out_end)) - paletteIndex = PaletteIndex::Punctuation; + paletteIndex = PaletteIndex::NumericLiteral; + else if (TokenizeCStyleSeparator(in_begin, in_end, out_begin, out_end)) + paletteIndex= PaletteIndex::Separator; + else if (TokenizeCStyleOperator(in_begin, in_end, out_begin, out_end)) + paletteIndex = PaletteIndex::Operator; return paletteIndex != PaletteIndex::Max; }; langDef.mCommentStart = "/*"; langDef.mCommentEnd = "*/"; - langDef.mSingleLineComment = "//"; + langDef.mSingleLineComment = "//"; langDef.mCaseSensitive = true; langDef.mAutoIndentation = true; @@ -3620,19 +3613,19 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::HLSL() { langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Preprocessor)); - langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\!\\%\\^\\&\\*\\(\\)\\-\\+\\=\\~\\|\\<\\>\\?\\/\\;\\,\\.]", PaletteIndex::Punctuation)); - + langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Directive)); + langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::StringLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\!\\%\\^\\&\\*\\-\\+\\=\\~\\|\\<\\>\\?\\/]", PaletteIndex::Operator)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\(\\)\\;\\,\\.]", PaletteIndex::Separator)); langDef.mCommentStart = "/*"; langDef.mCommentEnd = "*/"; - langDef.mSingleLineComment = "//"; + langDef.mSingleLineComment = "//"; langDef.mCaseSensitive = true; langDef.mAutoIndentation = true; @@ -3663,19 +3656,19 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::GLSL() { langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Preprocessor)); - langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\!\\%\\^\\&\\*\\(\\)\\-\\+\\=\\~\\|\\<\\>\\?\\/\\;\\,\\.]", PaletteIndex::Punctuation)); - + langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Directive)); + langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::StringLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\!\\%\\^\\&\\*\\-\\+\\=\\~\\|\\<\\>\\?\\/]", PaletteIndex::Operator)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\(\\)\\;\\,\\.]", PaletteIndex::Separator)); langDef.mCommentStart = "/*"; langDef.mCommentEnd = "*/"; - langDef.mSingleLineComment = "//"; + langDef.mSingleLineComment = "//"; langDef.mCaseSensitive = true; langDef.mAutoIndentation = true; @@ -3706,7 +3699,7 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::C() { langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenize = [](const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end, PaletteIndex &paletteIndex) -> bool { + langDef.mTokenize = [](strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end, PaletteIndex &paletteIndex) -> bool { paletteIndex = PaletteIndex::Max; while (in_begin < in_end && isascii(*in_begin) && isblank(*in_begin)) @@ -3717,22 +3710,24 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::C() { out_end = in_end; paletteIndex = PaletteIndex::Default; } else if (TokenizeCStyleString(in_begin, in_end, out_begin, out_end)) - paletteIndex = PaletteIndex::String; + paletteIndex = PaletteIndex::StringLiteral; else if (TokenizeCStyleCharacterLiteral(in_begin, in_end, out_begin, out_end)) paletteIndex = PaletteIndex::CharLiteral; else if (TokenizeCStyleIdentifier(in_begin, in_end, out_begin, out_end)) paletteIndex = PaletteIndex::Identifier; else if (TokenizeCStyleNumber(in_begin, in_end, out_begin, out_end)) - paletteIndex = PaletteIndex::Number; - else if (TokenizeCStylePunctuation(in_begin, in_end, out_begin, out_end)) - paletteIndex = PaletteIndex::Punctuation; + paletteIndex = PaletteIndex::NumericLiteral; + else if (TokenizeCStyleOperator(in_begin, in_end, out_begin, out_end)) + paletteIndex = PaletteIndex::Operator; + else if (TokenizeCStyleSeparator(in_begin, in_end, out_begin, out_end)) + paletteIndex = PaletteIndex::Separator; return paletteIndex != PaletteIndex::Max; }; langDef.mCommentStart = "/*"; langDef.mCommentEnd = "*/"; - langDef.mSingleLineComment = "//"; + langDef.mSingleLineComment = "//"; langDef.mCaseSensitive = true; langDef.mAutoIndentation = true; @@ -3764,18 +3759,20 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::SQL() { langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("\\\'[^\\\']*\\\'", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\!\\%\\^\\&\\*\\(\\)\\-\\+\\=\\~\\|\\<\\>\\?\\/\\;\\,\\.]", PaletteIndex::Punctuation)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Directive)); + langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::StringLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\!\\%\\^\\&\\*\\-\\+\\=\\~\\|\\<\\>\\?\\/]", PaletteIndex::Operator)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\(\\)\\;\\,\\.]", PaletteIndex::Separator)); langDef.mCommentStart = "/*"; langDef.mCommentEnd = "*/"; - langDef.mSingleLineComment = "//"; + langDef.mSingleLineComment = "//"; langDef.mCaseSensitive = false; langDef.mAutoIndentation = false; @@ -3807,14 +3804,16 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::AngelScrip langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\!\\%\\^\\&\\*\\(\\)\\-\\+\\=\\~\\|\\<\\>\\?\\/\\;\\,\\.]", PaletteIndex::Punctuation)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Directive)); + langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::StringLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\!\\%\\^\\&\\*\\-\\+\\=\\~\\|\\<\\>\\?\\/]", PaletteIndex::Operator)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\(\\)\\;\\,\\.]", PaletteIndex::Separator)); langDef.mCommentStart = "/*"; langDef.mCommentEnd = "*/"; @@ -3850,13 +3849,16 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::Lua() { langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("\\\'[^\\\']*\\\'", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\!\\%\\^\\&\\*\\(\\)\\-\\+\\=\\~\\|\\<\\>\\?\\/\\;\\,\\.]", PaletteIndex::Punctuation)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Directive)); + langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::StringLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\!\\%\\^\\&\\*\\-\\+\\=\\~\\|\\<\\>\\?\\/]", PaletteIndex::Operator)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\(\\)\\;\\,\\.]", PaletteIndex::Separator)); langDef.mCommentStart = "--[["; langDef.mCommentEnd = "]]"; diff --git a/main/gui/source/window/window.cpp b/main/gui/source/window/window.cpp index 0da16b789..f5cd13ae5 100644 --- a/main/gui/source/window/window.cpp +++ b/main/gui/source/window/window.cpp @@ -2,8 +2,6 @@ #include -#include - #include #include #include diff --git a/plugins/builtin/CMakeLists.txt b/plugins/builtin/CMakeLists.txt index a4f1c877a..eee8cc8ef 100644 --- a/plugins/builtin/CMakeLists.txt +++ b/plugins/builtin/CMakeLists.txt @@ -119,6 +119,8 @@ add_imhex_plugin( source/content/views/view_achievements.cpp source/content/views/view_highlight_rules.cpp source/content/views/view_tutorials.cpp + + source/content/text_highlighting/pattern_language.cpp INCLUDES include diff --git a/plugins/builtin/include/content/text_highlighting/pattern_language.hpp b/plugins/builtin/include/content/text_highlighting/pattern_language.hpp new file mode 100644 index 000000000..c107d0b89 --- /dev/null +++ b/plugins/builtin/include/content/text_highlighting/pattern_language.hpp @@ -0,0 +1,419 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace pl { + class PatternLanguage; +} + +namespace hex::plugin::builtin { + class ViewPatternEditor; + class TextHighlighter { + public: + class Interval; + using Token = pl::core::Token; + using ASTNode = pl::core::ast::ASTNode; + using ExcludedLocation = pl::core::Preprocessor::ExcludedLocation; + using CompileError = pl::core::err::CompileError; + using Identifier = Token::Identifier; + using IdentifierType = Identifier::IdentifierType; + using UnorderedBlocks = std::map; + using OrderedBlocks = std::map; + using Scopes = std::set; + using Location = pl::core::Location; + using TokenIter = pl::hlp::SafeIterator::const_iterator>; + using VariableScopes = std::map; + using Inheritances = std::map>; + using IdentifierTypeColor = std::map; + using TokenTypeColor = std::map; + using TokenColor = std::map; + + struct ParentDefinition; + struct Definition { + Definition()= default; + Definition(IdentifierType identifierType, std::string typeStr,i32 tokenId, Location location) : idType(identifierType), typeStr(typeStr), tokenIndex(tokenId),location(location) {} + IdentifierType idType; + std::string typeStr; + i32 tokenIndex; + Location location; + }; + + struct ParentDefinition { + ParentDefinition() = default; + ParentDefinition(IdentifierType identifierType, i32 tokenId, Location location) : idType(identifierType), tokenIndex(tokenId), location(location) {} + IdentifierType idType; + i32 tokenIndex; + Location location; + }; + /// to define functions and types + using Definitions = std::map; + /// to define global variables + using Variables = std::map>; + /// to define UDT and function variables + using VariableMap = std::map; + private: + std::string m_text; + std::vector m_lines; + std::vector m_firstTokenIdOfLine; + ViewPatternEditor *m_viewPatternEditor; + std::vector m_excludedLocations; + std::vector m_tokens; + TokenColor m_tokenColors; + std::unique_ptr *patternLanguage; + std::vector m_compileErrors; + std::map> m_instances; + Definitions m_UDTDefinitions; + Definitions m_functionDefinitions; + + OrderedBlocks m_namespaceTokenRange; + UnorderedBlocks m_UDTTokenRange; + UnorderedBlocks m_functionTokenRange; + Scopes m_globalTokenRange; + + VariableMap m_UDTVariables; + VariableMap m_functionVariables; + Variables m_globalVariables; + + std::map m_attributeFunctionArgumentType; + std::map m_typeDefMap; + std::map m_typeDefInvMap; + std::vector m_nameSpaces; + std::vector m_UDTs; + std::set m_taggedIdentifiers; + std::set m_memberChains; + std::set m_scopeChains; + + TokenIter m_curr; + TokenIter m_startToken, m_originalPosition, m_partOriginalPosition; + + VariableScopes m_UDTBlocks; + VariableScopes m_functionBlocks; + Scopes m_globalBlocks; + Inheritances m_inheritances; + const static IdentifierTypeColor m_identifierTypeColor; + const static TokenTypeColor m_tokenTypeColor; + + i32 m_runningColorizers=0; + public: + + /// Intervals are the sets finite contiguous non-negative integer that + /// are described by their endpoints. The sets must have the following + /// properties: + /// 1. Any two elements of the set can either have an empty intersection or + /// 2. their intersection is equal to one of the two sets (i.e. one is + /// a subset of the other). + /// An interval is defined to be smaller than another if: + /// 1. The interval lies entirely to the left of the other interval or + /// 2. The interval is a proper subset of the other interval. + /// Two intervals are equal if they have identical start and end values. + /// This ordering is used for things like code blocks or the token + /// ranges that are defined by the blocks. + class Interval { + public: + i32 start; + i32 end; + Interval() : start(0), end(0) {} + Interval(i32 start, i32 end) : start(start), end(end) { + if (start > end) + throw std::invalid_argument("Interval start must be less than or equal to end"); + } + bool operator<(const Interval &other) const { + return other.end > end; + } + bool operator>(const Interval &other) const { + return end > other.end; + } + bool operator==(const Interval &other) const { + return start == other.start && end == other.end; + } + bool operator!=(const Interval &other) const { + return start != other.start || end != other.end; + } + bool operator<=(const Interval &other) const { + return other.end >= end; + } + bool operator>=(const Interval &other) const { + return end >= other.end; + } + bool contains(const Interval &other) const { + return other.start >= start && other.end <= end; + } + bool contains(i32 value) const { + return value >= start && value <= end; + } + bool contiguous(const Interval &other) const { + return ((start - other.end) == 1 || (other.start - end) == 1); + } + }; + std::atomic m_needsToUpdateColors = true; + std::atomic m_wasInterrupted = false; + + TextHighlighter(ViewPatternEditor *viewPatternEditor, std::unique_ptr *patternLanguage ) : + m_viewPatternEditor(viewPatternEditor), patternLanguage(patternLanguage), m_needsToUpdateColors(true) {} + /** + * @brief Entry point to syntax highlighting + */ + void highlightSourceCode(); + + /** + * @brief Syntax highlighting from parser + */ + void setInitialColors(); + /** + * @brief Create data to pass to text editor + */ + void setRequestedIdentifierColors(); + /** + * @brief Set the color of a token + */ + void setColor(i32 tokenId=-1, const IdentifierType &type = IdentifierType::Unknown); + void setIdentifierColor(i32 tokenId=-1, const IdentifierType &type = IdentifierType::Unknown); + /** + * @brief Only identifiers not in chains should remain + */ + void colorRemainingIdentifierTokens(); + /** + * @brief Renders compile errors in real time + */ + void renderErrors(); + /// A token range is the set of token indices of a definition. The namespace token + /// ranges are obtained first because they are needed to obtain unique identifiers. + void getAllTokenRanges(IdentifierType idtype); + /// The global scope is the complement of the union of all the function and UDT token ranges + void getGlobalTokenRanges(); + /// If the current token is a function or UDT, creates a map entry from the name to the token range. These are ordered alphabetically by name. + /// If the current token is a namespace, creates a map entry from the token range to the name. Namespace entries are stored in the order they occur in the source code. + bool getTokenRange(std::vector keywords,UnorderedBlocks &tokenRange, OrderedBlocks &tokenRangeInv, bool fullName, VariableScopes *blocks); + /// Global variables are the variables that are not inside a function or UDT + void fixGlobalVariables(); + /// Creates the definition maps for UDTs, functions, their variables and global variables + void getDefinitions(); + void loadGlobalDefinitions(Scopes tokenRangeSet, std::vector identifierTypes, Variables &variables); + void loadVariableDefinitions(UnorderedBlocks tokenRangeMap, Token delimiter1, Token delimiter2, std::vector identifierTypes, bool isArgument, VariableMap &variableMap); + void loadTypeDefinitions(UnorderedBlocks tokenRangeMap, std::vector identifierTypes, Definitions &types); + std::string getArgumentTypeName(i32 rangeStart, Token delimiter2); + std::string getVariableTypeName(); + /// Append the variable definitions of the parent to the child + void appendInheritances(); + void recurseInheritances(std::string name); + ///Loads a map of identifiers to their token id instances + void loadInstances(); + /// Replace auto with the actual type for template arguments and function parameters + void fixAutos(); + void resolveAutos(VariableMap &variableMap, UnorderedBlocks &tokenRange); + /// Chains are sequences of identifiers separated by scope resolution or dot operators. + void fixChains(); + bool colorSeparatorScopeChain(); + bool colorOperatorDotChain(); + /// Returns the next/previous valid source code line + u32 nextLine(u32 line); + u32 previousLine(u32 line); + /// Loads the source code and calculates the first token index of each line + void loadText(); + /// Used to obtain the color to be applied. + TextEditor::PaletteIndex getPaletteIndex(Token::Literal *literal); + /// The complement of a set is also known as its inverse + void invertGlobalTokenRange(); + /// Starting at the identifier, it tracks all the scope resolution and dot operators and returns the full chain without arrays, templates, pointers,... + bool getFullName(std::string &identifierName, std::vector &identifiers, bool preserveCurr = true); + /// Returns the identifier value. + bool getIdentifierName(std::string &identifierName, Identifier *identifier); + /// Adds namespaces to the full name if they exist + bool getQualifiedName(std::string &identifierName, std::vector &identifiers, bool useDefinitions = false, bool preserveCurr = true); + /// As it moves forward it loads the result to the argument. Used by getFullName + bool forwardIdentifierName(std::string &identifierName, std::vector &identifiers, bool preserveCurr = true); + /// Takes as input the full name and returns the type of the last element. + bool resolveIdentifierType(Definition &result, std::string identifierName); + /// like previous functions but returns the type of the variable that is a member of a UDT + std::string findIdentifierTypeStr(const std::string &identifierName, std::string context=""); + IdentifierType findIdentifierType(const std::string &identifierName, std::string context); + /// If context is empty search for the variable, if it isnt use the variable map. + bool findOrContains(std::string &context, UnorderedBlocks tokenRange, VariableMap variableMap); + /// Search for instances inside some block + void setBlockInstancesColor(const std::string &name, const Definition &definition, const Interval &block); + /// Convenience functions. + void skipAttribute(); + void skipArray(i32 maxSkipCount, bool forward = true); + void skipTemplate(i32 maxSkipCount, bool forward = true); + void skipDelimiters(i32 maxSkipCount, Token delimiter[2], i8 increment); + void skipToken(Token token, i8 step=1); + /// from given or current names find the corresponding definition + bool findIdentifierDefinition(Definition &result, const std::string &optionalIdentifierName = "", std::string optionalName = "", bool optional = false); + /// To deal with the Parent keyword + std::optional setChildrenTypes(); + bool findParentTypes(std::vector &parentTypes, const std::string &optionalName=""); + bool findAllParentTypes(std::vector &parentTypes, std::vector &identifiers, std::string &optionalFullName); + bool tryParentType(const std::string &parentType, std::string &variableName, std::optional &result, std::vector &identifiers); + /// Convenience function + bool isTokenIdValid(i32 tokenId); + bool isLocationValid(Location location); + /// Returns the name of the context where the current or given token is located + bool findScope(std::string &name, const UnorderedBlocks &map, i32 optionalTokenId=-1); + /// Returns the name of the namespace where the current or given token is located + bool findNamespace(std::string &nameSpace, i32 optionalTokenId=-1); + /// Calculate the source code, line and column numbers of a token index + pl::core::Location getLocation(i32 tokenId); + /// Calculate the token index of a source code, line and column numbers + i32 getTokenId(pl::core::Location location); + /// Calculate the function or template argument position from token indices + i32 getArgumentNumber(i32 start,i32 arg); + /// Calculate the token index of a function or template argument position + void getTokenIdForArgument(i32 start, i32 argNumber, Token delimiter); + ///Creates a map from function name to argument type + void linkAttribute(); + /// Comment and strings usethese function to determine their coordinates + template TextEditor::Coordinates commentCoordinates(Token *token); + TextEditor::Coordinates stringCoordinates(); + /// Returns the number of tasks highlighting code. Shouldn't be > 1 + i32 getRunningColorizers() { + return m_runningColorizers; + } + + enum class HighlightStage { + Starting, + NamespaceTokenRanges, + UDTTokenRanges, + FunctionTokenRanges, + GlobalTokenRanges, + FixGlobalVariables, + SetInitialColors, + LoadInstances, + AttributeTokenRanges, + Definitions, + FixAutos, + FixChains, + ExcludedLocations, + ColorRemainingTokens, + SetRequestedIdentifierColors, + Stage1, + Stage2, + Stage3, + Stage4, + Stage5, + Stage6, + Stage7, + Stage8, + Stage9, + Stage10, + Stage11, + }; + + HighlightStage m_highlightStage = HighlightStage::Starting; + + /// The following functions were copied from the parser and some were modified + + template + T *getValue(const i32 index) { + return const_cast(std::get_if(&m_curr[index].value)); + } + + void next(i32 count = 1) { + if (count == 0) + return; + i32 id = getTokenId(m_curr->location); + i32 maxChange; + if (count > 0) + maxChange = std::min(count,static_cast(m_tokens.size() - id)); + else + maxChange = -std::min(-count,id); + m_curr += maxChange; + } + constexpr static u32 Normal = 0; + constexpr static u32 Not = 1; + + bool begin() { + m_originalPosition = m_curr; + + return true; + } + + void partBegin() { + m_partOriginalPosition = m_curr; + } + + void reset() { + m_curr = m_originalPosition; + } + + void partReset() { + m_curr = m_partOriginalPosition; + } + + bool resetIfFailed(const bool value) { + if (!value) reset(); + + return value; + } + + template + bool sequenceImpl() { + if constexpr (S == Normal) + return true; + else if constexpr (S == Not) + return false; + else + std::unreachable(); + } + + template + bool matchOne(const Token &token) { + if constexpr (S == Normal) { + if (!peek(token)) { + partReset(); + return false; + } + + next(); + return true; + } else if constexpr (S == Not) { + if (!peek(token)) + return true; + + next(); + partReset(); + return false; + } else + std::unreachable(); + } + + template + bool sequenceImpl(const auto &... args) { + return (matchOne(args) && ...); + } + + template + bool sequence(const Token &token, const auto &... args) { + partBegin(); + return sequenceImpl(token, args...); + } + + bool isValid() { + Token token; + try { + token = m_curr[0]; + } + catch (const std::out_of_range &e) { + auto t = e.what(); + if (t == nullptr) + return false; + return false; + } + if (!isLocationValid(token.location)) + return false; + return true; + } + + bool peek(const Token &token, const i32 index = 0) { + if (!isValid()) + return false; + i32 id = getTokenId(m_curr->location); + if (id+index < 0 || id+index >= (i32)m_tokens.size()) + return false; + return m_curr[index].type == token.type && m_curr[index] == token.value; + } + + }; +} \ No newline at end of file diff --git a/plugins/builtin/include/content/views/view_pattern_editor.hpp b/plugins/builtin/include/content/views/view_pattern_editor.hpp index c075e1560..7429b7ba3 100644 --- a/plugins/builtin/include/content/views/view_pattern_editor.hpp +++ b/plugins/builtin/include/content/views/view_pattern_editor.hpp @@ -19,6 +19,7 @@ #include #include +#include namespace pl::ptrn { class Pattern; } @@ -69,6 +70,30 @@ namespace hex::plugin::builtin { ~ViewPatternEditor() override; void drawAlwaysVisibleContent() override; + std::unique_ptr *getPatternLanguage() { + return &m_editorRuntime; + } + + TextEditor &getTextEditor() { + return m_textEditor; + } + + bool getChangesWereParsed() const { + return m_changesWereParsed; + } + + u32 getRunningParsers () const { + return m_runningParsers; + } + + u32 getRunningEvaluators () const { + return m_runningEvaluators; + } + + void setChangesWereParsed(bool changesWereParsed) { + m_changesWereParsed = changesWereParsed; + } + void drawContent() override; [[nodiscard]] ImGuiWindowFlags getWindowFlags() const override { return ImGuiWindowFlags_NoScrollbar | ImGuiWindowFlags_NoScrollWithMouse; @@ -234,6 +259,7 @@ namespace hex::plugin::builtin { std::atomic m_runningEvaluators = 0; std::atomic m_runningParsers = 0; + bool m_changesWereParsed = false; PerProvider m_hasUnevaluatedChanges; std::chrono::time_point m_lastEditorChangeTime; @@ -308,6 +334,7 @@ namespace hex::plugin::builtin { static inline u32 m_replaceHistorySize = 0; static inline u32 m_replaceHistoryIndex = 0; + TextHighlighter m_textHighlighter = TextHighlighter(this,&this->m_editorRuntime); private: void drawConsole(ImVec2 size); void drawEnvVars(ImVec2 size, std::list &envVars); diff --git a/plugins/builtin/romfs/themes/classic.json b/plugins/builtin/romfs/themes/classic.json index 8c365c00c..48d590a2b 100644 --- a/plugins/builtin/romfs/themes/classic.json +++ b/plugins/builtin/romfs/themes/classic.json @@ -147,27 +147,51 @@ "title-text": "#E5E5E5FF" }, "text-editor": { + "attribute": "#FFFF00FF", "background": "#000080FF", + "multi-line-comment": "#404040FF", "breakpoint": "#FF200080", + "known-identifier": "#4DC69BFF", + "calculated-pointer": "#FFFF00FF", "char-literal": "#008080FF", "comment": "#808080FF", "current-line-edge": "#00000040", "current-line-fill": "#00000040", "current-line-fill-inactive": "#80808040", "cursor": "#FF8000FF", + "debug-text": "#8A8A8AFF", "default": "#FFFF00FF", + "default-text": "#FFFF00FF", + "preprocessor": "#7FBF00FF", + "doc-block-comment": "#206020FF", + "doc-comment": "#206020FF", + "doc-global-comment": "#206020FF", "error-marker": "#FF0000A0", + "error-text": "#FF200080", + "function": "#FFFF00FF", + "function-parameter": "#FFFF00FF", + "function-variable": "#FFFF00FF", + "global-variable": "#FFFF00FF", "identifier": "#FFFF00FF", "keyword": "#00FFFFFF", - "known-identifier": "#FFFFFFFF", "line-number": "#008080FF", - "multi-line-comment": "#404040FF", + "local-variable": "#FFFF00FF", + "namespace": "#FFFF00FF", "number": "#00FF00FF", - "preproc-identifier": "#FF00FFFF", - "preprocessor": "#008000FF", - "punctuation": "#FFFFFFFF", + "punctuation": "#FFFF00FF", + "pattern-variable": "#FFFF00FF", + "placed-variable": "#FFFF00FF", + "preprocessor-deactivated": "#4F4F4F45", + "preproc-identifier": "#7FBF00FF", "selection": "*#00FFFF80", - "string": "#008080FF" + "separator": "#FFFF00FF", + "string": "#008080FF", + "template-variable": "#FFFF00FF", + "typedef": "#FFFF00FF", + "unknown-identifier": "#FC2C2CFE", + "user-defined-type": "#FFFF00FF", + "view": "#FFFF00FF", + "warning-text": "#FFFF00FF" } }, "image_theme": "dark", diff --git a/plugins/builtin/romfs/themes/dark.json b/plugins/builtin/romfs/themes/dark.json index 4db34e6b2..f9ec75bac 100644 --- a/plugins/builtin/romfs/themes/dark.json +++ b/plugins/builtin/romfs/themes/dark.json @@ -147,27 +147,51 @@ "title-text": "#FFFFFFFF" }, "text-editor": { + "attribute": "#AAAAAAFF", "background": "#101010FF", + "multi-line-comment": "#206040FF", "breakpoint": "#FF200040", + "known-identifier": "#4DC69BFF", + "calculated-pointer": "#AAAAAAFF", "char-literal": "#E0A070FF", "comment": "#206020FF", "current-line-edge": "#A0A0A040", "current-line-fill": "#00000040", "current-line-fill-inactive": "#80808040", "cursor": "#E0E0E0FF", + "debug-text": "#8A8A8AFF", "default": "#7F7F7FFF", + "default-text": "#7F7F7FFF", + "preprocessor": "#808060FF", + "doc-block-comment": "#206020FF", + "doc-comment": "#206020FF", + "doc-global-comment": "#206020FF", "error-marker": "#FF200080", + "error-text": "#FF200080", + "function": "#AAAAAAFF", + "function-parameter": "#AAAAAAFF", + "function-variable": "#AAAAAAFF", + "global-variable": "#AAAAAAFF", "identifier": "#AAAAAAFF", "keyword": "#569CD6FF", - "known-identifier": "#4DC69BFF", "line-number": "#007070FF", - "multi-line-comment": "#206040FF", + "local-variable": "#AAAAAAFF", + "namespace": "#AAAAAAFF", "number": "#00FF00FF", - "preproc-identifier": "#A040C0FF", - "preprocessor": "#808040FF", - "punctuation": "#FFFFFFFF", + "punctuation": "#7F7F7FFF", + "pattern-variable": "#AAAAAAFF", + "placed-variable": "#AAAAAAFF", + "preprocessor-deactivated": "#4F4F4F45", + "preproc-identifier": "#808060FF", "selection": "*#2060A080", - "string": "#E07070FF" + "separator": "#7F7F7FFF", + "string": "#E07070FF", + "template-variable": "#AAAAAAFF", + "typedef": "#AAAAAAFF", + "unknown-identifier": "#FC2C2CFE", + "user-defined-type": "#AAAAAAFF", + "view": "#AAAAAAFF", + "warning-text": "#FFFF00FF" } }, "image_theme": "dark", diff --git a/plugins/builtin/romfs/themes/light.json b/plugins/builtin/romfs/themes/light.json index 77783b840..7570b0183 100644 --- a/plugins/builtin/romfs/themes/light.json +++ b/plugins/builtin/romfs/themes/light.json @@ -147,27 +147,50 @@ "title-text": "#000000FF" }, "text-editor": { + "attribute": "#404040FF", "background": "#FFFFFFFF", + "multi-line-comment": "#205040FF", "breakpoint": "#FF200080", + "known-identifier": "#106060FF", + "calculated-pointer": "#404040FF", "char-literal": "#704030FF", "comment": "#205020FF", "current-line-edge": "#00000040", "current-line-fill": "#00000040", "current-line-fill-inactive": "#80808040", "cursor": "#000000FF", + "debug-text": "#8A8A8AFF", "default": "#7F7F7FFF", + "default-text": "#7F7F7FFF", + "preprocessor": "#6F6F5FFF", + "doc-block-comment": "#205040FF", + "doc-comment": "#205020FF", + "doc-global-comment": "#205040FF", "error-marker": "#FF1000A0", - "identifier": "#404040FF", + "error-text": "#FF200080", + "function": "#404040FF", + "function-parameter": "#404040FF", + "function-variable": "#404040FF", + "global-variable": "#404040FF", "keyword": "#060CFFFF", - "known-identifier": "#106060FF", "line-number": "#005050FF", - "multi-line-comment": "#205040FF", + "local-variable": "#404040FF", + "namespace": "#404040FF", "number": "#008000FF", - "preproc-identifier": "#A040C0FF", - "preprocessor": "#606040FF", - "punctuation": "#000000FF", + "punctuation": "#7F7F7FFF", + "pattern-variable": "#404040FF", + "placed-variable": "#404040FF", + "preprocessor-deactivated": "#4F4F4F45", + "preproc-identifier": "#6F6F5FFF", "selection": "*#00006080", - "string": "#A02020FF" + "separator": "#7F7F7FFF", + "string": "#A02020FF", + "template-variable": "#404040FF", + "typedef": "#404040FF", + "unknown-identifier": "#FC2C2CFE", + "user-defined-type": "#404040FF", + "view": "#404040FF", + "warning-text": "#FFFF00FF" } }, "image_theme": "light", diff --git a/plugins/builtin/source/content/text_highlighting/pattern_language.cpp b/plugins/builtin/source/content/text_highlighting/pattern_language.cpp new file mode 100644 index 000000000..e71b149e9 --- /dev/null +++ b/plugins/builtin/source/content/text_highlighting/pattern_language.cpp @@ -0,0 +1,2332 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + + +namespace hex::plugin::builtin { + + using namespace pl::core; + using Identifier = Token::Identifier; + using Keyword = Token::Keyword; + using Separator = Token::Separator; + using Operator = Token::Operator; + using Comment = Token::Comment; + using DocComment = Token::DocComment; + using Literal = Token::Literal; + using ValueType = Token::ValueType; + + bool TextHighlighter::getIdentifierName(std::string &identifierName, Identifier *identifier) { + auto keyword = getValue(0); + identifier = getValue(0); + + if (identifier != nullptr) { + identifierName = identifier->get(); + return true; + } else if (keyword != nullptr) { + identifier = nullptr; + if (peek(tkn::Keyword::Parent)) { + identifierName = "Parent"; + return true; + } + + if (peek(tkn::Keyword::This)) { + identifierName = "This"; + return true; + } + } + identifier = nullptr; + return false; + } + +// Returns a chain of identifiers like a.b.c or a::b::c + bool TextHighlighter::getFullName(std::string &identifierName, std::vector &identifiers, bool preserveCurr) { + Identifier *identifier = nullptr; + + if (!peek(tkn::Literal::Identifier) || getTokenId(m_curr->location) < 1) + return getIdentifierName(identifierName, identifier); + + forwardIdentifierName(identifierName, identifiers, preserveCurr); + return true; + } + + bool TextHighlighter::forwardIdentifierName(std::string &identifierName, std::vector &identifiers, bool preserveCurr ) { + auto curr = m_curr; + Identifier *identifier = getValue(0); + std::string current; + + if (identifier != nullptr) { + identifiers.push_back(identifier); + identifierName += identifier->get(); + } else if (getIdentifierName(current, identifier)) { + identifiers.push_back(identifier); + identifierName += current; + } else { + m_curr = curr; + return false; + } + + skipArray(200, true); + + while ((peek(tkn::Operator::ScopeResolution, 1) || peek(tkn::Separator::Dot, 1))) { + next(); + + if (peek(tkn::Operator::ScopeResolution)) + identifierName += "::"; + else if (peek(tkn::Separator::Dot)) + identifierName += "."; + else { + m_curr = curr; + return false; + } + next(); + + if (getIdentifierName(current, identifier)) { + identifiers.push_back(identifier); + identifierName += current; + + skipArray(200, true); + + } else { + m_curr = curr; + return false; + } + } + if (preserveCurr) + m_curr = curr; + return true; + } + +// Adds namespace if it exists + bool TextHighlighter::getQualifiedName(std::string &identifierName, std::vector &identifiers, bool useDefinitions, bool preserveCurr) { + + std::string shortName; + std::string qualifiedName; + + if (!getFullName(identifierName, identifiers, preserveCurr)) + return false; + + if (std::ranges::find(m_UDTs, identifierName) != m_UDTs.end()) + return true; + std::vector vectorString; + if (identifierName.contains("::")) { + vectorString = hex::splitString(identifierName, "::"); + if (vectorString.size() > 1) { + shortName = vectorString.back(); + vectorString.pop_back(); + identifierName = hex::combineStrings(vectorString, "::"); + } + } + bool found = true; + for (auto name : vectorString) { + found = found || std::ranges::find(m_nameSpaces, name) != m_nameSpaces.end(); + } + if (found) { + if (!shortName.empty()) + identifierName = identifierName + "::" + shortName; + return true; + } + + if (useDefinitions) { + if (m_functionDefinitions.contains(identifierName) || m_UDTDefinitions.contains(identifierName)) { + if (!shortName.empty()) + identifierName = identifierName + "::" + shortName; + return true; + } + std::string nameSpace; + for (auto [name, definition]: m_UDTDefinitions) { + findNamespace(nameSpace, definition.tokenIndex); + + if (!nameSpace.empty() && !identifierName.contains(nameSpace)) { + qualifiedName = nameSpace + "::" + identifierName; + + if (name == qualifiedName) { + identifierName = qualifiedName; + if (!shortName.empty()) + identifierName = identifierName + "::" + shortName; + return true; + } + } + + if (name == identifierName) { + identifierName = name; + if (!shortName.empty()) + identifierName = identifierName + "::" + shortName; + return true; + } + } + } + + if (identifierName.empty()) + return false; + return true; + } + +// Finds the token range of a function, namespace or UDT + bool TextHighlighter::getTokenRange(std::vector keywords, + TextHighlighter::UnorderedBlocks &tokenRange, + TextHighlighter::OrderedBlocks &tokenRangeInv, + bool fullName, VariableScopes *blocks) { + + bool addArgumentBlock = !fullName; + std::vector tokenStack; + if (getTokenId(m_curr->location) < 1) + return false; + std::string name; + if (fullName) { + std::vector identifiers; + if (!getFullName(name, identifiers)) + return false; + } else { + Identifier *identifier = nullptr; + if (!getIdentifierName(name, identifier)) + return false; + std::string nameSpace; + findNamespace(nameSpace, getTokenId(m_curr->location)); + if (!nameSpace.empty()) + name = nameSpace + "::" + name; + } + + i32 tokenCount = m_tokens.size(); + auto saveCurr = m_curr - 1; + skipTemplate(200); + next(); + if (sequence(tkn::Operator::Colon)) { + while (peek(tkn::Literal::Identifier)) { + auto identifier = getValue(0); + if (identifier == nullptr) + break; + auto identifierName = identifier->get(); + if (std::ranges::find(m_inheritances[name], identifierName) == m_inheritances[name].end()) + m_inheritances[name].push_back(identifierName); + skipTemplate(200); + next(2); + } + } + + m_curr = saveCurr; + i32 index1 = getTokenId(m_curr->location); + bool result = true; + for (auto keyword: keywords) + result = result && !peek(keyword); + if (result) + return false; + u32 nestedLevel = 0; + next(); + auto endToken = TokenIter(m_tokens.begin() + tokenCount, m_tokens.end()); + while (endToken > m_curr) { + + if (sequence(tkn::Separator::LeftBrace)) { + auto tokenId = getTokenId(m_curr[-1].location); + tokenStack.push_back(tokenId); + nestedLevel++; + } else if (sequence(tkn::Separator::RightBrace)) { + nestedLevel--; + + if (tokenStack.empty()) + return false; + Interval range(tokenStack.back(), getTokenId(m_curr[-1].location)); + tokenStack.pop_back(); + + if (nestedLevel == 0) { + range.end -= 1; + if (blocks != nullptr) + blocks->operator[](name).insert(range); + skipAttribute(); + break; + } + if (blocks != nullptr) + blocks->operator[](name).insert(range); + } else if (sequence(tkn::Separator::EndOfProgram)) + return false; + else + next(); + } + i32 index2 = getTokenId(m_curr->location); + + if (index2 > index1 && index2 < tokenCount) { + if (fullName) { + tokenRangeInv[Interval(index1, index2)] = name; + } else { + tokenRange[name] = Interval(index1, index2); + } + if (blocks != nullptr) { + if (addArgumentBlock) { + auto tokenIndex = blocks->operator[](name).begin()->start; + blocks->operator[](name).insert(Interval(index1, tokenIndex)); + } + blocks->operator[](name).insert(Interval(index1, index2)); + } + return true; + } + return false; + } + +// Searches through tokens and loads all the ranges of one kind. First namespaces are searched. + void TextHighlighter::getAllTokenRanges(IdentifierType IdentifierTypeToSearch) { + + if (m_tokens.empty()) + return; + + Identifier *identifier; + IdentifierType identifierType; + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + auto endToken = TokenIter(m_tokens.end(), m_tokens.end()); + for (m_curr = m_startToken; endToken > m_curr; next()) { + auto curr = m_curr; + + if (peek(tkn::Literal::Identifier)) { + if (identifier = getValue(0); identifier != nullptr) { + identifierType = identifier->getType(); + std::string name = identifier->get(); + + if (identifierType == IdentifierTypeToSearch) { + switch (identifierType) { + case IdentifierType::Function: + getTokenRange({tkn::Keyword::Function}, m_functionTokenRange, m_namespaceTokenRange, false, &m_functionBlocks); + break; + case IdentifierType::NameSpace: + if (std::ranges::find(m_nameSpaces, name) == m_nameSpaces.end()) + m_nameSpaces.push_back(name); + getTokenRange({tkn::Keyword::Namespace}, m_functionTokenRange, m_namespaceTokenRange, true, nullptr); + break; + case IdentifierType::UDT: + getTokenRange({tkn::Keyword::Struct, tkn::Keyword::Union, tkn::Keyword::Enum, tkn::Keyword::Bitfield}, m_UDTTokenRange, m_namespaceTokenRange, false, &m_UDTBlocks); + break; + case IdentifierType::Attribute: + linkAttribute(); + break; + default: + break; + } + } + } + } else if (peek(tkn::Separator::EndOfProgram)) + return; + m_curr = curr; + } + } + + void TextHighlighter::skipDelimiters(i32 maxSkipCount, Token delimiter[2], i8 increment) { + auto curr = m_curr; + i32 skipCount = 0; + i32 depth = 0; + + if (!isValid()) + return; + i32 tokenId = getTokenId(m_curr->location); + auto tokenCount = m_tokens.size(); + + if (tokenId == -1 || tokenId >= (i32) tokenCount-1) + return; + i32 skipCountLimit = + increment > 0 ? std::min(maxSkipCount, (i32) tokenCount - 1 - tokenId) : std::min(maxSkipCount, tokenId); + next(increment); + + if (peek(delimiter[0])) { + next(increment); + while (skipCount < skipCountLimit) { + + if (peek(delimiter[1])) { + + if (depth == 0) + return; + depth--; + } else if (peek(delimiter[0])) + depth++; + else if (peek(tkn::Separator::Semicolon)) { + if (increment < 0) + m_curr = curr; + return; + } else if (peek(tkn::Literal::Identifier)) { + + if (peek(tkn::Separator::Dot,1) && peek(tkn::Literal::Identifier,2) ) + + m_memberChains.insert(getTokenId(m_curr->location)); + else if (peek(tkn::Operator::ScopeResolution,1) && peek(tkn::Literal::Identifier,2)) + + m_scopeChains.insert(getTokenId(m_curr->location)); + else + m_taggedIdentifiers.insert(getTokenId(m_curr->location)); + } + next(increment); + skipCount++; + } + } + m_curr = curr; + return; + } + + void TextHighlighter::skipTemplate(i32 maxSkipCount, bool forward) { + Token delimiters[2]; + + if (forward) { + delimiters[0] = Token(tkn::Operator::BoolLessThan); + delimiters[1] = Token(tkn::Operator::BoolGreaterThan); + } else { + delimiters[0] = Token(tkn::Operator::BoolGreaterThan); + delimiters[1] = Token(tkn::Operator::BoolLessThan); + } + skipDelimiters(maxSkipCount, delimiters, forward ? 1 : -1); + } + + void TextHighlighter::skipArray(i32 maxSkipCount, bool forward) { + Token delimiters[2]; + + if (forward) { + delimiters[0] = Token(tkn::Separator::LeftBracket); + delimiters[1] = Token(tkn::Separator::RightBracket); + } else { + delimiters[0] = Token(tkn::Separator::RightBracket); + delimiters[1] = Token(tkn::Separator::LeftBracket); + } + skipDelimiters(maxSkipCount, delimiters, forward ? 1 : -1); + } + +// Used to skip references,pointers,... + void TextHighlighter::skipToken(Token token, i8 step) { + + if (peek(token, step)) + next(step); + } + + void TextHighlighter::skipAttribute() { + + if (sequence(tkn::Separator::LeftBracket, tkn::Separator::LeftBracket)) { + while (!sequence(tkn::Separator::RightBracket, tkn::Separator::RightBracket)) + next(); + } + } + +// Takes an identifier chain resolves the type of the end from the rest iteratively. + bool TextHighlighter::resolveIdentifierType(Definition &result, std::string identifierName) { + std::string separator; + + if (identifierName.contains("::")) + separator = "::"; + else + separator = "."; + auto vectorString = hex::splitString(identifierName, separator); + + std::string nameSpace; + u32 index = 0; + std::string currentName = vectorString[index]; + index++; + std::string variableParentType; + + Definition definition; + + if (vectorString.size() > 1) { + + if (findIdentifierDefinition(definition, currentName)) { + variableParentType = definition.typeStr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, definition.idType); + skipArray(200, true); + next(); + } else + return false; + } + while (index < vectorString.size()) { + + if ( separator == ".") { + currentName = vectorString[index]; + next(); + + if (findIdentifierDefinition(result, currentName, variableParentType)) { + variableParentType = result.typeStr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, result.idType); + skipArray(200, true); + next(); + } else + return false; + + } else if (separator == "::") { + next(); + + if (std::ranges::find(m_nameSpaces, currentName) != m_nameSpaces.end()) { + nameSpace += currentName + "::"; + + variableParentType = vectorString[index]; + currentName = variableParentType; + + } else if (std::ranges::find(m_UDTs, currentName) != m_UDTs.end()) { + variableParentType = currentName; + + if (!nameSpace.empty() && !variableParentType.contains(nameSpace)) + variableParentType = nameSpace + variableParentType; + + else if (findNamespace(nameSpace) && !variableParentType.contains(nameSpace)) + variableParentType = nameSpace + "::" + variableParentType; + + currentName = vectorString[index]; + + if (findIdentifierDefinition(result, currentName, variableParentType)) { + variableParentType = result.typeStr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, result.idType); + skipArray(200, true); + next(); + } else + return false; + } + } + index++; + } + + return true; + } + +// If contex then find it otherwise check if it belongs in map + bool TextHighlighter::findOrContains(std::string &context, UnorderedBlocks tokenRange, VariableMap variableMap) { + + if (context.empty()) + return findScope(context, tokenRange); + else + return variableMap.contains(context); + } + + void TextHighlighter::setBlockInstancesColor(const std::string &name, const Definition &definition, const Interval &block) { + + if (definition.idType == IdentifierType::Unknown) + return; + for (auto instance: m_instances[name]) { + + if (block.contains(instance)) { + if (auto identifier = std::get_if(&m_tokens[instance].value); + identifier != nullptr && identifier->getType() == IdentifierType::Unknown) + setIdentifierColor(instance, definition.idType); + } + } + } + + bool TextHighlighter::findIdentifierDefinition( Definition &result, const std::string &optionalIdentifierName, std::string optionalName, bool setInstances) { + auto curr = m_curr; + bool isFunction = false; + auto tokenId = getTokenId(m_curr->location); + std::vector definitions; + std::string name = optionalName; + result.idType = IdentifierType::Unknown; + std::string identifierName = optionalIdentifierName; + + if (optionalIdentifierName == "") { + std::vector identifiers; + getFullName(identifierName, identifiers); + } + Interval tokenRange; + Scopes blocks; + Scopes::iterator blocksIterBegin, blocksIterEnd; + + if (findOrContains(name, m_UDTTokenRange, m_UDTVariables) && m_UDTVariables[name].contains(identifierName)) { + definitions = m_UDTVariables[name][identifierName]; + tokenRange = m_UDTTokenRange[name]; + blocksIterBegin = m_UDTBlocks[name].begin(); + blocksIterEnd = m_UDTBlocks[name].end(); + + } else if (findOrContains(name, m_functionTokenRange, m_functionVariables) && + m_functionVariables[name].contains(identifierName)) { + isFunction = true; + definitions = m_functionVariables[name][identifierName]; + tokenRange = m_functionTokenRange[name]; + blocksIterBegin = m_functionBlocks[name].begin(); + blocksIterEnd = m_functionBlocks[name].end(); + --blocksIterEnd; + + } else if (m_globalVariables.contains(identifierName)) { + definitions = m_globalVariables[identifierName]; + tokenRange = Interval(0, m_tokens.size()); + blocks.insert(tokenRange); + blocksIterBegin = blocks.begin(); + blocksIterEnd = blocks.end(); + } else if (name == "hex::type::Json" || name == "Object") { + result.idType = IdentifierType::LocalVariable; + result.typeStr = "Object"; + return true; + } + + if (isFunction) { + for (auto block = blocksIterBegin; block != blocksIterEnd; block++) { + + if (tokenId > block->start && tokenId < block->end) { + blocksIterBegin = block; + break; + } + } + for (auto definition : definitions) { + for (auto block = blocksIterBegin; block != blocksIterEnd; block++) { + + if (definition.tokenIndex > block->start && definition.tokenIndex < block->end) { + result = definition; + m_curr = curr; + + if (setInstances) + setBlockInstancesColor(identifierName, definition, *block); + return true; + } + } + } + auto it = std::find_if(definitions.begin(), definitions.end(), [&](const Definition &definition) { + return definition.tokenIndex > tokenRange.start && definition.tokenIndex < tokenRange.end; + }); + + if (it != definitions.end()) { + result = *it; + m_curr = curr; + + if (setInstances) + setBlockInstancesColor(identifierName, *it, tokenRange); + return true; + } + } else { + for (auto block = blocksIterBegin; block != blocksIterEnd; block++) { + + if (tokenId > block->start && tokenId < block->end) { + blocksIterBegin = block; + break; + } + } + for (auto block = blocksIterBegin; block != blocksIterEnd; block++) { + for (auto definition: definitions) { + + if (definition.tokenIndex > block->start && definition.tokenIndex < block->end) { + result = definition; + m_curr = curr; + + if (setInstances) + setBlockInstancesColor(identifierName, definition, *block); + return true; + } + } + } + } + m_curr = curr; + return false; + } + + using Definition = TextHighlighter::Definition; + + bool TextHighlighter::colorOperatorDotChain() { + std::vector identifiers; + std::string variableName; + auto tokenCount = m_tokens.size(); + + if (!getQualifiedName(variableName, identifiers, true)) + return false; + + auto vectorString = hex::splitString(variableName, "."); + u32 index = 0; + + + u32 currentLine = m_curr->location.line - 1; + u32 startingLineTokenIndex = m_firstTokenIdOfLine[currentLine]; + + if (startingLineTokenIndex == 0xFFFFFFFFu || startingLineTokenIndex > tokenCount) + return false; + + if (auto *keyword = std::get_if(&m_tokens[startingLineTokenIndex].value); + keyword != nullptr && *keyword == Keyword::Import) { + while (index < vectorString.size()) { + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, IdentifierType::NameSpace); + next(2); + index++; + } + return true; + } else { + std::string variableParentType; + Definition definition; + std::string currentName = vectorString[index]; + index++; + bool brokenChain = false; + + if (findIdentifierDefinition(definition, currentName)) { + variableParentType = definition.typeStr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, definition.idType); + skipArray(200, true); + next(); + } else { + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, IdentifierType::Unknown); + skipArray(200, true); + next(); + brokenChain = true; + } + + while (index < vectorString.size()) { + currentName = vectorString[index]; + next(); + Definition result=definition; + Definition parentDefinition = result; + + if (findIdentifierDefinition(result, currentName, variableParentType) && !brokenChain) { + variableParentType = result.typeStr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, result.idType); + skipArray(200, true); + next(); + } else if (auto udtVars = m_UDTVariables[result.typeStr];udtVars.contains(vectorString[index-1])) { + auto saveCurr = m_curr; + std::string templateName = ""; + auto instances = m_instances[variableParentType]; + for (auto instance : instances) { + if (auto *identifier = std::get_if(&m_tokens[instance].value); identifier != nullptr && identifier->getType() == IdentifierType::TemplateArgument) { + auto tokenRange = m_UDTTokenRange[result.typeStr]; + auto tokenIndex = m_firstTokenIdOfLine[getLocation(parentDefinition.tokenIndex).line - 1]; + i32 argNumber = getArgumentNumber(tokenRange.start, instance); + getTokenIdForArgument(tokenIndex, argNumber, tkn::Operator::BoolLessThan); + if (auto *identifier2 = std::get_if(&m_curr->value); identifier2 != nullptr) { + templateName = identifier2->get(); + break; + } + } + } + if (!templateName.empty() && findIdentifierDefinition(result, currentName, templateName) ) { + variableParentType = result.typeStr; + m_curr = saveCurr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, result.idType); + skipArray(200, true); + next(); + } else{ + if (m_typeDefMap.contains(variableParentType)) { + std::string typeName = ""; + instances = m_instances[variableParentType]; + for (auto instance: instances) { + if (auto *identifier = std::get_if(&m_tokens[instance].value); + identifier != nullptr && identifier->getType() == IdentifierType::Typedef) { + if (auto *identifier2 = std::get_if(&m_tokens[instance + 2].value); + identifier2 != nullptr) { + typeName = identifier2->get(); + break; + } + } + } + if (!typeName.empty() && findIdentifierDefinition(result, currentName, typeName)) { + variableParentType = result.typeStr; + m_curr = saveCurr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, result.idType); + skipArray(200, true); + next(); + } + } + } + } else { + brokenChain = true; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, IdentifierType::Unknown); + skipArray(200, true); + next(); + } + index++; + } + } + return true; + } + + bool TextHighlighter::colorSeparatorScopeChain() { + std::vector identifiers; + std::string identifierName; + + if (!getQualifiedName(identifierName, identifiers, true)) + return false; + auto tokenCount = m_tokens.size(); + auto vectorString = hex::splitString(identifierName, "::"); + auto vectorStringCount = vectorString.size(); + if (identifiers.size() != vectorStringCount) + return false; + auto curr = m_curr; + std::string nameSpace; + + for (u32 i = 0; i < vectorStringCount ; i++) { + auto name = vectorString[i]; + auto identifier = identifiers[i]; + + if (std::ranges::find(m_nameSpaces, name) != m_nameSpaces.end()) { + setIdentifierColor(-1, IdentifierType::NameSpace); + nameSpace += name + "::"; + } else if (m_UDTDefinitions.contains(nameSpace+name)) { + name = nameSpace + name; + auto udtDefinition = m_UDTDefinitions[name]; + auto definitionIndex = udtDefinition.tokenIndex-1; + if (auto *keyword = std::get_if(&m_tokens[definitionIndex].value); keyword != nullptr) { + setIdentifierColor(-1, IdentifierType::UDT); + if (*keyword == Keyword::Enum) { + next(); + if (!sequence(tkn::Operator::ScopeResolution) || vectorStringCount != i+2 || + !m_UDTVariables.contains(name)) + return false; + auto variableName = vectorString[i+1]; + if (!m_UDTVariables[name].contains(variableName)) + return false; + auto variableDefinition = m_UDTVariables[name][variableName][0]; + setIdentifierColor(-1, variableDefinition.idType); + return true; + } else + return true; + } else + return false; + + } else if (identifier->getType() == IdentifierType::Function) { + setIdentifierColor(-1, IdentifierType::Function); + return true; + } else if (std::ranges::find(m_UDTs, nameSpace+name) != m_UDTs.end()) { + setIdentifierColor(-1, IdentifierType::UDT); + if (vectorStringCount == i+1) + return true; + next(); + if (!sequence(tkn::Operator::ScopeResolution) || vectorStringCount != i+2) + return false; + setIdentifierColor(-1, IdentifierType::PatternVariable); + return true; + } else + return false; + next(2); + } + m_curr = curr; + + if (std::ranges::find(m_nameSpaces, identifierName) != m_nameSpaces.end()) { + setIdentifierColor(-1, IdentifierType::NameSpace); + return true; + } + + i32 index = getTokenId(m_curr->location); + + if (index < (i32) tokenCount - 1 && index > 2) { + auto nextToken = m_curr[1]; + auto *separator = std::get_if(&nextToken.value); + auto *operatortk = std::get_if(&nextToken.value); + + if ((separator != nullptr && *separator == Separator::Semicolon) || + (operatortk != nullptr && *operatortk == Operator::BoolLessThan)) { + auto previousToken = m_curr[-1]; + auto prevprevToken = m_curr[-2]; + operatortk = std::get_if(&previousToken.value); + auto *identifier2 = std::get_if(&prevprevToken.value); + + if (operatortk != nullptr && identifier2 != nullptr && *operatortk == Operator::ScopeResolution) { + + if (identifier2->getType() == IdentifierType::UDT) { + setIdentifierColor(-1, IdentifierType::LocalVariable); + return true; + + } else if (identifier2->getType() == IdentifierType::NameSpace) { + setIdentifierColor(-1, IdentifierType::UDT); + return true; + } + } + } + } + return false; + } + +// finds the name of the token range that the given or the current token index is in. + bool TextHighlighter::findScope(std::string &name, const UnorderedBlocks &map, i32 optionalTokenId) { + auto tokenId = optionalTokenId ==-1 ? getTokenId(m_curr->location) : optionalTokenId; + + for (auto [scopeName, range]: map) { + + if (range.contains(tokenId)) { + name = scopeName; + return true; + } + } + return false; + } + +// Finds the namespace of the given or the current token index. + bool TextHighlighter::findNamespace(std::string &nameSpace, i32 optionalTokenId) { + nameSpace = ""; + + for (auto [interval, name]: m_namespaceTokenRange) { + i32 tokenId = optionalTokenId == -1 ? getTokenId(m_curr->location) : optionalTokenId; + + if (tokenId > interval.start && tokenId < interval.end) { + + if (nameSpace == "") + nameSpace = name; + else + nameSpace = name + "::" + nameSpace; + } + } + + if (nameSpace != "") + return true; + return false; + } + +//The context is the name of the function or UDT that the variable is in. + std::string TextHighlighter::findIdentifierTypeStr(const std::string &identifierName, std::string context) { + Definition result; + findIdentifierDefinition(result, identifierName, context); + return result.typeStr; + } + + //The context is the name of the function or UDT that the variable is in. + TextHighlighter::IdentifierType TextHighlighter::findIdentifierType(const std::string &identifierName, std::string context) { + Definition result; + findIdentifierDefinition(result, identifierName, context); + return result.idType; + } + +// Creates a map from the attribute function to the type of the argument it takes. + void TextHighlighter::linkAttribute() { + auto curr = m_curr; + bool qualifiedAttribute = false; + using Types = std::map>; + auto parser = patternLanguage->get()->getInternals().parser.get(); + Types types = parser->getTypes(); + + while (sequence(tkn::Literal::Identifier, tkn::Operator::ScopeResolution)) + qualifiedAttribute = true; + + if (qualifiedAttribute) { + auto identifier = getValue(0); + + if (identifier != nullptr) + setIdentifierColor(-1, IdentifierType::Attribute); + m_curr = curr; + identifier = getValue(0); + + if (identifier != nullptr) + setIdentifierColor(-1, IdentifierType::NameSpace); + } else + m_curr = curr; + + std::string functionName; + next(); + + if (sequence(tkn::Separator::LeftParenthesis, tkn::Literal::String)) { + functionName = getValue(-1)->toString(false); + + if (!functionName.contains("::")) { + std::string namespaceName; + + if (findNamespace(namespaceName)) + functionName = namespaceName + "::" + functionName; + } else { + + auto vectorString = hex::splitString(functionName, "::"); + vectorString.pop_back(); + for (auto nameSpace: vectorString) { + + if (std::ranges::find(m_nameSpaces, nameSpace) == m_nameSpaces.end()) + m_nameSpaces.push_back(nameSpace); + } + } + } else + return; + + u32 line = m_curr->location.line; + i32 tokenIndex; + + while (!peek(tkn::Separator::Semicolon, -1)) { + + if (line = previousLine(line); line > m_firstTokenIdOfLine.size()-1) + return; + + if (tokenIndex = m_firstTokenIdOfLine[line]; !isTokenIdValid(tokenIndex)) + return; + + m_curr = m_startToken; + next(tokenIndex); + while (peek(tkn::Literal::Comment, -1) || peek(tkn::Literal::DocComment, -1)) + next(-1); + } + + while (peek(tkn::Literal::Comment) || peek(tkn::Literal::DocComment)) + next(); + + Identifier *identifier; + std::string UDTName; + while (sequence(tkn::Literal::Identifier, tkn::Operator::ScopeResolution)) { + identifier = getValue(-2); + UDTName += identifier->get() + "::"; + } + + if (sequence(tkn::Literal::Identifier)) { + identifier = getValue(-1); + UDTName += identifier->get(); + + if (!UDTName.contains("::")) { + std::string namespaceName; + + if (findNamespace(namespaceName)) + UDTName = namespaceName + "::" + UDTName; + } + if (types.contains(UDTName)) + m_attributeFunctionArgumentType[functionName] = UDTName; + + } else if (sequence(tkn::ValueType::Any)) { + auto valueType = getValue(-1); + m_attributeFunctionArgumentType[functionName] = Token::getTypeName(*valueType); + } else { + if (findScope(UDTName, m_UDTTokenRange) && !UDTName.empty()) + m_attributeFunctionArgumentType[functionName] = UDTName; + } + } + +// This function assumes that the first variable in the link that concatenates sequences including the Parent keyword started with Parent and was removed. Uses a function to find +// all the parents of a variable, If there are subsequent elements in the link that are Parent then for each parent it finds all the grandparents and puts them in a vector called +// parentTypes. It stops when a link that's not Parent is found amd only returns the last generation of parents. + bool TextHighlighter::findAllParentTypes(std::vector &parentTypes, + std::vector &identifiers, std::string &optionalFullName) { + auto fullName = optionalFullName; + + if (optionalFullName.empty()) + forwardIdentifierName(fullName, identifiers); + + auto nameParts = hex::splitString(fullName, "."); + std::vector grandpaTypes; + findParentTypes(parentTypes); + + if (parentTypes.empty()) { + auto curr = m_curr; + //must be a template instance of a type template instance + // find the name of the structure that is two < before. + u32 index = getTokenId(m_curr->location); + u32 maxIndex; + if (m_curr->location.line < m_firstTokenIdOfLine.size()) + maxIndex = m_firstTokenIdOfLine[m_curr->location.line]; + else + maxIndex = m_tokens.size() - 1; + u32 found = 0; + auto i=0; + do { + if (index >= maxIndex || i >= 200) { + m_curr = curr; + return false; + } + next(); + if (auto *operatortk = std::get_if(&m_curr->value);operatortk != nullptr && *operatortk == Operator::BoolGreaterThan) { + found++; + if (found == 2) + break; + } + index = getTokenId(m_curr->location); + i++; + } while (true); + + next(); + skipTemplate(200,false); + next(-1); + auto *identifier = std::get_if(&m_curr->value); + if (identifier != nullptr) { + auto identifierName = identifier->get(); + if (std::ranges::find(m_UDTs, identifierName) != m_UDTs.end()) { + if (std::ranges::find(parentTypes, identifierName) == parentTypes.end()) { + parentTypes.push_back(identifierName); + m_curr = curr; + return true; + } + } else if (std::ranges::find(m_nameSpaces, identifierName) == + m_nameSpaces.end()) { + m_nameSpaces.push_back(identifierName); + m_curr = curr; + return true; + } + } + } + if (parentTypes.empty()) + return false; + + auto currentName = nameParts[0]; + nameParts.erase(nameParts.begin()); + auto identifier = identifiers[0]; + identifiers.erase(identifiers.begin()); + + while (currentName == "Parent" && !nameParts.empty()) { + for (auto parentType: parentTypes) + findParentTypes(grandpaTypes, parentType); + + currentName = nameParts[0]; + nameParts.erase(nameParts.begin()); + identifier = identifiers[0]; + identifiers.erase(identifiers.begin()); + parentTypes = grandpaTypes; + grandpaTypes.clear(); + } + + nameParts.insert(nameParts.begin(), currentName); + identifiers.insert(identifiers.begin(), identifier); + optionalFullName = hex::combineStrings(nameParts, "."); + return true; + } + +// Searches for parents through every custom type,i.e. for structs that have members +// of the same type as the one being searched and places them in a vector called parentTypes. + bool TextHighlighter::findParentTypes(std::vector &parentTypes, const std::string &optionalUDTName) { + std::string UDTName; + std::string functionName; + bool isFunction = false; + if (optionalUDTName.empty()) { + if (!findScope(UDTName, m_UDTTokenRange)) { + if (!findScope(functionName, m_functionTokenRange)) { + return false; + } else { + isFunction = true; + } + } + } else + UDTName = optionalUDTName; + + bool found = false; + if (!isFunction) { + for (auto [name, variables]: m_UDTVariables) { + for (auto [variableName, definitions]: variables) { + for (auto definition: definitions) { + + if (definition.typeStr == UDTName) { + + if (std::ranges::find(parentTypes, name) == parentTypes.end()) { + parentTypes.push_back(name); + found = true; + } + } + } + } + } + } else { + auto curr = m_curr; + for (auto [name, range] : m_UDTTokenRange) { + auto startToken = TokenIter(m_tokens.begin()+range.start,m_tokens.begin()+range.end); + auto endToken = TokenIter(m_tokens.begin()+range.end,m_tokens.end()); + + for ( m_curr = startToken; endToken > m_curr; next()) { + if (auto *identifier = std::get_if(&m_curr->value); identifier != nullptr) { + auto identifierName = identifier->get(); + auto identifierType = identifier->getType(); + if (identifierName == functionName && identifierType == IdentifierType::Function) { + parentTypes.push_back(name); + found = true; + } + } + } + } + m_curr = curr; + } + return found; + } + +// this function searches all the parents recursively until it can match the variable name at the end of the chain +// and selects its type to colour the variable because the search only occurs pn type declarations which we know +// the types of. Once the end link is found then all the previous links are also assigned the types that were found +// for them during the search. + bool TextHighlighter::tryParentType(const std::string &parentType, std::string &variableName, + std::optional &result, + std::vector &identifiers) { + + auto vectorString = hex::splitString(variableName, "."); + auto count = vectorString.size(); + auto UDTName = parentType; + auto currentName = vectorString[0]; + + if (m_UDTVariables.contains(UDTName) && m_UDTVariables[UDTName].contains(currentName)) { + auto definitions = m_UDTVariables[UDTName][currentName]; + for (auto definition: definitions) { + UDTName = definition.typeStr; + + if (count == 1) { + setIdentifierColor(-1, definition.idType); + result = definition; + return true; + } + + vectorString.erase(vectorString.begin()); + variableName = hex::combineStrings(vectorString, "."); + Identifier *identifier = identifiers[0]; + identifiers.erase(identifiers.begin()); + skipArray(200, true); + next(2); + + if (tryParentType(UDTName, variableName, result, identifiers)) { + next(-1); + skipArray(200, false); + next(-1); + setIdentifierColor(-1, definition.idType); + return true; + } + + identifiers.insert(identifiers.begin(), identifier); + variableName += "." + currentName; + next(-1); + skipArray(200, false); + next(-1); + return false; + } + return false; + } else + return false; + return false; + } + +// Handles Parent keyword. + std::optional TextHighlighter::setChildrenTypes() { + auto curr = m_curr; + std::string fullName; + std::vector identifiers; + std::vector definitions; + std::optional result; + + forwardIdentifierName(fullName, identifiers); + + std::vector parentTypes; + auto vectorString = hex::splitString(fullName, "."); + if (vectorString[0] == "Parent") { + vectorString.erase(vectorString.begin()); + fullName = hex::combineStrings(vectorString, "."); + identifiers.erase(identifiers.begin()); + if (!findAllParentTypes(parentTypes, identifiers, fullName)) { + m_curr = curr; + return std::nullopt; + } + } else { + m_curr = curr; + return std::nullopt; + } + + for (auto parentType: parentTypes) { + m_curr = curr; + while (peek(tkn::Keyword::Parent)) + next(2); + + if (tryParentType(parentType, fullName, result, identifiers)) { + + if (result.has_value()) + definitions.push_back(result.value()); + } else { + m_curr = curr; + return std::nullopt; + } + } + // Todo: Are all definitions supposed to be the same? If not, which one should be used? + // for now, use the first one. + if (!definitions.empty()) + result = definitions[0]; + m_curr = curr; + return result; + } + + const TextHighlighter::TokenTypeColor TextHighlighter::m_tokenTypeColor = { + {Token::Type::Keyword, TextEditor::PaletteIndex::Keyword}, + {Token::Type::ValueType, TextEditor::PaletteIndex::BuiltInType}, + {Token::Type::Operator, TextEditor::PaletteIndex::Operator}, + {Token::Type::Separator, TextEditor::PaletteIndex::Separator}, + {Token::Type::String, TextEditor::PaletteIndex::StringLiteral}, + {Token::Type::Directive, TextEditor::PaletteIndex::Directive}, + {Token::Type::Comment, TextEditor::PaletteIndex::Comment}, + {Token::Type::Integer, TextEditor::PaletteIndex::NumericLiteral}, + {Token::Type::Identifier, TextEditor::PaletteIndex::Identifier}, + {Token::Type::DocComment, TextEditor::PaletteIndex::DocComment} + + }; + + const TextHighlighter::IdentifierTypeColor TextHighlighter::m_identifierTypeColor = { + {Identifier::IdentifierType::Macro, TextEditor::PaletteIndex::PreprocIdentifier}, + {Identifier::IdentifierType::UDT, TextEditor::PaletteIndex::UserDefinedType}, + {Identifier::IdentifierType::Function, TextEditor::PaletteIndex::Function}, + {Identifier::IdentifierType::Attribute, TextEditor::PaletteIndex::Attribute}, + {Identifier::IdentifierType::NameSpace, TextEditor::PaletteIndex::NameSpace}, + {Identifier::IdentifierType::Typedef, TextEditor::PaletteIndex::TypeDef}, + {Identifier::IdentifierType::PatternVariable, TextEditor::PaletteIndex::PatternVariable}, + {Identifier::IdentifierType::LocalVariable, TextEditor::PaletteIndex::LocalVariable}, + {Identifier::IdentifierType::CalculatedPointer, TextEditor::PaletteIndex::CalculatedPointer}, + {Identifier::IdentifierType::TemplateArgument, TextEditor::PaletteIndex::TemplateArgument}, + {Identifier::IdentifierType::PlacedVariable, TextEditor::PaletteIndex::PlacedVariable}, + {Identifier::IdentifierType::View, TextEditor::PaletteIndex::View}, + {Identifier::IdentifierType::FunctionVariable, TextEditor::PaletteIndex::FunctionVariable}, + {Identifier::IdentifierType::FunctionParameter, TextEditor::PaletteIndex::FunctionParameter}, + {Identifier::IdentifierType::Unknown, TextEditor::PaletteIndex::UnkIdentifier}, + {Identifier::IdentifierType::FunctionUnknown, TextEditor::PaletteIndex::UnkIdentifier}, + {Identifier::IdentifierType::MemberUnknown, TextEditor::PaletteIndex::UnkIdentifier}, + {Identifier::IdentifierType::ScopeResolutionUnknown, TextEditor::PaletteIndex::UnkIdentifier}, + {Identifier::IdentifierType::GlobalVariable, TextEditor::PaletteIndex::GlobalVariable}, + }; + +// Second paletteIndex called from processLineTokens to process literals + TextEditor::PaletteIndex TextHighlighter::getPaletteIndex(Literal *literal) { + + if (literal->isFloatingPoint() || literal->isSigned() || literal->isUnsigned()) + return TextEditor::PaletteIndex::NumericLiteral; + + else if (literal->isCharacter() || literal->isBoolean()) return TextEditor::PaletteIndex::CharLiteral; + + else if (literal->isString()) return TextEditor::PaletteIndex::StringLiteral; + + else return TextEditor::PaletteIndex::Default; + } + +// Render the compilation errors using squiggly lines + void TextHighlighter::renderErrors() { + const auto processMessage = [](const auto &message) { + auto lines = hex::splitString(message, "\n"); + + std::ranges::transform(lines, lines.begin(), [](auto line) { + + if (line.size() >= 128) + line = wolv::util::trim(line); + + return hex::limitStringLength(line, 128); + }); + + return hex::combineStrings(lines, "\n"); + }; + TextEditor::ErrorMarkers errorMarkers; + + if (!m_compileErrors.empty()) { + for (const auto &error: m_compileErrors) { + + if (isLocationValid(error.getLocation())) { + auto key = TextEditor::Coordinates(error.getLocation().line, error.getLocation().column); + + if (!errorMarkers.contains(key) || errorMarkers[key].first < error.getLocation().length) + errorMarkers[key] = std::make_pair(error.getLocation().length, processMessage(error.getMessage())); + } + } + } + m_viewPatternEditor->getTextEditor().SetErrorMarkers(errorMarkers); + } + +// creates a map from variable names to a vector of token indices +// od every instance of the variable name in the code. + void TextHighlighter::setInitialColors() { + + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + auto endToken = TokenIter(m_tokens.end(), m_tokens.end()); + for (m_curr = m_startToken; endToken > m_curr; next()) { + + if (peek(tkn::Literal::Identifier)) { + + if (auto identifier = getValue(0); identifier != nullptr) { + + if (auto identifierType = identifier->getType(); identifierType != IdentifierType::Unknown && identifierType != IdentifierType::MemberUnknown && + identifierType != IdentifierType::FunctionUnknown && identifierType != IdentifierType::ScopeResolutionUnknown) { + + setIdentifierColor(-1, identifierType); + } + } + } else if (peek(tkn::Separator::EndOfProgram)) + return; + } + } + + void TextHighlighter::loadInstances() { + std::map> instances; + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + auto endToken = TokenIter(m_tokens.end(), m_tokens.end()); + for (m_curr = m_startToken; endToken > m_curr; next()) { + + if (peek(tkn::Literal::Identifier)) { + std::string name; + + if (auto identifier = getValue(0); identifier != nullptr) { + + if (auto identifierType = identifier->getType(); identifierType != IdentifierType::Unknown && identifierType != IdentifierType::MemberUnknown && + identifierType != IdentifierType::FunctionUnknown && identifierType != IdentifierType::ScopeResolutionUnknown) { + name = identifier->get(); + + if (identifierType == IdentifierType::Typedef) { + auto curr = m_curr; + next(); + std::string typeName = ""; + if (sequence(tkn::Operator::Assign, tkn::Literal::Identifier)) { + auto identifier2 = getValue(-1); + if (identifier2 != nullptr) { + typeName = identifier2->get(); + if (!m_typeDefMap.contains(name) && !typeName.empty()) { + m_typeDefMap[name] = typeName; + m_typeDefInvMap[typeName] = name; + } + } + } + + m_curr = curr; + } + } else { + name = identifier->get(); + auto curr = m_curr; + auto tokenIndex = getTokenId(m_curr->location); + skipArray(200, true); + next(); + bool chainStarted = false; + while (sequence(tkn::Operator::ScopeResolution, tkn::Literal::Identifier)) { + + if (identifier = getValue(-1); identifier != nullptr) + name += "::" + identifier->get(); + + if (!chainStarted) { + chainStarted = true; + m_scopeChains.insert(tokenIndex); + } + curr = m_curr; + } + while (sequence(tkn::Separator::Dot, tkn::Literal::Identifier)) { + + if (identifier = getValue(-1); identifier != nullptr) + name += "." + identifier->get(); + + if (!chainStarted) { + chainStarted = true; + m_memberChains.insert(tokenIndex); + } + skipArray(200, true); + curr = m_curr; + } + m_curr = curr; + } + } + auto id = getTokenId(m_curr->location); + + if (instances.contains(name)) { + auto &nameInstances = instances[name]; + + if (std::ranges::find(nameInstances, id) == nameInstances.end()) + nameInstances.push_back(id); + } else + instances[name].push_back(id); + } else if (peek(tkn::Separator::EndOfProgram)) + break; + } + m_instances = std::move(instances); + } + +// Get the location of a given token index + pl::core::Location TextHighlighter::getLocation(i32 tokenId) { + + if (tokenId >= (i32) m_tokens.size()) + return Location::Empty(); + return m_tokens[tokenId].location; + } + +// Get the token index for a given location. + i32 TextHighlighter::getTokenId(pl::core::Location location) { + + if (!isLocationValid(location)) + return -1; + auto line1 = location.line - 1; + auto line2 = nextLine(line1); + auto tokenCount = m_tokens.size(); + i32 tokenStart = m_firstTokenIdOfLine[line1]; + i32 tokenEnd = m_firstTokenIdOfLine[line2] - 1; + + if (tokenEnd >= (i32) tokenCount) + tokenEnd = tokenCount - 1; + + if (tokenStart == -1 || tokenEnd == -1 || tokenStart >= (i32) tokenCount) + return -1; + + for (i32 i = tokenStart; i <= tokenEnd; i++) { + + if (m_tokens[i].location.column >= location.column) + return i; + } + return -1; + } + + void TextHighlighter::setIdentifierColor(i32 tokenId, const IdentifierType &type) { + Token *token; + + if (tokenId == -1) + token = const_cast(&m_curr[0]); + else + token = const_cast(&m_tokens[tokenId]); + + if (token->type == Token::Type::Identifier && (!m_tokenColors.contains(token) || m_tokenColors.at(token) == TextEditor::PaletteIndex::Default || m_tokenColors.at(token) == TextEditor::PaletteIndex::UnkIdentifier)) + m_tokenColors[token] = m_identifierTypeColor.at(type); + else if (!m_tokenColors.contains(token)) + m_tokenColors[token] = m_tokenTypeColor.at(token->type); + } + + void TextHighlighter::setColor(i32 tokenId, const IdentifierType &type) { + Token *token; + + if (tokenId == -1) + token = const_cast(&m_curr[0]); + else + token = const_cast(&m_tokens[tokenId]); + + if (token->type == Token::Type::Integer) { + auto literal = getValue(0); + + if (literal != nullptr && !m_tokenColors.contains(token)) + m_tokenColors[token] = getPaletteIndex(literal); + + + } else if (token->type == Token::Type::DocComment) { + auto docComment = getValue(0); + + if (docComment != nullptr && !m_tokenColors.contains(token)) { + + if (docComment->singleLine) + m_tokenColors[token] = TextEditor::PaletteIndex::DocComment; + else if (docComment->global) + m_tokenColors[token] = TextEditor::PaletteIndex::GlobalDocComment; + else + m_tokenColors[token] = TextEditor::PaletteIndex::DocBlockComment; + } + } else if (token->type == Token::Type::Comment) { + auto comment = getValue(0); + + if (comment != nullptr && !m_tokenColors.contains(token)) { + + if (comment->singleLine) + m_tokenColors[token] = TextEditor::PaletteIndex::Comment; + else + m_tokenColors[token] = TextEditor::PaletteIndex::BlockComment; + } + } else + setIdentifierColor(tokenId, type); + } + + void TextHighlighter::colorRemainingIdentifierTokens() { + std::vector taggedIdentifiers; + for (auto index: m_taggedIdentifiers) { + taggedIdentifiers.push_back(index); + } + m_taggedIdentifiers.clear(); + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + auto endToken = TokenIter(m_tokens.end(), m_tokens.end()); + m_curr = m_startToken; + + while (endToken > m_curr) { + if (peek(tkn::Separator::EndOfProgram)) + return; + i32 tokenId = getTokenId(m_curr->location); + + if (!taggedIdentifiers.empty() && tokenId > taggedIdentifiers.back()) { + next(taggedIdentifiers.back() - tokenId); + taggedIdentifiers.pop_back(); + } + + Token *token = const_cast(&m_curr[0]); + + if (sequence(tkn::Keyword::Import, tkn::Literal::Identifier)) { + next(-1); + do { + if (auto identifier = const_cast(getValue(0)); identifier != nullptr) { + setIdentifierColor(-1, IdentifierType::NameSpace); + if (std::ranges::find(m_nameSpaces, identifier->get()) == m_nameSpaces.end()) { + m_nameSpaces.push_back(identifier->get()); + } + } + } while (sequence(tkn::Literal::Identifier,tkn::Separator::Dot)); + next(); + if (sequence(tkn::Keyword::As, tkn::Literal::Identifier)) { + next(-1); + if (auto identifier = const_cast(getValue(0)); identifier != nullptr) { + setIdentifierColor(-1, IdentifierType::NameSpace); + if (std::ranges::find(m_nameSpaces, identifier->get()) == m_nameSpaces.end()) { + m_nameSpaces.push_back(identifier->get()); + } + } + } + } + if (peek(tkn::Literal::Identifier)) { + auto identifier = getValue(0); + Token::Identifier::IdentifierType identifierType; + if (identifier == nullptr) { + next(); + continue; + } + identifierType = identifier->getType(); + std::string variableName = identifier->get(); + + if (m_tokenColors.contains(token) && (m_tokenColors.at(token) != TextEditor::PaletteIndex::Default && identifierType != IdentifierType::Unknown)) { + next(); + continue; + } + Definition definition; + + if (peek(tkn::Keyword::Parent, -2)) { + auto save = m_curr; + next(-2); + while (peek(tkn::Keyword::Parent, -2)) + next(-2); + auto optional = setChildrenTypes(); + + if (optional.has_value()) + setIdentifierColor(-1, optional->idType); + else { + m_curr = save; + setIdentifierColor(-1, IdentifierType::Unknown); + next(); + continue; + } + m_curr = save; + next(); + continue; + } else if (peek(tkn::Operator::ScopeResolution, 1)) { + if (std::ranges::find(m_nameSpaces, variableName) != m_nameSpaces.end()) { + setIdentifierColor(-1, IdentifierType::NameSpace); + next(); + continue; + } + } else if (peek(tkn::Operator::ScopeResolution, -1)) { + auto save = m_curr; + next(-2); + if (auto parentIdentifier = const_cast(getValue(0)); parentIdentifier != nullptr) { + next(2); + if (parentIdentifier->getType() == IdentifierType::UDT) { + auto parentName = parentIdentifier->get(); + auto typeName = findIdentifierType(variableName, parentName); + setIdentifierColor(-1, typeName); + } + } + m_curr = save; + next(); + continue; + } else if (findIdentifierDefinition(definition)) { + identifierType = definition.idType; + setIdentifierColor(-1, identifierType); + next(); + continue; + } else if (std::ranges::find(m_UDTs, variableName) != m_UDTs.end()) { + if (m_typeDefMap.contains(variableName)) + setIdentifierColor(-1, IdentifierType::Typedef); + else + setIdentifierColor(-1, IdentifierType::UDT); + next(); + continue; + } else if (peek(tkn::Keyword::From, -1)) { + setIdentifierColor(-1, IdentifierType::GlobalVariable); + next(); + continue; + } else { + setIdentifierColor(-1, IdentifierType::Unknown); + next(); + continue; + } + } + next(); + } + } + + void TextHighlighter::setRequestedIdentifierColors() { + auto topLine = 0; + auto bottomLine = m_lines.size(); + for (u32 line = topLine; line < bottomLine; line = nextLine(line)) { + if (m_lines[line].empty()) + continue; + std::string lineOfColors = std::string(m_lines[line].size(), 0); + for (auto tokenIndex = m_firstTokenIdOfLine[line]; tokenIndex < m_firstTokenIdOfLine[nextLine(line)]; tokenIndex++) { + Token *token = const_cast(&m_tokens[tokenIndex]); + if (m_tokenColors.contains(token) && token->type == Token::Type::Identifier) { + u8 color = (u8) m_tokenColors.at(token); + u32 tokenLength = token->location.length; + u32 tokenOffset = token->location.column - 1; + if (token->location.line != line + 1) + continue; + if (tokenOffset + tokenLength - 1 >= m_lines[line].size()) + continue; + for (u32 j = 0; j < tokenLength; j++) + lineOfColors[tokenOffset + j] = color; + } + } + m_viewPatternEditor->getTextEditor().SetColorizedLine(line, lineOfColors); + } + } + + void TextHighlighter::recurseInheritances(std::string name) { + if (m_inheritances.contains(name)) { + for (auto inheritance: m_inheritances[name]) { + recurseInheritances(inheritance); + auto definitions = m_UDTVariables[inheritance]; + for (auto [variableName, variableDefinitions]: definitions) { + auto tokenRange = m_UDTTokenRange[name]; + u32 tokenIndex = tokenRange.start; + for (auto token = tokenRange.start; token < tokenRange.end; token++) { + + if (auto operatorTkn = std::get_if(&m_tokens.at(token).value); + operatorTkn != nullptr && *operatorTkn == Token::Operator::Colon) + tokenIndex = token + 1; + } + for (auto variableDefinition: variableDefinitions) { + variableDefinition.tokenIndex = tokenIndex; + m_UDTVariables[name][variableName].push_back(variableDefinition); + } + } + } + } + } + + void TextHighlighter::appendInheritances() { + for (auto [name, inheritances]: m_inheritances) + recurseInheritances(name); + } + +// Get the string of the argument type. This works on function arguments and non-type template arguments. + std::string TextHighlighter::getArgumentTypeName(i32 rangeStart, Token delimiter2) { + auto curr = m_curr; + i32 parameterIndex = getArgumentNumber(rangeStart, getTokenId(m_curr->location)); + Token delimiter; + std::string typeStr; + + if (parameterIndex > 0) + delimiter = tkn::Separator::Comma; + else + delimiter = delimiter2; + + while (!peek(delimiter)) + next(-1); + skipToken(tkn::Keyword::Reference); + next(); + + if (peek(tkn::ValueType::Any)) + typeStr = Token::getTypeName(*getValue(0)); + else if (peek(tkn::Literal::Identifier)) + typeStr = getValue(0)->get(); + + m_curr = curr; + return typeStr; + } + + bool TextHighlighter::isTokenIdValid(i32 tokenId) { + return tokenId >= 0 && tokenId < (i32) m_tokens.size(); + } + + bool TextHighlighter::isLocationValid(hex::plugin::builtin::TextHighlighter::Location location) { + const pl::api::Source *source; + try { + source = location.source; + } catch (const std::out_of_range &e) { + log::error("TextHighlighter::IsLocationValid: Out of range error: {}", e.what()); + return false; + } + if (source == nullptr || !source->mainSource) + return false; + i32 line = location.line - 1; + i32 col = location.column - 1; + i32 length = location.length; + + if ( line < 0 || line >= (i32) m_lines.size()) + return false; + + if ( col < 0 || col > (i32) m_lines[line].size()) + return false; + + if (length < 0 || length > (i32) m_lines[line].size()-col) + return false; + return true; + } + +// Find the string of the variable type. This works on function variables, views, +// local variables as well as on calculated pointers and pattern variables. + std::string TextHighlighter::getVariableTypeName() { + auto curr = m_curr; + auto varTokenId = getTokenId(m_curr->location); + + if (!isTokenIdValid(varTokenId)) + return ""; + + std::string typeStr; + skipToken(tkn::Operator::Star, -1); + + while (peek(tkn::Separator::Comma, -1)) + next(-2); + + if (peek(tkn::ValueType::Any, -1)) + typeStr = Token::getTypeName(*getValue(-1)); + else if (peek(tkn::Keyword::Signed, -1)) + typeStr = "signed"; + else if (peek(tkn::Keyword::Unsigned, -1)) + typeStr = "unsigned"; + else { + skipTemplate(200, false); + next(-1); + + if (peek(tkn::Literal::Identifier)) { + typeStr = getValue(0)->get(); + next(-1); + } + std::string nameSpace; + while (peek(tkn::Operator::ScopeResolution)) { + next(-1); + nameSpace = getValue(0)->get() + "::" + nameSpace; + next(-1); + } + typeStr = nameSpace + typeStr; + using Types = std::map>; + auto parser = patternLanguage->get()->getInternals().parser.get(); + Types types = parser->getTypes(); + + if (types.contains(typeStr)) { + m_curr = curr; + return typeStr; + } + std::vector candidates; + for (auto name: m_UDTs) { + auto vectorString = hex::splitString(name, "::"); + + if (typeStr == vectorString.back()) + candidates.push_back(name); + } + + if (candidates.size() == 1) { + m_curr = curr; + return candidates[0]; + } + } + m_curr = curr; + return typeStr; + } + +// Definitions of global variables and placed variables. + void TextHighlighter::loadGlobalDefinitions( Scopes tokenRangeSet, std::vector identifierTypes, Variables &variables) { + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + + for (auto range: tokenRangeSet) { + auto startToken = TokenIter(m_tokens.begin()+range.start,m_tokens.begin()+range.end); + auto endToken = TokenIter(m_tokens.begin()+range.end,m_tokens.end()); + + for ( m_curr = startToken; endToken > m_curr; next()) { + + if (peek(tkn::Literal::Identifier)) { + auto identifier = getValue(0); + auto identifierType = identifier->getType(); + auto identifierName = identifier->get(); + + if (std::ranges::find(identifierTypes, identifierType) != identifierTypes.end()) { + std::string typeStr = getVariableTypeName(); + + if (typeStr.empty()) + continue; + i32 tokenId = getTokenId(m_curr->location); + Definition definition(identifierType, typeStr, tokenId, m_curr->location); + variables[identifierName].push_back(definition); + continue; + } + } + } + } + } + +// Definitions of variables and arguments in functions and user defined types. + void TextHighlighter::loadVariableDefinitions(UnorderedBlocks tokenRangeMap, Token delimiter1, Token delimiter2, + std::vector identifierTypes, + bool isArgument, VariableMap &variableMap) { + for (auto [name, range]: tokenRangeMap) { + m_curr = m_startToken; + auto endToken = m_startToken; + next(range.start); + + if (isArgument) { + while (!peek(delimiter1)) { + + if (peek(tkn::Separator::LeftBrace)) + break; + next(); + } + + if (peek(tkn::Separator::LeftBrace)) + continue; + endToken = m_curr; + while (!peek(delimiter2)) { + + if (peek(tkn::Separator::LeftBrace)) + break; + next(); + } + + if (peek(tkn::Separator::LeftBrace)) + continue; + auto temp = m_curr; + m_curr = endToken; + endToken = temp; + } else + endToken = endToken + range.end; + + Keyword *keyword; + for (keyword = std::get_if(&m_tokens[range.start].value); endToken > m_curr; next()) { + + if (peek(tkn::Literal::Identifier)) { + auto identifier = getValue(0); + + if (identifier == nullptr) + continue; + auto identifierType = identifier->getType(); + auto identifierName = identifier->get(); + + if (std::ranges::find(identifierTypes, identifierType) != identifierTypes.end()) { + std::string typeStr; + + if (keyword != nullptr && (*keyword == Keyword::Enum)) { + typeStr = name; + } else if (isArgument) { + typeStr = getArgumentTypeName(range.start, delimiter1); + } else { + typeStr = getVariableTypeName(); + if (typeStr.empty() && keyword != nullptr && *keyword == Keyword::Bitfield) + typeStr = "bits"; + if (m_typeDefMap.contains(typeStr)) + typeStr = m_typeDefMap[typeStr]; + } + + if (typeStr.empty()) + continue; + Definition definition(identifierType, typeStr, getTokenId(m_curr->location), m_curr->location); + variableMap[name][identifierName].push_back(definition); + continue; + } + } + } + } + } + +// Definitions of user defined types and functions. + void TextHighlighter::loadTypeDefinitions( UnorderedBlocks tokenRangeMap, std::vector identifierTypes, Definitions &types) { + for (auto [name, range]: tokenRangeMap) { + + m_curr = m_startToken + range.start+1; + + if (!peek(tkn::Literal::Identifier)) + continue; + auto identifier = getValue(0); + if (identifier == nullptr) + continue; + auto identifierType = identifier->getType(); + + if (std::ranges::find(identifierTypes, identifierType) == identifierTypes.end()) + continue; + auto identifierName = identifier->get(); + if (!name.ends_with(identifierName)) + continue; + types[name] = ParentDefinition(identifierType, getTokenId(m_curr->location), m_curr->location); + } + } + +// Once types are loaded from parsed tokens we can create +// maps of variable names to their definitions. + void TextHighlighter::getDefinitions() { + using IdentifierType::LocalVariable; + using IdentifierType::PatternVariable; + using IdentifierType::CalculatedPointer; + using IdentifierType::GlobalVariable; + using IdentifierType::PlacedVariable; + using IdentifierType::View; + using IdentifierType::FunctionVariable; + using IdentifierType::FunctionParameter; + using IdentifierType::TemplateArgument; + using IdentifierType::UDT; + using IdentifierType::Function; + + if (!m_UDTDefinitions.empty()) + m_UDTDefinitions.clear(); + loadTypeDefinitions(m_UDTTokenRange, {UDT}, m_UDTDefinitions); + + if (!m_globalVariables.empty()) + m_globalVariables.clear(); + loadGlobalDefinitions(m_globalTokenRange, {GlobalVariable, PlacedVariable}, m_globalVariables); + + if (!m_UDTVariables.empty()) + m_UDTVariables.clear(); + loadVariableDefinitions(m_UDTTokenRange, tkn::Operator::BoolLessThan, tkn::Operator::BoolGreaterThan, + {TemplateArgument}, true, m_UDTVariables); + + loadVariableDefinitions(m_UDTTokenRange, tkn::Operator::BoolLessThan, tkn::Operator::BoolGreaterThan, + {LocalVariable, PatternVariable, CalculatedPointer}, false, m_UDTVariables); + appendInheritances(); + + if (!m_functionDefinitions.empty()) + m_functionDefinitions.clear(); + loadTypeDefinitions(m_functionTokenRange, {Function}, m_functionDefinitions); + + if (!m_functionVariables.empty()) + m_functionVariables.clear(); + loadVariableDefinitions(m_functionTokenRange, tkn::Separator::LeftParenthesis, tkn::Separator::RightParenthesis, + {FunctionParameter}, true, m_functionVariables); + + loadVariableDefinitions(m_functionTokenRange, tkn::Separator::LeftParenthesis, tkn::Separator::RightParenthesis, + {View,FunctionVariable}, false, m_functionVariables); + } + + +// Load the source code into the text highlighter, splits +// the text into lines and creates a lookup table for the +// first token id of each line. + void TextHighlighter::loadText() { + + if (!m_lines.empty()) + m_lines.clear(); + + if (m_text.empty()) + m_text = m_viewPatternEditor->getTextEditor().PreprocessText(m_viewPatternEditor->getTextEditor().GetText()); + + m_lines = hex::splitString(m_text, "\n"); + m_lines.push_back(""); + m_firstTokenIdOfLine.clear(); + m_firstTokenIdOfLine.resize(m_lines.size(), -1); + + i32 tokenId = 0; + i32 tokenCount = m_tokens.size(); + i32 index; + + if (tokenCount > 0) { + index = m_tokens[0].location.line - 1; + m_firstTokenIdOfLine[index] = 0; + } + i32 count = m_lines.size(); + for (i32 currentLine = 0; currentLine < count; currentLine++) { + for (index = m_tokens[tokenId].location.line - 1; index <= currentLine && tokenId + 1 < tokenCount; tokenId++) { + index = m_tokens[tokenId + 1].location.line - 1; + } + + if (index > currentLine) { + m_firstTokenIdOfLine[index] = tokenId; + } + } + + if (m_firstTokenIdOfLine.back() != tokenCount) + m_firstTokenIdOfLine.push_back(tokenCount); + } + +// Some tokens span many lines and some lines have no tokens. This +// function helps to find the next line number in the inner loop. + u32 TextHighlighter::nextLine(u32 line) { + auto currentTokenId = m_firstTokenIdOfLine[line]; + u32 i = 1; + while (line + i < m_lines.size() && + (m_firstTokenIdOfLine[line + i] == currentTokenId || m_firstTokenIdOfLine[line + i] == (i32) 0xFFFFFFFF)) + i++; + return i + line; + } + + u32 TextHighlighter::previousLine(u32 line) { + auto currentTokenId = m_firstTokenIdOfLine[line]; + u32 i = 1; + while (line - i < m_lines.size() && + (m_firstTokenIdOfLine[line - i] == currentTokenId || m_firstTokenIdOfLine[line - i] == (i32) 0xFFFFFFFF)) + i++; + return line - i; + } + +// global token ranges are the complement (aka inverse) of the union +// of the UDT and function token ranges + void TextHighlighter::invertGlobalTokenRange() { + std::set ranges; + auto size = m_globalTokenRange.size(); + auto tokenCount = m_tokens.size(); + + if (size == 0) { + ranges.insert(Interval(0, tokenCount)); + } else { + auto it = m_globalTokenRange.begin(); + auto it2 = std::next(it); + if (it->start != 0) + ranges.insert(Interval(0, it->start)); + while (it2 != m_globalTokenRange.end()) { + + if (it->end < it2->start) + ranges.insert(Interval(it->end, it2->start)); + else + ranges.insert(Interval(it->start, it2->end)); + it = it2; + it2 = std::next(it); + } + + if (it->end < (i32) (tokenCount-1)) + ranges.insert(Interval(it->end, tokenCount-1)); + } + m_globalTokenRange = ranges; + } + +// 0 for 1st argument, 1 for 2nd argument, etc. Obtained counting commas. + i32 TextHighlighter::getArgumentNumber(i32 start, i32 arg) { + i32 count = 0; + m_curr = m_startToken; + auto endToken = m_startToken + arg; + next(start); + while (endToken > m_curr) { + + if (peek(tkn::Separator::Comma)) + count++; + next(); + } + return count; + } + +// The inverse of getArgumentNumber. + void TextHighlighter::getTokenIdForArgument(i32 start, i32 argNumber, Token delimiter) { + m_curr = m_startToken; + next(start); + while (!peek(delimiter)) + next(); + next(); + i32 count = 0; + while (count < argNumber && !peek(tkn::Separator::EndOfProgram)) { + + if (peek(tkn::Separator::Comma)) + count++; + next(); + } + } + +// Changes auto type string in definitions to the actual type string. + void TextHighlighter::resolveAutos(VariableMap &variableMap, UnorderedBlocks &tokenRange) { + auto curr = m_curr; + std::string UDTName; + for (auto &[name, variables]: variableMap) { + for (auto &[variableName, definitions]: variables) { + for (auto &definition: definitions) { + + if (definition.typeStr == "auto" && (definition.idType == Token::Identifier::IdentifierType::TemplateArgument || definition.idType == Token::Identifier::IdentifierType::FunctionParameter)) { + auto argumentIndex = getArgumentNumber(tokenRange[name].start, definition.tokenIndex); + + if (tokenRange == m_UDTTokenRange || !m_attributeFunctionArgumentType.contains(name) || + m_attributeFunctionArgumentType[name].empty()) { + + auto instances = m_instances[name]; + for (auto instance: instances) { + + if (std::abs(definition.tokenIndex - instance) <= 5) + continue; + Token delimiter; + + if (tokenRange == m_UDTTokenRange) + delimiter = tkn::Operator::BoolLessThan; + else + delimiter = tkn::Separator::LeftParenthesis; + std::string fullName; + std::vector identifiers; + getTokenIdForArgument(instance, argumentIndex,delimiter); + forwardIdentifierName(fullName, identifiers); + + if (fullName.starts_with("Parent.")) { + auto fixedDefinition = setChildrenTypes(); + + if (fixedDefinition.has_value() && + m_UDTDefinitions.contains(fixedDefinition->typeStr)) { + definition.typeStr = fixedDefinition->typeStr; + continue; + } + } else if (fullName.contains(".")) { + Definition definitionTemp; + resolveIdentifierType(definitionTemp,fullName); + definition.typeStr = definitionTemp.typeStr; + } else { + auto typeName = findIdentifierTypeStr(fullName); + definition.typeStr = typeName; + } + } + } else { + UDTName = m_attributeFunctionArgumentType[name]; + if (m_UDTDefinitions.contains(UDTName)) { + definition.typeStr = UDTName; + continue; + } + } + } + } + } + } + m_curr = curr; + } + + void TextHighlighter::fixAutos() { + resolveAutos(m_functionVariables, m_functionTokenRange); + resolveAutos(m_UDTVariables, m_UDTTokenRange); + } + + void TextHighlighter::fixChains() { + + if (!m_scopeChains.empty()) { + for (auto chain: m_scopeChains) { + m_curr = m_startToken + chain; + colorSeparatorScopeChain(); + } + } + + if (!m_memberChains.empty()) { + for (auto chain: m_memberChains) { + m_curr = m_startToken + chain; + colorOperatorDotChain(); + } + } + } + +// Calculates the union of all the UDT and function token ranges +// and inverts the result. + void TextHighlighter::getGlobalTokenRanges() { + std::set ranges; + for (auto [name, range]: m_UDTTokenRange) + ranges.insert(range); + for (auto [name, range]: m_functionTokenRange) + ranges.insert(range); + + if (ranges.empty()) + return; + + auto it = ranges.begin(); + auto next = std::next(it); + while (next != ranges.end()) { + + if (next->start - it->end < 2) { + Interval &range = const_cast(*it); + range.end = next->end; + ranges.erase(next); + next = std::next(it); + } else { + it++; + next = std::next(it); + } + } + m_globalTokenRange = ranges; + invertGlobalTokenRange(); + for (auto tokenRange: m_globalTokenRange) { + + if ((u32) tokenRange.end == m_tokens.size()) { + tokenRange.end -= 1; + m_globalBlocks.insert(tokenRange); + } + } + } + +// Parser labels global variables that are not placed as +// function variables. + void TextHighlighter::fixGlobalVariables() { + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + for (auto range: m_globalTokenRange) { + auto startToken = TokenIter(m_tokens.begin() + range.start, m_tokens.begin() + range.end); + auto endToken = TokenIter(m_tokens.begin() + range.end, m_tokens.end()); + + for (m_curr = startToken; endToken > m_curr; next()) { + + if (auto identifier = getValue(0); identifier != nullptr) { + auto identifierType = identifier->getType(); + auto identifierName = identifier->get(); + + if (identifierType == IdentifierType::FunctionVariable) { + identifier->setType(IdentifierType::GlobalVariable, true); + setIdentifierColor(-1, IdentifierType::GlobalVariable); + } else if (identifierType == IdentifierType::View) { + identifier->setType(IdentifierType::PlacedVariable, true); + setIdentifierColor(-1,IdentifierType::PlacedVariable); + } else if (identifierType == IdentifierType::Unknown) { + if (std::ranges::find(m_UDTs,identifierName) != m_UDTs.end()) { + identifier->setType(IdentifierType::UDT, true); + setIdentifierColor(-1, IdentifierType::UDT); + } + } + } + } + } + } + +// Only update if needed. Must wait for the parser to finish first. + void TextHighlighter::highlightSourceCode() { + m_wasInterrupted = false; + ON_SCOPE_EXIT { + if (!m_tokenColors.empty()) + m_tokenColors.clear(); + m_runningColorizers--; + if (m_wasInterrupted) { + m_needsToUpdateColors = true; + m_viewPatternEditor->setChangesWereParsed(true); + } else { + m_needsToUpdateColors = false; + m_viewPatternEditor->setChangesWereParsed(false); + } + }; + try { + m_runningColorizers++; + auto preprocessor = patternLanguage->get()->getInternals().preprocessor.get(); + auto parser = patternLanguage->get()->getInternals().parser.get(); + using Types = std::map>; + Types types = parser->getTypes(); + + if (!m_UDTs.empty()) + m_UDTs.clear(); + for (auto &[name, type]: types) + m_UDTs.push_back(name); + m_tokens = preprocessor->getResult(); + if (m_tokens.empty()) + return; + auto lastToken = m_tokens.back(); + m_tokens.push_back(lastToken); + + m_text = m_viewPatternEditor->getTextEditor().PreprocessText( m_viewPatternEditor->getTextEditor().GetText()); + + if (m_text.empty() || m_text == "\n") + return; + loadText(); + + if (!m_globalTokenRange.empty()) + m_globalTokenRange.clear(); + m_globalTokenRange.insert(Interval(0, m_tokens.size()-1)); + + // Namespaces from included files. + m_nameSpaces.clear(); + m_nameSpaces = preprocessor->getNamespaces(); + + if (!m_inheritances.empty()) + m_inheritances.clear(); + + if (!m_namespaceTokenRange.empty()) + m_namespaceTokenRange.clear(); + getAllTokenRanges(IdentifierType::NameSpace); + + if (!m_UDTBlocks.empty()) + m_UDTBlocks.clear(); + + if (!m_UDTTokenRange.empty()) + m_UDTTokenRange.clear(); + getAllTokenRanges(IdentifierType::UDT); + + if (!m_functionBlocks.empty()) + m_functionBlocks.clear(); + + if (!m_functionTokenRange.empty()) + m_functionTokenRange.clear(); + getAllTokenRanges(IdentifierType::Function); + + if (!m_globalBlocks.empty()) + m_globalBlocks.clear(); + getGlobalTokenRanges(); + + fixGlobalVariables(); + if (!m_scopeChains.empty()) + m_scopeChains.clear(); + + if (!m_memberChains.empty()) + m_memberChains.clear(); + + setInitialColors(); + loadInstances(); + + if (!m_attributeFunctionArgumentType.empty()) + m_attributeFunctionArgumentType.clear(); + + getAllTokenRanges(IdentifierType::Attribute); + getDefinitions(); + fixAutos(); + fixChains(); + + m_excludedLocations = preprocessor->getExcludedLocations(); + + colorRemainingIdentifierTokens(); + setRequestedIdentifierColors(); + + m_viewPatternEditor->getTextEditor().SetTimeStamp(1); + m_viewPatternEditor->getTextEditor().ClearErrorMarkers(); + m_compileErrors = patternLanguage->get()->getCompileErrors(); + + if (!m_compileErrors.empty()) + renderErrors(); + else + m_viewPatternEditor->getTextEditor().ClearErrorMarkers(); + } catch (const std::out_of_range &e) { + log::debug("TextHighlighter::highlightSourceCode: Out of range error: {}", e.what()); + m_wasInterrupted = true; + return; + } + return; + } +} \ No newline at end of file diff --git a/plugins/builtin/source/content/themes.cpp b/plugins/builtin/source/content/themes.cpp index b859304b8..f2588a885 100644 --- a/plugins/builtin/source/content/themes.cpp +++ b/plugins/builtin/source/content/themes.cpp @@ -211,33 +211,53 @@ namespace hex::plugin::builtin { } ); } - { const static ThemeManager::ColorMap TextEditorColorMap = { - { "default", u32(TextEditor::PaletteIndex::Default) }, - { "keyword", u32(TextEditor::PaletteIndex::Keyword) }, - { "number", u32(TextEditor::PaletteIndex::Number) }, - { "string", u32(TextEditor::PaletteIndex::String) }, - { "char-literal", u32(TextEditor::PaletteIndex::CharLiteral) }, - { "punctuation", u32(TextEditor::PaletteIndex::Punctuation) }, - { "preprocessor", u32(TextEditor::PaletteIndex::Preprocessor) }, - { "identifier", u32(TextEditor::PaletteIndex::Identifier) }, - { "known-identifier", u32(TextEditor::PaletteIndex::KnownIdentifier) }, - { "preproc-identifier", u32(TextEditor::PaletteIndex::PreprocIdentifier) }, - { "global-doc-comment", u32(TextEditor::PaletteIndex::GlobalDocComment) }, - { "doc-comment", u32(TextEditor::PaletteIndex::DocComment) }, - { "comment", u32(TextEditor::PaletteIndex::Comment) }, - { "multi-line-comment", u32(TextEditor::PaletteIndex::MultiLineComment) }, - { "preprocessor-deactivated", u32(TextEditor::PaletteIndex::PreprocessorDeactivated) }, - { "background", u32(TextEditor::PaletteIndex::Background) }, - { "cursor", u32(TextEditor::PaletteIndex::Cursor) }, - { "selection", u32(TextEditor::PaletteIndex::Selection) }, - { "error-marker", u32(TextEditor::PaletteIndex::ErrorMarker) }, - { "breakpoint", u32(TextEditor::PaletteIndex::Breakpoint) }, - { "line-number", u32(TextEditor::PaletteIndex::LineNumber) }, - { "current-line-fill", u32(TextEditor::PaletteIndex::CurrentLineFill) }, - { "current-line-fill-inactive", u32(TextEditor::PaletteIndex::CurrentLineFillInactive) }, - { "current-line-edge", u32(TextEditor::PaletteIndex::CurrentLineEdge) } + { "attribute", u32(TextEditor::PaletteIndex::Attribute) }, + { "background", u32(TextEditor::PaletteIndex::Background) }, + { "breakpoint", u32(TextEditor::PaletteIndex::Breakpoint) }, + { "calculated-pointer", u32(TextEditor::PaletteIndex::CalculatedPointer) }, + { "char-literal", u32(TextEditor::PaletteIndex::CharLiteral) }, + { "comment", u32(TextEditor::PaletteIndex::Comment) }, + { "current-line-edge", u32(TextEditor::PaletteIndex::CurrentLineEdge) }, + { "current-line-fill", u32(TextEditor::PaletteIndex::CurrentLineFill) }, + { "current-line-fill-inactive", u32(TextEditor::PaletteIndex::CurrentLineFillInactive) }, + { "cursor", u32(TextEditor::PaletteIndex::Cursor) }, + { "debug-text", u32(TextEditor::PaletteIndex::DebugText) }, + { "default", u32(TextEditor::PaletteIndex::Default) }, + { "default-text", u32(TextEditor::PaletteIndex::DefaultText) }, + { "doc-block-comment", u32(TextEditor::PaletteIndex::DocBlockComment) }, + { "doc-comment", u32(TextEditor::PaletteIndex::DocComment) }, + { "doc-global-comment", u32(TextEditor::PaletteIndex::GlobalDocComment) }, + { "error-marker", u32(TextEditor::PaletteIndex::ErrorMarker) }, + { "error-text", u32(TextEditor::PaletteIndex::ErrorText) }, + { "function", u32(TextEditor::PaletteIndex::Function) }, + { "function-parameter", u32(TextEditor::PaletteIndex::FunctionParameter) }, + { "function-variable", u32(TextEditor::PaletteIndex::FunctionVariable) }, + { "global-variable", u32(TextEditor::PaletteIndex::GlobalVariable) }, + { "identifier" , u32(TextEditor::PaletteIndex::Identifier) }, + { "keyword", u32(TextEditor::PaletteIndex::Keyword) }, + { "known-identifier", u32(TextEditor::PaletteIndex::BuiltInType) }, + { "line-number", u32(TextEditor::PaletteIndex::LineNumber) }, + { "local-variable", u32(TextEditor::PaletteIndex::LocalVariable) }, + { "multi-line-comment", u32(TextEditor::PaletteIndex::BlockComment) }, + { "namespace", u32(TextEditor::PaletteIndex::NameSpace) }, + { "number", u32(TextEditor::PaletteIndex::NumericLiteral) }, + { "pattern-variable", u32(TextEditor::PaletteIndex::PatternVariable) }, + { "placed-variable", u32(TextEditor::PaletteIndex::PlacedVariable) }, + { "preprocessor", u32(TextEditor::PaletteIndex::Directive) }, + { "preprocessor-deactivated", u32(TextEditor::PaletteIndex::PreprocessorDeactivated) }, + { "preproc-identifier", u32(TextEditor::PaletteIndex::PreprocIdentifier) }, + { "punctuation", u32(TextEditor::PaletteIndex::Operator) }, + { "selection", u32(TextEditor::PaletteIndex::Selection) }, + { "separator", u32(TextEditor::PaletteIndex::Separator) }, + { "string", u32(TextEditor::PaletteIndex::StringLiteral) }, + { "template-variable", u32(TextEditor::PaletteIndex::TemplateArgument) }, + { "typedef", u32(TextEditor::PaletteIndex::TypeDef) }, + { "unknown-identifier", u32(TextEditor::PaletteIndex::UnkIdentifier) }, + { "user-defined-type", u32(TextEditor::PaletteIndex::UserDefinedType) }, + { "view", u32(TextEditor::PaletteIndex::View) }, + { "warning-text", u32(TextEditor::PaletteIndex::WarningText) } }; ThemeManager::addThemeHandler("text-editor", TextEditorColorMap, diff --git a/plugins/builtin/source/content/views/view_pattern_editor.cpp b/plugins/builtin/source/content/views/view_pattern_editor.cpp index e9963d764..efd58a69e 100644 --- a/plugins/builtin/source/content/views/view_pattern_editor.cpp +++ b/plugins/builtin/source/content/views/view_pattern_editor.cpp @@ -8,8 +8,7 @@ #include #include -#include -#include +#include #include #include @@ -47,7 +46,7 @@ namespace hex::plugin::builtin { static TextEditor::LanguageDefinition langDef; if (!initialized) { constexpr static std::array keywords = { - "using", "struct", "union", "enum", "bitfield", "be", "le", "if", "else", "match", "false", "true", "this", "parent", "addressof", "sizeof", "typenameof", "$", "while", "for", "fn", "return", "break", "continue", "namespace", "in", "out", "ref", "null", "const", "unsigned", "signed", "try", "catch", "import", "as", "from" + "using", "struct", "union", "enum", "bitfield", "be", "le", "if", "else", "match", "false", "true", "this", "parent", "addressof", "sizeof", "typenameof", "while", "for", "fn", "return", "break", "continue", "namespace", "in", "out", "ref", "null", "const", "unsigned", "signed", "try", "catch", "import", "as", "from" }; for (auto &k : keywords) langDef.mKeywords.insert(k); @@ -60,8 +59,15 @@ namespace hex::plugin::builtin { id.mDeclaration = ""; langDef.mIdentifiers.insert(std::make_pair(std::string(name), id)); } - - langDef.mTokenize = [](const char *inBegin, const char *inEnd, const char *&outBegin, const char *&outEnd, TextEditor::PaletteIndex &paletteIndex) -> bool { + constexpr static std::array directives = { + "include", "define", "ifdef", "ifndef", "endif", "undef", "pragma", "error" + }; + for (const auto name : directives) { + TextEditor::Identifier id; + id.mDeclaration = ""; + langDef.mPreprocIdentifiers.insert(std::make_pair(std::string(name), id)); + } + langDef.mTokenize = [](std::string::const_iterator inBegin, std::string::const_iterator inEnd, std::string::const_iterator &outBegin, std::string::const_iterator &outEnd, TextEditor::PaletteIndex &paletteIndex) -> bool { paletteIndex = TextEditor::PaletteIndex::Max; while (inBegin < inEnd && isascii(*inBegin) && std::isblank(*inBegin)) @@ -74,13 +80,16 @@ namespace hex::plugin::builtin { } else if (TokenizeCStyleIdentifier(inBegin, inEnd, outBegin, outEnd)) { paletteIndex = TextEditor::PaletteIndex::Identifier; } else if (TokenizeCStyleNumber(inBegin, inEnd, outBegin, outEnd)) { - paletteIndex = TextEditor::PaletteIndex::Number; + paletteIndex = TextEditor::PaletteIndex::NumericLiteral; } else if (TokenizeCStyleCharacterLiteral(inBegin, inEnd, outBegin, outEnd)) { paletteIndex = TextEditor::PaletteIndex::CharLiteral; } else if (TokenizeCStyleString(inBegin, inEnd, outBegin, outEnd)) { - paletteIndex = TextEditor::PaletteIndex::String; + paletteIndex = TextEditor::PaletteIndex::StringLiteral; + } else if (TokenizeCStyleSeparator(inBegin, inEnd, outBegin, outEnd)) { + paletteIndex = TextEditor::PaletteIndex::Separator; + } else if (TokenizeCStyleOperator(inBegin, inEnd, outBegin, outEnd)) { + paletteIndex = TextEditor::PaletteIndex::Operator; } - return paletteIndex != TextEditor::PaletteIndex::Max; }; @@ -92,8 +101,9 @@ namespace hex::plugin::builtin { langDef.mAutoIndentation = true; langDef.mPreprocChar = '#'; - langDef.mGlobalDocComment = "/*!"; - langDef.mDocComment = "/**"; + langDef.mGlobalDocComment = "/*!"; + langDef.mBlockDocComment = "/**"; + langDef.mDocComment = "///"; langDef.mName = "Pattern Language"; @@ -107,15 +117,16 @@ namespace hex::plugin::builtin { static bool initialized = false; static TextEditor::LanguageDefinition langDef; if (!initialized) { - langDef.mTokenize = [](const char *inBegin, const char *inEnd, const char *&outBegin, const char *&outEnd, TextEditor::PaletteIndex &paletteIndex) -> bool { - if (std::string_view(inBegin).starts_with("D: ")) - paletteIndex = TextEditor::PaletteIndex::Comment; - else if (std::string_view(inBegin).starts_with("I: ")) - paletteIndex = TextEditor::PaletteIndex::Default; - else if (std::string_view(inBegin).starts_with("W: ")) - paletteIndex = TextEditor::PaletteIndex::Preprocessor; - else if (std::string_view(inBegin).starts_with("E: ")) - paletteIndex = TextEditor::PaletteIndex::ErrorMarker; + langDef.mTokenize = [](std::string::const_iterator inBegin, std::string::const_iterator inEnd, std::string::const_iterator &outBegin, std::string::const_iterator &outEnd, TextEditor::PaletteIndex &paletteIndex) -> bool { + std::string_view inView(inBegin, inEnd); + if (inView.starts_with("D: ")) + paletteIndex = TextEditor::PaletteIndex::DefaultText; + else if (inView.starts_with("I: ")) + paletteIndex = TextEditor::PaletteIndex::DebugText; + else if (inView.starts_with("W: ")) + paletteIndex = TextEditor::PaletteIndex::WarningText; + else if (inView.starts_with("E: ")) + paletteIndex = TextEditor::PaletteIndex::ErrorText; else paletteIndex = TextEditor::PaletteIndex::Max; @@ -583,14 +594,18 @@ namespace hex::plugin::builtin { } } - if (m_textEditor.get(provider).IsTextChanged() && !m_hasUnevaluatedChanges.get(provider)) { - m_hasUnevaluatedChanges.get(provider) = true; + if (m_textEditor.get(provider).IsTextChanged()) { + m_textEditor.get(provider).SetTextChanged(false); + if (!m_hasUnevaluatedChanges.get(provider) ) { + m_hasUnevaluatedChanges.get(provider) = true; + m_changesWereParsed = false; + } m_lastEditorChangeTime = std::chrono::steady_clock::now(); ImHexApi::Provider::markDirty(); } - if (m_hasUnevaluatedChanges.get(provider) && m_runningEvaluators == 0 && m_runningParsers == 0) { - if ((std::chrono::steady_clock::now() - m_lastEditorChangeTime) > std::chrono::seconds(1LL)) { + if (m_hasUnevaluatedChanges.get(provider) && m_runningEvaluators == 0 && m_runningParsers == 0 && + (std::chrono::steady_clock::now() - m_lastEditorChangeTime) > std::chrono::seconds(1ll)) { auto code = m_textEditor.get(provider).GetText(); EventPatternEditorChanged::post(code); @@ -602,13 +617,21 @@ namespace hex::plugin::builtin { m_triggerAutoEvaluate = true; }); m_hasUnevaluatedChanges.get(provider) = false; - m_textEditor.get(provider).SetTextChanged(); - } } if (m_triggerAutoEvaluate.exchange(false)) { this->evaluatePattern(m_textEditor.get(provider).GetText(), provider); } + if (m_textHighlighter.m_needsToUpdateColors && m_changesWereParsed && (m_runningParsers + m_runningEvaluators == 0)) { + TaskHolder taskHolder; + if (m_textHighlighter.getRunningColorizers() == 0) { + m_textHighlighter.m_needsToUpdateColors = false; + m_changesWereParsed = false; + taskHolder = TaskManager::createBackgroundTask("HighlightSourceCode", [this](auto &) { m_textHighlighter.highlightSourceCode(); }); + } else { + taskHolder.interrupt(); + } + } } if (m_dangerousFunctionCalled && !ImGui::IsPopupOpen(ImGuiID(0), ImGuiPopupFlags_AnyPopup)) { @@ -1228,7 +1251,7 @@ namespace hex::plugin::builtin { const std::string label { "##" + name }; if (pl::core::Token::isSigned(variable.type)) { - i64 value = i64(hex::get_or(variable.value, 0)); + i64 value = i64(hex::get_or(variable.value, 0ll)); if (ImGui::InputScalar(label.c_str(), ImGuiDataType_S64, &value)) m_hasUnevaluatedChanges.get(provider) = true; variable.value = i128(value); @@ -1495,7 +1518,7 @@ namespace hex::plugin::builtin { }; TextEditor::ErrorMarkers errorMarkers; - if (!(*m_callStack)->empty()) { + if (*m_callStack != nullptr && !(*m_callStack)->empty()) { for (const auto &frame : **m_callStack | std::views::reverse) { auto location = frame.node->getLocation(); std::string message; @@ -1701,7 +1724,7 @@ namespace hex::plugin::builtin { const bool shiftHeld = ImGui::GetIO().KeyShift; ImGui::ColorButton(pattern->getVariableName().c_str(), ImColor(pattern->getColor()), ImGuiColorEditFlags_AlphaOpaque); ImGui::SameLine(0, 10); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "{} ", pattern->getFormattedName()); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "{} ", pattern->getFormattedName()); ImGui::SameLine(0, 5); ImGuiExt::TextFormatted("{}", pattern->getDisplayName()); ImGui::SameLine(); @@ -1844,6 +1867,7 @@ namespace hex::plugin::builtin { m_textEditor.get(provider).SetText(code, true); m_sourceCode.get(provider) = code; + m_textHighlighter.m_needsToUpdateColors = false; TaskManager::createBackgroundTask("hex.builtin.task.parsing_pattern", [this, code, provider](auto&) { this->parsePattern(code, provider); }); } } @@ -1885,6 +1909,8 @@ namespace hex::plugin::builtin { patternVariables = std::move(oldPatternVariables); } + m_textHighlighter.m_needsToUpdateColors = true; + m_changesWereParsed = true; m_runningParsers -= 1; } @@ -2039,6 +2065,7 @@ namespace hex::plugin::builtin { m_textEditor.get(provider).SetText(code); m_sourceCode.get(provider) = code; m_hasUnevaluatedChanges.get(provider) = true; + m_textHighlighter.m_needsToUpdateColors = false; }); ContentRegistry::Settings::onChange("hex.builtin.setting.general", "hex.builtin.setting.general.sync_pattern_source", [this](const ContentRegistry::Settings::SettingsValue &value) { @@ -2098,7 +2125,9 @@ namespace hex::plugin::builtin { m_cursorNeedsUpdate.get(newProvider) = true; m_consoleCursorNeedsUpdate.get(newProvider) = true; m_textEditor.get(newProvider).SetTextChanged(false); + m_hasUnevaluatedChanges.get(newProvider) = true; } + m_textHighlighter.m_needsToUpdateColors = false; }); @@ -2327,7 +2356,7 @@ namespace hex::plugin::builtin { m_textEditor.get(provider).SetText(sourceCode); m_hasUnevaluatedChanges.get(provider) = true; - + m_textHighlighter.m_needsToUpdateColors = false; return true; }, .store = [this](prv::Provider *provider, const std::fs::path &basePath, const Tar &tar) { diff --git a/plugins/ui/source/ui/pattern_drawer.cpp b/plugins/ui/source/ui/pattern_drawer.cpp index c976e643e..d8c6d4c0a 100644 --- a/plugins/ui/source/ui/pattern_drawer.cpp +++ b/plugins/ui/source/ui/pattern_drawer.cpp @@ -523,7 +523,7 @@ namespace hex::ui { // Draw type column ImGui::TableNextColumn(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "{}", pattern.getFormattedName().empty() ? pattern.getTypeName() : pattern.getFormattedName()); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "{}", pattern.getFormattedName().empty() ? pattern.getTypeName() : pattern.getFormattedName()); } void PatternDrawer::closeTreeNode(bool inlined) const { @@ -550,19 +550,19 @@ namespace hex::ui { if (dynamic_cast(&pattern) != nullptr) { ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::Keyword)], "signed"); ImGui::SameLine(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], pattern.getBitSize() == 1 ? "bit" : "bits"); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], pattern.getBitSize() == 1 ? "bit" : "bits"); } else if (dynamic_cast(&pattern) != nullptr) { ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::Keyword)], "enum"); ImGui::SameLine(); ImGui::TextUnformatted(pattern.getTypeName().c_str()); } else if (dynamic_cast(&pattern) != nullptr) { - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "bool"); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "bool"); ImGui::SameLine(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "bit"); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "bit"); } else { ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::Keyword)], "unsigned"); ImGui::SameLine(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], pattern.getBitSize() == 1 ? "bit" : "bits"); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], pattern.getBitSize() == 1 ? "bit" : "bits"); } if (!this->isEditingPattern(pattern)) { @@ -792,7 +792,7 @@ namespace hex::ui { drawOffsetColumns(pattern); drawSizeColumn(pattern); ImGui::TableNextColumn(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "{}", pattern.getFormattedName()); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "{}", pattern.getFormattedName()); drawValueColumn(pattern); drawCommentColumn(pattern); } @@ -1038,12 +1038,12 @@ namespace hex::ui { drawSizeColumn(pattern); ImGui::TableNextColumn(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "{0}", pattern.getTypeName()); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "{0}", pattern.getTypeName()); ImGui::SameLine(0, 0); ImGui::TextUnformatted("["); ImGui::SameLine(0, 0); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::Number)], "{0}", iterable.getEntryCount()); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::NumericLiteral)], "{0}", iterable.getEntryCount()); ImGui::SameLine(0, 0); ImGui::TextUnformatted("]"); @@ -1106,12 +1106,12 @@ namespace hex::ui { ImGui::TableNextColumn(); ImGuiExt::TextFormatted("{0} {1}", chunkSize, chunkSize == 1 ? "byte" : "bytes"); ImGui::TableNextColumn(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "{0}", pattern.getTypeName()); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "{0}", pattern.getTypeName()); ImGui::SameLine(0, 0); ImGui::TextUnformatted("["); ImGui::SameLine(0, 0); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::Number)], "{0}", endIndex - i); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::NumericLiteral)], "{0}", endIndex - i); ImGui::SameLine(0, 0); ImGui::TextUnformatted("]");