diff --git a/lib/third_party/imgui/ColorTextEditor/include/TextEditor.h b/lib/third_party/imgui/ColorTextEditor/include/TextEditor.h index faf810498..d0c16c544 100644 --- a/lib/third_party/imgui/ColorTextEditor/include/TextEditor.h +++ b/lib/third_party/imgui/ColorTextEditor/include/TextEditor.h @@ -9,71 +9,89 @@ #include #include #include +#include +#include #include "imgui.h" #include "imgui_internal.h" - +using strConstIter = std::string::const_iterator; +// https://en.wikipedia.org/wiki/UTF-8 +// We assume that the char is a standalone character (<128) or a leading byte of an UTF-8 code sequence (non-10xxxxxx code) +static int UTF8CharLength(uint8_t c) { + if ((c & 0xFE) == 0xFC) + return 6; + if ((c & 0xFC) == 0xF8) + return 5; + if ((c & 0xF8) == 0xF0) + return 4; + if ((c & 0xF0) == 0xE0) + return 3; + if ((c & 0xE0) == 0xC0) + return 2; + return 1; +} class TextEditor { public: enum class PaletteIndex { Default, - Keyword, - Number, - String, - CharLiteral, - Punctuation, - Preprocessor, Identifier, - KnownIdentifier, - PreprocIdentifier, - GlobalDocComment, - DocComment, - Comment, - MultiLineComment, - PreprocessorDeactivated, - Background, + Directive, + Operator, + Separator, + BuiltInType, + Keyword, + NumericLiteral, + StringLiteral, + CharLiteral, Cursor, - Selection, - ErrorMarker, - Breakpoint, + Background, LineNumber, + Selection, + Breakpoint, + ErrorMarker, + PreprocessorDeactivated, CurrentLineFill, CurrentLineFillInactive, CurrentLineEdge, + ErrorText, + WarningText, + DebugText, + DefaultText, + Attribute, + PatternVariable, + LocalVariable, + CalculatedPointer, + TemplateArgument, + Function, + View, + FunctionVariable, + FunctionParameter, + UserDefinedType, + PlacedVariable, + GlobalVariable, + NameSpace, + TypeDef, + UnkIdentifier, + DocComment, + DocBlockComment, + BlockComment, + GlobalDocComment, + Comment, + PreprocIdentifier, Max - }; + }; - - struct Breakpoint - { - int mLine; - bool mEnabled; - std::string mCondition; - - Breakpoint() - : mLine(-1) - , mEnabled(false) - {} - }; - - // Represents a character coordinate from the user's point of view, - // i. e. consider an uniform grid (assuming fixed-width font) on the - // screen as it is rendered, and each cell has its own coordinate, starting from 0. - // Tabs are counted as [1..mTabSize] count empty spaces, depending on - // how many space is necessary to reach the next tab stop. - // For example, coordinate (1, 5) represents the character 'B' in a line "\tABC", when mTabSize = 4, - // because it is rendered as " ABC" on the screen. + // indices of the arrays that contain the lines (vector) and the columns (a string) of the + // text editor. Negative values indicate the distance to the last element of the array. + // When comparing coordinates ensure they have the same sign because coordinates don't have + // information about the size of the array. Currently positive coordinates are always bigger + // than negatives even if that gives a wrong result. struct Coordinates { int mLine, mColumn; Coordinates() : mLine(0), mColumn(0) {} - Coordinates(int aLine, int aColumn) : mLine(aLine), mColumn(aColumn) - { - IM_ASSERT(aLine >= 0); - IM_ASSERT(aColumn >= 0); - } - static Coordinates Invalid() { static Coordinates invalid(-1, -1); return invalid; } + Coordinates(int aLine, int aColumn) : mLine(aLine), mColumn(aColumn) {} bool operator ==(const Coordinates& o) const { @@ -129,8 +147,8 @@ public: using Keywords = std::unordered_set ; using ErrorMarkers = std::map>; using Breakpoints = std::unordered_set; - using Palette = std::array; - using Char = uint8_t ; + using Palette = std::array; + using Glyph = uint8_t ; class ActionableBox { @@ -203,35 +221,381 @@ public: }; using ErrorHoverBoxes = std::map; - struct Glyph - { - Char mChar; - PaletteIndex mColorIndex = PaletteIndex::Default; - bool mComment : 1; - bool mMultiLineComment : 1; - bool mPreprocessor : 1; - bool mDocComment : 1; - bool mGlobalDocComment : 1; - bool mDeactivated : 1; + // A line of text in the pattern editor consists of three strings; the character encoding, the color encoding and the flags. + // The char encoding is utf-8, the color encoding are indices to the color palette and the flags are used to override the colors + // depending on priorities; e.g. comments, strings, etc. + + class Line { + public: + struct FlagBits { + bool mComment : 1; + bool mBlockComment : 1; + bool mDocComment : 1; + bool mBlockDocComment : 1; + bool mGlobalDocComment : 1; + bool mDeactivated : 1; + bool mPreprocessor : 1; + }; + union Flags { + Flags(char value) : mValue(value) {} + Flags(FlagBits bits) : mBits(bits) {} + FlagBits mBits; + char mValue; + }; + constexpr static char InComment = 31; - Glyph(Char aChar, PaletteIndex aColorIndex) : mChar(aChar), mColorIndex(aColorIndex), mComment(false), - mMultiLineComment(false), mPreprocessor(false), mDocComment(false), mGlobalDocComment(false), mDeactivated(false) {} - }; + class LineIterator { + public: + strConstIter mCharsIter; + strConstIter mColorsIter; + strConstIter mFlagsIter; - typedef std::vector Line; - typedef std::vector Lines; + LineIterator(const LineIterator &other) : mCharsIter(other.mCharsIter), mColorsIter(other.mColorsIter), mFlagsIter(other.mFlagsIter) {} + + LineIterator() = default; + + char operator*() { + return *mCharsIter; + } + + LineIterator operator++() { + LineIterator iter = *this; + ++iter.mCharsIter; + ++iter.mColorsIter; + ++iter.mFlagsIter; + return iter; + } + + LineIterator operator=(const LineIterator &other) { + mCharsIter = other.mCharsIter; + mColorsIter = other.mColorsIter; + mFlagsIter = other.mFlagsIter; + return *this; + } + + bool operator!=(const LineIterator &other) const { + return mCharsIter != other.mCharsIter || mColorsIter != other.mColorsIter || mFlagsIter != other.mFlagsIter; + } + + bool operator==(const LineIterator &other) const { + return mCharsIter == other.mCharsIter && mColorsIter == other.mColorsIter && mFlagsIter == other.mFlagsIter; + } + + LineIterator operator+(int n) { + LineIterator iter = *this; + iter.mCharsIter += n; + iter.mColorsIter += n; + iter.mFlagsIter += n; + return iter; + } + + int operator-(LineIterator l) { + return mCharsIter - l.mCharsIter; + } + }; + + LineIterator begin() const { + LineIterator iter; + iter.mCharsIter = mChars.begin(); + iter.mColorsIter = mColors.begin(); + iter.mFlagsIter = mFlags.begin(); + return iter; + } + + LineIterator end() const { + LineIterator iter; + iter.mCharsIter = mChars.end(); + iter.mColorsIter = mColors.end(); + iter.mFlagsIter = mFlags.end(); + return iter; + } + + std::string mChars; + std::string mColors; + std::string mFlags; + bool mColorized = false; + Line() : mChars(), mColors(), mFlags(), mColorized(false) {} + + explicit Line(const char *line) { + Line(std::string(line)); + } + + explicit Line(const std::string &line) : mChars(line), mColors(std::string(line.size(), 0x00)), mFlags(std::string(line.size(), 0x00)), mColorized(false) {} + Line(const Line &line) : mChars(line.mChars), mColors(line.mColors), mFlags(line.mFlags), mColorized(line.mColorized) {} + + LineIterator begin() { + LineIterator iter; + iter.mCharsIter = mChars.begin(); + iter.mColorsIter = mColors.begin(); + iter.mFlagsIter = mFlags.begin(); + return iter; + } + + LineIterator end() { + LineIterator iter; + iter.mCharsIter = mChars.end(); + iter.mColorsIter = mColors.end(); + iter.mFlagsIter = mFlags.end(); + return iter; + } + + Line &operator=(const Line &line) { + mChars = line.mChars; + mColors = line.mColors; + mFlags = line.mFlags; + mColorized = line.mColorized; + return *this; + } + + Line &operator=(Line &&line) noexcept { + mChars = std::move(line.mChars); + mColors = std::move(line.mColors); + mFlags = std::move(line.mFlags); + mColorized = line.mColorized; + return *this; + } + + size_t size() const { + return mChars.size(); + } + enum class LinePart { + Chars, + Colors, + Flags + }; + + char front(LinePart part = LinePart::Chars) const { + if (part == LinePart::Chars && !mChars.empty()) + return mChars.front(); + if (part == LinePart::Colors && !mColors.empty()) + return mColors.front(); + if (part == LinePart::Flags && !mFlags.empty()) + return mFlags.front(); + return 0x00; + } + + void push_back(char c) { + mChars.push_back(c); + mColors.push_back(0x00); + mFlags.push_back(0x00); + mColorized = false; + } + + bool empty() const { + return mChars.empty(); + } + + std::string substrUtf8(size_t start, size_t length = (size_t)-1, LinePart part = LinePart::Chars ) const { + if (start >= mChars.size()) + return ""; + if (length == (size_t)-1 || length >= mChars.size() - start) + length = mChars.size() - start; + size_t utf8Start= 0; + while (utf8Start= mChars.size()) + return ""; + if (start + length >= mChars.size()) + length = mChars.size() - start; + if (part == LinePart::Chars && length > 0) + return mChars.substr(start, length); + if (part == LinePart::Colors && length > 0) + return mColors.substr(start, length); + if (part == LinePart::Flags && length > 0) + return mFlags.substr(start, length); + return ""; + } + + template + char operator[](size_t index) const { + if (part == LinePart::Chars) + return mChars[index]; + if (part == LinePart::Colors) + return mColors[index]; + if (part == LinePart::Flags) + return mFlags[index]; + return mChars[index]; + } + + template + const char &operator[](size_t index) { + if (part == LinePart::Chars) + return mChars[index]; + if (part == LinePart::Colors) + return mColors[index]; + if (part == LinePart::Flags) + return mFlags[index]; + return mChars[index]; + } + + void SetNeedsUpdate(bool needsUpdate) { + mColorized = mColorized && !needsUpdate; + } + + void append(const char *text) { + append(std::string(text)); + } + + void append(const char text) { + append(std::string(1, text)); + } + + void append(const std::string &text) { + mChars.append(text); + mColors.append(text.size(), 0x00); + mFlags.append(text.size(), 0x00); + mColorized = false; + } + + void append(const Line &line) { + append(line.begin(), line.end()); + } + + void append(LineIterator begin, LineIterator end) { + if (begin.mCharsIter < end.mCharsIter) + mChars.append(begin.mCharsIter, end.mCharsIter); + if (begin.mColorsIter < end.mColorsIter) + mColors.append(begin.mColorsIter, end.mColorsIter); + if (begin.mFlagsIter < end.mFlagsIter) + mFlags.append(begin.mFlagsIter, end.mFlagsIter); + mColorized = false; + } + + void insert(LineIterator iter, const std::string &text) { + if (iter == end()) + append(text); + else + insert(iter, text.begin(), text.end()); + } + + void insert(LineIterator iter, const char text) { + if (iter == end()) + append(text); + else + insert(iter,std::string(1, text)); + } + + void insert(LineIterator iter, strConstIter beginString, strConstIter endString) { + if (iter == end()) + append(std::string(beginString, endString)); + else { + std::string charsString(beginString, endString); + mChars.insert(iter.mCharsIter, beginString, endString); + std::string colorString(charsString.size(), 0x00); + try { + mColors.insert(iter.mColorsIter, colorString.begin(), colorString.end()); + } catch (const std::exception &e) { + std::cerr << "Exception: " << e.what() << std::endl; + mColorized = false; + return; + } + std::string flagsString(charsString.size(), 0x00); + try { + mFlags.insert(iter.mFlagsIter, flagsString.begin(), flagsString.end()); + } catch (const std::exception &e) { + std::cerr << "Exception: " << e.what() << std::endl; + mColorized = false; + return; + } + mColorized = false; + } + } + + void insert(LineIterator iter,const Line &line) { + if (iter == end()) + append(line.begin(), line.end()); + else + insert(iter, line.begin(), line.end()); + } + + void insert(LineIterator iter,LineIterator beginLine, LineIterator endLine) { + if (iter == end()) + append(beginLine, endLine); + else { + mChars.insert(iter.mCharsIter, beginLine.mCharsIter, endLine.mCharsIter); + mColors.insert(iter.mColorsIter, beginLine.mColorsIter, endLine.mColorsIter); + mFlags.insert(iter.mFlagsIter, beginLine.mFlagsIter, endLine.mFlagsIter); + mColorized = false; + } + } + + void erase(LineIterator begin) { + mChars.erase(begin.mCharsIter); + mColors.erase(begin.mColorsIter); + mFlags.erase(begin.mFlagsIter); + mColorized = false; + } + + void erase(LineIterator begin, size_t count) { + if (count == (size_t) -1) + count = mChars.size() - (begin.mCharsIter - mChars.begin()); + mChars.erase(begin.mCharsIter, begin.mCharsIter + count); + mColors.erase(begin.mColorsIter, begin.mColorsIter + count); + mFlags.erase(begin.mFlagsIter, begin.mFlagsIter + count); + mColorized = false; + } + + void clear() { + mChars.clear(); + mColors.clear(); + mFlags.clear(); + mColorized = false; + } + + void SetLine(const std::string &text) { + mChars = text; + mColors = std::string(text.size(), 0x00); + mFlags = std::string(text.size(), 0x00); + mColorized = false; + } + + void SetLine(const Line &text) { + mChars = text.mChars; + mColors = text.mColors; + mFlags = text.mFlags; + mColorized = text.mColorized; + } + + + bool NeedsUpdate() const { + return !mColorized; + } + + }; + + using Lines = std::vector; struct LanguageDefinition { typedef std::pair TokenRegexString; typedef std::vector TokenRegexStrings; - typedef bool(*TokenizeCallback)(const char * in_begin, const char * in_end, const char *& out_begin, const char *& out_end, PaletteIndex & paletteIndex); + typedef bool(*TokenizeCallback)(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end, PaletteIndex &paletteIndex); std::string mName; Keywords mKeywords; Identifiers mIdentifiers; Identifiers mPreprocIdentifiers; - std::string mCommentStart, mCommentEnd, mSingleLineComment, mGlobalDocComment, mDocComment; + std::string mSingleLineComment, mCommentEnd, mCommentStart, mGlobalDocComment, mDocComment, mBlockDocComment; char mPreprocChar; bool mAutoIndentation; @@ -241,10 +605,8 @@ public: bool mCaseSensitive; - LanguageDefinition() - : mPreprocChar('#'), mAutoIndentation(true), mTokenize(nullptr), mCaseSensitive(true) - { - } + LanguageDefinition() : mName(""), mKeywords({}), mIdentifiers({}), mPreprocIdentifiers({}), mSingleLineComment(""), mCommentEnd(""), + mCommentStart(""), mGlobalDocComment(""), mDocComment(""), mBlockDocComment(""), mPreprocChar('#'), mAutoIndentation(true), mTokenize(nullptr), mTokenRegexStrings({}), mCaseSensitive(true) {} static const LanguageDefinition& CPlusPlus(); static const LanguageDefinition& HLSL(); @@ -303,10 +665,59 @@ public: } std::string GetText() const; bool isEmpty() const { - auto text = GetText(); - return text.empty() || text == "\n"; + if (mLines.empty()) + return true; + if (mLines.size() == 1) { + if (mLines[0].empty()) + return true; + if (mLines[0].size() == 1 && mLines[0].front() == '\n') + return true; + } + return false; } + void SetTopLine(); + uint32_t GetTopLine() const { + return static_cast(std::floor(mTopLine)); + } + uint32_t GetBottomLine() const { + return static_cast(std::ceil(mTopLine+mNumberOfLinesDisplayed)); + } + void SetNeedsUpdate (uint32_t line, bool needsUpdate) { + if (line < mLines.size()) + mLines[line].SetNeedsUpdate(needsUpdate); + } + + void SetColorizedLineSize(size_t line) { + if (line < mLines.size()) { + const auto &size = mLines[line].mChars.size(); + if (mLines[line].mColors.size() != size) { + mLines[line].mColors.resize(size); + std::fill(mLines[line].mColors.begin(), mLines[line].mColors.end(), 0x00); + } + } + } + + void SetColorizedLine(size_t line, const std::string &tokens) { + if (line < mLines.size()) { + auto &lineTokens = mLines[line].mColors; + if (lineTokens.size() != tokens.size()) { + lineTokens.resize(tokens.size()); + std::fill(lineTokens.begin(), lineTokens.end(), 0x00); + } + bool needsUpdate = false; + for (size_t i = 0; i < tokens.size(); ++i) { + if (tokens[i] != 0x00) { + if (tokens[i] != lineTokens[i]) { + lineTokens[i] = tokens[i]; + needsUpdate = true; + } + } + } + SetNeedsUpdate(line, needsUpdate); + } + } + void SetScrollY(); void SetTextLines(const std::vector& aLines); std::vector GetTextLines() const; @@ -345,14 +756,23 @@ public: void SetOverwrite(bool aValue) { mOverwrite = aValue; } std::string ReplaceStrings(std::string string, const std::string &search, const std::string &replace); - std::vector SplitString(const std::string &string, const std::string &delimiter, bool removeEmpty); + static std::vector SplitString(const std::string &string, const std::string &delimiter, bool removeEmpty); std::string ReplaceTabsWithSpaces(const std::string& string, uint32_t tabSize); std::string PreprocessText(const std::string &code); void SetReadOnly(bool aValue); + bool IsEndOfLine(const Coordinates &aCoordinates) const; + bool IsEndOfFile(const Coordinates &aCoordinates) const; bool IsReadOnly() const { return mReadOnly; } bool IsTextChanged() const { return mTextChanged; } - void SetTextChanged(bool aValue=false) { mTextChanged = aValue; } + void SetTextChanged(bool aValue) { mTextChanged = aValue; } + void SetTimeStamp(int code) { + auto now = std::chrono::high_resolution_clock::now(); + if (code == 0) + mLinesTimestamp = std::chrono::duration_cast(now.time_since_epoch()).count(); + else + mColorizedLinesTimestamp = std::chrono::duration_cast(now.time_since_epoch()).count(); + } bool IsCursorPositionChanged() const { return mCursorPositionChanged; } bool IsBreakpointsChanged() const { return mBreakPointsChanged; } void ClearBreakpointsChanged() { mBreakPointsChanged = false; } @@ -362,6 +782,7 @@ public: bool IsColorizerEnabled() const { return mColorizerEnabled; } void SetColorizerEnable(bool aValue); + void Colorize(); Coordinates GetCursorPosition() const { return GetActualCursorCoordinates(); } void SetCursorPosition(const Coordinates& aPosition); @@ -541,8 +962,7 @@ private: typedef std::vector UndoBuffer; void ProcessInputs(); - void Colorize(int aFromLine = 0, int aCount = -1); - void ColorizeRange(int aFromLine = 0, int aToLine = 0); + void ColorizeRange(); void ColorizeInternal(); float TextDistanceToLineStart(const Coordinates& aFrom) const; void EnsureCursorVisible(); @@ -553,35 +973,40 @@ private: void DeleteRange(const Coordinates& aStart, const Coordinates& aEnd); int InsertTextAt(Coordinates& aWhere, const std::string &aValue); void AddUndo(UndoRecord& aValue); - Coordinates ScreenPosToCoordinates(const ImVec2& aPosition) const; + Coordinates ScreenPosToCoordinates(const ImVec2& aPosition) const; Coordinates FindWordStart(const Coordinates& aFrom) const; Coordinates FindWordEnd(const Coordinates& aFrom) const; + Coordinates FindPreviousWord(const Coordinates& aFrom) const; Coordinates FindNextWord(const Coordinates& aFrom) const; Coordinates StringIndexToCoordinates(int aIndex, const std::string &str) const; int GetCharacterIndex(const Coordinates& aCoordinates) const; int GetCharacterColumn(int aLine, int aIndex) const; int GetLineCharacterCount(int aLine) const; - int Utf8CharsToBytes(const Coordinates &aCoordinates) const; - unsigned long long GetLineByteCount(int aLine) const; + int Utf8CharsToBytes(const Coordinates &aCoordinates) const; + static int Utf8CharsToBytes(std::string line, uint32_t start, uint32_t numChars); + unsigned long long GetLineByteCount(int aLine) const; int GetStringCharacterCount(std::string str) const; int GetLineMaxColumn(int aLine) const; bool IsOnWordBoundary(const Coordinates& aAt) const; void RemoveLine(int aStart, int aEnd); void RemoveLine(int aIndex); Line& InsertLine(int aIndex); - void EnterCharacter(ImWchar aChar, bool aShift); + void InsertLine(int aIndex, const std::string &aText); + void EnterCharacter(ImWchar aChar, bool aShift); void DeleteSelection(); std::string GetWordUnderCursor() const; std::string GetWordAt(const Coordinates& aCoords) const; - ImU32 GetGlyphColor(const Glyph& aGlyph) const; + TextEditor::PaletteIndex GetColorIndexFromFlags(Line::Flags flags); void ResetCursorBlinkTime(); - + uint32_t SkipSpaces(const Coordinates &aFrom); void HandleKeyboardInputs(); void HandleMouseInputs(); void RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPos, const ImVec2 &textEditorSize); void SetFocus(); float mLineSpacing = 1.0F; Lines mLines; + uint64_t mLinesTimestamp = 0; + uint64_t mColorizedLinesTimestamp = 0; EditorState mState = {}; UndoBuffer mUndoBuffer; int mUndoIndex = 0; @@ -616,7 +1041,7 @@ private: Palette mPalette = {}; LanguageDefinition mLanguageDefinition = {}; RegexList mRegexList; - bool mCheckComments = true; + bool mUpdateFlags = true; Breakpoints mBreakpoints = {}; ErrorMarkers mErrorMarkers = {}; ErrorHoverBoxes mErrorHoverBoxes = {}; @@ -646,8 +1071,9 @@ private: static const int sCursorBlinkOnTime; }; -bool TokenizeCStyleString(const char * in_begin, const char * in_end, const char *& out_begin, const char *& out_end); -bool TokenizeCStyleCharacterLiteral(const char * in_begin, const char * in_end, const char *& out_begin, const char *& out_end); -bool TokenizeCStyleIdentifier(const char * in_begin, const char * in_end, const char *& out_begin, const char *& out_end); -bool TokenizeCStyleNumber(const char * in_begin, const char * in_end, const char *& out_begin, const char *& out_end); -bool TokenizeCStylePunctuation(const char * in_begin, const char * in_end, const char *& out_begin, const char *& out_end); \ No newline at end of file +bool TokenizeCStyleString(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end); +bool TokenizeCStyleCharacterLiteral(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end); +bool TokenizeCStyleIdentifier(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end); +bool TokenizeCStyleNumber(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end); +bool TokenizeCStyleOperator(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end); +bool TokenizeCStyleSeparator(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end); \ No newline at end of file diff --git a/lib/third_party/imgui/ColorTextEditor/source/TextEditor.cpp b/lib/third_party/imgui/ColorTextEditor/source/TextEditor.cpp index ca543c954..6b1fa5028 100644 --- a/lib/third_party/imgui/ColorTextEditor/source/TextEditor.cpp +++ b/lib/third_party/imgui/ColorTextEditor/source/TextEditor.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "TextEditor.h" @@ -85,33 +86,40 @@ void TextEditor::SetPalette(const Palette &aValue) { sPaletteBase = aValue; } +bool TextEditor::IsEndOfLine(const Coordinates &aCoordinates) const { + if (aCoordinates.mLine < (int)mLines.size()) { + auto &line = mLines[aCoordinates.mLine]; + auto cindex = GetCharacterIndex(aCoordinates); + return cindex >= (int)line.size(); + } + return true; +} + +bool TextEditor::IsEndOfFile(const Coordinates &aCoordinates) const { + if (aCoordinates.mLine < (int)mLines.size()) { + auto &line = mLines[aCoordinates.mLine]; + auto cindex = GetCharacterIndex(aCoordinates); + return cindex >= (int)line.size() && aCoordinates.mLine == (int)mLines.size() - 1; + } + return true; +} + std::string TextEditor::GetText(const Coordinates &aStart, const Coordinates &aEnd) const { - std::string result; - - auto lstart = aStart.mLine; - auto lend = aEnd.mLine; - auto istart = GetCharacterIndex(aStart); - auto iend = GetCharacterIndex(aEnd); - size_t s = 0; - - for (size_t i = lstart; i < lend; i++) - s += mLines[i].size(); - - result.reserve(s + s / 8); - - while (istart < iend || lstart < lend) { - if (lstart >= (int)mLines.size()) - break; - - auto &line = mLines[lstart]; - if (istart < (int)line.size()) { - result += line[istart].mChar; - istart++; - } else { - istart = 0; - ++lstart; - result += '\n'; + std::string result=""; + auto start = SanitizeCoordinates(aStart); + auto end = SanitizeCoordinates(aEnd); + auto lineStart = start.mLine; + auto lineEnd = end.mLine; + auto startIndex = GetCharacterIndex(start); + auto endIndex = GetCharacterIndex(end); + if (lineStart == lineEnd) + result = mLines[lineStart].substr(startIndex,endIndex-startIndex); + else { + result = mLines[lineStart].substr(startIndex) + '\n'; + for (size_t i = lineStart+1; i < lineEnd; i++) { + result += mLines[i].mChars + '\n'; } + result += mLines[lineEnd].substr(0, endIndex); } return result; @@ -122,191 +130,145 @@ TextEditor::Coordinates TextEditor::GetActualCursorCoordinates() const { } TextEditor::Coordinates TextEditor::SanitizeCoordinates(const Coordinates &aValue) const { - auto line = aValue.mLine; - auto column = aValue.mColumn; - if (line >= (int)mLines.size()) { - if (isEmpty()) { - line = 0; - column = 0; - } else { - line = (int)mLines.size() - 1; - column = GetLineMaxColumn(line); - } - return Coordinates(line, column); - } else { - column = isEmpty() ? 0 : std::min(column, GetLineMaxColumn(line)); - return Coordinates(line, column); - } -} + Coordinates result = aValue; + if (aValue.mLine < 0) + result.mLine = mLines.size() + aValue.mLine; + if (aValue.mColumn < 0) + result.mColumn = GetLineMaxColumn(result.mLine) + aValue.mColumn + 1; -// https://en.wikipedia.org/wiki/UTF-8 -// We assume that the char is a standalone character (<128) or a leading byte of an UTF-8 code sequence (non-10xxxxxx code) -static int UTF8CharLength(TextEditor::Char c) { - if ((c & 0xFE) == 0xFC) - return 6; - if ((c & 0xFC) == 0xF8) - return 5; - if ((c & 0xF8) == 0xF0) - return 4; - else if ((c & 0xF0) == 0xE0) - return 3; - else if ((c & 0xE0) == 0xC0) - return 2; - return 1; + result.mLine = std::clamp(result.mLine, 0, (int)mLines.size()-1); + result.mColumn = std::clamp(result.mColumn, 0, GetLineMaxColumn(result.mLine)); + return result; } // "Borrowed" from ImGui source -static inline int ImTextCharToUtf8(char *buf, int buf_size, unsigned int c) { +static inline void ImTextCharToUtf8(std::string &buffer, unsigned int c) { if (c < 0x80) { - buf[0] = (char)c; - return 1; + buffer += (char) c; + return; } if (c < 0x800) { - if (buf_size < 2) return 0; - buf[0] = (char)(0xc0 + (c >> 6)); - buf[1] = (char)(0x80 + (c & 0x3f)); - return 2; - } - if (c >= 0xdc00 && c < 0xe000) { - return 0; + buffer += (char)(0xc0 + (c >> 6)); + buffer += (char)(0x80 + (c & 0x3f)); + return; } + if (c >= 0xdc00 && c < 0xe000) + return; if (c >= 0xd800 && c < 0xdc00) { - if (buf_size < 4) return 0; - buf[0] = (char)(0xf0 + (c >> 18)); - buf[1] = (char)(0x80 + ((c >> 12) & 0x3f)); - buf[2] = (char)(0x80 + ((c >> 6) & 0x3f)); - buf[3] = (char)(0x80 + ((c)&0x3f)); - return 4; + buffer += (char)(0xf0 + (c >> 18)); + buffer += (char)(0x80 + ((c >> 12) & 0x3f)); + buffer += (char)(0x80 + ((c >> 6) & 0x3f)); + buffer += (char)(0x80 + ((c)&0x3f)); + return; } // else if (c < 0x10000) { - if (buf_size < 3) return 0; - buf[0] = (char)(0xe0 + (c >> 12)); - buf[1] = (char)(0x80 + ((c >> 6) & 0x3f)); - buf[2] = (char)(0x80 + ((c)&0x3f)); - return 3; + buffer += (char)(0xe0 + (c >> 12)); + buffer += (char)(0x80 + ((c >> 6) & 0x3f)); + buffer += (char)(0x80 + ((c)&0x3f)); + return; } } +static inline int ImTextCharToUtf8(char *buffer, int buf_size, unsigned int c) { + std::string input; + ImTextCharToUtf8(input,c); + auto size = input.size(); + int i=0; + for (;i= aStart); - IM_ASSERT(!mReadOnly); - - // printf("D(%d.%d)-(%d.%d)\n", aStart.mLine, aStart.mColumn, aEnd.mLine, aEnd.mColumn); - - if (aEnd == aStart) + if (aEnd <= aStart || mReadOnly) return; - - auto start = GetCharacterIndex(aStart); - auto end = GetCharacterIndex(aEnd); - if (start == -1 || end == -1) - return; - - if (aStart.mLine == aEnd.mLine) { - auto &line = mLines[aStart.mLine]; - auto n = GetLineMaxColumn(aStart.mLine); - if (aEnd.mColumn >= n) - line.erase(line.begin() + start, line.end()); + auto start = SanitizeCoordinates(aStart); + auto end = SanitizeCoordinates(aEnd); + auto startIndex = GetCharacterIndex(start); + auto endIndex = GetCharacterIndex(end); + if (start.mLine == end.mLine) { + auto &line = mLines[start.mLine]; + if (endIndex >= GetLineMaxColumn(start.mLine)) + line.erase(line.begin() + startIndex, -1); else - line.erase(line.begin() + start, line.begin() + end); + line.erase(line.begin() + startIndex, endIndex-startIndex); + line.mColorized = false; } else { - auto &firstLine = mLines[aStart.mLine]; - auto &lastLine = mLines[aEnd.mLine]; + auto &firstLine = mLines[start.mLine]; + auto &lastLine = mLines[end.mLine]; - firstLine.erase(firstLine.begin() + start, firstLine.end()); - lastLine.erase(lastLine.begin(), lastLine.begin() + end); + firstLine.erase(firstLine.begin() + startIndex, -1); + lastLine.erase(lastLine.begin(), endIndex); - if (aStart.mLine < aEnd.mLine) + if (aStart.mLine < aEnd.mLine) { firstLine.insert(firstLine.end(), lastLine.begin(), lastLine.end()); - - if (aStart.mLine < aEnd.mLine) + firstLine.mColorized = false; RemoveLine(aStart.mLine + 1, aEnd.mLine); + } } mTextChanged = true; } void TextEditor::AppendLine(const std::string &aValue) { - if (mLines.size() != 1 || !mLines[0].empty()) - mLines.push_back(Line()); - Coordinates lastLine = {( int )mLines.size() - 1, 0}; - InsertTextAt(lastLine, aValue); - SetCursorPosition({( int )mLines.size() - 1, 0}); + auto text = PreprocessText(aValue); + if (text.empty()) + return; + if (isEmpty()) { + mLines[0].mChars = text; + mLines[0].mColors = std::string(text.size(),0); + mLines[0].mFlags = std::string(text.size(),0); + } else + mLines.push_back(Line(text)); + SetCursorPosition(Coordinates((int)mLines.size() - 1, 0)); + mLines.back().mColorized = false; EnsureCursorVisible(); mTextChanged = true; } -int TextEditor::InsertTextAt(Coordinates & /* inout */ aWhere, const std::string &aValueString) { - int cindex = GetCharacterIndex(aWhere); - int totalLines = 0; - auto aValue = aValueString.begin(); - while (aValue != aValueString.end()) { - if (mLines.empty()) { - mLines.push_back(Line()); - mTextChanged = true; - } - if (*aValue == '\r') { - // skip - ++aValue; - } else if (*aValue == '\t') { - auto &line = mLines[aWhere.mLine]; - auto c = GetCharacterColumn(aWhere.mLine, cindex); - auto r = c % mTabSize; - auto d = mTabSize - r; - auto i = d; - while (i-- > 0) - line.insert(line.begin() + cindex++, Glyph(' ', PaletteIndex::Default)); +int TextEditor::InsertTextAt(Coordinates & /* inout */ aWhere, const std::string &aValue) { + auto text = PreprocessText(aValue); + if (text.empty()) + return 0; + auto start = SanitizeCoordinates(aWhere); + auto &line = mLines[start.mLine]; + auto stringVector = SplitString(text, "\n", false); + auto lineCount = (int)stringVector.size(); + aWhere.mLine += lineCount - 1; + aWhere.mColumn += stringVector[lineCount-1].size(); + stringVector[lineCount - 1].append(line.substr(GetCharacterIndex(start))); + if (GetCharacterIndex(start) < (int)line.size()) + line.erase(line.begin() + GetCharacterIndex(start),-1); - aWhere.mColumn += d; - aValue++; - } else if (*aValue == '\n') { - if (cindex < (int)mLines[aWhere.mLine].size()) { - auto &newLine = InsertLine(aWhere.mLine + 1); - auto &line = mLines[aWhere.mLine]; - newLine.insert(newLine.begin(), line.begin() + cindex, line.end()); - line.erase(line.begin() + cindex, line.end()); - } else { - InsertLine(aWhere.mLine + 1); - } - ++aWhere.mLine; - aWhere.mColumn = 0; - cindex = 0; - ++totalLines; - ++aValue; - } else if (*aValue == 0) { - auto &line = mLines[aWhere.mLine]; - line.insert(line.begin() + cindex++, Glyph('.', PaletteIndex::Default)); - ++aWhere.mColumn; - ++aValue; - } else { - auto &line = mLines[aWhere.mLine]; - auto d = UTF8CharLength(*aValue); - while (d-- > 0 && aValue != aValueString.end()) - line.insert(line.begin() + cindex++, Glyph(*aValue++, PaletteIndex::Default)); - ++aWhere.mColumn; - } - - mTextChanged = true; + line.append(stringVector[0]); + line.mColorized = false; + for (int i = 1; i < lineCount; i++) { + InsertLine(start.mLine + i, stringVector[i]); + mLines[start.mLine + i].mColorized = false; } - - return totalLines; + mTextChanged = true; + return lineCount; } void TextEditor::AddUndo(UndoRecord &aValue) { @@ -324,66 +286,31 @@ void TextEditor::AddUndo(UndoRecord &aValue) { } TextEditor::Coordinates TextEditor::ScreenPosToCoordinates(const ImVec2 &aPosition) const { - ImVec2 origin = ImGui::GetCursorScreenPos(); - ImVec2 local(aPosition.x - origin.x, aPosition.y - origin.y); + ImVec2 local = aPosition - ImGui::GetCursorScreenPos(); int lineNo = std::max(0, (int)floor(local.y / mCharAdvance.y)); - if (local.x < mCharAdvance.x) - return Coordinates(lineNo, 0); - local.x -= mCharAdvance.x; + if (local.x < (mLeftMargin - 2) || lineNo >= (int)mLines.size() || mLines[lineNo].empty()) + return Coordinates(std::min(lineNo,(int)mLines.size()-1), 0); - int columnCoord = 0; + local.x -= (mLeftMargin - 2); + auto column = (int)floor(local.x / mCharAdvance.x); + auto rem = static_cast(local.x) % static_cast(mCharAdvance.x); + if (rem > mCharAdvance.x - 2) + column++; - if (lineNo >= 0 && lineNo < (int)mLines.size()) { - auto &line = mLines.at(lineNo); - - int columnIndex = 0; - float columnX = 0.0f; - - while ((size_t)columnIndex < line.size()) { - float columnWidth = 0.0f; - - if (line[columnIndex].mChar == '\t') { - float spaceSize = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, " ").x; - float oldX = columnX; - float newColumnX = (1.0f + std::floor((1.0f + columnX) / (float(mTabSize) * spaceSize))) * (float(mTabSize) * spaceSize); - columnWidth = newColumnX - oldX; - if (columnX + columnWidth > local.x) - break; - columnX = newColumnX; - columnCoord = (columnCoord / mTabSize) * mTabSize + mTabSize; - columnIndex++; - } else { - char buf[7]; - auto d = UTF8CharLength(line[columnIndex].mChar); - int i = 0; - while (i < 6 && d-- > 0) - buf[i++] = line[columnIndex++].mChar; - buf[i] = '\0'; - columnWidth = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, buf).x; - if (columnX + columnWidth > local.x) - break; - columnX += columnWidth; - columnCoord++; - } - } - } - - return SanitizeCoordinates(Coordinates(lineNo, columnCoord)); + return SanitizeCoordinates(Coordinates(lineNo, column)); } void TextEditor::DeleteWordLeft() { const auto wordEnd = GetCursorPosition(); - MoveLeft(); - const auto wordStart = FindWordStart(GetCursorPosition()); + const auto wordStart = FindPreviousWord(GetCursorPosition()); SetSelection(wordStart, wordEnd); Backspace(); } void TextEditor::DeleteWordRight() { const auto wordStart = GetCursorPosition(); - MoveRight(); - const auto wordEnd = FindWordEnd(GetCursorPosition()); + const auto wordEnd = FindNextWord(GetCursorPosition()); SetSelection(wordStart, wordEnd); Backspace(); } @@ -401,17 +328,16 @@ TextEditor::Coordinates TextEditor::FindWordStart(const Coordinates &aFrom) cons auto &line = mLines[at.mLine]; auto cindex = GetCharacterIndex(at); - if (cindex >= (int)line.size()) - return at; - - while (cindex > 0 && !isWordChar(line[cindex-1].mChar)) - --cindex; - - while (cindex > 0 && isWordChar(line[cindex - 1].mChar)) - --cindex; - - if (cindex==0 && line[cindex].mChar == '\"') - ++cindex; + if (isWordChar(line.mChars[cindex])) { + while (cindex > 0 && isWordChar(line.mChars[cindex-1])) + --cindex; + } else if (ispunct(line.mChars[cindex])) { + while (cindex > 0 && ispunct(line.mChars[cindex-1])) + --cindex; + } else if (isspace(line.mChars[cindex])) { + while (cindex > 0 && isspace(line.mChars[cindex-1])) + --cindex; + } return Coordinates(at.mLine, GetCharacterColumn(at.mLine, cindex)); } @@ -423,17 +349,16 @@ TextEditor::Coordinates TextEditor::FindWordEnd(const Coordinates &aFrom) const auto &line = mLines[at.mLine]; auto cindex = GetCharacterIndex(at); - if (cindex >= (int)line.size()) - return at; - - while (cindex < (line.size()) && !isWordChar(line[cindex].mChar)) - ++cindex; - while (cindex < (line.size()) && isWordChar(line[cindex].mChar)) - ++cindex; - - if (line[cindex-1].mChar == '\"') - --cindex; - + if (isWordChar(line.mChars[cindex])) { + while (cindex < (int)line.mChars.size() && isWordChar(line.mChars[cindex])) + ++cindex; + } else if (ispunct(line.mChars[cindex])) { + while (cindex < (int)line.mChars.size() && ispunct(line.mChars[cindex])) + ++cindex; + } else if (isspace(line.mChars[cindex])) { + while (cindex < (int)line.mChars.size() && isspace(line.mChars[cindex])) + ++cindex; + } return Coordinates(aFrom.mLine, GetCharacterColumn(aFrom.mLine, cindex)); } @@ -442,42 +367,57 @@ TextEditor::Coordinates TextEditor::FindNextWord(const Coordinates &aFrom) const if (at.mLine >= (int)mLines.size()) return at; - // skip to the next non-word character - auto cindex = GetCharacterIndex(aFrom); - bool isword = false; - bool skip = false; - if (cindex < (int)mLines[at.mLine].size()) { - const auto &line = mLines[at.mLine]; - isword = isalnum(line[cindex].mChar); - skip = isword; + auto &line = mLines[at.mLine]; + auto cindex = GetCharacterIndex(at); + + if (isspace(line.mChars[cindex])) { + while (cindex < (int)line.mChars.size() && isspace(line.mChars[cindex])) + ++cindex; } - - while (!isword || skip) { - if (at.mLine >= mLines.size()) { - auto l = std::max(0, (int)mLines.size() - 1); - return Coordinates(l, GetLineMaxColumn(l)); - } - - auto &line = mLines[at.mLine]; - if (cindex < (int)line.size()) { - isword = isalnum(line[cindex].mChar); - - if (isword && !skip) - return Coordinates(at.mLine, GetCharacterColumn(at.mLine, cindex)); - - if (!isword) - skip = false; - - cindex++; - } else { - cindex = 0; - ++at.mLine; - skip = false; - isword = false; - } + if (isWordChar(line.mChars[cindex])) { + while (cindex < (int)line.mChars.size() && (isWordChar(line.mChars[cindex]))) + ++cindex; + } else if (ispunct(line.mChars[cindex])) { + while (cindex < (int)line.mChars.size() && (ispunct(line.mChars[cindex]))) + ++cindex; } + return Coordinates(aFrom.mLine, GetCharacterColumn(aFrom.mLine, cindex)); +} - return at; +TextEditor::Coordinates TextEditor::FindPreviousWord(const Coordinates &aFrom) const { + Coordinates at = aFrom; + if (at.mLine >= (int)mLines.size()) + return at; + + auto &line = mLines[at.mLine]; + auto cindex = GetCharacterIndex(at); + + if (isspace(line.mChars[cindex-1])) { + while (cindex > 0 && isspace(line.mChars[cindex-1])) + --cindex; + } + if (isWordChar(line.mChars[cindex-1])) { + while (cindex > 0 && isWordChar(line.mChars[cindex-1])) + --cindex; + } else if (ispunct(line.mChars[cindex-1])) { + while (cindex > 0 && ispunct(line.mChars[cindex-1])) + --cindex; + } + return Coordinates(at.mLine, GetCharacterColumn(at.mLine, cindex)); +} + + + +int TextEditor::Utf8CharsToBytes(std::string line, uint32_t start, uint32_t numChars) { + if (line.empty()) + return 0; + int column = 0; + int index = 0; + while (start + index < line.size() && column < numChars) { + index += UTF8CharLength(line[start + index]); + ++column; + } + return index; } int TextEditor::Utf8CharsToBytes(const Coordinates &aCoordinates) const { @@ -489,8 +429,8 @@ int TextEditor::Utf8CharsToBytes(const Coordinates &aCoordinates) const { int c = 0; int i = 0; while (i < line.size() && c < aCoordinates.mColumn) { - i += UTF8CharLength(line[i].mChar); - if (line[i].mChar == '\t') + i += UTF8CharLength(line[i]); + if (line[i] == '\t') c = (c / mTabSize) * mTabSize + mTabSize; else ++c; @@ -518,7 +458,7 @@ int TextEditor::GetCharacterIndex(const Coordinates &aCoordinates) const { int column = 0; int index = 0; while (index < line.size() && column < aCoordinates.mColumn) { - const auto character = line[index].mChar; + const auto character = line[index]; index += UTF8CharLength(character); if (character == '\t') column = (column / mTabSize) * mTabSize + mTabSize; @@ -536,7 +476,7 @@ int TextEditor::GetCharacterColumn(int aLine, int aIndex) const { int col = 0; int i = 0; while (i < aIndex && i < (int)line.size()) { - auto c = line[i].mChar; + auto c = line[i]; i += UTF8CharLength(c); if (c == '\t') col = (col / mTabSize) * mTabSize + mTabSize; @@ -561,7 +501,7 @@ int TextEditor::GetLineCharacterCount(int aLine) const { auto &line = mLines[aLine]; int c = 0; for (unsigned i = 0; i < line.size(); c++) - i += UTF8CharLength(line[i].mChar); + i += UTF8CharLength(line[i]); return c; } @@ -578,7 +518,7 @@ int TextEditor::GetLineMaxColumn(int aLine) const { auto &line = mLines[aLine]; int col = 0; for (unsigned i = 0; i < line.size();) { - auto c = line[i].mChar; + auto c = line[i]; if (c == '\t') col = (col / mTabSize) * mTabSize + mTabSize; else @@ -597,15 +537,23 @@ bool TextEditor::IsOnWordBoundary(const Coordinates &aAt) const { if (cindex >= (int)line.size()) return true; - if (mColorizerEnabled) - return line[cindex].mColorIndex != line[size_t(cindex - 1)].mColorIndex; - - return isspace(line[cindex].mChar) != isspace(line[cindex - 1].mChar); + if (mColorizerEnabled) { + int wordStart = 0; + int newWordStart = 0; + int wordEnd = 0; + while (wordEnd < cindex) { + wordStart = newWordStart; + auto color = line.mColors[wordStart]; + int tokenLength = line.mColors.find_first_not_of(color, wordStart)-wordStart; + wordEnd = wordStart + tokenLength; + newWordStart = wordEnd + 1; + } + return cindex == wordStart || cindex == wordEnd; + } + return isspace(line[cindex]) != isspace(line[cindex - 1]); } void TextEditor::RemoveLine(int aStart, int aEnd) { - IM_ASSERT(!mReadOnly); - IM_ASSERT(aEnd >= aStart); ErrorMarkers etmp; for (auto &i : mErrorMarkers) { @@ -631,38 +579,24 @@ void TextEditor::RemoveLine(int aStart, int aEnd) { // use clamp to ensure valid results instead of assert. auto start = std::clamp(aStart, 0, (int)mLines.size()-1); auto end = std::clamp(aEnd, 0, (int)mLines.size()); + if (start > end) + std::swap(start, end); + mLines.erase(mLines.begin() + aStart, mLines.begin() + aEnd + 1); mTextChanged = true; } void TextEditor::RemoveLine(int aIndex) { - IM_ASSERT(!mReadOnly); - IM_ASSERT(mLines.size() > 1); - - ErrorMarkers etmp; - for (auto &i : mErrorMarkers) { - ErrorMarkers::value_type e(i.first.mLine > aIndex ? Coordinates(i.first.mLine - 1 ,i.first.mColumn) : i.first, i.second); - if (e.first.mLine - 1 == aIndex) - continue; - etmp.insert(e); - } - mErrorMarkers = std::move(etmp); - - Breakpoints btmp; - for (auto breakpoint : mBreakpoints) { - if (breakpoint > aIndex) { - btmp.insert(breakpoint - 1); - mBreakPointsChanged = true; - }else - btmp.insert(breakpoint); - } - if (mBreakPointsChanged) - mBreakpoints = std::move(btmp); - - mLines.erase(mLines.begin() + aIndex); - IM_ASSERT(!mLines.empty()); + RemoveLine(aIndex, aIndex); +} +void TextEditor::InsertLine(int aIndex, const std::string &aText) { + if (aIndex < 0 || aIndex > (int)mLines.size()) + return; + auto &line = InsertLine(aIndex); + line.append(aText); + line.mColorized = false; mTextChanged = true; } @@ -677,6 +611,7 @@ TextEditor::Line &TextEditor::InsertLine(int aIndex) { auto newLine = Line(); TextEditor::Line &result = *mLines.insert(mLines.begin() + aIndex, newLine); + result.mColorized = false; ErrorMarkers etmp; for (auto &i : mErrorMarkers) @@ -712,34 +647,39 @@ std::string TextEditor::GetWordAt(const Coordinates &aCoords) const { auto iend = GetCharacterIndex(end); for (auto it = istart; it < iend; ++it) - r.push_back(mLines[aCoords.mLine][it].mChar); + r.push_back(mLines[aCoords.mLine][it]); return r; } -ImU32 TextEditor::GetGlyphColor(const Glyph &aGlyph) const { - if (!mColorizerEnabled) - return mPalette[(int)PaletteIndex::Default]; - if (aGlyph.mGlobalDocComment) - return mPalette[(int)PaletteIndex::GlobalDocComment]; - if (aGlyph.mDocComment) - return mPalette[(int)PaletteIndex::DocComment]; - if (aGlyph.mComment) - return mPalette[(int)PaletteIndex::Comment]; - if (aGlyph.mMultiLineComment) - return mPalette[(int)PaletteIndex::MultiLineComment]; - if (aGlyph.mDeactivated) - return mPalette[(int)PaletteIndex::PreprocessorDeactivated]; - const auto color = mPalette[(int)aGlyph.mColorIndex]; - if (aGlyph.mPreprocessor) { - const auto ppcolor = mPalette[(int)PaletteIndex::Preprocessor]; - const int c0 = ((ppcolor & 0xff) + (color & 0xff)) / 2; - const int c1 = (((ppcolor >> 8) & 0xff) + ((color >> 8) & 0xff)) / 2; - const int c2 = (((ppcolor >> 16) & 0xff) + ((color >> 16) & 0xff)) / 2; - const int c3 = (((ppcolor >> 24) & 0xff) + ((color >> 24) & 0xff)) / 2; - return ImU32(c0 | (c1 << 8) | (c2 << 16) | (c3 << 24)); - } - return color; +uint8_t ByteCount(uint32_t number, std::string &bytes) { + bytes[0] = static_cast(number); + if (!(number & 0xFFFFFF00)) + return 1; + bytes[1] = static_cast(number >> 8); + if (!(number & 0xFFFF0000)) + return 2; + bytes[2] = static_cast(number >> 16); + bytes[3] = static_cast(number >> 24); + return 4; +} + +TextEditor::PaletteIndex TextEditor::GetColorIndexFromFlags(Line::Flags flags) { + if (flags.mBits.mGlobalDocComment) + return PaletteIndex::GlobalDocComment; + if (flags.mBits.mBlockDocComment ) + return PaletteIndex::DocBlockComment; + if (flags.mBits.mDocComment) + return PaletteIndex::DocComment; + if (flags.mBits.mBlockComment) + return PaletteIndex::BlockComment; + if (flags.mBits.mComment) + return PaletteIndex::Comment; + if (flags.mBits.mDeactivated) + return PaletteIndex::PreprocessorDeactivated; + if (flags.mBits.mPreprocessor) + return PaletteIndex::Directive; + return PaletteIndex::Default; } void TextEditor::HandleKeyboardInputs() { @@ -869,6 +809,22 @@ inline void TextUnformattedColoredAt(const ImVec2 &pos, const ImU32 &color, cons ImGui::TextUnformatted(text); ImGui::PopStyleColor(); } +uint32_t TextEditor::SkipSpaces(const Coordinates &aFrom) { + auto line = aFrom.mLine; + if (line >= mLines.size()) + return 0; + auto &lines = mLines[line].mChars; + auto &colors = mLines[line].mColors; + auto cindex = GetCharacterIndex(aFrom); + uint32_t s = 0; + while (cindex < (int)lines.size() && lines[cindex] == ' ' && colors[cindex] == 0x00) { + ++s; + ++cindex; + } + if (mUpdateFocus) + SetFocus(); + return s; +} void TextEditor::SetFocus() { mState.mCursorPosition = mFocusAtCoords; @@ -922,14 +878,8 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo auto lineMax = std::clamp(lineNo + mNumberOfLinesDisplayed, 0.0F, globalLineMax-1.0F); int totalDigitCount = std::floor(std::log10(globalLineMax)) + 1; - // Deduce mTextStart by evaluating mLines size (global lineMax) plus two spaces as text width char buf[16]; - if (mShowLineNumbers) - snprintf(buf, 16, " %d ", int(globalLineMax)); - else - buf[0] = '\0'; - mTextStart = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, buf, nullptr, nullptr).x + mLeftMargin; if (!mLines.empty()) { float spaceSize = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, " ", nullptr, nullptr).x; @@ -939,7 +889,7 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo ImVec2 textScreenPos = lineStartScreenPos; auto &line = mLines[lineNo]; - auto columnNo = 0; + auto colors = mLines[lineNo].mColors; Coordinates lineStartCoord(lineNo, 0); Coordinates lineEndCoord(lineNo, GetLineMaxColumn(lineNo)); @@ -985,28 +935,31 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo else mBreakpoints.insert(lineNo + 1); mBreakPointsChanged = true; + mState.mCursorPosition = Coordinates(lineNo, 0); JumpToCoords(mState.mCursorPosition); } - TextUnformattedColoredAt(ImVec2(mLeftMargin + lineNoStartScreenPos.x, lineStartScreenPos.y), mPalette[(int) PaletteIndex::LineNumber], lineNoStr.c_str()); - } - // Draw breakpoints - if (mBreakpoints.count(lineNo + 1) != 0) { - auto end = ImVec2(lineNoStartScreenPos.x + contentSize.x + mLineNumberFieldWidth, lineStartScreenPos.y + mCharAdvance.y); - drawList->AddRectFilled(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y), end, mPalette[(int)PaletteIndex::Breakpoint]); - - drawList->AddCircleFilled(start + ImVec2(0, mCharAdvance.y) / 2, mCharAdvance.y / 3, mPalette[(int)PaletteIndex::Breakpoint]); - drawList->AddCircle(start + ImVec2(0, mCharAdvance.y) / 2, mCharAdvance.y / 3, mPalette[(int)PaletteIndex::Default]); - } - - if (mState.mCursorPosition.mLine == lineNo && mShowCursor) { - - // Highlight the current line (where the cursor is) - if (!HasSelection()) { + // Draw breakpoints + if (mBreakpoints.count(lineNo + 1) != 0) { auto end = ImVec2(lineNoStartScreenPos.x + contentSize.x + mLineNumberFieldWidth, lineStartScreenPos.y + mCharAdvance.y); - drawList->AddRectFilled(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y), end, mPalette[(int)(focused ? PaletteIndex::CurrentLineFill : PaletteIndex::CurrentLineFillInactive)]); - drawList->AddRect(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y), end, mPalette[(int)PaletteIndex::CurrentLineEdge], 1.0f); + drawList->AddRectFilled(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y), end, mPalette[(int)PaletteIndex::Breakpoint]); + + drawList->AddCircleFilled(start + ImVec2(0, mCharAdvance.y) / 2, mCharAdvance.y / 3, mPalette[(int)PaletteIndex::Breakpoint]); + drawList->AddCircle(start + ImVec2(0, mCharAdvance.y) / 2, mCharAdvance.y / 3, mPalette[(int)PaletteIndex::Default]); + drawList->AddText(ImVec2(lineNoStartScreenPos.x + mLeftMargin, lineStartScreenPos.y),mPalette[(int) PaletteIndex::LineNumber], lineNoStr.c_str()); } + + if (mState.mCursorPosition.mLine == lineNo && mShowCursor) { + + // Highlight the current line (where the cursor is) + if (!HasSelection()) { + auto end = ImVec2(lineNoStartScreenPos.x + contentSize.x + mLineNumberFieldWidth, lineStartScreenPos.y + mCharAdvance.y); + drawList->AddRectFilled(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y), end, mPalette[(int)(focused ? PaletteIndex::CurrentLineFill : PaletteIndex::CurrentLineFillInactive)]); + drawList->AddRect(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y), end, mPalette[(int)PaletteIndex::CurrentLineEdge], 1.0f); + } + } + + TextUnformattedColoredAt(ImVec2(mLeftMargin + lineNoStartScreenPos.x, lineStartScreenPos.y), mPalette[(int) PaletteIndex::LineNumber], lineNoStr.c_str()); } if (mShowLineNumbers && !mIgnoreImGuiChild) ImGui::EndChild(); @@ -1024,13 +977,13 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo float cx = TextDistanceToLineStart(mState.mCursorPosition); if (mOverwrite && cindex < (int)line.size()) { - auto c = line[cindex].mChar; + auto c = line[cindex]; if (c == '\t') { auto x = (1.0f + std::floor((1.0f + cx) / (float(mTabSize) * spaceSize))) * (float(mTabSize) * spaceSize); width = x - cx; } else { char buf2[2]; - buf2[0] = line[cindex].mChar; + buf2[0] = line[cindex]; buf2[1] = '\0'; width = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, buf2).x; } @@ -1087,16 +1040,23 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo } // Render colorized text - auto prevColor = line.empty() ? mPalette[(int)PaletteIndex::Default] : GetGlyphColor(line[0]); - ImVec2 bufferOffset; - - if (mUpdateFocus) { - SetFocus(); + if (line.empty()) { + ImGui::Dummy(mCharAdvance); + lineNo = std::floor(lineNo + 1.0F); + if (mUpdateFocus) + SetFocus(); + continue; } - - for (int i = 0; i < line.size();) { - auto &glyph = line[i]; - auto color = GetGlyphColor(glyph); + int i = 0; + auto colorsSize = static_cast(colors.size()); + i += SkipSpaces(Coordinates(lineNo, i)); + while (i < colorsSize) { + char color = colors[i]; + uint32_t tokenLength = colors.find_first_not_of(color, i) - i; + if (mUpdateFocus) + SetFocus(); + color = std::clamp(color, (char)PaletteIndex::Default, (char)((uint8_t)PaletteIndex::Max-1)); + tokenLength = std::clamp(tokenLength, 1u, colorsSize - i); bool underwaved = false; ErrorMarkers::iterator errorIt; @@ -1104,13 +1064,14 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo underwaved = true; } - if ((color != prevColor || glyph.mChar == '\t' || glyph.mChar == ' ') && !mLineBuffer.empty()) { - const ImVec2 newOffset(textScreenPos.x + bufferOffset.x, textScreenPos.y + bufferOffset.y); - TextUnformattedColoredAt(newOffset, prevColor, mLineBuffer.c_str()); - auto textSize = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, mLineBuffer.c_str(), nullptr, nullptr); - bufferOffset.x += textSize.x; - mLineBuffer.clear(); - } + mLineBuffer = line.substr(i, tokenLength); + ImGui::PushStyleColor(ImGuiCol_Text, mPalette[(uint64_t) color]); + auto charsBefore = ImGui::CalcTextSize(line.mChars.substr(0, i).c_str()).x; + const ImVec2 textScreenPosition(lineStartScreenPos.x + charsBefore, lineStartScreenPos.y); + ImGui::SetCursorScreenPos(textScreenPosition); + ImGui::TextUnformatted(mLineBuffer.c_str()); + ImGui::PopStyleColor(); + mLineBuffer.clear(); if (underwaved) { auto textStart = TextDistanceToLineStart(Coordinates(lineNo, i)); auto begin = ImVec2(lineStartScreenPos.x + textStart, lineStartScreenPos.y); @@ -1129,67 +1090,33 @@ void TextEditor::RenderText(const char *aTitle, const ImVec2 &lineNumbersStartPo if (box.trigger()) box.callback(); } - prevColor = color; - if (glyph.mChar == '\t') { - auto oldX = bufferOffset.x; - bufferOffset.x = (1.0f + std::floor((1.0f + bufferOffset.x) / (float(mTabSize) * spaceSize))) * (float(mTabSize) * spaceSize); - ++i; + i += tokenLength; + i += SkipSpaces(Coordinates(lineNo, i)); - if (mShowWhitespaces) { - const auto s = ImGui::GetFontSize(); - const auto x1 = textScreenPos.x + oldX + 1.0f; - const auto x2 = textScreenPos.x + bufferOffset.x - 1.0f; - const auto y = textScreenPos.y + bufferOffset.y + s * 0.5f; - const ImVec2 p1(x1, y); - const ImVec2 p2(x2, y); - const ImVec2 p3(x2 - s * 0.2f, y - s * 0.2f); - const ImVec2 p4(x2 - s * 0.2f, y + s * 0.2f); - drawList->AddLine(p1, p2, 0x90909090); - drawList->AddLine(p2, p3, 0x90909090); - drawList->AddLine(p2, p4, 0x90909090); - } - } else if (glyph.mChar == ' ') { - if (mShowWhitespaces) { - const auto s = ImGui::GetFontSize(); - const auto x = textScreenPos.x + bufferOffset.x + spaceSize * 0.5f; - const auto y = textScreenPos.y + bufferOffset.y + s * 0.5f; - drawList->AddCircleFilled(ImVec2(x, y), 1.5f, 0x80808080, 4); - } - bufferOffset.x += spaceSize; - i++; - } else { - auto l = UTF8CharLength(glyph.mChar); - while (l-- > 0) - mLineBuffer.push_back(line[i++].mChar); - } - ++columnNo; - } - - if (!mLineBuffer.empty()) { - const ImVec2 newOffset(textScreenPos.x + bufferOffset.x, textScreenPos.y + bufferOffset.y); - TextUnformattedColoredAt(newOffset, prevColor, mLineBuffer.c_str()); - mLineBuffer.clear(); } lineNo = std::floor(lineNo + 1.0F); } } + ImVec2 lineStartScreenPos = ImVec2(cursorScreenPos.x + mLeftMargin, mTopMargin + cursorScreenPos.y + std::floor(lineNo) * mCharAdvance.y); if (!mIgnoreImGuiChild) ImGui::EndChild(); if (mShowLineNumbers && !mIgnoreImGuiChild) { ImGui::BeginChild("##lineNumbers"); + ImGui::SetCursorScreenPos(ImVec2(lineNumbersStartPos.x, lineStartScreenPos.y)); ImGui::Dummy(ImVec2(mLineNumberFieldWidth, (globalLineMax - lineMax - 1) * mCharAdvance.y + ImGui::GetCurrentWindow()->InnerClipRect.GetHeight() - mCharAdvance.y)); ImGui::EndChild(); } if (!mIgnoreImGuiChild) ImGui::BeginChild(aTitle); + ImGui::SetCursorScreenPos(lineStartScreenPos); if (mShowLineNumbers) - ImGui::Dummy(ImVec2(mLongestLineLength * mCharAdvance.x, (globalLineMax - lineMax - 2.0F) * mCharAdvance.y + ImGui::GetCurrentWindow()->InnerClipRect.GetHeight())); + ImGui::Dummy(ImVec2(mLongestLineLength * mCharAdvance.x + mCharAdvance.x, (globalLineMax - lineMax - 2.0F) * mCharAdvance.y + ImGui::GetCurrentWindow()->InnerClipRect.GetHeight())); else - ImGui::Dummy(ImVec2(mLongestLineLength * mCharAdvance.x, (globalLineMax - 1.0f - lineMax + GetPageSize() - 1.0f ) * mCharAdvance.y - 2 * ImGuiStyle().WindowPadding.y)); + ImGui::Dummy(ImVec2(mLongestLineLength * mCharAdvance.x + mCharAdvance.x, (globalLineMax - lineMax - 3.0F) * mCharAdvance.y + ImGui::GetCurrentWindow()->InnerClipRect.GetHeight() - 1.0f)); if (mScrollToCursor) EnsureCursorVisible(); @@ -1224,6 +1151,9 @@ void TextEditor::Render(const char *aTitle, const ImVec2 &aSize, bool aBorder) { mWithinRender = true; mCursorPositionChanged = false; + if (mLines.capacity() < 2*mLines.size()) + mLines.reserve(2*mLines.size()); + auto scrollBg = ImGui::GetStyleColorVec4(ImGuiCol_ScrollbarBg); scrollBg.w = 0.0f; auto scrollBarSize = ImGui::GetStyle().ScrollbarSize; @@ -1250,9 +1180,11 @@ void TextEditor::Render(const char *aTitle, const ImVec2 &aSize, bool aBorder) { ImVec2 textEditorSize = aSize; textEditorSize.x -= mLineNumberFieldWidth; - bool scroll_x = mLongestLineLength * mCharAdvance.x > textEditorSize.x; + + bool scroll_x = mLongestLineLength * mCharAdvance.x >= textEditorSize.x; + bool scroll_y = mLines.size() > 1; - if (!aBorder && scroll_y) + if (!aBorder) textEditorSize.x -= scrollBarSize; ImGui::SetCursorScreenPos(ImVec2(position.x + mLineNumberFieldWidth, position.y)); ImGuiChildFlags childFlags = aBorder ? ImGuiChildFlags_Borders : ImGuiChildFlags_None; @@ -1280,6 +1212,7 @@ void TextEditor::Render(const char *aTitle, const ImVec2 &aSize, bool aBorder) { if (mHandleMouseInputs) HandleMouseInputs(); + ColorizeInternal(); RenderText(aTitle, position, textEditorSize); @@ -1303,13 +1236,15 @@ void TextEditor::SetText(const std::string &aText, bool aUndo) { u.mRemovedEnd = Coordinates((int) mLines.size() - 1, GetLineMaxColumn((int) mLines.size() - 1)); } mLines.resize(1); - mLines[0].clear(); + mLines[0] = Line(); + mLines[0].mColorized = false; std::string text = PreprocessText(aText); for (auto chr : text) { - if (chr == '\n') + if (chr == '\n') { mLines.push_back(Line()); - else - mLines.back().push_back(Glyph(chr, PaletteIndex::Default)); + mLines.back().mColorized = false; + } else + mLines.back().push_back(Glyph(chr)); } if (!mReadOnly && aUndo) { u.mAdded = text; @@ -1330,23 +1265,21 @@ void TextEditor::SetText(const std::string &aText, bool aUndo) { void TextEditor::SetTextLines(const std::vector &aLines) { mLines.clear(); - if (isEmpty()) { + if (aLines.empty()) { mLines.emplace_back(Line()); + mLines[0].mColorized = false; } else { - mLines.resize(aLines.size()); - - for (size_t i = 0; i < aLines.size(); ++i) { - const std::string &aLine = PreprocessText(aLines[i]); - - mLines[i].reserve(aLine.size()); - for (size_t j = 0; j < aLine.size(); ++j) - mLines[i].emplace_back(Glyph(aLine[j], PaletteIndex::Default)); - } + auto lineCount = aLines.size(); + mLines.resize(lineCount); + for (size_t i = 0; i < lineCount; i++) { + mLines[i].SetLine(PreprocessText(aLines[i])); + mLines[i].mColorized = false; + } } mTextChanged = true; mScrollToTop = true; - + SetTimeStamp(0); mUndoBuffer.clear(); mUndoIndex = 0; @@ -1355,7 +1288,6 @@ void TextEditor::SetTextLines(const std::vector &aLines) { void TextEditor::EnterCharacter(ImWchar aChar, bool aShift) { IM_ASSERT(!mReadOnly); - UndoRecord u; u.mBefore = mState; @@ -1389,19 +1321,21 @@ void TextEditor::EnterCharacter(ImWchar aChar, bool aShift) { auto &line = mLines[i]; if (aShift) { if (!line.empty()) { - if (line.front().mChar == '\t') { + if (line.front() == '\t') { line.erase(line.begin()); modified = true; } else { - for (int j = 0; j < mTabSize && !line.empty() && line.front().mChar == ' '; j++) { + for (int j = 0; j < mTabSize && !line.empty() && line.front() == ' '; j++) { line.erase(line.begin()); modified = true; } } } } else { - for (int j = start.mColumn % mTabSize; j < mTabSize; j++) - line.insert(line.begin(), Glyph(' ', PaletteIndex::Background)); + auto spacesToInsert = mTabSize - (start.mColumn % mTabSize); + std::string spaces(spacesToInsert, ' '); + line.insert(line.begin(), spaces.begin(), spaces.end()); + line.mColorized = false; modified = true; } } @@ -1455,7 +1389,7 @@ void TextEditor::EnterCharacter(ImWchar aChar, bool aShift) { auto &newLine = mLines[coord.mLine + 1]; if (mLanguageDefinition.mAutoIndentation) - for (size_t it = 0; it < line.size() && isascii(line[it].mChar) && isblank(line[it].mChar); ++it) + for (size_t it = 0; it < line.size() && isascii(line[it]) && isblank(line[it]); ++it) newLine.push_back(line[it]); const size_t whitespaceSize = newLine.size(); @@ -1470,7 +1404,8 @@ void TextEditor::EnterCharacter(ImWchar aChar, bool aShift) { cpos = (int) whitespaceSize; } newLine.insert(newLine.end(), line.begin() + cstart, line.end()); - line.erase(line.begin() + cstart, line.begin() + line.size()); + line.erase(line.begin() + cstart,-1); + line.mColorized = false; SetCursorPosition(Coordinates(coord.mLine + 1, GetCharacterColumn(coord.mLine + 1, cpos))); u.mAdded = (char)aChar; } else if (aChar == '\t') { @@ -1479,60 +1414,57 @@ void TextEditor::EnterCharacter(ImWchar aChar, bool aShift) { if (!aShift) { auto spacesToInsert = mTabSize - (cindex % mTabSize); - for (int j = 0; j < spacesToInsert; j++) - line.insert(line.begin() + cindex, Glyph(' ', PaletteIndex::Background)); + std::string spaces(spacesToInsert, ' '); + line.insert(line.begin() + cindex, spaces.begin(), spaces.end()); + line.mColorized = false; SetCursorPosition(Coordinates(coord.mLine, GetCharacterColumn(coord.mLine, cindex + spacesToInsert))); } else { auto spacesToRemove = (cindex % mTabSize); if (spacesToRemove == 0) spacesToRemove = mTabSize; spacesToRemove = std::min(spacesToRemove, (int32_t) line.size()); for (int j = 0; j < spacesToRemove; j++) { - if ((line.begin() + cindex - 1)->mChar == ' ') { - line.erase(line.begin() + cindex - 1); + if (*(line.begin() + (cindex - 1)) == ' ') { + line.erase(line.begin() + (cindex - 1)); cindex -= 1; } } - + line.mColorized = false; SetCursorPosition(Coordinates(coord.mLine, GetCharacterColumn(coord.mLine, std::max(0, cindex)))); } } else { - char buf[7]; - int e = ImTextCharToUtf8(buf, 7, aChar); - if (e > 0) { - buf[e] = '\0'; + std::string buf = ""; + ImTextCharToUtf8(buf, aChar); + if (buf.size() > 0) { auto &line = mLines[coord.mLine]; auto cindex = GetCharacterIndex(coord); if (mOverwrite && cindex < (int)line.size()) { - auto d = UTF8CharLength(line[cindex].mChar); + auto d = UTF8CharLength(line[cindex]); u.mRemovedStart = mState.mCursorPosition; - u.mRemovedEnd = Coordinates(coord.mLine, GetCharacterColumn(coord.mLine, cindex + d)); - - while (d-- > 0 && cindex < (int)line.size()) { - u.mRemoved += line[cindex].mChar; - line.erase(line.begin() + cindex); - } + u.mRemovedEnd = Coordinates(coord.mLine, GetCharacterColumn(coord.mLine, cindex + d)); + u.mRemoved = std::string(line.mChars.begin() + cindex, line.mChars.begin() + cindex + d); + line.erase(line.begin() + cindex, d); + line.mColorized = false; } - - for (auto p = buf; *p != '\0'; p++, ++cindex) - line.insert(line.begin() + cindex, Glyph(*p, PaletteIndex::Default)); + line.insert(line.begin() + cindex, buf.begin(), buf.end()); + line.mColorized = false; u.mAdded = buf; - - SetCursorPosition(Coordinates(coord.mLine, GetCharacterColumn(coord.mLine, cindex))); + auto charCount = GetStringCharacterCount(buf); + SetCursorPosition(Coordinates(coord.mLine, GetCharacterColumn(coord.mLine, cindex + charCount))); } else return; } - mTextChanged = true; - u.mAddedEnd = GetActualCursorCoordinates(); u.mAfter = mState; + mTextChanged = true; + AddUndo(u); - Colorize(coord.mLine - 1, 3); + Colorize(); std::string findWord = mFindReplaceHandler.GetFindWord(); if (!findWord.empty()) { @@ -1603,10 +1535,10 @@ void TextEditor::InsertText(const char *aValue) { auto pos = GetActualCursorCoordinates(); auto start = std::min(pos, mState.mSelectionStart); - int totalLines = pos.mLine - start.mLine; auto text = PreprocessText(aValue); - totalLines += InsertTextAt(pos, text); + InsertTextAt(pos, text); + mLines[pos.mLine].mColorized = false; SetSelection(pos, pos); SetCursorPosition(pos); @@ -1616,7 +1548,7 @@ void TextEditor::InsertText(const char *aValue) { mFindReplaceHandler.resetMatches(); mFindReplaceHandler.FindAllMatches(this, findWord); } - Colorize(start.mLine - 1, totalLines + 2); + Colorize(); } void TextEditor::DeleteSelection() { @@ -1634,7 +1566,7 @@ void TextEditor::DeleteSelection() { mFindReplaceHandler.resetMatches(); mFindReplaceHandler.FindAllMatches(this, findWord); } - Colorize(mState.mSelectionStart.mLine, 1); + Colorize(); } void TextEditor::JumpToLine(int line) { @@ -1742,21 +1674,18 @@ void TextEditor::MoveLeft(int aAmount, bool aSelect, bool aWordMode) { else cindex = 0; } + } else if (aWordMode) { + mState.mCursorPosition = FindPreviousWord(mState.mCursorPosition); + cindex = GetCharacterIndex(mState.mCursorPosition); } else { --cindex; if (cindex > 0) { if ((int)mLines.size() > lindex) { - while (cindex > 0 && IsUTFSequence(line[cindex].mChar)) + while (cindex > 0 && IsUTFSequence(line[cindex])) --cindex; } } } - - mState.mCursorPosition = Coordinates(lindex, GetCharacterColumn(lindex, cindex)); - if (aWordMode) { - mState.mCursorPosition = FindWordStart(mState.mCursorPosition); - cindex = GetCharacterIndex(mState.mCursorPosition); - } } mState.mCursorPosition = Coordinates(lindex, GetCharacterColumn(lindex, cindex)); @@ -1773,6 +1702,7 @@ void TextEditor::MoveLeft(int aAmount, bool aSelect, bool aWordMode) { } } else mInteractiveStart = mInteractiveEnd = mState.mCursorPosition; + SetSelection(mInteractiveStart, mInteractiveEnd); EnsureCursorVisible(); @@ -1797,22 +1727,18 @@ void TextEditor::MoveRight(int aAmount, bool aSelect, bool aWordMode) { ++lindex; cindex = 0; } + } else if (aWordMode) { + mState.mCursorPosition = FindNextWord(mState.mCursorPosition); + cindex = GetCharacterIndex(mState.mCursorPosition); } else { ++cindex; if (cindex < (int)line.size()) { if ((int)mLines.size() > lindex) { - while (cindex < (int)line.size() && IsUTFSequence(line[cindex].mChar)) + while (cindex < (int)line.size() && IsUTFSequence(line[cindex])) ++cindex; } } } - - mState.mCursorPosition = Coordinates(lindex, GetCharacterColumn(lindex, cindex)); - - if (aWordMode) { - mState.mCursorPosition = FindWordEnd(mState.mCursorPosition); - cindex = GetCharacterIndex(mState.mCursorPosition); - } } mState.mCursorPosition = Coordinates(lindex, GetCharacterColumn(lindex, cindex)); @@ -1829,6 +1755,7 @@ void TextEditor::MoveRight(int aAmount, bool aSelect, bool aWordMode) { } } else mInteractiveStart = mInteractiveEnd = mState.mCursorPosition; + SetSelection(mInteractiveStart, mInteractiveEnd); EnsureCursorVisible(); @@ -1865,20 +1792,49 @@ void TextEditor::TextEditor::MoveBottom(bool aSelect) { void TextEditor::MoveHome(bool aSelect) { ResetCursorBlinkTime(); auto oldPos = mState.mCursorPosition; - auto &line = mLines[mState.mCursorPosition.mLine]; - if (line.size() == 0) + + auto &line = mLines[oldPos.mLine]; + auto prefix = line.substr(0, oldPos.mColumn); + auto postfix = line.substr(oldPos.mColumn); + if (prefix.empty() && postfix.empty()) return; - - auto limit = oldPos.mColumn != 0 ? oldPos.mColumn : line.size(); - auto home=0; - while (home < limit && isspace(line[home].mChar)) - home++; - - if (home == oldPos.mColumn) { - home = 0; + if (!prefix.empty()) { + auto idx = prefix.find_first_not_of(" "); + if (idx == std::string::npos) { + auto postIdx = postfix.find_first_of(" "); + if (postIdx == std::string::npos || postIdx == 0) + home=0; + else { + postIdx = postfix.find_first_not_of(" "); + if (postIdx == std::string::npos) + home = GetLineMaxColumn(oldPos.mLine); + else if (postIdx == 0) + home = 0; + else + home = oldPos.mColumn + postIdx; + } + } else + home = idx; + } else { + auto postIdx = postfix.find_first_of(" "); + if (postIdx == std::string::npos) + home = 0; + else { + postIdx = postfix.find_first_not_of(" "); + if (postIdx == std::string::npos) + home = GetLineMaxColumn(oldPos.mLine); + else + home = oldPos.mColumn + postIdx; + } } + + + //while (home < line.size() && isspace(line[home])) + // home++; + //if (home == oldPos.mColumn) + // home = 0; SetCursorPosition(Coordinates(mState.mCursorPosition.mLine, home)); if (mState.mCursorPosition != oldPos) { if (aSelect) { @@ -1931,7 +1887,6 @@ void TextEditor::Delete() { u.mRemoved = GetSelectedText(); u.mRemovedStart = mState.mSelectionStart; u.mRemovedEnd = mState.mSelectionEnd; - DeleteSelection(); } else { auto pos = GetActualCursorCoordinates(); @@ -1948,21 +1903,23 @@ void TextEditor::Delete() { auto &nextLine = mLines[pos.mLine + 1]; line.insert(line.end(), nextLine.begin(), nextLine.end()); + line.mColorized = false; RemoveLine(pos.mLine + 1); + } else { auto cindex = GetCharacterIndex(pos); u.mRemovedStart = u.mRemovedEnd = GetActualCursorCoordinates(); u.mRemovedEnd.mColumn++; u.mRemoved = GetText(u.mRemovedStart, u.mRemovedEnd); - auto d = UTF8CharLength(line[cindex].mChar); - while (d-- > 0 && cindex < (int)line.size()) - line.erase(line.begin() + cindex); + auto d = UTF8CharLength(line[cindex]); + line.erase(line.begin() + cindex, d); + line.mColorized = false; } mTextChanged = true; - Colorize(pos.mLine, 1); + Colorize(); } u.mAfter = mState; @@ -1976,9 +1933,7 @@ void TextEditor::Delete() { void TextEditor::Backspace() { ResetCursorBlinkTime(); - IM_ASSERT(!mReadOnly); - - if (isEmpty()) + if (isEmpty() || mReadOnly) return; UndoRecord u; @@ -1988,54 +1943,57 @@ void TextEditor::Backspace() { u.mRemoved = GetSelectedText(); u.mRemovedStart = mState.mSelectionStart; u.mRemovedEnd = mState.mSelectionEnd; - DeleteSelection(); } else { auto pos = GetActualCursorCoordinates(); SetCursorPosition(pos); + auto &line = mLines[pos.mLine]; - if (mState.mCursorPosition.mColumn == 0) { - if (mState.mCursorPosition.mLine == 0) + if (pos.mColumn == 0) { + if (pos.mLine == 0) return; u.mRemoved = '\n'; u.mRemovedStart = u.mRemovedEnd = Coordinates(pos.mLine - 1, GetLineMaxColumn(pos.mLine - 1)); Advance(u.mRemovedEnd); - auto &line = mLines[mState.mCursorPosition.mLine]; - auto &prevLine = mLines[mState.mCursorPosition.mLine - 1]; - auto prevSize = GetLineMaxColumn(mState.mCursorPosition.mLine - 1); - prevLine.insert(prevLine.end(), line.begin(), line.end()); + auto &prevLine = mLines[pos.mLine - 1]; + auto prevSize = GetLineMaxColumn(pos.mLine - 1); + if (prevSize == 0) + prevLine = line; + else + prevLine.insert(prevLine.end(), line.begin(), line.end()); + prevLine.mColorized = false; + ErrorMarkers etmp; for (auto &i : mErrorMarkers) etmp.insert(ErrorMarkers::value_type(i.first.mLine - 1 == mState.mCursorPosition.mLine ? Coordinates(i.first.mLine - 1,i.first.mColumn) : i.first, i.second)); mErrorMarkers = std::move(etmp); - RemoveLine(mState.mCursorPosition.mLine); --mState.mCursorPosition.mLine; mState.mCursorPosition.mColumn = prevSize; } else { - auto &line = mLines[mState.mCursorPosition.mLine]; auto cindex = GetCharacterIndex(pos) - 1; auto cend = cindex + 1; - while (cindex > 0 && IsUTFSequence(line[cindex].mChar)) + while (cindex > 0 && IsUTFSequence(line[cindex])) --cindex; u.mRemovedStart = u.mRemovedEnd = GetActualCursorCoordinates(); --u.mRemovedStart.mColumn; mState.mCursorPosition.mColumn = GetCharacterColumn(mState.mCursorPosition.mLine, cindex); - - while (cindex < line.size() && cend-- > cindex) { - u.mRemoved += line[cindex].mChar; - line.erase(line.begin() + cindex); - } + u.mRemoved = GetText(u.mRemovedStart, u.mRemovedEnd); + if (cend > cindex && cend < (int) line.size()) + line.erase(line.begin() + cindex, cend-cindex); + else + line.erase(line.begin() + cindex, -1); + line.mColorized = false; } mTextChanged = true; EnsureCursorVisible(); - Colorize(mState.mCursorPosition.mLine, 1); + Colorize(); } u.mAfter = mState; @@ -2066,9 +2024,8 @@ void TextEditor::Copy() { } else { if (!isEmpty()) { std::string str; - auto &line = mLines[GetActualCursorCoordinates().mLine]; - for (auto &g : line) - str.push_back(g.mChar); + const auto &line = mLines[GetActualCursorCoordinates().mLine]; + std::copy(line.mChars.begin(), line.mChars.end(), std::back_inserter(str)); ImGui::SetClipboardText(str.c_str()); } } @@ -2078,19 +2035,23 @@ void TextEditor::Cut() { if (IsReadOnly()) { Copy(); } else { - if (HasSelection()) { - UndoRecord u; - u.mBefore = mState; - u.mRemoved = GetSelectedText(); - u.mRemovedStart = mState.mSelectionStart; - u.mRemovedEnd = mState.mSelectionEnd; - - Copy(); - DeleteSelection(); - - u.mAfter = mState; - AddUndo(u); + if (!HasSelection()) { + auto lineIndex = GetActualCursorCoordinates().mLine; + if (lineIndex < 0 || lineIndex >= (int)mLines.size()) + return; + SetSelection(Coordinates(lineIndex, 0), Coordinates(lineIndex+1, 0)); } + UndoRecord u; + u.mBefore = mState; + u.mRemoved = GetSelectedText(); + u.mRemovedStart = mState.mSelectionStart; + u.mRemovedEnd = mState.mSelectionEnd; + + Copy(); + DeleteSelection(); + + u.mAfter = mState; + AddUndo(u); } std::string findWord = mFindReplaceHandler.GetFindWord(); if (!findWord.empty()) { @@ -2100,16 +2061,27 @@ void TextEditor::Cut() { } std::string TextEditor::ReplaceStrings(std::string string, const std::string &search, const std::string &replace) { - if (search.empty()) + std::string result; + if (string.empty()) return string; - - std::size_t pos = 0; - while ((pos = string.find(search, pos)) != std::string::npos) { - string.replace(pos, search.size(), replace); - pos += replace.size(); + if (search.empty()) { + for (auto c: string) { + if (c == '\0') { + result.append(replace); + } else { + result.push_back(c); + } + } + } else { + result = string; + std::size_t pos = 0; + while ((pos = result.find(search, pos)) != std::string::npos) { + result.replace(pos, search.size(), replace); + pos += replace.size(); + } } - return string; + return result; } std::vector TextEditor::SplitString(const std::string &string, const std::string &delimiter, bool removeEmpty) { @@ -2152,7 +2124,7 @@ std::string TextEditor::ReplaceTabsWithSpaces(const std::string& string, uint32_ while ((pos = line.find('\t', pos)) != std::string::npos) { auto spaces = tabSize - (pos % tabSize); line.replace(pos, 1, std::string(spaces, ' ')); - pos += tabSize - 1; + pos += spaces - 1; } result += line; if (i < size - 1) @@ -2165,6 +2137,7 @@ std::string TextEditor::ReplaceTabsWithSpaces(const std::string& string, uint32_ std::string TextEditor::PreprocessText(const std::string &code) { std::string result = ReplaceStrings(code, "\r\n", "\n"); result = ReplaceStrings(result, "\r", "\n"); + result = ReplaceStrings(result, "\000", "."); result = ReplaceTabsWithSpaces(result, 4); return result; @@ -2175,27 +2148,28 @@ void TextEditor::Paste() { return; auto clipText = ImGui::GetClipboardText(); - if (clipText != nullptr && strlen(clipText) > 0) { - std::string text = PreprocessText(clipText); + if (clipText != nullptr ) { + auto len = strlen(clipText); + if (len > 0 ) { + std::string text = PreprocessText(clipText); + UndoRecord u; + u.mBefore = mState; - UndoRecord u; - u.mBefore = mState; + if (HasSelection()) { + u.mRemoved = GetSelectedText(); + u.mRemovedStart = mState.mSelectionStart; + u.mRemovedEnd = mState.mSelectionEnd; + DeleteSelection(); + } - if (HasSelection()) { - u.mRemoved = GetSelectedText(); - u.mRemovedStart = mState.mSelectionStart; - u.mRemovedEnd = mState.mSelectionEnd; - DeleteSelection(); + u.mAdded = text; + u.mAddedStart = GetActualCursorCoordinates(); + InsertText(text); + + u.mAddedEnd = GetActualCursorCoordinates(); + u.mAfter = mState; + AddUndo(u); } - - u.mAdded = text; - u.mAddedStart = GetActualCursorCoordinates(); - - InsertText(text); - - u.mAddedEnd = GetActualCursorCoordinates(); - u.mAfter = mState; - AddUndo(u); } std::string findWord = mFindReplaceHandler.GetFindWord(); if (!findWord.empty()) { @@ -2239,7 +2213,7 @@ void TextEditor::FindReplaceHandler::SelectFound(TextEditor *editor, int index) auto selectionEnd = mMatches[index].mSelectionEnd; editor->SetSelection(selectionStart, selectionEnd); editor->SetCursorPosition(selectionEnd); - editor->mScrollToCursor = true; + editor->EnsureCursorVisible(); } // The returned index is shown in the form @@ -2366,8 +2340,7 @@ std::string make_wholeWord(const std::string &s) { bool TextEditor::FindReplaceHandler::FindNext(TextEditor *editor) { Coordinates curPos; curPos.mLine = mMatches.empty() ? editor->mState.mCursorPosition.mLine : mMatches.back().mCursorPosition.mLine; - curPos.mColumn = mMatches.empty() ? editor->mState.mCursorPosition.mColumn : editor->Utf8CharsToBytes( - mMatches.back().mCursorPosition); + curPos.mColumn = mMatches.empty() ? editor->mState.mCursorPosition.mColumn : editor->Utf8CharsToBytes(mMatches.back().mCursorPosition); unsigned long matchLength = editor->GetStringCharacterCount(mFindWord); size_t byteIndex = 0; @@ -2493,7 +2466,6 @@ void TextEditor::FindReplaceHandler::FindAllMatches(TextEditor *editor,std::stri bool TextEditor::FindReplaceHandler::Replace(TextEditor *editor, bool next) { - if (mMatches.empty() || mFindWord == mReplaceWord || mFindWord.empty()) return false; @@ -2519,7 +2491,6 @@ bool TextEditor::FindReplaceHandler::Replace(TextEditor *editor, bool next) { u.mRemoved = editor->GetSelectedText(); u.mRemovedStart = editor->mState.mSelectionStart; u.mRemovedEnd = editor->mState.mSelectionEnd; - editor->DeleteSelection(); if (GetFindRegEx()) { std::string replacedText = std::regex_replace(editor->GetText(), std::regex(mFindWord), mReplaceWord, std::regex_constants::format_first_only | std::regex_constants::format_no_copy); @@ -2528,8 +2499,8 @@ bool TextEditor::FindReplaceHandler::Replace(TextEditor *editor, bool next) { u.mAdded = mReplaceWord; u.mAddedStart = editor->GetActualCursorCoordinates(); - editor->InsertText(u.mAdded); + editor->SetCursorPosition(editor->mState.mSelectionEnd); u.mAddedEnd = editor->GetActualCursorCoordinates(); @@ -2654,7 +2625,7 @@ const TextEditor::Palette &TextEditor::GetRetroBluePalette() { std::string TextEditor::GetText() const { - return GetText(Coordinates(), Coordinates((int)mLines.size(), 0)); + return GetText(Coordinates(), Coordinates(-1, -1)); } std::vector TextEditor::GetTextLines() const { @@ -2663,13 +2634,7 @@ std::vector TextEditor::GetTextLines() const { result.reserve(mLines.size()); for (auto &line : mLines) { - std::string text; - - text.resize(line.size()); - - for (size_t i = 0; i < line.size(); ++i) - text[i] = line[i].mChar; - + std::string text = line.mChars; result.emplace_back(std::move(text)); } @@ -2692,102 +2657,136 @@ std::string TextEditor::GetLineText(int line) const { void TextEditor::ProcessInputs() { } -void TextEditor::Colorize(int aFromLine, int aLines) { - int toLine = aLines == -1 ? (int)mLines.size() : std::min((int)mLines.size(), aFromLine + aLines); - mColorRangeMin = std::min(mColorRangeMin, aFromLine); - mColorRangeMax = std::max(mColorRangeMax, toLine); - mColorRangeMin = std::max(0, mColorRangeMin); - mColorRangeMax = std::max(mColorRangeMin, mColorRangeMax); - mCheckComments = true; +void TextEditor::Colorize() { + mUpdateFlags = true; } -void TextEditor::ColorizeRange(int aFromLine, int aToLine) { - if (isEmpty() || aFromLine >= aToLine) +void TextEditor::ColorizeRange() { + + if (isEmpty()) return; - std::string buffer; - std::cmatch results; + std::smatch results; std::string id; - - int endLine = std::max(0, std::min((int)mLines.size(), aToLine)); - for (int i = aFromLine; i < endLine; ++i) { + if (mLanguageDefinition.mTokenize == nullptr) + mLanguageDefinition.mTokenize = []( strConstIter, strConstIter, strConstIter &, strConstIter &, PaletteIndex &) { return false; }; + auto linesSize = mLines.size(); + for (int i = 0; i < linesSize; ++i) { auto &line = mLines[i]; + auto size = line.size(); - if (line.empty()) - continue; - - buffer.resize(line.size()); - for (size_t j = 0; j < line.size(); ++j) { - auto &col = line[j]; - buffer[j] = col.mChar; - col.mColorIndex = PaletteIndex::Default; + if (line.mColors.size() != size) { + line.mColors.resize(size, 0); + line.mColorized = false; } - const char *bufferBegin = &buffer.front(); - const char *bufferEnd = bufferBegin + buffer.size(); + if (line.mColorized || line.empty()) + continue; - auto last = bufferEnd; + auto last = line.end(); - for (auto first = bufferBegin; first != last;) { - const char *token_begin = nullptr; - const char *token_end = nullptr; + auto first = line.begin(); + for (auto current = first;(current-first) < size;) { + strConstIter token_begin; + strConstIter token_end; PaletteIndex token_color = PaletteIndex::Default; - bool hasTokenizeResult = false; - - if (mLanguageDefinition.mTokenize != nullptr) { - if (mLanguageDefinition.mTokenize(first, last, token_begin, token_end, token_color)) - hasTokenizeResult = true; - } + bool hasTokenizeResult = mLanguageDefinition.mTokenize(current.mCharsIter, last.mCharsIter, token_begin, token_end, token_color); + auto token_offset = token_begin - first.mCharsIter; if (!hasTokenizeResult) { // todo : remove // printf("using regex for %.*s\n", first + 10 < last ? 10 : int(last - first), first); for (auto &p : mRegexList) { - if (std::regex_search(first, last, results, p.first, std::regex_constants::match_continuous)) { + if (std::regex_search(first.mCharsIter, last.mCharsIter, results, p.first, std::regex_constants::match_continuous)) { hasTokenizeResult = true; - auto &v = *results.begin(); - token_begin = v.first; - token_end = v.second; + const auto &v = results.begin(); + token_begin = v->first; + token_end = v->second; token_color = p.second; break; } } } - if (hasTokenizeResult == false) { - first++; - } else { - const size_t token_length = token_end - token_begin; + if (!hasTokenizeResult) + current=current + 1; + else { + current = first + token_offset; + size_t token_length=0; + Line::Flags flags(0); + flags.mValue = line.mFlags[token_offset]; + if (flags.mValue == 0) { + token_length = token_end - token_begin; + if (token_color == PaletteIndex::Identifier) { + id.assign(token_begin, token_end); - if (token_color == PaletteIndex::Identifier) { - id.assign(token_begin, token_end); - - // todo : allmost all language definitions use lower case to specify keywords, so shouldn't this use ::tolower ? - if (!mLanguageDefinition.mCaseSensitive) - std::transform(id.begin(), id.end(), id.begin(), ::toupper); - - if (!line[first - bufferBegin].mPreprocessor) { - if (mLanguageDefinition.mKeywords.count(id) != 0) + // todo : almost all language definitions use lower case to specify keywords, so shouldn't this use ::tolower ? + if (!mLanguageDefinition.mCaseSensitive) + std::transform(id.begin(), id.end(), id.begin(), ::toupper); + else if (mLanguageDefinition.mKeywords.count(id) != 0) token_color = PaletteIndex::Keyword; else if (mLanguageDefinition.mIdentifiers.count(id) != 0) - token_color = PaletteIndex::KnownIdentifier; - else if (mLanguageDefinition.mPreprocIdentifiers.count(id) != 0) - token_color = PaletteIndex::PreprocIdentifier; - } else { - if (mLanguageDefinition.mPreprocIdentifiers.count(id) != 0) + token_color = PaletteIndex::BuiltInType; + else if (id == "$") + token_color = PaletteIndex::GlobalVariable; + } + } else { + if ((token_color == PaletteIndex::Identifier || flags.mBits.mPreprocessor) && !flags.mBits.mDeactivated && !(flags.mValue & Line::InComment)) { + id.assign(token_begin, token_end); + if (mLanguageDefinition.mPreprocIdentifiers.count(id) != 0) { + token_color = PaletteIndex::Directive; + token_begin -= 1; + token_length = token_end - token_begin; + token_offset -= 1; + } else if (flags.mBits.mPreprocessor) { token_color = PaletteIndex::PreprocIdentifier; + token_length = token_end - token_begin; + } + } + if ((token_color != PaletteIndex::Directive && token_color != PaletteIndex::PreprocIdentifier) || flags.mBits.mDeactivated) { + if (flags.mBits.mDeactivated && flags.mBits.mPreprocessor) { + token_color = PaletteIndex::PreprocessorDeactivated; + token_begin -= 1; + token_offset -= 1; + } else if (id.assign(token_begin, token_end);flags.mValue & Line::InComment && mLanguageDefinition.mPreprocIdentifiers.count(id) != 0) { + token_color = GetColorIndexFromFlags(flags); + token_begin -= 1; + token_offset -= 1; + } + + auto flag = line.mFlags[token_offset]; + token_length = line.mFlags.find_first_not_of(flag, token_offset + 1); + if (token_length == std::string::npos) + token_length = line.size() - token_offset; + else + token_length -= token_offset; + + token_end = token_begin + token_length; + if (!flags.mBits.mPreprocessor || flags.mBits.mDeactivated) + token_color = GetColorIndexFromFlags(flags); } } - for (size_t j = 0; j < token_length; ++j) - line[(token_begin - bufferBegin) + j].mColorIndex = token_color; - - first = token_end; + if (token_color != PaletteIndex::Identifier || *current.mColorsIter == static_cast(PaletteIndex::Identifier)) { + if (token_offset + token_length >= (int)line.mColors.size()) { + auto colors = line.mColors; + line.mColors.resize(token_offset + token_length, static_cast(PaletteIndex::Default)); + std::copy(colors.begin(), colors.end(), line.mColors.begin()); + } + try { + line.mColors.replace(token_offset, token_length, token_length, static_cast(token_color)); + } catch (const std::exception &e) { + std::cerr << "Error replacing color: " << e.what() << std::endl; + return; + } + } + current = current + token_length; } } + line.mColorized = true; } } @@ -2795,218 +2794,197 @@ void TextEditor::ColorizeInternal() { if (isEmpty() || !mColorizerEnabled) return; - if (mCheckComments) { - auto endLine = mLines.size(); - auto endIndex = 0; - auto commentStartLine = endLine; - auto commentStartIndex = endIndex; - auto withinGlobalDocComment = false; - auto withinDocComment = false; - auto withinComment = false; - auto withinString = false; - auto withinSingleLineComment = false; - auto withinPreproc = false; - auto withinNotDef = false; - auto firstChar = true; // there is no other non-whitespace characters in the line before - auto currentLine = 0; - auto currentIndex = 0; - auto commentLength = 0; - auto &startStr = mLanguageDefinition.mCommentStart; - auto &singleStartStr = mLanguageDefinition.mSingleLineComment; - auto &docStartStr = mLanguageDefinition.mDocComment; - auto &globalStartStr = mLanguageDefinition.mGlobalDocComment; + if (mUpdateFlags) { + auto endLine = mLines.size(); + auto commentStartLine = endLine; + auto commentStartIndex = 0; + auto withinGlobalDocComment = false; + auto withinBlockDocComment = false; + auto withinString = false; + auto withinBlockComment = false; + auto withinNotDef = false; + auto currentLine = 0; + auto commentLength = 0; std::vector ifDefs; ifDefs.push_back(true); - - while (currentLine < endLine || currentIndex < endIndex) { + mDefines.push_back("__IMHEX__"); + for (currentLine = 0; currentLine < endLine; currentLine++) { auto &line = mLines[currentLine]; + auto lineLength = line.size(); - auto setGlyphFlags = [&](int index) { - line[index].mMultiLineComment = withinComment; - line[index].mComment = withinSingleLineComment; - line[index].mDocComment = withinDocComment; - line[index].mGlobalDocComment = withinGlobalDocComment; - line[index].mDeactivated = withinNotDef; - }; - - if (currentIndex == 0) { - withinSingleLineComment = false; - withinPreproc = false; - firstChar = true; + if (line.mFlags.size() != lineLength) { + line.mFlags.resize(lineLength, 0); + line.mColorized = false; } + //if (!line.mColorized) { - if (!line.empty()) { - auto &g = line[currentIndex]; - auto c = g.mChar; + auto withinComment = false; + auto withinDocComment = false; + auto withinPreproc = false; + auto firstChar = true; // there is no other non-whitespace characters in the line before - if (c != mLanguageDefinition.mPreprocChar && !isspace(c)) - firstChar = false; + auto setGlyphFlags = [&](int index) { + Line::Flags flags(0); + flags.mBits.mComment = withinComment; + flags.mBits.mBlockComment = withinBlockComment; + flags.mBits.mDocComment = withinDocComment; + flags.mBits.mGlobalDocComment = withinGlobalDocComment; + flags.mBits.mBlockDocComment = withinBlockDocComment; + flags.mBits.mDeactivated = withinNotDef; + if (mLines[currentLine].mFlags[index] != flags.mValue) { + mLines[currentLine].mColorized = false; + mLines[currentLine].mFlags[index] = flags.mValue; + } + }; - bool inComment = (commentStartLine < currentLine || (commentStartLine == currentLine && commentStartIndex <= currentIndex)); + auto currentIndex = 0; + if (line.empty()) + continue; + while (currentIndex < lineLength) { - if (withinString) { - setGlyphFlags(currentIndex); - if (c == '\\') { - currentIndex++; + auto &g = line[currentIndex]; + auto c = g; + + if (c != mLanguageDefinition.mPreprocChar && !isspace(c)) + firstChar = false; + + bool inComment = (commentStartLine < currentLine || (commentStartLine == currentLine && commentStartIndex <= currentIndex)); + + if (withinString) { setGlyphFlags(currentIndex); - } else if (c == '\"') - withinString = false; - } else { - if (firstChar && c == mLanguageDefinition.mPreprocChar) { - withinPreproc = true; - std::string directive; - auto start = currentIndex + 1; - while (start < (int) line.size() && !isspace(line[start].mChar)) { - directive += line[start].mChar; - start++; - } + if (c == '\\') { + currentIndex++; + setGlyphFlags(currentIndex); + } else if (c == '\"') + withinString = false; + } else { + if (firstChar && c == mLanguageDefinition.mPreprocChar && !inComment && !withinComment && !withinDocComment && !withinString) { + withinPreproc = true; + std::string directive; + auto start = currentIndex + 1; + while (start < (int) line.size() && !isspace(line[start])) { + directive += line[start]; + start++; + } - if (start < (int) line.size()) { + while (start < (int) line.size() && isspace(line[start])) + start++; - if (isspace(line[start].mChar)) { - start += 1; - if (directive == "define") { - while (start < (int) line.size() && isspace(line[start].mChar)) - start++; - std::string identifier; - while (start < (int) line.size() && !isspace(line[start].mChar)) { - identifier += line[start].mChar; - start++; - } - if (identifier.size() > 0 && !withinNotDef && std::find(mDefines.begin(),mDefines.end(),identifier) == mDefines.end()) - mDefines.push_back(identifier); - } else if (directive == "undef") { - while (start < (int) line.size() && isspace(line[start].mChar)) - start++; - std::string identifier; - while (start < (int) line.size() && !isspace(line[start].mChar)) { - identifier += line[start].mChar; - start++; - } - if (identifier.size() > 0 && !withinNotDef) - mDefines.erase(std::remove(mDefines.begin(), mDefines.end(), identifier), mDefines.end()); + if (directive == "endif" && !ifDefs.empty()) { + ifDefs.pop_back(); + withinNotDef = !ifDefs.back(); + } else { + std::string identifier; + while (start < (int) line.size() && !isspace(line[start])) { + identifier += line[start]; + start++; + } + if (directive == "define") { + if (identifier.size() > 0 && !withinNotDef && std::find(mDefines.begin(), mDefines.end(), identifier) == mDefines.end()) + mDefines.push_back(identifier); + } else if (directive == "undef") { + if (identifier.size() > 0 && !withinNotDef) + mDefines.erase(std::remove(mDefines.begin(), mDefines.end(), identifier), mDefines.end()); } else if (directive == "ifdef") { - while (start < (int) line.size() && isspace(line[start].mChar)) - start++; - std::string identifier; - while (start < (int) line.size() && !isspace(line[start].mChar)) { - identifier += line[start].mChar; - start++; - } if (!withinNotDef) { - bool isConditionMet = std::find(mDefines.begin(),mDefines.end(),identifier) != mDefines.end(); + bool isConditionMet = std::find(mDefines.begin(), mDefines.end(), identifier) != mDefines.end(); ifDefs.push_back(isConditionMet); } else ifDefs.push_back(false); } else if (directive == "ifndef") { - while (start < (int) line.size() && isspace(line[start].mChar)) - start++; - std::string identifier; - while (start < (int) line.size() && !isspace(line[start].mChar)) { - identifier += line[start].mChar; - start++; - } if (!withinNotDef) { - bool isConditionMet = std::find(mDefines.begin(),mDefines.end(),identifier) == mDefines.end(); + bool isConditionMet = std::find(mDefines.begin(), mDefines.end(), identifier) == mDefines.end(); ifDefs.push_back(isConditionMet); } else ifDefs.push_back(false); } } - } else { - if (directive == "endif") { - if (ifDefs.size() > 1) { - ifDefs.pop_back(); - withinNotDef = !ifDefs.back(); - } - } } - } - if (c == '\"') { - withinString = true; - setGlyphFlags(currentIndex); - } else { - auto pred = [](const char &a, const Glyph &b) { return a == b.mChar; }; + if (c == '\"' && !withinPreproc && !inComment && !withinComment && !withinDocComment) { + withinString = true; + setGlyphFlags(currentIndex); + } else { + auto pred = [](const char &a, const char &b) { return a == b; }; - auto compareForth = [&](const std::string &a, const std::vector &b) { - return !a.empty() && currentIndex + a.size() <= b.size() && equals(a.begin(), a.end(), - b.begin() + currentIndex, b.begin() + currentIndex + a.size(), pred); - }; + auto compareForth = [&](const std::string &a, const std::string &b) { + return !a.empty() && (currentIndex + a.size() <= b.size()) && equals(a.begin(), a.end(), b.begin() + currentIndex, b.begin() + (currentIndex + a.size()), pred); + }; - auto compareBack = [&](const std::string &a, const std::vector &b) { - return !a.empty() && currentIndex + 1 >= (int)a.size() && equals(a.begin(), a.end(), - b.begin() + currentIndex + 1 - a.size(), b.begin() + currentIndex + 1, pred); - }; + auto compareBack = [&](const std::string &a, const std::string &b) { + return !a.empty() && currentIndex + 1 >= (int) a.size() && equals(a.begin(), a.end(), b.begin() + (currentIndex + 1 - a.size()), b.begin() + (currentIndex + 1), pred); + }; - if (!inComment && !withinSingleLineComment && !withinPreproc) { - if (compareForth(singleStartStr, line)) { - withinSingleLineComment = !inComment; - } else { - bool isGlobalDocComment = compareForth(globalStartStr, line); - bool isDocComment = compareForth(docStartStr, line); - bool isComment = compareForth(startStr, line); - if (isGlobalDocComment || isDocComment || isComment) { - commentStartLine = currentLine; - commentStartIndex = currentIndex; - if (isGlobalDocComment) { - withinGlobalDocComment = true; - commentLength = 3; - } else if (isDocComment) { - withinDocComment = true; - commentLength = 3; - } else { - withinComment = true; - commentLength = 2; + if (!inComment && !withinComment && !withinDocComment && !withinPreproc && !withinString) { + if (compareForth(mLanguageDefinition.mDocComment, line.mChars)) { + withinDocComment = !inComment; + commentLength = 3; + } else if (compareForth(mLanguageDefinition.mSingleLineComment, line.mChars)) { + withinComment = !inComment; + commentLength = 2; + } else { + bool isGlobalDocComment = compareForth(mLanguageDefinition.mGlobalDocComment, line.mChars); + bool isBlockDocComment = compareForth(mLanguageDefinition.mBlockDocComment, line.mChars); + bool isBlockComment = compareForth(mLanguageDefinition.mCommentStart, line.mChars); + if (isGlobalDocComment || isBlockDocComment || isBlockComment) { + commentStartLine = currentLine; + commentStartIndex = currentIndex; + if (currentIndex < line.size() - 4 && isBlockComment && + line.mChars[currentIndex + 2] == '*' && + line.mChars[currentIndex + 3] == '/') { + withinBlockComment = true; + commentLength = 2; + } else if (isGlobalDocComment) { + withinGlobalDocComment = true; + commentLength = 3; + } else if (isBlockDocComment) { + withinBlockDocComment = true; + commentLength = 3; + } else { + withinBlockComment = true; + commentLength = 2; + } } } + inComment = (commentStartLine < currentLine || (commentStartLine == currentLine && commentStartIndex <= currentIndex)); } - inComment = (commentStartLine < currentLine || (commentStartLine == currentLine && commentStartIndex <= currentIndex)); - } - setGlyphFlags(currentIndex); + setGlyphFlags(currentIndex); - auto &endStr = mLanguageDefinition.mCommentEnd; - if (compareBack(endStr, line) && ((commentStartLine != currentLine) || (commentStartIndex + commentLength < currentIndex))) { - withinComment = false; - withinDocComment = false; - withinGlobalDocComment = false; - commentStartLine = endLine; - commentStartIndex = endIndex; - commentLength = 0; + if (compareBack(mLanguageDefinition.mCommentEnd, line.mChars) && ((commentStartLine != currentLine) || (commentStartIndex + commentLength < currentIndex))) { + withinBlockComment = false; + withinBlockDocComment = false; + withinGlobalDocComment = false; + commentStartLine = endLine; + commentStartIndex = 0; + commentLength = 0; + } } } + if (currentIndex < line.size()) { + Line::Flags flags(0); + flags.mValue = mLines[currentLine].mFlags[currentIndex]; + flags.mBits.mPreprocessor = withinPreproc; + mLines[currentLine].mFlags[currentIndex] = flags.mValue; + } + auto utf8CharLength = UTF8CharLength(c); + if (utf8CharLength > 1) { + Line::Flags flags(0); + flags.mValue = mLines[currentLine].mFlags[currentIndex]; + for (int j = 1; j < utf8CharLength; j++) { + currentIndex++; + mLines[currentLine].mFlags[currentIndex] = flags.mValue; + } + } + currentIndex++; } - if (currentIndex < line.size()) - line[currentIndex].mPreprocessor = withinPreproc; - - currentIndex += UTF8CharLength(c); - if (currentIndex >= (int)line.size()) { - withinNotDef = !ifDefs.back(); - currentIndex = 0; - ++currentLine; - } - } else { - currentIndex = 0; - ++currentLine; - } + withinNotDef = !ifDefs.back(); + // } + // mUpdateFlags = false; } mDefines.clear(); - mCheckComments = false; - } - - if (mColorRangeMin < mColorRangeMax) { - const int increment = (mLanguageDefinition.mTokenize == nullptr) ? 10 : 10000; - const int to = std::min(mColorRangeMin + increment, mColorRangeMax); - ColorizeRange(mColorRangeMin, to); - mColorRangeMin = to; - - if (mColorRangeMax == mColorRangeMin) { - mColorRangeMin = std::numeric_limits::max(); - mColorRangeMax = 0; - } - return; } + ColorizeRange(); } float TextEditor::TextDistanceToLineStart(const Coordinates &aFrom) const { @@ -3015,15 +2993,15 @@ float TextEditor::TextDistanceToLineStart(const Coordinates &aFrom) const { float spaceSize = ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, " ", nullptr, nullptr).x; int colIndex = GetCharacterIndex(aFrom); for (size_t it = 0u; it < line.size() && it < colIndex;) { - if (line[it].mChar == '\t') { + if (line[it] == '\t') { distance = (1.0f + std::floor((1.0f + distance) / (float(mTabSize) * spaceSize))) * (float(mTabSize) * spaceSize); ++it; } else { - auto d = UTF8CharLength(line[it].mChar); + auto d = UTF8CharLength(line[it]); char tempCString[7]; int i = 0; for (; i < 6 && d-- > 0 && it < (int)line.size(); i++, it++) - tempCString[i] = line[it].mChar; + tempCString[i] = line[it]; tempCString[i] = '\0'; distance += ImGui::GetFont()->CalcTextSizeA(ImGui::GetFontSize(), FLT_MAX, -1.0f, tempCString, nullptr, nullptr).x; @@ -3132,13 +3110,13 @@ TextEditor::UndoRecord::UndoRecord( void TextEditor::UndoRecord::Undo(TextEditor *aEditor) { if (!mAdded.empty()) { aEditor->DeleteRange(mAddedStart, mAddedEnd); - aEditor->Colorize(mAddedStart.mLine - 1, mAddedEnd.mLine - mAddedStart.mLine + 2); + aEditor->Colorize(); } if (!mRemoved.empty()) { auto start = mRemovedStart; - aEditor->InsertTextAt(start, mRemoved); - aEditor->Colorize(mRemovedStart.mLine - 1, mRemovedEnd.mLine - mRemovedStart.mLine + 2); + aEditor->InsertTextAt(start, mRemoved.c_str()); + aEditor->Colorize(); } aEditor->mState = mBefore; @@ -3148,13 +3126,13 @@ void TextEditor::UndoRecord::Undo(TextEditor *aEditor) { void TextEditor::UndoRecord::Redo(TextEditor *aEditor) { if (!mRemoved.empty()) { aEditor->DeleteRange(mRemovedStart, mRemovedEnd); - aEditor->Colorize(mRemovedStart.mLine - 1, mRemovedEnd.mLine - mRemovedStart.mLine + 1); + aEditor->Colorize(); } if (!mAdded.empty()) { auto start = mAddedStart; - aEditor->InsertTextAt(start, mAdded); - aEditor->Colorize(mAddedStart.mLine - 1, mAddedEnd.mLine - mAddedStart.mLine + 1); + aEditor->InsertTextAt(start, mAdded.c_str()); + aEditor->Colorize(); } aEditor->mState = mAfter; @@ -3162,8 +3140,8 @@ void TextEditor::UndoRecord::Redo(TextEditor *aEditor) { } -bool TokenizeCStyleString(const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end) { - const char *p = in_begin; +bool TokenizeCStyleString(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end) { + auto p = in_begin; if (*p == '"') { p++; @@ -3189,8 +3167,8 @@ bool TokenizeCStyleString(const char *in_begin, const char *in_end, const char * return false; } -bool TokenizeCStyleCharacterLiteral(const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end) { - const char *p = in_begin; +bool TokenizeCStyleCharacterLiteral(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end) { + auto p = in_begin; if (*p == '\'') { p++; @@ -3213,10 +3191,10 @@ bool TokenizeCStyleCharacterLiteral(const char *in_begin, const char *in_end, co return false; } -bool TokenizeCStyleIdentifier(const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end) { - const char *p = in_begin; +bool TokenizeCStyleIdentifier(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end) { + auto p = in_begin; - if ((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || *p == '_') { + if ((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || *p == '_' || *p == '$') { p++; while ((p < in_end) && ((*p >= 'a' && *p <= 'z') || (*p >= 'A' && *p <= 'Z') || (*p >= '0' && *p <= '9') || *p == '_')) @@ -3230,12 +3208,12 @@ bool TokenizeCStyleIdentifier(const char *in_begin, const char *in_end, const ch return false; } -bool TokenizeCStyleNumber(const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end) { - const char *p = in_begin; +bool TokenizeCStyleNumber(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end) { + auto p = in_begin; const bool startsWithNumber = *p >= '0' && *p <= '9'; - if (*p != '+' && *p != '-' && !startsWithNumber) + if (!startsWithNumber) return false; p++; @@ -3306,8 +3284,8 @@ bool TokenizeCStyleNumber(const char *in_begin, const char *in_end, const char * return false; } - // single precision floating point type - if (p < in_end && *p == 'f') + // single and double precision floating point type + if (p < in_end && (*p == 'f' || *p == 'F' || *p == 'd' || *p == 'D')) p++; } @@ -3322,21 +3300,15 @@ bool TokenizeCStyleNumber(const char *in_begin, const char *in_end, const char * return true; } -bool TokenizeCStylePunctuation(const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end) { +bool TokenizeCStyleOperator(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end) { (void)in_end; switch (*in_begin) { - case '[': - case ']': - case '{': - case '}': case '!': case '%': case '^': case '&': case '*': - case '(': - case ')': case '-': case '+': case '=': @@ -3347,6 +3319,25 @@ bool TokenizeCStylePunctuation(const char *in_begin, const char *in_end, const c case '?': case ':': case '/': + case '@': + out_begin = in_begin; + out_end = in_begin + 1; + return true; + } + + return false; +} + +bool TokenizeCStyleSeparator(strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end) { + (void)in_end; + + switch (*in_begin) { + case '[': + case ']': + case '{': + case '}': + case '(': + case ')': case ';': case ',': case '.': @@ -3377,7 +3368,7 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::CPlusPlus( langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenize = [](const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end, PaletteIndex &paletteIndex) -> bool { + langDef.mTokenize = [](strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end, PaletteIndex &paletteIndex) -> bool { paletteIndex = PaletteIndex::Max; while (in_begin < in_end && isascii(*in_begin) && isblank(*in_begin)) @@ -3388,22 +3379,24 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::CPlusPlus( out_end = in_end; paletteIndex = PaletteIndex::Default; } else if (TokenizeCStyleString(in_begin, in_end, out_begin, out_end)) - paletteIndex = PaletteIndex::String; + paletteIndex = PaletteIndex::StringLiteral; else if (TokenizeCStyleCharacterLiteral(in_begin, in_end, out_begin, out_end)) paletteIndex = PaletteIndex::CharLiteral; else if (TokenizeCStyleIdentifier(in_begin, in_end, out_begin, out_end)) paletteIndex = PaletteIndex::Identifier; else if (TokenizeCStyleNumber(in_begin, in_end, out_begin, out_end)) - paletteIndex = PaletteIndex::Number; - else if (TokenizeCStylePunctuation(in_begin, in_end, out_begin, out_end)) - paletteIndex = PaletteIndex::Punctuation; + paletteIndex = PaletteIndex::NumericLiteral; + else if (TokenizeCStyleSeparator(in_begin, in_end, out_begin, out_end)) + paletteIndex= PaletteIndex::Separator; + else if (TokenizeCStyleOperator(in_begin, in_end, out_begin, out_end)) + paletteIndex = PaletteIndex::Operator; return paletteIndex != PaletteIndex::Max; }; langDef.mCommentStart = "/*"; langDef.mCommentEnd = "*/"; - langDef.mSingleLineComment = "//"; + langDef.mSingleLineComment = "//"; langDef.mCaseSensitive = true; langDef.mAutoIndentation = true; @@ -3620,19 +3613,19 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::HLSL() { langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Preprocessor)); - langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\!\\%\\^\\&\\*\\(\\)\\-\\+\\=\\~\\|\\<\\>\\?\\/\\;\\,\\.]", PaletteIndex::Punctuation)); - + langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Directive)); + langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::StringLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\!\\%\\^\\&\\*\\-\\+\\=\\~\\|\\<\\>\\?\\/]", PaletteIndex::Operator)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\(\\)\\;\\,\\.]", PaletteIndex::Separator)); langDef.mCommentStart = "/*"; langDef.mCommentEnd = "*/"; - langDef.mSingleLineComment = "//"; + langDef.mSingleLineComment = "//"; langDef.mCaseSensitive = true; langDef.mAutoIndentation = true; @@ -3663,19 +3656,19 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::GLSL() { langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Preprocessor)); - langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\!\\%\\^\\&\\*\\(\\)\\-\\+\\=\\~\\|\\<\\>\\?\\/\\;\\,\\.]", PaletteIndex::Punctuation)); - + langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Directive)); + langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::StringLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\!\\%\\^\\&\\*\\-\\+\\=\\~\\|\\<\\>\\?\\/]", PaletteIndex::Operator)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\(\\)\\;\\,\\.]", PaletteIndex::Separator)); langDef.mCommentStart = "/*"; langDef.mCommentEnd = "*/"; - langDef.mSingleLineComment = "//"; + langDef.mSingleLineComment = "//"; langDef.mCaseSensitive = true; langDef.mAutoIndentation = true; @@ -3706,7 +3699,7 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::C() { langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenize = [](const char *in_begin, const char *in_end, const char *&out_begin, const char *&out_end, PaletteIndex &paletteIndex) -> bool { + langDef.mTokenize = [](strConstIter in_begin, strConstIter in_end, strConstIter &out_begin, strConstIter &out_end, PaletteIndex &paletteIndex) -> bool { paletteIndex = PaletteIndex::Max; while (in_begin < in_end && isascii(*in_begin) && isblank(*in_begin)) @@ -3717,22 +3710,24 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::C() { out_end = in_end; paletteIndex = PaletteIndex::Default; } else if (TokenizeCStyleString(in_begin, in_end, out_begin, out_end)) - paletteIndex = PaletteIndex::String; + paletteIndex = PaletteIndex::StringLiteral; else if (TokenizeCStyleCharacterLiteral(in_begin, in_end, out_begin, out_end)) paletteIndex = PaletteIndex::CharLiteral; else if (TokenizeCStyleIdentifier(in_begin, in_end, out_begin, out_end)) paletteIndex = PaletteIndex::Identifier; else if (TokenizeCStyleNumber(in_begin, in_end, out_begin, out_end)) - paletteIndex = PaletteIndex::Number; - else if (TokenizeCStylePunctuation(in_begin, in_end, out_begin, out_end)) - paletteIndex = PaletteIndex::Punctuation; + paletteIndex = PaletteIndex::NumericLiteral; + else if (TokenizeCStyleOperator(in_begin, in_end, out_begin, out_end)) + paletteIndex = PaletteIndex::Operator; + else if (TokenizeCStyleSeparator(in_begin, in_end, out_begin, out_end)) + paletteIndex = PaletteIndex::Separator; return paletteIndex != PaletteIndex::Max; }; langDef.mCommentStart = "/*"; langDef.mCommentEnd = "*/"; - langDef.mSingleLineComment = "//"; + langDef.mSingleLineComment = "//"; langDef.mCaseSensitive = true; langDef.mAutoIndentation = true; @@ -3764,18 +3759,20 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::SQL() { langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("\\\'[^\\\']*\\\'", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\!\\%\\^\\&\\*\\(\\)\\-\\+\\=\\~\\|\\<\\>\\?\\/\\;\\,\\.]", PaletteIndex::Punctuation)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Directive)); + langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::StringLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\!\\%\\^\\&\\*\\-\\+\\=\\~\\|\\<\\>\\?\\/]", PaletteIndex::Operator)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\(\\)\\;\\,\\.]", PaletteIndex::Separator)); langDef.mCommentStart = "/*"; langDef.mCommentEnd = "*/"; - langDef.mSingleLineComment = "//"; + langDef.mSingleLineComment = "//"; langDef.mCaseSensitive = false; langDef.mAutoIndentation = false; @@ -3807,14 +3804,16 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::AngelScrip langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\!\\%\\^\\&\\*\\(\\)\\-\\+\\=\\~\\|\\<\\>\\?\\/\\;\\,\\.]", PaletteIndex::Punctuation)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Directive)); + langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::StringLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\!\\%\\^\\&\\*\\-\\+\\=\\~\\|\\<\\>\\?\\/]", PaletteIndex::Operator)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\(\\)\\;\\,\\.]", PaletteIndex::Separator)); langDef.mCommentStart = "/*"; langDef.mCommentEnd = "*/"; @@ -3850,13 +3849,16 @@ const TextEditor::LanguageDefinition &TextEditor::LanguageDefinition::Lua() { langDef.mIdentifiers.insert(std::make_pair(std::string(k), id)); } - langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("\\\'[^\\\']*\\\'", PaletteIndex::String)); - langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::Number)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); - langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\!\\%\\^\\&\\*\\(\\)\\-\\+\\=\\~\\|\\<\\>\\?\\/\\;\\,\\.]", PaletteIndex::Punctuation)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[ \\t]*#[ \\t]*[a-zA-Z_]+", PaletteIndex::Directive)); + langDef.mTokenRegexStrings.push_back(std::make_pair("L?\\\"(\\\\.|[^\\\"])*\\\"", PaletteIndex::StringLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("\\'\\\\?[^\\']\\'", PaletteIndex::CharLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[fF]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[+-]?[0-9]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[0-7]+[Uu]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("0[xX][0-9a-fA-F]+[uU]?[lL]?[lL]?", PaletteIndex::NumericLiteral)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[a-zA-Z_][a-zA-Z0-9_]*", PaletteIndex::Identifier)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\!\\%\\^\\&\\*\\-\\+\\=\\~\\|\\<\\>\\?\\/]", PaletteIndex::Operator)); + langDef.mTokenRegexStrings.push_back(std::make_pair("[\\[\\]\\{\\}\\(\\)\\;\\,\\.]", PaletteIndex::Separator)); langDef.mCommentStart = "--[["; langDef.mCommentEnd = "]]"; diff --git a/main/gui/source/window/window.cpp b/main/gui/source/window/window.cpp index 0da16b789..f5cd13ae5 100644 --- a/main/gui/source/window/window.cpp +++ b/main/gui/source/window/window.cpp @@ -2,8 +2,6 @@ #include -#include - #include #include #include diff --git a/plugins/builtin/CMakeLists.txt b/plugins/builtin/CMakeLists.txt index a4f1c877a..eee8cc8ef 100644 --- a/plugins/builtin/CMakeLists.txt +++ b/plugins/builtin/CMakeLists.txt @@ -119,6 +119,8 @@ add_imhex_plugin( source/content/views/view_achievements.cpp source/content/views/view_highlight_rules.cpp source/content/views/view_tutorials.cpp + + source/content/text_highlighting/pattern_language.cpp INCLUDES include diff --git a/plugins/builtin/include/content/text_highlighting/pattern_language.hpp b/plugins/builtin/include/content/text_highlighting/pattern_language.hpp new file mode 100644 index 000000000..c107d0b89 --- /dev/null +++ b/plugins/builtin/include/content/text_highlighting/pattern_language.hpp @@ -0,0 +1,419 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace pl { + class PatternLanguage; +} + +namespace hex::plugin::builtin { + class ViewPatternEditor; + class TextHighlighter { + public: + class Interval; + using Token = pl::core::Token; + using ASTNode = pl::core::ast::ASTNode; + using ExcludedLocation = pl::core::Preprocessor::ExcludedLocation; + using CompileError = pl::core::err::CompileError; + using Identifier = Token::Identifier; + using IdentifierType = Identifier::IdentifierType; + using UnorderedBlocks = std::map; + using OrderedBlocks = std::map; + using Scopes = std::set; + using Location = pl::core::Location; + using TokenIter = pl::hlp::SafeIterator::const_iterator>; + using VariableScopes = std::map; + using Inheritances = std::map>; + using IdentifierTypeColor = std::map; + using TokenTypeColor = std::map; + using TokenColor = std::map; + + struct ParentDefinition; + struct Definition { + Definition()= default; + Definition(IdentifierType identifierType, std::string typeStr,i32 tokenId, Location location) : idType(identifierType), typeStr(typeStr), tokenIndex(tokenId),location(location) {} + IdentifierType idType; + std::string typeStr; + i32 tokenIndex; + Location location; + }; + + struct ParentDefinition { + ParentDefinition() = default; + ParentDefinition(IdentifierType identifierType, i32 tokenId, Location location) : idType(identifierType), tokenIndex(tokenId), location(location) {} + IdentifierType idType; + i32 tokenIndex; + Location location; + }; + /// to define functions and types + using Definitions = std::map; + /// to define global variables + using Variables = std::map>; + /// to define UDT and function variables + using VariableMap = std::map; + private: + std::string m_text; + std::vector m_lines; + std::vector m_firstTokenIdOfLine; + ViewPatternEditor *m_viewPatternEditor; + std::vector m_excludedLocations; + std::vector m_tokens; + TokenColor m_tokenColors; + std::unique_ptr *patternLanguage; + std::vector m_compileErrors; + std::map> m_instances; + Definitions m_UDTDefinitions; + Definitions m_functionDefinitions; + + OrderedBlocks m_namespaceTokenRange; + UnorderedBlocks m_UDTTokenRange; + UnorderedBlocks m_functionTokenRange; + Scopes m_globalTokenRange; + + VariableMap m_UDTVariables; + VariableMap m_functionVariables; + Variables m_globalVariables; + + std::map m_attributeFunctionArgumentType; + std::map m_typeDefMap; + std::map m_typeDefInvMap; + std::vector m_nameSpaces; + std::vector m_UDTs; + std::set m_taggedIdentifiers; + std::set m_memberChains; + std::set m_scopeChains; + + TokenIter m_curr; + TokenIter m_startToken, m_originalPosition, m_partOriginalPosition; + + VariableScopes m_UDTBlocks; + VariableScopes m_functionBlocks; + Scopes m_globalBlocks; + Inheritances m_inheritances; + const static IdentifierTypeColor m_identifierTypeColor; + const static TokenTypeColor m_tokenTypeColor; + + i32 m_runningColorizers=0; + public: + + /// Intervals are the sets finite contiguous non-negative integer that + /// are described by their endpoints. The sets must have the following + /// properties: + /// 1. Any two elements of the set can either have an empty intersection or + /// 2. their intersection is equal to one of the two sets (i.e. one is + /// a subset of the other). + /// An interval is defined to be smaller than another if: + /// 1. The interval lies entirely to the left of the other interval or + /// 2. The interval is a proper subset of the other interval. + /// Two intervals are equal if they have identical start and end values. + /// This ordering is used for things like code blocks or the token + /// ranges that are defined by the blocks. + class Interval { + public: + i32 start; + i32 end; + Interval() : start(0), end(0) {} + Interval(i32 start, i32 end) : start(start), end(end) { + if (start > end) + throw std::invalid_argument("Interval start must be less than or equal to end"); + } + bool operator<(const Interval &other) const { + return other.end > end; + } + bool operator>(const Interval &other) const { + return end > other.end; + } + bool operator==(const Interval &other) const { + return start == other.start && end == other.end; + } + bool operator!=(const Interval &other) const { + return start != other.start || end != other.end; + } + bool operator<=(const Interval &other) const { + return other.end >= end; + } + bool operator>=(const Interval &other) const { + return end >= other.end; + } + bool contains(const Interval &other) const { + return other.start >= start && other.end <= end; + } + bool contains(i32 value) const { + return value >= start && value <= end; + } + bool contiguous(const Interval &other) const { + return ((start - other.end) == 1 || (other.start - end) == 1); + } + }; + std::atomic m_needsToUpdateColors = true; + std::atomic m_wasInterrupted = false; + + TextHighlighter(ViewPatternEditor *viewPatternEditor, std::unique_ptr *patternLanguage ) : + m_viewPatternEditor(viewPatternEditor), patternLanguage(patternLanguage), m_needsToUpdateColors(true) {} + /** + * @brief Entry point to syntax highlighting + */ + void highlightSourceCode(); + + /** + * @brief Syntax highlighting from parser + */ + void setInitialColors(); + /** + * @brief Create data to pass to text editor + */ + void setRequestedIdentifierColors(); + /** + * @brief Set the color of a token + */ + void setColor(i32 tokenId=-1, const IdentifierType &type = IdentifierType::Unknown); + void setIdentifierColor(i32 tokenId=-1, const IdentifierType &type = IdentifierType::Unknown); + /** + * @brief Only identifiers not in chains should remain + */ + void colorRemainingIdentifierTokens(); + /** + * @brief Renders compile errors in real time + */ + void renderErrors(); + /// A token range is the set of token indices of a definition. The namespace token + /// ranges are obtained first because they are needed to obtain unique identifiers. + void getAllTokenRanges(IdentifierType idtype); + /// The global scope is the complement of the union of all the function and UDT token ranges + void getGlobalTokenRanges(); + /// If the current token is a function or UDT, creates a map entry from the name to the token range. These are ordered alphabetically by name. + /// If the current token is a namespace, creates a map entry from the token range to the name. Namespace entries are stored in the order they occur in the source code. + bool getTokenRange(std::vector keywords,UnorderedBlocks &tokenRange, OrderedBlocks &tokenRangeInv, bool fullName, VariableScopes *blocks); + /// Global variables are the variables that are not inside a function or UDT + void fixGlobalVariables(); + /// Creates the definition maps for UDTs, functions, their variables and global variables + void getDefinitions(); + void loadGlobalDefinitions(Scopes tokenRangeSet, std::vector identifierTypes, Variables &variables); + void loadVariableDefinitions(UnorderedBlocks tokenRangeMap, Token delimiter1, Token delimiter2, std::vector identifierTypes, bool isArgument, VariableMap &variableMap); + void loadTypeDefinitions(UnorderedBlocks tokenRangeMap, std::vector identifierTypes, Definitions &types); + std::string getArgumentTypeName(i32 rangeStart, Token delimiter2); + std::string getVariableTypeName(); + /// Append the variable definitions of the parent to the child + void appendInheritances(); + void recurseInheritances(std::string name); + ///Loads a map of identifiers to their token id instances + void loadInstances(); + /// Replace auto with the actual type for template arguments and function parameters + void fixAutos(); + void resolveAutos(VariableMap &variableMap, UnorderedBlocks &tokenRange); + /// Chains are sequences of identifiers separated by scope resolution or dot operators. + void fixChains(); + bool colorSeparatorScopeChain(); + bool colorOperatorDotChain(); + /// Returns the next/previous valid source code line + u32 nextLine(u32 line); + u32 previousLine(u32 line); + /// Loads the source code and calculates the first token index of each line + void loadText(); + /// Used to obtain the color to be applied. + TextEditor::PaletteIndex getPaletteIndex(Token::Literal *literal); + /// The complement of a set is also known as its inverse + void invertGlobalTokenRange(); + /// Starting at the identifier, it tracks all the scope resolution and dot operators and returns the full chain without arrays, templates, pointers,... + bool getFullName(std::string &identifierName, std::vector &identifiers, bool preserveCurr = true); + /// Returns the identifier value. + bool getIdentifierName(std::string &identifierName, Identifier *identifier); + /// Adds namespaces to the full name if they exist + bool getQualifiedName(std::string &identifierName, std::vector &identifiers, bool useDefinitions = false, bool preserveCurr = true); + /// As it moves forward it loads the result to the argument. Used by getFullName + bool forwardIdentifierName(std::string &identifierName, std::vector &identifiers, bool preserveCurr = true); + /// Takes as input the full name and returns the type of the last element. + bool resolveIdentifierType(Definition &result, std::string identifierName); + /// like previous functions but returns the type of the variable that is a member of a UDT + std::string findIdentifierTypeStr(const std::string &identifierName, std::string context=""); + IdentifierType findIdentifierType(const std::string &identifierName, std::string context); + /// If context is empty search for the variable, if it isnt use the variable map. + bool findOrContains(std::string &context, UnorderedBlocks tokenRange, VariableMap variableMap); + /// Search for instances inside some block + void setBlockInstancesColor(const std::string &name, const Definition &definition, const Interval &block); + /// Convenience functions. + void skipAttribute(); + void skipArray(i32 maxSkipCount, bool forward = true); + void skipTemplate(i32 maxSkipCount, bool forward = true); + void skipDelimiters(i32 maxSkipCount, Token delimiter[2], i8 increment); + void skipToken(Token token, i8 step=1); + /// from given or current names find the corresponding definition + bool findIdentifierDefinition(Definition &result, const std::string &optionalIdentifierName = "", std::string optionalName = "", bool optional = false); + /// To deal with the Parent keyword + std::optional setChildrenTypes(); + bool findParentTypes(std::vector &parentTypes, const std::string &optionalName=""); + bool findAllParentTypes(std::vector &parentTypes, std::vector &identifiers, std::string &optionalFullName); + bool tryParentType(const std::string &parentType, std::string &variableName, std::optional &result, std::vector &identifiers); + /// Convenience function + bool isTokenIdValid(i32 tokenId); + bool isLocationValid(Location location); + /// Returns the name of the context where the current or given token is located + bool findScope(std::string &name, const UnorderedBlocks &map, i32 optionalTokenId=-1); + /// Returns the name of the namespace where the current or given token is located + bool findNamespace(std::string &nameSpace, i32 optionalTokenId=-1); + /// Calculate the source code, line and column numbers of a token index + pl::core::Location getLocation(i32 tokenId); + /// Calculate the token index of a source code, line and column numbers + i32 getTokenId(pl::core::Location location); + /// Calculate the function or template argument position from token indices + i32 getArgumentNumber(i32 start,i32 arg); + /// Calculate the token index of a function or template argument position + void getTokenIdForArgument(i32 start, i32 argNumber, Token delimiter); + ///Creates a map from function name to argument type + void linkAttribute(); + /// Comment and strings usethese function to determine their coordinates + template TextEditor::Coordinates commentCoordinates(Token *token); + TextEditor::Coordinates stringCoordinates(); + /// Returns the number of tasks highlighting code. Shouldn't be > 1 + i32 getRunningColorizers() { + return m_runningColorizers; + } + + enum class HighlightStage { + Starting, + NamespaceTokenRanges, + UDTTokenRanges, + FunctionTokenRanges, + GlobalTokenRanges, + FixGlobalVariables, + SetInitialColors, + LoadInstances, + AttributeTokenRanges, + Definitions, + FixAutos, + FixChains, + ExcludedLocations, + ColorRemainingTokens, + SetRequestedIdentifierColors, + Stage1, + Stage2, + Stage3, + Stage4, + Stage5, + Stage6, + Stage7, + Stage8, + Stage9, + Stage10, + Stage11, + }; + + HighlightStage m_highlightStage = HighlightStage::Starting; + + /// The following functions were copied from the parser and some were modified + + template + T *getValue(const i32 index) { + return const_cast(std::get_if(&m_curr[index].value)); + } + + void next(i32 count = 1) { + if (count == 0) + return; + i32 id = getTokenId(m_curr->location); + i32 maxChange; + if (count > 0) + maxChange = std::min(count,static_cast(m_tokens.size() - id)); + else + maxChange = -std::min(-count,id); + m_curr += maxChange; + } + constexpr static u32 Normal = 0; + constexpr static u32 Not = 1; + + bool begin() { + m_originalPosition = m_curr; + + return true; + } + + void partBegin() { + m_partOriginalPosition = m_curr; + } + + void reset() { + m_curr = m_originalPosition; + } + + void partReset() { + m_curr = m_partOriginalPosition; + } + + bool resetIfFailed(const bool value) { + if (!value) reset(); + + return value; + } + + template + bool sequenceImpl() { + if constexpr (S == Normal) + return true; + else if constexpr (S == Not) + return false; + else + std::unreachable(); + } + + template + bool matchOne(const Token &token) { + if constexpr (S == Normal) { + if (!peek(token)) { + partReset(); + return false; + } + + next(); + return true; + } else if constexpr (S == Not) { + if (!peek(token)) + return true; + + next(); + partReset(); + return false; + } else + std::unreachable(); + } + + template + bool sequenceImpl(const auto &... args) { + return (matchOne(args) && ...); + } + + template + bool sequence(const Token &token, const auto &... args) { + partBegin(); + return sequenceImpl(token, args...); + } + + bool isValid() { + Token token; + try { + token = m_curr[0]; + } + catch (const std::out_of_range &e) { + auto t = e.what(); + if (t == nullptr) + return false; + return false; + } + if (!isLocationValid(token.location)) + return false; + return true; + } + + bool peek(const Token &token, const i32 index = 0) { + if (!isValid()) + return false; + i32 id = getTokenId(m_curr->location); + if (id+index < 0 || id+index >= (i32)m_tokens.size()) + return false; + return m_curr[index].type == token.type && m_curr[index] == token.value; + } + + }; +} \ No newline at end of file diff --git a/plugins/builtin/include/content/views/view_pattern_editor.hpp b/plugins/builtin/include/content/views/view_pattern_editor.hpp index c075e1560..7429b7ba3 100644 --- a/plugins/builtin/include/content/views/view_pattern_editor.hpp +++ b/plugins/builtin/include/content/views/view_pattern_editor.hpp @@ -19,6 +19,7 @@ #include #include +#include namespace pl::ptrn { class Pattern; } @@ -69,6 +70,30 @@ namespace hex::plugin::builtin { ~ViewPatternEditor() override; void drawAlwaysVisibleContent() override; + std::unique_ptr *getPatternLanguage() { + return &m_editorRuntime; + } + + TextEditor &getTextEditor() { + return m_textEditor; + } + + bool getChangesWereParsed() const { + return m_changesWereParsed; + } + + u32 getRunningParsers () const { + return m_runningParsers; + } + + u32 getRunningEvaluators () const { + return m_runningEvaluators; + } + + void setChangesWereParsed(bool changesWereParsed) { + m_changesWereParsed = changesWereParsed; + } + void drawContent() override; [[nodiscard]] ImGuiWindowFlags getWindowFlags() const override { return ImGuiWindowFlags_NoScrollbar | ImGuiWindowFlags_NoScrollWithMouse; @@ -234,6 +259,7 @@ namespace hex::plugin::builtin { std::atomic m_runningEvaluators = 0; std::atomic m_runningParsers = 0; + bool m_changesWereParsed = false; PerProvider m_hasUnevaluatedChanges; std::chrono::time_point m_lastEditorChangeTime; @@ -308,6 +334,7 @@ namespace hex::plugin::builtin { static inline u32 m_replaceHistorySize = 0; static inline u32 m_replaceHistoryIndex = 0; + TextHighlighter m_textHighlighter = TextHighlighter(this,&this->m_editorRuntime); private: void drawConsole(ImVec2 size); void drawEnvVars(ImVec2 size, std::list &envVars); diff --git a/plugins/builtin/romfs/themes/classic.json b/plugins/builtin/romfs/themes/classic.json index 8c365c00c..48d590a2b 100644 --- a/plugins/builtin/romfs/themes/classic.json +++ b/plugins/builtin/romfs/themes/classic.json @@ -147,27 +147,51 @@ "title-text": "#E5E5E5FF" }, "text-editor": { + "attribute": "#FFFF00FF", "background": "#000080FF", + "multi-line-comment": "#404040FF", "breakpoint": "#FF200080", + "known-identifier": "#4DC69BFF", + "calculated-pointer": "#FFFF00FF", "char-literal": "#008080FF", "comment": "#808080FF", "current-line-edge": "#00000040", "current-line-fill": "#00000040", "current-line-fill-inactive": "#80808040", "cursor": "#FF8000FF", + "debug-text": "#8A8A8AFF", "default": "#FFFF00FF", + "default-text": "#FFFF00FF", + "preprocessor": "#7FBF00FF", + "doc-block-comment": "#206020FF", + "doc-comment": "#206020FF", + "doc-global-comment": "#206020FF", "error-marker": "#FF0000A0", + "error-text": "#FF200080", + "function": "#FFFF00FF", + "function-parameter": "#FFFF00FF", + "function-variable": "#FFFF00FF", + "global-variable": "#FFFF00FF", "identifier": "#FFFF00FF", "keyword": "#00FFFFFF", - "known-identifier": "#FFFFFFFF", "line-number": "#008080FF", - "multi-line-comment": "#404040FF", + "local-variable": "#FFFF00FF", + "namespace": "#FFFF00FF", "number": "#00FF00FF", - "preproc-identifier": "#FF00FFFF", - "preprocessor": "#008000FF", - "punctuation": "#FFFFFFFF", + "punctuation": "#FFFF00FF", + "pattern-variable": "#FFFF00FF", + "placed-variable": "#FFFF00FF", + "preprocessor-deactivated": "#4F4F4F45", + "preproc-identifier": "#7FBF00FF", "selection": "*#00FFFF80", - "string": "#008080FF" + "separator": "#FFFF00FF", + "string": "#008080FF", + "template-variable": "#FFFF00FF", + "typedef": "#FFFF00FF", + "unknown-identifier": "#FC2C2CFE", + "user-defined-type": "#FFFF00FF", + "view": "#FFFF00FF", + "warning-text": "#FFFF00FF" } }, "image_theme": "dark", diff --git a/plugins/builtin/romfs/themes/dark.json b/plugins/builtin/romfs/themes/dark.json index 4db34e6b2..f9ec75bac 100644 --- a/plugins/builtin/romfs/themes/dark.json +++ b/plugins/builtin/romfs/themes/dark.json @@ -147,27 +147,51 @@ "title-text": "#FFFFFFFF" }, "text-editor": { + "attribute": "#AAAAAAFF", "background": "#101010FF", + "multi-line-comment": "#206040FF", "breakpoint": "#FF200040", + "known-identifier": "#4DC69BFF", + "calculated-pointer": "#AAAAAAFF", "char-literal": "#E0A070FF", "comment": "#206020FF", "current-line-edge": "#A0A0A040", "current-line-fill": "#00000040", "current-line-fill-inactive": "#80808040", "cursor": "#E0E0E0FF", + "debug-text": "#8A8A8AFF", "default": "#7F7F7FFF", + "default-text": "#7F7F7FFF", + "preprocessor": "#808060FF", + "doc-block-comment": "#206020FF", + "doc-comment": "#206020FF", + "doc-global-comment": "#206020FF", "error-marker": "#FF200080", + "error-text": "#FF200080", + "function": "#AAAAAAFF", + "function-parameter": "#AAAAAAFF", + "function-variable": "#AAAAAAFF", + "global-variable": "#AAAAAAFF", "identifier": "#AAAAAAFF", "keyword": "#569CD6FF", - "known-identifier": "#4DC69BFF", "line-number": "#007070FF", - "multi-line-comment": "#206040FF", + "local-variable": "#AAAAAAFF", + "namespace": "#AAAAAAFF", "number": "#00FF00FF", - "preproc-identifier": "#A040C0FF", - "preprocessor": "#808040FF", - "punctuation": "#FFFFFFFF", + "punctuation": "#7F7F7FFF", + "pattern-variable": "#AAAAAAFF", + "placed-variable": "#AAAAAAFF", + "preprocessor-deactivated": "#4F4F4F45", + "preproc-identifier": "#808060FF", "selection": "*#2060A080", - "string": "#E07070FF" + "separator": "#7F7F7FFF", + "string": "#E07070FF", + "template-variable": "#AAAAAAFF", + "typedef": "#AAAAAAFF", + "unknown-identifier": "#FC2C2CFE", + "user-defined-type": "#AAAAAAFF", + "view": "#AAAAAAFF", + "warning-text": "#FFFF00FF" } }, "image_theme": "dark", diff --git a/plugins/builtin/romfs/themes/light.json b/plugins/builtin/romfs/themes/light.json index 77783b840..7570b0183 100644 --- a/plugins/builtin/romfs/themes/light.json +++ b/plugins/builtin/romfs/themes/light.json @@ -147,27 +147,50 @@ "title-text": "#000000FF" }, "text-editor": { + "attribute": "#404040FF", "background": "#FFFFFFFF", + "multi-line-comment": "#205040FF", "breakpoint": "#FF200080", + "known-identifier": "#106060FF", + "calculated-pointer": "#404040FF", "char-literal": "#704030FF", "comment": "#205020FF", "current-line-edge": "#00000040", "current-line-fill": "#00000040", "current-line-fill-inactive": "#80808040", "cursor": "#000000FF", + "debug-text": "#8A8A8AFF", "default": "#7F7F7FFF", + "default-text": "#7F7F7FFF", + "preprocessor": "#6F6F5FFF", + "doc-block-comment": "#205040FF", + "doc-comment": "#205020FF", + "doc-global-comment": "#205040FF", "error-marker": "#FF1000A0", - "identifier": "#404040FF", + "error-text": "#FF200080", + "function": "#404040FF", + "function-parameter": "#404040FF", + "function-variable": "#404040FF", + "global-variable": "#404040FF", "keyword": "#060CFFFF", - "known-identifier": "#106060FF", "line-number": "#005050FF", - "multi-line-comment": "#205040FF", + "local-variable": "#404040FF", + "namespace": "#404040FF", "number": "#008000FF", - "preproc-identifier": "#A040C0FF", - "preprocessor": "#606040FF", - "punctuation": "#000000FF", + "punctuation": "#7F7F7FFF", + "pattern-variable": "#404040FF", + "placed-variable": "#404040FF", + "preprocessor-deactivated": "#4F4F4F45", + "preproc-identifier": "#6F6F5FFF", "selection": "*#00006080", - "string": "#A02020FF" + "separator": "#7F7F7FFF", + "string": "#A02020FF", + "template-variable": "#404040FF", + "typedef": "#404040FF", + "unknown-identifier": "#FC2C2CFE", + "user-defined-type": "#404040FF", + "view": "#404040FF", + "warning-text": "#FFFF00FF" } }, "image_theme": "light", diff --git a/plugins/builtin/source/content/text_highlighting/pattern_language.cpp b/plugins/builtin/source/content/text_highlighting/pattern_language.cpp new file mode 100644 index 000000000..e71b149e9 --- /dev/null +++ b/plugins/builtin/source/content/text_highlighting/pattern_language.cpp @@ -0,0 +1,2332 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + + +namespace hex::plugin::builtin { + + using namespace pl::core; + using Identifier = Token::Identifier; + using Keyword = Token::Keyword; + using Separator = Token::Separator; + using Operator = Token::Operator; + using Comment = Token::Comment; + using DocComment = Token::DocComment; + using Literal = Token::Literal; + using ValueType = Token::ValueType; + + bool TextHighlighter::getIdentifierName(std::string &identifierName, Identifier *identifier) { + auto keyword = getValue(0); + identifier = getValue(0); + + if (identifier != nullptr) { + identifierName = identifier->get(); + return true; + } else if (keyword != nullptr) { + identifier = nullptr; + if (peek(tkn::Keyword::Parent)) { + identifierName = "Parent"; + return true; + } + + if (peek(tkn::Keyword::This)) { + identifierName = "This"; + return true; + } + } + identifier = nullptr; + return false; + } + +// Returns a chain of identifiers like a.b.c or a::b::c + bool TextHighlighter::getFullName(std::string &identifierName, std::vector &identifiers, bool preserveCurr) { + Identifier *identifier = nullptr; + + if (!peek(tkn::Literal::Identifier) || getTokenId(m_curr->location) < 1) + return getIdentifierName(identifierName, identifier); + + forwardIdentifierName(identifierName, identifiers, preserveCurr); + return true; + } + + bool TextHighlighter::forwardIdentifierName(std::string &identifierName, std::vector &identifiers, bool preserveCurr ) { + auto curr = m_curr; + Identifier *identifier = getValue(0); + std::string current; + + if (identifier != nullptr) { + identifiers.push_back(identifier); + identifierName += identifier->get(); + } else if (getIdentifierName(current, identifier)) { + identifiers.push_back(identifier); + identifierName += current; + } else { + m_curr = curr; + return false; + } + + skipArray(200, true); + + while ((peek(tkn::Operator::ScopeResolution, 1) || peek(tkn::Separator::Dot, 1))) { + next(); + + if (peek(tkn::Operator::ScopeResolution)) + identifierName += "::"; + else if (peek(tkn::Separator::Dot)) + identifierName += "."; + else { + m_curr = curr; + return false; + } + next(); + + if (getIdentifierName(current, identifier)) { + identifiers.push_back(identifier); + identifierName += current; + + skipArray(200, true); + + } else { + m_curr = curr; + return false; + } + } + if (preserveCurr) + m_curr = curr; + return true; + } + +// Adds namespace if it exists + bool TextHighlighter::getQualifiedName(std::string &identifierName, std::vector &identifiers, bool useDefinitions, bool preserveCurr) { + + std::string shortName; + std::string qualifiedName; + + if (!getFullName(identifierName, identifiers, preserveCurr)) + return false; + + if (std::ranges::find(m_UDTs, identifierName) != m_UDTs.end()) + return true; + std::vector vectorString; + if (identifierName.contains("::")) { + vectorString = hex::splitString(identifierName, "::"); + if (vectorString.size() > 1) { + shortName = vectorString.back(); + vectorString.pop_back(); + identifierName = hex::combineStrings(vectorString, "::"); + } + } + bool found = true; + for (auto name : vectorString) { + found = found || std::ranges::find(m_nameSpaces, name) != m_nameSpaces.end(); + } + if (found) { + if (!shortName.empty()) + identifierName = identifierName + "::" + shortName; + return true; + } + + if (useDefinitions) { + if (m_functionDefinitions.contains(identifierName) || m_UDTDefinitions.contains(identifierName)) { + if (!shortName.empty()) + identifierName = identifierName + "::" + shortName; + return true; + } + std::string nameSpace; + for (auto [name, definition]: m_UDTDefinitions) { + findNamespace(nameSpace, definition.tokenIndex); + + if (!nameSpace.empty() && !identifierName.contains(nameSpace)) { + qualifiedName = nameSpace + "::" + identifierName; + + if (name == qualifiedName) { + identifierName = qualifiedName; + if (!shortName.empty()) + identifierName = identifierName + "::" + shortName; + return true; + } + } + + if (name == identifierName) { + identifierName = name; + if (!shortName.empty()) + identifierName = identifierName + "::" + shortName; + return true; + } + } + } + + if (identifierName.empty()) + return false; + return true; + } + +// Finds the token range of a function, namespace or UDT + bool TextHighlighter::getTokenRange(std::vector keywords, + TextHighlighter::UnorderedBlocks &tokenRange, + TextHighlighter::OrderedBlocks &tokenRangeInv, + bool fullName, VariableScopes *blocks) { + + bool addArgumentBlock = !fullName; + std::vector tokenStack; + if (getTokenId(m_curr->location) < 1) + return false; + std::string name; + if (fullName) { + std::vector identifiers; + if (!getFullName(name, identifiers)) + return false; + } else { + Identifier *identifier = nullptr; + if (!getIdentifierName(name, identifier)) + return false; + std::string nameSpace; + findNamespace(nameSpace, getTokenId(m_curr->location)); + if (!nameSpace.empty()) + name = nameSpace + "::" + name; + } + + i32 tokenCount = m_tokens.size(); + auto saveCurr = m_curr - 1; + skipTemplate(200); + next(); + if (sequence(tkn::Operator::Colon)) { + while (peek(tkn::Literal::Identifier)) { + auto identifier = getValue(0); + if (identifier == nullptr) + break; + auto identifierName = identifier->get(); + if (std::ranges::find(m_inheritances[name], identifierName) == m_inheritances[name].end()) + m_inheritances[name].push_back(identifierName); + skipTemplate(200); + next(2); + } + } + + m_curr = saveCurr; + i32 index1 = getTokenId(m_curr->location); + bool result = true; + for (auto keyword: keywords) + result = result && !peek(keyword); + if (result) + return false; + u32 nestedLevel = 0; + next(); + auto endToken = TokenIter(m_tokens.begin() + tokenCount, m_tokens.end()); + while (endToken > m_curr) { + + if (sequence(tkn::Separator::LeftBrace)) { + auto tokenId = getTokenId(m_curr[-1].location); + tokenStack.push_back(tokenId); + nestedLevel++; + } else if (sequence(tkn::Separator::RightBrace)) { + nestedLevel--; + + if (tokenStack.empty()) + return false; + Interval range(tokenStack.back(), getTokenId(m_curr[-1].location)); + tokenStack.pop_back(); + + if (nestedLevel == 0) { + range.end -= 1; + if (blocks != nullptr) + blocks->operator[](name).insert(range); + skipAttribute(); + break; + } + if (blocks != nullptr) + blocks->operator[](name).insert(range); + } else if (sequence(tkn::Separator::EndOfProgram)) + return false; + else + next(); + } + i32 index2 = getTokenId(m_curr->location); + + if (index2 > index1 && index2 < tokenCount) { + if (fullName) { + tokenRangeInv[Interval(index1, index2)] = name; + } else { + tokenRange[name] = Interval(index1, index2); + } + if (blocks != nullptr) { + if (addArgumentBlock) { + auto tokenIndex = blocks->operator[](name).begin()->start; + blocks->operator[](name).insert(Interval(index1, tokenIndex)); + } + blocks->operator[](name).insert(Interval(index1, index2)); + } + return true; + } + return false; + } + +// Searches through tokens and loads all the ranges of one kind. First namespaces are searched. + void TextHighlighter::getAllTokenRanges(IdentifierType IdentifierTypeToSearch) { + + if (m_tokens.empty()) + return; + + Identifier *identifier; + IdentifierType identifierType; + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + auto endToken = TokenIter(m_tokens.end(), m_tokens.end()); + for (m_curr = m_startToken; endToken > m_curr; next()) { + auto curr = m_curr; + + if (peek(tkn::Literal::Identifier)) { + if (identifier = getValue(0); identifier != nullptr) { + identifierType = identifier->getType(); + std::string name = identifier->get(); + + if (identifierType == IdentifierTypeToSearch) { + switch (identifierType) { + case IdentifierType::Function: + getTokenRange({tkn::Keyword::Function}, m_functionTokenRange, m_namespaceTokenRange, false, &m_functionBlocks); + break; + case IdentifierType::NameSpace: + if (std::ranges::find(m_nameSpaces, name) == m_nameSpaces.end()) + m_nameSpaces.push_back(name); + getTokenRange({tkn::Keyword::Namespace}, m_functionTokenRange, m_namespaceTokenRange, true, nullptr); + break; + case IdentifierType::UDT: + getTokenRange({tkn::Keyword::Struct, tkn::Keyword::Union, tkn::Keyword::Enum, tkn::Keyword::Bitfield}, m_UDTTokenRange, m_namespaceTokenRange, false, &m_UDTBlocks); + break; + case IdentifierType::Attribute: + linkAttribute(); + break; + default: + break; + } + } + } + } else if (peek(tkn::Separator::EndOfProgram)) + return; + m_curr = curr; + } + } + + void TextHighlighter::skipDelimiters(i32 maxSkipCount, Token delimiter[2], i8 increment) { + auto curr = m_curr; + i32 skipCount = 0; + i32 depth = 0; + + if (!isValid()) + return; + i32 tokenId = getTokenId(m_curr->location); + auto tokenCount = m_tokens.size(); + + if (tokenId == -1 || tokenId >= (i32) tokenCount-1) + return; + i32 skipCountLimit = + increment > 0 ? std::min(maxSkipCount, (i32) tokenCount - 1 - tokenId) : std::min(maxSkipCount, tokenId); + next(increment); + + if (peek(delimiter[0])) { + next(increment); + while (skipCount < skipCountLimit) { + + if (peek(delimiter[1])) { + + if (depth == 0) + return; + depth--; + } else if (peek(delimiter[0])) + depth++; + else if (peek(tkn::Separator::Semicolon)) { + if (increment < 0) + m_curr = curr; + return; + } else if (peek(tkn::Literal::Identifier)) { + + if (peek(tkn::Separator::Dot,1) && peek(tkn::Literal::Identifier,2) ) + + m_memberChains.insert(getTokenId(m_curr->location)); + else if (peek(tkn::Operator::ScopeResolution,1) && peek(tkn::Literal::Identifier,2)) + + m_scopeChains.insert(getTokenId(m_curr->location)); + else + m_taggedIdentifiers.insert(getTokenId(m_curr->location)); + } + next(increment); + skipCount++; + } + } + m_curr = curr; + return; + } + + void TextHighlighter::skipTemplate(i32 maxSkipCount, bool forward) { + Token delimiters[2]; + + if (forward) { + delimiters[0] = Token(tkn::Operator::BoolLessThan); + delimiters[1] = Token(tkn::Operator::BoolGreaterThan); + } else { + delimiters[0] = Token(tkn::Operator::BoolGreaterThan); + delimiters[1] = Token(tkn::Operator::BoolLessThan); + } + skipDelimiters(maxSkipCount, delimiters, forward ? 1 : -1); + } + + void TextHighlighter::skipArray(i32 maxSkipCount, bool forward) { + Token delimiters[2]; + + if (forward) { + delimiters[0] = Token(tkn::Separator::LeftBracket); + delimiters[1] = Token(tkn::Separator::RightBracket); + } else { + delimiters[0] = Token(tkn::Separator::RightBracket); + delimiters[1] = Token(tkn::Separator::LeftBracket); + } + skipDelimiters(maxSkipCount, delimiters, forward ? 1 : -1); + } + +// Used to skip references,pointers,... + void TextHighlighter::skipToken(Token token, i8 step) { + + if (peek(token, step)) + next(step); + } + + void TextHighlighter::skipAttribute() { + + if (sequence(tkn::Separator::LeftBracket, tkn::Separator::LeftBracket)) { + while (!sequence(tkn::Separator::RightBracket, tkn::Separator::RightBracket)) + next(); + } + } + +// Takes an identifier chain resolves the type of the end from the rest iteratively. + bool TextHighlighter::resolveIdentifierType(Definition &result, std::string identifierName) { + std::string separator; + + if (identifierName.contains("::")) + separator = "::"; + else + separator = "."; + auto vectorString = hex::splitString(identifierName, separator); + + std::string nameSpace; + u32 index = 0; + std::string currentName = vectorString[index]; + index++; + std::string variableParentType; + + Definition definition; + + if (vectorString.size() > 1) { + + if (findIdentifierDefinition(definition, currentName)) { + variableParentType = definition.typeStr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, definition.idType); + skipArray(200, true); + next(); + } else + return false; + } + while (index < vectorString.size()) { + + if ( separator == ".") { + currentName = vectorString[index]; + next(); + + if (findIdentifierDefinition(result, currentName, variableParentType)) { + variableParentType = result.typeStr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, result.idType); + skipArray(200, true); + next(); + } else + return false; + + } else if (separator == "::") { + next(); + + if (std::ranges::find(m_nameSpaces, currentName) != m_nameSpaces.end()) { + nameSpace += currentName + "::"; + + variableParentType = vectorString[index]; + currentName = variableParentType; + + } else if (std::ranges::find(m_UDTs, currentName) != m_UDTs.end()) { + variableParentType = currentName; + + if (!nameSpace.empty() && !variableParentType.contains(nameSpace)) + variableParentType = nameSpace + variableParentType; + + else if (findNamespace(nameSpace) && !variableParentType.contains(nameSpace)) + variableParentType = nameSpace + "::" + variableParentType; + + currentName = vectorString[index]; + + if (findIdentifierDefinition(result, currentName, variableParentType)) { + variableParentType = result.typeStr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, result.idType); + skipArray(200, true); + next(); + } else + return false; + } + } + index++; + } + + return true; + } + +// If contex then find it otherwise check if it belongs in map + bool TextHighlighter::findOrContains(std::string &context, UnorderedBlocks tokenRange, VariableMap variableMap) { + + if (context.empty()) + return findScope(context, tokenRange); + else + return variableMap.contains(context); + } + + void TextHighlighter::setBlockInstancesColor(const std::string &name, const Definition &definition, const Interval &block) { + + if (definition.idType == IdentifierType::Unknown) + return; + for (auto instance: m_instances[name]) { + + if (block.contains(instance)) { + if (auto identifier = std::get_if(&m_tokens[instance].value); + identifier != nullptr && identifier->getType() == IdentifierType::Unknown) + setIdentifierColor(instance, definition.idType); + } + } + } + + bool TextHighlighter::findIdentifierDefinition( Definition &result, const std::string &optionalIdentifierName, std::string optionalName, bool setInstances) { + auto curr = m_curr; + bool isFunction = false; + auto tokenId = getTokenId(m_curr->location); + std::vector definitions; + std::string name = optionalName; + result.idType = IdentifierType::Unknown; + std::string identifierName = optionalIdentifierName; + + if (optionalIdentifierName == "") { + std::vector identifiers; + getFullName(identifierName, identifiers); + } + Interval tokenRange; + Scopes blocks; + Scopes::iterator blocksIterBegin, blocksIterEnd; + + if (findOrContains(name, m_UDTTokenRange, m_UDTVariables) && m_UDTVariables[name].contains(identifierName)) { + definitions = m_UDTVariables[name][identifierName]; + tokenRange = m_UDTTokenRange[name]; + blocksIterBegin = m_UDTBlocks[name].begin(); + blocksIterEnd = m_UDTBlocks[name].end(); + + } else if (findOrContains(name, m_functionTokenRange, m_functionVariables) && + m_functionVariables[name].contains(identifierName)) { + isFunction = true; + definitions = m_functionVariables[name][identifierName]; + tokenRange = m_functionTokenRange[name]; + blocksIterBegin = m_functionBlocks[name].begin(); + blocksIterEnd = m_functionBlocks[name].end(); + --blocksIterEnd; + + } else if (m_globalVariables.contains(identifierName)) { + definitions = m_globalVariables[identifierName]; + tokenRange = Interval(0, m_tokens.size()); + blocks.insert(tokenRange); + blocksIterBegin = blocks.begin(); + blocksIterEnd = blocks.end(); + } else if (name == "hex::type::Json" || name == "Object") { + result.idType = IdentifierType::LocalVariable; + result.typeStr = "Object"; + return true; + } + + if (isFunction) { + for (auto block = blocksIterBegin; block != blocksIterEnd; block++) { + + if (tokenId > block->start && tokenId < block->end) { + blocksIterBegin = block; + break; + } + } + for (auto definition : definitions) { + for (auto block = blocksIterBegin; block != blocksIterEnd; block++) { + + if (definition.tokenIndex > block->start && definition.tokenIndex < block->end) { + result = definition; + m_curr = curr; + + if (setInstances) + setBlockInstancesColor(identifierName, definition, *block); + return true; + } + } + } + auto it = std::find_if(definitions.begin(), definitions.end(), [&](const Definition &definition) { + return definition.tokenIndex > tokenRange.start && definition.tokenIndex < tokenRange.end; + }); + + if (it != definitions.end()) { + result = *it; + m_curr = curr; + + if (setInstances) + setBlockInstancesColor(identifierName, *it, tokenRange); + return true; + } + } else { + for (auto block = blocksIterBegin; block != blocksIterEnd; block++) { + + if (tokenId > block->start && tokenId < block->end) { + blocksIterBegin = block; + break; + } + } + for (auto block = blocksIterBegin; block != blocksIterEnd; block++) { + for (auto definition: definitions) { + + if (definition.tokenIndex > block->start && definition.tokenIndex < block->end) { + result = definition; + m_curr = curr; + + if (setInstances) + setBlockInstancesColor(identifierName, definition, *block); + return true; + } + } + } + } + m_curr = curr; + return false; + } + + using Definition = TextHighlighter::Definition; + + bool TextHighlighter::colorOperatorDotChain() { + std::vector identifiers; + std::string variableName; + auto tokenCount = m_tokens.size(); + + if (!getQualifiedName(variableName, identifiers, true)) + return false; + + auto vectorString = hex::splitString(variableName, "."); + u32 index = 0; + + + u32 currentLine = m_curr->location.line - 1; + u32 startingLineTokenIndex = m_firstTokenIdOfLine[currentLine]; + + if (startingLineTokenIndex == 0xFFFFFFFFu || startingLineTokenIndex > tokenCount) + return false; + + if (auto *keyword = std::get_if(&m_tokens[startingLineTokenIndex].value); + keyword != nullptr && *keyword == Keyword::Import) { + while (index < vectorString.size()) { + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, IdentifierType::NameSpace); + next(2); + index++; + } + return true; + } else { + std::string variableParentType; + Definition definition; + std::string currentName = vectorString[index]; + index++; + bool brokenChain = false; + + if (findIdentifierDefinition(definition, currentName)) { + variableParentType = definition.typeStr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, definition.idType); + skipArray(200, true); + next(); + } else { + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, IdentifierType::Unknown); + skipArray(200, true); + next(); + brokenChain = true; + } + + while (index < vectorString.size()) { + currentName = vectorString[index]; + next(); + Definition result=definition; + Definition parentDefinition = result; + + if (findIdentifierDefinition(result, currentName, variableParentType) && !brokenChain) { + variableParentType = result.typeStr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, result.idType); + skipArray(200, true); + next(); + } else if (auto udtVars = m_UDTVariables[result.typeStr];udtVars.contains(vectorString[index-1])) { + auto saveCurr = m_curr; + std::string templateName = ""; + auto instances = m_instances[variableParentType]; + for (auto instance : instances) { + if (auto *identifier = std::get_if(&m_tokens[instance].value); identifier != nullptr && identifier->getType() == IdentifierType::TemplateArgument) { + auto tokenRange = m_UDTTokenRange[result.typeStr]; + auto tokenIndex = m_firstTokenIdOfLine[getLocation(parentDefinition.tokenIndex).line - 1]; + i32 argNumber = getArgumentNumber(tokenRange.start, instance); + getTokenIdForArgument(tokenIndex, argNumber, tkn::Operator::BoolLessThan); + if (auto *identifier2 = std::get_if(&m_curr->value); identifier2 != nullptr) { + templateName = identifier2->get(); + break; + } + } + } + if (!templateName.empty() && findIdentifierDefinition(result, currentName, templateName) ) { + variableParentType = result.typeStr; + m_curr = saveCurr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, result.idType); + skipArray(200, true); + next(); + } else{ + if (m_typeDefMap.contains(variableParentType)) { + std::string typeName = ""; + instances = m_instances[variableParentType]; + for (auto instance: instances) { + if (auto *identifier = std::get_if(&m_tokens[instance].value); + identifier != nullptr && identifier->getType() == IdentifierType::Typedef) { + if (auto *identifier2 = std::get_if(&m_tokens[instance + 2].value); + identifier2 != nullptr) { + typeName = identifier2->get(); + break; + } + } + } + if (!typeName.empty() && findIdentifierDefinition(result, currentName, typeName)) { + variableParentType = result.typeStr; + m_curr = saveCurr; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, result.idType); + skipArray(200, true); + next(); + } + } + } + } else { + brokenChain = true; + auto tokenIndex = getTokenId(m_curr->location); + setIdentifierColor(tokenIndex, IdentifierType::Unknown); + skipArray(200, true); + next(); + } + index++; + } + } + return true; + } + + bool TextHighlighter::colorSeparatorScopeChain() { + std::vector identifiers; + std::string identifierName; + + if (!getQualifiedName(identifierName, identifiers, true)) + return false; + auto tokenCount = m_tokens.size(); + auto vectorString = hex::splitString(identifierName, "::"); + auto vectorStringCount = vectorString.size(); + if (identifiers.size() != vectorStringCount) + return false; + auto curr = m_curr; + std::string nameSpace; + + for (u32 i = 0; i < vectorStringCount ; i++) { + auto name = vectorString[i]; + auto identifier = identifiers[i]; + + if (std::ranges::find(m_nameSpaces, name) != m_nameSpaces.end()) { + setIdentifierColor(-1, IdentifierType::NameSpace); + nameSpace += name + "::"; + } else if (m_UDTDefinitions.contains(nameSpace+name)) { + name = nameSpace + name; + auto udtDefinition = m_UDTDefinitions[name]; + auto definitionIndex = udtDefinition.tokenIndex-1; + if (auto *keyword = std::get_if(&m_tokens[definitionIndex].value); keyword != nullptr) { + setIdentifierColor(-1, IdentifierType::UDT); + if (*keyword == Keyword::Enum) { + next(); + if (!sequence(tkn::Operator::ScopeResolution) || vectorStringCount != i+2 || + !m_UDTVariables.contains(name)) + return false; + auto variableName = vectorString[i+1]; + if (!m_UDTVariables[name].contains(variableName)) + return false; + auto variableDefinition = m_UDTVariables[name][variableName][0]; + setIdentifierColor(-1, variableDefinition.idType); + return true; + } else + return true; + } else + return false; + + } else if (identifier->getType() == IdentifierType::Function) { + setIdentifierColor(-1, IdentifierType::Function); + return true; + } else if (std::ranges::find(m_UDTs, nameSpace+name) != m_UDTs.end()) { + setIdentifierColor(-1, IdentifierType::UDT); + if (vectorStringCount == i+1) + return true; + next(); + if (!sequence(tkn::Operator::ScopeResolution) || vectorStringCount != i+2) + return false; + setIdentifierColor(-1, IdentifierType::PatternVariable); + return true; + } else + return false; + next(2); + } + m_curr = curr; + + if (std::ranges::find(m_nameSpaces, identifierName) != m_nameSpaces.end()) { + setIdentifierColor(-1, IdentifierType::NameSpace); + return true; + } + + i32 index = getTokenId(m_curr->location); + + if (index < (i32) tokenCount - 1 && index > 2) { + auto nextToken = m_curr[1]; + auto *separator = std::get_if(&nextToken.value); + auto *operatortk = std::get_if(&nextToken.value); + + if ((separator != nullptr && *separator == Separator::Semicolon) || + (operatortk != nullptr && *operatortk == Operator::BoolLessThan)) { + auto previousToken = m_curr[-1]; + auto prevprevToken = m_curr[-2]; + operatortk = std::get_if(&previousToken.value); + auto *identifier2 = std::get_if(&prevprevToken.value); + + if (operatortk != nullptr && identifier2 != nullptr && *operatortk == Operator::ScopeResolution) { + + if (identifier2->getType() == IdentifierType::UDT) { + setIdentifierColor(-1, IdentifierType::LocalVariable); + return true; + + } else if (identifier2->getType() == IdentifierType::NameSpace) { + setIdentifierColor(-1, IdentifierType::UDT); + return true; + } + } + } + } + return false; + } + +// finds the name of the token range that the given or the current token index is in. + bool TextHighlighter::findScope(std::string &name, const UnorderedBlocks &map, i32 optionalTokenId) { + auto tokenId = optionalTokenId ==-1 ? getTokenId(m_curr->location) : optionalTokenId; + + for (auto [scopeName, range]: map) { + + if (range.contains(tokenId)) { + name = scopeName; + return true; + } + } + return false; + } + +// Finds the namespace of the given or the current token index. + bool TextHighlighter::findNamespace(std::string &nameSpace, i32 optionalTokenId) { + nameSpace = ""; + + for (auto [interval, name]: m_namespaceTokenRange) { + i32 tokenId = optionalTokenId == -1 ? getTokenId(m_curr->location) : optionalTokenId; + + if (tokenId > interval.start && tokenId < interval.end) { + + if (nameSpace == "") + nameSpace = name; + else + nameSpace = name + "::" + nameSpace; + } + } + + if (nameSpace != "") + return true; + return false; + } + +//The context is the name of the function or UDT that the variable is in. + std::string TextHighlighter::findIdentifierTypeStr(const std::string &identifierName, std::string context) { + Definition result; + findIdentifierDefinition(result, identifierName, context); + return result.typeStr; + } + + //The context is the name of the function or UDT that the variable is in. + TextHighlighter::IdentifierType TextHighlighter::findIdentifierType(const std::string &identifierName, std::string context) { + Definition result; + findIdentifierDefinition(result, identifierName, context); + return result.idType; + } + +// Creates a map from the attribute function to the type of the argument it takes. + void TextHighlighter::linkAttribute() { + auto curr = m_curr; + bool qualifiedAttribute = false; + using Types = std::map>; + auto parser = patternLanguage->get()->getInternals().parser.get(); + Types types = parser->getTypes(); + + while (sequence(tkn::Literal::Identifier, tkn::Operator::ScopeResolution)) + qualifiedAttribute = true; + + if (qualifiedAttribute) { + auto identifier = getValue(0); + + if (identifier != nullptr) + setIdentifierColor(-1, IdentifierType::Attribute); + m_curr = curr; + identifier = getValue(0); + + if (identifier != nullptr) + setIdentifierColor(-1, IdentifierType::NameSpace); + } else + m_curr = curr; + + std::string functionName; + next(); + + if (sequence(tkn::Separator::LeftParenthesis, tkn::Literal::String)) { + functionName = getValue(-1)->toString(false); + + if (!functionName.contains("::")) { + std::string namespaceName; + + if (findNamespace(namespaceName)) + functionName = namespaceName + "::" + functionName; + } else { + + auto vectorString = hex::splitString(functionName, "::"); + vectorString.pop_back(); + for (auto nameSpace: vectorString) { + + if (std::ranges::find(m_nameSpaces, nameSpace) == m_nameSpaces.end()) + m_nameSpaces.push_back(nameSpace); + } + } + } else + return; + + u32 line = m_curr->location.line; + i32 tokenIndex; + + while (!peek(tkn::Separator::Semicolon, -1)) { + + if (line = previousLine(line); line > m_firstTokenIdOfLine.size()-1) + return; + + if (tokenIndex = m_firstTokenIdOfLine[line]; !isTokenIdValid(tokenIndex)) + return; + + m_curr = m_startToken; + next(tokenIndex); + while (peek(tkn::Literal::Comment, -1) || peek(tkn::Literal::DocComment, -1)) + next(-1); + } + + while (peek(tkn::Literal::Comment) || peek(tkn::Literal::DocComment)) + next(); + + Identifier *identifier; + std::string UDTName; + while (sequence(tkn::Literal::Identifier, tkn::Operator::ScopeResolution)) { + identifier = getValue(-2); + UDTName += identifier->get() + "::"; + } + + if (sequence(tkn::Literal::Identifier)) { + identifier = getValue(-1); + UDTName += identifier->get(); + + if (!UDTName.contains("::")) { + std::string namespaceName; + + if (findNamespace(namespaceName)) + UDTName = namespaceName + "::" + UDTName; + } + if (types.contains(UDTName)) + m_attributeFunctionArgumentType[functionName] = UDTName; + + } else if (sequence(tkn::ValueType::Any)) { + auto valueType = getValue(-1); + m_attributeFunctionArgumentType[functionName] = Token::getTypeName(*valueType); + } else { + if (findScope(UDTName, m_UDTTokenRange) && !UDTName.empty()) + m_attributeFunctionArgumentType[functionName] = UDTName; + } + } + +// This function assumes that the first variable in the link that concatenates sequences including the Parent keyword started with Parent and was removed. Uses a function to find +// all the parents of a variable, If there are subsequent elements in the link that are Parent then for each parent it finds all the grandparents and puts them in a vector called +// parentTypes. It stops when a link that's not Parent is found amd only returns the last generation of parents. + bool TextHighlighter::findAllParentTypes(std::vector &parentTypes, + std::vector &identifiers, std::string &optionalFullName) { + auto fullName = optionalFullName; + + if (optionalFullName.empty()) + forwardIdentifierName(fullName, identifiers); + + auto nameParts = hex::splitString(fullName, "."); + std::vector grandpaTypes; + findParentTypes(parentTypes); + + if (parentTypes.empty()) { + auto curr = m_curr; + //must be a template instance of a type template instance + // find the name of the structure that is two < before. + u32 index = getTokenId(m_curr->location); + u32 maxIndex; + if (m_curr->location.line < m_firstTokenIdOfLine.size()) + maxIndex = m_firstTokenIdOfLine[m_curr->location.line]; + else + maxIndex = m_tokens.size() - 1; + u32 found = 0; + auto i=0; + do { + if (index >= maxIndex || i >= 200) { + m_curr = curr; + return false; + } + next(); + if (auto *operatortk = std::get_if(&m_curr->value);operatortk != nullptr && *operatortk == Operator::BoolGreaterThan) { + found++; + if (found == 2) + break; + } + index = getTokenId(m_curr->location); + i++; + } while (true); + + next(); + skipTemplate(200,false); + next(-1); + auto *identifier = std::get_if(&m_curr->value); + if (identifier != nullptr) { + auto identifierName = identifier->get(); + if (std::ranges::find(m_UDTs, identifierName) != m_UDTs.end()) { + if (std::ranges::find(parentTypes, identifierName) == parentTypes.end()) { + parentTypes.push_back(identifierName); + m_curr = curr; + return true; + } + } else if (std::ranges::find(m_nameSpaces, identifierName) == + m_nameSpaces.end()) { + m_nameSpaces.push_back(identifierName); + m_curr = curr; + return true; + } + } + } + if (parentTypes.empty()) + return false; + + auto currentName = nameParts[0]; + nameParts.erase(nameParts.begin()); + auto identifier = identifiers[0]; + identifiers.erase(identifiers.begin()); + + while (currentName == "Parent" && !nameParts.empty()) { + for (auto parentType: parentTypes) + findParentTypes(grandpaTypes, parentType); + + currentName = nameParts[0]; + nameParts.erase(nameParts.begin()); + identifier = identifiers[0]; + identifiers.erase(identifiers.begin()); + parentTypes = grandpaTypes; + grandpaTypes.clear(); + } + + nameParts.insert(nameParts.begin(), currentName); + identifiers.insert(identifiers.begin(), identifier); + optionalFullName = hex::combineStrings(nameParts, "."); + return true; + } + +// Searches for parents through every custom type,i.e. for structs that have members +// of the same type as the one being searched and places them in a vector called parentTypes. + bool TextHighlighter::findParentTypes(std::vector &parentTypes, const std::string &optionalUDTName) { + std::string UDTName; + std::string functionName; + bool isFunction = false; + if (optionalUDTName.empty()) { + if (!findScope(UDTName, m_UDTTokenRange)) { + if (!findScope(functionName, m_functionTokenRange)) { + return false; + } else { + isFunction = true; + } + } + } else + UDTName = optionalUDTName; + + bool found = false; + if (!isFunction) { + for (auto [name, variables]: m_UDTVariables) { + for (auto [variableName, definitions]: variables) { + for (auto definition: definitions) { + + if (definition.typeStr == UDTName) { + + if (std::ranges::find(parentTypes, name) == parentTypes.end()) { + parentTypes.push_back(name); + found = true; + } + } + } + } + } + } else { + auto curr = m_curr; + for (auto [name, range] : m_UDTTokenRange) { + auto startToken = TokenIter(m_tokens.begin()+range.start,m_tokens.begin()+range.end); + auto endToken = TokenIter(m_tokens.begin()+range.end,m_tokens.end()); + + for ( m_curr = startToken; endToken > m_curr; next()) { + if (auto *identifier = std::get_if(&m_curr->value); identifier != nullptr) { + auto identifierName = identifier->get(); + auto identifierType = identifier->getType(); + if (identifierName == functionName && identifierType == IdentifierType::Function) { + parentTypes.push_back(name); + found = true; + } + } + } + } + m_curr = curr; + } + return found; + } + +// this function searches all the parents recursively until it can match the variable name at the end of the chain +// and selects its type to colour the variable because the search only occurs pn type declarations which we know +// the types of. Once the end link is found then all the previous links are also assigned the types that were found +// for them during the search. + bool TextHighlighter::tryParentType(const std::string &parentType, std::string &variableName, + std::optional &result, + std::vector &identifiers) { + + auto vectorString = hex::splitString(variableName, "."); + auto count = vectorString.size(); + auto UDTName = parentType; + auto currentName = vectorString[0]; + + if (m_UDTVariables.contains(UDTName) && m_UDTVariables[UDTName].contains(currentName)) { + auto definitions = m_UDTVariables[UDTName][currentName]; + for (auto definition: definitions) { + UDTName = definition.typeStr; + + if (count == 1) { + setIdentifierColor(-1, definition.idType); + result = definition; + return true; + } + + vectorString.erase(vectorString.begin()); + variableName = hex::combineStrings(vectorString, "."); + Identifier *identifier = identifiers[0]; + identifiers.erase(identifiers.begin()); + skipArray(200, true); + next(2); + + if (tryParentType(UDTName, variableName, result, identifiers)) { + next(-1); + skipArray(200, false); + next(-1); + setIdentifierColor(-1, definition.idType); + return true; + } + + identifiers.insert(identifiers.begin(), identifier); + variableName += "." + currentName; + next(-1); + skipArray(200, false); + next(-1); + return false; + } + return false; + } else + return false; + return false; + } + +// Handles Parent keyword. + std::optional TextHighlighter::setChildrenTypes() { + auto curr = m_curr; + std::string fullName; + std::vector identifiers; + std::vector definitions; + std::optional result; + + forwardIdentifierName(fullName, identifiers); + + std::vector parentTypes; + auto vectorString = hex::splitString(fullName, "."); + if (vectorString[0] == "Parent") { + vectorString.erase(vectorString.begin()); + fullName = hex::combineStrings(vectorString, "."); + identifiers.erase(identifiers.begin()); + if (!findAllParentTypes(parentTypes, identifiers, fullName)) { + m_curr = curr; + return std::nullopt; + } + } else { + m_curr = curr; + return std::nullopt; + } + + for (auto parentType: parentTypes) { + m_curr = curr; + while (peek(tkn::Keyword::Parent)) + next(2); + + if (tryParentType(parentType, fullName, result, identifiers)) { + + if (result.has_value()) + definitions.push_back(result.value()); + } else { + m_curr = curr; + return std::nullopt; + } + } + // Todo: Are all definitions supposed to be the same? If not, which one should be used? + // for now, use the first one. + if (!definitions.empty()) + result = definitions[0]; + m_curr = curr; + return result; + } + + const TextHighlighter::TokenTypeColor TextHighlighter::m_tokenTypeColor = { + {Token::Type::Keyword, TextEditor::PaletteIndex::Keyword}, + {Token::Type::ValueType, TextEditor::PaletteIndex::BuiltInType}, + {Token::Type::Operator, TextEditor::PaletteIndex::Operator}, + {Token::Type::Separator, TextEditor::PaletteIndex::Separator}, + {Token::Type::String, TextEditor::PaletteIndex::StringLiteral}, + {Token::Type::Directive, TextEditor::PaletteIndex::Directive}, + {Token::Type::Comment, TextEditor::PaletteIndex::Comment}, + {Token::Type::Integer, TextEditor::PaletteIndex::NumericLiteral}, + {Token::Type::Identifier, TextEditor::PaletteIndex::Identifier}, + {Token::Type::DocComment, TextEditor::PaletteIndex::DocComment} + + }; + + const TextHighlighter::IdentifierTypeColor TextHighlighter::m_identifierTypeColor = { + {Identifier::IdentifierType::Macro, TextEditor::PaletteIndex::PreprocIdentifier}, + {Identifier::IdentifierType::UDT, TextEditor::PaletteIndex::UserDefinedType}, + {Identifier::IdentifierType::Function, TextEditor::PaletteIndex::Function}, + {Identifier::IdentifierType::Attribute, TextEditor::PaletteIndex::Attribute}, + {Identifier::IdentifierType::NameSpace, TextEditor::PaletteIndex::NameSpace}, + {Identifier::IdentifierType::Typedef, TextEditor::PaletteIndex::TypeDef}, + {Identifier::IdentifierType::PatternVariable, TextEditor::PaletteIndex::PatternVariable}, + {Identifier::IdentifierType::LocalVariable, TextEditor::PaletteIndex::LocalVariable}, + {Identifier::IdentifierType::CalculatedPointer, TextEditor::PaletteIndex::CalculatedPointer}, + {Identifier::IdentifierType::TemplateArgument, TextEditor::PaletteIndex::TemplateArgument}, + {Identifier::IdentifierType::PlacedVariable, TextEditor::PaletteIndex::PlacedVariable}, + {Identifier::IdentifierType::View, TextEditor::PaletteIndex::View}, + {Identifier::IdentifierType::FunctionVariable, TextEditor::PaletteIndex::FunctionVariable}, + {Identifier::IdentifierType::FunctionParameter, TextEditor::PaletteIndex::FunctionParameter}, + {Identifier::IdentifierType::Unknown, TextEditor::PaletteIndex::UnkIdentifier}, + {Identifier::IdentifierType::FunctionUnknown, TextEditor::PaletteIndex::UnkIdentifier}, + {Identifier::IdentifierType::MemberUnknown, TextEditor::PaletteIndex::UnkIdentifier}, + {Identifier::IdentifierType::ScopeResolutionUnknown, TextEditor::PaletteIndex::UnkIdentifier}, + {Identifier::IdentifierType::GlobalVariable, TextEditor::PaletteIndex::GlobalVariable}, + }; + +// Second paletteIndex called from processLineTokens to process literals + TextEditor::PaletteIndex TextHighlighter::getPaletteIndex(Literal *literal) { + + if (literal->isFloatingPoint() || literal->isSigned() || literal->isUnsigned()) + return TextEditor::PaletteIndex::NumericLiteral; + + else if (literal->isCharacter() || literal->isBoolean()) return TextEditor::PaletteIndex::CharLiteral; + + else if (literal->isString()) return TextEditor::PaletteIndex::StringLiteral; + + else return TextEditor::PaletteIndex::Default; + } + +// Render the compilation errors using squiggly lines + void TextHighlighter::renderErrors() { + const auto processMessage = [](const auto &message) { + auto lines = hex::splitString(message, "\n"); + + std::ranges::transform(lines, lines.begin(), [](auto line) { + + if (line.size() >= 128) + line = wolv::util::trim(line); + + return hex::limitStringLength(line, 128); + }); + + return hex::combineStrings(lines, "\n"); + }; + TextEditor::ErrorMarkers errorMarkers; + + if (!m_compileErrors.empty()) { + for (const auto &error: m_compileErrors) { + + if (isLocationValid(error.getLocation())) { + auto key = TextEditor::Coordinates(error.getLocation().line, error.getLocation().column); + + if (!errorMarkers.contains(key) || errorMarkers[key].first < error.getLocation().length) + errorMarkers[key] = std::make_pair(error.getLocation().length, processMessage(error.getMessage())); + } + } + } + m_viewPatternEditor->getTextEditor().SetErrorMarkers(errorMarkers); + } + +// creates a map from variable names to a vector of token indices +// od every instance of the variable name in the code. + void TextHighlighter::setInitialColors() { + + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + auto endToken = TokenIter(m_tokens.end(), m_tokens.end()); + for (m_curr = m_startToken; endToken > m_curr; next()) { + + if (peek(tkn::Literal::Identifier)) { + + if (auto identifier = getValue(0); identifier != nullptr) { + + if (auto identifierType = identifier->getType(); identifierType != IdentifierType::Unknown && identifierType != IdentifierType::MemberUnknown && + identifierType != IdentifierType::FunctionUnknown && identifierType != IdentifierType::ScopeResolutionUnknown) { + + setIdentifierColor(-1, identifierType); + } + } + } else if (peek(tkn::Separator::EndOfProgram)) + return; + } + } + + void TextHighlighter::loadInstances() { + std::map> instances; + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + auto endToken = TokenIter(m_tokens.end(), m_tokens.end()); + for (m_curr = m_startToken; endToken > m_curr; next()) { + + if (peek(tkn::Literal::Identifier)) { + std::string name; + + if (auto identifier = getValue(0); identifier != nullptr) { + + if (auto identifierType = identifier->getType(); identifierType != IdentifierType::Unknown && identifierType != IdentifierType::MemberUnknown && + identifierType != IdentifierType::FunctionUnknown && identifierType != IdentifierType::ScopeResolutionUnknown) { + name = identifier->get(); + + if (identifierType == IdentifierType::Typedef) { + auto curr = m_curr; + next(); + std::string typeName = ""; + if (sequence(tkn::Operator::Assign, tkn::Literal::Identifier)) { + auto identifier2 = getValue(-1); + if (identifier2 != nullptr) { + typeName = identifier2->get(); + if (!m_typeDefMap.contains(name) && !typeName.empty()) { + m_typeDefMap[name] = typeName; + m_typeDefInvMap[typeName] = name; + } + } + } + + m_curr = curr; + } + } else { + name = identifier->get(); + auto curr = m_curr; + auto tokenIndex = getTokenId(m_curr->location); + skipArray(200, true); + next(); + bool chainStarted = false; + while (sequence(tkn::Operator::ScopeResolution, tkn::Literal::Identifier)) { + + if (identifier = getValue(-1); identifier != nullptr) + name += "::" + identifier->get(); + + if (!chainStarted) { + chainStarted = true; + m_scopeChains.insert(tokenIndex); + } + curr = m_curr; + } + while (sequence(tkn::Separator::Dot, tkn::Literal::Identifier)) { + + if (identifier = getValue(-1); identifier != nullptr) + name += "." + identifier->get(); + + if (!chainStarted) { + chainStarted = true; + m_memberChains.insert(tokenIndex); + } + skipArray(200, true); + curr = m_curr; + } + m_curr = curr; + } + } + auto id = getTokenId(m_curr->location); + + if (instances.contains(name)) { + auto &nameInstances = instances[name]; + + if (std::ranges::find(nameInstances, id) == nameInstances.end()) + nameInstances.push_back(id); + } else + instances[name].push_back(id); + } else if (peek(tkn::Separator::EndOfProgram)) + break; + } + m_instances = std::move(instances); + } + +// Get the location of a given token index + pl::core::Location TextHighlighter::getLocation(i32 tokenId) { + + if (tokenId >= (i32) m_tokens.size()) + return Location::Empty(); + return m_tokens[tokenId].location; + } + +// Get the token index for a given location. + i32 TextHighlighter::getTokenId(pl::core::Location location) { + + if (!isLocationValid(location)) + return -1; + auto line1 = location.line - 1; + auto line2 = nextLine(line1); + auto tokenCount = m_tokens.size(); + i32 tokenStart = m_firstTokenIdOfLine[line1]; + i32 tokenEnd = m_firstTokenIdOfLine[line2] - 1; + + if (tokenEnd >= (i32) tokenCount) + tokenEnd = tokenCount - 1; + + if (tokenStart == -1 || tokenEnd == -1 || tokenStart >= (i32) tokenCount) + return -1; + + for (i32 i = tokenStart; i <= tokenEnd; i++) { + + if (m_tokens[i].location.column >= location.column) + return i; + } + return -1; + } + + void TextHighlighter::setIdentifierColor(i32 tokenId, const IdentifierType &type) { + Token *token; + + if (tokenId == -1) + token = const_cast(&m_curr[0]); + else + token = const_cast(&m_tokens[tokenId]); + + if (token->type == Token::Type::Identifier && (!m_tokenColors.contains(token) || m_tokenColors.at(token) == TextEditor::PaletteIndex::Default || m_tokenColors.at(token) == TextEditor::PaletteIndex::UnkIdentifier)) + m_tokenColors[token] = m_identifierTypeColor.at(type); + else if (!m_tokenColors.contains(token)) + m_tokenColors[token] = m_tokenTypeColor.at(token->type); + } + + void TextHighlighter::setColor(i32 tokenId, const IdentifierType &type) { + Token *token; + + if (tokenId == -1) + token = const_cast(&m_curr[0]); + else + token = const_cast(&m_tokens[tokenId]); + + if (token->type == Token::Type::Integer) { + auto literal = getValue(0); + + if (literal != nullptr && !m_tokenColors.contains(token)) + m_tokenColors[token] = getPaletteIndex(literal); + + + } else if (token->type == Token::Type::DocComment) { + auto docComment = getValue(0); + + if (docComment != nullptr && !m_tokenColors.contains(token)) { + + if (docComment->singleLine) + m_tokenColors[token] = TextEditor::PaletteIndex::DocComment; + else if (docComment->global) + m_tokenColors[token] = TextEditor::PaletteIndex::GlobalDocComment; + else + m_tokenColors[token] = TextEditor::PaletteIndex::DocBlockComment; + } + } else if (token->type == Token::Type::Comment) { + auto comment = getValue(0); + + if (comment != nullptr && !m_tokenColors.contains(token)) { + + if (comment->singleLine) + m_tokenColors[token] = TextEditor::PaletteIndex::Comment; + else + m_tokenColors[token] = TextEditor::PaletteIndex::BlockComment; + } + } else + setIdentifierColor(tokenId, type); + } + + void TextHighlighter::colorRemainingIdentifierTokens() { + std::vector taggedIdentifiers; + for (auto index: m_taggedIdentifiers) { + taggedIdentifiers.push_back(index); + } + m_taggedIdentifiers.clear(); + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + auto endToken = TokenIter(m_tokens.end(), m_tokens.end()); + m_curr = m_startToken; + + while (endToken > m_curr) { + if (peek(tkn::Separator::EndOfProgram)) + return; + i32 tokenId = getTokenId(m_curr->location); + + if (!taggedIdentifiers.empty() && tokenId > taggedIdentifiers.back()) { + next(taggedIdentifiers.back() - tokenId); + taggedIdentifiers.pop_back(); + } + + Token *token = const_cast(&m_curr[0]); + + if (sequence(tkn::Keyword::Import, tkn::Literal::Identifier)) { + next(-1); + do { + if (auto identifier = const_cast(getValue(0)); identifier != nullptr) { + setIdentifierColor(-1, IdentifierType::NameSpace); + if (std::ranges::find(m_nameSpaces, identifier->get()) == m_nameSpaces.end()) { + m_nameSpaces.push_back(identifier->get()); + } + } + } while (sequence(tkn::Literal::Identifier,tkn::Separator::Dot)); + next(); + if (sequence(tkn::Keyword::As, tkn::Literal::Identifier)) { + next(-1); + if (auto identifier = const_cast(getValue(0)); identifier != nullptr) { + setIdentifierColor(-1, IdentifierType::NameSpace); + if (std::ranges::find(m_nameSpaces, identifier->get()) == m_nameSpaces.end()) { + m_nameSpaces.push_back(identifier->get()); + } + } + } + } + if (peek(tkn::Literal::Identifier)) { + auto identifier = getValue(0); + Token::Identifier::IdentifierType identifierType; + if (identifier == nullptr) { + next(); + continue; + } + identifierType = identifier->getType(); + std::string variableName = identifier->get(); + + if (m_tokenColors.contains(token) && (m_tokenColors.at(token) != TextEditor::PaletteIndex::Default && identifierType != IdentifierType::Unknown)) { + next(); + continue; + } + Definition definition; + + if (peek(tkn::Keyword::Parent, -2)) { + auto save = m_curr; + next(-2); + while (peek(tkn::Keyword::Parent, -2)) + next(-2); + auto optional = setChildrenTypes(); + + if (optional.has_value()) + setIdentifierColor(-1, optional->idType); + else { + m_curr = save; + setIdentifierColor(-1, IdentifierType::Unknown); + next(); + continue; + } + m_curr = save; + next(); + continue; + } else if (peek(tkn::Operator::ScopeResolution, 1)) { + if (std::ranges::find(m_nameSpaces, variableName) != m_nameSpaces.end()) { + setIdentifierColor(-1, IdentifierType::NameSpace); + next(); + continue; + } + } else if (peek(tkn::Operator::ScopeResolution, -1)) { + auto save = m_curr; + next(-2); + if (auto parentIdentifier = const_cast(getValue(0)); parentIdentifier != nullptr) { + next(2); + if (parentIdentifier->getType() == IdentifierType::UDT) { + auto parentName = parentIdentifier->get(); + auto typeName = findIdentifierType(variableName, parentName); + setIdentifierColor(-1, typeName); + } + } + m_curr = save; + next(); + continue; + } else if (findIdentifierDefinition(definition)) { + identifierType = definition.idType; + setIdentifierColor(-1, identifierType); + next(); + continue; + } else if (std::ranges::find(m_UDTs, variableName) != m_UDTs.end()) { + if (m_typeDefMap.contains(variableName)) + setIdentifierColor(-1, IdentifierType::Typedef); + else + setIdentifierColor(-1, IdentifierType::UDT); + next(); + continue; + } else if (peek(tkn::Keyword::From, -1)) { + setIdentifierColor(-1, IdentifierType::GlobalVariable); + next(); + continue; + } else { + setIdentifierColor(-1, IdentifierType::Unknown); + next(); + continue; + } + } + next(); + } + } + + void TextHighlighter::setRequestedIdentifierColors() { + auto topLine = 0; + auto bottomLine = m_lines.size(); + for (u32 line = topLine; line < bottomLine; line = nextLine(line)) { + if (m_lines[line].empty()) + continue; + std::string lineOfColors = std::string(m_lines[line].size(), 0); + for (auto tokenIndex = m_firstTokenIdOfLine[line]; tokenIndex < m_firstTokenIdOfLine[nextLine(line)]; tokenIndex++) { + Token *token = const_cast(&m_tokens[tokenIndex]); + if (m_tokenColors.contains(token) && token->type == Token::Type::Identifier) { + u8 color = (u8) m_tokenColors.at(token); + u32 tokenLength = token->location.length; + u32 tokenOffset = token->location.column - 1; + if (token->location.line != line + 1) + continue; + if (tokenOffset + tokenLength - 1 >= m_lines[line].size()) + continue; + for (u32 j = 0; j < tokenLength; j++) + lineOfColors[tokenOffset + j] = color; + } + } + m_viewPatternEditor->getTextEditor().SetColorizedLine(line, lineOfColors); + } + } + + void TextHighlighter::recurseInheritances(std::string name) { + if (m_inheritances.contains(name)) { + for (auto inheritance: m_inheritances[name]) { + recurseInheritances(inheritance); + auto definitions = m_UDTVariables[inheritance]; + for (auto [variableName, variableDefinitions]: definitions) { + auto tokenRange = m_UDTTokenRange[name]; + u32 tokenIndex = tokenRange.start; + for (auto token = tokenRange.start; token < tokenRange.end; token++) { + + if (auto operatorTkn = std::get_if(&m_tokens.at(token).value); + operatorTkn != nullptr && *operatorTkn == Token::Operator::Colon) + tokenIndex = token + 1; + } + for (auto variableDefinition: variableDefinitions) { + variableDefinition.tokenIndex = tokenIndex; + m_UDTVariables[name][variableName].push_back(variableDefinition); + } + } + } + } + } + + void TextHighlighter::appendInheritances() { + for (auto [name, inheritances]: m_inheritances) + recurseInheritances(name); + } + +// Get the string of the argument type. This works on function arguments and non-type template arguments. + std::string TextHighlighter::getArgumentTypeName(i32 rangeStart, Token delimiter2) { + auto curr = m_curr; + i32 parameterIndex = getArgumentNumber(rangeStart, getTokenId(m_curr->location)); + Token delimiter; + std::string typeStr; + + if (parameterIndex > 0) + delimiter = tkn::Separator::Comma; + else + delimiter = delimiter2; + + while (!peek(delimiter)) + next(-1); + skipToken(tkn::Keyword::Reference); + next(); + + if (peek(tkn::ValueType::Any)) + typeStr = Token::getTypeName(*getValue(0)); + else if (peek(tkn::Literal::Identifier)) + typeStr = getValue(0)->get(); + + m_curr = curr; + return typeStr; + } + + bool TextHighlighter::isTokenIdValid(i32 tokenId) { + return tokenId >= 0 && tokenId < (i32) m_tokens.size(); + } + + bool TextHighlighter::isLocationValid(hex::plugin::builtin::TextHighlighter::Location location) { + const pl::api::Source *source; + try { + source = location.source; + } catch (const std::out_of_range &e) { + log::error("TextHighlighter::IsLocationValid: Out of range error: {}", e.what()); + return false; + } + if (source == nullptr || !source->mainSource) + return false; + i32 line = location.line - 1; + i32 col = location.column - 1; + i32 length = location.length; + + if ( line < 0 || line >= (i32) m_lines.size()) + return false; + + if ( col < 0 || col > (i32) m_lines[line].size()) + return false; + + if (length < 0 || length > (i32) m_lines[line].size()-col) + return false; + return true; + } + +// Find the string of the variable type. This works on function variables, views, +// local variables as well as on calculated pointers and pattern variables. + std::string TextHighlighter::getVariableTypeName() { + auto curr = m_curr; + auto varTokenId = getTokenId(m_curr->location); + + if (!isTokenIdValid(varTokenId)) + return ""; + + std::string typeStr; + skipToken(tkn::Operator::Star, -1); + + while (peek(tkn::Separator::Comma, -1)) + next(-2); + + if (peek(tkn::ValueType::Any, -1)) + typeStr = Token::getTypeName(*getValue(-1)); + else if (peek(tkn::Keyword::Signed, -1)) + typeStr = "signed"; + else if (peek(tkn::Keyword::Unsigned, -1)) + typeStr = "unsigned"; + else { + skipTemplate(200, false); + next(-1); + + if (peek(tkn::Literal::Identifier)) { + typeStr = getValue(0)->get(); + next(-1); + } + std::string nameSpace; + while (peek(tkn::Operator::ScopeResolution)) { + next(-1); + nameSpace = getValue(0)->get() + "::" + nameSpace; + next(-1); + } + typeStr = nameSpace + typeStr; + using Types = std::map>; + auto parser = patternLanguage->get()->getInternals().parser.get(); + Types types = parser->getTypes(); + + if (types.contains(typeStr)) { + m_curr = curr; + return typeStr; + } + std::vector candidates; + for (auto name: m_UDTs) { + auto vectorString = hex::splitString(name, "::"); + + if (typeStr == vectorString.back()) + candidates.push_back(name); + } + + if (candidates.size() == 1) { + m_curr = curr; + return candidates[0]; + } + } + m_curr = curr; + return typeStr; + } + +// Definitions of global variables and placed variables. + void TextHighlighter::loadGlobalDefinitions( Scopes tokenRangeSet, std::vector identifierTypes, Variables &variables) { + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + + for (auto range: tokenRangeSet) { + auto startToken = TokenIter(m_tokens.begin()+range.start,m_tokens.begin()+range.end); + auto endToken = TokenIter(m_tokens.begin()+range.end,m_tokens.end()); + + for ( m_curr = startToken; endToken > m_curr; next()) { + + if (peek(tkn::Literal::Identifier)) { + auto identifier = getValue(0); + auto identifierType = identifier->getType(); + auto identifierName = identifier->get(); + + if (std::ranges::find(identifierTypes, identifierType) != identifierTypes.end()) { + std::string typeStr = getVariableTypeName(); + + if (typeStr.empty()) + continue; + i32 tokenId = getTokenId(m_curr->location); + Definition definition(identifierType, typeStr, tokenId, m_curr->location); + variables[identifierName].push_back(definition); + continue; + } + } + } + } + } + +// Definitions of variables and arguments in functions and user defined types. + void TextHighlighter::loadVariableDefinitions(UnorderedBlocks tokenRangeMap, Token delimiter1, Token delimiter2, + std::vector identifierTypes, + bool isArgument, VariableMap &variableMap) { + for (auto [name, range]: tokenRangeMap) { + m_curr = m_startToken; + auto endToken = m_startToken; + next(range.start); + + if (isArgument) { + while (!peek(delimiter1)) { + + if (peek(tkn::Separator::LeftBrace)) + break; + next(); + } + + if (peek(tkn::Separator::LeftBrace)) + continue; + endToken = m_curr; + while (!peek(delimiter2)) { + + if (peek(tkn::Separator::LeftBrace)) + break; + next(); + } + + if (peek(tkn::Separator::LeftBrace)) + continue; + auto temp = m_curr; + m_curr = endToken; + endToken = temp; + } else + endToken = endToken + range.end; + + Keyword *keyword; + for (keyword = std::get_if(&m_tokens[range.start].value); endToken > m_curr; next()) { + + if (peek(tkn::Literal::Identifier)) { + auto identifier = getValue(0); + + if (identifier == nullptr) + continue; + auto identifierType = identifier->getType(); + auto identifierName = identifier->get(); + + if (std::ranges::find(identifierTypes, identifierType) != identifierTypes.end()) { + std::string typeStr; + + if (keyword != nullptr && (*keyword == Keyword::Enum)) { + typeStr = name; + } else if (isArgument) { + typeStr = getArgumentTypeName(range.start, delimiter1); + } else { + typeStr = getVariableTypeName(); + if (typeStr.empty() && keyword != nullptr && *keyword == Keyword::Bitfield) + typeStr = "bits"; + if (m_typeDefMap.contains(typeStr)) + typeStr = m_typeDefMap[typeStr]; + } + + if (typeStr.empty()) + continue; + Definition definition(identifierType, typeStr, getTokenId(m_curr->location), m_curr->location); + variableMap[name][identifierName].push_back(definition); + continue; + } + } + } + } + } + +// Definitions of user defined types and functions. + void TextHighlighter::loadTypeDefinitions( UnorderedBlocks tokenRangeMap, std::vector identifierTypes, Definitions &types) { + for (auto [name, range]: tokenRangeMap) { + + m_curr = m_startToken + range.start+1; + + if (!peek(tkn::Literal::Identifier)) + continue; + auto identifier = getValue(0); + if (identifier == nullptr) + continue; + auto identifierType = identifier->getType(); + + if (std::ranges::find(identifierTypes, identifierType) == identifierTypes.end()) + continue; + auto identifierName = identifier->get(); + if (!name.ends_with(identifierName)) + continue; + types[name] = ParentDefinition(identifierType, getTokenId(m_curr->location), m_curr->location); + } + } + +// Once types are loaded from parsed tokens we can create +// maps of variable names to their definitions. + void TextHighlighter::getDefinitions() { + using IdentifierType::LocalVariable; + using IdentifierType::PatternVariable; + using IdentifierType::CalculatedPointer; + using IdentifierType::GlobalVariable; + using IdentifierType::PlacedVariable; + using IdentifierType::View; + using IdentifierType::FunctionVariable; + using IdentifierType::FunctionParameter; + using IdentifierType::TemplateArgument; + using IdentifierType::UDT; + using IdentifierType::Function; + + if (!m_UDTDefinitions.empty()) + m_UDTDefinitions.clear(); + loadTypeDefinitions(m_UDTTokenRange, {UDT}, m_UDTDefinitions); + + if (!m_globalVariables.empty()) + m_globalVariables.clear(); + loadGlobalDefinitions(m_globalTokenRange, {GlobalVariable, PlacedVariable}, m_globalVariables); + + if (!m_UDTVariables.empty()) + m_UDTVariables.clear(); + loadVariableDefinitions(m_UDTTokenRange, tkn::Operator::BoolLessThan, tkn::Operator::BoolGreaterThan, + {TemplateArgument}, true, m_UDTVariables); + + loadVariableDefinitions(m_UDTTokenRange, tkn::Operator::BoolLessThan, tkn::Operator::BoolGreaterThan, + {LocalVariable, PatternVariable, CalculatedPointer}, false, m_UDTVariables); + appendInheritances(); + + if (!m_functionDefinitions.empty()) + m_functionDefinitions.clear(); + loadTypeDefinitions(m_functionTokenRange, {Function}, m_functionDefinitions); + + if (!m_functionVariables.empty()) + m_functionVariables.clear(); + loadVariableDefinitions(m_functionTokenRange, tkn::Separator::LeftParenthesis, tkn::Separator::RightParenthesis, + {FunctionParameter}, true, m_functionVariables); + + loadVariableDefinitions(m_functionTokenRange, tkn::Separator::LeftParenthesis, tkn::Separator::RightParenthesis, + {View,FunctionVariable}, false, m_functionVariables); + } + + +// Load the source code into the text highlighter, splits +// the text into lines and creates a lookup table for the +// first token id of each line. + void TextHighlighter::loadText() { + + if (!m_lines.empty()) + m_lines.clear(); + + if (m_text.empty()) + m_text = m_viewPatternEditor->getTextEditor().PreprocessText(m_viewPatternEditor->getTextEditor().GetText()); + + m_lines = hex::splitString(m_text, "\n"); + m_lines.push_back(""); + m_firstTokenIdOfLine.clear(); + m_firstTokenIdOfLine.resize(m_lines.size(), -1); + + i32 tokenId = 0; + i32 tokenCount = m_tokens.size(); + i32 index; + + if (tokenCount > 0) { + index = m_tokens[0].location.line - 1; + m_firstTokenIdOfLine[index] = 0; + } + i32 count = m_lines.size(); + for (i32 currentLine = 0; currentLine < count; currentLine++) { + for (index = m_tokens[tokenId].location.line - 1; index <= currentLine && tokenId + 1 < tokenCount; tokenId++) { + index = m_tokens[tokenId + 1].location.line - 1; + } + + if (index > currentLine) { + m_firstTokenIdOfLine[index] = tokenId; + } + } + + if (m_firstTokenIdOfLine.back() != tokenCount) + m_firstTokenIdOfLine.push_back(tokenCount); + } + +// Some tokens span many lines and some lines have no tokens. This +// function helps to find the next line number in the inner loop. + u32 TextHighlighter::nextLine(u32 line) { + auto currentTokenId = m_firstTokenIdOfLine[line]; + u32 i = 1; + while (line + i < m_lines.size() && + (m_firstTokenIdOfLine[line + i] == currentTokenId || m_firstTokenIdOfLine[line + i] == (i32) 0xFFFFFFFF)) + i++; + return i + line; + } + + u32 TextHighlighter::previousLine(u32 line) { + auto currentTokenId = m_firstTokenIdOfLine[line]; + u32 i = 1; + while (line - i < m_lines.size() && + (m_firstTokenIdOfLine[line - i] == currentTokenId || m_firstTokenIdOfLine[line - i] == (i32) 0xFFFFFFFF)) + i++; + return line - i; + } + +// global token ranges are the complement (aka inverse) of the union +// of the UDT and function token ranges + void TextHighlighter::invertGlobalTokenRange() { + std::set ranges; + auto size = m_globalTokenRange.size(); + auto tokenCount = m_tokens.size(); + + if (size == 0) { + ranges.insert(Interval(0, tokenCount)); + } else { + auto it = m_globalTokenRange.begin(); + auto it2 = std::next(it); + if (it->start != 0) + ranges.insert(Interval(0, it->start)); + while (it2 != m_globalTokenRange.end()) { + + if (it->end < it2->start) + ranges.insert(Interval(it->end, it2->start)); + else + ranges.insert(Interval(it->start, it2->end)); + it = it2; + it2 = std::next(it); + } + + if (it->end < (i32) (tokenCount-1)) + ranges.insert(Interval(it->end, tokenCount-1)); + } + m_globalTokenRange = ranges; + } + +// 0 for 1st argument, 1 for 2nd argument, etc. Obtained counting commas. + i32 TextHighlighter::getArgumentNumber(i32 start, i32 arg) { + i32 count = 0; + m_curr = m_startToken; + auto endToken = m_startToken + arg; + next(start); + while (endToken > m_curr) { + + if (peek(tkn::Separator::Comma)) + count++; + next(); + } + return count; + } + +// The inverse of getArgumentNumber. + void TextHighlighter::getTokenIdForArgument(i32 start, i32 argNumber, Token delimiter) { + m_curr = m_startToken; + next(start); + while (!peek(delimiter)) + next(); + next(); + i32 count = 0; + while (count < argNumber && !peek(tkn::Separator::EndOfProgram)) { + + if (peek(tkn::Separator::Comma)) + count++; + next(); + } + } + +// Changes auto type string in definitions to the actual type string. + void TextHighlighter::resolveAutos(VariableMap &variableMap, UnorderedBlocks &tokenRange) { + auto curr = m_curr; + std::string UDTName; + for (auto &[name, variables]: variableMap) { + for (auto &[variableName, definitions]: variables) { + for (auto &definition: definitions) { + + if (definition.typeStr == "auto" && (definition.idType == Token::Identifier::IdentifierType::TemplateArgument || definition.idType == Token::Identifier::IdentifierType::FunctionParameter)) { + auto argumentIndex = getArgumentNumber(tokenRange[name].start, definition.tokenIndex); + + if (tokenRange == m_UDTTokenRange || !m_attributeFunctionArgumentType.contains(name) || + m_attributeFunctionArgumentType[name].empty()) { + + auto instances = m_instances[name]; + for (auto instance: instances) { + + if (std::abs(definition.tokenIndex - instance) <= 5) + continue; + Token delimiter; + + if (tokenRange == m_UDTTokenRange) + delimiter = tkn::Operator::BoolLessThan; + else + delimiter = tkn::Separator::LeftParenthesis; + std::string fullName; + std::vector identifiers; + getTokenIdForArgument(instance, argumentIndex,delimiter); + forwardIdentifierName(fullName, identifiers); + + if (fullName.starts_with("Parent.")) { + auto fixedDefinition = setChildrenTypes(); + + if (fixedDefinition.has_value() && + m_UDTDefinitions.contains(fixedDefinition->typeStr)) { + definition.typeStr = fixedDefinition->typeStr; + continue; + } + } else if (fullName.contains(".")) { + Definition definitionTemp; + resolveIdentifierType(definitionTemp,fullName); + definition.typeStr = definitionTemp.typeStr; + } else { + auto typeName = findIdentifierTypeStr(fullName); + definition.typeStr = typeName; + } + } + } else { + UDTName = m_attributeFunctionArgumentType[name]; + if (m_UDTDefinitions.contains(UDTName)) { + definition.typeStr = UDTName; + continue; + } + } + } + } + } + } + m_curr = curr; + } + + void TextHighlighter::fixAutos() { + resolveAutos(m_functionVariables, m_functionTokenRange); + resolveAutos(m_UDTVariables, m_UDTTokenRange); + } + + void TextHighlighter::fixChains() { + + if (!m_scopeChains.empty()) { + for (auto chain: m_scopeChains) { + m_curr = m_startToken + chain; + colorSeparatorScopeChain(); + } + } + + if (!m_memberChains.empty()) { + for (auto chain: m_memberChains) { + m_curr = m_startToken + chain; + colorOperatorDotChain(); + } + } + } + +// Calculates the union of all the UDT and function token ranges +// and inverts the result. + void TextHighlighter::getGlobalTokenRanges() { + std::set ranges; + for (auto [name, range]: m_UDTTokenRange) + ranges.insert(range); + for (auto [name, range]: m_functionTokenRange) + ranges.insert(range); + + if (ranges.empty()) + return; + + auto it = ranges.begin(); + auto next = std::next(it); + while (next != ranges.end()) { + + if (next->start - it->end < 2) { + Interval &range = const_cast(*it); + range.end = next->end; + ranges.erase(next); + next = std::next(it); + } else { + it++; + next = std::next(it); + } + } + m_globalTokenRange = ranges; + invertGlobalTokenRange(); + for (auto tokenRange: m_globalTokenRange) { + + if ((u32) tokenRange.end == m_tokens.size()) { + tokenRange.end -= 1; + m_globalBlocks.insert(tokenRange); + } + } + } + +// Parser labels global variables that are not placed as +// function variables. + void TextHighlighter::fixGlobalVariables() { + m_startToken = m_originalPosition = m_partOriginalPosition = TokenIter(m_tokens.begin(), m_tokens.end()); + for (auto range: m_globalTokenRange) { + auto startToken = TokenIter(m_tokens.begin() + range.start, m_tokens.begin() + range.end); + auto endToken = TokenIter(m_tokens.begin() + range.end, m_tokens.end()); + + for (m_curr = startToken; endToken > m_curr; next()) { + + if (auto identifier = getValue(0); identifier != nullptr) { + auto identifierType = identifier->getType(); + auto identifierName = identifier->get(); + + if (identifierType == IdentifierType::FunctionVariable) { + identifier->setType(IdentifierType::GlobalVariable, true); + setIdentifierColor(-1, IdentifierType::GlobalVariable); + } else if (identifierType == IdentifierType::View) { + identifier->setType(IdentifierType::PlacedVariable, true); + setIdentifierColor(-1,IdentifierType::PlacedVariable); + } else if (identifierType == IdentifierType::Unknown) { + if (std::ranges::find(m_UDTs,identifierName) != m_UDTs.end()) { + identifier->setType(IdentifierType::UDT, true); + setIdentifierColor(-1, IdentifierType::UDT); + } + } + } + } + } + } + +// Only update if needed. Must wait for the parser to finish first. + void TextHighlighter::highlightSourceCode() { + m_wasInterrupted = false; + ON_SCOPE_EXIT { + if (!m_tokenColors.empty()) + m_tokenColors.clear(); + m_runningColorizers--; + if (m_wasInterrupted) { + m_needsToUpdateColors = true; + m_viewPatternEditor->setChangesWereParsed(true); + } else { + m_needsToUpdateColors = false; + m_viewPatternEditor->setChangesWereParsed(false); + } + }; + try { + m_runningColorizers++; + auto preprocessor = patternLanguage->get()->getInternals().preprocessor.get(); + auto parser = patternLanguage->get()->getInternals().parser.get(); + using Types = std::map>; + Types types = parser->getTypes(); + + if (!m_UDTs.empty()) + m_UDTs.clear(); + for (auto &[name, type]: types) + m_UDTs.push_back(name); + m_tokens = preprocessor->getResult(); + if (m_tokens.empty()) + return; + auto lastToken = m_tokens.back(); + m_tokens.push_back(lastToken); + + m_text = m_viewPatternEditor->getTextEditor().PreprocessText( m_viewPatternEditor->getTextEditor().GetText()); + + if (m_text.empty() || m_text == "\n") + return; + loadText(); + + if (!m_globalTokenRange.empty()) + m_globalTokenRange.clear(); + m_globalTokenRange.insert(Interval(0, m_tokens.size()-1)); + + // Namespaces from included files. + m_nameSpaces.clear(); + m_nameSpaces = preprocessor->getNamespaces(); + + if (!m_inheritances.empty()) + m_inheritances.clear(); + + if (!m_namespaceTokenRange.empty()) + m_namespaceTokenRange.clear(); + getAllTokenRanges(IdentifierType::NameSpace); + + if (!m_UDTBlocks.empty()) + m_UDTBlocks.clear(); + + if (!m_UDTTokenRange.empty()) + m_UDTTokenRange.clear(); + getAllTokenRanges(IdentifierType::UDT); + + if (!m_functionBlocks.empty()) + m_functionBlocks.clear(); + + if (!m_functionTokenRange.empty()) + m_functionTokenRange.clear(); + getAllTokenRanges(IdentifierType::Function); + + if (!m_globalBlocks.empty()) + m_globalBlocks.clear(); + getGlobalTokenRanges(); + + fixGlobalVariables(); + if (!m_scopeChains.empty()) + m_scopeChains.clear(); + + if (!m_memberChains.empty()) + m_memberChains.clear(); + + setInitialColors(); + loadInstances(); + + if (!m_attributeFunctionArgumentType.empty()) + m_attributeFunctionArgumentType.clear(); + + getAllTokenRanges(IdentifierType::Attribute); + getDefinitions(); + fixAutos(); + fixChains(); + + m_excludedLocations = preprocessor->getExcludedLocations(); + + colorRemainingIdentifierTokens(); + setRequestedIdentifierColors(); + + m_viewPatternEditor->getTextEditor().SetTimeStamp(1); + m_viewPatternEditor->getTextEditor().ClearErrorMarkers(); + m_compileErrors = patternLanguage->get()->getCompileErrors(); + + if (!m_compileErrors.empty()) + renderErrors(); + else + m_viewPatternEditor->getTextEditor().ClearErrorMarkers(); + } catch (const std::out_of_range &e) { + log::debug("TextHighlighter::highlightSourceCode: Out of range error: {}", e.what()); + m_wasInterrupted = true; + return; + } + return; + } +} \ No newline at end of file diff --git a/plugins/builtin/source/content/themes.cpp b/plugins/builtin/source/content/themes.cpp index b859304b8..f2588a885 100644 --- a/plugins/builtin/source/content/themes.cpp +++ b/plugins/builtin/source/content/themes.cpp @@ -211,33 +211,53 @@ namespace hex::plugin::builtin { } ); } - { const static ThemeManager::ColorMap TextEditorColorMap = { - { "default", u32(TextEditor::PaletteIndex::Default) }, - { "keyword", u32(TextEditor::PaletteIndex::Keyword) }, - { "number", u32(TextEditor::PaletteIndex::Number) }, - { "string", u32(TextEditor::PaletteIndex::String) }, - { "char-literal", u32(TextEditor::PaletteIndex::CharLiteral) }, - { "punctuation", u32(TextEditor::PaletteIndex::Punctuation) }, - { "preprocessor", u32(TextEditor::PaletteIndex::Preprocessor) }, - { "identifier", u32(TextEditor::PaletteIndex::Identifier) }, - { "known-identifier", u32(TextEditor::PaletteIndex::KnownIdentifier) }, - { "preproc-identifier", u32(TextEditor::PaletteIndex::PreprocIdentifier) }, - { "global-doc-comment", u32(TextEditor::PaletteIndex::GlobalDocComment) }, - { "doc-comment", u32(TextEditor::PaletteIndex::DocComment) }, - { "comment", u32(TextEditor::PaletteIndex::Comment) }, - { "multi-line-comment", u32(TextEditor::PaletteIndex::MultiLineComment) }, - { "preprocessor-deactivated", u32(TextEditor::PaletteIndex::PreprocessorDeactivated) }, - { "background", u32(TextEditor::PaletteIndex::Background) }, - { "cursor", u32(TextEditor::PaletteIndex::Cursor) }, - { "selection", u32(TextEditor::PaletteIndex::Selection) }, - { "error-marker", u32(TextEditor::PaletteIndex::ErrorMarker) }, - { "breakpoint", u32(TextEditor::PaletteIndex::Breakpoint) }, - { "line-number", u32(TextEditor::PaletteIndex::LineNumber) }, - { "current-line-fill", u32(TextEditor::PaletteIndex::CurrentLineFill) }, - { "current-line-fill-inactive", u32(TextEditor::PaletteIndex::CurrentLineFillInactive) }, - { "current-line-edge", u32(TextEditor::PaletteIndex::CurrentLineEdge) } + { "attribute", u32(TextEditor::PaletteIndex::Attribute) }, + { "background", u32(TextEditor::PaletteIndex::Background) }, + { "breakpoint", u32(TextEditor::PaletteIndex::Breakpoint) }, + { "calculated-pointer", u32(TextEditor::PaletteIndex::CalculatedPointer) }, + { "char-literal", u32(TextEditor::PaletteIndex::CharLiteral) }, + { "comment", u32(TextEditor::PaletteIndex::Comment) }, + { "current-line-edge", u32(TextEditor::PaletteIndex::CurrentLineEdge) }, + { "current-line-fill", u32(TextEditor::PaletteIndex::CurrentLineFill) }, + { "current-line-fill-inactive", u32(TextEditor::PaletteIndex::CurrentLineFillInactive) }, + { "cursor", u32(TextEditor::PaletteIndex::Cursor) }, + { "debug-text", u32(TextEditor::PaletteIndex::DebugText) }, + { "default", u32(TextEditor::PaletteIndex::Default) }, + { "default-text", u32(TextEditor::PaletteIndex::DefaultText) }, + { "doc-block-comment", u32(TextEditor::PaletteIndex::DocBlockComment) }, + { "doc-comment", u32(TextEditor::PaletteIndex::DocComment) }, + { "doc-global-comment", u32(TextEditor::PaletteIndex::GlobalDocComment) }, + { "error-marker", u32(TextEditor::PaletteIndex::ErrorMarker) }, + { "error-text", u32(TextEditor::PaletteIndex::ErrorText) }, + { "function", u32(TextEditor::PaletteIndex::Function) }, + { "function-parameter", u32(TextEditor::PaletteIndex::FunctionParameter) }, + { "function-variable", u32(TextEditor::PaletteIndex::FunctionVariable) }, + { "global-variable", u32(TextEditor::PaletteIndex::GlobalVariable) }, + { "identifier" , u32(TextEditor::PaletteIndex::Identifier) }, + { "keyword", u32(TextEditor::PaletteIndex::Keyword) }, + { "known-identifier", u32(TextEditor::PaletteIndex::BuiltInType) }, + { "line-number", u32(TextEditor::PaletteIndex::LineNumber) }, + { "local-variable", u32(TextEditor::PaletteIndex::LocalVariable) }, + { "multi-line-comment", u32(TextEditor::PaletteIndex::BlockComment) }, + { "namespace", u32(TextEditor::PaletteIndex::NameSpace) }, + { "number", u32(TextEditor::PaletteIndex::NumericLiteral) }, + { "pattern-variable", u32(TextEditor::PaletteIndex::PatternVariable) }, + { "placed-variable", u32(TextEditor::PaletteIndex::PlacedVariable) }, + { "preprocessor", u32(TextEditor::PaletteIndex::Directive) }, + { "preprocessor-deactivated", u32(TextEditor::PaletteIndex::PreprocessorDeactivated) }, + { "preproc-identifier", u32(TextEditor::PaletteIndex::PreprocIdentifier) }, + { "punctuation", u32(TextEditor::PaletteIndex::Operator) }, + { "selection", u32(TextEditor::PaletteIndex::Selection) }, + { "separator", u32(TextEditor::PaletteIndex::Separator) }, + { "string", u32(TextEditor::PaletteIndex::StringLiteral) }, + { "template-variable", u32(TextEditor::PaletteIndex::TemplateArgument) }, + { "typedef", u32(TextEditor::PaletteIndex::TypeDef) }, + { "unknown-identifier", u32(TextEditor::PaletteIndex::UnkIdentifier) }, + { "user-defined-type", u32(TextEditor::PaletteIndex::UserDefinedType) }, + { "view", u32(TextEditor::PaletteIndex::View) }, + { "warning-text", u32(TextEditor::PaletteIndex::WarningText) } }; ThemeManager::addThemeHandler("text-editor", TextEditorColorMap, diff --git a/plugins/builtin/source/content/views/view_pattern_editor.cpp b/plugins/builtin/source/content/views/view_pattern_editor.cpp index e9963d764..efd58a69e 100644 --- a/plugins/builtin/source/content/views/view_pattern_editor.cpp +++ b/plugins/builtin/source/content/views/view_pattern_editor.cpp @@ -8,8 +8,7 @@ #include #include -#include -#include +#include #include #include @@ -47,7 +46,7 @@ namespace hex::plugin::builtin { static TextEditor::LanguageDefinition langDef; if (!initialized) { constexpr static std::array keywords = { - "using", "struct", "union", "enum", "bitfield", "be", "le", "if", "else", "match", "false", "true", "this", "parent", "addressof", "sizeof", "typenameof", "$", "while", "for", "fn", "return", "break", "continue", "namespace", "in", "out", "ref", "null", "const", "unsigned", "signed", "try", "catch", "import", "as", "from" + "using", "struct", "union", "enum", "bitfield", "be", "le", "if", "else", "match", "false", "true", "this", "parent", "addressof", "sizeof", "typenameof", "while", "for", "fn", "return", "break", "continue", "namespace", "in", "out", "ref", "null", "const", "unsigned", "signed", "try", "catch", "import", "as", "from" }; for (auto &k : keywords) langDef.mKeywords.insert(k); @@ -60,8 +59,15 @@ namespace hex::plugin::builtin { id.mDeclaration = ""; langDef.mIdentifiers.insert(std::make_pair(std::string(name), id)); } - - langDef.mTokenize = [](const char *inBegin, const char *inEnd, const char *&outBegin, const char *&outEnd, TextEditor::PaletteIndex &paletteIndex) -> bool { + constexpr static std::array directives = { + "include", "define", "ifdef", "ifndef", "endif", "undef", "pragma", "error" + }; + for (const auto name : directives) { + TextEditor::Identifier id; + id.mDeclaration = ""; + langDef.mPreprocIdentifiers.insert(std::make_pair(std::string(name), id)); + } + langDef.mTokenize = [](std::string::const_iterator inBegin, std::string::const_iterator inEnd, std::string::const_iterator &outBegin, std::string::const_iterator &outEnd, TextEditor::PaletteIndex &paletteIndex) -> bool { paletteIndex = TextEditor::PaletteIndex::Max; while (inBegin < inEnd && isascii(*inBegin) && std::isblank(*inBegin)) @@ -74,13 +80,16 @@ namespace hex::plugin::builtin { } else if (TokenizeCStyleIdentifier(inBegin, inEnd, outBegin, outEnd)) { paletteIndex = TextEditor::PaletteIndex::Identifier; } else if (TokenizeCStyleNumber(inBegin, inEnd, outBegin, outEnd)) { - paletteIndex = TextEditor::PaletteIndex::Number; + paletteIndex = TextEditor::PaletteIndex::NumericLiteral; } else if (TokenizeCStyleCharacterLiteral(inBegin, inEnd, outBegin, outEnd)) { paletteIndex = TextEditor::PaletteIndex::CharLiteral; } else if (TokenizeCStyleString(inBegin, inEnd, outBegin, outEnd)) { - paletteIndex = TextEditor::PaletteIndex::String; + paletteIndex = TextEditor::PaletteIndex::StringLiteral; + } else if (TokenizeCStyleSeparator(inBegin, inEnd, outBegin, outEnd)) { + paletteIndex = TextEditor::PaletteIndex::Separator; + } else if (TokenizeCStyleOperator(inBegin, inEnd, outBegin, outEnd)) { + paletteIndex = TextEditor::PaletteIndex::Operator; } - return paletteIndex != TextEditor::PaletteIndex::Max; }; @@ -92,8 +101,9 @@ namespace hex::plugin::builtin { langDef.mAutoIndentation = true; langDef.mPreprocChar = '#'; - langDef.mGlobalDocComment = "/*!"; - langDef.mDocComment = "/**"; + langDef.mGlobalDocComment = "/*!"; + langDef.mBlockDocComment = "/**"; + langDef.mDocComment = "///"; langDef.mName = "Pattern Language"; @@ -107,15 +117,16 @@ namespace hex::plugin::builtin { static bool initialized = false; static TextEditor::LanguageDefinition langDef; if (!initialized) { - langDef.mTokenize = [](const char *inBegin, const char *inEnd, const char *&outBegin, const char *&outEnd, TextEditor::PaletteIndex &paletteIndex) -> bool { - if (std::string_view(inBegin).starts_with("D: ")) - paletteIndex = TextEditor::PaletteIndex::Comment; - else if (std::string_view(inBegin).starts_with("I: ")) - paletteIndex = TextEditor::PaletteIndex::Default; - else if (std::string_view(inBegin).starts_with("W: ")) - paletteIndex = TextEditor::PaletteIndex::Preprocessor; - else if (std::string_view(inBegin).starts_with("E: ")) - paletteIndex = TextEditor::PaletteIndex::ErrorMarker; + langDef.mTokenize = [](std::string::const_iterator inBegin, std::string::const_iterator inEnd, std::string::const_iterator &outBegin, std::string::const_iterator &outEnd, TextEditor::PaletteIndex &paletteIndex) -> bool { + std::string_view inView(inBegin, inEnd); + if (inView.starts_with("D: ")) + paletteIndex = TextEditor::PaletteIndex::DefaultText; + else if (inView.starts_with("I: ")) + paletteIndex = TextEditor::PaletteIndex::DebugText; + else if (inView.starts_with("W: ")) + paletteIndex = TextEditor::PaletteIndex::WarningText; + else if (inView.starts_with("E: ")) + paletteIndex = TextEditor::PaletteIndex::ErrorText; else paletteIndex = TextEditor::PaletteIndex::Max; @@ -583,14 +594,18 @@ namespace hex::plugin::builtin { } } - if (m_textEditor.get(provider).IsTextChanged() && !m_hasUnevaluatedChanges.get(provider)) { - m_hasUnevaluatedChanges.get(provider) = true; + if (m_textEditor.get(provider).IsTextChanged()) { + m_textEditor.get(provider).SetTextChanged(false); + if (!m_hasUnevaluatedChanges.get(provider) ) { + m_hasUnevaluatedChanges.get(provider) = true; + m_changesWereParsed = false; + } m_lastEditorChangeTime = std::chrono::steady_clock::now(); ImHexApi::Provider::markDirty(); } - if (m_hasUnevaluatedChanges.get(provider) && m_runningEvaluators == 0 && m_runningParsers == 0) { - if ((std::chrono::steady_clock::now() - m_lastEditorChangeTime) > std::chrono::seconds(1LL)) { + if (m_hasUnevaluatedChanges.get(provider) && m_runningEvaluators == 0 && m_runningParsers == 0 && + (std::chrono::steady_clock::now() - m_lastEditorChangeTime) > std::chrono::seconds(1ll)) { auto code = m_textEditor.get(provider).GetText(); EventPatternEditorChanged::post(code); @@ -602,13 +617,21 @@ namespace hex::plugin::builtin { m_triggerAutoEvaluate = true; }); m_hasUnevaluatedChanges.get(provider) = false; - m_textEditor.get(provider).SetTextChanged(); - } } if (m_triggerAutoEvaluate.exchange(false)) { this->evaluatePattern(m_textEditor.get(provider).GetText(), provider); } + if (m_textHighlighter.m_needsToUpdateColors && m_changesWereParsed && (m_runningParsers + m_runningEvaluators == 0)) { + TaskHolder taskHolder; + if (m_textHighlighter.getRunningColorizers() == 0) { + m_textHighlighter.m_needsToUpdateColors = false; + m_changesWereParsed = false; + taskHolder = TaskManager::createBackgroundTask("HighlightSourceCode", [this](auto &) { m_textHighlighter.highlightSourceCode(); }); + } else { + taskHolder.interrupt(); + } + } } if (m_dangerousFunctionCalled && !ImGui::IsPopupOpen(ImGuiID(0), ImGuiPopupFlags_AnyPopup)) { @@ -1228,7 +1251,7 @@ namespace hex::plugin::builtin { const std::string label { "##" + name }; if (pl::core::Token::isSigned(variable.type)) { - i64 value = i64(hex::get_or(variable.value, 0)); + i64 value = i64(hex::get_or(variable.value, 0ll)); if (ImGui::InputScalar(label.c_str(), ImGuiDataType_S64, &value)) m_hasUnevaluatedChanges.get(provider) = true; variable.value = i128(value); @@ -1495,7 +1518,7 @@ namespace hex::plugin::builtin { }; TextEditor::ErrorMarkers errorMarkers; - if (!(*m_callStack)->empty()) { + if (*m_callStack != nullptr && !(*m_callStack)->empty()) { for (const auto &frame : **m_callStack | std::views::reverse) { auto location = frame.node->getLocation(); std::string message; @@ -1701,7 +1724,7 @@ namespace hex::plugin::builtin { const bool shiftHeld = ImGui::GetIO().KeyShift; ImGui::ColorButton(pattern->getVariableName().c_str(), ImColor(pattern->getColor()), ImGuiColorEditFlags_AlphaOpaque); ImGui::SameLine(0, 10); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "{} ", pattern->getFormattedName()); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "{} ", pattern->getFormattedName()); ImGui::SameLine(0, 5); ImGuiExt::TextFormatted("{}", pattern->getDisplayName()); ImGui::SameLine(); @@ -1844,6 +1867,7 @@ namespace hex::plugin::builtin { m_textEditor.get(provider).SetText(code, true); m_sourceCode.get(provider) = code; + m_textHighlighter.m_needsToUpdateColors = false; TaskManager::createBackgroundTask("hex.builtin.task.parsing_pattern", [this, code, provider](auto&) { this->parsePattern(code, provider); }); } } @@ -1885,6 +1909,8 @@ namespace hex::plugin::builtin { patternVariables = std::move(oldPatternVariables); } + m_textHighlighter.m_needsToUpdateColors = true; + m_changesWereParsed = true; m_runningParsers -= 1; } @@ -2039,6 +2065,7 @@ namespace hex::plugin::builtin { m_textEditor.get(provider).SetText(code); m_sourceCode.get(provider) = code; m_hasUnevaluatedChanges.get(provider) = true; + m_textHighlighter.m_needsToUpdateColors = false; }); ContentRegistry::Settings::onChange("hex.builtin.setting.general", "hex.builtin.setting.general.sync_pattern_source", [this](const ContentRegistry::Settings::SettingsValue &value) { @@ -2098,7 +2125,9 @@ namespace hex::plugin::builtin { m_cursorNeedsUpdate.get(newProvider) = true; m_consoleCursorNeedsUpdate.get(newProvider) = true; m_textEditor.get(newProvider).SetTextChanged(false); + m_hasUnevaluatedChanges.get(newProvider) = true; } + m_textHighlighter.m_needsToUpdateColors = false; }); @@ -2327,7 +2356,7 @@ namespace hex::plugin::builtin { m_textEditor.get(provider).SetText(sourceCode); m_hasUnevaluatedChanges.get(provider) = true; - + m_textHighlighter.m_needsToUpdateColors = false; return true; }, .store = [this](prv::Provider *provider, const std::fs::path &basePath, const Tar &tar) { diff --git a/plugins/ui/source/ui/pattern_drawer.cpp b/plugins/ui/source/ui/pattern_drawer.cpp index c976e643e..d8c6d4c0a 100644 --- a/plugins/ui/source/ui/pattern_drawer.cpp +++ b/plugins/ui/source/ui/pattern_drawer.cpp @@ -523,7 +523,7 @@ namespace hex::ui { // Draw type column ImGui::TableNextColumn(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "{}", pattern.getFormattedName().empty() ? pattern.getTypeName() : pattern.getFormattedName()); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "{}", pattern.getFormattedName().empty() ? pattern.getTypeName() : pattern.getFormattedName()); } void PatternDrawer::closeTreeNode(bool inlined) const { @@ -550,19 +550,19 @@ namespace hex::ui { if (dynamic_cast(&pattern) != nullptr) { ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::Keyword)], "signed"); ImGui::SameLine(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], pattern.getBitSize() == 1 ? "bit" : "bits"); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], pattern.getBitSize() == 1 ? "bit" : "bits"); } else if (dynamic_cast(&pattern) != nullptr) { ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::Keyword)], "enum"); ImGui::SameLine(); ImGui::TextUnformatted(pattern.getTypeName().c_str()); } else if (dynamic_cast(&pattern) != nullptr) { - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "bool"); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "bool"); ImGui::SameLine(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "bit"); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "bit"); } else { ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::Keyword)], "unsigned"); ImGui::SameLine(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], pattern.getBitSize() == 1 ? "bit" : "bits"); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], pattern.getBitSize() == 1 ? "bit" : "bits"); } if (!this->isEditingPattern(pattern)) { @@ -792,7 +792,7 @@ namespace hex::ui { drawOffsetColumns(pattern); drawSizeColumn(pattern); ImGui::TableNextColumn(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "{}", pattern.getFormattedName()); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "{}", pattern.getFormattedName()); drawValueColumn(pattern); drawCommentColumn(pattern); } @@ -1038,12 +1038,12 @@ namespace hex::ui { drawSizeColumn(pattern); ImGui::TableNextColumn(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "{0}", pattern.getTypeName()); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "{0}", pattern.getTypeName()); ImGui::SameLine(0, 0); ImGui::TextUnformatted("["); ImGui::SameLine(0, 0); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::Number)], "{0}", iterable.getEntryCount()); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::NumericLiteral)], "{0}", iterable.getEntryCount()); ImGui::SameLine(0, 0); ImGui::TextUnformatted("]"); @@ -1106,12 +1106,12 @@ namespace hex::ui { ImGui::TableNextColumn(); ImGuiExt::TextFormatted("{0} {1}", chunkSize, chunkSize == 1 ? "byte" : "bytes"); ImGui::TableNextColumn(); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::KnownIdentifier)], "{0}", pattern.getTypeName()); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::BuiltInType)], "{0}", pattern.getTypeName()); ImGui::SameLine(0, 0); ImGui::TextUnformatted("["); ImGui::SameLine(0, 0); - ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::Number)], "{0}", endIndex - i); + ImGuiExt::TextFormattedColored(TextEditor::GetPalette()[u32(TextEditor::PaletteIndex::NumericLiteral)], "{0}", endIndex - i); ImGui::SameLine(0, 0); ImGui::TextUnformatted("]");