feat: Added stacktrace printing on crashes (#892)

* feat: Added simple stack trace output

* feat: Added backtrace wrapper

* build: Added libbacktrace include dirs to build

* build: Fixed libbacktrace variables

* build: More backtrace variable fixes

* build: Try to find backtrace include

* build: Get backtrace header from cmake script

* feat: Added backtrace support for execinfo

* feat: Added support for StackWalk on Windows

lib/libimhex/CMakeLists.txt

@@ -29,6 +29,7 @@ set(LIBIMHEX_SOURCES
     source/helpers/patches.cpp
     source/helpers/encoding_file.cpp
     source/helpers/logger.cpp
+    source/helpers/stacktrace.cpp
     source/helpers/tar.cpp
 
     source/providers/provider.cpp
@@ -59,7 +60,7 @@ set_target_properties(libimhex PROPERTIES POSITION_INDEPENDENT_CODE ON)
 setupCompilerWarnings(libimhex)
 
 
-target_include_directories(libimhex PUBLIC include ${XDGPP_INCLUDE_DIRS} ${MBEDTLS_INCLUDE_DIRS} ${CAPSTONE_INCLUDE_DIRS} ${MAGIC_INCLUDE_DIRS} ${LLVM_INCLUDE_DIRS} ${FMT_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS} ${YARA_INCLUDE_DIRS})
+target_include_directories(libimhex PUBLIC include ${XDGPP_INCLUDE_DIRS} ${MBEDTLS_INCLUDE_DIRS} ${CAPSTONE_INCLUDE_DIRS} ${MAGIC_INCLUDE_DIRS} ${LLVM_INCLUDE_DIRS} ${FMT_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS} ${YARA_INCLUDE_DIRS} ${LIBBACKTRACE_INCLUDE_DIRS})
 target_link_directories(libimhex PUBLIC ${MBEDTLS_LIBRARY_DIR} ${CAPSTONE_LIBRARY_DIRS} ${MAGIC_LIBRARY_DIRS})
 
 if (WIN32)
@@ -70,4 +71,4 @@ elseif (APPLE)
 endif ()
 
 target_link_libraries(libimhex PRIVATE ${FMT_LIBRARIES})
-target_link_libraries(libimhex PUBLIC dl imgui ${NFD_LIBRARIES} magic ${CAPSTONE_LIBRARIES} LLVMDemangle microtar ${NLOHMANN_JSON_LIBRARIES} ${YARA_LIBRARIES} ${LIBCURL_LIBRARIES} ${MBEDTLS_LIBRARIES} libpl intervaltree)
+target_link_libraries(libimhex PUBLIC dl imgui ${NFD_LIBRARIES} magic ${CAPSTONE_LIBRARIES} LLVMDemangle microtar ${NLOHMANN_JSON_LIBRARIES} ${YARA_LIBRARIES} ${LIBCURL_LIBRARIES} ${MBEDTLS_LIBRARIES} ${LIBBACKTRACE_LIBRARIES} libpl intervaltree)

lib/libimhex/include/hex/helpers/stacktrace.hpp

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <hex.hpp>
+
+#include <string>
+#include <vector>
+
+namespace hex::stacktrace {
+
+    struct StackFrame {
+        std::string file;
+        std::string function;
+        u32 line;
+    };
+
+    void initialize();
+
+    std::vector<StackFrame> getStackTrace(); 
+
+}

lib/libimhex/source/helpers/stacktrace.cpp

@@ -0,0 +1,176 @@
+#include <hex/helpers/stacktrace.hpp>
+#include <hex/helpers/fs.hpp>
+
+#if defined(OS_WINDOWS)
+
+    #include <windows.h>
+    #include <dbghelp.h>
+    #include <llvm/Demangle/Demangle.h>
+
+    namespace hex::stacktrace {
+
+        void initialize() {
+
+        }
+
+        std::vector<StackFrame> getStackTrace() {
+            std::vector<StackFrame> stackTrace;
+
+            HANDLE process = GetCurrentProcess();
+            HANDLE thread = GetCurrentThread();
+
+            CONTEXT context;
+            memset(&context, 0, sizeof(CONTEXT));
+            context.ContextFlags = CONTEXT_FULL;
+            RtlCaptureContext(&context);
+
+            SymSetOptions(SYMOPT_UNDNAME | SYMOPT_LOAD_LINES);
+            SymInitialize(process, nullptr, true);
+
+            DWORD image;
+            STACKFRAME64 stackFrame;
+            ZeroMemory(&stackFrame, sizeof(STACKFRAME64));
+
+            image = IMAGE_FILE_MACHINE_AMD64;
+            stackFrame.AddrPC.Offset = context.Rip;
+            stackFrame.AddrPC.Mode = AddrModeFlat;
+            stackFrame.AddrFrame.Offset = context.Rsp;
+            stackFrame.AddrFrame.Mode = AddrModeFlat;
+            stackFrame.AddrStack.Offset = context.Rsp;
+            stackFrame.AddrStack.Mode = AddrModeFlat;
+
+            while (true) {
+
+                if (!StackWalk64(
+                        image, process, thread,
+                        &stackFrame, &context, nullptr,
+                        SymFunctionTableAccess64, SymGetModuleBase64, nullptr))
+                    break;
+
+                char buffer[sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)];
+                auto symbol = (PSYMBOL_INFO)buffer;
+                symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+                symbol->MaxNameLen = MAX_SYM_NAME;
+
+                DWORD64 displacementSymbol = 0;
+                const char* symbolName;
+                if (SymFromAddr(process, stackFrame.AddrPC.Offset, &displacementSymbol, symbol)) {
+                    symbolName = symbol->Name;
+                } else {
+                    symbolName = "??";
+                }
+
+                IMAGEHLP_LINE64 line;
+                line.SizeOfStruct = sizeof(IMAGEHLP_LINE64);
+
+                DWORD displacementLine = 0;
+                u32 lineNumber;
+                const char* fileName;
+
+                if (SymGetLineFromAddr64(process, stackFrame.AddrPC.Offset, &displacementLine, &line)) {
+                    lineNumber = line.LineNumber;
+                    fileName = line.FileName;
+                } else {
+                    lineNumber = 0;
+                    fileName = "??";
+                }
+
+                std::string demangledName;
+                if (auto variant1 = llvm::demangle(symbolName); variant1 != symbolName)
+                    demangledName = variant1;
+                else if (auto variant2 = llvm::demangle(std::string("_") + symbolName); variant2 != std::string("_") + symbolName)
+                    demangledName = variant2;
+                else
+                    demangledName = symbolName;
+
+                stackTrace.push_back(StackFrame { fileName, demangledName, lineNumber });
+
+            }
+
+            SymCleanup(process);
+
+            return stackTrace;
+        }
+
+    }
+
+#elif defined(HEX_HAS_EXECINFO) && __has_include(BACKTRACE_HEADER)
+
+    #include BACKTRACE_HEADER
+    #include <llvm/Demangle/Demangle.h>
+    #include <hex/helpers/logger.hpp>
+    #include <hex/helpers/utils.hpp>
+
+    namespace hex::stacktrace {
+
+        void initialize() {
+
+        }
+
+        std::vector<StackFrame> getStackTrace() {
+            static std::vector<StackFrame> result;
+
+            std::array<void*, 128> addresses;
+            auto count = backtrace(addresses.data(), addresses.size());
+            auto functions = backtrace_symbols(addresses.data(), count);
+
+            for (i32 i = 0; i < count; i++)
+                result.push_back(StackFrame { "", functions[i], 0 });
+
+            return result;
+        }
+
+    }
+
+#elif defined(HEX_HAS_BACKTRACE) && __has_include(BACKTRACE_HEADER)
+
+    #include BACKTRACE_HEADER
+    #include <llvm/Demangle/Demangle.h>
+    #include <hex/helpers/logger.hpp>
+    #include <hex/helpers/utils.hpp>
+
+    namespace hex::stacktrace {
+
+        static struct backtrace_state *s_backtraceState;
+
+
+        void initialize() {
+            if (auto executablePath = fs::getExecutablePath(); executablePath.has_value()) {
+                static std::string path = executablePath->string();
+                s_backtraceState = backtrace_create_state(path.c_str(), 1, [](void *, const char *msg, int) { log::error("{}", msg); }, nullptr);
+            }
+        }
+
+        std::vector<StackFrame> getStackTrace() {
+            static std::vector<StackFrame> result;
+
+            result.clear();
+            if (s_backtraceState != nullptr) {
+                backtrace_full(s_backtraceState, 0, [](void *, uintptr_t, const char *fileName, int lineNumber, const char *function) -> int {
+                    if (fileName == nullptr)
+                        fileName = "??";
+                    if (function == nullptr)
+                        function = "??";
+
+                    result.push_back(StackFrame { std::fs::path(fileName).filename().string(), llvm::demangle(function), u32(lineNumber) });
+
+                    return 0;
+                }, nullptr, nullptr);
+
+            }
+
+            return result;
+        }
+
+    }
+
+#else
+
+    namespace hex::stacktrace {
+
+        void initialize() { }
+        std::vector<StackFrame> getStackTrace() { return { }; }
+
+    }
+    
+#endif