diff --git a/cmake/build_helpers.cmake b/cmake/build_helpers.cmake
index 70b05ddb4..5fc516a3a 100644
--- a/cmake/build_helpers.cmake
+++ b/cmake/build_helpers.cmake
@@ -606,17 +606,15 @@ function(downloadImHexPatternsFiles dest)
         install(CODE "set(imhex_patterns_SOURCE_DIR \"${imhex_patterns_SOURCE_DIR}\")")
         install(CODE [[
             message(STATUS "Downloading ImHex patterns from branch '${PATTERNS_BRANCH}'...")
-            if (EXISTS "${imhex_patterns_SOURCE_DIR}")
-                file(REMOVE_RECURSE "${imhex_patterns_SOURCE_DIR}")
-            else ()
+            if (NOT EXISTS "${imhex_patterns_SOURCE_DIR}")
                 file(MAKE_DIRECTORY "${imhex_patterns_SOURCE_DIR}")
+                
+                execute_process(
+                    COMMAND
+                        git clone --recurse-submodules --branch ${PATTERNS_BRANCH} https://github.com/WerWolv/ImHex-Patterns.git "${imhex_patterns_SOURCE_DIR}"
+                    COMMAND_ERROR_IS_FATAL ANY
+                )
             endif()
-
-            execute_process(
-                COMMAND
-                    git clone --recurse-submodules --branch ${PATTERNS_BRANCH} https://github.com/WerWolv/ImHex-Patterns.git "${imhex_patterns_SOURCE_DIR}"
-                COMMAND_ERROR_IS_FATAL ANY
-            )
         ]])
     else ()
         set(imhex_patterns_SOURCE_DIR "")