core: add stack smashing protection and Win32 bt

Add basic stack smashing protections and enable it for all engine
functions. Add proper backtrace support to the Windows build.

Signed-off-by: Danny Holman <dholman@gymli.org>

2 files changed, 47 insertions, 5 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 91f8a98..5bf751b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -57,13 +57,21 @@ list(APPEND LINK_LIBS
 
 if (WIN32)
         find_package(dlfcn-win32 REQUIRED)
+        find_package(DBGHELP REQUIRED)
         set(GLFW_LIBRARIES glfw3dll)
         set(DX12_LIBRARIES d3d12.lib dxgi.lib dxguid.lib)
         set(DL_LIBRARIES dlfcn-win32::dl)
-        list(APPEND LINK_LIBS ${DX12_LIBRARIES} ${DL_LIBRARIES})
-else()
+        list(APPEND LINK_LIBS ${DX12_LIBRARIES} ${DL_LIBRARIES} ${DBGHELP_LIBRARY})
+else ()
         set(GLFW_LIBRARIES glfw)
-endif()
+endif ()
+
+include(EnableCFLAG)
+if (MSVC)
+        enable_c_compiler_flag_if_supported("/GS")
+else ()
+        enable_c_compiler_flag_if_supported("-fstack-protector-all")
+endif ()
 
 list(APPEND LINK_LIBS ${GLFW_LIBRARIES})
 
diff --git a/core/abort.c b/core/abort.c
index edc7d29..292c1a1 100644
--- a/core/abort.c
+++ b/core/abort.c
@@ -7,8 +7,26 @@
 
 #ifdef _WIN32
 
+#include <windows.h>
+#include <dbghelp.h>
+
 void _stack_trace(void) {
-        log_output(LOG_ERROR, "Stack tracing is not supported on Windows");
+        void* buffer[MAX_TRACE_ITEMS];
+        HANDLE process = GetCurrentProcess();
+
+        SymInitialize(process, NULL, TRUE);
+        int num_links = CaptureStackBackTrace(0, MAX_TRACE_ITEMS, buffer, NULL);
+        SYMBOL_INFO *symbol = (SYMBOL_INFO*)rune_alloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char));
+        symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+        symbol->MaxNameLen = 255;
+
+        for (int i = 0; i < num_links; i++) {
+                SymFromAddr(process, (DWORD64)(buffer[i]), 0, symbol);
+                log_output(LOG_INFO, "#%d: %s", i, symbol->Name, symbol->Address);
+        }
+
+        rune_free(symbol);
+        SymCleanup(process);
 }
 
 #else
@@ -28,9 +46,25 @@ void _stack_trace(void) {
 
 #endif
 
-void rune_abort(void) {
+NORET void rune_abort(void) {
         log_output(LOG_INFO, "Abort called, printing stack trace");
         _stack_trace();
         rune_exit();
         exit(-1);
 }
+
+#ifdef MSVC
+
+NORET void __security_error_handler(void) {
+        log_output(LOG_FATAL, "Stack smashing detected in engine code");
+        rune_abort();
+}
+
+#else
+
+NORET void __stack_chk_fail(void) {
+        log_output(LOG_FATAL, "Stack smashing detected in engine code");
+        rune_abort();
+}
+
+#endif