core: add stack smashing protection and Win32 bt

Add basic stack smashing protections and enable it for all engine
functions. Add proper backtrace support to the Windows build.

Signed-off-by: Danny Holman <dholman@gymli.org>

1 files changed, 36 insertions, 2 deletions

diff --git a/core/abort.c b/core/abort.c
index edc7d29..292c1a1 100644
--- a/core/abort.c
+++ b/core/abort.c
@@ -7,8 +7,26 @@
 
 #ifdef _WIN32
 
+#include <windows.h>
+#include <dbghelp.h>
+
 void _stack_trace(void) {
-        log_output(LOG_ERROR, "Stack tracing is not supported on Windows");
+        void* buffer[MAX_TRACE_ITEMS];
+        HANDLE process = GetCurrentProcess();
+
+        SymInitialize(process, NULL, TRUE);
+        int num_links = CaptureStackBackTrace(0, MAX_TRACE_ITEMS, buffer, NULL);
+        SYMBOL_INFO *symbol = (SYMBOL_INFO*)rune_alloc(sizeof(SYMBOL_INFO) + 256 * sizeof(char));
+        symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+        symbol->MaxNameLen = 255;
+
+        for (int i = 0; i < num_links; i++) {
+                SymFromAddr(process, (DWORD64)(buffer[i]), 0, symbol);
+                log_output(LOG_INFO, "#%d: %s", i, symbol->Name, symbol->Address);
+        }
+
+        rune_free(symbol);
+        SymCleanup(process);
 }
 
 #else
@@ -28,9 +46,25 @@ void _stack_trace(void) {
 
 #endif
 
-void rune_abort(void) {
+NORET void rune_abort(void) {
         log_output(LOG_INFO, "Abort called, printing stack trace");
         _stack_trace();
         rune_exit();
         exit(-1);
 }
+
+#ifdef MSVC
+
+NORET void __security_error_handler(void) {
+        log_output(LOG_FATAL, "Stack smashing detected in engine code");
+        rune_abort();
+}
+
+#else
+
+NORET void __stack_chk_fail(void) {
+        log_output(LOG_FATAL, "Stack smashing detected in engine code");
+        rune_abort();
+}
+
+#endif