Add timeout to debugger captures

ext/compatibility.h

@@ -686,6 +686,8 @@ static inline zend_string *zend_ini_str(const char *name, size_t name_length, bo
     return return_value;
 }
 
+#define tsrm_is_managed_thread() (tsrm_get_ls_cache() != NULL)
+
 #define zend_zval_value_name zend_zval_type_name
 
 #define Z_PARAM_ZVAL_OR_NULL(dest) Z_PARAM_ZVAL_EX(dest, 1, 0)

ext/crashtracking_frames.c

@@ -24,6 +24,12 @@ static ddog_CharSlice dd_validate_zstr(zend_string *str) {
 }
 
 static void dd_frames_callback(void (*emit_frame)(const ddog_crasht_RuntimeStackFrame *)) {
+#ifdef ZTS
+    if (!tsrm_is_managed_thread()) {
+        return;
+    }
+#endif
+
     zend_execute_data *call;
 #if PHP_VERSION_ID >= 80400
     zend_execute_data *last_call = NULL;

ext/remote_config.c

@@ -7,7 +7,8 @@
 #include "threads.h"
 #include <tracer/tracer_api.h>
 #ifndef _WIN32
-#include <signal.h>
+#include <sys/time.h>
+#include <tracer/ddtrace_globals.h>
 #endif
 
 #if PHP_VERSION_ID < 70100
@@ -65,6 +66,50 @@ void datadog_check_for_new_config_now(void) {
 static void dd_sigvtalarm_handler(int signal, siginfo_t *siginfo, void *ctx) {
     UNUSED(signal, siginfo, ctx);
     datadog_set_all_thread_vm_interrupt();
+
+#if defined(__linux__) && defined(ZTS)
+    if (!tsrm_is_managed_thread()) {
+        return;
+    }
+#endif
+
+    uint64_t now_ns = 0;
+#if !defined(__linux__) && defined(ZTS)
+    // On macOS ZTS, setitimer is per-process; the signal may land on any thread - iterate all threads to check for expirations
+    uint64_t next_deadline = ~0ull;
+    tsrm_mutex_lock(datadog_threads_mutex);
+    void *TSRMLS_CACHE;
+    ZEND_HASH_FOREACH_PTR(&datadog_tls_bases, TSRMLS_CACHE) {
+#endif
+    // On Linux the signal gets delivered to the thread that set the timer, so we don't need to iterate all threads
+    uint64_t deadline = DDTRACE_G(capture_deadline_ns);
+    if (deadline) {
+        if (!now_ns) {
+            struct timespec now;
+            clock_gettime(CLOCK_THREAD_CPUTIME_ID, &now);
+            now_ns = (uint64_t)now.tv_sec * 1000000000ULL + (uint64_t)now.tv_nsec;
+        }
+        if (now_ns >= deadline) {
+            DDTRACE_G(debugger_capture_timed_out) = 1;
+        }
+#if !defined(__linux__) && defined(ZTS)
+        else {
+            next_deadline = MIN(deadline, next_deadline);
+        }
+#endif
+    }
+#if !defined(__linux__) && defined(ZTS)
+    } ZEND_HASH_FOREACH_END();
+    if (next_deadline != ~0ull) { // re-arm the timer, for ZTS concurrency
+        uint64_t usec = (next_deadline - now_ns) / 1000ull;
+        struct itimerval it = {
+            .it_value    = { .tv_sec = usec / 1000000, .tv_usec = usec % 1000000 },
+            .it_interval = { 0, 0 },
+        };
+        setitimer(ITIMER_VIRTUAL, &it, NULL);
+    }
+    tsrm_mutex_unlock(datadog_threads_mutex);
+#endif
 }
 #endif
 

metadata/supported-configurations.json

@@ -333,6 +333,13 @@
         "default": "http://localhost:8125"
       }
     ],
+    "DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS": [
+      {
+        "implementation": "A",
+        "type": "int",
+        "default": "15"
+      }
+    ],
     "DD_DYNAMIC_INSTRUMENTATION_ENABLED": [
       {
         "implementation": "A",

tests/ext/live-debugger/debugger_log_probe.phpt

@@ -8,6 +8,7 @@ DD_TRACE_AGENT_PORT=80
 DD_TRACE_GENERATE_ROOT_SPAN=0
 DD_DYNAMIC_INSTRUMENTATION_ENABLED=1
 DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS=0.1
+DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS=5000
 --INI--
 datadog.trace.agent_test_session_token=live-debugger/log_probe
 --FILE--

tests/ext/live-debugger/debugger_log_probe_capture_timeout.phpt

@@ -0,0 +1,67 @@
+--TEST--
+Live debugger log probe capture timeout with large data structure
+--SKIPIF--
+<?php include __DIR__ . '/../includes/skipif_no_dev_env.inc'; ?>
+--ENV--
+DD_AGENT_HOST=request-replayer
+DD_TRACE_AGENT_PORT=80
+DD_TRACE_GENERATE_ROOT_SPAN=0
+DD_DYNAMIC_INSTRUMENTATION_ENABLED=1
+DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS=0.1
+DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS=1
+--INI--
+datadog.trace.agent_test_session_token=live-debugger/log_probe_capture_timeout
+--FILE--
+<?php
+
+require __DIR__ . "/live_debugger.inc";
+
+reset_request_replayer();
+
+function large_capture($huge_array) {}
+
+await_probe_installation(function() {
+    build_log_probe([
+        "where" => ["methodName" => "large_capture"],
+        "captureSnapshot" => true,
+        "segments" => [["str" => "capture timeout test"]],
+    ]);
+    \DDTrace\start_span();
+});
+
+// 3-level array: 10 outer x 100 mid x 100 inner strings (100-char each)
+// ~100,000 capture operations: reliably exceeds the 1ms CPU-time timeout
+$data = [];
+for ($i = 0; $i < 99; $i++) {
+    $data[] = array_fill(0, 10, array_fill(0, 100, str_repeat('x', 100)));
+}
+$last = array_fill(0, 99, str_repeat('x', 100));
+$last[] = 'LAST_SENTINEL';
+$data[] = $last;
+
+large_capture($data);
+
+$dlr = new DebuggerLogReplayer;
+$log = $dlr->waitForDebuggerDataAndReplay();
+$captures = json_decode($log["body"], true)[0]["debugger"]["snapshot"]["captures"];
+$captures_json = json_encode($captures);
+
+// Snapshot was delivered with some captured data
+var_dump(!empty($captures));
+
+// Timeout reason must appear somewhere in the captured data
+var_dump(strpos($captures_json, '"timeout"') !== false);
+
+// The last element must NOT have been captured before the timeout fired
+var_dump(strpos($captures_json, 'LAST_SENTINEL') === false);
+
+?>
+--CLEAN--
+<?php
+require __DIR__ . "/live_debugger.inc";
+reset_request_replayer();
+?>
+--EXPECT--
+bool(true)
+bool(true)
+bool(true)

tests/ext/live-debugger/debugger_span_decoration_probe.phpt

@@ -8,6 +8,7 @@ DD_TRACE_AGENT_PORT=80
 DD_TRACE_GENERATE_ROOT_SPAN=0
 DD_DYNAMIC_INSTRUMENTATION_ENABLED=1
 DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS=0.1
+DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS=5000
 --INI--
 datadog.trace.agent_test_session_token=live-debugger/span_decoration_probe
 --FILE--

tests/ext/live-debugger/exception-replay_001.phpt

@@ -9,6 +9,7 @@ DD_TRACE_AGENT_PORT=80
 DD_TRACE_GENERATE_ROOT_SPAN=0
 DD_EXCEPTION_REPLAY_ENABLED=1
 DD_EXCEPTION_REPLAY_CAPTURE_INTERVAL_SECONDS=1
+DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS=5000
 --INI--
 datadog.trace.agent_test_session_token=live-debugger/exception-replay_001
 --FILE--

tests/ext/live-debugger/exception-replay_002.phpt

@@ -8,6 +8,7 @@ DD_TRACE_AGENT_PORT=80
 DD_TRACE_GENERATE_ROOT_SPAN=0
 DD_EXCEPTION_REPLAY_ENABLED=1
 DD_EXCEPTION_REPLAY_CAPTURE_INTERVAL_SECONDS=1
+DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS=5000
 --INI--
 datadog.trace.agent_test_session_token=live-debugger/exception-replay_002
 --FILE--

tests/ext/live-debugger/exception-replay_non_regression_2989_mysqli.phpt

@@ -10,6 +10,7 @@ DD_TRACE_AGENT_PORT=80
 DD_TRACE_GENERATE_ROOT_SPAN=0
 DD_EXCEPTION_REPLAY_ENABLED=1
 DD_EXCEPTION_REPLAY_CAPTURE_INTERVAL_SECONDS=1
+DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS=5000
 --INI--
 datadog.trace.agent_test_session_token=live-debugger/non_regression_2989_mysqli
 --FILE--

tracer/configuration.h

@@ -151,6 +151,7 @@
     CONFIG(BOOL, DD_APM_TRACING_ENABLED, "true")                                                               \
     CONFIG(SET, DD_DYNAMIC_INSTRUMENTATION_REDACTED_TYPES, "", .ini_change = zai_config_system_ini_change)     \
     CONFIG(SET, DD_DYNAMIC_INSTRUMENTATION_REDACTION_EXCLUDED_IDENTIFIERS, "", .ini_change = zai_config_system_ini_change) \
+    CONFIG(INT, DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS, "15", .ini_change = zai_config_system_ini_change) \
     CONFIG(INT, DD_TRACE_BAGGAGE_MAX_ITEMS, "64")                                                              \
     CONFIG(INT, DD_TRACE_BAGGAGE_MAX_BYTES, "8192")                                                            \
     CONFIG(BOOL, DD_TRACE_INFERRED_PROXY_SERVICES_ENABLED, "false")                                            \

tracer/ddtrace_globals.h

@@ -1,7 +1,9 @@
 #ifndef DDTRACE_GLOBALS_H
 #define DDTRACE_GLOBALS_H
+#include <signal.h>
 #ifndef _WIN32
 #include <dogstatsd_client/client.h>
+#include <time.h>
 #endif
 
 #include <ext/datadog.h>
@@ -71,6 +73,16 @@ typedef struct {
 
     dd_capture_arena debugger_capture_arena;
     ddog_Vec_DebuggerPayload exception_debugger_buffer;
+    volatile sig_atomic_t debugger_capture_timed_out;
+#ifndef _WIN32
+    volatile uint64_t capture_deadline_ns;
+#ifdef __linux__
+    timer_t capture_timer;
+    int capture_timer_active;
+#endif
+#else
+    HANDLE capture_timer_handle;
+#endif
     HashTable active_live_debugger_hooks;
     HashTable *agent_rate_by_service;
 

tracer/exception_serialize.c

@@ -111,6 +111,10 @@ static void ddtrace_capture_long_value(zend_long num, struct ddog_CaptureValue *
 }
 
 void ddtrace_create_capture_value(zval *zv, struct ddog_CaptureValue *value, const ddog_CaptureConfiguration *config, int remaining_nesting) {
+    if (UNEXPECTED(DDTRACE_G(debugger_capture_timed_out))) {
+        value->not_captured_reason = DDOG_CHARSLICE_C("timeout");
+        return;
+    }
     ZVAL_DEREF(zv);
     switch (Z_TYPE_P(zv)) {
         case IS_FALSE:
@@ -158,6 +162,10 @@ void ddtrace_create_capture_value(zval *zv, struct ddog_CaptureValue *value, con
             if (zend_array_is_list(Z_ARR_P(zv))) {
                 int remaining_fields = config->max_collection_size;
                 ZEND_HASH_FOREACH_VAL(Z_ARR_P(zv), val) {
+                    if (UNEXPECTED(DDTRACE_G(debugger_capture_timed_out))) {
+                        value->not_captured_reason = DDOG_CHARSLICE_C("timeout");
+                        break;
+                    }
                     if (remaining_fields-- == 0) {
                         value->not_captured_reason = DDOG_CHARSLICE_C("collectionSize");
                         break;
@@ -172,6 +180,10 @@ void ddtrace_create_capture_value(zval *zv, struct ddog_CaptureValue *value, con
                 zend_string *key;
                 int remaining_fields = config->max_collection_size;
                 ZEND_HASH_FOREACH_KEY_VAL(Z_ARR_P(zv), idx, key, val) {
+                    if (UNEXPECTED(DDTRACE_G(debugger_capture_timed_out))) {
+                        value->not_captured_reason = DDOG_CHARSLICE_C("timeout");
+                        break;
+                    }
                     if (remaining_fields-- == 0) {
                         value->not_captured_reason = DDOG_CHARSLICE_C("collectionSize");
                         break;
@@ -224,6 +236,10 @@ void ddtrace_create_capture_value(zval *zv, struct ddog_CaptureValue *value, con
                 break;
             }
             ZEND_HASH_REVERSE_FOREACH_STR_KEY_VAL(ht, key, val) {
+                if (UNEXPECTED(DDTRACE_G(debugger_capture_timed_out))) {
+                    value->not_captured_reason = DDOG_CHARSLICE_C("timeout");
+                    break;
+                }
                 if (!key) {
                     continue;
                 }
@@ -401,6 +417,8 @@ static void ddtrace_collect_exception_debug_data(zend_object *exception, zend_ob
 
     memset(&DDTRACE_G(exception_debugger_buffer), 0, sizeof(DDTRACE_G(exception_debugger_buffer)));
 
+    dd_start_debugger_timeout();
+
     zval *frame;
     int frame_num = 0;
     ZEND_HASH_FOREACH_NUM_KEY_VAL(Z_ARR_P(trace), frame_num, frame) {
@@ -480,6 +498,8 @@ static void ddtrace_collect_exception_debug_data(zend_object *exception, zend_ob
         }
     }
 
+    dd_stop_debugger_timeout();
+
     // Note: We MUST immediately send this, and not defer, as stuff may be freed during span processing. Including stuff potentially contained within the exception debugger payload.
     ddtrace_sidecar_send_debugger_data(DDTRACE_G(exception_debugger_buffer));