| @@ -0,0 +1,95 @@ | |||||
| # Auditoría de PCKILL y diagnóstico de bloqueos — 2026-09-28 | |||||
| Evidencia del operador: SIGKILL enviado correctamente con latencias 958.211, | |||||
| 1784.766, 1032.682, 709.773 y 946.776 ms, frente a SIGTERM de 0.006–0.013 ms. | |||||
| La fecha exacta de esas muestras y la revisión del kernel no fueron facilitadas. | |||||
| Los valores se conservan en tests/fixtures/pckill_latency_2026_09_28.csv; la | |||||
| fecha del archivo corresponde a esta revisión, no presupone la del episodio. | |||||
| ## Hallazgo del código del kernel | |||||
| En [illumos prcontrol.c](https://github.com/illumos/illumos-gate/blob/master/usr/src/uts/common/fs/proc/prcontrol.c), | |||||
| pr_control ejecuta pr_kill para PCKILL. Si SIGKILL tiene éxito, libera mediante | |||||
| prunlock y llama a pr_wait_die. Este espera con pidlock/cv_wait_sig mientras | |||||
| existe el proceso y no está SZOMB; una interrupción puede terminar la espera. | |||||
| SIGTERM no sigue esa rama. Es evidencia de espera síncrona, no una suposición | |||||
| basada sólo en las latencias. La rama examinada no condiciona esa espera a | |||||
| O_NONBLOCK. Referencia upstream consultada el 2026-09-28, no identificada como | |||||
| el binario exacto de producción ni como una particularidad de LX. | |||||
| ## Qué falta demostrar en el nodo | |||||
| La traza del guard delimita tiempo transcurrido alrededor de write, pero no | |||||
| reparte ese tiempo entre espera del kernel, locks, desprogramación o trabajo | |||||
| de salida de la víctima. No demuestra bloqueo por ZFS/pageout, liberación de | |||||
| memoria ni una causa propia de LX. Esas hipótesis requieren stacks y correlación. | |||||
| No se cambia a kill(2), no se reintenta una escritura ambigua ni se cambia | |||||
| la arquitectura del envío. Esta revisión mejora diagnóstico, no reduce por sí | |||||
| misma una espera síncrona del kernel. | |||||
| Para cerrar la atribución, en una observación autorizada posterior: | |||||
| 1. Conservar uname -a, versión de plataforma, binario del guard y revisión | |||||
| correspondiente de illumos-joyent. Comparar pr_control/pr_wait_die de ella. | |||||
| 2. Correlacionar entrada/retorno de write por PID/TID y descriptor ctl con | |||||
| kill_attempt_mono/kill_return_mono y la identidad de la víctima. | |||||
| 3. Medir pr_wait_die y la ruta de locks de /proc con las sondas disponibles | |||||
| de esa plataforma; capturar stacks del hilo llamador y de la víctima. | |||||
| 4. Separar tiempo on-CPU/off-CPU y correlacionar con memoria, scanner, pageout, | |||||
| ZFS y actividad LX. Comparar TERM/KILL con condiciones equivalentes. | |||||
| No se ejecutó esa observación ni se accedió al nodo en esta tarea. | |||||
| ## Call graph de la aplicación | |||||
| candidate_cache → action_plan → validation_acquire → trabajador platform_prepare: | |||||
| open(/proc/PID) → openat(psinfo) → openat(ctl) → pread → identidad/zona/política | |||||
| → freemem → resultado preparado. El descriptor ctl ya está abierto al salir | |||||
| de validación. Sólo el hilo de rescate llama platform_dispatch: | |||||
| preparar {PCKILL, señal} → timestamp → errno=0 → write → guardar errno | |||||
| → timestamp de retorno → resultado → release/counters → enqueue de diagnóstico. | |||||
| El cierre ocurre después en el trabajador. No hay open, validación ni logging | |||||
| entre las dos marcas que rodean write. | |||||
| ## Contrato temporal | |||||
| - proc_ctl_open_latency_ms: sólo openat(ctl), dentro de validación; es parte | |||||
| de proc_open_ms, no una duración que se deba sumar otra vez a validación. | |||||
| - pckill_prepare_latency_ms: entrada de platform_dispatch hasta marca pre-write. | |||||
| - pckill_write_latency_ms: marca pre-write hasta reloj inmediatamente posterior | |||||
| a guardar errno. Incluye syscall y desprogramación en ese intervalo, además | |||||
| del coste mínimo de reloj/captura; no equivale a tiempo de CPU del kernel. | |||||
| - pckill_total_latency_ms: prepare + write; excluye validación y apertura. | |||||
| - action_dispatch_latency_ms: decisión → retorno, incluye validación. | |||||
| - kill_latency_ms conserva el intervalo write y coincide con su nueva etiqueta. | |||||
| En v1.0.8 había dos asignaciones del resultado antes de leer el reloj de retorno; | |||||
| v1.0.9 mueve ese reloj justo después de guardar errno. No incluye logging ni | |||||
| espera del logger. Los timestamps no indican cuándo el proceso recibió la señal | |||||
| dentro de la syscall. signal_sent continúa exigiendo escritura completa. | |||||
| signal_dispatch_slow se encola después del envío si write supera estrictamente | |||||
| 100 ms, para TERM o KILL y tanto éxito como fallo. Es un umbral diagnóstico | |||||
| constante, sin efecto sobre política ni budgets. No se espera al logger. | |||||
| ## Bloqueos | |||||
| Todas las líneas action_blocked incluyen state/reason/free_pages/desfree/minfree, | |||||
| refresh_ok/candidate_source, refresh_errno/refresh_error/refresh_latency_ms. | |||||
| refresh_ok=1 significa muestra vigente; 0 fallo o caducidad; -1 no se intentó, | |||||
| por ejemplo warmup. Si falla, los valores son la última muestra del llamador, | |||||
| marcados sample_source=last_sample, nunca una muestra parcial presentada como válida. | |||||
| free_above_desfree significa free >= desfree, incluyendo igualdad por compatibilidad, | |||||
| refresh_ok=1 y LOG_INFO. refresh_failed distingue errno del backend, fallo sin errno | |||||
| y stale_sample (más de 2 s). Se captura errno antes de llamar al reloj. | |||||
| La supresión de acciones y todos los límites son los anteriores. | |||||
| ## Verificación y alcance | |||||
| Fixtures de recuperación y fallo, cinco latencias reales reproducidas mediante | |||||
| reloj simulado, TERM rápido, límite de 100 ms, TERM lento y medición del código | |||||
| real de dispatch con write simulado demorado. No se envía ninguna señal real. | |||||
| Pruebas portables no sustituyen la confirmación nativa del éxito/latencia del | |||||
| transporte en SmartOS. Ranking, state machine, configuración funcional, budgets, | |||||
| allow/protect y arquitectura async quedan sin cambios. | |||||
| @@ -1,5 +1,16 @@ | |||||
| # Changelog | # Changelog | ||||
| ## 1.0.9 — 2026-09-28 — PCKILL phases and distinct refresh diagnostics | |||||
| - Medidas separadas de openat(ctl), preparación, write y total de dispatch. | |||||
| Reloj de retorno inmediatamente después de capturar errno. | |||||
| - Evento asíncrono signal_dispatch_slow para write >100 ms, sin alterar rescate. | |||||
| - action_blocked distingue recuperación válida, fallo y muestra caducada, con | |||||
| contexto de memoria, fuente de candidato y resultado/latencia del refresco. | |||||
| - Auditoría upstream de la rama SIGKILL/pr_wait_die; atribución nativa pendiente. | |||||
| - Fixtures TERM/KILL, recuperación sin segunda señal y fallo de refresco. | |||||
| - Sin cambios de política, ranking, budgets, state machine o arquitectura. | |||||
| ## 1.0.8 — 2026-09-23 — Bounded preparation and exact syscall results | ## 1.0.8 — 2026-09-23 — Bounded preparation and exact syscall results | ||||
| - Auditado FLOOR486: v1.0.7 usa write(PCKILL), repite validación y puede devolver | - Auditado FLOOR486: v1.0.7 usa write(PCKILL), repite validación y puede devolver | ||||
| @@ -811,3 +811,14 @@ Informe: `INCIDENT_FLOOR_KILL_FAILED_2026_09_23.md`. | |||||
| Fixture: `tests/fixtures/floor_free_pages_486_2026_09_23.csv`. | Fixture: `tests/fixtures/floor_free_pages_486_2026_09_23.csv`. | ||||
| La suite añade esperas reales de 1283 ms, saturación, errno y código nativo con | La suite añade esperas reales de 1283 ms, saturación, errno y código nativo con | ||||
| I/O simulado. Source-only; validación nativa pendiente, sin despliegue. | I/O simulado. Source-only; validación nativa pendiente, sin despliegue. | ||||
| ## 22. Addendum v1.0.9 — PCKILL lento y refresco desambiguado | |||||
| El operador aporta cinco SIGKILL enviados con latencias 709,773–1784,766 ms. | |||||
| Se añade medición ctl-open/prepare/write/total, evento lento >100 ms y bloqueos | |||||
| separados: free_above_desfree (informativo, >=) y refresh_failed (incluye stale). | |||||
| Cada bloqueo conserva contexto y marca la fuente de la muestra. | |||||
| La rama upstream SIGKILL llama pr_wait_die; su correspondencia con el kernel | |||||
| desplegado y la distribución entre espera, locks y off-CPU requieren observación. | |||||
| No se atribuye la latencia a LX/ZFS ni se cambia transporte, ranking o política. | |||||
| Auditoría: AUDIT_PCKILL_LATENCY_2026_09_28.md. Entrega source-only. | |||||
| @@ -21,6 +21,8 @@ test-guard: tests/test_guard.c tests/fixtures/pageout_deadman_real_incident_2026 | |||||
| test-runtime: tests/test_runtime.c tests/fixtures/global_victim_ranking_smartos_2026_09.csv tests/fixtures/panic32_daemon_stall_2026_09_22.csv tests/fixtures/floor_free_pages_486_2026_09_23.csv src/main.c src/core.c src/config.c src/result.c src/guard.h | test-runtime: tests/test_runtime.c tests/fixtures/global_victim_ranking_smartos_2026_09.csv tests/fixtures/panic32_daemon_stall_2026_09_22.csv tests/fixtures/floor_free_pages_486_2026_09_23.csv src/main.c src/core.c src/config.c src/result.c src/guard.h | ||||
| $(CC) $(TESTFLAGS) -o $@ tests/test_runtime.c src/core.c src/config.c src/result.c | $(CC) $(TESTFLAGS) -o $@ tests/test_runtime.c src/core.c src/config.c src/result.c | ||||
| test-runtime: tests/fixtures/pckill_latency_2026_09_28.csv tests/fixtures/refresh_recovery_and_failure.csv | |||||
| test-distribution: tests/test_distribution.sh smf/o6h-memory-guard.xml etc/o6h-memory-guard.conf src/main.c | test-distribution: tests/test_distribution.sh smf/o6h-memory-guard.xml etc/o6h-memory-guard.conf src/main.c | ||||
| sh tests/test_distribution.sh | sh tests/test_distribution.sh | ||||
| @@ -1,4 +1,4 @@ | |||||
| # o6h-memory-guard 1.0.8 | |||||
| # o6h-memory-guard 1.0.9 | |||||
| Primera implementación para SmartOS global zone: daemon C de 64 bits, libkstat, | Primera implementación para SmartOS global zone: daemon C de 64 bits, libkstat, | ||||
| /proc, identidad UUID + generación de zona + PID + instante de creación, | /proc, identidad UUID + generación de zona + PID + instante de creación, | ||||
| @@ -49,12 +49,12 @@ Si no hay herramientas de desarrollo, compilar en una **zona nativa SmartOS** | |||||
| de la misma plataforma (no LX), ejecutar allí `gmake test`, y copiar el binario | de la misma plataforma (no LX), ejecutar allí `gmake test`, y copiar el binario | ||||
| junto a este árbol al host. No instalar un compilador en un nodo bajo presión. | junto a este árbol al host. No instalar un compilador en un nodo bajo presión. | ||||
| Copiar el archivo distribuido a `/var/tmp/o6h-memory-guard-1.0.8.tar.gz`: | |||||
| Copiar el archivo distribuido a `/var/tmp/o6h-memory-guard-1.0.9.tar.gz`: | |||||
| ```sh | ```sh | ||||
| mkdir -p /opt/custom/src | mkdir -p /opt/custom/src | ||||
| cd /opt/custom/src | cd /opt/custom/src | ||||
| gzip -dc /var/tmp/o6h-memory-guard-1.0.8.tar.gz | tar -xf - | |||||
| gzip -dc /var/tmp/o6h-memory-guard-1.0.9.tar.gz | tar -xf - | |||||
| cd /opt/custom/src/o6h-memory-guard | cd /opt/custom/src/o6h-memory-guard | ||||
| gmake CC=gcc | gmake CC=gcc | ||||
| gmake test CC=gcc | gmake test CC=gcc | ||||
| @@ -297,6 +297,21 @@ de escritura de señal consumen presupuesto para evitar reintentos ambiguos. | |||||
| ## Recursos y fallos | ## Recursos y fallos | ||||
| v1.0.9 añade pckill_timing con proc_ctl_open_latency_ms (durante validación), | |||||
| pckill_prepare_latency_ms, pckill_write_latency_ms y pckill_total_latency_ms. | |||||
| El total es prepare+write; no incluye validación ni apertura. El intervalo write | |||||
| incluye desprogramación y no demuestra por sí mismo una causa del kernel. | |||||
| Si supera 100 ms se encola signal_dispatch_slow, sin cambiar la acción. | |||||
| La auditoría y el procedimiento para atribuir la espera están en | |||||
| AUDIT_PCKILL_LATENCY_2026_09_28.md. | |||||
| action_blocked distingue free_above_desfree (free >= desfree, muestra válida, | |||||
| informativo) de refresh_failed (fallo o muestra caducada). Incluye memoria, | |||||
| refresh_ok, candidate_source, error y latencia. refresh_ok=-1 indica que no | |||||
| se intentó el refresco. Con fallo, sample_source=last_sample identifica los | |||||
| datos anteriores del llamador. Se sigue suprimiendo toda acción cuya presión | |||||
| no pueda confirmarse. | |||||
| Desde v1.0.8, la preparación ocurre en 16 trabajadores preasignados. La espera | Desde v1.0.8, la preparación ocurre en 16 trabajadores preasignados. La espera | ||||
| del hilo de rescate se limita a 250 ms en CRITICAL, 100 ms en EMERGENCY y 50 ms | del hilo de rescate se limita a 250 ms en CRITICAL, 100 ms en EMERGENCY y 50 ms | ||||
| en FLOOR. Si vence, abandona ese candidato y prueba el siguiente. Los | en FLOOR. Si vence, abandona ese candidato y prueba el siguiente. Los | ||||
| @@ -436,7 +451,7 @@ Alloy/Loki puede configurarse fuera del guard, pero esta entrega no lo hace. | |||||
| ## Modelo de activación real (operación futura; no ejecutada en esta entrega) | ## Modelo de activación real (operación futura; no ejecutada en esta entrega) | ||||
| v1.0.8 debe compilarse y observarse primero en SmartOS con `ACTION_MODE dry-run`. | |||||
| v1.0.9 debe compilarse y observarse primero en SmartOS con `ACTION_MODE dry-run`. | |||||
| El manifiesto ya no contiene una segunda puerta oculta. Cuando exista aprobación | El manifiesto ya no contiene una segunda puerta oculta. Cuando exista aprobación | ||||
| operativa explícita, el único cambio de modo es editar la configuración: | operativa explícita, el único cambio de modo es editar la configuración: | ||||
| @@ -1,31 +1,32 @@ | |||||
| ab22dd5e73f360b7f08839f95e35e970ce4401e4541071e802cf2d24e3d8e2a6 .gitignore | ab22dd5e73f360b7f08839f95e35e970ce4401e4541071e802cf2d24e3d8e2a6 .gitignore | ||||
| b6e4b74d694eec1214446e0bfc369e582fd28069a2547eabe9571135cb27d669 CHANGELOG.md | |||||
| 1d366afb5d27b3eb001e3534f9cfff2dd422d4e5524780dbb866a120d80507e4 CHECKPOINT.md | |||||
| bba839eb1ece469050db8beb6a7eccdbe07a51eadc2a920c0cdf0f707330b800 AUDIT_PCKILL_LATENCY_2026_09_28.md | |||||
| bffc45b4707f96ee78e59c03a06ce06bffbf7c500d6b3b3939690ad30ff7da4c CHANGELOG.md | |||||
| 8b762e7767e62aeb9188cb5988ef575985caa72f00d6c866a05b5e37a1556419 CHECKPOINT.md | |||||
| 4d0b3983bbb0d2a51c61ce0f99d778cc79768ed371bdb5ced382c0fb5dfcaf5f INCIDENT_FLOOR_KILL_FAILED_2026_09_23.md | 4d0b3983bbb0d2a51c61ce0f99d778cc79768ed371bdb5ced382c0fb5dfcaf5f INCIDENT_FLOOR_KILL_FAILED_2026_09_23.md | ||||
| a0f96300e751d08ceeef1360325ed54008b703fa6bd07df9d5368674e580f113 INCIDENT_PAGEOUT_DEADMAN_2026_09.md | a0f96300e751d08ceeef1360325ed54008b703fa6bd07df9d5368674e580f113 INCIDENT_PAGEOUT_DEADMAN_2026_09.md | ||||
| fb6b65a2655ccd808d102fd917b4866cbe939f24fac4e4aeef226a6aeda6d588 INCIDENT_PAGEOUT_DEADMAN_AVOIDED_2026_09_21.md | fb6b65a2655ccd808d102fd917b4866cbe939f24fac4e4aeef226a6aeda6d588 INCIDENT_PAGEOUT_DEADMAN_AVOIDED_2026_09_21.md | ||||
| 0f18d6dd3e0d027c1e8dbfce05ce6ac3beb70f3f66d1a77a8d0c5e0f4ae2a039 INCIDENT_PAGEOUT_DEADMAN_PANIC32_2026_09_22.md | 0f18d6dd3e0d027c1e8dbfce05ce6ac3beb70f3f66d1a77a8d0c5e0f4ae2a039 INCIDENT_PAGEOUT_DEADMAN_PANIC32_2026_09_22.md | ||||
| 3f46724f39a0b91fa6cb56e6d2dc79f195894f085954c11b575cb1f611476138 Makefile | |||||
| d56adedb3baf4b7d126cae06dc6f94129b77f9590753b5870b7d89b51106b97d README.md | |||||
| 8afc5ef6c25abf43bc32f56833ad761dc85a495a06aadc79f2b94da890f4693d VALIDATION.md | |||||
| b5a762b6e405dbaa408b8a6a7f5cdfaf9d5b04c3220cdc6787c25e96dba314e1 etc/o6h-memory-guard.conf | |||||
| 87912109e1dd3750888d484aa2387e5ddea90827082871c77bf3ee2e1017d298 Makefile | |||||
| 3d38133e31d5e71f8ef97aae51fb042ac7223f9ad82ea58455482834c57aa20d README.md | |||||
| c8d35d9e65df672e080786c85dcae676540f0fe7e702242392b19d769a15dc98 VALIDATION.md | |||||
| dd3d5ef89dda5a37a51d85e693b4c03d1f322d39d64bc1b9d78140f22d1d162d etc/o6h-memory-guard.conf | |||||
| f6565d0e1ece0a08721c9893f9ed071d2ab0a2818fa93ed0a14e0050d358ce99 install.sh | f6565d0e1ece0a08721c9893f9ed071d2ab0a2818fa93ed0a14e0050d358ce99 install.sh | ||||
| 24c1b9bc39fda71d459d6fad6a8d889b0d9b92be89d303677643b3d0b93caf4b logs/distribution-tests.log | 24c1b9bc39fda71d459d6fad6a8d889b0d9b92be89d303677643b3d0b93caf4b logs/distribution-tests.log | ||||
| 50626dec1924e19b4ca92eec30416fb39c3fe7cff28d98a8e120ccf6247d2834 logs/example.log | 50626dec1924e19b4ca92eec30416fb39c3fe7cff28d98a8e120ccf6247d2834 logs/example.log | ||||
| 812482297ae6dfb2a0881998781459ed77288ef073f202a18f22055437518fa6 logs/policy-tests.log | 812482297ae6dfb2a0881998781459ed77288ef073f202a18f22055437518fa6 logs/policy-tests.log | ||||
| d60b67d7c8b0307593fe28fc13025778f0c7175d9bdf0cc2fa4f5edfcc18fb6c logs/portability.log | d60b67d7c8b0307593fe28fc13025778f0c7175d9bdf0cc2fa4f5edfcc18fb6c logs/portability.log | ||||
| f0ebf6b604648fa1cddb15b55c9ae7f2f5428cd952bcd41778171a1e6b1dc79d logs/prepare-tests.log | |||||
| 84a2c4ba87494915e8225bae692c9458417131919fed8d8c1c5c116da6a38554 logs/runtime-tests.log | |||||
| 3d707eaf3b6b306a363abbc696226a3764069099bd3c6809585769a5f63a626a logs/sanitizers.log | |||||
| 437b5853cf6bdc05a660620ce44b14a10901fca1428218942ed87897cfef6d17 logs/validation-tests.log | |||||
| fa1dad7c69b95259bff4e18a98946618c68f68642cf654e97984ea80028d0f19 logs/prepare-tests.log | |||||
| 5c21b4cfcde65fc393ec3e3825beab61b4aa0e56818ace9b6363955f26a764a9 logs/runtime-tests.log | |||||
| c863ebc2d6fbe6a2c96f4a8040a5dc28a61e669d2d12879055c410665ae2ef6c logs/sanitizers.log | |||||
| ec90a51222ef270ab0294532bc9396862afec1679824026848c2bc61165348be logs/validation-tests.log | |||||
| 55cf49840cf1d22e4e02bedf579a10207856a0a5b40c86ae2cf6ee2f00d3f972 smf/o6h-memory-guard.xml | 55cf49840cf1d22e4e02bedf579a10207856a0a5b40c86ae2cf6ee2f00d3f972 smf/o6h-memory-guard.xml | ||||
| 99e1fb6a90a7bd2c1002ad09320cd3f21c2deb2499212eff2baddc14f3bdf0e6 src/config.c | 99e1fb6a90a7bd2c1002ad09320cd3f21c2deb2499212eff2baddc14f3bdf0e6 src/config.c | ||||
| 659da2facd27577c763484694a2ff62eef42aea48ebc496b878853b0164b6539 src/core.c | 659da2facd27577c763484694a2ff62eef42aea48ebc496b878853b0164b6539 src/core.c | ||||
| c4a18ce71f8d5e29e861c5d2507ca811b4d803fda6f00c7a872950a7d32445b7 src/guard.h | |||||
| f6bc7848998e626a822c6ab119af958727c46d67669fbb63ec6313ee6a595e17 src/illumos.c | |||||
| ce063766de0739504172bcf272f1713739c6674861a11f0ac215601229d6aaf6 src/main.c | |||||
| 8133cad598f1e14e341c0611586f3cd506b640e13d90a261a8e256274e347875 src/guard.h | |||||
| 8ebc01c7c1a5a58cb1a40cc355505777940e1dc95f29c650f5e066e10a183f3e src/illumos.c | |||||
| e58568de0a12af9df6773ac8d4755328b823480cd001e0423ad541cff202271c src/main.c | |||||
| 5bb4e0980de83b29e05bb10ea87d6c04f10df884dc92f74cd6b3b025d41d3332 src/result.c | 5bb4e0980de83b29e05bb10ea87d6c04f10df884dc92f74cd6b3b025d41d3332 src/result.c | ||||
| ff0ec972bfbb0bbdc9bd7dd205407da7c0f2b2102263626ce10210773d2da7e8 src/signal.c | |||||
| 948bfd760d5a4e1cd301da8b8aa270cfa894d2ec3334d000bd687ef023788388 src/signal.c | |||||
| 1f09afd342ba86580a21f6c04a030d9ea2551a60b669e257295fbf4f43b2fad7 src/support.c | 1f09afd342ba86580a21f6c04a030d9ea2551a60b669e257295fbf4f43b2fad7 src/support.c | ||||
| dfa4926c9bba9b0707c683dd4e1d8dc4bfa044da5221b0bf25e5a04fe007aac8 src/validation.c | dfa4926c9bba9b0707c683dd4e1d8dc4bfa044da5221b0bf25e5a04fe007aac8 src/validation.c | ||||
| f754e71d91d65403ab8e84e9701cfbb89bdd91e13766e668445e689e6890b369 tests/api-sketch/kstat.h | f754e71d91d65403ab8e84e9701cfbb89bdd91e13766e668445e689e6890b369 tests/api-sketch/kstat.h | ||||
| @@ -37,9 +38,11 @@ f0d06353d2e326c7bdce776bee6f0a8606cda50d5de957dfdd586b6848fca8da tests/api-sket | |||||
| 4aa08ded8e98cf25d2a8762543de7a9273f0ebf00b4c7e5b359603ceda46eb56 tests/fixtures/global_victim_ranking_smartos_2026_09.csv | 4aa08ded8e98cf25d2a8762543de7a9273f0ebf00b4c7e5b359603ceda46eb56 tests/fixtures/global_victim_ranking_smartos_2026_09.csv | ||||
| c5d9d7fc7a39354474d30cd316ea9f122205c2b38e57f927f4a08306b8a62c81 tests/fixtures/pageout_deadman_real_incident_2026_09.csv | c5d9d7fc7a39354474d30cd316ea9f122205c2b38e57f927f4a08306b8a62c81 tests/fixtures/pageout_deadman_real_incident_2026_09.csv | ||||
| 025983b24045e24a96532c933715b8760edd8c0b0e11b9f0e019481c1c52a50d tests/fixtures/panic32_daemon_stall_2026_09_22.csv | 025983b24045e24a96532c933715b8760edd8c0b0e11b9f0e019481c1c52a50d tests/fixtures/panic32_daemon_stall_2026_09_22.csv | ||||
| b80c56d558d98c7aae6883f5d51ad4652be46999385170e79fb50da17ade325f tests/fixtures/pckill_latency_2026_09_28.csv | |||||
| ccf983995eff740e26feb66200de6e2339a3f31f997d74a9af3815ce35722c86 tests/fixtures/refresh_recovery_and_failure.csv | |||||
| facc4580ffadef29cdce84bf1226a696cba3880bbbec80161beaceb2752bfd77 tests/fixtures/successful_pageout_deadman_avoidance_2026_09_21.csv | facc4580ffadef29cdce84bf1226a696cba3880bbbec80161beaceb2752bfd77 tests/fixtures/successful_pageout_deadman_avoidance_2026_09_21.csv | ||||
| 84b3e9b9dbab27a6ed18b69d9f11bc7ca6a84105cd80bb543411146cdba57a00 tests/test_distribution.sh | 84b3e9b9dbab27a6ed18b69d9f11bc7ca6a84105cd80bb543411146cdba57a00 tests/test_distribution.sh | ||||
| 7c0cfe5fc6719d27258bb87b9e68388be7b580d7031a5ed6deac34ddb3050f25 tests/test_guard.c | 7c0cfe5fc6719d27258bb87b9e68388be7b580d7031a5ed6deac34ddb3050f25 tests/test_guard.c | ||||
| a485433d8c0f1ee1a538a520a2e6a4a82db5a7e6b3896351a7403e0eeb4d2c90 tests/test_prepare.c | |||||
| bbdd14d320368507afeab712f0c8aa4afaa4b31d325233b19d546e78685966b7 tests/test_runtime.c | |||||
| bb6e78e1b4ea22d703f738a3beaaee6c192bfaf103d75af7e381143052818b05 tests/test_validation.c | |||||
| 1210e59dd2fc57df15dc9a4254dbc652059895ed6d5ea6e2f82f79a5b046107a tests/test_prepare.c | |||||
| fad190cce29e49bc0d3549a6dfd9648f98129d8ae170e0fc7926b236cdb9ad7e tests/test_runtime.c | |||||
| 9576bb5d5dcb525c7c89ec5dc929b896e8c5b61beffe4d7313f89d823476d957 tests/test_validation.c | |||||
| @@ -1,11 +1,18 @@ | |||||
| # Validación de la entrega v1.0.8 | |||||
| # Validación de la entrega v1.0.9 | |||||
| Fecha: 2026-09-23. Entrega source-only. **No se desplegó, no se modificó el | |||||
| Fecha: 2026-09-28. Entrega source-only. **No se desplegó, no se modificó el | |||||
| nodo armado y no se enviaron señales reales.** La configuración conserva | nodo armado y no se enviaron señales reales.** La configuración conserva | ||||
| `ACTION_MODE dry-run`, syslog `local5`, cero UUID `allow` y SMF deshabilitado. | `ACTION_MODE dry-run`, syslog `local5`, cero UUID `allow` y SMF deshabilitado. | ||||
| ## Motivo | ## Motivo | ||||
| Revisión diagnóstica de SIGKILL enviado con 709–1785 ms de latencia y bloqueos | |||||
| ambiguos de refresco. Se separan apertura ctl/preparación/write/total y causas de | |||||
| bloqueo sin modificar decisiones. AUDIT_PCKILL_LATENCY_2026_09_28.md documenta | |||||
| la espera pr_wait_die hallada upstream y los límites de atribución en producción. | |||||
| No se afirma haber confirmado la revisión de kernel del nodo ni causalidad LX/ZFS. | |||||
| Se conservan como regresiones los incidentes anteriores: | |||||
| FLOOR486 de v1.0.7: free=486, nscan=1548386, validación 1283,513 ms, | FLOOR486 de v1.0.7: free=486, nscan=1548386, validación 1283,513 ms, | ||||
| acción failed errno=11 y recuperación posterior con el candidato aún vivo. | acción failed errno=11 y recuperación posterior con el candidato aún vivo. | ||||
| No hubo panic en esa traza. El logger aisló escrituras de hasta 6655 ms: | No hubo panic en esa traza. El logger aisló escrituras de hasta 6655 ms: | ||||
| @@ -56,19 +63,26 @@ no está confirmado si lo causó syslog, guard.log u otra syscall. Véase | |||||
| ## Pruebas ejecutadas | ## Pruebas ejecutadas | ||||
| - Política: 175 comprobaciones; incluye los fixtures previos y no enlaza backend. | - Política: 175 comprobaciones; incluye los fixtures previos y no enlaza backend. | ||||
| - Runtime simulado: 452 comprobaciones; ninguna señal nativa. El número cambia | |||||
| - Runtime simulado: 503 comprobaciones; ninguna señal nativa. El número cambia | |||||
| al eliminar las comprobaciones duplicadas del mock de doble validación. | al eliminar las comprobaciones duplicadas del mock de doble validación. | ||||
| - Preparación concurrente: 66 comprobaciones. Espera real inyectada de 1283 ms; | |||||
| - Preparación concurrente/dispatch: 70 comprobaciones. Espera real inyectada de 1283 ms; | |||||
| abandono con presupuesto de 50 ms, fallback a otro trabajador, descarte tardío, | abandono con presupuesto de 50 ms, fallback a otro trabajador, descarte tardío, | ||||
| deduplicación, saturación y recuperación del pool. Tolerancias del test local: | deduplicación, saturación y recuperación del pool. Tolerancias del test local: | ||||
| abandono <150 ms y fallback completado <200 ms, muy inferiores a 1 s; el | abandono <150 ms y fallback completado <200 ms, muy inferiores a 1 s; el | ||||
| presupuesto programado sigue siendo 50 ms. No es garantía de tiempo real. | presupuesto programado sigue siendo 50 ms. No es garantía de tiempo real. | ||||
| - Código nativo de preparación con I/O simulado: 15 comprobaciones de PID/start, | |||||
| - Código nativo de preparación con I/O simulado: 16 comprobaciones de PID/start, | |||||
| generación, protect, errno, memoria recuperada, subpasos y dry-run sin ctl. | generación, protect, errno, memoria recuperada, subpasos y dry-run sin ctl. | ||||
| - Código real de dispatch con write simulado: éxito, ESRCH, EPERM, EINVAL, EAGAIN, | - Código real de dispatch con write simulado: éxito, ESRCH, EPERM, EINVAL, EAGAIN, | ||||
| errno previo EAGAIN/ENOMEM, reloj que altera errno y escritura parcial. | errno previo EAGAIN/ENOMEM, reloj que altera errno y escritura parcial. | ||||
| - Fixture `floor_free_pages_486_2026_09_23`: NORMAL→EMERGENCY/FLOOR y recuperación; | - Fixture `floor_free_pages_486_2026_09_23`: NORMAL→EMERGENCY/FLOOR y recuperación; | ||||
| timeout rank1 → rank2, rechazo sin syscall y resultados sin errno heredado. | timeout rank1 → rank2, rechazo sin syscall y resultados sin errno heredado. | ||||
| - Fixture recovery/failure: una víctima floor, recuperación y supresión de segunda | |||||
| señal; después fallo de refresco. Validación de errno, muestra caducada, igualdad | |||||
| con desfree y valores de memoria conocidos frente a muestra parcial inválida. | |||||
| - Cinco latencias SIGKILL reproducidas con reloj simulado; TERM rápido, TERM lento | |||||
| y frontera estricta de 100 ms para signal_dispatch_slow. | |||||
| - Dispatch real con write simulado demorado 10/80 ms: separación temporal y | |||||
| total=prepare+write; apertura ctl medida por separado del resto de /proc. | |||||
| - Fixture `panic32_daemon_stall_2026_09_22`: conserva el intervalo de 145 s. | - Fixture `panic32_daemon_stall_2026_09_22`: conserva el intervalo de 145 s. | ||||
| - Logger/syslog detenidos 180 s: TERM y SIGKILL se solicitan sin esperar y el | - Logger/syslog detenidos 180 s: TERM y SIGKILL se solicitan sin esperar y el | ||||
| tiempo real de cada llamada a `act()` queda por debajo de 100 ms. | tiempo real de cada llamada a `act()` queda por debajo de 100 ms. | ||||
| @@ -123,7 +137,9 @@ semántica monotónica de `sampling_overrun`. | |||||
| 7. Revisar `logging_queue_drop`, `logging_delivery` y `logging_stall` bajo carga. | 7. Revisar `logging_queue_drop`, `logging_delivery` y `logging_stall` bajo carga. | ||||
| 8. Inyectar lecturas lentas por subpaso y comprobar abandono, descarte tardío, | 8. Inyectar lecturas lentas por subpaso y comprobar abandono, descarte tardío, | ||||
| recuperación y saturación del pool. Verificar semántica nativa O_NONBLOCK/ctl. | recuperación y saturación del pool. Verificar semántica nativa O_NONBLOCK/ctl. | ||||
| 9. Mantener dry-run hasta revisión humana explícita. v1.0.8 no se considera apta | |||||
| 9. Medir entry/return de write, pr_wait_die, locks y off-CPU en la revisión nativa | |||||
| correspondiente para atribuir los 0,7–1,8 s; no se ejecutó en esta entrega. | |||||
| 10. Mantener dry-run hasta revisión humana explícita. v1.0.9 no se considera apta | |||||
| para producción sólo por superar las pruebas portables. | para producción sólo por superar las pruebas portables. | ||||
| ## Límites | ## Límites | ||||
| @@ -1,4 +1,4 @@ | |||||
| # o6h-memory-guard 1.0.8 -- whitespace-separated, no shell evaluation. | |||||
| # o6h-memory-guard 1.0.9 -- whitespace-separated, no shell evaluation. | |||||
| # Restart required for configuration changes. Duplicate scalar keys are errors. | # Restart required for configuration changes. Duplicate scalar keys are errors. | ||||
| ACTION_MODE dry-run | ACTION_MODE dry-run | ||||
| SYSLOG_ENABLED yes | SYSLOG_ENABLED yes | ||||
| @@ -1 +1 @@ | |||||
| PASS 15 native preparation source checks with mocked proc/kstat/zone I/O; no native ABI claim or signals | |||||
| PASS 16 native preparation source checks with mocked proc/kstat/zone I/O; no native ABI claim or signals | |||||
| @@ -12,4 +12,5 @@ PASS 16-entry stale rescue cache supplies all 8 EMERGENCY victims without scan | |||||
| PASS 16-entry stale rescue cache supplies all 12 FLOOR victims without scan | PASS 16-entry stale rescue cache supplies all 12 FLOOR victims without scan | ||||
| PASS saturated/locked event queue drops telemetry without delaying the signal | PASS saturated/locked event queue drops telemetry without delaying the signal | ||||
| PASS floor486 fixture, 50ms timeout fallback rank1->2, syscall errno attribution and validation substep logs | PASS floor486 fixture, 50ms timeout fallback rank1->2, syscall errno attribution and validation substep logs | ||||
| PASS 452 runtime checks; all platform calls mocked, no signals sent | |||||
| PASS recovery/failure/stale refresh reasons, desfree equality, no second victim, PCKILL subphases and slow-event threshold | |||||
| PASS 503 runtime checks; all platform calls mocked, no signals sent | |||||
| @@ -28,10 +28,11 @@ PASS 16-entry stale rescue cache supplies all 8 EMERGENCY victims without scan | |||||
| PASS 16-entry stale rescue cache supplies all 12 FLOOR victims without scan | PASS 16-entry stale rescue cache supplies all 12 FLOOR victims without scan | ||||
| PASS saturated/locked event queue drops telemetry without delaying the signal | PASS saturated/locked event queue drops telemetry without delaying the signal | ||||
| PASS floor486 fixture, 50ms timeout fallback rank1->2, syscall errno attribution and validation substep logs | PASS floor486 fixture, 50ms timeout fallback rank1->2, syscall errno attribution and validation substep logs | ||||
| PASS 452 runtime checks; all platform calls mocked, no signals sent | |||||
| PASS recovery/failure/stale refresh reasons, desfree equality, no second victim, PCKILL subphases and slow-event threshold | |||||
| PASS 503 runtime checks; all platform calls mocked, no signals sent | |||||
| gcc -std=c99 -O1 -g -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -Isrc -Wall -Wextra -Werror -Wformat=2 -pedantic -pthread -Itests/api-sketch -fsanitize=address,undefined -fno-omit-frame-pointer -o test-sanitize-validation tests/test_validation.c | gcc -std=c99 -O1 -g -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -Isrc -Wall -Wextra -Werror -Wformat=2 -pedantic -pthread -Itests/api-sketch -fsanitize=address,undefined -fno-omit-frame-pointer -o test-sanitize-validation tests/test_validation.c | ||||
| ./test-sanitize-validation | ./test-sanitize-validation | ||||
| PASS 66 validation checks: real 1283ms worker abandoned, fallback, late disposal, pool saturation; actual dispatch code with mocked write; no signals | |||||
| PASS 70 validation checks: real 1283ms worker abandoned, fallback, late disposal, pool saturation; actual dispatch code with mocked write; no signals | |||||
| gcc -std=c99 -O1 -g -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -Isrc -Wall -Wextra -Werror -Wformat=2 -pedantic -pthread -Itests/api-sketch -fsanitize=address,undefined -fno-omit-frame-pointer -o test-sanitize-prepare tests/test_prepare.c src/core.c src/config.c | gcc -std=c99 -O1 -g -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -Isrc -Wall -Wextra -Werror -Wformat=2 -pedantic -pthread -Itests/api-sketch -fsanitize=address,undefined -fno-omit-frame-pointer -o test-sanitize-prepare tests/test_prepare.c src/core.c src/config.c | ||||
| ./test-sanitize-prepare | ./test-sanitize-prepare | ||||
| PASS 15 native preparation source checks with mocked proc/kstat/zone I/O; no native ABI claim or signals | |||||
| PASS 16 native preparation source checks with mocked proc/kstat/zone I/O; no native ABI claim or signals | |||||
| @@ -1 +1 @@ | |||||
| PASS 66 validation checks: real 1283ms worker abandoned, fallback, late disposal, pool saturation; actual dispatch code with mocked write; no signals | |||||
| PASS 70 validation checks: real 1283ms worker abandoned, fallback, late disposal, pool saturation; actual dispatch code with mocked write; no signals | |||||
| @@ -13,7 +13,7 @@ | |||||
| #define CONF_PATH "/opt/custom/etc/o6h-memory-guard.conf" | #define CONF_PATH "/opt/custom/etc/o6h-memory-guard.conf" | ||||
| #define LOG_DIR "/opt/custom/var/log/o6h-memory-guard" | #define LOG_DIR "/opt/custom/var/log/o6h-memory-guard" | ||||
| #define RUN_DIR "/opt/custom/var/run" | #define RUN_DIR "/opt/custom/var/run" | ||||
| #define VERSION "1.0.8" | |||||
| #define VERSION "1.0.9" | |||||
| enum state { NORMAL, PRE_PRESSURE, PRESSURE, CRITICAL, EMERGENCY, RECOVERY }; | enum state { NORMAL, PRE_PRESSURE, PRESSURE, CRITICAL, EMERGENCY, RECOVERY }; | ||||
| enum policy { OBSERVE, ALLOW, PROTECT }; | enum policy { OBSERVE, ALLOW, PROTECT }; | ||||
| @@ -98,12 +98,13 @@ int platform_walk(int (*)(const process_t *, void *), void *, unsigned); | |||||
| typedef struct { | typedef struct { | ||||
| double pid_ms, start_ms, zone_ms, generation_ms, policy_ms, protected_ms; | double pid_ms, start_ms, zone_ms, generation_ms, policy_ms, protected_ms; | ||||
| double cooldown_ms, open_ms, read_ms, parse_ms, total_ms, memory_ms; | double cooldown_ms, open_ms, read_ms, parse_ms, total_ms, memory_ms; | ||||
| double ctl_open_ms; | |||||
| const char *stage; | const char *stage; | ||||
| } validation_t; | } validation_t; | ||||
| typedef struct { | typedef struct { | ||||
| int attempted, error; | int attempted, error; | ||||
| long rc; | long rc; | ||||
| double attempt, returned; | |||||
| double attempt, returned, prepare_ms, write_ms, total_ms; | |||||
| } signal_result_t; | } signal_result_t; | ||||
| /* Worker-only prepare/dispose; only the rescue thread may dispatch. */ | /* Worker-only prepare/dispose; only the rescue thread may dispatch. */ | ||||
| int platform_validation_init(unsigned); | int platform_validation_init(unsigned); | ||||
| @@ -281,7 +281,8 @@ int platform_prepare(unsigned slot, const config_t *c, const process_t *expected | |||||
| start = monotime(); | start = monotime(); | ||||
| ctl = openat(dir, "ctl", O_WRONLY | O_EXCL | O_CLOEXEC | O_NONBLOCK); | ctl = openat(dir, "ctl", O_WRONLY | O_EXCL | O_CLOEXEC | O_NONBLOCK); | ||||
| saved = ctl < 0 ? errno : 0; | saved = ctl < 0 ? errno : 0; | ||||
| timing->open_ms += 1000 * (monotime() - start); errno = saved; | |||||
| timing->ctl_open_ms = 1000 * (monotime() - start); | |||||
| timing->open_ms += timing->ctl_open_ms; errno = saved; | |||||
| if (ctl < 0) goto done; | if (ctl < 0) goto done; | ||||
| } | } | ||||
| start = monotime(); | start = monotime(); | ||||
| @@ -699,6 +699,27 @@ static void log_rejections(const rejection_t *rejections, unsigned count) | |||||
| rejections[i].stage ? rejections[i].stage : "pending_or_unknown"); | rejections[i].stage ? rejections[i].stage : "pending_or_unknown"); | ||||
| } | } | ||||
| /* Diagnostic threshold only: does not change the rescue policy. */ | |||||
| #define SIGNAL_DISPATCH_SLOW_MS 100.0 | |||||
| static void action_blocked(const char *reason, enum state state, | |||||
| const sample_t *s, int refresh_ok, int refresh_errno, | |||||
| const char *refresh_error, double refresh_ms, const char *source) | |||||
| { | |||||
| char warmup[96] = ""; | |||||
| if (!strcmp(reason, "armed_warmup")) | |||||
| snprintf(warmup, sizeof (warmup), " elapsed_seconds=%.1f required_seconds=60", | |||||
| monotime() - started); | |||||
| emit_important(!strcmp(reason, "free_above_desfree") ? LOG_INFO : LOG_WARNING, | |||||
| "event=action_blocked reason=%s state=%s free_pages=%" PRIu64 | |||||
| " desfree=%" PRIu64 " minfree=%" PRIu64 " refresh_ok=%d" | |||||
| " candidate_source=%s refresh_errno=%d refresh_error=%s refresh_latency_ms=%.3f" | |||||
| " sample_source=%s scan_complete=%d latest_complete_age_ms=%.1f%s", | |||||
| reason, state_name(state), s->free, s->des, s->min, refresh_ok, | |||||
| source, refresh_errno, refresh_error, refresh_ms, | |||||
| refresh_ok == 1 ? "refresh" : "last_sample", scan_complete, | |||||
| latest_complete_scan_at > 0 ? 1000 * (s->at - latest_complete_scan_at) : -1.0, warmup); | |||||
| } | |||||
| static void act(sample_t *sample) | static void act(sample_t *sample) | ||||
| { | { | ||||
| plan_t plan; | plan_t plan; | ||||
| @@ -709,6 +730,9 @@ static void act(sample_t *sample) | |||||
| double validation_ms = 0, kill_attempt_mono = 0, kill_return_mono = 0; | double validation_ms = 0, kill_attempt_mono = 0, kill_return_mono = 0; | ||||
| double dispatch_ms = 0, kill_ms = 0, enqueue_start, enqueue_ms; | double dispatch_ms = 0, kill_ms = 0, enqueue_start, enqueue_ms; | ||||
| double policy_start, cooldown_ms = 0; | double policy_start, cooldown_ms = 0; | ||||
| double refresh_start, refresh_ms; | |||||
| int refresh_rc, refresh_errno; | |||||
| const char *initial_source = controller.has_pending ? "pending" : "not_selected"; | |||||
| unsigned i, rank = 0, rejected = 0; | unsigned i, rank = 0, rejected = 0; | ||||
| unsigned budget_ms; | unsigned budget_ms; | ||||
| int ticket = -1; | int ticket = -1; | ||||
| @@ -722,15 +746,23 @@ static void act(sample_t *sample) | |||||
| return; | return; | ||||
| } | } | ||||
| if (cfg.armed && monotime() - started < 60) { | if (cfg.armed && monotime() - started < 60) { | ||||
| emit_important(LOG_WARNING, | |||||
| "event=action_blocked reason=armed_warmup state=%s elapsed_seconds=%.1f" | |||||
| " required_seconds=60", state_name(engine.state), monotime() - started); return; | |||||
| action_blocked("armed_warmup", engine.state, sample, -1, 0, | |||||
| "not_attempted", 0, initial_source); return; | |||||
| } | } | ||||
| /* O(1) critical path: kstat refresh + cached identity validation, no walk. */ | /* O(1) critical path: kstat refresh + cached identity validation, no walk. */ | ||||
| if (platform_sample(&fresh) || monotime() - fresh.at > 2 || fresh.free >= fresh.des) { | |||||
| emit_important(LOG_WARNING, | |||||
| "event=action_blocked reason=refresh_failed_or_above_desfree state=%s", | |||||
| state_name(engine.state)); return; | |||||
| refresh_start = monotime(); | |||||
| errno = 0; | |||||
| refresh_rc = platform_sample(&fresh); | |||||
| refresh_errno = refresh_rc ? errno : 0; | |||||
| refresh_ms = 1000 * (monotime() - refresh_start); | |||||
| if (refresh_rc || monotime() - fresh.at > 2) { | |||||
| action_blocked("refresh_failed", engine.state, sample, 0, refresh_errno, | |||||
| refresh_rc ? (refresh_errno ? signal_errno_name(refresh_errno) : "backend_failure") : | |||||
| "stale_sample", refresh_ms, initial_source); return; | |||||
| } | |||||
| if (fresh.free >= fresh.des) { | |||||
| action_blocked("free_above_desfree", engine.state, &fresh, 1, 0, | |||||
| "none", refresh_ms, initial_source); return; | |||||
| } | } | ||||
| current_engine = engine; | current_engine = engine; | ||||
| current_engine.floor = fresh.free <= fresh.min / 2; | current_engine.floor = fresh.free <= fresh.min / 2; | ||||
| @@ -745,9 +777,8 @@ retry_candidate: | |||||
| plan = action_plan(&controller, &cfg, ¤t_engine, &fresh, NULL); | plan = action_plan(&controller, &cfg, ¤t_engine, &fresh, NULL); | ||||
| cooldown_ms = 1000 * (monotime() - policy_start); | cooldown_ms = 1000 * (monotime() - policy_start); | ||||
| if (plan.action == NONE) { | if (plan.action == NONE) { | ||||
| emit_important(LOG_WARNING, | |||||
| "event=action_blocked reason=%s state=%s", plan.reason, | |||||
| state_name(current_engine.state)); return; | |||||
| action_blocked(plan.reason, current_engine.state, &fresh, 1, 0, | |||||
| "none", refresh_ms, "pending"); return; | |||||
| } | } | ||||
| decision_mono = validation_start = monotime(); | decision_mono = validation_start = monotime(); | ||||
| if (validation_acquire(&cfg, &plan.target, cfg.armed, | if (validation_acquire(&cfg, &plan.target, cfg.armed, | ||||
| @@ -793,12 +824,8 @@ retry_candidate: | |||||
| } | } | ||||
| if (!validated) { | if (!validated) { | ||||
| log_rejections(rejections, rejected); | log_rejections(rejections, rejected); | ||||
| emit_important(LOG_WARNING, | |||||
| "event=action_blocked reason=%s state=%s scan_complete=%d" | |||||
| " latest_complete_age_ms=%.1f", blocked_reason, | |||||
| state_name(current_engine.state), scan_complete, | |||||
| latest_complete_scan_at > 0 ? | |||||
| 1000 * (fresh.at - latest_complete_scan_at) : -1.0); | |||||
| action_blocked(blocked_reason, current_engine.state, &fresh, 1, 0, | |||||
| "none", refresh_ms, "cache_exhausted"); | |||||
| return; | return; | ||||
| } | } | ||||
| } | } | ||||
| @@ -853,6 +880,21 @@ retry_candidate: | |||||
| rc ? "failed" : "sent", err, signal_result.attempted, | rc ? "failed" : "sent", err, signal_result.attempted, | ||||
| signal_result.rc, signal_errno_name(err), | signal_result.rc, signal_errno_name(err), | ||||
| !rc ? "complete" : signal_result.rc < 0 ? "error" : "short_write"); | !rc ? "complete" : signal_result.rc < 0 ? "error" : "short_write"); | ||||
| emit("event=pckill_timing signal=%s pid=%d state=%s result=%s" | |||||
| " proc_ctl_open_latency_ms=%.3f pckill_prepare_latency_ms=%.3f" | |||||
| " pckill_write_latency_ms=%.3f pckill_total_latency_ms=%.3f" | |||||
| " kill_attempt_mono=%.9f kill_return_mono=%.9f", | |||||
| plan.action == TERM ? "SIGTERM" : "SIGKILL", plan.target.pid, | |||||
| state_name(current_engine.state), rc ? "failed" : "sent", | |||||
| timing.ctl_open_ms, signal_result.prepare_ms, signal_result.write_ms, | |||||
| signal_result.total_ms, signal_result.attempt, signal_result.returned); | |||||
| if (signal_result.write_ms > SIGNAL_DISPATCH_SLOW_MS) | |||||
| emit_important(LOG_WARNING, | |||||
| "event=signal_dispatch_slow signal=%s pid=%d state=%s latency_ms=%.3f" | |||||
| " threshold_ms=%.3f syscall=write operation=PCKILL result=%s", | |||||
| plan.action == TERM ? "SIGTERM" : "SIGKILL", plan.target.pid, | |||||
| state_name(current_engine.state), signal_result.write_ms, | |||||
| SIGNAL_DISPATCH_SLOW_MS, rc ? "failed" : "sent"); | |||||
| } else { | } else { | ||||
| emit("event=action_decision state=%s candidate_source=%s rank=%u" | emit("event=action_decision state=%s candidate_source=%s rank=%u" | ||||
| " scan_complete=%d candidate_age_ms=%.1f victim_budget_override=%s" | " scan_complete=%d candidate_age_ms=%.1f victim_budget_override=%s" | ||||
| @@ -8,6 +8,7 @@ | |||||
| void platform_dispatch(int handle, enum action action, signal_result_t *out) | void platform_dispatch(int handle, enum action action, signal_result_t *out) | ||||
| { | { | ||||
| double begin = monotime(), returned; | |||||
| long command[2] = {PCKILL, action == TERM ? SIGTERM : SIGKILL}; | long command[2] = {PCKILL, action == TERM ? SIGTERM : SIGKILL}; | ||||
| ssize_t written; | ssize_t written; | ||||
| int saved; | int saved; | ||||
| @@ -16,7 +17,11 @@ void platform_dispatch(int handle, enum action action, signal_result_t *out) | |||||
| errno = 0; | errno = 0; | ||||
| written = write(handle, command, sizeof (command)); | written = write(handle, command, sizeof (command)); | ||||
| saved = errno; /* MUST be before clock/log/close or any other call. */ | saved = errno; /* MUST be before clock/log/close or any other call. */ | ||||
| returned = monotime(); | |||||
| out->rc = (long)written; | out->rc = (long)written; | ||||
| out->error = written < 0 ? saved : 0; | out->error = written < 0 ? saved : 0; | ||||
| out->returned = monotime(); | |||||
| out->returned = returned; | |||||
| out->prepare_ms = 1000 * (out->attempt - begin); | |||||
| out->write_ms = 1000 * (returned - out->attempt); | |||||
| out->total_ms = 1000 * (returned - begin); | |||||
| } | } | ||||
| @@ -0,0 +1,6 @@ | |||||
| time,latency_ms | |||||
| 02:26:42,958.211 | |||||
| 03:25:12,1784.766 | |||||
| 07:30:07,1032.682 | |||||
| 13:33:21,709.773 | |||||
| 13:33:35,946.776 | |||||
| @@ -0,0 +1,4 @@ | |||||
| free_pages,refresh_failed,expected_signals | |||||
| 20000,0,1 | |||||
| 70000,0,1 | |||||
| 20000,1,1 | |||||
| @@ -90,6 +90,7 @@ int main(void) | |||||
| CHECK(platform_prepare(0, &c, &p, 1, 24539, &handle, &t) == 0); | CHECK(platform_prepare(0, &c, &p, 1, 24539, &handle, &t) == 0); | ||||
| CHECK(handle == 12 && errno == 0 && ctl_opens == 1 && closed == 2); | CHECK(handle == 12 && errno == 0 && ctl_opens == 1 && closed == 2); | ||||
| CHECK(t.open_ms > 4.9 && t.read_ms > 2.9 && t.zone_ms > 3.9); | CHECK(t.open_ms > 4.9 && t.read_ms > 2.9 && t.zone_ms > 3.9); | ||||
| CHECK(t.ctl_open_ms > 1.9 && t.ctl_open_ms < 2.1 && t.ctl_open_ms < t.open_ms); | |||||
| CHECK(t.generation_ms > 4.9 && t.memory_ms > 5.9); | CHECK(t.generation_ms > 4.9 && t.memory_ms > 5.9); | ||||
| platform_dispose(handle); | platform_dispose(handle); | ||||
| available = 214354; memset(&t, 0, sizeof (t)); | available = 214354; memset(&t, 0, sizeof (t)); | ||||
| @@ -4,6 +4,7 @@ | |||||
| #undef main | #undef main | ||||
| static double fake_now = 100; | static double fake_now = 100; | ||||
| static double refresh_age, dispatch_delay; | |||||
| static sample_t fake_sample; | static sample_t fake_sample; | ||||
| static process_t fixtures[32]; | static process_t fixtures[32]; | ||||
| static unsigned nfixtures, real_requests, validation_requests, walk_requests, checks; | static unsigned nfixtures, real_requests, validation_requests, walk_requests, checks; | ||||
| @@ -41,7 +42,9 @@ int platform_syslog_write(int priority, const char *message) | |||||
| } | } | ||||
| void platform_syslog_close(void) { syslog_close_count++; } | void platform_syslog_close(void) { syslog_close_count++; } | ||||
| int platform_sample(sample_t *s) | int platform_sample(sample_t *s) | ||||
| { *s = fake_sample; s->at = fake_now; return fail_sample ? -1 : 0; } | |||||
| { *s = fake_sample; s->at = fake_now - refresh_age; | |||||
| if (fail_sample) { memset(s, 0, sizeof (*s)); errno = EIO; return -1; } | |||||
| return 0; } | |||||
| int platform_walk(int (*fn)(const process_t *, void *), void *arg, unsigned budget) | int platform_walk(int (*fn)(const process_t *, void *), void *arg, unsigned budget) | ||||
| { | { | ||||
| unsigned j; (void)budget; | unsigned j; (void)budget; | ||||
| @@ -77,7 +80,8 @@ void platform_dispatch(int handle, enum action action, signal_result_t *out) | |||||
| CHECK(handle == mock_ticket_pid && (action == TERM || action == KILL)); | CHECK(handle == mock_ticket_pid && (action == TERM || action == KILL)); | ||||
| real_requests++; | real_requests++; | ||||
| memset(out, 0, sizeof (*out)); out->attempted = 1; | memset(out, 0, sizeof (*out)); out->attempted = 1; | ||||
| out->attempt = out->returned = fake_now; | |||||
| out->attempt = fake_now; fake_now += dispatch_delay; out->returned = fake_now; | |||||
| out->write_ms = out->total_ms = 1000 * dispatch_delay; | |||||
| out->rc = mock_signal_errno ? -1 : (long)(2 * sizeof (long)); | out->rc = mock_signal_errno ? -1 : (long)(2 * sizeof (long)); | ||||
| out->error = mock_signal_errno; | out->error = mock_signal_errno; | ||||
| } | } | ||||
| @@ -103,6 +107,7 @@ static void reset(void) | |||||
| syslog_count = syslog_open_count = syslog_close_count = 0; | syslog_count = syslog_open_count = syslog_close_count = 0; | ||||
| syslog_facility = -1; fail_syslog = 0; syslog_ident[0] = 0; | syslog_facility = -1; fail_syslog = 0; syslog_ident[0] = 0; | ||||
| syslog_delay_seconds = 0; | syslog_delay_seconds = 0; | ||||
| refresh_age = dispatch_delay = 0; | |||||
| mock_signal_errno = mock_validation_errno = mock_timeout_pid = 0; | mock_signal_errno = mock_validation_errno = mock_timeout_pid = 0; | ||||
| scan_generation = 0; scan_complete = published_scan_complete = 0; | scan_generation = 0; scan_complete = published_scan_complete = 0; | ||||
| scanner_latest_complete_scan_at = latest_complete_scan_at = 0; | scanner_latest_complete_scan_at = latest_complete_scan_at = 0; | ||||
| @@ -286,6 +291,48 @@ static void floor_errno_fixture(void) | |||||
| puts("PASS floor486 fixture, 50ms timeout fallback rank1->2, syscall errno attribution and validation substep logs"); | puts("PASS floor486 fixture, 50ms timeout fallback rank1->2, syscall errno attribution and validation substep logs"); | ||||
| } | } | ||||
| static void diagnostic_fixtures(void) | |||||
| { | |||||
| FILE *f; char line[128], stamp[32], expected[128]; | |||||
| unsigned signals, failure, rows = 0; uint64_t free_pages; double ms; | |||||
| reset(); cfg.armed = 1; async_logging = 1; engine.state = EMERGENCY; | |||||
| f = fopen("tests/fixtures/refresh_recovery_and_failure.csv", "r"); | |||||
| CHECK(f && fgets(line, sizeof (line), f)); | |||||
| while (fscanf(f, "%" SCNu64 ",%u,%u", &free_pages, &failure, &signals) == 3) { | |||||
| fake_sample.free = free_pages; fail_sample = (int)failure; | |||||
| act(&fake_sample); CHECK(real_requests == signals); rows++; | |||||
| } | |||||
| fclose(f); CHECK(rows == 3); | |||||
| CHECK(queued_contains("reason=free_above_desfree state=EMERGENCY free_pages=70000 desfree=65437 minfree=49077 refresh_ok=1")); | |||||
| CHECK(queued_contains("reason=refresh_failed state=EMERGENCY free_pages=20000 desfree=65437 minfree=49077 refresh_ok=0")); | |||||
| CHECK(queued_contains("refresh_errno=5 refresh_error=EIO")); | |||||
| reset(); cfg.armed = 1; async_logging = 1; refresh_age = 2.1; | |||||
| act(&fake_sample); CHECK(real_requests == 0); | |||||
| CHECK(queued_contains("reason=refresh_failed")); CHECK(queued_contains("refresh_error=stale_sample")); | |||||
| reset(); cfg.armed = 1; async_logging = 1; fake_sample.free = fake_sample.des; | |||||
| act(&fake_sample); CHECK(real_requests == 0 && queued_contains("reason=free_above_desfree")); | |||||
| f = fopen("tests/fixtures/pckill_latency_2026_09_28.csv", "r"); | |||||
| CHECK(f && fgets(line, sizeof (line), f)); rows = 0; | |||||
| while (fscanf(f, "%31[^,],%lf\n", stamp, &ms) == 2) { | |||||
| reset(); cfg.armed = 1; async_logging = 1; fake_sample.free = 486; | |||||
| engine.state = EMERGENCY; dispatch_delay = ms / 1000; | |||||
| act(&fake_sample); | |||||
| CHECK(real_requests == 1 && queued_contains("event=signal_sent")); | |||||
| CHECK(queued_contains("event=signal_dispatch_slow signal=SIGKILL")); | |||||
| snprintf(expected, sizeof (expected), "pckill_write_latency_ms=%.3f", ms); | |||||
| CHECK(queued_contains(expected)); rows++; | |||||
| } | |||||
| fclose(f); CHECK(rows == 5); | |||||
| reset(); cfg.armed = 1; async_logging = 1; dispatch_delay = .000010; | |||||
| act(&fake_sample); CHECK(queued_contains("pckill_write_latency_ms=0.010")); | |||||
| CHECK(!queued_contains("event=signal_dispatch_slow")); | |||||
| reset(); cfg.armed = 1; async_logging = 1; dispatch_delay = .100; | |||||
| act(&fake_sample); CHECK(!queued_contains("event=signal_dispatch_slow")); | |||||
| reset(); cfg.armed = 1; async_logging = 1; dispatch_delay = .101; | |||||
| act(&fake_sample); CHECK(queued_contains("event=signal_dispatch_slow signal=SIGTERM")); | |||||
| puts("PASS recovery/failure/stale refresh reasons, desfree equality, no second victim, PCKILL subphases and slow-event threshold"); | |||||
| } | |||||
| static void load_global_ranking_fixture(void) | static void load_global_ranking_fixture(void) | ||||
| { | { | ||||
| FILE *f; | FILE *f; | ||||
| @@ -507,7 +554,7 @@ int main(void) | |||||
| syslog_open_count == 1); | syslog_open_count == 1); | ||||
| emit_important(LOG_INFO, "event=start version=%s mode=dry-run", VERSION); | emit_important(LOG_INFO, "event=start version=%s mode=dry-run", VERSION); | ||||
| CHECK(syslog_count == 1 && syslog_records[0].priority == LOG_INFO && | CHECK(syslog_count == 1 && syslog_records[0].priority == LOG_INFO && | ||||
| strstr(syslog_records[0].message, "event=start version=1.0.8") != NULL); | |||||
| strstr(syslog_records[0].message, "event=start version=1.0.9") != NULL); | |||||
| log_sample(&fake_sample); scan_processes(0); | log_sample(&fake_sample); scan_processes(0); | ||||
| CHECK(syslog_count == 1); /* High-volume sample/delta/proc/cache events stay local. */ | CHECK(syslog_count == 1); /* High-volume sample/delta/proc/cache events stay local. */ | ||||
| { | { | ||||
| @@ -560,6 +607,7 @@ int main(void) | |||||
| puts("PASS selective native syslog ident/facility/severity, exclusions, signal ordering, disable and failure isolation"); | puts("PASS selective native syslog ident/facility/severity, exclusions, signal ordering, disable and failure isolation"); | ||||
| critical_path_faults(); | critical_path_faults(); | ||||
| floor_errno_fixture(); | floor_errno_fixture(); | ||||
| diagnostic_fixtures(); | |||||
| fclose(log); | fclose(log); | ||||
| printf("PASS %u runtime checks; all platform calls mocked, no signals sent\n", checks); | printf("PASS %u runtime checks; all platform calls mocked, no signals sent\n", checks); | ||||
| return 0; | return 0; | ||||
| @@ -12,6 +12,7 @@ static unsigned checks; | |||||
| static volatile int disposed, gate; | static volatile int disposed, gate; | ||||
| static int injected_errno, inherited_errno, calls, poison_clock; | static int injected_errno, inherited_errno, calls, poison_clock; | ||||
| static long injected_rc; | static long injected_rc; | ||||
| static unsigned write_delay_ms; | |||||
| double monotime(void) | double monotime(void) | ||||
| { | { | ||||
| struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); | struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); | ||||
| @@ -42,7 +43,7 @@ void validation_late_result(int pid, const validation_t *t, int error) | |||||
| static ssize_t injected_write(int fd, const void *buf, size_t n) | static ssize_t injected_write(int fd, const void *buf, size_t n) | ||||
| { | { | ||||
| (void)fd; (void)buf; (void)n; | (void)fd; (void)buf; (void)n; | ||||
| calls++; errno = injected_errno; return injected_rc; | |||||
| calls++; pause_ms(write_delay_ms); errno = injected_errno; return injected_rc; | |||||
| } | } | ||||
| #define write injected_write | #define write injected_write | ||||
| #include "../src/signal.c" | #include "../src/signal.c" | ||||
| @@ -86,6 +87,15 @@ int main(void) | |||||
| platform_dispatch(10, TERM, &r); | platform_dispatch(10, TERM, &r); | ||||
| CHECK(r.error == 0 && r.rc == injected_rc); | CHECK(r.error == 0 && r.rc == injected_rc); | ||||
| __sync_lock_test_and_set(&poison_clock, 0); | __sync_lock_test_and_set(&poison_clock, 0); | ||||
| write_delay_ms = 10; | |||||
| platform_dispatch(10, TERM, &r); | |||||
| CHECK(r.write_ms >= 9 && r.write_ms < 200); | |||||
| CHECK(r.total_ms >= r.write_ms && r.total_ms - r.write_ms - r.prepare_ms < .001); | |||||
| write_delay_ms = 80; | |||||
| platform_dispatch(10, KILL, &r); | |||||
| CHECK(r.write_ms >= 79 && r.write_ms < 300); | |||||
| CHECK(r.returned > r.attempt && r.error == 0); | |||||
| write_delay_ms = 0; | |||||
| __sync_lock_test_and_set(&gate, 1); | __sync_lock_test_and_set(&gate, 1); | ||||
| for (i = 0; i < 16; i++) { | for (i = 0; i < 16; i++) { | ||||
| p.pid = 100 + i; | p.pid = 100 + i; | ||||