diff --git a/CHANGELOG.md b/CHANGELOG.md index a65271c..6d045f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 1.0.11 — 2026-10-09 — panic34: staged persistent minfree risk + +- Fixed VM sample window, validated configurable margin/duration/sample counts. + Intermittent nscan is tolerated; anonymous growth evaluated between samples. +- Persistent minfree risk opens8→12, then12→16 after three additional seconds + of continuous risk. FLOOR alone still opens only12; existing persistent FLOOR + authorization remains. No immediate FLOOR-to16 path; configuration rejects + MAX_FLOOR_VICTIMS_EPISODE above12 so it cannot bypass this invariant. +- Brief recovery preserves episode victims, override stage and window; stable + RECOVERY→NORMAL resets them. Old evidence cannot grant new authorization. +- No dependency on proc scan completeness. Missing usable cache candidates + produce no_revalidatable_candidate with explicit cache/rejection context. +- Replace ambiguous repeated *_used with total/tier/limits/stage; change-only + deadman_risk and budget_override events, rate-limited blocked diagnostics. +- Preserve thresholds, state transitions, ranking/weights, protection, grace, + PCKILL and async logging. No swap/SLOG changes. Source-only, no publication. +- panic34 synthetic fixture plus negative/window/recovery/single-FLOOR tests; + panic33 and all previous fixtures retained. + ## 1.0.10 — 2026-10-08 — panic33: persistent FLOOR and incremental cache - Normal budgets stay 4/8/12; finite hard ceiling defaults to 16 (configuration diff --git a/CHECKPOINT.md b/CHECKPOINT.md index 9f2299f..cb08030 100644 --- a/CHECKPOINT.md +++ b/CHECKPOINT.md @@ -836,3 +836,19 @@ candidatos parciales y continúa cursor; un bloqueo dentro de syscall aún puede detener descubrimiento, no rescate. Nuevos eventos victim_budget/budget_override. Suite 957 comprobaciones (240+620+70+27), sanitizadores y portabilidad. Source-only, sin despliegue, señales reales, credenciales ni push remoto. + +## 24. Addendum v1.0.11 — panic34 (2026-10-09) + +El operador reporta minfree=49077, free cerca de48500–49077, floor=0, presupuesto +agotado en8, nscan y anon pageout activos, /proc incompleto con más de376 s de +edad. Dump: freemem=643 y pageout en asignación de memoria para ZFS. Swap4K +reportado, no se cambia swap ni se atribuye causa al SLOG. Informe dedicado: +INCIDENT_PAGEOUT_DEADMAN_PANIC34_2026_10_09.md. +Ventana VM preasignada con parámetros5 s/6 muestras/margen1024; near5/6, +nscan3/6, anon4 incrementos. Escalones8→12 y12→16 tras3 s adicionales continuos. +La aclaración posterior del operador prevalece: ninguna muestra FLOOR aislada +abre16; se conserva la vía FLOOR persistente de v1.0.10. Recuperación breve +conserva contexto y presupuestos; sólo estable hasta NORMAL cierra el episodio. +Total/tier/límites explícitos, eventos asíncronos de cambio y bloqueos limitados. +Suite1267 (369+801+70+27), sanitizadores/analyzer/C99/shell/XML y checksums. +Source-only: sin publicación, despliegue, señales reales ni cambio ACTION_MODE. diff --git a/INCIDENT_PAGEOUT_DEADMAN_PANIC34_2026_10_09.md b/INCIDENT_PAGEOUT_DEADMAN_PANIC34_2026_10_09.md new file mode 100644 index 0000000..ef2fea8 --- /dev/null +++ b/INCIDENT_PAGEOUT_DEADMAN_PANIC34_2026_10_09.md @@ -0,0 +1,99 @@ +# panic34 — revisión 2026-10-09 + +Fuente: especificación suministrada por el operador. No se adquirió ni analizó +independientemente el dump ni logs completos. «Confirmado» abajo significa +reportado como confirmado por el operador, no validación nativa de esta entrega. + +## Panicinfo y memstat aportados + +```text +pageout_deadman: stuck pushing the same page for 90 seconds +freemem=643 +pagesize=4096 +Kernel 7332 MiB 22% +ZFS File Data 1756 MiB 5% +Anon 23193 MiB 71% +Free 2 MiB +``` + +643 páginas corresponden a2,51 MiB. Tramo final persistido del guard: +minfree=49077, free_pages aproximadamente48500–49077, EMERGENCY, floor=0, +persistent_floor_samples=0 y hard_floor_ready=0. El valor643 del dump es +posterior; no se debe convertir en una muestra FLOOR que el guard hubiera visto. + +## Stack de pageout/1 tal como se suministró + +```text +pageout +→ swap_putpage +→ swap_putapage +→ zvol_strategy +→ dmu_write +→ dmu_write_impl +→ dmu_buf_will_fill +→ dbuf_noread +→ arc_alloc_buf +→ arc_hdr_alloc +→ arc_get_data_abd +→ abd_alloc +→ segkmem_zio_alloc +→ segkmem_page_create +→ page_create_va +→ búsqueda de página física +``` + +La última línea es una descripción del operador, no un símbolo C. Se conserva +esta secuencia sin inventar frames, direcciones ni timestamps. Hace visible +la dependencia de asignación de memoria de ZFS en la ruta del paging; no prueba +por sí sola cuánto tiempo se pasó en cada frame ni una causa raíz única. + +## Swap y observaciones del guard + +Propiedades reportadas: volblocksize=4K, primarycache=metadata, +secondarycache=none, logbias=throughput, sync=always, compression=lz4. +La alineación nominal de bloque/página no elimina la necesidad de memoria para +ARC/ABD/ZFS. No se cambió ninguna propiedad ni se investigó hardware en esta tarea. + +Se reporta v1.0.10 activa, presupuesto agotado en8 y ningún budget_override. +nscan alcanzó6772446, anon pageout y swap físico crecían, mientras +proc_scan complete=0 y latest_complete_age_ms aproximadamente376000–392000. +El guard bloqueaba con emergency_victim_budget_exhausted y +candidate_source=not_selected, antes de intentar seleccionar otra víctima. + +En v1.0.10, FLOOR aislado sí permitía9–12, y FLOOR persistente13–16. Alrededor +de minfree sin FLOOR no existía ninguna de esas vías. Los cuatro campos *_used=8 +eran el mismo consumo acumulado del episodio, no cuatro presupuestos consumidos. + +## Confirmado en la evidencia suministrada + +- v1.0.10 activa y swap con volblocksize=4K. +- Presupuesto agotado en8 y override no activado. +- floor nunca observado en el tramo final persistido, cerca de minfree. +- Pageout observado intentando asignar memoria para ZFS; freemem final643. +- Lecturas VM seguían funcionando mientras el scanner /proc estaba degradado. + +## No demostrado + +- Que las víctimas9–16 hubieran evitado con certeza el panic. +- Instante exacto en que pageout dejó de progresar. +- Que un SLOG, disco, controlador u otro subsistema fuera la causa raíz. + +SLOG/ZIL queda fuera de alcance. Esta entrega no atribuye el incidente al SLOG +ni deriva de sus propiedades una garantía sobre la ruta de I/O. Cualquier +investigación de errores SMART/SATA, resets o contención requiere evidencia +específica y separada. + +## Respuesta v1.0.11 + +Ventana VM preasignada y tiempo monotónico: por defecto5/6 muestras cerca de +minfree,3/6 con nscan activo y4 incrementos anónimos en5 s. El riesgo abre8→12; +tres segundos adicionales continuos abren12→16. No exige scan /proc completo. +Se respeta la aclaración del operador: FLOOR inmediato sólo12; llegar a16 exige +la persistencia FLOOR anterior o la nueva persistencia escalonada, nunca una +muestra FLOOR aislada. Recuperación breve conserva contexto; estable cierra. + +Fixture: tests/fixtures/panic34_persistent_minfree_budget_exhaustion_2026_10_09.csv. +Es una reconstrucción sintética de las condiciones aportadas, no un log exacto. +Se preserva la limitación: una syscall /proc bloqueada impide nuevos +descubrimientos mientras no retorne, aunque rescate use candidatos ya publicados. +Entrega source-only, sin despliegue, publicación, cambios de swap ni señales reales. diff --git a/Makefile b/Makefile index d10ede8..47b98b1 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,7 @@ test-runtime: tests/test_runtime.c tests/fixtures/global_victim_ranking_smartos_ test-runtime: tests/fixtures/pckill_latency_2026_09_28.csv tests/fixtures/refresh_recovery_and_failure.csv test-runtime: tests/fixtures/panic33_persistent_floor_budget_exhaustion_2026_10_08.csv +test-runtime: tests/fixtures/panic34_persistent_minfree_budget_exhaustion_2026_10_09.csv test-distribution: tests/test_distribution.sh smf/o6h-memory-guard.xml etc/o6h-memory-guard.conf src/main.c sh tests/test_distribution.sh diff --git a/README.md b/README.md index e2b2f92..900b3ea 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,89 @@ -# o6h-memory-guard 1.0.10 +# o6h-memory-guard 1.0.11 -## panic33: hard-floor y caché incremental +## panic34: riesgo VM persistente alrededor de minfree + +v1.0.11 añade `persistent_minfree_deadman_risk` sin cambiar los umbrales de +estado, ranking, protecciones, gracias TERM/KILL ni transporte PCKILL. +**Una única muestra FLOOR nunca abre16:** FLOOR inmediato sigue limitado a12. +Para13–16 se necesita FLOOR persistente según v1.0.10, o el nuevo camino +escalonado descrito abajo. No se aplica la propuesta de FLOOR inmediato a16 +del documento inicial: la aclaración del operador la descartó expresamente. + +Configuración distribuida: + +```text +DEADMAN_RISK_MARGIN_PAGES 1024 +DEADMAN_RISK_SECONDS 5 +DEADMAN_RISK_SAMPLES 6 +DEADMAN_RISK_NSCAN_ACTIVE_SAMPLES 3 +DEADMAN_RISK_ANONPGOUT_ACTIVE_SAMPLES 4 +``` + +Ventana fija preasignada (máximo32 muestras): por defecto al menos5 de6 cerca +de minfree (free<=minfree+1024 y free0 y4 incrementos +anónimos entre pares consecutivos, durante al menos5 segundos monotónicos. +Debe haber EMERGENCY y escasez actual dentro de la banda; no crea un estado +nuevo. Un nscan=0 aislado no borra la ventana. Anon debe ser válido, sin reset +de contador en la ventana; swap físico se registra como corroboración, no es +trigger ni requisito de crecimiento. El delta físico es firmado (es un gauge). + +Al agotarse8 y confirmarse el riesgo, el escalón1 autoriza hasta12. Después de +tres segundos adicionales de riesgo continuo desde ese primer override, +escalón2 autoriza hasta16. Una recuperación pausa esa continuidad: exige tres +segundos nuevos de riesgo para promover de1 a2. Si2 ya se ganó, se conserva +pero sólo se usa con evidencia actual de riesgo. FLOOR persistente conserva su +vía independiente a16; FLOOR aislado no salta el segundo escalón. + +RECOVERY breve conserva víctimas, etapa de override, caché y muestras (no +permisos de actuar con memoria recuperada). RECOVERY estable hasta NORMAL +reinicia ventana/override/presupuesto, conservando la ventana móvil de rate +limit existente. Los overruns no borran el historial. Para no autorizar con +datos antiguos se rechaza una ventana que abarque más de3 veces +DEADMAN_RISK_SECONDS y un permiso VM de más de1,5 segundos. El refresco actual +debe seguir en la banda, con contador anónimo válido y sin retroceder. + +Rangos validados: margen0–65536 páginas, duración5–60 s, muestras6–32, +nscan activo1–N y anon activo1–(N-1); duplicados/valores inválidos se rechazan. +Con muestreo de1 s conviene N>=duración+1: por ejemplo,6 muestras para5 s. +Valores incompatibles con la cadencia pueden impedir el override, no relajarlo. + +El scanner **VM** se distingue del scanner **/proc**: `vm_page_scanner_active` +es la corroboración de nscan por ventana; `proc_scanner_complete`/ +`proc_scan_complete` indican la situación del recorrido de procesos. Un scan +incompleto o antiguo no inhibe el override si hay identidades revalidables. +Si no las hay: `action_blocked reason=no_revalidatable_candidate`, con +cache_entries/rejected_entries/latest_complete_age_ms/proc_scan_complete. + +Observabilidad asíncrona, después del rescate: + +- `deadman_risk active=0|1`: sólo cambios, con contadores de ventana y deltas + anónimo/físico y sus flags de validez. +- `budget_override reason=persistent_minfree_deadman_risk`, from_limit=8, + to_limit=12; segundo escalón: reason=persistent_minfree_deadman_risk_continues, + from_limit=12, to_limit=16. Sólo cambios de escalón. +- `budget_override_blocked`: persistencia insuficiente, scanner VM inactivo o + anon sin crecimiento; como máximo uno cada30 s. +- Presupuesto: episode_victims_total/current_budget_tier/current_limit, + normal_limit/emergency_limit/floor_limit/absolute_limit y override_stage. + Se retiran los cuatro *_used ambiguos: eran el mismo total repetido, no cuotas. + Los límites siguen siendo acumulados, nunca sumables. El tier floor/absolute + nombra el techo vigente, aunque se alcance por riesgo VM sin floor=1. + override_stage describe sólo el camino de riesgo VM; la vía FLOOR persistente + conserva su evento persistent_floor independiente. + +Fixture sintético: `panic34_persistent_minfree_budget_exhaustion_2026_10_09.csv`. +Informe: `INCIDENT_PAGEOUT_DEADMAN_PANIC34_2026_10_09.md`. No se promete que más +víctimas hubieran evitado el panic. Source-only; no se cambian swap ni SLOG. + +## panic33: vía FLOOR y caché incremental conservadas Presupuestos acumulados del mismo episodio: normal 4, emergency 8, floor 12 y `MAX_HARD_FLOOR_VICTIMS_EPISODE 16`. No son cuotas sumables. Se exige -normal <= emergency <= floor <= hard <=16. TERM+KILL cuenta una sola identidad. +normal <= emergency <= floor <= hard <=16, con floor<=12 desde v1.0.11. +Se rechaza una configuración FLOOR16 para no habilitar16 sin persistencia. +TERM+KILL cuenta una sola identidad. -El tramo adicional requiere EMERGENCY, FLOOR existente (free<=minfree/2), +En la vía FLOOR, el tramo adicional requiere EMERGENCY, FLOOR (free<=minfree/2), floor normal agotado, nscan>0 y delta reciente positivo/válido de anon_pageout. Debe persistir cinco segundos/seis muestras, con separación máxima de 1,5 s. Una muestra aislada, hueco, contador reseteado/delta no positivo, pageout @@ -38,12 +115,11 @@ bloqueo siguen disponibles, pero no se promete descubrir otros hasta que retorne. Rescate no espera el mutex del scanner (trylock); no se crean hilos ilimitados ni se altera PCKILL, logger o syslog asíncrono. -Eventos nuevos: `victim_budget tier=normal|emergency|floor|hard_floor` con +Eventos conservados: `victim_budget tier=normal|emergency|floor|hard_floor|absolute` con used/limit/remaining/mode al consumir víctima o cerrar episodio; `budget_override reason=persistent_floor` (LOG_CRIT) por víctima adicional. -Bloqueos: normal_used/emergency_used/floor_used/hard_floor_used representan el -mismo consumo acumulado frente a los cuatro techos, no cuatro contadores -independientes; añaden nscan y anonpgout_delta/anonpgout_delta_ok. Las muestras +Bloqueos: el total y los límites se separan explícitamente desde v1.0.11; +añaden nscan y anonpgout_delta/anonpgout_delta_ok. Las muestras incluyen hard_floor_ready y persistent_floor_samples. Informe: `INCIDENT_PAGEOUT_DEADMAN_PANIC33_2026_10_08.md`. @@ -100,12 +176,12 @@ Si no hay herramientas de desarrollo, compilar en una **zona nativa SmartOS** de la misma plataforma (no LX), ejecutar allí `gmake test`, y copiar el binario junto a este árbol al host. No instalar un compilador en un nodo bajo presión. -Copiar el archivo distribuido a `/var/tmp/o6h-memory-guard-1.0.10.tar.gz`: +Copiar el archivo distribuido a `/var/tmp/o6h-memory-guard-1.0.11.tar.gz`: ```sh mkdir -p /opt/custom/src cd /opt/custom/src -gzip -dc /var/tmp/o6h-memory-guard-1.0.10.tar.gz | tar -xf - +gzip -dc /var/tmp/o6h-memory-guard-1.0.11.tar.gz | tar -xf - cd /opt/custom/src/o6h-memory-guard gmake CC=gcc gmake test CC=gcc @@ -505,7 +581,7 @@ Alloy/Loki puede configurarse fuera del guard, pero esta entrega no lo hace. ## Modelo de activación real (operación futura; no ejecutada en esta entrega) -v1.0.10 debe compilarse y observarse primero en SmartOS con `ACTION_MODE dry-run`. +v1.0.11 debe compilarse y observarse primero en SmartOS con `ACTION_MODE dry-run`. El manifiesto ya no contiene una segunda puerta oculta. Cuando exista aprobación operativa explícita, el único cambio de modo es editar la configuración: diff --git a/SHA256SUMS b/SHA256SUMS index bf8e7fc..b038fe7 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -1,31 +1,32 @@ ab22dd5e73f360b7f08839f95e35e970ce4401e4541071e802cf2d24e3d8e2a6 .gitignore bba839eb1ece469050db8beb6a7eccdbe07a51eadc2a920c0cdf0f707330b800 AUDIT_PCKILL_LATENCY_2026_09_28.md -28a8fa917b95065c76d5c91ac350fee3e5cf6fa2fcafedb272125dda70c471f0 CHANGELOG.md -d395a34c4806489ea8e9cf79220f708442d47e500ce5e695d64d9a629e1cd33b CHECKPOINT.md +08ae485b8e0b92170ce433c6f487e7c15fddfbfd489ef50251803d8baf2f6157 CHANGELOG.md +74b29b7f9575ecbef1983828dbae399640414b196a10c254f7c832755d67ee0f CHECKPOINT.md 4d0b3983bbb0d2a51c61ce0f99d778cc79768ed371bdb5ced382c0fb5dfcaf5f INCIDENT_FLOOR_KILL_FAILED_2026_09_23.md a0f96300e751d08ceeef1360325ed54008b703fa6bd07df9d5368674e580f113 INCIDENT_PAGEOUT_DEADMAN_2026_09.md fb6b65a2655ccd808d102fd917b4866cbe939f24fac4e4aeef226a6aeda6d588 INCIDENT_PAGEOUT_DEADMAN_AVOIDED_2026_09_21.md 0f18d6dd3e0d027c1e8dbfce05ce6ac3beb70f3f66d1a77a8d0c5e0f4ae2a039 INCIDENT_PAGEOUT_DEADMAN_PANIC32_2026_09_22.md 153d3a57333dabc3227e3660fd8830c1a38663c0aa0190e2ea83b581cfbf1559 INCIDENT_PAGEOUT_DEADMAN_PANIC33_2026_10_08.md -78a5e8cb5a04b633dd0f0512984c5e77d5a8638eef36e66deb3e5f6150801677 Makefile -46bea1f497107d7825839e9b8ae9ede28255068fe298b3de5113b97a07ca3bc3 README.md -2ab1c9c33f0cb059720101be72262574012dd9e50b8d2b78c075c6aa3cdce6ba VALIDATION.md -1e65ea7af3b643052219190a9f3dd28afef78edb2b5e318fbbab4c69085b0cb0 etc/o6h-memory-guard.conf +b4f82d35598412e81233daa2a7749763e78f73d08107ba96b0469793e85ba68f INCIDENT_PAGEOUT_DEADMAN_PANIC34_2026_10_09.md +a9d7fdd61ad4c6587708187c6886e4889bd917b565679d9dd1cf2aa1cf33c7d1 Makefile +a2b99c6793c43e7a7e5861f9adef10d0e5468038659fd958b1f722669e3692e9 README.md +785f14c3df352dba978b64fd9d67c3720af8ad841c6b67cff28311c7ceaca6d3 VALIDATION.md +196705e87bf3096a1ad476f2b190d084e6ae3bf719b3fb124eb06a4082ec46a8 etc/o6h-memory-guard.conf f6565d0e1ece0a08721c9893f9ed071d2ab0a2818fa93ed0a14e0050d358ce99 install.sh 92faeb31da43244631f25bb915be07720d662b205d5872dc3db7702ecb883f37 logs/distribution-tests.log 50626dec1924e19b4ca92eec30416fb39c3fe7cff28d98a8e120ccf6247d2834 logs/example.log -c2533657a871956bd36243b16c02a63bbf4137bcad444a495cb367376f64b478 logs/policy-tests.log +e7b285886ccd1b0498149f4728900e38dedfc191f6d95cc3517aa9e6a5f12a0c logs/policy-tests.log c5e2dd83262f119b1515526bb2209937598f33b613c0fed82362385e6b6ef27d logs/portability.log 180733f8807f47abd6bcd90dca6fdc19c2555b64f205940e885edaa48b5642e3 logs/prepare-tests.log -0fe337f74419a60f6ed6c4c949e1aeaf0711dc80741e03051c9bd6dab6a45fd8 logs/runtime-tests.log -b463307c75d3e24338b1dde62c993b03db4cf9010ff1c56b4330b69d7aa14150 logs/sanitizers.log +3974867b0470605993e274ee7aa8544ca707b99332d19d6e96814f634dd008e3 logs/runtime-tests.log +73802be6ce5a6f5ba0bc10c5cec4775f26a6bbcef6496c4cd1e32c920f6ef8b8 logs/sanitizers.log ec90a51222ef270ab0294532bc9396862afec1679824026848c2bc61165348be logs/validation-tests.log 55cf49840cf1d22e4e02bedf579a10207856a0a5b40c86ae2cf6ee2f00d3f972 smf/o6h-memory-guard.xml -b0ecefbadfb583a9dc06e85705cd4951e689e3099f4b9a5daa2da99fb66e01ee src/config.c -532a369d0275dfddffd1f6bfb0c0bd819487760a059c634bf0060b07bb16d287 src/core.c -10cf9ab6ab8df2a94a732d56dbf166552e2710e6e732c122ae19e614472a39aa src/guard.h +e6110feaef8ccd9259db23a2913b7b73ae81c1c26684a6c9ddf4809cebb4ca51 src/config.c +f7700ee774beb644c995792be2c3fa8a9f523744b883ba01a79ab1b1ce11a086 src/core.c +47cbff8948451cfe0f82cae317a7595344d5b9f644a18a965ebb63d0dbb29d7c src/guard.h ad57cd80c8ae56ffbdc075abfd5d2b68bc115ab1355e5ee60e79359353cd1220 src/illumos.c -9f224c509cc006f1e7b8adfca0faf6fb03f6d13a49bc7aa1d53739b5ebe8e666 src/main.c +87f9fa16b7a1d8b6984d9e405394916afa5689f2ad469e7173ebcc1f73b51723 src/main.c 5bb4e0980de83b29e05bb10ea87d6c04f10df884dc92f74cd6b3b025d41d3332 src/result.c 948bfd760d5a4e1cd301da8b8aa270cfa894d2ec3334d000bd687ef023788388 src/signal.c 1f09afd342ba86580a21f6c04a030d9ea2551a60b669e257295fbf4f43b2fad7 src/support.c @@ -40,11 +41,12 @@ f0d06353d2e326c7bdce776bee6f0a8606cda50d5de957dfdd586b6848fca8da tests/api-sket c5d9d7fc7a39354474d30cd316ea9f122205c2b38e57f927f4a08306b8a62c81 tests/fixtures/pageout_deadman_real_incident_2026_09.csv 025983b24045e24a96532c933715b8760edd8c0b0e11b9f0e019481c1c52a50d tests/fixtures/panic32_daemon_stall_2026_09_22.csv a154f01d5fe41a7986e7ce86cdd0917e0dd2127ed0d295fa6321143e28c8a3d5 tests/fixtures/panic33_persistent_floor_budget_exhaustion_2026_10_08.csv +4e2d187c85cddfb35c2c502e8d634d49f9a5b573c6a563dd5d9c1ee829d9d599 tests/fixtures/panic34_persistent_minfree_budget_exhaustion_2026_10_09.csv b80c56d558d98c7aae6883f5d51ad4652be46999385170e79fb50da17ade325f tests/fixtures/pckill_latency_2026_09_28.csv ccf983995eff740e26feb66200de6e2339a3f31f997d74a9af3815ce35722c86 tests/fixtures/refresh_recovery_and_failure.csv facc4580ffadef29cdce84bf1226a696cba3880bbbec80161beaceb2752bfd77 tests/fixtures/successful_pageout_deadman_avoidance_2026_09_21.csv -fc1b7e17ac3ef1275edf248ddc7e62f0ef85e79d433715aec38e4d4719a5b699 tests/test_distribution.sh -13224898d9850eaa33ef9e5070238f25ffb759d19b77cdceef0094fb637373f5 tests/test_guard.c +fa23a478e69f63bcf429fc95d618cbaba8102f92dbea7fe8776544571644be15 tests/test_distribution.sh +c5fa32a3bf5f39ef5880769c7e73f45e4e9fcda80b8ee63658702c0629021725 tests/test_guard.c debaacd3890a8b65d163c83ffcd61126fd45ebbc66c3e5c69a1202766ff39d68 tests/test_prepare.c -c6d8093904c632e11ee1139170d1c4f68f693fb9634cca75fceb9f388631560e tests/test_runtime.c +48d899fde732acf449f96930d1a432621f10775802208dfc3ee330d473e81076 tests/test_runtime.c 9576bb5d5dcb525c7c89ec5dc929b896e8c5b61beffe4d7313f89d823476d957 tests/test_validation.c diff --git a/VALIDATION.md b/VALIDATION.md index 8b43043..64221ea 100644 --- a/VALIDATION.md +++ b/VALIDATION.md @@ -1,11 +1,25 @@ -# Validación de la entrega v1.0.10 +# Validación de la entrega v1.0.11 -Fecha: 2026-10-08. Entrega source-only. **No se desplegó, no se modificó el +Fecha: 2026-10-09. Entrega source-only. **No se desplegó, no se modificó el nodo armado y no se enviaron señales reales.** La configuración conserva `ACTION_MODE dry-run`, syslog `local5`, cero UUID `allow` y SMF deshabilitado. ## Motivo +panic34: presupuesto agotado en8 con floor=0 cerca de minfree. Ventana de riesgo +VM independiente de /proc, escalones8→12→16 y contrato de presupuestos explícito. +FLOOR aislado mantiene12; llegar a16 exige persistencia. Informe: +INCIDENT_PAGEOUT_DEADMAN_PANIC34_2026_10_09.md. Fixture sintético, no dump/log real. +Se prueba el orden real de ticks (rescate retenido antes de engine_step), una +muestra nscan=0, caché con último scan completo >376 s y revalidación simulada. +Casos negativos: sin pageout, sin escasez, pico de dos muestras, contador que +retrocede, pageout inválido y evidencia demasiado antigua. Ventana máxima32, +rollback de reloj, overruns de2 s, recuperación breve/estable, deduplicación, +sin candidatos y techo16. Tests de una sola muestra FLOOR en episodio nuevo, +escalón1 con ventana ya válida y ruta runtime: nunca habilita16 por sí sola. + +Regresión conservada de v1.0.10: + panic33: presión extrema, presupuesto agotado y caché degradada. Se añade techo hard-floor 16 condicionado a persistencia y publicación incremental. Informe: INCIDENT_PAGEOUT_DEADMAN_PANIC33_2026_10_08.md. Fixture reconstruido sintéticamente. @@ -74,12 +88,12 @@ no está confirmado si lo causó syslog, guard.log u otra syscall. Véase ## Pruebas ejecutadas -Total: **957 comprobaciones C** (240 + 620 + 70 + 27), más contrato de +Total: **1267 comprobaciones C** (369 + 801 + 70 + 27), más contrato de distribución, análisis estático, shell, XML y checksums. No equivale a prueba nativa SMF/ABI: se usan sketches de API e I/O simulado en Linux. -- Política: 240 comprobaciones; incluye los fixtures previos y no enlaza backend. -- Runtime simulado: 620 comprobaciones; ninguna señal nativa. +- Política: 369 comprobaciones; incluye los fixtures previos y no enlaza backend. +- Runtime simulado: 801 comprobaciones; ninguna señal nativa. - Preparación concurrente/dispatch: 70 comprobaciones. Espera real inyectada de 1283 ms; abandono con presupuesto de 50 ms, fallback a otro trabajador, descarte tardío, deduplicación, saturación y recuperación del pool. Tolerancias del test local: @@ -154,7 +168,7 @@ semántica monotónica de `sampling_overrun`. recuperación y saturación del pool. Verificar semántica nativa O_NONBLOCK/ctl. 9. Medir entry/return de write, pr_wait_die, locks y off-CPU en la revisión nativa correspondiente para atribuir los 0,7–1,8 s; no se ejecutó en esta entrega. -10. Mantener dry-run hasta revisión humana explícita. v1.0.10 no se considera apta +10. Mantener dry-run hasta revisión humana explícita. v1.0.11 no se considera apta para producción sólo por superar las pruebas portables. ## Límites diff --git a/etc/o6h-memory-guard.conf b/etc/o6h-memory-guard.conf index 2d3b30a..d84ce44 100644 --- a/etc/o6h-memory-guard.conf +++ b/etc/o6h-memory-guard.conf @@ -1,4 +1,4 @@ -# o6h-memory-guard 1.0.10 -- whitespace-separated, no shell evaluation. +# o6h-memory-guard 1.0.11 -- whitespace-separated, no shell evaluation. # Restart required for configuration changes. Duplicate scalar keys are errors. ACTION_MODE dry-run SYSLOG_ENABLED yes @@ -19,8 +19,13 @@ MAX_VICTIMS_EPISODE 4 MAX_VICTIMS_MINUTE 4 MAX_EMERGENCY_VICTIMS_EPISODE 8 MAX_FLOOR_VICTIMS_EPISODE 12 -# Absolute ceiling; only persistent corroborated FLOOR may exceed the normal 12. +# Absolute ceiling; persistent FLOOR or staged VM deadman risk required above12. MAX_HARD_FLOOR_VICTIMS_EPISODE 16 +DEADMAN_RISK_MARGIN_PAGES 1024 +DEADMAN_RISK_SECONDS 5 +DEADMAN_RISK_SAMPLES 6 +DEADMAN_RISK_NSCAN_ACTIVE_SAMPLES 3 +DEADMAN_RISK_ANONPGOUT_ACTIVE_SAMPLES 4 MIN_RSS_KIB 65536 LOG_MAX_MIB 10 LOG_KEEP 5 diff --git a/logs/policy-tests.log b/logs/policy-tests.log index d2341e7..17ccd1b 100644 --- a/logs/policy-tests.log +++ b/logs/policy-tests.log @@ -9,4 +9,5 @@ SIMULATED t=105 state=CRITICAL pid=123 WOULD_ACTION=SIGKILL reason=memory_not_re SIMULATED state=EMERGENCY floor=1 WOULD_ACTION=SIGKILL reason=floor PASS TERM/KILL sequencing, emergency grace, floor, cooldown, budgets and no-action protections PASS hard-floor persistence, finite ceiling, missing corroboration, gap, brief/stable recovery -PASS 240 checks; no native backend linked, no signals sent +PASS deadman window, staged8/12/16, recovery, overruns, negatives and single FLOOR never16 +PASS 369 checks; no native backend linked, no signals sent diff --git a/logs/runtime-tests.log b/logs/runtime-tests.log index 8341117..2d130df 100644 --- a/logs/runtime-tests.log +++ b/logs/runtime-tests.log @@ -15,4 +15,5 @@ PASS floor486 fixture, 50ms timeout fallback rank1->2, syscall errno attribution PASS recovery/failure/stale refresh reasons, desfree equality, no second victim, PCKILL subphases and slow-event threshold PASS panic33: partial recoveries, budgets, dead PID replacement, incomplete scan, hard ceiling16 PASS incremental pre-return publication, survivor ordering, bounded merge and nonblocking handoff -PASS 620 runtime checks; all platform calls mocked, no signals sent +PASS panic34 real-loop order: intermittent VM scanner, stale proc cache,8->12->16, finite and deduplicated +PASS 801 runtime checks; all platform calls mocked, no signals sent diff --git a/logs/sanitizers.log b/logs/sanitizers.log index 7ff7641..73f34a0 100644 --- a/logs/sanitizers.log +++ b/logs/sanitizers.log @@ -11,7 +11,8 @@ SIMULATED t=105 state=CRITICAL pid=123 WOULD_ACTION=SIGKILL reason=memory_not_re SIMULATED state=EMERGENCY floor=1 WOULD_ACTION=SIGKILL reason=floor PASS TERM/KILL sequencing, emergency grace, floor, cooldown, budgets and no-action protections PASS hard-floor persistence, finite ceiling, missing corroboration, gap, brief/stable recovery -PASS 240 checks; no native backend linked, no signals sent +PASS deadman window, staged8/12/16, recovery, overruns, negatives and single FLOOR never16 +PASS 369 checks; no native backend linked, no signals sent gcc -std=c99 -O1 -g -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -Isrc -Wall -Wextra -Werror -Wformat=2 -pedantic -pthread -fsanitize=address,undefined -fno-omit-frame-pointer -o test-sanitize-runtime tests/test_runtime.c src/core.c src/config.c src/result.c ./test-sanitize-runtime PASS actual daemon dry-run path records WOULD_ACTION=SIGTERM/SIGKILL; zero real requests @@ -31,7 +32,8 @@ PASS floor486 fixture, 50ms timeout fallback rank1->2, syscall errno attribution PASS recovery/failure/stale refresh reasons, desfree equality, no second victim, PCKILL subphases and slow-event threshold PASS panic33: partial recoveries, budgets, dead PID replacement, incomplete scan, hard ceiling16 PASS incremental pre-return publication, survivor ordering, bounded merge and nonblocking handoff -PASS 620 runtime checks; all platform calls mocked, no signals sent +PASS panic34 real-loop order: intermittent VM scanner, stale proc cache,8->12->16, finite and deduplicated +PASS 801 runtime checks; all platform calls mocked, no signals sent gcc -std=c99 -O1 -g -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -Isrc -Wall -Wextra -Werror -Wformat=2 -pedantic -pthread -Itests/api-sketch -fsanitize=address,undefined -fno-omit-frame-pointer -o test-sanitize-validation tests/test_validation.c ./test-sanitize-validation PASS 70 validation checks: real 1283ms worker abandoned, fallback, late disposal, pool saturation; actual dispatch code with mocked write; no signals diff --git a/src/config.c b/src/config.c index 32c3317..292cc86 100644 --- a/src/config.c +++ b/src/config.c @@ -16,6 +16,8 @@ void defaults(config_t *c) c->max_victims = 4; c->max_per_minute = 4; c->max_emergency_victims = 8; c->max_floor_victims = 12; c->max_hard_floor_victims = 16; + c->risk_margin = 1024; c->risk_seconds = 5; c->risk_samples = 6; + c->risk_scan_samples = 3; c->risk_anon_samples = 4; c->min_rss_kib = 65536; c->log_mib = 10; c->log_keep = 5; } @@ -165,15 +167,22 @@ int config_read(FILE *f, config_t *c, char *error, size_t cap) OPT("FLOOR_KILL_DELAY_SECONDS", floor_kill_delay_s, 0, 5, 15) OPT("MAX_CACHED_CANDIDATE_AGE_SECONDS", max_cached_candidate_age_s, 5, 60, 16) OPT("MAX_EMERGENCY_VICTIMS_EPISODE", max_emergency_victims, 1, MAX_VICTIMS, 17) - OPT("MAX_FLOOR_VICTIMS_EPISODE", max_floor_victims, 1, MAX_VICTIMS, 18) + OPT("MAX_FLOOR_VICTIMS_EPISODE", max_floor_victims, 1, 12, 18) OPT("MAX_HARD_FLOOR_VICTIMS_EPISODE", max_hard_floor_victims, 1, 16, 21) + OPT("DEADMAN_RISK_MARGIN_PAGES", risk_margin, 0, 65536, 22) + OPT("DEADMAN_RISK_SECONDS", risk_seconds, 5, 60, 23) + OPT("DEADMAN_RISK_SAMPLES", risk_samples, 6, RISK_SAMPLES_MAX, 24) + OPT("DEADMAN_RISK_NSCAN_ACTIVE_SAMPLES", risk_scan_samples, 1, RISK_SAMPLES_MAX, 25) + OPT("DEADMAN_RISK_ANONPGOUT_ACTIVE_SAMPLES", risk_anon_samples, 1, RISK_SAMPLES_MAX - 1, 26) #undef OPT goto invalid; } if (ferror(f) || c->pressure_s < c->pre_s || c->max_emergency_victims < c->max_victims || c->max_floor_victims < c->max_emergency_victims || - c->max_hard_floor_victims < c->max_floor_victims) goto invalid; + c->max_hard_floor_victims < c->max_floor_victims || + c->risk_scan_samples > c->risk_samples || + c->risk_anon_samples >= c->risk_samples) goto invalid; if (c->armed) { size_t i; int allow = 0; for (i = 0; i < c->nzones; i++) allow |= c->zones[i].policy == ALLOW; diff --git a/src/core.c b/src/core.c index f5201b0..d762002 100644 --- a/src/core.c +++ b/src/core.c @@ -68,6 +68,91 @@ void engine_gap(engine_t *e) e->danger = 0; e->floor = 0; e->persistent_floor_since = -1; e->persistent_floor_samples = 0; e->hard_floor_ready = 0; + e->risk.ready = 0; e->risk.since = -1; +} + +static int near_minfree(const sample_t *s, const config_t *c) +{ + /* Subtraction avoids overflow in minfree + configurable margin. */ + return s->free < s->des && (s->free <= s->min || s->free - s->min <= c->risk_margin); +} + +static void deadman_risk_step(engine_t *e, const config_t *c, const sample_t *s) +{ + deadman_risk_t *r = &e->risk; + unsigned i, first; + int anon_valid = 1, was_ready = r->ready; + const sample_t *oldest, *previous = NULL; + if (e->state == NORMAL && e->recovery_since >= 0 && + s->at - e->recovery_since >= c->recovery_s) { + memset(r, 0, sizeof (*r)); r->since = -1; return; + } + if (r->count && s->at <= r->samples[(r->next + c->risk_samples - 1) % c->risk_samples].at) { + memset(r, 0, sizeof (*r)); was_ready = 0; + } + r->samples[r->next] = *s; + r->next = (r->next + 1) % c->risk_samples; + if (r->count < c->risk_samples) r->count++; + first = (r->next + c->risk_samples - r->count) % c->risk_samples; + oldest = &r->samples[first]; + r->near_count = r->scan_count = r->anon_count = 0; + memset(&r->anon_delta, 0, sizeof (r->anon_delta)); + memset(&r->swap_delta, 0, sizeof (r->swap_delta)); + for (i = 0; i < r->count; i++) { + const sample_t *p = &r->samples[(first + i) % c->risk_samples]; + if (near_minfree(p, c)) r->near_count++; + if (p->scan > 0) r->scan_count++; + if (!p->pageout_ok) anon_valid = 0; + if (previous && previous->pageout_ok && p->pageout_ok) { + if (p->anon_pageout_pages > previous->anon_pageout_pages) r->anon_count++; + if (p->anon_pageout_pages < previous->anon_pageout_pages) anon_valid = 0; + } + previous = p; + } + r->anon_delta.age = r->swap_delta.age = s->at - oldest->at; + if (anon_valid && s->anon_pageout_pages >= oldest->anon_pageout_pages && + s->anon_pageout_pages - oldest->anon_pageout_pages <= INT64_MAX) { + r->anon_delta.valid = 1; + r->anon_delta.value = (int64_t)(s->anon_pageout_pages - oldest->anon_pageout_pages); + } + if (s->swap_physical_ok && oldest->swap_physical_ok && + s->swap_physical_used_bytes <= INT64_MAX && oldest->swap_physical_used_bytes <= INT64_MAX) { + r->swap_delta.valid = 1; + r->swap_delta.value = (int64_t)s->swap_physical_used_bytes - (int64_t)oldest->swap_physical_used_bytes; + } + /* Fixed bounded window. Overruns do not erase it, but ancient evidence + * (>3*configured duration) cannot authorize a new budget. */ + r->ready = e->state == EMERGENCY && near_minfree(s, c) && + r->count == c->risk_samples && r->anon_delta.age >= c->risk_seconds && + r->anon_delta.age <= 3.0 * c->risk_seconds && + r->near_count >= c->risk_samples - 1 && r->scan_count >= c->risk_scan_samples && + r->anon_count >= c->risk_anon_samples && r->anon_delta.valid && r->anon_delta.value > 0; + if (!r->ready) r->since = -1; + else if (!was_ready) r->since = s->at; +} + +int deadman_risk_current(const engine_t *e, const config_t *c, const sample_t *s) +{ + return e->state == EMERGENCY && e->risk.ready && near_minfree(s, c) && + e->risk.count && s->anon_pageout_pages >= + e->risk.samples[(e->risk.next + c->risk_samples - 1) % c->risk_samples].anon_pageout_pages && + s->pageout_ok && s->at >= e->last_at && s->at - e->last_at <= 1.5; +} + +unsigned action_limit(const controller_t *a, const config_t *c, + const engine_t *e, const sample_t *s) +{ + unsigned limit = e->floor ? c->max_floor_victims : + e->state == EMERGENCY ? c->max_emergency_victims : c->max_victims; + if (e->floor && e->hard_floor_ready && e->state == EMERGENCY && + s->free <= s->min / 2 && s->scan > 0 && s->pageout_ok && + s->at >= e->last_at && s->at - e->last_at <= 1.5) + limit = c->max_hard_floor_victims; + if (deadman_risk_current(e, c, s) && a->override_stage) { + unsigned risk_limit = a->override_stage == 2 ? c->max_hard_floor_victims : c->max_floor_victims; + if (risk_limit > limit) limit = risk_limit; + } + return limit > 16 ? 16 : limit; } void engine_step(engine_t *e, const config_t *c, const sample_t *s, @@ -141,6 +226,7 @@ void engine_step(engine_t *e, const config_t *c, const sample_t *s, } else { e->persistent_floor_since = -1; e->persistent_floor_samples = 0; } + deadman_risk_step(e, c, s); e->last_at = s->at; e->last_free = s->free; e->have_last = 1; } @@ -170,7 +256,15 @@ plan_t action_plan(controller_t *a, const config_t *c, const engine_t *e, if (s->at - a->rate[i] < 60) a->rate[n++] = a->rate[i]; a->nrate = n; if (e->state == NORMAL && e->recovery_since >= 0 && - s->at - e->recovery_since >= c->recovery_s) a->nvictims = 0; + s->at - e->recovery_since >= c->recovery_s) { + a->nvictims = 0; a->override_stage = 0; a->override_since = 0; + } + if (a->nvictims >= c->max_emergency_victims && deadman_risk_current(e, c, s)) { + if (!a->override_stage) { a->override_stage = 1; a->override_since = s->at; } + else if (a->override_stage == 1 && s->at - a->override_since >= 3 && + e->risk.since >= 0 && s->at - e->risk.since >= 3) + a->override_stage = 2; + } if (s->free >= s->lots) a->has_pending = 0; if (!e->danger) { a->has_pending = 0; return p; } if (a->has_pending) { @@ -189,20 +283,23 @@ plan_t action_plan(controller_t *a, const config_t *c, const engine_t *e, if (a->nvictims >= c->max_hard_floor_victims) { p.reason = "hard_floor_victim_budget_exhausted"; return p; } - if (!e->hard_floor_ready || e->state != EMERGENCY || - s->free > s->min / 2 || s->scan == 0 || !s->pageout_ok || - s->at < e->last_at || s->at - e->last_at > 1.5) { + if (action_limit(a, c, e, s) <= a->nvictims) { p.reason = "absolute_victim_budget_exhausted"; return p; } - p.budget_override = "hard_floor"; + p.budget_override = e->hard_floor_ready ? "hard_floor" : "deadman_risk"; } else if (a->nvictims >= c->max_victims || a->nrate >= c->max_per_minute) p.budget_override = "floor"; } else if (e->state == EMERGENCY) { if (a->nvictims >= c->max_emergency_victims) { - p.reason = "emergency_victim_budget_exhausted"; return p; + if (a->nvictims >= action_limit(a, c, e, s)) { + p.reason = a->nvictims >= c->max_hard_floor_victims ? + "hard_floor_victim_budget_exhausted" : "emergency_victim_budget_exhausted"; + return p; + } + p.budget_override = "deadman_risk"; } - if (a->nvictims >= c->max_victims || a->nrate >= c->max_per_minute) + else if (a->nvictims >= c->max_victims || a->nrate >= c->max_per_minute) p.budget_override = "emergency"; } else if (a->nvictims >= c->max_victims || a->nrate >= c->max_per_minute) { p.reason = "victim_budget_exhausted"; return p; diff --git a/src/guard.h b/src/guard.h index 39b94e4..dc5c907 100644 --- a/src/guard.h +++ b/src/guard.h @@ -13,7 +13,8 @@ #define CONF_PATH "/opt/custom/etc/o6h-memory-guard.conf" #define LOG_DIR "/opt/custom/var/log/o6h-memory-guard" #define RUN_DIR "/opt/custom/var/run" -#define VERSION "1.0.10" +#define VERSION "1.0.11" +#define RISK_SAMPLES_MAX 32 enum state { NORMAL, PRE_PRESSURE, PRESSURE, CRITICAL, EMERGENCY, RECOVERY }; enum policy { OBSERVE, ALLOW, PROTECT }; @@ -37,6 +38,7 @@ typedef struct { unsigned normal_scan_s, scan_budget_ms, cooldown_s, max_victims; unsigned max_emergency_victims, max_floor_victims, max_hard_floor_victims; unsigned max_per_minute, log_mib, log_keep; + unsigned risk_margin, risk_seconds, risk_samples, risk_scan_samples, risk_anon_samples; uint64_t min_rss_kib; zone_rule_t zones[MAX_ZONES]; size_t nzones; char protected_names[MAX_RULES][32]; size_t nnames; @@ -49,6 +51,13 @@ typedef struct { uint64_t zone_generation, rss_kib; char uuid[64], name[32]; } process_t; +typedef struct { + sample_t samples[RISK_SAMPLES_MAX]; + unsigned next, count, near_count, scan_count, anon_count; + int ready; + double since; + delta_t anon_delta, swap_delta; +} deadman_risk_t; typedef struct { enum state state; double low_since, critical_since, recovery_since, last_at; @@ -59,6 +68,7 @@ typedef struct { unsigned persistent_floor_samples; int hard_floor_ready; delta_t pageout_delta, physical_delta; + deadman_risk_t risk; } engine_t; typedef struct { process_t pending; @@ -66,6 +76,8 @@ typedef struct { double term_at, next_at; process_t victims[MAX_VICTIMS]; unsigned nvictims; double rate[64]; unsigned nrate; + unsigned override_stage; + double override_since; } controller_t; typedef struct { enum action action; @@ -93,6 +105,8 @@ int was_victim(const controller_t *, const process_t *); plan_t action_plan(controller_t *, const config_t *, const engine_t *, const sample_t *, const process_t *); void action_commit(controller_t *, const config_t *, const sample_t *, const plan_t *); +int deadman_risk_current(const engine_t *, const config_t *, const sample_t *); +unsigned action_limit(const controller_t *, const config_t *, const engine_t *, const sample_t *); /* Native backend. No backend is linked into the portable policy tests. */ int platform_init(void); diff --git a/src/main.c b/src/main.c index f19af61..c8d4b97 100644 --- a/src/main.c +++ b/src/main.c @@ -69,6 +69,9 @@ static const char *const metric_names[METRIC_COUNT] = { }; static engine_t engine; static controller_t controller; +static unsigned logged_override_stage, action_rejected_entries; +static int logged_deadman_risk; +static double override_blocked_logged_at = -1; static unsigned scan_generation; static double scan_at, scanner_latest_complete_scan_at, latest_complete_scan_at, started; static int scan_complete; @@ -761,11 +764,63 @@ static void log_rejections(const rejection_t *rejections, unsigned count) /* Diagnostic threshold only: does not change the rescue policy. */ #define SIGNAL_DISPATCH_SLOW_MS 100.0 +static const char *budget_tier(unsigned limit) +{ + if (limit == cfg.max_hard_floor_victims) return "absolute"; + if (limit == cfg.max_floor_victims) return "floor"; + if (limit == cfg.max_emergency_victims) return "emergency"; + return "normal"; +} + +/* Called after rescue, never used as a prerequisite for a signal. */ +static void log_deadman_status(const sample_t *s) +{ + const deadman_risk_t *r = &engine.risk; + if (logged_deadman_risk != r->ready) { + logged_deadman_risk = r->ready; + emit_important(LOG_WARNING, "event=deadman_risk active=%d state=%s free_pages=%" PRIu64 + " minfree=%" PRIu64 " margin_pages=%u near_minfree_samples=%u window_samples=%u" + " nscan_active_samples=%u anonpgout_active_samples=%u" + " anonpgout_window_delta=%" PRId64 " anonpgout_window_delta_ok=%d" + " swap_physical_window_delta=%" PRId64 " swap_physical_window_delta_ok=%d" + " vm_page_scanner_active=%d proc_scanner_complete=%d proc_scan_complete=%d" + " latest_complete_age_ms=%.1f", r->ready, state_name(engine.state), s->free, s->min, + cfg.risk_margin, r->near_count, r->count, r->scan_count, r->anon_count, + r->anon_delta.value, r->anon_delta.valid, r->swap_delta.value, r->swap_delta.valid, + r->scan_count >= cfg.risk_scan_samples, scan_complete, scan_complete, + latest_complete_scan_at > 0 ? 1000 * (s->at - latest_complete_scan_at) : -1.0); + } + if (controller.override_stage > logged_override_stage) { + emit_important(LOG_CRIT, "event=budget_override reason=%s from_limit=%u to_limit=%u" + " episode_victims_total=%u override_stage=%u mode=%s", + controller.override_stage == 1 ? "persistent_minfree_deadman_risk" : + "persistent_minfree_deadman_risk_continues", + controller.override_stage == 1 ? cfg.max_emergency_victims : cfg.max_floor_victims, + controller.override_stage == 1 ? cfg.max_floor_victims : cfg.max_hard_floor_victims, + controller.nvictims, controller.override_stage, cfg.armed ? "armed" : "dry-run"); + } + logged_override_stage = controller.override_stage; + if (engine.state == EMERGENCY && controller.nvictims >= cfg.max_emergency_victims && + !r->ready && (override_blocked_logged_at < 0 || s->at - override_blocked_logged_at >= 30)) { + const char *reason = r->count < cfg.risk_samples || r->near_count < cfg.risk_samples - 1 || + r->anon_delta.age < cfg.risk_seconds || r->anon_delta.age > 3.0 * cfg.risk_seconds ? + "insufficient_persistence" : r->scan_count < cfg.risk_scan_samples ? + "page_scanner_inactive" : "anonpgout_not_growing"; + emit("event=budget_override_blocked reason=%s", reason); + override_blocked_logged_at = s->at; + } +} + static void action_blocked(const char *reason, enum state state, const sample_t *s, int refresh_ok, int refresh_errno, const char *refresh_error, double refresh_ms, const char *source) { char warmup[96] = ""; + engine_t current = engine; + unsigned i, entries = 0, limit; + current.state = state; current.floor = s->free <= s->min / 2; + limit = action_limit(&controller, &cfg, ¤t, s); + for (i = 0; i < CACHED_CANDIDATES; i++) entries += candidate_cache[i].valid != 0; if (!strcmp(reason, "armed_warmup")) snprintf(warmup, sizeof (warmup), " elapsed_seconds=%.1f required_seconds=60", monotime() - started); @@ -774,13 +829,17 @@ static void action_blocked(const char *reason, enum state state, " desfree=%" PRIu64 " minfree=%" PRIu64 " refresh_ok=%d" " candidate_source=%s refresh_errno=%d refresh_error=%s refresh_latency_ms=%.3f" " sample_source=%s scan_complete=%d latest_complete_age_ms=%.1f%s" - " normal_used=%u emergency_used=%u floor_used=%u hard_floor_used=%u" + " episode_victims_total=%u current_budget_tier=%s current_limit=%u" + " normal_limit=%u emergency_limit=%u floor_limit=%u absolute_limit=%u override_stage=%u" + " cache_entries=%u rejected_entries=%u proc_scan_complete=%d" " nscan=%" PRIu64 " anonpgout_delta=%" PRId64 " anonpgout_delta_ok=%d", reason, state_name(state), s->free, s->des, s->min, refresh_ok, source, refresh_errno, refresh_error, refresh_ms, refresh_ok == 1 ? "refresh" : "last_sample", scan_complete, latest_complete_scan_at > 0 ? 1000 * (s->at - latest_complete_scan_at) : -1.0, warmup, - controller.nvictims, controller.nvictims, controller.nvictims, controller.nvictims, + controller.nvictims, budget_tier(limit), limit, cfg.max_victims, cfg.max_emergency_victims, + cfg.max_floor_victims, cfg.max_hard_floor_victims, controller.override_stage, + entries, action_rejected_entries, scan_complete, s->scan, engine.pageout_delta.value, engine.pageout_delta.valid); } @@ -789,7 +848,7 @@ static void act(sample_t *sample) plan_t plan; sample_t fresh; engine_t current_engine; - const char *source = "pending", *blocked_reason = "no_valid_cached_candidate"; + const char *source = "pending", *blocked_reason = "no_revalidatable_candidate"; double candidate_age = -1, decision_mono = 0, validation_start = 0; double validation_ms = 0, kill_attempt_mono = 0, kill_return_mono = 0; double dispatch_ms = 0, kill_ms = 0, enqueue_start, enqueue_ms; @@ -805,6 +864,7 @@ static void act(sample_t *sample) signal_result_t signal_result; rejection_t rejections[CACHED_CANDIDATES + 1]; int err = 0, rc = 0, validated = 0, rejected_pid = 0, rejected_errno = 0; + action_rejected_entries = 0; if (stopping) return; if (!engine.danger) { (void)action_plan(&controller, &cfg, &engine, sample, NULL); @@ -860,6 +920,7 @@ retry_candidate: rejections[rejected].ms = timing.total_ms; rejections[rejected].stage = timing.stage; rejected++; rejected_pid = plan.target.pid; rejected_errno = err; + action_rejected_entries = rejected; goto retry_candidate; } validation_ms = 1000 * (monotime() - validation_start); @@ -891,6 +952,7 @@ retry_candidate: rejections[rejected].ms = timing.total_ms; rejections[rejected].stage = timing.stage; rejected++; rejected_pid = plan.target.pid; rejected_errno = err; + action_rejected_entries = rejected; continue; } validation_ms = 1000 * (monotime() - validation_start); @@ -927,9 +989,8 @@ retry_candidate: if (controller.nvictims != victims_before) { const char *tier = !strcmp(plan.budget_override, "hard_floor") ? "hard_floor" : current_engine.floor ? "floor" : current_engine.state == EMERGENCY ? "emergency" : "normal"; - unsigned limit = !strcmp(tier, "hard_floor") ? cfg.max_hard_floor_victims : - current_engine.floor ? cfg.max_floor_victims : - current_engine.state == EMERGENCY ? cfg.max_emergency_victims : cfg.max_victims; + unsigned limit = action_limit(&controller, &cfg, ¤t_engine, &fresh); + if (!strcmp(plan.budget_override, "deadman_risk")) tier = budget_tier(limit); emit_important(LOG_WARNING, "event=victim_budget tier=%s used=%u limit=%u remaining=%u mode=%s", tier, controller.nvictims, limit, limit > controller.nvictims ? limit - controller.nvictims : 0, cfg.armed ? "armed" : "dry-run"); @@ -1171,6 +1232,7 @@ int main(int argc, char **argv) request_scan(0); last_scan = sample.at; } /* All telemetry is queued only after the rescue path has returned. */ + log_deadman_status(&sample); if (engine.state != previous) (void)transition_message(transition, sizeof (transition), previous, engine.state, &sample, engine.low_samples); diff --git a/tests/fixtures/panic34_persistent_minfree_budget_exhaustion_2026_10_09.csv b/tests/fixtures/panic34_persistent_minfree_budget_exhaustion_2026_10_09.csv new file mode 100644 index 0000000..84daa43 --- /dev/null +++ b/tests/fixtures/panic34_persistent_minfree_budget_exhaustion_2026_10_09.csv @@ -0,0 +1,26 @@ +# Synthetic reconstruction of supplied panic34 conditions; not raw host telemetry. +# monotonic,free_pages,nscan,anon_pageout_pages,swap_physical_used_bytes +500,48500,6772446,100000,1000000 +501,49077,6000000,100100,1004096 +502,48800,0,100200,1008192 +503,48700,6200000,100300,1012288 +504,49000,6400000,100400,1016384 +505,48900,6700000,100500,1020480 +506,48500,6700000,100600,1024576 +507,49077,6700000,100700,1028672 +508,48800,6700000,100800,1032768 +509,48700,6700000,100900,1036864 +510,49000,6700000,101000,1040960 +511,48900,6700000,101100,1045056 +512,48500,6700000,101200,1049152 +513,49077,6700000,101300,1053248 +514,48800,6700000,101400,1057344 +515,48700,6700000,101500,1061440 +516,49000,6700000,101600,1065536 +517,48900,6700000,101700,1069632 +518,48500,6700000,101800,1073728 +519,49077,6700000,101900,1077824 +520,48800,6700000,102000,1081920 +521,48700,6700000,102100,1086016 +522,49000,6700000,102200,1090112 +523,48900,6700000,102300,1094208 diff --git a/tests/test_distribution.sh b/tests/test_distribution.sh index c1686f8..a2fa0ae 100644 --- a/tests/test_distribution.sh +++ b/tests/test_distribution.sh @@ -31,6 +31,11 @@ MAX_VICTIMS_MINUTE 4 MAX_EMERGENCY_VICTIMS_EPISODE 8 MAX_FLOOR_VICTIMS_EPISODE 12 MAX_HARD_FLOOR_VICTIMS_EPISODE 16 +DEADMAN_RISK_MARGIN_PAGES 1024 +DEADMAN_RISK_SECONDS 5 +DEADMAN_RISK_SAMPLES 6 +DEADMAN_RISK_NSCAN_ACTIVE_SAMPLES 3 +DEADMAN_RISK_ANONPGOUT_ACTIVE_SAMPLES 4 EOF grep -Fq -- '"--force-dry-run"' src/main.c diff --git a/tests/test_guard.c b/tests/test_guard.c index 3e13035..d78fa1b 100644 --- a/tests/test_guard.c +++ b/tests/test_guard.c @@ -66,8 +66,19 @@ static void configs(void) "FLOOR_KILL_DELAY_SECONDS 0\nMAX_CACHED_CANDIDATE_AGE_SECONDS 15\n", &c) == 0); CHECK(parse("MAX_EMERGENCY_VICTIMS_EPISODE 3\n", &c) != 0); CHECK(parse("MAX_FLOOR_VICTIMS_EPISODE 7\n", &c) != 0); + CHECK(parse("MAX_FLOOR_VICTIMS_EPISODE 16\n", &c) != 0); CHECK(parse("MAX_HARD_FLOOR_VICTIMS_EPISODE 16\n", &c) == 0 && c.max_hard_floor_victims == 16); CHECK(parse("MAX_HARD_FLOOR_VICTIMS_EPISODE 17\n", &c) != 0); + CHECK(parse("DEADMAN_RISK_MARGIN_PAGES 1024\nDEADMAN_RISK_SECONDS 5\n" + "DEADMAN_RISK_SAMPLES 6\nDEADMAN_RISK_NSCAN_ACTIVE_SAMPLES 3\n" + "DEADMAN_RISK_ANONPGOUT_ACTIVE_SAMPLES 4\n", &c) == 0); + CHECK(parse("DEADMAN_RISK_SAMPLES 0\n", &c) != 0); + CHECK(parse("DEADMAN_RISK_SAMPLES 33\n", &c) != 0); + CHECK(parse("DEADMAN_RISK_SECONDS 0\n", &c) != 0); + CHECK(parse("DEADMAN_RISK_MARGIN_PAGES -1\n", &c) != 0); + CHECK(parse("DEADMAN_RISK_NSCAN_ACTIVE_SAMPLES 7\n", &c) != 0); + CHECK(parse("DEADMAN_RISK_ANONPGOUT_ACTIVE_SAMPLES 6\n", &c) != 0); + CHECK(parse("DEADMAN_RISK_SECONDS 5\nDEADMAN_RISK_SECONDS 5\n", &c) != 0); CHECK(parse("MAX_HARD_FLOOR_VICTIMS_EPISODE 11\n", &c) != 0); CHECK(parse("MAX_HARD_FLOOR_VICTIMS_EPISODE 16\nMAX_HARD_FLOOR_VICTIMS_EPISODE 16\n", &c) != 0); CHECK(parse("UNKNOWN foo\n", &c) != 0); @@ -343,10 +354,113 @@ static void hard_floor_policy(void) puts("PASS hard-floor persistence, finite ceiling, missing corroboration, gap, brief/stable recovery"); } +static void deadman_policy(void) +{ + config_t c = allowed(); + controller_t a; + engine_t e; + sample_t s; + process_t p = process(9000); + delta_t none = {0, 0, 0}, growing = {1, 5, 100}; + unsigned i, mode; + memset(&a, 0, sizeof (a)); engine_init(&e); a.nvictims = 8; + for (i = 0; i < 9; i++) { + s = sample(100 + i, i == 6 ? 100 : 48500 + i, i == 2 ? 0 : 1000000); + s.pageout_ok = s.swap_physical_ok = 1; + s.anon_pageout_pages = 1000 + i; s.swap_physical_used_bytes = 10000 + i; + engine_step(&e, &c, &s, none, growing, growing); + (void)action_plan(&a, &c, &e, &s, &p); + CHECK(e.floor == (i == 6)); + CHECK(e.risk.ready == (i >= 5)); + CHECK(a.override_stage == (i < 5 ? 0u : i < 8 ? 1u : 2u)); + if (i == 5) { + CHECK(action_limit(&a, &c, &e, &s) == 12); + a.nvictims = 12; + } + if (i == 6 || i == 7) CHECK(action_plan(&a, &c, &e, &s, &p).action == NONE); + } + CHECK(action_limit(&a, &c, &e, &s) == 16); + CHECK(action_plan(&a, &c, &e, &s, &p).action == TERM); + s.free = s.min + 1024; + CHECK(deadman_risk_current(&e, &c, &s)); + s.free++; + CHECK(!deadman_risk_current(&e, &c, &s)); + s.free = 48500; + a.nvictims = 16; + CHECK(action_plan(&a, &c, &e, &s, &p).action == NONE); + a.nvictims = 12; + s.at++; s.free = s.des + 1; + engine_step(&e, &c, &s, none, none, none); + CHECK(!e.risk.ready && e.risk.count == 6); + (void)action_plan(&a, &c, &e, &s, &p); + CHECK(a.nvictims == 12 && a.override_stage == 2); + s.at += 2; s.free = 48500; s.anon_pageout_pages++; + engine_step(&e, &c, &s, none, growing, growing); + (void)action_plan(&a, &c, &e, &s, &p); + CHECK(a.nvictims == 12 && a.override_stage == 2 && e.risk.count == 6); + for (i = 0; i <= c.recovery_s; i++) { + s.at++; s.free = s.lots + 1; s.scan = 0; + engine_step(&e, &c, &s, none, none, none); + (void)action_plan(&a, &c, &e, &s, &p); + } + CHECK(e.state == NORMAL && !e.risk.count && !a.nvictims && !a.override_stage); + + /* FLOOR alone: neither a fresh episode nor risk stage1 may jump to16. */ + for (mode = 0; mode < 2; mode++) { + memset(&a, 0, sizeof (a)); engine_init(&e); + a.nvictims = 12; a.override_stage = mode; a.override_since = 100; + s = sample(100, 100, 1000000); s.pageout_ok = 1; s.anon_pageout_pages = 100; + engine_step(&e, &c, &s, none, growing, growing); + CHECK(e.floor && !e.hard_floor_ready && !e.risk.ready); + CHECK(action_limit(&a, &c, &e, &s) == 12); + CHECK(action_plan(&a, &c, &e, &s, &p).action == NONE); + a.nvictims = 8; + CHECK(action_plan(&a, &c, &e, &s, &p).action == KILL); + } + /* Negative windows: no paging, no scarcity, brief dip, reset/invalid counter. */ + for (mode = 0; mode < 5; mode++) { + memset(&a, 0, sizeof (a)); engine_init(&e); a.nvictims = 8; + for (i = 0; i < 10; i++) { + s = sample(100 + i, mode == 1 || (mode == 2 && i > 1) ? 140000 : 48500, + mode == 0 ? 0 : 1000000); + s.pageout_ok = mode != 4; + s.anon_pageout_pages = mode == 0 ? 100 : mode == 3 ? 1000 - i : 1000 + i; + engine_step(&e, &c, &s, none, none, none); + (void)action_plan(&a, &c, &e, &s, &p); + CHECK(!e.risk.ready && !a.override_stage); + } + } + /* Real monotonic overruns retain evidence; very old windows cannot authorize. */ + engine_init(&e); + for (i = 0; i < 6; i++) { + s = sample(100 + 2 * i, 48500, i == 3 ? 0 : 1000000); + s.pageout_ok = 1; s.anon_pageout_pages = 100 + i; + engine_step(&e, &c, &s, none, none, none); + } + CHECK(e.risk.ready); + s.at += 30; s.anon_pageout_pages++; + engine_step(&e, &c, &s, none, none, none); + CHECK(!e.risk.ready && e.risk.count == 6); + c.risk_samples = 32; c.risk_seconds = 31; + c.risk_scan_samples = c.risk_anon_samples = 30; + engine_init(&e); + for (i = 0; i < 40; i++) { + s = sample(100 + i, 48500, 1000000); + s.pageout_ok = 1; s.anon_pageout_pages = 1000 + i; + engine_step(&e, &c, &s, none, none, none); + } + CHECK(e.risk.ready && e.risk.count == 32); + s.anon_pageout_pages = 0; + CHECK(!deadman_risk_current(&e, &c, &s)); + s.at = 90; engine_step(&e, &c, &s, none, none, none); + CHECK(!e.risk.ready && e.risk.count == 1); + puts("PASS deadman window, staged8/12/16, recovery, overruns, negatives and single FLOOR never16"); +} + int main(void) { configs(); protections(); histories(); states(); real_incident_fixture(); - successful_avoidance_fixture(); actions(); hard_floor_policy(); + successful_avoidance_fixture(); actions(); hard_floor_policy(); deadman_policy(); printf("PASS %u checks; no native backend linked, no signals sent\n", checks); return 0; } diff --git a/tests/test_runtime.c b/tests/test_runtime.c index 407f4df..dbedfe2 100644 --- a/tests/test_runtime.c +++ b/tests/test_runtime.c @@ -98,6 +98,8 @@ static void reset(void) memset(published_cache, 0, sizeof (published_cache)); memset(rejected_identity, 0, sizeof (rejected_identity)); rejected_identity_next = rejected_identity_count = 0; + logged_override_stage = action_rejected_entries = 0; + logged_deadman_risk = 0; override_blocked_logged_at = -1; memset(metrics, 0, sizeof (metrics)); memset(&controller, 0, sizeof (controller)); memset(fixtures, 0, sizeof (fixtures)); engine_init(&engine); engine.state = CRITICAL; engine.danger = 1; @@ -198,6 +200,60 @@ static void panic33_fixture(void) puts("PASS panic33: partial recoveries, budgets, dead PID replacement, incomplete scan, hard ceiling16"); } +static void panic34_fixture(void) +{ + FILE *f; + char line[256]; + unsigned i, rows = 0, walks, before; + delta_t none = {0, 0, 0}; + populate_many_candidates(16); cfg.armed = 1; async_logging = 1; + for (i = 0; i < 8; i++) controller.victims[i] = fixtures[i]; + controller.nvictims = 8; + scan_complete = 0; latest_complete_scan_at = 123; + walks = walk_requests; engine_init(&engine); + f = fopen("tests/fixtures/panic34_persistent_minfree_budget_exhaustion_2026_10_09.csv", "r"); + CHECK(f != NULL); + while (fgets(line, sizeof (line), f)) { + int acted = 0; + if (line[0] == '#') continue; + CHECK(sscanf(line, "%lf,%" SCNu64 ",%" SCNu64 ",%" SCNu64 ",%" SCNu64, + &fake_now, &fake_sample.free, &fake_sample.scan, + &fake_sample.anon_pageout_pages, &fake_sample.swap_physical_used_bytes) == 5); + fake_sample.at = fake_now; fake_sample.pageout_ok = fake_sample.swap_physical_ok = 1; + /* Same order as production: retained rescue before the next engine step. */ + if (engine.danger) { act(&fake_sample); acted = 1; } + engine_step(&engine, &cfg, &fake_sample, none, none, none); + if (engine.danger && !acted) act(&fake_sample); + log_deadman_status(&fake_sample); + CHECK(!engine.floor && !scan_complete && fake_now - latest_complete_scan_at > 376); + CHECK(engine.risk.ready == (rows >= 5)); + CHECK(controller.override_stage == (rows < 6 ? 0u : rows < 9 ? 1u : 2u)); + CHECK(controller.nvictims <= 16 && walk_requests == walks); + rows++; + } + fclose(f); + CHECK(rows == 24 && controller.nvictims == 16 && real_requests == 16); + CHECK(queued_contains("reason=persistent_minfree_deadman_risk from_limit=8 to_limit=12")); + CHECK(queued_contains("reason=persistent_minfree_deadman_risk_continues from_limit=12 to_limit=16")); + CHECK(queued_contains("event=deadman_risk active=1")); + CHECK(queued_contains("proc_scanner_complete=0")); + CHECK(queued_contains("episode_victims_total=16 current_budget_tier=absolute current_limit=16")); + before = event_count; log_deadman_status(&fake_sample); CHECK(event_count == before); + /* No usable candidates is distinct from exhausted budget. */ + controller.nvictims = 8; controller.has_pending = 0; + memset(candidate_cache, 0, sizeof (candidate_cache)); + act(&fake_sample); + CHECK(queued_contains("reason=no_revalidatable_candidate")); + CHECK(queued_contains("cache_entries=0 rejected_entries=0 proc_scan_complete=0")); + reset(); cfg.armed = 1; async_logging = 1; controller.nvictims = 12; + fake_sample.free = 100; fake_sample.pageout_ok = 1; fake_sample.scan = 1000000; + engine_step(&engine, &cfg, &fake_sample, none, none, none); + act(&fake_sample); + CHECK(engine.floor && real_requests == 0 && controller.nvictims == 12); + CHECK(action_limit(&controller, &cfg, &engine, &fake_sample) == 12); + puts("PASS panic34 real-loop order: intermittent VM scanner, stale proc cache,8->12->16, finite and deduplicated"); +} + static void partial_cache_tests(void) { process_t replacement, survivor; @@ -691,6 +747,7 @@ int main(void) diagnostic_fixtures(); panic33_fixture(); partial_cache_tests(); + panic34_fixture(); fclose(log); printf("PASS %u runtime checks; all platform calls mocked, no signals sent\n", checks); return 0;