diff --git a/CHANGELOG.md b/CHANGELOG.md index 5217a29..a65271c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 1.0.10 — 2026-10-08 — panic33: persistent FLOOR and incremental cache + +- Normal budgets stay 4/8/12; finite hard ceiling defaults to 16 (configuration + rejects >16 or inconsistent ordering). Five continuous seconds/six fresh + FLOOR samples, active scanner and positive recent anonymous pageout required. +- Partial scanner publication and directory cursor continuation; survivor merge + preserves better entries, rejected identities removed, rescue handoff trylock. +- Brief RECOVERY and sampling failures do not reset episode budgets. Only stable + RECOVERY to NORMAL resets. No change to thresholds, ranking weights, grace, + allow/protect, async logger/syslog or PCKILL implementation. +- victim_budget and critical budget_override diagnostics; panic33 synthetic + fixture and incident report distinguish supplied evidence from open questions. +- Source-only; no deployment, real signals, swap changes or remote publication. + ## 1.0.9 — 2026-09-28 — PCKILL phases and distinct refresh diagnostics - Medidas separadas de openat(ctl), preparación, write y total de dispatch. diff --git a/CHECKPOINT.md b/CHECKPOINT.md index 26e6421..9f2299f 100644 --- a/CHECKPOINT.md +++ b/CHECKPOINT.md @@ -822,3 +822,17 @@ La rama upstream SIGKILL llama pr_wait_die; su correspondencia con el kernel desplegado y la distribución entre espera, locks y off-CPU requieren observación. No se atribuye la latencia a LX/ZFS ni se cambia transporte, ranking o política. Auditoría: AUDIT_PCKILL_LATENCY_2026_09_28.md. Entrega source-only. + +## 23. Addendum v1.0.10 — panic33 (2026-10-08) + +Evidencia suministrada: pageout_deadman con freemem=47959, no cero; stack en +swap/zvol/ZIL/rangelock, múltiples TERM/KILL, presupuestos agotados y caché +degradada. No demuestra que más víctimas hubieran evitado el panic. +Informe: INCIDENT_PAGEOUT_DEADMAN_PANIC33_2026_10_08.md. +Techos 4/8/12 preservados, hard-floor16 finito condicionado a cinco segundos/ +seis muestras FLOOR con scanner activo y crecimiento anónimo reciente válido. +Recuperación breve y errores de muestreo conservan presupuesto. Scanner publica +candidatos parciales y continúa cursor; un bloqueo dentro de syscall aún puede +detener descubrimiento, no rescate. Nuevos eventos victim_budget/budget_override. +Suite 957 comprobaciones (240+620+70+27), sanitizadores y portabilidad. +Source-only, sin despliegue, señales reales, credenciales ni push remoto. diff --git a/INCIDENT_PAGEOUT_DEADMAN_PANIC33_2026_10_08.md b/INCIDENT_PAGEOUT_DEADMAN_PANIC33_2026_10_08.md new file mode 100644 index 0000000..0d13763 --- /dev/null +++ b/INCIDENT_PAGEOUT_DEADMAN_PANIC33_2026_10_08.md @@ -0,0 +1,88 @@ +# panic33 — evidencia aportada por el operador + +Fecha de revisión: 2026-10-08. Este informe transcribe el resumen aportado; +no se ha adquirido ni analizado independientemente el dump o el log completo. +El fixture es una reconstrucción sintética, no una cronología exacta. + +## Panicinfo y memstat + +```text +pageout_deadman: stuck pushing the same page for 90 seconds +(freemem is 47959) +pagesize = 4096 +minfree = 49077 pages +freemem = 47959 pages +Kernel 7314 MiB 22% +ZFS File Data 1714 MiB 5% +Anon 23045 MiB 70% +Free ~187 MiB +``` + +47959 páginas son 187,34 MiB: el deadman no exige RAM libre cero. + +## Stack exacto aportado de pageout/1 + +```text +pageout +fop_putpage +swap_putpage +swap_putapage +fop_pageio +spec_pageio +spec_startio +bdev_strategy +zvol_strategy +zil_commit +zil_commit_impl +zil_commit_writer +zil_process_commit_list +zil_lwb_commit +zvol_get_data +rangelock_enter +rangelock_enter_reader +cv_wait +``` + +Otras hebras aparecen en page_create_throttle, page_create_va, +segkmem_page_create, segkmem_zio_alloc, abd_alloc, arc_alloc_buf y dbuf_read. + +## Correlación con el guard + +En unos 80 segundos se observaron múltiples acciones reales y recuperaciones +parciales. Víctimas citadas (no inventario exhaustivo ni orden exacto): + +- SIGKILL: 03.base0.es. +- SIGTERM/SIGKILL: sailandfun.bz2, servidor.correccionencatala.cat, + servidor.consultaerte.com y web5.open6hosting.com. + +Después se repite emergency_victim_budget_exhausted, con free_pages aproximados +49009, 53342, 59645, 10719 (floor), 29578, 62974, 27290, 6513 (floor), 29899 +y 42129; nscan alcanza millones. Rechazos proc_ctl_open errno=2 preceden a +no_valid_cached_candidate. proc_scan complete=0 y la edad del último recorrido +completo crece a decenas de segundos. No se conocen aquí timestamps completos +que permitan ordenar todas las acciones respecto al inicio del bloqueo kernel. + +## Confirmado en la evidencia aportada + +El guard actuó múltiples veces, se enviaron TERM/KILL, se agotó el presupuesto, +la caché perdió candidatos utilizables y el scanner se degradó. El stack sitúa +pageout en la ruta ZIL/rangelock al observar el dump. La combinación defendible +es presión física extrema, pageout escribiendo swap sobre zvol ZFS y ausencia +de progreso que termina en pageout_deadman. No identifica por sí sola la causa +raíz ni demuestra que pageout estuviera 90 segundos en ese mismo frame. + +## Pendiente + +- Instante exacto en que pageout dejó de progresar. +- Si más víctimas habrían evitado con seguridad el panic (no demostrado). +- Cuánto contribuye volblocksize=8K; no se modifica el swap desde este proyecto. +- Validación nativa del nuevo scanner/política en la versión exacta de SmartOS. + +## Respuesta source-only + +v1.0.10 conserva 4/8/12 y añade techo absoluto 16, condicionado a FLOOR +persistente y pageout corroborado. Conserva presupuestos durante recuperación +breve, publica candidatos parciales y reemplaza identidades desaparecidas sin +esperar un recorrido global. Ninguna de estas medidas garantiza evitar un panic. +Fixture: tests/fixtures/panic33_persistent_floor_budget_exhaustion_2026_10_08.csv. +No se desplegó, no se alteró el nodo, no se enviaron señales reales. diff --git a/Makefile b/Makefile index e6f93b1..d10ede8 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ test-runtime: tests/test_runtime.c tests/fixtures/global_victim_ranking_smartos_ $(CC) $(TESTFLAGS) -o $@ tests/test_runtime.c src/core.c src/config.c src/result.c test-runtime: tests/fixtures/pckill_latency_2026_09_28.csv tests/fixtures/refresh_recovery_and_failure.csv +test-runtime: tests/fixtures/panic33_persistent_floor_budget_exhaustion_2026_10_08.csv test-distribution: tests/test_distribution.sh smf/o6h-memory-guard.xml etc/o6h-memory-guard.conf src/main.c sh tests/test_distribution.sh diff --git a/README.md b/README.md index 46e689d..e2b2f92 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,55 @@ -# o6h-memory-guard 1.0.9 +# o6h-memory-guard 1.0.10 + +## panic33: hard-floor y caché incremental + +Presupuestos acumulados del mismo episodio: normal 4, emergency 8, floor 12 y +`MAX_HARD_FLOOR_VICTIMS_EPISODE 16`. No son cuotas sumables. Se exige +normal <= emergency <= floor <= hard <=16. TERM+KILL cuenta una sola identidad. + +El tramo adicional requiere EMERGENCY, FLOOR existente (free<=minfree/2), +floor normal agotado, nscan>0 y delta reciente positivo/válido de anon_pageout. +Debe persistir cinco segundos/seis muestras, con separación máxima de 1,5 s. +Una muestra aislada, hueco, contador reseteado/delta no positivo, pageout +desconocido o recuperación invalida el permiso. El refresco antes de actuar +debe seguir en FLOOR, con pageout válido, scanner activo y autorización de +antigüedad <=1,5 s. Nunca se mata sólo por métricas de swap/pageout. +Swap físico sigue teniendo deltas 5/15/60: es un gauge que puede bajar y no se +exige su crecimiento para autorizar el override. Los relojes son monotónicos. + +RECOVERY breve conserva víctimas, rate limit y caché, pero exige nueva +persistencia para otro override. Sólo RECOVERY estable durante RECOVERY_SECONDS +(free>=lotsfree, nscan=0) hasta NORMAL reinicia el presupuesto. Un fallo de +muestreo inhibe acciones sin regalar un episodio nuevo. + +El scanner publica cada candidato elegible antes de continuar, fusionando +16 entradas y conservando supervivientes mejores. ENOENT/ESRCH/ESTALE invalidan +identidades; hay memoria acotada de 64 rechazos, reiniciada tras recuperación +estable. Toda acción sigue revalidando PID/start/UUID/generación/allow/protect. +En presión se solicita scan cada segundo sin esperar. Al agotar SCAN_BUDGET_MS +se conserva el cursor para no comenzar siempre por los mismos PID. Un recorrido +reanudado sigue marcado incompleto incluso al llegar a EOF: no renueva la fecha +del último scan completo. Ranking y pesos no cambian; el componente de zona de +un parcial usa los datos disponibles y puede subestimar su total. Un parcial +no reemplaza una entrada mejor por una peor. + +**Límite temporal:** el presupuesto es cooperativo entre operaciones. No +interrumpe una syscall /proc bloqueada. Los candidatos publicados antes del +bloqueo siguen disponibles, pero no se promete descubrir otros hasta que +retorne. Rescate no espera el mutex del scanner (trylock); no se crean hilos +ilimitados ni se altera PCKILL, logger o syslog asíncrono. + +Eventos nuevos: `victim_budget tier=normal|emergency|floor|hard_floor` con +used/limit/remaining/mode al consumir víctima o cerrar episodio; +`budget_override reason=persistent_floor` (LOG_CRIT) por víctima adicional. +Bloqueos: normal_used/emergency_used/floor_used/hard_floor_used representan el +mismo consumo acumulado frente a los cuatro techos, no cuatro contadores +independientes; añaden nscan y anonpgout_delta/anonpgout_delta_ok. Las muestras +incluyen hard_floor_ready y persistent_floor_samples. + +Informe: `INCIDENT_PAGEOUT_DEADMAN_PANIC33_2026_10_08.md`. +Fixture sintético: `panic33_persistent_floor_budget_exhaustion_2026_10_08.csv`. +No garantiza evitar un deadman. Source-only, sin cambios de swap/zvol, dry-run +por defecto y sin UUID autorizados. Primera implementación para SmartOS global zone: daemon C de 64 bits, libkstat, /proc, identidad UUID + generación de zona + PID + instante de creación, @@ -49,12 +100,12 @@ Si no hay herramientas de desarrollo, compilar en una **zona nativa SmartOS** de la misma plataforma (no LX), ejecutar allí `gmake test`, y copiar el binario junto a este árbol al host. No instalar un compilador en un nodo bajo presión. -Copiar el archivo distribuido a `/var/tmp/o6h-memory-guard-1.0.9.tar.gz`: +Copiar el archivo distribuido a `/var/tmp/o6h-memory-guard-1.0.10.tar.gz`: ```sh mkdir -p /opt/custom/src cd /opt/custom/src -gzip -dc /var/tmp/o6h-memory-guard-1.0.9.tar.gz | tar -xf - +gzip -dc /var/tmp/o6h-memory-guard-1.0.10.tar.gz | tar -xf - cd /opt/custom/src/o6h-memory-guard gmake CC=gcc gmake test CC=gcc @@ -214,10 +265,11 @@ el PID se prueba el siguiente candidato cacheado sin recorrer `/proc` completo. En NORMAL/PRESSURE/CRITICAL rigen 4 víctimas por episodio y 4 por ventana móvil de 60 s. EMERGENCY puede superar esos límites normales hasta 8 víctimas por -episodio; floor tiene un límite absoluto de 12. Cada override se registra como +episodio; floor tiene un límite normal de 12 y hard-floor un techo absoluto de 16. Cada override se registra como `victim_budget_override=emergency|floor`. TERM+KILL del mismo proceso cuentan como una víctima. Los límites 4/8/12 son configurables pero deben mantener ese -orden. El episodio sólo se reinicia al volver a NORMAL. En armed, los primeros +orden y no superar el techo hard-floor de 16. El episodio sólo se reinicia tras +RECOVERY estable hasta NORMAL, no tras un error de muestreo. En armed, los primeros 60 s desde el arranque monotónico del daemon inhiben únicamente las señales, también en EMERGENCY, para evitar eludir el límite por minuto mediante reinicios. La detección, el muestreo, los scans permitidos por estado y la reconstrucción @@ -355,14 +407,16 @@ su cadena. No se promete cero asignaciones ni tiempo real estricto. El recorrido /proc tiene presupuesto cooperativo de 200 ms (hasta 500 ms por configuración) y se ejecuta en un scanner independiente. Un scan incompleto o -bloqueado no sustituye la última caché completa ni detiene el muestreo. NORMAL -solicita actualización cada 5 s; PRE-PRESSURE/PRESSURE, cada segundo. +bloqueado no borra los supervivientes ni detiene el muestreo: publica candidatos +parciales según se descubren. NORMAL solicita actualización cada 5 s; en presión, +incluyendo CRITICAL/EMERGENCY/FLOOR, se solicita cada segundo. CRITICAL/EMERGENCY/FLOOR nunca esperan al scanner: recorren como máximo 16 entradas precomputadas y revalidan sólo el proceso elegido. Los 15 s de frescura siguen aplicándose en NORMAL/PRE/PRESSURE. En rescate, una entrada más antigua puede usarse como `stale_cache_revalidated` si PID, inicio, UUID, generación, regla `allow`, nombre y protecciones todavía coinciden. La caché sobrevive a -EMERGENCY → RECOVERY → EMERGENCY y sólo un scan completo sano la reemplaza. +EMERGENCY → RECOVERY → EMERGENCY; scans parciales pueden renovar sus entradas, +mientras que un scan completo sano reconstruye el ranking global. Muestras obligatorias ausentes/invalidas o demasiado lentas inhiben señales; ARC/swap/pageout opcionales ausentes se marcan con `*_ok=0` y no bloquean por sí @@ -451,7 +505,7 @@ Alloy/Loki puede configurarse fuera del guard, pero esta entrega no lo hace. ## Modelo de activación real (operación futura; no ejecutada en esta entrega) -v1.0.9 debe compilarse y observarse primero en SmartOS con `ACTION_MODE dry-run`. +v1.0.10 debe compilarse y observarse primero en SmartOS con `ACTION_MODE dry-run`. El manifiesto ya no contiene una segunda puerta oculta. Cuando exista aprobación operativa explícita, el único cambio de modo es editar la configuración: diff --git a/SHA256SUMS b/SHA256SUMS index 63647c0..bf8e7fc 100644 --- a/SHA256SUMS +++ b/SHA256SUMS @@ -1,30 +1,31 @@ ab22dd5e73f360b7f08839f95e35e970ce4401e4541071e802cf2d24e3d8e2a6 .gitignore bba839eb1ece469050db8beb6a7eccdbe07a51eadc2a920c0cdf0f707330b800 AUDIT_PCKILL_LATENCY_2026_09_28.md -bffc45b4707f96ee78e59c03a06ce06bffbf7c500d6b3b3939690ad30ff7da4c CHANGELOG.md -8b762e7767e62aeb9188cb5988ef575985caa72f00d6c866a05b5e37a1556419 CHECKPOINT.md +28a8fa917b95065c76d5c91ac350fee3e5cf6fa2fcafedb272125dda70c471f0 CHANGELOG.md +d395a34c4806489ea8e9cf79220f708442d47e500ce5e695d64d9a629e1cd33b CHECKPOINT.md 4d0b3983bbb0d2a51c61ce0f99d778cc79768ed371bdb5ced382c0fb5dfcaf5f INCIDENT_FLOOR_KILL_FAILED_2026_09_23.md a0f96300e751d08ceeef1360325ed54008b703fa6bd07df9d5368674e580f113 INCIDENT_PAGEOUT_DEADMAN_2026_09.md fb6b65a2655ccd808d102fd917b4866cbe939f24fac4e4aeef226a6aeda6d588 INCIDENT_PAGEOUT_DEADMAN_AVOIDED_2026_09_21.md 0f18d6dd3e0d027c1e8dbfce05ce6ac3beb70f3f66d1a77a8d0c5e0f4ae2a039 INCIDENT_PAGEOUT_DEADMAN_PANIC32_2026_09_22.md -87912109e1dd3750888d484aa2387e5ddea90827082871c77bf3ee2e1017d298 Makefile -3d38133e31d5e71f8ef97aae51fb042ac7223f9ad82ea58455482834c57aa20d README.md -c8d35d9e65df672e080786c85dcae676540f0fe7e702242392b19d769a15dc98 VALIDATION.md -dd3d5ef89dda5a37a51d85e693b4c03d1f322d39d64bc1b9d78140f22d1d162d etc/o6h-memory-guard.conf +153d3a57333dabc3227e3660fd8830c1a38663c0aa0190e2ea83b581cfbf1559 INCIDENT_PAGEOUT_DEADMAN_PANIC33_2026_10_08.md +78a5e8cb5a04b633dd0f0512984c5e77d5a8638eef36e66deb3e5f6150801677 Makefile +46bea1f497107d7825839e9b8ae9ede28255068fe298b3de5113b97a07ca3bc3 README.md +2ab1c9c33f0cb059720101be72262574012dd9e50b8d2b78c075c6aa3cdce6ba VALIDATION.md +1e65ea7af3b643052219190a9f3dd28afef78edb2b5e318fbbab4c69085b0cb0 etc/o6h-memory-guard.conf f6565d0e1ece0a08721c9893f9ed071d2ab0a2818fa93ed0a14e0050d358ce99 install.sh -24c1b9bc39fda71d459d6fad6a8d889b0d9b92be89d303677643b3d0b93caf4b logs/distribution-tests.log +92faeb31da43244631f25bb915be07720d662b205d5872dc3db7702ecb883f37 logs/distribution-tests.log 50626dec1924e19b4ca92eec30416fb39c3fe7cff28d98a8e120ccf6247d2834 logs/example.log -812482297ae6dfb2a0881998781459ed77288ef073f202a18f22055437518fa6 logs/policy-tests.log -d60b67d7c8b0307593fe28fc13025778f0c7175d9bdf0cc2fa4f5edfcc18fb6c logs/portability.log -fa1dad7c69b95259bff4e18a98946618c68f68642cf654e97984ea80028d0f19 logs/prepare-tests.log -5c21b4cfcde65fc393ec3e3825beab61b4aa0e56818ace9b6363955f26a764a9 logs/runtime-tests.log -c863ebc2d6fbe6a2c96f4a8040a5dc28a61e669d2d12879055c410665ae2ef6c logs/sanitizers.log +c2533657a871956bd36243b16c02a63bbf4137bcad444a495cb367376f64b478 logs/policy-tests.log +c5e2dd83262f119b1515526bb2209937598f33b613c0fed82362385e6b6ef27d logs/portability.log +180733f8807f47abd6bcd90dca6fdc19c2555b64f205940e885edaa48b5642e3 logs/prepare-tests.log +0fe337f74419a60f6ed6c4c949e1aeaf0711dc80741e03051c9bd6dab6a45fd8 logs/runtime-tests.log +b463307c75d3e24338b1dde62c993b03db4cf9010ff1c56b4330b69d7aa14150 logs/sanitizers.log ec90a51222ef270ab0294532bc9396862afec1679824026848c2bc61165348be logs/validation-tests.log 55cf49840cf1d22e4e02bedf579a10207856a0a5b40c86ae2cf6ee2f00d3f972 smf/o6h-memory-guard.xml -99e1fb6a90a7bd2c1002ad09320cd3f21c2deb2499212eff2baddc14f3bdf0e6 src/config.c -659da2facd27577c763484694a2ff62eef42aea48ebc496b878853b0164b6539 src/core.c -8133cad598f1e14e341c0611586f3cd506b640e13d90a261a8e256274e347875 src/guard.h -8ebc01c7c1a5a58cb1a40cc355505777940e1dc95f29c650f5e066e10a183f3e src/illumos.c -e58568de0a12af9df6773ac8d4755328b823480cd001e0423ad541cff202271c src/main.c +b0ecefbadfb583a9dc06e85705cd4951e689e3099f4b9a5daa2da99fb66e01ee src/config.c +532a369d0275dfddffd1f6bfb0c0bd819487760a059c634bf0060b07bb16d287 src/core.c +10cf9ab6ab8df2a94a732d56dbf166552e2710e6e732c122ae19e614472a39aa src/guard.h +ad57cd80c8ae56ffbdc075abfd5d2b68bc115ab1355e5ee60e79359353cd1220 src/illumos.c +9f224c509cc006f1e7b8adfca0faf6fb03f6d13a49bc7aa1d53739b5ebe8e666 src/main.c 5bb4e0980de83b29e05bb10ea87d6c04f10df884dc92f74cd6b3b025d41d3332 src/result.c 948bfd760d5a4e1cd301da8b8aa270cfa894d2ec3334d000bd687ef023788388 src/signal.c 1f09afd342ba86580a21f6c04a030d9ea2551a60b669e257295fbf4f43b2fad7 src/support.c @@ -38,11 +39,12 @@ f0d06353d2e326c7bdce776bee6f0a8606cda50d5de957dfdd586b6848fca8da tests/api-sket 4aa08ded8e98cf25d2a8762543de7a9273f0ebf00b4c7e5b359603ceda46eb56 tests/fixtures/global_victim_ranking_smartos_2026_09.csv c5d9d7fc7a39354474d30cd316ea9f122205c2b38e57f927f4a08306b8a62c81 tests/fixtures/pageout_deadman_real_incident_2026_09.csv 025983b24045e24a96532c933715b8760edd8c0b0e11b9f0e019481c1c52a50d tests/fixtures/panic32_daemon_stall_2026_09_22.csv +a154f01d5fe41a7986e7ce86cdd0917e0dd2127ed0d295fa6321143e28c8a3d5 tests/fixtures/panic33_persistent_floor_budget_exhaustion_2026_10_08.csv b80c56d558d98c7aae6883f5d51ad4652be46999385170e79fb50da17ade325f tests/fixtures/pckill_latency_2026_09_28.csv ccf983995eff740e26feb66200de6e2339a3f31f997d74a9af3815ce35722c86 tests/fixtures/refresh_recovery_and_failure.csv facc4580ffadef29cdce84bf1226a696cba3880bbbec80161beaceb2752bfd77 tests/fixtures/successful_pageout_deadman_avoidance_2026_09_21.csv -84b3e9b9dbab27a6ed18b69d9f11bc7ca6a84105cd80bb543411146cdba57a00 tests/test_distribution.sh -7c0cfe5fc6719d27258bb87b9e68388be7b580d7031a5ed6deac34ddb3050f25 tests/test_guard.c -1210e59dd2fc57df15dc9a4254dbc652059895ed6d5ea6e2f82f79a5b046107a tests/test_prepare.c -fad190cce29e49bc0d3549a6dfd9648f98129d8ae170e0fc7926b236cdb9ad7e tests/test_runtime.c +fc1b7e17ac3ef1275edf248ddc7e62f0ef85e79d433715aec38e4d4719a5b699 tests/test_distribution.sh +13224898d9850eaa33ef9e5070238f25ffb759d19b77cdceef0094fb637373f5 tests/test_guard.c +debaacd3890a8b65d163c83ffcd61126fd45ebbc66c3e5c69a1202766ff39d68 tests/test_prepare.c +c6d8093904c632e11ee1139170d1c4f68f693fb9634cca75fceb9f388631560e tests/test_runtime.c 9576bb5d5dcb525c7c89ec5dc929b896e8c5b61beffe4d7313f89d823476d957 tests/test_validation.c diff --git a/VALIDATION.md b/VALIDATION.md index 593c1e4..8b43043 100644 --- a/VALIDATION.md +++ b/VALIDATION.md @@ -1,11 +1,23 @@ -# Validación de la entrega v1.0.9 +# Validación de la entrega v1.0.10 -Fecha: 2026-09-28. Entrega source-only. **No se desplegó, no se modificó el +Fecha: 2026-10-08. Entrega source-only. **No se desplegó, no se modificó el nodo armado y no se enviaron señales reales.** La configuración conserva `ACTION_MODE dry-run`, syslog `local5`, cero UUID `allow` y SMF deshabilitado. ## Motivo +panic33: presión extrema, presupuesto agotado y caché degradada. Se añade techo +hard-floor 16 condicionado a persistencia y publicación incremental. Informe: +INCIDENT_PAGEOUT_DEADMAN_PANIC33_2026_10_08.md. Fixture reconstruido sintéticamente. +Pruebas nuevas: cinco segundos/seis muestras, muestra aislada, huecos, pageout +ausente, scanner inactivo, techo finito, recuperación breve/estable, fallo de +muestreo sin reset, reemplazo parcial de PID desaparecido, publicación antes +del retorno y mutex de caché ocupado. Recorrido nativo con I/O simulado: tres +slices con cursor conservado, EOF incompleto y siguiente recorrido completo. +El presupuesto del scanner no puede interrumpir una syscall kernel bloqueada. + +Contexto conservado de v1.0.9: + Revisión diagnóstica de SIGKILL enviado con 709–1785 ms de latencia y bloqueos ambiguos de refresco. Se separan apertura ctl/preparación/write/total y causas de bloqueo sin modificar decisiones. AUDIT_PCKILL_LATENCY_2026_09_28.md documenta @@ -39,7 +51,7 @@ no está confirmado si lo causó syslog, guard.log u otra syscall. Véase 3. Guard log, rotación y syslog sólo ocurren en el worker. Sus stalls no detienen muestras ni nuevas acciones. 4. El scanner `/proc` es otro worker. El hilo de muestreo solicita trabajo sin - esperar y conserva la última caché completa publicada. + esperar y combina supervivientes con candidatos parciales publicados. 5. La caché global contiene 16 candidatos; el log sigue mostrando top-3. 6. CRITICAL/EMERGENCY/FLOOR pueden revalidar caché antigua mediante PID, inicio, UUID, generación, zona `allow`, nombre y protecciones. La fuente se registra @@ -62,15 +74,18 @@ no está confirmado si lo causó syslog, guard.log u otra syscall. Véase ## Pruebas ejecutadas -- Política: 175 comprobaciones; incluye los fixtures previos y no enlaza backend. -- Runtime simulado: 503 comprobaciones; ninguna señal nativa. El número cambia - al eliminar las comprobaciones duplicadas del mock de doble validación. +Total: **957 comprobaciones C** (240 + 620 + 70 + 27), más contrato de +distribución, análisis estático, shell, XML y checksums. No equivale a prueba +nativa SMF/ABI: se usan sketches de API e I/O simulado en Linux. + +- Política: 240 comprobaciones; incluye los fixtures previos y no enlaza backend. +- Runtime simulado: 620 comprobaciones; ninguna señal nativa. - Preparación concurrente/dispatch: 70 comprobaciones. Espera real inyectada de 1283 ms; abandono con presupuesto de 50 ms, fallback a otro trabajador, descarte tardío, deduplicación, saturación y recuperación del pool. Tolerancias del test local: abandono <150 ms y fallback completado <200 ms, muy inferiores a 1 s; el presupuesto programado sigue siendo 50 ms. No es garantía de tiempo real. -- Código nativo de preparación con I/O simulado: 16 comprobaciones de PID/start, +- Código nativo de preparación/recorrido con I/O simulado: 27 comprobaciones de PID/start, generación, protect, errno, memoria recuperada, subpasos y dry-run sin ctl. - Código real de dispatch con write simulado: éxito, ESRCH, EPERM, EINVAL, EAGAIN, errno previo EAGAIN/ENOMEM, reloj que altera errno y escritura parcial. @@ -139,7 +154,7 @@ semántica monotónica de `sampling_overrun`. recuperación y saturación del pool. Verificar semántica nativa O_NONBLOCK/ctl. 9. Medir entry/return de write, pr_wait_die, locks y off-CPU en la revisión nativa correspondiente para atribuir los 0,7–1,8 s; no se ejecutó en esta entrega. -10. Mantener dry-run hasta revisión humana explícita. v1.0.9 no se considera apta +10. Mantener dry-run hasta revisión humana explícita. v1.0.10 no se considera apta para producción sólo por superar las pruebas portables. ## Límites diff --git a/etc/o6h-memory-guard.conf b/etc/o6h-memory-guard.conf index 6f8ec50..2d3b30a 100644 --- a/etc/o6h-memory-guard.conf +++ b/etc/o6h-memory-guard.conf @@ -1,4 +1,4 @@ -# o6h-memory-guard 1.0.9 -- whitespace-separated, no shell evaluation. +# o6h-memory-guard 1.0.10 -- whitespace-separated, no shell evaluation. # Restart required for configuration changes. Duplicate scalar keys are errors. ACTION_MODE dry-run SYSLOG_ENABLED yes @@ -19,6 +19,8 @@ MAX_VICTIMS_EPISODE 4 MAX_VICTIMS_MINUTE 4 MAX_EMERGENCY_VICTIMS_EPISODE 8 MAX_FLOOR_VICTIMS_EPISODE 12 +# Absolute ceiling; only persistent corroborated FLOOR may exceed the normal 12. +MAX_HARD_FLOOR_VICTIMS_EPISODE 16 MIN_RSS_KIB 65536 LOG_MAX_MIB 10 LOG_KEEP 5 diff --git a/logs/distribution-tests.log b/logs/distribution-tests.log index e8c363f..61d27d2 100644 --- a/logs/distribution-tests.log +++ b/logs/distribution-tests.log @@ -1,2 +1 @@ -sh tests/test_distribution.sh PASS distribution defaults to ACTION_MODE dry-run; production SMF does not force it diff --git a/logs/policy-tests.log b/logs/policy-tests.log index eeb2712..d2341e7 100644 --- a/logs/policy-tests.log +++ b/logs/policy-tests.log @@ -8,4 +8,5 @@ SIMULATED t=100 state=CRITICAL UUID=11111111-1111-4111-8111-111111111111 pid=123 SIMULATED t=105 state=CRITICAL pid=123 WOULD_ACTION=SIGKILL reason=memory_not_recovered SIMULATED state=EMERGENCY floor=1 WOULD_ACTION=SIGKILL reason=floor PASS TERM/KILL sequencing, emergency grace, floor, cooldown, budgets and no-action protections -PASS 175 checks; no native backend linked, no signals sent +PASS hard-floor persistence, finite ceiling, missing corroboration, gap, brief/stable recovery +PASS 240 checks; no native backend linked, no signals sent diff --git a/logs/portability.log b/logs/portability.log index 6649bc4..c5cf770 100644 --- a/logs/portability.log +++ b/logs/portability.log @@ -1,5 +1,3 @@ gcc -std=c99 -O1 -g -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -Isrc -Wall -Wextra -Werror -Wformat=2 -pedantic -pthread -Itests/api-sketch -fsyntax-only src/main.c src/core.c src/config.c src/support.c src/illumos.c src/validation.c src/result.c src/signal.c sh -n install.sh tests/test_distribution.sh xmllint --nonet --noout smf/o6h-memory-guard.xml -PASS GCC -fanalyzer for all SOURCES (silent Makefile loop) -API sketches only, not native ABI validation. diff --git a/logs/prepare-tests.log b/logs/prepare-tests.log index ee78399..855d595 100644 --- a/logs/prepare-tests.log +++ b/logs/prepare-tests.log @@ -1 +1 @@ -PASS 16 native preparation source checks with mocked proc/kstat/zone I/O; no native ABI claim or signals +PASS 27 native preparation source checks with mocked proc/kstat/zone I/O; no native ABI claim or signals diff --git a/logs/runtime-tests.log b/logs/runtime-tests.log index d06aad1..8341117 100644 --- a/logs/runtime-tests.log +++ b/logs/runtime-tests.log @@ -13,4 +13,6 @@ PASS 16-entry stale rescue cache supplies all 12 FLOOR victims without scan PASS saturated/locked event queue drops telemetry without delaying the signal PASS floor486 fixture, 50ms timeout fallback rank1->2, syscall errno attribution and validation substep logs PASS recovery/failure/stale refresh reasons, desfree equality, no second victim, PCKILL subphases and slow-event threshold -PASS 503 runtime checks; all platform calls mocked, no signals sent +PASS panic33: partial recoveries, budgets, dead PID replacement, incomplete scan, hard ceiling16 +PASS incremental pre-return publication, survivor ordering, bounded merge and nonblocking handoff +PASS 620 runtime checks; all platform calls mocked, no signals sent diff --git a/logs/sanitizers.log b/logs/sanitizers.log index f263086..7ff7641 100644 --- a/logs/sanitizers.log +++ b/logs/sanitizers.log @@ -1,4 +1,3 @@ -ASAN_OPTIONS=detect_leaks=0 (LeakSanitizer disabled in this environment) gcc -std=c99 -O1 -g -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -Isrc -Wall -Wextra -Werror -Wformat=2 -pedantic -pthread -fsanitize=address,undefined -fno-omit-frame-pointer -o test-sanitize-guard tests/test_guard.c src/core.c src/config.c ./test-sanitize-guard PASS strict configuration, safe defaults, explicit UUID authorization @@ -11,7 +10,8 @@ SIMULATED t=100 state=CRITICAL UUID=11111111-1111-4111-8111-111111111111 pid=123 SIMULATED t=105 state=CRITICAL pid=123 WOULD_ACTION=SIGKILL reason=memory_not_recovered SIMULATED state=EMERGENCY floor=1 WOULD_ACTION=SIGKILL reason=floor PASS TERM/KILL sequencing, emergency grace, floor, cooldown, budgets and no-action protections -PASS 175 checks; no native backend linked, no signals sent +PASS hard-floor persistence, finite ceiling, missing corroboration, gap, brief/stable recovery +PASS 240 checks; no native backend linked, no signals sent gcc -std=c99 -O1 -g -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -Isrc -Wall -Wextra -Werror -Wformat=2 -pedantic -pthread -fsanitize=address,undefined -fno-omit-frame-pointer -o test-sanitize-runtime tests/test_runtime.c src/core.c src/config.c src/result.c ./test-sanitize-runtime PASS actual daemon dry-run path records WOULD_ACTION=SIGTERM/SIGKILL; zero real requests @@ -29,10 +29,12 @@ PASS 16-entry stale rescue cache supplies all 12 FLOOR victims without scan PASS saturated/locked event queue drops telemetry without delaying the signal PASS floor486 fixture, 50ms timeout fallback rank1->2, syscall errno attribution and validation substep logs PASS recovery/failure/stale refresh reasons, desfree equality, no second victim, PCKILL subphases and slow-event threshold -PASS 503 runtime checks; all platform calls mocked, no signals sent +PASS panic33: partial recoveries, budgets, dead PID replacement, incomplete scan, hard ceiling16 +PASS incremental pre-return publication, survivor ordering, bounded merge and nonblocking handoff +PASS 620 runtime checks; all platform calls mocked, no signals sent gcc -std=c99 -O1 -g -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -Isrc -Wall -Wextra -Werror -Wformat=2 -pedantic -pthread -Itests/api-sketch -fsanitize=address,undefined -fno-omit-frame-pointer -o test-sanitize-validation tests/test_validation.c ./test-sanitize-validation PASS 70 validation checks: real 1283ms worker abandoned, fallback, late disposal, pool saturation; actual dispatch code with mocked write; no signals gcc -std=c99 -O1 -g -D_POSIX_C_SOURCE=200809L -D_XOPEN_SOURCE=700 -Isrc -Wall -Wextra -Werror -Wformat=2 -pedantic -pthread -Itests/api-sketch -fsanitize=address,undefined -fno-omit-frame-pointer -o test-sanitize-prepare tests/test_prepare.c src/core.c src/config.c ./test-sanitize-prepare -PASS 16 native preparation source checks with mocked proc/kstat/zone I/O; no native ABI claim or signals +PASS 27 native preparation source checks with mocked proc/kstat/zone I/O; no native ABI claim or signals diff --git a/src/config.c b/src/config.c index dadfa5f..32c3317 100644 --- a/src/config.c +++ b/src/config.c @@ -15,6 +15,7 @@ void defaults(config_t *c) c->scan_budget_ms = 200; c->cooldown_s = 10; c->max_victims = 4; c->max_per_minute = 4; c->max_emergency_victims = 8; c->max_floor_victims = 12; + c->max_hard_floor_victims = 16; c->min_rss_kib = 65536; c->log_mib = 10; c->log_keep = 5; } @@ -165,12 +166,14 @@ int config_read(FILE *f, config_t *c, char *error, size_t cap) OPT("MAX_CACHED_CANDIDATE_AGE_SECONDS", max_cached_candidate_age_s, 5, 60, 16) OPT("MAX_EMERGENCY_VICTIMS_EPISODE", max_emergency_victims, 1, MAX_VICTIMS, 17) OPT("MAX_FLOOR_VICTIMS_EPISODE", max_floor_victims, 1, MAX_VICTIMS, 18) + OPT("MAX_HARD_FLOOR_VICTIMS_EPISODE", max_hard_floor_victims, 1, 16, 21) #undef OPT goto invalid; } if (ferror(f) || c->pressure_s < c->pre_s || c->max_emergency_victims < c->max_victims || - c->max_floor_victims < c->max_emergency_victims) goto invalid; + c->max_floor_victims < c->max_emergency_victims || + c->max_hard_floor_victims < c->max_floor_victims) goto invalid; if (c->armed) { size_t i; int allow = 0; for (i = 0; i < c->nzones; i++) allow |= c->zones[i].policy == ALLOW; diff --git a/src/core.c b/src/core.c index dab571f..f5201b0 100644 --- a/src/core.c +++ b/src/core.c @@ -66,6 +66,8 @@ void engine_gap(engine_t *e) e->low_since = e->critical_since = e->recovery_since = -1; e->state = NORMAL; e->low_samples = 0; e->have_last = 0; e->danger = 0; e->floor = 0; + e->persistent_floor_since = -1; e->persistent_floor_samples = 0; + e->hard_floor_ready = 0; } void engine_step(engine_t *e, const config_t *c, const sample_t *s, @@ -121,6 +123,24 @@ void engine_step(engine_t *e, const config_t *c, const sample_t *s, } else e->recovery_since = -1; } e->danger = e->state == CRITICAL || e->state == EMERGENCY; + /* New authorization requires five seconds of fresh, consecutive FLOOR + * samples with active scanning and increasing anonymous pageout. Recovery + * retains episode budgets, but never retains authorization to override. */ + e->pageout_delta = pageout_d5; e->physical_delta = physical_swap_d5; + e->hard_floor_ready = 0; + if (e->state == EMERGENCY && e->floor && s->scan > 0 && + s->pageout_ok && pageout_d5.valid && pageout_d5.value > 0) { + if (e->persistent_floor_since < 0 || !e->have_last || + s->at - e->last_at > 1.5) { + e->persistent_floor_since = s->at; + e->persistent_floor_samples = 0; + } + if (e->persistent_floor_samples < UINT_MAX) e->persistent_floor_samples++; + e->hard_floor_ready = e->persistent_floor_samples >= 6 && + s->at - e->persistent_floor_since >= 5; + } else { + e->persistent_floor_since = -1; e->persistent_floor_samples = 0; + } e->last_at = s->at; e->last_free = s->free; e->have_last = 1; } @@ -149,7 +169,8 @@ plan_t action_plan(controller_t *a, const config_t *c, const engine_t *e, for (i = 0; i < a->nrate; i++) if (s->at - a->rate[i] < 60) a->rate[n++] = a->rate[i]; a->nrate = n; - if (e->state == NORMAL) a->nvictims = 0; + if (e->state == NORMAL && e->recovery_since >= 0 && + s->at - e->recovery_since >= c->recovery_s) a->nvictims = 0; if (s->free >= s->lots) a->has_pending = 0; if (!e->danger) { a->has_pending = 0; return p; } if (a->has_pending) { @@ -165,9 +186,17 @@ plan_t action_plan(controller_t *a, const config_t *c, const engine_t *e, } if (e->floor) { if (a->nvictims >= c->max_floor_victims) { - p.reason = "absolute_victim_budget_exhausted"; return p; + if (a->nvictims >= c->max_hard_floor_victims) { + p.reason = "hard_floor_victim_budget_exhausted"; return p; + } + if (!e->hard_floor_ready || e->state != EMERGENCY || + s->free > s->min / 2 || s->scan == 0 || !s->pageout_ok || + s->at < e->last_at || s->at - e->last_at > 1.5) { + p.reason = "absolute_victim_budget_exhausted"; return p; + } + p.budget_override = "hard_floor"; } - if (a->nvictims >= c->max_victims || a->nrate >= c->max_per_minute) + else if (a->nvictims >= c->max_victims || a->nrate >= c->max_per_minute) p.budget_override = "floor"; } else if (e->state == EMERGENCY) { if (a->nvictims >= c->max_emergency_victims) { diff --git a/src/guard.h b/src/guard.h index 96f9202..39b94e4 100644 --- a/src/guard.h +++ b/src/guard.h @@ -13,7 +13,7 @@ #define CONF_PATH "/opt/custom/etc/o6h-memory-guard.conf" #define LOG_DIR "/opt/custom/var/log/o6h-memory-guard" #define RUN_DIR "/opt/custom/var/run" -#define VERSION "1.0.9" +#define VERSION "1.0.10" enum state { NORMAL, PRE_PRESSURE, PRESSURE, CRITICAL, EMERGENCY, RECOVERY }; enum policy { OBSERVE, ALLOW, PROTECT }; @@ -35,7 +35,7 @@ typedef struct { unsigned pre_s, pressure_s, critical_s, recovery_s, critical_term_s; unsigned emergency_term_s, floor_kill_delay_s, max_cached_candidate_age_s; unsigned normal_scan_s, scan_budget_ms, cooldown_s, max_victims; - unsigned max_emergency_victims, max_floor_victims; + unsigned max_emergency_victims, max_floor_victims, max_hard_floor_victims; unsigned max_per_minute, log_mib, log_keep; uint64_t min_rss_kib; zone_rule_t zones[MAX_ZONES]; size_t nzones; @@ -55,6 +55,10 @@ typedef struct { uint64_t last_free; unsigned low_samples; int have_last, danger, floor; + double persistent_floor_since; + unsigned persistent_floor_samples; + int hard_floor_ready; + delta_t pageout_delta, physical_delta; } engine_t; typedef struct { process_t pending; diff --git a/src/illumos.c b/src/illumos.c index a58a563..8af0e68 100644 --- a/src/illumos.c +++ b/src/illumos.c @@ -186,8 +186,10 @@ int platform_walk(int (*visit)(const process_t *, void *), void *arg, unsigned b { struct dirent *de; double deadline = monotime() + (double)budget_ms / 1000; - int partial = 0; - nzones = 0; rewinddir(procdir); + static int resume; + int partial = resume; + nzones = 0; + if (!resume) rewinddir(procdir); for (;;) { char path[96], *end; long pid; @@ -197,9 +199,9 @@ int platform_walk(int (*visit)(const process_t *, void *), void *arg, unsigned b process_t p; zone_cache_t *z = NULL; size_t i; + if (monotime() >= deadline) { partial = 1; resume = 1; break; } errno = 0; de = readdir(procdir); - if (!de) { if (errno) partial = 1; break; } - if (monotime() >= deadline) { partial = 1; break; } + if (!de) { if (errno) partial = 1; resume = 0; break; } if (!isdigit((unsigned char)de->d_name[0])) continue; errno = 0; pid = strtol(de->d_name, &end, 10); if (errno || *end || pid <= 0 || pid > 2147483647L) continue; @@ -221,7 +223,7 @@ int platform_walk(int (*visit)(const process_t *, void *), void *arg, unsigned b } if (!z) partial = 1; convert(&info, z, &p); - if (visit(&p, arg)) { partial = 1; break; } + if (visit(&p, arg)) { partial = 1; resume = 1; break; } } return partial ? -1 : 0; } diff --git a/src/main.c b/src/main.c index 2c5e76c..f19af61 100644 --- a/src/main.c +++ b/src/main.c @@ -40,6 +40,9 @@ static tracked_t processes[MAX_PROCS]; static zone_t zone_stats[MAX_ZONES]; static cached_candidate_t candidate_cache[CACHED_CANDIDATES]; static cached_candidate_t published_cache[CACHED_CANDIDATES]; +/* Rescue-owned rejection memory; scanner never reads it without cache handoff. */ +static process_t rejected_identity[64]; +static unsigned rejected_identity_next, rejected_identity_count; static pthread_mutex_t cache_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_mutex_t scan_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t scan_cond = PTHREAD_COND_INITIALIZER; @@ -350,6 +353,7 @@ static void score_candidate(cached_candidate_t *entry, const zone_t *z) entry->zone_component); } +static void publish_partial_candidate(const tracked_t *, const zone_t *); static int visit(const process_t *p, void *unused) { unsigned i, slot = MAX_PROCS, zslot = MAX_ZONES; @@ -388,6 +392,7 @@ static int visit(const process_t *p, void *unused) z->seen = scan_generation; z->last = scan_at; if (UINT64_MAX - z->rss < p->rss_kib) return -1; z->rss += p->rss_kib; + publish_partial_candidate(t, z); return 0; } @@ -416,6 +421,56 @@ static int cache_better(const cached_candidate_t *a, const cached_candidate_t *b return a->p.pid < b->p.pid; } +static void publish_partial_candidate(const tracked_t *t, const zone_t *z) +{ + cached_candidate_t entry; + unsigned i, j; + if (protected_process(&cfg, &t->p)) return; + memset(&entry, 0, sizeof (entry)); + entry.p = t->p; entry.scan_time = scan_at; + entry.valid = entry.eligible = 1; + entry.d5 = history_delta(&t->rss, scan_at, 5); + entry.d15 = history_delta(&t->rss, scan_at, 15); + entry.d60 = history_delta(&t->rss, scan_at, 60); + score_candidate(&entry, z); + snprintf(entry.alias, sizeof (entry.alias), "%s", alias_for(t->p.uuid)); + pthread_mutex_lock(&cache_mutex); + for (i = 0, j = 0; i < CACHED_CANDIDATES; i++) + if (published_cache[i].valid) published_cache[j++] = published_cache[i]; + while (j < CACHED_CANDIDATES) memset(&published_cache[j++], 0, sizeof (entry)); + for (i = 0; i < CACHED_CANDIDATES; i++) + if (published_cache[i].valid && same_process(&entry.p, &published_cache[i].p)) break; + /* Partial observations never replace a better survivor. */ + if (i < CACHED_CANDIDATES) { + if (!cache_better(&entry, &published_cache[i])) { + pthread_mutex_unlock(&cache_mutex); return; + } + for (j = i; j + 1 < CACHED_CANDIDATES; j++) published_cache[j] = published_cache[j + 1]; + memset(&published_cache[CACHED_CANDIDATES - 1], 0, sizeof (entry)); + } + for (i = 0; i < CACHED_CANDIDATES; i++) { + if (!cache_better(&entry, &published_cache[i])) continue; + for (j = CACHED_CANDIDATES - 1; j > i; j--) published_cache[j] = published_cache[j - 1]; + published_cache[i] = entry; break; + } + for (j = 0; j < CACHED_CANDIDATES; j++) published_cache[j].rank = j + 1; + published_scan_complete = 0; + if (!scanner_async) memcpy(candidate_cache, published_cache, sizeof (candidate_cache)); + pthread_mutex_unlock(&cache_mutex); +} + +static void invalidate_candidate(const process_t *p) +{ + unsigned i; + rejected_identity[rejected_identity_next] = *p; + rejected_identity_next = (rejected_identity_next + 1) % 64; + if (rejected_identity_count < 64) rejected_identity_count++; + if (pthread_mutex_trylock(&cache_mutex) != 0) return; + for (i = 0; i < CACHED_CANDIDATES; i++) + if (same_process(p, &published_cache[i].p)) published_cache[i].valid = 0; + pthread_mutex_unlock(&cache_mutex); +} + static void publish_candidate_cache(const cached_candidate_t *next, int complete) { pthread_mutex_lock(&cache_mutex); @@ -436,7 +491,12 @@ static void publish_candidate_cache(const cached_candidate_t *next, int complete static void refresh_candidate_cache(void) { - if (!scanner_async || pthread_mutex_trylock(&cache_mutex) != 0) return; + unsigned i, j; + if (pthread_mutex_trylock(&cache_mutex) != 0) return; + for (i = 0; i < CACHED_CANDIDATES; i++) + for (j = 0; j < rejected_identity_count; j++) + if (same_process(&published_cache[i].p, &rejected_identity[j])) + published_cache[i].valid = 0; memcpy(candidate_cache, published_cache, sizeof (candidate_cache)); scan_complete = published_scan_complete; latest_complete_scan_at = published_latest_complete_scan_at; @@ -660,13 +720,13 @@ static void log_sample(const sample_t *s) " swap_physical_used_ok=%d swap_physical_used_bytes=%" PRIu64 " anon_pageout_ok=%d anon_pageout_pages=%" PRIu64 " swap_alloc_pages=%" PRIu64 " swap_reserved_pages=%" PRIu64 - " swap_available_pages=%" PRIu64, + " swap_available_pages=%" PRIu64 " hard_floor_ready=%d persistent_floor_samples=%u", state_name(engine.state), engine.danger, engine.floor, s->free, s->lots, s->des, s->min, s->scan, s->kernel, s->locked, s->arc_ok, s->arc, s->swap_virtual_ok, s->swap_alloc + s->swap_reserved, s->swap_physical_ok, s->swap_physical_used_bytes, s->pageout_ok, s->anon_pageout_pages, s->swap_alloc, - s->swap_reserved, s->swap_available); + s->swap_reserved, s->swap_available, engine.hard_floor_ready, engine.persistent_floor_samples); for (i = 0; i < METRIC_COUNT; i++) { char d5[64], d15[64], d60[64]; delta_text(d5, sizeof (d5), history_delta(&metrics[i], s->at, 5)); @@ -713,11 +773,15 @@ static void action_blocked(const char *reason, enum state state, "event=action_blocked reason=%s state=%s free_pages=%" PRIu64 " desfree=%" PRIu64 " minfree=%" PRIu64 " refresh_ok=%d" " candidate_source=%s refresh_errno=%d refresh_error=%s refresh_latency_ms=%.3f" - " sample_source=%s scan_complete=%d latest_complete_age_ms=%.1f%s", + " sample_source=%s scan_complete=%d latest_complete_age_ms=%.1f%s" + " normal_used=%u emergency_used=%u floor_used=%u hard_floor_used=%u" + " nscan=%" PRIu64 " anonpgout_delta=%" PRId64 " anonpgout_delta_ok=%d", reason, state_name(state), s->free, s->des, s->min, refresh_ok, source, refresh_errno, refresh_error, refresh_ms, refresh_ok == 1 ? "refresh" : "last_sample", scan_complete, - latest_complete_scan_at > 0 ? 1000 * (s->at - latest_complete_scan_at) : -1.0, warmup); + latest_complete_scan_at > 0 ? 1000 * (s->at - latest_complete_scan_at) : -1.0, warmup, + controller.nvictims, controller.nvictims, controller.nvictims, controller.nvictims, + s->scan, engine.pageout_delta.value, engine.pageout_delta.valid); } static void act(sample_t *sample) @@ -735,6 +799,7 @@ static void act(sample_t *sample) const char *initial_source = controller.has_pending ? "pending" : "not_selected"; unsigned i, rank = 0, rejected = 0; unsigned budget_ms; + unsigned victims_before = controller.nvictims; int ticket = -1; validation_t timing; signal_result_t signal_result; @@ -743,6 +808,12 @@ static void act(sample_t *sample) if (stopping) return; if (!engine.danger) { (void)action_plan(&controller, &cfg, &engine, sample, NULL); + if (victims_before && !controller.nvictims) { + rejected_identity_next = rejected_identity_count = 0; + emit_important(LOG_INFO, "event=victim_budget tier=normal used=0 limit=%u" + " remaining=%u reason=stable_recovery mode=%s", cfg.max_victims, + cfg.max_victims, cfg.armed ? "armed" : "dry-run"); + } return; } if (cfg.armed && monotime() - started < 60) { @@ -794,6 +865,11 @@ retry_candidate: validation_ms = 1000 * (monotime() - validation_start); validated = 1; } else { + plan = action_plan(&controller, &cfg, ¤t_engine, &fresh, NULL); + if (strstr(plan.reason, "budget_exhausted")) { + action_blocked(plan.reason, current_engine.state, &fresh, 1, 0, + "none", refresh_ms, "not_selected"); return; + } for (i = 0; i < CACHED_CANDIDATES; i++) { cached_candidate_t *entry = &candidate_cache[i]; candidate_age = fresh.at - entry->scan_time; @@ -809,6 +885,8 @@ retry_candidate: current_engine.floor ? fresh.min / 2 + 1 : fresh.des, budget_ms, &ticket, &timing)) { err = errno; entry->valid = 0; + if (err == ENOENT || err == ESRCH || err == ESTALE) + invalidate_candidate(&entry->p); rejections[rejected].rank = i + 1; rejections[rejected].error = err; rejections[rejected].ms = timing.total_ms; rejections[rejected].stage = timing.stage; @@ -843,8 +921,24 @@ retry_candidate: validation_release(ticket); /* State and budgets are memory-only and are updated after the signal attempt. */ action_commit(&controller, &cfg, &fresh, &plan); + invalidate_candidate(&plan.target); if (cfg.armed && rc) controller.has_pending = 0; enqueue_start = monotime(); + if (controller.nvictims != victims_before) { + const char *tier = !strcmp(plan.budget_override, "hard_floor") ? "hard_floor" : + current_engine.floor ? "floor" : current_engine.state == EMERGENCY ? "emergency" : "normal"; + unsigned limit = !strcmp(tier, "hard_floor") ? cfg.max_hard_floor_victims : + current_engine.floor ? cfg.max_floor_victims : + current_engine.state == EMERGENCY ? cfg.max_emergency_victims : cfg.max_victims; + emit_important(LOG_WARNING, "event=victim_budget tier=%s used=%u limit=%u remaining=%u mode=%s", + tier, controller.nvictims, limit, limit > controller.nvictims ? limit - controller.nvictims : 0, + cfg.armed ? "armed" : "dry-run"); + if (!strcmp(plan.budget_override, "hard_floor")) + emit_important(LOG_CRIT, "event=budget_override reason=persistent_floor" + " used=%u limit=%u free_pages=%" PRIu64 " nscan=%" PRIu64 + " anonpgout_delta=%" PRId64, + controller.nvictims, limit, fresh.free, fresh.scan, engine.pageout_delta.value); + } log_rejections(rejections, rejected); if (rejected) emit("event=action_rejected reason=identity_or_protection rejected_count=%u" @@ -1045,6 +1139,7 @@ int main(int argc, char **argv) if (platform_sample(&sample) || monotime() - sample.at > 2) { int err = errno; engine_gap(&engine); controller.has_pending = 0; scan_complete = 0; + if (controller.nvictims) engine.state = RECOVERY; emit("event=sample_error errno=%d actions=inhibited", err); syslog_event(LOG_ERR, "event=internal_error reason=sample_error errno=%d actions=inhibited", err); @@ -1072,6 +1167,9 @@ int main(int argc, char **argv) } } record_auxiliary_metrics(&sample); + if (engine.danger && sample.at - last_scan >= 1) { + request_scan(0); last_scan = sample.at; + } /* All telemetry is queued only after the rescue path has returned. */ if (engine.state != previous) (void)transition_message(transition, sizeof (transition), previous, diff --git a/tests/fixtures/panic33_persistent_floor_budget_exhaustion_2026_10_08.csv b/tests/fixtures/panic33_persistent_floor_budget_exhaustion_2026_10_08.csv new file mode 100644 index 0000000..dba0543 --- /dev/null +++ b/tests/fixtures/panic33_persistent_floor_budget_exhaustion_2026_10_08.csv @@ -0,0 +1,9 @@ +# Synthetic reconstruction from supplied panic33 evidence, NOT raw host telemetry. +# free_pages,seconds,partial_scan,expected_total_victims +40000,16,0,8 +70000,2,0,8 +40000,2,0,8 +10719,2,1,10 +29578,1,1,10 +6513,15,1,16 +42129,2,1,16 diff --git a/tests/test_distribution.sh b/tests/test_distribution.sh index 4980486..c1686f8 100644 --- a/tests/test_distribution.sh +++ b/tests/test_distribution.sh @@ -30,6 +30,7 @@ MAX_VICTIMS_EPISODE 4 MAX_VICTIMS_MINUTE 4 MAX_EMERGENCY_VICTIMS_EPISODE 8 MAX_FLOOR_VICTIMS_EPISODE 12 +MAX_HARD_FLOOR_VICTIMS_EPISODE 16 EOF grep -Fq -- '"--force-dry-run"' src/main.c diff --git a/tests/test_guard.c b/tests/test_guard.c index 8d4f19f..3e13035 100644 --- a/tests/test_guard.c +++ b/tests/test_guard.c @@ -66,6 +66,10 @@ static void configs(void) "FLOOR_KILL_DELAY_SECONDS 0\nMAX_CACHED_CANDIDATE_AGE_SECONDS 15\n", &c) == 0); CHECK(parse("MAX_EMERGENCY_VICTIMS_EPISODE 3\n", &c) != 0); CHECK(parse("MAX_FLOOR_VICTIMS_EPISODE 7\n", &c) != 0); + CHECK(parse("MAX_HARD_FLOOR_VICTIMS_EPISODE 16\n", &c) == 0 && c.max_hard_floor_victims == 16); + CHECK(parse("MAX_HARD_FLOOR_VICTIMS_EPISODE 17\n", &c) != 0); + CHECK(parse("MAX_HARD_FLOOR_VICTIMS_EPISODE 11\n", &c) != 0); + CHECK(parse("MAX_HARD_FLOOR_VICTIMS_EPISODE 16\nMAX_HARD_FLOOR_VICTIMS_EPISODE 16\n", &c) != 0); CHECK(parse("UNKNOWN foo\n", &c) != 0); CHECK(parse("PRE_SECONDS 20\nPRESSURE_SECONDS 10\n", &c) != 0); CHECK(parse("zone global global allow\n", &c) != 0); @@ -255,6 +259,7 @@ static void actions(void) plan = action_plan(&a, &c, &e, &s, &p); CHECK(plan.action == KILL); CHECK(!strcmp(plan.budget_override, "floor")); e.state = NORMAL; e.danger = 0; s.free = 160000; + e.recovery_since = s.at - c.recovery_s; (void)action_plan(&a, &c, &e, &s, &p); CHECK(a.nvictims == 0); e.state = EMERGENCY; e.danger = 1; e.floor = 0; s.free = 40000; a.nvictims = c.max_victims; a.nrate = c.max_per_minute; @@ -285,10 +290,63 @@ static void actions(void) } puts("PASS TERM/KILL sequencing, emergency grace, floor, cooldown, budgets and no-action protections"); } +static void hard_floor_policy(void) +{ + config_t c = allowed(); + controller_t a; + engine_t e; + sample_t s = sample(100, 6513, 1000000); + process_t p = process(3000); + delta_t growing = {1, 5, 100}, invalid = {0, 0, 0}; + unsigned i; + memset(&a, 0, sizeof (a)); engine_init(&e); + a.nvictims = c.max_floor_victims; s.pageout_ok = 1; + for (i = 0; i < 6; i++) { + s.at = 100 + i; + engine_step(&e, &c, &s, invalid, growing, growing); + CHECK(e.hard_floor_ready == (i == 5)); + CHECK((action_plan(&a, &c, &e, &s, &p).action == KILL) == (i == 5)); + } + CHECK(!strcmp(action_plan(&a, &c, &e, &s, &p).budget_override, "hard_floor")); + a.nvictims = 16; + CHECK(!strcmp(action_plan(&a, &c, &e, &s, &p).reason, "hard_floor_victim_budget_exhausted")); + a.nvictims = 12; s.at += 2; + engine_step(&e, &c, &s, invalid, growing, growing); + CHECK(!e.hard_floor_ready); + for (i = 0; i < 8; i++) { + s.at++; engine_step(&e, &c, &s, invalid, growing, invalid); + CHECK(!e.hard_floor_ready); + } + s.scan = 0; + for (i = 0; i < 8; i++) { + s.at++; engine_step(&e, &c, &s, invalid, growing, growing); + CHECK(!e.hard_floor_ready); + } + s.free = 70000; s.at++; + engine_step(&e, &c, &s, invalid, invalid, invalid); + CHECK(e.state == RECOVERY); + (void)action_plan(&a, &c, &e, &s, &p); CHECK(a.nvictims == 12); + s.at += 2; s.free = 6513; + engine_step(&e, &c, &s, invalid, invalid, invalid); + CHECK(e.state == EMERGENCY && !e.hard_floor_ready); + (void)action_plan(&a, &c, &e, &s, &p); CHECK(a.nvictims == 12); + engine_gap(&e); s.at++; s.free = s.lots; + (void)action_plan(&a, &c, &e, &s, &p); CHECK(a.nvictims == 12); + s.free = 6513; s.at++; engine_step(&e, &c, &s, invalid, invalid, invalid); + s.free = s.lots; + for (i = 0; i <= c.recovery_s; i++) { + s.at++; engine_step(&e, &c, &s, invalid, invalid, invalid); + (void)action_plan(&a, &c, &e, &s, &p); + CHECK(a.nvictims == (i == c.recovery_s ? 0u : 12u)); + } + CHECK(e.state == NORMAL); + puts("PASS hard-floor persistence, finite ceiling, missing corroboration, gap, brief/stable recovery"); +} + int main(void) { configs(); protections(); histories(); states(); real_incident_fixture(); - successful_avoidance_fixture(); actions(); + successful_avoidance_fixture(); actions(); hard_floor_policy(); printf("PASS %u checks; no native backend linked, no signals sent\n", checks); return 0; } diff --git a/tests/test_prepare.c b/tests/test_prepare.c index 2cfe5af..6023910 100644 --- a/tests/test_prepare.c +++ b/tests/test_prepare.c @@ -7,6 +7,9 @@ #include #include #include +#include +static struct dirent *mock_readdir(DIR *); +static void mock_rewinddir(DIR *); static int mock_open(const char *, int, ...); static int mock_openat(int, const char *, int, ...); static ssize_t mock_pread(int, void *, size_t, off_t); @@ -15,16 +18,35 @@ static int mock_close(int); #define openat mock_openat #define pread mock_pread #define close mock_close +#define readdir mock_readdir +#define rewinddir mock_rewinddir #include "../src/illumos.c" #undef open #undef openat #undef pread #undef close +#undef readdir +#undef rewinddir static unsigned checks; #define CHECK(x) do { if (!(x)) { fprintf(stderr, "FAIL prepare %d: %s\n", __LINE__, #x); exit(1); } checks++; } while (0) static double now; static psinfo_t ps; +static unsigned dir_cursor, dir_rewinds, walked; +static struct dirent mock_dirent; +static struct dirent *mock_readdir(DIR *dir) +{ + (void)dir; + if (dir_cursor == 3) return NULL; + ps.pr_pid = 123 + (int)dir_cursor++; + snprintf(mock_dirent.d_name, sizeof (mock_dirent.d_name), "%d", ps.pr_pid); + return &mock_dirent; +} +static void mock_rewinddir(DIR *dir) { (void)dir; dir_cursor = 0; dir_rewinds++; } +static int walk_visit(const process_t *p, void *arg) +{ + (void)arg; CHECK(p->pid == 123 + (int)(walked % 3)); walked++; return 0; +} static uint64_t generation = 20, available = 486; static int ctl_opens, closed, fail_open, fail_read, fail_generation; static kstat_ctl_t fake_kc; @@ -111,6 +133,11 @@ int main(void) fail_generation = 0; ctl_opens = 0; CHECK(platform_prepare(0, &c, &p, 0, 24539, &handle, &t) == 0); CHECK(handle == -1 && ctl_opens == 0); + CHECK(platform_walk(walk_visit, NULL, 1) == -1 && walked == 1); + CHECK(platform_walk(walk_visit, NULL, 1) == -1 && walked == 2); + CHECK(platform_walk(walk_visit, NULL, 1) == -1 && walked == 3); + CHECK(platform_walk(walk_visit, NULL, 1) == -1 && walked == 3 && dir_rewinds == 1); + CHECK(platform_walk(walk_visit, NULL, 200) == 0 && walked == 6 && dir_rewinds == 2); printf("PASS %u native preparation source checks with mocked proc/kstat/zone I/O; no native ABI claim or signals\n", checks); return 0; } diff --git a/tests/test_runtime.c b/tests/test_runtime.c index 9a70cc6..407f4df 100644 --- a/tests/test_runtime.c +++ b/tests/test_runtime.c @@ -96,6 +96,8 @@ static void reset(void) memset(processes, 0, sizeof (processes)); memset(zone_stats, 0, sizeof (zone_stats)); memset(candidate_cache, 0, sizeof (candidate_cache)); memset(published_cache, 0, sizeof (published_cache)); + memset(rejected_identity, 0, sizeof (rejected_identity)); + rejected_identity_next = rejected_identity_count = 0; memset(metrics, 0, sizeof (metrics)); memset(&controller, 0, sizeof (controller)); memset(fixtures, 0, sizeof (fixtures)); engine_init(&engine); engine.state = CRITICAL; engine.danger = 1; @@ -149,6 +151,84 @@ static int queued_contains(const char *text) return 0; } +static void panic33_fixture(void) +{ + FILE *f; + char line[256]; + unsigned seconds, incomplete, expected, j, rows = 0; + uint64_t free_pages; + delta_t d = {1, 5, 100}; + int removed = 0; + populate_many_candidates(16); + cfg.armed = 1; async_logging = 1; + engine_init(&engine); + f = fopen("tests/fixtures/panic33_persistent_floor_budget_exhaustion_2026_10_08.csv", "r"); + CHECK(f != NULL); + while (fgets(line, sizeof (line), f)) { + if (line[0] == '#') continue; + CHECK(sscanf(line, "%" SCNu64 ",%u,%u,%u", &free_pages, &seconds, &incomplete, &expected) == 4); + for (j = 0; j < seconds; j++) { + fake_now++; fake_sample.at = fake_now; fake_sample.free = free_pages; + fake_sample.scan = 1000000; fake_sample.pageout_ok = 1; + if (incomplete && !removed) { + /* Cached PID disappeared; its replacement is smaller than survivors. */ + fixtures[8].pid = 2000; fixtures[8].start_sec++; + fixtures[8].rss_kib = 100000; removed = 1; + } + engine_step(&engine, &cfg, &fake_sample, d, d, d); + refresh_candidate_cache(); + act(&fake_sample); + partial = (int)incomplete; + scan_processes(0); + } + CHECK(controller.nvictims == expected); + rows++; + } + fclose(f); + CHECK(rows == 7 && !scan_complete); + CHECK(queued_contains("reason=emergency_victim_budget_exhausted")); + CHECK(queued_contains("event=budget_override reason=persistent_floor")); + CHECK(queued_contains("event=victim_budget tier=hard_floor used=16 limit=16 remaining=0")); + CHECK(was_victim(&controller, &fixtures[8])); + CHECK(real_requests > 16); /* TERM escalation plus direct FLOOR KILL. */ + fake_sample.free = 6513; fake_sample.at = ++fake_now; + act(&fake_sample); + CHECK(queued_contains("reason=hard_floor_victim_budget_exhausted")); + CHECK(controller.nvictims == 16); + puts("PASS panic33: partial recoveries, budgets, dead PID replacement, incomplete scan, hard ceiling16"); +} + +static void partial_cache_tests(void) +{ + process_t replacement, survivor; + unsigned i; + int found = 0; + populate_many_candidates(16); + survivor = candidate_cache[1].p; + invalidate_candidate(&candidate_cache[0].p); + scanner_async = 1; + replacement = fixtures[0]; replacement.pid = 9000; replacement.start_sec++; + replacement.rss_kib = 70000; + scan_at = ++fake_now; scan_generation++; + /* Callback publishes before platform_walk returns (or blocks). */ + CHECK(visit(&replacement, NULL) == 0); + refresh_candidate_cache(); + for (i = 0; i < CACHED_CANDIDATES; i++) + if (candidate_cache[i].valid && same_process(&candidate_cache[i].p, &replacement)) found = 1; + CHECK(found && !scan_complete); + CHECK(same_process(&candidate_cache[0].p, &survivor)); + CHECK(pthread_mutex_lock(&cache_mutex) == 0); + refresh_candidate_cache(); /* trylock only: must return while scanner holds lock. */ + CHECK(pthread_mutex_unlock(&cache_mutex) == 0); + replacement.pid = 9001; replacement.rss_kib = 65536; + CHECK(visit(&replacement, NULL) == 0); + refresh_candidate_cache(); + CHECK(same_process(&candidate_cache[0].p, &survivor)); + CHECK(candidate_cache[15].p.pid == 9000); + scanner_async = 0; + puts("PASS incremental pre-return publication, survivor ordering, bounded merge and nonblocking handoff"); +} + static void critical_path_faults(void) { struct timespec before, after; double elapsed_ms; unsigned walks; @@ -554,7 +634,7 @@ int main(void) syslog_open_count == 1); emit_important(LOG_INFO, "event=start version=%s mode=dry-run", VERSION); CHECK(syslog_count == 1 && syslog_records[0].priority == LOG_INFO && - strstr(syslog_records[0].message, "event=start version=1.0.9") != NULL); + strstr(syslog_records[0].message, "event=start version=" VERSION) != NULL); log_sample(&fake_sample); scan_processes(0); CHECK(syslog_count == 1); /* High-volume sample/delta/proc/cache events stay local. */ { @@ -567,7 +647,7 @@ int main(void) CHECK(syslog_count == 2 && syslog_records[1].priority == LOG_CRIT); } cfg.armed = 0; act(&fake_sample); - CHECK(syslog_count == 2 && real_requests == 0); + CHECK(syslog_count == 3 && real_requests == 0); reset(); cfg.armed = 1; started = 80; fake_now = 100; CHECK(platform_syslog_open("o6h-memory-guard", cfg.syslog_facility) == 0); syslog_ready = 1; act(&fake_sample); @@ -577,19 +657,20 @@ int main(void) reset(); cfg.armed = 1; CHECK(platform_syslog_open("o6h-memory-guard", cfg.syslog_facility) == 0); syslog_ready = 1; act(&fake_sample); - CHECK(real_requests == 1 && syslog_count == 2); - CHECK(syslog_records[0].priority == LOG_ERR && - strstr(syslog_records[0].message, "event=action_decision state=CRITICAL") != NULL && - strstr(syslog_records[0].message, "result=sent") != NULL); + CHECK(real_requests == 1 && syslog_count == 3); + CHECK(strstr(syslog_records[0].message, "event=victim_budget tier=normal") != NULL); CHECK(syslog_records[1].priority == LOG_ERR && - strstr(syslog_records[1].message, "event=signal_sent state=CRITICAL") != NULL && - strstr(syslog_records[1].message, "signal=SIGTERM") != NULL && - strstr(syslog_records[1].message, "UUID=") != NULL && - strstr(syslog_records[1].message, "alias=web-a") != NULL && - strstr(syslog_records[1].message, "pid=123") != NULL && - strstr(syslog_records[1].message, "comm=php-fpm") != NULL && - strstr(syslog_records[1].message, "rss_kib=100000") != NULL && - strstr(syslog_records[1].message, "free_pages=60000") != NULL); + strstr(syslog_records[1].message, "event=action_decision state=CRITICAL") != NULL && + strstr(syslog_records[1].message, "result=sent") != NULL); + CHECK(syslog_records[2].priority == LOG_ERR && + strstr(syslog_records[2].message, "event=signal_sent state=CRITICAL") != NULL && + strstr(syslog_records[2].message, "signal=SIGTERM") != NULL && + strstr(syslog_records[2].message, "UUID=") != NULL && + strstr(syslog_records[2].message, "alias=web-a") != NULL && + strstr(syslog_records[2].message, "pid=123") != NULL && + strstr(syslog_records[2].message, "comm=php-fpm") != NULL && + strstr(syslog_records[2].message, "rss_kib=100000") != NULL && + strstr(syslog_records[2].message, "free_pages=60000") != NULL); reset(); cfg.armed = 1; fail_syslog = 1; syslog_ready = 1; act(&fake_sample); CHECK(real_requests == 1 && controller.has_pending); reset(); cfg.syslog_enabled = 0; syslog_ready = 0; @@ -608,6 +689,8 @@ int main(void) critical_path_faults(); floor_errno_fixture(); diagnostic_fixtures(); + panic33_fixture(); + partial_cache_tests(); fclose(log); printf("PASS %u runtime checks; all platform calls mocked, no signals sent\n", checks); return 0;