Conservative SmartOS global-zone memory pressure guard with dry-run-first policy and SMF integration.
Nevar pievienot vairāk kā 25 tēmas Tēmai ir jāsākas ar burtu vai ciparu, tā var saturēt domu zīmes ('-') un var būt līdz 35 simboliem gara.
 
 
 
 

615 rindas
32 KiB

  1. /* Exercise the actual daemon selection/action path. Native backend NOT linked. */
  2. #define main guard_program_main
  3. #include "../src/main.c"
  4. #undef main
  5. static double fake_now = 100;
  6. static double refresh_age, dispatch_delay;
  7. static sample_t fake_sample;
  8. static process_t fixtures[32];
  9. static unsigned nfixtures, real_requests, validation_requests, walk_requests, checks;
  10. static int fail_validation, fail_sample, partial, invalid_pid;
  11. typedef struct { int priority; char message[1536]; } syslog_record_t;
  12. static syslog_record_t syslog_records[64];
  13. static unsigned syslog_count, syslog_open_count, syslog_close_count;
  14. static int syslog_facility, fail_syslog;
  15. static double syslog_delay_seconds;
  16. static char syslog_ident[64];
  17. #define CHECK(x) do { if (!(x)) { fprintf(stderr, "FAIL runtime line %d: %s\n", __LINE__, #x); exit(1); } checks++; } while (0)
  18. double monotime(void) { return fake_now; }
  19. int secure_file(const char *p, int flags, unsigned mode)
  20. { (void)p; (void)flags; (void)mode; errno = EPERM; return -1; }
  21. int secure_directory(const char *p) { (void)p; return -1; }
  22. int platform_init(void) { return 0; }
  23. void platform_fini(void) { }
  24. int platform_syslog_open(const char *ident, int facility)
  25. {
  26. syslog_open_count++; syslog_facility = facility;
  27. snprintf(syslog_ident, sizeof (syslog_ident), "%s", ident);
  28. return fail_syslog ? -1 : 0;
  29. }
  30. int platform_syslog_write(int priority, const char *message)
  31. {
  32. if (fail_syslog) { errno = EIO; return -1; }
  33. fake_now += syslog_delay_seconds;
  34. CHECK(syslog_count < sizeof (syslog_records) / sizeof (syslog_records[0]));
  35. syslog_records[syslog_count].priority = priority;
  36. snprintf(syslog_records[syslog_count].message,
  37. sizeof (syslog_records[syslog_count].message), "%s", message);
  38. syslog_count++;
  39. return 0;
  40. }
  41. void platform_syslog_close(void) { syslog_close_count++; }
  42. int platform_sample(sample_t *s)
  43. { *s = fake_sample; s->at = fake_now - refresh_age;
  44. if (fail_sample) { memset(s, 0, sizeof (*s)); errno = EIO; return -1; }
  45. return 0; }
  46. int platform_walk(int (*fn)(const process_t *, void *), void *arg, unsigned budget)
  47. {
  48. unsigned j; (void)budget;
  49. walk_requests++;
  50. for (j = 0; j < nfixtures; j++) if (fn(&fixtures[j], arg)) return -1;
  51. return partial ? -1 : 0;
  52. }
  53. static int mock_signal_errno, mock_validation_errno, mock_timeout_pid;
  54. static int mock_ticket_pid;
  55. int validation_start(void) { return 0; }
  56. int validation_handle(int ticket) { return ticket; }
  57. void validation_release(int ticket) { (void)ticket; }
  58. int validation_acquire(const config_t *c, const process_t *p, int real,
  59. uint64_t ceiling, unsigned budget_ms, int *ticket, validation_t *timing)
  60. {
  61. unsigned j; int found = 0;
  62. (void)real; (void)ceiling;
  63. memset(timing, 0, sizeof (*timing));
  64. CHECK(!protected_process(c, p));
  65. for (j = 0; j < nfixtures; j++)
  66. if (fixtures[j].pid == p->pid && same_process(&fixtures[j], p)) { found = 1; break; }
  67. validation_requests++;
  68. if (p->pid == mock_timeout_pid) {
  69. fake_now += budget_ms / 1000.0; timing->total_ms = budget_ms;
  70. errno = ETIMEDOUT; return -1;
  71. }
  72. if (fail_validation || p->pid == invalid_pid || !found) { errno = ESTALE; return -1; }
  73. *ticket = p->pid; mock_ticket_pid = p->pid; errno = mock_validation_errno;
  74. return 0; /* This test backend cannot send any signal. */
  75. }
  76. void platform_dispatch(int handle, enum action action, signal_result_t *out)
  77. {
  78. CHECK(handle == mock_ticket_pid && (action == TERM || action == KILL));
  79. real_requests++;
  80. memset(out, 0, sizeof (*out)); out->attempted = 1;
  81. out->attempt = fake_now; fake_now += dispatch_delay; out->returned = fake_now;
  82. out->write_ms = out->total_ms = 1000 * dispatch_delay;
  83. out->rc = mock_signal_errno ? -1 : (long)(2 * sizeof (long));
  84. out->error = mock_signal_errno;
  85. }
  86. static void reset(void)
  87. {
  88. defaults(&cfg); cfg.log_mib = 100;
  89. cfg.nzones = 2;
  90. strcpy(cfg.zones[0].uuid, "11111111-1111-4111-8111-111111111111");
  91. strcpy(cfg.zones[0].alias, "web-a"); cfg.zones[0].policy = ALLOW;
  92. strcpy(cfg.zones[1].uuid, "22222222-2222-4222-8222-222222222222");
  93. strcpy(cfg.zones[1].alias, "web-b"); cfg.zones[1].policy = ALLOW;
  94. memset(processes, 0, sizeof (processes)); memset(zone_stats, 0, sizeof (zone_stats));
  95. memset(candidate_cache, 0, sizeof (candidate_cache));
  96. memset(published_cache, 0, sizeof (published_cache));
  97. memset(metrics, 0, sizeof (metrics));
  98. memset(&controller, 0, sizeof (controller)); memset(fixtures, 0, sizeof (fixtures));
  99. engine_init(&engine); engine.state = CRITICAL; engine.danger = 1;
  100. stopping = 0; started = 0; log_ok = 1; log_bytes = 0;
  101. async_logging = scanner_async = 0;
  102. event_head = event_tail = event_count = dropped_events = 0;
  103. event_sequence = 0; scan_requested = scan_snapshot_requested = 0;
  104. syslog_ready = syslog_init_failed = 0;
  105. syslog_count = syslog_open_count = syslog_close_count = 0;
  106. syslog_facility = -1; fail_syslog = 0; syslog_ident[0] = 0;
  107. syslog_delay_seconds = 0;
  108. refresh_age = dispatch_delay = 0;
  109. mock_signal_errno = mock_validation_errno = mock_timeout_pid = 0;
  110. scan_generation = 0; scan_complete = published_scan_complete = 0;
  111. scanner_latest_complete_scan_at = latest_complete_scan_at = 0;
  112. published_latest_complete_scan_at = 0;
  113. fake_now = 100; partial = fail_sample = fail_validation = invalid_pid = 0;
  114. real_requests = validation_requests = walk_requests = 0;
  115. memset(&fake_sample, 0, sizeof (fake_sample)); fake_sample.at = fake_now;
  116. fake_sample.free = 60000; fake_sample.lots = 130875;
  117. fake_sample.des = 65437; fake_sample.min = 49077;
  118. fixtures[0].pid = 123; fixtures[0].ppid = 100; fixtures[0].zid = 9;
  119. fixtures[0].initpid = 99; fixtures[0].nlwp = 1; fixtures[0].start_sec = 1000;
  120. fixtures[0].zone_generation = 20; fixtures[0].rss_kib = 100000;
  121. strcpy(fixtures[0].uuid, cfg.zones[0].uuid); strcpy(fixtures[0].name, "php-fpm");
  122. nfixtures = 1;
  123. scan_processes(0);
  124. }
  125. static void populate_many_candidates(unsigned count)
  126. {
  127. unsigned i;
  128. reset();
  129. CHECK(count <= sizeof (fixtures) / sizeof (fixtures[0]));
  130. for (i = 0; i < count; i++) {
  131. fixtures[i] = fixtures[0]; fixtures[i].pid = 1000 + (int)i;
  132. fixtures[i].start_sec += i; fixtures[i].start_nsec += i;
  133. fixtures[i].rss_kib = 2000000 - 1000 * i;
  134. }
  135. nfixtures = count; fake_now = 101; scan_processes(0);
  136. for (i = 0; i < count; i++) CHECK(candidate_cache[i].valid);
  137. }
  138. static int queued_contains(const char *text)
  139. {
  140. unsigned i;
  141. for (i = 0; i < event_count; i++) {
  142. unsigned slot = (event_head + i) % EVENT_QUEUE_SIZE;
  143. if (strstr(event_queue[slot].body, text)) return 1;
  144. }
  145. return 0;
  146. }
  147. static void critical_path_faults(void)
  148. {
  149. struct timespec before, after; double elapsed_ms; unsigned walks;
  150. struct stat st_before, st_after;
  151. {
  152. FILE *fixture; char line[160], event[64];
  153. unsigned seconds, rows = 0, decision_at = 0, sent_at = 0;
  154. uint64_t free_pages, nscan;
  155. fixture = fopen("tests/fixtures/panic32_daemon_stall_2026_09_22.csv", "r");
  156. CHECK(fixture != NULL); CHECK(fgets(line, sizeof (line), fixture) != NULL);
  157. while (fscanf(fixture, "%u,%" SCNu64 ",%" SCNu64 ",%63s",
  158. &seconds, &free_pages, &nscan, event) == 4) {
  159. rows++;
  160. if (!strcmp(event, "kill_decision")) decision_at = seconds;
  161. if (!strcmp(event, "kill_sent_after_145s")) sent_at = seconds;
  162. }
  163. fclose(fixture);
  164. CHECK(rows == 7 && sent_at - decision_at == 145);
  165. }
  166. {
  167. queued_event_t event;
  168. reset(); async_logging = 1; syslog_ready = 1;
  169. syslog_delay_seconds = 0.200;
  170. memset(&event, 0, sizeof (event));
  171. strcpy(event.body, "event=test_syslog_stall");
  172. event.wall_time = time(NULL); event.mono_time = fake_now;
  173. event.syslog_channel = 1; event.priority = LOG_WARNING; event.sequence = 1;
  174. CHECK(write_event_now(&event) == 0);
  175. CHECK(queued_contains("event=logging_stall"));
  176. CHECK(queued_contains("channel=syslog"));
  177. CHECK(queued_contains("syslog_latency_ms=200.000"));
  178. }
  179. populate_many_candidates(16); cfg.armed = 1; engine.state = EMERGENCY;
  180. engine.danger = 1; fake_sample.free = 40000; fake_now += 180;
  181. async_logging = 1; fail_syslog = 1;
  182. CHECK(fstat(logfd, &st_before) == 0); walks = walk_requests;
  183. clock_gettime(CLOCK_MONOTONIC, &before); act(&fake_sample);
  184. clock_gettime(CLOCK_MONOTONIC, &after);
  185. elapsed_ms = 1000.0 * (after.tv_sec - before.tv_sec) +
  186. (after.tv_nsec - before.tv_nsec) / 1000000.0;
  187. CHECK(real_requests == 1 && validation_requests == 1);
  188. CHECK(elapsed_ms < 100.0 && walk_requests == walks && syslog_count == 0);
  189. CHECK(fstat(logfd, &st_after) == 0 && st_after.st_size == st_before.st_size);
  190. CHECK(queued_contains("candidate_source=stale_cache_revalidated"));
  191. CHECK(queued_contains("dispatch_latency_ms="));
  192. CHECK(queued_contains("candidate_validate_latency_ms="));
  193. CHECK(queued_contains("kill_latency_ms="));
  194. CHECK(queued_contains("log_enqueue_latency_ms="));
  195. fake_now += cfg.emergency_term_s;
  196. clock_gettime(CLOCK_MONOTONIC, &before); act(&fake_sample);
  197. clock_gettime(CLOCK_MONOTONIC, &after);
  198. elapsed_ms = 1000.0 * (after.tv_sec - before.tv_sec) +
  199. (after.tv_nsec - before.tv_nsec) / 1000000.0;
  200. CHECK(real_requests == 2 && validation_requests == 2 && elapsed_ms < 100.0);
  201. CHECK(queued_contains("signal=SIGKILL"));
  202. puts("PASS panic32 fault: logger/syslog stopped 180s, stale cache revalidated, signal dispatch <100ms");
  203. populate_many_candidates(16); cfg.armed = 1; engine.state = EMERGENCY;
  204. engine.danger = 1; fake_sample.free = 40000; async_logging = scanner_async = 1;
  205. scan_requested = 1; walks = walk_requests;
  206. act(&fake_sample);
  207. CHECK(real_requests == 1 && walk_requests == walks);
  208. puts("PASS stalled/incomplete scanner cannot delay or trigger a rescue-path proc scan");
  209. populate_many_candidates(16); cfg.armed = 1; engine.state = EMERGENCY;
  210. engine.danger = 1; fake_sample.free = 40000; async_logging = 1;
  211. walks = walk_requests;
  212. while (controller.nvictims < cfg.max_emergency_victims) {
  213. act(&fake_sample);
  214. if (controller.has_pending) { fake_now += cfg.emergency_term_s; act(&fake_sample); }
  215. fake_now += 0.01;
  216. }
  217. CHECK(controller.nvictims == 8 && real_requests == 16 && walk_requests == walks);
  218. act(&fake_sample); CHECK(real_requests == 16);
  219. puts("PASS 16-entry stale rescue cache supplies all 8 EMERGENCY victims without scan");
  220. populate_many_candidates(16); cfg.armed = 1; engine.state = EMERGENCY;
  221. engine.danger = engine.floor = 1; fake_sample.free = 20000; async_logging = 1;
  222. walks = walk_requests;
  223. while (controller.nvictims < cfg.max_floor_victims) {
  224. act(&fake_sample); fake_now += 0.01;
  225. }
  226. CHECK(controller.nvictims == 12 && real_requests == 12 && walk_requests == walks);
  227. act(&fake_sample); CHECK(real_requests == 12);
  228. puts("PASS 16-entry stale rescue cache supplies all 12 FLOOR victims without scan");
  229. populate_many_candidates(16); cfg.armed = 1; engine.state = EMERGENCY;
  230. engine.danger = 1; fake_sample.free = 40000; async_logging = 1;
  231. CHECK(pthread_mutex_lock(&event_mutex) == 0);
  232. clock_gettime(CLOCK_MONOTONIC, &before); act(&fake_sample);
  233. clock_gettime(CLOCK_MONOTONIC, &after);
  234. CHECK(pthread_mutex_unlock(&event_mutex) == 0);
  235. elapsed_ms = 1000.0 * (after.tv_sec - before.tv_sec) +
  236. (after.tv_nsec - before.tv_nsec) / 1000000.0;
  237. CHECK(real_requests == 1 && elapsed_ms < 100.0 && dropped_events > 0);
  238. puts("PASS saturated/locked event queue drops telemetry without delaying the signal");
  239. }
  240. static void floor_errno_fixture(void)
  241. {
  242. FILE *f; char line[160], event[80];
  243. unsigned seconds, rows = 0, i, j;
  244. uint64_t free_pages, scans;
  245. delta_t zero = {0, 0, 0};
  246. const int prior[] = {EAGAIN, ENOMEM};
  247. const int errors[] = {0, ESRCH, EPERM, EINVAL, EAGAIN};
  248. populate_many_candidates(16); cfg.armed = 1; async_logging = 1;
  249. engine_init(&engine);
  250. f = fopen("tests/fixtures/floor_free_pages_486_2026_09_23.csv", "r");
  251. CHECK(f != NULL && fgets(line, sizeof (line), f) != NULL);
  252. while (fscanf(f, "%u,%" SCNu64 ",%" SCNu64 ",%79s", &seconds,
  253. &free_pages, &scans, event) == 4) {
  254. fake_now = 200 + seconds; fake_sample.at = fake_now;
  255. fake_sample.free = free_pages; fake_sample.scan = scans;
  256. engine_step(&engine, &cfg, &fake_sample, zero, zero, zero);
  257. if (seconds == 1) {
  258. CHECK(engine.state == EMERGENCY && engine.floor);
  259. mock_timeout_pid = candidate_cache[0].p.pid;
  260. act(&fake_sample);
  261. CHECK(real_requests == 1 && mock_ticket_pid == candidate_cache[1].p.pid);
  262. CHECK(queued_contains("reason=validation_timeout rank=1 latency_ms=50.000"));
  263. CHECK(queued_contains("rank=2"));
  264. CHECK(fake_now - fake_sample.at < 0.100);
  265. }
  266. rows++;
  267. }
  268. fclose(f); CHECK(rows == 3 && engine.state == RECOVERY);
  269. for (i = 0; i < 2; i++) for (j = 0; j < 5; j++) {
  270. char expected[120];
  271. reset(); cfg.armed = 1; async_logging = 1;
  272. mock_validation_errno = prior[i]; mock_signal_errno = errors[j];
  273. act(&fake_sample);
  274. snprintf(expected, sizeof (expected), "result=%s errno=%d syscall=write operation=PCKILL attempted=1 rc=%ld errno_name=%s",
  275. errors[j] ? "failed" : "sent", errors[j],
  276. errors[j] ? -1L : (long)(2 * sizeof (long)), signal_errno_name(errors[j]));
  277. CHECK(queued_contains(expected));
  278. CHECK(queued_contains("event=signal_sent") == (errors[j] == 0));
  279. CHECK(queued_contains("validate_start_time_ms="));
  280. }
  281. puts("PASS floor486 fixture, 50ms timeout fallback rank1->2, syscall errno attribution and validation substep logs");
  282. }
  283. static void diagnostic_fixtures(void)
  284. {
  285. FILE *f; char line[128], stamp[32], expected[128];
  286. unsigned signals, failure, rows = 0; uint64_t free_pages; double ms;
  287. reset(); cfg.armed = 1; async_logging = 1; engine.state = EMERGENCY;
  288. f = fopen("tests/fixtures/refresh_recovery_and_failure.csv", "r");
  289. CHECK(f && fgets(line, sizeof (line), f));
  290. while (fscanf(f, "%" SCNu64 ",%u,%u", &free_pages, &failure, &signals) == 3) {
  291. fake_sample.free = free_pages; fail_sample = (int)failure;
  292. act(&fake_sample); CHECK(real_requests == signals); rows++;
  293. }
  294. fclose(f); CHECK(rows == 3);
  295. CHECK(queued_contains("reason=free_above_desfree state=EMERGENCY free_pages=70000 desfree=65437 minfree=49077 refresh_ok=1"));
  296. CHECK(queued_contains("reason=refresh_failed state=EMERGENCY free_pages=20000 desfree=65437 minfree=49077 refresh_ok=0"));
  297. CHECK(queued_contains("refresh_errno=5 refresh_error=EIO"));
  298. reset(); cfg.armed = 1; async_logging = 1; refresh_age = 2.1;
  299. act(&fake_sample); CHECK(real_requests == 0);
  300. CHECK(queued_contains("reason=refresh_failed")); CHECK(queued_contains("refresh_error=stale_sample"));
  301. reset(); cfg.armed = 1; async_logging = 1; fake_sample.free = fake_sample.des;
  302. act(&fake_sample); CHECK(real_requests == 0 && queued_contains("reason=free_above_desfree"));
  303. f = fopen("tests/fixtures/pckill_latency_2026_09_28.csv", "r");
  304. CHECK(f && fgets(line, sizeof (line), f)); rows = 0;
  305. while (fscanf(f, "%31[^,],%lf\n", stamp, &ms) == 2) {
  306. reset(); cfg.armed = 1; async_logging = 1; fake_sample.free = 486;
  307. engine.state = EMERGENCY; dispatch_delay = ms / 1000;
  308. act(&fake_sample);
  309. CHECK(real_requests == 1 && queued_contains("event=signal_sent"));
  310. CHECK(queued_contains("event=signal_dispatch_slow signal=SIGKILL"));
  311. snprintf(expected, sizeof (expected), "pckill_write_latency_ms=%.3f", ms);
  312. CHECK(queued_contains(expected)); rows++;
  313. }
  314. fclose(f); CHECK(rows == 5);
  315. reset(); cfg.armed = 1; async_logging = 1; dispatch_delay = .000010;
  316. act(&fake_sample); CHECK(queued_contains("pckill_write_latency_ms=0.010"));
  317. CHECK(!queued_contains("event=signal_dispatch_slow"));
  318. reset(); cfg.armed = 1; async_logging = 1; dispatch_delay = .100;
  319. act(&fake_sample); CHECK(!queued_contains("event=signal_dispatch_slow"));
  320. reset(); cfg.armed = 1; async_logging = 1; dispatch_delay = .101;
  321. act(&fake_sample); CHECK(queued_contains("event=signal_dispatch_slow signal=SIGTERM"));
  322. puts("PASS recovery/failure/stale refresh reasons, desfree equality, no second victim, PCKILL subphases and slow-event threshold");
  323. }
  324. static void load_global_ranking_fixture(void)
  325. {
  326. FILE *f;
  327. char line[256], uuid[64], alias[64], name[32];
  328. int zid, pid;
  329. uint64_t generation, rss;
  330. reset();
  331. cfg.nzones = cfg.nnames = 0;
  332. strcpy(cfg.protected_names[cfg.nnames++], "mysqld");
  333. strcpy(cfg.protected_names[cfg.nnames++], "zone-overhead");
  334. memset(processes, 0, sizeof (processes));
  335. memset(zone_stats, 0, sizeof (zone_stats));
  336. memset(candidate_cache, 0, sizeof (candidate_cache));
  337. memset(&controller, 0, sizeof (controller));
  338. memset(fixtures, 0, sizeof (fixtures));
  339. scan_generation = 0; scan_complete = 0; latest_complete_scan_at = 0;
  340. nfixtures = 0; fake_now = 100;
  341. f = fopen("tests/fixtures/global_victim_ranking_smartos_2026_09.csv", "r");
  342. CHECK(f != NULL); CHECK(fgets(line, sizeof (line), f) != NULL);
  343. while (fscanf(f, " %63[^,],%63[^,],%d,%" SCNu64 ",%d,%31[^,],%" SCNu64,
  344. uuid, alias, &zid, &generation, &pid, name, &rss) == 7) {
  345. process_t *p;
  346. const zone_rule_t *rule = zone_rule(&cfg, uuid);
  347. CHECK(nfixtures < sizeof (fixtures) / sizeof (fixtures[0]));
  348. if (!rule) {
  349. zone_rule_t *z;
  350. CHECK(cfg.nzones < MAX_ZONES); z = &cfg.zones[cfg.nzones++];
  351. strcpy(z->uuid, uuid); strcpy(z->alias, alias); z->policy = ALLOW;
  352. }
  353. p = &fixtures[nfixtures++];
  354. p->pid = pid; p->ppid = 100; p->zid = zid; p->initpid = 9000 + zid;
  355. p->nlwp = 1; p->start_sec = 1000 + pid; p->start_nsec = pid;
  356. p->zone_generation = generation; p->rss_kib = rss;
  357. strcpy(p->uuid, uuid); strcpy(p->name, name);
  358. }
  359. fclose(f); CHECK(nfixtures == 16 && cfg.nzones == 5);
  360. scan_processes(1);
  361. }
  362. int main(void)
  363. {
  364. FILE *log = tmpfile(); char contents[65536]; size_t n;
  365. const process_t *chosen;
  366. unsigned second;
  367. CHECK(log != NULL); logfd = fileno(log);
  368. reset(); act(&fake_sample);
  369. CHECK(real_requests == 0 && validation_requests == 1 && controller.has_pending);
  370. fake_now += 5; scan_processes(0); act(&fake_sample);
  371. CHECK(real_requests == 0 && !controller.has_pending);
  372. rewind(log); n = fread(contents, 1, sizeof (contents) - 1, log); contents[n] = 0;
  373. CHECK(strstr(contents, "WOULD_ACTION=SIGTERM") != NULL);
  374. CHECK(strstr(contents, "WOULD_ACTION=SIGKILL") != NULL);
  375. CHECK(strstr(contents, "ACTION_INTENT=") == NULL);
  376. CHECK(fseek(log, 0, SEEK_END) == 0);
  377. puts("PASS actual daemon dry-run path records WOULD_ACTION=SIGTERM/SIGKILL; zero real requests");
  378. reset();
  379. fake_sample.arc_ok = 1;
  380. fake_sample.swap_virtual_ok = fake_sample.swap_physical_ok = fake_sample.pageout_ok = 1;
  381. for (second = 0; second <= 60; second++) {
  382. fake_sample.at = 100 + second;
  383. fake_sample.arc = UINT64_C(6000000000) + 1000 * second;
  384. fake_sample.swap_alloc = 7000000 + 3 * second;
  385. fake_sample.swap_reserved = 480000 + 2 * second;
  386. fake_sample.swap_available = 6200000 - second;
  387. fake_sample.swap_physical_used_bytes = UINT64_C(4143648768) + 4096 * second;
  388. fake_sample.anon_pageout_pages = 1000 + 2 * second;
  389. record_pressure_metrics(&fake_sample);
  390. record_auxiliary_metrics(&fake_sample);
  391. }
  392. log_sample(&fake_sample);
  393. CHECK(fflush(log) == 0 && fseek(log, 0, SEEK_SET) == 0);
  394. n = fread(contents, 1, sizeof (contents) - 1, log); contents[n] = 0;
  395. CHECK(strstr(contents, "swap_committed_ok=1 swap_committed_pages=7480300") != NULL);
  396. CHECK(strstr(contents, "swap_physical_used_ok=1 swap_physical_used_bytes=4143894528") != NULL);
  397. CHECK(strstr(contents, "anon_pageout_ok=1 anon_pageout_pages=1120") != NULL);
  398. CHECK(strstr(contents, "swap_alloc_pages=7000180 swap_reserved_pages=480120 swap_available_pages=6199940") != NULL);
  399. CHECK(strstr(contents, "metric=swap_committed_pages d5=+25@5.00s d15=+75@15.00s d60=+300@60.00s") != NULL);
  400. CHECK(strstr(contents, "metric=swap_physical_used_bytes d5=+20480@5.00s d15=+61440@15.00s d60=+245760@60.00s") != NULL);
  401. CHECK(strstr(contents, "metric=anon_pageout_pages d5=+10@5.00s d15=+30@15.00s d60=+120@60.00s") != NULL);
  402. CHECK(strstr(contents, "swap_used_pages") == NULL);
  403. CHECK(fseek(log, 0, SEEK_END) == 0);
  404. puts("PASS continuous sample contract and 5/15/60 deltas for commitment, physical swap and pageout");
  405. reset(); cfg.armed = 1; act(&fake_sample);
  406. CHECK(real_requests == 1);
  407. fake_now += 5; scan_processes(0); act(&fake_sample); CHECK(real_requests == 2);
  408. reset(); cfg.armed = 1; started = 80; fake_now = 101; scan_processes(0);
  409. CHECK(candidate_cache[0].valid && engine.danger && walk_requests == 2);
  410. act(&fake_sample); CHECK(real_requests == 0 && candidate_cache[0].valid);
  411. CHECK(fflush(log) == 0 && fseek(log, 0, SEEK_SET) == 0);
  412. n = fread(contents, 1, sizeof (contents) - 1, log); contents[n] = 0;
  413. CHECK(strstr(contents, "event=action_blocked reason=armed_warmup") != NULL);
  414. CHECK(fseek(log, 0, SEEK_END) == 0);
  415. reset(); cfg.armed = 1; log_ok = 0; act(&fake_sample); CHECK(real_requests == 1);
  416. reset(); cfg.armed = 1; partial = 1; fake_now++; scan_processes(0);
  417. act(&fake_sample); CHECK(real_requests == 1 && !scan_complete);
  418. reset(); cfg.armed = 1; fail_sample = 1; act(&fake_sample); CHECK(real_requests == 0);
  419. reset(); cfg.armed = 1; fail_validation = 1; act(&fake_sample); CHECK(real_requests == 0);
  420. reset(); cfg.armed = 1; fake_sample.free = 150000; act(&fake_sample); CHECK(real_requests == 0);
  421. reset(); cfg.armed = 1; fake_now += 16; act(&fake_sample); CHECK(real_requests == 1);
  422. reset(); cfg.armed = 1; stopping = 1; act(&fake_sample); CHECK(real_requests == 0);
  423. reset(); fake_sample.free = 20000; act(&fake_sample);
  424. CHECK(!controller.has_pending && controller.nvictims == 1);
  425. puts("PASS armed-path mock requests, explicit warmup, stale cache, logging/sampling failures, stop and fresh floor");
  426. reset(); fixtures[1] = fixtures[0]; fixtures[1].pid = 124;
  427. fixtures[1].rss_kib = 900000; fixtures[1].zid = 10;
  428. strcpy(fixtures[1].uuid, cfg.zones[1].uuid); nfixtures = 2;
  429. fake_now = 101; scan_processes(0); chosen = candidate(); CHECK(chosen && chosen->pid == 124);
  430. /* Small zone grows fast enough to outweigh a larger steady zone. */
  431. fixtures[0].rss_kib = 600000; fake_now = 161; scan_processes(0);
  432. chosen = candidate(); CHECK(chosen && chosen->pid == 123);
  433. cfg.zones[0].policy = PROTECT; fake_now = 162; scan_processes(0);
  434. chosen = candidate(); CHECK(chosen && chosen->pid == 124);
  435. fixtures[1].zid = 0; fake_now = 163; scan_processes(0); CHECK(candidate() == NULL);
  436. puts("PASS global score uses process RSS/growth; protected zones and global zone excluded");
  437. load_global_ranking_fixture();
  438. CHECK(candidate_cache[0].valid && candidate_cache[1].valid && candidate_cache[2].valid);
  439. CHECK(!strcmp(candidate_cache[0].p.name, "clamd"));
  440. CHECK(!strcmp(candidate_cache[1].p.name, "clamd"));
  441. CHECK(!strcmp(candidate_cache[2].p.name, "clamd"));
  442. CHECK(candidate_cache[0].p.rss_kib > 1300000 &&
  443. candidate_cache[2].p.rss_kib > 1300000);
  444. CHECK(strcmp(candidate_cache[0].p.uuid, candidate_cache[1].p.uuid));
  445. CHECK(strcmp(candidate_cache[1].p.uuid, candidate_cache[2].p.uuid));
  446. for (second = 0; second < CACHED_CANDIDATES; second++) {
  447. CHECK(strcmp(candidate_cache[second].p.name, "mysqld"));
  448. CHECK(strcmp(candidate_cache[second].p.name, "zone-overhead"));
  449. }
  450. for (second = 0; second < nfixtures; second++)
  451. if (!strcmp(fixtures[second].name, "mysqld"))
  452. CHECK(protected_process(&cfg, &fixtures[second]));
  453. CHECK(fflush(log) == 0 && fseek(log, 0, SEEK_SET) == 0);
  454. n = fread(contents, 1, sizeof (contents) - 1, log); contents[n] = 0;
  455. {
  456. char *mysql = strstr(contents, "comm=mysqld");
  457. char *ineligible = mysql ? strstr(mysql, "eligible=0") : NULL;
  458. char *line_end = mysql ? strchr(mysql, '\n') : NULL;
  459. CHECK(mysql && ineligible && line_end && ineligible < line_end);
  460. }
  461. CHECK(fseek(log, 0, SEEK_END) == 0);
  462. fixtures[nfixtures] = fixtures[0]; fixtures[nfixtures].pid = 9001;
  463. fixtures[nfixtures].zid = 90; fixtures[nfixtures].zone_generation = 900;
  464. fixtures[nfixtures].rss_kib = UINT64_C(9000000);
  465. strcpy(fixtures[nfixtures].uuid, "66666666-6666-4666-8666-666666666666");
  466. strcpy(fixtures[nfixtures].name, "unmapped-hog"); nfixtures++;
  467. fixtures[nfixtures] = fixtures[0]; fixtures[nfixtures].pid = 9002;
  468. fixtures[nfixtures].zid = 0; fixtures[nfixtures].rss_kib = UINT64_C(11000000);
  469. strcpy(fixtures[nfixtures].name, "global-hog"); nfixtures++;
  470. fake_now = 101; scan_processes(0);
  471. for (second = 0; second < CACHED_CANDIDATES; second++)
  472. CHECK(candidate_cache[second].p.pid != 9001 && candidate_cache[second].p.pid != 9002);
  473. for (second = 0; second < nfixtures; second++)
  474. if (fixtures[second].pid == 1103) fixtures[second].rss_kib = 600000;
  475. fake_now = 106; scan_processes(0);
  476. CHECK(candidate_cache[0].p.pid == 1103);
  477. CHECK(candidate_cache[0].growth5_component > 0 &&
  478. candidate_cache[0].total_score > candidate_cache[1].total_score);
  479. {
  480. cached_candidate_t scored;
  481. zone_t zone;
  482. memset(&scored, 0, sizeof (scored)); memset(&zone, 0, sizeof (zone));
  483. scored.p.rss_kib = 100000; scored.d5.valid = scored.d15.valid = 1;
  484. scored.d60.valid = 1; scored.d5.value = 1000;
  485. scored.d15.value = 2000; scored.d60.value = 3000;
  486. scan_at = 60; history_add(&zone.history, 0, 1000000);
  487. history_add(&zone.history, 60, 1100000); zone.rss = 1100000;
  488. score_candidate(&scored, &zone);
  489. CHECK(scored.growth5_component == 4000);
  490. CHECK(scored.growth15_component == 4000);
  491. CHECK(scored.growth60_component == 3000);
  492. CHECK(scored.zone_component <= scored.p.rss_kib / 4);
  493. }
  494. CHECK(fflush(log) == 0 && fseek(log, 0, SEEK_SET) == 0);
  495. n = fread(contents, 1, sizeof (contents) - 1, log); contents[n] = 0;
  496. CHECK(strstr(contents, "rss_component=") != NULL);
  497. CHECK(strstr(contents, "growth5_component=") != NULL);
  498. CHECK(strstr(contents, "growth15_component=") != NULL);
  499. CHECK(strstr(contents, "growth60_component=") != NULL);
  500. CHECK(strstr(contents, "zone_component=") != NULL);
  501. CHECK(strstr(contents, "total_score=") != NULL);
  502. CHECK(fseek(log, 0, SEEK_END) == 0);
  503. puts("PASS SmartOS fixture produces a global cross-zone top-3 and bounded zone context");
  504. reset(); fixtures[1] = fixtures[0]; fixtures[1].pid = 124;
  505. fixtures[1].rss_kib = 900000; fixtures[1].zid = 10;
  506. strcpy(fixtures[1].uuid, cfg.zones[1].uuid);
  507. fixtures[2] = fixtures[0]; fixtures[2].pid = 125; fixtures[2].rss_kib = 800000;
  508. nfixtures = 3; fake_now = 101; scan_processes(0);
  509. CHECK(candidate_cache[0].valid && candidate_cache[1].valid && candidate_cache[2].valid);
  510. invalid_pid = candidate_cache[0].p.pid;
  511. partial = 1; fake_now = 102; scan_processes(0);
  512. CHECK(!scan_complete && candidate_cache[0].p.pid == invalid_pid);
  513. {
  514. unsigned walks_before = walk_requests;
  515. engine.state = EMERGENCY; engine.danger = 1; fake_sample.free = 40000;
  516. act(&fake_sample);
  517. CHECK(walk_requests == walks_before); /* O(1): no /proc walk in action path. */
  518. CHECK(controller.has_pending && controller.pending.pid != invalid_pid);
  519. CHECK(validation_requests >= 2 && real_requests == 0);
  520. }
  521. CHECK(fflush(log) == 0 && fseek(log, 0, SEEK_SET) == 0);
  522. n = fread(contents, 1, sizeof (contents) - 1, log); contents[n] = 0;
  523. CHECK(strstr(contents, "candidate_source=fallback_candidate") != NULL);
  524. CHECK(strstr(contents, "scan_complete=0") != NULL);
  525. CHECK(fseek(log, 0, SEEK_END) == 0);
  526. puts("PASS incomplete scan preserves cache; emergency uses O(1) fallback without a new walk");
  527. reset(); fixtures[1] = fixtures[0]; fixtures[1].pid = 124;
  528. fixtures[1].rss_kib = 900000; fixtures[1].zid = 10;
  529. strcpy(fixtures[1].uuid, cfg.zones[1].uuid); nfixtures = 2;
  530. fake_now = 101; scan_processes(0);
  531. for (second = 0; second < nfixtures; second++)
  532. if (fixtures[second].pid == candidate_cache[0].p.pid) fixtures[second].start_nsec++;
  533. engine.state = EMERGENCY; engine.danger = 1; fake_sample.free = 40000;
  534. act(&fake_sample);
  535. CHECK(controller.has_pending && !same_process(&controller.pending, &candidate_cache[0].p));
  536. puts("PASS PID/start-time reuse rejects rank 1 and selects cached fallback");
  537. reset();
  538. CHECK(platform_syslog_open("o6h-memory-guard", cfg.syslog_facility) == 0);
  539. syslog_ready = 1;
  540. CHECK(!strcmp(syslog_ident, "o6h-memory-guard") && syslog_facility == 5 &&
  541. syslog_open_count == 1);
  542. emit_important(LOG_INFO, "event=start version=%s mode=dry-run", VERSION);
  543. CHECK(syslog_count == 1 && syslog_records[0].priority == LOG_INFO &&
  544. strstr(syslog_records[0].message, "event=start version=1.0.9") != NULL);
  545. log_sample(&fake_sample); scan_processes(0);
  546. CHECK(syslog_count == 1); /* High-volume sample/delta/proc/cache events stay local. */
  547. {
  548. char message[512]; sample_t s = fake_sample;
  549. s.free = s.min;
  550. CHECK(!transition_message(message, sizeof (message), NORMAL, EMERGENCY, &s, 1));
  551. CHECK(strstr(message, "reason=threshold_jump") != NULL);
  552. CHECK(strstr(message, "skipped_states=PRE-PRESSURE,PRESSURE,CRITICAL") != NULL);
  553. syslog_event(state_priority(EMERGENCY, 0), "%s", message);
  554. CHECK(syslog_count == 2 && syslog_records[1].priority == LOG_CRIT);
  555. }
  556. cfg.armed = 0; act(&fake_sample);
  557. CHECK(syslog_count == 2 && real_requests == 0);
  558. reset(); cfg.armed = 1; started = 80; fake_now = 100;
  559. CHECK(platform_syslog_open("o6h-memory-guard", cfg.syslog_facility) == 0);
  560. syslog_ready = 1; act(&fake_sample);
  561. CHECK(real_requests == 0 && syslog_count == 1 &&
  562. syslog_records[0].priority == LOG_WARNING &&
  563. strstr(syslog_records[0].message, "event=action_blocked reason=armed_warmup") != NULL);
  564. reset(); cfg.armed = 1;
  565. CHECK(platform_syslog_open("o6h-memory-guard", cfg.syslog_facility) == 0);
  566. syslog_ready = 1; act(&fake_sample);
  567. CHECK(real_requests == 1 && syslog_count == 2);
  568. CHECK(syslog_records[0].priority == LOG_ERR &&
  569. strstr(syslog_records[0].message, "event=action_decision state=CRITICAL") != NULL &&
  570. strstr(syslog_records[0].message, "result=sent") != NULL);
  571. CHECK(syslog_records[1].priority == LOG_ERR &&
  572. strstr(syslog_records[1].message, "event=signal_sent state=CRITICAL") != NULL &&
  573. strstr(syslog_records[1].message, "signal=SIGTERM") != NULL &&
  574. strstr(syslog_records[1].message, "UUID=") != NULL &&
  575. strstr(syslog_records[1].message, "alias=web-a") != NULL &&
  576. strstr(syslog_records[1].message, "pid=123") != NULL &&
  577. strstr(syslog_records[1].message, "comm=php-fpm") != NULL &&
  578. strstr(syslog_records[1].message, "rss_kib=100000") != NULL &&
  579. strstr(syslog_records[1].message, "free_pages=60000") != NULL);
  580. reset(); cfg.armed = 1; fail_syslog = 1; syslog_ready = 1;
  581. act(&fake_sample); CHECK(real_requests == 1 && controller.has_pending);
  582. reset(); cfg.syslog_enabled = 0; syslog_ready = 0;
  583. emit_important(LOG_INFO, "event=start disabled_test=yes"); CHECK(syslog_count == 0);
  584. CHECK(state_priority(PRE_PRESSURE, 0) == LOG_NOTICE);
  585. CHECK(state_priority(PRESSURE, 0) == LOG_WARNING);
  586. CHECK(state_priority(CRITICAL, 0) == LOG_ERR);
  587. CHECK(state_priority(EMERGENCY, 0) == LOG_CRIT);
  588. CHECK(state_priority(RECOVERY, 0) == LOG_INFO);
  589. CHECK(state_priority(NORMAL, 1) == LOG_CRIT);
  590. log_sampling_overrun(4.459, &fake_sample);
  591. CHECK(fflush(log) == 0 && fseek(log, 0, SEEK_SET) == 0);
  592. n = fread(contents, 1, sizeof (contents) - 1, log); contents[n] = 0;
  593. CHECK(strstr(contents, "event=sampling_overrun late_seconds=4.459 state=CRITICAL free_pages=60000") != NULL);
  594. puts("PASS selective native syslog ident/facility/severity, exclusions, signal ordering, disable and failure isolation");
  595. critical_path_faults();
  596. floor_errno_fixture();
  597. diagnostic_fixtures();
  598. fclose(log);
  599. printf("PASS %u runtime checks; all platform calls mocked, no signals sent\n", checks);
  600. return 0;
  601. }