Co-creation-projects/zjzhou-SREOnCallAgent/main.ipynb
AI-powered incident triage, root cause investigation, and post-mortem generation
When a production alert fires at 3am, an SRE on-call engineer must:
This project automates that workflow with a three-agent AI pipeline:
Alert JSON
│
▼
┌─────────────────────────────┐
│ Stage 1: TriageAgent │ Plan-and-Solve
│ "What should I investigate?"│ → ordered investigation plan
└─────────────┬───────────────┘
│
▼
┌─────────────────────────────┐
│ Stage 2: InvestigationAgent│ ReAct loop
│ log_search / metric_query │ → root cause hypothesis
│ / runbook_lookup │
└─────────────┬───────────────┘
│
▼
┌─────────────────────────────┐
│ Stage 3: PostmortemAgent │ Reflection
│ draft → critique → revise │ → final RCA report
└─────────────────────────────┘
# Install dependencies
!pip install -q openai fastapi uvicorn pyyaml pydantic python-dotenv
import os
import sys
import json
import time
import re
import ast
import glob
import yaml
from pathlib import Path
from typing import Dict, Any, List
from dotenv import load_dotenv
from openai import OpenAI
# Add project root to path so src/ imports work
project_root = Path().resolve()
if str(project_root) not in sys.path:
sys.path.insert(0, str(project_root))
load_dotenv()
print("✅ Environment loaded")
print(f" LLM_MODEL_ID: {os.getenv('LLM_MODEL_ID', 'NOT SET')}")
print(f" LLM_BASE_URL: {os.getenv('LLM_BASE_URL', 'NOT SET')}")
print(f" LLM_API_KEY: {'SET' if os.getenv('LLM_API_KEY') else 'NOT SET'}")
Three tools give the InvestigationAgent eyes into the incident:
| Tool | Input | What it does |
|---|---|---|
LogSearchTool | keyword/regex | Searches incident log entries |
MetricQueryTool | metric name keyword | Returns time-series metric data |
RunbookLookupTool | error pattern | Fetches runbook remediation steps |
from src.tools.log_search_tool import LogSearchTool
from src.tools.metric_query_tool import MetricQueryTool
from src.tools.runbook_tool import RunbookLookupTool
# --- Quick tool smoke test ---
incident_path = project_root / "data" / "incidents" / "db_pool_exhaustion.json"
with open(incident_path) as f:
sample_incident = json.load(f)
log_tool = LogSearchTool(sample_incident)
metric_tool = MetricQueryTool(sample_incident)
runbook_tool = RunbookLookupTool(
service="checkout-service",
runbooks_dir=str(project_root / "data" / "runbooks")
)
print("🔧 LogSearchTool — searching for 'pool':")
print(log_tool.run("pool"))
print()
print("📊 MetricQueryTool — querying 'db_pool':")
print(metric_tool.run("db_pool"))
print()
print("📖 RunbookLookupTool — pattern 'pool exhausted':")
print(runbook_tool.run("pool exhausted"))
Three agents implementing different paradigms from the Hello-Agents curriculum:
from src.core.llm_client import HelloAgentsLLM
from src.agents.triage_agent import TriageAgent
from src.agents.investigation_agent import InvestigationAgent
from src.agents.postmortem_agent import PostmortemAgent
from src.agents.pipeline import run_pipeline, list_incidents, load_incident
# Verify LLM connection
llm = HelloAgentsLLM(verbose=True)
test_response = llm.think([{"role": "user", "content": "Reply with exactly: LLM OK"}])
print(f"\nLLM connection test: {test_response.strip()}")
print(f"\nAvailable incident fixtures: {list_incidents()}")
Alert: checkout-service P99 latency 8.3s > threshold 1.0s
Root cause (ground truth): Missing index on orders.user_id → full table scan → DB pool exhaustion
# Show the incident alert
incident_1 = load_incident("db_pool_exhaustion")
print("📣 Incident Alert:")
print(json.dumps({
"incident_id": incident_1["incident_id"],
"service": incident_1["service"],
"severity": incident_1["severity"],
"alert": incident_1["alert"]
}, indent=2))
# Run the full pipeline
start = time.time()
result_1 = run_pipeline("db_pool_exhaustion", verbose=True)
elapsed_1 = round(time.time() - start, 1)
print(f"\n⏱️ Pipeline completed in {elapsed_1}s")
# Display the generated post-mortem report
from IPython.display import Markdown, display
display(Markdown(result_1["report"]))
Alert: payment-service success rate 23.4% < threshold 95%
Root cause (ground truth): Promotional traffic spike + no exponential backoff → retry storm → Stripe 429 cascade
start = time.time()
result_3 = run_pipeline("external_api_ratelimit", verbose=True)
elapsed_3 = round(time.time() - start, 1)
print(f"\n⏱️ Pipeline completed in {elapsed_3}s")
display(Markdown(result_3["report"]))
Run all 3 incident fixtures and measure: pipeline speed, root cause accuracy, and report quality.
GROUND_TRUTH = {
"db_pool_exhaustion": "missing index",
"memory_leak_oom": "no TTL",
"external_api_ratelimit": "retry storm",
}
eval_results = []
for incident_id in list_incidents():
t0 = time.time()
res = run_pipeline(incident_id, verbose=False)
elapsed = round(time.time() - t0, 1)
ground_truth_keyword = GROUND_TRUTH.get(incident_id, "")
root_cause_text = res["findings"].get("root_cause", "").lower()
correct = ground_truth_keyword.lower() in root_cause_text
eval_results.append({
"incident_id": incident_id,
"service": res["service"],
"severity": res["severity"],
"elapsed_s": elapsed,
"root_cause_correct": "✅" if correct else "❌",
"root_cause_found": res["findings"].get("root_cause", "")[:80] + "...",
})
print("\n=== Evaluation Results ===")
for r in eval_results:
print(f" [{r['root_cause_correct']}] {r['incident_id']:30s} "
f"| {r['elapsed_s']:5.1f}s | {r['root_cause_found']}")
accuracy = sum(1 for r in eval_results if r["root_cause_correct"] == "✅") / len(eval_results)
avg_time = sum(r["elapsed_s"] for r in eval_results) / len(eval_results)
print(f"\nAccuracy: {accuracy:.0%} Avg pipeline time: {avg_time:.1f}s")
| Agent | Paradigm | Chapter Reference |
|---|---|---|
| TriageAgent | Plan-and-Solve | Chapter 4 |
| InvestigationAgent | ReAct | Chapter 4 |
| PostmortemAgent | Reflection | Chapter 4 |