8c6e7831e9
Major additions: - marketplace/: Agent template registry with FTS5 search, ratings, versioning - observability/: Prometheus metrics, distributed tracing, structured logging - ledger/migrations/: Database migration scripts for multi-tenant support - tests/governance/: 15 new test files for phases 6-12 (295 total tests) - bin/validate-phases: Full 12-phase validation script New features: - Multi-tenant support with tenant isolation and quota enforcement - Agent marketplace with semantic versioning and search - Observability with metrics, tracing, and log correlation - Tier-1 agent bootstrap scripts Updated components: - ledger/api.py: Extended API for tenants, marketplace, observability - ledger/schema.sql: Added tenant, project, marketplace tables - testing/framework.ts: Enhanced test framework - checkpoint/checkpoint.py: Improved checkpoint management Archived: - External integrations (Slack/GitHub/PagerDuty) moved to .archive/ - Old checkpoint files cleaned up Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1746 lines
66 KiB
Python
Executable File
1746 lines
66 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Context Checkpoint Skill
|
|
========================
|
|
Preserves long-running session context and reduces token usage when
|
|
orchestrating sub-agents. Part of Phase 5: Agent Bootstrapping.
|
|
|
|
Features:
|
|
- Periodic state capture (phase, tasks, dependencies, variables, outputs)
|
|
- Token-aware sub-agent context summarization
|
|
- CLI integration for manual and automatic checkpoints
|
|
- Extensible storage (local JSON, future: remote sync)
|
|
"""
|
|
|
|
import json
|
|
import hashlib
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import os
|
|
import time
|
|
import difflib
|
|
from dataclasses import dataclass, field, asdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Optional
|
|
import redis
|
|
|
|
|
|
# =============================================================================
|
|
# Configuration
|
|
# =============================================================================
|
|
|
|
CHECKPOINT_DIR = Path("/opt/agent-governance/checkpoint/storage")
|
|
LEDGER_DB = Path("/opt/agent-governance/ledger/governance.db")
|
|
ORCHESTRATOR_DIR = Path("/opt/agent-governance/orchestrator")
|
|
MAX_CHECKPOINTS = 50 # Keep last N checkpoints
|
|
AUTO_CHECKPOINT_INTERVAL = 300 # seconds (5 minutes)
|
|
|
|
# Automated orchestration settings
|
|
AUTO_ORCHESTRATE_ENABLED = os.environ.get("AUTO_AGENT_MODE", "disabled") != "disabled"
|
|
|
|
|
|
# =============================================================================
|
|
# Data Classes
|
|
# =============================================================================
|
|
|
|
@dataclass
|
|
class ProjectPhase:
|
|
"""Current project phase information"""
|
|
name: str
|
|
number: int
|
|
status: str # in_progress, complete, pending
|
|
started_at: Optional[str] = None
|
|
completed_at: Optional[str] = None
|
|
notes: str = ""
|
|
|
|
|
|
@dataclass
|
|
class TaskState:
|
|
"""State of a single task"""
|
|
id: str
|
|
subject: str
|
|
status: str # pending, in_progress, completed
|
|
owner: Optional[str] = None
|
|
blocks: list = field(default_factory=list)
|
|
blocked_by: list = field(default_factory=list)
|
|
metadata: dict = field(default_factory=dict)
|
|
|
|
|
|
@dataclass
|
|
class Dependency:
|
|
"""External dependency state"""
|
|
name: str
|
|
type: str # service, database, api, file
|
|
status: str # available, unavailable, degraded
|
|
endpoint: Optional[str] = None
|
|
last_checked: Optional[str] = None
|
|
|
|
|
|
@dataclass
|
|
class PendingInstruction:
|
|
"""Instruction waiting to be executed in automated mode"""
|
|
id: str
|
|
instruction: str
|
|
command_type: str # shell, checkpoint, plan, execute, review
|
|
priority: int = 0
|
|
requires_confirmation: bool = False
|
|
created_at: str = ""
|
|
expires_at: Optional[str] = None
|
|
|
|
|
|
@dataclass
|
|
class DirectoryStatusEntry:
|
|
"""Status entry for a directory from STATUS.md"""
|
|
path: str
|
|
phase: str # complete, in_progress, blocked, needs_review, not_started
|
|
last_updated: Optional[str] = None
|
|
has_readme: bool = False
|
|
has_status: bool = False
|
|
tasks_total: int = 0
|
|
tasks_done: int = 0
|
|
|
|
|
|
@dataclass
|
|
class MemoryRef:
|
|
"""Lightweight reference to a memory entry (for embedding in checkpoints)."""
|
|
id: str
|
|
type: str
|
|
summary: str
|
|
tokens: int
|
|
|
|
def to_inline(self) -> str:
|
|
"""Format for inline prompt inclusion."""
|
|
return f"[Memory:{self.id}] {self.summary[:50]}... ({self.tokens} tokens)"
|
|
|
|
|
|
@dataclass
|
|
class ContextCheckpoint:
|
|
"""
|
|
Complete context checkpoint capturing session state.
|
|
Designed to be reloadable after token window reset or CLI restart.
|
|
"""
|
|
checkpoint_id: str
|
|
created_at: str
|
|
session_id: Optional[str] = None
|
|
|
|
# Project State
|
|
phase: Optional[ProjectPhase] = None
|
|
phases_completed: list = field(default_factory=list)
|
|
|
|
# Task State
|
|
tasks: list = field(default_factory=list) # List of TaskState
|
|
active_task_id: Optional[str] = None
|
|
|
|
# Dependencies
|
|
dependencies: list = field(default_factory=list) # List of Dependency
|
|
|
|
# Key Variables (agent-specific context)
|
|
variables: dict = field(default_factory=dict)
|
|
|
|
# Recent Outputs (summaries, not full content)
|
|
recent_outputs: list = field(default_factory=list)
|
|
|
|
# Agent State
|
|
agent_id: Optional[str] = None
|
|
agent_tier: int = 0
|
|
|
|
# Checksums for change detection
|
|
content_hash: str = ""
|
|
parent_checkpoint_id: Optional[str] = None
|
|
|
|
# Token metrics
|
|
estimated_tokens: int = 0
|
|
|
|
# Automated Orchestration
|
|
orchestration_mode: str = "disabled"
|
|
pending_instructions: list = field(default_factory=list) # List of PendingInstruction
|
|
last_model_response: Optional[str] = None
|
|
|
|
# Directory Status Snapshot (integrated with status system)
|
|
directory_statuses: list = field(default_factory=list) # List of DirectoryStatusEntry
|
|
status_summary: dict = field(default_factory=dict) # Aggregated counts by phase
|
|
|
|
# Memory Layer References (links to external memory for large content)
|
|
memory_refs: list = field(default_factory=list) # List of MemoryRef
|
|
memory_summary: dict = field(default_factory=dict) # Aggregated memory stats
|
|
|
|
def to_dict(self) -> dict:
|
|
"""Convert to dictionary for serialization"""
|
|
return {
|
|
"checkpoint_id": self.checkpoint_id,
|
|
"created_at": self.created_at,
|
|
"session_id": self.session_id,
|
|
"phase": asdict(self.phase) if self.phase else None,
|
|
"phases_completed": self.phases_completed,
|
|
"tasks": [asdict(t) if isinstance(t, TaskState) else t for t in self.tasks],
|
|
"active_task_id": self.active_task_id,
|
|
"dependencies": [asdict(d) if isinstance(d, Dependency) else d for d in self.dependencies],
|
|
"variables": self.variables,
|
|
"recent_outputs": self.recent_outputs,
|
|
"agent_id": self.agent_id,
|
|
"agent_tier": self.agent_tier,
|
|
"content_hash": self.content_hash,
|
|
"parent_checkpoint_id": self.parent_checkpoint_id,
|
|
"estimated_tokens": self.estimated_tokens,
|
|
"orchestration_mode": self.orchestration_mode,
|
|
"pending_instructions": [asdict(i) if isinstance(i, PendingInstruction) else i
|
|
for i in self.pending_instructions],
|
|
"last_model_response": self.last_model_response,
|
|
"directory_statuses": [asdict(d) if isinstance(d, DirectoryStatusEntry) else d
|
|
for d in self.directory_statuses],
|
|
"status_summary": self.status_summary,
|
|
"memory_refs": [asdict(m) if isinstance(m, MemoryRef) else m
|
|
for m in self.memory_refs],
|
|
"memory_summary": self.memory_summary
|
|
}
|
|
|
|
@classmethod
|
|
def from_dict(cls, data: dict) -> 'ContextCheckpoint':
|
|
"""Create from dictionary"""
|
|
phase = None
|
|
if data.get("phase"):
|
|
phase = ProjectPhase(**data["phase"])
|
|
|
|
tasks = []
|
|
for t in data.get("tasks", []):
|
|
if isinstance(t, dict):
|
|
tasks.append(TaskState(**t))
|
|
else:
|
|
tasks.append(t)
|
|
|
|
dependencies = []
|
|
for d in data.get("dependencies", []):
|
|
if isinstance(d, dict):
|
|
dependencies.append(Dependency(**d))
|
|
else:
|
|
dependencies.append(d)
|
|
|
|
pending_instructions = []
|
|
for i in data.get("pending_instructions", []):
|
|
if isinstance(i, dict):
|
|
pending_instructions.append(PendingInstruction(**i))
|
|
else:
|
|
pending_instructions.append(i)
|
|
|
|
directory_statuses = []
|
|
for d in data.get("directory_statuses", []):
|
|
if isinstance(d, dict):
|
|
directory_statuses.append(DirectoryStatusEntry(**d))
|
|
else:
|
|
directory_statuses.append(d)
|
|
|
|
memory_refs = []
|
|
for m in data.get("memory_refs", []):
|
|
if isinstance(m, dict):
|
|
memory_refs.append(MemoryRef(**m))
|
|
else:
|
|
memory_refs.append(m)
|
|
|
|
return cls(
|
|
checkpoint_id=data["checkpoint_id"],
|
|
created_at=data["created_at"],
|
|
session_id=data.get("session_id"),
|
|
phase=phase,
|
|
phases_completed=data.get("phases_completed", []),
|
|
tasks=tasks,
|
|
active_task_id=data.get("active_task_id"),
|
|
dependencies=dependencies,
|
|
variables=data.get("variables", {}),
|
|
recent_outputs=data.get("recent_outputs", []),
|
|
agent_id=data.get("agent_id"),
|
|
agent_tier=data.get("agent_tier", 0),
|
|
content_hash=data.get("content_hash", ""),
|
|
parent_checkpoint_id=data.get("parent_checkpoint_id"),
|
|
estimated_tokens=data.get("estimated_tokens", 0),
|
|
orchestration_mode=data.get("orchestration_mode", "disabled"),
|
|
pending_instructions=pending_instructions,
|
|
last_model_response=data.get("last_model_response"),
|
|
directory_statuses=directory_statuses,
|
|
status_summary=data.get("status_summary", {}),
|
|
memory_refs=memory_refs,
|
|
memory_summary=data.get("memory_summary", {})
|
|
)
|
|
|
|
|
|
# =============================================================================
|
|
# Checkpoint Manager
|
|
# =============================================================================
|
|
|
|
class CheckpointManager:
|
|
"""
|
|
Manages context checkpoints for long-running agent sessions.
|
|
"""
|
|
|
|
def __init__(self, storage_dir: Path = CHECKPOINT_DIR):
|
|
self.storage_dir = storage_dir
|
|
self.storage_dir.mkdir(parents=True, exist_ok=True)
|
|
self.redis = self._get_redis()
|
|
|
|
def _now(self) -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
def _generate_id(self) -> str:
|
|
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
|
|
suffix = hashlib.sha256(f"{timestamp}-{os.getpid()}".encode()).hexdigest()[:8]
|
|
return f"ckpt-{timestamp}-{suffix}"
|
|
|
|
def _get_redis(self) -> Optional[redis.Redis]:
|
|
"""Get DragonflyDB connection"""
|
|
try:
|
|
with open("/opt/vault/init-keys.json") as f:
|
|
token = json.load(f)["root_token"]
|
|
|
|
result = subprocess.run([
|
|
"curl", "-sk",
|
|
"-H", f"X-Vault-Token: {token}",
|
|
"https://127.0.0.1:8200/v1/secret/data/services/dragonfly"
|
|
], capture_output=True, text=True)
|
|
|
|
creds = json.loads(result.stdout)["data"]["data"]
|
|
return redis.Redis(
|
|
host=creds["host"],
|
|
port=int(creds["port"]),
|
|
password=creds["password"],
|
|
decode_responses=True
|
|
)
|
|
except:
|
|
return None
|
|
|
|
def _estimate_tokens(self, data: dict) -> int:
|
|
"""Rough token estimation (4 chars ~= 1 token)"""
|
|
json_str = json.dumps(data)
|
|
return len(json_str) // 4
|
|
|
|
def _compute_hash(self, data: dict) -> str:
|
|
"""Compute content hash for change detection"""
|
|
# Exclude volatile fields
|
|
stable_data = {k: v for k, v in data.items()
|
|
if k not in ["checkpoint_id", "created_at", "content_hash", "estimated_tokens"]}
|
|
return hashlib.sha256(json.dumps(stable_data, sort_keys=True).encode()).hexdigest()[:16]
|
|
|
|
# -------------------------------------------------------------------------
|
|
# State Collection
|
|
# -------------------------------------------------------------------------
|
|
|
|
def collect_phase_state(self) -> Optional[ProjectPhase]:
|
|
"""Collect current project phase from implementation plan"""
|
|
plan_file = Path("/root/agent-taxonomy-implementation-plan.md")
|
|
if not plan_file.exists():
|
|
return None
|
|
|
|
content = plan_file.read_text()
|
|
|
|
# Phase definitions with names
|
|
phase_names = {
|
|
1: "Foundation (Vault + Basic Infrastructure)",
|
|
2: "Vault Policy Engine",
|
|
3: "Execution Pipeline",
|
|
4: "Promotion and Revocation Engine",
|
|
5: "Agent Bootstrapping",
|
|
6: "Pipeline DSL, Agent Templates, Testing Framework",
|
|
7: "Hierarchical Teams & Learning System",
|
|
8: "Production Hardening",
|
|
9: "External Integrations",
|
|
10: "Multi-Tenant Support",
|
|
11: "Agent Marketplace",
|
|
12: "Observability",
|
|
}
|
|
|
|
# Parse phases from markdown - check for COMPLETE marker
|
|
phases_status = {}
|
|
for num in range(1, 13):
|
|
marker = f"Phase {num}:"
|
|
if marker in content:
|
|
try:
|
|
section = content.split(marker)[1].split("##")[0]
|
|
if "COMPLETE" in section or "✅" in section:
|
|
phases_status[num] = "complete"
|
|
elif "IN_PROGRESS" in section or "🚧" in section:
|
|
phases_status[num] = "in_progress"
|
|
else:
|
|
phases_status[num] = "pending"
|
|
except IndexError:
|
|
phases_status[num] = "pending"
|
|
else:
|
|
phases_status[num] = "not_defined"
|
|
|
|
# Find current phase (first non-complete, or highest complete + 1)
|
|
current_phase = 8 # Default to Phase 8
|
|
for num in range(12, 0, -1):
|
|
if phases_status.get(num) == "in_progress":
|
|
current_phase = num
|
|
break
|
|
elif phases_status.get(num) == "complete":
|
|
current_phase = num + 1
|
|
break
|
|
|
|
# Cap at 12
|
|
current_phase = min(current_phase, 12)
|
|
|
|
return ProjectPhase(
|
|
name=f"Phase {current_phase}: {phase_names.get(current_phase, 'Unknown')}",
|
|
number=current_phase,
|
|
status="in_progress",
|
|
started_at=self._now()
|
|
)
|
|
|
|
def collect_tasks_from_db(self) -> list[TaskState]:
|
|
"""Collect task state from governance database"""
|
|
tasks = []
|
|
|
|
if not LEDGER_DB.exists():
|
|
return tasks
|
|
|
|
try:
|
|
conn = sqlite3.connect(LEDGER_DB)
|
|
conn.row_factory = sqlite3.Row
|
|
cursor = conn.cursor()
|
|
|
|
# Check if tasks table exists
|
|
cursor.execute("""
|
|
SELECT name FROM sqlite_master
|
|
WHERE type='table' AND name='tasks'
|
|
""")
|
|
|
|
if cursor.fetchone():
|
|
cursor.execute("SELECT * FROM tasks ORDER BY id DESC LIMIT 20")
|
|
for row in cursor.fetchall():
|
|
tasks.append(TaskState(
|
|
id=str(row['id']),
|
|
subject=row['subject'],
|
|
status=row['status'],
|
|
owner=row.get('owner'),
|
|
metadata=json.loads(row['metadata']) if row.get('metadata') else {}
|
|
))
|
|
|
|
conn.close()
|
|
except Exception as e:
|
|
print(f"Warning: Could not read tasks from DB: {e}")
|
|
|
|
return tasks
|
|
|
|
def collect_tasks_from_redis(self) -> list[TaskState]:
|
|
"""Collect task state from DragonflyDB"""
|
|
tasks = []
|
|
|
|
if not self.redis:
|
|
return tasks
|
|
|
|
try:
|
|
# Look for task keys
|
|
keys = self.redis.keys("task:*:state")
|
|
for key in keys:
|
|
data = self.redis.get(key)
|
|
if data:
|
|
task_data = json.loads(data)
|
|
tasks.append(TaskState(
|
|
id=task_data.get("id", key.split(":")[1]),
|
|
subject=task_data.get("subject", ""),
|
|
status=task_data.get("status", "pending"),
|
|
owner=task_data.get("owner"),
|
|
blocks=task_data.get("blocks", []),
|
|
blocked_by=task_data.get("blocked_by", [])
|
|
))
|
|
except Exception as e:
|
|
print(f"Warning: Could not read tasks from Redis: {e}")
|
|
|
|
return tasks
|
|
|
|
def collect_dependencies(self) -> list[Dependency]:
|
|
"""Collect dependency states"""
|
|
dependencies = []
|
|
|
|
# Check Vault
|
|
try:
|
|
result = subprocess.run(
|
|
["docker", "exec", "vault", "vault", "status"],
|
|
capture_output=True, text=True, timeout=5
|
|
)
|
|
dependencies.append(Dependency(
|
|
name="vault",
|
|
type="service",
|
|
status="available" if result.returncode == 0 else "unavailable",
|
|
endpoint="https://127.0.0.1:8200",
|
|
last_checked=self._now()
|
|
))
|
|
except:
|
|
dependencies.append(Dependency(
|
|
name="vault", type="service", status="unavailable",
|
|
last_checked=self._now()
|
|
))
|
|
|
|
# Check DragonflyDB
|
|
if self.redis:
|
|
try:
|
|
self.redis.ping()
|
|
dependencies.append(Dependency(
|
|
name="dragonfly",
|
|
type="database",
|
|
status="available",
|
|
endpoint="redis://127.0.0.1:6379",
|
|
last_checked=self._now()
|
|
))
|
|
except:
|
|
dependencies.append(Dependency(
|
|
name="dragonfly", type="database", status="unavailable",
|
|
last_checked=self._now()
|
|
))
|
|
|
|
# Check SQLite
|
|
if LEDGER_DB.exists():
|
|
dependencies.append(Dependency(
|
|
name="ledger",
|
|
type="database",
|
|
status="available",
|
|
endpoint=str(LEDGER_DB),
|
|
last_checked=self._now()
|
|
))
|
|
|
|
return dependencies
|
|
|
|
def collect_agent_state(self) -> tuple[Optional[str], int]:
|
|
"""Collect current agent ID and tier"""
|
|
agent_id = os.environ.get("AGENT_ID")
|
|
agent_tier = int(os.environ.get("AGENT_TIER", "0"))
|
|
|
|
# Try to get from Redis
|
|
if self.redis and agent_id:
|
|
try:
|
|
state = self.redis.get(f"agent:{agent_id}:state")
|
|
if state:
|
|
data = json.loads(state)
|
|
agent_tier = data.get("tier", agent_tier)
|
|
except:
|
|
pass
|
|
|
|
return agent_id, agent_tier
|
|
|
|
def collect_orchestration_state(self) -> tuple[str, list]:
|
|
"""Collect current orchestration mode and pending instructions"""
|
|
mode = os.environ.get("AUTO_AGENT_MODE", "disabled")
|
|
pending = []
|
|
|
|
# Try to get pending instructions from Redis
|
|
if self.redis:
|
|
try:
|
|
# Get from instruction queue
|
|
queue_data = self.redis.lrange("orchestration:instructions", 0, -1)
|
|
for item in queue_data:
|
|
data = json.loads(item)
|
|
pending.append(PendingInstruction(
|
|
id=data.get("id", ""),
|
|
instruction=data.get("instruction", ""),
|
|
command_type=data.get("command_type", "shell"),
|
|
priority=data.get("priority", 0),
|
|
requires_confirmation=data.get("requires_confirmation", False),
|
|
created_at=data.get("created_at", ""),
|
|
expires_at=data.get("expires_at")
|
|
))
|
|
except:
|
|
pass
|
|
|
|
return mode, pending
|
|
|
|
def collect_recent_outputs(self, max_items: int = 10) -> list[dict]:
|
|
"""Collect recent command outputs (summaries only)"""
|
|
outputs = []
|
|
|
|
# Check evidence packages
|
|
evidence_dir = Path("/opt/agent-governance/evidence/packages")
|
|
if evidence_dir.exists():
|
|
packages = sorted(evidence_dir.iterdir(), reverse=True)[:max_items]
|
|
for pkg in packages:
|
|
meta_file = pkg / "evidence.json"
|
|
if meta_file.exists():
|
|
try:
|
|
meta = json.loads(meta_file.read_text())
|
|
outputs.append({
|
|
"type": "evidence",
|
|
"id": meta.get("package_id"),
|
|
"action": meta.get("action_type"),
|
|
"success": meta.get("results", {}).get("execution_success"),
|
|
"timestamp": meta.get("created_at")
|
|
})
|
|
except:
|
|
pass
|
|
|
|
return outputs
|
|
|
|
def collect_directory_statuses(self) -> tuple[list[DirectoryStatusEntry], dict]:
|
|
"""
|
|
Collect status from all directory STATUS.md files.
|
|
Returns (list of entries, summary dict by phase).
|
|
"""
|
|
import re
|
|
|
|
PROJECT_ROOT = Path("/opt/agent-governance")
|
|
SKIP_DIRS = {"__pycache__", "node_modules", ".git", "logs", "storage",
|
|
"dragonfly-data", "credentials", "workspace", ".claude"}
|
|
|
|
entries = []
|
|
summary = {
|
|
"complete": 0,
|
|
"in_progress": 0,
|
|
"blocked": 0,
|
|
"needs_review": 0,
|
|
"not_started": 0,
|
|
"total": 0
|
|
}
|
|
|
|
def should_skip(dir_path: Path) -> bool:
|
|
name = dir_path.name
|
|
return name.startswith(".") or name in SKIP_DIRS or "evd-" in name
|
|
|
|
def parse_phase(content: str) -> str:
|
|
"""Parse phase status from STATUS.md content.
|
|
|
|
Looks for explicit status in 'Current Phase' section first,
|
|
then falls back to checking the header area only (first 500 chars).
|
|
This prevents activity log entries from triggering false status.
|
|
"""
|
|
# First, try to find explicit status in Current Phase section
|
|
# Pattern: **STATUS** or **STATUS:** or **PHASE X: NAME** (Status)
|
|
phase_pattern = r'\*\*\s*(COMPLETE|BLOCKED|IN[_\s]?PROGRESS|NEEDS[_\s]?REVIEW|NOT[_\s]?STARTED)\s*\*\*'
|
|
phase_match = re.search(phase_pattern, content[:1000], re.IGNORECASE)
|
|
if phase_match:
|
|
status = phase_match.group(1).lower().replace(" ", "_").replace("-", "_")
|
|
if "complete" in status:
|
|
return "complete"
|
|
if "progress" in status:
|
|
return "in_progress"
|
|
if "block" in status:
|
|
return "blocked"
|
|
if "review" in status:
|
|
return "needs_review"
|
|
return "not_started"
|
|
|
|
# Check for status in parentheses after phase name: (Complete) or (Blocked)
|
|
paren_pattern = r'\((?:status:\s*)?(Complete|Blocked|In[_\s]?Progress|Needs[_\s]?Review)\)'
|
|
paren_match = re.search(paren_pattern, content[:1000], re.IGNORECASE)
|
|
if paren_match:
|
|
status = paren_match.group(1).lower()
|
|
if "complete" in status:
|
|
return "complete"
|
|
if "progress" in status:
|
|
return "in_progress"
|
|
if "block" in status:
|
|
return "blocked"
|
|
if "review" in status:
|
|
return "needs_review"
|
|
|
|
# Fallback: check header area only (first 500 chars) for keywords
|
|
header = content[:500].lower()
|
|
if "blocked" in header:
|
|
return "blocked"
|
|
if "in_progress" in header or "in progress" in header:
|
|
return "in_progress"
|
|
if "needs_review" in header or "needs review" in header:
|
|
return "needs_review"
|
|
if "complete" in header:
|
|
return "complete"
|
|
return "not_started"
|
|
|
|
def parse_timestamp(content: str) -> Optional[str]:
|
|
pattern = r'(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2})'
|
|
matches = re.findall(pattern, content)
|
|
return matches[-1] if matches else None
|
|
|
|
def count_tasks(content: str) -> tuple[int, int]:
|
|
total = 0
|
|
done = 0
|
|
pattern = r'\|\s*([x✓☐✗ -])\s*\|'
|
|
for match in re.finditer(pattern, content, re.IGNORECASE):
|
|
char = match.group(1).strip()
|
|
if char and char not in ['-', '']:
|
|
total += 1
|
|
if char.lower() in ['x', '✓']:
|
|
done += 1
|
|
return total, done
|
|
|
|
# Walk directory tree
|
|
for root, subdirs, files in os.walk(PROJECT_ROOT):
|
|
root_path = Path(root)
|
|
subdirs[:] = [d for d in subdirs if not should_skip(root_path / d)]
|
|
|
|
if should_skip(root_path):
|
|
continue
|
|
|
|
rel_path = str(root_path.relative_to(PROJECT_ROOT))
|
|
if rel_path == ".":
|
|
rel_path = "."
|
|
|
|
readme_path = root_path / "README.md"
|
|
status_path = root_path / "STATUS.md"
|
|
|
|
entry = DirectoryStatusEntry(
|
|
path=rel_path,
|
|
phase="not_started",
|
|
has_readme=readme_path.exists(),
|
|
has_status=status_path.exists()
|
|
)
|
|
|
|
if status_path.exists():
|
|
try:
|
|
content = status_path.read_text()
|
|
entry.phase = parse_phase(content)
|
|
entry.last_updated = parse_timestamp(content)
|
|
entry.tasks_total, entry.tasks_done = count_tasks(content)
|
|
except:
|
|
pass
|
|
|
|
entries.append(entry)
|
|
summary[entry.phase] = summary.get(entry.phase, 0) + 1
|
|
summary["total"] += 1
|
|
|
|
return entries, summary
|
|
|
|
def collect_memory_refs(self, limit: int = 20) -> tuple[list, dict]:
|
|
"""
|
|
Collect references to recent memory entries.
|
|
Returns (list of MemoryRef, summary dict).
|
|
"""
|
|
refs = []
|
|
summary = {"total_entries": 0, "total_tokens": 0, "by_type": {}}
|
|
|
|
try:
|
|
# Import memory manager
|
|
sys.path.insert(0, "/opt/agent-governance/memory")
|
|
from memory import MemoryManager, MemoryStatus
|
|
|
|
manager = MemoryManager()
|
|
entries = manager.list_entries(status=MemoryStatus.ACTIVE, limit=limit)
|
|
|
|
for entry in entries:
|
|
refs.append(MemoryRef(
|
|
id=entry.id,
|
|
type=entry.type.value if hasattr(entry.type, 'value') else entry.type,
|
|
summary=entry.summary or "(no summary)",
|
|
tokens=entry.tokens_estimate
|
|
))
|
|
summary["total_entries"] += 1
|
|
summary["total_tokens"] += entry.tokens_estimate
|
|
|
|
entry_type = entry.type.value if hasattr(entry.type, 'value') else entry.type
|
|
summary["by_type"][entry_type] = summary["by_type"].get(entry_type, 0) + 1
|
|
|
|
except Exception as e:
|
|
# Memory layer not available - return empty
|
|
pass
|
|
|
|
return refs, summary
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Checkpoint Operations
|
|
# -------------------------------------------------------------------------
|
|
|
|
def create_checkpoint(
|
|
self,
|
|
session_id: Optional[str] = None,
|
|
variables: dict = None,
|
|
notes: str = "",
|
|
include_orchestration: bool = True,
|
|
include_directory_status: bool = True,
|
|
include_memory: bool = True
|
|
) -> ContextCheckpoint:
|
|
"""Create a new checkpoint capturing current state"""
|
|
|
|
# Get previous checkpoint for parent reference
|
|
previous = self.get_latest_checkpoint()
|
|
|
|
# Collect state
|
|
phase = self.collect_phase_state()
|
|
tasks = self.collect_tasks_from_db() or self.collect_tasks_from_redis()
|
|
dependencies = self.collect_dependencies()
|
|
agent_id, agent_tier = self.collect_agent_state()
|
|
recent_outputs = self.collect_recent_outputs()
|
|
|
|
# Collect orchestration state if enabled
|
|
orchestration_mode = "disabled"
|
|
pending_instructions = []
|
|
if include_orchestration:
|
|
orchestration_mode, pending_instructions = self.collect_orchestration_state()
|
|
|
|
# Collect directory statuses (integrated with status system)
|
|
directory_statuses = []
|
|
status_summary = {}
|
|
if include_directory_status:
|
|
directory_statuses, status_summary = self.collect_directory_statuses()
|
|
|
|
# Collect memory references (links to external memory)
|
|
memory_refs = []
|
|
memory_summary = {}
|
|
if include_memory:
|
|
memory_refs, memory_summary = self.collect_memory_refs()
|
|
|
|
# Build checkpoint
|
|
checkpoint = ContextCheckpoint(
|
|
checkpoint_id=self._generate_id(),
|
|
created_at=self._now(),
|
|
session_id=session_id or os.environ.get("SESSION_ID"),
|
|
phase=phase,
|
|
phases_completed=[1, 2, 3, 4] if phase and phase.number == 5 else [],
|
|
tasks=tasks,
|
|
dependencies=dependencies,
|
|
variables=variables or {},
|
|
recent_outputs=recent_outputs,
|
|
agent_id=agent_id,
|
|
agent_tier=agent_tier,
|
|
parent_checkpoint_id=previous.checkpoint_id if previous else None,
|
|
orchestration_mode=orchestration_mode,
|
|
pending_instructions=pending_instructions,
|
|
directory_statuses=directory_statuses,
|
|
status_summary=status_summary,
|
|
memory_refs=memory_refs,
|
|
memory_summary=memory_summary
|
|
)
|
|
|
|
# Add notes to phase if provided
|
|
if notes and checkpoint.phase:
|
|
checkpoint.phase.notes = notes
|
|
|
|
# Compute hash and token estimate
|
|
data = checkpoint.to_dict()
|
|
checkpoint.content_hash = self._compute_hash(data)
|
|
checkpoint.estimated_tokens = self._estimate_tokens(data)
|
|
|
|
# Save checkpoint
|
|
self.save_checkpoint(checkpoint)
|
|
|
|
# Store in Redis for fast access
|
|
if self.redis:
|
|
self.redis.set("checkpoint:latest", json.dumps(checkpoint.to_dict()))
|
|
self.redis.set(f"checkpoint:{checkpoint.checkpoint_id}", json.dumps(checkpoint.to_dict()))
|
|
|
|
return checkpoint
|
|
|
|
def save_checkpoint(self, checkpoint: ContextCheckpoint) -> Path:
|
|
"""Save checkpoint to disk"""
|
|
filename = f"{checkpoint.checkpoint_id}.json"
|
|
filepath = self.storage_dir / filename
|
|
|
|
with open(filepath, "w") as f:
|
|
json.dump(checkpoint.to_dict(), f, indent=2)
|
|
|
|
# Prune old checkpoints
|
|
self.prune_checkpoints()
|
|
|
|
return filepath
|
|
|
|
def load_checkpoint(self, checkpoint_id: str) -> Optional[ContextCheckpoint]:
|
|
"""Load a specific checkpoint"""
|
|
# Try Redis first
|
|
if self.redis:
|
|
data = self.redis.get(f"checkpoint:{checkpoint_id}")
|
|
if data:
|
|
return ContextCheckpoint.from_dict(json.loads(data))
|
|
|
|
# Fall back to disk
|
|
filepath = self.storage_dir / f"{checkpoint_id}.json"
|
|
if filepath.exists():
|
|
with open(filepath) as f:
|
|
return ContextCheckpoint.from_dict(json.load(f))
|
|
|
|
return None
|
|
|
|
def get_latest_checkpoint(self) -> Optional[ContextCheckpoint]:
|
|
"""Get the most recent checkpoint"""
|
|
# Try Redis first
|
|
if self.redis:
|
|
data = self.redis.get("checkpoint:latest")
|
|
if data:
|
|
return ContextCheckpoint.from_dict(json.loads(data))
|
|
|
|
# Fall back to disk
|
|
files = sorted(self.storage_dir.glob("ckpt-*.json"), reverse=True)
|
|
if files:
|
|
with open(files[0]) as f:
|
|
return ContextCheckpoint.from_dict(json.load(f))
|
|
|
|
return None
|
|
|
|
def list_checkpoints(self, limit: int = 20) -> list[dict]:
|
|
"""List available checkpoints"""
|
|
checkpoints = []
|
|
|
|
files = sorted(self.storage_dir.glob("ckpt-*.json"), reverse=True)[:limit]
|
|
for f in files:
|
|
try:
|
|
with open(f) as fp:
|
|
data = json.load(fp)
|
|
checkpoints.append({
|
|
"id": data["checkpoint_id"],
|
|
"created_at": data["created_at"],
|
|
"phase": data.get("phase", {}).get("name") if data.get("phase") else None,
|
|
"tasks": len(data.get("tasks", [])),
|
|
"tokens": data.get("estimated_tokens", 0)
|
|
})
|
|
except:
|
|
pass
|
|
|
|
return checkpoints
|
|
|
|
def prune_checkpoints(self, keep: int = MAX_CHECKPOINTS):
|
|
"""Remove old checkpoints beyond the limit"""
|
|
files = sorted(self.storage_dir.glob("ckpt-*.json"), reverse=True)
|
|
|
|
for f in files[keep:]:
|
|
f.unlink()
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Diff Operations
|
|
# -------------------------------------------------------------------------
|
|
|
|
def diff_checkpoints(
|
|
self,
|
|
checkpoint_a: ContextCheckpoint,
|
|
checkpoint_b: ContextCheckpoint
|
|
) -> dict:
|
|
"""Compare two checkpoints and return differences"""
|
|
diff = {
|
|
"from_id": checkpoint_a.checkpoint_id,
|
|
"to_id": checkpoint_b.checkpoint_id,
|
|
"time_delta": None,
|
|
"changes": []
|
|
}
|
|
|
|
# Time delta
|
|
try:
|
|
from_time = datetime.fromisoformat(checkpoint_a.created_at.replace("Z", "+00:00"))
|
|
to_time = datetime.fromisoformat(checkpoint_b.created_at.replace("Z", "+00:00"))
|
|
diff["time_delta"] = str(to_time - from_time)
|
|
except:
|
|
pass
|
|
|
|
# Phase change
|
|
if checkpoint_a.phase and checkpoint_b.phase:
|
|
if checkpoint_a.phase.number != checkpoint_b.phase.number:
|
|
diff["changes"].append({
|
|
"type": "phase_change",
|
|
"from": f"Phase {checkpoint_a.phase.number}",
|
|
"to": f"Phase {checkpoint_b.phase.number}"
|
|
})
|
|
elif checkpoint_a.phase.status != checkpoint_b.phase.status:
|
|
diff["changes"].append({
|
|
"type": "phase_status",
|
|
"from": checkpoint_a.phase.status,
|
|
"to": checkpoint_b.phase.status
|
|
})
|
|
|
|
# Task changes
|
|
old_tasks = {t.id: t for t in checkpoint_a.tasks}
|
|
new_tasks = {t.id: t for t in checkpoint_b.tasks}
|
|
|
|
for task_id, task in new_tasks.items():
|
|
if task_id not in old_tasks:
|
|
diff["changes"].append({
|
|
"type": "task_added",
|
|
"task_id": task_id,
|
|
"subject": task.subject
|
|
})
|
|
elif old_tasks[task_id].status != task.status:
|
|
diff["changes"].append({
|
|
"type": "task_status",
|
|
"task_id": task_id,
|
|
"from": old_tasks[task_id].status,
|
|
"to": task.status
|
|
})
|
|
|
|
for task_id in old_tasks:
|
|
if task_id not in new_tasks:
|
|
diff["changes"].append({
|
|
"type": "task_removed",
|
|
"task_id": task_id
|
|
})
|
|
|
|
# Dependency changes
|
|
old_deps = {d.name: d.status for d in checkpoint_a.dependencies}
|
|
new_deps = {d.name: d.status for d in checkpoint_b.dependencies}
|
|
|
|
for name, status in new_deps.items():
|
|
if name not in old_deps:
|
|
diff["changes"].append({
|
|
"type": "dependency_added",
|
|
"name": name,
|
|
"status": status
|
|
})
|
|
elif old_deps[name] != status:
|
|
diff["changes"].append({
|
|
"type": "dependency_status",
|
|
"name": name,
|
|
"from": old_deps[name],
|
|
"to": status
|
|
})
|
|
|
|
# Variable changes
|
|
old_vars = checkpoint_a.variables
|
|
new_vars = checkpoint_b.variables
|
|
|
|
for key in set(list(old_vars.keys()) + list(new_vars.keys())):
|
|
if key not in old_vars:
|
|
diff["changes"].append({
|
|
"type": "variable_added",
|
|
"key": key
|
|
})
|
|
elif key not in new_vars:
|
|
diff["changes"].append({
|
|
"type": "variable_removed",
|
|
"key": key
|
|
})
|
|
elif old_vars[key] != new_vars[key]:
|
|
diff["changes"].append({
|
|
"type": "variable_changed",
|
|
"key": key
|
|
})
|
|
|
|
# Content hash change
|
|
diff["content_changed"] = checkpoint_a.content_hash != checkpoint_b.content_hash
|
|
|
|
return diff
|
|
|
|
|
|
# =============================================================================
|
|
# Token-Aware Context Summarizer
|
|
# =============================================================================
|
|
|
|
class ContextSummarizer:
|
|
"""
|
|
Creates minimal context summaries for sub-agent calls.
|
|
Reduces token usage while preserving essential information.
|
|
"""
|
|
|
|
# Token budgets for different summary levels
|
|
BUDGET_MINIMAL = 500
|
|
BUDGET_COMPACT = 1000
|
|
BUDGET_STANDARD = 2000
|
|
BUDGET_FULL = 4000
|
|
|
|
def __init__(self, checkpoint: ContextCheckpoint):
|
|
self.checkpoint = checkpoint
|
|
|
|
def _estimate_tokens(self, text: str) -> int:
|
|
return len(text) // 4
|
|
|
|
def minimal_summary(self) -> str:
|
|
"""Absolute minimum context (~500 tokens)"""
|
|
lines = []
|
|
|
|
# Phase
|
|
if self.checkpoint.phase:
|
|
lines.append(f"Phase: {self.checkpoint.phase.name} ({self.checkpoint.phase.status})")
|
|
|
|
# Active task
|
|
if self.checkpoint.active_task_id:
|
|
task = next((t for t in self.checkpoint.tasks if t.id == self.checkpoint.active_task_id), None)
|
|
if task:
|
|
lines.append(f"Active Task: {task.subject}")
|
|
|
|
# Agent
|
|
if self.checkpoint.agent_id:
|
|
lines.append(f"Agent: {self.checkpoint.agent_id} (Tier {self.checkpoint.agent_tier})")
|
|
|
|
# Key deps
|
|
available = [d.name for d in self.checkpoint.dependencies if d.status == "available"]
|
|
if available:
|
|
lines.append(f"Available: {', '.join(available)}")
|
|
|
|
return "\n".join(lines)
|
|
|
|
def compact_summary(self) -> str:
|
|
"""Compact context (~1000 tokens)"""
|
|
lines = [self.minimal_summary(), ""]
|
|
|
|
# Recent tasks
|
|
pending = [t for t in self.checkpoint.tasks if t.status == "pending"][:5]
|
|
in_progress = [t for t in self.checkpoint.tasks if t.status == "in_progress"]
|
|
|
|
if in_progress:
|
|
lines.append("In Progress:")
|
|
for t in in_progress:
|
|
lines.append(f" - {t.subject}")
|
|
|
|
if pending:
|
|
lines.append(f"Pending Tasks: {len(pending)}")
|
|
|
|
# Key variables
|
|
if self.checkpoint.variables:
|
|
lines.append("\nKey Variables:")
|
|
for k, v in list(self.checkpoint.variables.items())[:5]:
|
|
lines.append(f" {k}: {str(v)[:50]}")
|
|
|
|
return "\n".join(lines)
|
|
|
|
def standard_summary(self) -> str:
|
|
"""Standard context (~2000 tokens)"""
|
|
lines = [self.compact_summary(), ""]
|
|
|
|
# All tasks with status
|
|
lines.append("\nAll Tasks:")
|
|
for t in self.checkpoint.tasks[:15]:
|
|
status_icon = {"pending": "[ ]", "in_progress": "[>]", "completed": "[x]"}.get(t.status, "[?]")
|
|
lines.append(f" {status_icon} {t.subject}")
|
|
|
|
# Dependencies
|
|
lines.append("\nDependencies:")
|
|
for d in self.checkpoint.dependencies:
|
|
status_icon = {"available": "+", "unavailable": "-", "degraded": "~"}.get(d.status, "?")
|
|
lines.append(f" [{status_icon}] {d.name}: {d.endpoint or 'N/A'}")
|
|
|
|
# Recent outputs
|
|
if self.checkpoint.recent_outputs:
|
|
lines.append("\nRecent Outputs:")
|
|
for o in self.checkpoint.recent_outputs[:5]:
|
|
result = "OK" if o.get("success") else "FAIL"
|
|
lines.append(f" - {o.get('action', 'unknown')} [{result}]")
|
|
|
|
return "\n".join(lines)
|
|
|
|
def full_summary(self) -> str:
|
|
"""Full context for complex operations (~4000 tokens)"""
|
|
lines = [self.standard_summary(), ""]
|
|
|
|
# All variables
|
|
lines.append("\nAll Variables:")
|
|
for k, v in self.checkpoint.variables.items():
|
|
lines.append(f" {k}: {json.dumps(v)[:100]}")
|
|
|
|
# Phases completed
|
|
if self.checkpoint.phases_completed:
|
|
lines.append(f"\nCompleted Phases: {self.checkpoint.phases_completed}")
|
|
|
|
# Checkpoint metadata
|
|
lines.append(f"\nCheckpoint: {self.checkpoint.checkpoint_id}")
|
|
lines.append(f"Created: {self.checkpoint.created_at}")
|
|
lines.append(f"Token Estimate: {self.checkpoint.estimated_tokens}")
|
|
|
|
return "\n".join(lines)
|
|
|
|
def for_subagent(
|
|
self,
|
|
task_type: str,
|
|
relevant_keys: list[str] = None,
|
|
max_tokens: int = BUDGET_COMPACT
|
|
) -> str:
|
|
"""
|
|
Generate task-specific context for a sub-agent.
|
|
Only includes information relevant to the task type.
|
|
"""
|
|
lines = []
|
|
|
|
# Always include phase and agent
|
|
if self.checkpoint.phase:
|
|
lines.append(f"Phase: {self.checkpoint.phase.name}")
|
|
if self.checkpoint.agent_id:
|
|
lines.append(f"Agent: {self.checkpoint.agent_id} (T{self.checkpoint.agent_tier})")
|
|
|
|
# Task-specific context
|
|
if task_type in ["terraform", "ansible", "infrastructure"]:
|
|
# Include relevant deps
|
|
infra_deps = [d for d in self.checkpoint.dependencies
|
|
if d.type in ["service", "api"]]
|
|
if infra_deps:
|
|
lines.append("\nInfrastructure:")
|
|
for d in infra_deps:
|
|
lines.append(f" {d.name}: {d.status}")
|
|
|
|
elif task_type in ["database", "query", "ledger"]:
|
|
# Include database deps
|
|
db_deps = [d for d in self.checkpoint.dependencies
|
|
if d.type == "database"]
|
|
if db_deps:
|
|
lines.append("\nDatabases:")
|
|
for d in db_deps:
|
|
lines.append(f" {d.name}: {d.endpoint}")
|
|
|
|
elif task_type in ["promotion", "revocation", "governance"]:
|
|
# Include agent tier info
|
|
lines.append(f"\nGovernance Context:")
|
|
lines.append(f" Current Tier: {self.checkpoint.agent_tier}")
|
|
|
|
# Include relevant variables
|
|
gov_vars = {k: v for k, v in self.checkpoint.variables.items()
|
|
if any(x in k.lower() for x in ["tier", "promotion", "violation"])}
|
|
if gov_vars:
|
|
for k, v in gov_vars.items():
|
|
lines.append(f" {k}: {v}")
|
|
|
|
# Include requested variables
|
|
if relevant_keys:
|
|
lines.append("\nRelevant Variables:")
|
|
for key in relevant_keys:
|
|
if key in self.checkpoint.variables:
|
|
lines.append(f" {key}: {self.checkpoint.variables[key]}")
|
|
|
|
result = "\n".join(lines)
|
|
|
|
# Truncate if over budget
|
|
while self._estimate_tokens(result) > max_tokens and lines:
|
|
lines.pop()
|
|
result = "\n".join(lines)
|
|
|
|
return result
|
|
|
|
|
|
# =============================================================================
|
|
# CLI Interface
|
|
# =============================================================================
|
|
|
|
def cli():
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="Context Checkpoint Skill",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
checkpoint now Create checkpoint
|
|
checkpoint now --notes "Phase 5" Create with notes
|
|
checkpoint load Load latest checkpoint
|
|
checkpoint load ckpt-20260123-... Load specific checkpoint
|
|
checkpoint diff Compare latest vs previous
|
|
checkpoint list List all checkpoints
|
|
checkpoint summary Show context summary
|
|
checkpoint summary --level full Full context summary
|
|
checkpoint prune --keep 10 Keep only 10 checkpoints
|
|
"""
|
|
)
|
|
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
# now
|
|
now_parser = subparsers.add_parser("now", help="Create checkpoint now")
|
|
now_parser.add_argument("--notes", help="Add notes to checkpoint")
|
|
now_parser.add_argument("--var", action="append", nargs=2, metavar=("KEY", "VALUE"),
|
|
help="Add variable (can repeat)")
|
|
now_parser.add_argument("--json", action="store_true", help="Output JSON")
|
|
|
|
# load
|
|
load_parser = subparsers.add_parser("load", help="Load checkpoint")
|
|
load_parser.add_argument("checkpoint_id", nargs="?", help="Checkpoint ID (default: latest)")
|
|
load_parser.add_argument("--json", action="store_true", help="Output JSON")
|
|
|
|
# diff
|
|
diff_parser = subparsers.add_parser("diff", help="Compare checkpoints")
|
|
diff_parser.add_argument("--from", dest="from_id", help="From checkpoint ID")
|
|
diff_parser.add_argument("--to", dest="to_id", help="To checkpoint ID")
|
|
diff_parser.add_argument("--json", action="store_true", help="Output JSON")
|
|
|
|
# list
|
|
list_parser = subparsers.add_parser("list", help="List checkpoints")
|
|
list_parser.add_argument("--limit", type=int, default=20)
|
|
list_parser.add_argument("--json", action="store_true", help="Output JSON")
|
|
|
|
# summary
|
|
summary_parser = subparsers.add_parser("summary", help="Show context summary")
|
|
summary_parser.add_argument("--level", choices=["minimal", "compact", "standard", "full"],
|
|
default="compact")
|
|
summary_parser.add_argument("--for", dest="task_type", help="Task-specific summary")
|
|
|
|
# prune
|
|
prune_parser = subparsers.add_parser("prune", help="Remove old checkpoints")
|
|
prune_parser.add_argument("--keep", type=int, default=MAX_CHECKPOINTS)
|
|
|
|
# report (new - integrated checkpoint + status view)
|
|
report_parser = subparsers.add_parser("report", help="Show checkpoint + directory status report")
|
|
report_parser.add_argument("--checkpoint", help="Checkpoint ID (default: latest)")
|
|
report_parser.add_argument("--phase", help="Filter by status phase")
|
|
report_parser.add_argument("--json", action="store_true", help="Output JSON")
|
|
|
|
# timeline (new - show checkpoint history with status changes)
|
|
timeline_parser = subparsers.add_parser("timeline", help="Show checkpoint timeline with status changes")
|
|
timeline_parser.add_argument("--limit", type=int, default=10, help="Number of checkpoints")
|
|
timeline_parser.add_argument("--json", action="store_true", help="Output JSON")
|
|
|
|
# auto-orchestrate
|
|
auto_parser = subparsers.add_parser("auto-orchestrate", help="Run in automated orchestration mode")
|
|
auto_parser.add_argument("--model", choices=["minimax", "gemini", "gemini-pro"],
|
|
default="minimax", help="Model to use")
|
|
auto_parser.add_argument("--instruction", "-i", action="append",
|
|
help="Instruction to execute (can repeat)")
|
|
auto_parser.add_argument("--confirm", action="store_true",
|
|
help="Require confirmation before execution")
|
|
auto_parser.add_argument("--dry-run", action="store_true",
|
|
help="Show what would be executed without running")
|
|
|
|
# queue
|
|
queue_parser = subparsers.add_parser("queue", help="Manage instruction queue")
|
|
queue_parser.add_argument("action", choices=["list", "add", "clear", "pop"])
|
|
queue_parser.add_argument("--instruction", help="Instruction to add")
|
|
queue_parser.add_argument("--type", default="shell", help="Command type")
|
|
queue_parser.add_argument("--priority", type=int, default=0, help="Priority (higher = first)")
|
|
|
|
args = parser.parse_args()
|
|
|
|
manager = CheckpointManager()
|
|
|
|
# -------------------------------------------------------------------------
|
|
|
|
if args.command == "now":
|
|
variables = {}
|
|
if args.var:
|
|
for key, value in args.var:
|
|
variables[key] = value
|
|
|
|
checkpoint = manager.create_checkpoint(
|
|
variables=variables,
|
|
notes=args.notes or ""
|
|
)
|
|
|
|
if args.json:
|
|
print(json.dumps(checkpoint.to_dict(), indent=2))
|
|
else:
|
|
print(f"\n{'='*60}")
|
|
print("CHECKPOINT CREATED")
|
|
print(f"{'='*60}")
|
|
print(f"ID: {checkpoint.checkpoint_id}")
|
|
print(f"Time: {checkpoint.created_at}")
|
|
if checkpoint.phase:
|
|
print(f"Phase: {checkpoint.phase.name}")
|
|
print(f"Tasks: {len(checkpoint.tasks)}")
|
|
print(f"Dependencies: {len(checkpoint.dependencies)}")
|
|
print(f"Est. Tokens: {checkpoint.estimated_tokens}")
|
|
print(f"Hash: {checkpoint.content_hash}")
|
|
print(f"{'='*60}")
|
|
|
|
elif args.command == "load":
|
|
if args.checkpoint_id:
|
|
checkpoint = manager.load_checkpoint(args.checkpoint_id)
|
|
else:
|
|
checkpoint = manager.get_latest_checkpoint()
|
|
|
|
if not checkpoint:
|
|
print("No checkpoint found")
|
|
sys.exit(1)
|
|
|
|
if args.json:
|
|
print(json.dumps(checkpoint.to_dict(), indent=2))
|
|
else:
|
|
print(f"\n{'='*60}")
|
|
print("CHECKPOINT LOADED")
|
|
print(f"{'='*60}")
|
|
print(f"ID: {checkpoint.checkpoint_id}")
|
|
print(f"Created: {checkpoint.created_at}")
|
|
|
|
if checkpoint.phase:
|
|
print(f"\nPhase: {checkpoint.phase.name}")
|
|
print(f"Status: {checkpoint.phase.status}")
|
|
|
|
if checkpoint.tasks:
|
|
print(f"\nTasks ({len(checkpoint.tasks)}):")
|
|
for t in checkpoint.tasks[:10]:
|
|
status_icon = {"pending": "[ ]", "in_progress": "[>]", "completed": "[x]"}.get(t.status, "[?]")
|
|
print(f" {status_icon} {t.subject}")
|
|
if len(checkpoint.tasks) > 10:
|
|
print(f" ... and {len(checkpoint.tasks) - 10} more")
|
|
|
|
if checkpoint.dependencies:
|
|
print(f"\nDependencies:")
|
|
for d in checkpoint.dependencies:
|
|
icon = {"available": "+", "unavailable": "-"}.get(d.status, "?")
|
|
print(f" [{icon}] {d.name}")
|
|
|
|
print(f"\nTokens: {checkpoint.estimated_tokens}")
|
|
print(f"{'='*60}")
|
|
|
|
elif args.command == "diff":
|
|
if args.from_id and args.to_id:
|
|
checkpoint_a = manager.load_checkpoint(args.from_id)
|
|
checkpoint_b = manager.load_checkpoint(args.to_id)
|
|
else:
|
|
# Compare latest with previous
|
|
checkpoints = manager.list_checkpoints(limit=2)
|
|
if len(checkpoints) < 2:
|
|
print("Need at least 2 checkpoints to diff")
|
|
sys.exit(1)
|
|
|
|
checkpoint_b = manager.load_checkpoint(checkpoints[0]["id"])
|
|
checkpoint_a = manager.load_checkpoint(checkpoints[1]["id"])
|
|
|
|
if not checkpoint_a or not checkpoint_b:
|
|
print("Could not load checkpoints")
|
|
sys.exit(1)
|
|
|
|
diff = manager.diff_checkpoints(checkpoint_a, checkpoint_b)
|
|
|
|
if args.json:
|
|
print(json.dumps(diff, indent=2))
|
|
else:
|
|
print(f"\n{'='*60}")
|
|
print("CHECKPOINT DIFF")
|
|
print(f"{'='*60}")
|
|
print(f"From: {diff['from_id']}")
|
|
print(f"To: {diff['to_id']}")
|
|
if diff['time_delta']:
|
|
print(f"Time: {diff['time_delta']}")
|
|
|
|
print(f"\nChanges ({len(diff['changes'])}):")
|
|
if not diff['changes']:
|
|
print(" No changes detected")
|
|
else:
|
|
for change in diff['changes']:
|
|
ctype = change['type']
|
|
if ctype == "phase_change":
|
|
print(f" [PHASE] {change['from']} -> {change['to']}")
|
|
elif ctype == "task_added":
|
|
print(f" [+TASK] {change['subject']}")
|
|
elif ctype == "task_removed":
|
|
print(f" [-TASK] {change['task_id']}")
|
|
elif ctype == "task_status":
|
|
print(f" [TASK] {change['task_id']}: {change['from']} -> {change['to']}")
|
|
elif ctype == "dependency_status":
|
|
print(f" [DEP] {change['name']}: {change['from']} -> {change['to']}")
|
|
elif ctype == "variable_added":
|
|
print(f" [+VAR] {change['key']}")
|
|
elif ctype == "variable_changed":
|
|
print(f" [VAR] {change['key']} changed")
|
|
else:
|
|
print(f" [{ctype}] {change}")
|
|
|
|
print(f"\nContent Changed: {diff['content_changed']}")
|
|
print(f"{'='*60}")
|
|
|
|
elif args.command == "list":
|
|
checkpoints = manager.list_checkpoints(limit=args.limit)
|
|
|
|
if args.json:
|
|
print(json.dumps(checkpoints, indent=2))
|
|
else:
|
|
print(f"\n{'='*60}")
|
|
print("CHECKPOINTS")
|
|
print(f"{'='*60}")
|
|
|
|
if not checkpoints:
|
|
print("No checkpoints found")
|
|
else:
|
|
for ckpt in checkpoints:
|
|
phase = ckpt['phase'] or "N/A"
|
|
print(f"\n {ckpt['id']}")
|
|
print(f" Created: {ckpt['created_at']}")
|
|
print(f" Phase: {phase}")
|
|
print(f" Tasks: {ckpt['tasks']}, Tokens: {ckpt['tokens']}")
|
|
|
|
print(f"{'='*60}")
|
|
|
|
elif args.command == "summary":
|
|
checkpoint = manager.get_latest_checkpoint()
|
|
|
|
if not checkpoint:
|
|
print("No checkpoint found. Run 'checkpoint now' first.")
|
|
sys.exit(1)
|
|
|
|
summarizer = ContextSummarizer(checkpoint)
|
|
|
|
if args.task_type:
|
|
summary = summarizer.for_subagent(args.task_type)
|
|
elif args.level == "minimal":
|
|
summary = summarizer.minimal_summary()
|
|
elif args.level == "compact":
|
|
summary = summarizer.compact_summary()
|
|
elif args.level == "standard":
|
|
summary = summarizer.standard_summary()
|
|
else:
|
|
summary = summarizer.full_summary()
|
|
|
|
print(summary)
|
|
|
|
elif args.command == "prune":
|
|
manager.prune_checkpoints(keep=args.keep)
|
|
print(f"Pruned checkpoints, keeping last {args.keep}")
|
|
|
|
elif args.command == "report":
|
|
# Load checkpoint
|
|
if args.checkpoint:
|
|
checkpoint = manager.load_checkpoint(args.checkpoint)
|
|
else:
|
|
checkpoint = manager.get_latest_checkpoint()
|
|
|
|
if not checkpoint:
|
|
print("No checkpoint found. Run 'checkpoint now' first.")
|
|
sys.exit(1)
|
|
|
|
if args.json:
|
|
report_data = {
|
|
"checkpoint_id": checkpoint.checkpoint_id,
|
|
"created_at": checkpoint.created_at,
|
|
"phase": asdict(checkpoint.phase) if checkpoint.phase else None,
|
|
"status_summary": checkpoint.status_summary,
|
|
"directory_statuses": [asdict(d) if isinstance(d, DirectoryStatusEntry) else d
|
|
for d in checkpoint.directory_statuses],
|
|
"dependencies": [asdict(d) if isinstance(d, Dependency) else d
|
|
for d in checkpoint.dependencies]
|
|
}
|
|
print(json.dumps(report_data, indent=2))
|
|
else:
|
|
print(f"\n{'='*70}")
|
|
print("CHECKPOINT + DIRECTORY STATUS REPORT")
|
|
print(f"{'='*70}")
|
|
|
|
# Checkpoint info
|
|
print(f"\n[CHECKPOINT]")
|
|
print(f" ID: {checkpoint.checkpoint_id}")
|
|
print(f" Created: {checkpoint.created_at}")
|
|
if checkpoint.phase:
|
|
print(f" Phase: {checkpoint.phase.name}")
|
|
if checkpoint.phase.notes:
|
|
print(f" Notes: {checkpoint.phase.notes}")
|
|
|
|
# Dependencies
|
|
print(f"\n[DEPENDENCIES]")
|
|
for d in checkpoint.dependencies:
|
|
icon = {"available": "✓", "unavailable": "✗", "degraded": "~"}.get(d.status, "?")
|
|
print(f" {icon} {d.name}: {d.endpoint or 'N/A'}")
|
|
|
|
# Status summary
|
|
summary = checkpoint.status_summary
|
|
if summary:
|
|
total = summary.get("total", 0)
|
|
complete = summary.get("complete", 0)
|
|
pct = (complete / total * 100) if total > 0 else 0
|
|
|
|
print(f"\n[DIRECTORY STATUS SUMMARY]")
|
|
bar_width = 30
|
|
filled = int(bar_width * complete / total) if total > 0 else 0
|
|
bar = "█" * filled + "░" * (bar_width - filled)
|
|
print(f" Progress: [{bar}] {pct:.1f}%")
|
|
print(f" ✅ Complete: {summary.get('complete', 0):3d}")
|
|
print(f" 🚧 In Progress: {summary.get('in_progress', 0):3d}")
|
|
print(f" ❗ Blocked: {summary.get('blocked', 0):3d}")
|
|
print(f" ⚠️ Needs Review: {summary.get('needs_review', 0):3d}")
|
|
print(f" ⬜ Not Started: {summary.get('not_started', 0):3d}")
|
|
|
|
# Check project state for blockers
|
|
project_state_path = Path("/opt/agent-governance/project_state.yaml")
|
|
if project_state_path.exists():
|
|
try:
|
|
import yaml
|
|
with open(project_state_path) as f:
|
|
project_state = yaml.safe_load(f)
|
|
|
|
overall = project_state.get("project", {}).get("overall_status", "unknown")
|
|
blockers = project_state.get("blockers", [])
|
|
pending_blockers = [b for b in blockers if b.get("status") == "pending"]
|
|
|
|
if overall != "complete" or pending_blockers:
|
|
print(f"\n[PROJECT STATE WARNING]")
|
|
print(f" Overall Status: {overall.upper()}")
|
|
if pending_blockers:
|
|
print(f" Pending Blockers: {len(pending_blockers)}")
|
|
for b in pending_blockers:
|
|
print(f" ❗ {b.get('id', 'unknown')}: {b.get('description', 'No description')}")
|
|
if b.get("resolution_doc"):
|
|
print(f" See: {b.get('resolution_doc')}")
|
|
except ImportError:
|
|
pass # yaml not available, skip
|
|
except Exception as e:
|
|
print(f"\n[PROJECT STATE ERROR] Could not load: {e}")
|
|
|
|
# Active directories (non-complete)
|
|
active_dirs = [d for d in checkpoint.directory_statuses
|
|
if d.phase in ("in_progress", "blocked", "needs_review")]
|
|
|
|
# Filter by phase if requested
|
|
if args.phase:
|
|
active_dirs = [d for d in checkpoint.directory_statuses
|
|
if d.phase == args.phase]
|
|
print(f"\n[DIRECTORIES: {args.phase.upper()}]")
|
|
elif active_dirs:
|
|
print(f"\n[ACTIVE DIRECTORIES]")
|
|
|
|
phase_icons = {
|
|
"complete": "✅", "in_progress": "🚧", "blocked": "❗",
|
|
"needs_review": "⚠️", "not_started": "⬜"
|
|
}
|
|
|
|
for d in active_dirs[:20]:
|
|
icon = phase_icons.get(d.phase, "?")
|
|
tasks_str = f" [{d.tasks_done}/{d.tasks_total} tasks]" if d.tasks_total > 0 else ""
|
|
updated = f" (updated: {d.last_updated[:10]})" if d.last_updated else ""
|
|
print(f" {icon} {d.path}{tasks_str}{updated}")
|
|
|
|
if len(active_dirs) > 20:
|
|
print(f" ... and {len(active_dirs) - 20} more")
|
|
|
|
print(f"\n{'='*70}")
|
|
|
|
elif args.command == "timeline":
|
|
checkpoints = manager.list_checkpoints(limit=args.limit)
|
|
|
|
if args.json:
|
|
# Load full details for each checkpoint
|
|
timeline_data = []
|
|
for ckpt in checkpoints:
|
|
full_ckpt = manager.load_checkpoint(ckpt["id"])
|
|
if full_ckpt:
|
|
timeline_data.append({
|
|
"checkpoint_id": full_ckpt.checkpoint_id,
|
|
"created_at": full_ckpt.created_at,
|
|
"phase_notes": full_ckpt.phase.notes if full_ckpt.phase else None,
|
|
"status_summary": full_ckpt.status_summary
|
|
})
|
|
print(json.dumps(timeline_data, indent=2))
|
|
else:
|
|
print(f"\n{'='*70}")
|
|
print("CHECKPOINT TIMELINE")
|
|
print(f"{'='*70}")
|
|
|
|
if not checkpoints:
|
|
print("No checkpoints found")
|
|
else:
|
|
for i, ckpt in enumerate(checkpoints):
|
|
full_ckpt = manager.load_checkpoint(ckpt["id"])
|
|
if not full_ckpt:
|
|
continue
|
|
|
|
# Timeline marker
|
|
marker = "●" if i == 0 else "○"
|
|
print(f"\n{marker} {full_ckpt.created_at}")
|
|
print(f" ID: {full_ckpt.checkpoint_id}")
|
|
|
|
# Phase notes
|
|
if full_ckpt.phase and full_ckpt.phase.notes:
|
|
notes = full_ckpt.phase.notes[:60]
|
|
if len(full_ckpt.phase.notes) > 60:
|
|
notes += "..."
|
|
print(f" Notes: {notes}")
|
|
|
|
# Status summary
|
|
summary = full_ckpt.status_summary
|
|
if summary:
|
|
complete = summary.get("complete", 0)
|
|
total = summary.get("total", 0)
|
|
in_prog = summary.get("in_progress", 0)
|
|
print(f" Status: {complete}/{total} complete, {in_prog} in progress")
|
|
|
|
# Show connector for non-last items
|
|
if i < len(checkpoints) - 1:
|
|
print(" │")
|
|
|
|
print(f"\n{'='*70}")
|
|
|
|
elif args.command == "auto-orchestrate":
|
|
# Import model controller
|
|
try:
|
|
sys.path.insert(0, str(ORCHESTRATOR_DIR))
|
|
from model_controller import OrchestrationManager, OrchestrationMode
|
|
except ImportError as e:
|
|
print(f"Error: Could not import model_controller: {e}")
|
|
print("Make sure /opt/agent-governance/orchestrator/model_controller.py exists")
|
|
sys.exit(1)
|
|
|
|
orchestrator = OrchestrationManager()
|
|
|
|
# Enable the selected model
|
|
if not orchestrator.set_mode(args.model):
|
|
print(f"Error: Could not enable {args.model} mode")
|
|
sys.exit(1)
|
|
|
|
print(f"\n{'='*60}")
|
|
print(f"AUTO-ORCHESTRATE MODE: {args.model.upper()}")
|
|
print(f"{'='*60}")
|
|
|
|
if args.dry_run:
|
|
print("[DRY RUN - No commands will be executed]")
|
|
print()
|
|
|
|
# Create pre-run checkpoint
|
|
checkpoint = manager.create_checkpoint(
|
|
notes=f"Auto-orchestrate session started with {args.model}"
|
|
)
|
|
print(f"Checkpoint created: {checkpoint.checkpoint_id}")
|
|
|
|
# Execute instructions
|
|
if args.instruction:
|
|
print(f"\nExecuting {len(args.instruction)} instruction(s):")
|
|
for i, instruction in enumerate(args.instruction, 1):
|
|
print(f"\n[{i}] {instruction}")
|
|
|
|
if args.dry_run:
|
|
print(" [SKIPPED - Dry run]")
|
|
continue
|
|
|
|
if args.confirm:
|
|
confirm = input(" Execute? [y/N]: ")
|
|
if confirm.lower() != 'y':
|
|
print(" [SKIPPED - User declined]")
|
|
continue
|
|
|
|
# Delegate to model
|
|
response = orchestrator.delegate_command(
|
|
instruction,
|
|
command_type="shell",
|
|
require_confirmation=args.confirm
|
|
)
|
|
|
|
if response.success:
|
|
print(f" [OK] Model: {response.model_used}, Tokens: {response.tokens_used}")
|
|
print(f" Output: {response.output[:200]}...")
|
|
else:
|
|
print(f" [FAILED] {response.error}")
|
|
|
|
# Create post-run checkpoint
|
|
final_checkpoint = manager.create_checkpoint(
|
|
notes=f"Auto-orchestrate session completed"
|
|
)
|
|
print(f"\nFinal checkpoint: {final_checkpoint.checkpoint_id}")
|
|
print(f"{'='*60}")
|
|
|
|
elif args.command == "queue":
|
|
if args.action == "list":
|
|
if manager.redis:
|
|
items = manager.redis.lrange("orchestration:instructions", 0, -1)
|
|
print(f"\n{'='*60}")
|
|
print("INSTRUCTION QUEUE")
|
|
print(f"{'='*60}")
|
|
if not items:
|
|
print("Queue is empty")
|
|
else:
|
|
for i, item in enumerate(items, 1):
|
|
data = json.loads(item)
|
|
print(f"\n[{i}] {data.get('instruction', 'N/A')}")
|
|
print(f" Type: {data.get('command_type', 'shell')}")
|
|
print(f" Priority: {data.get('priority', 0)}")
|
|
print(f"{'='*60}")
|
|
else:
|
|
print("Error: DragonflyDB not available")
|
|
|
|
elif args.action == "add":
|
|
if not args.instruction:
|
|
print("Error: --instruction is required")
|
|
sys.exit(1)
|
|
|
|
if manager.redis:
|
|
entry = {
|
|
"id": hashlib.sha256(f"{time.time()}-{args.instruction}".encode()).hexdigest()[:12],
|
|
"instruction": args.instruction,
|
|
"command_type": args.type,
|
|
"priority": args.priority,
|
|
"created_at": datetime.now(timezone.utc).isoformat()
|
|
}
|
|
manager.redis.lpush("orchestration:instructions", json.dumps(entry))
|
|
print(f"Added to queue: {args.instruction}")
|
|
else:
|
|
print("Error: DragonflyDB not available")
|
|
|
|
elif args.action == "clear":
|
|
if manager.redis:
|
|
manager.redis.delete("orchestration:instructions")
|
|
print("Queue cleared")
|
|
else:
|
|
print("Error: DragonflyDB not available")
|
|
|
|
elif args.action == "pop":
|
|
if manager.redis:
|
|
item = manager.redis.rpop("orchestration:instructions")
|
|
if item:
|
|
data = json.loads(item)
|
|
print(f"Popped: {data.get('instruction', 'N/A')}")
|
|
else:
|
|
print("Queue is empty")
|
|
else:
|
|
print("Error: DragonflyDB not available")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cli()
|