#!/usr/bin/env python3 """ Context Checkpoint Skill ======================== Preserves long-running session context and reduces token usage when orchestrating sub-agents. Part of Phase 5: Agent Bootstrapping. Features: - Periodic state capture (phase, tasks, dependencies, variables, outputs) - Token-aware sub-agent context summarization - CLI integration for manual and automatic checkpoints - Extensible storage (local JSON, future: remote sync) """ import json import hashlib import sqlite3 import subprocess import sys import os import time import difflib from dataclasses import dataclass, field, asdict from datetime import datetime, timezone from pathlib import Path from typing import Any, Optional import redis # ============================================================================= # Configuration # ============================================================================= CHECKPOINT_DIR = Path("/opt/agent-governance/checkpoint/storage") LEDGER_DB = Path("/opt/agent-governance/ledger/governance.db") ORCHESTRATOR_DIR = Path("/opt/agent-governance/orchestrator") MAX_CHECKPOINTS = 50 # Keep last N checkpoints AUTO_CHECKPOINT_INTERVAL = 300 # seconds (5 minutes) # Automated orchestration settings AUTO_ORCHESTRATE_ENABLED = os.environ.get("AUTO_AGENT_MODE", "disabled") != "disabled" # ============================================================================= # Data Classes # ============================================================================= @dataclass class ProjectPhase: """Current project phase information""" name: str number: int status: str # in_progress, complete, pending started_at: Optional[str] = None completed_at: Optional[str] = None notes: str = "" @dataclass class TaskState: """State of a single task""" id: str subject: str status: str # pending, in_progress, completed owner: Optional[str] = None blocks: list = field(default_factory=list) blocked_by: list = field(default_factory=list) metadata: dict = field(default_factory=dict) @dataclass class Dependency: """External dependency state""" name: str type: str # service, database, api, file status: str # available, unavailable, degraded endpoint: Optional[str] = None last_checked: Optional[str] = None @dataclass class PendingInstruction: """Instruction waiting to be executed in automated mode""" id: str instruction: str command_type: str # shell, checkpoint, plan, execute, review priority: int = 0 requires_confirmation: bool = False created_at: str = "" expires_at: Optional[str] = None @dataclass class DirectoryStatusEntry: """Status entry for a directory from STATUS.md""" path: str phase: str # complete, in_progress, blocked, needs_review, not_started last_updated: Optional[str] = None has_readme: bool = False has_status: bool = False tasks_total: int = 0 tasks_done: int = 0 @dataclass class MemoryRef: """Lightweight reference to a memory entry (for embedding in checkpoints).""" id: str type: str summary: str tokens: int def to_inline(self) -> str: """Format for inline prompt inclusion.""" return f"[Memory:{self.id}] {self.summary[:50]}... ({self.tokens} tokens)" @dataclass class ContextCheckpoint: """ Complete context checkpoint capturing session state. Designed to be reloadable after token window reset or CLI restart. """ checkpoint_id: str created_at: str session_id: Optional[str] = None # Project State phase: Optional[ProjectPhase] = None phases_completed: list = field(default_factory=list) # Task State tasks: list = field(default_factory=list) # List of TaskState active_task_id: Optional[str] = None # Dependencies dependencies: list = field(default_factory=list) # List of Dependency # Key Variables (agent-specific context) variables: dict = field(default_factory=dict) # Recent Outputs (summaries, not full content) recent_outputs: list = field(default_factory=list) # Agent State agent_id: Optional[str] = None agent_tier: int = 0 # Checksums for change detection content_hash: str = "" parent_checkpoint_id: Optional[str] = None # Token metrics estimated_tokens: int = 0 # Automated Orchestration orchestration_mode: str = "disabled" pending_instructions: list = field(default_factory=list) # List of PendingInstruction last_model_response: Optional[str] = None # Directory Status Snapshot (integrated with status system) directory_statuses: list = field(default_factory=list) # List of DirectoryStatusEntry status_summary: dict = field(default_factory=dict) # Aggregated counts by phase # Memory Layer References (links to external memory for large content) memory_refs: list = field(default_factory=list) # List of MemoryRef memory_summary: dict = field(default_factory=dict) # Aggregated memory stats def to_dict(self) -> dict: """Convert to dictionary for serialization""" return { "checkpoint_id": self.checkpoint_id, "created_at": self.created_at, "session_id": self.session_id, "phase": asdict(self.phase) if self.phase else None, "phases_completed": self.phases_completed, "tasks": [asdict(t) if isinstance(t, TaskState) else t for t in self.tasks], "active_task_id": self.active_task_id, "dependencies": [asdict(d) if isinstance(d, Dependency) else d for d in self.dependencies], "variables": self.variables, "recent_outputs": self.recent_outputs, "agent_id": self.agent_id, "agent_tier": self.agent_tier, "content_hash": self.content_hash, "parent_checkpoint_id": self.parent_checkpoint_id, "estimated_tokens": self.estimated_tokens, "orchestration_mode": self.orchestration_mode, "pending_instructions": [asdict(i) if isinstance(i, PendingInstruction) else i for i in self.pending_instructions], "last_model_response": self.last_model_response, "directory_statuses": [asdict(d) if isinstance(d, DirectoryStatusEntry) else d for d in self.directory_statuses], "status_summary": self.status_summary, "memory_refs": [asdict(m) if isinstance(m, MemoryRef) else m for m in self.memory_refs], "memory_summary": self.memory_summary } @classmethod def from_dict(cls, data: dict) -> 'ContextCheckpoint': """Create from dictionary""" phase = None if data.get("phase"): phase = ProjectPhase(**data["phase"]) tasks = [] for t in data.get("tasks", []): if isinstance(t, dict): tasks.append(TaskState(**t)) else: tasks.append(t) dependencies = [] for d in data.get("dependencies", []): if isinstance(d, dict): dependencies.append(Dependency(**d)) else: dependencies.append(d) pending_instructions = [] for i in data.get("pending_instructions", []): if isinstance(i, dict): pending_instructions.append(PendingInstruction(**i)) else: pending_instructions.append(i) directory_statuses = [] for d in data.get("directory_statuses", []): if isinstance(d, dict): directory_statuses.append(DirectoryStatusEntry(**d)) else: directory_statuses.append(d) memory_refs = [] for m in data.get("memory_refs", []): if isinstance(m, dict): memory_refs.append(MemoryRef(**m)) else: memory_refs.append(m) return cls( checkpoint_id=data["checkpoint_id"], created_at=data["created_at"], session_id=data.get("session_id"), phase=phase, phases_completed=data.get("phases_completed", []), tasks=tasks, active_task_id=data.get("active_task_id"), dependencies=dependencies, variables=data.get("variables", {}), recent_outputs=data.get("recent_outputs", []), agent_id=data.get("agent_id"), agent_tier=data.get("agent_tier", 0), content_hash=data.get("content_hash", ""), parent_checkpoint_id=data.get("parent_checkpoint_id"), estimated_tokens=data.get("estimated_tokens", 0), orchestration_mode=data.get("orchestration_mode", "disabled"), pending_instructions=pending_instructions, last_model_response=data.get("last_model_response"), directory_statuses=directory_statuses, status_summary=data.get("status_summary", {}), memory_refs=memory_refs, memory_summary=data.get("memory_summary", {}) ) # ============================================================================= # Checkpoint Manager # ============================================================================= class CheckpointManager: """ Manages context checkpoints for long-running agent sessions. """ def __init__(self, storage_dir: Path = CHECKPOINT_DIR): self.storage_dir = storage_dir self.storage_dir.mkdir(parents=True, exist_ok=True) self.redis = self._get_redis() def _now(self) -> str: return datetime.now(timezone.utc).isoformat() def _generate_id(self) -> str: timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S") suffix = hashlib.sha256(f"{timestamp}-{os.getpid()}".encode()).hexdigest()[:8] return f"ckpt-{timestamp}-{suffix}" def _get_redis(self) -> Optional[redis.Redis]: """Get DragonflyDB connection""" try: with open("/opt/vault/init-keys.json") as f: token = json.load(f)["root_token"] result = subprocess.run([ "curl", "-sk", "-H", f"X-Vault-Token: {token}", "https://127.0.0.1:8200/v1/secret/data/services/dragonfly" ], capture_output=True, text=True) creds = json.loads(result.stdout)["data"]["data"] return redis.Redis( host=creds["host"], port=int(creds["port"]), password=creds["password"], decode_responses=True ) except: return None def _estimate_tokens(self, data: dict) -> int: """Rough token estimation (4 chars ~= 1 token)""" json_str = json.dumps(data) return len(json_str) // 4 def _compute_hash(self, data: dict) -> str: """Compute content hash for change detection""" # Exclude volatile fields stable_data = {k: v for k, v in data.items() if k not in ["checkpoint_id", "created_at", "content_hash", "estimated_tokens"]} return hashlib.sha256(json.dumps(stable_data, sort_keys=True).encode()).hexdigest()[:16] # ------------------------------------------------------------------------- # State Collection # ------------------------------------------------------------------------- def collect_phase_state(self) -> Optional[ProjectPhase]: """Collect current project phase from implementation plan""" plan_file = Path("/root/agent-taxonomy-implementation-plan.md") if not plan_file.exists(): return None content = plan_file.read_text() # Phase definitions with names phase_names = { 1: "Foundation (Vault + Basic Infrastructure)", 2: "Vault Policy Engine", 3: "Execution Pipeline", 4: "Promotion and Revocation Engine", 5: "Agent Bootstrapping", 6: "Pipeline DSL, Agent Templates, Testing Framework", 7: "Hierarchical Teams & Learning System", 8: "Production Hardening", 9: "External Integrations", 10: "Multi-Tenant Support", 11: "Agent Marketplace", 12: "Observability", } # Parse phases from markdown - check for COMPLETE marker phases_status = {} for num in range(1, 13): marker = f"Phase {num}:" if marker in content: try: section = content.split(marker)[1].split("##")[0] if "COMPLETE" in section or "✅" in section: phases_status[num] = "complete" elif "IN_PROGRESS" in section or "🚧" in section: phases_status[num] = "in_progress" else: phases_status[num] = "pending" except IndexError: phases_status[num] = "pending" else: phases_status[num] = "not_defined" # Find current phase (first non-complete, or highest complete + 1) current_phase = 8 # Default to Phase 8 for num in range(12, 0, -1): if phases_status.get(num) == "in_progress": current_phase = num break elif phases_status.get(num) == "complete": current_phase = num + 1 break # Cap at 12 current_phase = min(current_phase, 12) return ProjectPhase( name=f"Phase {current_phase}: {phase_names.get(current_phase, 'Unknown')}", number=current_phase, status="in_progress", started_at=self._now() ) def collect_tasks_from_db(self) -> list[TaskState]: """Collect task state from governance database""" tasks = [] if not LEDGER_DB.exists(): return tasks try: conn = sqlite3.connect(LEDGER_DB) conn.row_factory = sqlite3.Row cursor = conn.cursor() # Check if tasks table exists cursor.execute(""" SELECT name FROM sqlite_master WHERE type='table' AND name='tasks' """) if cursor.fetchone(): cursor.execute("SELECT * FROM tasks ORDER BY id DESC LIMIT 20") for row in cursor.fetchall(): tasks.append(TaskState( id=str(row['id']), subject=row['subject'], status=row['status'], owner=row.get('owner'), metadata=json.loads(row['metadata']) if row.get('metadata') else {} )) conn.close() except Exception as e: print(f"Warning: Could not read tasks from DB: {e}") return tasks def collect_tasks_from_redis(self) -> list[TaskState]: """Collect task state from DragonflyDB""" tasks = [] if not self.redis: return tasks try: # Look for task keys keys = self.redis.keys("task:*:state") for key in keys: data = self.redis.get(key) if data: task_data = json.loads(data) tasks.append(TaskState( id=task_data.get("id", key.split(":")[1]), subject=task_data.get("subject", ""), status=task_data.get("status", "pending"), owner=task_data.get("owner"), blocks=task_data.get("blocks", []), blocked_by=task_data.get("blocked_by", []) )) except Exception as e: print(f"Warning: Could not read tasks from Redis: {e}") return tasks def collect_dependencies(self) -> list[Dependency]: """Collect dependency states""" dependencies = [] # Check Vault try: result = subprocess.run( ["docker", "exec", "vault", "vault", "status"], capture_output=True, text=True, timeout=5 ) dependencies.append(Dependency( name="vault", type="service", status="available" if result.returncode == 0 else "unavailable", endpoint="https://127.0.0.1:8200", last_checked=self._now() )) except: dependencies.append(Dependency( name="vault", type="service", status="unavailable", last_checked=self._now() )) # Check DragonflyDB if self.redis: try: self.redis.ping() dependencies.append(Dependency( name="dragonfly", type="database", status="available", endpoint="redis://127.0.0.1:6379", last_checked=self._now() )) except: dependencies.append(Dependency( name="dragonfly", type="database", status="unavailable", last_checked=self._now() )) # Check SQLite if LEDGER_DB.exists(): dependencies.append(Dependency( name="ledger", type="database", status="available", endpoint=str(LEDGER_DB), last_checked=self._now() )) return dependencies def collect_agent_state(self) -> tuple[Optional[str], int]: """Collect current agent ID and tier""" agent_id = os.environ.get("AGENT_ID") agent_tier = int(os.environ.get("AGENT_TIER", "0")) # Try to get from Redis if self.redis and agent_id: try: state = self.redis.get(f"agent:{agent_id}:state") if state: data = json.loads(state) agent_tier = data.get("tier", agent_tier) except: pass return agent_id, agent_tier def collect_orchestration_state(self) -> tuple[str, list]: """Collect current orchestration mode and pending instructions""" mode = os.environ.get("AUTO_AGENT_MODE", "disabled") pending = [] # Try to get pending instructions from Redis if self.redis: try: # Get from instruction queue queue_data = self.redis.lrange("orchestration:instructions", 0, -1) for item in queue_data: data = json.loads(item) pending.append(PendingInstruction( id=data.get("id", ""), instruction=data.get("instruction", ""), command_type=data.get("command_type", "shell"), priority=data.get("priority", 0), requires_confirmation=data.get("requires_confirmation", False), created_at=data.get("created_at", ""), expires_at=data.get("expires_at") )) except: pass return mode, pending def collect_recent_outputs(self, max_items: int = 10) -> list[dict]: """Collect recent command outputs (summaries only)""" outputs = [] # Check evidence packages evidence_dir = Path("/opt/agent-governance/evidence/packages") if evidence_dir.exists(): packages = sorted(evidence_dir.iterdir(), reverse=True)[:max_items] for pkg in packages: meta_file = pkg / "evidence.json" if meta_file.exists(): try: meta = json.loads(meta_file.read_text()) outputs.append({ "type": "evidence", "id": meta.get("package_id"), "action": meta.get("action_type"), "success": meta.get("results", {}).get("execution_success"), "timestamp": meta.get("created_at") }) except: pass return outputs def collect_directory_statuses(self) -> tuple[list[DirectoryStatusEntry], dict]: """ Collect status from all directory STATUS.md files. Returns (list of entries, summary dict by phase). """ import re PROJECT_ROOT = Path("/opt/agent-governance") SKIP_DIRS = {"__pycache__", "node_modules", ".git", "logs", "storage", "dragonfly-data", "credentials", "workspace", ".claude"} entries = [] summary = { "complete": 0, "in_progress": 0, "blocked": 0, "needs_review": 0, "not_started": 0, "total": 0 } def should_skip(dir_path: Path) -> bool: name = dir_path.name return name.startswith(".") or name in SKIP_DIRS or "evd-" in name def parse_phase(content: str) -> str: content_lower = content.lower() if "complete" in content_lower and "in_progress" not in content_lower: return "complete" if "in_progress" in content_lower or "in progress" in content_lower: return "in_progress" if "blocked" in content_lower: return "blocked" if "needs_review" in content_lower or "needs review" in content_lower: return "needs_review" return "not_started" def parse_timestamp(content: str) -> Optional[str]: pattern = r'(\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2})' matches = re.findall(pattern, content) return matches[-1] if matches else None def count_tasks(content: str) -> tuple[int, int]: total = 0 done = 0 pattern = r'\|\s*([x✓☐✗ -])\s*\|' for match in re.finditer(pattern, content, re.IGNORECASE): char = match.group(1).strip() if char and char not in ['-', '']: total += 1 if char.lower() in ['x', '✓']: done += 1 return total, done # Walk directory tree for root, subdirs, files in os.walk(PROJECT_ROOT): root_path = Path(root) subdirs[:] = [d for d in subdirs if not should_skip(root_path / d)] if should_skip(root_path): continue rel_path = str(root_path.relative_to(PROJECT_ROOT)) if rel_path == ".": rel_path = "." readme_path = root_path / "README.md" status_path = root_path / "STATUS.md" entry = DirectoryStatusEntry( path=rel_path, phase="not_started", has_readme=readme_path.exists(), has_status=status_path.exists() ) if status_path.exists(): try: content = status_path.read_text() entry.phase = parse_phase(content) entry.last_updated = parse_timestamp(content) entry.tasks_total, entry.tasks_done = count_tasks(content) except: pass entries.append(entry) summary[entry.phase] = summary.get(entry.phase, 0) + 1 summary["total"] += 1 return entries, summary def collect_memory_refs(self, limit: int = 20) -> tuple[list, dict]: """ Collect references to recent memory entries. Returns (list of MemoryRef, summary dict). """ refs = [] summary = {"total_entries": 0, "total_tokens": 0, "by_type": {}} try: # Import memory manager sys.path.insert(0, "/opt/agent-governance/memory") from memory import MemoryManager, MemoryStatus manager = MemoryManager() entries = manager.list_entries(status=MemoryStatus.ACTIVE, limit=limit) for entry in entries: refs.append(MemoryRef( id=entry.id, type=entry.type.value if hasattr(entry.type, 'value') else entry.type, summary=entry.summary or "(no summary)", tokens=entry.tokens_estimate )) summary["total_entries"] += 1 summary["total_tokens"] += entry.tokens_estimate entry_type = entry.type.value if hasattr(entry.type, 'value') else entry.type summary["by_type"][entry_type] = summary["by_type"].get(entry_type, 0) + 1 except Exception as e: # Memory layer not available - return empty pass return refs, summary # ------------------------------------------------------------------------- # Checkpoint Operations # ------------------------------------------------------------------------- def create_checkpoint( self, session_id: Optional[str] = None, variables: dict = None, notes: str = "", include_orchestration: bool = True, include_directory_status: bool = True, include_memory: bool = True ) -> ContextCheckpoint: """Create a new checkpoint capturing current state""" # Get previous checkpoint for parent reference previous = self.get_latest_checkpoint() # Collect state phase = self.collect_phase_state() tasks = self.collect_tasks_from_db() or self.collect_tasks_from_redis() dependencies = self.collect_dependencies() agent_id, agent_tier = self.collect_agent_state() recent_outputs = self.collect_recent_outputs() # Collect orchestration state if enabled orchestration_mode = "disabled" pending_instructions = [] if include_orchestration: orchestration_mode, pending_instructions = self.collect_orchestration_state() # Collect directory statuses (integrated with status system) directory_statuses = [] status_summary = {} if include_directory_status: directory_statuses, status_summary = self.collect_directory_statuses() # Collect memory references (links to external memory) memory_refs = [] memory_summary = {} if include_memory: memory_refs, memory_summary = self.collect_memory_refs() # Build checkpoint checkpoint = ContextCheckpoint( checkpoint_id=self._generate_id(), created_at=self._now(), session_id=session_id or os.environ.get("SESSION_ID"), phase=phase, phases_completed=[1, 2, 3, 4] if phase and phase.number == 5 else [], tasks=tasks, dependencies=dependencies, variables=variables or {}, recent_outputs=recent_outputs, agent_id=agent_id, agent_tier=agent_tier, parent_checkpoint_id=previous.checkpoint_id if previous else None, orchestration_mode=orchestration_mode, pending_instructions=pending_instructions, directory_statuses=directory_statuses, status_summary=status_summary, memory_refs=memory_refs, memory_summary=memory_summary ) # Add notes to phase if provided if notes and checkpoint.phase: checkpoint.phase.notes = notes # Compute hash and token estimate data = checkpoint.to_dict() checkpoint.content_hash = self._compute_hash(data) checkpoint.estimated_tokens = self._estimate_tokens(data) # Save checkpoint self.save_checkpoint(checkpoint) # Store in Redis for fast access if self.redis: self.redis.set("checkpoint:latest", json.dumps(checkpoint.to_dict())) self.redis.set(f"checkpoint:{checkpoint.checkpoint_id}", json.dumps(checkpoint.to_dict())) return checkpoint def save_checkpoint(self, checkpoint: ContextCheckpoint) -> Path: """Save checkpoint to disk""" filename = f"{checkpoint.checkpoint_id}.json" filepath = self.storage_dir / filename with open(filepath, "w") as f: json.dump(checkpoint.to_dict(), f, indent=2) # Prune old checkpoints self.prune_checkpoints() return filepath def load_checkpoint(self, checkpoint_id: str) -> Optional[ContextCheckpoint]: """Load a specific checkpoint""" # Try Redis first if self.redis: data = self.redis.get(f"checkpoint:{checkpoint_id}") if data: return ContextCheckpoint.from_dict(json.loads(data)) # Fall back to disk filepath = self.storage_dir / f"{checkpoint_id}.json" if filepath.exists(): with open(filepath) as f: return ContextCheckpoint.from_dict(json.load(f)) return None def get_latest_checkpoint(self) -> Optional[ContextCheckpoint]: """Get the most recent checkpoint""" # Try Redis first if self.redis: data = self.redis.get("checkpoint:latest") if data: return ContextCheckpoint.from_dict(json.loads(data)) # Fall back to disk files = sorted(self.storage_dir.glob("ckpt-*.json"), reverse=True) if files: with open(files[0]) as f: return ContextCheckpoint.from_dict(json.load(f)) return None def list_checkpoints(self, limit: int = 20) -> list[dict]: """List available checkpoints""" checkpoints = [] files = sorted(self.storage_dir.glob("ckpt-*.json"), reverse=True)[:limit] for f in files: try: with open(f) as fp: data = json.load(fp) checkpoints.append({ "id": data["checkpoint_id"], "created_at": data["created_at"], "phase": data.get("phase", {}).get("name") if data.get("phase") else None, "tasks": len(data.get("tasks", [])), "tokens": data.get("estimated_tokens", 0) }) except: pass return checkpoints def prune_checkpoints(self, keep: int = MAX_CHECKPOINTS): """Remove old checkpoints beyond the limit""" files = sorted(self.storage_dir.glob("ckpt-*.json"), reverse=True) for f in files[keep:]: f.unlink() # ------------------------------------------------------------------------- # Diff Operations # ------------------------------------------------------------------------- def diff_checkpoints( self, checkpoint_a: ContextCheckpoint, checkpoint_b: ContextCheckpoint ) -> dict: """Compare two checkpoints and return differences""" diff = { "from_id": checkpoint_a.checkpoint_id, "to_id": checkpoint_b.checkpoint_id, "time_delta": None, "changes": [] } # Time delta try: from_time = datetime.fromisoformat(checkpoint_a.created_at.replace("Z", "+00:00")) to_time = datetime.fromisoformat(checkpoint_b.created_at.replace("Z", "+00:00")) diff["time_delta"] = str(to_time - from_time) except: pass # Phase change if checkpoint_a.phase and checkpoint_b.phase: if checkpoint_a.phase.number != checkpoint_b.phase.number: diff["changes"].append({ "type": "phase_change", "from": f"Phase {checkpoint_a.phase.number}", "to": f"Phase {checkpoint_b.phase.number}" }) elif checkpoint_a.phase.status != checkpoint_b.phase.status: diff["changes"].append({ "type": "phase_status", "from": checkpoint_a.phase.status, "to": checkpoint_b.phase.status }) # Task changes old_tasks = {t.id: t for t in checkpoint_a.tasks} new_tasks = {t.id: t for t in checkpoint_b.tasks} for task_id, task in new_tasks.items(): if task_id not in old_tasks: diff["changes"].append({ "type": "task_added", "task_id": task_id, "subject": task.subject }) elif old_tasks[task_id].status != task.status: diff["changes"].append({ "type": "task_status", "task_id": task_id, "from": old_tasks[task_id].status, "to": task.status }) for task_id in old_tasks: if task_id not in new_tasks: diff["changes"].append({ "type": "task_removed", "task_id": task_id }) # Dependency changes old_deps = {d.name: d.status for d in checkpoint_a.dependencies} new_deps = {d.name: d.status for d in checkpoint_b.dependencies} for name, status in new_deps.items(): if name not in old_deps: diff["changes"].append({ "type": "dependency_added", "name": name, "status": status }) elif old_deps[name] != status: diff["changes"].append({ "type": "dependency_status", "name": name, "from": old_deps[name], "to": status }) # Variable changes old_vars = checkpoint_a.variables new_vars = checkpoint_b.variables for key in set(list(old_vars.keys()) + list(new_vars.keys())): if key not in old_vars: diff["changes"].append({ "type": "variable_added", "key": key }) elif key not in new_vars: diff["changes"].append({ "type": "variable_removed", "key": key }) elif old_vars[key] != new_vars[key]: diff["changes"].append({ "type": "variable_changed", "key": key }) # Content hash change diff["content_changed"] = checkpoint_a.content_hash != checkpoint_b.content_hash return diff # ============================================================================= # Token-Aware Context Summarizer # ============================================================================= class ContextSummarizer: """ Creates minimal context summaries for sub-agent calls. Reduces token usage while preserving essential information. """ # Token budgets for different summary levels BUDGET_MINIMAL = 500 BUDGET_COMPACT = 1000 BUDGET_STANDARD = 2000 BUDGET_FULL = 4000 def __init__(self, checkpoint: ContextCheckpoint): self.checkpoint = checkpoint def _estimate_tokens(self, text: str) -> int: return len(text) // 4 def minimal_summary(self) -> str: """Absolute minimum context (~500 tokens)""" lines = [] # Phase if self.checkpoint.phase: lines.append(f"Phase: {self.checkpoint.phase.name} ({self.checkpoint.phase.status})") # Active task if self.checkpoint.active_task_id: task = next((t for t in self.checkpoint.tasks if t.id == self.checkpoint.active_task_id), None) if task: lines.append(f"Active Task: {task.subject}") # Agent if self.checkpoint.agent_id: lines.append(f"Agent: {self.checkpoint.agent_id} (Tier {self.checkpoint.agent_tier})") # Key deps available = [d.name for d in self.checkpoint.dependencies if d.status == "available"] if available: lines.append(f"Available: {', '.join(available)}") return "\n".join(lines) def compact_summary(self) -> str: """Compact context (~1000 tokens)""" lines = [self.minimal_summary(), ""] # Recent tasks pending = [t for t in self.checkpoint.tasks if t.status == "pending"][:5] in_progress = [t for t in self.checkpoint.tasks if t.status == "in_progress"] if in_progress: lines.append("In Progress:") for t in in_progress: lines.append(f" - {t.subject}") if pending: lines.append(f"Pending Tasks: {len(pending)}") # Key variables if self.checkpoint.variables: lines.append("\nKey Variables:") for k, v in list(self.checkpoint.variables.items())[:5]: lines.append(f" {k}: {str(v)[:50]}") return "\n".join(lines) def standard_summary(self) -> str: """Standard context (~2000 tokens)""" lines = [self.compact_summary(), ""] # All tasks with status lines.append("\nAll Tasks:") for t in self.checkpoint.tasks[:15]: status_icon = {"pending": "[ ]", "in_progress": "[>]", "completed": "[x]"}.get(t.status, "[?]") lines.append(f" {status_icon} {t.subject}") # Dependencies lines.append("\nDependencies:") for d in self.checkpoint.dependencies: status_icon = {"available": "+", "unavailable": "-", "degraded": "~"}.get(d.status, "?") lines.append(f" [{status_icon}] {d.name}: {d.endpoint or 'N/A'}") # Recent outputs if self.checkpoint.recent_outputs: lines.append("\nRecent Outputs:") for o in self.checkpoint.recent_outputs[:5]: result = "OK" if o.get("success") else "FAIL" lines.append(f" - {o.get('action', 'unknown')} [{result}]") return "\n".join(lines) def full_summary(self) -> str: """Full context for complex operations (~4000 tokens)""" lines = [self.standard_summary(), ""] # All variables lines.append("\nAll Variables:") for k, v in self.checkpoint.variables.items(): lines.append(f" {k}: {json.dumps(v)[:100]}") # Phases completed if self.checkpoint.phases_completed: lines.append(f"\nCompleted Phases: {self.checkpoint.phases_completed}") # Checkpoint metadata lines.append(f"\nCheckpoint: {self.checkpoint.checkpoint_id}") lines.append(f"Created: {self.checkpoint.created_at}") lines.append(f"Token Estimate: {self.checkpoint.estimated_tokens}") return "\n".join(lines) def for_subagent( self, task_type: str, relevant_keys: list[str] = None, max_tokens: int = BUDGET_COMPACT ) -> str: """ Generate task-specific context for a sub-agent. Only includes information relevant to the task type. """ lines = [] # Always include phase and agent if self.checkpoint.phase: lines.append(f"Phase: {self.checkpoint.phase.name}") if self.checkpoint.agent_id: lines.append(f"Agent: {self.checkpoint.agent_id} (T{self.checkpoint.agent_tier})") # Task-specific context if task_type in ["terraform", "ansible", "infrastructure"]: # Include relevant deps infra_deps = [d for d in self.checkpoint.dependencies if d.type in ["service", "api"]] if infra_deps: lines.append("\nInfrastructure:") for d in infra_deps: lines.append(f" {d.name}: {d.status}") elif task_type in ["database", "query", "ledger"]: # Include database deps db_deps = [d for d in self.checkpoint.dependencies if d.type == "database"] if db_deps: lines.append("\nDatabases:") for d in db_deps: lines.append(f" {d.name}: {d.endpoint}") elif task_type in ["promotion", "revocation", "governance"]: # Include agent tier info lines.append(f"\nGovernance Context:") lines.append(f" Current Tier: {self.checkpoint.agent_tier}") # Include relevant variables gov_vars = {k: v for k, v in self.checkpoint.variables.items() if any(x in k.lower() for x in ["tier", "promotion", "violation"])} if gov_vars: for k, v in gov_vars.items(): lines.append(f" {k}: {v}") # Include requested variables if relevant_keys: lines.append("\nRelevant Variables:") for key in relevant_keys: if key in self.checkpoint.variables: lines.append(f" {key}: {self.checkpoint.variables[key]}") result = "\n".join(lines) # Truncate if over budget while self._estimate_tokens(result) > max_tokens and lines: lines.pop() result = "\n".join(lines) return result # ============================================================================= # CLI Interface # ============================================================================= def cli(): import argparse parser = argparse.ArgumentParser( description="Context Checkpoint Skill", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: checkpoint now Create checkpoint checkpoint now --notes "Phase 5" Create with notes checkpoint load Load latest checkpoint checkpoint load ckpt-20260123-... Load specific checkpoint checkpoint diff Compare latest vs previous checkpoint list List all checkpoints checkpoint summary Show context summary checkpoint summary --level full Full context summary checkpoint prune --keep 10 Keep only 10 checkpoints """ ) subparsers = parser.add_subparsers(dest="command", required=True) # now now_parser = subparsers.add_parser("now", help="Create checkpoint now") now_parser.add_argument("--notes", help="Add notes to checkpoint") now_parser.add_argument("--var", action="append", nargs=2, metavar=("KEY", "VALUE"), help="Add variable (can repeat)") now_parser.add_argument("--json", action="store_true", help="Output JSON") # load load_parser = subparsers.add_parser("load", help="Load checkpoint") load_parser.add_argument("checkpoint_id", nargs="?", help="Checkpoint ID (default: latest)") load_parser.add_argument("--json", action="store_true", help="Output JSON") # diff diff_parser = subparsers.add_parser("diff", help="Compare checkpoints") diff_parser.add_argument("--from", dest="from_id", help="From checkpoint ID") diff_parser.add_argument("--to", dest="to_id", help="To checkpoint ID") diff_parser.add_argument("--json", action="store_true", help="Output JSON") # list list_parser = subparsers.add_parser("list", help="List checkpoints") list_parser.add_argument("--limit", type=int, default=20) list_parser.add_argument("--json", action="store_true", help="Output JSON") # summary summary_parser = subparsers.add_parser("summary", help="Show context summary") summary_parser.add_argument("--level", choices=["minimal", "compact", "standard", "full"], default="compact") summary_parser.add_argument("--for", dest="task_type", help="Task-specific summary") # prune prune_parser = subparsers.add_parser("prune", help="Remove old checkpoints") prune_parser.add_argument("--keep", type=int, default=MAX_CHECKPOINTS) # report (new - integrated checkpoint + status view) report_parser = subparsers.add_parser("report", help="Show checkpoint + directory status report") report_parser.add_argument("--checkpoint", help="Checkpoint ID (default: latest)") report_parser.add_argument("--phase", help="Filter by status phase") report_parser.add_argument("--json", action="store_true", help="Output JSON") # timeline (new - show checkpoint history with status changes) timeline_parser = subparsers.add_parser("timeline", help="Show checkpoint timeline with status changes") timeline_parser.add_argument("--limit", type=int, default=10, help="Number of checkpoints") timeline_parser.add_argument("--json", action="store_true", help="Output JSON") # auto-orchestrate auto_parser = subparsers.add_parser("auto-orchestrate", help="Run in automated orchestration mode") auto_parser.add_argument("--model", choices=["minimax", "gemini", "gemini-pro"], default="minimax", help="Model to use") auto_parser.add_argument("--instruction", "-i", action="append", help="Instruction to execute (can repeat)") auto_parser.add_argument("--confirm", action="store_true", help="Require confirmation before execution") auto_parser.add_argument("--dry-run", action="store_true", help="Show what would be executed without running") # queue queue_parser = subparsers.add_parser("queue", help="Manage instruction queue") queue_parser.add_argument("action", choices=["list", "add", "clear", "pop"]) queue_parser.add_argument("--instruction", help="Instruction to add") queue_parser.add_argument("--type", default="shell", help="Command type") queue_parser.add_argument("--priority", type=int, default=0, help="Priority (higher = first)") args = parser.parse_args() manager = CheckpointManager() # ------------------------------------------------------------------------- if args.command == "now": variables = {} if args.var: for key, value in args.var: variables[key] = value checkpoint = manager.create_checkpoint( variables=variables, notes=args.notes or "" ) if args.json: print(json.dumps(checkpoint.to_dict(), indent=2)) else: print(f"\n{'='*60}") print("CHECKPOINT CREATED") print(f"{'='*60}") print(f"ID: {checkpoint.checkpoint_id}") print(f"Time: {checkpoint.created_at}") if checkpoint.phase: print(f"Phase: {checkpoint.phase.name}") print(f"Tasks: {len(checkpoint.tasks)}") print(f"Dependencies: {len(checkpoint.dependencies)}") print(f"Est. Tokens: {checkpoint.estimated_tokens}") print(f"Hash: {checkpoint.content_hash}") print(f"{'='*60}") elif args.command == "load": if args.checkpoint_id: checkpoint = manager.load_checkpoint(args.checkpoint_id) else: checkpoint = manager.get_latest_checkpoint() if not checkpoint: print("No checkpoint found") sys.exit(1) if args.json: print(json.dumps(checkpoint.to_dict(), indent=2)) else: print(f"\n{'='*60}") print("CHECKPOINT LOADED") print(f"{'='*60}") print(f"ID: {checkpoint.checkpoint_id}") print(f"Created: {checkpoint.created_at}") if checkpoint.phase: print(f"\nPhase: {checkpoint.phase.name}") print(f"Status: {checkpoint.phase.status}") if checkpoint.tasks: print(f"\nTasks ({len(checkpoint.tasks)}):") for t in checkpoint.tasks[:10]: status_icon = {"pending": "[ ]", "in_progress": "[>]", "completed": "[x]"}.get(t.status, "[?]") print(f" {status_icon} {t.subject}") if len(checkpoint.tasks) > 10: print(f" ... and {len(checkpoint.tasks) - 10} more") if checkpoint.dependencies: print(f"\nDependencies:") for d in checkpoint.dependencies: icon = {"available": "+", "unavailable": "-"}.get(d.status, "?") print(f" [{icon}] {d.name}") print(f"\nTokens: {checkpoint.estimated_tokens}") print(f"{'='*60}") elif args.command == "diff": if args.from_id and args.to_id: checkpoint_a = manager.load_checkpoint(args.from_id) checkpoint_b = manager.load_checkpoint(args.to_id) else: # Compare latest with previous checkpoints = manager.list_checkpoints(limit=2) if len(checkpoints) < 2: print("Need at least 2 checkpoints to diff") sys.exit(1) checkpoint_b = manager.load_checkpoint(checkpoints[0]["id"]) checkpoint_a = manager.load_checkpoint(checkpoints[1]["id"]) if not checkpoint_a or not checkpoint_b: print("Could not load checkpoints") sys.exit(1) diff = manager.diff_checkpoints(checkpoint_a, checkpoint_b) if args.json: print(json.dumps(diff, indent=2)) else: print(f"\n{'='*60}") print("CHECKPOINT DIFF") print(f"{'='*60}") print(f"From: {diff['from_id']}") print(f"To: {diff['to_id']}") if diff['time_delta']: print(f"Time: {diff['time_delta']}") print(f"\nChanges ({len(diff['changes'])}):") if not diff['changes']: print(" No changes detected") else: for change in diff['changes']: ctype = change['type'] if ctype == "phase_change": print(f" [PHASE] {change['from']} -> {change['to']}") elif ctype == "task_added": print(f" [+TASK] {change['subject']}") elif ctype == "task_removed": print(f" [-TASK] {change['task_id']}") elif ctype == "task_status": print(f" [TASK] {change['task_id']}: {change['from']} -> {change['to']}") elif ctype == "dependency_status": print(f" [DEP] {change['name']}: {change['from']} -> {change['to']}") elif ctype == "variable_added": print(f" [+VAR] {change['key']}") elif ctype == "variable_changed": print(f" [VAR] {change['key']} changed") else: print(f" [{ctype}] {change}") print(f"\nContent Changed: {diff['content_changed']}") print(f"{'='*60}") elif args.command == "list": checkpoints = manager.list_checkpoints(limit=args.limit) if args.json: print(json.dumps(checkpoints, indent=2)) else: print(f"\n{'='*60}") print("CHECKPOINTS") print(f"{'='*60}") if not checkpoints: print("No checkpoints found") else: for ckpt in checkpoints: phase = ckpt['phase'] or "N/A" print(f"\n {ckpt['id']}") print(f" Created: {ckpt['created_at']}") print(f" Phase: {phase}") print(f" Tasks: {ckpt['tasks']}, Tokens: {ckpt['tokens']}") print(f"{'='*60}") elif args.command == "summary": checkpoint = manager.get_latest_checkpoint() if not checkpoint: print("No checkpoint found. Run 'checkpoint now' first.") sys.exit(1) summarizer = ContextSummarizer(checkpoint) if args.task_type: summary = summarizer.for_subagent(args.task_type) elif args.level == "minimal": summary = summarizer.minimal_summary() elif args.level == "compact": summary = summarizer.compact_summary() elif args.level == "standard": summary = summarizer.standard_summary() else: summary = summarizer.full_summary() print(summary) elif args.command == "prune": manager.prune_checkpoints(keep=args.keep) print(f"Pruned checkpoints, keeping last {args.keep}") elif args.command == "report": # Load checkpoint if args.checkpoint: checkpoint = manager.load_checkpoint(args.checkpoint) else: checkpoint = manager.get_latest_checkpoint() if not checkpoint: print("No checkpoint found. Run 'checkpoint now' first.") sys.exit(1) if args.json: report_data = { "checkpoint_id": checkpoint.checkpoint_id, "created_at": checkpoint.created_at, "phase": asdict(checkpoint.phase) if checkpoint.phase else None, "status_summary": checkpoint.status_summary, "directory_statuses": [asdict(d) if isinstance(d, DirectoryStatusEntry) else d for d in checkpoint.directory_statuses], "dependencies": [asdict(d) if isinstance(d, Dependency) else d for d in checkpoint.dependencies] } print(json.dumps(report_data, indent=2)) else: print(f"\n{'='*70}") print("CHECKPOINT + DIRECTORY STATUS REPORT") print(f"{'='*70}") # Checkpoint info print(f"\n[CHECKPOINT]") print(f" ID: {checkpoint.checkpoint_id}") print(f" Created: {checkpoint.created_at}") if checkpoint.phase: print(f" Phase: {checkpoint.phase.name}") if checkpoint.phase.notes: print(f" Notes: {checkpoint.phase.notes}") # Dependencies print(f"\n[DEPENDENCIES]") for d in checkpoint.dependencies: icon = {"available": "✓", "unavailable": "✗", "degraded": "~"}.get(d.status, "?") print(f" {icon} {d.name}: {d.endpoint or 'N/A'}") # Status summary summary = checkpoint.status_summary if summary: total = summary.get("total", 0) complete = summary.get("complete", 0) pct = (complete / total * 100) if total > 0 else 0 print(f"\n[DIRECTORY STATUS SUMMARY]") bar_width = 30 filled = int(bar_width * complete / total) if total > 0 else 0 bar = "█" * filled + "░" * (bar_width - filled) print(f" Progress: [{bar}] {pct:.1f}%") print(f" ✅ Complete: {summary.get('complete', 0):3d}") print(f" 🚧 In Progress: {summary.get('in_progress', 0):3d}") print(f" ❗ Blocked: {summary.get('blocked', 0):3d}") print(f" ⚠️ Needs Review: {summary.get('needs_review', 0):3d}") print(f" ⬜ Not Started: {summary.get('not_started', 0):3d}") # Active directories (non-complete) active_dirs = [d for d in checkpoint.directory_statuses if d.phase in ("in_progress", "blocked", "needs_review")] # Filter by phase if requested if args.phase: active_dirs = [d for d in checkpoint.directory_statuses if d.phase == args.phase] print(f"\n[DIRECTORIES: {args.phase.upper()}]") elif active_dirs: print(f"\n[ACTIVE DIRECTORIES]") phase_icons = { "complete": "✅", "in_progress": "🚧", "blocked": "❗", "needs_review": "⚠️", "not_started": "⬜" } for d in active_dirs[:20]: icon = phase_icons.get(d.phase, "?") tasks_str = f" [{d.tasks_done}/{d.tasks_total} tasks]" if d.tasks_total > 0 else "" updated = f" (updated: {d.last_updated[:10]})" if d.last_updated else "" print(f" {icon} {d.path}{tasks_str}{updated}") if len(active_dirs) > 20: print(f" ... and {len(active_dirs) - 20} more") print(f"\n{'='*70}") elif args.command == "timeline": checkpoints = manager.list_checkpoints(limit=args.limit) if args.json: # Load full details for each checkpoint timeline_data = [] for ckpt in checkpoints: full_ckpt = manager.load_checkpoint(ckpt["id"]) if full_ckpt: timeline_data.append({ "checkpoint_id": full_ckpt.checkpoint_id, "created_at": full_ckpt.created_at, "phase_notes": full_ckpt.phase.notes if full_ckpt.phase else None, "status_summary": full_ckpt.status_summary }) print(json.dumps(timeline_data, indent=2)) else: print(f"\n{'='*70}") print("CHECKPOINT TIMELINE") print(f"{'='*70}") if not checkpoints: print("No checkpoints found") else: for i, ckpt in enumerate(checkpoints): full_ckpt = manager.load_checkpoint(ckpt["id"]) if not full_ckpt: continue # Timeline marker marker = "●" if i == 0 else "○" print(f"\n{marker} {full_ckpt.created_at}") print(f" ID: {full_ckpt.checkpoint_id}") # Phase notes if full_ckpt.phase and full_ckpt.phase.notes: notes = full_ckpt.phase.notes[:60] if len(full_ckpt.phase.notes) > 60: notes += "..." print(f" Notes: {notes}") # Status summary summary = full_ckpt.status_summary if summary: complete = summary.get("complete", 0) total = summary.get("total", 0) in_prog = summary.get("in_progress", 0) print(f" Status: {complete}/{total} complete, {in_prog} in progress") # Show connector for non-last items if i < len(checkpoints) - 1: print(" │") print(f"\n{'='*70}") elif args.command == "auto-orchestrate": # Import model controller try: sys.path.insert(0, str(ORCHESTRATOR_DIR)) from model_controller import OrchestrationManager, OrchestrationMode except ImportError as e: print(f"Error: Could not import model_controller: {e}") print("Make sure /opt/agent-governance/orchestrator/model_controller.py exists") sys.exit(1) orchestrator = OrchestrationManager() # Enable the selected model if not orchestrator.set_mode(args.model): print(f"Error: Could not enable {args.model} mode") sys.exit(1) print(f"\n{'='*60}") print(f"AUTO-ORCHESTRATE MODE: {args.model.upper()}") print(f"{'='*60}") if args.dry_run: print("[DRY RUN - No commands will be executed]") print() # Create pre-run checkpoint checkpoint = manager.create_checkpoint( notes=f"Auto-orchestrate session started with {args.model}" ) print(f"Checkpoint created: {checkpoint.checkpoint_id}") # Execute instructions if args.instruction: print(f"\nExecuting {len(args.instruction)} instruction(s):") for i, instruction in enumerate(args.instruction, 1): print(f"\n[{i}] {instruction}") if args.dry_run: print(" [SKIPPED - Dry run]") continue if args.confirm: confirm = input(" Execute? [y/N]: ") if confirm.lower() != 'y': print(" [SKIPPED - User declined]") continue # Delegate to model response = orchestrator.delegate_command( instruction, command_type="shell", require_confirmation=args.confirm ) if response.success: print(f" [OK] Model: {response.model_used}, Tokens: {response.tokens_used}") print(f" Output: {response.output[:200]}...") else: print(f" [FAILED] {response.error}") # Create post-run checkpoint final_checkpoint = manager.create_checkpoint( notes=f"Auto-orchestrate session completed" ) print(f"\nFinal checkpoint: {final_checkpoint.checkpoint_id}") print(f"{'='*60}") elif args.command == "queue": if args.action == "list": if manager.redis: items = manager.redis.lrange("orchestration:instructions", 0, -1) print(f"\n{'='*60}") print("INSTRUCTION QUEUE") print(f"{'='*60}") if not items: print("Queue is empty") else: for i, item in enumerate(items, 1): data = json.loads(item) print(f"\n[{i}] {data.get('instruction', 'N/A')}") print(f" Type: {data.get('command_type', 'shell')}") print(f" Priority: {data.get('priority', 0)}") print(f"{'='*60}") else: print("Error: DragonflyDB not available") elif args.action == "add": if not args.instruction: print("Error: --instruction is required") sys.exit(1) if manager.redis: entry = { "id": hashlib.sha256(f"{time.time()}-{args.instruction}".encode()).hexdigest()[:12], "instruction": args.instruction, "command_type": args.type, "priority": args.priority, "created_at": datetime.now(timezone.utc).isoformat() } manager.redis.lpush("orchestration:instructions", json.dumps(entry)) print(f"Added to queue: {args.instruction}") else: print("Error: DragonflyDB not available") elif args.action == "clear": if manager.redis: manager.redis.delete("orchestration:instructions") print("Queue cleared") else: print("Error: DragonflyDB not available") elif args.action == "pop": if manager.redis: item = manager.redis.rpop("orchestration:instructions") if item: data = json.loads(item) print(f"Popped: {data.get('instruction', 'N/A')}") else: print("Queue is empty") else: print("Error: DragonflyDB not available") if __name__ == "__main__": cli()