Phase 8 Production Hardening with complete governance infrastructure: - Vault integration with tiered policies (T0-T4) - DragonflyDB state management - SQLite audit ledger - Pipeline DSL and templates - Promotion/revocation engine - Checkpoint system for session persistence - Health manager and circuit breaker for fault tolerance - GitHub/Slack integrations - Architectural test pipeline with bug watcher, suggestion engine, council review - Multi-agent chaos testing framework Test Results: - Governance tests: 68/68 passing - E2E workflow: 16/16 passing - Phase 2 Vault: 14/14 passing - Integration tests: 27/27 passing Coverage: 57.6% average across 12 phases Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1010 lines
34 KiB
Python
Executable File
1010 lines
34 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Model Controller - Automated Orchestration Layer
|
|
=================================================
|
|
Delegates CLI commands to Minimax/Gemini via OpenRouter for "humanless mode".
|
|
|
|
Features:
|
|
- Model selection (Minimax or Gemini)
|
|
- Command delegation with context
|
|
- Checkpoint integration (before/after each action)
|
|
- Fallback behavior on model failure
|
|
- Full audit logging
|
|
|
|
Models:
|
|
- Minimax: minimax/minimax-01 - High capability, cost-effective
|
|
- Gemini: google/gemini-2.0-flash-thinking-exp-1219 - Fast, experimental thinking
|
|
- Gemini Pro: google/gemini-2.5-pro-preview-03-25 - Highest capability
|
|
|
|
Part of Phase 5: Agent Bootstrapping - Automated Checkpoint & Self-Healing Pipeline
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
import time
|
|
import hashlib
|
|
import sqlite3
|
|
import subprocess
|
|
import logging
|
|
from dataclasses import dataclass, asdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Optional, Any
|
|
from enum import Enum
|
|
|
|
try:
|
|
import requests
|
|
except ImportError:
|
|
requests = None
|
|
|
|
try:
|
|
import redis
|
|
except ImportError:
|
|
redis = None
|
|
|
|
|
|
# =============================================================================
|
|
# Configuration
|
|
# =============================================================================
|
|
|
|
OPENROUTER_API_URL = "https://openrouter.ai/api/v1/chat/completions"
|
|
LEDGER_DB = Path("/opt/agent-governance/ledger/governance.db")
|
|
CHECKPOINT_DIR = Path("/opt/agent-governance/checkpoint/storage")
|
|
LOG_DIR = Path("/opt/agent-governance/orchestrator/logs")
|
|
|
|
# Model configurations
|
|
MODELS = {
|
|
"minimax": {
|
|
"model_id": "minimax/minimax-01",
|
|
"display_name": "Minimax-01",
|
|
"max_tokens": 100000,
|
|
"cost_per_1k_input": 0.0004,
|
|
"cost_per_1k_output": 0.0016,
|
|
"capabilities": ["code", "reasoning", "long_context"],
|
|
"fallback": "gemini"
|
|
},
|
|
"gemini": {
|
|
"model_id": "google/gemini-2.0-flash-thinking-exp-1219",
|
|
"display_name": "Gemini 2.0 Flash Thinking",
|
|
"max_tokens": 65536,
|
|
"cost_per_1k_input": 0.0, # Free tier
|
|
"cost_per_1k_output": 0.0,
|
|
"capabilities": ["code", "reasoning", "thinking"],
|
|
"fallback": "gemini-pro"
|
|
},
|
|
"gemini-pro": {
|
|
"model_id": "google/gemini-2.5-pro-preview-03-25",
|
|
"display_name": "Gemini 2.5 Pro",
|
|
"max_tokens": 65536,
|
|
"cost_per_1k_input": 0.00125,
|
|
"cost_per_1k_output": 0.01,
|
|
"capabilities": ["code", "reasoning", "complex_tasks"],
|
|
"fallback": "minimax"
|
|
}
|
|
}
|
|
|
|
DEFAULT_MODEL = "minimax"
|
|
|
|
|
|
# =============================================================================
|
|
# Enums
|
|
# =============================================================================
|
|
|
|
class OrchestrationMode(Enum):
|
|
DISABLED = "disabled"
|
|
MINIMAX = "minimax"
|
|
GEMINI = "gemini"
|
|
GEMINI_PRO = "gemini-pro"
|
|
|
|
|
|
class CommandType(Enum):
|
|
CHECKPOINT = "checkpoint"
|
|
SHELL = "shell"
|
|
PLAN = "plan"
|
|
EXECUTE = "execute"
|
|
REVIEW = "review"
|
|
|
|
|
|
# =============================================================================
|
|
# Data Classes
|
|
# =============================================================================
|
|
|
|
@dataclass
|
|
class OrchestrationConfig:
|
|
"""Configuration for automated orchestration"""
|
|
mode: OrchestrationMode = OrchestrationMode.DISABLED
|
|
model_override: Optional[str] = None
|
|
api_key: Optional[str] = None
|
|
max_retries: int = 3
|
|
checkpoint_before_action: bool = True
|
|
checkpoint_after_action: bool = True
|
|
fallback_to_human: bool = True
|
|
max_tokens_per_request: int = 4096
|
|
timeout_seconds: int = 120
|
|
|
|
@classmethod
|
|
def from_env(cls) -> 'OrchestrationConfig':
|
|
"""Load configuration from environment variables"""
|
|
mode_str = os.environ.get("AUTO_AGENT_MODE", "disabled").lower()
|
|
|
|
try:
|
|
mode = OrchestrationMode(mode_str)
|
|
except ValueError:
|
|
mode = OrchestrationMode.DISABLED
|
|
|
|
return cls(
|
|
mode=mode,
|
|
model_override=os.environ.get("OPENROUTER_MODEL_OVERRIDE"),
|
|
api_key=os.environ.get("OPENROUTER_API_KEY"),
|
|
max_retries=int(os.environ.get("AUTO_AGENT_RETRIES", "3")),
|
|
checkpoint_before_action=os.environ.get("AUTO_AGENT_CHECKPOINT_BEFORE", "true").lower() == "true",
|
|
checkpoint_after_action=os.environ.get("AUTO_AGENT_CHECKPOINT_AFTER", "true").lower() == "true",
|
|
fallback_to_human=os.environ.get("AUTO_AGENT_FALLBACK_HUMAN", "true").lower() == "true",
|
|
max_tokens_per_request=int(os.environ.get("AUTO_AGENT_MAX_TOKENS", "4096")),
|
|
timeout_seconds=int(os.environ.get("AUTO_AGENT_TIMEOUT", "120"))
|
|
)
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"mode": self.mode.value,
|
|
"model_override": self.model_override,
|
|
"max_retries": self.max_retries,
|
|
"checkpoint_before": self.checkpoint_before_action,
|
|
"checkpoint_after": self.checkpoint_after_action,
|
|
"fallback_to_human": self.fallback_to_human,
|
|
"max_tokens": self.max_tokens_per_request,
|
|
"timeout": self.timeout_seconds
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class OrchestrationContext:
|
|
"""Context passed to the model for command delegation"""
|
|
phase: str
|
|
phase_status: str
|
|
active_tasks: list
|
|
pending_tasks: list
|
|
constraints: dict
|
|
recent_outputs: list
|
|
agent_id: Optional[str] = None
|
|
agent_tier: int = 0
|
|
session_id: Optional[str] = None
|
|
pending_instructions: list = None
|
|
|
|
def to_prompt(self) -> str:
|
|
"""Convert to a prompt string for the model"""
|
|
lines = [
|
|
"# Current Session Context",
|
|
"",
|
|
f"## Phase: {self.phase}",
|
|
f"Status: {self.phase_status}",
|
|
""
|
|
]
|
|
|
|
if self.agent_id:
|
|
lines.append(f"## Agent: {self.agent_id} (Tier {self.agent_tier})")
|
|
lines.append("")
|
|
|
|
if self.active_tasks:
|
|
lines.append("## Active Tasks:")
|
|
for task in self.active_tasks[:5]:
|
|
lines.append(f"- [{task.get('status', 'pending')}] {task.get('subject', 'Unknown')}")
|
|
lines.append("")
|
|
|
|
if self.pending_tasks:
|
|
lines.append(f"## Pending Tasks: {len(self.pending_tasks)}")
|
|
for task in self.pending_tasks[:3]:
|
|
lines.append(f"- {task.get('subject', 'Unknown')}")
|
|
if len(self.pending_tasks) > 3:
|
|
lines.append(f"- ... and {len(self.pending_tasks) - 3} more")
|
|
lines.append("")
|
|
|
|
if self.constraints:
|
|
lines.append("## Constraints:")
|
|
for key, value in self.constraints.items():
|
|
lines.append(f"- {key}: {value}")
|
|
lines.append("")
|
|
|
|
if self.pending_instructions:
|
|
lines.append("## Pending Instructions:")
|
|
for instr in self.pending_instructions:
|
|
lines.append(f"- {instr}")
|
|
lines.append("")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
@dataclass
|
|
class CommandRequest:
|
|
"""Request to execute a command via the model"""
|
|
command_type: CommandType
|
|
command: str
|
|
context: OrchestrationContext
|
|
require_confirmation: bool = False
|
|
timeout: int = 120
|
|
|
|
|
|
@dataclass
|
|
class CommandResponse:
|
|
"""Response from model command execution"""
|
|
success: bool
|
|
output: str
|
|
model_used: str
|
|
tokens_used: int
|
|
execution_time: float
|
|
checkpoint_before_id: Optional[str] = None
|
|
checkpoint_after_id: Optional[str] = None
|
|
error: Optional[str] = None
|
|
fallback_triggered: bool = False
|
|
|
|
|
|
# =============================================================================
|
|
# Model Controller
|
|
# =============================================================================
|
|
|
|
class ModelController:
|
|
"""
|
|
Controls model selection and command delegation.
|
|
Integrates with checkpoint system for state management.
|
|
"""
|
|
|
|
def __init__(self, config: OrchestrationConfig = None):
|
|
self.config = config or OrchestrationConfig.from_env()
|
|
self.logger = self._setup_logging()
|
|
self.redis = self._get_redis()
|
|
self._session_id = None
|
|
self._command_count = 0
|
|
|
|
def _setup_logging(self) -> logging.Logger:
|
|
"""Set up logging for orchestration"""
|
|
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
logger = logging.getLogger("orchestrator")
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
# File handler
|
|
log_file = LOG_DIR / f"orchestrator-{datetime.now().strftime('%Y%m%d')}.log"
|
|
fh = logging.FileHandler(log_file)
|
|
fh.setLevel(logging.DEBUG)
|
|
fh.setFormatter(logging.Formatter(
|
|
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
|
))
|
|
logger.addHandler(fh)
|
|
|
|
# Console handler (only warnings and above)
|
|
ch = logging.StreamHandler()
|
|
ch.setLevel(logging.WARNING)
|
|
ch.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
|
|
logger.addHandler(ch)
|
|
|
|
return logger
|
|
|
|
def _get_redis(self):
|
|
"""Get DragonflyDB connection"""
|
|
if not redis:
|
|
return None
|
|
|
|
try:
|
|
with open("/opt/vault/init-keys.json") as f:
|
|
token = json.load(f)["root_token"]
|
|
|
|
result = subprocess.run([
|
|
"curl", "-sk",
|
|
"-H", f"X-Vault-Token: {token}",
|
|
"https://127.0.0.1:8200/v1/secret/data/services/dragonfly"
|
|
], capture_output=True, text=True)
|
|
|
|
creds = json.loads(result.stdout)["data"]["data"]
|
|
return redis.Redis(
|
|
host=creds["host"],
|
|
port=int(creds["port"]),
|
|
password=creds["password"],
|
|
decode_responses=True
|
|
)
|
|
except Exception as e:
|
|
self.logger.warning(f"Could not connect to DragonflyDB: {e}")
|
|
return None
|
|
|
|
def _get_api_key(self) -> Optional[str]:
|
|
"""Get OpenRouter API key from config or Vault"""
|
|
if self.config.api_key:
|
|
return self.config.api_key
|
|
|
|
# Try to get from Vault
|
|
try:
|
|
with open("/opt/vault/init-keys.json") as f:
|
|
token = json.load(f)["root_token"]
|
|
|
|
result = subprocess.run([
|
|
"curl", "-sk",
|
|
"-H", f"X-Vault-Token: {token}",
|
|
"https://127.0.0.1:8200/v1/secret/data/services/openrouter"
|
|
], capture_output=True, text=True)
|
|
|
|
data = json.loads(result.stdout)
|
|
return data.get("data", {}).get("data", {}).get("api_key")
|
|
except:
|
|
return None
|
|
|
|
def _get_model_config(self) -> dict:
|
|
"""Get the model configuration based on current settings"""
|
|
if self.config.model_override and self.config.model_override in MODELS:
|
|
return MODELS[self.config.model_override]
|
|
|
|
if self.config.mode == OrchestrationMode.MINIMAX:
|
|
return MODELS["minimax"]
|
|
elif self.config.mode == OrchestrationMode.GEMINI:
|
|
return MODELS["gemini"]
|
|
elif self.config.mode == OrchestrationMode.GEMINI_PRO:
|
|
return MODELS["gemini-pro"]
|
|
|
|
return MODELS[DEFAULT_MODEL]
|
|
|
|
def is_enabled(self) -> bool:
|
|
"""Check if automated orchestration is enabled"""
|
|
return self.config.mode != OrchestrationMode.DISABLED
|
|
|
|
def get_mode(self) -> str:
|
|
"""Get current orchestration mode"""
|
|
return self.config.mode.value
|
|
|
|
def get_model_name(self) -> str:
|
|
"""Get the display name of the current model"""
|
|
model_config = self._get_model_config()
|
|
return model_config["display_name"]
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Checkpoint Integration
|
|
# -------------------------------------------------------------------------
|
|
|
|
def _create_checkpoint(self, notes: str = "") -> Optional[str]:
|
|
"""Create a checkpoint before/after action"""
|
|
try:
|
|
result = subprocess.run(
|
|
["/opt/agent-governance/bin/checkpoint", "now", "--notes", notes, "--json"],
|
|
capture_output=True, text=True, timeout=30
|
|
)
|
|
|
|
if result.returncode == 0:
|
|
data = json.loads(result.stdout)
|
|
return data.get("checkpoint_id")
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to create checkpoint: {e}")
|
|
|
|
return None
|
|
|
|
def _load_latest_checkpoint(self) -> Optional[dict]:
|
|
"""Load the latest checkpoint for context"""
|
|
try:
|
|
result = subprocess.run(
|
|
["/opt/agent-governance/bin/checkpoint", "load", "--json"],
|
|
capture_output=True, text=True, timeout=30
|
|
)
|
|
|
|
if result.returncode == 0:
|
|
return json.loads(result.stdout)
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to load checkpoint: {e}")
|
|
|
|
return None
|
|
|
|
def _build_context(self) -> OrchestrationContext:
|
|
"""Build orchestration context from checkpoint and environment"""
|
|
checkpoint = self._load_latest_checkpoint()
|
|
|
|
phase = "Unknown"
|
|
phase_status = "unknown"
|
|
active_tasks = []
|
|
pending_tasks = []
|
|
|
|
if checkpoint:
|
|
if checkpoint.get("phase"):
|
|
phase = checkpoint["phase"].get("name", "Unknown")
|
|
phase_status = checkpoint["phase"].get("status", "unknown")
|
|
|
|
for task in checkpoint.get("tasks", []):
|
|
if task.get("status") == "in_progress":
|
|
active_tasks.append(task)
|
|
elif task.get("status") == "pending":
|
|
pending_tasks.append(task)
|
|
|
|
# Build constraints based on agent tier
|
|
agent_tier = int(os.environ.get("AGENT_TIER", "0"))
|
|
constraints = {
|
|
"max_tier": agent_tier,
|
|
"can_execute": agent_tier >= 1,
|
|
"can_modify_prod": agent_tier >= 3,
|
|
"requires_approval": agent_tier < 2
|
|
}
|
|
|
|
return OrchestrationContext(
|
|
phase=phase,
|
|
phase_status=phase_status,
|
|
active_tasks=active_tasks,
|
|
pending_tasks=pending_tasks,
|
|
constraints=constraints,
|
|
recent_outputs=checkpoint.get("recent_outputs", []) if checkpoint else [],
|
|
agent_id=os.environ.get("AGENT_ID"),
|
|
agent_tier=agent_tier,
|
|
session_id=os.environ.get("SESSION_ID")
|
|
)
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Model Interaction
|
|
# -------------------------------------------------------------------------
|
|
|
|
def _call_model(
|
|
self,
|
|
prompt: str,
|
|
system_prompt: str = None,
|
|
model_config: dict = None
|
|
) -> tuple[str, int, str]:
|
|
"""
|
|
Call the model via OpenRouter API.
|
|
Returns (response_text, tokens_used, model_id)
|
|
"""
|
|
if not requests:
|
|
raise RuntimeError("requests library not available")
|
|
|
|
api_key = self._get_api_key()
|
|
if not api_key:
|
|
raise RuntimeError("OpenRouter API key not configured")
|
|
|
|
model_config = model_config or self._get_model_config()
|
|
model_id = model_config["model_id"]
|
|
|
|
messages = []
|
|
if system_prompt:
|
|
messages.append({"role": "system", "content": system_prompt})
|
|
messages.append({"role": "user", "content": prompt})
|
|
|
|
headers = {
|
|
"Authorization": f"Bearer {api_key}",
|
|
"Content-Type": "application/json",
|
|
"HTTP-Referer": "https://agent-governance.local",
|
|
"X-Title": "Agent Governance Orchestrator"
|
|
}
|
|
|
|
payload = {
|
|
"model": model_id,
|
|
"messages": messages,
|
|
"max_tokens": self.config.max_tokens_per_request,
|
|
"temperature": 0.3, # Lower temperature for more deterministic output
|
|
}
|
|
|
|
self.logger.info(f"Calling model: {model_id}")
|
|
|
|
response = requests.post(
|
|
OPENROUTER_API_URL,
|
|
headers=headers,
|
|
json=payload,
|
|
timeout=self.config.timeout_seconds
|
|
)
|
|
|
|
if response.status_code != 200:
|
|
error_msg = f"API error: {response.status_code} - {response.text}"
|
|
self.logger.error(error_msg)
|
|
raise RuntimeError(error_msg)
|
|
|
|
data = response.json()
|
|
|
|
if "error" in data:
|
|
raise RuntimeError(f"Model error: {data['error']}")
|
|
|
|
content = data["choices"][0]["message"]["content"]
|
|
tokens_used = data.get("usage", {}).get("total_tokens", 0)
|
|
|
|
self.logger.info(f"Model response received: {tokens_used} tokens")
|
|
|
|
return content, tokens_used, model_id
|
|
|
|
def _call_with_fallback(
|
|
self,
|
|
prompt: str,
|
|
system_prompt: str = None
|
|
) -> tuple[str, int, str, bool]:
|
|
"""
|
|
Call model with fallback chain.
|
|
Returns (response_text, tokens_used, model_id, fallback_triggered)
|
|
"""
|
|
model_config = self._get_model_config()
|
|
fallback_triggered = False
|
|
|
|
for attempt in range(self.config.max_retries):
|
|
try:
|
|
response, tokens, model_id = self._call_model(
|
|
prompt, system_prompt, model_config
|
|
)
|
|
return response, tokens, model_id, fallback_triggered
|
|
|
|
except Exception as e:
|
|
self.logger.warning(f"Model call failed (attempt {attempt + 1}): {e}")
|
|
|
|
# Try fallback model
|
|
fallback_name = model_config.get("fallback")
|
|
if fallback_name and fallback_name in MODELS:
|
|
self.logger.info(f"Falling back to {fallback_name}")
|
|
model_config = MODELS[fallback_name]
|
|
fallback_triggered = True
|
|
continue
|
|
|
|
# No more fallbacks
|
|
if attempt == self.config.max_retries - 1:
|
|
raise
|
|
|
|
time.sleep(2 ** attempt) # Exponential backoff
|
|
|
|
raise RuntimeError("All model calls failed")
|
|
|
|
# -------------------------------------------------------------------------
|
|
# Command Execution
|
|
# -------------------------------------------------------------------------
|
|
|
|
def execute_command(self, request: CommandRequest) -> CommandResponse:
|
|
"""
|
|
Execute a command via the model with full checkpoint integration.
|
|
"""
|
|
start_time = time.time()
|
|
checkpoint_before_id = None
|
|
checkpoint_after_id = None
|
|
|
|
self._command_count += 1
|
|
self.logger.info(f"Executing command #{self._command_count}: {request.command_type.value}")
|
|
|
|
# Pre-action checkpoint
|
|
if self.config.checkpoint_before_action:
|
|
checkpoint_before_id = self._create_checkpoint(
|
|
f"Pre-action checkpoint for {request.command_type.value}"
|
|
)
|
|
|
|
# Build system prompt
|
|
system_prompt = self._build_system_prompt(request)
|
|
|
|
# Build user prompt with context
|
|
user_prompt = self._build_user_prompt(request)
|
|
|
|
try:
|
|
# Call model with fallback
|
|
response, tokens_used, model_id, fallback_triggered = self._call_with_fallback(
|
|
user_prompt, system_prompt
|
|
)
|
|
|
|
# Parse and execute the model's response
|
|
execution_result = self._execute_model_response(response, request)
|
|
|
|
# Post-action checkpoint
|
|
if self.config.checkpoint_after_action:
|
|
checkpoint_after_id = self._create_checkpoint(
|
|
f"Post-action checkpoint for {request.command_type.value}"
|
|
)
|
|
|
|
# Log to audit
|
|
self._log_to_audit(request, response, model_id, tokens_used, True)
|
|
|
|
execution_time = time.time() - start_time
|
|
|
|
return CommandResponse(
|
|
success=True,
|
|
output=execution_result,
|
|
model_used=model_id,
|
|
tokens_used=tokens_used,
|
|
execution_time=execution_time,
|
|
checkpoint_before_id=checkpoint_before_id,
|
|
checkpoint_after_id=checkpoint_after_id,
|
|
fallback_triggered=fallback_triggered
|
|
)
|
|
|
|
except Exception as e:
|
|
execution_time = time.time() - start_time
|
|
error_msg = str(e)
|
|
|
|
self.logger.error(f"Command execution failed: {error_msg}")
|
|
|
|
# Log failure to audit
|
|
self._log_to_audit(request, error_msg, "N/A", 0, False)
|
|
|
|
# Fallback to human if configured
|
|
if self.config.fallback_to_human:
|
|
self.logger.warning("Falling back to human intervention")
|
|
self._request_human_intervention(request, error_msg)
|
|
|
|
return CommandResponse(
|
|
success=False,
|
|
output="",
|
|
model_used="N/A",
|
|
tokens_used=0,
|
|
execution_time=execution_time,
|
|
checkpoint_before_id=checkpoint_before_id,
|
|
error=error_msg,
|
|
fallback_triggered=True
|
|
)
|
|
|
|
def _build_system_prompt(self, request: CommandRequest) -> str:
|
|
"""Build the system prompt for the model"""
|
|
return f"""You are an automated agent orchestrator for a governance system.
|
|
|
|
Your role is to execute commands safely and report results clearly.
|
|
|
|
Current Mode: {self.config.mode.value}
|
|
Agent Tier: {request.context.agent_tier}
|
|
|
|
Constraints:
|
|
- Only execute commands within the allowed scope for Tier {request.context.agent_tier}
|
|
- Always verify actions before execution
|
|
- Report any errors or unexpected behavior
|
|
- Do not attempt to bypass governance controls
|
|
|
|
Response Format:
|
|
- Start with ACTION: followed by the specific action to take
|
|
- Then RATIONALE: explaining why
|
|
- Finally RESULT: with the expected or actual outcome
|
|
|
|
If the command cannot be safely executed, respond with:
|
|
ACTION: BLOCKED
|
|
RATIONALE: [reason]
|
|
RESULT: Requires human intervention
|
|
"""
|
|
|
|
def _build_user_prompt(self, request: CommandRequest) -> str:
|
|
"""Build the user prompt with context"""
|
|
context_prompt = request.context.to_prompt()
|
|
|
|
return f"""{context_prompt}
|
|
|
|
## Command Request
|
|
|
|
Type: {request.command_type.value}
|
|
Command: {request.command}
|
|
Requires Confirmation: {request.require_confirmation}
|
|
|
|
Please analyze this command and provide your response in the specified format.
|
|
"""
|
|
|
|
def _execute_model_response(self, response: str, request: CommandRequest) -> str:
|
|
"""Parse and execute the model's response"""
|
|
# Parse response for ACTION
|
|
action_line = None
|
|
for line in response.split("\n"):
|
|
if line.startswith("ACTION:"):
|
|
action_line = line[7:].strip()
|
|
break
|
|
|
|
if not action_line:
|
|
return f"Model response (no explicit action):\n{response}"
|
|
|
|
if action_line == "BLOCKED":
|
|
return f"Command blocked by model:\n{response}"
|
|
|
|
# For now, return the response without actually executing
|
|
# Real execution would happen here based on command type
|
|
return f"Model analysis:\n{response}"
|
|
|
|
def _log_to_audit(
|
|
self,
|
|
request: CommandRequest,
|
|
response: str,
|
|
model_id: str,
|
|
tokens_used: int,
|
|
success: bool
|
|
):
|
|
"""Log the command execution to audit trail"""
|
|
if not LEDGER_DB.exists():
|
|
return
|
|
|
|
try:
|
|
conn = sqlite3.connect(LEDGER_DB)
|
|
cursor = conn.cursor()
|
|
|
|
# Check if orchestration_log table exists
|
|
cursor.execute("""
|
|
SELECT name FROM sqlite_master
|
|
WHERE type='table' AND name='orchestration_log'
|
|
""")
|
|
|
|
if not cursor.fetchone():
|
|
# Create the table
|
|
cursor.execute("""
|
|
CREATE TABLE orchestration_log (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
timestamp TEXT NOT NULL,
|
|
session_id TEXT,
|
|
agent_id TEXT,
|
|
orchestration_mode TEXT NOT NULL,
|
|
model_id TEXT,
|
|
command_type TEXT NOT NULL,
|
|
command TEXT NOT NULL,
|
|
response TEXT,
|
|
tokens_used INTEGER,
|
|
success INTEGER,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
cursor.execute("""
|
|
INSERT INTO orchestration_log
|
|
(timestamp, session_id, agent_id, orchestration_mode, model_id,
|
|
command_type, command, response, tokens_used, success)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
""", (
|
|
datetime.now(timezone.utc).isoformat(),
|
|
request.context.session_id,
|
|
request.context.agent_id,
|
|
self.config.mode.value,
|
|
model_id,
|
|
request.command_type.value,
|
|
request.command,
|
|
response[:10000], # Truncate long responses
|
|
tokens_used,
|
|
1 if success else 0
|
|
))
|
|
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
except Exception as e:
|
|
self.logger.error(f"Failed to log to audit: {e}")
|
|
|
|
# Also log to DragonflyDB
|
|
if self.redis:
|
|
try:
|
|
log_entry = {
|
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
"mode": self.config.mode.value,
|
|
"model": model_id,
|
|
"command": request.command[:500],
|
|
"success": success
|
|
}
|
|
self.redis.lpush("orchestration:log", json.dumps(log_entry))
|
|
self.redis.ltrim("orchestration:log", 0, 999) # Keep last 1000
|
|
except:
|
|
pass
|
|
|
|
def _request_human_intervention(self, request: CommandRequest, error: str):
|
|
"""Request human intervention when automated execution fails"""
|
|
intervention_request = {
|
|
"type": "human_intervention_required",
|
|
"timestamp": datetime.now(timezone.utc).isoformat(),
|
|
"command_type": request.command_type.value,
|
|
"command": request.command,
|
|
"error": error,
|
|
"context": request.context.to_prompt()
|
|
}
|
|
|
|
# Store in DragonflyDB for dashboard pickup
|
|
if self.redis:
|
|
try:
|
|
self.redis.lpush("intervention:queue", json.dumps(intervention_request))
|
|
self.redis.publish("intervention:new", json.dumps({
|
|
"type": "intervention_required",
|
|
"timestamp": intervention_request["timestamp"]
|
|
}))
|
|
except:
|
|
pass
|
|
|
|
# Also write to file
|
|
intervention_dir = Path("/opt/agent-governance/orchestrator/interventions")
|
|
intervention_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
filename = f"intervention-{datetime.now().strftime('%Y%m%d-%H%M%S')}.json"
|
|
with open(intervention_dir / filename, "w") as f:
|
|
json.dump(intervention_request, f, indent=2)
|
|
|
|
self.logger.warning(f"Human intervention requested: {intervention_dir / filename}")
|
|
|
|
|
|
# =============================================================================
|
|
# Orchestration Manager
|
|
# =============================================================================
|
|
|
|
class OrchestrationManager:
|
|
"""
|
|
High-level manager for automated orchestration.
|
|
Handles mode switching, status reporting, and manual override.
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.controller = ModelController()
|
|
self.redis = self.controller.redis
|
|
|
|
def get_status(self) -> dict:
|
|
"""Get current orchestration status"""
|
|
return {
|
|
"enabled": self.controller.is_enabled(),
|
|
"mode": self.controller.get_mode(),
|
|
"model": self.controller.get_model_name() if self.controller.is_enabled() else None,
|
|
"config": self.controller.config.to_dict(),
|
|
"command_count": self.controller._command_count
|
|
}
|
|
|
|
def set_mode(self, mode: str) -> bool:
|
|
"""Set orchestration mode"""
|
|
try:
|
|
new_mode = OrchestrationMode(mode.lower())
|
|
self.controller.config.mode = new_mode
|
|
|
|
# Store in DragonflyDB
|
|
if self.redis:
|
|
self.redis.set("orchestration:mode", mode.lower())
|
|
|
|
return True
|
|
except ValueError:
|
|
return False
|
|
|
|
def take_control(self) -> bool:
|
|
"""Manual override - switch to human control"""
|
|
self.controller.config.mode = OrchestrationMode.DISABLED
|
|
|
|
if self.redis:
|
|
self.redis.set("orchestration:mode", "disabled")
|
|
self.redis.publish("orchestration:control", json.dumps({
|
|
"event": "manual_override",
|
|
"timestamp": datetime.now(timezone.utc).isoformat()
|
|
}))
|
|
|
|
return True
|
|
|
|
def delegate_command(
|
|
self,
|
|
command: str,
|
|
command_type: str = "shell",
|
|
require_confirmation: bool = False
|
|
) -> CommandResponse:
|
|
"""Delegate a command to the model"""
|
|
if not self.controller.is_enabled():
|
|
return CommandResponse(
|
|
success=False,
|
|
output="",
|
|
model_used="N/A",
|
|
tokens_used=0,
|
|
execution_time=0,
|
|
error="Orchestration mode is disabled"
|
|
)
|
|
|
|
try:
|
|
cmd_type = CommandType(command_type.lower())
|
|
except ValueError:
|
|
cmd_type = CommandType.SHELL
|
|
|
|
context = self.controller._build_context()
|
|
|
|
request = CommandRequest(
|
|
command_type=cmd_type,
|
|
command=command,
|
|
context=context,
|
|
require_confirmation=require_confirmation
|
|
)
|
|
|
|
return self.controller.execute_command(request)
|
|
|
|
|
|
# =============================================================================
|
|
# CLI Interface
|
|
# =============================================================================
|
|
|
|
def cli():
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="Model Controller - Automated Orchestration",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
model-controller status Show orchestration status
|
|
model-controller enable minimax Enable Minimax model
|
|
model-controller enable gemini Enable Gemini model
|
|
model-controller disable Disable automated mode
|
|
model-controller delegate "ls -la" Delegate command to model
|
|
model-controller override Manual override (take control)
|
|
"""
|
|
)
|
|
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
# status
|
|
subparsers.add_parser("status", help="Show orchestration status")
|
|
|
|
# enable
|
|
enable_parser = subparsers.add_parser("enable", help="Enable automated mode")
|
|
enable_parser.add_argument("model", choices=["minimax", "gemini", "gemini-pro"],
|
|
help="Model to use")
|
|
|
|
# disable
|
|
subparsers.add_parser("disable", help="Disable automated mode")
|
|
|
|
# delegate
|
|
delegate_parser = subparsers.add_parser("delegate", help="Delegate command to model")
|
|
delegate_parser.add_argument("cmd", help="Command to delegate")
|
|
delegate_parser.add_argument("--type", choices=["shell", "checkpoint", "plan", "execute", "review"],
|
|
default="shell", help="Command type")
|
|
delegate_parser.add_argument("--confirm", action="store_true", help="Require confirmation")
|
|
|
|
# override
|
|
subparsers.add_parser("override", help="Manual override (take control)")
|
|
|
|
# config
|
|
config_parser = subparsers.add_parser("config", help="Show/set configuration")
|
|
config_parser.add_argument("--set", nargs=2, metavar=("KEY", "VALUE"),
|
|
help="Set configuration value")
|
|
|
|
args = parser.parse_args()
|
|
|
|
manager = OrchestrationManager()
|
|
|
|
if args.command == "status":
|
|
status = manager.get_status()
|
|
print(f"\n{'='*60}")
|
|
print("ORCHESTRATION STATUS")
|
|
print(f"{'='*60}")
|
|
print(f"Enabled: {status['enabled']}")
|
|
print(f"Mode: {status['mode']}")
|
|
if status['model']:
|
|
print(f"Model: {status['model']}")
|
|
print(f"Commands Executed: {status['command_count']}")
|
|
print(f"\nConfiguration:")
|
|
for key, value in status['config'].items():
|
|
print(f" {key}: {value}")
|
|
print(f"{'='*60}")
|
|
|
|
elif args.command == "enable":
|
|
if manager.set_mode(args.model):
|
|
print(f"Orchestration enabled with {args.model}")
|
|
else:
|
|
print(f"Failed to enable orchestration")
|
|
sys.exit(1)
|
|
|
|
elif args.command == "disable":
|
|
manager.set_mode("disabled")
|
|
print("Orchestration disabled")
|
|
|
|
elif args.command == "delegate":
|
|
if not manager.controller.is_enabled():
|
|
print("Error: Orchestration is disabled. Enable it first with 'enable' command.")
|
|
sys.exit(1)
|
|
|
|
response = manager.delegate_command(
|
|
args.cmd,
|
|
command_type=args.type,
|
|
require_confirmation=args.confirm
|
|
)
|
|
|
|
print(f"\n{'='*60}")
|
|
print("COMMAND DELEGATION RESULT")
|
|
print(f"{'='*60}")
|
|
print(f"Success: {response.success}")
|
|
print(f"Model: {response.model_used}")
|
|
print(f"Tokens: {response.tokens_used}")
|
|
print(f"Time: {response.execution_time:.2f}s")
|
|
if response.checkpoint_before_id:
|
|
print(f"Checkpoint Before: {response.checkpoint_before_id}")
|
|
if response.checkpoint_after_id:
|
|
print(f"Checkpoint After: {response.checkpoint_after_id}")
|
|
if response.fallback_triggered:
|
|
print("Fallback: Triggered")
|
|
if response.error:
|
|
print(f"Error: {response.error}")
|
|
print(f"\nOutput:\n{response.output}")
|
|
print(f"{'='*60}")
|
|
|
|
elif args.command == "override":
|
|
manager.take_control()
|
|
print("Manual override activated - you are now in control")
|
|
|
|
elif args.command == "config":
|
|
if args.set:
|
|
key, value = args.set
|
|
# Would need to implement config setting
|
|
print(f"Configuration setting not yet implemented: {key}={value}")
|
|
else:
|
|
config = manager.controller.config.to_dict()
|
|
print(f"\n{'='*60}")
|
|
print("CONFIGURATION")
|
|
print(f"{'='*60}")
|
|
for key, value in config.items():
|
|
print(f"{key}: {value}")
|
|
print(f"{'='*60}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
cli()
|