77655c298c
Phase 8 Production Hardening with complete governance infrastructure: - Vault integration with tiered policies (T0-T4) - DragonflyDB state management - SQLite audit ledger - Pipeline DSL and templates - Promotion/revocation engine - Checkpoint system for session persistence - Health manager and circuit breaker for fault tolerance - GitHub/Slack integrations - Architectural test pipeline with bug watcher, suggestion engine, council review - Multi-agent chaos testing framework Test Results: - Governance tests: 68/68 passing - E2E workflow: 16/16 passing - Phase 2 Vault: 14/14 passing - Integration tests: 27/27 passing Coverage: 57.6% average across 12 phases Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
466 lines
16 KiB
Python
Executable File
466 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Evidence Packaging System
|
|
=========================
|
|
Auto-generates comprehensive evidence packages for audit.
|
|
Part of Phase 3: Execution Pipeline - Post-Execution Verification.
|
|
|
|
Evidence Package Contents:
|
|
- Plan artifact (original plan)
|
|
- Apply/execution logs
|
|
- State diff (before/after)
|
|
- Health check results
|
|
- Timing information
|
|
- Agent identity
|
|
- Checksums and signatures
|
|
"""
|
|
|
|
import json
|
|
import hashlib
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import io
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
|
|
@dataclass
|
|
class EvidencePackage:
|
|
"""
|
|
Complete evidence package for an execution.
|
|
"""
|
|
package_id: str
|
|
agent_id: str
|
|
agent_tier: int
|
|
task_id: str
|
|
action_type: str # terraform, ansible, docker, etc.
|
|
created_at: str
|
|
|
|
# Timing
|
|
preflight_start: Optional[str] = None
|
|
preflight_end: Optional[str] = None
|
|
execution_start: Optional[str] = None
|
|
execution_end: Optional[str] = None
|
|
verification_start: Optional[str] = None
|
|
verification_end: Optional[str] = None
|
|
|
|
# Artifacts
|
|
plan_artifact_id: Optional[str] = None
|
|
plan_checksum: Optional[str] = None
|
|
execution_log: Optional[str] = None
|
|
state_before: Optional[dict] = None
|
|
state_after: Optional[dict] = None
|
|
state_diff: Optional[dict] = None
|
|
|
|
# Health checks
|
|
health_checks: list = field(default_factory=list)
|
|
|
|
# Verification
|
|
preflight_passed: bool = False
|
|
execution_success: bool = False
|
|
verification_passed: bool = False
|
|
|
|
# Errors and notes
|
|
errors: list = field(default_factory=list)
|
|
warnings: list = field(default_factory=list)
|
|
notes: list = field(default_factory=list)
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"package_id": self.package_id,
|
|
"agent_id": self.agent_id,
|
|
"agent_tier": self.agent_tier,
|
|
"task_id": self.task_id,
|
|
"action_type": self.action_type,
|
|
"created_at": self.created_at,
|
|
"timing": {
|
|
"preflight_start": self.preflight_start,
|
|
"preflight_end": self.preflight_end,
|
|
"execution_start": self.execution_start,
|
|
"execution_end": self.execution_end,
|
|
"verification_start": self.verification_start,
|
|
"verification_end": self.verification_end
|
|
},
|
|
"artifacts": {
|
|
"plan_artifact_id": self.plan_artifact_id,
|
|
"plan_checksum": self.plan_checksum
|
|
},
|
|
"state": {
|
|
"before": self.state_before,
|
|
"after": self.state_after,
|
|
"diff": self.state_diff
|
|
},
|
|
"health_checks": self.health_checks,
|
|
"results": {
|
|
"preflight_passed": self.preflight_passed,
|
|
"execution_success": self.execution_success,
|
|
"verification_passed": self.verification_passed
|
|
},
|
|
"errors": self.errors,
|
|
"warnings": self.warnings,
|
|
"notes": self.notes
|
|
}
|
|
|
|
|
|
class EvidenceCollector:
|
|
"""
|
|
Collects and packages execution evidence.
|
|
"""
|
|
|
|
EVIDENCE_DIR = Path("/opt/agent-governance/evidence")
|
|
|
|
def __init__(self, agent_id: str, agent_tier: int, task_id: str, action_type: str):
|
|
self.agent_id = agent_id
|
|
self.agent_tier = agent_tier
|
|
self.task_id = task_id
|
|
self.action_type = action_type
|
|
self.package = self._create_package()
|
|
|
|
def _now(self) -> str:
|
|
return datetime.now(timezone.utc).isoformat()
|
|
|
|
def _generate_package_id(self) -> str:
|
|
timestamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
|
|
suffix = hashlib.sha256(f"{self.task_id}-{timestamp}".encode()).hexdigest()[:8]
|
|
return f"evd-{timestamp}-{suffix}"
|
|
|
|
def _create_package(self) -> EvidencePackage:
|
|
return EvidencePackage(
|
|
package_id=self._generate_package_id(),
|
|
agent_id=self.agent_id,
|
|
agent_tier=self.agent_tier,
|
|
task_id=self.task_id,
|
|
action_type=self.action_type,
|
|
created_at=self._now()
|
|
)
|
|
|
|
# Timing methods
|
|
def start_preflight(self):
|
|
self.package.preflight_start = self._now()
|
|
|
|
def end_preflight(self, passed: bool):
|
|
self.package.preflight_end = self._now()
|
|
self.package.preflight_passed = passed
|
|
|
|
def start_execution(self):
|
|
self.package.execution_start = self._now()
|
|
|
|
def end_execution(self, success: bool):
|
|
self.package.execution_end = self._now()
|
|
self.package.execution_success = success
|
|
|
|
def start_verification(self):
|
|
self.package.verification_start = self._now()
|
|
|
|
def end_verification(self, passed: bool):
|
|
self.package.verification_end = self._now()
|
|
self.package.verification_passed = passed
|
|
|
|
# Artifact methods
|
|
def set_plan_artifact(self, artifact_id: str, checksum: str = None):
|
|
self.package.plan_artifact_id = artifact_id
|
|
self.package.plan_checksum = checksum
|
|
|
|
def set_execution_log(self, log: str):
|
|
self.package.execution_log = log
|
|
|
|
def set_state_before(self, state: dict):
|
|
self.package.state_before = state
|
|
|
|
def set_state_after(self, state: dict):
|
|
self.package.state_after = state
|
|
self._compute_state_diff()
|
|
|
|
def _compute_state_diff(self):
|
|
"""Compute diff between before and after states"""
|
|
if self.package.state_before and self.package.state_after:
|
|
before = self.package.state_before
|
|
after = self.package.state_after
|
|
|
|
diff = {
|
|
"added": {},
|
|
"removed": {},
|
|
"changed": {}
|
|
}
|
|
|
|
# Find added and changed
|
|
for key, value in after.items():
|
|
if key not in before:
|
|
diff["added"][key] = value
|
|
elif before[key] != value:
|
|
diff["changed"][key] = {
|
|
"before": before[key],
|
|
"after": value
|
|
}
|
|
|
|
# Find removed
|
|
for key, value in before.items():
|
|
if key not in after:
|
|
diff["removed"][key] = value
|
|
|
|
self.package.state_diff = diff
|
|
|
|
# Health check methods
|
|
def add_health_check(self, name: str, passed: bool, message: str, details: dict = None):
|
|
self.package.health_checks.append({
|
|
"name": name,
|
|
"passed": passed,
|
|
"message": message,
|
|
"details": details or {},
|
|
"timestamp": self._now()
|
|
})
|
|
|
|
# Error/warning/note methods
|
|
def add_error(self, error: str):
|
|
self.package.errors.append({
|
|
"error": error,
|
|
"timestamp": self._now()
|
|
})
|
|
|
|
def add_warning(self, warning: str):
|
|
self.package.warnings.append({
|
|
"warning": warning,
|
|
"timestamp": self._now()
|
|
})
|
|
|
|
def add_note(self, note: str):
|
|
self.package.notes.append({
|
|
"note": note,
|
|
"timestamp": self._now()
|
|
})
|
|
|
|
# Package methods
|
|
def finalize(self) -> dict:
|
|
"""Finalize and return the evidence package"""
|
|
return self.package.to_dict()
|
|
|
|
def save(self) -> Path:
|
|
"""Save evidence package to disk"""
|
|
package_dir = self.EVIDENCE_DIR / "packages" / self.package.package_id
|
|
package_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Save main package JSON
|
|
package_file = package_dir / "evidence.json"
|
|
with open(package_file, "w") as f:
|
|
json.dump(self.package.to_dict(), f, indent=2)
|
|
|
|
# Save execution log if present
|
|
if self.package.execution_log:
|
|
log_file = package_dir / "execution.log"
|
|
with open(log_file, "w") as f:
|
|
f.write(self.package.execution_log)
|
|
|
|
# Create manifest
|
|
manifest = {
|
|
"package_id": self.package.package_id,
|
|
"files": [],
|
|
"created_at": self._now()
|
|
}
|
|
|
|
for file in package_dir.iterdir():
|
|
manifest["files"].append({
|
|
"name": file.name,
|
|
"size": file.stat().st_size,
|
|
"checksum": hashlib.sha256(file.read_bytes()).hexdigest()
|
|
})
|
|
|
|
manifest_file = package_dir / "MANIFEST.json"
|
|
with open(manifest_file, "w") as f:
|
|
json.dump(manifest, f, indent=2)
|
|
|
|
return package_dir
|
|
|
|
def create_archive(self) -> Path:
|
|
"""Create a compressed archive of the evidence"""
|
|
package_dir = self.save()
|
|
archive_path = self.EVIDENCE_DIR / "archives" / f"{self.package.package_id}.tar.gz"
|
|
archive_path.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
with tarfile.open(archive_path, "w:gz") as tar:
|
|
tar.add(package_dir, arcname=self.package.package_id)
|
|
|
|
return archive_path
|
|
|
|
|
|
class HealthChecker:
|
|
"""
|
|
Performs health checks on execution results.
|
|
"""
|
|
|
|
@staticmethod
|
|
def check_service_running(service_name: str) -> tuple[bool, str]:
|
|
"""Check if a systemd service is running"""
|
|
try:
|
|
result = subprocess.run(
|
|
["systemctl", "is-active", service_name],
|
|
capture_output=True,
|
|
text=True
|
|
)
|
|
if result.stdout.strip() == "active":
|
|
return True, f"Service {service_name} is running"
|
|
return False, f"Service {service_name} is not running: {result.stdout.strip()}"
|
|
except Exception as e:
|
|
return False, f"Failed to check service: {str(e)}"
|
|
|
|
@staticmethod
|
|
def check_port_open(host: str, port: int) -> tuple[bool, str]:
|
|
"""Check if a port is open"""
|
|
import socket
|
|
try:
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
sock.settimeout(5)
|
|
result = sock.connect_ex((host, port))
|
|
sock.close()
|
|
if result == 0:
|
|
return True, f"Port {port} on {host} is open"
|
|
return False, f"Port {port} on {host} is closed"
|
|
except Exception as e:
|
|
return False, f"Failed to check port: {str(e)}"
|
|
|
|
@staticmethod
|
|
def check_http_endpoint(url: str, expected_code: int = 200) -> tuple[bool, str]:
|
|
"""Check HTTP endpoint health"""
|
|
try:
|
|
result = subprocess.run([
|
|
"curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}",
|
|
"--connect-timeout", "5",
|
|
url
|
|
], capture_output=True, text=True, timeout=10)
|
|
|
|
status_code = int(result.stdout.strip())
|
|
if status_code == expected_code:
|
|
return True, f"HTTP {url} returned {status_code}"
|
|
return False, f"HTTP {url} returned {status_code}, expected {expected_code}"
|
|
except Exception as e:
|
|
return False, f"Failed to check HTTP endpoint: {str(e)}"
|
|
|
|
@staticmethod
|
|
def check_file_exists(path: str) -> tuple[bool, str]:
|
|
"""Check if a file exists"""
|
|
p = Path(path)
|
|
if p.exists():
|
|
return True, f"File {path} exists"
|
|
return False, f"File {path} does not exist"
|
|
|
|
@staticmethod
|
|
def check_command_success(command: list[str]) -> tuple[bool, str]:
|
|
"""Check if a command executes successfully"""
|
|
try:
|
|
result = subprocess.run(command, capture_output=True, text=True, timeout=30)
|
|
if result.returncode == 0:
|
|
return True, f"Command succeeded: {' '.join(command)}"
|
|
return False, f"Command failed (exit {result.returncode}): {result.stderr[:200]}"
|
|
except Exception as e:
|
|
return False, f"Command error: {str(e)}"
|
|
|
|
|
|
# =============================================================================
|
|
# CLI
|
|
# =============================================================================
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
|
|
parser = argparse.ArgumentParser(description="Evidence Package Manager")
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
# Create command
|
|
create_parser = subparsers.add_parser("create", help="Create a new evidence package")
|
|
create_parser.add_argument("--agent-id", required=True)
|
|
create_parser.add_argument("--tier", type=int, required=True)
|
|
create_parser.add_argument("--task-id", required=True)
|
|
create_parser.add_argument("--action", required=True)
|
|
|
|
# List command
|
|
list_parser = subparsers.add_parser("list", help="List evidence packages")
|
|
list_parser.add_argument("--json", action="store_true")
|
|
|
|
# Show command
|
|
show_parser = subparsers.add_parser("show", help="Show evidence package")
|
|
show_parser.add_argument("package_id")
|
|
|
|
# Health command
|
|
health_parser = subparsers.add_parser("health", help="Run health checks")
|
|
health_parser.add_argument("--service", help="Check systemd service")
|
|
health_parser.add_argument("--port", help="Check port (host:port)")
|
|
health_parser.add_argument("--http", help="Check HTTP endpoint")
|
|
health_parser.add_argument("--file", help="Check file exists")
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.command == "create":
|
|
collector = EvidenceCollector(
|
|
agent_id=args.agent_id,
|
|
agent_tier=args.tier,
|
|
task_id=args.task_id,
|
|
action_type=args.action
|
|
)
|
|
|
|
# Simulate a collection
|
|
collector.start_preflight()
|
|
collector.end_preflight(True)
|
|
collector.start_execution()
|
|
collector.add_note("Demo evidence package")
|
|
collector.end_execution(True)
|
|
|
|
package_dir = collector.save()
|
|
print(f"Evidence package created: {collector.package.package_id}")
|
|
print(f"Saved to: {package_dir}")
|
|
|
|
elif args.command == "list":
|
|
packages_dir = Path("/opt/agent-governance/evidence/packages")
|
|
if packages_dir.exists():
|
|
packages = sorted(packages_dir.iterdir(), reverse=True)[:20]
|
|
|
|
if args.json:
|
|
print(json.dumps([p.name for p in packages]))
|
|
else:
|
|
print("Recent Evidence Packages:")
|
|
print("-" * 40)
|
|
for p in packages:
|
|
meta_file = p / "evidence.json"
|
|
if meta_file.exists():
|
|
with open(meta_file) as f:
|
|
meta = json.load(f)
|
|
print(f" {p.name}")
|
|
print(f" Agent: {meta.get('agent_id')} (Tier {meta.get('agent_tier')})")
|
|
print(f" Action: {meta.get('action_type')}")
|
|
print(f" Success: {meta.get('results', {}).get('execution_success')}")
|
|
print()
|
|
else:
|
|
print("No evidence packages found")
|
|
|
|
elif args.command == "show":
|
|
package_dir = Path("/opt/agent-governance/evidence/packages") / args.package_id
|
|
evidence_file = package_dir / "evidence.json"
|
|
|
|
if evidence_file.exists():
|
|
with open(evidence_file) as f:
|
|
print(json.dumps(json.load(f), indent=2))
|
|
else:
|
|
print(f"Package not found: {args.package_id}")
|
|
sys.exit(1)
|
|
|
|
elif args.command == "health":
|
|
checker = HealthChecker()
|
|
|
|
if args.service:
|
|
passed, msg = checker.check_service_running(args.service)
|
|
print(f"{'[PASS]' if passed else '[FAIL]'} {msg}")
|
|
|
|
if args.port:
|
|
host, port = args.port.split(":")
|
|
passed, msg = checker.check_port_open(host, int(port))
|
|
print(f"{'[PASS]' if passed else '[FAIL]'} {msg}")
|
|
|
|
if args.http:
|
|
passed, msg = checker.check_http_endpoint(args.http)
|
|
print(f"{'[PASS]' if passed else '[FAIL]'} {msg}")
|
|
|
|
if args.file:
|
|
passed, msg = checker.check_file_exists(args.file)
|
|
print(f"{'[PASS]' if passed else '[FAIL]'} {msg}")
|