8c6e7831e9
Major additions: - marketplace/: Agent template registry with FTS5 search, ratings, versioning - observability/: Prometheus metrics, distributed tracing, structured logging - ledger/migrations/: Database migration scripts for multi-tenant support - tests/governance/: 15 new test files for phases 6-12 (295 total tests) - bin/validate-phases: Full 12-phase validation script New features: - Multi-tenant support with tenant isolation and quota enforcement - Agent marketplace with semantic versioning and search - Observability with metrics, tracing, and log correlation - Tier-1 agent bootstrap scripts Updated components: - ledger/api.py: Extended API for tenants, marketplace, observability - ledger/schema.sql: Added tenant, project, marketplace tables - testing/framework.ts: Enhanced test framework - checkpoint/checkpoint.py: Improved checkpoint management Archived: - External integrations (Slack/GitHub/PagerDuty) moved to .archive/ - Old checkpoint files cleaned up Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
431 lines
15 KiB
Python
431 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Phase 11: Agent Marketplace Tests
|
|
=================================
|
|
Validates marketplace schema, template management, search, and ratings.
|
|
"""
|
|
|
|
import json
|
|
import sqlite3
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
|
|
|
DB_PATH = Path("/opt/agent-governance/ledger/governance.db")
|
|
|
|
|
|
def get_db():
|
|
conn = sqlite3.connect(DB_PATH)
|
|
conn.row_factory = sqlite3.Row
|
|
return conn
|
|
|
|
|
|
def test_agent_templates_table_exists():
|
|
"""Verify agent_templates table exists"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(agent_templates)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
required = {"template_id", "name", "slug", "author_id", "category", "is_public"}
|
|
assert required.issubset(columns), f"Missing columns: {required - columns}"
|
|
print("[PASS] agent_templates table exists with required columns")
|
|
|
|
|
|
def test_template_versions_table_exists():
|
|
"""Verify template_versions table exists"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(template_versions)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
required = {"version_id", "template_id", "version", "version_number", "template_content"}
|
|
assert required.issubset(columns), f"Missing columns: {required - columns}"
|
|
print("[PASS] template_versions table exists with required columns")
|
|
|
|
|
|
def test_template_ratings_table_exists():
|
|
"""Verify template_ratings table exists"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(template_ratings)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
required = {"template_id", "tenant_id", "rating", "review_text"}
|
|
assert required.issubset(columns), f"Missing columns: {required - columns}"
|
|
print("[PASS] template_ratings table exists with required columns")
|
|
|
|
|
|
def test_template_stats_table_exists():
|
|
"""Verify template_stats table exists"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(template_stats)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
required = {"template_id", "total_downloads", "avg_rating", "active_installs"}
|
|
assert required.issubset(columns), f"Missing columns: {required - columns}"
|
|
print("[PASS] template_stats table exists with required columns")
|
|
|
|
|
|
def test_template_installations_table_exists():
|
|
"""Verify template_installations table exists"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(template_installations)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
required = {"template_id", "version_id", "tenant_id", "project_id"}
|
|
assert required.issubset(columns), f"Missing columns: {required - columns}"
|
|
print("[PASS] template_installations table exists with required columns")
|
|
|
|
|
|
def test_template_dependencies_table_exists():
|
|
"""Verify template_dependencies table exists"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(template_dependencies)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
required = {"template_id", "version_id", "depends_on_template"}
|
|
assert required.issubset(columns), f"Missing columns: {required - columns}"
|
|
print("[PASS] template_dependencies table exists with required columns")
|
|
|
|
|
|
def test_marketplace_categories_table_exists():
|
|
"""Verify marketplace_categories table exists with defaults"""
|
|
conn = get_db()
|
|
cursor = conn.execute("SELECT * FROM marketplace_categories ORDER BY sort_order")
|
|
categories = [dict(row) for row in cursor.fetchall()]
|
|
conn.close()
|
|
|
|
assert len(categories) >= 5, f"Expected at least 5 categories, found {len(categories)}"
|
|
|
|
category_ids = {c["category_id"] for c in categories}
|
|
expected = {"general", "automation", "monitoring", "security", "devops"}
|
|
assert expected.issubset(category_ids), f"Missing categories: {expected - category_ids}"
|
|
print(f"[PASS] marketplace_categories has {len(categories)} default categories")
|
|
|
|
|
|
def test_template_search_fts_exists():
|
|
"""Verify FTS5 search index exists"""
|
|
conn = get_db()
|
|
cursor = conn.execute("""
|
|
SELECT name FROM sqlite_master
|
|
WHERE type='table' AND name='template_search'
|
|
""")
|
|
row = cursor.fetchone()
|
|
conn.close()
|
|
|
|
assert row is not None, "template_search FTS table not found"
|
|
print("[PASS] template_search FTS5 index exists")
|
|
|
|
|
|
def test_create_template():
|
|
"""Test creating a template"""
|
|
conn = get_db()
|
|
template_id = "tpl-test-validate"
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM agent_templates WHERE template_id = ?", (template_id,))
|
|
conn.commit()
|
|
|
|
# Create template
|
|
conn.execute("""
|
|
INSERT INTO agent_templates
|
|
(template_id, name, slug, author_id, category, is_public, tags, capabilities)
|
|
VALUES (?, 'Test Template', 'test-validate', 'default', 'general', 1, '["test"]', '["basic"]')
|
|
""", (template_id,))
|
|
conn.commit()
|
|
|
|
# Verify
|
|
cursor = conn.execute("SELECT * FROM agent_templates WHERE template_id = ?", (template_id,))
|
|
row = cursor.fetchone()
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM agent_templates WHERE template_id = ?", (template_id,))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
assert row is not None, "Failed to create template"
|
|
assert row["name"] == "Test Template"
|
|
assert row["is_public"] == 1
|
|
print("[PASS] template creation works")
|
|
|
|
|
|
def test_create_template_version():
|
|
"""Test creating a template version"""
|
|
conn = get_db()
|
|
template_id = "tpl-test-version"
|
|
version_id = "ver-test-validate"
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM template_versions WHERE version_id = ?", (version_id,))
|
|
conn.execute("DELETE FROM agent_templates WHERE template_id = ?", (template_id,))
|
|
conn.commit()
|
|
|
|
# Create template and version
|
|
conn.execute("""
|
|
INSERT INTO agent_templates (template_id, name, slug, author_id, category)
|
|
VALUES (?, 'Version Test', 'version-test', 'default', 'general')
|
|
""", (template_id,))
|
|
|
|
template_content = json.dumps({"agent_type": "tier0", "constraints": []})
|
|
conn.execute("""
|
|
INSERT INTO template_versions
|
|
(version_id, template_id, version, version_number, template_content, is_stable)
|
|
VALUES (?, ?, '1.0.0', 10000, ?, 1)
|
|
""", (version_id, template_id, template_content))
|
|
conn.commit()
|
|
|
|
# Verify
|
|
cursor = conn.execute("""
|
|
SELECT v.*, t.name as template_name
|
|
FROM template_versions v
|
|
JOIN agent_templates t ON v.template_id = t.template_id
|
|
WHERE v.version_id = ?
|
|
""", (version_id,))
|
|
row = cursor.fetchone()
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM template_versions WHERE version_id = ?", (version_id,))
|
|
conn.execute("DELETE FROM agent_templates WHERE template_id = ?", (template_id,))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
assert row is not None, "Failed to create version"
|
|
assert row["version"] == "1.0.0"
|
|
assert row["version_number"] == 10000
|
|
print("[PASS] template version creation works")
|
|
|
|
|
|
def test_template_stats_tracking():
|
|
"""Test template statistics tracking"""
|
|
conn = get_db()
|
|
template_id = "tpl-test-stats"
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM template_stats WHERE template_id = ?", (template_id,))
|
|
conn.execute("DELETE FROM agent_templates WHERE template_id = ?", (template_id,))
|
|
conn.commit()
|
|
|
|
# Create template and stats
|
|
conn.execute("""
|
|
INSERT INTO agent_templates (template_id, name, slug, author_id, category)
|
|
VALUES (?, 'Stats Test', 'stats-test', 'default', 'general')
|
|
""", (template_id,))
|
|
|
|
conn.execute("""
|
|
INSERT INTO template_stats (template_id, total_downloads, avg_rating, rating_count)
|
|
VALUES (?, 100, 4.5, 20)
|
|
""", (template_id,))
|
|
conn.commit()
|
|
|
|
# Verify
|
|
cursor = conn.execute("SELECT * FROM template_stats WHERE template_id = ?", (template_id,))
|
|
row = cursor.fetchone()
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM template_stats WHERE template_id = ?", (template_id,))
|
|
conn.execute("DELETE FROM agent_templates WHERE template_id = ?", (template_id,))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
assert row is not None, "Failed to track stats"
|
|
assert row["total_downloads"] == 100
|
|
assert row["avg_rating"] == 4.5
|
|
print("[PASS] template stats tracking works")
|
|
|
|
|
|
def test_template_rating():
|
|
"""Test template rating submission"""
|
|
conn = get_db()
|
|
template_id = "tpl-test-rating"
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM template_ratings WHERE template_id = ?", (template_id,))
|
|
conn.execute("DELETE FROM agent_templates WHERE template_id = ?", (template_id,))
|
|
conn.commit()
|
|
|
|
# Create template and rating
|
|
conn.execute("""
|
|
INSERT INTO agent_templates (template_id, name, slug, author_id, category)
|
|
VALUES (?, 'Rating Test', 'rating-test', 'default', 'general')
|
|
""", (template_id,))
|
|
|
|
conn.execute("""
|
|
INSERT INTO template_ratings (template_id, tenant_id, rating, review_title, review_text)
|
|
VALUES (?, 'default', 5, 'Great template!', 'Works perfectly for my use case.')
|
|
""", (template_id,))
|
|
conn.commit()
|
|
|
|
# Verify
|
|
cursor = conn.execute("SELECT * FROM template_ratings WHERE template_id = ?", (template_id,))
|
|
row = cursor.fetchone()
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM template_ratings WHERE template_id = ?", (template_id,))
|
|
conn.execute("DELETE FROM agent_templates WHERE template_id = ?", (template_id,))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
assert row is not None, "Failed to submit rating"
|
|
assert row["rating"] == 5
|
|
assert row["review_title"] == "Great template!"
|
|
print("[PASS] template rating submission works")
|
|
|
|
|
|
def test_template_installation():
|
|
"""Test template installation tracking"""
|
|
conn = get_db()
|
|
template_id = "tpl-test-install"
|
|
version_id = "ver-test-install"
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM template_installations WHERE template_id = ?", (template_id,))
|
|
conn.execute("DELETE FROM template_versions WHERE version_id = ?", (version_id,))
|
|
conn.execute("DELETE FROM agent_templates WHERE template_id = ?", (template_id,))
|
|
conn.commit()
|
|
|
|
# Create template, version, and installation
|
|
conn.execute("""
|
|
INSERT INTO agent_templates (template_id, name, slug, author_id, category)
|
|
VALUES (?, 'Install Test', 'install-test', 'default', 'general')
|
|
""", (template_id,))
|
|
|
|
conn.execute("""
|
|
INSERT INTO template_versions (version_id, template_id, version, version_number, template_content)
|
|
VALUES (?, ?, '1.0.0', 10000, '{}')
|
|
""", (version_id, template_id))
|
|
|
|
conn.execute("""
|
|
INSERT INTO template_installations (template_id, version_id, tenant_id, project_id)
|
|
VALUES (?, ?, 'default', 'default')
|
|
""", (template_id, version_id))
|
|
conn.commit()
|
|
|
|
# Verify
|
|
cursor = conn.execute("""
|
|
SELECT i.*, t.name as template_name, v.version
|
|
FROM template_installations i
|
|
JOIN agent_templates t ON i.template_id = t.template_id
|
|
JOIN template_versions v ON i.version_id = v.version_id
|
|
WHERE i.template_id = ?
|
|
""", (template_id,))
|
|
row = cursor.fetchone()
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM template_installations WHERE template_id = ?", (template_id,))
|
|
conn.execute("DELETE FROM template_versions WHERE version_id = ?", (version_id,))
|
|
conn.execute("DELETE FROM agent_templates WHERE template_id = ?", (template_id,))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
assert row is not None, "Failed to track installation"
|
|
assert row["tenant_id"] == "default"
|
|
assert row["version"] == "1.0.0"
|
|
print("[PASS] template installation tracking works")
|
|
|
|
|
|
def test_category_filtering():
|
|
"""Test filtering templates by category"""
|
|
conn = get_db()
|
|
|
|
# Query by category
|
|
cursor = conn.execute("""
|
|
SELECT COUNT(*) as count FROM agent_templates
|
|
WHERE category = 'general' AND is_public = 1
|
|
""")
|
|
row = cursor.fetchone()
|
|
conn.close()
|
|
|
|
# Should not error - just verify query works
|
|
assert row is not None, "Category filter query failed"
|
|
print(f"[PASS] category filtering works (found {row['count']} templates)")
|
|
|
|
|
|
def test_public_templates_query():
|
|
"""Test querying public templates"""
|
|
conn = get_db()
|
|
|
|
cursor = conn.execute("""
|
|
SELECT t.*, ts.total_downloads, ts.avg_rating
|
|
FROM agent_templates t
|
|
LEFT JOIN template_stats ts ON t.template_id = ts.template_id
|
|
WHERE t.is_public = 1
|
|
ORDER BY ts.total_downloads DESC NULLS LAST
|
|
LIMIT 10
|
|
""")
|
|
templates = [dict(row) for row in cursor.fetchall()]
|
|
conn.close()
|
|
|
|
# Query should work (may be empty)
|
|
assert isinstance(templates, list), "Public templates query failed"
|
|
print(f"[PASS] public templates query works ({len(templates)} found)")
|
|
|
|
|
|
def test_marketplace_api_module_exists():
|
|
"""Verify marketplace API module exists"""
|
|
api_path = Path("/opt/agent-governance/marketplace/api.py")
|
|
assert api_path.exists(), "Marketplace API module not found"
|
|
|
|
content = api_path.read_text()
|
|
assert "list_templates" in content, "list_templates endpoint not found"
|
|
assert "create_template" in content, "create_template endpoint not found"
|
|
assert "install_template" in content or "install" in content, "install endpoint not found"
|
|
print("[PASS] marketplace API module exists with required endpoints")
|
|
|
|
|
|
def run_all_tests():
|
|
"""Run all Phase 11 tests"""
|
|
print("=" * 60)
|
|
print("PHASE 11: AGENT MARKETPLACE TESTS")
|
|
print("=" * 60)
|
|
print()
|
|
|
|
tests = [
|
|
test_agent_templates_table_exists,
|
|
test_template_versions_table_exists,
|
|
test_template_ratings_table_exists,
|
|
test_template_stats_table_exists,
|
|
test_template_installations_table_exists,
|
|
test_template_dependencies_table_exists,
|
|
test_marketplace_categories_table_exists,
|
|
test_template_search_fts_exists,
|
|
test_create_template,
|
|
test_create_template_version,
|
|
test_template_stats_tracking,
|
|
test_template_rating,
|
|
test_template_installation,
|
|
test_category_filtering,
|
|
test_public_templates_query,
|
|
test_marketplace_api_module_exists,
|
|
]
|
|
|
|
passed = 0
|
|
failed = 0
|
|
|
|
for test in tests:
|
|
try:
|
|
test()
|
|
passed += 1
|
|
except AssertionError as e:
|
|
print(f"[FAIL] {test.__name__}: {e}")
|
|
failed += 1
|
|
except Exception as e:
|
|
print(f"[ERROR] {test.__name__}: {e}")
|
|
failed += 1
|
|
|
|
print()
|
|
print("=" * 60)
|
|
print(f"RESULTS: {passed} passed, {failed} failed")
|
|
print("=" * 60)
|
|
|
|
return failed == 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
success = run_all_tests()
|
|
sys.exit(0 if success else 1)
|