8c6e7831e9
Major additions: - marketplace/: Agent template registry with FTS5 search, ratings, versioning - observability/: Prometheus metrics, distributed tracing, structured logging - ledger/migrations/: Database migration scripts for multi-tenant support - tests/governance/: 15 new test files for phases 6-12 (295 total tests) - bin/validate-phases: Full 12-phase validation script New features: - Multi-tenant support with tenant isolation and quota enforcement - Agent marketplace with semantic versioning and search - Observability with metrics, tracing, and log correlation - Tier-1 agent bootstrap scripts Updated components: - ledger/api.py: Extended API for tenants, marketplace, observability - ledger/schema.sql: Added tenant, project, marketplace tables - testing/framework.ts: Enhanced test framework - checkpoint/checkpoint.py: Improved checkpoint management Archived: - External integrations (Slack/GitHub/PagerDuty) moved to .archive/ - Old checkpoint files cleaned up Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
357 lines
11 KiB
Python
357 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Phase 10: Multi-Tenant Support Tests
|
|
====================================
|
|
Validates multi-tenant schema, isolation, quotas, and RBAC.
|
|
"""
|
|
|
|
import json
|
|
import sqlite3
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Add parent to path
|
|
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
|
|
|
|
DB_PATH = Path("/opt/agent-governance/ledger/governance.db")
|
|
|
|
|
|
def get_db():
|
|
conn = sqlite3.connect(DB_PATH)
|
|
conn.row_factory = sqlite3.Row
|
|
return conn
|
|
|
|
|
|
def test_tenants_table_exists():
|
|
"""Verify tenants table exists with required columns"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(tenants)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
required = {"tenant_id", "name", "slug", "subscription_tier", "is_active"}
|
|
assert required.issubset(columns), f"Missing columns: {required - columns}"
|
|
print("[PASS] tenants table exists with required columns")
|
|
|
|
|
|
def test_projects_table_exists():
|
|
"""Verify projects table exists with required columns"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(projects)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
required = {"project_id", "tenant_id", "name", "slug", "is_active"}
|
|
assert required.issubset(columns), f"Missing columns: {required - columns}"
|
|
print("[PASS] projects table exists with required columns")
|
|
|
|
|
|
def test_tenant_quotas_table_exists():
|
|
"""Verify tenant_quotas table exists with quota fields"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(tenant_quotas)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
required = {"tenant_id", "max_projects", "max_agents_per_project", "max_api_calls_per_day"}
|
|
assert required.issubset(columns), f"Missing columns: {required - columns}"
|
|
print("[PASS] tenant_quotas table exists with quota fields")
|
|
|
|
|
|
def test_tenant_usage_table_exists():
|
|
"""Verify tenant_usage table for tracking consumption"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(tenant_usage)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
required = {"tenant_id", "period_start", "api_calls", "tokens_used"}
|
|
assert required.issubset(columns), f"Missing columns: {required - columns}"
|
|
print("[PASS] tenant_usage table exists for consumption tracking")
|
|
|
|
|
|
def test_api_keys_table_exists():
|
|
"""Verify api_keys table for tenant authentication"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(api_keys)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
required = {"key_id", "tenant_id", "key_hash", "is_active"}
|
|
assert required.issubset(columns), f"Missing columns: {required - columns}"
|
|
print("[PASS] api_keys table exists for tenant authentication")
|
|
|
|
|
|
def test_default_tenant_exists():
|
|
"""Verify default tenant was created"""
|
|
conn = get_db()
|
|
cursor = conn.execute("SELECT * FROM tenants WHERE tenant_id = 'default'")
|
|
row = cursor.fetchone()
|
|
conn.close()
|
|
|
|
assert row is not None, "Default tenant not found"
|
|
assert row["is_active"] == 1, "Default tenant should be active"
|
|
print("[PASS] default tenant exists and is active")
|
|
|
|
|
|
def test_default_project_exists():
|
|
"""Verify default project was created"""
|
|
conn = get_db()
|
|
cursor = conn.execute("SELECT * FROM projects WHERE project_id = 'default'")
|
|
row = cursor.fetchone()
|
|
conn.close()
|
|
|
|
assert row is not None, "Default project not found"
|
|
assert row["tenant_id"] == "default", "Default project should belong to default tenant"
|
|
print("[PASS] default project exists under default tenant")
|
|
|
|
|
|
def test_default_quotas_exist():
|
|
"""Verify default tenant has quotas"""
|
|
conn = get_db()
|
|
cursor = conn.execute("SELECT * FROM tenant_quotas WHERE tenant_id = 'default'")
|
|
row = cursor.fetchone()
|
|
conn.close()
|
|
|
|
assert row is not None, "Default tenant quotas not found"
|
|
assert row["max_projects"] > 0, "Should have positive project quota"
|
|
assert row["max_api_calls_per_day"] > 0, "Should have positive API call quota"
|
|
print("[PASS] default tenant has quotas configured")
|
|
|
|
|
|
def test_agent_metrics_has_tenant_columns():
|
|
"""Verify agent_metrics table has tenant isolation columns"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(agent_metrics)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
assert "tenant_id" in columns, "agent_metrics missing tenant_id"
|
|
assert "project_id" in columns, "agent_metrics missing project_id"
|
|
print("[PASS] agent_metrics has tenant isolation columns")
|
|
|
|
|
|
def test_agent_actions_has_tenant_columns():
|
|
"""Verify agent_actions table has tenant isolation columns"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(agent_actions)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
assert "tenant_id" in columns, "agent_actions missing tenant_id"
|
|
assert "project_id" in columns, "agent_actions missing project_id"
|
|
print("[PASS] agent_actions has tenant isolation columns")
|
|
|
|
|
|
def test_violations_has_tenant_columns():
|
|
"""Verify violations table has tenant isolation columns"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(violations)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
assert "tenant_id" in columns, "violations missing tenant_id"
|
|
assert "project_id" in columns, "violations missing project_id"
|
|
print("[PASS] violations has tenant isolation columns")
|
|
|
|
|
|
def test_promotions_has_tenant_columns():
|
|
"""Verify promotions table has tenant isolation columns"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(promotions)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
assert "tenant_id" in columns, "promotions missing tenant_id"
|
|
assert "project_id" in columns, "promotions missing project_id"
|
|
print("[PASS] promotions has tenant isolation columns")
|
|
|
|
|
|
def test_project_members_table_exists():
|
|
"""Verify project_members table for RBAC"""
|
|
conn = get_db()
|
|
cursor = conn.execute("PRAGMA table_info(project_members)")
|
|
columns = {row[1] for row in cursor.fetchall()}
|
|
conn.close()
|
|
|
|
required = {"project_id", "user_id", "role"}
|
|
assert required.issubset(columns), f"Missing columns: {required - columns}"
|
|
print("[PASS] project_members table exists for RBAC")
|
|
|
|
|
|
def test_tenant_index_exists():
|
|
"""Verify tenant indexes exist for performance"""
|
|
conn = get_db()
|
|
cursor = conn.execute("SELECT name FROM sqlite_master WHERE type='index' AND name LIKE '%tenant%'")
|
|
indexes = [row[0] for row in cursor.fetchall()]
|
|
conn.close()
|
|
|
|
assert len(indexes) > 0, "No tenant indexes found"
|
|
print(f"[PASS] tenant indexes exist: {len(indexes)} found")
|
|
|
|
|
|
def test_create_tenant():
|
|
"""Test creating a new tenant"""
|
|
conn = get_db()
|
|
test_id = "test-tenant-validate"
|
|
|
|
# Clean up first
|
|
conn.execute("DELETE FROM tenants WHERE tenant_id = ?", (test_id,))
|
|
conn.commit()
|
|
|
|
# Create tenant
|
|
conn.execute("""
|
|
INSERT INTO tenants (tenant_id, name, slug, subscription_tier)
|
|
VALUES (?, 'Test Tenant', 'test-validate', 'free')
|
|
""", (test_id,))
|
|
conn.commit()
|
|
|
|
# Verify
|
|
cursor = conn.execute("SELECT * FROM tenants WHERE tenant_id = ?", (test_id,))
|
|
row = cursor.fetchone()
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM tenants WHERE tenant_id = ?", (test_id,))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
assert row is not None, "Failed to create tenant"
|
|
assert row["name"] == "Test Tenant"
|
|
print("[PASS] tenant creation works")
|
|
|
|
|
|
def test_create_project_with_tenant():
|
|
"""Test creating a project under a tenant"""
|
|
conn = get_db()
|
|
tenant_id = "test-tenant-proj"
|
|
project_id = "test-project-validate"
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM projects WHERE project_id = ?", (project_id,))
|
|
conn.execute("DELETE FROM tenants WHERE tenant_id = ?", (tenant_id,))
|
|
conn.commit()
|
|
|
|
# Create tenant and project
|
|
conn.execute("""
|
|
INSERT INTO tenants (tenant_id, name, slug) VALUES (?, 'Test', 'test')
|
|
""", (tenant_id,))
|
|
conn.execute("""
|
|
INSERT INTO projects (project_id, tenant_id, name, slug)
|
|
VALUES (?, ?, 'Test Project', 'test-proj')
|
|
""", (project_id, tenant_id))
|
|
conn.commit()
|
|
|
|
# Verify foreign key relationship
|
|
cursor = conn.execute("""
|
|
SELECT p.*, t.name as tenant_name
|
|
FROM projects p
|
|
JOIN tenants t ON p.tenant_id = t.tenant_id
|
|
WHERE p.project_id = ?
|
|
""", (project_id,))
|
|
row = cursor.fetchone()
|
|
|
|
# Clean up
|
|
conn.execute("DELETE FROM projects WHERE project_id = ?", (project_id,))
|
|
conn.execute("DELETE FROM tenants WHERE tenant_id = ?", (tenant_id,))
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
assert row is not None, "Failed to create project"
|
|
assert row["tenant_name"] == "Test"
|
|
print("[PASS] project creation with tenant relationship works")
|
|
|
|
|
|
def test_tenant_isolation_query():
|
|
"""Test that queries can filter by tenant"""
|
|
conn = get_db()
|
|
|
|
# Query agents with tenant filter
|
|
cursor = conn.execute("""
|
|
SELECT COUNT(*) as count FROM agent_metrics
|
|
WHERE tenant_id = 'default' AND project_id = 'default'
|
|
""")
|
|
row = cursor.fetchone()
|
|
conn.close()
|
|
|
|
# Should not error - just verify query works
|
|
assert row is not None, "Tenant-filtered query failed"
|
|
print(f"[PASS] tenant isolation query works (found {row['count']} agents)")
|
|
|
|
|
|
def test_quota_tracking_insert():
|
|
"""Test inserting usage tracking records"""
|
|
conn = get_db()
|
|
|
|
# Insert usage record
|
|
conn.execute("""
|
|
INSERT OR REPLACE INTO tenant_usage (tenant_id, period_start, api_calls, tokens_used)
|
|
VALUES ('default', date('now'), 100, 5000)
|
|
""")
|
|
conn.commit()
|
|
|
|
# Verify
|
|
cursor = conn.execute("""
|
|
SELECT * FROM tenant_usage WHERE tenant_id = 'default' AND period_start = date('now')
|
|
""")
|
|
row = cursor.fetchone()
|
|
conn.close()
|
|
|
|
assert row is not None, "Failed to track usage"
|
|
assert row["api_calls"] == 100
|
|
print("[PASS] quota usage tracking works")
|
|
|
|
|
|
def run_all_tests():
|
|
"""Run all Phase 10 tests"""
|
|
print("=" * 60)
|
|
print("PHASE 10: MULTI-TENANT SUPPORT TESTS")
|
|
print("=" * 60)
|
|
print()
|
|
|
|
tests = [
|
|
test_tenants_table_exists,
|
|
test_projects_table_exists,
|
|
test_tenant_quotas_table_exists,
|
|
test_tenant_usage_table_exists,
|
|
test_api_keys_table_exists,
|
|
test_default_tenant_exists,
|
|
test_default_project_exists,
|
|
test_default_quotas_exist,
|
|
test_agent_metrics_has_tenant_columns,
|
|
test_agent_actions_has_tenant_columns,
|
|
test_violations_has_tenant_columns,
|
|
test_promotions_has_tenant_columns,
|
|
test_project_members_table_exists,
|
|
test_tenant_index_exists,
|
|
test_create_tenant,
|
|
test_create_project_with_tenant,
|
|
test_tenant_isolation_query,
|
|
test_quota_tracking_insert,
|
|
]
|
|
|
|
passed = 0
|
|
failed = 0
|
|
|
|
for test in tests:
|
|
try:
|
|
test()
|
|
passed += 1
|
|
except AssertionError as e:
|
|
print(f"[FAIL] {test.__name__}: {e}")
|
|
failed += 1
|
|
except Exception as e:
|
|
print(f"[ERROR] {test.__name__}: {e}")
|
|
failed += 1
|
|
|
|
print()
|
|
print("=" * 60)
|
|
print(f"RESULTS: {passed} passed, {failed} failed")
|
|
print("=" * 60)
|
|
|
|
return failed == 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
success = run_all_tests()
|
|
sys.exit(0 if success else 1)
|