import asyncio import json import logging import uuid from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any import httpx import pytest from alembic import command from alembic.config import Config from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from lead_scoring.aihub.client import AIHubClient from lead_scoring.db.models.outbox_event import OutboxEvent from lead_scoring.db.models.scoring_run import ScoringRun from lead_scoring.execution import execute_scoring_run from lead_scoring.scoring import ( EffortProfile, FallbackPolicy, RuleMetric, ScoringConfig, ScoringRule, ) def get_alembic_config(database_url: str) -> Config: root_dir = Path(__file__).resolve().parent.parent config = Config(str(root_dir / "alembic.ini")) config.attributes["sqlalchemy_url"] = database_url return config def make_snapshot_data(ems_lead_id: str = "lead-m-1") -> dict[str, Any]: return { "version": "1.0.0", "scope": "linked_contact_history", "captured_at": "2026-08-27T12:00:00Z", "lead_profile": { "ems_lead_id": ems_lead_id, "name": "Nguyen Van C", "phone": "+84987654321", "email": "nvc@example.com", "status": "New", }, "conversations": [ { "conversation_id": "conv-m-1", "messages": [ { "message_id": "msg-m-1", "sender_type": "contact", "content": "Toi can trien khai giai phap AI cho 100 nhan vien", "sent_at": "2026-08-27T11:00:00Z", } ], } ], "identity_context": { "match_quality": "direct", "contact_id": "cnt-m-1", }, "trigger_metadata": { "trigger_type": "manual_omni", "trigger_conversation_id": "conv-m-1", "actor_id": "user-m-1", }, "hitl_policy": { "confidence_threshold": 0.80, "policy_version": "1.0.0", }, } @pytest.mark.asyncio async def test_medium_execution_success_completed(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="medium-config-1", base_score=50, confidence=0.90, confidence_threshold=0.80, hitl_policy_version="policy-1", rules=(ScoringRule(metric=RuleMetric.PROFILE_FIELD_COUNT, minimum=2, points=30),), rule_weight=0.4, ai_weight=0.6, effort_profile=EffortProfile.MEDIUM, ai_model="gpt-4o-mini", fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-med-1", subject_ref="lead-m-1", scoring_config_version="medium-config-1", idempotency_key="idemp-med-1", payload_hash="a" * 64, status="pending", snapshot_data=make_snapshot_data("lead-m-1"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() def handler(request: httpx.Request) -> httpx.Response: assert request.url.path == "/v1/chat/completions" body = json.loads(request.content.decode("utf-8")) schema_name = body.get("response_format", {}).get("json_schema", {}).get("name") if schema_name == "evidence_extraction": return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "summary": "Lead needs AI solution for 100 staff", "intent": "high", "key_signals": ["100 staff", "enterprise solution"], } ) } } ] }, ) elif schema_name == "lead_scoring": return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "score": 90, "confidence": 0.95, "reasoning": "Large enterprise scale and urgent need.", "positive_factors": ["100 staff size"], "negative_factors": [], } ) } } ] }, ) return httpx.Response(404) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) success = await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey-token", lease_seconds=30.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=2), ) assert success is True # Rule score = 50 + 30 = 80 # AI score = 90 # Final score = round(80 * 0.4 + 90 * 0.6) = round(32 + 54) = 86 # Combined confidence = 0.90 * 0.4 + 0.95 * 0.6 = 0.36 + 0.57 = 0.93 >= 0.80 -> completed async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "completed" assert run.rule_score == 80 assert run.ai_score == 90 assert run.final_score == 86 assert run.scoring_mode == "medium_ai" assert run.degraded is False assert run.confidence == 0.93 assert run.ai_metadata is not None assert run.ai_metadata["model"] == "gpt-4o-mini" assert run.ai_metadata["pass_count"] == 2 assert "evidence_summary" not in run.ai_metadata assert "reasoning" not in run.ai_metadata events = list( (await session.execute(select(OutboxEvent).where(OutboxEvent.run_id == run_id))) .scalars() .all() ) assert len(events) == 1 data = events[0].payload["data"] assert data["status"] == "completed" assert data["rule_score"] == 80 assert data["confidence"] == 0.93 assert data["error_code"] is None assert data["retryable"] is None await engine.dispose() @pytest.mark.asyncio async def test_medium_execution_success_review_required_due_to_confidence(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="medium-config-2", base_score=40, confidence=0.60, confidence_threshold=0.80, hitl_policy_version="policy-1", rule_weight=0.4, ai_weight=0.6, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-med-2", subject_ref="lead-m-2", scoring_config_version="medium-config-2", idempotency_key="idemp-med-2", payload_hash="b" * 64, status="pending", snapshot_data=make_snapshot_data("lead-m-2"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() def handler(request: httpx.Request) -> httpx.Response: assert request.url.path == "/v1/chat/completions" body = json.loads(request.content.decode("utf-8")) schema_name = body.get("response_format", {}).get("json_schema", {}).get("name") if schema_name == "evidence_extraction": return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( {"summary": "Limited evidence", "intent": "low"} ) } } ] }, ) elif schema_name == "lead_scoring": return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "score": 45, "confidence": 0.50, "reasoning": "Unclear requirements.", "positive_factors": [], "negative_factors": ["limited info"], } ) } } ] }, ) return httpx.Response(404) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey-token", lease_seconds=30.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=2), ) # Combined confidence = 0.60 * 0.4 + 0.50 * 0.6 = 0.24 + 0.30 = 0.54 < 0.80 -> review_required async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "completed" assert run.final_score == round(40 * 0.4 + 45 * 0.6) assert run.confidence == 0.54 assert run.degraded is False await engine.dispose() @pytest.mark.asyncio async def test_medium_execution_fallback_rules_only_review_on_ai_failure(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="medium-config-3", base_score=70, confidence=0.85, confidence_threshold=0.80, hitl_policy_version="policy-1", fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-med-3", subject_ref="lead-m-3", scoring_config_version="medium-config-3", idempotency_key="idemp-med-3", payload_hash="c" * 64, status="pending", snapshot_data=make_snapshot_data("lead-m-3"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(500, json={"error": "AIHub Gateway Error"}) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey-token", lease_seconds=30.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=2), ) # Fallback to rules_only_review: # status = review_required # final_score = rule_score (70) # ai_score = None (discarded) # degraded = True # scoring_mode = "rules_only_fallback" async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "completed" assert run.rule_score == 70 assert run.ai_score is None assert run.final_score == 70 assert run.scoring_mode == "rules_only_fallback" assert run.degraded is True assert run.ai_metadata is not None assert run.ai_metadata["error_code"] == "aihub_server_error" events = list( (await session.execute(select(OutboxEvent).where(OutboxEvent.run_id == run_id))) .scalars() .all() ) assert len(events) == 1 data = events[0].payload["data"] assert data["status"] == "completed" assert data["rule_score"] == 70 assert data["confidence"] == 0.85 assert data["error_code"] == "aihub_server_error" assert data["retryable"] is False await engine.dispose() @pytest.mark.asyncio async def test_medium_execution_fallback_fail_run(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="medium-config-4", base_score=70, confidence=0.85, confidence_threshold=0.80, hitl_policy_version="policy-1", fallback_policy=FallbackPolicy.FAIL_RUN, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-med-4", subject_ref="lead-m-4", scoring_config_version="medium-config-4", idempotency_key="idemp-med-4", payload_hash="d" * 64, status="pending", snapshot_data=make_snapshot_data("lead-m-4"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(429, json={"error": "Rate limit exceeded"}) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey-token", lease_seconds=30.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=2), ) async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "failed" assert run.error_code == "aihub_quota_exceeded" assert run.retryable is False await engine.dispose() @pytest.mark.asyncio async def test_medium_execution_vkey_sentinel_never_persisted(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) sentinel_vkey = "sentinel-super-secret-vkey-xyz-999" config = ScoringConfig( version="medium-config-5", confidence=0.9, confidence_threshold=0.8, hitl_policy_version="policy-1", ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-med-5", subject_ref="lead-m-5", scoring_config_version="medium-config-5", idempotency_key="idemp-med-5", payload_hash="e" * 64, status="pending", snapshot_data=make_snapshot_data("lead-m-5"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() def handler(request: httpx.Request) -> httpx.Response: assert request.url.path == "/v1/chat/completions" body = json.loads(request.content.decode("utf-8")) schema_name = body.get("response_format", {}).get("json_schema", {}).get("name") if schema_name == "evidence_extraction": return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( {"summary": "Enterprise lead", "intent": "high"} ) } } ] }, ) elif schema_name == "lead_scoring": return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "score": 85, "confidence": 0.9, "reasoning": "Good fit", "positive_factors": ["enterprise"], "negative_factors": [], } ) } } ] }, ) return httpx.Response(404) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) await execute_scoring_run( session_factory, run_id, tenant_vkey=sentinel_vkey, lease_seconds=30.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=2), ) # Check database row and outbox event async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None # Convert entire model and dicts to string and check sentinel run_str = str(run.__dict__) assert sentinel_vkey not in run_str assert sentinel_vkey not in str(run.snapshot_data) assert sentinel_vkey not in str(run.effective_config) assert sentinel_vkey not in str(run.ai_metadata) events = list( (await session.execute(select(OutboxEvent).where(OutboxEvent.run_id == run_id))) .scalars() .all() ) assert len(events) == 1 assert sentinel_vkey not in str(events[0].payload) await engine.dispose() @pytest.mark.asyncio async def test_medium_execution_unconfigured_aihub_client_triggers_fallback(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="medium-config-unconf", base_score=75, confidence=0.9, confidence_threshold=0.8, hitl_policy_version="policy-1", effort_profile=EffortProfile.MEDIUM, fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-unconf", subject_ref="lead-unconf", scoring_config_version="medium-config-unconf", idempotency_key="idemp-unconf", payload_hash="f" * 64, status="pending", snapshot_data=make_snapshot_data("lead-unconf"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() # Pass aihub_client=None explicitly await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey-token", lease_seconds=30.0, aihub_client=None, clock=lambda: created_at + timedelta(seconds=2), ) async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "completed" assert run.rule_score == 75 assert run.ai_score is None assert run.final_score == 75 assert run.scoring_mode == "rules_only_fallback" assert run.degraded is True assert run.ai_metadata is not None assert run.ai_metadata["error_code"] == "aihub_client_unconfigured" await engine.dispose() @pytest.mark.asyncio async def test_medium_execution_non_cancellation_exception_uses_canned_error_and_no_sentinel_leak( temp_postgres_db: str, caplog: pytest.LogCaptureFixture, capsys: pytest.CaptureFixture[str], ): caplog.set_level(logging.DEBUG) await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) sentinel_secret = "sentinel-leak-in-exception-1234567890" config = ScoringConfig( version="medium-config-crash", base_score=60, confidence=0.85, confidence_threshold=0.8, hitl_policy_version="policy-1", effort_profile=EffortProfile.MEDIUM, fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-crash", subject_ref="lead-crash", scoring_config_version="medium-config-crash", idempotency_key="idemp-crash", payload_hash="1" * 64, status="pending", snapshot_data=make_snapshot_data("lead-crash"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() class CrashingClient: async def extract_evidence(self, *args, **kwargs): raise RuntimeError(f"Crashing with secret {sentinel_secret}") async def score_lead(self, *args, **kwargs): raise RuntimeError(f"Crashing with secret {sentinel_secret}") async def analyze_and_score_lead(self, *args, **kwargs): raise RuntimeError(f"Crashing with secret {sentinel_secret}") async def critique_evidence(self, *args, **kwargs): raise RuntimeError(f"Crashing with secret {sentinel_secret}") async def score_lead_heavy(self, *args, **kwargs): raise RuntimeError(f"Crashing with secret {sentinel_secret}") await execute_scoring_run( session_factory, run_id, tenant_vkey=sentinel_secret, lease_seconds=30.0, aihub_client=CrashingClient(), clock=lambda: created_at + timedelta(seconds=2), ) async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "completed" assert run.rule_score == 60 assert run.final_score == 60 assert run.scoring_mode == "rules_only_fallback" assert run.degraded is True assert run.ai_metadata is not None assert run.ai_metadata["error_code"] == "aihub_unknown_error" assert run.ai_metadata["detail"] == "Unexpected internal execution error" # Verify sentinel_secret NEVER appears anywhere in DB record or outbox assert sentinel_secret not in str(run.__dict__) assert sentinel_secret not in str(run.ai_metadata) assert sentinel_secret not in str(run.effective_config) events = list( (await session.execute(select(OutboxEvent).where(OutboxEvent.run_id == run_id))) .scalars() .all() ) assert len(events) == 1 assert sentinel_secret not in str(events[0].payload) # Verify sentinel_secret NEVER appears anywhere in captured logs, stdout, or stderr assert sentinel_secret not in caplog.text captured = capsys.readouterr() assert sentinel_secret not in captured.err assert sentinel_secret not in captured.out await engine.dispose()