import asyncio import json import logging import uuid from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any import httpx import pytest from alembic import command from alembic.config import Config from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from lead_scoring.aihub.client import AIHubClient from lead_scoring.aihub.errors import AIHubError from lead_scoring.aihub.schemas import EvidenceExtractionResult from lead_scoring.db.models.outbox_event import OutboxEvent from lead_scoring.db.models.scoring_run import ScoringRun from lead_scoring.execution import execute_scoring_run from lead_scoring.scoring import ( EffortProfile, FallbackPolicy, RuleMetric, ScoringConfig, ScoringMode, ScoringRule, ) def get_alembic_config(database_url: str) -> Config: root_dir = Path(__file__).resolve().parent.parent config = Config(str(root_dir / "alembic.ini")) config.attributes["sqlalchemy_url"] = database_url return config def make_snapshot_data(ems_lead_id: str = "lead-p-1") -> dict[str, Any]: return { "version": "1.0.0", "scope": "linked_contact_history", "captured_at": "2026-08-27T12:00:00Z", "lead_profile": { "ems_lead_id": ems_lead_id, "name": "Le Thi D", "phone": "+84901234567", "email": "ltd@example.com", "status": "New", }, "conversations": [ { "conversation_id": "conv-p-1", "messages": [ { "message_id": "msg-p-1", "sender_type": "contact", "content": "Toi can tu van goi AI Lead Scoring cho cong ty", "sent_at": "2026-08-27T11:00:00Z", } ], } ], "identity_context": { "match_quality": "direct", "contact_id": "cnt-p-1", }, "trigger_metadata": { "trigger_type": "manual_omni", "trigger_conversation_id": "conv-p-1", "actor_id": "user-p-1", }, "hitl_policy": { "confidence_threshold": 0.80, "policy_version": "1.0.0", }, } @pytest.mark.asyncio async def test_light_execution_success_completed(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="light-config-1", base_score=50, confidence=0.85, confidence_threshold=0.80, hitl_policy_version="policy-1", rules=(ScoringRule(metric=RuleMetric.PROFILE_FIELD_COUNT, minimum=2, points=30),), rule_weight=0.3, ai_weight=0.7, effort_profile=EffortProfile.LIGHT, ai_model="openai/gpt-4o-mini", fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-light-1", subject_ref="lead-p-1", scoring_config_version="light-config-1", idempotency_key="idemp-light-1", payload_hash="1" * 64, status="pending", snapshot_data=make_snapshot_data("lead-p-1"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() call_count = 0 def handler(request: httpx.Request) -> httpx.Response: nonlocal call_count call_count += 1 assert request.url.path == "/v1/chat/completions" body = json.loads(request.content.decode("utf-8")) schema_name = body.get("response_format", {}).get("json_schema", {}).get("name") assert schema_name == "light_lead_scoring" return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "score": 85, "confidence": 0.90, "reasoning": "Single pass light evaluation.", "positive_factors": ["Direct contact interest"], "negative_factors": [], } ) } } ] }, ) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) success = await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey-token", lease_seconds=30.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=2), ) assert success is True assert call_count == 1 # Rule score = 50 + 30 = 80 # AI score = 85 # Final score = round(80 * 0.3 + 85 * 0.7) = round(24 + 59.5) = 84 # Combined confidence = 0.85 * 0.3 + 0.90 * 0.7 = 0.255 + 0.63 = 0.885 >= 0.80 -> completed async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "completed" assert run.rule_score == 80 assert run.ai_score == 85 assert run.final_score == 84 assert run.scoring_mode == ScoringMode.LIGHT_AI.value assert run.degraded is False assert run.confidence == 0.885 assert run.ai_metadata == { "model": "openai/gpt-4o-mini", "effort_profile": "light", "pass_count": 1, } events = list( (await session.execute(select(OutboxEvent).where(OutboxEvent.run_id == run_id))) .scalars() .all() ) assert len(events) == 1 data = events[0].payload["data"] assert data["status"] == "completed" assert data["rule_score"] == 80 assert data["confidence"] == 0.885 assert data["error_code"] is None assert data["retryable"] is None await engine.dispose() @pytest.mark.asyncio async def test_heavy_execution_success_completed(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="heavy-config-1", base_score=40, confidence=0.90, confidence_threshold=0.80, hitl_policy_version="policy-1", rules=(ScoringRule(metric=RuleMetric.PROFILE_FIELD_COUNT, minimum=2, points=20),), rule_weight=0.2, ai_weight=0.8, effort_profile=EffortProfile.HEAVY, ai_model="openai/gpt-4o-mini", fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-heavy-1", subject_ref="lead-p-2", scoring_config_version="heavy-config-1", idempotency_key="idemp-heavy-1", payload_hash="2" * 64, status="pending", snapshot_data=make_snapshot_data("lead-p-2"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=40), ) ) await session.commit() call_sequence: list[str] = [] def handler(request: httpx.Request) -> httpx.Response: assert request.url.path == "/v1/chat/completions" body = json.loads(request.content.decode("utf-8")) schema_name = body.get("response_format", {}).get("json_schema", {}).get("name") call_sequence.append(schema_name) if schema_name == "evidence_extraction": return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "summary": "Enterprise interest in AI Platform", "intent": "high", "key_signals": ["Enterprise lead"], } ) } } ] }, ) elif schema_name == "evidence_critique": return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "critique_summary": ( "High value potential with strong buy-in" ), "risks": [], "strengths": ["Enterprise scale"], "cross_check_notes": "Validated across conversations", } ) } } ] }, ) elif schema_name == "heavy_lead_scoring": return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "score": 95, "confidence": 0.95, "reasoning": "High confidence enterprise lead", "positive_factors": ["Enterprise scale", "Strong buy-in"], "negative_factors": [], } ) } } ] }, ) return httpx.Response(404) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) success = await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey-token", lease_seconds=40.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=3), ) assert success is True assert call_sequence == ["evidence_extraction", "evidence_critique", "heavy_lead_scoring"] # Rule score = 40 + 20 = 60 # AI score = 95 # Final score = round(60 * 0.2 + 95 * 0.8) = round(12 + 76) = 88 # Combined confidence = 0.90 * 0.2 + 0.95 * 0.8 = 0.18 + 0.76 = 0.94 >= 0.80 -> completed async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "completed" assert run.rule_score == 60 assert run.ai_score == 95 assert run.final_score == 88 assert run.scoring_mode == ScoringMode.HEAVY_AI.value assert run.degraded is False assert run.confidence == 0.94 assert run.ai_metadata == { "model": "openai/gpt-4o-mini", "effort_profile": "heavy", "pass_count": 3, } await engine.dispose() @pytest.mark.asyncio async def test_heavy_execution_fallback_rules_only_review_on_pass_2_failure(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="heavy-config-fallback-1", base_score=65, confidence=0.85, confidence_threshold=0.80, hitl_policy_version="policy-1", effort_profile=EffortProfile.HEAVY, fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-heavy-fb", subject_ref="lead-p-3", scoring_config_version="heavy-config-fallback-1", idempotency_key="idemp-heavy-fb", payload_hash="3" * 64, status="pending", snapshot_data=make_snapshot_data("lead-p-3"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=40), ) ) await session.commit() call_sequence: list[str] = [] def handler(request: httpx.Request) -> httpx.Response: body = json.loads(request.content.decode("utf-8")) schema_name = body.get("response_format", {}).get("json_schema", {}).get("name") call_sequence.append(schema_name) if schema_name == "evidence_extraction": return httpx.Response( 200, json={"choices": [{"message": {"content": json.dumps({"summary": "Ok"})}}]}, ) elif schema_name == "evidence_critique": return httpx.Response(429, json={"error": "Rate limit exceeded in Pass 2"}) return httpx.Response(404) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey-token", lease_seconds=40.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=3), ) # Exactly 2 calls (Pass 3 was never called) assert call_sequence == ["evidence_extraction", "evidence_critique"] # Fallback to rules_only_review: async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "completed" assert run.rule_score == 65 assert run.ai_score is None # Partial results discarded assert run.final_score == 65 assert run.scoring_mode == ScoringMode.RULES_ONLY_FALLBACK.value assert run.degraded is True assert run.confidence == 0.85 assert run.ai_metadata == { "error_code": "aihub_quota_exceeded", "detail": "Quota or rate limit exceeded on AIHub", "fallback_policy": "rules_only_review", } await engine.dispose() @pytest.mark.asyncio async def test_heavy_execution_fallback_fail_run_on_pass_3_timeout(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="heavy-config-fail-1", base_score=65, confidence=0.85, confidence_threshold=0.80, hitl_policy_version="policy-1", effort_profile=EffortProfile.HEAVY, fallback_policy=FallbackPolicy.FAIL_RUN, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-heavy-fail", subject_ref="lead-p-4", scoring_config_version="heavy-config-fail-1", idempotency_key="idemp-heavy-fail", payload_hash="4" * 64, status="pending", snapshot_data=make_snapshot_data("lead-p-4"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=40), ) ) await session.commit() call_sequence: list[str] = [] def handler(request: httpx.Request) -> httpx.Response: body = json.loads(request.content.decode("utf-8")) schema_name = body.get("response_format", {}).get("json_schema", {}).get("name") call_sequence.append(schema_name) if schema_name == "evidence_extraction": return httpx.Response( 200, json={"choices": [{"message": {"content": json.dumps({"summary": "Ok"})}}]}, ) elif schema_name == "evidence_critique": return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "critique_summary": "Critique ok", "risks": [], "strengths": [], } ) } } ] }, ) elif schema_name == "heavy_lead_scoring": raise httpx.ReadTimeout("Pass 3 timed out after 10s") return httpx.Response(404) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey-token", lease_seconds=40.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=3), ) assert call_sequence == ["evidence_extraction", "evidence_critique", "heavy_lead_scoring"] # Fallback to fail_run: async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "failed" assert run.rule_score is None assert run.ai_score is None assert run.final_score is None assert run.scoring_mode is None assert run.confidence is None assert run.degraded is True assert run.error_code == "aihub_timeout" assert run.retryable is False assert run.ai_metadata == { "error_code": "aihub_timeout", "detail": "Request to AIHub timed out", "fallback_policy": "fail_run", } events = list( (await session.execute(select(OutboxEvent).where(OutboxEvent.run_id == run_id))) .scalars() .all() ) assert len(events) == 1 data = events[0].payload["data"] assert data["status"] == "failed" assert data["error_code"] == "aihub_timeout" assert data["retryable"] is False await engine.dispose() @pytest.mark.asyncio async def test_sentinel_vkey_never_persisted_in_light_and_heavy_profiles( temp_postgres_db: str, caplog: pytest.LogCaptureFixture, capsys: pytest.CaptureFixture[str], ): caplog.set_level(logging.DEBUG) await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) sentinel_vkey_light = "sentinel-light-secret-vkey-8888" sentinel_vkey_heavy = "sentinel-heavy-secret-vkey-9999" # 1. Test Light profile with Sentinel run_id_light = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config_light = ScoringConfig( version="light-sentinel-1", confidence=0.85, confidence_threshold=0.80, hitl_policy_version="policy-1", effort_profile=EffortProfile.LIGHT, ) async with session_factory() as session: session.add( ScoringRun( id=run_id_light, tenant_id="tenant-s-light", subject_ref="lead-s-1", scoring_config_version="light-sentinel-1", idempotency_key="idemp-s-light", payload_hash="s1" * 32, status="pending", snapshot_data=make_snapshot_data("lead-s-1"), effective_config=config_light.model_dump(mode="json"), confidence_threshold=config_light.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() def handler_light(request: httpx.Request) -> httpx.Response: return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "score": 80, "confidence": 0.85, "reasoning": "Light sentinel run", "positive_factors": [], "negative_factors": [], } ) } } ] }, ) transport_light = httpx.MockTransport(handler_light) async with httpx.AsyncClient(transport=transport_light) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) await execute_scoring_run( session_factory, run_id_light, tenant_vkey=sentinel_vkey_light, lease_seconds=30.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=2), ) # 2. Test Heavy profile with Sentinel (failing under fail_run) run_id_heavy = str(uuid.uuid4()) config_heavy = ScoringConfig( version="heavy-sentinel-1", confidence=0.85, confidence_threshold=0.80, hitl_policy_version="policy-1", effort_profile=EffortProfile.HEAVY, fallback_policy=FallbackPolicy.FAIL_RUN, ) async with session_factory() as session: session.add( ScoringRun( id=run_id_heavy, tenant_id="tenant-s-heavy", subject_ref="lead-s-2", scoring_config_version="heavy-sentinel-1", idempotency_key="idemp-s-heavy", payload_hash="s2" * 32, status="pending", snapshot_data=make_snapshot_data("lead-s-2"), effective_config=config_heavy.model_dump(mode="json"), confidence_threshold=config_heavy.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=40), ) ) await session.commit() def handler_heavy(request: httpx.Request) -> httpx.Response: return httpx.Response(500, json={"error": "Sentinel error"}) transport_heavy = httpx.MockTransport(handler_heavy) async with httpx.AsyncClient(transport=transport_heavy) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) await execute_scoring_run( session_factory, run_id_heavy, tenant_vkey=sentinel_vkey_heavy, lease_seconds=40.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=2), ) # Verify sentinel secrets are absent everywhere in DB and Outbox async with session_factory() as session: run_l = await session.get(ScoringRun, run_id_light) assert run_l is not None assert sentinel_vkey_light not in str(run_l.__dict__) assert sentinel_vkey_light not in str(run_l.ai_metadata) run_h = await session.get(ScoringRun, run_id_heavy) assert run_h is not None assert sentinel_vkey_heavy not in str(run_h.__dict__) assert sentinel_vkey_heavy not in str(run_h.ai_metadata) events = list((await session.execute(select(OutboxEvent))).scalars().all()) for event in events: assert sentinel_vkey_light not in str(event.payload) assert sentinel_vkey_heavy not in str(event.payload) # Verify sentinel secrets are absent from captured logs, stdout, and stderr assert sentinel_vkey_light not in caplog.text assert sentinel_vkey_heavy not in caplog.text captured = capsys.readouterr() assert sentinel_vkey_light not in captured.out assert sentinel_vkey_light not in captured.err assert sentinel_vkey_heavy not in captured.out assert sentinel_vkey_heavy not in captured.err await engine.dispose() @pytest.mark.asyncio async def test_light_execution_review_required_due_to_confidence(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="light-conf-rr", base_score=40, confidence=0.60, confidence_threshold=0.80, hitl_policy_version="policy-1", rule_weight=0.4, ai_weight=0.6, effort_profile=EffortProfile.LIGHT, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-light-rr", subject_ref="lead-p-rr", scoring_config_version="light-conf-rr", idempotency_key="idemp-light-rr", payload_hash="5" * 64, status="pending", snapshot_data=make_snapshot_data("lead-p-rr"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() def handler(request: httpx.Request) -> httpx.Response: return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "score": 50, "confidence": 0.50, "reasoning": "Uncertain light evaluation", "positive_factors": [], "negative_factors": ["Sparse context"], } ) } } ] }, ) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey", lease_seconds=30.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=2), ) # Combined confidence = 0.60 * 0.4 + 0.50 * 0.6 = 0.24 + 0.30 = 0.54 < 0.80 -> review_required async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "completed" assert run.rule_score == 40 assert run.ai_score == 50 assert run.final_score == round(40 * 0.4 + 50 * 0.6) assert run.scoring_mode == ScoringMode.LIGHT_AI.value assert run.degraded is False assert run.confidence == 0.54 await engine.dispose() @pytest.mark.asyncio async def test_heavy_execution_review_required_due_to_confidence(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="heavy-conf-rr", base_score=40, confidence=0.50, confidence_threshold=0.80, hitl_policy_version="policy-1", rule_weight=0.3, ai_weight=0.7, effort_profile=EffortProfile.HEAVY, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-heavy-rr", subject_ref="lead-p-hrr", scoring_config_version="heavy-conf-rr", idempotency_key="idemp-heavy-rr", payload_hash="6" * 64, status="pending", snapshot_data=make_snapshot_data("lead-p-hrr"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=40), ) ) await session.commit() def handler(request: httpx.Request) -> httpx.Response: body = json.loads(request.content.decode("utf-8")) schema_name = body.get("response_format", {}).get("json_schema", {}).get("name") if schema_name == "evidence_extraction": return httpx.Response( 200, json={"choices": [{"message": {"content": json.dumps({"summary": "Weak info"})}}]}, ) elif schema_name == "evidence_critique": return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "critique_summary": "High risk", "risks": ["Vague requirements"], "strengths": [], } ) } } ] }, ) elif schema_name == "heavy_lead_scoring": return httpx.Response( 200, json={ "choices": [ { "message": { "content": json.dumps( { "score": 35, "confidence": 0.40, "reasoning": "Low confidence due to risks", "positive_factors": [], "negative_factors": ["High risk"], } ) } } ] }, ) return httpx.Response(404) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey", lease_seconds=40.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=3), ) # Combined confidence = 0.50 * 0.3 + 0.40 * 0.7 = 0.15 + 0.28 = 0.43 < 0.80 -> review_required async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "completed" assert run.rule_score == 40 assert run.ai_score == 35 assert run.final_score == round(40 * 0.3 + 35 * 0.7) assert run.scoring_mode == ScoringMode.HEAVY_AI.value assert run.degraded is False assert run.confidence == 0.43 await engine.dispose() @pytest.mark.asyncio async def test_light_execution_fallback_rules_only_review_on_5xx(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="light-fb-5xx", base_score=72, confidence=0.88, confidence_threshold=0.80, hitl_policy_version="policy-1", effort_profile=EffortProfile.LIGHT, fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-l-5xx", subject_ref="lead-p-5xx", scoring_config_version="light-fb-5xx", idempotency_key="idemp-l-5xx", payload_hash="7" * 64, status="pending", snapshot_data=make_snapshot_data("lead-p-5xx"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(500, json={"error": "AIHub Server Error"}) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey", lease_seconds=30.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=2), ) async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "completed" assert run.rule_score == 72 assert run.ai_score is None assert run.final_score == 72 assert run.scoring_mode == ScoringMode.RULES_ONLY_FALLBACK.value assert run.degraded is True assert run.ai_metadata == { "error_code": "aihub_server_error", "detail": "AIHub returned server error", "fallback_policy": "rules_only_review", } await engine.dispose() @pytest.mark.asyncio async def test_light_execution_fallback_fail_run_on_quota(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config = ScoringConfig( version="light-fail-429", base_score=72, confidence=0.88, confidence_threshold=0.80, hitl_policy_version="policy-1", effort_profile=EffortProfile.LIGHT, fallback_policy=FallbackPolicy.FAIL_RUN, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-l-429", subject_ref="lead-p-429", scoring_config_version="light-fail-429", idempotency_key="idemp-l-429", payload_hash="8" * 64, status="pending", snapshot_data=make_snapshot_data("lead-p-429"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(429, json={"error": "Rate limit exceeded"}) transport = httpx.MockTransport(handler) async with httpx.AsyncClient(transport=transport) as http_client: aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client) await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey", lease_seconds=30.0, aihub_client=aihub_client, clock=lambda: created_at + timedelta(seconds=2), ) async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "failed" assert run.rule_score is None assert run.ai_score is None assert run.final_score is None assert run.scoring_mode is None assert run.error_code == "aihub_quota_exceeded" assert run.retryable is False assert run.ai_metadata == { "error_code": "aihub_quota_exceeded", "detail": "Quota or rate limit exceeded on AIHub", "fallback_policy": "fail_run", } await engine.dispose() @pytest.mark.asyncio async def test_missing_vkey_light_execution_falls_back_to_rules_only(temp_postgres_db: str): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) # Test Light with unconfigured client run_id_l = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) config_l = ScoringConfig( version="unconf-light", base_score=60, confidence=0.8, confidence_threshold=0.8, hitl_policy_version="policy-1", effort_profile=EffortProfile.LIGHT, fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW, ) async with session_factory() as session: session.add( ScoringRun( id=run_id_l, tenant_id="tenant-unconf-l", subject_ref="lead-unconf-l", scoring_config_version="unconf-light", idempotency_key="idemp-unconf-l", payload_hash="9" * 64, status="pending", snapshot_data=make_snapshot_data("lead-unconf-l"), effective_config=config_l.model_dump(mode="json"), confidence_threshold=config_l.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() await execute_scoring_run( session_factory, run_id_l, tenant_vkey=None, lease_seconds=30.0, aihub_client=None, clock=lambda: created_at + timedelta(seconds=2), ) async with session_factory() as session: run = await session.get(ScoringRun, run_id_l) assert run is not None assert run.status == "completed" assert run.rule_score == 60 assert run.ai_score is None assert run.final_score == 60 assert run.scoring_mode == ScoringMode.RULES_ONLY_FALLBACK.value assert run.degraded is True assert run.error_code == "aihub_vkey_missing" assert run.ai_metadata == { "error_code": "aihub_vkey_missing", "detail": "Tenant AIHub credential is missing", "fallback_policy": "rules_only_review", } await engine.dispose() @pytest.mark.asyncio async def test_non_cancellation_exception_light_and_heavy_uses_canned_error( temp_postgres_db: str, caplog: pytest.LogCaptureFixture, capsys: pytest.CaptureFixture[str], ): caplog.set_level(logging.DEBUG) await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) sentinel_secret = "sentinel-leak-crash-exception-7777" config = ScoringConfig( version="heavy-crash-cfg", base_score=55, confidence=0.85, confidence_threshold=0.8, hitl_policy_version="policy-1", effort_profile=EffortProfile.HEAVY, fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-h-crash", subject_ref="lead-h-crash", scoring_config_version="heavy-crash-cfg", idempotency_key="idemp-h-crash", payload_hash="a" * 64, status="pending", snapshot_data=make_snapshot_data("lead-h-crash"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=40), ) ) await session.commit() class CrashingClient: async def extract_evidence(self, *args, **kwargs): return EvidenceExtractionResult(summary="ok") async def critique_evidence(self, *args, **kwargs): raise RuntimeError(f"Crashing during heavy critique with secret {sentinel_secret}") async def score_lead_heavy(self, *args, **kwargs): raise RuntimeError(f"Crashing during heavy scoring with secret {sentinel_secret}") async def analyze_and_score_lead(self, *args, **kwargs): raise RuntimeError(f"Crashing during light with secret {sentinel_secret}") async def score_lead(self, *args, **kwargs): raise RuntimeError(f"Crashing with secret {sentinel_secret}") await execute_scoring_run( session_factory, run_id, tenant_vkey=sentinel_secret, lease_seconds=40.0, aihub_client=CrashingClient(), clock=lambda: created_at + timedelta(seconds=2), ) async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None assert run.status == "completed" assert run.rule_score == 55 assert run.final_score == 55 assert run.scoring_mode == ScoringMode.RULES_ONLY_FALLBACK.value assert run.degraded is True assert run.ai_metadata == { "error_code": "aihub_unknown_error", "detail": "Unexpected internal execution error", "fallback_policy": "rules_only_review", } # Verify sentinel_secret NEVER appears anywhere in DB record or outbox assert sentinel_secret not in str(run.__dict__) assert sentinel_secret not in str(run.ai_metadata) assert sentinel_secret not in str(run.effective_config) # Verify sentinel_secret NEVER appears anywhere in captured logs, stdout, or stderr assert sentinel_secret not in caplog.text captured = capsys.readouterr() assert sentinel_secret not in captured.err assert sentinel_secret not in captured.out await engine.dispose() @pytest.mark.asyncio async def test_execution_fallback_allowed_false_forces_failed_status_under_rules_only_review( temp_postgres_db: str, ): await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head") engine = create_async_engine(temp_postgres_db) session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) run_id = str(uuid.uuid4()) created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC) # Config has rules_only_review fallback policy config = ScoringConfig( version="fb-not-allowed-cfg", base_score=70, confidence=0.85, confidence_threshold=0.8, hitl_policy_version="policy-1", effort_profile=EffortProfile.MEDIUM, fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW, ) async with session_factory() as session: session.add( ScoringRun( id=run_id, tenant_id="tenant-no-fb", subject_ref="lead-no-fb", scoring_config_version="fb-not-allowed-cfg", idempotency_key="idemp-no-fb", payload_hash="b" * 64, status="pending", snapshot_data=make_snapshot_data("lead-no-fb"), effective_config=config.model_dump(mode="json"), confidence_threshold=config.confidence_threshold, created_at=created_at, updated_at=created_at, lease_expires_at=created_at + timedelta(seconds=30), ) ) await session.commit() class NonFallbackErrorClient: async def extract_evidence(self, *args, **kwargs): raise AIHubError( "Critical auth failure not allowed to fallback", error_code="aihub_non_fallbackable_error", retryable=False, fallback_allowed=False, detail="Non-fallbackable failure", ) async def score_lead(self, *args, **kwargs): raise AIHubError("Score error", fallback_allowed=False) async def analyze_and_score_lead(self, *args, **kwargs): raise AIHubError("Light error", fallback_allowed=False) async def critique_evidence(self, *args, **kwargs): raise AIHubError("Critique error", fallback_allowed=False) async def score_lead_heavy(self, *args, **kwargs): raise AIHubError("Heavy error", fallback_allowed=False) await execute_scoring_run( session_factory, run_id, tenant_vkey="secret-vkey", lease_seconds=30.0, aihub_client=NonFallbackErrorClient(), clock=lambda: created_at + timedelta(seconds=2), ) async with session_factory() as session: run = await session.get(ScoringRun, run_id) assert run is not None # MUST be failed because fallback_allowed is False, even though config has rules_only_review assert run.status == "failed" assert run.rule_score is None assert run.ai_score is None assert run.final_score is None assert run.error_code == "aihub_non_fallbackable_error" assert run.retryable is False assert run.ai_metadata == { "error_code": "aihub_non_fallbackable_error", "detail": "Non-fallbackable failure", "fallback_policy": "rules_only_review", } await engine.dispose()