755 lines
27 KiB
Python
755 lines
27 KiB
Python
import asyncio
|
|
import json
|
|
import logging
|
|
import uuid
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
from alembic import command
|
|
from alembic.config import Config
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from lead_scoring.aihub.client import AIHubClient
|
|
from lead_scoring.db.models.outbox_event import OutboxEvent
|
|
from lead_scoring.db.models.scoring_run import ScoringRun
|
|
from lead_scoring.execution import execute_scoring_run
|
|
from lead_scoring.scoring import (
|
|
EffortProfile,
|
|
FallbackPolicy,
|
|
RuleMetric,
|
|
ScoringConfig,
|
|
ScoringRule,
|
|
)
|
|
|
|
|
|
def get_alembic_config(database_url: str) -> Config:
|
|
root_dir = Path(__file__).resolve().parent.parent
|
|
config = Config(str(root_dir / "alembic.ini"))
|
|
config.attributes["sqlalchemy_url"] = database_url
|
|
return config
|
|
|
|
|
|
def make_snapshot_data(ems_lead_id: str = "lead-m-1") -> dict[str, Any]:
|
|
return {
|
|
"version": "1.0.0",
|
|
"scope": "linked_contact_history",
|
|
"captured_at": "2026-08-27T12:00:00Z",
|
|
"lead_profile": {
|
|
"ems_lead_id": ems_lead_id,
|
|
"name": "Nguyen Van C",
|
|
"phone": "+84987654321",
|
|
"email": "nvc@example.com",
|
|
"status": "New",
|
|
},
|
|
"conversations": [
|
|
{
|
|
"conversation_id": "conv-m-1",
|
|
"messages": [
|
|
{
|
|
"message_id": "msg-m-1",
|
|
"sender_type": "contact",
|
|
"content": "Toi can trien khai giai phap AI cho 100 nhan vien",
|
|
"sent_at": "2026-08-27T11:00:00Z",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
"identity_context": {
|
|
"match_quality": "direct",
|
|
"contact_id": "cnt-m-1",
|
|
},
|
|
"trigger_metadata": {
|
|
"trigger_type": "manual_omni",
|
|
"trigger_conversation_id": "conv-m-1",
|
|
"actor_id": "user-m-1",
|
|
},
|
|
"hitl_policy": {
|
|
"confidence_threshold": 0.80,
|
|
"policy_version": "1.0.0",
|
|
},
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_medium_execution_success_completed(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="medium-config-1",
|
|
base_score=50,
|
|
confidence=0.90,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
rules=(ScoringRule(metric=RuleMetric.PROFILE_FIELD_COUNT, minimum=2, points=30),),
|
|
rule_weight=0.4,
|
|
ai_weight=0.6,
|
|
effort_profile=EffortProfile.MEDIUM,
|
|
ai_model="gpt-4o-mini",
|
|
fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-med-1",
|
|
subject_ref="lead-m-1",
|
|
scoring_config_version="medium-config-1",
|
|
idempotency_key="idemp-med-1",
|
|
payload_hash="a" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-m-1"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
assert request.url.path == "/v1/chat/completions"
|
|
body = json.loads(request.content.decode("utf-8"))
|
|
schema_name = body.get("response_format", {}).get("json_schema", {}).get("name")
|
|
if schema_name == "evidence_extraction":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"summary": "Lead needs AI solution for 100 staff",
|
|
"intent": "high",
|
|
"key_signals": ["100 staff", "enterprise solution"],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
elif schema_name == "lead_scoring":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"score": 90,
|
|
"confidence": 0.95,
|
|
"reasoning": "Large enterprise scale and urgent need.",
|
|
"positive_factors": ["100 staff size"],
|
|
"negative_factors": [],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
return httpx.Response(404)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
|
|
success = await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey-token",
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
assert success is True
|
|
|
|
# Rule score = 50 + 30 = 80
|
|
# AI score = 90
|
|
# Final score = round(80 * 0.4 + 90 * 0.6) = round(32 + 54) = 86
|
|
# Combined confidence = 0.90 * 0.4 + 0.95 * 0.6 = 0.36 + 0.57 = 0.93 >= 0.80 -> completed
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.rule_score == 80
|
|
assert run.ai_score == 90
|
|
assert run.final_score == 86
|
|
assert run.scoring_mode == "medium_ai"
|
|
assert run.degraded is False
|
|
assert run.confidence == 0.93
|
|
assert run.ai_metadata is not None
|
|
assert run.ai_metadata["model"] == "gpt-4o-mini"
|
|
assert run.ai_metadata["pass_count"] == 2
|
|
assert "evidence_summary" not in run.ai_metadata
|
|
assert "reasoning" not in run.ai_metadata
|
|
|
|
events = list(
|
|
(await session.execute(select(OutboxEvent).where(OutboxEvent.run_id == run_id)))
|
|
.scalars()
|
|
.all()
|
|
)
|
|
assert len(events) == 1
|
|
data = events[0].payload["data"]
|
|
assert data["status"] == "completed"
|
|
assert data["rule_score"] == 80
|
|
assert data["confidence"] == 0.93
|
|
assert data["error_code"] is None
|
|
assert data["retryable"] is None
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_medium_execution_success_review_required_due_to_confidence(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="medium-config-2",
|
|
base_score=40,
|
|
confidence=0.60,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
rule_weight=0.4,
|
|
ai_weight=0.6,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-med-2",
|
|
subject_ref="lead-m-2",
|
|
scoring_config_version="medium-config-2",
|
|
idempotency_key="idemp-med-2",
|
|
payload_hash="b" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-m-2"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
assert request.url.path == "/v1/chat/completions"
|
|
body = json.loads(request.content.decode("utf-8"))
|
|
schema_name = body.get("response_format", {}).get("json_schema", {}).get("name")
|
|
if schema_name == "evidence_extraction":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{"summary": "Limited evidence", "intent": "low"}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
elif schema_name == "lead_scoring":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"score": 45,
|
|
"confidence": 0.50,
|
|
"reasoning": "Unclear requirements.",
|
|
"positive_factors": [],
|
|
"negative_factors": ["limited info"],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
return httpx.Response(404)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey-token",
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
# Combined confidence = 0.60 * 0.4 + 0.50 * 0.6 = 0.24 + 0.30 = 0.54 < 0.80 -> review_required
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.final_score == round(40 * 0.4 + 45 * 0.6)
|
|
assert run.confidence == 0.54
|
|
assert run.degraded is False
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_medium_execution_fallback_rules_only_review_on_ai_failure(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="medium-config-3",
|
|
base_score=70,
|
|
confidence=0.85,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-med-3",
|
|
subject_ref="lead-m-3",
|
|
scoring_config_version="medium-config-3",
|
|
idempotency_key="idemp-med-3",
|
|
payload_hash="c" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-m-3"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(500, json={"error": "AIHub Gateway Error"})
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey-token",
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
# Fallback to rules_only_review:
|
|
# status = review_required
|
|
# final_score = rule_score (70)
|
|
# ai_score = None (discarded)
|
|
# degraded = True
|
|
# scoring_mode = "rules_only_fallback"
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.rule_score == 70
|
|
assert run.ai_score is None
|
|
assert run.final_score == 70
|
|
assert run.scoring_mode == "rules_only_fallback"
|
|
assert run.degraded is True
|
|
assert run.ai_metadata is not None
|
|
assert run.ai_metadata["error_code"] == "aihub_server_error"
|
|
|
|
events = list(
|
|
(await session.execute(select(OutboxEvent).where(OutboxEvent.run_id == run_id)))
|
|
.scalars()
|
|
.all()
|
|
)
|
|
assert len(events) == 1
|
|
data = events[0].payload["data"]
|
|
assert data["status"] == "completed"
|
|
assert data["rule_score"] == 70
|
|
assert data["confidence"] == 0.85
|
|
assert data["error_code"] == "aihub_server_error"
|
|
assert data["retryable"] is False
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_medium_execution_fallback_fail_run(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="medium-config-4",
|
|
base_score=70,
|
|
confidence=0.85,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
fallback_policy=FallbackPolicy.FAIL_RUN,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-med-4",
|
|
subject_ref="lead-m-4",
|
|
scoring_config_version="medium-config-4",
|
|
idempotency_key="idemp-med-4",
|
|
payload_hash="d" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-m-4"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(429, json={"error": "Rate limit exceeded"})
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey-token",
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "failed"
|
|
assert run.error_code == "aihub_quota_exceeded"
|
|
assert run.retryable is False
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_medium_execution_vkey_sentinel_never_persisted(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
sentinel_vkey = "sentinel-super-secret-vkey-xyz-999"
|
|
|
|
config = ScoringConfig(
|
|
version="medium-config-5",
|
|
confidence=0.9,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="policy-1",
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-med-5",
|
|
subject_ref="lead-m-5",
|
|
scoring_config_version="medium-config-5",
|
|
idempotency_key="idemp-med-5",
|
|
payload_hash="e" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-m-5"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
assert request.url.path == "/v1/chat/completions"
|
|
body = json.loads(request.content.decode("utf-8"))
|
|
schema_name = body.get("response_format", {}).get("json_schema", {}).get("name")
|
|
if schema_name == "evidence_extraction":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{"summary": "Enterprise lead", "intent": "high"}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
elif schema_name == "lead_scoring":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"score": 85,
|
|
"confidence": 0.9,
|
|
"reasoning": "Good fit",
|
|
"positive_factors": ["enterprise"],
|
|
"negative_factors": [],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
return httpx.Response(404)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey=sentinel_vkey,
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
# Check database row and outbox event
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
|
|
# Convert entire model and dicts to string and check sentinel
|
|
run_str = str(run.__dict__)
|
|
assert sentinel_vkey not in run_str
|
|
assert sentinel_vkey not in str(run.snapshot_data)
|
|
assert sentinel_vkey not in str(run.effective_config)
|
|
assert sentinel_vkey not in str(run.ai_metadata)
|
|
|
|
events = list(
|
|
(await session.execute(select(OutboxEvent).where(OutboxEvent.run_id == run_id)))
|
|
.scalars()
|
|
.all()
|
|
)
|
|
assert len(events) == 1
|
|
assert sentinel_vkey not in str(events[0].payload)
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_medium_execution_unconfigured_aihub_client_triggers_fallback(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="medium-config-unconf",
|
|
base_score=75,
|
|
confidence=0.9,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="policy-1",
|
|
effort_profile=EffortProfile.MEDIUM,
|
|
fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-unconf",
|
|
subject_ref="lead-unconf",
|
|
scoring_config_version="medium-config-unconf",
|
|
idempotency_key="idemp-unconf",
|
|
payload_hash="f" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-unconf"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
# Pass aihub_client=None explicitly
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey-token",
|
|
lease_seconds=30.0,
|
|
aihub_client=None,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.rule_score == 75
|
|
assert run.ai_score is None
|
|
assert run.final_score == 75
|
|
assert run.scoring_mode == "rules_only_fallback"
|
|
assert run.degraded is True
|
|
assert run.ai_metadata is not None
|
|
assert run.ai_metadata["error_code"] == "aihub_client_unconfigured"
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_medium_execution_non_cancellation_exception_uses_canned_error_and_no_sentinel_leak(
|
|
temp_postgres_db: str,
|
|
caplog: pytest.LogCaptureFixture,
|
|
capsys: pytest.CaptureFixture[str],
|
|
):
|
|
caplog.set_level(logging.DEBUG)
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
sentinel_secret = "sentinel-leak-in-exception-1234567890"
|
|
|
|
config = ScoringConfig(
|
|
version="medium-config-crash",
|
|
base_score=60,
|
|
confidence=0.85,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="policy-1",
|
|
effort_profile=EffortProfile.MEDIUM,
|
|
fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-crash",
|
|
subject_ref="lead-crash",
|
|
scoring_config_version="medium-config-crash",
|
|
idempotency_key="idemp-crash",
|
|
payload_hash="1" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-crash"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
class CrashingClient:
|
|
async def extract_evidence(self, *args, **kwargs):
|
|
raise RuntimeError(f"Crashing with secret {sentinel_secret}")
|
|
|
|
async def score_lead(self, *args, **kwargs):
|
|
raise RuntimeError(f"Crashing with secret {sentinel_secret}")
|
|
|
|
async def analyze_and_score_lead(self, *args, **kwargs):
|
|
raise RuntimeError(f"Crashing with secret {sentinel_secret}")
|
|
|
|
async def critique_evidence(self, *args, **kwargs):
|
|
raise RuntimeError(f"Crashing with secret {sentinel_secret}")
|
|
|
|
async def score_lead_heavy(self, *args, **kwargs):
|
|
raise RuntimeError(f"Crashing with secret {sentinel_secret}")
|
|
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey=sentinel_secret,
|
|
lease_seconds=30.0,
|
|
aihub_client=CrashingClient(),
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.rule_score == 60
|
|
assert run.final_score == 60
|
|
assert run.scoring_mode == "rules_only_fallback"
|
|
assert run.degraded is True
|
|
assert run.ai_metadata is not None
|
|
assert run.ai_metadata["error_code"] == "aihub_unknown_error"
|
|
assert run.ai_metadata["detail"] == "Unexpected internal execution error"
|
|
|
|
# Verify sentinel_secret NEVER appears anywhere in DB record or outbox
|
|
assert sentinel_secret not in str(run.__dict__)
|
|
assert sentinel_secret not in str(run.ai_metadata)
|
|
assert sentinel_secret not in str(run.effective_config)
|
|
|
|
events = list(
|
|
(await session.execute(select(OutboxEvent).where(OutboxEvent.run_id == run_id)))
|
|
.scalars()
|
|
.all()
|
|
)
|
|
assert len(events) == 1
|
|
assert sentinel_secret not in str(events[0].payload)
|
|
|
|
# Verify sentinel_secret NEVER appears anywhere in captured logs, stdout, or stderr
|
|
assert sentinel_secret not in caplog.text
|
|
captured = capsys.readouterr()
|
|
assert sentinel_secret not in captured.err
|
|
assert sentinel_secret not in captured.out
|
|
|
|
await engine.dispose()
|