1328 lines
49 KiB
Python
1328 lines
49 KiB
Python
import asyncio
|
|
import json
|
|
import logging
|
|
import uuid
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
from alembic import command
|
|
from alembic.config import Config
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from lead_scoring.aihub.client import AIHubClient
|
|
from lead_scoring.aihub.errors import AIHubError
|
|
from lead_scoring.aihub.schemas import EvidenceExtractionResult
|
|
from lead_scoring.db.models.outbox_event import OutboxEvent
|
|
from lead_scoring.db.models.scoring_run import ScoringRun
|
|
from lead_scoring.execution import execute_scoring_run
|
|
from lead_scoring.scoring import (
|
|
EffortProfile,
|
|
FallbackPolicy,
|
|
RuleMetric,
|
|
ScoringConfig,
|
|
ScoringMode,
|
|
ScoringRule,
|
|
)
|
|
|
|
|
|
def get_alembic_config(database_url: str) -> Config:
|
|
root_dir = Path(__file__).resolve().parent.parent
|
|
config = Config(str(root_dir / "alembic.ini"))
|
|
config.attributes["sqlalchemy_url"] = database_url
|
|
return config
|
|
|
|
|
|
def make_snapshot_data(ems_lead_id: str = "lead-p-1") -> dict[str, Any]:
|
|
return {
|
|
"version": "1.0.0",
|
|
"scope": "linked_contact_history",
|
|
"captured_at": "2026-08-27T12:00:00Z",
|
|
"lead_profile": {
|
|
"ems_lead_id": ems_lead_id,
|
|
"name": "Le Thi D",
|
|
"phone": "+84901234567",
|
|
"email": "ltd@example.com",
|
|
"status": "New",
|
|
},
|
|
"conversations": [
|
|
{
|
|
"conversation_id": "conv-p-1",
|
|
"messages": [
|
|
{
|
|
"message_id": "msg-p-1",
|
|
"sender_type": "contact",
|
|
"content": "Toi can tu van goi AI Lead Scoring cho cong ty",
|
|
"sent_at": "2026-08-27T11:00:00Z",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
"identity_context": {
|
|
"match_quality": "direct",
|
|
"contact_id": "cnt-p-1",
|
|
},
|
|
"trigger_metadata": {
|
|
"trigger_type": "manual_omni",
|
|
"trigger_conversation_id": "conv-p-1",
|
|
"actor_id": "user-p-1",
|
|
},
|
|
"hitl_policy": {
|
|
"confidence_threshold": 0.80,
|
|
"policy_version": "1.0.0",
|
|
},
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_light_execution_success_completed(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="light-config-1",
|
|
base_score=50,
|
|
confidence=0.85,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
rules=(ScoringRule(metric=RuleMetric.PROFILE_FIELD_COUNT, minimum=2, points=30),),
|
|
rule_weight=0.3,
|
|
ai_weight=0.7,
|
|
effort_profile=EffortProfile.LIGHT,
|
|
ai_model="openai/gpt-4o-mini",
|
|
fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-light-1",
|
|
subject_ref="lead-p-1",
|
|
scoring_config_version="light-config-1",
|
|
idempotency_key="idemp-light-1",
|
|
payload_hash="1" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-p-1"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
call_count = 0
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
nonlocal call_count
|
|
call_count += 1
|
|
assert request.url.path == "/v1/chat/completions"
|
|
body = json.loads(request.content.decode("utf-8"))
|
|
schema_name = body.get("response_format", {}).get("json_schema", {}).get("name")
|
|
assert schema_name == "light_lead_scoring"
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"score": 85,
|
|
"confidence": 0.90,
|
|
"reasoning": "Single pass light evaluation.",
|
|
"positive_factors": ["Direct contact interest"],
|
|
"negative_factors": [],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
|
|
success = await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey-token",
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
assert success is True
|
|
assert call_count == 1
|
|
|
|
# Rule score = 50 + 30 = 80
|
|
# AI score = 85
|
|
# Final score = round(80 * 0.3 + 85 * 0.7) = round(24 + 59.5) = 84
|
|
# Combined confidence = 0.85 * 0.3 + 0.90 * 0.7 = 0.255 + 0.63 = 0.885 >= 0.80 -> completed
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.rule_score == 80
|
|
assert run.ai_score == 85
|
|
assert run.final_score == 84
|
|
assert run.scoring_mode == ScoringMode.LIGHT_AI.value
|
|
assert run.degraded is False
|
|
assert run.confidence == 0.885
|
|
assert run.ai_metadata == {
|
|
"model": "openai/gpt-4o-mini",
|
|
"effort_profile": "light",
|
|
"pass_count": 1,
|
|
}
|
|
|
|
events = list(
|
|
(await session.execute(select(OutboxEvent).where(OutboxEvent.run_id == run_id)))
|
|
.scalars()
|
|
.all()
|
|
)
|
|
assert len(events) == 1
|
|
data = events[0].payload["data"]
|
|
assert data["status"] == "completed"
|
|
assert data["rule_score"] == 80
|
|
assert data["confidence"] == 0.885
|
|
assert data["error_code"] is None
|
|
assert data["retryable"] is None
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_heavy_execution_success_completed(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="heavy-config-1",
|
|
base_score=40,
|
|
confidence=0.90,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
rules=(ScoringRule(metric=RuleMetric.PROFILE_FIELD_COUNT, minimum=2, points=20),),
|
|
rule_weight=0.2,
|
|
ai_weight=0.8,
|
|
effort_profile=EffortProfile.HEAVY,
|
|
ai_model="openai/gpt-4o-mini",
|
|
fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-heavy-1",
|
|
subject_ref="lead-p-2",
|
|
scoring_config_version="heavy-config-1",
|
|
idempotency_key="idemp-heavy-1",
|
|
payload_hash="2" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-p-2"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=40),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
call_sequence: list[str] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
assert request.url.path == "/v1/chat/completions"
|
|
body = json.loads(request.content.decode("utf-8"))
|
|
schema_name = body.get("response_format", {}).get("json_schema", {}).get("name")
|
|
call_sequence.append(schema_name)
|
|
|
|
if schema_name == "evidence_extraction":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"summary": "Enterprise interest in AI Platform",
|
|
"intent": "high",
|
|
"key_signals": ["Enterprise lead"],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
elif schema_name == "evidence_critique":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"critique_summary": (
|
|
"High value potential with strong buy-in"
|
|
),
|
|
"risks": [],
|
|
"strengths": ["Enterprise scale"],
|
|
"cross_check_notes": "Validated across conversations",
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
elif schema_name == "heavy_lead_scoring":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"score": 95,
|
|
"confidence": 0.95,
|
|
"reasoning": "High confidence enterprise lead",
|
|
"positive_factors": ["Enterprise scale", "Strong buy-in"],
|
|
"negative_factors": [],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
return httpx.Response(404)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
|
|
success = await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey-token",
|
|
lease_seconds=40.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=3),
|
|
)
|
|
assert success is True
|
|
assert call_sequence == ["evidence_extraction", "evidence_critique", "heavy_lead_scoring"]
|
|
|
|
# Rule score = 40 + 20 = 60
|
|
# AI score = 95
|
|
# Final score = round(60 * 0.2 + 95 * 0.8) = round(12 + 76) = 88
|
|
# Combined confidence = 0.90 * 0.2 + 0.95 * 0.8 = 0.18 + 0.76 = 0.94 >= 0.80 -> completed
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.rule_score == 60
|
|
assert run.ai_score == 95
|
|
assert run.final_score == 88
|
|
assert run.scoring_mode == ScoringMode.HEAVY_AI.value
|
|
assert run.degraded is False
|
|
assert run.confidence == 0.94
|
|
assert run.ai_metadata == {
|
|
"model": "openai/gpt-4o-mini",
|
|
"effort_profile": "heavy",
|
|
"pass_count": 3,
|
|
}
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_heavy_execution_fallback_rules_only_review_on_pass_2_failure(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="heavy-config-fallback-1",
|
|
base_score=65,
|
|
confidence=0.85,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
effort_profile=EffortProfile.HEAVY,
|
|
fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-heavy-fb",
|
|
subject_ref="lead-p-3",
|
|
scoring_config_version="heavy-config-fallback-1",
|
|
idempotency_key="idemp-heavy-fb",
|
|
payload_hash="3" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-p-3"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=40),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
call_sequence: list[str] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
body = json.loads(request.content.decode("utf-8"))
|
|
schema_name = body.get("response_format", {}).get("json_schema", {}).get("name")
|
|
call_sequence.append(schema_name)
|
|
|
|
if schema_name == "evidence_extraction":
|
|
return httpx.Response(
|
|
200,
|
|
json={"choices": [{"message": {"content": json.dumps({"summary": "Ok"})}}]},
|
|
)
|
|
elif schema_name == "evidence_critique":
|
|
return httpx.Response(429, json={"error": "Rate limit exceeded in Pass 2"})
|
|
return httpx.Response(404)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey-token",
|
|
lease_seconds=40.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=3),
|
|
)
|
|
# Exactly 2 calls (Pass 3 was never called)
|
|
assert call_sequence == ["evidence_extraction", "evidence_critique"]
|
|
|
|
# Fallback to rules_only_review:
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.rule_score == 65
|
|
assert run.ai_score is None # Partial results discarded
|
|
assert run.final_score == 65
|
|
assert run.scoring_mode == ScoringMode.RULES_ONLY_FALLBACK.value
|
|
assert run.degraded is True
|
|
assert run.confidence == 0.85
|
|
assert run.ai_metadata == {
|
|
"error_code": "aihub_quota_exceeded",
|
|
"detail": "Quota or rate limit exceeded on AIHub",
|
|
"fallback_policy": "rules_only_review",
|
|
}
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_heavy_execution_fallback_fail_run_on_pass_3_timeout(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="heavy-config-fail-1",
|
|
base_score=65,
|
|
confidence=0.85,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
effort_profile=EffortProfile.HEAVY,
|
|
fallback_policy=FallbackPolicy.FAIL_RUN,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-heavy-fail",
|
|
subject_ref="lead-p-4",
|
|
scoring_config_version="heavy-config-fail-1",
|
|
idempotency_key="idemp-heavy-fail",
|
|
payload_hash="4" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-p-4"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=40),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
call_sequence: list[str] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
body = json.loads(request.content.decode("utf-8"))
|
|
schema_name = body.get("response_format", {}).get("json_schema", {}).get("name")
|
|
call_sequence.append(schema_name)
|
|
|
|
if schema_name == "evidence_extraction":
|
|
return httpx.Response(
|
|
200,
|
|
json={"choices": [{"message": {"content": json.dumps({"summary": "Ok"})}}]},
|
|
)
|
|
elif schema_name == "evidence_critique":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"critique_summary": "Critique ok",
|
|
"risks": [],
|
|
"strengths": [],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
elif schema_name == "heavy_lead_scoring":
|
|
raise httpx.ReadTimeout("Pass 3 timed out after 10s")
|
|
return httpx.Response(404)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey-token",
|
|
lease_seconds=40.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=3),
|
|
)
|
|
assert call_sequence == ["evidence_extraction", "evidence_critique", "heavy_lead_scoring"]
|
|
|
|
# Fallback to fail_run:
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "failed"
|
|
assert run.rule_score is None
|
|
assert run.ai_score is None
|
|
assert run.final_score is None
|
|
assert run.scoring_mode is None
|
|
assert run.confidence is None
|
|
assert run.degraded is True
|
|
assert run.error_code == "aihub_timeout"
|
|
assert run.retryable is False
|
|
assert run.ai_metadata == {
|
|
"error_code": "aihub_timeout",
|
|
"detail": "Request to AIHub timed out",
|
|
"fallback_policy": "fail_run",
|
|
}
|
|
|
|
events = list(
|
|
(await session.execute(select(OutboxEvent).where(OutboxEvent.run_id == run_id)))
|
|
.scalars()
|
|
.all()
|
|
)
|
|
assert len(events) == 1
|
|
data = events[0].payload["data"]
|
|
assert data["status"] == "failed"
|
|
assert data["error_code"] == "aihub_timeout"
|
|
assert data["retryable"] is False
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_sentinel_vkey_never_persisted_in_light_and_heavy_profiles(
|
|
temp_postgres_db: str,
|
|
caplog: pytest.LogCaptureFixture,
|
|
capsys: pytest.CaptureFixture[str],
|
|
):
|
|
caplog.set_level(logging.DEBUG)
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
sentinel_vkey_light = "sentinel-light-secret-vkey-8888"
|
|
sentinel_vkey_heavy = "sentinel-heavy-secret-vkey-9999"
|
|
|
|
# 1. Test Light profile with Sentinel
|
|
run_id_light = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
config_light = ScoringConfig(
|
|
version="light-sentinel-1",
|
|
confidence=0.85,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
effort_profile=EffortProfile.LIGHT,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id_light,
|
|
tenant_id="tenant-s-light",
|
|
subject_ref="lead-s-1",
|
|
scoring_config_version="light-sentinel-1",
|
|
idempotency_key="idemp-s-light",
|
|
payload_hash="s1" * 32,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-s-1"),
|
|
effective_config=config_light.model_dump(mode="json"),
|
|
confidence_threshold=config_light.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler_light(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"score": 80,
|
|
"confidence": 0.85,
|
|
"reasoning": "Light sentinel run",
|
|
"positive_factors": [],
|
|
"negative_factors": [],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
|
|
transport_light = httpx.MockTransport(handler_light)
|
|
async with httpx.AsyncClient(transport=transport_light) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id_light,
|
|
tenant_vkey=sentinel_vkey_light,
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
# 2. Test Heavy profile with Sentinel (failing under fail_run)
|
|
run_id_heavy = str(uuid.uuid4())
|
|
config_heavy = ScoringConfig(
|
|
version="heavy-sentinel-1",
|
|
confidence=0.85,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
effort_profile=EffortProfile.HEAVY,
|
|
fallback_policy=FallbackPolicy.FAIL_RUN,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id_heavy,
|
|
tenant_id="tenant-s-heavy",
|
|
subject_ref="lead-s-2",
|
|
scoring_config_version="heavy-sentinel-1",
|
|
idempotency_key="idemp-s-heavy",
|
|
payload_hash="s2" * 32,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-s-2"),
|
|
effective_config=config_heavy.model_dump(mode="json"),
|
|
confidence_threshold=config_heavy.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=40),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler_heavy(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(500, json={"error": "Sentinel error"})
|
|
|
|
transport_heavy = httpx.MockTransport(handler_heavy)
|
|
async with httpx.AsyncClient(transport=transport_heavy) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id_heavy,
|
|
tenant_vkey=sentinel_vkey_heavy,
|
|
lease_seconds=40.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
# Verify sentinel secrets are absent everywhere in DB and Outbox
|
|
async with session_factory() as session:
|
|
run_l = await session.get(ScoringRun, run_id_light)
|
|
assert run_l is not None
|
|
assert sentinel_vkey_light not in str(run_l.__dict__)
|
|
assert sentinel_vkey_light not in str(run_l.ai_metadata)
|
|
|
|
run_h = await session.get(ScoringRun, run_id_heavy)
|
|
assert run_h is not None
|
|
assert sentinel_vkey_heavy not in str(run_h.__dict__)
|
|
assert sentinel_vkey_heavy not in str(run_h.ai_metadata)
|
|
|
|
events = list((await session.execute(select(OutboxEvent))).scalars().all())
|
|
for event in events:
|
|
assert sentinel_vkey_light not in str(event.payload)
|
|
assert sentinel_vkey_heavy not in str(event.payload)
|
|
|
|
# Verify sentinel secrets are absent from captured logs, stdout, and stderr
|
|
assert sentinel_vkey_light not in caplog.text
|
|
assert sentinel_vkey_heavy not in caplog.text
|
|
captured = capsys.readouterr()
|
|
assert sentinel_vkey_light not in captured.out
|
|
assert sentinel_vkey_light not in captured.err
|
|
assert sentinel_vkey_heavy not in captured.out
|
|
assert sentinel_vkey_heavy not in captured.err
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_light_execution_review_required_due_to_confidence(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="light-conf-rr",
|
|
base_score=40,
|
|
confidence=0.60,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
rule_weight=0.4,
|
|
ai_weight=0.6,
|
|
effort_profile=EffortProfile.LIGHT,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-light-rr",
|
|
subject_ref="lead-p-rr",
|
|
scoring_config_version="light-conf-rr",
|
|
idempotency_key="idemp-light-rr",
|
|
payload_hash="5" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-p-rr"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"score": 50,
|
|
"confidence": 0.50,
|
|
"reasoning": "Uncertain light evaluation",
|
|
"positive_factors": [],
|
|
"negative_factors": ["Sparse context"],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey",
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
# Combined confidence = 0.60 * 0.4 + 0.50 * 0.6 = 0.24 + 0.30 = 0.54 < 0.80 -> review_required
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.rule_score == 40
|
|
assert run.ai_score == 50
|
|
assert run.final_score == round(40 * 0.4 + 50 * 0.6)
|
|
assert run.scoring_mode == ScoringMode.LIGHT_AI.value
|
|
assert run.degraded is False
|
|
assert run.confidence == 0.54
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_heavy_execution_review_required_due_to_confidence(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="heavy-conf-rr",
|
|
base_score=40,
|
|
confidence=0.50,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
rule_weight=0.3,
|
|
ai_weight=0.7,
|
|
effort_profile=EffortProfile.HEAVY,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-heavy-rr",
|
|
subject_ref="lead-p-hrr",
|
|
scoring_config_version="heavy-conf-rr",
|
|
idempotency_key="idemp-heavy-rr",
|
|
payload_hash="6" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-p-hrr"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=40),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
body = json.loads(request.content.decode("utf-8"))
|
|
schema_name = body.get("response_format", {}).get("json_schema", {}).get("name")
|
|
if schema_name == "evidence_extraction":
|
|
return httpx.Response(
|
|
200,
|
|
json={"choices": [{"message": {"content": json.dumps({"summary": "Weak info"})}}]},
|
|
)
|
|
elif schema_name == "evidence_critique":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"critique_summary": "High risk",
|
|
"risks": ["Vague requirements"],
|
|
"strengths": [],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
elif schema_name == "heavy_lead_scoring":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"score": 35,
|
|
"confidence": 0.40,
|
|
"reasoning": "Low confidence due to risks",
|
|
"positive_factors": [],
|
|
"negative_factors": ["High risk"],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
return httpx.Response(404)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey",
|
|
lease_seconds=40.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=3),
|
|
)
|
|
|
|
# Combined confidence = 0.50 * 0.3 + 0.40 * 0.7 = 0.15 + 0.28 = 0.43 < 0.80 -> review_required
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.rule_score == 40
|
|
assert run.ai_score == 35
|
|
assert run.final_score == round(40 * 0.3 + 35 * 0.7)
|
|
assert run.scoring_mode == ScoringMode.HEAVY_AI.value
|
|
assert run.degraded is False
|
|
assert run.confidence == 0.43
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_light_execution_fallback_rules_only_review_on_5xx(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="light-fb-5xx",
|
|
base_score=72,
|
|
confidence=0.88,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
effort_profile=EffortProfile.LIGHT,
|
|
fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-l-5xx",
|
|
subject_ref="lead-p-5xx",
|
|
scoring_config_version="light-fb-5xx",
|
|
idempotency_key="idemp-l-5xx",
|
|
payload_hash="7" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-p-5xx"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(500, json={"error": "AIHub Server Error"})
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey",
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.rule_score == 72
|
|
assert run.ai_score is None
|
|
assert run.final_score == 72
|
|
assert run.scoring_mode == ScoringMode.RULES_ONLY_FALLBACK.value
|
|
assert run.degraded is True
|
|
assert run.ai_metadata == {
|
|
"error_code": "aihub_server_error",
|
|
"detail": "AIHub returned server error",
|
|
"fallback_policy": "rules_only_review",
|
|
}
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_light_execution_fallback_fail_run_on_quota(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="light-fail-429",
|
|
base_score=72,
|
|
confidence=0.88,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="policy-1",
|
|
effort_profile=EffortProfile.LIGHT,
|
|
fallback_policy=FallbackPolicy.FAIL_RUN,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-l-429",
|
|
subject_ref="lead-p-429",
|
|
scoring_config_version="light-fail-429",
|
|
idempotency_key="idemp-l-429",
|
|
payload_hash="8" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-p-429"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(429, json={"error": "Rate limit exceeded"})
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey",
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "failed"
|
|
assert run.rule_score is None
|
|
assert run.ai_score is None
|
|
assert run.final_score is None
|
|
assert run.scoring_mode is None
|
|
assert run.error_code == "aihub_quota_exceeded"
|
|
assert run.retryable is False
|
|
assert run.ai_metadata == {
|
|
"error_code": "aihub_quota_exceeded",
|
|
"detail": "Quota or rate limit exceeded on AIHub",
|
|
"fallback_policy": "fail_run",
|
|
}
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_missing_vkey_light_execution_falls_back_to_rules_only(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
# Test Light with unconfigured client
|
|
run_id_l = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
config_l = ScoringConfig(
|
|
version="unconf-light",
|
|
base_score=60,
|
|
confidence=0.8,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="policy-1",
|
|
effort_profile=EffortProfile.LIGHT,
|
|
fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id_l,
|
|
tenant_id="tenant-unconf-l",
|
|
subject_ref="lead-unconf-l",
|
|
scoring_config_version="unconf-light",
|
|
idempotency_key="idemp-unconf-l",
|
|
payload_hash="9" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-unconf-l"),
|
|
effective_config=config_l.model_dump(mode="json"),
|
|
confidence_threshold=config_l.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id_l,
|
|
tenant_vkey=None,
|
|
lease_seconds=30.0,
|
|
aihub_client=None,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id_l)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.rule_score == 60
|
|
assert run.ai_score is None
|
|
assert run.final_score == 60
|
|
assert run.scoring_mode == ScoringMode.RULES_ONLY_FALLBACK.value
|
|
assert run.degraded is True
|
|
assert run.error_code == "aihub_vkey_missing"
|
|
assert run.ai_metadata == {
|
|
"error_code": "aihub_vkey_missing",
|
|
"detail": "Tenant AIHub credential is missing",
|
|
"fallback_policy": "rules_only_review",
|
|
}
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_non_cancellation_exception_light_and_heavy_uses_canned_error(
|
|
temp_postgres_db: str,
|
|
caplog: pytest.LogCaptureFixture,
|
|
capsys: pytest.CaptureFixture[str],
|
|
):
|
|
caplog.set_level(logging.DEBUG)
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
sentinel_secret = "sentinel-leak-crash-exception-7777"
|
|
|
|
config = ScoringConfig(
|
|
version="heavy-crash-cfg",
|
|
base_score=55,
|
|
confidence=0.85,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="policy-1",
|
|
effort_profile=EffortProfile.HEAVY,
|
|
fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-h-crash",
|
|
subject_ref="lead-h-crash",
|
|
scoring_config_version="heavy-crash-cfg",
|
|
idempotency_key="idemp-h-crash",
|
|
payload_hash="a" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-h-crash"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=40),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
class CrashingClient:
|
|
async def extract_evidence(self, *args, **kwargs):
|
|
return EvidenceExtractionResult(summary="ok")
|
|
|
|
async def critique_evidence(self, *args, **kwargs):
|
|
raise RuntimeError(f"Crashing during heavy critique with secret {sentinel_secret}")
|
|
|
|
async def score_lead_heavy(self, *args, **kwargs):
|
|
raise RuntimeError(f"Crashing during heavy scoring with secret {sentinel_secret}")
|
|
|
|
async def analyze_and_score_lead(self, *args, **kwargs):
|
|
raise RuntimeError(f"Crashing during light with secret {sentinel_secret}")
|
|
|
|
async def score_lead(self, *args, **kwargs):
|
|
raise RuntimeError(f"Crashing with secret {sentinel_secret}")
|
|
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey=sentinel_secret,
|
|
lease_seconds=40.0,
|
|
aihub_client=CrashingClient(),
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.rule_score == 55
|
|
assert run.final_score == 55
|
|
assert run.scoring_mode == ScoringMode.RULES_ONLY_FALLBACK.value
|
|
assert run.degraded is True
|
|
assert run.ai_metadata == {
|
|
"error_code": "aihub_unknown_error",
|
|
"detail": "Unexpected internal execution error",
|
|
"fallback_policy": "rules_only_review",
|
|
}
|
|
|
|
# Verify sentinel_secret NEVER appears anywhere in DB record or outbox
|
|
assert sentinel_secret not in str(run.__dict__)
|
|
assert sentinel_secret not in str(run.ai_metadata)
|
|
assert sentinel_secret not in str(run.effective_config)
|
|
|
|
# Verify sentinel_secret NEVER appears anywhere in captured logs, stdout, or stderr
|
|
assert sentinel_secret not in caplog.text
|
|
captured = capsys.readouterr()
|
|
assert sentinel_secret not in captured.err
|
|
assert sentinel_secret not in captured.out
|
|
|
|
await engine.dispose()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execution_fallback_allowed_false_forces_failed_status_under_rules_only_review(
|
|
temp_postgres_db: str,
|
|
):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 27, 12, 0, tzinfo=UTC)
|
|
|
|
# Config has rules_only_review fallback policy
|
|
config = ScoringConfig(
|
|
version="fb-not-allowed-cfg",
|
|
base_score=70,
|
|
confidence=0.85,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="policy-1",
|
|
effort_profile=EffortProfile.MEDIUM,
|
|
fallback_policy=FallbackPolicy.RULES_ONLY_REVIEW,
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-no-fb",
|
|
subject_ref="lead-no-fb",
|
|
scoring_config_version="fb-not-allowed-cfg",
|
|
idempotency_key="idemp-no-fb",
|
|
payload_hash="b" * 64,
|
|
status="pending",
|
|
snapshot_data=make_snapshot_data("lead-no-fb"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
class NonFallbackErrorClient:
|
|
async def extract_evidence(self, *args, **kwargs):
|
|
raise AIHubError(
|
|
"Critical auth failure not allowed to fallback",
|
|
error_code="aihub_non_fallbackable_error",
|
|
retryable=False,
|
|
fallback_allowed=False,
|
|
detail="Non-fallbackable failure",
|
|
)
|
|
|
|
async def score_lead(self, *args, **kwargs):
|
|
raise AIHubError("Score error", fallback_allowed=False)
|
|
|
|
async def analyze_and_score_lead(self, *args, **kwargs):
|
|
raise AIHubError("Light error", fallback_allowed=False)
|
|
|
|
async def critique_evidence(self, *args, **kwargs):
|
|
raise AIHubError("Critique error", fallback_allowed=False)
|
|
|
|
async def score_lead_heavy(self, *args, **kwargs):
|
|
raise AIHubError("Heavy error", fallback_allowed=False)
|
|
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey",
|
|
lease_seconds=30.0,
|
|
aihub_client=NonFallbackErrorClient(),
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
# MUST be failed because fallback_allowed is False, even though config has rules_only_review
|
|
assert run.status == "failed"
|
|
assert run.rule_score is None
|
|
assert run.ai_score is None
|
|
assert run.final_score is None
|
|
assert run.error_code == "aihub_non_fallbackable_error"
|
|
assert run.retryable is False
|
|
assert run.ai_metadata == {
|
|
"error_code": "aihub_non_fallbackable_error",
|
|
"detail": "Non-fallbackable failure",
|
|
"fallback_policy": "rules_only_review",
|
|
}
|
|
|
|
await engine.dispose()
|