900 lines
33 KiB
Python
900 lines
33 KiB
Python
import asyncio
|
|
import json
|
|
import uuid
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
from alembic import command
|
|
from alembic.config import Config
|
|
from pydantic import SecretStr
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from lead_scoring.aihub.client import AIHubClient
|
|
from lead_scoring.config import ServiceCallerConfig, Settings
|
|
from lead_scoring.db.models.scoring_run import ScoringRun
|
|
from lead_scoring.execution import ExecutionRuntime, execute_scoring_run
|
|
from lead_scoring.main import create_app
|
|
from lead_scoring.schemas.scoring_config import (
|
|
EffortProfile,
|
|
ScoringConfig,
|
|
)
|
|
from lead_scoring.schemas.snapshot import CanonicalSnapshot
|
|
from lead_scoring.scoring import (
|
|
RuleEvaluationStatus,
|
|
derive_outcome_fields,
|
|
evaluate_education_rules,
|
|
)
|
|
|
|
|
|
def get_alembic_config(database_url: str) -> Config:
|
|
from pathlib import Path
|
|
|
|
root_dir = Path(__file__).resolve().parent.parent
|
|
config = Config(str(root_dir / "alembic.ini"))
|
|
config.attributes["sqlalchemy_url"] = database_url
|
|
return config
|
|
|
|
|
|
def make_education_snapshot(
|
|
custom_fields: dict[str, Any],
|
|
conversations: list[dict[str, Any]] | None = None,
|
|
) -> CanonicalSnapshot:
|
|
if conversations is None:
|
|
conversations = [
|
|
{
|
|
"conversation_id": "conv-1",
|
|
"channel": "zalo",
|
|
"messages": [
|
|
{
|
|
"message_id": "msg-1",
|
|
"sender_type": "contact",
|
|
"sent_at": "2026-08-28T11:00:00Z",
|
|
"content": "Toi muon hoi hoc phi khoa IELTS",
|
|
}
|
|
],
|
|
}
|
|
]
|
|
return CanonicalSnapshot.model_validate(
|
|
{
|
|
"version": "1.0.0",
|
|
"scope": "linked_contact_history",
|
|
"captured_at": "2026-08-28T12:00:00Z",
|
|
"lead_profile": {
|
|
"ems_lead_id": "ems-lead-edu-1",
|
|
"name": "Nguyen Van An",
|
|
"phone": "+84901234567",
|
|
"email": "an.nguyen@example.com",
|
|
"status": "new",
|
|
"custom_fields": custom_fields,
|
|
},
|
|
"conversations": conversations,
|
|
"hitl_policy": {
|
|
"policy_version": "1.0.0",
|
|
"confidence_threshold": 0.8,
|
|
},
|
|
}
|
|
)
|
|
|
|
|
|
# ============================================================================
|
|
# SEAM 1: Fixed Education Scoring Rules & Breakdown
|
|
# ============================================================================
|
|
|
|
|
|
def test_education_age_exact_bands():
|
|
config = ScoringConfig(
|
|
version="edu-age-v1",
|
|
base_score=0,
|
|
confidence=0.85,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="1.0.0",
|
|
education_demo_enabled=True,
|
|
)
|
|
|
|
# 1. Age 3-15 -> 40 points
|
|
snap_10 = make_education_snapshot({"student_age": 10}, conversations=[])
|
|
score_10, breakdown_10, _, _ = evaluate_education_rules(snap_10, config)
|
|
assert score_10 == 40
|
|
age_item_10 = next(b for b in breakdown_10 if b.rule_name == "student_age")
|
|
assert age_item_10.status == RuleEvaluationStatus.MATCHED
|
|
assert age_item_10.points_awarded == 40
|
|
|
|
# 2. Age 16-18 -> 20 points
|
|
snap_17 = make_education_snapshot({"student_age": 17}, conversations=[])
|
|
score_17, breakdown_17, _, _ = evaluate_education_rules(snap_17, config)
|
|
assert score_17 == 20
|
|
age_item_17 = next(b for b in breakdown_17 if b.rule_name == "student_age")
|
|
assert age_item_17.status == RuleEvaluationStatus.MATCHED
|
|
assert age_item_17.points_awarded == 20
|
|
|
|
# 3. Age outside [3, 18] -> 0 points (not matched)
|
|
snap_25 = make_education_snapshot({"student_age": 25}, conversations=[])
|
|
score_25, breakdown_25, _, _ = evaluate_education_rules(snap_25, config)
|
|
assert score_25 == 0
|
|
age_item_25 = next(b for b in breakdown_25 if b.rule_name == "student_age")
|
|
assert age_item_25.status == RuleEvaluationStatus.NOT_MATCHED
|
|
assert age_item_25.points_awarded == 0
|
|
|
|
# 4. Age missing -> 0 points (skipped missing data)
|
|
snap_none = make_education_snapshot({}, conversations=[])
|
|
score_none, breakdown_none, _, _ = evaluate_education_rules(snap_none, config)
|
|
assert score_none == 0
|
|
age_item_none = next(b for b in breakdown_none if b.rule_name == "student_age")
|
|
assert age_item_none.status == RuleEvaluationStatus.SKIPPED_MISSING_DATA
|
|
assert age_item_none.points_awarded == 0
|
|
|
|
|
|
def test_education_recency_exact_bands():
|
|
config = ScoringConfig(
|
|
version="edu-rec-v1",
|
|
base_score=0,
|
|
confidence=0.85,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="1.0.0",
|
|
education_demo_enabled=True,
|
|
)
|
|
|
|
def snap_at(hours_ago: float) -> CanonicalSnapshot:
|
|
captured = datetime(2026, 8, 28, 12, 0, tzinfo=UTC)
|
|
sent = captured - timedelta(hours=hours_ago)
|
|
return CanonicalSnapshot.model_validate(
|
|
{
|
|
"version": "1.0.0",
|
|
"scope": "linked_contact_history",
|
|
"captured_at": captured.isoformat(),
|
|
"lead_profile": {
|
|
"ems_lead_id": "ems-lead-rec-1",
|
|
"custom_fields": {},
|
|
},
|
|
"conversations": [
|
|
{
|
|
"conversation_id": "conv-rec",
|
|
"messages": [
|
|
{
|
|
"message_id": "msg-rec",
|
|
"sender_type": "contact",
|
|
"sent_at": sent.isoformat(),
|
|
"content": "hello",
|
|
}
|
|
],
|
|
}
|
|
],
|
|
}
|
|
)
|
|
|
|
# 0-3d (e.g. 1d = 24h) -> 40 points
|
|
s1, b1, _, _ = evaluate_education_rules(snap_at(24.0), config)
|
|
assert s1 == 40
|
|
assert next(b for b in b1 if b.rule_name == "contact_message_recency").points_awarded == 40
|
|
|
|
# 4-7d (e.g. 5d = 120h) -> 25 points
|
|
s2, b2, _, _ = evaluate_education_rules(snap_at(120.0), config)
|
|
assert s2 == 25
|
|
assert next(b for b in b2 if b.rule_name == "contact_message_recency").points_awarded == 25
|
|
|
|
# 8-14d (e.g. 10d = 240h) -> 15 points
|
|
s3, b3, _, _ = evaluate_education_rules(snap_at(240.0), config)
|
|
assert s3 == 15
|
|
assert next(b for b in b3 if b.rule_name == "contact_message_recency").points_awarded == 15
|
|
|
|
# 15-30d (e.g. 20d = 480h) -> 5 points
|
|
s4, b4, _, _ = evaluate_education_rules(snap_at(480.0), config)
|
|
assert s4 == 5
|
|
assert next(b for b in b4 if b.rule_name == "contact_message_recency").points_awarded == 5
|
|
|
|
# >30d (e.g. 35d = 840h) -> 0 points (not matched)
|
|
s5, b5, _, _ = evaluate_education_rules(snap_at(840.0), config)
|
|
assert s5 == 0
|
|
item_5 = next(b for b in b5 if b.rule_name == "contact_message_recency")
|
|
assert item_5.status == RuleEvaluationStatus.NOT_MATCHED
|
|
assert item_5.points_awarded == 0
|
|
|
|
# No customer message -> skipped_missing_data (0 points)
|
|
s6, b6, _, _ = evaluate_education_rules(make_education_snapshot({}, conversations=[]), config)
|
|
assert s6 == 0
|
|
item_6 = next(b for b in b6 if b.rule_name == "contact_message_recency")
|
|
assert item_6.status == RuleEvaluationStatus.SKIPPED_MISSING_DATA
|
|
assert item_6.points_awarded == 0
|
|
|
|
|
|
def test_education_source_exact_bands():
|
|
config = ScoringConfig(
|
|
version="edu-src-v1",
|
|
base_score=0,
|
|
confidence=0.85,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="1.0.0",
|
|
education_demo_enabled=True,
|
|
)
|
|
|
|
# Referral/Walk-in = 20
|
|
s_ref, b_ref, _, _ = evaluate_education_rules(
|
|
make_education_snapshot({"lead_source": "Walk-in"}, conversations=[]), config
|
|
)
|
|
assert s_ref == 20
|
|
assert next(b for b in b_ref if b.rule_name == "lead_source").points_awarded == 20
|
|
|
|
# Call-in = 15
|
|
s_call, b_call, _, _ = evaluate_education_rules(
|
|
make_education_snapshot({"lead_source": "hotline"}, conversations=[]), config
|
|
)
|
|
assert s_call == 15
|
|
assert next(b for b in b_call if b.rule_name == "lead_source").points_awarded == 15
|
|
|
|
# Digital = 10
|
|
s_dig, b_dig, _, _ = evaluate_education_rules(
|
|
make_education_snapshot({"lead_source": "zalo"}, conversations=[]), config
|
|
)
|
|
assert s_dig == 10
|
|
assert next(b for b in b_dig if b.rule_name == "lead_source").points_awarded == 10
|
|
|
|
# Campaign = 8
|
|
s_cam, b_cam, _, _ = evaluate_education_rules(
|
|
make_education_snapshot({"lead_source": "ads"}, conversations=[]), config
|
|
)
|
|
assert s_cam == 8
|
|
assert next(b for b in b_cam if b.rule_name == "lead_source").points_awarded == 8
|
|
|
|
# other nonempty = 5
|
|
s_oth, b_oth, _, _ = evaluate_education_rules(
|
|
make_education_snapshot({"lead_source": "alumni_network"}, conversations=[]), config
|
|
)
|
|
assert s_oth == 5
|
|
assert next(b for b in b_oth if b.rule_name == "lead_source").points_awarded == 5
|
|
|
|
# missing = 0 (skipped)
|
|
s_mis, b_mis, _, _ = evaluate_education_rules(
|
|
make_education_snapshot({}, conversations=[]), config
|
|
)
|
|
assert s_mis == 0
|
|
assert next(b for b in b_mis if b.rule_name == "lead_source").points_awarded == 0
|
|
|
|
|
|
def test_education_completeness_four_groups():
|
|
config = ScoringConfig(
|
|
version="edu-comp-v1",
|
|
base_score=0,
|
|
confidence=0.85,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="1.0.0",
|
|
education_demo_enabled=True,
|
|
)
|
|
|
|
# All 4 present: age, course_interest, customer_message, source -> 100% (1.0)
|
|
snap_full = make_education_snapshot(
|
|
{
|
|
"student_age": 12,
|
|
"course_interest": "IELTS Junior",
|
|
"lead_source": "facebook",
|
|
}
|
|
)
|
|
_, _, _, comp_full = evaluate_education_rules(snap_full, config)
|
|
assert comp_full.completeness_ratio == 1.0
|
|
assert set(comp_full.present_fields) == {
|
|
"age",
|
|
"course_interest",
|
|
"customer_message",
|
|
"source",
|
|
}
|
|
assert len(comp_full.missing_fields) == 0
|
|
|
|
# 2 of 4 present: customer_message and source -> 50% (0.50)
|
|
snap_half = make_education_snapshot({"lead_source": "facebook"})
|
|
_, _, _, comp_half = evaluate_education_rules(snap_half, config)
|
|
assert comp_half.completeness_ratio == 0.50
|
|
assert set(comp_half.present_fields) == {"customer_message", "source"}
|
|
assert set(comp_half.missing_fields) == {"age", "course_interest"}
|
|
|
|
# 1 of 4 present: course_interest only -> 25% (0.25)
|
|
snap_quarter = make_education_snapshot({"course_interest": "General English"}, conversations=[])
|
|
_, _, _, comp_quarter = evaluate_education_rules(snap_quarter, config)
|
|
assert comp_quarter.completeness_ratio == 0.25
|
|
assert set(comp_quarter.present_fields) == {"course_interest"}
|
|
|
|
|
|
# ============================================================================
|
|
# SEAM 2: Configured Medium System Prompts
|
|
# ============================================================================
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_medium_pipeline_receives_configured_prompts(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 28, 12, 0, tzinfo=UTC)
|
|
|
|
custom_extract = "CUSTOM SYSTEM PROMPT PASS 1: Extract academic goals only."
|
|
custom_score = "CUSTOM SYSTEM PROMPT PASS 2: Score strictly on enrollment urgency."
|
|
|
|
config = ScoringConfig(
|
|
version="custom-prompts-v1",
|
|
base_score=50,
|
|
confidence=0.9,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="1.0.0",
|
|
education_demo_enabled=True,
|
|
effort_profile=EffortProfile.MEDIUM,
|
|
medium_extract_prompt=custom_extract,
|
|
medium_score_prompt=custom_score,
|
|
)
|
|
|
|
snapshot = make_education_snapshot(
|
|
{"student_age": 15, "course_interest": "IELTS", "lead_source": "zalo"}
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-edu-prompt",
|
|
subject_ref="ems-lead-edu-1",
|
|
scoring_config_version="custom-prompts-v1",
|
|
idempotency_key="idemp-edu-prompt",
|
|
payload_hash="p" * 64,
|
|
status="pending",
|
|
snapshot_data=snapshot.model_dump(mode="json"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
observed_prompts: list[str] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
body = json.loads(request.content.decode("utf-8"))
|
|
system_msg = next(m["content"] for m in body["messages"] if m["role"] == "system")
|
|
observed_prompts.append(system_msg)
|
|
|
|
schema_name = body.get("response_format", {}).get("json_schema", {}).get("name")
|
|
if schema_name == "evidence_extraction":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{"summary": "Student wants IELTS", "intent": "high"}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
elif schema_name == "lead_scoring":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"score": 88,
|
|
"confidence": 0.92,
|
|
"reasoning": "High enrollment intent.",
|
|
"positive_factors": ["IELTS target"],
|
|
"negative_factors": [],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
return httpx.Response(404)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
success = await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-vkey-token-123",
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
assert success is True
|
|
|
|
assert len(observed_prompts) == 2
|
|
assert observed_prompts[0] == custom_extract
|
|
assert observed_prompts[1] == custom_score
|
|
|
|
|
|
# ============================================================================
|
|
# SEAM 3: Execution Trace Secret Exclusion & Low Completeness Forced Review
|
|
# ============================================================================
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execution_trace_persisted_and_excludes_secrets(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 28, 12, 0, tzinfo=UTC)
|
|
secret_vkey = "top-secret-tenant-vkey-xyz-789"
|
|
|
|
config = ScoringConfig(
|
|
version="trace-test-v1",
|
|
base_score=20,
|
|
confidence=0.85,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="1.0.0",
|
|
education_demo_enabled=True,
|
|
effort_profile=EffortProfile.MEDIUM,
|
|
)
|
|
snapshot = make_education_snapshot(
|
|
{
|
|
"student_age": 17,
|
|
"course_interest": "IELTS",
|
|
"lead_source": "website",
|
|
}
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-trace-sec",
|
|
subject_ref="ems-lead-edu-1",
|
|
scoring_config_version="trace-test-v1",
|
|
idempotency_key="idemp-trace-sec",
|
|
payload_hash="s" * 64,
|
|
status="pending",
|
|
snapshot_data=snapshot.model_dump(mode="json"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
body = json.loads(request.content.decode("utf-8"))
|
|
schema_name = body.get("response_format", {}).get("json_schema", {}).get("name")
|
|
if schema_name == "evidence_extraction":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{"summary": "Evidence extracted", "intent": "medium"}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
elif schema_name == "lead_scoring":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"score": 82,
|
|
"confidence": 0.88,
|
|
"reasoning": "Strong match for intensive class.",
|
|
"positive_factors": ["High intent"],
|
|
"negative_factors": [],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
return httpx.Response(404)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey=secret_vkey,
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.execution_trace is not None
|
|
|
|
trace = run.execution_trace
|
|
trace_json = json.dumps(trace)
|
|
|
|
# Critical Security Assertions:
|
|
assert secret_vkey not in trace_json
|
|
assert "Authorization" not in trace_json
|
|
assert "Bearer" not in trace_json
|
|
assert config.medium_extract_prompt not in trace_json
|
|
|
|
# Required fields in trace:
|
|
assert "normalized_features" in trace
|
|
assert trace["normalized_features"]["student_age"] == 17
|
|
assert "rule_breakdown" in trace
|
|
assert len(trace["rule_breakdown"]) >= 3
|
|
assert "data_completeness" in trace
|
|
assert trace["data_completeness"]["completeness_ratio"] == 1.0
|
|
assert "ai_evaluation" in trace
|
|
assert trace["ai_evaluation"]["reasoning"] == "Strong match for intensive class."
|
|
assert "calculation" in trace
|
|
assert "terminal_status_reason" in trace
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_low_completeness_forces_review_required(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
created_at = datetime(2026, 8, 28, 12, 0, tzinfo=UTC)
|
|
|
|
config = ScoringConfig(
|
|
version="trace-low-comp-v1",
|
|
base_score=50,
|
|
confidence=0.95,
|
|
confidence_threshold=0.80,
|
|
hitl_policy_version="1.0.0",
|
|
education_demo_enabled=True,
|
|
effort_profile=EffortProfile.LIGHT,
|
|
completeness_threshold=0.50, # 50% required
|
|
)
|
|
# Only 1 of 4 completeness groups present (customer_message in conv): 25% < 50%
|
|
snapshot = make_education_snapshot({})
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-low-comp",
|
|
subject_ref="ems-lead-edu-1",
|
|
scoring_config_version="trace-low-comp-v1",
|
|
idempotency_key="idemp-low-comp",
|
|
payload_hash="c" * 64,
|
|
status="pending",
|
|
snapshot_data=snapshot.model_dump(mode="json"),
|
|
effective_config=config.model_dump(mode="json"),
|
|
confidence_threshold=config.confidence_threshold,
|
|
created_at=created_at,
|
|
updated_at=created_at,
|
|
lease_expires_at=created_at + timedelta(seconds=30),
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"choices": [
|
|
{
|
|
"message": {
|
|
"content": json.dumps(
|
|
{
|
|
"score": 90,
|
|
"confidence": 0.95,
|
|
"reasoning": "High confidence AI score.",
|
|
"positive_factors": [],
|
|
"negative_factors": [],
|
|
}
|
|
)
|
|
}
|
|
}
|
|
]
|
|
},
|
|
)
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as http_client:
|
|
aihub_client = AIHubClient(base_url="https://aihub.internal", http_client=http_client)
|
|
await execute_scoring_run(
|
|
session_factory,
|
|
run_id,
|
|
tenant_vkey="secret-key",
|
|
lease_seconds=30.0,
|
|
aihub_client=aihub_client,
|
|
clock=lambda: created_at + timedelta(seconds=2),
|
|
)
|
|
|
|
async with session_factory() as session:
|
|
run = await session.get(ScoringRun, run_id)
|
|
assert run is not None
|
|
assert run.status == "completed"
|
|
assert run.execution_trace is not None
|
|
|
|
|
|
def test_public_outcome_fields_deterministic_derivation():
|
|
# 1. High priority outcome
|
|
cat_hi, exp_hi, act_hi = derive_outcome_fields(
|
|
status="completed",
|
|
effective_score=85,
|
|
final_score=85,
|
|
rule_score=80,
|
|
ai_reasoning="Student expressed urgency for upcoming exam.",
|
|
)
|
|
assert cat_hi == "hot"
|
|
assert exp_hi == "Lead đạt 85/100 sau khi kết hợp bộ quy tắc và phân tích hội thoại bằng AI."
|
|
assert act_hi is not None and "24 giờ" in act_hi
|
|
|
|
# 2. Medium priority outcome
|
|
cat_med, exp_med, act_med = derive_outcome_fields(
|
|
status="completed",
|
|
effective_score=65,
|
|
final_score=65,
|
|
rule_score=60,
|
|
)
|
|
assert cat_med == "warm"
|
|
assert exp_med is not None and "65/100" in exp_med
|
|
assert act_med is not None and "khóa học" in act_med.lower()
|
|
|
|
# 3. Fallback when AI reasoning is None (rule-only / degraded)
|
|
cat_low, exp_low, act_low = derive_outcome_fields(
|
|
status="review_required",
|
|
effective_score=None,
|
|
final_score=40,
|
|
rule_score=40,
|
|
ai_reasoning=None,
|
|
)
|
|
assert cat_low == "warm"
|
|
assert exp_low is not None and "40/100" in exp_low
|
|
assert act_low is not None and "chăm sóc" in act_low.lower()
|
|
|
|
|
|
# ============================================================================
|
|
# SEAM 4: Minimal Authenticated Internal GET Endpoint & Admission Validation
|
|
# ============================================================================
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_admission_rejects_insufficient_evidence_and_accepts_valid(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
config = ScoringConfig(
|
|
version="1.0.0",
|
|
confidence=0.85,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="1.0.0",
|
|
education_demo_enabled=True,
|
|
)
|
|
|
|
settings = Settings(
|
|
environment="development",
|
|
database_url=temp_postgres_db,
|
|
scoring_configs=[config],
|
|
service_callers=[
|
|
ServiceCallerConfig(
|
|
caller_id="test-caller",
|
|
token=SecretStr("valid-token"),
|
|
allowed_apps=["omni"],
|
|
allowed_features=["lead-scoring"],
|
|
allowed_tenants=["tenant-adm-test"],
|
|
)
|
|
],
|
|
tenant_allowlist=["tenant-adm-test"],
|
|
)
|
|
app = create_app(settings=settings)
|
|
app.state.session_factory = session_factory
|
|
app.state.execution_runtime = ExecutionRuntime(session_factory, 30.0, 5.0)
|
|
|
|
async with httpx.AsyncClient(
|
|
transport=httpx.ASGITransport(app=app), base_url="http://test"
|
|
) as client:
|
|
# 1. No course_interest AND no customer message -> 422 insufficient_scoring_evidence
|
|
payload_insufficient = {
|
|
"tenant_id": "tenant-adm-test",
|
|
"subject_ref": "ems-lead-no-ev",
|
|
"scoring_config_version": "1.0.0",
|
|
"snapshot": {
|
|
"version": "1.0.0",
|
|
"scope": "linked_contact_history",
|
|
"captured_at": "2026-08-28T12:00:00Z",
|
|
"lead_profile": {
|
|
"ems_lead_id": "ems-lead-no-ev",
|
|
"custom_fields": {"student_age": 15}, # no course_interest
|
|
},
|
|
"conversations": [], # no customer message
|
|
},
|
|
}
|
|
resp = await client.post(
|
|
"/v1/scoring-runs",
|
|
json=payload_insufficient,
|
|
headers={
|
|
"Authorization": "Bearer valid-token",
|
|
"X-Tenant-VKey": "vkey-123",
|
|
"Idempotency-Key": "adm-idem-1",
|
|
"X-DotB-App": "omni",
|
|
"X-DotB-Feature": "lead-scoring",
|
|
},
|
|
)
|
|
assert resp.status_code == 422
|
|
assert resp.json()["error_code"] == "insufficient_scoring_evidence"
|
|
|
|
# 2. Has course_interest without customer message -> 202 accepted
|
|
payload_with_course = {
|
|
"tenant_id": "tenant-adm-test",
|
|
"subject_ref": "ems-lead-course-only",
|
|
"scoring_config_version": "1.0.0",
|
|
"snapshot": {
|
|
"version": "1.0.0",
|
|
"scope": "linked_contact_history",
|
|
"captured_at": "2026-08-28T12:00:00Z",
|
|
"lead_profile": {
|
|
"ems_lead_id": "ems-lead-course-only",
|
|
"custom_fields": {"course_interest": "IELTS 7.0+"},
|
|
},
|
|
"conversations": [],
|
|
},
|
|
}
|
|
resp_ok = await client.post(
|
|
"/v1/scoring-runs",
|
|
json=payload_with_course,
|
|
headers={
|
|
"Authorization": "Bearer valid-token",
|
|
"X-Tenant-VKey": "vkey-123",
|
|
"Idempotency-Key": "adm-idem-2",
|
|
"X-DotB-App": "omni",
|
|
"X-DotB-Feature": "lead-scoring",
|
|
},
|
|
)
|
|
assert resp_ok.status_code == 202
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_internal_trace_endpoint_access_and_rejection(temp_postgres_db: str):
|
|
await asyncio.to_thread(command.upgrade, get_alembic_config(temp_postgres_db), "head")
|
|
engine = create_async_engine(temp_postgres_db)
|
|
session_factory = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
run_id = str(uuid.uuid4())
|
|
now = datetime(2026, 8, 28, 12, 0, tzinfo=UTC)
|
|
|
|
test_trace = {
|
|
"run_id": run_id,
|
|
"status": "completed",
|
|
"calculation": {"final_score": 85},
|
|
}
|
|
|
|
async with session_factory() as session:
|
|
session.add(
|
|
ScoringRun(
|
|
id=run_id,
|
|
tenant_id="tenant-trace-ep",
|
|
subject_ref="lead-trace-ep",
|
|
scoring_config_version="1.0.0",
|
|
idempotency_key="idemp-trace-ep",
|
|
payload_hash="t" * 64,
|
|
status="completed",
|
|
snapshot_data={},
|
|
execution_trace=test_trace,
|
|
created_at=now,
|
|
updated_at=now,
|
|
)
|
|
)
|
|
await session.commit()
|
|
|
|
# 1. Non-production environment: authorized caller should receive trace
|
|
settings_dev = Settings(
|
|
environment="development",
|
|
database_url=temp_postgres_db,
|
|
service_callers=[
|
|
ServiceCallerConfig(
|
|
caller_id="test-caller",
|
|
token=SecretStr("valid-token"),
|
|
allowed_apps=["omni"],
|
|
allowed_features=["lead-scoring"],
|
|
allowed_tenants=["tenant-trace-ep"],
|
|
)
|
|
],
|
|
tenant_allowlist=["tenant-trace-ep"],
|
|
)
|
|
app_dev = create_app(settings=settings_dev)
|
|
app_dev.state.session_factory = session_factory
|
|
|
|
async with httpx.AsyncClient(
|
|
transport=httpx.ASGITransport(app=app_dev), base_url="http://test"
|
|
) as client:
|
|
# Success path
|
|
resp = await client.get(
|
|
f"/v1/scoring-runs/{run_id}/trace",
|
|
headers={
|
|
"Authorization": "Bearer valid-token",
|
|
"X-Tenant-Id": "tenant-trace-ep",
|
|
},
|
|
)
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert data["run_id"] == run_id
|
|
assert data["calculation"]["final_score"] == 85
|
|
|
|
# Cross-tenant forbidden path
|
|
cross_resp = await client.get(
|
|
f"/v1/scoring-runs/{run_id}/trace",
|
|
headers={
|
|
"Authorization": "Bearer valid-token",
|
|
"X-Tenant-Id": "other-tenant",
|
|
},
|
|
)
|
|
assert cross_resp.status_code == 403
|
|
|
|
# Unauthorized (no token)
|
|
unauth_resp = await client.get(
|
|
f"/v1/scoring-runs/{run_id}/trace",
|
|
headers={"X-Tenant-Id": "tenant-trace-ep"},
|
|
)
|
|
assert unauth_resp.status_code == 401
|
|
|
|
# 2. Production environment: MUST reject access with 403
|
|
settings_prod = Settings(
|
|
environment="production",
|
|
database_url=temp_postgres_db,
|
|
aihub_base_url="https://aihub.internal",
|
|
service_callers=[
|
|
ServiceCallerConfig(
|
|
caller_id="test-caller",
|
|
token=SecretStr("valid-token"),
|
|
allowed_apps=["omni"],
|
|
allowed_features=["lead-scoring"],
|
|
allowed_tenants=["tenant-trace-ep"],
|
|
)
|
|
],
|
|
tenant_allowlist=["tenant-trace-ep"],
|
|
)
|
|
app_prod = create_app(settings=settings_prod)
|
|
app_prod.state.session_factory = session_factory
|
|
|
|
async with httpx.AsyncClient(
|
|
transport=httpx.ASGITransport(app=app_prod), base_url="http://test"
|
|
) as client:
|
|
prod_resp = await client.get(
|
|
f"/v1/scoring-runs/{run_id}/trace",
|
|
headers={
|
|
"Authorization": "Bearer valid-token",
|
|
"X-Tenant-Id": "tenant-trace-ep",
|
|
},
|
|
)
|
|
assert prod_resp.status_code == 403
|
|
assert prod_resp.json()["error_code"] == "trace_disabled_in_production"
|
|
|
|
|
|
def test_education_rule_points_come_from_selected_config():
|
|
config = ScoringConfig(
|
|
version="edu-custom-points-v1",
|
|
base_score=0,
|
|
confidence=0.85,
|
|
confidence_threshold=0.8,
|
|
hitl_policy_version="1.0.0",
|
|
education_demo_enabled=True,
|
|
age_primary_points=31,
|
|
)
|
|
snapshot = make_education_snapshot({"student_age": 10}, conversations=[])
|
|
|
|
score, breakdown, _, _ = evaluate_education_rules(snapshot, config)
|
|
|
|
assert score == 31
|
|
assert next(item for item in breakdown if item.rule_name == "student_age").points_awarded == 31
|