如何為業務選模型：先按任務難度分檔，再用自己的資料評測

import os
import time
from openai import OpenAI

client = OpenAI(base_url="https://www.tutujin.com/v1", api_key=os.environ["TUTUJIN_API_KEY"])

candidates = [
    "gpt-5.4-mini",
    "deepseek-v4.1-flash",
    "qwen3.5-plus",
]
samples = [
    {"text": "快递三天了还没到，怎么回事", "label": "物流"},
    {"text": "这件衣服能退吗", "label": "售后"},
    # 换成你自己的 50 到 200 条样本
]
system = "把用户消息归入以下类别之一：物流、售后、商品咨询、其他。只输出类别名。"

for model in candidates:
    correct, seconds, tokens_in, tokens_out = 0, 0.0, 0, 0
    for s in samples:
        t0 = time.time()
        resp = client.chat.completions.create(
            model=model,
            messages=[
                {"role": "system", "content": system},
                {"role": "user", "content": s["text"]},
            ],
        )
        seconds += time.time() - t0
        correct += resp.choices[0].message.content.strip() == s["label"]
        tokens_in += resp.usage.prompt_tokens
        tokens_out += resp.usage.completion_tokens
    n = len(samples)
    print(f"{model}: 正确率 {correct / n:.0%}，平均耗时 {seconds / n:.2f}s，平均 tokens {tokens_in / n:.0f} / {tokens_out / n:.0f}")