Text and image content moderation

import base64
import json
import os
from openai import APIError, OpenAI

client = OpenAI(base_url="https://www.tutujin.com/v1", api_key=os.environ["TUTUJIN_API_KEY"])

POLICY = """You moderate posts in a community app. Reply with one JSON object only:
{"decision": "allow" | "block" | "review", "category": "none" | "spam" | "abuse" | "sexual" | "violence" | "illegal" | "privacy", "reason": "one short sentence"}
spam: ads, or contact details to move a deal off the platform. abuse: insults or threats aimed at a person.
privacy: someone else's phone number, address or ID number. Judge only by these rules. If you are not sure, answer "review"."""
REVIEW = {"decision": "review", "category": "unknown"}
review_queue = []


def moderate(content, model):
    """Anything other than a clear, parseable answer goes to a person, never to "allow"."""
    try:
        resp = client.chat.completions.create(
            model=model,
            messages=[{"role": "system", "content": POLICY}, {"role": "user", "content": content}],
        )
    except APIError as e:  # refused or blocked by the model provider
        return {**REVIEW, "reason": "request failed: " + type(e).__name__}
    choice = resp.choices[0]
    text = choice.message.content or ""
    if choice.finish_reason == "content_filter" or "{" not in text:
        return {**REVIEW, "reason": "model declined"}
    try:
        result = json.loads(text[text.find("{"): text.rfind("}") + 1])
    except json.JSONDecodeError:
        return {**REVIEW, "reason": "unparseable answer"}
    if result.get("decision") not in ("allow", "block", "review"):
        return {**REVIEW, "reason": "unknown label"}
    return result


def moderate_text(text):
    return moderate(text, "gemini-2.5-flash-lite")


def moderate_image(path):
    with open(path, "rb") as f:  # scale images down before this: fewer tokens
        url = "data:image/jpeg;base64," + base64.b64encode(f.read()).decode()
    content = [
        {"type": "text", "text": "Moderate this image, including any text in it."},
        {"type": "image_url", "image_url": {"url": url}},
    ]
    return moderate(content, "claude-haiku-5-5")


for item_id, result in [
    ("post-1", moderate_text("DM me on another app for cheap followers!!")),
    ("avatar-7", moderate_image("avatar.jpg")),
]:
    if result["decision"] == "review":
        # a person decides; qwen3.5-plus can gather the context for the reviewer
        review_queue.append({"id": item_id, **result})
    print(item_id, result)