Social science text analysis and qualitative coding

import json
import os

from openai import OpenAI

client = OpenAI(base_url="https://www.tutujin.com/v1", api_key=os.environ["TUTUJIN_API_KEY"], max_retries=3)

CODEBOOK = {  # written and versioned by the research team
    "COST": "Money, prices or affordability given as a reason for a decision.",
    "TRUST": "Trust or distrust in an institution, a platform or a person.",
    "FAMILY": "Family members influence or take part in the decision.",
}
SYSTEM = (
    "You are a qualitative coder. Apply only these codes, as defined:\n"
    + "\n".join(f"{code}: {definition}" for code, definition in CODEBOOK.items())
    + "\nFor every code that applies, copy the exact words of the segment that justify it. "
    'Reply with JSON only: {"codes": [{"code": "COST", "quote": "exact words"}]}, or {"codes": []} if none apply.'
)


def code_segment(segment, model="qwen3.5-plus"):
    resp = client.chat.completions.create(
        model=model,
        messages=[{"role": "system", "content": SYSTEM}, {"role": "user", "content": segment["text"]}],
        temperature=0,
    )
    raw = resp.choices[0].message.content
    found = json.loads(raw[raw.find("{"): raw.rfind("}") + 1]).get("codes", [])
    # keep a code only if it is in the codebook and its quote appears word for word in the segment
    kept = [c for c in found if c.get("code") in CODEBOOK and c.get("quote") and c["quote"] in segment["text"]]
    return {"id": segment["id"], "model": resp.model, "codes": sorted({c["code"] for c in kept}),
            "quotes": kept, "to_check": len(found) - len(kept)}


def cohen_kappa(a, b):
    """Agreement of two coders on one code; a and b are 0/1 lists over the same segments."""
    n = len(a)
    observed = sum(x == y for x, y in zip(a, b)) / n
    pa, pb = sum(a) / n, sum(b) / n
    expected = pa * pb + (1 - pa) * (1 - pb)
    return float("nan") if expected == 1 else (observed - expected) / (1 - expected)


# the agreement sample: a few hundred segments in practice, de-identified before sending
sample = [
    {"id": "INT07-P012", "text": "My mother said the rent there was too high, so we stayed where we were."},
    {"id": "INT11-P003", "text": "I never believed what the agency told us about the subsidy."},
]
human = {"INT07-P012": {"COST", "FAMILY"}, "INT11-P003": {"TRUST"}}  # coded without seeing the model output
first = [code_segment(s) for s in sample]
second = [code_segment(s, model="gpt-5.4-mini") for s in sample]

for code in CODEBOOK:
    m1 = [int(code in r["codes"]) for r in first]
    m2 = [int(code in r["codes"]) for r in second]
    h = [int(code in human[r["id"]]) for r in first]
    print(code, "model vs human:", round(cohen_kappa(m1, h), 2), "| model vs second model:", round(cohen_kappa(m1, m2), 2))
# Exploring themes first: embed answers or posts with text-embedding-3-large and cluster the vectors.