ShadowTest (A/B Testing)

from tokensense.harness import ShadowTest
import openai
import anthropic

# 1. Define the models you want to test
clients = {
    "current": openai.OpenAI(), # e.g. gpt-4o
    "candidate": anthropic.Anthropic() # e.g. claude-3-haiku
}

# 2. Define the prompts
prompts = [
    {
        "model_current": "gpt-4o",
        "model_candidate": "claude-3-haiku-20240307",
        "messages": [{"role": "user", "content": "Return the JSON for a user profile."}],
        "expected_format": "json"
    }
]

# 3. Run the test
test = ShadowTest(clients, prompts, scoring="format-check")
report = test.run()

print(report.summary())

ShadowTest(..., scoring="exact-match")
# Requires "expected_output" in your prompt dicts

ShadowTest(..., scoring="format-check")
# Requires "expected_format": "json" in your prompt dicts

def my_embed_fn(text: str) -> list[float]:
    # call your embedding model here
    pass

ShadowTest(..., scoring="similarity", embedding_fn=my_embed_fn, similarity_threshold=0.85)

ShadowTest(
    ..., 
    scoring="llm-judge", 
    judge=openai.OpenAI(), 
    judge_rubric="The answer must be polite and cover all the same facts as the reference."
)