Commands Reference

Quick reference for all API calls and function signatures.

OpenAI Evals API

Create Eval

eval_config = client.evals.create(
    name: str,                          # Eval name
    data_source_config: dict,           # Schema configuration
    testing_criteria: list[dict],       # List of graders
)
# Returns: EvalConfig with .id attribute

Example:

eval_config = client.evals.create(
    name="summarization-eval",
    data_source_config={
        "type": "custom",
        "item_schema": {
            "type": "object",
            "properties": {
                "section": {"type": "string"},
                "summary": {"type": "string"},
            },
            "required": ["section", "summary"],
        },
    },
    testing_criteria=[grader1, grader2, grader3, grader4],
)
EVAL_ID = eval_config.id

Create Eval Run

run = client.evals.runs.create(
    eval_id: str,                       # Eval ID
    name: str,                          # Run name (optional)
    data_source: dict,                  # Data to evaluate
)
# Returns: EvalRun with .id, .status attributes

Example:

run = client.evals.runs.create(
    eval_id=EVAL_ID,
    name="self-evolving-eval",
    data_source={
        "type": "jsonl",
        "source": {
            "type": "file_content",
            "content": [
                {"item": {"section": section, "summary": summary}}
            ],
        },
    },
)

Retrieve Eval Run

run = client.evals.runs.retrieve(
    eval_id: str,                       # Eval ID
    run_id: str,                        # Run ID
)
# Returns: EvalRun with .status ("pending", "running", "completed")

List Run Output Items

items = client.evals.runs.output_items.list(
    eval_id: str,                       # Eval ID
    run_id: str,                        # Run ID
)
# Returns: List of output items with .data attribute

Agents SDK

Create Agent

from agents import Agent

agent = Agent(
    name: str,                          # Agent name
    instructions: str,                  # System prompt
    model: str = "gpt-5",              # Model ID
)

Run Agent

from agents import Runner

result = await Runner.run(
    agent: Agent,                       # Agent instance
    input: str,                         # User input
)
# Returns: RunResult with .final_output attribute

Full Example:

from agents import Agent, Runner
import asyncio

agent = Agent(
    name="SummarizationAgent",
    instructions="You are a summarization assistant.",
    model="gpt-5"
)

async def summarize(text: str) -> str:
    result = await Runner.run(agent, text)
    return result.final_output

# Run
summary = asyncio.run(summarize("Your text here"))

Helper Functions

run_eval()

def run_eval(eval_id: str, section: str, summary: str):
    """Create an eval run for a section-summary pair."""
    return client.evals.runs.create(
        eval_id=eval_id,
        name="self-evolving-eval",
        data_source={
            "type": "jsonl",
            "source": {
                "type": "file_content",
                "content": [{"item": {"section": section, "summary": summary}}],
            },
        },
    )

poll_eval_run()

def poll_eval_run(eval_id: str, run_id: str, max_polls: int = 10):
    """Poll until eval run completes."""
    for attempt in range(1, max_polls + 1):
        run = client.evals.runs.retrieve(eval_id=eval_id, run_id=run_id)
        if run.status == "completed":
            break
        if attempt == max_polls:
            raise TimeoutError("Eval run did not complete")
        time.sleep(5)

    return client.evals.runs.output_items.list(eval_id=eval_id, run_id=run_id)

parse_eval_run_output()

def parse_eval_run_output(items) -> list[dict]:
    """Extract grader scores from eval output."""
    results = []
    for item in items.data:
        for result in item.results:
            results.append({
                "grader_name": result.name,
                "score": result.score,
                "passed": result.passed,
                "reasoning": extract_reasoning(result),
            })
    return results

calculate_grader_score()

def calculate_grader_score(grader_scores: list[dict]) -> float:
    """Calculate average score across all graders."""
    if not grader_scores:
        return 0.0
    return sum(e.get("score", 0.0) for e in grader_scores) / len(grader_scores)

is_lenient_pass()

def is_lenient_pass(grader_scores: list[dict], average_score: float) -> bool:
    """Check if evaluation passes lenient criteria."""
    if not grader_scores:
        return False

    passed_count = sum(1 for e in grader_scores if e.get("passed"))
    total = len(grader_scores)

    # Pass if 75% of graders pass OR average >= 85%
    if total and (passed_count / total) >= 0.75:
        return True
    return average_score >= 0.85

collect_grader_feedback()

def collect_grader_feedback(grader_scores: list[dict]) -> str:
    """Consolidate grader feedback for metaprompt."""
    feedback_lines = []

    for entry in grader_scores:
        grader = entry.get("grader_name", "")
        passed = entry.get("passed", False)
        reasoning = entry.get("reasoning")

        if not passed:
            if "chemical_name" in grader:
                feedback_lines.append("Chemical names missing.")
            elif "word_length" in grader:
                feedback_lines.append("Summary length incorrect.")
            elif "cosine" in grader:
                feedback_lines.append("Summary not similar enough.")
            elif "llm_as_judge" in grader and reasoning:
                feedback_lines.append(reasoning)

    return " ".join(feedback_lines) if feedback_lines else "All graders passed."

GEPA Commands

Install GEPA

pip install gepa litellm

Run GEPA Optimization

import gepa

result = gepa.optimize(
    seed_candidate: dict,               # Starting prompt
    trainset: list[dict],              # Training data
    valset: list[dict],                # Validation data
    adapter: object,                   # Adapter instance
    reflection_lm: str = "gpt-5",      # Reflection model
    max_metric_calls: int = 10,        # Max iterations
    track_best_outputs: bool = True,
    display_progress_bar: bool = True,
)
# Returns: OptimizationResult with .best_candidate, .best_score

Example:

seed_candidate = {
    "system_prompt": "You are a summarization assistant."
}

result = gepa.optimize(
    seed_candidate=seed_candidate,
    trainset=trainset,
    valset=valset,
    adapter=EvalsBackedSummarizationAdapter(client, EVAL_ID),
    reflection_lm="gpt-5",
    max_metric_calls=10,
)

best_prompt = result.best_candidate["system_prompt"]