openai-python/src/openai/types/evals/run_create_params.py at main · sweshgit/openai-python

371 lines (266 loc) · 14.5 KB
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
from __future__ import annotations
from typing import Dict, Union, Iterable, Optional
from typing_extensions import Literal, Required, TypeAlias, TypedDict
from ..._types import SequenceNotStr
from ..responses.tool_param import ToolParam
from ..shared_params.metadata import Metadata
from ..shared.reasoning_effort import ReasoningEffort
from ..graders.grader_inputs_param import GraderInputsParam
from ..responses.response_input_text_param import ResponseInputTextParam
from ..responses.response_input_audio_param import ResponseInputAudioParam
from .create_eval_jsonl_run_data_source_param import CreateEvalJSONLRunDataSourceParam
from ..responses.response_format_text_config_param import ResponseFormatTextConfigParam
from .create_eval_completions_run_data_source_param import CreateEvalCompletionsRunDataSourceParam
__all__ = [
    "RunCreateParams",
    "DataSource",
    "DataSourceCreateEvalResponsesRunDataSource",
    "DataSourceCreateEvalResponsesRunDataSourceSource",
    "DataSourceCreateEvalResponsesRunDataSourceSourceFileContent",
    "DataSourceCreateEvalResponsesRunDataSourceSourceFileContentContent",
    "DataSourceCreateEvalResponsesRunDataSourceSourceFileID",
    "DataSourceCreateEvalResponsesRunDataSourceSourceResponses",
    "DataSourceCreateEvalResponsesRunDataSourceInputMessages",
    "DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplate",
    "DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplate",
    "DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateChatMessage",
    "DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItem",
    "DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItemContent",
    "DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItemContentOutputText",
    "DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItemContentInputImage",
    "DataSourceCreateEvalResponsesRunDataSourceInputMessagesItemReference",
    "DataSourceCreateEvalResponsesRunDataSourceSamplingParams",
    "DataSourceCreateEvalResponsesRunDataSourceSamplingParamsText",
class RunCreateParams(TypedDict, total=False):
    data_source: Required[DataSource]
    """Details about the run's data source."""
    metadata: Optional[Metadata]
    """Set of 16 key-value pairs that can be attached to an object.
    This can be useful for storing additional information about the object in a
    structured format, and querying for objects via API or the dashboard.
    Keys are strings with a maximum length of 64 characters. Values are strings with
    a maximum length of 512 characters.
    name: str
    """The name of the run."""
class DataSourceCreateEvalResponsesRunDataSourceSourceFileContentContent(TypedDict, total=False):
    item: Required[Dict[str, object]]
    sample: Dict[str, object]
class DataSourceCreateEvalResponsesRunDataSourceSourceFileContent(TypedDict, total=False):
    content: Required[Iterable[DataSourceCreateEvalResponsesRunDataSourceSourceFileContentContent]]
    """The content of the jsonl file."""
    type: Required[Literal["file_content"]]
    """The type of jsonl source. Always `file_content`."""
class DataSourceCreateEvalResponsesRunDataSourceSourceFileID(TypedDict, total=False):
    id: Required[str]
    """The identifier of the file."""
    type: Required[Literal["file_id"]]
    """The type of jsonl source. Always `file_id`."""
class DataSourceCreateEvalResponsesRunDataSourceSourceResponses(TypedDict, total=False):
    """A EvalResponsesSource object describing a run data source configuration."""
    type: Required[Literal["responses"]]
    """The type of run data source. Always `responses`."""
    created_after: Optional[int]
    """Only include items created after this timestamp (inclusive).
    This is a query parameter used to select responses.
    created_before: Optional[int]
    """Only include items created before this timestamp (inclusive).
    This is a query parameter used to select responses.
    instructions_search: Optional[str]
    """Optional string to search the 'instructions' field.
    This is a query parameter used to select responses.
    metadata: Optional[object]
    """Metadata filter for the responses.
    This is a query parameter used to select responses.
    model: Optional[str]
    """The name of the model to find responses for.
    This is a query parameter used to select responses.
    reasoning_effort: Optional[ReasoningEffort]
    Constrains effort on reasoning for
    [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
    supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
    Reducing reasoning effort can result in faster responses and fewer tokens used
    on reasoning in a response.
    - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
      reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
      calls are supported for all reasoning values in gpt-5.1.
    - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
      support `none`.
    - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
    - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
    temperature: Optional[float]
    """Sampling temperature. This is a query parameter used to select responses."""
    tools: Optional[SequenceNotStr[str]]
    """List of tool names. This is a query parameter used to select responses."""
    top_p: Optional[float]
    """Nucleus sampling parameter. This is a query parameter used to select responses."""
    users: Optional[SequenceNotStr[str]]
    """List of user identifiers. This is a query parameter used to select responses."""
DataSourceCreateEvalResponsesRunDataSourceSource: TypeAlias = Union[
    DataSourceCreateEvalResponsesRunDataSourceSourceFileContent,
    DataSourceCreateEvalResponsesRunDataSourceSourceFileID,
    DataSourceCreateEvalResponsesRunDataSourceSourceResponses,
class DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateChatMessage(TypedDict, total=False):
    content: Required[str]
    """The content of the message."""
    role: Required[str]
    """The role of the message (e.g. "system", "assistant", "user")."""
class DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItemContentOutputText(
    TypedDict, total=False
    """A text output from the model."""
    text: Required[str]
    """The text output from the model."""
    type: Required[Literal["output_text"]]
    """The type of the output text. Always `output_text`."""
class DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItemContentInputImage(
    TypedDict, total=False
    """An image input block used within EvalItem content arrays."""
    image_url: Required[str]
    """The URL of the image input."""
    type: Required[Literal["input_image"]]
    """The type of the image input. Always `input_image`."""
    detail: str
    """The detail level of the image to be sent to the model.
    One of `high`, `low`, or `auto`. Defaults to `auto`.
DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItemContent: TypeAlias = Union[
    ResponseInputTextParam,
    DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItemContentOutputText,
    DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItemContentInputImage,
    ResponseInputAudioParam,
    GraderInputsParam,
class DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItem(TypedDict, total=False):
    A message input to the model with a role indicating instruction following
    hierarchy. Instructions given with the `developer` or `system` role take
    precedence over instructions given with the `user` role. Messages with the
    `assistant` role are presumed to have been generated by the model in previous
    interactions.
    content: Required[DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItemContent]
    """Inputs to the model - can contain template strings.
    Supports text, output text, input images, and input audio, either as a single
    item or an array of items.
    role: Required[Literal["user", "assistant", "system", "developer"]]
    """The role of the message input.
    One of `user`, `assistant`, `system`, or `developer`.
    type: Literal["message"]
    """The type of the message input. Always `message`."""
DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplate: TypeAlias = Union[
    DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateChatMessage,
    DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplateEvalItem,
class DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplate(TypedDict, total=False):
    template: Required[Iterable[DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplateTemplate]]
    """A list of chat messages forming the prompt or context.
    May include variable references to the `item` namespace, ie {{item.name}}.
    type: Required[Literal["template"]]
    """The type of input messages. Always `template`."""
class DataSourceCreateEvalResponsesRunDataSourceInputMessagesItemReference(TypedDict, total=False):
    item_reference: Required[str]
    """A reference to a variable in the `item` namespace. Ie, "item.name" """
    type: Required[Literal["item_reference"]]
    """The type of input messages. Always `item_reference`."""
DataSourceCreateEvalResponsesRunDataSourceInputMessages: TypeAlias = Union[
    DataSourceCreateEvalResponsesRunDataSourceInputMessagesTemplate,
    DataSourceCreateEvalResponsesRunDataSourceInputMessagesItemReference,
class DataSourceCreateEvalResponsesRunDataSourceSamplingParamsText(TypedDict, total=False):
    """Configuration options for a text response from the model.
    Can be plain
    text or structured JSON data. Learn more:
    - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
    - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
    format: ResponseFormatTextConfigParam
    """An object specifying the format that the model must output.
    Configuring `{ "type": "json_schema" }` enables Structured Outputs, which
    ensures the model will match your supplied JSON schema. Learn more in the
    [Structured Outputs guide](https://platform.openai.com/docs/guides/structured-outputs).
    The default format is `{ "type": "text" }` with no additional options.
    **Not recommended for gpt-4o and newer models:**
    Setting to `{ "type": "json_object" }` enables the older JSON mode, which
    ensures the message the model generates is valid JSON. Using `json_schema` is
    preferred for models that support it.
class DataSourceCreateEvalResponsesRunDataSourceSamplingParams(TypedDict, total=False):
    max_completion_tokens: int
    """The maximum number of tokens in the generated output."""
    reasoning_effort: Optional[ReasoningEffort]
    Constrains effort on reasoning for
    [reasoning models](https://platform.openai.com/docs/guides/reasoning). Currently
    supported values are `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`.
    Reducing reasoning effort can result in faster responses and fewer tokens used
    on reasoning in a response.
    - `gpt-5.1` defaults to `none`, which does not perform reasoning. The supported
      reasoning values for `gpt-5.1` are `none`, `low`, `medium`, and `high`. Tool
      calls are supported for all reasoning values in gpt-5.1.
    - All models before `gpt-5.1` default to `medium` reasoning effort, and do not
      support `none`.
    - The `gpt-5-pro` model defaults to (and only supports) `high` reasoning effort.
    - `xhigh` is supported for all models after `gpt-5.1-codex-max`.
    seed: int
    """A seed value to initialize the randomness, during sampling."""
    temperature: float
    """A higher temperature increases randomness in the outputs."""
    text: DataSourceCreateEvalResponsesRunDataSourceSamplingParamsText
    """Configuration options for a text response from the model.
    Can be plain text or structured JSON data. Learn more:
    - [Text inputs and outputs](https://platform.openai.com/docs/guides/text)
    - [Structured Outputs](https://platform.openai.com/docs/guides/structured-outputs)
    tools: Iterable[ToolParam]
    """An array of tools the model may call while generating a response.
    You can specify which tool to use by setting the `tool_choice` parameter.
    The two categories of tools you can provide the model are:
    - **Built-in tools**: Tools that are provided by OpenAI that extend the model's
      capabilities, like
      [web search](https://platform.openai.com/docs/guides/tools-web-search) or
      [file search](https://platform.openai.com/docs/guides/tools-file-search).
      Learn more about
      [built-in tools](https://platform.openai.com/docs/guides/tools).
    - **Function calls (custom tools)**: Functions that are defined by you, enabling
      the model to call your own code. Learn more about
      [function calling](https://platform.openai.com/docs/guides/function-calling).
    top_p: float
    """An alternative to temperature for nucleus sampling; 1.0 includes all tokens."""
class DataSourceCreateEvalResponsesRunDataSource(TypedDict, total=False):
    """A ResponsesRunDataSource object describing a model sampling configuration."""
    source: Required[DataSourceCreateEvalResponsesRunDataSourceSource]
    """Determines what populates the `item` namespace in this run's data source."""
    type: Required[Literal["responses"]]
    """The type of run data source. Always `responses`."""
    input_messages: DataSourceCreateEvalResponsesRunDataSourceInputMessages
    """Used when sampling from a model.
    Dictates the structure of the messages passed into the model. Can either be a
    reference to a prebuilt trajectory (ie, `item.input_trajectory`), or a template
    with variable references to the `item` namespace.
    model: str
    """The name of the model to use for generating completions (e.g. "o3-mini")."""
    sampling_params: DataSourceCreateEvalResponsesRunDataSourceSamplingParams
DataSource: TypeAlias = Union[
    CreateEvalJSONLRunDataSourceParam,
    CreateEvalCompletionsRunDataSourceParam,
    DataSourceCreateEvalResponsesRunDataSource,
Provide feedback

Saved searches

Use saved searches to filter your results more quickly

FilesExpand file tree

run_create_params.py

Latest commit

History

run_create_params.py

File metadata and controls
