Evals
Reference for client.evals in the Future AGI Python and TypeScript SDKs: evals.test, evals.configs.create, evals.configs.list, evals.groups.get, evals.groups.list, evals.tasks.create and 9 more.
client.evals on fi.Client (Python) and client.evals on Client from @future-agi/sdk (TypeScript), with the nested resources client.evals.configs, client.evals.groups, client.evals.tasks, client.evals.templates. The models these methods return are on Datasets & evals response models; how arguments, pagination, errors and strict mode work is in the overview.
Methods
| Method | Description |
|---|---|
evals.test | Execute an evaluation template against supplied test inputs. |
evals.configs.create | Create a custom evaluation configuration from an eval template so it can be attached to an online evaluation task. |
evals.configs.list | List the custom evaluation configurations available for online evaluation tasks. |
evals.groups.get | Return an evaluation group by its identifier. |
evals.groups.list | List evaluation groups in the current workspace. |
evals.tasks.create | Create an online evaluation task. |
evals.tasks.get | Return an online evaluation task by its identifier. |
evals.tasks.list | List online evaluation tasks in the current workspace. |
evals.tasks.pause | Pause a running online evaluation task by eval_task_id. |
evals.tasks.resume | Resume a paused online evaluation task by eval_task_id, scheduling remaining evaluations. |
evals.tasks.update | Update and rerun an online evaluation task using eval_task_id and edit_type. |
evals.templates.create | Create an evaluation template. |
evals.templates.get | Return an evaluation template by its identifier. |
evals.templates.list | List evaluation templates using the supplied filters. |
evals.templates.update | Update an existing evaluation template. |
client.evals
evals.test
Execute an evaluation template against supplied test inputs.
/model-hub/test-evaluation/ MCP tool: test_evaluation
result = client.evals.test(
config={},
name="your-name",
output_type="your-output-type",
)const result = await client.evals.test({
config: {},
name: "your-name",
output_type: "your-output-type",
}); Signature
client.evals.test(
*,
config: dict[str, Any],
name: str,
output_type: str,
model: str | None | NotGiven = NOT_GIVEN,
eval_tags: list[str] | None | NotGiven = NOT_GIVEN,
criteria: str | None | NotGiven = NOT_GIVEN,
multi_choice: bool | None | NotGiven = NOT_GIVEN,
choices: dict[str, Any] | None | NotGiven = NOT_GIVEN,
input_data_types: dict[str, Any] | None | NotGiven = NOT_GIVEN,
description: str | None | NotGiven = NOT_GIVEN,
check_internet: bool | None | NotGiven = NOT_GIVEN,
required_keys: list[str] | None | NotGiven = NOT_GIVEN,
template_type: str | None | NotGiven = NOT_GIVEN,
eval_type_id: str | None | NotGiven = NOT_GIVEN,
template_id: str | None | NotGiven = NOT_GIVEN,
eval_type: str | None | NotGiven = NOT_GIVEN,
error_localizer: bool | None | NotGiven = NOT_GIVEN,
reason_column: bool | None | NotGiven = NOT_GIVEN,
optional_keys: list[str] | None | NotGiven = NOT_GIVEN,
variable_keys: Any | None | NotGiven = NOT_GIVEN,
run_prompt_column: bool | None | NotGiven = NOT_GIVEN,
template_name: str | None | NotGiven = NOT_GIVEN,
mapping: dict[str, Any] | None | NotGiven = NOT_GIVEN,
output: str | None | NotGiven = NOT_GIVEN,
config_params_desc: dict[str, Any] | None | NotGiven = NOT_GIVEN,
config_params_option: dict[str, Any] | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.EvalExecutionResponseResultclient.evals.test(params: EvalsTestParams, options?: RequestOptions): Promise<models.EvalExecutionResponseResult>
export interface EvalsTestParams {
config: Record<string, unknown>;
name: string;
output_type: string;
model?: string | null;
eval_tags?: Array<string> | null;
criteria?: string | null;
multi_choice?: boolean | null;
choices?: Record<string, unknown> | null;
input_data_types?: Record<string, unknown> | null;
description?: string | null;
check_internet?: boolean | null;
required_keys?: Array<string> | null;
template_type?: string | null;
eval_type_id?: string | null;
template_id?: string | null;
eval_type?: string | null;
error_localizer?: boolean | null;
reason_column?: boolean | null;
optional_keys?: Array<string> | null;
variable_keys?: JsonValue | null;
run_prompt_column?: boolean | null;
template_name?: string | null;
mapping?: Record<string, unknown> | null;
output?: string | null;
config_params_desc?: Record<string, unknown> | null;
config_params_option?: Record<string, unknown> | null;
} Parameters
Returns
client.evals.configs
TypeScript: client.evals.configs.
evals.configs.create
Create a custom evaluation configuration from an eval template so it can be attached to an online evaluation task. Use when list_eval_configs has no entry for the template you want to run over spans.
/tracer/custom-eval-config/ MCP tool: create_eval_config
result = client.evals.configs.create(eval_template="your-eval-template", project="your-project")const result = await client.evals.configs.create({
eval_template: "your-eval-template",
project: "your-project",
}); Signature
client.evals.configs.create(
*,
eval_template: str,
project: str,
name: str | None | NotGiven = NOT_GIVEN,
config: dict[str, Any] | None | NotGiven = NOT_GIVEN,
mapping: dict[str, Any] | None | NotGiven = NOT_GIVEN,
filters: dict[str, Any] | None | NotGiven = NOT_GIVEN,
error_localizer: bool | None | NotGiven = NOT_GIVEN,
kb_id: str | None | NotGiven = NOT_GIVEN,
model: str | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.EvalsConfigsCreateResultclient.evals.configs.create(params: EvalsConfigsCreateParams, options?: RequestOptions): Promise<models.EvalsConfigsCreateResult>
export interface EvalsConfigsCreateParams {
eval_template: string;
project: string;
name?: string | null;
config?: Record<string, unknown> | null;
mapping?: Record<string, unknown> | null;
filters?: Record<string, unknown> | null;
error_localizer?: boolean | null;
kb_id?: string | null;
model?: string | null;
} Parameters
Returns
evals.configs.list
List the custom evaluation configurations available for online evaluation tasks. The ids returned here are what create_eval_task takes in its evals argument - eval templates are a different object and their ids are not accepted there.
/tracer/custom-eval-config/ Paginated: returns a pager (how pagination works) · MCP tool: list_eval_configs
for item in client.evals.configs.list():
print(item)for await (const item of client.evals.configs.list()) {
console.log(item);
} Signature
client.evals.configs.list(
*,
limit: int | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> SyncPager[models.CustomEvalConfig]client.evals.configs.list(params: EvalsConfigsListParams = {}, options?: RequestOptions): Pager<models.CustomEvalConfig>
export interface EvalsConfigsListParams {
/** Number of results to return per page. */
limit?: number | null;
} Parameters
Number of results to return per page.
The pager sends page itself; pass limit to change the page size.
Returns
A pager of CustomEvalConfig items: iterate it, or call to_list() (Python) / toArray() (TypeScript).
client.evals.groups
TypeScript: client.evals.groups.
evals.groups.get
Return an evaluation group by its identifier.
/model-hub/eval-groups/{id}/ MCP tool: get_eval_group
result = client.evals.groups.get("your-id")const result = await client.evals.groups.get("your-id"); Signature
client.evals.groups.get(
id: str,
*,
name: str | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.EvalGroupclient.evals.groups.get(id: string, params: EvalsGroupsGetParams = {}, options?: RequestOptions): Promise<models.EvalGroup>
export interface EvalsGroupsGetParams {
name?: string | null;
} Parameters
Positional argument id.
Returns
evals.groups.list
List evaluation groups in the current workspace.
/model-hub/eval-groups/ MCP tool: list_eval_groups
result = client.evals.groups.list()const result = await client.evals.groups.list(); Signature
client.evals.groups.list(
*,
name: str | None | NotGiven = NOT_GIVEN,
page_number: int | None | NotGiven = NOT_GIVEN,
page_size: int | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> Anyclient.evals.groups.list(params: EvalsGroupsListParams = {}, options?: RequestOptions): Promise<unknown>
export interface EvalsGroupsListParams {
name?: string | null;
page_number?: number | null;
page_size?: number | null;
} Parameters
Returns
The decoded response. The contract does not describe its shape, so it is untyped.
client.evals.tasks
TypeScript: client.evals.tasks.
evals.tasks.create
Create an online evaluation task.
/tracer/eval-task/ MCP tool: create_eval_task
result = client.evals.tasks.create(
project="your-project",
name="your-name",
sampling_rate=1.0,
run_type="continuous",
evals=[],
)const result = await client.evals.tasks.create({
project: "your-project",
name: "your-name",
sampling_rate: 1.0,
run_type: "continuous",
evals: [],
}); Signature
client.evals.tasks.create(
*,
project: str,
name: str,
sampling_rate: float,
run_type: Literal['continuous', 'historical'],
evals: list[str],
filters: dict[str, Any] | None | NotGiven = NOT_GIVEN,
last_run: str | None | NotGiven = NOT_GIVEN,
spans_limit: int | None | NotGiven = NOT_GIVEN,
row_type: Literal['spans', 'traces', 'sessions', 'voiceCalls'] | None | NotGiven = NOT_GIVEN,
status: Literal['pending', 'running', 'completed', 'failed', 'paused', 'deleted'] | None | NotGiven = NOT_GIVEN,
start_time: str | None | NotGiven = NOT_GIVEN,
end_time: str | None | NotGiven = NOT_GIVEN,
evals_details: dict[str, Any] | None | NotGiven = NOT_GIVEN,
failed_spans: dict[str, Any] | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.EvalTaskCreateResultclient.evals.tasks.create(params: EvalsTasksCreateParams, options?: RequestOptions): Promise<models.EvalTaskCreateResult>
export interface EvalsTasksCreateParams {
project: string;
name: string;
sampling_rate: number;
run_type: "continuous" | "historical";
evals: Array<string>;
filters?: Record<string, unknown> | null;
last_run?: string | null;
spans_limit?: number | null;
row_type?: "spans" | "traces" | "sessions" | "voiceCalls" | null;
status?: "pending" | "running" | "completed" | "failed" | "paused" | "deleted" | null;
start_time?: string | null;
end_time?: string | null;
evals_details?: Record<string, unknown> | null;
failed_spans?: Record<string, unknown> | null;
} Parameters
continuoushistorical spanstracessessionsvoiceCalls pendingrunningcompletedfailedpauseddeleted Returns
evals.tasks.get
Return an online evaluation task by its identifier.
/tracer/eval-task/{id}/ MCP tool: get_eval_task
result = client.evals.tasks.get("your-id")const result = await client.evals.tasks.get("your-id"); Signature
client.evals.tasks.get(
id: str,
*,
extra_query: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.EvalTaskclient.evals.tasks.get(id: string, options?: RequestOptions): Promise<models.EvalTask> Parameters
Positional argument id.
Returns
evals.tasks.list
List online evaluation tasks in the current workspace.
/tracer/eval-task/list_eval_tasks/ MCP tool: list_eval_tasks
result = client.evals.tasks.list()const result = await client.evals.tasks.list(); Signature
client.evals.tasks.list(
*,
project_id: str | None | NotGiven = NOT_GIVEN,
name: str | None | NotGiven = NOT_GIVEN,
filters: str | None | NotGiven = NOT_GIVEN,
sort_params: str | None | NotGiven = NOT_GIVEN,
page_number: int | None | NotGiven = NOT_GIVEN,
page_size: int | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models._EvalTaskListResultResponseclient.evals.tasks.list(params: EvalsTasksListParams = {}, options?: RequestOptions): Promise<models._EvalTaskListResultResponse>
export interface EvalsTasksListParams {
project_id?: string | null;
name?: string | null;
filters?: string | null;
sort_params?: string | null;
page_number?: number | null;
page_size?: number | null;
} Parameters
Returns
evals.tasks.pause
Pause a running online evaluation task by eval_task_id. Already-launched evaluations may finish before the worker stops.
/tracer/eval-task/pause_eval_task/ MCP tool: pause_eval_task
result = client.evals.tasks.pause(eval_task_id="your-eval-task-id")const result = await client.evals.tasks.pause({ eval_task_id: "your-eval-task-id" }); Signature
client.evals.tasks.pause(
*,
eval_task_id: str,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.EvalTaskMessageResultclient.evals.tasks.pause(params: EvalsTasksPauseParams, options?: RequestOptions): Promise<models.EvalTaskMessageResult>
export interface EvalsTasksPauseParams {
eval_task_id: string;
} Parameters
Returns
evals.tasks.resume
Resume a paused online evaluation task by eval_task_id, scheduling remaining evaluations. May incur evaluation charges.
/tracer/eval-task/unpause_eval_task/ MCP tool: resume_eval_task
result = client.evals.tasks.resume(eval_task_id="your-eval-task-id")const result = await client.evals.tasks.resume({ eval_task_id: "your-eval-task-id" }); Signature
client.evals.tasks.resume(
*,
eval_task_id: str,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.EvalTaskMessageResultclient.evals.tasks.resume(params: EvalsTasksResumeParams, options?: RequestOptions): Promise<models.EvalTaskMessageResult>
export interface EvalsTasksResumeParams {
eval_task_id: string;
} Parameters
Returns
evals.tasks.update
Update and rerun an online evaluation task using eval_task_id and edit_type. Pause a running task first. edit_rerun preserves existing results; fresh_run DELETES previous results and starts again. May incur evaluation charges.
/tracer/eval-task/update_eval_task/ MCP tool: update_eval_task
result = client.evals.tasks.update(edit_type="edit_rerun", eval_task_id="your-eval-task-id")const result = await client.evals.tasks.update({
edit_type: "edit_rerun",
eval_task_id: "your-eval-task-id",
}); Signature
client.evals.tasks.update(
*,
edit_type: Literal['edit_rerun', 'fresh_run'],
eval_task_id: str,
name: str | None | NotGiven = NOT_GIVEN,
filters: dict[str, Any] | None | NotGiven = NOT_GIVEN,
sampling_rate: float | None | NotGiven = NOT_GIVEN,
spans_limit: int | None | NotGiven = NOT_GIVEN,
run_type: Literal['continuous', 'historical'] | None | NotGiven = NOT_GIVEN,
row_type: Literal['spans', 'traces', 'sessions', 'voiceCalls'] | None | NotGiven = NOT_GIVEN,
status: Literal['pending', 'running', 'completed', 'failed', 'paused', 'deleted'] | None | NotGiven = NOT_GIVEN,
evals: list[str] | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.EvalTaskUpdateResultclient.evals.tasks.update(params: EvalsTasksUpdateParams, options?: RequestOptions): Promise<models.EvalTaskUpdateResult>
export interface EvalsTasksUpdateParams {
edit_type: "edit_rerun" | "fresh_run";
eval_task_id: string;
name?: string | null;
filters?: Record<string, unknown> | null;
sampling_rate?: number | null;
spans_limit?: number | null;
run_type?: "continuous" | "historical" | null;
row_type?: "spans" | "traces" | "sessions" | "voiceCalls" | null;
status?: "pending" | "running" | "completed" | "failed" | "paused" | "deleted" | null;
evals?: Array<string> | null;
} Parameters
edit_rerunfresh_run continuoushistorical spanstracessessionsvoiceCalls pendingrunningcompletedfailedpauseddeleted Returns
client.evals.templates
TypeScript: client.evals.templates.
evals.templates.create
Create an evaluation template.
/model-hub/eval-templates/create-v2/ MCP tool: create_eval_template
result = client.evals.templates.create()const result = await client.evals.templates.create(); Signature
client.evals.templates.create(
*,
name: str | None | NotGiven = NOT_GIVEN,
is_draft: bool | None | NotGiven = NOT_GIVEN,
eval_type: Literal['llm', 'code', 'agent'] | None | NotGiven = NOT_GIVEN,
instructions: str | None | NotGiven = NOT_GIVEN,
model: str | None | NotGiven = NOT_GIVEN,
output_type: Literal['pass_fail', 'percentage', 'deterministic'] | None | NotGiven = NOT_GIVEN,
pass_threshold: float | None | NotGiven = NOT_GIVEN,
choice_scores: dict[str, Any] | None | NotGiven = NOT_GIVEN,
description: str | None | NotGiven = NOT_GIVEN,
tags: list[str] | None | NotGiven = NOT_GIVEN,
check_internet: bool | None | NotGiven = NOT_GIVEN,
code: str | None | NotGiven = NOT_GIVEN,
code_language: Literal['python', 'javascript'] | None | NotGiven = NOT_GIVEN,
messages: list[dict[str, Any]] | None | NotGiven = NOT_GIVEN,
few_shot_examples: list[dict[str, Any]] | None | NotGiven = NOT_GIVEN,
mode: Literal['auto', 'agent', 'quick'] | None | NotGiven = NOT_GIVEN,
tools: dict[str, Any] | None | NotGiven = NOT_GIVEN,
knowledge_bases: list[str] | None | NotGiven = NOT_GIVEN,
data_injection: dict[str, Any] | None | NotGiven = NOT_GIVEN,
summary: dict[str, Any] | None | NotGiven = NOT_GIVEN,
error_localizer_enabled: bool | None | NotGiven = NOT_GIVEN,
template_format: Literal['mustache', 'jinja'] | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.EvalTemplateCreateResponseResultclient.evals.templates.create(params: EvalsTemplatesCreateParams = {}, options?: RequestOptions): Promise<models.EvalTemplateCreateResponseResult>
export interface EvalsTemplatesCreateParams {
name?: string | null;
is_draft?: boolean | null;
eval_type?: "llm" | "code" | "agent" | null;
instructions?: string | null;
model?: string | null;
output_type?: "pass_fail" | "percentage" | "deterministic" | null;
pass_threshold?: number | null;
choice_scores?: Record<string, unknown> | null;
description?: string | null;
tags?: Array<string> | null;
check_internet?: boolean | null;
code?: string | null;
code_language?: "python" | "javascript" | null;
messages?: Array<Record<string, unknown>> | null;
few_shot_examples?: Array<Record<string, unknown>> | null;
mode?: "auto" | "agent" | "quick" | null;
tools?: Record<string, unknown> | null;
knowledge_bases?: Array<string> | null;
data_injection?: Record<string, unknown> | null;
summary?: Record<string, unknown> | null;
error_localizer_enabled?: boolean | null;
template_format?: "mustache" | "jinja" | null;
} Parameters
llmcodeagent pass_failpercentagedeterministic pythonjavascript autoagentquick mustachejinja Returns
EvalTemplateCreateResponseResult.
evals.templates.get
Return an evaluation template by its identifier.
/model-hub/eval-templates/{template_id}/detail/ MCP tool: get_eval_template
result = client.evals.templates.get("your-template-id")const result = await client.evals.templates.get("your-template-id"); Signature
client.evals.templates.get(
template_id: str,
*,
extra_query: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.EvalTemplateDetailResponseResultclient.evals.templates.get(templateId: string, options?: RequestOptions): Promise<models.EvalTemplateDetailResponseResult> Parameters
Positional argument template_id.
Returns
EvalTemplateDetailResponseResult.
evals.templates.list
List evaluation templates using the supplied filters.
/model-hub/eval-templates/list/ MCP tool: list_eval_templates
result = client.evals.templates.list()const result = await client.evals.templates.list(); Signature
client.evals.templates.list(
*,
page: int | None | NotGiven = NOT_GIVEN,
page_size: int | None | NotGiven = NOT_GIVEN,
search: str | None | NotGiven = NOT_GIVEN,
owner_filter: Literal['all', 'user', 'system'] | None | NotGiven = NOT_GIVEN,
filters: dict[str, Any] | None | NotGiven = NOT_GIVEN,
sort_by: Literal['name', 'updated_at', 'created_at'] | None | NotGiven = NOT_GIVEN,
sort_order: Literal['asc', 'desc'] | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.EvalTemplateListResponseResultclient.evals.templates.list(params: EvalsTemplatesListParams = {}, options?: RequestOptions): Promise<models.EvalTemplateListResponseResult>
export interface EvalsTemplatesListParams {
page?: number | null;
page_size?: number | null;
search?: string | null;
owner_filter?: "all" | "user" | "system" | null;
filters?: Record<string, unknown> | null;
sort_by?: "name" | "updated_at" | "created_at" | null;
sort_order?: "asc" | "desc" | null;
} Parameters
allusersystem nameupdated_atcreated_at ascdesc Returns
EvalTemplateListResponseResult.
evals.templates.update
Update an existing evaluation template.
/model-hub/eval-templates/{template_id}/update/ MCP tool: update_eval_template
result = client.evals.templates.update("your-template-id")const result = await client.evals.templates.update("your-template-id"); Signature
client.evals.templates.update(
template_id: str,
*,
name: str | None | NotGiven = NOT_GIVEN,
eval_type: Literal['llm', 'code', 'agent'] | None | NotGiven = NOT_GIVEN,
instructions: str | None | NotGiven = NOT_GIVEN,
model: str | None | NotGiven = NOT_GIVEN,
output_type: Literal['pass_fail', 'percentage', 'deterministic'] | None | NotGiven = NOT_GIVEN,
pass_threshold: float | None | NotGiven = NOT_GIVEN,
choice_scores: dict[str, Any] | None | NotGiven = NOT_GIVEN,
multi_choice: bool | None | NotGiven = NOT_GIVEN,
description: str | None | NotGiven = NOT_GIVEN,
tags: list[str] | None | NotGiven = NOT_GIVEN,
check_internet: bool | None | NotGiven = NOT_GIVEN,
code: str | None | NotGiven = NOT_GIVEN,
code_language: Literal['python', 'javascript'] | None | NotGiven = NOT_GIVEN,
messages: list[dict[str, Any]] | None | NotGiven = NOT_GIVEN,
few_shot_examples: list[dict[str, Any]] | None | NotGiven = NOT_GIVEN,
mode: Literal['auto', 'agent', 'quick'] | None | NotGiven = NOT_GIVEN,
tools: dict[str, Any] | None | NotGiven = NOT_GIVEN,
knowledge_bases: list[str] | None | NotGiven = NOT_GIVEN,
data_injection: dict[str, Any] | None | NotGiven = NOT_GIVEN,
summary: dict[str, Any] | None | NotGiven = NOT_GIVEN,
error_localizer_enabled: bool | None | NotGiven = NOT_GIVEN,
publish: bool | None | NotGiven = NOT_GIVEN,
template_format: Literal['mustache', 'jinja'] | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.EvalTemplateUpdateResponseResultclient.evals.templates.update(templateId: string, params: EvalsTemplatesUpdateParams = {}, options?: RequestOptions): Promise<models.EvalTemplateUpdateResponseResult>
export interface EvalsTemplatesUpdateParams {
name?: string | null;
eval_type?: "llm" | "code" | "agent" | null;
instructions?: string | null;
model?: string | null;
output_type?: "pass_fail" | "percentage" | "deterministic" | null;
pass_threshold?: number | null;
choice_scores?: Record<string, unknown> | null;
multi_choice?: boolean | null;
description?: string | null;
tags?: Array<string> | null;
check_internet?: boolean | null;
code?: string | null;
code_language?: "python" | "javascript" | null;
messages?: Array<Record<string, unknown>> | null;
few_shot_examples?: Array<Record<string, unknown>> | null;
mode?: "auto" | "agent" | "quick" | null;
tools?: Record<string, unknown> | null;
knowledge_bases?: Array<string> | null;
data_injection?: Record<string, unknown> | null;
summary?: Record<string, unknown> | null;
error_localizer_enabled?: boolean | null;
publish?: boolean | null;
template_format?: "mustache" | "jinja" | null;
} Parameters
Positional argument template_id.
llmcodeagent pass_failpercentagedeterministic pythonjavascript autoagentquick mustachejinja Returns
Questions & Discussion