Scores
Reference for client.scores in the Future AGI Python and TypeScript SDKs: scores.create, scores.create_bulk, scores.list_for_source.
client.scores on fi.Client (Python) and client.scores on Client from @future-agi/sdk (TypeScript). The models these methods return are on Annotation response models; how arguments, pagination, errors and strict mode work is in the overview.
Methods
| Method | Description |
|---|---|
scores.create | Record one score (annotation value) against a source such as a span or dataset row. |
scores.create_bulk | Record several scores against one source in a single call. |
scores.list_for_source | List the scores recorded against one source. |
client.scores
scores.create
Record one score (annotation value) against a source such as a span or dataset row.
/model-hub/scores/ result = client.scores.create(
source_type="dataset_row",
source_id="your-source-id",
label_id="your-label-id",
value="your-value",
)const result = await client.scores.create({
source_type: "dataset_row",
source_id: "your-source-id",
label_id: "your-label-id",
value: "your-value",
}); Signature
client.scores.create(
*,
source_type: Literal['dataset_row', 'trace', 'observation_span', 'prototype_run', 'call_execution', 'trace_session'],
source_id: str,
label_id: str,
value: Any,
notes: str | None | NotGiven = NOT_GIVEN,
score_source: Literal['human', 'api', 'auto', 'imported'] | None | NotGiven = NOT_GIVEN,
queue_item_id: str | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.Scoreclient.scores.create(params: ScoresCreateParams, options?: RequestOptions): Promise<models.Score>
export interface ScoresCreateParams {
source_type: "dataset_row" | "trace" | "observation_span" | "prototype_run" | "call_execution" | "trace_session";
source_id: string;
label_id: string;
value: JsonValue;
notes?: string | null;
score_source?: "human" | "api" | "auto" | "imported" | null;
queue_item_id?: string | null;
} Parameters
dataset_rowtraceobservation_spanprototype_runcall_executiontrace_session humanapiautoimported Returns
scores.create_bulk
Record several scores against one source in a single call.
/model-hub/scores/bulk/ result = client.scores.create_bulk(
source_type="dataset_row",
source_id="your-source-id",
scores=[],
)const result = await client.scores.createBulk({
source_type: "dataset_row",
source_id: "your-source-id",
scores: [],
}); Signature
client.scores.create_bulk(
*,
source_type: Literal['dataset_row', 'trace', 'observation_span', 'prototype_run', 'call_execution', 'trace_session'],
source_id: str,
scores: list[dict[str, Any]],
notes: str | None | NotGiven = NOT_GIVEN,
span_notes: str | None | NotGiven = NOT_GIVEN,
span_notes_source_id: str | None | NotGiven = NOT_GIVEN,
queue_item_id: str | None | NotGiven = NOT_GIVEN,
project_id: str | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
extra_body: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> models.BulkCreateScoresResultclient.scores.createBulk(params: ScoresCreateBulkParams, options?: RequestOptions): Promise<models.BulkCreateScoresResult>
export interface ScoresCreateBulkParams {
source_type: "dataset_row" | "trace" | "observation_span" | "prototype_run" | "call_execution" | "trace_session";
source_id: string;
scores: Array<Record<string, unknown>>;
notes?: string | null;
span_notes?: string | null;
span_notes_source_id?: string | null;
queue_item_id?: string | null;
project_id?: string | null;
} Parameters
dataset_rowtraceobservation_spanprototype_runcall_executiontrace_session Returns
scores.list_for_source
List the scores recorded against one source.
/model-hub/scores/for-source/ result = client.scores.list_for_source(source_type="dataset_row", source_id="your-source-id")const result = await client.scores.listForSource({
source_type: "dataset_row",
source_id: "your-source-id",
}); Signature
client.scores.list_for_source(
*,
source_type: Literal['dataset_row', 'trace', 'observation_span', 'prototype_run', 'call_execution', 'trace_session'],
source_id: str,
page: int | None | NotGiven = NOT_GIVEN,
limit: int | None | NotGiven = NOT_GIVEN,
project_id: str | None | NotGiven = NOT_GIVEN,
extra_query: dict[str, Any] | None = None,
options: RequestOptions | None = None,
) -> List[models.Score]client.scores.listForSource(params: ScoresListForSourceParams, options?: RequestOptions): Promise<Array<models.Score>>
export interface ScoresListForSourceParams {
source_type: "dataset_row" | "trace" | "observation_span" | "prototype_run" | "call_execution" | "trace_session";
source_id: string;
/** A page number within the paginated result set. */
page?: number | null;
/** Number of results to return per page. */
limit?: number | null;
/** Tracer project the trace / span was opened from. The same id can exist in several projects; when supplied, only that project's scores are listed. */
project_id?: string | null;
} Parameters
dataset_rowtraceobservation_spanprototype_runcall_executiontrace_session A page number within the paginated result set.
Number of results to return per page.
Tracer project the trace / span was opened from. The same id can exist in several projects; when supplied, only that project’s scores are listed.
Returns
A list of Score.
Questions & Discussion