Scores

Reference for client.scores in the Future AGI Python and TypeScript SDKs: scores.create, scores.create_bulk, scores.list_for_source.

client.scores on fi.Client (Python) and client.scores on Client from @future-agi/sdk (TypeScript). The models these methods return are on Annotation response models; how arguments, pagination, errors and strict mode work is in the overview.

Methods

MethodDescription
scores.createRecord one score (annotation value) against a source such as a span or dataset row.
scores.create_bulkRecord several scores against one source in a single call.
scores.list_for_sourceList the scores recorded against one source.

client.scores

scores.create

Record one score (annotation value) against a source such as a span or dataset row.

POST /model-hub/scores/
result = client.scores.create(
    source_type="dataset_row",
    source_id="your-source-id",
    label_id="your-label-id",
    value="your-value",
)
const result = await client.scores.create({
  source_type: "dataset_row",
  source_id: "your-source-id",
  label_id: "your-label-id",
  value: "your-value",
});
Signature
client.scores.create(
    *,
    source_type: Literal['dataset_row', 'trace', 'observation_span', 'prototype_run', 'call_execution', 'trace_session'],
    source_id: str,
    label_id: str,
    value: Any,
    notes: str | None | NotGiven = NOT_GIVEN,
    score_source: Literal['human', 'api', 'auto', 'imported'] | None | NotGiven = NOT_GIVEN,
    queue_item_id: str | None | NotGiven = NOT_GIVEN,
    extra_query: dict[str, Any] | None = None,
    extra_body: dict[str, Any] | None = None,
    options: RequestOptions | None = None,
) -> models.Score
client.scores.create(params: ScoresCreateParams, options?: RequestOptions): Promise<models.Score>

export interface ScoresCreateParams {
  source_type: "dataset_row" | "trace" | "observation_span" | "prototype_run" | "call_execution" | "trace_session";
  source_id: string;
  label_id: string;
  value: JsonValue;
  notes?: string | null;
  score_source?: "human" | "api" | "auto" | "imported" | null;
  queue_item_id?: string | null;
}

Parameters

source_type string Required
Allowed values:
dataset_rowtraceobservation_spanprototype_runcall_executiontrace_session
source_id string Required
label_id string Required
value any JSON value Required
notes string Optional
score_source string Optional
Allowed values:
humanapiautoimported
queue_item_id string Optional

Returns

Score.

scores.create_bulk

Record several scores against one source in a single call.

POST /model-hub/scores/bulk/
result = client.scores.create_bulk(
    source_type="dataset_row",
    source_id="your-source-id",
    scores=[],
)
const result = await client.scores.createBulk({
  source_type: "dataset_row",
  source_id: "your-source-id",
  scores: [],
});
Signature
client.scores.create_bulk(
    *,
    source_type: Literal['dataset_row', 'trace', 'observation_span', 'prototype_run', 'call_execution', 'trace_session'],
    source_id: str,
    scores: list[dict[str, Any]],
    notes: str | None | NotGiven = NOT_GIVEN,
    span_notes: str | None | NotGiven = NOT_GIVEN,
    span_notes_source_id: str | None | NotGiven = NOT_GIVEN,
    queue_item_id: str | None | NotGiven = NOT_GIVEN,
    project_id: str | None | NotGiven = NOT_GIVEN,
    extra_query: dict[str, Any] | None = None,
    extra_body: dict[str, Any] | None = None,
    options: RequestOptions | None = None,
) -> models.BulkCreateScoresResult
client.scores.createBulk(params: ScoresCreateBulkParams, options?: RequestOptions): Promise<models.BulkCreateScoresResult>

export interface ScoresCreateBulkParams {
  source_type: "dataset_row" | "trace" | "observation_span" | "prototype_run" | "call_execution" | "trace_session";
  source_id: string;
  scores: Array<Record<string, unknown>>;
  notes?: string | null;
  span_notes?: string | null;
  span_notes_source_id?: string | null;
  queue_item_id?: string | null;
  project_id?: string | null;
}

Parameters

source_type string Required
Allowed values:
dataset_rowtraceobservation_spanprototype_runcall_executiontrace_session
source_id string Required
scores object[] Required
notes string Optional
span_notes string Optional
span_notes_source_id string Optional
queue_item_id string Optional
project_id string Optional

Returns

BulkCreateScoresResult.

scores.list_for_source

List the scores recorded against one source.

GET /model-hub/scores/for-source/
result = client.scores.list_for_source(source_type="dataset_row", source_id="your-source-id")
const result = await client.scores.listForSource({
  source_type: "dataset_row",
  source_id: "your-source-id",
});
Signature
client.scores.list_for_source(
    *,
    source_type: Literal['dataset_row', 'trace', 'observation_span', 'prototype_run', 'call_execution', 'trace_session'],
    source_id: str,
    page: int | None | NotGiven = NOT_GIVEN,
    limit: int | None | NotGiven = NOT_GIVEN,
    project_id: str | None | NotGiven = NOT_GIVEN,
    extra_query: dict[str, Any] | None = None,
    options: RequestOptions | None = None,
) -> List[models.Score]
client.scores.listForSource(params: ScoresListForSourceParams, options?: RequestOptions): Promise<Array<models.Score>>

export interface ScoresListForSourceParams {
  source_type: "dataset_row" | "trace" | "observation_span" | "prototype_run" | "call_execution" | "trace_session";
  source_id: string;
  /** A page number within the paginated result set. */
  page?: number | null;
  /** Number of results to return per page. */
  limit?: number | null;
  /** Tracer project the trace / span was opened from. The same id can exist in several projects; when supplied, only that project's scores are listed. */
  project_id?: string | null;
}

Parameters

source_type string Required
Allowed values:
dataset_rowtraceobservation_spanprototype_runcall_executiontrace_session
source_id string Required
page integer Optional

A page number within the paginated result set.

limit integer Optional

Number of results to return per page.

project_id string Optional

Tracer project the trace / span was opened from. The same id can exist in several projects; when supplied, only that project’s scores are listed.

Returns

A list of Score.

Was this page helpful?

Questions & Discussion