Experiment
Anthropic
Introduction
Evaluation
- Overview
- Quickstart
- Concept
- How To
- Eval Definition
Build
- Overview
- Concept
- How To
Experimentation
Observability
Admin & Settings
Experiment
Anthropic
1. Installation
First install the FutureAGI package to access the observability framework
pip install futureagi
pip install anthropic
import os
os.environ["FI_API_KEY"] = FI_API_KEY
os.environ["FI_SECRET_KEY"] = FI_SECRET_KEY
os.environ["ANTHROPIC_API_KEY"] = ANTHROPIC_API_KEY
import anthropic
from fi.integrations.otel import AnthropicInstrumentor, register
from fi.integrations.otel.types import (
EvalName,
EvalSpanKind,
EvalTag,
EvalTagType,
ProjectType,
)
eval_tags = [
EvalTag(
eval_name=EvalName.DETERMINISTIC_EVALS,
value=EvalSpanKind.TOOL,
type=EvalTagType.OBSERVATION_SPAN,
config={
"multi_choice": False,
"choices": ["Yes", "No"],
"rule_prompt": "Evaluate if the response is correct",
},
custom_eval_name="det_eval_anthropic_1"
)
]
trace_provider = register(
project_type=ProjectType.EXPERIMENT,
eval_tags=eval_tags,
project_name="FUTURE_AGI432423",
project_version_name="v1",
)
AnthropicInstrumentor().instrument(tracer_provider=trace_provider)
client = anthropic.Anthropic()
response = client.messages.create(
model="claude-3-5-sonnet-20240620",
max_tokens=1024,
tools=[
{
"name": "get_weather",
"description": "Get the current weather in a given location",
"input_schema": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA",
},
"unit": {
"type": "string",
"enum": ["celsius", "fahrenheit"],
"description": "The unit of temperature, either 'celsius' or 'fahrenheit'",
},
},
"required": ["location"],
},
},
{
"name": "get_time",
"description": "Get the current time in a given time zone",
"input_schema": {
"type": "object",
"properties": {
"timezone": {
"type": "string",
"description": "The IANA time zone name, e.g. America/Los_Angeles",
}
},
"required": ["timezone"],
},
},
],
messages=[
{
"role": "user",
"content": "What is the weather like right now in New York?"
" Also what time is it there? Use necessary tools simultaneously.",
}
],
)
print(response)
On this page