Mastering Evaluation of AI Agents

pip install futureagi==0.3.7

import pandas as pd

dataset = pd.read_csv("functiondata.csv")
pd.set_option('display.max_colwidth', None)
dataset.fillna('{}')
dataset.head(2)

from getpass import getpass
from fi.evals import Evaluator

evaluator = Evaluator(
    getpass("Enter your Future AGI API key: ")
)

from fi.evals import LLMFunctionCalling, InstructionAdherence
from fi.testcase import TestCase

agentic_function_eval = LLMFunctionCalling(config={"model": "gpt-4o-mini"})

results_1 = []
for index, row in dataset.iterrows():
    test_case_1 = TestCase(
        input=row['input'],
        output=row['function_calling']
    )
    result_1 = evaluator.evaluate(eval_templates=[agentic_function_eval], inputs=[test_case_1])
    option_1 = result_1.eval_results[0].metrics[0].value
    results_1.append(option_1)

agentic_instruction_eval = InstructionAdherence(config={"model": "gpt-4o-mini"})

results_2 = []
for index, row in dataset.iterrows():
    # Create a test case with input and output
    test_case_2 = TestCase(
        input=row['input'],
        output=row['output']
    )
    # Evaluate the test case
    result_2 = evaluator.evaluate(eval_templates=[agentic_instruction_eval], inputs=[test_case_2])
    # Get the first evaluation result
    option_2 = result_2.eval_results[0]
    # Create a dictionary with prompt identifier, failure, and reason
    result_dict = {
        'value': option_2.metrics[0].value,
        'reason': option_2.reason,
    }
    # Append the dictionary to results_2
    results_2.append(result_dict)

from fi.evals import Tone, Toxicity, ContextRelevance

agentic_tone_eval = Tone(config={"model": "gpt-4o-mini"})
results_3 = []
for index, row in dataset.iterrows():
    test_case_3 = TestCase(
        input=row['output']
    )
    result_3 = evaluator.evaluate(eval_templates=[agentic_tone_eval], inputs=[test_case_3])
    option_3 = result_3.eval_results[0]
    results_dict = {}
    # Check if option_3.data is not empty before accessing its elements
    if option_3.data:
        results_dict = {
            'tone': option_3.data,
        }
    else:
        # Handle the case where option_3.data is empty (e.g., assign a default value)
        results_dict = {
            'tone': 'N/A',  # or any other appropriate value
        }

    results_3.append(results_dict)

agentic_toxicity_eval = Toxicity(config={"model": "gpt-4o-mini"})
results_4 = []
for index, row in dataset.iterrows():
    test_case_4 = TestCase(
        input=row['output']
    )
    result_4 = evaluator.evaluate(eval_templates=[agentic_toxicity_eval], inputs=[test_case_4])
    option_4 = result_4.eval_results[0]
    results_dict = {
        'toxicity': option_4.data[0],
    }
    results_4.append(results_dict)

agentic_context_eval = ContextRelevance(config={"model": "gpt-4o-mini", "check_internet": False})
results_5 = []
for index, row in dataset.iterrows():
    test_case_5 = TestCase(
        input=row['input'],
        context=row['output']
    )
    result_5 = evaluator.evaluate(eval_templates=[agentic_context_eval], inputs=[test_case_5])
    option_5 = result_5.eval_results[0]
    results_dict = {
        'context': option_5.metrics[0].value,
    }
    results_5.append(results_dict)

import pandas as pd

functioncalling_df = pd.DataFrame(results_1)
instructionadherance_df = pd.DataFrame(results_2)
tone_df = pd.DataFrame(results_3)
toxicity_df = pd.DataFrame(results_4)
context_df = pd.DataFrame(results_5)

combined_df = pd.concat([dataset, functioncalling_df, instructionadherance_df, tone_df, toxicity_df, context_df], axis=1)
combined_df.columns = ['input', 'function_calling', 'output', 'function_calling_eval', 'instruction_adherence_eval', 'tone_eval', 'toxicity_eval', 'context_eval']
combined_df