Image Instruction Evaluation

from fi.evals import ImageInstruction, EvalClient
from fi.testcases import MLLMTestCase, MLLMImage

# Initialize the evaluation client
evaluator = EvalClient(
    fi_api_key="your_api_key",
    fi_secret_key="your_secret_key"
)

# Create a test case using MLLMImage object
test_case = MLLMTestCase(
    input="The image shows two brown dogs with their heads down, appearing to sniff or eat something in the grass. They are in a grassy area with a blurred background, possibly a fence or garden.",
    image_url=MLLMImage(url="https://fastly.picsum.photos/id/169/200/200.jpg?hmac=MquoCIcsCP_IxfteFmd8LfVF7NCoRre282nO9gVD0Yc")
)

# Or create a test case using direct URL
test_case_with_url = MLLMTestCase(
    input="The image shows two brown dogs with their heads down, appearing to sniff or eat something in the grass. They are in a grassy area with a blurred background, possibly a fence or garden.",
    image_url="https://fastly.picsum.photos/id/169/200/200.jpg?hmac=MquoCIcsCP_IxfteFmd8LfVF7NCoRre282nO9gVD0Yc"
)

# Initialize the image instruction evaluator
image_eval = ImageInstruction(
    config = {
        "criteria": """
        Evaluate the image based on:
        1. Accuracy of object representation (apple)
        2. Setting accuracy (wooden table)
        3. Color accuracy (red)
        4. Overall composition
        5. Image quality and realism
        """
    }
)

# Run the evaluation
result = evaluator.evaluate(image_eval, test_case)
print(result)  # Will return a score between 0 and 1