path = '/content/data.json'# Open and load the JSON filewith open(path, 'r') as file: datapoints = json.load(file)
Copy
# Sample Datapointdatapoint = { 'id': 'masked_id', 'image_url': './images/output_8_0.png', 'output_image_url': './images/output_26_0.png', 'prompt': 'an asian man, closeup, on new york city street', 'type': 'T2I', 'category': 'Ethnicity', 'question': 'Does the image follow the Ethnicity mentioned in the prompt?'}
Copy
# Sample Imageresponse = requests.get(datapoint['image_url'])# Display the image in the notebookif response.status_code == 200: display(Image(response.content))else: print("Failed to fetch the image.")
Initializing the FutureAGI Evaluator Class and Deterministic Eval
Copy
from getpass import getpassfrom fi.evals import Evaluatorfi_api_key = getpass("Enter your FI API Key: ")fi_secret_key = getpass("Enter your FI Secret Key: ")evaluator = Evaluator( fi_api_key=fi_api_key, fi_secret_key=fi_secret_key, fi_base_url="https://api.futureagi.com")print("Evaluator client initialized successfully!")
image_input_output_eval = ImageInputOutput(config={ "criteria": """ Evaluate the output image based on: 1. Adherence to input instruction 2. Preservation of key elements from input image 3. Quality of color modification 4. Image quality and realism """})
Copy
class ImageInputOutputTestCase(MLLMTestCase): input: str input_image_url: str output_image_url: str
Copy
response = requests.get(datapoint['output_image_url'])# Display the image in the notebookif response.status_code == 200: display(Image(response.content))else: print("Failed to fetch the image.")
Output:
Copy
test_case_image_input_output = ImageInputOutputTestCase( input='Replace the man with a man of African ethnicity', input_image_url=datapoint['image_url'], output_image_url=datapoint['output_image_url'])
The output image accurately replaces the man with one of African ethnicity while preserving all key elements, maintaining high image quality and realism.