path = '/content/data.json'# Open and load the JSON filewith open(path, 'r') as file: datapoints = json.load(file)
Copy
# Sample Datapointdatapoint = { 'id': 'masked_id', 'image_url': './images/output_8_0.png', 'output_image_url': './images/output_26_0.png', 'prompt': 'an asian man, closeup, on new york city street', 'type': 'T2I', 'category': 'Ethnicity', 'question': 'Does the image follow the Ethnicity mentioned in the prompt?'}
Copy
# Sample Imageresponse = requests.get(datapoint['image_url'])# Display the image in the notebookif response.status_code == 200: display(Image(response.content))else: print("Failed to fetch the image.")
Initializing the FutureAGI Evaluator Class and Deterministic Eval
Copy
from getpass import getpassfrom fi.evals import Evaluatorfi_api_key = getpass("Enter your FI API Key: ")fi_secret_key = getpass("Enter your FI Secret Key: ")evaluator = Evaluator( fi_api_key=fi_api_key, fi_secret_key=fi_secret_key, fi_base_url="https://api.futureagi.com")print("Evaluator client initialized successfully!")
image_input_output_eval = ImageInputOutput(config={ "criteria": """ Evaluate the output image based on: 1. Adherence to input instruction 2. Preservation of key elements from input image 3. Quality of color modification 4. Image quality and realism """})
Copy
class ImageInputOutputTestCase(MLLMTestCase): input: str input_image_url: str output_image_url: str
Copy
response = requests.get(datapoint['output_image_url'])# Display the image in the notebookif response.status_code == 200: display(Image(response.content))else: print("Failed to fetch the image.")
Output:
Copy
test_case_image_input_output = ImageInputOutputTestCase( input='Replace the man with a man of African ethnicity', input_image_url=datapoint['image_url'], output_image_url=datapoint['output_image_url'])
The output image accurately replaces the man with one of African ethnicity while preserving all key elements, maintaining high image quality and realism.
Assistant
Responses are generated using AI and may contain mistakes.