How to Decrease RAG Hallucinations with Future AGI

project/
├── data.csv             # Dataset used in this experiment in CSV format
├── config.yaml          # Configuration file defining experiment parameters
└── rag_experiment.py    # Main script to run RAG setup and evaluation

future_agi:
  api_key: "API_KEY"
  secret_key: "SECRET_KEY"
  base_url: "https://api.futureagi.com"
  project_name: "Experiment_RAG_Evaluation"
  project_version: "RecursiveCharacterTextSplitter_similarity_map_reduce"

openai:
  api_key: "OPENAI_API_KEY"
  llm_model: "gpt-4o-mini"
  llm_temperature: 0.5
  embedding_model: "text-embedding-3-small"

# --- Data Loading ---
data:
  file_path: "./data.csv"
  encoding: "utf-8"

# --- Chunking Strategy ---
chunking:
  enabled: true # Set to false to load documents whole (1 doc per CSV row)
  # Options: RecursiveCharacterTextSplitter, CharacterTextSplitter
  splitter_type: "RecursiveCharacterTextSplitter"
  chunk_size: 1000
  chunk_overlap: 150

# --- Retrieval Strategy ---
retrieval:
  # Options: "similarity", "mmr" (Maximal Marginal Relevance)
  search_type: "similarity"
  k: 3 # Number of documents to retrieve and pass to the LLM

# --- Chain Strategy ---
chain:
  # Options: "stuff", "map_reduce", "refine", "map_rerank"
  type: "map_reduce"
  return_source_documents: true

# --- Evaluation ---
evaluation:
  queries:
    - "Who found the answer to a search query collar george herbert essay?"
    - "What are some of the potential negative impacts of charity as discussed in the context?"
    - "Who were the three stars in the NHL game between Buffalo Sabres and Edmonton Oilers?"
    - "What services does Pearl Moving Company in Santa Clarita, 91390 offer?"
    - "What are the responsibilities of a Senior Planning Engineer in London, United Kingdom?"

pip install pyyaml
pip install langchain-openai
pip install langchain-community

pip install traceAI-langchain

import os
import csv
import yaml
import argparse
import traceback

from langchain_openai import ChatOpenAI
from langchain_core.prompts import PromptTemplate
from langchain_core.output_parsers import StrOutputParser
from langchain_community.document_loaders.csv_loader import CSVLoader
from langchain_community.vectorstores import FAISS
from langchain_openai import OpenAIEmbeddings
from langchain.text_splitter import (
    CharacterTextSplitter,
    RecursiveCharacterTextSplitter
)
from langchain.chains import RetrievalQA
from traceai_langchain import LangChainInstrumentor
from fi_instrumentation import register
from fi_instrumentation.fi_types import (
    EvalName,
    EvalSpanKind,
    EvalTag,
    EvalTagType,
    ProjectType
)

def load_config(config_path: str) -> dict:
    try:
        with open(config_path, 'r') as f:
            config = yaml.safe_load(f)
        print(f"Configuration loaded successfully from {config_path}")
        return config
    except FileNotFoundError:
        print(f"Error: Configuration file not found at {config_path}")
        exit(1)
    except yaml.YAMLError as e:
        print(f"Error parsing YAML file {config_path}: {e}")
        exit(1)
    except Exception as e:
        print(f"An unexpected error occurred while loading config: {e}")
        exit(1)

def setup_environment(config: dict):
    os.environ["FI_API_KEY"] = config['future_agi'].get('api_key')
    os.environ["FI_SECRET_KEY"] = config['future_agi'].get('secret_key')
    os.environ["OPENAI_API_KEY"] = config['openai'].get('api_key')
    os.environ["FI_BASE_URL"] = config['future_agi'].get('base_url', os.environ.get('FI_BASE_URL', 'https://api.futureagi.com'))

def setup_instrumentation(config: dict)
    eval_tags=[
        EvalTag(
            type=EvalTagType.OBSERVATION_SPAN,
            value=EvalSpanKind.LLM,
            eval_name=EvalName.GROUNDEDNESS,
            config={},
            mapping={
                "input": "llm.input_messages.1.message.content",
                "output": "llm.output_messages.0.message.content"
            },
            custom_eval_name="Groundedness"
        ),
        EvalTag(
            type=EvalTagType.OBSERVATION_SPAN,
            value=EvalSpanKind.LLM,
            eval_name=EvalName.CONTEXT_ADHERENCE,
            config={},
            mapping={
                "context": "llm.input_messages.0.message.content",
                "output": "llm.output_messages.0.message.content"
            },
            custom_eval_name="Context_Adherence"
        ),
        EvalTag(
            type=EvalTagType.OBSERVATION_SPAN,
            value=EvalSpanKind.LLM,
            eval_name=EvalName.EVAL_CONTEXT_RETRIEVAL_QUALITY,
            config={
                "criteria": "Evaluate if the context is relevant and sufficient to support the output."
            },
            mapping={
                "input": "llm.input_messages.1.message.content",
                "output": "llm.output_messages.0.message.content",
                "context": "llm.input_messages.0.message.content"
            },
            custom_eval_name="Context_Retrieval_Quality"
        )
    ]

    trace_provider = register(
        project_type=ProjectType.EXPERIMENT,
        project_name=config['future_agi']['project_name'],
        project_version_name=config['future_agi']['project_version'],
        eval_tags=eval_tags
    )
    LangChainInstrumentor().instrument(tracer_provider=trace_provider)
    print(f"FutureAGI instrumentation setup for Project: {config['future_agi']['project_name']}, Version: {config['future_agi']['project_version']}")

def setup_rag(config: dict):
    data_config = config['data']
    chunking_config = config['chunking']
    retrieval_config = config['retrieval']
    chain_config = config['chain']
    openai_config = config['openai']

    print(f"--- RAG Setup using Configuration ---")
    print(f"Data Path: {data_config['file_path']}")
    print(f"Chunking Enabled: {chunking_config['enabled']}")
    if chunking_config['enabled']:
        print(f"Chunker: {chunking_config['splitter_type']}, Size: {chunking_config['chunk_size']}, Overlap: {chunking_config['chunk_overlap']}")
    print(f"Retrieval Type: {retrieval_config['search_type']}, k: {retrieval_config['k']}")
    if retrieval_config['search_type'] == 'mmr':
         print(f"MMR Fetch K: {retrieval_config.get('fetch_k', 20)}, Lambda: {retrieval_config.get('lambda_mult', 0.5)}")
    print(f"Chain Type: {chain_config['type']}")
    print(f"LLM Model: {openai_config['llm_model']}, Temp: {openai_config['llm_temperature']}")
    print(f"Embedding Model: {openai_config.get('embedding_model', 'Default')}")
    print("-" * 30)

    try:
        # 1. Load Documents
        loader_args = {
            "file_path": data_config['file_path'],
            "encoding": data_config['encoding'],
        }
        if data_config.get('source_column'):
            loader_args['source_column'] = data_config['source_column']
        if data_config.get('metadata_columns'):
             loader_args['csv_args'] = {'fieldnames': data_config['metadata_columns']}

        loader = CSVLoader(**loader_args)
        documents = loader.load()
        print(f"Loaded {len(documents)} documents.")

        if not documents:
            print(f"No documents loaded. Check file content and CSVLoader configuration.")
            return None

        # 2. Chunk Documents (if enabled)
        if chunking_config['enabled']:
            splitter_type = chunking_config['splitter_type']
            if splitter_type == "RecursiveCharacterTextSplitter":
                text_splitter = RecursiveCharacterTextSplitter(
                    chunk_size=chunking_config['chunk_size'],
                    chunk_overlap=chunking_config['chunk_overlap'],
                    length_function=len,
                    add_start_index=True,
                )
            elif splitter_type == "CharacterTextSplitter":
                 text_splitter = CharacterTextSplitter(
                    separator="\n\n",
                    chunk_size=chunking_config['chunk_size'],
                    chunk_overlap=chunking_config['chunk_overlap'],
                    length_function=len,
                )
            else:
                print(f"Warning: Unknown splitter_type '{splitter_type}'. Defaulting to RecursiveCharacterTextSplitter.")
                text_splitter = RecursiveCharacterTextSplitter(
                    chunk_size=chunking_config['chunk_size'],
                    chunk_overlap=chunking_config['chunk_overlap']
                )

            docs_to_index = text_splitter.split_documents(documents)
            print(f"Split into {len(docs_to_index)} chunks.")
        else:
            docs_to_index = documents
            print("Chunking disabled, indexing whole documents.")

        # 3. Create Embeddings
        embedding_model_name = openai_config.get('embedding_model')
        if embedding_model_name:
            embeddings = OpenAIEmbeddings(model=embedding_model_name)
        else:
            embeddings = OpenAIEmbeddings()

        # 4. Create Vector Store
        print("Creating vector store...")
        vectorstore = FAISS.from_documents(docs_to_index, embeddings)
        print("Vector store created successfully.")

        # 5. Create Retriever
        retriever_kwargs = {"k": retrieval_config['k']}
        search_type = retrieval_config['search_type']
        if search_type == "mmr":
            retriever_kwargs['fetch_k'] = retrieval_config.get('fetch_k', 20)
            retriever_kwargs['lambda_mult'] = retrieval_config.get('lambda_mult', 0.5)

        retriever = vectorstore.as_retriever(
            search_type=search_type,
            search_kwargs=retriever_kwargs
        )

        # 6. Create LLM
        llm = ChatOpenAI(
            temperature=openai_config['llm_temperature'],
            model=openai_config['llm_model']
        )

        # 7. Create RetrievalQA Chain
        rag_chain = RetrievalQA.from_chain_type(
            llm=llm,
            chain_type=chain_config['type'],
            retriever=retriever,
            return_source_documents=chain_config['return_source_documents']
        )
        print("RAG chain setup complete.")
        return rag_chain

    except ValueError as ve:
        print(f"ValueError during RAG setup: {ve}")
        if "got an unexpected keyword argument 'fieldnames'" in str(ve):
             print("Hint: Check 'metadata_columns' in config.yaml. CSVLoader might expect them differently or they might not exist.")
        elif "must have a source_column" in str(ve):
             print("Hint: Check 'source_column' in config.yaml. It might be missing or incorrect.")
        else:
             print("This might relate to CSV column names specified in config.yaml (source_column, metadata_columns) not matching data.csv.")
        traceback.print_exc()
        return None
    except Exception as e:
        print(f"Error setting up RAG system: {str(e)}")
        traceback.print_exc()
        return None

def process_query(rag_chain, query: str, data_file_path: str):
    if rag_chain is None:
         return f"Sorry, the knowledge base from '{data_file_path}' could not be loaded. RAG chain is None."

    try:
        print(f"Invoking RAG chain for query: '{query}'")
        result = rag_chain.invoke({"query": query})
        response = result.get("result", "No answer could be generated based on the documents.")

        if rag_chain.return_source_documents:
            source_docs = result.get("source_documents", [])
            print(f"Retrieved {len(source_docs)} source documents for the answer.")
        return response

    except Exception as e:
        print(f"Error processing RAG query: {str(e)}")
        traceback.print_exc()
        return f"Sorry, I encountered an error during retrieval or generation: {str(e)}"

def run_evaluation_queries(config: dict):
    print("\n--- Initializing RAG based on Configuration ---")
    rag_chain = setup_rag(config)

    if rag_chain is None:
        print("\n--- RAG Setup Failed. Cannot run evaluation queries. Please check errors above. ---")
        return {}

    print("\n--- Starting RAG Evaluation Queries ---")
    queries = config['evaluation']['queries']
    data_file_path = config['data']['file_path']

    if not queries or any("[Your Column Name]" in q for q in queries):
        print("\n*** WARNING: Please replace placeholder queries in config.yaml under 'evaluation.queries'")
        print("*** with questions relevant to your specific data.csv file for meaningful evaluation! ***\n")

    results = {}
    for i, query in enumerate(queries):
        print(f"\n--- Query {i+1}/{len(queries)} ---")
        print(f"Query: {query}")
        response = process_query(rag_chain, query, data_file_path)
        print(f"Response: {response}")
        results[query] = response
        print("-" * 20)

    print("\n--- RAG Evaluation Queries Finished ---")
    print("Check the FutureAGI platform for traces and evaluation results.")
    print(f"Project: {config['future_agi']['project_name']}, Version: {config['future_agi']['project_version']}")
    return results

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Run RAG evaluation with configuration from a YAML file.")
    parser.add_argument(
        "-c", "--config",
        default="config.yaml",
        help="Path to the YAML configuration file (default: config.yaml)"
    )
    args = parser.parse_args()

    # Load Configuration
    config = load_config(args.config)

    # Setup Environment (API Keys etc.)
    setup_environment(config)

    # Setup FutureAGI Instrumentation
    setup_instrumentation(config)

    # Run Evaluation
    run_evaluation_queries(config)

    print("\nScript finished.")