Pourquoi l'Expansion de Requêtes ?

Problème : La requête utilisateur est trop courte ou utilise des mots différents

Exemple :

• Utilisateur : "ML models"
• Documents pertinents utilisent : "machine learning algorithms", "neural networks", "deep learning"

L'expansion de requêtes reformule les requêtes pour correspondre à plus de documents.

Méthode 1 : Expansion par Synonymes

DEVELOPERpython
from nltk.corpus import wordnet

def expand_with_synonyms(query):
    words = query.split()
    expanded_queries = [query]  # Inclure l'original

    for word in words:
        synonyms = []
        for syn in wordnet.synsets(word):
            for lemma in syn.lemmas():
                if lemma.name() != word:
                    synonyms.append(lemma.name().replace('_', ' '))

        # Ajouter des variations de synonymes
        if synonyms:
            expanded = query.replace(word, synonyms[0])
            expanded_queries.append(expanded)

    return list(set(expanded_queries))

# Exemple
queries = expand_with_synonyms("fast car")
# ["fast car", "quick car", "fast automobile", "quick automobile"]

Méthode 2 : Reformulation par LLM

DEVELOPERpython
import openai

def expand_with_llm(query):
    response = openai.ChatCompletion.create(
        model="gpt-4-turbo",
        messages=[{
            "role": "system",
            "content": "Generate 3 alternative phrasings of the user's query. Output as JSON array."
        }, {
            "role": "user",
            "content": query
        }],
        response_format={"type": "json_object"}
    )

    variations = json.loads(response.choices[0].message.content)
    return [query] + variations["alternatives"]

# Exemple
queries = expand_with_llm("How to reduce costs?")
# [
#   "How to reduce costs?",
#   "What are cost reduction strategies?",
#   "Ways to lower expenses",
#   "Best practices for cutting costs"
# ]

Méthode 3 : Récupération Multi-Requêtes

Rechercher avec toutes les variations, fusionner les résultats :

DEVELOPERpython
def multi_query_retrieval(query, vector_db):
    # Générer les variations
    queries = expand_with_llm(query)

    # Récupérer pour chacune
    all_results = []
    for q in queries:
        q_emb = embed(q)
        results = vector_db.search(q_emb, limit=20)
        all_results.extend(results)

    # Dédupliquer et classer par fréquence
    doc_scores = {}
    for doc in all_results:
        if doc.id not in doc_scores:
            doc_scores[doc.id] = 0
        doc_scores[doc.id] += doc.score

    # Trier par score combiné
    ranked = sorted(doc_scores.items(), key=lambda x: x[1], reverse=True)

    return ranked[:10]

Méthode 4 : HyDE (Hypothetical Document Embeddings)

Générer une fausse réponse, la rechercher :

DEVELOPERpython
def hyde_retrieval(query):
    # Générer une réponse hypothétique
    hypothetical_doc = openai.ChatCompletion.create(
        model="gpt-4-turbo",
        messages=[{
            "role": "system",
            "content": "Write a detailed answer to this question as if it were a Wikipedia article."
        }, {
            "role": "user",
            "content": query
        }]
    ).choices[0].message.content

    # Créer l'embedding du document hypothétique (pas de la requête !)
    doc_embedding = embed(hypothetical_doc)

    # Rechercher des documents similaires
    results = vector_db.search(doc_embedding, limit=10)

    return results

Méthode 5 : Step-Back Prompting

Poser d'abord une question plus large :

DEVELOPERpython
def step_back_expansion(query):
    # Générer une question plus large
    step_back = openai.ChatCompletion.create(
        model="gpt-4-turbo",
        messages=[{
            "role": "system",
            "content": "Given a specific question, generate a broader, more general question."
        }, {
            "role": "user",
            "content": query
        }]
    ).choices[0].message.content

    return [query, step_back]

# Exemple
queries = step_back_expansion("What is the capital of France?")
# [
#   "What is the capital of France?",
#   "What are capitals of European countries?"
# ]

Méthode 6 : Décomposition en Sous-Requêtes

Diviser les requêtes complexes en parties :

DEVELOPERpython
def decompose_query(query):
    response = openai.ChatCompletion.create(
        model="gpt-4-turbo",
        messages=[{
            "role": "system",
            "content": "Break this complex question into 2-3 simpler sub-questions. Return JSON array."
        }, {
            "role": "user",
            "content": query
        }],
        response_format={"type": "json_object"}
    )

    sub_queries = json.loads(response.choices[0].message.content)["sub_questions"]

    # Récupérer pour chaque sous-requête
    all_results = []
    for sq in sub_queries:
        results = vector_db.search(embed(sq), limit=5)
        all_results.extend(results)

    return deduplicate(all_results)

# Exemple
sub_queries = decompose_query("How does photosynthesis affect climate change?")
# [
#   "What is photosynthesis?",
#   "How do plants remove CO2?",
#   "What is the relationship between CO2 and climate?"
# ]

Implémentation Langchain

DEVELOPERpython
from langchain.retrievers.multi_query import MultiQueryRetriever
from langchain.llms import OpenAI

retriever = MultiQueryRetriever.from_llm(
    retriever=vector_store.as_retriever(),
    llm=OpenAI(temperature=0)
)

# Expansion automatique de la requête
docs = retriever.get_relevant_documents("How to train neural networks?")

Évaluation

DEVELOPERpython
# Mesurer l'amélioration du recall
def evaluate_expansion(queries, ground_truth_docs):
    recall_baseline = []
    recall_expanded = []

    for query, relevant_docs in zip(queries, ground_truth_docs):
        # Baseline
        base_results = vector_db.search(embed(query), limit=10)
        base_recall = len(set(base_results) & set(relevant_docs)) / len(relevant_docs)
        recall_baseline.append(base_recall)

        # Étendu
        expanded_results = multi_query_retrieval(query, vector_db)
        exp_recall = len(set(expanded_results) & set(relevant_docs)) / len(relevant_docs)
        recall_expanded.append(exp_recall)

    print(f"Baseline recall: {np.mean(recall_baseline):.2f}")
    print(f"Expanded recall: {np.mean(recall_expanded):.2f}")

L'expansion de requêtes est peu coûteuse et à fort impact. Augmentez le recall de 30-50% instantanément.