Reduce RAG Latency: From 2000ms to 200ms

Latency Breakdown

Typical RAG pipeline (2000ms):

1. Embed query: 50ms
2. Vector search: 100ms
3. Rerank: 300ms
4. LLM generation: 1500ms

Optimized (200ms):

1. Embed query: 20ms (cached)
2. Vector search: 30ms (optimized index)
3. Rerank: 50ms (parallel)
4. LLM generation: 100ms (streaming)

1. Parallel Retrieval

DEVELOPERpython
import asyncio

async def parallel_rag(query):
    # Run embedding + search in parallel
    embed_task = asyncio.create_task(embed_async(query))
    
    # Can also search multiple indices in parallel
    search_tasks = [
        asyncio.create_task(vector_db1.search(query)),
        asyncio.create_task(vector_db2.search(query))
    ]
    
    # Wait for all
    query_emb = await embed_task
    results = await asyncio.gather(*search_tasks)
    
    # Merge and rerank
    combined = merge_results(results)
    return await rerank_async(query, combined)

2. Streaming Responses

Don't wait for full generation:

DEVELOPERpython
def stream_rag(query):
    # Fast retrieval
    context = retrieve(query)  # 100ms
    
    # Stream LLM response
    for chunk in openai.ChatCompletion.create(
        model="gpt-4-turbo",
        messages=[{
            "role": "user",
            "content": f"Context: {context}\n\nQuestion: {query}"
        }],
        stream=True
    ):
        yield chunk.choices[0].delta.content

User sees first token in 150ms instead of waiting 1500ms.

3. Approximate Nearest Neighbors

Use HNSW for 10x faster search:

DEVELOPERpython
# Qdrant with HNSW
client.update_collection(
    collection_name="docs",
    hnsw_config={
        "m": 16,  # Lower = faster but less accurate
        "ef_construct": 100
    }
)

# Search with speed priority
results = client.search(
    collection_name="docs",
    query_vector=embedding,
    search_params={"hnsw_ef": 32},  # Lower = faster
    limit=10
)

4. Smaller Reranking Models

DEVELOPERpython
# Fast reranker (50ms for 20 docs)
from sentence_transformers import CrossEncoder

model = CrossEncoder('cross-encoder/ms-marco-TinyBERT-L-2-v2')  # Tiny!

def fast_rerank(query, docs):
    pairs = [[query, doc] for doc in docs]
    scores = model.predict(pairs)  # 50ms
    return sorted(zip(docs, scores), reverse=True)[:10]

5. Reduce Context Size

Fewer retrieved docs = faster LLM:

DEVELOPERpython
# Instead of 10 long docs, use 5 short ones
context = "\n\n".join([
    doc[:200]  # First 200 chars only
    for doc in retrieve(query, k=5)
])

6. Edge Caching

CDN-level caching for popular queries:

DEVELOPERpython
# Cloudflare Workers
async function handleRequest(request) {
    const cache = caches.default
    const cachedResponse = await cache.match(request)
    
    if (cachedResponse) {
        return cachedResponse  // < 10ms
    }
    
    const response = await ragPipeline(request)
    await cache.put(request, response.clone())
    
    return response
}

Complete Optimized Pipeline

DEVELOPERpython
async def optimized_rag(query):
    # 1. Check cache (10ms)
    cached = await redis_get(query)
    if cached:
        return cached
    
    # 2. Parallel embed + search (50ms)
    embed_task = embed_async(query)
    search_task = vector_db.search_async(query, k=20)
    
    query_emb, candidates = await asyncio.gather(embed_task, search_task)
    
    # 3. Fast rerank (50ms)
    reranked = fast_rerank(query, candidates[:20])
    
    # 4. Stream response (100ms to first token)
    context = "\n".join([d[:300] for d in reranked[:5]])
    
    async for chunk in stream_llm(query, context):
        yield chunk
    
    # Total: ~200ms to first token

From 2000ms to 200ms - 10x faster with smart optimizations.