Chapter 4: RAG (Retrieval-Augmented Generation)

Learning Objectives

• Understand the RAG architecture and why it's important
• Load and process documents for retrieval
• Create embeddings and store them in vector databases
• Build retrieval chains for question answering
• Apply best practices for production RAG systems

4.1 What is RAG?

RAG (Retrieval-Augmented Generation) is a technique that enhances LLM responses by retrieving relevant information from a knowledge base before generating answers.

Why RAG?

4.2 Document Loading

The first step is loading documents from various sources:

pip install beautifulsoup4 pypdf  # For WebBaseLoader and PyPDFLoader
pip install chromadb              # For Chroma vector store

from langchain_community.document_loaders import (
    TextLoader,
    PyPDFLoader,
    WebBaseLoader,
    DirectoryLoader
)

# Load a text file
text_loader = TextLoader("./data/document.txt")
text_docs = text_loader.load()

# Load a PDF
pdf_loader = PyPDFLoader("./data/paper.pdf")
pdf_docs = pdf_loader.load()

# Load a web page
web_loader = WebBaseLoader("https://example.com/article")
web_docs = web_loader.load()

# Load all files in a directory
dir_loader = DirectoryLoader("./data/", glob="**/*.txt")
all_docs = dir_loader.load()

# Each document has content and metadata
for doc in text_docs:
    print(f"Content: {doc.page_content[:100]}...")
    print(f"Metadata: {doc.metadata}")

4.3 Text Splitting

Documents need to be split into chunks for efficient retrieval:

from langchain_text_splitters import (
    RecursiveCharacterTextSplitter,
    CharacterTextSplitter
)

# Recommended: RecursiveCharacterTextSplitter
# Tries to split at natural boundaries (paragraphs, sentences, words)
splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,      # Maximum characters per chunk
    chunk_overlap=200,    # Overlap between chunks for context
    separators=["\n\n", "\n", " ", ""]  # Split priority
)

# Split documents
chunks = splitter.split_documents(text_docs)

print(f"Original documents: {len(text_docs)}")
print(f"After splitting: {len(chunks)} chunks")

# Each chunk maintains metadata
for i, chunk in enumerate(chunks[:3]):
    print(f"\nChunk {i}: {len(chunk.page_content)} chars")
    print(chunk.page_content[:100] + "...")

4.4 Embeddings and Vector Stores

Embeddings convert text into numerical vectors for similarity search:

from langchain_openai import OpenAIEmbeddings
from langchain_community.vectorstores import Chroma

# Create embeddings model
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")

# Test embedding
test_embedding = embeddings.embed_query("What is machine learning?")
print(f"Embedding dimension: {len(test_embedding)}")  # 1536

# Create vector store from documents
vectorstore = Chroma.from_documents(
    documents=chunks,
    embedding=embeddings,
    persist_directory="./chroma_db"  # Persist to disk
)

# Query the vector store
results = vectorstore.similarity_search(
    "How does neural network training work?",
    k=3  # Return top 3 results
)

for doc in results:
    print(f"Score: {doc.metadata.get('score', 'N/A')}")
    print(f"Content: {doc.page_content[:200]}...\n")

Popular Vector Stores

4.5 Building a RAG Chain

from langchain_openai import ChatOpenAI, OpenAIEmbeddings
from langchain_community.vectorstores import Chroma
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnablePassthrough

# Setup components
embeddings = OpenAIEmbeddings()
vectorstore = Chroma(persist_directory="./chroma_db", embedding_function=embeddings)
retriever = vectorstore.as_retriever(search_kwargs={"k": 3})
llm = ChatOpenAI(model="gpt-4")

# Create RAG prompt
rag_prompt = ChatPromptTemplate.from_messages([
    ("system", """You are a helpful assistant. Answer the question based only on the following context.
If the context doesn't contain relevant information, say so.

Context:
{context}"""),
    ("human", "{question}")
])

# Helper function to format documents
def format_docs(docs):
    return "\n\n".join(doc.page_content for doc in docs)

# Build RAG chain with LCEL
rag_chain = (
    {"context": retriever | format_docs, "question": RunnablePassthrough()}
    | rag_prompt
    | llm
    | StrOutputParser()
)

# Use the chain
response = rag_chain.invoke("What are the main types of machine learning?")
print(response)

4.6 Advanced RAG Patterns

from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.runnables import RunnablePassthrough
from pydantic import BaseModel, Field
from typing import List

# Define structured output with citations
class AnswerWithSources(BaseModel):
    answer: str = Field(description="The answer to the question")
    sources: List[str] = Field(description="List of source documents used")

# Create chain with structured output
llm_with_sources = ChatOpenAI(model="gpt-4").with_structured_output(AnswerWithSources)

rag_prompt_with_sources = ChatPromptTemplate.from_messages([
    ("system", """Answer based on the context. Include which sources you used.

Context:
{context}"""),
    ("human", "{question}")
])

def format_docs_with_sources(docs):
    formatted = []
    for i, doc in enumerate(docs):
        source = doc.metadata.get("source", f"doc_{i}")
        formatted.append(f"[Source: {source}]\n{doc.page_content}")
    return "\n\n".join(formatted)

rag_chain_with_sources = (
    {"context": retriever | format_docs_with_sources, "question": RunnablePassthrough()}
    | rag_prompt_with_sources
    | llm_with_sources
)

result = rag_chain_with_sources.invoke("What is deep learning?")
print(f"Answer: {result.answer}")
print(f"Sources: {result.sources}")

from langchain.retrievers import EnsembleRetriever
from langchain_community.retrievers import BM25Retriever

# Create keyword-based retriever
bm25_retriever = BM25Retriever.from_documents(chunks)
bm25_retriever.k = 3

# Create vector retriever
vector_retriever = vectorstore.as_retriever(search_kwargs={"k": 3})

# Combine with ensemble (hybrid search)
ensemble_retriever = EnsembleRetriever(
    retrievers=[bm25_retriever, vector_retriever],
    weights=[0.4, 0.6]  # 40% keyword, 60% semantic
)

# Use in RAG chain
results = ensemble_retriever.invoke("neural network optimization techniques")
for doc in results:
    print(doc.page_content[:100] + "...")

Exercise

Summary

• RAG grounds LLM responses in retrieved documents
• Document loaders ingest data from files, PDFs, web pages, etc.
• Text splitters break documents into searchable chunks
• Embeddings convert text to vectors for similarity search
• Vector stores enable efficient semantic search
• Use LCEL to compose retrieval chains
• Hybrid search combines vector and keyword matching for better results