Chapter 2: Vision-Language Models

2.1 Introduction to Vision-Language Models

Vision-Language Models (VLMs) are multimodal AI systems that can understand and reason about both images and text. They represent a critical advancement in AI, enabling applications from image captioning to visual question answering.

2.2 CLIP: Contrastive Language-Image Pre-training

CLIP, released by OpenAI in 2021, revolutionized vision-language models by demonstrating that contrastive learning at scale creates powerful cross-modal representations.

CLIP Architecture

Contrastive Learning Objective

CLIP learns by maximizing similarity between matched image-text pairs while minimizing similarity with unmatched pairs:

CLIP Implementation

# CLIP for Zero-Shot Image Classification
from transformers import CLIPProcessor, CLIPModel
from PIL import Image
import requests
import torch

# Load CLIP model
model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")

# Load image
url = "https://images.unsplash.com/photo-1560807707-8cc77767d783?w=400"
image = Image.open(requests.get(url, stream=True).raw)

# Define candidate labels
labels = ["a photo of a dog", "a photo of a cat", "a photo of a bird"]

# Process inputs
inputs = processor(
    text=labels,
    images=image,
    return_tensors="pt",
    padding=True
)

# Get similarity scores
outputs = model(**inputs)
logits_per_image = outputs.logits_per_image
probs = logits_per_image.softmax(dim=1)

# Print results
for label, prob in zip(labels, probs[0]):
    print(f"{label}: {prob.item():.2%}")
# Output: "a photo of a dog: 98.5%"

CLIP for Image-Text Retrieval

# Image-Text Retrieval with CLIP
import torch
from transformers import CLIPProcessor, CLIPModel

class CLIPRetriever:
    def __init__(self, model_name="openai/clip-vit-base-patch32"):
        self.model = CLIPModel.from_pretrained(model_name)
        self.processor = CLIPProcessor.from_pretrained(model_name)
        self.image_embeddings = None
        self.images = []

    def index_images(self, images):
        """Pre-compute embeddings for image database"""
        self.images = images
        inputs = self.processor(images=images, return_tensors="pt", padding=True)

        with torch.no_grad():
            self.image_embeddings = self.model.get_image_features(**inputs)
            self.image_embeddings = self.image_embeddings / self.image_embeddings.norm(dim=-1, keepdim=True)

    def search(self, query_text, top_k=5):
        """Find most similar images to text query"""
        inputs = self.processor(text=[query_text], return_tensors="pt", padding=True)

        with torch.no_grad():
            text_embedding = self.model.get_text_features(**inputs)
            text_embedding = text_embedding / text_embedding.norm(dim=-1, keepdim=True)

        # Compute similarities
        similarities = (text_embedding @ self.image_embeddings.T).squeeze()
        top_indices = similarities.argsort(descending=True)[:top_k]

        return [(self.images[i], similarities[i].item()) for i in top_indices]

# Usage
retriever = CLIPRetriever()
retriever.index_images(image_database)
results = retriever.search("a sunset over the ocean")

2.3 BLIP and BLIP-2: Bootstrapped Vision-Language

BLIP (Bootstrapping Language-Image Pre-training) improves upon CLIP by combining contrastive learning with generative objectives.

BLIP-2 Architecture: Q-Former

Q-Former: The Key Innovation

The Q-Former (Querying Transformer) bridges frozen image encoders and LLMs:

• Learned Queries: 32 learnable query embeddings that extract visual information
• Cross-Attention: Queries attend to visual features from frozen ViT
• Self-Attention: Queries interact with each other and text tokens
• Efficiency: Only Q-Former is trained, keeping vision encoder and LLM frozen

# BLIP-2 for Image Captioning
from transformers import Blip2Processor, Blip2ForConditionalGeneration
from PIL import Image
import torch

# Load BLIP-2 model
processor = Blip2Processor.from_pretrained("Salesforce/blip2-opt-2.7b")
model = Blip2ForConditionalGeneration.from_pretrained(
    "Salesforce/blip2-opt-2.7b",
    torch_dtype=torch.float16
)
model.to("cuda")

# Process image
image = Image.open("example.jpg")
inputs = processor(image, return_tensors="pt").to("cuda", torch.float16)

# Generate caption
generated_ids = model.generate(**inputs, max_new_tokens=50)
caption = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
print(f"Caption: {caption}")

BLIP-2 for Visual Question Answering

# Visual QA with BLIP-2
def ask_about_image(image_path, question):
    image = Image.open(image_path)

    # Format prompt for QA
    prompt = f"Question: {question} Answer:"

    inputs = processor(image, prompt, return_tensors="pt").to("cuda", torch.float16)

    generated_ids = model.generate(
        **inputs,
        max_new_tokens=30,
        num_beams=5,
        early_stopping=True
    )

    answer = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
    return answer.strip()

# Example usage
answer = ask_about_image("street_scene.jpg", "How many people are in this image?")
print(f"Answer: {answer}")  # "There are 5 people in this image"

2.4 LLaVA: Visual Instruction Tuning

LLaVA (Large Language and Vision Assistant) demonstrates that simple architectures can achieve impressive results through visual instruction tuning.

LLaVA Architecture

LLaVA Training Strategy

LLaVA Implementation

# Using LLaVA for Visual Conversation
from transformers import LlavaProcessor, LlavaForConditionalGeneration
from PIL import Image
import torch

# Load LLaVA model
model_id = "llava-hf/llava-1.5-7b-hf"
processor = LlavaProcessor.from_pretrained(model_id)
model = LlavaForConditionalGeneration.from_pretrained(
    model_id,
    torch_dtype=torch.float16,
    device_map="auto"
)

def chat_with_image(image_path, user_message):
    image = Image.open(image_path)

    # LLaVA conversation format
    conversation = [
        {
            "role": "user",
            "content": [
                {"type": "image"},
                {"type": "text", "text": user_message}
            ]
        }
    ]

    prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
    inputs = processor(prompt, image, return_tensors="pt").to("cuda")

    output = model.generate(**inputs, max_new_tokens=200)
    response = processor.decode(output[0], skip_special_tokens=True)

    return response

# Multi-turn conversation
response1 = chat_with_image("kitchen.jpg", "What do you see in this image?")
print(response1)

response2 = chat_with_image("kitchen.jpg", "What could I cook with the ingredients visible?")
print(response2)

2.5 Visual Tokenization Strategies

How images are converted to tokens significantly impacts VLM performance:

Patch-Based Tokenization (ViT-style)

# Patch-based visual tokenization
import torch
import torch.nn as nn

class PatchEmbedding(nn.Module):
    def __init__(self, img_size=224, patch_size=16, in_channels=3, embed_dim=768):
        super().__init__()
        self.img_size = img_size
        self.patch_size = patch_size
        self.n_patches = (img_size // patch_size) ** 2  # 196 for 224/16

        # Project patches to embedding dimension
        self.proj = nn.Conv2d(
            in_channels,
            embed_dim,
            kernel_size=patch_size,
            stride=patch_size
        )

        # Learnable position embeddings
        self.pos_embed = nn.Parameter(torch.randn(1, self.n_patches + 1, embed_dim))
        self.cls_token = nn.Parameter(torch.randn(1, 1, embed_dim))

    def forward(self, x):
        B = x.shape[0]

        # Create patch embeddings: (B, C, H, W) -> (B, embed_dim, H/P, W/P)
        x = self.proj(x)

        # Flatten: (B, embed_dim, n_patches^0.5, n_patches^0.5) -> (B, n_patches, embed_dim)
        x = x.flatten(2).transpose(1, 2)

        # Add CLS token
        cls_tokens = self.cls_token.expand(B, -1, -1)
        x = torch.cat([cls_tokens, x], dim=1)

        # Add position embeddings
        x = x + self.pos_embed

        return x  # (B, n_patches + 1, embed_dim)

# Example: 224x224 image -> 197 tokens (196 patches + 1 CLS)
patch_embed = PatchEmbedding()
image = torch.randn(1, 3, 224, 224)
tokens = patch_embed(image)
print(f"Visual tokens shape: {tokens.shape}")  # (1, 197, 768)

Comparison of Tokenization Approaches

2.6 Cross-Attention Mechanisms in VLMs

Cross-attention allows one modality to query information from another:

# Cross-Attention Implementation
class CrossAttention(nn.Module):
    def __init__(self, dim, num_heads=8):
        super().__init__()
        self.num_heads = num_heads
        self.head_dim = dim // num_heads
        self.scale = self.head_dim ** -0.5

        # Query from one modality, Key/Value from another
        self.q_proj = nn.Linear(dim, dim)
        self.k_proj = nn.Linear(dim, dim)
        self.v_proj = nn.Linear(dim, dim)
        self.out_proj = nn.Linear(dim, dim)

    def forward(self, query_tokens, kv_tokens):
        """
        query_tokens: (B, N_q, D) - e.g., text tokens
        kv_tokens: (B, N_kv, D) - e.g., image tokens
        """
        B, N_q, D = query_tokens.shape
        N_kv = kv_tokens.shape[1]

        # Project to Q, K, V
        Q = self.q_proj(query_tokens).view(B, N_q, self.num_heads, self.head_dim).transpose(1, 2)
        K = self.k_proj(kv_tokens).view(B, N_kv, self.num_heads, self.head_dim).transpose(1, 2)
        V = self.v_proj(kv_tokens).view(B, N_kv, self.num_heads, self.head_dim).transpose(1, 2)

        # Attention: text queries attend to image keys/values
        attn = (Q @ K.transpose(-2, -1)) * self.scale
        attn = attn.softmax(dim=-1)

        # Aggregate values
        out = (attn @ V).transpose(1, 2).reshape(B, N_q, D)
        return self.out_proj(out)

# Example: Text tokens (10) attend to image tokens (196)
cross_attn = CrossAttention(dim=768)
text_tokens = torch.randn(1, 10, 768)
image_tokens = torch.randn(1, 196, 768)
attended = cross_attn(text_tokens, image_tokens)
print(f"Output shape: {attended.shape}")  # (1, 10, 768)

2.7 Unified Embedding Spaces

VLMs create shared embedding spaces where images and text can be compared directly:

# Visualizing CLIP's Unified Embedding Space
import numpy as np
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt

def visualize_clip_space(images, texts, model, processor):
    """Visualize how images and texts cluster in CLIP space"""

    # Get image embeddings
    image_inputs = processor(images=images, return_tensors="pt", padding=True)
    with torch.no_grad():
        image_embeds = model.get_image_features(**image_inputs)
        image_embeds = image_embeds / image_embeds.norm(dim=-1, keepdim=True)

    # Get text embeddings
    text_inputs = processor(text=texts, return_tensors="pt", padding=True)
    with torch.no_grad():
        text_embeds = model.get_text_features(**text_inputs)
        text_embeds = text_embeds / text_embeds.norm(dim=-1, keepdim=True)

    # Combine and reduce dimensionality
    all_embeds = torch.cat([image_embeds, text_embeds], dim=0).numpy()
    tsne = TSNE(n_components=2, perplexity=min(30, len(all_embeds)-1))
    reduced = tsne.fit_transform(all_embeds)

    # Plot
    n_images = len(images)
    plt.figure(figsize=(10, 8))
    plt.scatter(reduced[:n_images, 0], reduced[:n_images, 1], c='blue', label='Images', s=100)
    plt.scatter(reduced[n_images:, 0], reduced[n_images:, 1], c='red', label='Texts', s=100)

    # Draw lines between matched pairs
    for i in range(min(n_images, len(texts))):
        plt.plot([reduced[i, 0], reduced[n_images+i, 0]],
                 [reduced[i, 1], reduced[n_images+i, 1]], 'g--', alpha=0.5)

    plt.legend()
    plt.title("CLIP Unified Embedding Space")
    plt.show()

2.8 Summary

Exercises

Exercise 1: Zero-Shot Classification

Use CLIP to classify images into custom categories (e.g., "professional photo", "amateur photo", "AI-generated image"). Compare results with different text prompt formulations.

Exercise 2: Q-Former Analysis

Explain why BLIP-2 uses a fixed number of query tokens (32) regardless of image resolution. What are the trade-offs of this design choice?

Exercise 3: LLaVA Conversation

Implement a multi-turn visual conversation where you ask follow-up questions about an image. How does context from previous turns affect answers?

Exercise 4: Embedding Space Visualization

Create a visualization showing how CLIP embeddings cluster for images of different categories. Do semantically similar images cluster together?