Chapter 4: Training and Evaluation

4.1 Contrastive Learning for Multimodal Models

Contrastive learning is the foundation of modern vision-language models. It learns representations by maximizing similarity between matched pairs while minimizing similarity with unmatched pairs.

InfoNCE Loss

# Implementing Contrastive Loss
import torch
import torch.nn as nn
import torch.nn.functional as F

class ContrastiveLoss(nn.Module):
    def __init__(self, temperature=0.07):
        super().__init__()
        self.temperature = nn.Parameter(torch.tensor(temperature))

    def forward(self, image_embeds, text_embeds):
        """
        image_embeds: (N, D) normalized image embeddings
        text_embeds: (N, D) normalized text embeddings
        """
        # Compute similarity matrix
        logits = image_embeds @ text_embeds.T / self.temperature.exp()

        # Labels: diagonal elements are positive pairs
        labels = torch.arange(len(image_embeds), device=logits.device)

        # Cross-entropy loss in both directions
        loss_i2t = F.cross_entropy(logits, labels)
        loss_t2i = F.cross_entropy(logits.T, labels)

        return (loss_i2t + loss_t2i) / 2

# Training loop example
def train_step(model, images, texts, optimizer, criterion):
    optimizer.zero_grad()

    # Get normalized embeddings
    image_embeds = F.normalize(model.encode_image(images), dim=-1)
    text_embeds = F.normalize(model.encode_text(texts), dim=-1)

    # Compute loss
    loss = criterion(image_embeds, text_embeds)

    loss.backward()
    optimizer.step()

    return loss.item()

Large Batch Sizes: Why They Matter

Contrastive learning benefits significantly from large batch sizes:

# Gradient accumulation for effective large batches
def train_with_accumulation(model, dataloader, optimizer, criterion,
                            accumulation_steps=8):
    """Simulate large batch with gradient accumulation"""
    model.train()
    optimizer.zero_grad()

    for i, (images, texts) in enumerate(dataloader):
        # Forward pass
        image_embeds = F.normalize(model.encode_image(images), dim=-1)
        text_embeds = F.normalize(model.encode_text(texts), dim=-1)

        # Scale loss by accumulation steps
        loss = criterion(image_embeds, text_embeds) / accumulation_steps
        loss.backward()

        # Update weights every accumulation_steps
        if (i + 1) % accumulation_steps == 0:
            optimizer.step()
            optimizer.zero_grad()

    return loss.item() * accumulation_steps

4.2 Pre-training Objectives

Modern multimodal models combine multiple pre-training objectives:

BLIP-style Multi-Objective Training

# Multi-objective pre-training
class MultimodalPretraining(nn.Module):
    def __init__(self, model):
        super().__init__()
        self.model = model
        self.itc_loss = ContrastiveLoss()
        self.itm_head = nn.Linear(model.hidden_dim, 2)  # Binary classification
        self.lm_head = nn.Linear(model.hidden_dim, model.vocab_size)

    def forward(self, images, texts, text_labels):
        # 1. Image-Text Contrastive (ITC)
        image_embeds = self.model.encode_image(images)
        text_embeds = self.model.encode_text(texts)
        itc_loss = self.itc_loss(
            F.normalize(image_embeds, dim=-1),
            F.normalize(text_embeds, dim=-1)
        )

        # 2. Image-Text Matching (ITM) with hard negatives
        # Create negative pairs by shuffling
        neg_images = images[torch.randperm(len(images))]
        pos_hidden = self.model.fuse(images, texts)
        neg_hidden = self.model.fuse(neg_images, texts)

        pos_logits = self.itm_head(pos_hidden[:, 0])  # CLS token
        neg_logits = self.itm_head(neg_hidden[:, 0])

        itm_labels = torch.cat([
            torch.ones(len(images)),
            torch.zeros(len(images))
        ]).long().to(images.device)

        itm_loss = F.cross_entropy(
            torch.cat([pos_logits, neg_logits]),
            itm_labels
        )

        # 3. Language Modeling (conditioned on image)
        lm_hidden = self.model.generate_hidden(images, texts[:, :-1])
        lm_logits = self.lm_head(lm_hidden)
        lm_loss = F.cross_entropy(
            lm_logits.view(-1, self.model.vocab_size),
            text_labels[:, 1:].contiguous().view(-1),
            ignore_index=-100
        )

        # Combine losses
        total_loss = itc_loss + itm_loss + lm_loss
        return total_loss, {"itc": itc_loss, "itm": itm_loss, "lm": lm_loss}

4.3 Datasets for Multimodal Pre-training

4.4 Multimodal Hallucination

Hallucination is a critical challenge where models generate text inconsistent with visual content:

Root Causes

1. Language Prior Dominance: Model relies too heavily on learned language patterns
2. Co-occurrence Bias: Objects frequently seen together in training data
3. Weak Visual Grounding: Insufficient connection between text and image regions
4. Training Data Noise: Misaligned image-text pairs in web-scraped data

Detection and Mitigation

# Hallucination Detection using CLIP
import torch
from transformers import CLIPProcessor, CLIPModel

class HallucinationDetector:
    def __init__(self, threshold=0.3):
        self.model = CLIPModel.from_pretrained("openai/clip-vit-large-patch14")
        self.processor = CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14")
        self.threshold = threshold

    def check_claim(self, image, claim):
        """
        Check if a claim about an image is likely hallucinated.
        Returns True if claim is supported by image, False if likely hallucinated.
        """
        # Encode image and claim
        inputs = self.processor(
            text=[claim, f"not {claim}"],
            images=image,
            return_tensors="pt",
            padding=True
        )

        with torch.no_grad():
            outputs = self.model(**inputs)
            probs = outputs.logits_per_image.softmax(dim=1)

        # High probability for claim = likely true
        claim_prob = probs[0, 0].item()
        return claim_prob > self.threshold, claim_prob

    def analyze_caption(self, image, caption):
        """Break down caption and check each claim"""
        # Simple sentence splitting (use NLP library for production)
        claims = caption.replace(",", ".").split(".")
        claims = [c.strip() for c in claims if c.strip()]

        results = []
        for claim in claims:
            is_valid, confidence = self.check_claim(image, claim)
            results.append({
                "claim": claim,
                "valid": is_valid,
                "confidence": confidence
            })

        return results

# Usage
detector = HallucinationDetector()
results = detector.analyze_caption(image, generated_caption)
for r in results:
    status = "Valid" if r["valid"] else "Hallucinated?"
    print(f"{status} ({r['confidence']:.2f}): {r['claim']}")

Mitigation Strategies

4.5 Evaluation Benchmarks

Understanding Benchmarks

Generation Benchmarks

# Computing CLIP Score for Generated Images
from torchmetrics.multimodal import CLIPScore

def evaluate_generation(generated_images, prompts):
    """Compute CLIP score for text-to-image generation"""
    metric = CLIPScore(model_name_or_path="openai/clip-vit-base-patch16")

    scores = []
    for image, prompt in zip(generated_images, prompts):
        # Image should be uint8 tensor (B, C, H, W)
        if isinstance(image, Image.Image):
            image = transforms.ToTensor()(image).unsqueeze(0) * 255
            image = image.to(torch.uint8)

        score = metric(image, prompt)
        scores.append(score.item())

    return {
        "mean_clip_score": sum(scores) / len(scores),
        "individual_scores": scores
    }

4.6 Fine-tuning Multimodal Models

# LoRA Fine-tuning for Vision-Language Models
from peft import LoraConfig, get_peft_model
from transformers import LlavaForConditionalGeneration

def setup_lora_finetuning(model_name, target_modules=None):
    """Configure LoRA for efficient VLM fine-tuning"""

    # Load base model
    model = LlavaForConditionalGeneration.from_pretrained(
        model_name,
        torch_dtype=torch.float16
    )

    # Default target modules for LLaVA
    if target_modules is None:
        target_modules = [
            "q_proj", "v_proj",  # Attention
            "mm_projector"       # Vision-language connector
        ]

    # LoRA configuration
    lora_config = LoraConfig(
        r=16,                    # Rank
        lora_alpha=32,           # Scaling
        target_modules=target_modules,
        lora_dropout=0.05,
        bias="none",
        task_type="CAUSAL_LM"
    )

    # Apply LoRA
    model = get_peft_model(model, lora_config)

    # Print trainable parameters
    trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
    total = sum(p.numel() for p in model.parameters())
    print(f"Trainable: {trainable:,} / {total:,} ({100*trainable/total:.2f}%)")

    return model

# Training configuration
from transformers import TrainingArguments, Trainer

training_args = TrainingArguments(
    output_dir="./llava-finetuned",
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    learning_rate=2e-4,
    warmup_ratio=0.03,
    lr_scheduler_type="cosine",
    fp16=True,
    logging_steps=10,
    save_strategy="epoch"
)

4.7 Summary

Exercises

Exercise 1: Implement Contrastive Loss

Implement the symmetric contrastive loss and verify it produces the correct gradients for matched vs. unmatched pairs.

Exercise 2: Hallucination Analysis

Generate captions for 10 images using different VLMs. Manually annotate hallucinations and calculate the hallucination rate per model.

Exercise 3: Benchmark Evaluation

Run a VLM on the VQAv2 validation set (subset). Compare accuracy across different question types (yes/no, counting, other).

Exercise 4: LoRA Fine-tuning

Fine-tune a VLM on a custom domain (e.g., medical images, product photos) using LoRA. Measure performance before and after fine-tuning.