Advanced Methods in Natural Language Processing - Session 8¶

Table of Contents¶

1. Part 1: Fine-Tuning BERT

   • 1.1. Loading and exploring AG News
   • 1.2. TF-IDF baseline
   • 1.3. Tokenization for BERT
   • 1.4. Adding a classification head
   • 1.5. Training with the HF Trainer
   • 1.6. Frozen-encoder variant

2. Part 2: Few-Shot Learning with SetFit

   • 2.1. SetFit: Theory
   • 2.2. Setup
   • 2.3. Training SetFit with 8 examples / class
   • 2.4. Apples-to-apples comparison (TF-IDF / BERT-200 / SetFit-8)
   • 2.5. Visualizing how SetFit re-shapes the embedding space
   • 2.6. Augmenting the training set with zero-shot prompting
   • 2.7. Overall evaluation

3. Part 3: Biases in BERT models: WinoGender schemas

   • 3.1. Filling pronouns with BERT-large / BERT / DistilBERT
   • 3.2. Visualizing the gender bias per occupation

Part 0: Metrics Functions to Consider¶

Before diving into the model building and training, we set up a small Metrics helper that stores the accuracy / precision / recall / F1 of every model we train, so we can compare them side-by-side at the end of each Part.

In [ ]:

import matplotlib.pyplot as plt
import numpy as np
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score


class Metrics:
    """Tiny container to log and plot classification metrics across runs."""

    def __init__(self):
        self.results = {}

    def run(self, y_true, y_pred, method_name, average='macro'):
        self.results[method_name] = {
            'accuracy':  accuracy_score(y_true, y_pred),
            'precision': precision_score(y_true, y_pred, average=average, zero_division=0),
            'recall':    recall_score(y_true, y_pred, average=average, zero_division=0),
            'f1':        f1_score(y_true, y_pred, average=average, zero_division=0),
        }

    def plot(self):
        fig, axs = plt.subplots(2, 2, figsize=(15, 10))
        for i, metric in enumerate(['accuracy', 'precision', 'recall', 'f1']):
            ax = axs[i // 2, i % 2]
            values = [res[metric] * 100 for res in self.results.values()]
            ax.bar(self.results.keys(), values)
            ax.set_title(metric)
            ax.set_ylim(0, 100)
            for j, v in enumerate(values):
                ax.text(j, v + 0.5, f'{v:.2f}', ha='center', va='bottom')
            plt.setp(ax.get_xticklabels(), rotation=20, ha='right')
        plt.tight_layout()
        plt.show()

metrics_val = Metrics()

import matplotlib.pyplot as plt import numpy as np from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score class Metrics: """Tiny container to log and plot classification metrics across runs.""" def __init__(self): self.results = {} def run(self, y_true, y_pred, method_name, average='macro'): self.results[method_name] = { 'accuracy': accuracy_score(y_true, y_pred), 'precision': precision_score(y_true, y_pred, average=average, zero_division=0), 'recall': recall_score(y_true, y_pred, average=average, zero_division=0), 'f1': f1_score(y_true, y_pred, average=average, zero_division=0), } def plot(self): fig, axs = plt.subplots(2, 2, figsize=(15, 10)) for i, metric in enumerate(['accuracy', 'precision', 'recall', 'f1']): ax = axs[i // 2, i % 2] values = [res[metric] * 100 for res in self.results.values()] ax.bar(self.results.keys(), values) ax.set_title(metric) ax.set_ylim(0, 100) for j, v in enumerate(values): ax.text(j, v + 0.5, f'{v:.2f}', ha='center', va='bottom') plt.setp(ax.get_xticklabels(), rotation=20, ha='right') plt.tight_layout() plt.show() metrics_val = Metrics()

Part 1: Fine-tuning BERT ¶

We fine-tune a small DistilBERT on AG News (4 classes: World / Sports / Business / Sci-Tech). We use the HuggingFace Trainer (PyTorch) which is the modern, recommended API.

1.1 Loading and exploring AG News¶

We load AG News from the HuggingFace Hub and build a small stratified split so that training stays fast on a laptop / Colab.

In [ ]:

from datasets import load_dataset
from sklearn.model_selection import train_test_split

# ✅ TASK: load AG News and build small stratified train (~10k) / valid (~2k) splits.
#
# 🪜 Instructions:
# 1. dataset = load_dataset('ag_news')
# 2. Extract `data, labels` from the train split (and `test_data, test_labels` for later).
# 3. Stratified split into (train, valid).
# 4. Subsample train -> 10k, valid -> 2k (still stratified, random_state=42).

dataset =
data, labels =
test_data, test_labels =

train_data, valid_data, train_labels, valid_labels =

train_data, _, train_labels, _ =
valid_data, _, valid_labels, _ =

from datasets import load_dataset from sklearn.model_selection import train_test_split # ✅ TASK: load AG News and build small stratified train (~10k) / valid (~2k) splits. # # 🪜 Instructions: # 1. dataset = load_dataset('ag_news') # 2. Extract `data, labels` from the train split (and `test_data, test_labels` for later). # 3. Stratified split into (train, valid). # 4. Subsample train -> 10k, valid -> 2k (still stratified, random_state=42). dataset = data, labels = test_data, test_labels = train_data, valid_data, train_labels, valid_labels = train_data, _, train_labels, _ = valid_data, _, valid_labels, _ =

In [ ]:

# ✅ TASK: explore the class balance and topic vocabulary.
#
# 🧰 Tools: collections.Counter, defaultdict, wordcloud.WordCloud, nltk stopwords.
#
# 🪜 Instructions:
# 1. Build {label_id -> concatenated text} for the 4 classes.
# 2. Plot a 2x2 grid of WordClouds (one per class).
# 3. Print Counter(train_labels) to confirm the split is balanced.

from collections import Counter, defaultdict
from wordcloud import WordCloud
import nltk
from nltk.corpus import stopwords

nltk.download('stopwords', quiet=True)
stop_words = set(stopwords.words('english'))
labels_map = {0: 'World', 1: 'Sports', 2: 'Business', 3: 'Sci/Tech'}

label_text =

fig, axs = plt.subplots(2, 2, figsize=(10, 6))
# ... (plot wordclouds)

Counter(train_labels)

# ✅ TASK: explore the class balance and topic vocabulary. # # 🧰 Tools: collections.Counter, defaultdict, wordcloud.WordCloud, nltk stopwords. # # 🪜 Instructions: # 1. Build {label_id -> concatenated text} for the 4 classes. # 2. Plot a 2x2 grid of WordClouds (one per class). # 3. Print Counter(train_labels) to confirm the split is balanced. from collections import Counter, defaultdict from wordcloud import WordCloud import nltk from nltk.corpus import stopwords nltk.download('stopwords', quiet=True) stop_words = set(stopwords.words('english')) labels_map = {0: 'World', 1: 'Sports', 2: 'Business', 3: 'Sci/Tech'} label_text = fig, axs = plt.subplots(2, 2, figsize=(10, 6)) # ... (plot wordclouds) Counter(train_labels)

1.2 TF-IDF baseline¶

We start with a plain TF-IDF + Logistic Regression pipeline trained on the full 10k training split. This is the strong classical baseline SetFit will eventually need to beat with only a handful of examples.

In [ ]:

# ✅ TASK: train a TF-IDF + Logistic Regression baseline on the full 10k training set.
#
# 🧰 Tools: TfidfVectorizer, LogisticRegression, Pipeline.
#
# 🪜 Instructions:
# 1. Build a Pipeline(tfidf -> logistic regression).
# 2. Fit on (train_data, train_labels).
# 3. Predict on `valid_data` and log into metrics_val as 'TF-IDF (full 10k)'.

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline

tfidf_pipeline = Pipeline([
    # ('tfidf', TfidfVectorizer(...)),
    # ('clf',   LogisticRegression(...)),
])
tfidf_pipeline

valid_preds =
metrics_val.run(valid_labels, valid_preds, 'TF-IDF (full 10k)')
metrics_val.plot()

# ✅ TASK: train a TF-IDF + Logistic Regression baseline on the full 10k training set. # # 🧰 Tools: TfidfVectorizer, LogisticRegression, Pipeline. # # 🪜 Instructions: # 1. Build a Pipeline(tfidf -> logistic regression). # 2. Fit on (train_data, train_labels). # 3. Predict on `valid_data` and log into metrics_val as 'TF-IDF (full 10k)'. from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.linear_model import LogisticRegression from sklearn.pipeline import Pipeline tfidf_pipeline = Pipeline([ # ('tfidf', TfidfVectorizer(...)), # ('clf', LogisticRegression(...)), ]) tfidf_pipeline valid_preds = metrics_val.run(valid_labels, valid_preds, 'TF-IDF (full 10k)') metrics_val.plot()

1.3 Tokenization for BERT¶

Unlike TF-IDF, BERT does not take raw text. Each sentence must first be turned into a fixed-length sequence of integer token-ids that match BERT's WordPiece vocabulary.

The HuggingFace tokenizer takes care of three things at once:

1. WordPiece splitting, rare words are broken into known sub-words. For example tokenization → [tokeni, ##zation].
2. Special tokens, every input is wrapped as [CLS] ... [SEP]. The hidden state of [CLS] is what we use for classification.
3. Padding + attention mask, sentences are padded (or truncated) to a fixed max_length so we can batch them as a tensor. The attention_mask (1s for real tokens, 0s for padding) tells the model which positions to ignore.

Concretely, you get back two arrays per sentence:

Tensor	Shape	What it holds
input_ids	(B, L)	token ids, padded to max_length=L
attention_mask	(B, L)	1 where there is content, 0 for padding

Below we tokenize two toy sentences to see exactly what comes out.

In [ ]:

# ✅ TASK: tokenize two demo sentences with DistilBERT and inspect what you get back.
#
# 🧰 Tools: transformers.AutoTokenizer.
#
# 🪜 Instructions:
# 1. Load 'distilbert-base-uncased' tokenizer into `tokenizer`.
# 2. Tokenize ['I am a BSE student', 'I am a [MASK] student'] with
#    max_length=64, padding='max_length', truncation=True, return_tensors='pt'.
# 3. Print the shape of input_ids / attention_mask + the first 12 ids of each.
# 4. Use `tokenizer.decode(...)` to confirm round-tripping.

from transformers import AutoTokenizer
checkpoint = 'distilbert-base-uncased'
tokenizer =
max_length = 64

demo =
# print shapes, decode, etc.

# ✅ TASK: tokenize two demo sentences with DistilBERT and inspect what you get back. # # 🧰 Tools: transformers.AutoTokenizer. # # 🪜 Instructions: # 1. Load 'distilbert-base-uncased' tokenizer into `tokenizer`. # 2. Tokenize ['I am a BSE student', 'I am a [MASK] student'] with # max_length=64, padding='max_length', truncation=True, return_tensors='pt'. # 3. Print the shape of input_ids / attention_mask + the first 12 ids of each. # 4. Use `tokenizer.decode(...)` to confirm round-tripping. from transformers import AutoTokenizer checkpoint = 'distilbert-base-uncased' tokenizer = max_length = 64 demo = # print shapes, decode, etc.

1.4 Adding a classification head¶

DistilBERT outputs, for every input token, a 768-dimensional contextual vector. To turn this into a probability distribution over our 4 classes, we add a small classification head on top.

The standard recipe (and the one HuggingFace's AutoModelForSequenceClassification implements out of the box) is:

1. Take the last-layer hidden state of the [CLS] token (shape (768,)).
2. (Optional) apply dropout to fight overfitting on small datasets.
3. Project it through a linear layer Linear(768, num_labels).
4. Apply softmax to get class probabilities (the loss does this implicitly).

During fine-tuning, gradients flow all the way back through the encoder, so BERT's weights are adapted to AG News (we'll also try a frozen-encoder variant in 1.6 to see how much that matters).

We do not have to write this head ourselves, AutoModelForSequenceClassification loads the pretrained DistilBERT encoder, attaches a randomly initialised classification head with num_labels outputs, and wires up the cross-entropy loss automatically.

1.5 Training with the HuggingFace Trainer¶

We tokenize the whole train/valid split once, wrap everything as a HuggingFace Dataset, and let Trainer handle the optimization loop (gradient accumulation, mixed precision, logging, checkpointing).

In [ ]:

# ✅ TASK: write a helper `train_bert(...)` that fine-tunes DistilBERT with HF Trainer.
#
# 🧰 Tools: AutoModelForSequenceClassification, Trainer, TrainingArguments,
#          DataCollatorWithPadding, datasets.Dataset.
#
# 🪜 Instructions:
# 1. Wrap (texts, labels) into a HF Dataset and tokenize it batched.
# 2. Load AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=...).
# 3. If freeze_encoder=True, set requires_grad=False on `model.distilbert`.
# 4. Configure TrainingArguments (batch_size=32, lr=5e-5, no checkpointing, no logging to W&B).
# 5. Trainer.train(), then trainer.predict(valid_ds).predictions.argmax(-1).
# 6. Return (model, valid_preds).

import torch
from datasets import Dataset
from transformers import (
    AutoModelForSequenceClassification,
    Trainer,
    TrainingArguments,
    DataCollatorWithPadding,
)

device = 'cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu')
num_labels = len(set(train_labels))

def to_hf_dataset(texts, labels):
    return Dataset.from_dict({'text': list(texts), 'label': list(labels)})

def tokenize(batch):
    return tokenizer(batch['text'], truncation=True, max_length=max_length)

def train_bert(train_texts, train_labels, valid_texts, num_train_epochs=1,
               freeze_encoder=False, output_dir='./tmp_bert'):
    """Train DistilBERT on (train_texts, train_labels), return (model, valid_preds)."""
    train_ds =
    valid_ds =

    model =
    if freeze_encoder:
        # freeze the encoder, keep classification head trainable
        ...

    args = TrainingArguments(
        # output_dir=..., num_train_epochs=..., per_device_train_batch_size=32, ...
    )
    trainer = Trainer(
        # model=..., args=..., train_dataset=..., tokenizer=tokenizer,
        # data_collator=DataCollatorWithPadding(tokenizer),
    )
    trainer

    preds =
    return model, preds

# ✅ TASK: write a helper `train_bert(...)` that fine-tunes DistilBERT with HF Trainer. # # 🧰 Tools: AutoModelForSequenceClassification, Trainer, TrainingArguments, # DataCollatorWithPadding, datasets.Dataset. # # 🪜 Instructions: # 1. Wrap (texts, labels) into a HF Dataset and tokenize it batched. # 2. Load AutoModelForSequenceClassification.from_pretrained(checkpoint, num_labels=...). # 3. If freeze_encoder=True, set requires_grad=False on `model.distilbert`. # 4. Configure TrainingArguments (batch_size=32, lr=5e-5, no checkpointing, no logging to W&B). # 5. Trainer.train(), then trainer.predict(valid_ds).predictions.argmax(-1). # 6. Return (model, valid_preds). import torch from datasets import Dataset from transformers import ( AutoModelForSequenceClassification, Trainer, TrainingArguments, DataCollatorWithPadding, ) device = 'cuda' if torch.cuda.is_available() else ('mps' if torch.backends.mps.is_available() else 'cpu') num_labels = len(set(train_labels)) def to_hf_dataset(texts, labels): return Dataset.from_dict({'text': list(texts), 'label': list(labels)}) def tokenize(batch): return tokenizer(batch['text'], truncation=True, max_length=max_length) def train_bert(train_texts, train_labels, valid_texts, num_train_epochs=1, freeze_encoder=False, output_dir='./tmp_bert'): """Train DistilBERT on (train_texts, train_labels), return (model, valid_preds).""" train_ds = valid_ds = model = if freeze_encoder: # freeze the encoder, keep classification head trainable ... args = TrainingArguments( # output_dir=..., num_train_epochs=..., per_device_train_batch_size=32, ... ) trainer = Trainer( # model=..., args=..., train_dataset=..., tokenizer=tokenizer, # data_collator=DataCollatorWithPadding(tokenizer), ) trainer preds = return model, preds

In [ ]:

# ✅ TASK: fine-tune DistilBERT on the full 10k training set.
#
# 🪜 Instructions:
# 1. Call train_bert(train_data, train_labels, valid_data, num_train_epochs=1,
#                    freeze_encoder=False, output_dir='./tmp_bert_full').
# 2. Log results under 'DistilBERT (full 10k)' in metrics_val.

bert_model, valid_preds_bert =
metrics_val.run(valid_labels, valid_preds_bert, 'DistilBERT (full 10k)')
metrics_val.plot()

# ✅ TASK: fine-tune DistilBERT on the full 10k training set. # # 🪜 Instructions: # 1. Call train_bert(train_data, train_labels, valid_data, num_train_epochs=1, # freeze_encoder=False, output_dir='./tmp_bert_full'). # 2. Log results under 'DistilBERT (full 10k)' in metrics_val. bert_model, valid_preds_bert = metrics_val.run(valid_labels, valid_preds_bert, 'DistilBERT (full 10k)') metrics_val.plot()

1.6 Frozen-encoder variant¶

What if we freeze BERT's weights and only train the classification head? This shows how much the pretrained representations already encode about the task, vs how much we gain from actually fine-tuning the encoder.

In [ ]:

# ✅ TASK: same as above, but with a **frozen** encoder. Train for more epochs since
# only the classification head learns.

_, valid_preds_bert_frozen =
metrics_val.run(valid_labels, valid_preds_bert_frozen, 'DistilBERT frozen')
metrics_val.plot()

# ✅ TASK: same as above, but with a **frozen** encoder. Train for more epochs since # only the classification head learns. _, valid_preds_bert_frozen = metrics_val.run(valid_labels, valid_preds_bert_frozen, 'DistilBERT frozen') metrics_val.plot()

Part 2: Few-Shot Learning with SetFit ¶

In Part 1 we fine-tuned a full transformer on 10k labelled examples. In real-world projects labels are scarce and expensive. SetFit (Tunstall et al., 2022) is a few-shot text-classification recipe that needs only 8 to 64 labelled examples per class and runs on a single GPU in minutes.

2.1 SetFit: Theory¶

SetFit builds on top of a frozen Sentence Transformer backbone (a BERT-like encoder trained to produce a single sentence-level vector). It works in two stages:

Stage 1: Contrastive fine-tuning of the encoder¶

The trick is to generate many training pairs from very few labelled examples.

Given N labelled examples we form pairs:

Pair type	Construction	Target
Positive	Two sentences sharing the same label	similar embeddings
Negative	Two sentences from different labels	distant embeddings

With R pairs per example and K classes, only a few labelled sentences yield thousands of pairs, enough signal to fine-tune the encoder with a contrastive loss (cosine-similarity / triplet loss, à la SimCSE and FaceNet).

The encoder is updated so that:

• sentences of the same class end up close in embedding space,
• sentences of different classes are pushed apart.

Stage 2: Lightweight classification head¶

The fine-tuned encoder is used to embed the same few labelled examples. A simple logistic regression is then fit on top of those embeddings. No prompt, no template, no MLM head.

Why it works so well¶

• No prompts → no hand-engineering, no template sensitivity.
• Frozen architecture → very fast to train and tiny memory footprint.
• Geometry first → the contrastive objective directly shapes a separable embedding space, which is exactly what a linear head needs.
• Sample-efficient → quadratic blow-up of pair counts amplifies the signal of a handful of labels.

When to reach for SetFit¶

Setting	Use SetFit?
≤ 64 labelled examples / class, single GPU	✅ ideal
Thousands of labelled examples	❌ prefer full fine-tuning
Need calibrated probabilities	⚠️ logistic head is OK but not great
Multi-label / very long documents	⚠️ requires adaptation

References:

• Tunstall et al. (2022), Efficient Few-Shot Learning Without Prompts, arXiv:2209.11055
• Reimers & Gurevych (2019), Sentence-BERT, arXiv:1908.10084
• Schroff et al. (2015), FaceNet / triplet loss, arXiv:1503.03832

2.2 Setup¶

In [ ]:

# pip install -q setfit umap-learn   # uncomment in Colab
from setfit import SetFitModel, Trainer as SetFitTrainer, TrainingArguments as SetFitArgs, sample_dataset
from datasets import Dataset

# Small but solid Sentence Transformer (22M params, dim=384). About 5x faster
# than `paraphrase-mpnet-base-v2` on MPS / CPU, with marginal accuracy loss on AG News.
SETFIT_CKPT = 'sentence-transformers/all-MiniLM-L6-v2'

full_train_dataset = Dataset.from_dict({'text': train_data, 'label': train_labels})

# TASK: write two helpers.
#
# Tools:
# - `sample_dataset(dataset, label_column='label', num_samples=N, seed=...)`
# - `SetFitModel.from_pretrained(SETFIT_CKPT)`
# - `SetFitTrainer(model=..., args=..., train_dataset=..., metric='accuracy')`
#
# Instructions:
# 1. `make_few_shot(dataset, n_per_class, seed=42)` -> balanced few-shot subset.
# 2. `train_setfit(train_ds, num_epochs=1, num_iterations=10, batch_size=16)`
#    -> trains a fresh SetFitModel and returns it.

def make_few_shot(dataset, n_per_class, seed=42):
    return

def train_setfit(train_ds, num_epochs=1, num_iterations=10, batch_size=16):
    model =
    args =
    trainer =
    # train
    return model

# pip install -q setfit umap-learn # uncomment in Colab from setfit import SetFitModel, Trainer as SetFitTrainer, TrainingArguments as SetFitArgs, sample_dataset from datasets import Dataset # Small but solid Sentence Transformer (22M params, dim=384). About 5x faster # than `paraphrase-mpnet-base-v2` on MPS / CPU, with marginal accuracy loss on AG News. SETFIT_CKPT = 'sentence-transformers/all-MiniLM-L6-v2' full_train_dataset = Dataset.from_dict({'text': train_data, 'label': train_labels}) # TASK: write two helpers. # # Tools: # - `sample_dataset(dataset, label_column='label', num_samples=N, seed=...)` # - `SetFitModel.from_pretrained(SETFIT_CKPT)` # - `SetFitTrainer(model=..., args=..., train_dataset=..., metric='accuracy')` # # Instructions: # 1. `make_few_shot(dataset, n_per_class, seed=42)` -> balanced few-shot subset. # 2. `train_setfit(train_ds, num_epochs=1, num_iterations=10, batch_size=16)` # -> trains a fresh SetFitModel and returns it. def make_few_shot(dataset, n_per_class, seed=42): return def train_setfit(train_ds, num_epochs=1, num_iterations=10, batch_size=16): model = args = trainer = # train return model

2.3 Training SetFit with 8 examples per class¶

In [ ]:

# TASK: train SetFit with 8 examples per class and log results.
#
# Instructions:
# 1. Build `train_ds_8 = make_few_shot(full_train_dataset, n_per_class=8)`.
# 2. Train via `train_setfit(...)`.
# 3. Predict on `valid_data` (note: `.predict(...)` returns a tensor; call .cpu().numpy()).
# 4. Log under 'SetFit (8/class)'.

train_ds_8 =
model_setfit_8 =
valid_preds_setfit_8 =
metrics_val.run(valid_labels, valid_preds_setfit_8, 'SetFit (8/class)')
metrics_val.plot()

# TASK: train SetFit with 8 examples per class and log results. # # Instructions: # 1. Build `train_ds_8 = make_few_shot(full_train_dataset, n_per_class=8)`. # 2. Train via `train_setfit(...)`. # 3. Predict on `valid_data` (note: `.predict(...)` returns a tensor; call .cpu().numpy()). # 4. Log under 'SetFit (8/class)'. train_ds_8 = model_setfit_8 = valid_preds_setfit_8 = metrics_val.run(valid_labels, valid_preds_setfit_8, 'SetFit (8/class)') metrics_val.plot()

2.4 Apples-to-apples comparison at low label budget¶

We compare three models on the same validation set:

Model	Training data
TF-IDF + Logistic Regression	full 10k training set (already logged in 1.2)
DistilBERT fine-tuned	only 200 labelled examples
SetFit	only 8 examples / class (= 32 total, reused from 2.3)

Goal: show that SetFit competes with, or beats, a fine-tuned BERT at equal-or-larger label budget, while being orders of magnitude cheaper to train than the TF-IDF baseline.

In [ ]:

# TASK: train DistilBERT on a small ~200-example stratified subset, for the
# apples-to-apples comparison against SetFit at 8 examples / class.
#
# Instructions:
# 1. Stratified subsample: train_size=200 from (train_data, train_labels).
# 2. Train BERT with `train_bert(...)` for 3 epochs.
# 3. Log under 'DistilBERT (~200 ex)'.
# 4. ALSO log the SetFit predictions from section 2.3 (`valid_preds_setfit_8`)
#    under 'SetFit (8/class) [compare]', so the comparison plot shows BERT-200
#    vs SetFit-8 vs TF-IDF-full side by side. NO retraining needed.

small_train_texts, _, small_train_labels, _ =
_, valid_preds_bert_small =
metrics_val.run(valid_labels, valid_preds_bert_small, 'DistilBERT (~200 ex)')

# Re-log the SetFit-8 model from 2.3 under a comparison-friendly name:
metrics_val.run(valid_labels, valid_preds_setfit_8, 'SetFit (8/class) [compare]')
metrics_val.plot()

# TASK: train DistilBERT on a small ~200-example stratified subset, for the # apples-to-apples comparison against SetFit at 8 examples / class. # # Instructions: # 1. Stratified subsample: train_size=200 from (train_data, train_labels). # 2. Train BERT with `train_bert(...)` for 3 epochs. # 3. Log under 'DistilBERT (~200 ex)'. # 4. ALSO log the SetFit predictions from section 2.3 (`valid_preds_setfit_8`) # under 'SetFit (8/class) [compare]', so the comparison plot shows BERT-200 # vs SetFit-8 vs TF-IDF-full side by side. NO retraining needed. small_train_texts, _, small_train_labels, _ = _, valid_preds_bert_small = metrics_val.run(valid_labels, valid_preds_bert_small, 'DistilBERT (~200 ex)') # Re-log the SetFit-8 model from 2.3 under a comparison-friendly name: metrics_val.run(valid_labels, valid_preds_setfit_8, 'SetFit (8/class) [compare]') metrics_val.plot()

2.5 Visualizing how SetFit re-shapes the embedding space¶

Why does SetFit work so well with so few examples? Because the contrastive stage moves the embedding space: same-class sentences get pulled together, others get pushed apart.

Recipe:

1. Embed a fixed sample of validation sentences with the base Sentence Transformer.
2. Fit a PCA(50) → UMAP(2) pipeline once on those base embeddings, and freeze it.
3. Re-train SetFit at N = 4 / 8 / 16 examples / class.
4. Re-embed the same validation sentences with each fine-tuned encoder.
5. Transform (do not refit) through the frozen pipeline and scatter-plot.

Because we reuse the same projection across all runs, movements on the plot reflect real movements in embedding space, not refitting artefacts.

In [ ]:

# ✅ TASK: prepare a *frozen* PCA(50) -> UMAP(2) projection on the **base** Sentence
# Transformer's embeddings. We'll re-use this same projection across all later runs
# so that movement on the scatter plot reflects movement in embedding space.
#
# 🧰 Tools: sklearn.decomposition.PCA, umap.UMAP, sklearn.pipeline.Pipeline.
#
# 🪜 Instructions:
# 1. Build `viz_texts` / `viz_labels`: stratified sample of ~600 validation sentences
#    (150 per class), using a fixed seed.
# 2. Define `embed_fn(model, texts)` -> np.ndarray, using `model.model_body.encode(...)`.
# 3. Instantiate `base_model = SetFitModel.from_pretrained(SETFIT_CKPT)`.
# 4. `base_emb = embed_fn(base_model, viz_texts)`.
# 5. Build a Pipeline PCA(50) -> UMAP(2, random_state=42) and FIT it on `base_emb`.
# 6. Define `project(emb) = proj.transform(emb)` (never refit).

from sklearn.decomposition import PCA
from sklearn.pipeline import Pipeline
import umap

valid_texts_np = np.array(valid_data)
valid_labels_np = np.array(valid_labels)

viz_texts =
viz_labels =

def embed_fn(model, texts, batch_size=64):
    return

base_model =
base_emb =

proj = Pipeline([
    # ('pca',  PCA(...)),
    # ('umap', umap.UMAP(...)),
])
# proj.fit(...)

def project(emb):
    return

# ✅ TASK: prepare a *frozen* PCA(50) -> UMAP(2) projection on the **base** Sentence # Transformer's embeddings. We'll re-use this same projection across all later runs # so that movement on the scatter plot reflects movement in embedding space. # # 🧰 Tools: sklearn.decomposition.PCA, umap.UMAP, sklearn.pipeline.Pipeline. # # 🪜 Instructions: # 1. Build `viz_texts` / `viz_labels`: stratified sample of ~600 validation sentences # (150 per class), using a fixed seed. # 2. Define `embed_fn(model, texts)` -> np.ndarray, using `model.model_body.encode(...)`. # 3. Instantiate `base_model = SetFitModel.from_pretrained(SETFIT_CKPT)`. # 4. `base_emb = embed_fn(base_model, viz_texts)`. # 5. Build a Pipeline PCA(50) -> UMAP(2, random_state=42) and FIT it on `base_emb`. # 6. Define `project(emb) = proj.transform(emb)` (never refit). from sklearn.decomposition import PCA from sklearn.pipeline import Pipeline import umap valid_texts_np = np.array(valid_data) valid_labels_np = np.array(valid_labels) viz_texts = viz_labels = def embed_fn(model, texts, batch_size=64): return base_model = base_emb = proj = Pipeline([ # ('pca', PCA(...)), # ('umap', umap.UMAP(...)), ]) # proj.fit(...) def project(emb): return

In [ ]:

# TASK: re-train SetFit at N = 4, 8, 16 examples per class.
# For each N, embed `viz_texts` with the fine-tuned encoder, project through
# the FROZEN PCA+UMAP pipeline, and store the 2-D coordinates.
#
# Instructions:
# 1. Initialise `runs = {0: project(base_emb)}` and `setfit_models = {}`.
# 2. For each N in [4, 8, 16]:
#       build the few-shot dataset, call `train_setfit(...)`,
#       embed `viz_texts`, project, store into `runs[n]` and `setfit_models[n]`.

runs = {0: project(base_emb)}
setfit_models = {}

for n in [4, 8, 16]:
    ds_n =
    model_n =
    setfit_models[n] = model_n
    emb_n =
    runs[n] =

# TASK: re-train SetFit at N = 4, 8, 16 examples per class. # For each N, embed `viz_texts` with the fine-tuned encoder, project through # the FROZEN PCA+UMAP pipeline, and store the 2-D coordinates. # # Instructions: # 1. Initialise `runs = {0: project(base_emb)}` and `setfit_models = {}`. # 2. For each N in [4, 8, 16]: # build the few-shot dataset, call `train_setfit(...)`, # embed `viz_texts`, project, store into `runs[n]` and `setfit_models[n]`. runs = {0: project(base_emb)} setfit_models = {} for n in [4, 8, 16]: ds_n = model_n = setfit_models[n] = model_n emb_n = runs[n] =

In [ ]:

# TASK: plot the four projections on a 1x4 grid (N = 0, 4, 8, 16).
#
# Instructions:
# 1. fig, axes = plt.subplots(1, 4, figsize=(20, 5), sharex=True, sharey=True).
# 2. For (n, ax) in zip([0, 4, 8, 16], axes):
#       scatter runs[n] coloured by viz_labels.
#       title 'N = 0 (base)' / 'N = K / class'.
# 3. Add a legend with class names.
# 4. Discuss in 2-3 sentences: how do the clusters move from N=0 to N=16?

fig, axes = plt.subplots(1, 4, figsize=(20, 5), sharex=True, sharey=True)
palette = ['#4C72B0', '#DD8452', '#55A467', '#C44E52']

for n, ax in zip([0, 4, 8, 16], axes):
    # scatter ...
    ax.set_title('N = 0 (base)' if n == 0 else f'N = {n} / class')
    ax.set_xticks([]); ax.set_yticks([])

plt.tight_layout(); plt.show()

# TASK: plot the four projections on a 1x4 grid (N = 0, 4, 8, 16). # # Instructions: # 1. fig, axes = plt.subplots(1, 4, figsize=(20, 5), sharex=True, sharey=True). # 2. For (n, ax) in zip([0, 4, 8, 16], axes): # scatter runs[n] coloured by viz_labels. # title 'N = 0 (base)' / 'N = K / class'. # 3. Add a legend with class names. # 4. Discuss in 2-3 sentences: how do the clusters move from N=0 to N=16? fig, axes = plt.subplots(1, 4, figsize=(20, 5), sharex=True, sharey=True) palette = ['#4C72B0', '#DD8452', '#55A467', '#C44E52'] for n, ax in zip([0, 4, 8, 16], axes): # scatter ... ax.set_title('N = 0 (base)' if n == 0 else f'N = {n} / class') ax.set_xticks([]); ax.set_yticks([]) plt.tight_layout(); plt.show()

Part 3: Bias in BERT models with WinoGender schemas ¶

We probe gender bias with the WinoGender schemas (Rudinger et al., 2018), a curated set of sentences where an ambiguous pronoun could grammatically refer to either of two noun phrases. We mask the pronoun and ask BERT to fill in.

In [ ]:

import pandas as pd
import re
from transformers import pipeline

URL = 'https://raw.githubusercontent.com/rudinger/winogender-schemas/master/data/templates.tsv'
df = pd.read_csv(URL, sep='\t')
df['whole_sentence'] = df.apply(
    lambda x: x['sentence']
        .replace('$OCCUPATION', x['occupation(0)'])
        .replace('$PARTICIPANT', x['other-participant(1)']),
    axis=1,
)

regex = r'\$(\w+)'
df['pronoun'] = df['whole_sentence'].apply(lambda x: re.findall(regex, x)[0])
words_to_replace = ['\\$' + p for p in df['pronoun'].unique()]
replace_regex = r'(?:{})'.format('|'.join(words_to_replace))
df['sentence_mask'] = df['whole_sentence'].apply(lambda x: re.sub(replace_regex, '[MASK]', x))
df.head()

import pandas as pd import re from transformers import pipeline URL = 'https://raw.githubusercontent.com/rudinger/winogender-schemas/master/data/templates.tsv' df = pd.read_csv(URL, sep='\t') df['whole_sentence'] = df.apply( lambda x: x['sentence'] .replace('$OCCUPATION', x['occupation(0)']) .replace('$PARTICIPANT', x['other-participant(1)']), axis=1, ) regex = r'\$(\w+)' df['pronoun'] = df['whole_sentence'].apply(lambda x: re.findall(regex, x)[0]) words_to_replace = ['\\$' + p for p in df['pronoun'].unique()] replace_regex = r'(?:{})'.format('|'.join(words_to_replace)) df['sentence_mask'] = df['whole_sentence'].apply(lambda x: re.sub(replace_regex, '[MASK]', x)) df.head()

In [ ]:

pronouns = {
    'ACC_PRONOUN':  ['her', 'him'],
    'NOM_PRONOUN':  ['she', 'he'],
    'POSS_PRONOUN': ['her', 'his'],
}


def score_model(checkpoint, df, pronouns):
    """Return per-row P(female-pronoun)."""
    classifier = pipeline(
        'fill-mask', model=checkpoint,
        device=0 if device == 'cuda' else -1,
    )
    out = df.copy()
    col = checkpoint.split('/')[-1] + '-female'
    out[col] = np.nan
    for key, targets in pronouns.items():
        sub = out[out['pronoun'] == key]
        texts = sub['sentence_mask'].tolist()
        if not texts:
            continue
        preds = classifier(texts, targets=targets, top_k=2)
        probas = []
        for x in preds:
            if x[0]['token_str'] == targets[0]:
                p = x[0]['score'] / (x[0]['score'] + x[1]['score'])
            else:
                p = x[1]['score'] / (x[0]['score'] + x[1]['score'])
            probas.append(p)
        out.loc[sub.index, col] = probas
    return out, col

df, col_l   = score_model('bert-large-uncased',    df, pronouns)
df, col_b   = score_model('bert-base-uncased',     df, pronouns)
df, col_d   = score_model('distilbert-base-uncased', df, pronouns)

pronouns = { 'ACC_PRONOUN': ['her', 'him'], 'NOM_PRONOUN': ['she', 'he'], 'POSS_PRONOUN': ['her', 'his'], } def score_model(checkpoint, df, pronouns): """Return per-row P(female-pronoun).""" classifier = pipeline( 'fill-mask', model=checkpoint, device=0 if device == 'cuda' else -1, ) out = df.copy() col = checkpoint.split('/')[-1] + '-female' out[col] = np.nan for key, targets in pronouns.items(): sub = out[out['pronoun'] == key] texts = sub['sentence_mask'].tolist() if not texts: continue preds = classifier(texts, targets=targets, top_k=2) probas = [] for x in preds: if x[0]['token_str'] == targets[0]: p = x[0]['score'] / (x[0]['score'] + x[1]['score']) else: p = x[1]['score'] / (x[0]['score'] + x[1]['score']) probas.append(p) out.loc[sub.index, col] = probas return out, col df, col_l = score_model('bert-large-uncased', df, pronouns) df, col_b = score_model('bert-base-uncased', df, pronouns) df, col_d = score_model('distilbert-base-uncased', df, pronouns)

In [ ]:

# Aggregate per-occupation P(female) across the three models
probas = {}
for occ, part, answer, l, b, d in df[[
    'occupation(0)', 'other-participant(1)', 'answer', col_l, col_b, col_d,
]].values:
    target = occ if answer == 0 else part
    probas.setdefault(target, []).append([l, b, d])
for k in probas:
    probas[k] = np.nanmean(probas[k], axis=0)
probas = sorted(probas.items(), key=lambda x: x[1][0], reverse=True)

fig, ax = plt.subplots(figsize=(30, 8))
ax.scatter(np.arange(len(probas)), [x[1][0] for x in probas], label='BERT-large')
ax.scatter(np.arange(len(probas)), [x[1][1] for x in probas], label='BERT')
ax.scatter(np.arange(len(probas)), [x[1][2] for x in probas], label='DistilBERT')
ax.hlines(0.5, 0, len(probas), colors='g')
ax.set_xticks(np.arange(len(probas)))
ax.set_xticklabels([x[0] for x in probas], rotation=90)
ax.set_xlabel('occupations')
ax.set_ylabel('probability to be women')
ax.legend()
plt.tight_layout(); plt.show()

# Aggregate per-occupation P(female) across the three models probas = {} for occ, part, answer, l, b, d in df[[ 'occupation(0)', 'other-participant(1)', 'answer', col_l, col_b, col_d, ]].values: target = occ if answer == 0 else part probas.setdefault(target, []).append([l, b, d]) for k in probas: probas[k] = np.nanmean(probas[k], axis=0) probas = sorted(probas.items(), key=lambda x: x[1][0], reverse=True) fig, ax = plt.subplots(figsize=(30, 8)) ax.scatter(np.arange(len(probas)), [x[1][0] for x in probas], label='BERT-large') ax.scatter(np.arange(len(probas)), [x[1][1] for x in probas], label='BERT') ax.scatter(np.arange(len(probas)), [x[1][2] for x in probas], label='DistilBERT') ax.hlines(0.5, 0, len(probas), colors='g') ax.set_xticks(np.arange(len(probas))) ax.set_xticklabels([x[0] for x in probas], rotation=90) ax.set_xlabel('occupations') ax.set_ylabel('probability to be women') ax.legend() plt.tight_layout(); plt.show()