The trap accuracy walks into, and how the assignment wants a confusion matrix read.
The notebook carries the source, the install step and every run below, so nothing has to be on your machine. Open it, then choose Runtime › Run all.
Colab opens it read only. Click Copy to Drive to keep
your changes. You can also download
classification.ipynb, or run it
locally: python3 classification.py.
python3 classification.py
The whole evaluation, from counts to macro-F1.
python3 classification.py --trap
A four-class model scoring 0.90 accuracy and 0.24 macro-F1 by always answering the majority class. Three classes score exactly zero and accuracy does not notice.
python3 classification.py --read
The hardest class and the most confused pair, computed rather than judged. government has the lowest F1 at 0.6364, and news and government exchange 5 documents.
Download classification.py
· served verbatim at https://nlp.jcrlabz.com/code/classification.py
"""Evaluating a classifier, worked as Chapter 14 works it.
Four things, all reproducing the chapter's tables:
the accuracy trap a useless model with 90 per cent accuracy
the confusion matrix per-class precision, recall and F1, computed
reading the matrix hardest class and most-confused pair, derived
macro against micro why they differ, and when they cannot
python3 classification.py # all four
python3 classification.py --trap # the accuracy trap
python3 classification.py --matrix # the full per-class table
python3 classification.py --read # hardest class, confused pairs
python3 classification.py --averages # macro, micro and accuracy
Install: nothing, the Python standard library is enough
"""
import argparse
# The genre confusion matrix from the lectures. Rows are the true class,
# columns the predicted class. Twelve documents of each genre.
GENRES = ["news", "fiction", "government", "learned"]
MATRIX = [
[9, 0, 2, 1], # news
[0, 11, 0, 1], # fiction
[3, 0, 7, 2], # government
[1, 1, 1, 9], # learned
]
def counts(M, i):
"""TP, FP, FN for class i, treating the task as one against the rest."""
tp = M[i][i]
fp = sum(M[r][i] for r in range(len(M))) - tp
fn = sum(M[i]) - tp
return tp, fp, fn
def prf(tp, fp, fn):
p = tp / (tp + fp) if tp + fp else 0.0
r = tp / (tp + fn) if tp + fn else 0.0
f = 2 * p * r / (p + r) if p + r else 0.0
return p, r, f
def macro_f1(M):
return sum(prf(*counts(M, i))[2] for i in range(len(M))) / len(M)
def accuracy(M):
return sum(M[i][i] for i in range(len(M))) / sum(sum(r) for r in M)
# ------------------------------------------------------------ the trap
def show_trap():
"""Imbalanced classes, and a model that has learned nothing."""
support = [90, 5, 3, 2]
labels = ["news", "fiction", "government", "learned"]
# Everything predicted as the majority class.
M = [[0] * 4 for _ in range(4)]
for i, n in enumerate(support):
M[i][0] = n
print("\nSTEP 1 why accuracy is not enough\n")
print(" A hundred test documents, unevenly distributed:")
print(" " + ", ".join(f"{l} {n}" for l, n in zip(labels, support)))
print("\n The model is a single line of code: always answer 'news'.\n")
print(f" {'true class':<12}" + "".join(f"{l[:7]:>10}" for l in labels))
print(" " + "-" * 52)
for i, l in enumerate(labels):
print(f" {l:<12}" + "".join(f"{v:>10}" for v in M[i]))
print(f"\n {'class':<12}{'P':>9}{'R':>9}{'F1':>9}")
print(" " + "-" * 40)
for i, l in enumerate(labels):
p, r, f = prf(*counts(M, i))
print(f" {l:<12}{p:>9.4f}{r:>9.4f}{f:>9.4f}")
print(f"\n accuracy {accuracy(M):.4f}")
print(f" macro-F1 {macro_f1(M):.4f}")
print("\n Ninety per cent accuracy, and the model cannot tell any two")
print(" documents apart. Three of the four classes score exactly zero.")
print("\n Macro-F1 sees it immediately, because it averages the per-class")
print(" scores and a class the model ignores contributes 0 to that mean.")
print("\n This is why Assignment 5's leaderboard uses macro-F1. You cannot")
print(" win it by chasing the majority class.")
# -------------------------------------------------------- the real matrix
def show_matrix():
print("\n\nSTEP 2 a real confusion matrix, scored\n")
print(" Rows are the true class, columns the prediction.")
print(" The diagonal is correct. Everything else is an error.\n")
print(f" {'true / pred':<13}" + "".join(f"{g[:7]:>10}" for g in GENRES)
+ f"{'total':>9}")
print(" " + "-" * 62)
for i, g in enumerate(GENRES):
cells = "".join(f"{v:>10}" for v in MATRIX[i])
print(f" {g:<13}{cells}{sum(MATRIX[i]):>9}")
print(f"\n Now one row per class, treating each as one against the rest.\n")
print(f" {'class':<12}{'TP':>5}{'FP':>5}{'FN':>5}{'P':>9}{'R':>9}"
f"{'F1':>9}")
print(" " + "-" * 56)
for i, g in enumerate(GENRES):
tp, fp, fn = counts(MATRIX, i)
p, r, f = prf(tp, fp, fn)
print(f" {g:<12}{tp:>5}{fp:>5}{fn:>5}{p:>9.4f}{r:>9.4f}{f:>9.4f}")
print(f"\n accuracy {accuracy(MATRIX):.4f}")
print(f" macro-F1 {macro_f1(MATRIX):.4f}")
print("\n Here the classes are balanced at twelve each, so accuracy is")
print(" not misleading. It is still less informative: it says 0.75 and")
print(" stops, where the per-class table says which genre is failing.")
print("\n Look at 'government'. Its recall is 0.5833, the lowest in the")
print(" table, because five of its twelve documents went elsewhere.")
print(" Precision is 0.7000, because three documents from other genres")
print(" were labelled government. Both directions are wrong at once.")
# ------------------------------------------------------- reading the matrix
def show_read():
print("\n\nSTEP 3 two diagnoses the matrix hands you\n")
f1 = [(prf(*counts(MATRIX, i))[2], GENRES[i]) for i in range(len(GENRES))]
f1.sort()
print(" HARDEST CLASS is the lowest per-class F1.\n")
for f, g in f1:
print(f" {g:<12}{f:>9.4f}")
print(f"\n hardest: {f1[0][1]}")
print("\n MOST-CONFUSED PAIR is the unordered pair with the most cross")
print(" errors. Add the two off-diagonal cells that join them.\n")
pairs = []
for i in range(len(GENRES)):
for j in range(i + 1, len(GENRES)):
total = MATRIX[i][j] + MATRIX[j][i]
pairs.append((total, GENRES[i], GENRES[j],
MATRIX[i][j], MATRIX[j][i]))
pairs.sort(reverse=True)
print(f" {'pair':<26}{'i->j':>6}{'j->i':>6}{'total':>8}")
print(" " + "-" * 48)
for t, a, b, ab, ba in pairs:
print(f" {a + ' / ' + b:<26}{ab:>6}{ba:>6}{t:>8}")
print(f"\n most confused: {pairs[0][1]} and {pairs[0][2]}, {pairs[0][0]}"
f" errors")
print("\n Both answers are properties of the matrix, not opinions. Two")
print(" people reading the same matrix must reach the same pair.")
print("\n Note that the errors are directional. Government loses 3")
print(" documents to news and takes 2 back, which are different")
print(" mistakes with different causes.")
# --------------------------------------------------------- macro and micro
def show_averages():
print("\n\nSTEP 4 macro, micro, and why one of them is redundant\n")
tp = sum(counts(MATRIX, i)[0] for i in range(len(GENRES)))
fp = sum(counts(MATRIX, i)[1] for i in range(len(GENRES)))
fn = sum(counts(MATRIX, i)[2] for i in range(len(GENRES)))
micro = prf(tp, fp, fn)
print(" MACRO averages the per-class F1 scores. Every class counts once,")
print(" however rare it is.\n")
print(" MICRO pools every decision first, then computes one F1. Frequent")
print(" classes therefore dominate it.\n")
print(f" pooled TP = {tp}, FP = {fp}, FN = {fn}")
print(f"\n {'measure':<14}{'value':>9}")
print(" " + "-" * 26)
print(f" {'accuracy':<14}{accuracy(MATRIX):>9.4f}")
print(f" {'micro-F1':<14}{micro[2]:>9.4f}")
print(f" {'macro-F1':<14}{macro_f1(MATRIX):>9.4f}")
print("\n Micro-F1 and accuracy are the same number, and that is not a")
print(" coincidence. When every document gets exactly one label, every")
print(" error is one FP and one FN at the same time, so the pooled")
print(" precision and recall are both the fraction correct.")
print("\n So on single-label tasks micro-F1 tells you nothing accuracy")
print(" did not. Report macro-F1, or report the per-class table.")
print("\n\n ABLATION DISCIPLINE\n")
runs = [("unigrams, raw counts", 0.612),
("unigrams, tf-idf", 0.681),
("unigrams + bigrams, tf-idf", 0.724),
("+ min document frequency 2", 0.748),
("+ lowercasing", 0.741)]
print(f" {'configuration':<32}{'validation macro-F1':>20}")
print(" " + "-" * 54)
best = max(runs, key=lambda r: r[1])
for name, score in runs:
mark = " <- best" if (name, score) == best else ""
print(f" {name:<32}{score:>20.3f}{mark}")
print("\n One factor changes per row. That is what makes it an ablation")
print(" rather than a list of guesses.")
print("\n The last row went down, and reporting it is the point. An")
print(" ablation is a record of runs you did, not of runs that worked.")
print("\n The best row must be the model you actually submit. If it is")
print(" not, your experiment log and your system have diverged.")
def main():
ap = argparse.ArgumentParser(description=__doc__)
for f in ("trap", "matrix", "read", "averages"):
ap.add_argument(f"--{f}", action="store_true")
a = ap.parse_args()
picked = a.trap or a.matrix or a.read or a.averages
if a.trap or not picked:
show_trap()
if a.matrix or not picked:
show_matrix()
if a.read or not picked:
show_read()
if a.averages or not picked:
show_averages()
print()
if __name__ == "__main__":
main()