77 lines
3 KiB
Go
77 lines
3 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// The regression this guards: a SIDtoday piece called "The Life of an Exec"
|
|
// used to list five FISC court filings as related, because their titles say
|
|
// "of" eight times and every occurrence scored like a shared codename. A
|
|
// shared word is worth only as much as it is rare, and it counts once.
|
|
func TestRelatedIgnoresFillerTitleWords(t *testing.T) {
|
|
filler := strings.Repeat("Motion of the Court for Disclosure of Records of the Court of Appeal ", 3)
|
|
docs := []Document{
|
|
{ID: "exec", Title: "The Life of an Exec (repost)", Collection: "sidtoday-archive"},
|
|
{ID: "sibling", Title: "A Day in the Life of an Exec", Collection: "sidtoday-archive"},
|
|
{ID: "filing", Title: filler, Collection: "foia"},
|
|
}
|
|
// Enough other titles saying "of" and "court" that those words are
|
|
// common - which is exactly what the real corpus looks like.
|
|
for i := 0; i < 30; i++ {
|
|
docs = append(docs, Document{
|
|
ID: fmt.Sprintf("noise-%02d", i),
|
|
Title: fmt.Sprintf("Order of the Court of Review number %d", i),
|
|
Collection: "foia",
|
|
})
|
|
}
|
|
|
|
got := corpus().buildRelatedDocs(docs[:1], docs)["exec"]
|
|
if len(got) == 0 || got[0] != "sibling" {
|
|
t.Fatalf("related = %v, want sibling first", got)
|
|
}
|
|
for _, id := range got {
|
|
if id == "filing" || strings.HasPrefix(id, "noise-") {
|
|
t.Errorf("related = %v, want no documents matched only on filler words", got)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
// Shared metadata counts each distinct value once: a tag listed twice in one
|
|
// document is one tag, not two matches.
|
|
func TestRelatedScoreCountsEachValueOnce(t *testing.T) {
|
|
a := Document{ID: "a", Title: "Alpha", Tags: []string{"PRISM", "prism", "SSO"}}
|
|
b := Document{ID: "b", Title: "Beta", Tags: []string{"PRISM", "PRISM"}}
|
|
if got := corpus().relatedScore(&a, &b, titleWeights{}); got != 2.0 {
|
|
t.Errorf("score = %v, want 2 (one shared tag)", got)
|
|
}
|
|
|
|
// A shared program, tag, agency and country each still count.
|
|
a.DocMeta.Programs, a.DocMeta.Creator, a.DocMeta.Countries = []string{"XKEYSCORE"}, "NSA", []string{"Norway"}
|
|
b.DocMeta.Programs, b.DocMeta.Creator, b.DocMeta.Countries = []string{"xkeyscore"}, "nsa", []string{"NORWAY", "Norway"}
|
|
if got := corpus().relatedScore(&a, &b, titleWeights{}); got != 2.0+4.0+1.0+0.5 {
|
|
t.Errorf("score = %v, want 7.5", got)
|
|
}
|
|
}
|
|
|
|
// A word in most titles is worth almost nothing; one in a couple of titles is
|
|
// worth nearly full marks.
|
|
func TestTitleWeights(t *testing.T) {
|
|
tw := titleWeights{n: 1000, df: map[string]int{"of": 800, "exec": 2}}
|
|
if w := tw.weight("of"); w > 0.05 {
|
|
t.Errorf("weight(of) = %v, want near 0", w)
|
|
}
|
|
if w := tw.weight("exec"); w < 0.8 {
|
|
t.Errorf("weight(exec) = %v, want near 1", w)
|
|
}
|
|
// An unknown word is treated as unique, and a corpus too small to have
|
|
// frequencies weighs everything equally.
|
|
if w := tw.weight("never-seen"); w < 0.99 {
|
|
t.Errorf("weight(unseen) = %v, want ~1", w)
|
|
}
|
|
if w := (titleWeights{}).weight("anything"); w != 1 {
|
|
t.Errorf("empty weights = %v, want 1", w)
|
|
}
|
|
}
|