package main import ( "fmt" "strings" "testing" ) // The regression this guards: a SIDtoday piece called "The Life of an Exec" // used to list five FISC court filings as related, because their titles say // "of" eight times and every occurrence scored like a shared codename. A // shared word is worth only as much as it is rare, and it counts once. func TestRelatedIgnoresFillerTitleWords(t *testing.T) { filler := strings.Repeat("Motion of the Court for Disclosure of Records of the Court of Appeal ", 3) docs := []Document{ {ID: "exec", Title: "The Life of an Exec (repost)", Collection: "sidtoday-archive"}, {ID: "sibling", Title: "A Day in the Life of an Exec", Collection: "sidtoday-archive"}, {ID: "filing", Title: filler, Collection: "foia"}, } // Enough other titles saying "of" and "court" that those words are // common - which is exactly what the real corpus looks like. for i := 0; i < 30; i++ { docs = append(docs, Document{ ID: fmt.Sprintf("noise-%02d", i), Title: fmt.Sprintf("Order of the Court of Review number %d", i), Collection: "foia", }) } got := corpus().buildRelatedDocs(docs[:1], docs)["exec"] if len(got) == 0 || got[0] != "sibling" { t.Fatalf("related = %v, want sibling first", got) } for _, id := range got { if id == "filing" || strings.HasPrefix(id, "noise-") { t.Errorf("related = %v, want no documents matched only on filler words", got) break } } } // Shared metadata counts each distinct value once: a tag listed twice in one // document is one tag, not two matches. func TestRelatedScoreCountsEachValueOnce(t *testing.T) { a := Document{ID: "a", Title: "Alpha", Tags: []string{"PRISM", "prism", "SSO"}} b := Document{ID: "b", Title: "Beta", Tags: []string{"PRISM", "PRISM"}} if got := corpus().relatedScore(&a, &b, titleWeights{}); got != 2.0 { t.Errorf("score = %v, want 2 (one shared tag)", got) } // A shared program, tag, agency and country each still count. a.DocMeta.Programs, a.DocMeta.Creator, a.DocMeta.Countries = []string{"XKEYSCORE"}, "NSA", []string{"Norway"} b.DocMeta.Programs, b.DocMeta.Creator, b.DocMeta.Countries = []string{"xkeyscore"}, "nsa", []string{"NORWAY", "Norway"} if got := corpus().relatedScore(&a, &b, titleWeights{}); got != 2.0+4.0+1.0+0.5 { t.Errorf("score = %v, want 7.5", got) } } // A word in most titles is worth almost nothing; one in a couple of titles is // worth nearly full marks. func TestTitleWeights(t *testing.T) { tw := titleWeights{n: 1000, df: map[string]int{"of": 800, "exec": 2}} if w := tw.weight("of"); w > 0.05 { t.Errorf("weight(of) = %v, want near 0", w) } if w := tw.weight("exec"); w < 0.8 { t.Errorf("weight(exec) = %v, want near 1", w) } // An unknown word is treated as unique, and a corpus too small to have // frequencies weighs everything equally. if w := tw.weight("never-seen"); w < 0.99 { t.Errorf("weight(unseen) = %v, want ~1", w) } if w := (titleWeights{}).weight("anything"); w != 1 { t.Errorf("empty weights = %v, want 1", w) } }