first commit
This commit is contained in:
commit
8c4dcae475
89 changed files with 24623 additions and 0 deletions
77
tests/related_test.go
Normal file
77
tests/related_test.go
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The regression this guards: a SIDtoday piece called "The Life of an Exec"
|
||||
// used to list five FISC court filings as related, because their titles say
|
||||
// "of" eight times and every occurrence scored like a shared codename. A
|
||||
// shared word is worth only as much as it is rare, and it counts once.
|
||||
func TestRelatedIgnoresFillerTitleWords(t *testing.T) {
|
||||
filler := strings.Repeat("Motion of the Court for Disclosure of Records of the Court of Appeal ", 3)
|
||||
docs := []Document{
|
||||
{ID: "exec", Title: "The Life of an Exec (repost)", Collection: "sidtoday-archive"},
|
||||
{ID: "sibling", Title: "A Day in the Life of an Exec", Collection: "sidtoday-archive"},
|
||||
{ID: "filing", Title: filler, Collection: "foia"},
|
||||
}
|
||||
// Enough other titles saying "of" and "court" that those words are
|
||||
// common - which is exactly what the real corpus looks like.
|
||||
for i := 0; i < 30; i++ {
|
||||
docs = append(docs, Document{
|
||||
ID: fmt.Sprintf("noise-%02d", i),
|
||||
Title: fmt.Sprintf("Order of the Court of Review number %d", i),
|
||||
Collection: "foia",
|
||||
})
|
||||
}
|
||||
|
||||
got := corpus().buildRelatedDocs(docs[:1], docs)["exec"]
|
||||
if len(got) == 0 || got[0] != "sibling" {
|
||||
t.Fatalf("related = %v, want sibling first", got)
|
||||
}
|
||||
for _, id := range got {
|
||||
if id == "filing" || strings.HasPrefix(id, "noise-") {
|
||||
t.Errorf("related = %v, want no documents matched only on filler words", got)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Shared metadata counts each distinct value once: a tag listed twice in one
|
||||
// document is one tag, not two matches.
|
||||
func TestRelatedScoreCountsEachValueOnce(t *testing.T) {
|
||||
a := Document{ID: "a", Title: "Alpha", Tags: []string{"PRISM", "prism", "SSO"}}
|
||||
b := Document{ID: "b", Title: "Beta", Tags: []string{"PRISM", "PRISM"}}
|
||||
if got := corpus().relatedScore(&a, &b, titleWeights{}); got != 2.0 {
|
||||
t.Errorf("score = %v, want 2 (one shared tag)", got)
|
||||
}
|
||||
|
||||
// A shared program, tag, agency and country each still count.
|
||||
a.DocMeta.Programs, a.DocMeta.Creator, a.DocMeta.Countries = []string{"XKEYSCORE"}, "NSA", []string{"Norway"}
|
||||
b.DocMeta.Programs, b.DocMeta.Creator, b.DocMeta.Countries = []string{"xkeyscore"}, "nsa", []string{"NORWAY", "Norway"}
|
||||
if got := corpus().relatedScore(&a, &b, titleWeights{}); got != 2.0+4.0+1.0+0.5 {
|
||||
t.Errorf("score = %v, want 7.5", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A word in most titles is worth almost nothing; one in a couple of titles is
|
||||
// worth nearly full marks.
|
||||
func TestTitleWeights(t *testing.T) {
|
||||
tw := titleWeights{n: 1000, df: map[string]int{"of": 800, "exec": 2}}
|
||||
if w := tw.weight("of"); w > 0.05 {
|
||||
t.Errorf("weight(of) = %v, want near 0", w)
|
||||
}
|
||||
if w := tw.weight("exec"); w < 0.8 {
|
||||
t.Errorf("weight(exec) = %v, want near 1", w)
|
||||
}
|
||||
// An unknown word is treated as unique, and a corpus too small to have
|
||||
// frequencies weighs everything equally.
|
||||
if w := tw.weight("never-seen"); w < 0.99 {
|
||||
t.Errorf("weight(unseen) = %v, want ~1", w)
|
||||
}
|
||||
if w := (titleWeights{}).weight("anything"); w != 1 {
|
||||
t.Errorf("empty weights = %v, want 1", w)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue