package main import ( "encoding/json" "net/url" "reflect" "strconv" "strings" "testing" ) func mustQuery(t *testing.T, u string) url.Values { t.Helper() parsed, err := url.Parse(u) if err != nil { t.Fatal(err) } return parsed.Query() } // TestStemUnifiesForms pins the stemmer's contract: the inflections a // reader is likely to type must land on the same token as the base word, // because the index and the query both go through stem(). func TestStemUnifiesForms(t *testing.T) { groups := [][]string{ {"operation", "operations", "operational"}, {"communication", "communications"}, {"collection", "collections"}, {"collecting", "collected"}, {"agency", "agencies"}, {"surveillance", "surveilling"}, {"target", "targets", "targeting", "targeted"}, {"meeting", "meetings"}, {"computer", "computers"}, {"process", "processes"}, {"study", "studies"}, {"tie", "ties", "tied"}, {"lie", "lies", "lied"}, {"spy", "spies", "spied", "spying"}, {"fly", "flies", "flying", "flyer"}, {"study", "studying", "studied"}, {"control", "controlled", "controlling", "controls"}, {"install", "installed", "installing"}, {"cancel", "cancelled", "canceled"}, {"fall", "falling", "falls"}, {"kill", "killed", "killing"}, // A plural in -ies takes its singular's path through the -ity and // -ly rules, and a word in -ie meets its plural. {"capability", "capabilities"}, {"vulnerability", "vulnerabilities"}, {"activity", "activities"}, {"family", "families"}, {"anomaly", "anomalies"}, {"cookie", "cookies"}, // Verbs in -ply are not adverbs. {"apply", "applies", "applied", "applying"}, {"supply", "supplies", "supplied", "supplying", "supplier"}, {"comply", "complies", "complied", "complying", "compliance"}, {"reply", "replies", "replied"}, } for _, g := range groups { want := stem(g[0]) for _, w := range g[1:] { if got := stem(w); got != want { t.Errorf("stem(%q) = %q, stem(%q) = %q; want equal", w, got, g[0], want) } } } // Words a stripped ending must not turn into each other. for _, p := range [][2]string{ {"apply", "app"}, {"reply", "rep"}, {"supply", "supp"}, {"early", "ear"}, {"italy", "ita"}, {"july", "julie"}, } { if stem(p[0]) == stem(p[1]) { t.Errorf("stem(%q) = stem(%q) = %q; want them apart", p[0], p[1], stem(p[0])) } } // Rules that only exist for suffixes ending in "s"/"ss" must be // reachable: the plural pass no longer returns early. if stem("happiness") == "happiness" || stem("useless") == "useless" { t.Errorf("-ness/-less rules unreachable: %q %q", stem("happiness"), stem("useless")) } // Domain terms the conservative stemmer must leave recognizable. for w, want := range map[string]string{"sigint": "sigint", "nsa": "nsa", "class": "class", "yes": "yes", "tdy": "tdy", "tdi": "tdi"} { if got := stem(w); got != want { t.Errorf("stem(%q) = %q, want %q", w, got, want) } } } func TestTokenizeSplitsLikeConcatenation(t *testing.T) { // loadData relies on tokenize(a+" "+b) == tokenize(a) ++ tokenize(b). a, b := "Müller-Maguhn's XKEYSCORE deck, 2013", "collecting the metadata of targets" joined := tokenize(a + " " + b) parts := append(tokenize(a), tokenize(b)...) if !reflect.DeepEqual(joined, parts) { t.Errorf("tokenize(a+b) = %v, parts = %v", joined, parts) } } func TestParseSimpleQuery(t *testing.T) { q, f, c, d := parseSimpleQuery(`program:XKEYSCORE -tempora "cable tapping"~5 date:2013..2015 upstream`) wantQ := []string{"XKEYSCORE", `"cable tapping"~5`, "upstream", "tempora"} wantF := []string{"PR", "TE", "TE", "TE"} wantC := []string{"and", "and", "and", "not"} if !reflect.DeepEqual(q, wantQ) || !reflect.DeepEqual(f, wantF) || !reflect.DeepEqual(c, wantC) { t.Errorf("got %v %v %v, want %v %v %v", q, f, c, wantQ, wantF, wantC) } if d.DocStart != "2013" || d.DocEnd != "2015-12-31" { t.Errorf("dates = %+v", d) } // Pure-negative queries get a match-all anchor. q, _, c, _ = parseSimpleQuery("-draft") if !reflect.DeepEqual(q, []string{"", "draft"}) || !reflect.DeepEqual(c, []string{"and", "not"}) { t.Errorf("pure negative: %v %v", q, c) } } func TestNormalizeClauses(t *testing.T) { q, f, c := normalizeClauses([]string{"a", "b"}, []string{"TI"}, nil) if !reflect.DeepEqual(f, []string{"TI", "TE"}) || !reflect.DeepEqual(c, []string{"and", "and"}) || len(q) != 2 { t.Errorf("got %v %v %v", q, f, c) } many := make([]string, maxSearchClauses+10) q, f, c = normalizeClauses(many, nil, nil) if len(q) != maxSearchClauses || len(f) != maxSearchClauses || len(c) != maxSearchClauses { t.Errorf("cap not applied: %d %d %d", len(q), len(f), len(c)) } } func TestPaginate(t *testing.T) { cases := []struct { total, page, perPage int start, end, wantPage, wantTotalPages int }{ {45, 1, 20, 0, 20, 1, 3}, {45, 3, 20, 40, 45, 3, 3}, {45, 99, 20, 40, 45, 3, 3}, {45, -5, 20, 0, 20, 1, 3}, {0, 7, 20, 0, 0, 1, 0}, {45, 2, 0, 0, 45, 1, 1}, } for _, c := range cases { s, e, p, tp := paginate(c.total, c.page, c.perPage) if s != c.start || e != c.end || p != c.wantPage || tp != c.wantTotalPages { t.Errorf("paginate(%d,%d,%d) = %d,%d,%d,%d; want %d,%d,%d,%d", c.total, c.page, c.perPage, s, e, p, tp, c.start, c.end, c.wantPage, c.wantTotalPages) } } } func TestSafeBackPath(t *testing.T) { for in, want := range map[string]string{ "/doc/x": "/doc/x", "//evil.com": "/", "/\\evil.com": "/", "https://evil": "/", "": "/", "/search?q=a\r\n": "/", } { if got := safeBackPath(in); got != want { t.Errorf("safeBackPath(%q) = %q, want %q", in, got, want) } } } // Tags are curated in the metadata itself, so the // server does nothing to them beyond the normalization every list field // gets: trim, drop empties, fold case-duplicates keeping the first casing. func TestNormalizeListOnTags(t *testing.T) { got := normalizeList([]string{" XKS ", "xks", "", "PRISM", "prism"}) want := []string{"XKS", "PRISM"} if !reflect.DeepEqual(got, want) { t.Errorf("normalizeList = %v, want %v", got, want) } if normalizeList(nil) != nil { t.Error("nil in should stay nil (JSON null, not [])") } } func TestClassSortKeyFoldsAliases(t *testing.T) { if classSortKey([]string{"TS", "SI", "TK"}) != classSortKey([]string{"TOP SECRET", "COMINT", "Talent Keyhole"}) { t.Error("abbreviations and full names must rank identically") } if classSortKey([]string{"TOP SECRET", "SI"}) <= classSortKey([]string{"TOP SECRET"}) { t.Error("a level with SI must outrank the bare level") } if classSortKey([]string{"NOFORN"}) != "" { t.Error("only unranked components must sort with the empties") } } func TestMarkTermsEscapes(t *testing.T) { hl := buildHighlightTerms([]string{"müller says"}, []string{"and"}) out := string(markTerms(`Müller says hi`, hl)) if out != `Müller says <b>hi</b>` { t.Errorf("markTerms = %s", out) } } func TestHeadingLevel(t *testing.T) { for in, want := range map[string]int{ "# Title": 1, "### Deep": 3, "######\tSix": 6, "#": 1, "#1": 0, "#hashtag": 0, "####### seven": 0, "no": 0, "##########################": 0, } { if got := headingLevel(in); got != want { t.Errorf("headingLevel(%q) = %d, want %d", in, got, want) } } if got := Render("#1 priority\n#2 next"); got != "
#1 priority
\n#2 next