package main // query_test.go - what the search engine promises, as queries and the // documents they must return. The unit tests cover the pieces (tokenizer, // wildcards, ranges); this covers the sentence a reader types. // // Results come from /api/search, which runs the same engine as the page, so // a refactor that changes what search FINDS fails here even when every page // still renders. import ( "encoding/json" "net/url" "sort" "strings" "testing" ) func searchIDs(t *testing.T, q string) []string { t.Helper() w := get(t, "/api/search?per_page=all&q="+url.QueryEscape(q)) if w.Code != 200 { t.Fatalf("%q: status %d", q, w.Code) } var body struct { Results []struct { ID string `json:"id"` } `json:"results"` } if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { t.Fatalf("%q: %v", q, err) } ids := make([]string, 0, len(body.Results)) for _, r := range body.Results { ids = append(ids, r.ID) } sort.Strings(ids) return ids } func TestQuerySuite(t *testing.T) { buildFixture(t) const ( prism = "prism-slides" companion = "prism-companion" sid = "2003-05-29_SIDToday_-_Fixture_Issue" foia = "FISC_-_fixture_order" undated = "undated-fixture" ) cases := []struct { query string want []string why string }{ // --- words, stems and case -------------------------------------- {"xkeyscore", []string{prism}, "a plain word in the transcript"}, {"XKEYSCORE", []string{prism}, "matching ignores case"}, {"collections", []string{prism, sid}, "stemming: plural finds the singular"}, {"analysts", []string{prism, sid}, "plural finds singular"}, // The stemmer deliberately leaves -tion alone (see stem), so // "collecting" and "collection" are different words here. This is // the behaviour, recorded so a change to the stemmer is a decision // rather than a surprise. {"collecting", []string{prism}, "-ing does not reach -tion"}, {"zzzznotacorpusword", nil, "a word nobody wrote"}, // --- phrases and proximity --------------------------------------- {`"upstream collection"`, []string{prism}, "a quoted phrase is adjacency, and SIDtoday has the words apart"}, {`"upstream collection"~12`, []string{prism, sid}, "proximity widens it to a window"}, {`"cable tapping"`, []string{prism}, "a phrase that is also a tag"}, // --- punctuation and accents -------------------------------------- {"RT-RG", []string{prism}, "a hyphenated term matches its parts in sequence"}, {"muller", []string{prism}, "accent folding: muller finds Müller-Maguhn"}, {"Müller-Maguhn", []string{prism}, "and the accented form finds itself"}, {"sao paulo", []string{prism}, "folding works across a phrase too"}, // --- wildcards ----------------------------------------------------- {"*keyscore", []string{prism}, "a leading wildcard"}, {"tapp*", []string{prism, sid}, "a trailing wildcard reaches tapping"}, {"*ollectio*", []string{prism, sid}, "both ends open"}, // --- exact values vs substrings ------------------------------------- {`classification:="TOP SECRET"`, []string{prism}, "an exact value is the whole value"}, {`classification:="SECRET"`, []string{foia}, "so SECRET does not match TOP SECRET"}, {"classification:secret", []string{foia, prism}, "unquoted, it is a word match on both"}, // --- fielded search --------------------------------------------------- {"program:PRISM", []string{companion, prism}, "a metadata field"}, {"agency:FISC", []string{foia}, "creator"}, {"agency:NSA", []string{companion, prism, sid}, "creator across collections"}, {`journalist:"Greenwald, Glenn"`, []string{companion, prism}, "a quoted byline"}, {"publisher:NRK", nil, "a reporting-only partner is not a publisher"}, {"publisher:\"The Guardian\"", []string{companion, prism}, "the outlet that published it"}, {"tag:collection", []string{prism}, "tags are their own field"}, {"country:Brazil", []string{prism}, "countries"}, {"people:Snowden", []string{prism}, "people, by surname"}, {"type:FOIA", []string{foia}, "document type"}, {"series:\"(U) A Day in the Life of...\"", []string{sid}, "a series name with punctuation"}, // --- boolean and negation --------------------------------------------- {"prism -companion", []string{prism}, "inline negation excludes"}, {"upstream -xkeyscore", []string{sid}, "negation against a word in the other document"}, {"prism -x", []string{companion, prism}, "a single letter is a literal word, not an empty clause that excludes everything"}, {"u.s.", nil, "a dotted abbreviation is matched as written, not dropped"}, // --- a phrase stays inside one value ------------------------------------ {`"united states"`, []string{prism}, "a phrase inside one country"}, {`"andy united"`, nil, "a phrase cannot run from one metadata value into the next"}, {`"slides slides describing"`, nil, "nor from the title into the description"}, {"program:PRISM tag:collection", []string{prism}, "two fields are an AND"}, // --- dates ------------------------------------------------------------- {"date:2013", []string{companion, foia, prism}, "document year, across collections"}, {"date:2013-04-02", []string{companion}, "an exact document date"}, {"date:2003..2013", []string{companion, foia, prism, sid}, "a range"}, {"published:2016", []string{sid}, "publication year, not document year"}, // --- the corner the archive cares about --------------------------------- {"fairview", []string{prism}, "text that exists only in another published copy"}, {"guardian", []string{companion, prism}, "an outlet name is searchable as text"}, {"prism-slides", []string{prism}, "the document id itself"}, {"prism-slides-guardian", []string{prism}, "and the name the same file was published under"}, } for _, c := range cases { got := searchIDs(t, c.query) want := append([]string(nil), c.want...) sort.Strings(want) if strings.Join(got, ",") != strings.Join(want, ",") { t.Errorf("%-34s got %v, want %v (%s)", c.query, got, want, c.why) } } } // Relevance is not just membership: a title hit must outrank a body hit, // and the phrase bonus must beat loose words. func TestQueryRanking(t *testing.T) { buildFixture(t) first := func(q string) string { ids := func() []string { w := get(t, "/api/search?q="+url.QueryEscape(q)) var body struct { Results []struct { ID string `json:"id"` } `json:"results"` } json.Unmarshal(w.Body.Bytes(), &body) out := make([]string, 0, len(body.Results)) for _, r := range body.Results { out = append(out, r.ID) } return out }() if len(ids) == 0 { t.Fatalf("%q returned nothing", q) } return ids[0] } if got := first("collection"); got != "prism-slides" { t.Errorf("collection: first result %s, want the document with it in the title and the tags", got) } if got := first("fixture"); got == "" { t.Error("fixture: no results") } } // TestQueryRowParts: a structured row is read as ?q= reads its text - each // quoted phrase is a part of its own and the bare words are one more, all // ANDed. A row with two phrases used to be one substring search for the // text between the outer quotes, quotes included, and found nothing. func TestQueryRowParts(t *testing.T) { buildFixture(t) for _, c := range []struct { row string want int }{ {`"upstream collection" "cable tapping"`, 1}, {`"upstream collection" xkeyscore`, 1}, {`"collection upstream" xkeyscore`, 0}, } { w := get(t, "/api/search?per_page=all&fqf=TE&fqc=and&fqv="+url.QueryEscape(c.row)) var body struct { Total int `json:"total"` } if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { t.Fatalf("%s: %v", c.row, err) } if body.Total != c.want { t.Errorf("row %s: %d results, want %d", c.row, body.Total, c.want) } } }