package main import ( "cmp" "html/template" "log" "math" "net/http" "net/url" "slices" "sort" "strconv" "strings" "unicode" "unicode/utf8" ) type TFIDFIndex struct { DocStats map[string]map[string]float64 DocFreq map[string]int N int } // stopWords are excluded from tokenization to avoid recreating the map on // every tokenize() call. var stopWords = map[string]bool{ "the": true, "and": true, "for": true, "that": true, "this": true, "with": true, "from": true, "are": true, "was": true, "were": true, "has": true, "have": true, "not": true, "but": true, "they": true, "their": true, "will": true, "been": true, "its": true, "also": true, "into": true, "which": true, "such": true, "other": true, } // --- Stemmer --- // Porter-lite: suffix stripping for English. Applied at both index time and // query time so "surveillance" and "surveilling" map to the same token. // Intentionally conservative - only strips high-confidence suffixes to avoid // over-stemming domain-specific terms like "SIGINT" or "collection". func stem(w string) string { // The last two steps apply whichever suffix rule returned: "flying" and // "flyer" leave "fly", which must meet "fly" itself. s := yToI(stripSuffixes(w)) // fixStem keeps a doubled l ("falling" -> "fall"), so "controlled" and // "controller" reach "controll" while "control" stays "control". A final // "ll" after more than one vowel-consonant run is undoubled in every // word alike (Porter's step 5b): control/controlled -> "control", // install/installing -> "instal", while fall/falling stay "fall". if strings.HasSuffix(s, "ll") && measure(s) > 1 { return s[:len(s)-1] } return s } func stripSuffixes(w string) string { switch { case strings.HasSuffix(w, "sses"): w = w[:len(w)-2] case strings.HasSuffix(w, "ies") && len(w) > 4: w = w[:len(w)-3] + "y" // "studies" -> "study", "spies" -> "spy" case strings.HasSuffix(w, "ies"): w = w[:len(w)-1] // "ties" -> "tie": one letter before it, a word in "ie" case strings.HasSuffix(w, "ss"): // "process", "class": not a plural case strings.HasSuffix(w, "s") && len(w) > 3: w = w[:len(w)-1] } switch { case strings.HasSuffix(w, "ied") && len(w) == 4: return w[:3] // "tied" -> "tie", like "ties" case strings.HasSuffix(w, "ied") && len(w) > 4: w = w[:len(w)-3] + "y" // "studied" -> "study", "spied" -> "spy" case strings.HasSuffix(w, "eed") && len(w) > 4: return w[:len(w)-1] case strings.HasSuffix(w, "ed") && len(w) > 4: s := w[:len(w)-2] return fixStem(s) case strings.HasSuffix(w, "ing") && len(w) > 5: s := w[:len(w)-3] return fixStem(s) } switch { case strings.HasSuffix(w, "ational") && len(w) > 8: return w[:len(w)-5] // "operational" -> "operat", same as "operation" case strings.HasSuffix(w, "ation") && len(w) > 6: return w[:len(w)-3] case strings.HasSuffix(w, "ence") && len(w) > 5: return w[:len(w)-4] case strings.HasSuffix(w, "ance") && len(w) > 5: return w[:len(w)-4] case strings.HasSuffix(w, "ment") && len(w) > 5: return w[:len(w)-4] case strings.HasSuffix(w, "ness") && len(w) > 5: return w[:len(w)-4] case strings.HasSuffix(w, "ity") && len(w) > 5: return w[:len(w)-3] case strings.HasSuffix(w, "ism") && len(w) > 5: return w[:len(w)-3] case strings.HasSuffix(w, "ist") && len(w) > 5: return w[:len(w)-3] case strings.HasSuffix(w, "ize") && len(w) > 5: return w[:len(w)-3] case strings.HasSuffix(w, "ise") && len(w) > 5: return w[:len(w)-3] case strings.HasSuffix(w, "ful") && len(w) > 5: return w[:len(w)-3] case strings.HasSuffix(w, "less") && len(w) > 5: return w[:len(w)-4] case strings.HasSuffix(w, "ly") && len(w) > 5 && w[len(w)-3] != 'p': return w[:len(w)-2] case strings.HasSuffix(w, "er") && len(w) > 4: return w[:len(w)-2] case strings.HasSuffix(w, "al") && len(w) > 4: return w[:len(w)-2] } return w } // yToI turns a final consonant+y into i, so singular/plural pairs unify // with the "ies" rule: "study" -> "studi", "studies" -> "studi"; // "agency" -> "agenci", "agencies" -> "agenci". func yToI(w string) string { if strings.HasSuffix(w, "y") && len(w) > 3 && !isVowel(rune(w[len(w)-2])) { return w[:len(w)-1] + "i" } if strings.HasSuffix(w, "ie") && len(w) > 5 { return w[:len(w)-1] } return w } // measure is Porter's m: the number of vowel-consonant runs in w, with y a // vowel after a consonant ("controll" 2, "fall" 1, "tree" 0). func measure(w string) int { m := 0 prevVowel := false for i := 0; i < len(w); i++ { v := isVowel(rune(w[i])) || (w[i] == 'y' && i > 0 && !prevVowel) if prevVowel && !v { m++ } prevVowel = v } return m } // fixStem handles consonant doubling after stripping -ed/-ing // e.g. "hopping" -> "hopp" -> "hop". func fixStem(s string) string { if len(s) == 0 { return s } if len(s) >= 2 { last := s[len(s)-1] secondLast := s[len(s)-2] if last == secondLast && !isVowel(rune(last)) && last != 'l' && last != 's' && last != 'z' { return s[:len(s)-1] } } return s } func isVowel(r rune) bool { return strings.ContainsRune("aeiou", r) } // foldTable maps accented Latin characters to their ASCII base form so // "muller" finds "Müller-Maguhn" and "Sao Paulo" finds "São Paulo". The // fold runs on BOTH the index and the query side, so it's symmetric. var foldTable = map[rune]string{ 'À': "A", 'Á': "A", 'Â': "A", 'Ã': "A", 'Ä': "A", 'Å': "A", 'Æ': "AE", 'Ç': "C", 'È': "E", 'É': "E", 'Ê': "E", 'Ë': "E", 'Ì': "I", 'Í': "I", 'Î': "I", 'Ï': "I", 'Ð': "D", 'Ñ': "N", 'Ò': "O", 'Ó': "O", 'Ô': "O", 'Õ': "O", 'Ö': "O", 'Ø': "O", 'Ù': "U", 'Ú': "U", 'Û': "U", 'Ü': "U", 'Ý': "Y", 'Þ': "TH", 'ß': "ss", 'à': "a", 'á': "a", 'â': "a", 'ã': "a", 'ä': "a", 'å': "a", 'æ': "ae", 'ç': "c", 'è': "e", 'é': "e", 'ê': "e", 'ë': "e", 'ì': "i", 'í': "i", 'î': "i", 'ï': "i", 'ð': "d", 'ñ': "n", 'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o", 'ö': "o", 'ø': "o", 'ù': "u", 'ú': "u", 'û': "u", 'ü': "u", 'ý': "y", 'þ': "th", 'ÿ': "y", 'Ā': "A", 'ā': "a", 'Ă': "A", 'ă': "a", 'Ą': "A", 'ą': "a", 'Ć': "C", 'ć': "c", 'Ĉ': "C", 'ĉ': "c", 'Ċ': "C", 'ċ': "c", 'Č': "C", 'č': "c", 'Ď': "D", 'ď': "d", 'Đ': "D", 'đ': "d", 'Ē': "E", 'ē': "e", 'Ĕ': "E", 'ĕ': "e", 'Ė': "E", 'ė': "e", 'Ę': "E", 'ę': "e", 'Ě': "E", 'ě': "e", 'Ĝ': "G", 'ĝ': "g", 'Ğ': "G", 'ğ': "g", 'Ġ': "G", 'ġ': "g", 'Ģ': "G", 'ģ': "g", 'Ĥ': "H", 'ĥ': "h", 'Ħ': "H", 'ħ': "h", 'Ĩ': "I", 'ĩ': "i", 'Ī': "I", 'ī': "i", 'Ĭ': "I", 'ĭ': "i", 'Į': "I", 'į': "i", 'İ': "I", 'ı': "i", 'Ĵ': "J", 'ĵ': "j", 'Ķ': "K", 'ķ': "k", 'Ĺ': "L", 'ĺ': "l", 'Ļ': "L", 'ļ': "l", 'Ľ': "L", 'ľ': "l", 'Ŀ': "L", 'ŀ': "l", 'Ł': "L", 'ł': "l", 'Ń': "N", 'ń': "n", 'Ņ': "N", 'ņ': "n", 'Ň': "N", 'ň': "n", 'ʼn': "n", 'Ō': "O", 'ō': "o", 'Ŏ': "O", 'ŏ': "o", 'Ő': "O", 'ő': "o", 'Œ': "OE", 'œ': "oe", 'Ŕ': "R", 'ŕ': "r", 'Ŗ': "R", 'ŗ': "r", 'Ř': "R", 'ř': "r", 'Ś': "S", 'ś': "s", 'Ŝ': "S", 'ŝ': "s", 'Ş': "S", 'ş': "s", 'Š': "S", 'š': "s", 'Ţ': "T", 'ţ': "t", 'Ť': "T", 'ť': "t", 'Ŧ': "T", 'ŧ': "t", 'Ũ': "U", 'ũ': "u", 'Ū': "U", 'ū': "u", 'Ŭ': "U", 'ŭ': "u", 'Ů': "U", 'ů': "u", 'Ű': "U", 'ű': "u", 'Ų': "U", 'ų': "u", 'Ŵ': "W", 'ŵ': "w", 'Ŷ': "Y", 'ŷ': "y", 'Ÿ': "Y", 'Ź': "Z", 'ź': "z", 'Ż': "Z", 'ż': "z", 'Ž': "Z", 'ž': "z", 'Ș': "S", 'ș': "s", 'Ț': "T", 'ț': "t", 'ẞ': "SS", } // foldMin is the smallest rune foldTable maps, every rune below it folds // to itself so the folds skip the table lookup for it (all of ASCII). var foldMin = func() rune { m := utf8.MaxRune for r := range foldTable { m = min(m, r) } return m }() // foldAccents replaces accented characters with their base form, // preserving case for characters not in the table. func foldAccents(s string) string { needs := false for _, r := range s { if r < foldMin { continue } if _, ok := foldTable[r]; ok { needs = true break } } if !needs { return s } var b strings.Builder b.Grow(len(s)) for _, r := range s { if r >= foldMin { if f, ok := foldTable[r]; ok { b.WriteString(f) continue } } b.WriteRune(r) } return b.String() } func normFoldString(s string) string { return strings.ToLower(foldAccents(s)) } // foldRune is normFoldString for one rune. func foldRune(r rune) string { if r < utf8.RuneSelf { if 'A' <= r && r <= 'Z' { r += 'a' - 'A' } return string(r) } if f, ok := foldTable[r]; ok { return strings.ToLower(f) } return strings.ToLower(string(r)) } func foldedLen(r rune) int { if r < utf8.RuneSelf { return 1 } if f, ok := foldTable[r]; ok { return len(f) } return utf8.RuneLen(unicode.ToLower(r)) } // normFold returns the normal form of s plus a position map: posMap[i] is // the byte offset in s of the rune that produced norm byte i (with a final // entry of len(s)). Lets match highlighting (markTerms) mark on the normal // form but slice the ORIGINAL text, so a "muller" query highlights "Müller" // as written. func normFold(s string) (string, []int) { var b strings.Builder b.Grow(len(s)) posMap := make([]int, 0, len(s)+1) for i, r := range s { if r < utf8.RuneSelf { if 'A' <= r && r <= 'Z' { r += 'a' - 'A' } b.WriteByte(byte(r)) posMap = append(posMap, i) continue } f := foldRune(r) for j := 0; j < len(f); j++ { posMap = append(posMap, i) } b.WriteString(f) } posMap = append(posMap, len(s)) return b.String(), posMap } // splitWords splits s into alphanumeric runs (it does not change case). func splitWords(s string) []string { return strings.FieldsFunc(s, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) }) } func tokenize(s string) []string { words := splitWords(normFoldString(s)) filtered := words[:0] for _, w := range words { if len(w) > 1 && !stopWords[w] { filtered = append(filtered, stem(w)) } } return filtered } // eachToken calls f with every token of a stemmed stream (tokenize's // output joined by single spaces) in order. func eachToken(stream string, f func(tok string)) { for i := 0; i < len(stream); { j := strings.IndexByte(stream[i:], ' ') if j < 0 { j = len(stream) - i } if j > 0 { f(stream[i : i+j]) } i += j + 1 } } func buildTFIDF(docs []Document) TFIDFIndex { idx := TFIDFIndex{ DocStats: make(map[string]map[string]float64, len(docs)), DocFreq: make(map[string]int), N: len(docs), } for i := range docs { d := &docs[i] // Title 3x, fulltext 2x, descriptions + metadata 1x. The metadata // is every field the all-fields search reads (searchMeta), so a doc // that matches only via metadata still gets a relevance score // instead of always sorting last. Title, descriptions and body come // from the stemmed forms loadData already built (tokenize() output // joined by spaces). tf := make(map[string]float64) total := 0.0 add := func(weight float64) func(string) { return func(w string) { if w == valueSep { return } tf[w] += weight total += weight } } eachToken(d.StemmedTitle, add(3)) eachToken(d.StemmedFull, add(2)) eachToken(d.StemmedDesc, add(1)) for _, w := range tokenize(searchMeta(d)) { add(1)(w) } for w := range tf { tf[w] /= total // total > 0 whenever tf has a key idx.DocFreq[w]++ } idx.DocStats[d.ID] = tf } log.Printf("TF-IDF index built: %d docs, %d unique terms", len(docs), len(idx.DocFreq)) return idx } // scorer reads one positive term of a query into its relevance score for a // document. The term's words go through the same tokenize() pipeline as the // index (stemming + stopword removal), and matching against stemmed fields // is by whole token. func (idx TFIDFIndex) scorer(query string) func(doc *Document) float64 { // A proximity clause scores as its phrase - the window is a matcher // concern, not a relevance signal (and the raw "~10" tail would // otherwise leak a junk "10" term into the tokenizer). if _, phrase, _, ok := parseProximity(query); ok { query = phrase } // An exact-value clause scores as its phrase with the phrase bonuses on // whole words; quoted text earns them wherever it occurs. contains := strings.Contains if v, ok := exactValue(query); ok { query, contains = `"`+v+`"`, containsWord } // Folded so the phrase bonuses are accent-consistent with matching, and // so the content check can use the pre-folded body (no per-doc fold). clean := strings.Trim(normFoldString(strings.TrimSpace(query)), `"`) // Wildcard words score by where they occur, not through their stems: // +5 when the title has one, else +1 when the document does. var wild []queryWord if !isQuotedQuery(query) { var kept []string for _, w := range strings.Fields(clean) { if _, _, _, isWild := wildcard(w); isWild { wild = append(wild, literalWord(w)) } else { kept = append(kept, w) } } clean = strings.Join(kept, " ") } terms := tokenize(clean) // A term's weight in a document is its share of the document times how // rare it is in the corpus; a term no document holds weighs nothing. idf := make([]float64, len(terms)) for i, w := range terms { if df := idx.DocFreq[w]; df != 0 { idf[i] = math.Log(float64(idx.N) / float64(df)) } } isPhrase := len(strings.Fields(clean)) > 1 return func(doc *Document) float64 { stats, ok := idx.DocStats[doc.ID] if !ok || (len(terms) == 0 && len(wild) == 0) { return 0 } title := "" // for the wildcard and phrase bonuses if len(wild) > 0 || isPhrase { title = doc.titleFold() } var score float64 for i := range wild { switch { case wild[i].kind == wordNever: // matches nothing (see wildcard), so earns nothing case wild[i].in(title): score += 5 case wild[i].in(doc.SearchText): score++ } } if len(terms) == 0 { return score } for i, w := range terms { score += stats[w] * idf[i] } // bonus is whole when the field holds the exact phrase (or, for one // word, the word), else all when it holds every term somewhere. bonus := func(phraseIn string, holds func(term string) bool, whole, all float64) float64 { if isPhrase && contains(phraseIn, clean) { return whole } for _, w := range terms { if !holds(w) { return 0 } } if isPhrase { return all } return whole } // Title +10 or +5; the whole document +3 or +1. The document's // entry in the index holds exactly the terms of its all-fields // stream. inTitle := func(w string) bool { return containsTokens(doc.StemmedTitle, w) } inDoc := func(w string) bool { _, ok := stats[w]; return ok } return score + bonus(title, inTitle, 10, 5) + bonus(doc.FoldedFull, inDoc, 3, 1) } } // similarity returns cosine similarity between two docs' TF-IDF vectors. // Returns 0-1; used by relatedScore in data.go. // TODO split this file or move this away from here? func (idx TFIDFIndex) similarity(aID, bID string) float64 { a, ok1 := idx.DocStats[aID] b, ok2 := idx.DocStats[bID] if !ok1 || !ok2 { return 0 } var dot, normA, normB float64 for w, tfA := range a { df := idx.DocFreq[w] if df == 0 { continue } idf := math.Log(float64(idx.N) / float64(df)) tfidfA := tfA * idf normA += tfidfA * tfidfA if tfB, ok := b[w]; ok { tfidfB := tfB * idf dot += tfidfA * tfidfB } } for w, tfB := range b { df := idx.DocFreq[w] if df == 0 { continue } idf := math.Log(float64(idx.N) / float64(df)) tfidfB := tfB * idf normB += tfidfB * tfidfB } if normA == 0 || normB == 0 { return 0 } return dot / (math.Sqrt(normA) * math.Sqrt(normB)) } // --- Clause values --- // isQuotedQuery returns true for "exact phrase" queries. func isQuotedQuery(q string) bool { q = strings.TrimSpace(q) return len(q) >= 2 && strings.HasPrefix(q, `"`) && strings.HasSuffix(q, `"`) } // --- Exact values and word boundaries --- // // A clause value has four shapes: words (cable tapping, matched with their // other forms), quoted text ("cable tapping", found anywhere, like grep -i), // an exact value (="cable tapping", or =tap for one word) and a wildcard // (tap*, *apping). An exact value means "exactly this, nothing attached", on // a metadata field it is one whole value, so agency:=NSA does not match // "NSA, GCHQ"; on the text fields (all fields, title, content, description) // it is whole words exactly as typed, so =tap finds "tap" but not "taps" or // "strap". // exactValue reports whether q is an exact-value clause ("=") followed by // a quoted value or a single bare word - and returns the value. func exactValue(q string) (string, bool) { q = strings.TrimSpace(q) if !strings.HasPrefix(q, "=") { return "", false } v := strings.TrimSpace(q[1:]) if isQuotedQuery(v) { v = strings.TrimSpace(v[1 : len(v)-1]) } else if strings.ContainsAny(v, spaceBytes+`"`) { return "", false } return v, v != "" } // exactClause builds the ="..." clause value the generated links use, so a // count and its click compare whole values the same way. func exactClause(v string) string { return `="` + v + `"` } // isWordRune is the word-character test splitWords uses. func isWordRune(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) } // indexWord returns the byte offset of the first occurrence of needle in s, // at or after from, that stands on word boundaries (no letter or digit // directly before or after it) or -1. A needle edge that is not a word // character ("u.s.") needs no boundary on that side. s and needle must // already share a normal form (normFoldString). func indexWord(s, needle string, from int) int { first, _ := utf8.DecodeRuneInString(needle) last, _ := utf8.DecodeLastRuneInString(needle) return indexBounded(s, needle, from, isWordRune(first), isWordRune(last)) } // wildMinRunes is the fewest letters or digits a fragment behind a leading // "*" must have: "*a" or "*ab" would match nearly every document. const wildMinRunes = 3 // wildcard reports whether w is a wildcard word (a fragment with "*") at // its end (tap*), its start (*apping) or both (*keys*), and returns the // fragment in normal form plus which sides may run on into more of the // word. func wildcard(w string) (frag string, openStart, openEnd, ok bool) { openStart, openEnd = strings.HasPrefix(w, "*"), strings.HasSuffix(w, "*") if !openStart && !openEnd { return "", false, false, false } frag = normFoldString(strings.Trim(w, "*")) n := 0 for _, r := range frag { if isWordRune(r) { n++ } } if n == 0 || (openStart && n < wildMinRunes) || strings.Contains(frag, "*") { return "", openStart, openEnd, true } return frag, openStart, openEnd, true } // indexWild finds a wildcard fragment in s, both in normal form. A closed // side must sit on a word boundary and an open side may continue into the // word, so tap* finds "tapping" but not "strap". func indexWild(s, frag string, from int, openStart, openEnd bool) int { first, _ := utf8.DecodeRuneInString(frag) last, _ := utf8.DecodeLastRuneInString(frag) return indexBounded(s, frag, from, !openStart && isWordRune(first), !openEnd && isWordRune(last)) } // wordBounds widens [start, end) in s to the whole word around it. func wordBounds(s string, start, end int) (int, int) { for start > 0 { r, size := utf8.DecodeLastRuneInString(s[:start]) if !isWordRune(r) { break } start -= size } for end < len(s) { r, size := utf8.DecodeRuneInString(s[end:]) if !isWordRune(r) { break } end += size } return start, end } // indexBounded is indexWord with the boundary checks chosen by the caller: // checkStart requires no letter or digit directly before the match, // checkEnd none directly after it. func indexBounded(s, needle string, from int, checkStart, checkEnd bool) int { if from < 0 { from = 0 } if needle == "" { if from <= len(s) { return from } return -1 } for from <= len(s)-len(needle) { j := strings.Index(s[from:], needle) if j < 0 { return -1 } i := from + j end := i + len(needle) ok := true if checkStart && i > 0 { if r, _ := utf8.DecodeLastRuneInString(s[:i]); isWordRune(r) { ok = false } } if ok && checkEnd && end < len(s) { if r, _ := utf8.DecodeRuneInString(s[end:]); isWordRune(r) { ok = false } } if ok { return i } _, size := utf8.DecodeRuneInString(s[i:]) from = i + size } return -1 } // containsWord reports whether needle occurs in s on word boundaries. func containsWord(s, needle string) bool { return indexWord(s, needle, 0) >= 0 } // containsTokens reports whether phrase (one or more stemmed tokens joined // by single spaces) occurs in a space-joined token stream as WHOLE tokens. func containsTokens(stream, phrase string) bool { if phrase == "" { return true } for i := 0; i+len(phrase) <= len(stream); { j := strings.Index(stream[i:], phrase) if j < 0 { return false } p := i + j end := p + len(phrase) if (p == 0 || stream[p-1] == ' ') && (end == len(stream) || stream[end] == ' ') { return true } i = p + 1 } return false } // valueSep joins a field's values for word and phrase matching. It is not a // word character, so words still split on it, and nobody types it, so a // quoted phrase cannot run from one value into the next. const valueSep = "\x1f" func valueKey(v string) string { return strings.ToLower(strings.TrimSpace(v)) } func sameValue(a, b string) bool { a, b = strings.TrimSpace(a), strings.TrimSpace(b) if !isASCII(a) || !isASCII(b) { return strings.ToLower(a) == strings.ToLower(b) } if len(a) != len(b) { return false } for i := 0; i < len(a); i++ { x, y := a[i], b[i] if 'A' <= x && x <= 'Z' { x += 'a' - 'A' } if 'A' <= y && y <= 'Z' { y += 'a' - 'A' } if x != y { return false } } return true } func isASCII(s string) bool { for i := 0; i < len(s); i++ { if s[i] >= utf8.RuneSelf { return false } } return true } // --- Compiled clauses --- // // A search reads each clause once, into a test, and the scan runs the tests // on every document. // queryWord is one word of an unquoted clause. type queryWord struct { kind int text string // the fragment (wordWild), the word (wordWhole, wordSub) or its stemmed tokens joined by spaces (wordStems) openStart, openEnd bool // wordWild: which sides of the fragment had "*" tokens []string // wordStems: the tokens of text } const ( wordNever = iota // a wildcard with nothing to search for (wildcard): it matches nothing wordWild // tap*, *apping, *keys*: part of a word, in the folded text wordWhole // a whole word as written, in the folded text wordSub // a word with internal punctuation ("u.s."), as written, anywhere in the folded text wordStems // the word's stemmed tokens in sequence, in the token stream ) // literalWord reads a folded query word the way folded text can hold it: a // wildcard, a whole word or, with internal punctuation, which word matching // can't represent, a substring. func literalWord(w string) queryWord { if frag, openStart, openEnd, ok := wildcard(w); ok { if frag == "" { return queryWord{kind: wordNever} } return queryWord{kind: wordWild, text: frag, openStart: openStart, openEnd: openEnd} } if parts := splitWords(w); len(parts) == 1 { return queryWord{kind: wordWhole, text: parts[0]} } return queryWord{kind: wordSub, text: w} } // in reports whether the word, read literally, occurs in folded text. func (w *queryWord) in(folded string) bool { switch w.kind { case wordWild: return indexWild(folded, w.text, 0, w.openStart, w.openEnd) >= 0 case wordWhole: return containsWord(folded, w.text) case wordSub: return strings.Contains(folded, w.text) } return false } // literalTest is a clause read for folded text that is not tokenized (the // values of a metadata field). Quoted text is found anywhere, like grep -i // ("ard" inside "guardian"); otherwise every word must occur as literalWord // reads it. type literalTest struct { quoted bool needle string words []queryWord } func compileLiteral(q string) literalTest { q = normFoldString(strings.TrimSpace(q)) if isQuotedQuery(q) { return literalTest{quoted: true, needle: strings.Trim(q, `"`)} } var t literalTest for _, w := range strings.Fields(q) { t.words = append(t.words, literalWord(w)) } return t } func (t *literalTest) match(folded string) bool { if t.quoted { return strings.Contains(folded, t.needle) } for i := range t.words { if !t.words[i].in(folded) { return false } } return true } // valuesTest is a clause read for a metadata field's values: ="..." is one // whole value (the equality the browse pages and the refine tallies use // the valueKey) and anything else the values' words and phrases // (literalTest). type valuesTest struct { exact bool key string // exact: the value's valueKey text literalTest } func compileValues(q string) valuesTest { if v, ok := exactValue(q); ok { return valuesTest{exact: true, key: valueKey(v)} } return valuesTest{text: compileLiteral(q)} } // match tests the clause on a list of values. func (t *valuesTest) match(vals []string) bool { if t.exact { for _, x := range vals { if sameValue(x, t.key) { return true } } return false } return t.text.match(normFoldString(strings.Join(vals, valueSep))) } func (t *valuesTest) matchIndexed(keys []string, folded string) bool { if t.exact { return slices.Contains(keys, t.key) } return t.text.match(folded) } // compileWords reads an unquoted clause for a text field. Its words run // through the same tokenize pipeline as the index, so stemming stays // consistent with it. func compileWords(q string) []queryWord { var stems, literal []queryWord for _, w := range strings.Fields(strings.ToLower(strings.TrimSpace(q))) { if _, _, _, ok := wildcard(w); ok { literal = append(literal, literalWord(w)) } else if toks := tokenize(w); len(toks) == 0 { literal = append(literal, literalWord(normFoldString(w))) } else { stems = append(stems, queryWord{kind: wordStems, text: strings.Join(toks, " "), tokens: toks}) } } return append(stems, literal...) } // compile reads clause q (one negation-free value of a row) into the // test of it on the field. func (f *searchField) compile(q string) func(*Document) bool { if terms, phrase, win, ok := parseProximity(q); ok { // "..."~N - see the proximity block below. if f.Stemmed != nil && len(terms) >= 2 { return func(d *Document) bool { return d.hasTerms(terms) && proximityMatch(f.Stemmed(d), terms, win) } } // A metadata field, or a phrase that boiled down to fewer than two // searchable words, so it adds nothing, match the plain // quoted phrase on the requested field instead. q = phrase } if f.Values != nil { test := compileValues(q) return func(d *Document) bool { if d.fields == nil { return test.match(f.read(d)) } return test.matchIndexed(d.fields.keys[f.index], d.fields.folded[f.index]) } } // A text field: an exact value (=word) is a whole-word test and quoted // text a substring test, both on the folded text; anything else runs on // the stemmed tokens, and reads the folded text only for its wildcard // and literal words. if v, ok := exactValue(q); ok { needle := normFoldString(v) return func(d *Document) bool { return containsWord(f.Folded(d), needle) } } if isQuotedQuery(q) { needle := strings.Trim(normFoldString(strings.TrimSpace(q)), `"`) return func(d *Document) bool { return strings.Contains(f.Folded(d), needle) } } words := compileWords(q) everyField := f.Code == "TE" return func(d *Document) bool { for i := range words { w := &words[i] switch { case w.kind != wordStems: if !w.in(f.Folded(d)) { return false } case !d.hasTerms(w.tokens): return false case everyField && len(w.tokens) == 1 && d.terms != nil: // stream needs no walking case !containsTokens(f.Stemmed(d), w.text): return false } } return true } } // --- Proximity operator ---------------------------------------------------- // // "upstream collection"~10 matches documents where every quoted word occurs // within a 10-word window, unordered and counted on the same stemmed, // stopword-free token streams the bare-word matcher uses. // parseProximity reports whether q has the "..."~N shape (the whole clause, // nothing after the digits). terms are the tokenized, stemmed, deduplicated // phrase words; phrase is the plain quoted phrase for callers that degrade; // win is N, capped at 1000 (wider than any window that could still mean // anything beyond plain AND). func parseProximity(q string) (terms []string, phrase string, win int, ok bool) { q = strings.TrimSpace(q) if len(q) < 4 || q[0] != '"' { return nil, "", 0, false } end := strings.LastIndexByte(q, '"') if end == 0 || end+2 >= len(q) || q[end+1] != '~' { return nil, "", 0, false } n, err := strconv.Atoi(q[end+2:]) if err != nil || n < 1 { return nil, "", 0, false } if n > 1000 { n = 1000 } inner := q[1:end] seen := map[string]bool{} for _, t := range tokenize(inner) { if !seen[t] { seen[t] = true terms = append(terms, t) } } return terms, `"` + inner + `"`, n, true } // proximityMatch walks a space-joined stemmed stream once, tracking the most // recent position of each term; whenever every term has been seen and the // span from the oldest to the newest of those positions fits the window, a // valid window exists. Last-seen positions minimize the span at every step, // so no qualifying window is missed. One linear pass, no string splitting. func proximityMatch(stream string, terms []string, win int) bool { if len(terms) == 0 { return false } last := make([]int, len(terms)) for i := range last { last[i] = -1 } need := len(terms) pos := 0 i := 0 for i < len(stream) { for i < len(stream) && stream[i] == ' ' { i++ } if i >= len(stream) { break } j := i for j < len(stream) && stream[j] != ' ' { j++ } tok := stream[i:j] for t, term := range terms { if tok != term { continue } if last[t] < 0 { need-- } last[t] = pos if need == 0 { lo, hi := last[0], last[0] for _, p := range last[1:] { if p < lo { lo = p } if p > hi { hi = p } } if hi-lo <= win { return true } } } pos++ i = j } return false } // --- Snippet helpers --- // getSnippets returns up to max non-overlapping +-60-char windows around // query matches, in document order. The match runs on the normal form // (folded + lowered) and each window's bounds are resolved back to the // ORIGINAL text (one forward walk, see below), so accents and casing // survive into the snippet. func getSnippets(content, query string, max int) []string { return snippetsIn(content, normFoldString(content), query, max) } // bodySnippets is getSnippets over the document's transcript, on the fold // of it the loader already holds. func (d *Document) bodySnippets(query string, max int) []string { return snippetsIn(d.FullText, d.foldedBody(), query, max) } // anchorSource yields one needle's matches in a text, in document order and // one at a time, so a search for three windows reads no further than them. type anchorSource struct { next func(from int) (pos, n, resume int) // pos < 0: no match from there on from int pos, n int // the match not yet taken; pos < 0 once there is none } func (a *anchorSource) advance() { a.pos, a.n, a.from = a.next(a.from) } // snippetsIn is getSnippets for a caller that holds the normal form of // content (normFoldString). func snippetsIn(content, norm, query string, max int) []string { if query == "" || content == "" || max <= 0 { return nil } searchQuery := strings.TrimSpace(query) bounded := false // an exact value anchors on whole words only if v, ok := exactValue(searchQuery); ok { searchQuery, bounded = `"`+v+`"`, true } isExact := isQuotedQuery(searchQuery) cleanQuery := normFoldString(strings.Trim(searchQuery, `"`)) if cleanQuery == "" { return nil } var sources []*anchorSource add := func(next func(from int) (pos, n, resume int)) { src := &anchorSource{next: next} src.advance() sources = append(sources, src) } addAll := func(needle string, wordsOnly bool) { if needle == "" { return } add(func(from int) (int, int, int) { i := -1 if wordsOnly { i = indexWord(norm, needle, from) } else if j := strings.Index(norm[from:], needle); j >= 0 { i = from + j } return i, len(needle), i + len(needle) }) } found := func() bool { for _, src := range sources { if src.pos >= 0 { return true } } return false } if isExact || strings.Contains(cleanQuery, " ") { addAll(cleanQuery, !isExact || bounded) } if !isExact { var literal []string // stopwords, single letters, "u.s.": anchors of last resort for _, w := range strings.Fields(cleanQuery) { if frag, openStart, openEnd, ok := wildcard(w); ok { if frag != "" { add(func(from int) (int, int, int) { i := indexWild(norm, frag, from, openStart, openEnd) return i, len(frag), i + len(frag) }) } continue } kept := 0 for _, p := range splitWords(w) { if len(p) <= 1 || stopWords[p] { continue } kept++ addAll(p, true) s := stem(p) n := 0 for n < len(s) && n < len(p) && s[n] == p[n] { n++ } if s == p || n < 2 { continue } add(func(from int) (int, int, int) { for { i := indexWild(norm, p[:n], from, false, true) if i < 0 { return -1, 0, from } start, end := wordBounds(norm, i, i+n) if word := norm[start:end]; word != p && stem(word) == s { return start, end - start, end } from = end } }) } if kept == 0 { literal = append(literal, w) } } if !found() { for _, w := range literal { parts := splitWords(w) if len(parts) == 1 { w = parts[0] } addAll(w, len(parts) == 1) } } } if !found() { return nil } // First pass. Choose up to max non-overlapping +-60-byte windows, in // NORM offsets, taking the anchors in document order. type window struct{ start, end int } var wins []window lastEnd := -1 for len(wins) < max { var first *anchorSource for _, src := range sources { if src.pos >= 0 && (first == nil || src.pos < first.pos) { first = src } } if first == nil { break } pos, n := first.pos, first.n first.advance() if pos < lastEnd { continue // already inside the previous window } start := pos - 60 if start < 0 { start = 0 } end := pos + n + 60 if end > len(norm) { end = len(norm) } lastEnd = end wins = append(wins, window{start, end}) } // Second pass. Resolve those few norm offsets back to offsets in the // original content with ONE forward walk. A norm offset resolves to the // start of the rune whose folded form covers it; the offset just past // the end resolves to len(content). bounds := make([]int, 0, len(wins)*2) for _, wnd := range wins { bounds = append(bounds, wnd.start, wnd.end) } sort.Ints(bounds) // window paddings may overlap, so bounds aren't sorted yet orig := make(map[int]int, len(bounds)) bi, normOff := 0, 0 for oi, r := range content { if bi == len(bounds) { break } l := foldedLen(r) for bi < len(bounds) && bounds[bi] < normOff+l { orig[bounds[bi]] = oi bi++ } normOff += l } for ; bi < len(bounds); bi++ { orig[bounds[bi]] = len(content) // the past-the-end boundary } out := make([]string, 0, len(wins)) for _, wnd := range wins { oStart, oEnd := orig[wnd.start], orig[wnd.end] prefix := "... " if oStart == 0 { prefix = "" } suffix := " ..." if oEnd == len(content) { suffix = "" } out = append(out, prefix+content[oStart:oEnd]+suffix) } return out } // --- Search-result highlighting --- // Highlight term kinds const ( hlWord = iota // matches whole words hlWild // marks the whole word a wildcard fragment matched in (tap*, *keys*) hlPhrase // matches an exact value (=tap) at word boundaries hlSub // matches raw substrings (quoted text, and punctuated terms) ) // hlTerm is one positive query term prepared for markTerms. type hlTerm struct { kind int needle string // in normal form (folded + lowered) stemmed string // the needle's stem when word stemming applies openStart, openEnd bool // which sides of the fragment had "*" } // buildHighlightTerms extracts the positive terms of the clause list in the // shape markTerms consumes, mirroring how the matchers read a clause // (literalWord, compileWords). A quoted value is one exact phrase, plain // words match whole words (stem-unified, like the index), "word*" matches // word prefixes, and words with internal punctuation fall back to substring // matching. Negated clauses are skipped (their terms are what the results // DON'T contain). func buildHighlightTerms(queries, conjs []string) []hlTerm { var out []hlTerm seen := make(map[string]bool) add := func(t hlTerm) { k := strconv.Itoa(t.kind) + strconv.FormatBool(t.openStart) + strconv.FormatBool(t.openEnd) + "\x00" + t.needle if t.needle == "" || seen[k] { return } seen[k] = true out = append(out, t) } for _, q := range positiveTerms(queries, conjs) { if _, phrase, _, ok := parseProximity(q); ok { // Proximity clause. The match is word-based, not an exact // substring, so highlight its individual words (with the // same stemming eligibility bare words get below). q = strings.Trim(phrase, `"`) } if v, ok := exactValue(q); ok { add(hlTerm{kind: hlPhrase, needle: normFoldString(v)}) continue } q = normFoldString(strings.TrimSpace(q)) if q == "" { continue } if isQuotedQuery(q) { add(hlTerm{kind: hlSub, needle: strings.Trim(q, `"`)}) continue } for _, w := range strings.Fields(q) { if frag, openStart, openEnd, ok := wildcard(w); ok { if frag != "" { add(hlTerm{kind: hlWild, needle: frag, openStart: openStart, openEnd: openEnd}) } continue } parts := splitWords(w) if len(parts) != 1 { // Internal punctuation. The matchers read these as // substrings, so highlight them the same way. add(hlTerm{kind: hlSub, needle: w}) continue } w = parts[0] t := hlTerm{kind: hlWord, needle: w} // Stem eligibility mirrors tokenize, so "taps" highlights // "tapped" exactly when the index unified them. if len(w) > 1 && !stopWords[w] { t.stemmed = stem(w) } add(t) } } return out } // markTerms HTML-escapes s and wraps every occurrence of the given terms in // tags, for the search results page. // // Matching runs on the normal form (normFold) so casing and accents can't // hide a match, and the marked ranges are resolved back to the ORIGINAL // text through normFold's position map (e.g. a "muller" query highlights // "Müller" as written). Word terms match whole words, or words whose stem // equals the term's stem (the same unification the index does); prefix // terms match the start of a word; substring terms match anywhere. func markTerms(s string, terms []hlTerm) template.HTML { if s == "" || len(terms) == 0 { return template.HTML(template.HTMLEscapeString(s)) } norm, posMap := normFold(s) var wordTerms, subTerms []hlTerm for _, t := range terms { if t.kind != hlWord { subTerms = append(subTerms, t) } else { wordTerms = append(wordTerms, t) } } type span struct{ start, end int } // byte offsets in norm var spans []span if len(wordTerms) > 0 { needStem := false for _, t := range wordTerms { if t.stemmed != "" { needStem = true break } } ws := -1 // current word start; -1 = between words flush := func(we int) { if ws < 0 { return } word := norm[ws:we] wstem := "" if needStem { wstem = stem(word) } for _, t := range wordTerms { if word == t.needle || (t.stemmed != "" && wstem == t.stemmed) { spans = append(spans, span{ws, we}) break } } ws = -1 } for i, r := range norm { if unicode.IsLetter(r) || unicode.IsDigit(r) { if ws < 0 { ws = i } } else { flush(i) } } flush(len(norm)) } for _, t := range subTerms { for from := 0; ; { i := -1 switch t.kind { case hlPhrase: i = indexWord(norm, t.needle, from) case hlWild: i = indexWild(norm, t.needle, from, t.openStart, t.openEnd) default: if j := strings.Index(norm[from:], t.needle); j >= 0 { i = from + j } } if i < 0 { break } start, end := i, i+len(t.needle) from = end if t.kind == hlWild { start, end = wordBounds(norm, start, end) // mark the whole word } spans = append(spans, span{start, end}) } } if len(spans) == 0 { return template.HTML(template.HTMLEscapeString(s)) } // Merge overlapping and touching spans so nested or adjacent matches // render as one contiguous highlight. sort.Slice(spans, func(i, j int) bool { return spans[i].start < spans[j].start }) merged := spans[:1] for _, sp := range spans[1:] { if last := &merged[len(merged)-1]; sp.start <= last.end { if sp.end > last.end { last.end = sp.end } } else { merged = append(merged, sp) } } // Emit escaped gap / / escaped match / runs. Norm // offsets resolve to original offsets via posMap; when a span boundary // falls inside one rune's multi-byte fold (e.g. 'ẞ' -> "ss"), the whole // rune is included. var b strings.Builder last := 0 for _, sp := range merged { oStart := posMap[sp.start] j := sp.end for j < len(norm) && posMap[j] == posMap[sp.end-1] { j++ } oEnd := posMap[j] if oStart < last { oStart = last // rune snapping can nudge a span into the previous one } if oEnd <= oStart { continue } b.WriteString(template.HTMLEscapeString(s[last:oStart])) b.WriteString("") b.WriteString(template.HTMLEscapeString(s[oStart:oEnd])) b.WriteString("") last = oEnd } b.WriteString(template.HTMLEscapeString(s[last:])) return template.HTML(b.String()) } // --- Search execution (shared by the HTML page and the JSON API) --- // ScoredDoc is one match, with its relevance score. Doc points into archive.Docs, which is // read-only while serving: the match set is walked, filtered and sorted as // pointers, and the page a reader sees wraps them in searchResults, which // carry the highlighting. type ScoredDoc struct { Doc *Document Score float64 } // searchResult is one row of the search page: a document and what the // query marked in it. type searchResult struct { Doc *Document Title, Desc template.HTML // escaped, query matches in (markTerms) Snippets []template.HTML // up to 3 match-context windows, -highlighted VersionSnippets []VersionSnippet // windows from versions' new lines, labelled with their copy MatchesIn []*DocVersion // versions that matched when the main transcript didn't } const maxSnippetDocs = 100 const maxSearchClauses = 24 const maxSearchWords = 64 func searchWords(queries []string) int { n := 0 for _, q := range queries { n += len(strings.Fields(q)) } return n } // normalizeClauses caps the clause count and pads/truncates fields and // conjs to the length of queries. func normalizeClauses(queries, fields, conjs []string) ([]string, []string, []string) { if len(queries) > maxSearchClauses { queries = queries[:maxSearchClauses] } trimmed := make([]string, len(queries)) for i, q := range queries { trimmed[i] = strings.TrimSpace(q) } queries = trimmed if len(fields) > len(queries) { fields = fields[:len(queries)] } for len(fields) < len(queries) { fields = append(fields, "TE") } if len(conjs) > len(queries) { conjs = conjs[:len(queries)] } for len(conjs) < len(queries) { conjs = append(conjs, "and") } return queries, fields, conjs } // searchField describes one searchable field: its wire code (the fqf // value), UI label, simple-box operator aliases, and what it searches. This // table is the single source of truth. type searchField struct { Code string Label string Aliases []string Stemmed, Folded func(d *Document) string Values func(d *Document) []string read func(d *Document) []string index int // the field's place in searchFields } type docFields struct { values [][]string keys [][]string folded []string pubYears []string // pubYearsOf } func (co *Corpus) buildDocFields() { for i := range co.archive.Docs { d := &co.archive.Docs[i] d.fields = nil // the readers below work everything out f := &docFields{ values: make([][]string, len(searchFields)), keys: make([][]string, len(searchFields)), folded: make([]string, len(searchFields)), pubYears: pubYearsOf(d), } for fi := range searchFields { sf := &searchFields[fi] if sf.read == nil { continue } vals := sf.read(d) keys := make([]string, len(vals)) for vi, v := range vals { keys[vi] = valueKey(v) } f.values[fi], f.keys[fi] = vals, keys f.folded[fi] = normFoldString(strings.Join(vals, valueSep)) } d.fields = f } } // searchFields is in the form dropdown's display order. var searchFields = []searchField{ // SearchText is stored pre-folded, so quoted text and exact words are // direct tests against it. {Code: "TE", Label: "All fields", Aliases: []string{"text"}, Stemmed: func(d *Document) string { return d.StemmedText }, Folded: func(d *Document) string { return d.SearchText }}, {Code: "TI", Label: "Title", Aliases: []string{"title"}, Stemmed: func(d *Document) string { return d.StemmedTitle }, Folded: (*Document).titleFold}, {Code: "FT", Label: "Content", Aliases: []string{"content"}, Stemmed: func(d *Document) string { return d.StemmedFull }, Folded: func(d *Document) string { return d.FoldedFull }}, {Code: "DE", Label: "Description", Aliases: []string{"description"}, Stemmed: func(d *Document) string { return d.StemmedDesc }, Folded: (*Document).descFold}, {Code: "CR", Label: "Agency", Aliases: []string{"agency"}, Values: func(d *Document) []string { return single(d.DocMeta.Creator) }}, {Code: "RE", Label: "Journalist", Aliases: []string{"journalist"}, Values: journalistsOf}, {Code: "PU", Label: "Publisher", Aliases: []string{"publisher"}, Values: publishersOf}, {Code: "CL", Label: "Classification", Aliases: []string{"classification"}, Values: func(d *Document) []string { return d.DocMeta.Classification }}, {Code: "PR", Label: "Program", Aliases: []string{"program"}, Values: func(d *Document) []string { return d.DocMeta.Programs }}, {Code: "DI", Label: "Dissemination", Aliases: []string{"dissemination"}, Values: func(d *Document) []string { return d.DocMeta.Dissemination }}, {Code: "BA", Label: "Banner", Aliases: []string{"banner"}, Values: func(d *Document) []string { return single(d.DocMeta.Banner) }}, {Code: "GE", Label: "Genre", Aliases: []string{"genre"}, Values: func(d *Document) []string { return single(d.DocMeta.Genre) }}, {Code: "SE", Label: "Series", Aliases: []string{"series"}, Values: func(d *Document) []string { return single(d.DocMeta.Series.Name) }}, {Code: "CO", Label: "Country", Aliases: []string{"country"}, Values: func(d *Document) []string { return d.DocMeta.Countries }}, {Code: "PE", Label: "People", Aliases: []string{"people"}, Values: func(d *Document) []string { return d.DocMeta.People }}, {Code: "TA", Label: "Tags", Aliases: []string{"tag"}, Values: func(d *Document) []string { return d.Tags }}, {Code: "TY", Label: "Type", Aliases: []string{"type"}, Values: func(d *Document) []string { return single(d.Type) }}, {Code: "PL", Label: "Portion level", Values: func(d *Document) []string { return d.PortionLevels }}, {Code: "PM", Label: "Portion marking", Values: func(d *Document) []string { return d.PortionMarkings }}, {Code: "PC", Label: "Portion control", Values: func(d *Document) []string { return d.PortionControls }}, } func single(v string) []string { if v == "" { return nil } return []string{v} } var ( fieldByCode = map[string]*searchField{} simpleOps = map[string]string{} ) func init() { for i := range searchFields { f := &searchFields[i] f.index = i if read := f.Values; read != nil { f.read = read f.Values = func(d *Document) []string { if d.fields != nil { return d.fields.values[i] } return read(d) } } fieldByCode[f.Code] = f for _, a := range f.Aliases { simpleOps[a] = f.Code } } } type simpleDates struct { DocStart, DocEnd, PubStart, PubEnd string } // padDateEnd widens a partial date for use as an inclusive range end: // 2013 -> 2013-12-31, 2013-06 -> 2013-06-31 (string comparison makes the // over-wide day harmless). func padDateEnd(v string) string { switch len(v) { case 4: return v + "-12-31" case 7: return v + "-31" } return v } // parseSimpleQuery turns a one-line query into fielded clauses plus date // filters. // // program:XKEYSCORE country:Germany cable tapping // // becomes three AND clauses: PR=XKEYSCORE, CO=Germany, TE="cable tapping". // Values can be double-quoted (journalist:"Poitras, Laura"); the quotes are // KEPT on the clause value, so a quoted value has the same exact-match // (substring/phrase) semantics it has when typed into the structured form, // and a bare "quoted phrase" becomes its own exact all-fields clause rather // than dissolving into the free-text words. // // A leading '-' negates: -term and -program:X become "and not" clauses // (ordered after the positive ones, since the boolean evaluation is // left-to-right). date:2013, date:2013-06, date:2013..2015 filter on the // document date; published: does the same for publication dates. func parseSimpleQuery(s string) (queries, fields, conjs []string, dates simpleDates) { var freeText []string type clause struct{ q, f string } var negs []clause for i, neg, ok := nextTerm(s, 0); ok; i, neg, ok = nextTerm(s, i) { // try operator: word characters followed by ':' opEnd := i for opEnd < len(s) && (isWordByte(s[opEnd])) { opEnd++ } field := "" dateOp := "" valStart := i if opEnd > i && opEnd < len(s) && s[opEnd] == ':' { name := strings.ToLower(s[i:opEnd]) if code, ok := simpleOps[name]; ok { field = code valStart = opEnd + 1 } else if name == "date" || name == "published" { dateOp = name valStart = opEnd + 1 } } clauseVal, val, quoted, exact, next := readValue(s, valStart) i = next if val == "" { continue } if dateOp != "" { // date filters can't be negated; '-' before them is ignored. start, end := val, val if a, b, ok := strings.Cut(val, ".."); ok { start, end = strings.TrimSpace(a), strings.TrimSpace(b) } end = padDateEnd(end) if dateOp == "date" { dates.DocStart, dates.DocEnd = start, end } else { dates.PubStart, dates.PubEnd = start, end } continue } val = clauseVal switch { case field != "" && neg: negs = append(negs, clause{val, field}) case field != "": queries = append(queries, val) fields = append(fields, field) case neg: negs = append(negs, clause{val, "TE"}) case quoted || exact: queries = append(queries, val) fields = append(fields, "TE") default: freeText = append(freeText, val) } } if len(freeText) > 0 { queries = append(queries, strings.Join(freeText, " ")) fields = append(fields, "TE") } if len(queries) == 0 { queries = []string{""} fields = []string{"TE"} } conjs = make([]string, len(queries)) for i := range conjs { conjs[i] = "and" } for _, n := range negs { queries = append(queries, n.q) fields = append(fields, n.f) conjs = append(conjs, "not") } return queries, fields, conjs, dates } func isWordByte(b byte) bool { return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_' } const spaceBytes = " \t\n\v\f\r" func isSpaceByte(b byte) bool { return strings.IndexByte(spaceBytes, b) >= 0 } func proxSuffix(s string, i int) (string, int) { if i >= len(s) || s[i] != '~' { return "", i } j := i + 1 for j < len(s) && s[j] >= '0' && s[j] <= '9' { j++ } if j == i+1 || (j < len(s) && !isSpaceByte(s[j])) { return "", i } return s[i:j], j } // readToken reads one search-value token starting at s[i]: either a // "quoted span" returned with the quotes stripped and quoted=true, or // a bare run up to the next whitespace. next is the index just past the // token. This is THE tokenizer for search values: parseSimpleQuery and // rowTerms both use it, behind nextTerm, so neither the quoting rule nor // the negation rule can drift between them. func readToken(s string, i int) (val string, quoted bool, next int) { if i < len(s) && s[i] == '"' { j := i + 1 if k := strings.IndexByte(s[j:], '"'); k >= 0 { return s[j : j+k], true, j + k + 1 } return s[i+1:], true, len(s) } j := i for j < len(s) && !isSpaceByte(s[j]) { j++ } return s[i:j], false, j } // nextTerm finds the next term of a query from s[i] on: it skips the // whitespace, and a leading '-' negates the term when it is attached to // one. i is where the term's value (or its field: operator) starts; ok is // false at the end of the string. func nextTerm(s string, i int) (start int, neg, ok bool) { for i < len(s) && isSpaceByte(s[i]) { i++ } if i >= len(s) { return i, false, false } if s[i] == '-' && i+1 < len(s) && !isSpaceByte(s[i+1]) { return i + 1, true, true } return i, false, true } // readValue reads one search value at s[i] func readValue(s string, i int) (clause, inner string, quoted, exact bool, next int) { if i < len(s) && s[i] == '=' { exact = true i++ } inner, quoted, next = readToken(s, i) prox := "" if quoted { prox, next = proxSuffix(s, next) } inner = strings.TrimSpace(inner) switch { case inner == "": case exact && quoted: clause = `="` + inner + `"` case exact: clause = "=" + inner case quoted: clause = `"` + inner + `"` + prox default: clause = inner } return clause, inner, quoted, exact, next } // rowTerms splits one search row into what it requires and what it // excludes, the way parseSimpleQuery splits ?q=: each quoted phrase, // proximity clause or exact value is a positive part of its own, the bare // words together are one more (last), and google-style -term and -"phrase" // tokens are exclusions on the row's own field. // // This is THE reading of a row: the matcher, the scorer, the snippet anchor // and the highlighter all go through it, so "beta -gamma" is (beta AND NOT // gamma) as one unit wherever the row sits. func rowTerms(s string) (positives, negatives []string) { var words []string for i, neg, ok := nextTerm(s, 0); ok; i, neg, ok = nextTerm(s, i) { tok, _, quoted, exact, next := readValue(s, i) i = next switch { case tok == "" || tok == "-": case neg: negatives = append(negatives, tok) case quoted || exact: positives = append(positives, tok) default: words = append(words, tok) } } if len(words) > 0 { positives = append(positives, strings.Join(words, " ")) } return positives, negatives } // executeSearch is the search pipeline the page and the API both run: over // the whole archive, the collection filter, the dates and the boolean // clause list, then the type filter, and the survivors ranked and sorted // by p.Sort/p.Order, as a fresh slice the caller owns. The two filters are // the browse pages' (inCollections, typeSelected). typeCounts tallies the // types BEFORE the type filter. Only the documents that pass every filter // are scored. func (co *Corpus) executeSearch(p searchParams) (matches []ScoredDoc, typeCounts map[string]int) { dateDocEnd := padDateEnd(p.DateDocEnd) datePubEnd := padDateEnd(p.DatePubEnd) query := compileQuery(p.Queries, p.Fields, p.Conjs) byEntry := entryRows(p.Queries, p.Fields, p.Conjs) var scorers []func(*Document) float64 for _, q := range positiveTerms(p.Queries, p.Conjs) { scorers = append(scorers, co.tfidf.scorer(q)) } typeCounts = make(map[string]int) for i := range co.archive.Docs { doc := &co.archive.Docs[i] if !inCollections(doc, p.Coll) || !matchDates(doc, p.DateDocStart, dateDocEnd, p.DatePubStart, datePubEnd, byEntry) || !query.matches(doc) { continue } typeCounts[doc.Type]++ if !typeSelected(doc, p.Types) { continue } totalScore := 0.0 for _, score := range scorers { totalScore += score(doc) } matches = append(matches, ScoredDoc{Doc: doc, Score: totalScore}) } slices.SortStableFunc(matches, func(a, b ScoredDoc) int { return cmp.Compare(b.Score, a.Score) }) return sortScored(matches, p.Sort, p.Order), typeCounts } // --- Search handler (HTML) --- // --- Faceted search refinement ----------------------------------------------- type searchFacet struct { Label string Values []facetEntry } func topFacetCounts(m map[string]int, total, n int, baseURL, field string) []facetEntry { var out []facetEntry for _, v := range rankCounts(m, -1) { if len(out) == n { break } if m[v] >= total { continue } // ="value" -> a whole-value match on the field, the equality the // tally counted, so the number is the number the click finds. out = append(out, facetEntry{Name: v, Count: m[v], URL: appendClause(baseURL, field, exactClause(v))}) } return out } func facetValues(d *Document, code string) []string { if f := fieldByCode[code]; f != nil && f.Values != nil { return f.Values(d) } return nil } type facetGroup struct { Label string Code string } func facetGroups(docs []*Document, groups []facetGroup, baseURL string, values func(*Document, string) []string) []searchFacet { type tally struct{ count, first int } tallies := make([]map[string]*tally, len(groups)) for i := range tallies { tallies[i] = map[string]*tally{} } seq := 0 for i := range docs { for gi, g := range groups { for _, v := range values(docs[i], g.Code) { t := tallies[gi][v] if t == nil { t = &tally{first: seq} tallies[gi][v] = t } t.count++ seq++ } } } out := make([]searchFacet, 0, len(groups)) for gi, g := range groups { type folded struct { name string count, first int } byKey := make(map[string]*folded, len(tallies[gi])) for v, t := range tallies[gi] { k := valueKey(v) f := byKey[k] if f == nil { byKey[k] = &folded{v, t.count, t.first} continue } f.count += t.count if t.first < f.first { f.name, f.first = v, t.first } } counts := make(map[string]int, len(byKey)) for _, f := range byKey { counts[f.name] = f.count } if vals := topFacetCounts(counts, len(docs), 8, baseURL, g.Code); len(vals) > 0 { out = append(out, searchFacet{Label: g.Label, Values: vals}) } } return out } func facetsWithinWords(facets []searchFacet, room int) []searchFacet { out := facets[:0] for _, f := range facets { vals := f.Values[:0] for _, v := range f.Values { if searchWords([]string{exactClause(v.Name)}) <= room { vals = append(vals, v) } } if len(vals) > 0 { f.Values = vals out = append(out, f) } } return out } func computeSearchFacets(docs []*Document, baseURL string, values func(*Document, string) []string) []searchFacet { return facetGroups(docs, []facetGroup{ {"Programs", "PR"}, {"Agencies", "CR"}, {"Countries", "CO"}, {"Journalists", "RE"}, }, baseURL, values) } func collectionFilter(v url.Values) map[string]bool { sel := map[string]bool{} for _, c := range v["coll"] { if validCollections[c] { sel[c] = true } } if len(sel) == 0 || len(sel) == len(validCollections) { return nil } return sel } func snippetAnchor(queries, fields, conjs []string) (term, field string) { for i, q := range queries { if q == "" || conjs[i] == "not" { continue } pos, _ := rowTerms(q) if len(pos) == 0 { continue // only exclusions: nothing in the results to anchor on } q = pos[0] if _, phrase, _, ok := parseProximity(q); ok { if ws := strings.Fields(strings.Trim(phrase, `"`)); len(ws) > 0 { return ws[0], fields[i] } } return q, fields[i] } return "", "" } // positiveTerms is what the query asks the results to contain: each // non-empty row's positive part, skipping NOT-rows and every row's // exclusions (rowTerms). The scorer and the highlighter read the query // through it, so neither rewards or marks a term the results were selected // for NOT containing. func positiveTerms(queries, conjs []string) []string { var out []string for i, q := range queries { if q == "" || (i < len(conjs) && conjs[i] == "not") { continue } pos, _ := rowTerms(q) out = append(out, pos...) } return out } // searchParams is one search's effective inputs as read by readSearchParams. type searchParams struct { Queries, Fields, Conjs []string DateDocStart, DateDocEnd, DatePubStart, DatePubEnd string FromQ bool // a ?q= shortcut replaced the structured params Coll map[string]bool // collectionFilter: nil filters nothing Types map[string]bool // selectedTypes: empty filters nothing Sort, Order string // Sort defaults to "relevance" OverClauses int // the requested clause count when it exceeds maxSearchClauses: the search must not run OverWords int // the word count (searchWords) when it exceeds maxSearchWords: the search must not run } func (p searchParams) refused() bool { return p.OverClauses > 0 || p.OverWords > 0 } // readSearchParams reads a search from URL params. this intake is shared by // the HTML page and the JSON API. // // A non-empty ?q= replaces any structured clause params. Plain words become // a single all-fields clause. Field operators like program:XKEYSCORE or // journalist:"Poitras, Laura" become their own AND clauses (see // parseSimpleQuery). Its parsed dates override the URL's when either end is // set - date:..2015 has an empty start but is still a real filter. func readSearchParams(v url.Values) searchParams { p := searchParams{ Queries: v["fqv"], Fields: v["fqf"], Conjs: v["fqc"], DateDocStart: v.Get("date_doc_start"), DateDocEnd: v.Get("date_doc_end"), DatePubStart: v.Get("date_pub_start"), DatePubEnd: v.Get("date_pub_end"), Coll: collectionFilter(v), Sort: sortToken(v.Get("sort")), Order: sortToken(v.Get("order")), } p.Types, _ = selectedTypes(v) if docSortByKey[p.Sort] == nil { p.Sort, p.Order = "relevance", "" } if s := v.Get("q"); s != "" { var sd simpleDates p.Queries, p.Fields, p.Conjs, sd = parseSimpleQuery(s) if sd.DocStart != "" || sd.DocEnd != "" { p.DateDocStart, p.DateDocEnd = sd.DocStart, sd.DocEnd } if sd.PubStart != "" || sd.PubEnd != "" { p.DatePubStart, p.DatePubEnd = sd.PubStart, sd.PubEnd } p.FromQ = true } if n := len(p.Queries); n > maxSearchClauses { p.OverClauses = n } else if n := searchWords(p.Queries); n > maxSearchWords { p.OverWords = n } p.Queries, p.Fields, p.Conjs = normalizeClauses(p.Queries, p.Fields, p.Conjs) return p } func (p searchParams) dates() [4]string { return [4]string{p.DateDocStart, p.DateDocEnd, p.DatePubStart, p.DatePubEnd} } // canonicalSearchQuery rebuilds the query params from the EFFECTIVE state // of a search (the post-add/remove clause rows, the collection and type // filters, the dates that a ?q= shortcut may have parsed out of the box) // for every link the page generates (sorting, pagination, the histogram // bars, the refine box, the export and the recent/pinned entries). func canonicalSearchQuery(doSearch bool, p searchParams) url.Values { q := url.Values{} if doSearch { // A ?q= entry has no do_search param, so the canonical form adds it // explicitly. q.Set("do_search", "1") } for i := range p.Queries { q.Add("fqv", p.Queries[i]) q.Add("fqf", p.Fields[i]) q.Add("fqc", p.Conjs[i]) } if colls := collectionParams(p.Coll); len(colls) > 0 { q["coll"] = colls } if len(p.Types) > 0 { types := make([]string, 0, len(p.Types)) for t := range p.Types { types = append(types, t) } sort.Strings(types) q["type"] = types } dates := p.dates() for i, k := range []string{"date_doc_start", "date_doc_end", "date_pub_start", "date_pub_end"} { if dates[i] != "" { q.Set(k, dates[i]) } } return q } // searchDateHistogram is the year-bar strip under a result set. func searchDateHistogram(results []*Document, q url.Values) []timelineBar { counts := make(map[string]int) undated := 0 for i := range results { if y := docYear(results[i]); y != "" { counts[y]++ } else { undated++ } } _, maxCount := sortedYears(counts) if undated > maxCount { maxCount = undated // the undated bar shares the scale } start, end := q.Get("date_doc_start"), q.Get("date_doc_end") bars := yearBars(counts, func(y string) string { from, to := y, y if start > y { from = start } if end != "" && padDateEnd(end) < padDateEnd(y) { to = end } yq := cloneValues(q) yq.Set("date_doc_start", from) yq.Set("date_doc_end", to) return "/search?" + yq.Encode() }) for i := range bars { bars[i].Pct = barPct(bars[i].Count, maxCount, 1) } if undated > 0 { bars = append(bars, timelineBar{ Year: "-", Count: undated, Pct: barPct(undated, maxCount, 1), }) } return bars } // decorateResults marks the query's terms in the titles, descriptions and // body snippets of the results on the page. func decorateResults(docs []*Document, queries, fields, conjs []string) (results []searchResult, snippetsCapped bool) { snippetTerm, snippetField := snippetAnchor(queries, fields, conjs) hl := buildHighlightTerms(queries, conjs) results = make([]searchResult, len(docs)) for i, doc := range docs { res := &results[i] res.Doc = doc res.Title = markTerms(doc.Title, hl) res.Desc = markTerms(doc.Description, hl) if snippetTerm == "" || i >= maxSnippetDocs { continue } var raw []string var hits []versionHit mainHit := false if snippetField == "TA" { raw = getSnippets(strings.Join(doc.Tags, ", "), snippetTerm, 3) } else { raw = doc.bodySnippets(snippetTerm, 3) mainHit = len(raw) > 0 if len(raw) < 3 { hits = versionHits(doc, snippetTerm, 3) } if !mainHit && len(hits) == 0 { raw = getSnippets(doc.Description, snippetTerm, 1) } } for _, s := range raw { res.Snippets = append(res.Snippets, markTerms(s, hl)) } room := 3 - len(raw) for _, h := range hits { if !mainHit { res.MatchesIn = append(res.MatchesIn, h.Version) } for _, s := range h.Snippets { if room <= 0 { break } res.VersionSnippets = append(res.VersionSnippets, VersionSnippet{Slug: h.Version.Slug, Label: h.Version.Label, HTML: markTerms(s, hl)}) room-- } } } return results, snippetTerm != "" && len(docs) > maxSnippetDocs } func (co *Corpus) handleSearch(w http.ResponseWriter, r *http.Request) { params := r.URL.Query() action := params.Get("action") sp := readSearchParams(params) prefs := readPrefs(r) scope := pageScope(params, prefs, true) sp.Coll = scope.filter queries, fields, conjs := sp.Queries, sp.Fields, sp.Conjs sortBy, order := sp.Sort, sp.Order doSearch := (params.Get("do_search") == "1" || sp.FromQ) && !sp.refused() if len(queries) == 0 { queries = []string{""} fields = []string{"TE"} conjs = []string{"and"} } switch { case strings.HasPrefix(action, "remove-"): idxStr := strings.TrimPrefix(action, "remove-") removeIdx, err := strconv.Atoi(idxStr) if err == nil && removeIdx > 0 && removeIdx < len(queries) { queries = append(queries[:removeIdx], queries[removeIdx+1:]...) fields = append(fields[:removeIdx], fields[removeIdx+1:]...) conjs = append(conjs[:removeIdx], conjs[removeIdx+1:]...) } case action == "add" && len(queries) < maxSearchClauses: queries = append(queries, "") fields = append(fields, "TE") conjs = append(conjs, "and") } sp.Queries, sp.Fields, sp.Conjs = queries, fields, conjs var scored []ScoredDoc searchTypeCounts := map[string]int{} if doSearch { scored, searchTypeCounts = co.executeSearch(sp) } selColl := scope.shown collFiltered := scope.explicit() selTypeSet := sp.Types results := make([]*Document, 0, len(scored)) for _, s := range scored { results = append(results, s.Doc) } q := canonicalSearchQuery(doSearch, sp) q["coll"] = scope.colls canonicalQS := q.Encode() searchBaseURL := "/search?" + canonicalQS + "&" formParams := cloneValues(q) if sortBy != "relevance" { formParams.Set("sort", sortBy) } if order != "" { formParams.Set("order", order) } page, _ := strconv.Atoi(params.Get("p")) start, end, page, totalPages := paginate(len(results), page, prefs.cardsPerPage()) pageResults, snippetsCapped := decorateResults(results[start:end], queries, fields, conjs) labelQ := q if scope.defaulted { labelQ = cloneValues(q) delete(labelQ, "coll") } queryLabel, named := "", false // the page title's name for the search if doSearch { queryLabel, named = co.searchLabel(labelQ) } savable := named && len(canonicalQS) <= searchQSMax var recorded []string if prefs.TrackSearches && savable && (action == "" || action == "search") && !crossSiteFetch(r) { recorded = recordRecentSearch(w, r, canonicalQS) } var recentSearches []recentSearch if prefs.TrackSearches { if recorded != nil { recentSearches = co.parseSearchEntries(recorded) } else { recentSearches = co.readRecentSearches(r) } } searchPinned := savable && isPinnedSearch(r, canonicalQS) var dateHist []timelineBar if doSearch && len(results) > 0 { dateHist = searchDateHistogram(results, q) } var facets []searchFacet if doSearch && len(results) >= 2 && len(queries) < maxSearchClauses { values := facetValues if !hasOrRow(queries, conjs) { values = refineValues(sp.DatePubStart, sp.DatePubEnd) } facets = facetsWithinWords(computeSearchFacets(results, searchBaseURL, values), maxSearchWords-searchWords(queries)) } if len(recentSearches) > 0 { noStore(w) } render(w, r, &searchPage{ FieldOptions: searchFields, Facets: facets, Queries: queries, Fields: fields, Conjs: conjs, Results: pageResults, SearchTypeCounts: searchTypeCounts, SelTypeSet: selTypeSet, FormParams: formParams, Savable: savable, ShowThumbs: prefs.thumbsOn(), FOIAQS: prefs.foiaLinkQS(), Base: pageWith(prefs), Total: len(results), Page: page, TotalPages: totalPages, RecentSearches: recentSearches, SelColl: selColl, CollFiltered: collFiltered, CanonicalQS: canonicalQS, SearchPinned: searchPinned, Searched: doSearch, QueryLabel: queryLabel, DateDocStart: sp.DateDocStart, DateDocEnd: sp.DateDocEnd, DatePubStart: sp.DatePubStart, DatePubEnd: sp.DatePubEnd, Sort: sortBy, Order: order, SearchBaseURL: searchBaseURL, DateHist: dateHist, SnippetsCapped: snippetsCapped, SnippetLimit: maxSnippetDocs, OverClauses: sp.OverClauses, ClauseLimit: maxSearchClauses, OverWords: sp.OverWords, WordLimit: maxSearchWords, }) } // --- Date helpers --- func inDateRange(date, start, end string) bool { if date == "" { return start == "" && end == "" } if start != "" && date < start { return false } if end != "" && date > end { return false } return true } func matchDates(doc *Document, dateDocStart, dateDocEnd, datePubStart, datePubEnd string, byEntry []entryRow) bool { if dateDocStart != "" || dateDocEnd != "" { if !inDateRange(doc.DocMeta.DateCreated, dateDocStart, dateDocEnd) { return false } } if datePubStart == "" && datePubEnd == "" { return true } inRange := func(i int) bool { return inDateRange(doc.Publication[i].DatePublished, datePubStart, datePubEnd) } if len(byEntry) == 0 { for i := range doc.Publication { if inRange(i) { return true } } return false } for _, r := range byEntry { found := false for i := range doc.Publication { if inRange(i) && r.matches(doc, i) { found = true break } } if !found { return false } } return true } // entryRow is a journalist or publisher row a publication date is tied // to, parsed once per search. type entryRow struct { field string // "RE" or "PU" pos, negs []valuesTest } func entryRows(queries, fields, conjs []string) []entryRow { if hasOrRow(queries, conjs) { return nil } var rows []entryRow for i, q := range queries { if q == "" { continue } if conjs[i] != "not" && (fields[i] == "RE" || fields[i] == "PU") { pos, negs := rowTerms(q) rows = append(rows, entryRow{fields[i], entryTests(pos), entryTests(negs)}) } } return rows } func hasOrRow(queries, conjs []string) bool { for i, q := range queries { if q != "" && conjs[i] == "or" { return true } } return false } func entryTests(terms []string) []valuesTest { tests := make([]valuesTest, len(terms)) for i, t := range terms { if _, phrase, _, ok := parseProximity(t); ok { t = phrase } tests[i] = compileValues(t) } return tests } func entryValues(doc *Document, i int, field string) []string { p := &doc.Publication[i] switch { case field == "RE": return p.Journalists case !p.ReportingOnly: return []string{p.Publisher} } return nil } func refineValues(pubStart, pubEnd string) func(*Document, string) []string { if pubStart == "" && pubEnd == "" { return facetValues } pubEnd = padDateEnd(pubEnd) return func(d *Document, code string) []string { if code != "RE" && code != "PU" { return facetValues(d, code) } var vals []string for i := range d.Publication { if inDateRange(d.Publication[i].DatePublished, pubStart, pubEnd) { vals = append(vals, entryValues(d, i, code)...) } } return normalizeList(vals) } } // matches reports whether publication entry i matches the row on its own // journalists or publisher (entryValues). func (r entryRow) matches(doc *Document, i int) bool { vals := entryValues(doc, i, r.field) for t := range r.pos { if !r.pos[t].match(vals) { return false } } for t := range r.negs { if r.negs[t].match(vals) { return false } } return true } // --- The boolean clause list --- // compiledQuery is a search's rows read once. The non-empty ones, in order, // each with the tests of its positive values and of its exclusions on its // field. type compiledQuery []compiledRow type compiledRow struct { conj string pos, negs []func(*Document) bool } func compileQuery(queries, fields, conjs []string) compiledQuery { var rows compiledQuery for i, q := range queries { if q == "" { continue } f := fieldByCode[fields[i]] if f == nil { f = fieldByCode["TE"] } row := compiledRow{conj: conjs[i]} pos, negs := rowTerms(q) for _, v := range pos { row.pos = append(row.pos, f.compile(v)) } for _, v := range negs { row.negs = append(row.negs, f.compile(v)) } rows = append(rows, row) } return rows } // matches evaluates the rows against a document, left to right, each row // as one unit. func (rows compiledQuery) matches(d *Document) bool { overall := true for i := range rows { row := &rows[i] if i == 0 { overall = row.matches(d) != (row.conj == "not") continue } switch row.conj { case "or": overall = overall || row.matches(d) case "not": overall = overall && !row.matches(d) default: // "and" overall = overall && row.matches(d) } } return overall } // matches reports whether a document matches one row. func (row *compiledRow) matches(d *Document) bool { for _, test := range row.pos { if !test(d) { return false } } for _, test := range row.negs { if test(d) { return false } } return true } // versionHit is one of a document's versions whose new lines match a query, // with snippet windows from those lines. type versionHit struct { Version *DocVersion Snippets []string } // versionHits returns the versions of doc whose new lines (NovelText) match // term in version order. func versionHits(doc *Document, term string, max int) []versionHit { var out []versionHit for i := range doc.Versions { v := &doc.Versions[i] if s := getSnippets(v.NovelText, term, max); len(s) > 0 { out = append(out, versionHit{Version: v, Snippets: s}) } } return out }