2644 lines
75 KiB
Go
2644 lines
75 KiB
Go
package main
|
||
|
||
import (
|
||
"cmp"
|
||
"html/template"
|
||
"log"
|
||
"math"
|
||
"net/http"
|
||
"net/url"
|
||
"slices"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"unicode"
|
||
"unicode/utf8"
|
||
)
|
||
|
||
type TFIDFIndex struct {
|
||
DocStats map[string]map[string]float64
|
||
DocFreq map[string]int
|
||
N int
|
||
}
|
||
|
||
// stopWords are excluded from tokenization to avoid recreating the map on
|
||
// every tokenize() call.
|
||
var stopWords = map[string]bool{
|
||
"the": true, "and": true, "for": true, "that": true,
|
||
"this": true, "with": true, "from": true, "are": true,
|
||
"was": true, "were": true, "has": true, "have": true,
|
||
"not": true, "but": true, "they": true, "their": true,
|
||
"will": true, "been": true, "its": true, "also": true,
|
||
"into": true, "which": true, "such": true, "other": true,
|
||
}
|
||
|
||
// --- Stemmer ---
|
||
// Porter-lite: suffix stripping for English. Applied at both index time and
|
||
// query time so "surveillance" and "surveilling" map to the same token.
|
||
// Intentionally conservative - only strips high-confidence suffixes to avoid
|
||
// over-stemming domain-specific terms like "SIGINT" or "collection".
|
||
func stem(w string) string {
|
||
// The last two steps apply whichever suffix rule returned: "flying" and
|
||
// "flyer" leave "fly", which must meet "fly" itself.
|
||
s := yToI(stripSuffixes(w))
|
||
// fixStem keeps a doubled l ("falling" -> "fall"), so "controlled" and
|
||
// "controller" reach "controll" while "control" stays "control". A final
|
||
// "ll" after more than one vowel-consonant run is undoubled in every
|
||
// word alike (Porter's step 5b): control/controlled -> "control",
|
||
// install/installing -> "instal", while fall/falling stay "fall".
|
||
if strings.HasSuffix(s, "ll") && measure(s) > 1 {
|
||
return s[:len(s)-1]
|
||
}
|
||
return s
|
||
}
|
||
|
||
func stripSuffixes(w string) string {
|
||
switch {
|
||
case strings.HasSuffix(w, "sses"):
|
||
w = w[:len(w)-2]
|
||
case strings.HasSuffix(w, "ies") && len(w) > 4:
|
||
w = w[:len(w)-3] + "y" // "studies" -> "study", "spies" -> "spy"
|
||
case strings.HasSuffix(w, "ies"):
|
||
w = w[:len(w)-1] // "ties" -> "tie": one letter before it, a word in "ie"
|
||
case strings.HasSuffix(w, "ss"):
|
||
// "process", "class": not a plural
|
||
case strings.HasSuffix(w, "s") && len(w) > 3:
|
||
w = w[:len(w)-1]
|
||
}
|
||
switch {
|
||
case strings.HasSuffix(w, "ied") && len(w) == 4:
|
||
return w[:3] // "tied" -> "tie", like "ties"
|
||
case strings.HasSuffix(w, "ied") && len(w) > 4:
|
||
w = w[:len(w)-3] + "y" // "studied" -> "study", "spied" -> "spy"
|
||
case strings.HasSuffix(w, "eed") && len(w) > 4:
|
||
return w[:len(w)-1]
|
||
case strings.HasSuffix(w, "ed") && len(w) > 4:
|
||
s := w[:len(w)-2]
|
||
return fixStem(s)
|
||
case strings.HasSuffix(w, "ing") && len(w) > 5:
|
||
s := w[:len(w)-3]
|
||
return fixStem(s)
|
||
}
|
||
switch {
|
||
case strings.HasSuffix(w, "ational") && len(w) > 8:
|
||
return w[:len(w)-5] // "operational" -> "operat", same as "operation"
|
||
case strings.HasSuffix(w, "ation") && len(w) > 6:
|
||
return w[:len(w)-3]
|
||
case strings.HasSuffix(w, "ence") && len(w) > 5:
|
||
return w[:len(w)-4]
|
||
case strings.HasSuffix(w, "ance") && len(w) > 5:
|
||
return w[:len(w)-4]
|
||
case strings.HasSuffix(w, "ment") && len(w) > 5:
|
||
return w[:len(w)-4]
|
||
case strings.HasSuffix(w, "ness") && len(w) > 5:
|
||
return w[:len(w)-4]
|
||
case strings.HasSuffix(w, "ity") && len(w) > 5:
|
||
return w[:len(w)-3]
|
||
case strings.HasSuffix(w, "ism") && len(w) > 5:
|
||
return w[:len(w)-3]
|
||
case strings.HasSuffix(w, "ist") && len(w) > 5:
|
||
return w[:len(w)-3]
|
||
case strings.HasSuffix(w, "ize") && len(w) > 5:
|
||
return w[:len(w)-3]
|
||
case strings.HasSuffix(w, "ise") && len(w) > 5:
|
||
return w[:len(w)-3]
|
||
case strings.HasSuffix(w, "ful") && len(w) > 5:
|
||
return w[:len(w)-3]
|
||
case strings.HasSuffix(w, "less") && len(w) > 5:
|
||
return w[:len(w)-4]
|
||
case strings.HasSuffix(w, "ly") && len(w) > 5 && w[len(w)-3] != 'p':
|
||
return w[:len(w)-2]
|
||
case strings.HasSuffix(w, "er") && len(w) > 4:
|
||
return w[:len(w)-2]
|
||
case strings.HasSuffix(w, "al") && len(w) > 4:
|
||
return w[:len(w)-2]
|
||
}
|
||
return w
|
||
}
|
||
|
||
// yToI turns a final consonant+y into i, so singular/plural pairs unify
|
||
// with the "ies" rule: "study" -> "studi", "studies" -> "studi";
|
||
// "agency" -> "agenci", "agencies" -> "agenci".
|
||
func yToI(w string) string {
|
||
if strings.HasSuffix(w, "y") && len(w) > 3 && !isVowel(rune(w[len(w)-2])) {
|
||
return w[:len(w)-1] + "i"
|
||
}
|
||
if strings.HasSuffix(w, "ie") && len(w) > 5 {
|
||
return w[:len(w)-1]
|
||
}
|
||
return w
|
||
}
|
||
|
||
// measure is Porter's m: the number of vowel-consonant runs in w, with y a
|
||
// vowel after a consonant ("controll" 2, "fall" 1, "tree" 0).
|
||
func measure(w string) int {
|
||
m := 0
|
||
prevVowel := false
|
||
for i := 0; i < len(w); i++ {
|
||
v := isVowel(rune(w[i])) || (w[i] == 'y' && i > 0 && !prevVowel)
|
||
if prevVowel && !v {
|
||
m++
|
||
}
|
||
prevVowel = v
|
||
}
|
||
return m
|
||
}
|
||
|
||
// fixStem handles consonant doubling after stripping -ed/-ing
|
||
// e.g. "hopping" -> "hopp" -> "hop".
|
||
func fixStem(s string) string {
|
||
if len(s) == 0 {
|
||
return s
|
||
}
|
||
if len(s) >= 2 {
|
||
last := s[len(s)-1]
|
||
secondLast := s[len(s)-2]
|
||
if last == secondLast && !isVowel(rune(last)) && last != 'l' && last != 's' && last != 'z' {
|
||
return s[:len(s)-1]
|
||
}
|
||
}
|
||
return s
|
||
}
|
||
|
||
func isVowel(r rune) bool {
|
||
return strings.ContainsRune("aeiou", r)
|
||
}
|
||
|
||
// foldTable maps accented Latin characters to their ASCII base form so
|
||
// "muller" finds "Müller-Maguhn" and "Sao Paulo" finds "São Paulo". The
|
||
// fold runs on BOTH the index and the query side, so it's symmetric.
|
||
var foldTable = map[rune]string{
|
||
'À': "A", 'Á': "A", 'Â': "A", 'Ã': "A", 'Ä': "A",
|
||
'Å': "A", 'Æ': "AE", 'Ç': "C", 'È': "E", 'É': "E", 'Ê': "E",
|
||
'Ë': "E", 'Ì': "I", 'Í': "I", 'Î': "I", 'Ï': "I", 'Ð': "D",
|
||
'Ñ': "N", 'Ò': "O", 'Ó': "O", 'Ô': "O", 'Õ': "O", 'Ö': "O",
|
||
'Ø': "O", 'Ù': "U", 'Ú': "U", 'Û': "U", 'Ü': "U", 'Ý': "Y",
|
||
'Þ': "TH", 'ß': "ss", 'à': "a", 'á': "a", 'â': "a", 'ã': "a",
|
||
'ä': "a", 'å': "a", 'æ': "ae", 'ç': "c", 'è': "e", 'é': "e",
|
||
'ê': "e", 'ë': "e", 'ì': "i", 'í': "i", 'î': "i", 'ï': "i",
|
||
'ð': "d", 'ñ': "n", 'ò': "o", 'ó': "o", 'ô': "o", 'õ': "o",
|
||
'ö': "o", 'ø': "o", 'ù': "u", 'ú': "u", 'û': "u", 'ü': "u",
|
||
'ý': "y", 'þ': "th", 'ÿ': "y", 'Ā': "A", 'ā': "a", 'Ă': "A",
|
||
'ă': "a", 'Ą': "A", 'ą': "a", 'Ć': "C", 'ć': "c", 'Ĉ': "C",
|
||
'ĉ': "c", 'Ċ': "C", 'ċ': "c", 'Č': "C", 'č': "c", 'Ď': "D",
|
||
'ď': "d", 'Đ': "D", 'đ': "d", 'Ē': "E", 'ē': "e", 'Ĕ': "E",
|
||
'ĕ': "e", 'Ė': "E", 'ė': "e", 'Ę': "E", 'ę': "e", 'Ě': "E",
|
||
'ě': "e", 'Ĝ': "G", 'ĝ': "g", 'Ğ': "G", 'ğ': "g", 'Ġ': "G",
|
||
'ġ': "g", 'Ģ': "G", 'ģ': "g", 'Ĥ': "H", 'ĥ': "h", 'Ħ': "H",
|
||
'ħ': "h", 'Ĩ': "I", 'ĩ': "i", 'Ī': "I", 'ī': "i", 'Ĭ': "I",
|
||
'ĭ': "i", 'Į': "I", 'į': "i", 'İ': "I", 'ı': "i", 'Ĵ': "J",
|
||
'ĵ': "j", 'Ķ': "K", 'ķ': "k", 'Ĺ': "L", 'ĺ': "l", 'Ļ': "L",
|
||
'ļ': "l", 'Ľ': "L", 'ľ': "l", 'Ŀ': "L", 'ŀ': "l", 'Ł': "L",
|
||
'ł': "l", 'Ń': "N", 'ń': "n", 'Ņ': "N", 'ņ': "n", 'Ň': "N",
|
||
'ň': "n", 'ʼn': "n", 'Ō': "O", 'ō': "o", 'Ŏ': "O", 'ŏ': "o",
|
||
'Ő': "O", 'ő': "o", 'Œ': "OE", 'œ': "oe", 'Ŕ': "R", 'ŕ': "r",
|
||
'Ŗ': "R", 'ŗ': "r", 'Ř': "R", 'ř': "r", 'Ś': "S", 'ś': "s",
|
||
'Ŝ': "S", 'ŝ': "s", 'Ş': "S", 'ş': "s", 'Š': "S", 'š': "s",
|
||
'Ţ': "T", 'ţ': "t", 'Ť': "T", 'ť': "t", 'Ŧ': "T", 'ŧ': "t",
|
||
'Ũ': "U", 'ũ': "u", 'Ū': "U", 'ū': "u", 'Ŭ': "U", 'ŭ': "u",
|
||
'Ů': "U", 'ů': "u", 'Ű': "U", 'ű': "u", 'Ų': "U", 'ų': "u",
|
||
'Ŵ': "W", 'ŵ': "w", 'Ŷ': "Y", 'ŷ': "y", 'Ÿ': "Y", 'Ź': "Z",
|
||
'ź': "z", 'Ż': "Z", 'ż': "z", 'Ž': "Z", 'ž': "z", 'Ș': "S",
|
||
'ș': "s", 'Ț': "T", 'ț': "t", 'ẞ': "SS",
|
||
}
|
||
|
||
// foldMin is the smallest rune foldTable maps, every rune below it folds
|
||
// to itself so the folds skip the table lookup for it (all of ASCII).
|
||
var foldMin = func() rune {
|
||
m := utf8.MaxRune
|
||
for r := range foldTable {
|
||
m = min(m, r)
|
||
}
|
||
return m
|
||
}()
|
||
|
||
// foldAccents replaces accented characters with their base form,
|
||
// preserving case for characters not in the table.
|
||
func foldAccents(s string) string {
|
||
needs := false
|
||
for _, r := range s {
|
||
if r < foldMin {
|
||
continue
|
||
}
|
||
if _, ok := foldTable[r]; ok {
|
||
needs = true
|
||
break
|
||
}
|
||
}
|
||
if !needs {
|
||
return s
|
||
}
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
for _, r := range s {
|
||
if r >= foldMin {
|
||
if f, ok := foldTable[r]; ok {
|
||
b.WriteString(f)
|
||
continue
|
||
}
|
||
}
|
||
b.WriteRune(r)
|
||
}
|
||
return b.String()
|
||
}
|
||
|
||
func normFoldString(s string) string {
|
||
return strings.ToLower(foldAccents(s))
|
||
}
|
||
|
||
// foldRune is normFoldString for one rune.
|
||
func foldRune(r rune) string {
|
||
if r < utf8.RuneSelf {
|
||
if 'A' <= r && r <= 'Z' {
|
||
r += 'a' - 'A'
|
||
}
|
||
return string(r)
|
||
}
|
||
if f, ok := foldTable[r]; ok {
|
||
return strings.ToLower(f)
|
||
}
|
||
return strings.ToLower(string(r))
|
||
}
|
||
|
||
func foldedLen(r rune) int {
|
||
if r < utf8.RuneSelf {
|
||
return 1
|
||
}
|
||
if f, ok := foldTable[r]; ok {
|
||
return len(f)
|
||
}
|
||
return utf8.RuneLen(unicode.ToLower(r))
|
||
}
|
||
|
||
// normFold returns the normal form of s plus a position map: posMap[i] is
|
||
// the byte offset in s of the rune that produced norm byte i (with a final
|
||
// entry of len(s)). Lets match highlighting (markTerms) mark on the normal
|
||
// form but slice the ORIGINAL text, so a "muller" query highlights "Müller"
|
||
// as written.
|
||
func normFold(s string) (string, []int) {
|
||
var b strings.Builder
|
||
b.Grow(len(s))
|
||
posMap := make([]int, 0, len(s)+1)
|
||
for i, r := range s {
|
||
if r < utf8.RuneSelf {
|
||
if 'A' <= r && r <= 'Z' {
|
||
r += 'a' - 'A'
|
||
}
|
||
b.WriteByte(byte(r))
|
||
posMap = append(posMap, i)
|
||
continue
|
||
}
|
||
f := foldRune(r)
|
||
for j := 0; j < len(f); j++ {
|
||
posMap = append(posMap, i)
|
||
}
|
||
b.WriteString(f)
|
||
}
|
||
posMap = append(posMap, len(s))
|
||
return b.String(), posMap
|
||
}
|
||
|
||
// splitWords splits s into alphanumeric runs (it does not change case).
|
||
func splitWords(s string) []string {
|
||
return strings.FieldsFunc(s, func(r rune) bool {
|
||
return !unicode.IsLetter(r) && !unicode.IsDigit(r)
|
||
})
|
||
}
|
||
|
||
func tokenize(s string) []string {
|
||
words := splitWords(normFoldString(s))
|
||
filtered := words[:0]
|
||
for _, w := range words {
|
||
if len(w) > 1 && !stopWords[w] {
|
||
filtered = append(filtered, stem(w))
|
||
}
|
||
}
|
||
return filtered
|
||
}
|
||
|
||
// eachToken calls f with every token of a stemmed stream (tokenize's
|
||
// output joined by single spaces) in order.
|
||
func eachToken(stream string, f func(tok string)) {
|
||
for i := 0; i < len(stream); {
|
||
j := strings.IndexByte(stream[i:], ' ')
|
||
if j < 0 {
|
||
j = len(stream) - i
|
||
}
|
||
if j > 0 {
|
||
f(stream[i : i+j])
|
||
}
|
||
i += j + 1
|
||
}
|
||
}
|
||
|
||
func buildTFIDF(docs []Document) TFIDFIndex {
|
||
idx := TFIDFIndex{
|
||
DocStats: make(map[string]map[string]float64, len(docs)),
|
||
DocFreq: make(map[string]int),
|
||
N: len(docs),
|
||
}
|
||
|
||
for i := range docs {
|
||
d := &docs[i]
|
||
// Title 3x, fulltext 2x, descriptions + metadata 1x. The metadata
|
||
// is every field the all-fields search reads (searchMeta), so a doc
|
||
// that matches only via metadata still gets a relevance score
|
||
// instead of always sorting last. Title, descriptions and body come
|
||
// from the stemmed forms loadData already built (tokenize() output
|
||
// joined by spaces).
|
||
tf := make(map[string]float64)
|
||
total := 0.0
|
||
add := func(weight float64) func(string) {
|
||
return func(w string) {
|
||
if w == valueSep {
|
||
return
|
||
}
|
||
tf[w] += weight
|
||
total += weight
|
||
}
|
||
}
|
||
eachToken(d.StemmedTitle, add(3))
|
||
eachToken(d.StemmedFull, add(2))
|
||
eachToken(d.StemmedDesc, add(1))
|
||
for _, w := range tokenize(searchMeta(d)) {
|
||
add(1)(w)
|
||
}
|
||
|
||
for w := range tf {
|
||
tf[w] /= total // total > 0 whenever tf has a key
|
||
idx.DocFreq[w]++
|
||
}
|
||
idx.DocStats[d.ID] = tf
|
||
}
|
||
|
||
log.Printf("TF-IDF index built: %d docs, %d unique terms", len(docs), len(idx.DocFreq))
|
||
return idx
|
||
}
|
||
|
||
// scorer reads one positive term of a query into its relevance score for a
|
||
// document. The term's words go through the same tokenize() pipeline as the
|
||
// index (stemming + stopword removal), and matching against stemmed fields
|
||
// is by whole token.
|
||
func (idx TFIDFIndex) scorer(query string) func(doc *Document) float64 {
|
||
// A proximity clause scores as its phrase - the window is a matcher
|
||
// concern, not a relevance signal (and the raw "~10" tail would
|
||
// otherwise leak a junk "10" term into the tokenizer).
|
||
if _, phrase, _, ok := parseProximity(query); ok {
|
||
query = phrase
|
||
}
|
||
// An exact-value clause scores as its phrase with the phrase bonuses on
|
||
// whole words; quoted text earns them wherever it occurs.
|
||
contains := strings.Contains
|
||
if v, ok := exactValue(query); ok {
|
||
query, contains = `"`+v+`"`, containsWord
|
||
}
|
||
// Folded so the phrase bonuses are accent-consistent with matching, and
|
||
// so the content check can use the pre-folded body (no per-doc fold).
|
||
clean := strings.Trim(normFoldString(strings.TrimSpace(query)), `"`)
|
||
// Wildcard words score by where they occur, not through their stems:
|
||
// +5 when the title has one, else +1 when the document does.
|
||
var wild []queryWord
|
||
if !isQuotedQuery(query) {
|
||
var kept []string
|
||
for _, w := range strings.Fields(clean) {
|
||
if _, _, _, isWild := wildcard(w); isWild {
|
||
wild = append(wild, literalWord(w))
|
||
} else {
|
||
kept = append(kept, w)
|
||
}
|
||
}
|
||
clean = strings.Join(kept, " ")
|
||
}
|
||
terms := tokenize(clean)
|
||
// A term's weight in a document is its share of the document times how
|
||
// rare it is in the corpus; a term no document holds weighs nothing.
|
||
idf := make([]float64, len(terms))
|
||
for i, w := range terms {
|
||
if df := idx.DocFreq[w]; df != 0 {
|
||
idf[i] = math.Log(float64(idx.N) / float64(df))
|
||
}
|
||
}
|
||
isPhrase := len(strings.Fields(clean)) > 1
|
||
|
||
return func(doc *Document) float64 {
|
||
stats, ok := idx.DocStats[doc.ID]
|
||
if !ok || (len(terms) == 0 && len(wild) == 0) {
|
||
return 0
|
||
}
|
||
title := "" // for the wildcard and phrase bonuses
|
||
if len(wild) > 0 || isPhrase {
|
||
title = doc.titleFold()
|
||
}
|
||
|
||
var score float64
|
||
for i := range wild {
|
||
switch {
|
||
case wild[i].kind == wordNever: // matches nothing (see wildcard), so earns nothing
|
||
case wild[i].in(title):
|
||
score += 5
|
||
case wild[i].in(doc.SearchText):
|
||
score++
|
||
}
|
||
}
|
||
if len(terms) == 0 {
|
||
return score
|
||
}
|
||
for i, w := range terms {
|
||
score += stats[w] * idf[i]
|
||
}
|
||
|
||
// bonus is whole when the field holds the exact phrase (or, for one
|
||
// word, the word), else all when it holds every term somewhere.
|
||
bonus := func(phraseIn string, holds func(term string) bool, whole, all float64) float64 {
|
||
if isPhrase && contains(phraseIn, clean) {
|
||
return whole
|
||
}
|
||
for _, w := range terms {
|
||
if !holds(w) {
|
||
return 0
|
||
}
|
||
}
|
||
if isPhrase {
|
||
return all
|
||
}
|
||
return whole
|
||
}
|
||
// Title +10 or +5; the whole document +3 or +1. The document's
|
||
// entry in the index holds exactly the terms of its all-fields
|
||
// stream.
|
||
inTitle := func(w string) bool { return containsTokens(doc.StemmedTitle, w) }
|
||
inDoc := func(w string) bool { _, ok := stats[w]; return ok }
|
||
return score + bonus(title, inTitle, 10, 5) + bonus(doc.FoldedFull, inDoc, 3, 1)
|
||
}
|
||
}
|
||
|
||
// similarity returns cosine similarity between two docs' TF-IDF vectors.
|
||
// Returns 0-1; used by relatedScore in data.go.
|
||
// TODO split this file or move this away from here?
|
||
func (idx TFIDFIndex) similarity(aID, bID string) float64 {
|
||
a, ok1 := idx.DocStats[aID]
|
||
b, ok2 := idx.DocStats[bID]
|
||
if !ok1 || !ok2 {
|
||
return 0
|
||
}
|
||
|
||
var dot, normA, normB float64
|
||
for w, tfA := range a {
|
||
df := idx.DocFreq[w]
|
||
if df == 0 {
|
||
continue
|
||
}
|
||
idf := math.Log(float64(idx.N) / float64(df))
|
||
tfidfA := tfA * idf
|
||
normA += tfidfA * tfidfA
|
||
if tfB, ok := b[w]; ok {
|
||
tfidfB := tfB * idf
|
||
dot += tfidfA * tfidfB
|
||
}
|
||
}
|
||
for w, tfB := range b {
|
||
df := idx.DocFreq[w]
|
||
if df == 0 {
|
||
continue
|
||
}
|
||
idf := math.Log(float64(idx.N) / float64(df))
|
||
tfidfB := tfB * idf
|
||
normB += tfidfB * tfidfB
|
||
}
|
||
|
||
if normA == 0 || normB == 0 {
|
||
return 0
|
||
}
|
||
return dot / (math.Sqrt(normA) * math.Sqrt(normB))
|
||
}
|
||
|
||
// --- Clause values ---
|
||
|
||
// isQuotedQuery returns true for "exact phrase" queries.
|
||
func isQuotedQuery(q string) bool {
|
||
q = strings.TrimSpace(q)
|
||
return len(q) >= 2 && strings.HasPrefix(q, `"`) && strings.HasSuffix(q, `"`)
|
||
}
|
||
|
||
// --- Exact values and word boundaries ---
|
||
//
|
||
// A clause value has four shapes: words (cable tapping, matched with their
|
||
// other forms), quoted text ("cable tapping", found anywhere, like grep -i),
|
||
// an exact value (="cable tapping", or =tap for one word) and a wildcard
|
||
// (tap*, *apping). An exact value means "exactly this, nothing attached", on
|
||
// a metadata field it is one whole value, so agency:=NSA does not match
|
||
// "NSA, GCHQ"; on the text fields (all fields, title, content, description)
|
||
// it is whole words exactly as typed, so =tap finds "tap" but not "taps" or
|
||
// "strap".
|
||
|
||
// exactValue reports whether q is an exact-value clause ("=") followed by
|
||
// a quoted value or a single bare word - and returns the value.
|
||
func exactValue(q string) (string, bool) {
|
||
q = strings.TrimSpace(q)
|
||
if !strings.HasPrefix(q, "=") {
|
||
return "", false
|
||
}
|
||
v := strings.TrimSpace(q[1:])
|
||
if isQuotedQuery(v) {
|
||
v = strings.TrimSpace(v[1 : len(v)-1])
|
||
} else if strings.ContainsAny(v, spaceBytes+`"`) {
|
||
return "", false
|
||
}
|
||
return v, v != ""
|
||
}
|
||
|
||
// exactClause builds the ="..." clause value the generated links use, so a
|
||
// count and its click compare whole values the same way.
|
||
func exactClause(v string) string { return `="` + v + `"` }
|
||
|
||
// isWordRune is the word-character test splitWords uses.
|
||
func isWordRune(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) }
|
||
|
||
// indexWord returns the byte offset of the first occurrence of needle in s,
|
||
// at or after from, that stands on word boundaries (no letter or digit
|
||
// directly before or after it) or -1. A needle edge that is not a word
|
||
// character ("u.s.") needs no boundary on that side. s and needle must
|
||
// already share a normal form (normFoldString).
|
||
func indexWord(s, needle string, from int) int {
|
||
first, _ := utf8.DecodeRuneInString(needle)
|
||
last, _ := utf8.DecodeLastRuneInString(needle)
|
||
return indexBounded(s, needle, from, isWordRune(first), isWordRune(last))
|
||
}
|
||
|
||
// wildMinRunes is the fewest letters or digits a fragment behind a leading
|
||
// "*" must have: "*a" or "*ab" would match nearly every document.
|
||
const wildMinRunes = 3
|
||
|
||
// wildcard reports whether w is a wildcard word (a fragment with "*") at
|
||
// its end (tap*), its start (*apping) or both (*keys*), and returns the
|
||
// fragment in normal form plus which sides may run on into more of the
|
||
// word.
|
||
func wildcard(w string) (frag string, openStart, openEnd, ok bool) {
|
||
openStart, openEnd = strings.HasPrefix(w, "*"), strings.HasSuffix(w, "*")
|
||
if !openStart && !openEnd {
|
||
return "", false, false, false
|
||
}
|
||
frag = normFoldString(strings.Trim(w, "*"))
|
||
n := 0
|
||
for _, r := range frag {
|
||
if isWordRune(r) {
|
||
n++
|
||
}
|
||
}
|
||
if n == 0 || (openStart && n < wildMinRunes) || strings.Contains(frag, "*") {
|
||
return "", openStart, openEnd, true
|
||
}
|
||
return frag, openStart, openEnd, true
|
||
}
|
||
|
||
// indexWild finds a wildcard fragment in s, both in normal form. A closed
|
||
// side must sit on a word boundary and an open side may continue into the
|
||
// word, so tap* finds "tapping" but not "strap".
|
||
func indexWild(s, frag string, from int, openStart, openEnd bool) int {
|
||
first, _ := utf8.DecodeRuneInString(frag)
|
||
last, _ := utf8.DecodeLastRuneInString(frag)
|
||
return indexBounded(s, frag, from, !openStart && isWordRune(first), !openEnd && isWordRune(last))
|
||
}
|
||
|
||
// wordBounds widens [start, end) in s to the whole word around it.
|
||
func wordBounds(s string, start, end int) (int, int) {
|
||
for start > 0 {
|
||
r, size := utf8.DecodeLastRuneInString(s[:start])
|
||
if !isWordRune(r) {
|
||
break
|
||
}
|
||
start -= size
|
||
}
|
||
for end < len(s) {
|
||
r, size := utf8.DecodeRuneInString(s[end:])
|
||
if !isWordRune(r) {
|
||
break
|
||
}
|
||
end += size
|
||
}
|
||
return start, end
|
||
}
|
||
|
||
// indexBounded is indexWord with the boundary checks chosen by the caller:
|
||
// checkStart requires no letter or digit directly before the match,
|
||
// checkEnd none directly after it.
|
||
func indexBounded(s, needle string, from int, checkStart, checkEnd bool) int {
|
||
if from < 0 {
|
||
from = 0
|
||
}
|
||
if needle == "" {
|
||
if from <= len(s) {
|
||
return from
|
||
}
|
||
return -1
|
||
}
|
||
for from <= len(s)-len(needle) {
|
||
j := strings.Index(s[from:], needle)
|
||
if j < 0 {
|
||
return -1
|
||
}
|
||
i := from + j
|
||
end := i + len(needle)
|
||
ok := true
|
||
if checkStart && i > 0 {
|
||
if r, _ := utf8.DecodeLastRuneInString(s[:i]); isWordRune(r) {
|
||
ok = false
|
||
}
|
||
}
|
||
if ok && checkEnd && end < len(s) {
|
||
if r, _ := utf8.DecodeRuneInString(s[end:]); isWordRune(r) {
|
||
ok = false
|
||
}
|
||
}
|
||
if ok {
|
||
return i
|
||
}
|
||
_, size := utf8.DecodeRuneInString(s[i:])
|
||
from = i + size
|
||
}
|
||
return -1
|
||
}
|
||
|
||
// containsWord reports whether needle occurs in s on word boundaries.
|
||
func containsWord(s, needle string) bool { return indexWord(s, needle, 0) >= 0 }
|
||
|
||
// containsTokens reports whether phrase (one or more stemmed tokens joined
|
||
// by single spaces) occurs in a space-joined token stream as WHOLE tokens.
|
||
func containsTokens(stream, phrase string) bool {
|
||
if phrase == "" {
|
||
return true
|
||
}
|
||
for i := 0; i+len(phrase) <= len(stream); {
|
||
j := strings.Index(stream[i:], phrase)
|
||
if j < 0 {
|
||
return false
|
||
}
|
||
p := i + j
|
||
end := p + len(phrase)
|
||
if (p == 0 || stream[p-1] == ' ') && (end == len(stream) || stream[end] == ' ') {
|
||
return true
|
||
}
|
||
i = p + 1
|
||
}
|
||
return false
|
||
}
|
||
|
||
// valueSep joins a field's values for word and phrase matching. It is not a
|
||
// word character, so words still split on it, and nobody types it, so a
|
||
// quoted phrase cannot run from one value into the next.
|
||
const valueSep = "\x1f"
|
||
|
||
func valueKey(v string) string { return strings.ToLower(strings.TrimSpace(v)) }
|
||
|
||
func sameValue(a, b string) bool {
|
||
a, b = strings.TrimSpace(a), strings.TrimSpace(b)
|
||
if !isASCII(a) || !isASCII(b) {
|
||
return strings.ToLower(a) == strings.ToLower(b)
|
||
}
|
||
if len(a) != len(b) {
|
||
return false
|
||
}
|
||
for i := 0; i < len(a); i++ {
|
||
x, y := a[i], b[i]
|
||
if 'A' <= x && x <= 'Z' {
|
||
x += 'a' - 'A'
|
||
}
|
||
if 'A' <= y && y <= 'Z' {
|
||
y += 'a' - 'A'
|
||
}
|
||
if x != y {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
func isASCII(s string) bool {
|
||
for i := 0; i < len(s); i++ {
|
||
if s[i] >= utf8.RuneSelf {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// --- Compiled clauses ---
|
||
//
|
||
// A search reads each clause once, into a test, and the scan runs the tests
|
||
// on every document.
|
||
|
||
// queryWord is one word of an unquoted clause.
|
||
type queryWord struct {
|
||
kind int
|
||
text string // the fragment (wordWild), the word (wordWhole, wordSub) or its stemmed tokens joined by spaces (wordStems)
|
||
openStart, openEnd bool // wordWild: which sides of the fragment had "*"
|
||
tokens []string // wordStems: the tokens of text
|
||
}
|
||
|
||
const (
|
||
wordNever = iota // a wildcard with nothing to search for (wildcard): it matches nothing
|
||
wordWild // tap*, *apping, *keys*: part of a word, in the folded text
|
||
wordWhole // a whole word as written, in the folded text
|
||
wordSub // a word with internal punctuation ("u.s."), as written, anywhere in the folded text
|
||
wordStems // the word's stemmed tokens in sequence, in the token stream
|
||
)
|
||
|
||
// literalWord reads a folded query word the way folded text can hold it: a
|
||
// wildcard, a whole word or, with internal punctuation, which word matching
|
||
// can't represent, a substring.
|
||
func literalWord(w string) queryWord {
|
||
if frag, openStart, openEnd, ok := wildcard(w); ok {
|
||
if frag == "" {
|
||
return queryWord{kind: wordNever}
|
||
}
|
||
return queryWord{kind: wordWild, text: frag, openStart: openStart, openEnd: openEnd}
|
||
}
|
||
if parts := splitWords(w); len(parts) == 1 {
|
||
return queryWord{kind: wordWhole, text: parts[0]}
|
||
}
|
||
return queryWord{kind: wordSub, text: w}
|
||
}
|
||
|
||
// in reports whether the word, read literally, occurs in folded text.
|
||
func (w *queryWord) in(folded string) bool {
|
||
switch w.kind {
|
||
case wordWild:
|
||
return indexWild(folded, w.text, 0, w.openStart, w.openEnd) >= 0
|
||
case wordWhole:
|
||
return containsWord(folded, w.text)
|
||
case wordSub:
|
||
return strings.Contains(folded, w.text)
|
||
}
|
||
return false
|
||
}
|
||
|
||
// literalTest is a clause read for folded text that is not tokenized (the
|
||
// values of a metadata field). Quoted text is found anywhere, like grep -i
|
||
// ("ard" inside "guardian"); otherwise every word must occur as literalWord
|
||
// reads it.
|
||
type literalTest struct {
|
||
quoted bool
|
||
needle string
|
||
words []queryWord
|
||
}
|
||
|
||
func compileLiteral(q string) literalTest {
|
||
q = normFoldString(strings.TrimSpace(q))
|
||
if isQuotedQuery(q) {
|
||
return literalTest{quoted: true, needle: strings.Trim(q, `"`)}
|
||
}
|
||
var t literalTest
|
||
for _, w := range strings.Fields(q) {
|
||
t.words = append(t.words, literalWord(w))
|
||
}
|
||
return t
|
||
}
|
||
|
||
func (t *literalTest) match(folded string) bool {
|
||
if t.quoted {
|
||
return strings.Contains(folded, t.needle)
|
||
}
|
||
for i := range t.words {
|
||
if !t.words[i].in(folded) {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// valuesTest is a clause read for a metadata field's values: ="..." is one
|
||
// whole value (the equality the browse pages and the refine tallies use
|
||
// the valueKey) and anything else the values' words and phrases
|
||
// (literalTest).
|
||
type valuesTest struct {
|
||
exact bool
|
||
key string // exact: the value's valueKey
|
||
text literalTest
|
||
}
|
||
|
||
func compileValues(q string) valuesTest {
|
||
if v, ok := exactValue(q); ok {
|
||
return valuesTest{exact: true, key: valueKey(v)}
|
||
}
|
||
return valuesTest{text: compileLiteral(q)}
|
||
}
|
||
|
||
// match tests the clause on a list of values.
|
||
func (t *valuesTest) match(vals []string) bool {
|
||
if t.exact {
|
||
for _, x := range vals {
|
||
if sameValue(x, t.key) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
return t.text.match(normFoldString(strings.Join(vals, valueSep)))
|
||
}
|
||
|
||
func (t *valuesTest) matchIndexed(keys []string, folded string) bool {
|
||
if t.exact {
|
||
return slices.Contains(keys, t.key)
|
||
}
|
||
return t.text.match(folded)
|
||
}
|
||
|
||
// compileWords reads an unquoted clause for a text field. Its words run
|
||
// through the same tokenize pipeline as the index, so stemming stays
|
||
// consistent with it.
|
||
func compileWords(q string) []queryWord {
|
||
var stems, literal []queryWord
|
||
for _, w := range strings.Fields(strings.ToLower(strings.TrimSpace(q))) {
|
||
if _, _, _, ok := wildcard(w); ok {
|
||
literal = append(literal, literalWord(w))
|
||
} else if toks := tokenize(w); len(toks) == 0 {
|
||
literal = append(literal, literalWord(normFoldString(w)))
|
||
} else {
|
||
stems = append(stems, queryWord{kind: wordStems, text: strings.Join(toks, " "), tokens: toks})
|
||
}
|
||
}
|
||
return append(stems, literal...)
|
||
}
|
||
|
||
// compile reads clause q (one negation-free value of a row) into the
|
||
// test of it on the field.
|
||
func (f *searchField) compile(q string) func(*Document) bool {
|
||
if terms, phrase, win, ok := parseProximity(q); ok {
|
||
// "..."~N - see the proximity block below.
|
||
if f.Stemmed != nil && len(terms) >= 2 {
|
||
return func(d *Document) bool {
|
||
return d.hasTerms(terms) && proximityMatch(f.Stemmed(d), terms, win)
|
||
}
|
||
}
|
||
// A metadata field, or a phrase that boiled down to fewer than two
|
||
// searchable words, so it adds nothing, match the plain
|
||
// quoted phrase on the requested field instead.
|
||
q = phrase
|
||
}
|
||
if f.Values != nil {
|
||
test := compileValues(q)
|
||
return func(d *Document) bool {
|
||
if d.fields == nil {
|
||
return test.match(f.read(d))
|
||
}
|
||
return test.matchIndexed(d.fields.keys[f.index], d.fields.folded[f.index])
|
||
}
|
||
}
|
||
|
||
// A text field: an exact value (=word) is a whole-word test and quoted
|
||
// text a substring test, both on the folded text; anything else runs on
|
||
// the stemmed tokens, and reads the folded text only for its wildcard
|
||
// and literal words.
|
||
if v, ok := exactValue(q); ok {
|
||
needle := normFoldString(v)
|
||
return func(d *Document) bool { return containsWord(f.Folded(d), needle) }
|
||
}
|
||
if isQuotedQuery(q) {
|
||
needle := strings.Trim(normFoldString(strings.TrimSpace(q)), `"`)
|
||
return func(d *Document) bool { return strings.Contains(f.Folded(d), needle) }
|
||
}
|
||
words := compileWords(q)
|
||
everyField := f.Code == "TE"
|
||
return func(d *Document) bool {
|
||
for i := range words {
|
||
w := &words[i]
|
||
switch {
|
||
case w.kind != wordStems:
|
||
if !w.in(f.Folded(d)) {
|
||
return false
|
||
}
|
||
case !d.hasTerms(w.tokens):
|
||
return false
|
||
case everyField && len(w.tokens) == 1 && d.terms != nil:
|
||
// stream needs no walking
|
||
case !containsTokens(f.Stemmed(d), w.text):
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
}
|
||
|
||
// --- Proximity operator ----------------------------------------------------
|
||
//
|
||
// "upstream collection"~10 matches documents where every quoted word occurs
|
||
// within a 10-word window, unordered and counted on the same stemmed,
|
||
// stopword-free token streams the bare-word matcher uses.
|
||
|
||
// parseProximity reports whether q has the "..."~N shape (the whole clause,
|
||
// nothing after the digits). terms are the tokenized, stemmed, deduplicated
|
||
// phrase words; phrase is the plain quoted phrase for callers that degrade;
|
||
// win is N, capped at 1000 (wider than any window that could still mean
|
||
// anything beyond plain AND).
|
||
func parseProximity(q string) (terms []string, phrase string, win int, ok bool) {
|
||
q = strings.TrimSpace(q)
|
||
if len(q) < 4 || q[0] != '"' {
|
||
return nil, "", 0, false
|
||
}
|
||
end := strings.LastIndexByte(q, '"')
|
||
if end == 0 || end+2 >= len(q) || q[end+1] != '~' {
|
||
return nil, "", 0, false
|
||
}
|
||
n, err := strconv.Atoi(q[end+2:])
|
||
if err != nil || n < 1 {
|
||
return nil, "", 0, false
|
||
}
|
||
if n > 1000 {
|
||
n = 1000
|
||
}
|
||
inner := q[1:end]
|
||
seen := map[string]bool{}
|
||
for _, t := range tokenize(inner) {
|
||
if !seen[t] {
|
||
seen[t] = true
|
||
terms = append(terms, t)
|
||
}
|
||
}
|
||
return terms, `"` + inner + `"`, n, true
|
||
}
|
||
|
||
// proximityMatch walks a space-joined stemmed stream once, tracking the most
|
||
// recent position of each term; whenever every term has been seen and the
|
||
// span from the oldest to the newest of those positions fits the window, a
|
||
// valid window exists. Last-seen positions minimize the span at every step,
|
||
// so no qualifying window is missed. One linear pass, no string splitting.
|
||
func proximityMatch(stream string, terms []string, win int) bool {
|
||
if len(terms) == 0 {
|
||
return false
|
||
}
|
||
last := make([]int, len(terms))
|
||
for i := range last {
|
||
last[i] = -1
|
||
}
|
||
need := len(terms)
|
||
pos := 0
|
||
i := 0
|
||
for i < len(stream) {
|
||
for i < len(stream) && stream[i] == ' ' {
|
||
i++
|
||
}
|
||
if i >= len(stream) {
|
||
break
|
||
}
|
||
j := i
|
||
for j < len(stream) && stream[j] != ' ' {
|
||
j++
|
||
}
|
||
tok := stream[i:j]
|
||
for t, term := range terms {
|
||
if tok != term {
|
||
continue
|
||
}
|
||
if last[t] < 0 {
|
||
need--
|
||
}
|
||
last[t] = pos
|
||
if need == 0 {
|
||
lo, hi := last[0], last[0]
|
||
for _, p := range last[1:] {
|
||
if p < lo {
|
||
lo = p
|
||
}
|
||
if p > hi {
|
||
hi = p
|
||
}
|
||
}
|
||
if hi-lo <= win {
|
||
return true
|
||
}
|
||
}
|
||
}
|
||
pos++
|
||
i = j
|
||
}
|
||
return false
|
||
}
|
||
|
||
// --- Snippet helpers ---
|
||
|
||
// getSnippets returns up to max non-overlapping +-60-char windows around
|
||
// query matches, in document order. The match runs on the normal form
|
||
// (folded + lowered) and each window's bounds are resolved back to the
|
||
// ORIGINAL text (one forward walk, see below), so accents and casing
|
||
// survive into the snippet.
|
||
func getSnippets(content, query string, max int) []string {
|
||
return snippetsIn(content, normFoldString(content), query, max)
|
||
}
|
||
|
||
// bodySnippets is getSnippets over the document's transcript, on the fold
|
||
// of it the loader already holds.
|
||
func (d *Document) bodySnippets(query string, max int) []string {
|
||
return snippetsIn(d.FullText, d.foldedBody(), query, max)
|
||
}
|
||
|
||
// anchorSource yields one needle's matches in a text, in document order and
|
||
// one at a time, so a search for three windows reads no further than them.
|
||
type anchorSource struct {
|
||
next func(from int) (pos, n, resume int) // pos < 0: no match from there on
|
||
from int
|
||
pos, n int // the match not yet taken; pos < 0 once there is none
|
||
}
|
||
|
||
func (a *anchorSource) advance() { a.pos, a.n, a.from = a.next(a.from) }
|
||
|
||
// snippetsIn is getSnippets for a caller that holds the normal form of
|
||
// content (normFoldString).
|
||
func snippetsIn(content, norm, query string, max int) []string {
|
||
if query == "" || content == "" || max <= 0 {
|
||
return nil
|
||
}
|
||
|
||
searchQuery := strings.TrimSpace(query)
|
||
bounded := false // an exact value anchors on whole words only
|
||
if v, ok := exactValue(searchQuery); ok {
|
||
searchQuery, bounded = `"`+v+`"`, true
|
||
}
|
||
isExact := isQuotedQuery(searchQuery)
|
||
cleanQuery := normFoldString(strings.Trim(searchQuery, `"`))
|
||
if cleanQuery == "" {
|
||
return nil
|
||
}
|
||
|
||
var sources []*anchorSource
|
||
add := func(next func(from int) (pos, n, resume int)) {
|
||
src := &anchorSource{next: next}
|
||
src.advance()
|
||
sources = append(sources, src)
|
||
}
|
||
addAll := func(needle string, wordsOnly bool) {
|
||
if needle == "" {
|
||
return
|
||
}
|
||
add(func(from int) (int, int, int) {
|
||
i := -1
|
||
if wordsOnly {
|
||
i = indexWord(norm, needle, from)
|
||
} else if j := strings.Index(norm[from:], needle); j >= 0 {
|
||
i = from + j
|
||
}
|
||
return i, len(needle), i + len(needle)
|
||
})
|
||
}
|
||
found := func() bool {
|
||
for _, src := range sources {
|
||
if src.pos >= 0 {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
if isExact || strings.Contains(cleanQuery, " ") {
|
||
addAll(cleanQuery, !isExact || bounded)
|
||
}
|
||
if !isExact {
|
||
var literal []string // stopwords, single letters, "u.s.": anchors of last resort
|
||
for _, w := range strings.Fields(cleanQuery) {
|
||
if frag, openStart, openEnd, ok := wildcard(w); ok {
|
||
if frag != "" {
|
||
add(func(from int) (int, int, int) {
|
||
i := indexWild(norm, frag, from, openStart, openEnd)
|
||
return i, len(frag), i + len(frag)
|
||
})
|
||
}
|
||
continue
|
||
}
|
||
kept := 0
|
||
for _, p := range splitWords(w) {
|
||
if len(p) <= 1 || stopWords[p] {
|
||
continue
|
||
}
|
||
kept++
|
||
addAll(p, true)
|
||
s := stem(p)
|
||
n := 0
|
||
for n < len(s) && n < len(p) && s[n] == p[n] {
|
||
n++
|
||
}
|
||
if s == p || n < 2 {
|
||
continue
|
||
}
|
||
add(func(from int) (int, int, int) {
|
||
for {
|
||
i := indexWild(norm, p[:n], from, false, true)
|
||
if i < 0 {
|
||
return -1, 0, from
|
||
}
|
||
start, end := wordBounds(norm, i, i+n)
|
||
if word := norm[start:end]; word != p && stem(word) == s {
|
||
return start, end - start, end
|
||
}
|
||
from = end
|
||
}
|
||
})
|
||
}
|
||
if kept == 0 {
|
||
literal = append(literal, w)
|
||
}
|
||
}
|
||
if !found() {
|
||
for _, w := range literal {
|
||
parts := splitWords(w)
|
||
if len(parts) == 1 {
|
||
w = parts[0]
|
||
}
|
||
addAll(w, len(parts) == 1)
|
||
}
|
||
}
|
||
}
|
||
if !found() {
|
||
return nil
|
||
}
|
||
|
||
// First pass. Choose up to max non-overlapping +-60-byte windows, in
|
||
// NORM offsets, taking the anchors in document order.
|
||
type window struct{ start, end int }
|
||
var wins []window
|
||
lastEnd := -1
|
||
for len(wins) < max {
|
||
var first *anchorSource
|
||
for _, src := range sources {
|
||
if src.pos >= 0 && (first == nil || src.pos < first.pos) {
|
||
first = src
|
||
}
|
||
}
|
||
if first == nil {
|
||
break
|
||
}
|
||
pos, n := first.pos, first.n
|
||
first.advance()
|
||
if pos < lastEnd {
|
||
continue // already inside the previous window
|
||
}
|
||
start := pos - 60
|
||
if start < 0 {
|
||
start = 0
|
||
}
|
||
end := pos + n + 60
|
||
if end > len(norm) {
|
||
end = len(norm)
|
||
}
|
||
lastEnd = end
|
||
wins = append(wins, window{start, end})
|
||
}
|
||
|
||
// Second pass. Resolve those few norm offsets back to offsets in the
|
||
// original content with ONE forward walk. A norm offset resolves to the
|
||
// start of the rune whose folded form covers it; the offset just past
|
||
// the end resolves to len(content).
|
||
bounds := make([]int, 0, len(wins)*2)
|
||
for _, wnd := range wins {
|
||
bounds = append(bounds, wnd.start, wnd.end)
|
||
}
|
||
sort.Ints(bounds) // window paddings may overlap, so bounds aren't sorted yet
|
||
orig := make(map[int]int, len(bounds))
|
||
bi, normOff := 0, 0
|
||
for oi, r := range content {
|
||
if bi == len(bounds) {
|
||
break
|
||
}
|
||
l := foldedLen(r)
|
||
for bi < len(bounds) && bounds[bi] < normOff+l {
|
||
orig[bounds[bi]] = oi
|
||
bi++
|
||
}
|
||
normOff += l
|
||
}
|
||
for ; bi < len(bounds); bi++ {
|
||
orig[bounds[bi]] = len(content) // the past-the-end boundary
|
||
}
|
||
|
||
out := make([]string, 0, len(wins))
|
||
for _, wnd := range wins {
|
||
oStart, oEnd := orig[wnd.start], orig[wnd.end]
|
||
prefix := "... "
|
||
if oStart == 0 {
|
||
prefix = ""
|
||
}
|
||
suffix := " ..."
|
||
if oEnd == len(content) {
|
||
suffix = ""
|
||
}
|
||
out = append(out, prefix+content[oStart:oEnd]+suffix)
|
||
}
|
||
return out
|
||
}
|
||
|
||
// --- Search-result highlighting ---
|
||
|
||
// Highlight term kinds
|
||
const (
|
||
hlWord = iota // matches whole words
|
||
hlWild // marks the whole word a wildcard fragment matched in (tap*, *keys*)
|
||
hlPhrase // matches an exact value (=tap) at word boundaries
|
||
hlSub // matches raw substrings (quoted text, and punctuated terms)
|
||
)
|
||
|
||
// hlTerm is one positive query term prepared for markTerms.
|
||
type hlTerm struct {
|
||
kind int
|
||
needle string // in normal form (folded + lowered)
|
||
stemmed string // the needle's stem when word stemming applies
|
||
openStart, openEnd bool // which sides of the fragment had "*"
|
||
}
|
||
|
||
// buildHighlightTerms extracts the positive terms of the clause list in the
|
||
// shape markTerms consumes, mirroring how the matchers read a clause
|
||
// (literalWord, compileWords). A quoted value is one exact phrase, plain
|
||
// words match whole words (stem-unified, like the index), "word*" matches
|
||
// word prefixes, and words with internal punctuation fall back to substring
|
||
// matching. Negated clauses are skipped (their terms are what the results
|
||
// DON'T contain).
|
||
func buildHighlightTerms(queries, conjs []string) []hlTerm {
|
||
var out []hlTerm
|
||
seen := make(map[string]bool)
|
||
add := func(t hlTerm) {
|
||
k := strconv.Itoa(t.kind) + strconv.FormatBool(t.openStart) + strconv.FormatBool(t.openEnd) + "\x00" + t.needle
|
||
if t.needle == "" || seen[k] {
|
||
return
|
||
}
|
||
seen[k] = true
|
||
out = append(out, t)
|
||
}
|
||
|
||
for _, q := range positiveTerms(queries, conjs) {
|
||
if _, phrase, _, ok := parseProximity(q); ok {
|
||
// Proximity clause. The match is word-based, not an exact
|
||
// substring, so highlight its individual words (with the
|
||
// same stemming eligibility bare words get below).
|
||
q = strings.Trim(phrase, `"`)
|
||
}
|
||
if v, ok := exactValue(q); ok {
|
||
add(hlTerm{kind: hlPhrase, needle: normFoldString(v)})
|
||
continue
|
||
}
|
||
q = normFoldString(strings.TrimSpace(q))
|
||
if q == "" {
|
||
continue
|
||
}
|
||
if isQuotedQuery(q) {
|
||
add(hlTerm{kind: hlSub, needle: strings.Trim(q, `"`)})
|
||
continue
|
||
}
|
||
for _, w := range strings.Fields(q) {
|
||
if frag, openStart, openEnd, ok := wildcard(w); ok {
|
||
if frag != "" {
|
||
add(hlTerm{kind: hlWild, needle: frag, openStart: openStart, openEnd: openEnd})
|
||
}
|
||
continue
|
||
}
|
||
parts := splitWords(w)
|
||
if len(parts) != 1 {
|
||
// Internal punctuation. The matchers read these as
|
||
// substrings, so highlight them the same way.
|
||
add(hlTerm{kind: hlSub, needle: w})
|
||
continue
|
||
}
|
||
w = parts[0]
|
||
t := hlTerm{kind: hlWord, needle: w}
|
||
// Stem eligibility mirrors tokenize, so "taps" highlights
|
||
// "tapped" exactly when the index unified them.
|
||
if len(w) > 1 && !stopWords[w] {
|
||
t.stemmed = stem(w)
|
||
}
|
||
add(t)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
// markTerms HTML-escapes s and wraps every occurrence of the given terms in
|
||
// <mark> tags, for the search results page.
|
||
//
|
||
// Matching runs on the normal form (normFold) so casing and accents can't
|
||
// hide a match, and the marked ranges are resolved back to the ORIGINAL
|
||
// text through normFold's position map (e.g. a "muller" query highlights
|
||
// "Müller" as written). Word terms match whole words, or words whose stem
|
||
// equals the term's stem (the same unification the index does); prefix
|
||
// terms match the start of a word; substring terms match anywhere.
|
||
func markTerms(s string, terms []hlTerm) template.HTML {
|
||
if s == "" || len(terms) == 0 {
|
||
return template.HTML(template.HTMLEscapeString(s))
|
||
}
|
||
|
||
norm, posMap := normFold(s)
|
||
|
||
var wordTerms, subTerms []hlTerm
|
||
for _, t := range terms {
|
||
if t.kind != hlWord {
|
||
subTerms = append(subTerms, t)
|
||
} else {
|
||
wordTerms = append(wordTerms, t)
|
||
}
|
||
}
|
||
|
||
type span struct{ start, end int } // byte offsets in norm
|
||
var spans []span
|
||
|
||
if len(wordTerms) > 0 {
|
||
needStem := false
|
||
for _, t := range wordTerms {
|
||
if t.stemmed != "" {
|
||
needStem = true
|
||
break
|
||
}
|
||
}
|
||
ws := -1 // current word start; -1 = between words
|
||
flush := func(we int) {
|
||
if ws < 0 {
|
||
return
|
||
}
|
||
word := norm[ws:we]
|
||
wstem := ""
|
||
if needStem {
|
||
wstem = stem(word)
|
||
}
|
||
for _, t := range wordTerms {
|
||
if word == t.needle || (t.stemmed != "" && wstem == t.stemmed) {
|
||
spans = append(spans, span{ws, we})
|
||
break
|
||
}
|
||
}
|
||
ws = -1
|
||
}
|
||
for i, r := range norm {
|
||
if unicode.IsLetter(r) || unicode.IsDigit(r) {
|
||
if ws < 0 {
|
||
ws = i
|
||
}
|
||
} else {
|
||
flush(i)
|
||
}
|
||
}
|
||
flush(len(norm))
|
||
}
|
||
|
||
for _, t := range subTerms {
|
||
for from := 0; ; {
|
||
i := -1
|
||
switch t.kind {
|
||
case hlPhrase:
|
||
i = indexWord(norm, t.needle, from)
|
||
case hlWild:
|
||
i = indexWild(norm, t.needle, from, t.openStart, t.openEnd)
|
||
default:
|
||
if j := strings.Index(norm[from:], t.needle); j >= 0 {
|
||
i = from + j
|
||
}
|
||
}
|
||
if i < 0 {
|
||
break
|
||
}
|
||
start, end := i, i+len(t.needle)
|
||
from = end
|
||
if t.kind == hlWild {
|
||
start, end = wordBounds(norm, start, end) // mark the whole word
|
||
}
|
||
spans = append(spans, span{start, end})
|
||
}
|
||
}
|
||
|
||
if len(spans) == 0 {
|
||
return template.HTML(template.HTMLEscapeString(s))
|
||
}
|
||
|
||
// Merge overlapping and touching spans so nested or adjacent matches
|
||
// render as one contiguous highlight.
|
||
sort.Slice(spans, func(i, j int) bool { return spans[i].start < spans[j].start })
|
||
merged := spans[:1]
|
||
for _, sp := range spans[1:] {
|
||
if last := &merged[len(merged)-1]; sp.start <= last.end {
|
||
if sp.end > last.end {
|
||
last.end = sp.end
|
||
}
|
||
} else {
|
||
merged = append(merged, sp)
|
||
}
|
||
}
|
||
|
||
// Emit escaped gap / <mark> / escaped match / </mark> runs. Norm
|
||
// offsets resolve to original offsets via posMap; when a span boundary
|
||
// falls inside one rune's multi-byte fold (e.g. 'ẞ' -> "ss"), the whole
|
||
// rune is included.
|
||
var b strings.Builder
|
||
last := 0
|
||
for _, sp := range merged {
|
||
oStart := posMap[sp.start]
|
||
j := sp.end
|
||
for j < len(norm) && posMap[j] == posMap[sp.end-1] {
|
||
j++
|
||
}
|
||
oEnd := posMap[j]
|
||
if oStart < last {
|
||
oStart = last // rune snapping can nudge a span into the previous one
|
||
}
|
||
if oEnd <= oStart {
|
||
continue
|
||
}
|
||
b.WriteString(template.HTMLEscapeString(s[last:oStart]))
|
||
b.WriteString("<mark>")
|
||
b.WriteString(template.HTMLEscapeString(s[oStart:oEnd]))
|
||
b.WriteString("</mark>")
|
||
last = oEnd
|
||
}
|
||
b.WriteString(template.HTMLEscapeString(s[last:]))
|
||
return template.HTML(b.String())
|
||
}
|
||
|
||
// --- Search execution (shared by the HTML page and the JSON API) ---
|
||
|
||
// ScoredDoc is one match, with its relevance score. Doc points into archive.Docs, which is
|
||
// read-only while serving: the match set is walked, filtered and sorted as
|
||
// pointers, and the page a reader sees wraps them in searchResults, which
|
||
// carry the highlighting.
|
||
type ScoredDoc struct {
|
||
Doc *Document
|
||
Score float64
|
||
}
|
||
|
||
// searchResult is one row of the search page: a document and what the
|
||
// query marked in it.
|
||
type searchResult struct {
|
||
Doc *Document
|
||
Title, Desc template.HTML // escaped, query matches in <mark> (markTerms)
|
||
Snippets []template.HTML // up to 3 match-context windows, <mark>-highlighted
|
||
VersionSnippets []VersionSnippet // windows from versions' new lines, labelled with their copy
|
||
MatchesIn []*DocVersion // versions that matched when the main transcript didn't
|
||
}
|
||
|
||
const maxSnippetDocs = 100
|
||
const maxSearchClauses = 24
|
||
const maxSearchWords = 64
|
||
|
||
func searchWords(queries []string) int {
|
||
n := 0
|
||
for _, q := range queries {
|
||
n += len(strings.Fields(q))
|
||
}
|
||
return n
|
||
}
|
||
|
||
// normalizeClauses caps the clause count and pads/truncates fields and
|
||
// conjs to the length of queries.
|
||
func normalizeClauses(queries, fields, conjs []string) ([]string, []string, []string) {
|
||
if len(queries) > maxSearchClauses {
|
||
queries = queries[:maxSearchClauses]
|
||
}
|
||
trimmed := make([]string, len(queries))
|
||
for i, q := range queries {
|
||
trimmed[i] = strings.TrimSpace(q)
|
||
}
|
||
queries = trimmed
|
||
if len(fields) > len(queries) {
|
||
fields = fields[:len(queries)]
|
||
}
|
||
for len(fields) < len(queries) {
|
||
fields = append(fields, "TE")
|
||
}
|
||
if len(conjs) > len(queries) {
|
||
conjs = conjs[:len(queries)]
|
||
}
|
||
for len(conjs) < len(queries) {
|
||
conjs = append(conjs, "and")
|
||
}
|
||
return queries, fields, conjs
|
||
}
|
||
|
||
// searchField describes one searchable field: its wire code (the fqf
|
||
// value), UI label, simple-box operator aliases, and what it searches. This
|
||
// table is the single source of truth.
|
||
type searchField struct {
|
||
Code string
|
||
Label string
|
||
Aliases []string
|
||
Stemmed, Folded func(d *Document) string
|
||
Values func(d *Document) []string
|
||
read func(d *Document) []string
|
||
index int // the field's place in searchFields
|
||
}
|
||
|
||
type docFields struct {
|
||
values [][]string
|
||
keys [][]string
|
||
folded []string
|
||
pubYears []string // pubYearsOf
|
||
}
|
||
|
||
func (co *Corpus) buildDocFields() {
|
||
for i := range co.archive.Docs {
|
||
d := &co.archive.Docs[i]
|
||
d.fields = nil // the readers below work everything out
|
||
f := &docFields{
|
||
values: make([][]string, len(searchFields)),
|
||
keys: make([][]string, len(searchFields)),
|
||
folded: make([]string, len(searchFields)),
|
||
pubYears: pubYearsOf(d),
|
||
}
|
||
for fi := range searchFields {
|
||
sf := &searchFields[fi]
|
||
if sf.read == nil {
|
||
continue
|
||
}
|
||
vals := sf.read(d)
|
||
keys := make([]string, len(vals))
|
||
for vi, v := range vals {
|
||
keys[vi] = valueKey(v)
|
||
}
|
||
f.values[fi], f.keys[fi] = vals, keys
|
||
f.folded[fi] = normFoldString(strings.Join(vals, valueSep))
|
||
}
|
||
d.fields = f
|
||
}
|
||
}
|
||
|
||
// searchFields is in the form dropdown's display order.
|
||
var searchFields = []searchField{
|
||
// SearchText is stored pre-folded, so quoted text and exact words are
|
||
// direct tests against it.
|
||
{Code: "TE", Label: "All fields", Aliases: []string{"text"},
|
||
Stemmed: func(d *Document) string { return d.StemmedText },
|
||
Folded: func(d *Document) string { return d.SearchText }},
|
||
{Code: "TI", Label: "Title", Aliases: []string{"title"},
|
||
Stemmed: func(d *Document) string { return d.StemmedTitle },
|
||
Folded: (*Document).titleFold},
|
||
{Code: "FT", Label: "Content", Aliases: []string{"content"},
|
||
Stemmed: func(d *Document) string { return d.StemmedFull },
|
||
Folded: func(d *Document) string { return d.FoldedFull }},
|
||
{Code: "DE", Label: "Description", Aliases: []string{"description"},
|
||
Stemmed: func(d *Document) string { return d.StemmedDesc },
|
||
Folded: (*Document).descFold},
|
||
{Code: "CR", Label: "Agency", Aliases: []string{"agency"},
|
||
Values: func(d *Document) []string { return single(d.DocMeta.Creator) }},
|
||
{Code: "RE", Label: "Journalist", Aliases: []string{"journalist"},
|
||
Values: journalistsOf},
|
||
{Code: "PU", Label: "Publisher", Aliases: []string{"publisher"},
|
||
Values: publishersOf},
|
||
{Code: "CL", Label: "Classification", Aliases: []string{"classification"},
|
||
Values: func(d *Document) []string { return d.DocMeta.Classification }},
|
||
{Code: "PR", Label: "Program", Aliases: []string{"program"},
|
||
Values: func(d *Document) []string { return d.DocMeta.Programs }},
|
||
{Code: "DI", Label: "Dissemination", Aliases: []string{"dissemination"},
|
||
Values: func(d *Document) []string { return d.DocMeta.Dissemination }},
|
||
{Code: "BA", Label: "Banner", Aliases: []string{"banner"},
|
||
Values: func(d *Document) []string { return single(d.DocMeta.Banner) }},
|
||
{Code: "GE", Label: "Genre", Aliases: []string{"genre"},
|
||
Values: func(d *Document) []string { return single(d.DocMeta.Genre) }},
|
||
{Code: "SE", Label: "Series", Aliases: []string{"series"},
|
||
Values: func(d *Document) []string { return single(d.DocMeta.Series.Name) }},
|
||
{Code: "CO", Label: "Country", Aliases: []string{"country"},
|
||
Values: func(d *Document) []string { return d.DocMeta.Countries }},
|
||
{Code: "PE", Label: "People", Aliases: []string{"people"},
|
||
Values: func(d *Document) []string { return d.DocMeta.People }},
|
||
{Code: "TA", Label: "Tags", Aliases: []string{"tag"},
|
||
Values: func(d *Document) []string { return d.Tags }},
|
||
{Code: "TY", Label: "Type", Aliases: []string{"type"},
|
||
Values: func(d *Document) []string { return single(d.Type) }},
|
||
{Code: "PL", Label: "Portion level",
|
||
Values: func(d *Document) []string { return d.PortionLevels }},
|
||
{Code: "PM", Label: "Portion marking",
|
||
Values: func(d *Document) []string { return d.PortionMarkings }},
|
||
{Code: "PC", Label: "Portion control",
|
||
Values: func(d *Document) []string { return d.PortionControls }},
|
||
}
|
||
|
||
func single(v string) []string {
|
||
if v == "" {
|
||
return nil
|
||
}
|
||
return []string{v}
|
||
}
|
||
|
||
var (
|
||
fieldByCode = map[string]*searchField{}
|
||
simpleOps = map[string]string{}
|
||
)
|
||
|
||
func init() {
|
||
for i := range searchFields {
|
||
f := &searchFields[i]
|
||
f.index = i
|
||
if read := f.Values; read != nil {
|
||
f.read = read
|
||
f.Values = func(d *Document) []string {
|
||
if d.fields != nil {
|
||
return d.fields.values[i]
|
||
}
|
||
return read(d)
|
||
}
|
||
}
|
||
fieldByCode[f.Code] = f
|
||
for _, a := range f.Aliases {
|
||
simpleOps[a] = f.Code
|
||
}
|
||
}
|
||
}
|
||
|
||
type simpleDates struct {
|
||
DocStart, DocEnd, PubStart, PubEnd string
|
||
}
|
||
|
||
// padDateEnd widens a partial date for use as an inclusive range end:
|
||
// 2013 -> 2013-12-31, 2013-06 -> 2013-06-31 (string comparison makes the
|
||
// over-wide day harmless).
|
||
func padDateEnd(v string) string {
|
||
switch len(v) {
|
||
case 4:
|
||
return v + "-12-31"
|
||
case 7:
|
||
return v + "-31"
|
||
}
|
||
return v
|
||
}
|
||
|
||
// parseSimpleQuery turns a one-line query into fielded clauses plus date
|
||
// filters.
|
||
//
|
||
// program:XKEYSCORE country:Germany cable tapping
|
||
//
|
||
// becomes three AND clauses: PR=XKEYSCORE, CO=Germany, TE="cable tapping".
|
||
// Values can be double-quoted (journalist:"Poitras, Laura"); the quotes are
|
||
// KEPT on the clause value, so a quoted value has the same exact-match
|
||
// (substring/phrase) semantics it has when typed into the structured form,
|
||
// and a bare "quoted phrase" becomes its own exact all-fields clause rather
|
||
// than dissolving into the free-text words.
|
||
//
|
||
// A leading '-' negates: -term and -program:X become "and not" clauses
|
||
// (ordered after the positive ones, since the boolean evaluation is
|
||
// left-to-right). date:2013, date:2013-06, date:2013..2015 filter on the
|
||
// document date; published: does the same for publication dates.
|
||
func parseSimpleQuery(s string) (queries, fields, conjs []string, dates simpleDates) {
|
||
var freeText []string
|
||
type clause struct{ q, f string }
|
||
var negs []clause
|
||
|
||
for i, neg, ok := nextTerm(s, 0); ok; i, neg, ok = nextTerm(s, i) {
|
||
// try operator: word characters followed by ':'
|
||
opEnd := i
|
||
for opEnd < len(s) && (isWordByte(s[opEnd])) {
|
||
opEnd++
|
||
}
|
||
field := ""
|
||
dateOp := ""
|
||
valStart := i
|
||
if opEnd > i && opEnd < len(s) && s[opEnd] == ':' {
|
||
name := strings.ToLower(s[i:opEnd])
|
||
if code, ok := simpleOps[name]; ok {
|
||
field = code
|
||
valStart = opEnd + 1
|
||
} else if name == "date" || name == "published" {
|
||
dateOp = name
|
||
valStart = opEnd + 1
|
||
}
|
||
}
|
||
|
||
clauseVal, val, quoted, exact, next := readValue(s, valStart)
|
||
i = next
|
||
if val == "" {
|
||
continue
|
||
}
|
||
|
||
if dateOp != "" {
|
||
// date filters can't be negated; '-' before them is ignored.
|
||
start, end := val, val
|
||
if a, b, ok := strings.Cut(val, ".."); ok {
|
||
start, end = strings.TrimSpace(a), strings.TrimSpace(b)
|
||
}
|
||
end = padDateEnd(end)
|
||
if dateOp == "date" {
|
||
dates.DocStart, dates.DocEnd = start, end
|
||
} else {
|
||
dates.PubStart, dates.PubEnd = start, end
|
||
}
|
||
continue
|
||
}
|
||
|
||
val = clauseVal
|
||
|
||
switch {
|
||
case field != "" && neg:
|
||
negs = append(negs, clause{val, field})
|
||
case field != "":
|
||
queries = append(queries, val)
|
||
fields = append(fields, field)
|
||
case neg:
|
||
negs = append(negs, clause{val, "TE"})
|
||
case quoted || exact:
|
||
queries = append(queries, val)
|
||
fields = append(fields, "TE")
|
||
default:
|
||
freeText = append(freeText, val)
|
||
}
|
||
}
|
||
|
||
if len(freeText) > 0 {
|
||
queries = append(queries, strings.Join(freeText, " "))
|
||
fields = append(fields, "TE")
|
||
}
|
||
if len(queries) == 0 {
|
||
queries = []string{""}
|
||
fields = []string{"TE"}
|
||
}
|
||
conjs = make([]string, len(queries))
|
||
for i := range conjs {
|
||
conjs[i] = "and"
|
||
}
|
||
for _, n := range negs {
|
||
queries = append(queries, n.q)
|
||
fields = append(fields, n.f)
|
||
conjs = append(conjs, "not")
|
||
}
|
||
return queries, fields, conjs, dates
|
||
}
|
||
|
||
func isWordByte(b byte) bool {
|
||
return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_'
|
||
}
|
||
|
||
const spaceBytes = " \t\n\v\f\r"
|
||
|
||
func isSpaceByte(b byte) bool { return strings.IndexByte(spaceBytes, b) >= 0 }
|
||
|
||
func proxSuffix(s string, i int) (string, int) {
|
||
if i >= len(s) || s[i] != '~' {
|
||
return "", i
|
||
}
|
||
j := i + 1
|
||
for j < len(s) && s[j] >= '0' && s[j] <= '9' {
|
||
j++
|
||
}
|
||
if j == i+1 || (j < len(s) && !isSpaceByte(s[j])) {
|
||
return "", i
|
||
}
|
||
return s[i:j], j
|
||
}
|
||
|
||
// readToken reads one search-value token starting at s[i]: either a
|
||
// "quoted span" returned with the quotes stripped and quoted=true, or
|
||
// a bare run up to the next whitespace. next is the index just past the
|
||
// token. This is THE tokenizer for search values: parseSimpleQuery and
|
||
// rowTerms both use it, behind nextTerm, so neither the quoting rule nor
|
||
// the negation rule can drift between them.
|
||
func readToken(s string, i int) (val string, quoted bool, next int) {
|
||
if i < len(s) && s[i] == '"' {
|
||
j := i + 1
|
||
if k := strings.IndexByte(s[j:], '"'); k >= 0 {
|
||
return s[j : j+k], true, j + k + 1
|
||
}
|
||
return s[i+1:], true, len(s)
|
||
}
|
||
j := i
|
||
for j < len(s) && !isSpaceByte(s[j]) {
|
||
j++
|
||
}
|
||
return s[i:j], false, j
|
||
}
|
||
|
||
// nextTerm finds the next term of a query from s[i] on: it skips the
|
||
// whitespace, and a leading '-' negates the term when it is attached to
|
||
// one. i is where the term's value (or its field: operator) starts; ok is
|
||
// false at the end of the string.
|
||
func nextTerm(s string, i int) (start int, neg, ok bool) {
|
||
for i < len(s) && isSpaceByte(s[i]) {
|
||
i++
|
||
}
|
||
if i >= len(s) {
|
||
return i, false, false
|
||
}
|
||
if s[i] == '-' && i+1 < len(s) && !isSpaceByte(s[i+1]) {
|
||
return i + 1, true, true
|
||
}
|
||
return i, false, true
|
||
}
|
||
|
||
// readValue reads one search value at s[i]
|
||
func readValue(s string, i int) (clause, inner string, quoted, exact bool, next int) {
|
||
if i < len(s) && s[i] == '=' {
|
||
exact = true
|
||
i++
|
||
}
|
||
inner, quoted, next = readToken(s, i)
|
||
prox := ""
|
||
if quoted {
|
||
prox, next = proxSuffix(s, next)
|
||
}
|
||
inner = strings.TrimSpace(inner)
|
||
switch {
|
||
case inner == "":
|
||
case exact && quoted:
|
||
clause = `="` + inner + `"`
|
||
case exact:
|
||
clause = "=" + inner
|
||
case quoted:
|
||
clause = `"` + inner + `"` + prox
|
||
default:
|
||
clause = inner
|
||
}
|
||
return clause, inner, quoted, exact, next
|
||
}
|
||
|
||
// rowTerms splits one search row into what it requires and what it
|
||
// excludes, the way parseSimpleQuery splits ?q=: each quoted phrase,
|
||
// proximity clause or exact value is a positive part of its own, the bare
|
||
// words together are one more (last), and google-style -term and -"phrase"
|
||
// tokens are exclusions on the row's own field.
|
||
//
|
||
// This is THE reading of a row: the matcher, the scorer, the snippet anchor
|
||
// and the highlighter all go through it, so "beta -gamma" is (beta AND NOT
|
||
// gamma) as one unit wherever the row sits.
|
||
func rowTerms(s string) (positives, negatives []string) {
|
||
var words []string
|
||
for i, neg, ok := nextTerm(s, 0); ok; i, neg, ok = nextTerm(s, i) {
|
||
tok, _, quoted, exact, next := readValue(s, i)
|
||
i = next
|
||
switch {
|
||
case tok == "" || tok == "-":
|
||
case neg:
|
||
negatives = append(negatives, tok)
|
||
case quoted || exact:
|
||
positives = append(positives, tok)
|
||
default:
|
||
words = append(words, tok)
|
||
}
|
||
}
|
||
if len(words) > 0 {
|
||
positives = append(positives, strings.Join(words, " "))
|
||
}
|
||
return positives, negatives
|
||
}
|
||
|
||
// executeSearch is the search pipeline the page and the API both run: over
|
||
// the whole archive, the collection filter, the dates and the boolean
|
||
// clause list, then the type filter, and the survivors ranked and sorted
|
||
// by p.Sort/p.Order, as a fresh slice the caller owns. The two filters are
|
||
// the browse pages' (inCollections, typeSelected). typeCounts tallies the
|
||
// types BEFORE the type filter. Only the documents that pass every filter
|
||
// are scored.
|
||
func (co *Corpus) executeSearch(p searchParams) (matches []ScoredDoc, typeCounts map[string]int) {
|
||
dateDocEnd := padDateEnd(p.DateDocEnd)
|
||
datePubEnd := padDateEnd(p.DatePubEnd)
|
||
|
||
query := compileQuery(p.Queries, p.Fields, p.Conjs)
|
||
byEntry := entryRows(p.Queries, p.Fields, p.Conjs)
|
||
var scorers []func(*Document) float64
|
||
for _, q := range positiveTerms(p.Queries, p.Conjs) {
|
||
scorers = append(scorers, co.tfidf.scorer(q))
|
||
}
|
||
typeCounts = make(map[string]int)
|
||
for i := range co.archive.Docs {
|
||
doc := &co.archive.Docs[i]
|
||
if !inCollections(doc, p.Coll) ||
|
||
!matchDates(doc, p.DateDocStart, dateDocEnd, p.DatePubStart, datePubEnd, byEntry) ||
|
||
!query.matches(doc) {
|
||
continue
|
||
}
|
||
typeCounts[doc.Type]++
|
||
if !typeSelected(doc, p.Types) {
|
||
continue
|
||
}
|
||
totalScore := 0.0
|
||
for _, score := range scorers {
|
||
totalScore += score(doc)
|
||
}
|
||
matches = append(matches, ScoredDoc{Doc: doc, Score: totalScore})
|
||
}
|
||
|
||
slices.SortStableFunc(matches, func(a, b ScoredDoc) int { return cmp.Compare(b.Score, a.Score) })
|
||
|
||
return sortScored(matches, p.Sort, p.Order), typeCounts
|
||
}
|
||
|
||
// --- Search handler (HTML) ---
|
||
|
||
// --- Faceted search refinement -----------------------------------------------
|
||
|
||
type searchFacet struct {
|
||
Label string
|
||
Values []facetEntry
|
||
}
|
||
|
||
func topFacetCounts(m map[string]int, total, n int, baseURL, field string) []facetEntry {
|
||
var out []facetEntry
|
||
for _, v := range rankCounts(m, -1) {
|
||
if len(out) == n {
|
||
break
|
||
}
|
||
if m[v] >= total {
|
||
continue
|
||
}
|
||
// ="value" -> a whole-value match on the field, the equality the
|
||
// tally counted, so the number is the number the click finds.
|
||
out = append(out, facetEntry{Name: v, Count: m[v], URL: appendClause(baseURL, field, exactClause(v))})
|
||
}
|
||
return out
|
||
}
|
||
|
||
func facetValues(d *Document, code string) []string {
|
||
if f := fieldByCode[code]; f != nil && f.Values != nil {
|
||
return f.Values(d)
|
||
}
|
||
return nil
|
||
}
|
||
|
||
type facetGroup struct {
|
||
Label string
|
||
Code string
|
||
}
|
||
|
||
func facetGroups(docs []*Document, groups []facetGroup, baseURL string, values func(*Document, string) []string) []searchFacet {
|
||
type tally struct{ count, first int }
|
||
tallies := make([]map[string]*tally, len(groups))
|
||
for i := range tallies {
|
||
tallies[i] = map[string]*tally{}
|
||
}
|
||
seq := 0
|
||
for i := range docs {
|
||
for gi, g := range groups {
|
||
for _, v := range values(docs[i], g.Code) {
|
||
t := tallies[gi][v]
|
||
if t == nil {
|
||
t = &tally{first: seq}
|
||
tallies[gi][v] = t
|
||
}
|
||
t.count++
|
||
seq++
|
||
}
|
||
}
|
||
}
|
||
out := make([]searchFacet, 0, len(groups))
|
||
for gi, g := range groups {
|
||
type folded struct {
|
||
name string
|
||
count, first int
|
||
}
|
||
byKey := make(map[string]*folded, len(tallies[gi]))
|
||
for v, t := range tallies[gi] {
|
||
k := valueKey(v)
|
||
f := byKey[k]
|
||
if f == nil {
|
||
byKey[k] = &folded{v, t.count, t.first}
|
||
continue
|
||
}
|
||
f.count += t.count
|
||
if t.first < f.first {
|
||
f.name, f.first = v, t.first
|
||
}
|
||
}
|
||
counts := make(map[string]int, len(byKey))
|
||
for _, f := range byKey {
|
||
counts[f.name] = f.count
|
||
}
|
||
if vals := topFacetCounts(counts, len(docs), 8, baseURL, g.Code); len(vals) > 0 {
|
||
out = append(out, searchFacet{Label: g.Label, Values: vals})
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func facetsWithinWords(facets []searchFacet, room int) []searchFacet {
|
||
out := facets[:0]
|
||
for _, f := range facets {
|
||
vals := f.Values[:0]
|
||
for _, v := range f.Values {
|
||
if searchWords([]string{exactClause(v.Name)}) <= room {
|
||
vals = append(vals, v)
|
||
}
|
||
}
|
||
if len(vals) > 0 {
|
||
f.Values = vals
|
||
out = append(out, f)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
|
||
func computeSearchFacets(docs []*Document, baseURL string, values func(*Document, string) []string) []searchFacet {
|
||
return facetGroups(docs, []facetGroup{
|
||
{"Programs", "PR"}, {"Agencies", "CR"}, {"Countries", "CO"}, {"Journalists", "RE"},
|
||
}, baseURL, values)
|
||
}
|
||
|
||
func collectionFilter(v url.Values) map[string]bool {
|
||
sel := map[string]bool{}
|
||
for _, c := range v["coll"] {
|
||
if validCollections[c] {
|
||
sel[c] = true
|
||
}
|
||
}
|
||
if len(sel) == 0 || len(sel) == len(validCollections) {
|
||
return nil
|
||
}
|
||
return sel
|
||
}
|
||
|
||
func snippetAnchor(queries, fields, conjs []string) (term, field string) {
|
||
for i, q := range queries {
|
||
if q == "" || conjs[i] == "not" {
|
||
continue
|
||
}
|
||
pos, _ := rowTerms(q)
|
||
if len(pos) == 0 {
|
||
continue // only exclusions: nothing in the results to anchor on
|
||
}
|
||
q = pos[0]
|
||
if _, phrase, _, ok := parseProximity(q); ok {
|
||
if ws := strings.Fields(strings.Trim(phrase, `"`)); len(ws) > 0 {
|
||
return ws[0], fields[i]
|
||
}
|
||
}
|
||
return q, fields[i]
|
||
}
|
||
return "", ""
|
||
}
|
||
|
||
// positiveTerms is what the query asks the results to contain: each
|
||
// non-empty row's positive part, skipping NOT-rows and every row's
|
||
// exclusions (rowTerms). The scorer and the highlighter read the query
|
||
// through it, so neither rewards or marks a term the results were selected
|
||
// for NOT containing.
|
||
func positiveTerms(queries, conjs []string) []string {
|
||
var out []string
|
||
for i, q := range queries {
|
||
if q == "" || (i < len(conjs) && conjs[i] == "not") {
|
||
continue
|
||
}
|
||
pos, _ := rowTerms(q)
|
||
out = append(out, pos...)
|
||
}
|
||
return out
|
||
}
|
||
|
||
// searchParams is one search's effective inputs as read by readSearchParams.
|
||
type searchParams struct {
|
||
Queries, Fields, Conjs []string
|
||
DateDocStart, DateDocEnd, DatePubStart, DatePubEnd string
|
||
FromQ bool // a ?q= shortcut replaced the structured params
|
||
Coll map[string]bool // collectionFilter: nil filters nothing
|
||
Types map[string]bool // selectedTypes: empty filters nothing
|
||
Sort, Order string // Sort defaults to "relevance"
|
||
OverClauses int // the requested clause count when it exceeds maxSearchClauses: the search must not run
|
||
OverWords int // the word count (searchWords) when it exceeds maxSearchWords: the search must not run
|
||
}
|
||
|
||
func (p searchParams) refused() bool { return p.OverClauses > 0 || p.OverWords > 0 }
|
||
|
||
// readSearchParams reads a search from URL params. this intake is shared by
|
||
// the HTML page and the JSON API.
|
||
//
|
||
// A non-empty ?q= replaces any structured clause params. Plain words become
|
||
// a single all-fields clause. Field operators like program:XKEYSCORE or
|
||
// journalist:"Poitras, Laura" become their own AND clauses (see
|
||
// parseSimpleQuery). Its parsed dates override the URL's when either end is
|
||
// set - date:..2015 has an empty start but is still a real filter.
|
||
func readSearchParams(v url.Values) searchParams {
|
||
p := searchParams{
|
||
Queries: v["fqv"],
|
||
Fields: v["fqf"],
|
||
Conjs: v["fqc"],
|
||
DateDocStart: v.Get("date_doc_start"),
|
||
DateDocEnd: v.Get("date_doc_end"),
|
||
DatePubStart: v.Get("date_pub_start"),
|
||
DatePubEnd: v.Get("date_pub_end"),
|
||
Coll: collectionFilter(v),
|
||
Sort: sortToken(v.Get("sort")),
|
||
Order: sortToken(v.Get("order")),
|
||
}
|
||
p.Types, _ = selectedTypes(v)
|
||
if docSortByKey[p.Sort] == nil {
|
||
p.Sort, p.Order = "relevance", ""
|
||
}
|
||
if s := v.Get("q"); s != "" {
|
||
var sd simpleDates
|
||
p.Queries, p.Fields, p.Conjs, sd = parseSimpleQuery(s)
|
||
if sd.DocStart != "" || sd.DocEnd != "" {
|
||
p.DateDocStart, p.DateDocEnd = sd.DocStart, sd.DocEnd
|
||
}
|
||
if sd.PubStart != "" || sd.PubEnd != "" {
|
||
p.DatePubStart, p.DatePubEnd = sd.PubStart, sd.PubEnd
|
||
}
|
||
p.FromQ = true
|
||
}
|
||
if n := len(p.Queries); n > maxSearchClauses {
|
||
p.OverClauses = n
|
||
} else if n := searchWords(p.Queries); n > maxSearchWords {
|
||
p.OverWords = n
|
||
}
|
||
p.Queries, p.Fields, p.Conjs = normalizeClauses(p.Queries, p.Fields, p.Conjs)
|
||
return p
|
||
}
|
||
|
||
func (p searchParams) dates() [4]string {
|
||
return [4]string{p.DateDocStart, p.DateDocEnd, p.DatePubStart, p.DatePubEnd}
|
||
}
|
||
|
||
// canonicalSearchQuery rebuilds the query params from the EFFECTIVE state
|
||
// of a search (the post-add/remove clause rows, the collection and type
|
||
// filters, the dates that a ?q= shortcut may have parsed out of the box)
|
||
// for every link the page generates (sorting, pagination, the histogram
|
||
// bars, the refine box, the export and the recent/pinned entries).
|
||
func canonicalSearchQuery(doSearch bool, p searchParams) url.Values {
|
||
q := url.Values{}
|
||
if doSearch {
|
||
// A ?q= entry has no do_search param, so the canonical form adds it
|
||
// explicitly.
|
||
q.Set("do_search", "1")
|
||
}
|
||
for i := range p.Queries {
|
||
q.Add("fqv", p.Queries[i])
|
||
q.Add("fqf", p.Fields[i])
|
||
q.Add("fqc", p.Conjs[i])
|
||
}
|
||
if colls := collectionParams(p.Coll); len(colls) > 0 {
|
||
q["coll"] = colls
|
||
}
|
||
if len(p.Types) > 0 {
|
||
types := make([]string, 0, len(p.Types))
|
||
for t := range p.Types {
|
||
types = append(types, t)
|
||
}
|
||
sort.Strings(types)
|
||
q["type"] = types
|
||
}
|
||
dates := p.dates()
|
||
for i, k := range []string{"date_doc_start", "date_doc_end", "date_pub_start", "date_pub_end"} {
|
||
if dates[i] != "" {
|
||
q.Set(k, dates[i])
|
||
}
|
||
}
|
||
return q
|
||
}
|
||
|
||
// searchDateHistogram is the year-bar strip under a result set.
|
||
func searchDateHistogram(results []*Document, q url.Values) []timelineBar {
|
||
counts := make(map[string]int)
|
||
undated := 0
|
||
for i := range results {
|
||
if y := docYear(results[i]); y != "" {
|
||
counts[y]++
|
||
} else {
|
||
undated++
|
||
}
|
||
}
|
||
_, maxCount := sortedYears(counts)
|
||
if undated > maxCount {
|
||
maxCount = undated // the undated bar shares the scale
|
||
}
|
||
start, end := q.Get("date_doc_start"), q.Get("date_doc_end")
|
||
bars := yearBars(counts, func(y string) string {
|
||
from, to := y, y
|
||
if start > y {
|
||
from = start
|
||
}
|
||
if end != "" && padDateEnd(end) < padDateEnd(y) {
|
||
to = end
|
||
}
|
||
yq := cloneValues(q)
|
||
yq.Set("date_doc_start", from)
|
||
yq.Set("date_doc_end", to)
|
||
return "/search?" + yq.Encode()
|
||
})
|
||
for i := range bars {
|
||
bars[i].Pct = barPct(bars[i].Count, maxCount, 1)
|
||
}
|
||
if undated > 0 {
|
||
bars = append(bars, timelineBar{
|
||
Year: "-",
|
||
Count: undated,
|
||
Pct: barPct(undated, maxCount, 1),
|
||
})
|
||
}
|
||
return bars
|
||
}
|
||
|
||
// decorateResults marks the query's terms in the titles, descriptions and
|
||
// body snippets of the results on the page.
|
||
func decorateResults(docs []*Document, queries, fields, conjs []string) (results []searchResult, snippetsCapped bool) {
|
||
snippetTerm, snippetField := snippetAnchor(queries, fields, conjs)
|
||
hl := buildHighlightTerms(queries, conjs)
|
||
results = make([]searchResult, len(docs))
|
||
for i, doc := range docs {
|
||
res := &results[i]
|
||
res.Doc = doc
|
||
res.Title = markTerms(doc.Title, hl)
|
||
res.Desc = markTerms(doc.Description, hl)
|
||
if snippetTerm == "" || i >= maxSnippetDocs {
|
||
continue
|
||
}
|
||
var raw []string
|
||
var hits []versionHit
|
||
mainHit := false
|
||
if snippetField == "TA" {
|
||
raw = getSnippets(strings.Join(doc.Tags, ", "), snippetTerm, 3)
|
||
} else {
|
||
raw = doc.bodySnippets(snippetTerm, 3)
|
||
mainHit = len(raw) > 0
|
||
if len(raw) < 3 {
|
||
hits = versionHits(doc, snippetTerm, 3)
|
||
}
|
||
if !mainHit && len(hits) == 0 {
|
||
raw = getSnippets(doc.Description, snippetTerm, 1)
|
||
}
|
||
}
|
||
for _, s := range raw {
|
||
res.Snippets = append(res.Snippets, markTerms(s, hl))
|
||
}
|
||
room := 3 - len(raw)
|
||
for _, h := range hits {
|
||
if !mainHit {
|
||
res.MatchesIn = append(res.MatchesIn, h.Version)
|
||
}
|
||
for _, s := range h.Snippets {
|
||
if room <= 0 {
|
||
break
|
||
}
|
||
res.VersionSnippets = append(res.VersionSnippets,
|
||
VersionSnippet{Slug: h.Version.Slug, Label: h.Version.Label, HTML: markTerms(s, hl)})
|
||
room--
|
||
}
|
||
}
|
||
}
|
||
return results, snippetTerm != "" && len(docs) > maxSnippetDocs
|
||
}
|
||
|
||
func (co *Corpus) handleSearch(w http.ResponseWriter, r *http.Request) {
|
||
params := r.URL.Query()
|
||
action := params.Get("action")
|
||
sp := readSearchParams(params)
|
||
prefs := readPrefs(r)
|
||
scope := pageScope(params, prefs, true)
|
||
sp.Coll = scope.filter
|
||
queries, fields, conjs := sp.Queries, sp.Fields, sp.Conjs
|
||
sortBy, order := sp.Sort, sp.Order
|
||
|
||
doSearch := (params.Get("do_search") == "1" || sp.FromQ) && !sp.refused()
|
||
|
||
if len(queries) == 0 {
|
||
queries = []string{""}
|
||
fields = []string{"TE"}
|
||
conjs = []string{"and"}
|
||
}
|
||
|
||
switch {
|
||
case strings.HasPrefix(action, "remove-"):
|
||
idxStr := strings.TrimPrefix(action, "remove-")
|
||
removeIdx, err := strconv.Atoi(idxStr)
|
||
if err == nil && removeIdx > 0 && removeIdx < len(queries) {
|
||
queries = append(queries[:removeIdx], queries[removeIdx+1:]...)
|
||
fields = append(fields[:removeIdx], fields[removeIdx+1:]...)
|
||
conjs = append(conjs[:removeIdx], conjs[removeIdx+1:]...)
|
||
}
|
||
case action == "add" && len(queries) < maxSearchClauses:
|
||
queries = append(queries, "")
|
||
fields = append(fields, "TE")
|
||
conjs = append(conjs, "and")
|
||
}
|
||
|
||
sp.Queries, sp.Fields, sp.Conjs = queries, fields, conjs
|
||
|
||
var scored []ScoredDoc
|
||
searchTypeCounts := map[string]int{}
|
||
if doSearch {
|
||
scored, searchTypeCounts = co.executeSearch(sp)
|
||
}
|
||
|
||
selColl := scope.shown
|
||
collFiltered := scope.explicit()
|
||
selTypeSet := sp.Types
|
||
|
||
results := make([]*Document, 0, len(scored))
|
||
for _, s := range scored {
|
||
results = append(results, s.Doc)
|
||
}
|
||
|
||
q := canonicalSearchQuery(doSearch, sp)
|
||
q["coll"] = scope.colls
|
||
canonicalQS := q.Encode()
|
||
searchBaseURL := "/search?" + canonicalQS + "&"
|
||
|
||
formParams := cloneValues(q)
|
||
if sortBy != "relevance" {
|
||
formParams.Set("sort", sortBy)
|
||
}
|
||
if order != "" {
|
||
formParams.Set("order", order)
|
||
}
|
||
|
||
page, _ := strconv.Atoi(params.Get("p"))
|
||
start, end, page, totalPages := paginate(len(results), page, prefs.cardsPerPage())
|
||
pageResults, snippetsCapped := decorateResults(results[start:end], queries, fields, conjs)
|
||
|
||
labelQ := q
|
||
if scope.defaulted {
|
||
labelQ = cloneValues(q)
|
||
delete(labelQ, "coll")
|
||
}
|
||
queryLabel, named := "", false // the page title's name for the search
|
||
if doSearch {
|
||
queryLabel, named = co.searchLabel(labelQ)
|
||
}
|
||
savable := named && len(canonicalQS) <= searchQSMax
|
||
var recorded []string
|
||
if prefs.TrackSearches && savable &&
|
||
(action == "" || action == "search") && !crossSiteFetch(r) {
|
||
recorded = recordRecentSearch(w, r, canonicalQS)
|
||
}
|
||
var recentSearches []recentSearch
|
||
if prefs.TrackSearches {
|
||
if recorded != nil {
|
||
recentSearches = co.parseSearchEntries(recorded)
|
||
} else {
|
||
recentSearches = co.readRecentSearches(r)
|
||
}
|
||
}
|
||
|
||
searchPinned := savable && isPinnedSearch(r, canonicalQS)
|
||
|
||
var dateHist []timelineBar
|
||
if doSearch && len(results) > 0 {
|
||
dateHist = searchDateHistogram(results, q)
|
||
}
|
||
|
||
var facets []searchFacet
|
||
if doSearch && len(results) >= 2 && len(queries) < maxSearchClauses {
|
||
values := facetValues
|
||
if !hasOrRow(queries, conjs) {
|
||
values = refineValues(sp.DatePubStart, sp.DatePubEnd)
|
||
}
|
||
facets = facetsWithinWords(computeSearchFacets(results, searchBaseURL, values),
|
||
maxSearchWords-searchWords(queries))
|
||
}
|
||
|
||
if len(recentSearches) > 0 {
|
||
noStore(w)
|
||
}
|
||
render(w, r, &searchPage{
|
||
FieldOptions: searchFields,
|
||
Facets: facets,
|
||
Queries: queries,
|
||
Fields: fields,
|
||
Conjs: conjs,
|
||
Results: pageResults,
|
||
SearchTypeCounts: searchTypeCounts,
|
||
SelTypeSet: selTypeSet,
|
||
FormParams: formParams,
|
||
Savable: savable,
|
||
ShowThumbs: prefs.thumbsOn(),
|
||
FOIAQS: prefs.foiaLinkQS(),
|
||
Base: pageWith(prefs),
|
||
Total: len(results),
|
||
Page: page,
|
||
TotalPages: totalPages,
|
||
RecentSearches: recentSearches,
|
||
SelColl: selColl,
|
||
CollFiltered: collFiltered,
|
||
CanonicalQS: canonicalQS,
|
||
SearchPinned: searchPinned,
|
||
Searched: doSearch,
|
||
QueryLabel: queryLabel,
|
||
DateDocStart: sp.DateDocStart,
|
||
DateDocEnd: sp.DateDocEnd,
|
||
DatePubStart: sp.DatePubStart,
|
||
DatePubEnd: sp.DatePubEnd,
|
||
Sort: sortBy,
|
||
Order: order,
|
||
SearchBaseURL: searchBaseURL,
|
||
DateHist: dateHist,
|
||
SnippetsCapped: snippetsCapped,
|
||
SnippetLimit: maxSnippetDocs,
|
||
OverClauses: sp.OverClauses,
|
||
ClauseLimit: maxSearchClauses,
|
||
OverWords: sp.OverWords,
|
||
WordLimit: maxSearchWords,
|
||
})
|
||
}
|
||
|
||
// --- Date helpers ---
|
||
|
||
func inDateRange(date, start, end string) bool {
|
||
if date == "" {
|
||
return start == "" && end == ""
|
||
}
|
||
if start != "" && date < start {
|
||
return false
|
||
}
|
||
if end != "" && date > end {
|
||
return false
|
||
}
|
||
return true
|
||
}
|
||
|
||
func matchDates(doc *Document, dateDocStart, dateDocEnd, datePubStart, datePubEnd string, byEntry []entryRow) bool {
|
||
if dateDocStart != "" || dateDocEnd != "" {
|
||
if !inDateRange(doc.DocMeta.DateCreated, dateDocStart, dateDocEnd) {
|
||
return false
|
||
}
|
||
}
|
||
if datePubStart == "" && datePubEnd == "" {
|
||
return true
|
||
}
|
||
inRange := func(i int) bool {
|
||
return inDateRange(doc.Publication[i].DatePublished, datePubStart, datePubEnd)
|
||
}
|
||
if len(byEntry) == 0 {
|
||
for i := range doc.Publication {
|
||
if inRange(i) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
for _, r := range byEntry {
|
||
found := false
|
||
for i := range doc.Publication {
|
||
if inRange(i) && r.matches(doc, i) {
|
||
found = true
|
||
break
|
||
}
|
||
}
|
||
if !found {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// entryRow is a journalist or publisher row a publication date is tied
|
||
// to, parsed once per search.
|
||
type entryRow struct {
|
||
field string // "RE" or "PU"
|
||
pos, negs []valuesTest
|
||
}
|
||
|
||
func entryRows(queries, fields, conjs []string) []entryRow {
|
||
if hasOrRow(queries, conjs) {
|
||
return nil
|
||
}
|
||
var rows []entryRow
|
||
for i, q := range queries {
|
||
if q == "" {
|
||
continue
|
||
}
|
||
if conjs[i] != "not" && (fields[i] == "RE" || fields[i] == "PU") {
|
||
pos, negs := rowTerms(q)
|
||
rows = append(rows, entryRow{fields[i], entryTests(pos), entryTests(negs)})
|
||
}
|
||
}
|
||
return rows
|
||
}
|
||
|
||
func hasOrRow(queries, conjs []string) bool {
|
||
for i, q := range queries {
|
||
if q != "" && conjs[i] == "or" {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
func entryTests(terms []string) []valuesTest {
|
||
tests := make([]valuesTest, len(terms))
|
||
for i, t := range terms {
|
||
if _, phrase, _, ok := parseProximity(t); ok {
|
||
t = phrase
|
||
}
|
||
tests[i] = compileValues(t)
|
||
}
|
||
return tests
|
||
}
|
||
|
||
func entryValues(doc *Document, i int, field string) []string {
|
||
p := &doc.Publication[i]
|
||
switch {
|
||
case field == "RE":
|
||
return p.Journalists
|
||
case !p.ReportingOnly:
|
||
return []string{p.Publisher}
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func refineValues(pubStart, pubEnd string) func(*Document, string) []string {
|
||
if pubStart == "" && pubEnd == "" {
|
||
return facetValues
|
||
}
|
||
pubEnd = padDateEnd(pubEnd)
|
||
return func(d *Document, code string) []string {
|
||
if code != "RE" && code != "PU" {
|
||
return facetValues(d, code)
|
||
}
|
||
var vals []string
|
||
for i := range d.Publication {
|
||
if inDateRange(d.Publication[i].DatePublished, pubStart, pubEnd) {
|
||
vals = append(vals, entryValues(d, i, code)...)
|
||
}
|
||
}
|
||
return normalizeList(vals)
|
||
}
|
||
}
|
||
|
||
// matches reports whether publication entry i matches the row on its own
|
||
// journalists or publisher (entryValues).
|
||
func (r entryRow) matches(doc *Document, i int) bool {
|
||
vals := entryValues(doc, i, r.field)
|
||
for t := range r.pos {
|
||
if !r.pos[t].match(vals) {
|
||
return false
|
||
}
|
||
}
|
||
for t := range r.negs {
|
||
if r.negs[t].match(vals) {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// --- The boolean clause list ---
|
||
|
||
// compiledQuery is a search's rows read once. The non-empty ones, in order,
|
||
// each with the tests of its positive values and of its exclusions on its
|
||
// field.
|
||
type compiledQuery []compiledRow
|
||
|
||
type compiledRow struct {
|
||
conj string
|
||
pos, negs []func(*Document) bool
|
||
}
|
||
|
||
func compileQuery(queries, fields, conjs []string) compiledQuery {
|
||
var rows compiledQuery
|
||
for i, q := range queries {
|
||
if q == "" {
|
||
continue
|
||
}
|
||
f := fieldByCode[fields[i]]
|
||
if f == nil {
|
||
f = fieldByCode["TE"]
|
||
}
|
||
row := compiledRow{conj: conjs[i]}
|
||
pos, negs := rowTerms(q)
|
||
for _, v := range pos {
|
||
row.pos = append(row.pos, f.compile(v))
|
||
}
|
||
for _, v := range negs {
|
||
row.negs = append(row.negs, f.compile(v))
|
||
}
|
||
rows = append(rows, row)
|
||
}
|
||
return rows
|
||
}
|
||
|
||
// matches evaluates the rows against a document, left to right, each row
|
||
// as one unit.
|
||
func (rows compiledQuery) matches(d *Document) bool {
|
||
overall := true
|
||
for i := range rows {
|
||
row := &rows[i]
|
||
if i == 0 {
|
||
overall = row.matches(d) != (row.conj == "not")
|
||
continue
|
||
}
|
||
switch row.conj {
|
||
case "or":
|
||
overall = overall || row.matches(d)
|
||
case "not":
|
||
overall = overall && !row.matches(d)
|
||
default: // "and"
|
||
overall = overall && row.matches(d)
|
||
}
|
||
}
|
||
return overall
|
||
}
|
||
|
||
// matches reports whether a document matches one row.
|
||
func (row *compiledRow) matches(d *Document) bool {
|
||
for _, test := range row.pos {
|
||
if !test(d) {
|
||
return false
|
||
}
|
||
}
|
||
for _, test := range row.negs {
|
||
if test(d) {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// versionHit is one of a document's versions whose new lines match a query,
|
||
// with snippet windows from those lines.
|
||
type versionHit struct {
|
||
Version *DocVersion
|
||
Snippets []string
|
||
}
|
||
|
||
// versionHits returns the versions of doc whose new lines (NovelText) match
|
||
// term in version order.
|
||
func versionHits(doc *Document, term string, max int) []versionHit {
|
||
var out []versionHit
|
||
for i := range doc.Versions {
|
||
v := &doc.Versions[i]
|
||
if s := getSnippets(v.NovelText, term, max); len(s) > 0 {
|
||
out = append(out, versionHit{Version: v, Snippets: s})
|
||
}
|
||
}
|
||
return out
|
||
}
|