snowchive-server/codewords.go
2026-11-19 04:40:59 +02:00

298 lines
7.3 KiB
Go

package main
import (
"html/template"
"net/http"
"net/url"
"sort"
"strings"
)
// The codeword concordance: an automatically extracted index of ALL-CAPS
// tokens across the corpus - program names, compartments, org designators.
var codewordStoplist = map[string]bool{
"REDACTED": true,
"DYNAMIC": true,
"PAGE": true,
"HIGHEST": true,
"POSSIBLE": true,
"DERIVED": true,
"DECLASSIFY": true,
"JAN": true,
"ADDED": true,
"CLASSIFICATION": true,
}
type codewordEntry struct {
Token string
Count int // total occurrences across the corpus
Docs int // number of documents containing it
First string // earliest document year containing it ("" if only undated docs)
Last string // latest such year
}
func isCodewordToken(w string) bool {
if len(w) < 3 || w[0] < 'A' || w[0] > 'Z' {
return false
}
letters := 0
var seen [26]bool
distinct := 0
for i := 0; i < len(w); i++ {
c := w[i]
switch {
case c >= 'A' && c <= 'Z':
letters++
if !seen[c-'A'] {
seen[c-'A'] = true
distinct++
}
case c >= '0' && c <= '9':
// digits allowed
default:
return false
}
}
return letters >= 3 && distinct >= 2
}
func scanRuns(text string, mention func(w string)) {
start := -1
for j := 0; j <= len(text); j++ {
alnum := false
if j < len(text) {
ch := text[j]
alnum = (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9')
}
if alnum {
if start < 0 {
start = j
}
continue
}
if start >= 0 {
mention(text[start:j])
start = -1
}
}
}
func (co *Corpus) buildCodewords() {
co.codewords, co.codewordTable = nil, nil
caps := map[string]int{}
other := map[string]int{}
docCaps := make([][]string, len(co.archive.Docs))
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
here := map[string]bool{}
count := func(w string) {
switch {
case codewordStoplist[w]:
// boilerplate or archive markup (codewordStoplist)
case isCodewordToken(w):
caps[w]++
here[w] = true
default:
other[strings.ToLower(w)]++
}
}
scanRuns(d.Title, count)
scanRuns(d.FullText, count)
for w := range here {
docCaps[i] = append(docCaps[i], w)
}
}
for w, cnt := range caps {
if cnt < 2 || cnt <= other[strings.ToLower(w)] {
continue
}
co.codewords = append(co.codewords, codewordEntry{Token: w, Count: cnt})
}
kept := make(map[string]*codewordEntry, len(co.codewords))
for i := range co.codewords {
kept[co.codewords[i].Token] = &co.codewords[i]
}
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
year := docYear(d)
inDoc := map[string]bool{}
exact := func(w string) {
if e := kept[w]; e != nil {
inDoc[e.Token] = true
}
}
folded := func(w string) {
if e := kept[strings.ToUpper(w)]; e != nil {
inDoc[e.Token] = true
}
}
for _, w := range docCaps[i] {
exact(w)
}
scanRuns(d.Description, exact)
scanRuns(d.LongDescription, exact)
for _, p := range d.DocMeta.Programs {
scanRuns(p, folded)
}
for _, t := range d.Tags {
scanRuns(t, folded)
}
for w := range inDoc {
e := kept[w]
e.Docs++
if year == "" {
continue
}
if e.First == "" || year < e.First {
e.First = year
}
if year > e.Last {
e.Last = year
}
}
}
sort.Slice(co.codewords, func(i, j int) bool {
if co.codewords[i].Count != co.codewords[j].Count {
return co.codewords[i].Count > co.codewords[j].Count
}
return co.codewords[i].Token < co.codewords[j].Token
})
lowerProg := make(map[string]string, len(co.archive.Facets["program"]))
for p := range co.archive.Facets["program"] {
lowerProg[valueKey(p)] = p
}
shown := co.codewords
if len(shown) > codewordRows {
shown = shown[:codewordRows]
}
co.codewordTable = make([]codewordRow, 0, len(shown))
for _, e := range shown {
row := codewordRow{codewordEntry: e, Gloss: co.glossaryLookup(e.Token), SearchURL: tokenSearchURL(e.Token)}
if p, ok := lowerProg[valueKey(e.Token)]; ok {
row.ProgURL = wholeArchive.link(categoryValueURL("program", p))
}
if e.First != "" {
row.FirstURL = yearTokenURL(e.Token, e.First)
}
if e.Last != "" {
row.LastURL = yearTokenURL(e.Token, e.Last)
}
if row.Gloss != nil {
row.GlossURL = template.URL("/glossary#term-" + url.PathEscape(row.Gloss.Term))
}
co.codewordTable = append(co.codewordTable, row)
}
co.buildCodewordViews()
}
func yearTokenURL(token, year string) template.URL {
return template.URL(wholeArchive.link(dateRangeSearchURL("TE", exactClause(token), [2]string{"date_doc", year})))
}
func tokenSearchURL(token string) template.URL {
return template.URL(wholeArchive.link("/search?q=" + url.QueryEscape(exactClause(token))))
}
type codewordRow struct {
codewordEntry
Gloss *GlossaryEntry
ProgURL string
SearchURL template.URL
FirstURL, LastURL template.URL
GlossURL template.URL
}
// codewordRows caps the page
const codewordRows = 500
var codewordCols = []sortCol{
{Key: "token", Label: "Token"},
{Key: "count", Label: "Occurrences", DescFirst: true},
{Key: "docs", Label: "Documents", DescFirst: true},
{Key: "first", Label: "First seen"},
{Key: "last", Label: "Last seen", DescFirst: true},
{Key: "gloss", Label: "Glossary"},
}
func (co *Corpus) handleCodewords(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
sortBy := sortToken(q.Get("sort"))
order := sortToken(q.Get("order"))
view, ok := co.codewordViews[sortBy+"/"+order]
if !ok {
view, ok = co.codewordViews[sortBy+"/"+codewordDefaultOrder(sortBy)]
}
if !ok {
view = co.codewordViews["count/desc"]
}
render(w, r, &codewordsPage{
Rows: view.Rows,
RowsHTML: co.preHTML("codewords:" + view.Key),
Total: len(co.codewords),
Shown: len(view.Rows),
Heads: view.Heads,
})
}
type codewordView struct {
Key string
Rows []codewordRow
Heads []sortHead
}
func codewordDefaultOrder(sortBy string) string {
for _, col := range codewordCols {
if col.Key == sortBy {
return orderParam(col.DescFirst)
}
}
return ""
}
func (co *Corpus) buildCodewordViews() {
co.codewordViews = make(map[string]codewordView, 2*len(codewordCols))
for _, sc := range codewordCols {
for _, order := range []string{"asc", "desc"} {
rows := append([]codewordRow(nil), co.codewordTable...)
sortCodewordRows(rows, sc.Key, order == "desc")
heads := sortHeads("?", codewordCols, sc.Key, order == "desc")
co.codewordViews[sc.Key+"/"+order] = codewordView{sc.Key + "/" + order, rows, heads}
}
}
}
func sortCodewordRows(rows []codewordRow, sortBy string, desc bool) {
sort.SliceStable(rows, func(i, j int) bool {
var less, eq bool
switch sortBy {
case "token":
less, eq = rows[i].Token < rows[j].Token, rows[i].Token == rows[j].Token
case "docs":
less, eq = rows[i].Docs < rows[j].Docs, rows[i].Docs == rows[j].Docs
case "first", "last":
a, b := rows[i].First, rows[j].First
if sortBy == "last" {
a, b = rows[i].Last, rows[j].Last
}
if (a == "") != (b == "") {
return b == ""
}
less, eq = a < b, a == b
case "gloss":
a, b := rows[i].Gloss != nil, rows[j].Gloss != nil
less, eq = !a && b, a == b // missing before defined, ascending
default:
less, eq = rows[i].Count < rows[j].Count, rows[i].Count == rows[j].Count
}
if eq {
return rows[i].Token < rows[j].Token
}
if desc {
return !less
}
return less
})
}