package main import ( "html/template" "net/http" "net/url" "sort" "strings" ) // The codeword concordance: an automatically extracted index of ALL-CAPS // tokens across the corpus - program names, compartments, org designators. var codewordStoplist = map[string]bool{ "REDACTED": true, "DYNAMIC": true, "PAGE": true, "HIGHEST": true, "POSSIBLE": true, "DERIVED": true, "DECLASSIFY": true, "JAN": true, "ADDED": true, "CLASSIFICATION": true, } type codewordEntry struct { Token string Count int // total occurrences across the corpus Docs int // number of documents containing it First string // earliest document year containing it ("" if only undated docs) Last string // latest such year } func isCodewordToken(w string) bool { if len(w) < 3 || w[0] < 'A' || w[0] > 'Z' { return false } letters := 0 var seen [26]bool distinct := 0 for i := 0; i < len(w); i++ { c := w[i] switch { case c >= 'A' && c <= 'Z': letters++ if !seen[c-'A'] { seen[c-'A'] = true distinct++ } case c >= '0' && c <= '9': // digits allowed default: return false } } return letters >= 3 && distinct >= 2 } func scanRuns(text string, mention func(w string)) { start := -1 for j := 0; j <= len(text); j++ { alnum := false if j < len(text) { ch := text[j] alnum = (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9') } if alnum { if start < 0 { start = j } continue } if start >= 0 { mention(text[start:j]) start = -1 } } } func (co *Corpus) buildCodewords() { co.codewords, co.codewordTable = nil, nil caps := map[string]int{} other := map[string]int{} docCaps := make([][]string, len(co.archive.Docs)) for i := range co.archive.Docs { d := &co.archive.Docs[i] here := map[string]bool{} count := func(w string) { switch { case codewordStoplist[w]: // boilerplate or archive markup (codewordStoplist) case isCodewordToken(w): caps[w]++ here[w] = true default: other[strings.ToLower(w)]++ } } scanRuns(d.Title, count) scanRuns(d.FullText, count) for w := range here { docCaps[i] = append(docCaps[i], w) } } for w, cnt := range caps { if cnt < 2 || cnt <= other[strings.ToLower(w)] { continue } co.codewords = append(co.codewords, codewordEntry{Token: w, Count: cnt}) } kept := make(map[string]*codewordEntry, len(co.codewords)) for i := range co.codewords { kept[co.codewords[i].Token] = &co.codewords[i] } for i := range co.archive.Docs { d := &co.archive.Docs[i] year := docYear(d) inDoc := map[string]bool{} exact := func(w string) { if e := kept[w]; e != nil { inDoc[e.Token] = true } } folded := func(w string) { if e := kept[strings.ToUpper(w)]; e != nil { inDoc[e.Token] = true } } for _, w := range docCaps[i] { exact(w) } scanRuns(d.Description, exact) scanRuns(d.LongDescription, exact) for _, p := range d.DocMeta.Programs { scanRuns(p, folded) } for _, t := range d.Tags { scanRuns(t, folded) } for w := range inDoc { e := kept[w] e.Docs++ if year == "" { continue } if e.First == "" || year < e.First { e.First = year } if year > e.Last { e.Last = year } } } sort.Slice(co.codewords, func(i, j int) bool { if co.codewords[i].Count != co.codewords[j].Count { return co.codewords[i].Count > co.codewords[j].Count } return co.codewords[i].Token < co.codewords[j].Token }) lowerProg := make(map[string]string, len(co.archive.Facets["program"])) for p := range co.archive.Facets["program"] { lowerProg[valueKey(p)] = p } shown := co.codewords if len(shown) > codewordRows { shown = shown[:codewordRows] } co.codewordTable = make([]codewordRow, 0, len(shown)) for _, e := range shown { row := codewordRow{codewordEntry: e, Gloss: co.glossaryLookup(e.Token), SearchURL: tokenSearchURL(e.Token)} if p, ok := lowerProg[valueKey(e.Token)]; ok { row.ProgURL = wholeArchive.link(categoryValueURL("program", p)) } if e.First != "" { row.FirstURL = yearTokenURL(e.Token, e.First) } if e.Last != "" { row.LastURL = yearTokenURL(e.Token, e.Last) } if row.Gloss != nil { row.GlossURL = template.URL("/glossary#term-" + url.PathEscape(row.Gloss.Term)) } co.codewordTable = append(co.codewordTable, row) } co.buildCodewordViews() } func yearTokenURL(token, year string) template.URL { return template.URL(wholeArchive.link(dateRangeSearchURL("TE", exactClause(token), [2]string{"date_doc", year}))) } func tokenSearchURL(token string) template.URL { return template.URL(wholeArchive.link("/search?q=" + url.QueryEscape(exactClause(token)))) } type codewordRow struct { codewordEntry Gloss *GlossaryEntry ProgURL string SearchURL template.URL FirstURL, LastURL template.URL GlossURL template.URL } // codewordRows caps the page const codewordRows = 500 var codewordCols = []sortCol{ {Key: "token", Label: "Token"}, {Key: "count", Label: "Occurrences", DescFirst: true}, {Key: "docs", Label: "Documents", DescFirst: true}, {Key: "first", Label: "First seen"}, {Key: "last", Label: "Last seen", DescFirst: true}, {Key: "gloss", Label: "Glossary"}, } func (co *Corpus) handleCodewords(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() sortBy := sortToken(q.Get("sort")) order := sortToken(q.Get("order")) view, ok := co.codewordViews[sortBy+"/"+order] if !ok { view, ok = co.codewordViews[sortBy+"/"+codewordDefaultOrder(sortBy)] } if !ok { view = co.codewordViews["count/desc"] } render(w, r, &codewordsPage{ Rows: view.Rows, RowsHTML: co.preHTML("codewords:" + view.Key), Total: len(co.codewords), Shown: len(view.Rows), Heads: view.Heads, }) } type codewordView struct { Key string Rows []codewordRow Heads []sortHead } func codewordDefaultOrder(sortBy string) string { for _, col := range codewordCols { if col.Key == sortBy { return orderParam(col.DescFirst) } } return "" } func (co *Corpus) buildCodewordViews() { co.codewordViews = make(map[string]codewordView, 2*len(codewordCols)) for _, sc := range codewordCols { for _, order := range []string{"asc", "desc"} { rows := append([]codewordRow(nil), co.codewordTable...) sortCodewordRows(rows, sc.Key, order == "desc") heads := sortHeads("?", codewordCols, sc.Key, order == "desc") co.codewordViews[sc.Key+"/"+order] = codewordView{sc.Key + "/" + order, rows, heads} } } } func sortCodewordRows(rows []codewordRow, sortBy string, desc bool) { sort.SliceStable(rows, func(i, j int) bool { var less, eq bool switch sortBy { case "token": less, eq = rows[i].Token < rows[j].Token, rows[i].Token == rows[j].Token case "docs": less, eq = rows[i].Docs < rows[j].Docs, rows[i].Docs == rows[j].Docs case "first", "last": a, b := rows[i].First, rows[j].First if sortBy == "last" { a, b = rows[i].Last, rows[j].Last } if (a == "") != (b == "") { return b == "" } less, eq = a < b, a == b case "gloss": a, b := rows[i].Gloss != nil, rows[j].Gloss != nil less, eq = !a && b, a == b // missing before defined, ascending default: less, eq = rows[i].Count < rows[j].Count, rows[i].Count == rows[j].Count } if eq { return rows[i].Token < rows[j].Token } if desc { return !less } return less }) }