first commit
This commit is contained in:
commit
8c4dcae475
89 changed files with 24623 additions and 0 deletions
298
codewords.go
Normal file
298
codewords.go
Normal file
|
|
@ -0,0 +1,298 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"html/template"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The codeword concordance: an automatically extracted index of ALL-CAPS
|
||||
// tokens across the corpus - program names, compartments, org designators.
|
||||
|
||||
var codewordStoplist = map[string]bool{
|
||||
"REDACTED": true,
|
||||
"DYNAMIC": true,
|
||||
"PAGE": true,
|
||||
"HIGHEST": true,
|
||||
"POSSIBLE": true,
|
||||
"DERIVED": true,
|
||||
"DECLASSIFY": true,
|
||||
"JAN": true,
|
||||
"ADDED": true,
|
||||
"CLASSIFICATION": true,
|
||||
}
|
||||
|
||||
type codewordEntry struct {
|
||||
Token string
|
||||
Count int // total occurrences across the corpus
|
||||
Docs int // number of documents containing it
|
||||
First string // earliest document year containing it ("" if only undated docs)
|
||||
Last string // latest such year
|
||||
}
|
||||
|
||||
func isCodewordToken(w string) bool {
|
||||
if len(w) < 3 || w[0] < 'A' || w[0] > 'Z' {
|
||||
return false
|
||||
}
|
||||
letters := 0
|
||||
var seen [26]bool
|
||||
distinct := 0
|
||||
for i := 0; i < len(w); i++ {
|
||||
c := w[i]
|
||||
switch {
|
||||
case c >= 'A' && c <= 'Z':
|
||||
letters++
|
||||
if !seen[c-'A'] {
|
||||
seen[c-'A'] = true
|
||||
distinct++
|
||||
}
|
||||
case c >= '0' && c <= '9':
|
||||
// digits allowed
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
return letters >= 3 && distinct >= 2
|
||||
}
|
||||
|
||||
func scanRuns(text string, mention func(w string)) {
|
||||
start := -1
|
||||
for j := 0; j <= len(text); j++ {
|
||||
alnum := false
|
||||
if j < len(text) {
|
||||
ch := text[j]
|
||||
alnum = (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9')
|
||||
}
|
||||
if alnum {
|
||||
if start < 0 {
|
||||
start = j
|
||||
}
|
||||
continue
|
||||
}
|
||||
if start >= 0 {
|
||||
mention(text[start:j])
|
||||
start = -1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (co *Corpus) buildCodewords() {
|
||||
co.codewords, co.codewordTable = nil, nil
|
||||
caps := map[string]int{}
|
||||
other := map[string]int{}
|
||||
docCaps := make([][]string, len(co.archive.Docs))
|
||||
for i := range co.archive.Docs {
|
||||
d := &co.archive.Docs[i]
|
||||
here := map[string]bool{}
|
||||
count := func(w string) {
|
||||
switch {
|
||||
case codewordStoplist[w]:
|
||||
// boilerplate or archive markup (codewordStoplist)
|
||||
case isCodewordToken(w):
|
||||
caps[w]++
|
||||
here[w] = true
|
||||
default:
|
||||
other[strings.ToLower(w)]++
|
||||
}
|
||||
}
|
||||
scanRuns(d.Title, count)
|
||||
scanRuns(d.FullText, count)
|
||||
for w := range here {
|
||||
docCaps[i] = append(docCaps[i], w)
|
||||
}
|
||||
}
|
||||
for w, cnt := range caps {
|
||||
if cnt < 2 || cnt <= other[strings.ToLower(w)] {
|
||||
continue
|
||||
}
|
||||
co.codewords = append(co.codewords, codewordEntry{Token: w, Count: cnt})
|
||||
}
|
||||
kept := make(map[string]*codewordEntry, len(co.codewords))
|
||||
for i := range co.codewords {
|
||||
kept[co.codewords[i].Token] = &co.codewords[i]
|
||||
}
|
||||
|
||||
for i := range co.archive.Docs {
|
||||
d := &co.archive.Docs[i]
|
||||
year := docYear(d)
|
||||
inDoc := map[string]bool{}
|
||||
exact := func(w string) {
|
||||
if e := kept[w]; e != nil {
|
||||
inDoc[e.Token] = true
|
||||
}
|
||||
}
|
||||
folded := func(w string) {
|
||||
if e := kept[strings.ToUpper(w)]; e != nil {
|
||||
inDoc[e.Token] = true
|
||||
}
|
||||
}
|
||||
for _, w := range docCaps[i] {
|
||||
exact(w)
|
||||
}
|
||||
scanRuns(d.Description, exact)
|
||||
scanRuns(d.LongDescription, exact)
|
||||
for _, p := range d.DocMeta.Programs {
|
||||
scanRuns(p, folded)
|
||||
}
|
||||
for _, t := range d.Tags {
|
||||
scanRuns(t, folded)
|
||||
}
|
||||
for w := range inDoc {
|
||||
e := kept[w]
|
||||
e.Docs++
|
||||
if year == "" {
|
||||
continue
|
||||
}
|
||||
if e.First == "" || year < e.First {
|
||||
e.First = year
|
||||
}
|
||||
if year > e.Last {
|
||||
e.Last = year
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(co.codewords, func(i, j int) bool {
|
||||
if co.codewords[i].Count != co.codewords[j].Count {
|
||||
return co.codewords[i].Count > co.codewords[j].Count
|
||||
}
|
||||
return co.codewords[i].Token < co.codewords[j].Token
|
||||
})
|
||||
|
||||
lowerProg := make(map[string]string, len(co.archive.Facets["program"]))
|
||||
for p := range co.archive.Facets["program"] {
|
||||
lowerProg[valueKey(p)] = p
|
||||
}
|
||||
shown := co.codewords
|
||||
if len(shown) > codewordRows {
|
||||
shown = shown[:codewordRows]
|
||||
}
|
||||
co.codewordTable = make([]codewordRow, 0, len(shown))
|
||||
for _, e := range shown {
|
||||
row := codewordRow{codewordEntry: e, Gloss: co.glossaryLookup(e.Token), SearchURL: tokenSearchURL(e.Token)}
|
||||
if p, ok := lowerProg[valueKey(e.Token)]; ok {
|
||||
row.ProgURL = wholeArchive.link(categoryValueURL("program", p))
|
||||
}
|
||||
if e.First != "" {
|
||||
row.FirstURL = yearTokenURL(e.Token, e.First)
|
||||
}
|
||||
if e.Last != "" {
|
||||
row.LastURL = yearTokenURL(e.Token, e.Last)
|
||||
}
|
||||
if row.Gloss != nil {
|
||||
row.GlossURL = template.URL("/glossary#term-" + url.PathEscape(row.Gloss.Term))
|
||||
}
|
||||
co.codewordTable = append(co.codewordTable, row)
|
||||
}
|
||||
co.buildCodewordViews()
|
||||
}
|
||||
|
||||
func yearTokenURL(token, year string) template.URL {
|
||||
return template.URL(wholeArchive.link(dateRangeSearchURL("TE", exactClause(token), [2]string{"date_doc", year})))
|
||||
}
|
||||
|
||||
func tokenSearchURL(token string) template.URL {
|
||||
return template.URL(wholeArchive.link("/search?q=" + url.QueryEscape(exactClause(token))))
|
||||
}
|
||||
|
||||
type codewordRow struct {
|
||||
codewordEntry
|
||||
Gloss *GlossaryEntry
|
||||
ProgURL string
|
||||
SearchURL template.URL
|
||||
FirstURL, LastURL template.URL
|
||||
GlossURL template.URL
|
||||
}
|
||||
|
||||
// codewordRows caps the page
|
||||
const codewordRows = 500
|
||||
|
||||
var codewordCols = []sortCol{
|
||||
{Key: "token", Label: "Token"},
|
||||
{Key: "count", Label: "Occurrences", DescFirst: true},
|
||||
{Key: "docs", Label: "Documents", DescFirst: true},
|
||||
{Key: "first", Label: "First seen"},
|
||||
{Key: "last", Label: "Last seen", DescFirst: true},
|
||||
{Key: "gloss", Label: "Glossary"},
|
||||
}
|
||||
|
||||
func (co *Corpus) handleCodewords(w http.ResponseWriter, r *http.Request) {
|
||||
q := r.URL.Query()
|
||||
sortBy := sortToken(q.Get("sort"))
|
||||
order := sortToken(q.Get("order"))
|
||||
view, ok := co.codewordViews[sortBy+"/"+order]
|
||||
if !ok {
|
||||
view, ok = co.codewordViews[sortBy+"/"+codewordDefaultOrder(sortBy)]
|
||||
}
|
||||
if !ok {
|
||||
view = co.codewordViews["count/desc"]
|
||||
}
|
||||
render(w, r, &codewordsPage{
|
||||
Rows: view.Rows,
|
||||
RowsHTML: co.preHTML("codewords:" + view.Key),
|
||||
Total: len(co.codewords),
|
||||
Shown: len(view.Rows),
|
||||
Heads: view.Heads,
|
||||
})
|
||||
}
|
||||
|
||||
type codewordView struct {
|
||||
Key string
|
||||
Rows []codewordRow
|
||||
Heads []sortHead
|
||||
}
|
||||
|
||||
func codewordDefaultOrder(sortBy string) string {
|
||||
for _, col := range codewordCols {
|
||||
if col.Key == sortBy {
|
||||
return orderParam(col.DescFirst)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (co *Corpus) buildCodewordViews() {
|
||||
co.codewordViews = make(map[string]codewordView, 2*len(codewordCols))
|
||||
for _, sc := range codewordCols {
|
||||
for _, order := range []string{"asc", "desc"} {
|
||||
rows := append([]codewordRow(nil), co.codewordTable...)
|
||||
sortCodewordRows(rows, sc.Key, order == "desc")
|
||||
heads := sortHeads("?", codewordCols, sc.Key, order == "desc")
|
||||
co.codewordViews[sc.Key+"/"+order] = codewordView{sc.Key + "/" + order, rows, heads}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sortCodewordRows(rows []codewordRow, sortBy string, desc bool) {
|
||||
sort.SliceStable(rows, func(i, j int) bool {
|
||||
var less, eq bool
|
||||
switch sortBy {
|
||||
case "token":
|
||||
less, eq = rows[i].Token < rows[j].Token, rows[i].Token == rows[j].Token
|
||||
case "docs":
|
||||
less, eq = rows[i].Docs < rows[j].Docs, rows[i].Docs == rows[j].Docs
|
||||
case "first", "last":
|
||||
a, b := rows[i].First, rows[j].First
|
||||
if sortBy == "last" {
|
||||
a, b = rows[i].Last, rows[j].Last
|
||||
}
|
||||
if (a == "") != (b == "") {
|
||||
return b == ""
|
||||
}
|
||||
less, eq = a < b, a == b
|
||||
case "gloss":
|
||||
a, b := rows[i].Gloss != nil, rows[j].Gloss != nil
|
||||
less, eq = !a && b, a == b // missing before defined, ascending
|
||||
default:
|
||||
less, eq = rows[i].Count < rows[j].Count, rows[i].Count == rows[j].Count
|
||||
}
|
||||
if eq {
|
||||
return rows[i].Token < rows[j].Token
|
||||
}
|
||||
if desc {
|
||||
return !less
|
||||
}
|
||||
return less
|
||||
})
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue