first commit

This commit is contained in:
libroot 2026-11-19 04:40:59 +02:00
commit 8c4dcae475
Signed by: libroot
GPG key ID: 2AC68C029EBE9ACE
89 changed files with 24623 additions and 0 deletions

230
portions.go Normal file
View file

@ -0,0 +1,230 @@
// portions.go - the portion markings inside the transcripts.
package main
import (
"html/template"
"log"
"net/http"
"regexp"
"strings"
)
// A marking opens a line, optionally behind a bullet, a heading hash or a
// list number: "(U)", "* (S//SI) ...", "## (U//FOUO) Heading".
var rePortion = regexp.MustCompile(`^[ \t]{0,6}(?:[-*+][ \t]+|#{1,6}[ \t]+|\d+[.)][ \t]+)?\(([^)\n]{1,42})\)`)
var portionLevels = map[string]string{
"U": "U", "UNCLASSIFIED": "U", "FOUO": "U",
"C": "C", "CONFIDENTIAL": "C",
"S": "S", "SECRET": "S",
"TS": "TS", "TOP SECRET": "TS",
}
var portionControls = map[string]bool{
"SI": true, "TK": true, "GAMMA": true, "SI-GAMMA": true, "STLW": true, "TSP": true,
"HCS": true, "KLONDIKE": true, "ECI": true, "SI-ECI": true, "COMINT": true,
"FOUO": true, "NF": true, "NOFORN": true, "OC": true, "ORCON": true,
"PROPIN": true, "PR": true, "RELIDO": true, "RSEN": true, "FISA": true,
"IMCON": true, "DSEN": true, "REL": true, "FVEY": true, "NATO": true,
"X1": true,
}
var portionLevelNames = []string{"U", "C", "S", "TS"}
var portionLevelLabels = map[string]string{
"U": "UNCLASSIFIED", "C": "CONFIDENTIAL", "S": "SECRET", "TS": "TOP SECRET",
}
// reSlashRun folds "/", "//" and "///" to one separator.
var reSlashRun = regexp.MustCompile(`[ \t]*/+[ \t]*`)
// portionMark is one marking, canonicalised.
type portionMark struct {
Raw string // as the transcript writes it, without the parentheses
Canonical string // "TS//SI//NF"
Level string // "TS"
Controls []string // ["SI", "NF"], in the order the marking writes them
OK bool // every part recognised
}
func parsePortion(raw string) portionMark {
m := portionMark{Raw: strings.TrimSpace(raw)}
written := strings.Split(reSlashRun.ReplaceAllString(strings.Join(strings.Fields(m.Raw), " "), "//"), "//")
if lvl := strings.TrimSpace(written[0]); lvl == "" || lvl != strings.ToUpper(lvl) {
return m
}
parts := make([]string, len(written))
for i := range written {
parts[i] = strings.ToUpper(strings.TrimSpace(written[i]))
}
lvl, isLevel := portionLevels[parts[0]]
if !isLevel {
return m
}
m.Level = lvl
if parts[0] == "FOUO" {
m.Controls = append(m.Controls, "FOUO")
}
known := true
for _, p := range parts[1:] {
if p == "" {
continue
}
m.Controls = append(m.Controls, p)
if !portionControlKnown(p) {
known = false
}
}
m.OK = known
m.Canonical = m.Level
if len(m.Controls) > 0 {
m.Canonical += "//" + strings.Join(m.Controls, "//")
}
return m
}
func portionControlKnown(p string) bool {
if portionControls[p] {
return true
}
if strings.HasPrefix(p, "REL") {
return true
}
if strings.HasPrefix(p, "ECI ") || strings.HasPrefix(p, "SI-ECI") {
return true
}
return false
}
func extractPortions(text string) []portionMark {
var out []portionMark
fenced := false
for _, line := range strings.Split(text, "\n") {
t := strings.TrimSpace(line)
if strings.HasPrefix(t, "```") || strings.HasPrefix(t, "~~~") {
fenced = !fenced
continue
}
if fenced || strings.HasPrefix(t, ">") {
continue
}
m := rePortion.FindStringSubmatch(line)
if m == nil {
continue
}
if p := parsePortion(m[1]); p.Level != "" {
out = append(out, p)
}
}
return out
}
// --- the corpus view -------------------------------------------------------
type portionRow struct {
Name string // the canonical marking, a level, or a control
Label string // the spelled-out level, where there is one
Marks int // occurrences across the corpus
Docs int // documents carrying it
SearchURL template.URL
}
type portionSummary struct {
Docs int // documents carrying at least one marking
Marks int // markings found
Distinct int // distinct canonical markings
Unrecog int // markings whose parts are not all known
UnrecogEg []portionRow
Levels []portionRow
Markings []portionRow
Controls []portionRow
}
const portionRows = 200
func (co *Corpus) buildPortions() {
co.portionStats = portionSummary{}
markMarks, lvlMarks, ctlMarks := map[string]int{}, map[string]int{}, map[string]int{}
docsBy := map[string]map[string]int{"PL": {}, "PM": {}, "PC": {}}
unrecogMarks := map[string]int{}
for i := range co.archive.Docs {
d := &co.archive.Docs[i]
d.PortionLevels, d.PortionMarkings, d.PortionControls = nil, nil, nil
marks := extractPortions(d.FullText)
if len(marks) == 0 {
continue
}
co.portionStats.Docs++
co.portionStats.Marks += len(marks)
for _, m := range marks {
if !m.OK {
co.portionStats.Unrecog++
unrecogMarks[m.Raw]++
continue
}
markMarks[m.Canonical]++
lvlMarks[m.Level]++
d.PortionMarkings = append(d.PortionMarkings, m.Canonical)
d.PortionLevels = append(d.PortionLevels, m.Level)
for _, c := range m.Controls {
ctlMarks[c]++
d.PortionControls = append(d.PortionControls, c)
}
}
d.PortionLevels = normalizeList(d.PortionLevels)
d.PortionMarkings = normalizeList(d.PortionMarkings)
d.PortionControls = normalizeList(d.PortionControls)
for code, docs := range docsBy {
for _, v := range facetValues(d, code) {
docs[v]++
}
}
}
co.portionStats.Distinct = len(markMarks)
for _, l := range portionLevelNames {
if lvlMarks[l] == 0 {
continue
}
co.portionStats.Levels = append(co.portionStats.Levels, portionRow{
Name: l, Label: portionLevelLabels[l], Marks: lvlMarks[l], Docs: docsBy["PL"][l],
SearchURL: portionSearchURL("PL", l),
})
}
co.portionStats.Markings = portionRowsOf(markMarks, docsBy["PM"], portionRows)
for i := range co.portionStats.Markings {
r := &co.portionStats.Markings[i]
r.SearchURL = portionSearchURL("PM", r.Name)
}
co.portionStats.Controls = portionRowsOf(ctlMarks, docsBy["PC"], portionRows)
for i := range co.portionStats.Controls {
r := &co.portionStats.Controls[i]
r.SearchURL = portionSearchURL("PC", r.Name)
}
co.portionStats.UnrecogEg = portionRowsOf(unrecogMarks, nil, 25)
log.Printf("Portion markings: %d in %d documents, %d distinct (%d unrecognised)",
co.portionStats.Marks, co.portionStats.Docs, co.portionStats.Distinct, co.portionStats.Unrecog)
}
func portionSearchURL(code, value string) template.URL {
return template.URL(wholeArchive.link(searchURL([]searchClause{{Field: code, Value: exactClause(value)}})))
}
func portionRowsOf(marks, docs map[string]int, n int) []portionRow {
names := rankCounts(marks, n)
out := make([]portionRow, len(names))
for i, k := range names {
out[i] = portionRow{Name: k, Marks: marks[k], Docs: docs[k]}
}
return out
}
func (co *Corpus) handlePortions(w http.ResponseWriter, r *http.Request) {
render(w, r, &portionsPage{
Title: "Portion markings",
Stats: co.portionStats,
RowLimit: portionRows,
})
}