package main import ( "reflect" "testing" ) func TestParsePortion(t *testing.T) { cases := []struct { in string canonical string level string ok bool }{ {"U", "U", "U", true}, {"U//FOUO", "U//FOUO", "U", true}, {"TS//SI//NF", "TS//SI//NF", "TS", true}, {"S//REL TO USA, FVEY", "S//REL TO USA, FVEY", "S", true}, // OCR writes the separator every way; one marking, one canonical form. {"S/NF", "S//NF", "S", true}, {"U/FOUO", "U//FOUO", "U", true}, {"TS/SI//REL", "TS//SI//REL", "TS", true}, {"TS//SI/NF", "TS//SI//NF", "TS", true}, {"TS//STLW//SI//OC/NF", "TS//STLW//SI//OC//NF", "TS", true}, {"TS//SI// ECI SOL", "TS//SI//ECI SOL", "TS", true}, // FOUO alone is a control on unclassified material. {"FOUO", "U//FOUO", "U", true}, // Spelled out. {"TOP SECRET//COMINT", "TS//COMINT", "TS", true}, // A level with a part no rule covers: recognised as a marking, but // not counted - the Explore page lists these separately. {"S//SI) (DNI collection", "", "S", false}, {"S//WIBBLE", "S//WIBBLE", "S", false}, } for _, c := range cases { got := parsePortion(c.in) if got.Level != c.level { t.Errorf("parsePortion(%q).Level = %q, want %q", c.in, got.Level, c.level) } if got.OK != c.ok { t.Errorf("parsePortion(%q).OK = %v, want %v", c.in, got.OK, c.ok) } if c.ok && got.Canonical != c.canonical { t.Errorf("parsePortion(%q).Canonical = %q, want %q", c.in, got.Canonical, c.canonical) } } } // The things that merely look like markings: list enumerators, FOIA // exemption citations, asides. Each of these appears hundreds of times. func TestParsePortionRejectsNonMarkings(t *testing.T) { for _, s := range []string{"1", "2", "a", "b", "iii", "d", "Cont'd.", "continued...", "Dollars in Thousands", "b)(1), (b)(3", "A", "", " "} { if got := parsePortion(s); got.Level != "" { t.Errorf("parsePortion(%q) read a level %q - it is not a marking", s, got.Level) } } } func TestExtractPortions(t *testing.T) { text := "" + "TOP SECRET//COMINT\n" + "\n" + "(U) Some Tips on Portion Marking\n" + "\n" + "(U//FOUO) The lead-in sentence MUST ALWAYS be portion-marked.\n" + "* (S//SI) a bullet carrying a complete thought\n" + "* an unmarked bullet, covered by its lead-in\n" + "## (TS//SI//NF) A marked heading\n" + "1. (C) a numbered portion\n" + "(1) not a marking, a list number\n" + "text mentioning (U//FOUO) mid-line is not a portion mark\n" got := make([]string, 0, 5) for _, m := range extractPortions(text) { got = append(got, m.Canonical) } want := []string{"U", "U//FOUO", "S//SI", "TS//SI//NF", "C"} if !reflect.DeepEqual(got, want) { t.Errorf("extractPortions = %v, want %v", got, want) } } // A document quoting a marking is not marked by it. Fenced blocks and block // quotes are where a transcript puts an example, and they are the curator's // correction surface: wrap the example, and it stops being counted. func TestExtractPortionsSkipsQuotedExamples(t *testing.T) { text := "" + "(U) Wrong way:\n" + "\n" + "> (U) All good men and women should come to the aid of their country through:\n" + "> * (U//FOUO) volunteering for a tdy to the Middle East\n" + "\n" + "```\n" + "(TS//SI//NF) an example inside a fenced block\n" + "```\n" + "(S) a real portion after the block\n" var got []string for _, m := range extractPortions(text) { got = append(got, m.Canonical) } want := []string{"U", "S"} if !reflect.DeepEqual(got, want) { t.Errorf("extractPortions = %v, want %v (quoted and fenced examples must not count)", got, want) } } // Markings are written in capitals. Court filings enumerate subsections // "(a)", "(b)", "(c)" at the start of a line, and "(c)" is not CONFIDENTIAL. func TestParsePortionNeedsCapitals(t *testing.T) { for _, s := range []string{"c", "s", "u", "ts", "s//si", "Ts", "secret"} { if got := parsePortion(s); got.Level != "" { t.Errorf("parsePortion(%q) read level %q from a lower-case label", s, got.Level) } } // The controls after a capital level are read whatever their case. if got := parsePortion("S//si"); got.Canonical != "S//SI" || !got.OK { t.Errorf("parsePortion(S//si) = %+v, want S//SI", got) } }