From 8c4dcae475e4378063a8d91d33da092b3557c81d Mon Sep 17 00:00:00 2001
From: libroot
Date: Thu, 19 Nov 2026 04:40:59 +0200
Subject: [PATCH] first commit
---
LICENSE | 661 ++++++++
README.md | 177 ++
agencies.go | 60 +
api.go | 335 ++++
articles.go | 211 +++
assemblies.go | 273 +++
browse.go | 2273 +++++++++++++++++++++++++
changelog.go | 102 ++
codewords.go | 298 ++++
corpus.go | 162 ++
data.go | 1470 +++++++++++++++++
explore.go | 377 +++++
explore_ids.go | 114 ++
glossary.go | 135 ++
go.mod | 3 +
history.go | 194 +++
main.go | 773 +++++++++
mdparser.go | 591 +++++++
outlets.go | 132 ++
pages.go | 393 +++++
pins.go | 145 ++
portions.go | 230 +++
prefs.go | 262 +++
search.go | 2644 ++++++++++++++++++++++++++++++
seo.go | 233 +++
static/main.js | 415 +++++
static/style.css | 2353 ++++++++++++++++++++++++++
templates/404.html | 14 +
templates/about.html | 544 ++++++
templates/article.html | 25 +
templates/articles.html | 39 +
templates/assemblies.html | 32 +
templates/assembly.html | 60 +
templates/base.html | 35 +
templates/browse.html | 155 ++
templates/category.html | 59 +
templates/changelog.html | 25 +
templates/classpatterns.html | 50 +
templates/codewords.html | 38 +
templates/doc.html | 272 +++
templates/explore.html | 50 +
templates/explore_table.html | 35 +
templates/glossary.html | 36 +
templates/ids.html | 63 +
templates/index.html | 150 ++
templates/list.html | 95 ++
templates/outlets.html | 54 +
templates/pages.html | 40 +
templates/partials.html | 412 +++++
templates/pinned.html | 46 +
templates/portions.html | 82 +
templates/preferences.html | 97 ++
templates/search.html | 280 ++++
templates/version.html | 59 +
tests/api_test.go | 122 ++
tests/assemblies_test.go | 211 +++
tests/bench_test.go | 312 ++++
tests/bugs_test.go | 78 +
tests/consistency_test.go | 538 ++++++
tests/corpus_test.go | 199 +++
tests/data_test.go | 123 ++
tests/fixture_test.go | 295 ++++
tests/flags_test.go | 167 ++
tests/foia_test.go | 272 +++
tests/go.mod | 5 +
tests/headings_test.go | 53 +
tests/http_test.go | 231 +++
tests/ids_test.go | 99 ++
tests/image_sizes_test.go | 53 +
tests/invariants_test.go | 277 ++++
tests/loader_test.go | 166 ++
tests/main_test.go | 74 +
tests/mdparser_test.go | 145 ++
tests/outlets_test.go | 54 +
tests/portions_test.go | 119 ++
tests/prefs_matrix_test.go | 153 ++
tests/prefs_test.go | 117 ++
tests/probes_test.go | 126 ++
tests/published_together_test.go | 43 +
tests/query_test.go | 193 +++
tests/related_test.go | 77 +
tests/reporting_only_test.go | 49 +
tests/routes_test.go | 705 ++++++++
tests/run.sh | 22 +
tests/search_test.go | 1000 +++++++++++
tests/sidtoday_scope_test.go | 58 +
tests/tmplcheck_test.go | 464 ++++++
tests/versions_pages_test.go | 59 +
tests/versions_search_test.go | 101 ++
89 files changed, 24623 insertions(+)
create mode 100644 LICENSE
create mode 100644 README.md
create mode 100644 agencies.go
create mode 100644 api.go
create mode 100644 articles.go
create mode 100644 assemblies.go
create mode 100644 browse.go
create mode 100644 changelog.go
create mode 100644 codewords.go
create mode 100644 corpus.go
create mode 100644 data.go
create mode 100644 explore.go
create mode 100644 explore_ids.go
create mode 100644 glossary.go
create mode 100644 go.mod
create mode 100644 history.go
create mode 100644 main.go
create mode 100644 mdparser.go
create mode 100644 outlets.go
create mode 100644 pages.go
create mode 100644 pins.go
create mode 100644 portions.go
create mode 100644 prefs.go
create mode 100644 search.go
create mode 100644 seo.go
create mode 100644 static/main.js
create mode 100644 static/style.css
create mode 100644 templates/404.html
create mode 100644 templates/about.html
create mode 100644 templates/article.html
create mode 100644 templates/articles.html
create mode 100644 templates/assemblies.html
create mode 100644 templates/assembly.html
create mode 100644 templates/base.html
create mode 100644 templates/browse.html
create mode 100644 templates/category.html
create mode 100644 templates/changelog.html
create mode 100644 templates/classpatterns.html
create mode 100644 templates/codewords.html
create mode 100644 templates/doc.html
create mode 100644 templates/explore.html
create mode 100644 templates/explore_table.html
create mode 100644 templates/glossary.html
create mode 100644 templates/ids.html
create mode 100644 templates/index.html
create mode 100644 templates/list.html
create mode 100644 templates/outlets.html
create mode 100644 templates/pages.html
create mode 100644 templates/partials.html
create mode 100644 templates/pinned.html
create mode 100644 templates/portions.html
create mode 100644 templates/preferences.html
create mode 100644 templates/search.html
create mode 100644 templates/version.html
create mode 100644 tests/api_test.go
create mode 100644 tests/assemblies_test.go
create mode 100644 tests/bench_test.go
create mode 100644 tests/bugs_test.go
create mode 100644 tests/consistency_test.go
create mode 100644 tests/corpus_test.go
create mode 100644 tests/data_test.go
create mode 100644 tests/fixture_test.go
create mode 100644 tests/flags_test.go
create mode 100644 tests/foia_test.go
create mode 100644 tests/go.mod
create mode 100644 tests/headings_test.go
create mode 100644 tests/http_test.go
create mode 100644 tests/ids_test.go
create mode 100644 tests/image_sizes_test.go
create mode 100644 tests/invariants_test.go
create mode 100644 tests/loader_test.go
create mode 100644 tests/main_test.go
create mode 100644 tests/mdparser_test.go
create mode 100644 tests/outlets_test.go
create mode 100644 tests/portions_test.go
create mode 100644 tests/prefs_matrix_test.go
create mode 100644 tests/prefs_test.go
create mode 100644 tests/probes_test.go
create mode 100644 tests/published_together_test.go
create mode 100644 tests/query_test.go
create mode 100644 tests/related_test.go
create mode 100644 tests/reporting_only_test.go
create mode 100644 tests/routes_test.go
create mode 100755 tests/run.sh
create mode 100644 tests/search_test.go
create mode 100644 tests/sidtoday_scope_test.go
create mode 100644 tests/tmplcheck_test.go
create mode 100644 tests/versions_pages_test.go
create mode 100644 tests/versions_search_test.go
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..be3f7b2
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,661 @@
+ GNU AFFERO GENERAL PUBLIC LICENSE
+ Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+ A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+ The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+ An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU Affero General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Remote Network Interaction; Use with the GNU General Public License.
+
+ Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software. This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time. Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+
+ Copyright (C)
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+.
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..f392a81
--- /dev/null
+++ b/README.md
@@ -0,0 +1,177 @@
+# Snowchive server
+
+A self-contained web server for browsing and searching an archive of published
+Snowden documents in Snowchive website. The server includes faceted browsing, a
+boolean/fielded search engine with stemming and accent folding, a SIDtoday
+serial reader, a read-only JSON API, sitemaps, and per-visitor preferences, pins
+and history that live only in the visitor's cookies. The server uses just Go's
+standard library.
+
+No JavaScript is required to use the website. A file in `static/main.js` adds
+progressive enhancements only. No database and no external services.
+
+See it in action: https://snowchive.org
+
+If you are looking for Snowchive data or metadata:
+
+Data (PDFs, page images):
+
+https://snowchive.org/about#downloading-all-data
+
+Metadata (with extracted texts):
+
+https://git.disroot.org/libroot/snowchive-meta
+
+## Run
+
+Full Snowchive data is required (containing all the PDFs and metadata) to run
+the server. If you don't yet have the data, download it first:
+https://snowchive.org/about#downloading-all-data
+
+```sh
+go run . -root=/path/to/snowchive-data/
+```
+
+## Building
+
+If you want to build a binary to deploy one executable for production use:
+
+```sh
+go build -ldflags "-X main.buildCommit=$(git rev-parse --short HEAD)" -o snowchive .
+./snowchive -root=/path/to/snowchive-data/ -base=https://snowchive.example.org
+```
+
+## Flags
+
+| flag | purpose |
+|---|---|
+| `-root` | Snowchive data directory holding `data/` (the corpus, one directory per document) and `project/` (`glossary.json`, `assemblies.json`, `changelog.json`, `related.json`) (default: the working directory, `$SNOWCHIVE_ROOT`) |
+| `-listen` | TCP address (default `127.0.0.1:8080`, `$SNOWCHIVE_LISTEN`). |
+| `-base` | canonical `scheme://host` for absolute URLs in robots.txt and the sitemap; set it on anything internet-facing. Without it those URLs follow each request's host. |
+| `-no-api` | disable the JSON API (`$SNOWCHIVE_NO_API=1`) |
+| `-accel-files PREFIX` | behind nginx only: answer document files with an `X-Accel-Redirect` into this internal location (e.g. `/_files/`, an `alias` of the data directory), so nginx sends the bytes and this server only decides whether the file is served (`$SNOWCHIVE_ACCEL_FILES`) |
+| `-accel-expires DURATION` | let the proxy cache a page served to a reader without cookies for this long (e.g. `10m`, sent as `X-Accel-Expires`); browsers keep the 10-second `max-age`. Clear the proxy cache after a restart with new data (`$SNOWCHIVE_ACCEL_EXPIRES`) |
+| `-dev` | serve `templates/` and `static/` from the working directory, re-parse per request, verbose errors. Never in production. |
+| `-check` | load the corpus, print the warnings and the tally, then exit non-zero if anything was reported - run it after a curation pass or in CI. |
+| `-rebuild-related` | compute `related.json` entries for documents that lack one, or whose list links to a document no longer in the corpus, drop the entries of documents that are gone, then exit. It only fills gaps, so after a scoring change delete `related.json` first to rebuild every list. |
+
+## What it costs to run
+
+Measured on a 2-core VM against the 3,106-document corpus: ~7 seconds to load
+(warm page cache), ~260 MB of live heap, and ~750 MB of address space under
+sustained concurrent load.
+
+## Directory layout
+
+```
+data/
+ / one directory per document
+ metadata.json document metadata
+ .pdf document PDF file
+ .md extracted text transcript from PDF
+ thumb.jpg, thumb.gif document thumbnail images
+ images/page-N.jpg PDF page scans
+ versions// another published copy that differs (e.g. less
+ redacted, extra pages): its own PDF,
+ .md and images/ (optional)
+ FOIA// the FOIA document collection
+ SIDtoday-archive// the bulk SIDtoday document collection
+```
+
+The files about the archive as a whole are in the project directory
+(`-project`):
+
+```
+project/
+ glossary.json format documented in glossary.go
+ assemblies.json see "Assemblies" below
+ changelog.json snowchive changelog, format in changelog.go
+ related.json documents relating to each other, generated by
+ -rebuild-related
+```
+
+The corpus is read exactly once at startup and everything derived from it is
+computed there. All of it lives in one `Corpus` value (`corpus.go`), which is
+built whole and pre-rendered. The server writes nothing while it runs. Only
+`-rebuild-related` writes a file (`related.json`, in the project directory).
+
+## About metadata
+
+`metadata.json` carries the title, description, tags, the `document`
+block (creator, date_created, programs, classification, dissemination,
+banner, genre, series, countries, people) and the `publication` list
+(publisher, title, link, reporters, contributors, date_published, plus
+`doc_source` and `doc_url` for the copy that entry published, and `note`
+and `version` when that copy differs from the primary one). Every field
+except `title` is optional. There is no document-level `doc_source`/`doc_url`,
+because where a copy came from is a property of the publication that published
+it.
+
+A publication entry is an outlet that published the document. A partner in a
+joint investigation that published the story but no documents is the one
+exception (they have entry `"reporting_only": true`).
+
+A publication that released a different copy of the same document is in
+`version` directory. The directory `versions//` is in the document
+directory, holding that copy's PDF, `.md` and `images/`. The document page shows
+the publication's `note` and links the copy, which has its own page at
+`/doc//versions/`. Versions never count as documents. Search covers
+the lines of a version's transcript that the main transcript lacks; results
+label those matches with the copy they came from. A version no publication names
+is listed as another copy.
+
+The `versions` object records, per version directory, the SHA-256 of that copy's
+PDF and the pages that differ from the main document, by the number in the page
+image's filename:
+`"versions": {"": {"pdf_sha256": "...", "changed_pages": [7, 12]}}`.
+
+`aka_id` lists other names the document was published under, as an array of
+objects: `{"id": "...", "doc_source": "...", "doc_url": "..."}`. An aka is a
+pointer and stores no file, which is what separates it from a `versions/` entry.
+
+More info about the metadata: https://git.disroot.org/libroot/snowchive-meta
+
+## Assemblies
+
+`assemblies.json` is a hand-maintained list of sets of pages drawn from several
+documents and versions, for material that was published in pieces, such as a
+slide deck. Each gets a page at `/explore/assemblies/` showing all its
+slides in one grid and lightbox, listed at `/explore/assemblies`
+
+```json
+[{
+ "id": "prism-overview",
+ "title": "PRISM/US-984XN briefing",
+ "note": "Shown at the top: how the set was put together and what is unproven.",
+ "slides": [
+ {"doc": "", "pages": "1-7"},
+ {"doc": "", "version": "", "pages": "8", "note": "Shown under the slide."},
+ {"doc": "", "pages": "9,11",
+ "alt": [{"doc": "", "version": "", "pages": "1-2"}]}
+ ]
+}]
+```
+An assembly is the archive's own arrangement, it's not an original document and
+stores no files, and is neither counted nor searched.
+
+## Tests
+
+./tests/run.sh
+
+TODO. The tests are currently written largely by Claude.
+
+## Other/contact
+
+The maintainers of this project do not endorse any public Snowchive instance
+other than the official snowchive.org instance (unless you are a reputable
+organization that wants to host a public instance).
+
+Please submit issues/feedback on the Snowchive server's repository:
+
+https://git.disroot.org/libroot/snowchive-server
+
+or to Codeberg (mirror): https://codeberg.org/libroot/snowchive-server
+
+Contact the maintainers via email:
+
+https://libroot.org/about
diff --git a/agencies.go b/agencies.go
new file mode 100644
index 0000000..a20d594
--- /dev/null
+++ b/agencies.go
@@ -0,0 +1,60 @@
+// agencies.go - display names for the acronyms used as document creators
+
+package main
+
+import "strings"
+
+var agencyNames = map[string]string{
+ "CSE": "Communications Security Establishment (Canada)",
+ "DFS": "Directorate for Signals Intelligence (Japan)",
+ "DNI": "Director of National Intelligence (US)",
+ "DOD": "Department of Defense (US)",
+ "DOJ": "Department of Justice (US)",
+ "DOS": "Department of State (US)",
+ "DSD": "Defence Signals Directorate (Australia; now ASD)",
+ "FISC": "Foreign Intelligence Surveillance Court (US)",
+ "FVEY": "Five Eyes (US, UK, Canada, Australia, New Zealand)",
+ "GCHQ": "Government Communications Headquarters (UK)",
+ "GCSB": "Government Communications Security Bureau (New Zealand)",
+ "MI5": "Security Service (UK)",
+ "NATO": "North Atlantic Treaty Organization",
+ "NCTC": "National Counterterrorism Center (US)",
+ "NSA": "National Security Agency (US)",
+ "ODNI": "Office of the Director of National Intelligence (US)",
+ "SIS": "Secret Intelligence Service (UK)",
+ "SWAC": "Southwest Analysis Center",
+ "USG": "United States Government",
+}
+
+func agencyName(creator string) string {
+ var out strings.Builder
+ known := false
+ start := 0
+ flush := func(end int) {
+ part := creator[start:end]
+ key := strings.ToUpper(strings.TrimSpace(part))
+ if name, ok := agencyNames[key]; ok {
+ known = true
+ out.WriteString(name)
+ } else {
+ out.WriteString(strings.TrimSpace(part))
+ }
+ }
+ for i := 0; i < len(creator); i++ {
+ switch creator[i] {
+ case '/':
+ flush(i)
+ out.WriteString(" / ")
+ start = i + 1
+ case ',':
+ flush(i)
+ out.WriteString(", ")
+ start = i + 1
+ }
+ }
+ flush(len(creator))
+ if !known {
+ return ""
+ }
+ return out.String()
+}
diff --git a/api.go b/api.go
new file mode 100644
index 0000000..4afc0e0
--- /dev/null
+++ b/api.go
@@ -0,0 +1,335 @@
+// api.go - read-only JSON API mirroring the UI's features.
+//
+// Endpoints (all GET):
+//
+// /api/docs list documents; params: coll (repeatable:
+// news, sidtoday-archive, foia), type
+// (repeatable), sort (any list-page sort: title,
+// agency, date, pubdate, classification, ...; see
+// docSorts), order (asc|desc), p, per_page (max
+// 500, or "all" for exports)
+// /api/doc/{id} full document: metadata, full text, images,
+// related IDs, published-together IDs,
+// version_copies (other published copies: PDF,
+// page images, transcript, publication indices)
+// /api/search params: q (simple) OR fqv/fqf/fqc clause
+// triples (same field codes and value syntax as
+// the UI: word, "text", =exact, *wildcard*),
+// plus coll and type (both repeatable, as on
+// /api/docs; no coll means every collection),
+// date_doc_start/_end, date_pub_start/_end
+// (partial END dates are inclusive: 2013 means
+// through 2013-12-31), sort, order, and the same
+// p / per_page pagination as /api/docs (default
+// 50, max 500, "all" for every match without
+// snippets see apiPageSize). results come
+// as one flat list, mirroring the search page:
+// all collections rank equally. a result whose match
+// is in another published copy of the document
+// lists that copy in matched_versions (slug,
+// label, url, snippet)
+// /api/facets facet maps (whole corpus, like the UI; the
+// one news-scoped map is news_types) plus
+// collection counts
+package main
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "log"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+ "time"
+)
+
+const (
+ apiPerPageDefault = 50
+ apiMaxPerPage = 500
+)
+
+func writeJSON(w http.ResponseWriter, status int, v interface{}) {
+ b, err := json.Marshal(v)
+ if err != nil {
+ log.Printf("API: marshal error: %v", err)
+ w.Header().Set("Cache-Control", "no-store")
+ w.Header().Set("Content-Type", "application/json; charset=utf-8")
+ w.WriteHeader(http.StatusInternalServerError)
+ w.Write([]byte(`{"error":"internal error"}`))
+ return
+ }
+ writeJSONBytes(w, status, b)
+}
+
+func writeJSONBytes(w http.ResponseWriter, status int, b []byte) {
+ setJSONHeaders(w)
+ w.WriteHeader(status)
+ w.Write(b)
+}
+
+func setJSONHeaders(w http.ResponseWriter) {
+ cc := w.Header().Get("Cache-Control")
+ if cc == "" {
+ cc = "public, max-age=60"
+ w.Header().Set("Cache-Control", cc)
+ }
+ w.Header().Set("Content-Type", "application/json; charset=utf-8")
+
+ if !strings.Contains(cc, "private") {
+ w.Header().Set("Access-Control-Allow-Origin", "*")
+ }
+}
+
+func apiError(w http.ResponseWriter, status int, msg string) {
+ writeJSON(w, status, map[string]string{"error": msg})
+}
+
+type apiDocSummary struct {
+ Document
+ URL string `json:"url"`
+ PDF string `json:"pdf"`
+}
+
+func docSummary(d Document) apiDocSummary {
+ return apiDocSummary{
+ Document: d,
+ URL: docPath(d.ID),
+ PDF: filePath(d.ID, d.PDFFile),
+ }
+}
+
+type apiDocDetail struct {
+ apiDocSummary
+ FullText string `json:"full_text,omitempty"`
+ Images []string `json:"images,omitempty"`
+ Related []string `json:"related,omitempty"`
+ PublishedTogether []string `json:"published_together,omitempty"`
+ VersionCopies []apiVersion `json:"version_copies,omitempty"`
+}
+
+type apiSearchResult struct {
+ apiDocSummary
+ Score float64 `json:"score"`
+ Snippet string `json:"snippet,omitempty"`
+ MatchedVersions []apiMatchedVersion `json:"matched_versions,omitempty"`
+}
+
+func (co *Corpus) handleAPIDocs(w http.ResponseWriter, r *http.Request) {
+ q := r.URL.Query()
+
+ typeFilter, _ := selectedTypes(q)
+
+ // coll=, repeatable, as /api/search and the pages take it.
+ pool := filterByCollection(co.everyDoc, collectionFilter(q))
+ docs := filterByType(append([]*Document(nil), pool...), typeFilter)
+
+ sortDocs(docs, q.Get("sort"), q.Get("order"))
+
+ perPage := apiPageSize(q)
+ page, _ := strconv.Atoi(q.Get("p"))
+ start, end, page, totalPages := paginate(len(docs), page, perPage)
+
+ out := make([]apiDocSummary, 0, end-start)
+ for _, d := range docs[start:end] {
+ out = append(out, docSummary(*d))
+ }
+
+ writeJSON(w, http.StatusOK, map[string]interface{}{
+ "total": len(docs),
+ "page": page,
+ "per_page": apiRespPerPage(perPage, len(docs)),
+ "total_pages": totalPages,
+ "docs": out,
+ })
+}
+
+func apiPageSize(q url.Values) int {
+ if q.Get("per_page") == "all" {
+ return 0
+ }
+ if v, err := strconv.Atoi(q.Get("per_page")); err == nil && v > 0 {
+ if v > apiMaxPerPage {
+ return apiMaxPerPage
+ }
+ return v
+ }
+ return apiPerPageDefault
+}
+
+func apiRespPerPage(perPage, total int) int {
+ if perPage > 0 {
+ return perPage
+ }
+ if total == 0 {
+ return 1
+ }
+ return total
+}
+
+func (co *Corpus) handleAPIDoc(w http.ResponseWriter, r *http.Request) {
+ id := r.PathValue("id")
+ doc := co.docIndex[id]
+ if doc == nil {
+ apiError(w, http.StatusNotFound, "document not found")
+ return
+ }
+
+ var pubIDs []string
+ for _, d := range co.publishedTogether(doc) {
+ pubIDs = append(pubIDs, d.ID)
+ }
+
+ writeJSON(w, http.StatusOK, apiDocDetail{
+ apiDocSummary: docSummary(*doc),
+ FullText: doc.FullText,
+ Images: doc.Images,
+ Related: co.archive.RelatedDocs[id],
+ PublishedTogether: pubIDs,
+ VersionCopies: apiVersions(doc),
+ })
+}
+
+func (co *Corpus) handleAPISearch(w http.ResponseWriter, r *http.Request) {
+ q := r.URL.Query()
+
+ sp := readSearchParams(q)
+ if len(sp.Queries) == 0 {
+ apiError(w, http.StatusBadRequest,
+ "missing query: pass ?q=term or fqv/fqf/fqc clause parameters")
+ return
+ }
+ if sp.OverClauses > 0 {
+ apiError(w, http.StatusBadRequest, fmt.Sprintf(
+ "too many clauses: %d (the limit is %d)", sp.OverClauses, maxSearchClauses))
+ return
+ }
+ if sp.OverWords > 0 {
+ apiError(w, http.StatusBadRequest, fmt.Sprintf(
+ "too many words: %d (the limit is %d)", sp.OverWords, maxSearchWords))
+ return
+ }
+ scored, _ := co.executeSearch(sp)
+ snippetTerm, _ := snippetAnchor(sp.Queries, sp.Fields, sp.Conjs)
+
+ perPage := apiPageSize(q)
+ page, _ := strconv.Atoi(q.Get("p"))
+ start, end, page, totalPages := paginate(len(scored), page, perPage)
+ withSnippets := snippetTerm != "" && perPage > 0
+
+ results := make([]apiSearchResult, 0, end-start)
+ for _, s := range scored[start:end] {
+ res := apiSearchResult{apiDocSummary: docSummary(*s.Doc), Score: s.Score}
+ if withSnippets {
+ if body := s.Doc.bodySnippets(snippetTerm, 1); len(body) > 0 {
+ res.Snippet = body[0]
+ }
+ for _, h := range versionHits(s.Doc, snippetTerm, 1) {
+ res.MatchedVersions = append(res.MatchedVersions, apiMatchedVersion{
+ Slug: h.Version.Slug,
+ Label: h.Version.Label,
+ URL: versionPath(s.Doc.ID, h.Version.Slug),
+ Snippet: h.Snippets[0],
+ })
+ }
+ }
+ results = append(results, res)
+ }
+
+ body := map[string]interface{}{
+ "total": len(scored),
+ "page": page,
+ "per_page": apiRespPerPage(perPage, len(scored)),
+ "total_pages": totalPages,
+ "results": results,
+ }
+ if snippetTerm != "" && !withSnippets {
+ body["snippets_omitted"] = true
+ }
+ writeJSON(w, http.StatusOK, body)
+}
+
+func (co *Corpus) buildFacetsJSON() {
+ b, err := json.Marshal(map[string]interface{}{
+ "note": "facet counts cover the whole corpus; news_types covers the news collection only",
+ "collections": co.archive.CollectionCounts,
+ "sidtoday_total": co.archive.SIDTodayCount,
+ "types": co.archive.Types,
+ "news_types": co.archive.NewsTypes,
+ "agencies": co.archive.Facets["agency"],
+ "programs": co.archive.Facets["program"],
+ "journalists": co.archive.Facets["journalist"],
+ "publishers": co.archive.Facets["publisher"],
+ "countries": co.archive.Facets["country"],
+ "people": co.archive.Facets["people"],
+ "classification": co.archive.Facets["classification"],
+ "dissemination": co.archive.Facets["dissemination"],
+ "banners": co.archive.Facets["banner"],
+ "genres": co.archive.Facets["genre"],
+ "series": co.archive.Facets["series"],
+ "tags": co.archive.Facets["tag"],
+ "pub_years": co.archive.Facets["published"],
+ "doc_years": co.archive.Facets["docdate"],
+ "by_collection": co.archive.FacetsByColl,
+ })
+ if err != nil {
+ log.Fatalf("facets: marshal failed: %v", err)
+ }
+ co.facetsJSON = b
+ sum := sha256.Sum256(b)
+ co.facetsETag = `"` + hex.EncodeToString(sum[:8]) + `"`
+ log.Printf("Facets JSON pre-built: %d bytes", len(co.facetsJSON))
+}
+
+func (co *Corpus) handleAPIFacets(w http.ResponseWriter, r *http.Request) {
+ setJSONHeaders(w)
+ w.Header().Set("ETag", co.facetsETag)
+ http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(co.facetsJSON))
+}
+
+// "version" as in document versions (metadata.json can have "versions" object)
+type apiVersion struct {
+ Slug string `json:"slug"`
+ URL string `json:"url"`
+ PDF string `json:"pdf"`
+ PDFSha256 string `json:"pdf_sha256,omitempty"`
+ Images []string `json:"images,omitempty"`
+ FullText string `json:"full_text,omitempty"`
+ Publications []int `json:"publications,omitempty"`
+ ChangedPages []int `json:"changed_pages,omitempty"`
+}
+
+func apiVersions(doc *Document) []apiVersion {
+ var out []apiVersion
+ for _, v := range doc.Versions {
+ out = append(out, apiVersion{
+ Slug: v.Slug,
+ URL: versionPath(doc.ID, v.Slug),
+ PDF: versionFilePath(doc.ID, v.Slug, v.PDFFile),
+ PDFSha256: v.SHA,
+ Images: v.Images,
+ FullText: v.FullText,
+ Publications: v.PubIndex,
+ ChangedPages: changedPageNumbers(v.ChangedPages),
+ })
+ }
+ return out
+}
+
+type apiMatchedVersion struct {
+ Slug string `json:"slug"`
+ Label string `json:"label"`
+ URL string `json:"url"`
+ Snippet string `json:"snippet"`
+}
+
+func changedPageNumbers(pages []ChangedPage) []int {
+ var out []int
+ for _, p := range pages {
+ out = append(out, p.Page)
+ }
+ return out
+}
diff --git a/articles.go b/articles.go
new file mode 100644
index 0000000..aead1c8
--- /dev/null
+++ b/articles.go
@@ -0,0 +1,211 @@
+// articles.go - news articles as first-class objects.
+//
+// An article's identity is its publication LINK. Giving a publication entry
+// a title is what promotes its article to a story.
+
+package main
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "net/http"
+ "net/url"
+ "slices"
+ "sort"
+ "strings"
+)
+
+type Article struct {
+ Key string // short stable hash of the link (the /article/{key} path)
+ Title string
+ Link string
+ Host string // display fallback when Title is empty
+ Publisher string
+ Date string // earliest date_published seen
+ Journalists []string // union across docs, first-seen order
+ DocIDs []string
+ Book bool // published by a book publisher, not a news outlet
+}
+
+// bookPublishers are the publication entries that are books, not news
+// articles (Greenwald's No Place to Hide, Gellman's Dark Mirror, Poitras's
+// Astro Noise).
+var bookPublishers = map[string]bool{
+ "Metropolitan Books": true,
+ "Penguin Press": true,
+ "Yale University Press": true,
+}
+
+// articleKeyFor derives the stable URL key for a link
+func articleKeyFor(link string) string {
+ h := sha256.Sum256([]byte(link))
+ return hex.EncodeToString(h[:])[:12]
+}
+
+func hostOf(link string) string {
+ u, err := url.Parse(link)
+ if err != nil || u.Host == "" {
+ return link
+ }
+ return strings.TrimPrefix(u.Host, "www.")
+}
+
+// buildArticles indexes every publication entry that has a link
+func (co *Corpus) buildArticles() {
+ co.articleByLink = make(map[string]*Article)
+ co.articleIndex = make(map[string]*Article)
+ co.articles = nil
+
+ for _, d := range co.archive.Docs {
+ for _, p := range d.Publication {
+ if p.Link == "" {
+ continue
+ }
+ a := co.articleByLink[p.Link]
+ if a == nil {
+ a = &Article{
+ Key: articleKeyFor(p.Link),
+ Link: p.Link,
+ Host: hostOf(p.Link),
+ }
+ co.articleByLink[p.Link] = a
+ co.articleIndex[a.Key] = a
+ }
+ if a.Title == "" && p.Title != "" {
+ a.Title = p.Title
+ }
+ if a.Publisher == "" && p.Publisher != "" {
+ a.Publisher = p.Publisher
+ a.Book = bookPublishers[p.Publisher]
+ }
+ if p.DatePublished != "" && (a.Date == "" || p.DatePublished < a.Date) {
+ a.Date = p.DatePublished
+ }
+ for _, j := range p.Journalists {
+ if j == "" {
+ continue
+ }
+ if !slices.ContainsFunc(a.Journalists, func(x string) bool { return sameValue(x, j) }) {
+ a.Journalists = append(a.Journalists, j)
+ }
+ }
+ dup := false
+ for _, id := range a.DocIDs {
+ if id == d.ID {
+ dup = true
+ break
+ }
+ }
+ if !dup {
+ a.DocIDs = append(a.DocIDs, d.ID)
+ }
+ }
+ }
+
+ for _, a := range co.articleByLink {
+ co.articles = append(co.articles, *a)
+ }
+ // Newest first; undated articles sort last; ties by title then link so
+ // the order is deterministic.
+ sort.Slice(co.articles, func(i, j int) bool {
+ di, dj := co.articles[i].Date, co.articles[j].Date
+ if di == "" && dj != "" {
+ return false
+ }
+ if di != "" && dj == "" {
+ return true
+ }
+ if di != dj {
+ return di > dj
+ }
+ ti := strings.ToLower(displayTitle(co.articles[i]))
+ tj := strings.ToLower(displayTitle(co.articles[j]))
+ if ti != tj {
+ return ti < tj
+ }
+ return co.articles[i].Link < co.articles[j].Link
+ })
+
+ co.buildArticleGroups()
+}
+
+// inCollection reports whether one of the article's documents is in coll.
+func (a *Article) inCollection(co *Corpus, coll string) bool {
+ for _, id := range a.DocIDs {
+ if d := co.docIndex[id]; d != nil && d.Collection == coll {
+ return true
+ }
+ }
+ return false
+}
+
+func displayTitle(a Article) string {
+ if a.Title != "" {
+ return a.Title
+ }
+ return a.Host
+}
+
+type articleYearGroup struct {
+ Year string
+ Articles []Article
+}
+
+func (co *Corpus) buildArticleGroups() {
+ co.articleGroups, co.articlesTitled, co.articlesBooks = nil, 0, 0
+ for _, a := range co.articles {
+ if a.Title == "" {
+ continue
+ }
+ co.articlesTitled++
+ if a.Book {
+ co.articlesBooks++
+ }
+ year := "Undated"
+ if len(a.Date) >= 4 {
+ year = a.Date[:4]
+ }
+ if len(co.articleGroups) == 0 || co.articleGroups[len(co.articleGroups)-1].Year != year {
+ co.articleGroups = append(co.articleGroups, articleYearGroup{Year: year})
+ }
+ co.articleGroups[len(co.articleGroups)-1].Articles = append(co.articleGroups[len(co.articleGroups)-1].Articles, a)
+ }
+}
+
+func (co *Corpus) handleArticles(w http.ResponseWriter, r *http.Request) {
+ render(w, r, &articlesPage{
+ Groups: co.articleGroups,
+ GroupsHTML: co.preHTML("articles"),
+ Total: co.articlesTitled - co.articlesBooks,
+ Books: co.articlesBooks,
+ })
+}
+
+func (co *Corpus) handleArticleView(w http.ResponseWriter, r *http.Request) {
+ a := co.articleIndex[r.PathValue("key")]
+ if a == nil {
+ notFound(w, r)
+ return
+ }
+ prefs := readPrefs(r)
+ scope := pageScope(nil, prefs, false)
+ docs := make([]*Document, 0, len(a.DocIDs))
+ for _, id := range a.DocIDs {
+ if d := co.docIndex[id]; d != nil {
+ docs = append(docs, d)
+ scope = scope.including(d.Collection)
+ }
+ }
+ data := &articlePage{
+ Base: pageWith(prefs),
+ Article: *a,
+ Total: len(docs),
+ BrowseQS: scope.suffix(),
+ Cards: newCardView(prefs, docs),
+ }
+ // A partner that only reported the story has no publisher page.
+ if scope.hasValue(co, "publisher", a.Publisher) {
+ data.PublisherURL = scope.link(categoryValueURL("publisher", a.Publisher))
+ }
+ render(w, r, data)
+}
diff --git a/assemblies.go b/assemblies.go
new file mode 100644
index 0000000..3e12058
--- /dev/null
+++ b/assemblies.go
@@ -0,0 +1,273 @@
+// assemblies.go - hand-built sets of pages drawn from several documents,
+// loaded from data/assemblies.json.
+
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "log"
+ "net/http"
+ "net/url"
+ "os"
+ "strconv"
+ "strings"
+)
+
+type assemblyRef struct {
+ Doc string `json:"doc"`
+ Version string `json:"version,omitempty"`
+ Pages string `json:"pages"`
+}
+
+type assemblySlideSpec struct {
+ assemblyRef
+ Note string `json:"note,omitempty"`
+ Alt []assemblyRef `json:"alt,omitempty"`
+}
+
+type assemblySpec struct {
+ ID string `json:"id"`
+ Title string `json:"title"`
+ Note string `json:"note,omitempty"`
+ Slides []assemblySlideSpec `json:"slides"`
+}
+
+type AssemblyCopy struct {
+ Image string
+ Size imageSize
+ DocID string
+ DocTitle string
+ Version string
+ Page int
+ Source string // the outlets that published this copy
+ Path string // the document or version page it came from
+}
+
+type AssemblySlide struct {
+ AssemblyCopy
+ Pos int
+ Note string
+ Alts []AssemblyCopy
+}
+
+type Assembly struct {
+ ID string
+ Title string
+ Note string
+ Slides []AssemblySlide
+ DocList []string // documents the slides are shown from, in slide order
+ Copies int // copies the slides are shown from: a document or one of its versions
+}
+
+// parsePageList expands "1-3,7" into page numbers, keeping the order given.
+func parsePageList(s string) ([]int, error) {
+ var out []int
+ for _, part := range strings.Split(s, ",") {
+ part = strings.TrimSpace(part)
+ first, last, isRange := strings.Cut(part, "-")
+ if !isRange {
+ last = first
+ }
+ a, err := strconv.Atoi(strings.TrimSpace(first))
+ if err != nil {
+ return nil, fmt.Errorf("%q is not a page or a page range", part)
+ }
+ b, err := strconv.Atoi(strings.TrimSpace(last))
+ if err != nil {
+ return nil, fmt.Errorf("%q is not a page or a page range", part)
+ }
+ if a < 1 || b < a {
+ return nil, fmt.Errorf("%q is not a page or a page range", part)
+ }
+ for n := a; n <= b; n++ {
+ out = append(out, n)
+ }
+ }
+ return out, nil
+}
+
+// copySource names the outlets that published a copy, oldest entry first as
+// the metadata lists them. Partners that published no documents are left out.
+func copySource(d *Document, slug string) string {
+ var names []string
+ seen := map[string]bool{}
+ for _, p := range d.Publication {
+ key := valueKey(p.Publisher)
+ if p.ReportingOnly || p.Version != slug || key == "" || seen[key] {
+ continue
+ }
+ seen[key] = true
+ names = append(names, p.Publisher)
+ }
+ return strings.Join(names, ", ")
+}
+
+func (co *Corpus) resolveAssemblyRef(label string, r assemblyRef) ([]*AssemblyCopy, bool) {
+ pages, err := parsePageList(r.Pages)
+ if err != nil {
+ warnf("%s: %s: pages: %v - skipped", label, r.Doc, err)
+ return nil, false
+ }
+ out := make([]*AssemblyCopy, len(pages))
+ d := co.docIndex[r.Doc]
+ if d == nil {
+ warnf("%s: no document %q - skipped", label, r.Doc)
+ return out, true
+ }
+ images, sizes, base, path := d.Images, d.ImageSizes, imageBase(d.ID, ""), docPath(d.ID)
+ if r.Version != "" {
+ v := d.VersionFor(r.Version)
+ if v == nil {
+ warnf("%s: %s has no version %q - skipped", label, r.Doc, r.Version)
+ return out, true
+ }
+ images, sizes, base, path = v.Images, v.ImageSizes, imageBase(d.ID, v.Slug), versionPath(d.ID, v.Slug)
+ }
+ byNumber := make(map[int]int, len(images))
+ for i, name := range images {
+ if n, err := pageNumber(name); err == nil {
+ byNumber[n] = i
+ }
+ }
+ source := copySource(d, r.Version)
+ for i, n := range pages {
+ k, ok := byNumber[n]
+ if !ok {
+ warnf("%s: %s %s has no page image %d - skipped", label, r.Doc, r.Version, n)
+ continue
+ }
+ out[i] = &AssemblyCopy{
+ Image: base + url.PathEscape(images[k]), Size: sizes[k], DocID: d.ID, DocTitle: d.Title,
+ Version: r.Version, Page: n, Source: source, Path: path,
+ }
+ }
+ return out, true
+}
+
+func (co *Corpus) buildAssembly(spec assemblySpec) *Assembly {
+ label := "assemblies.json: " + spec.ID
+ a := &Assembly{ID: spec.ID, Title: strings.TrimSpace(spec.Title), Note: strings.TrimSpace(spec.Note)}
+ seenImage := map[string]bool{}
+ seenDoc := map[string]bool{}
+ seenCopy := map[[2]string]bool{}
+ for _, s := range spec.Slides {
+ copies, ok := co.resolveAssemblyRef(label, s.assemblyRef)
+ if !ok {
+ continue
+ }
+ alts := make([][]*AssemblyCopy, 0, len(s.Alt))
+ for _, ref := range s.Alt {
+ c, ok := co.resolveAssemblyRef(label, ref)
+ if !ok {
+ continue
+ }
+ if len(c) != len(copies) {
+ warnf("%s: alt %s lists %d pages for a run of %d - skipped", label, ref.Doc, len(c), len(copies))
+ continue
+ }
+ alts = append(alts, c)
+ }
+ for i, c := range copies {
+ if c == nil {
+ continue
+ }
+ if seenImage[c.Image] {
+ warnf("%s: %s page %d is listed twice", label, c.DocID, c.Page)
+ }
+ seenImage[c.Image] = true
+ if !seenDoc[c.DocID] {
+ seenDoc[c.DocID] = true
+ a.DocList = append(a.DocList, c.DocID)
+ }
+ slide := AssemblySlide{AssemblyCopy: *c, Pos: len(a.Slides) + 1, Note: s.Note}
+ seenCopy[[2]string{c.DocID, c.Version}] = true
+ for _, alt := range alts {
+ if alt[i] != nil {
+ slide.Alts = append(slide.Alts, *alt[i])
+ }
+ }
+ a.Slides = append(a.Slides, slide)
+ }
+ }
+ a.Copies = len(seenCopy)
+ return a
+}
+
+func (co *Corpus) loadAssemblies() {
+ co.assemblies = nil
+ co.assemblyIndex = map[string]*Assembly{}
+ co.assembliesByDoc = map[string][]*Assembly{}
+
+ p := projectPath("assemblies.json")
+ b, err := os.ReadFile(p)
+ if err != nil {
+ log.Printf("Note: %s not found - no assemblies.", p)
+ return
+ }
+ var specs []assemblySpec
+ if err := json.Unmarshal(b, &specs); err != nil {
+ warnf("%s is malformed: %v - assemblies disabled.", p, err)
+ return
+ }
+ for _, spec := range specs {
+ spec.ID = strings.TrimSpace(spec.ID)
+ if spec.ID == "" || !docIDSafe(spec.ID) {
+ warnf("assemblies.json: id %q is empty or unsafe in a URL - skipped", spec.ID)
+ continue
+ }
+ if co.assemblyIndex[spec.ID] != nil {
+ warnf("assemblies.json: id %q appears twice - keeping the first", spec.ID)
+ continue
+ }
+ a := co.buildAssembly(spec)
+ if len(a.Slides) == 0 {
+ warnf("assemblies.json: %s has no slides that can be shown - skipped", spec.ID)
+ continue
+ }
+ co.assemblies = append(co.assemblies, a)
+ co.assemblyIndex[a.ID] = a
+ linked := map[string]bool{}
+ link := func(id string) {
+ if !linked[id] {
+ linked[id] = true
+ co.assembliesByDoc[id] = append(co.assembliesByDoc[id], a)
+ }
+ }
+ for _, s := range a.Slides {
+ link(s.DocID)
+ for _, alt := range s.Alts {
+ link(alt.DocID)
+ }
+ }
+ }
+ log.Printf("Assemblies loaded: %d", len(co.assemblies))
+}
+
+// assemblyPath is an assembly's page; assemblies live under /explore.
+const assembliesPath = "/explore/assemblies"
+
+func assemblyPath(id string) string { return assembliesPath + "/" + url.PathEscape(id) }
+
+func (co *Corpus) handleAssemblies(w http.ResponseWriter, r *http.Request) {
+ prefs := readPrefs(r)
+ render(w, r, &assembliesPage{
+ Base: pageWith(prefs),
+ Assemblies: co.assemblies,
+ TextOnly: prefs.TextOnly,
+ })
+}
+
+func (co *Corpus) handleAssemblyView(w http.ResponseWriter, r *http.Request) {
+ a := co.assemblyIndex[r.PathValue("id")]
+ if a == nil {
+ notFound(w, r)
+ return
+ }
+ prefs := readPrefs(r)
+ render(w, r, &assemblyPage{
+ Base: pageWith(prefs),
+ Assembly: a,
+ TextOnly: prefs.TextOnly,
+ })
+}
diff --git a/browse.go b/browse.go
new file mode 100644
index 0000000..528a8dd
--- /dev/null
+++ b/browse.go
@@ -0,0 +1,2273 @@
+package main
+
+import (
+ "bytes"
+ "cmp"
+ "fmt"
+ "html/template"
+ "math/rand"
+ "net/http"
+ "net/url"
+ "regexp"
+ "slices"
+ "sort"
+ "strconv"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+)
+
+type facetEntry struct {
+ Name string
+ Count int
+ URL string
+}
+
+// facetLinks renders a metadata field's values as links to their category
+// pages, wherever a card, a table row, a search result or a document page
+// lists them. It takes the template's dict:
+//
+// Vals the values ("-" stands for none)
+// Cat the category's slug
+// Sep between the links: "dot", "space", or a comma when not given
+// AsList one
per link instead
+// Class the links' class
+// Active the value of the page the links are on, when it is this
+// category's: its own link is marked active
+// QS the collection scope the links carry ("?coll=..."), when the page
+// they open would otherwise leave out the document they come from
+//
+// The output is what html/template made of the same markup: every value
+// goes through htmlEscape.
+func facetLinks(p map[string]interface{}) (template.HTML, error) {
+ vals, ok := p["Vals"].([]string)
+ if !ok && p["Vals"] != nil {
+ return "", fmt.Errorf("facetLinks: Vals is %T, not a list of values", p["Vals"])
+ }
+ if len(vals) == 0 {
+ return "-", nil
+ }
+ cat, _ := p["Cat"].(string)
+ sep, _ := p["Sep"].(string)
+ class, _ := p["Class"].(string)
+ active, _ := p["Active"].(string)
+ qs, _ := p["QS"].(string)
+ asList, _ := p["AsList"].(bool)
+
+ var b strings.Builder
+ if asList {
+ b.WriteString(`
`)
+ }
+ for i, v := range vals {
+ switch {
+ case asList:
+ b.WriteString("
")
+ case i == 0:
+ case sep == "dot":
+ b.WriteString(` · `)
+ case sep == "space":
+ b.WriteString(" ")
+ default:
+ b.WriteString(", ")
+ }
+ on := active != "" && sameValue(v, active)
+ b.WriteString(`")
+ b.WriteString(htmlEscape(v))
+ b.WriteString("")
+ if asList {
+ b.WriteString("
")
+ }
+ }
+ if asList {
+ b.WriteString("
")
+ }
+ return template.HTML(b.String()), nil
+}
+
+func htmlEscape(s string) string {
+ if !strings.ContainsAny(s, "\x00\"&'+<>") && isASCII(s) {
+ return s
+ }
+ var b strings.Builder
+ b.Grow(len(s) + 16)
+ for i := 0; i < len(s); {
+ r, w := utf8.DecodeRuneInString(s[i:])
+ switch {
+ case r == 0:
+ b.WriteString("\uFFFD")
+ case r == '"':
+ b.WriteString(""")
+ case r == '&':
+ b.WriteString("&")
+ case r == '\'':
+ b.WriteString("'")
+ case r == '+':
+ b.WriteString("+")
+ case r == '<':
+ b.WriteString("<")
+ case r == '>':
+ b.WriteString(">")
+ case r >= 0xfdd0 && r <= 0xfdef, r >= 0xfff0 && r <= 0xffff:
+ fmt.Fprintf(&b, "%x;", r)
+ default:
+ b.WriteString(s[i : i+w])
+ }
+ i += w
+ }
+ return b.String()
+}
+
+// rankCounts returns the keys of a count map in rank order (most counted
+// first, ties alphabetically ignoring accents and case, then by the raw
+// spelling)
+func rankCounts(m map[string]int, n int) []string {
+ type keyed struct {
+ name, fold string
+ count int
+ }
+ all := make([]keyed, 0, len(m))
+ for k, v := range m {
+ all = append(all, keyed{k, normFoldString(k), v})
+ }
+ sort.Slice(all, func(i, j int) bool {
+ a, b := &all[i], &all[j]
+ if a.count != b.count {
+ return a.count > b.count
+ }
+ if a.fold != b.fold {
+ return a.fold < b.fold
+ }
+ return a.name < b.name
+ })
+ if n >= 0 && len(all) > n {
+ all = all[:n]
+ }
+ out := make([]string, len(all))
+ for i := range all {
+ out[i] = all[i].name
+ }
+ return out
+}
+
+func topFacets(m map[string]int, n int) []facetEntry {
+ names := rankCounts(m, n)
+ out := make([]facetEntry, len(names))
+ for i, k := range names {
+ out[i] = facetEntry{Name: k, Count: m[k]}
+ }
+ return out
+}
+
+func barPct(count, denom, floor int) int {
+ if denom <= 0 || count <= 0 {
+ return 0
+ }
+ pct := count * 100 / denom
+ if pct < floor {
+ pct = floor
+ }
+ return pct
+}
+
+func sortedYears(m map[string]int) (years []string, maxCount int) {
+ years = make([]string, 0, len(m))
+ for y, c := range m {
+ years = append(years, y)
+ if c > maxCount {
+ maxCount = c
+ }
+ }
+ sort.Strings(years)
+ return years, maxCount
+}
+
+func yearBars(m map[string]int, urlFor func(y string) string) []timelineBar {
+ years, maxCount := sortedYears(m)
+ if maxCount == 0 {
+ return nil
+ }
+ bars := make([]timelineBar, 0, len(years))
+ for _, y := range years {
+ bars = append(bars, timelineBar{
+ Year: y,
+ Count: m[y],
+ Pct: barPct(m[y], maxCount, 1),
+ URL: urlFor(y),
+ })
+ }
+ return bars
+}
+
+type homeFigures struct {
+ TopAgencies, TopPrograms, TopCountries, TopPeople,
+ TopJournalists, TopPublishers, TopTags []facetEntry
+ DocHist, PubHist []timelineBar
+}
+
+func (co *Corpus) buildHomeStats() {
+ co.homeStats = co.homeFigures(false)
+ co.homeStatsSIDtoday = co.homeFigures(true)
+ co.buildRandomPools()
+}
+
+func (co *Corpus) homeFigures(sidtoday bool) homeFigures {
+ scope := scopeOf(map[string]bool{collectionNews: true, collectionArchive: sidtoday})
+ top := func(slug string, n int) []facetEntry {
+ entries := topFacets(co.homeFacet(slug, sidtoday), n)
+ for i := range entries {
+ entries[i].URL = scope.link(categoryValueURL(slug, entries[i].Name))
+ }
+ return entries
+ }
+ f := homeFigures{
+ TopAgencies: top("agency", 10),
+ TopPrograms: top("program", 20),
+ TopCountries: top("country", 15),
+ TopPeople: top("people", 15),
+ TopJournalists: top("journalist", 12),
+ TopPublishers: top("publisher", 10),
+ TopTags: top("tag", 25),
+ }
+ f.DocHist, f.PubHist = co.homeHistograms(scope)
+ return f
+}
+
+func (co *Corpus) homeFacet(slug string, sidtoday bool) map[string]int {
+ out := make(map[string]int)
+ for value, c := range co.archive.FacetsByColl[slug] {
+ n := c.News
+ if sidtoday {
+ n += c.Archive
+ }
+ if n > 0 {
+ out[value] = n
+ }
+ }
+ return out
+}
+
+func (co *Corpus) homeHistograms(scope collScope) (docHist, pubHist []timelineBar) {
+ docYears := map[string]int{}
+ pubYears := map[string]int{}
+ for i := range co.archive.Docs {
+ d := &co.archive.Docs[i]
+ if !scope.shows(d) {
+ continue
+ }
+ if y := docYear(d); y != "" {
+ docYears[y]++
+ }
+ if y := d.FirstPubYear(); y != "" {
+ pubYears[y]++
+ }
+ }
+ link := func(slug string) func(string) string {
+ return func(y string) string { return scope.link(categoryValueURL(slug, y)) }
+ }
+ return yearBars(docYears, link("docdate")), yearBars(pubYears, link("published"))
+}
+
+func (co *Corpus) addFacetNav(b *Base, prefs Prefs) {
+ b.ShowCatBoxes = true
+ b.CatCounts = co.categoryValueCounts[prefs.IncludeFOIA]
+}
+
+func (co *Corpus) handleIndex(w http.ResponseWriter, r *http.Request) {
+ prefs := readPrefs(r)
+ var recentDocs []*Document
+ if prefs.TrackRecent {
+ for _, id := range readIDListCookie(r, recentDocsCookie, recentDocsMax) {
+ if d := co.docIndex[id]; d != nil {
+ recentDocs = append(recentDocs, d)
+ }
+ }
+ }
+ if len(recentDocs) > 0 {
+ noStore(w)
+ }
+
+ withSIDtoday := r.URL.Query().Get("sidtoday") == "1"
+ stats := co.homeStats
+ if withSIDtoday {
+ stats = co.homeStatsSIDtoday
+ }
+ render(w, r, &indexPage{
+ Base: pageWith(prefs),
+ homeFigures: stats,
+ TrackRecent: prefs.TrackRecent,
+ RecentDocs: recentDocs,
+ SnowdenTotal: co.archive.CollectionCounts[collectionNews] + co.archive.CollectionCounts[collectionArchive],
+ ArchiveTotal: co.archive.CollectionCounts[collectionArchive],
+ FOIATotal: co.archive.CollectionCounts[collectionFOIA],
+ IncludeFOIA: prefs.IncludeFOIA,
+ ReportedTotal: co.archive.CollectionCounts[collectionNews],
+ WithSIDtoday: withSIDtoday,
+ })
+}
+
+// randomWeights says how often /random lands in each collection
+var randomWeights = []struct {
+ coll string
+ weight int
+}{
+ {collectionNews, 65},
+ {collectionArchive, 25},
+ {collectionFOIA, 10},
+}
+
+func (co *Corpus) buildRandomPools() {
+ co.randomPools = map[string][]int{}
+ for i, d := range co.archive.Docs {
+ co.randomPools[d.Collection] = append(co.randomPools[d.Collection], i)
+ }
+}
+
+func (co *Corpus) buildCollectionDocs() {
+ co.everyDoc = make([]*Document, len(co.archive.Docs))
+ co.collectionDocs = make(map[string][]*Document, len(validCollections))
+ co.sidtodayDocs = nil
+ for i := range co.archive.Docs {
+ d := &co.archive.Docs[i]
+ co.everyDoc[i] = d
+ co.collectionDocs[d.Collection] = append(co.collectionDocs[d.Collection], d)
+ if d.Type == "SIDTODAY" {
+ co.sidtodayDocs = append(co.sidtodayDocs, d)
+ }
+ }
+}
+
+func (co *Corpus) randomDoc(includeFOIA bool) *Document {
+ pool := func(coll string) []int {
+ if coll == collectionFOIA && !includeFOIA {
+ return nil
+ }
+ return co.randomPools[coll]
+ }
+ total := 0
+ for _, rw := range randomWeights {
+ if len(pool(rw.coll)) > 0 {
+ total += rw.weight
+ }
+ }
+ if total == 0 {
+ return nil
+ }
+ n := rand.Intn(total)
+ for _, rw := range randomWeights {
+ p := pool(rw.coll)
+ if len(p) == 0 {
+ continue
+ }
+ if n < rw.weight {
+ return &co.archive.Docs[p[rand.Intn(len(p))]]
+ }
+ n -= rw.weight
+ }
+ return nil
+}
+
+func (co *Corpus) handleRandom(w http.ResponseWriter, r *http.Request) {
+ d := co.randomDoc(readPrefs(r).IncludeFOIA)
+ if d == nil {
+ notFound(w, r)
+ return
+ }
+ http.Redirect(w, r, docPath(d.ID), http.StatusFound)
+}
+
+func handleAbout(w http.ResponseWriter, r *http.Request) {
+ render(w, r, &aboutPage{})
+}
+
+func paginate(total, page, perPage int) (start, end, clampedPage, totalPages int) {
+ if perPage <= 0 {
+ return 0, total, 1, 1
+ }
+ totalPages = (total + perPage - 1) / perPage
+
+ if page < 1 {
+ page = 1
+ }
+ maxPage := totalPages
+ if maxPage < 1 {
+ maxPage = 1
+ }
+ if page > maxPage {
+ page = maxPage
+ }
+ start = (page - 1) * perPage
+ if start < 0 || start > total {
+ start = total
+ }
+ end = start + perPage
+ if end < start || end > total {
+ end = total
+ }
+ return start, end, page, totalPages
+}
+
+// buildListView assembles everything the shared doc-cards / doc-table
+// layouts need: view, sort, pagination, the link bases their partials
+// build on, and the card display prefs.
+//
+// The page embeds what is returned and adds its own fields, including the
+// cards' ActiveCategory and ActiveValue when it has a filter value for them
+// to highlight.
+func (co *Corpus) buildListView(q url.Values, prefs Prefs, docs []*Document, basePath string, params url.Values, defaultSort string, explicitView bool) listView {
+ return co.buildListViewBoosted(q, prefs, docs, basePath, params, defaultSort, explicitView, "")
+}
+
+func (co *Corpus) buildListViewBoosted(q url.Values, prefs Prefs, docs []*Document, basePath string, params url.Values, defaultSort string, explicitView bool, boost string) listView {
+ view := q.Get("view")
+ sortBy := sortToken(q.Get("sort"))
+ order := sortToken(q.Get("order"))
+ if docSortByKey[sortBy] == nil {
+ sortBy = ""
+ }
+
+ effSort := cmp.Or(sortBy, defaultSort)
+ sortDocs(docs, effSort, order)
+ if sortBy == "" && boost != "" {
+ boostTitleMatches(docs, boost)
+ }
+
+ page, _ := strconv.Atoi(q.Get("p"))
+ finalDocs := docs
+ totalPages := 1
+ if view == "table" {
+ page = 1
+ } else {
+ var start, end int
+ start, end, page, totalPages = paginate(len(docs), page, prefs.cardsPerPage())
+ finalDocs = docs[start:end]
+ }
+
+ baseQ := cloneValues(params)
+ sortQ := cloneValues(params)
+ if view != "" && (explicitView || view == "table") {
+ baseQ.Set("view", view)
+ }
+
+ lv := listView{
+ Docs: finalDocs,
+ Total: len(docs),
+ Page: page,
+ TotalPages: totalPages,
+ View: view,
+ Sort: sortBy,
+ Order: order,
+ DefaultSort: defaultSort,
+ BaseURL: queryBase(basePath, baseQ),
+ SortURL: queryBase(basePath, sortQ),
+ TableHeads: sortHeads(queryBase(basePath, baseQ), docTableCols, effSort, order == "desc"),
+ Cards: newCardView(prefs, finalDocs),
+ }
+ if view == "table" {
+ lv.RowsHTML = co.buildDocRows(finalDocs, false)
+ }
+ return lv
+}
+
+// selectedTypes reads the ?type= params into a set, and reports whether the
+// request actually filters (no params means everything, not nothing).
+func selectedTypes(v url.Values) (map[string]bool, bool) {
+ sel := make(map[string]bool, len(v["type"]))
+ for _, t := range v["type"] {
+ if t = strings.ToUpper(strings.TrimSpace(t)); t != "" {
+ sel[t] = true
+ }
+ }
+ return sel, len(sel) > 0
+}
+
+func typeSelected(d *Document, sel map[string]bool) bool {
+ return len(sel) == 0 || sel[d.Type]
+}
+
+func typeCounts(docs []*Document) map[string]int {
+ counts := make(map[string]int)
+ for i := range docs {
+ counts[docs[i].Type]++
+ }
+ return counts
+}
+
+func filterByType(docs []*Document, sel map[string]bool) []*Document {
+ if len(sel) == 0 {
+ return docs
+ }
+ kept := docs[:0]
+ for _, d := range docs {
+ if typeSelected(d, sel) {
+ kept = append(kept, d)
+ }
+ }
+ return kept
+}
+
+func allTypesSelected(counts map[string]int) map[string]bool {
+ sel := make(map[string]bool, len(counts))
+ for t := range counts {
+ sel[t] = true
+ }
+ return sel
+}
+
+var validCollections = map[string]bool{
+ collectionNews: true,
+ collectionArchive: true,
+ collectionFOIA: true,
+}
+
+var collectionOrder = []string{collectionNews, collectionArchive, collectionFOIA}
+
+func collectionParams(sel map[string]bool) []string {
+ var out []string
+ for _, c := range collectionOrder {
+ if sel[c] {
+ out = append(out, c)
+ }
+ }
+ return out
+}
+
+// defaultCollections is what a page shows when its URL names no
+// collection: FOIA releases are opt-in (Prefs.IncludeFOIA).
+var defaultCollections = map[string]bool{collectionNews: true, collectionArchive: true}
+
+// collScope is a page's collection scope: the documents it shows and what
+// its own links carry. Every link a page builds from its documents goes
+// through link, so the number on a link is the number its click returns.
+type collScope struct {
+ filter map[string]bool // the documents shown; nil: every collection
+ shown map[string]bool // the same as a set, never nil (the form's boxes)
+ defaulted bool // the URL named no collection: the preference decided
+ colls []string // the coll= the page's links carry; nil: none
+ extra string // more scope for the links (a publisher page's year)
+}
+
+// scopeOf is an explicit scope: sel, with links that name it, so a count
+// holds whatever the next reader's FOIA preference.
+func scopeOf(sel map[string]bool) collScope {
+ s := collScope{shown: sel, colls: collectionParams(sel)}
+ if len(s.colls) < len(validCollections) {
+ s.filter = sel
+ }
+ return s
+}
+
+// wholeArchive scopes links to all three collections: a count taken over
+// the whole archive must not change with the reader's FOIA preference.
+var wholeArchive = scopeOf(validCollections)
+
+// pageScope reads a page's scope: the coll= its URL names or, when it
+// names none, the reader's preference, FOIA releases only when opted in.
+func pageScope(v url.Values, p Prefs, pinDefault bool) collScope {
+ if _, given := v["coll"]; given {
+ if f := collectionFilter(v); f != nil {
+ return scopeOf(f)
+ }
+ return wholeArchive
+ }
+ s := scopeOf(defaultCollections)
+ if p.IncludeFOIA {
+ s = wholeArchive
+ }
+ s.defaulted = true
+ if !pinDefault {
+ s.colls = nil
+ }
+ return s
+}
+
+func (s collScope) shows(d *Document) bool { return inCollections(d, s.filter) }
+
+func (s collScope) hasValue(co *Corpus, slug, value string) bool {
+ c := co.archive.FacetsByColl[slug][co.categoryNames[slug][valueKey(value)]]
+ if c == nil {
+ return false
+ }
+ in := func(coll string, n int) bool { return n > 0 && (s.filter == nil || s.filter[coll]) }
+ return in(collectionNews, c.News) || in(collectionArchive, c.Archive) || in(collectionFOIA, c.FOIA)
+}
+
+func (s collScope) docs(docs []*Document) []*Document { return filterByCollection(docs, s.filter) }
+
+func (s collScope) explicit() bool { return s.filter != nil && !s.defaulted }
+
+func (s collScope) including(coll string) collScope {
+ if s.filter == nil || s.filter[coll] {
+ return s
+ }
+ return wholeArchive
+}
+
+func (s collScope) and(qs string) collScope {
+ if s.extra != "" {
+ qs = s.extra + "&" + qs
+ }
+ s.extra = qs
+ return s
+}
+
+func (s collScope) qs() string {
+ q := ""
+ if len(s.colls) > 0 {
+ q = url.Values{"coll": s.colls}.Encode()
+ }
+ switch {
+ case q == "":
+ return s.extra
+ case s.extra != "":
+ return q + "&" + s.extra
+ }
+ return q
+}
+
+func (s collScope) link(u string) string { return withQuery(u, s.qs()) }
+
+func (s collScope) suffix() string {
+ if q := s.qs(); q != "" {
+ return "?" + q
+ }
+ return ""
+}
+
+func withQuery(link, qs string) string {
+ switch {
+ case qs == "" || link == "":
+ return link
+ case strings.Contains(link, "?"):
+ return link + "&" + qs
+ }
+ return link + "?" + qs
+}
+
+func cloneValues(v url.Values) url.Values {
+ out := make(url.Values, len(v))
+ for k, vs := range v {
+ out[k] = append([]string(nil), vs...)
+ }
+ return out
+}
+
+func (co *Corpus) handleBrowseRoot(w http.ResponseWriter, r *http.Request) {
+ q := r.URL.Query()
+ collection := q.Get("collection")
+ if !validCollections[collection] {
+ collection = collectionNews
+ }
+
+ if collection == collectionArchive {
+ co.renderArchiveBrowse(w, r, q)
+ return
+ }
+
+ prefs := readPrefs(r)
+
+ selTypes, typeFiltered := selectedTypes(q)
+ typeFiltered = typeFiltered && collection == collectionNews
+ selectedTypeSet := map[string]bool{}
+ if collection == collectionNews {
+ selectedTypeSet = selTypes
+ if !typeFiltered {
+ selectedTypeSet = allTypesSelected(co.archive.NewsTypes)
+ }
+ }
+
+ docs := append([]*Document(nil), co.collectionDocs[collection]...)
+ if collection == collectionNews {
+ docs = filterByType(docs, selectedTypeSet)
+ }
+
+ params := url.Values{}
+ if collection != collectionNews {
+ params.Set("collection", collection)
+ }
+ if typeFiltered {
+ params["type"] = q["type"]
+ }
+
+ data := &browsePage{
+ Base: pageWith(prefs),
+ listView: co.buildListView(q, prefs, docs, "/browse", params, "title", false),
+ Collection: collection,
+ CollectionCounts: co.archive.CollectionCounts,
+ SIDTodayCount: co.archive.SIDTodayCount,
+ SelectedTypes: selectedTypeSet,
+ AllTypes: co.archive.NewsTypes,
+ Params: q,
+ }
+ co.addFacetNav(&data.Base, prefs)
+ render(w, r, data)
+}
+
+// --- SIDtoday archive: chronological view ---
+type archiveMonth struct {
+ Name string
+ Docs []*Document
+}
+
+type archiveYear struct {
+ Year string
+ Count int
+ Months []archiveMonth
+}
+
+var monthNames = [...]string{"", "January", "February", "March", "April",
+ "May", "June", "July", "August", "September", "October", "November",
+ "December"}
+
+func (co *Corpus) buildArchiveTimeline() {
+ co.archiveTimeline = nil
+ docs := append([]*Document(nil), co.sidtodayDocs...)
+
+ sort.SliceStable(docs, func(i, j int) bool {
+ a, b := docs[i].DocMeta.DateCreated, docs[j].DocMeta.DateCreated
+ if (a == "") != (b == "") {
+ return b == ""
+ }
+ if a != b {
+ return a < b
+ }
+ ta, tb := strings.ToLower(docs[i].Title), strings.ToLower(docs[j].Title)
+ if ta == tb {
+ return false
+ }
+ return ta < tb
+ })
+
+ for _, d := range docs {
+ y, m := "Undated", ""
+ date := d.DocMeta.DateCreated
+ if len(date) >= 4 {
+ y = date[:4]
+ }
+ if len(date) >= 7 {
+ if n, err := strconv.Atoi(date[5:7]); err == nil && n >= 1 && n <= 12 {
+ m = monthNames[n]
+ }
+ }
+
+ if len(co.archiveTimeline) == 0 || co.archiveTimeline[len(co.archiveTimeline)-1].Year != y {
+ co.archiveTimeline = append(co.archiveTimeline, archiveYear{Year: y})
+ }
+ yr := &co.archiveTimeline[len(co.archiveTimeline)-1]
+ yr.Count++
+ if len(yr.Months) == 0 || yr.Months[len(yr.Months)-1].Name != m {
+ yr.Months = append(yr.Months, archiveMonth{Name: m})
+ }
+ mo := &yr.Months[len(yr.Months)-1]
+ mo.Docs = append(mo.Docs, d)
+ }
+}
+
+func (co *Corpus) renderArchiveBrowse(w http.ResponseWriter, r *http.Request, q url.Values) {
+ prefs := readPrefs(r)
+ view := q.Get("view")
+
+ data := &browsePage{
+ Base: pageWith(prefs),
+ Collection: collectionArchive,
+ CollectionCounts: co.archive.CollectionCounts,
+ SIDTodayCount: co.archive.SIDTodayCount,
+ SeriesList: co.seriesBySize[prefs.IncludeFOIA],
+ }
+ co.addFacetNav(&data.Base, prefs)
+
+ if view == "card" || view == "table" {
+ docs := append([]*Document(nil), co.sidtodayDocs...)
+ params := url.Values{"collection": {collectionArchive}}
+ data.listView = co.buildListView(q, prefs, docs, "/browse", params, "date", true)
+ data.MarkReported, data.Cards.MarkReported = true, true
+ if data.RowsHTML != "" {
+ data.RowsHTML = co.buildDocRows(data.Docs, true)
+ }
+ render(w, r, data)
+ return
+ }
+
+ data.Total = co.archive.SIDTodayCount
+ data.View = view
+ data.ArchiveYears = co.archiveTimeline
+ data.TimelineHTML = co.preHTML("archive-timeline")
+ render(w, r, data)
+}
+
+func docPath(id string) string { return "/doc/" + url.PathEscape(id) }
+
+func filePath(id, name string) string {
+ return "/files/" + url.PathEscape(id) + "/" + url.PathEscape(name)
+}
+
+func versionPath(id, slug string) string {
+ return docPath(id) + "/versions/" + url.PathEscape(slug)
+}
+
+func versionFilePath(id, slug, name string) string {
+ return "/files/" + url.PathEscape(id) + "/versions/" + url.PathEscape(slug) + "/" + url.PathEscape(name)
+}
+
+func imageBase(id, slug string) string {
+ if slug == "" {
+ return "/files/" + url.PathEscape(id) + "/images/"
+ }
+ return "/files/" + url.PathEscape(id) + "/versions/" + url.PathEscape(slug) + "/images/"
+}
+
+func (co *Corpus) publishedTogether(doc *Document) []*Document {
+ var out []*Document
+ seen := map[string]bool{doc.ID: true}
+ for _, p := range doc.Publication {
+ if p.Link == "" {
+ continue
+ }
+ a := co.articleByLink[p.Link]
+ if a == nil {
+ continue
+ }
+ for _, id := range a.DocIDs {
+ if seen[id] {
+ continue
+ }
+ seen[id] = true
+ if d := co.docIndex[id]; d != nil {
+ out = append(out, d)
+ }
+ }
+ }
+ return out
+}
+
+// ogImageAbs is the document's first page image, or its thumbnail when it
+// has no page images.
+func ogImageAbs(r *http.Request, doc *Document) string {
+ switch {
+ case len(doc.Images) > 0:
+ return requestBase(r) + imageBase(doc.ID, "") + url.PathEscape(doc.Images[0])
+ case doc.HasThumb:
+ return requestBase(r) + filePath(doc.ID, "thumb.jpg")
+ }
+ return ""
+}
+
+// queryBase is path plus q, ready for one more parameter to be appended.
+func queryBase(path string, q url.Values) string {
+ if enc := q.Encode(); enc != "" {
+ return path + "?" + enc + "&"
+ }
+ return path + "?"
+}
+
+func (co *Corpus) handleDocPages(w http.ResponseWriter, r *http.Request) {
+ id := r.PathValue("id")
+ doc := co.docIndex[id]
+ if doc == nil {
+ notFound(w, r)
+ return
+ }
+ prefs := readPrefs(r)
+ render(w, r, &pagesPage{
+ Base: pageWith(prefs),
+ viewer: viewer{Images: doc.Images, Sizes: doc.ImageSizes, ImageBase: imageBase(doc.ID, "")},
+ Doc: doc,
+ TextOnly: prefs.TextOnly,
+ })
+}
+
+func (co *Corpus) handleDocView(w http.ResponseWriter, r *http.Request) {
+ id := r.PathValue("id")
+
+ doc := co.docIndex[id]
+ if doc == nil {
+ notFound(w, r)
+ return
+ }
+
+ together := co.publishedTogether(doc)
+ shownTogether := together
+ var moreArticles []*Article
+ if len(together) > pubTogetherCap {
+ shownTogether = together[:pubTogetherCap]
+ moreArticles = co.docArticles(doc)
+ }
+
+ prefs := readPrefs(r)
+ scope := pageScope(nil, prefs, false).including(doc.Collection)
+ var relatedDocs []*Document
+ for _, relID := range co.archive.RelatedDocs[id] {
+ if related := co.docIndex[relID]; related != nil && scope.shows(related) {
+ relatedDocs = append(relatedDocs, related)
+ }
+ }
+
+ if prefs.TrackRecent && !crossSiteFetch(r) {
+ pushRecentDoc(w, r, id)
+ }
+
+ var serialPrevDoc, serialNextDoc *Document
+ if doc.Type == "SIDTODAY" {
+ if id, ok := co.serialPrev[doc.ID]; ok {
+ serialPrevDoc = co.docIndex[id]
+ }
+ if id, ok := co.serialNext[doc.ID]; ok {
+ serialNextDoc = co.docIndex[id]
+ }
+ }
+
+ embedBase := requestBase(r)
+ absURL := embedBase + docPath(doc.ID)
+ pdfURL := embedBase + filePath(doc.ID, doc.PDFFile)
+
+ render(w, r, &docPage{
+ Base: pageWith(prefs),
+ viewer: viewer{
+ Images: doc.Images,
+ Sizes: doc.ImageSizes,
+ ImageBase: imageBase(doc.ID, ""),
+ Content: doc.HTMLContent,
+ },
+ Doc: doc,
+ BrowseQS: scope.suffix(),
+ OGImageURL: ogImageAbs(r, doc),
+ SeriesGroup: co.seriesIndex[valueKey(doc.DocMeta.Series.Name)],
+ Assemblies: co.assembliesByDoc[doc.ID],
+ DocAbsURL: absURL,
+ Embeds: buildDocEmbeds(doc, absURL, pdfURL),
+ SerialPrev: serialPrevDoc,
+ SerialNext: serialNextDoc,
+ PublishedTogether: together,
+ PublishedShown: len(shownTogether),
+ PublishedMore: moreArticles,
+ RelatedDocs: relatedDocs,
+ PublishedCards: newCardView(prefs, shownTogether),
+ RelatedCards: newCardView(prefs, relatedDocs),
+ Pinned: isPinned(r, id),
+ })
+}
+
+func sortToken(s string) string {
+ for i := 0; i < len(s); i++ {
+ if s[i] < 'a' || s[i] > 'z' {
+ return ""
+ }
+ }
+ return s
+}
+
+// firstLetter returns the uppercase A-Z bucket for a facet value, folding
+// accents first ("Çağlar" -> "C"), with "#" for anything non-alphabetic.
+func firstLetter(s string) string {
+ s = strings.ToUpper(strings.TrimSpace(foldAccents(s)))
+ if s != "" && s[0] >= 'A' && s[0] <= 'Z' {
+ return string(s[0])
+ }
+ return "#"
+}
+
+// TODO split this huge ass file to separate files, browse_cateogry etc?
+
+// category describes one browsable dimension: its URL slug, the two page
+// headings, and the search field code its values map to
+type category struct {
+ Slug string
+ Title string // list page e.g. "Browse by Programs"
+ Label string // value page e.g. "Program: XKEYSCORE"
+ Field string // search field code (fqf), "" for date categories
+ DateParam string // "date_doc" / "date_pub" for the date categories, else ""
+ Blurb string
+}
+
+func (c *category) isDate() bool { return c.DateParam != "" }
+
+// facet = category's corpus counts (value -> documents)
+func (c *category) Facet(co *Corpus) map[string]int { return co.archive.Facets[c.Slug] }
+
+var categories = []category{
+ {"program", "Programs", "Program", "PR", "",
+ "Codenames and programs recorded for each document."},
+ {"journalist", "Journalists", "Journalist", "RE", "",
+ "Journalists bylined on the stories that published these documents. Contributors credited without a byline are searchable but not listed here."},
+ {"agency", "Agencies", "Agency", "CR", "",
+ "The agency that WROTE each document."},
+ {"publisher", "Publishers", "Publisher", "PU", "",
+ "Who published each document. Mostly a news outlet, but sometimes a book. A partner outlet that reported the story without publishing the documents keeps its byline and its article, but is not a publisher here."},
+ {"country", "Countries", "Country", "CO", "",
+ "Countries mentioned in each document. A country may be mentioned directly (e.g., France) or inferred from a related reference, such as a city (Paris -> France) or a domain name (.fr -> France)."},
+ {"people", "People", "Person", "PE", "",
+ "People who appear in these documents, named in the text or recognised in an image. An appearance implies nothing about the person beyond that appearance."},
+ {"classification", "Classification Levels", "Classification", "CL", "",
+ "Each document's classification level, as component values. If a document contains multiple classifications, the highest is used here."},
+ {"dissemination", "Dissemination Controls", "Dissemination control", "DI", "",
+ "Handling controls assessed for each document, as component values. If a document contains multiple disseminations, the most restrictive is used here."},
+ {"banner", "Classification Banners", "Banner", "BA", "",
+ "The marking line each document prints."},
+ {"genre", "Genres", "Genre", "GE", "",
+ "What kind of document it is: slides, report, wiki, manual, guide, newsletter."},
+ {"series", "Document Series", "Series", "SE", "",
+ "Recurring columns and multi-part runs in SIDtoday documents. A series lists its parts in the order the documents print. A missing part shows as a gap."},
+ {"published", "Publication Years", "Publication year", "", "date_pub",
+ "When a publisher first published each document. If a document is published multiple times, only the oldest one is used here."},
+ {"docdate", "Document Dates", "Document year", "", "date_doc",
+ "When each document was written, from its own dateline. Undated documents don't appear here."},
+ {"tag", "Tags", "Tag", "TA", "",
+ "Free-text subjects the curators added. Unlike the other axes these are not necessarily from the documents' own vocabulary, so they are the loosest thing to browse by (and the most forgiving)."},
+}
+
+// pubTogetherCap = on doc view, how many published together docs are shown, max
+const pubTogetherCap = 10
+
+func (co *Corpus) docArticles(doc *Document) []*Article {
+ var out []*Article
+ for _, p := range doc.Publication {
+ a := co.articleByLink[p.Link]
+ if a != nil && len(a.DocIDs) > 1 && !slices.Contains(out, a) {
+ out = append(out, a)
+ }
+ }
+ return out
+}
+
+func categoryValueURL(slug, value string) string {
+ return "/browse/" + slug + "/" + url.PathEscape(value)
+}
+
+func inCollections(d *Document, sel map[string]bool) bool {
+ return sel == nil || sel[d.Collection]
+}
+
+func filterByCollection(docs []*Document, sel map[string]bool) []*Document {
+ if sel == nil {
+ return docs
+ }
+ out := make([]*Document, 0, len(docs))
+ for _, d := range docs {
+ if inCollections(d, sel) {
+ out = append(out, d)
+ }
+ }
+ return out
+}
+
+func categoryBySlug(slug string) *category {
+ for i := range categories {
+ if categories[i].Slug == slug {
+ return &categories[i]
+ }
+ }
+ return nil
+}
+
+const azMinEntries = 50
+
+func (co *Corpus) prerenderCategoryRows() {
+ t := tmplSet["category.html"]
+ if t == nil {
+ return
+ }
+ exec := func(block string, data interface{}) (template.HTML, bool) {
+ var buf bytes.Buffer
+ if err := t.ExecuteTemplate(&buf, block, data); err != nil {
+ warnf("prerender category %s: %v - rendering rows per request", block, err)
+ return "", false
+ }
+ return template.HTML(buf.String()), true
+ }
+ co.categoryRowHTML = make(map[string][]template.HTML, len(co.categoryTables))
+ co.categoryRowSize = make(map[string]int, len(co.categoryTables))
+ co.azRowHTML = map[string]template.HTML{}
+ for ci := range categories {
+ for _, key := range []string{categories[ci].Slug, noFOIAKey(categories[ci].Slug)} {
+ entries := co.categoryTables[key].Entries
+ rows := make([]template.HTML, len(entries))
+ for i, e := range entries {
+ h, ok := exec("category-row", e)
+ if !ok {
+ return
+ }
+ rows[i] = h
+ co.categoryRowSize[key] += len(h)
+ if _, seen := co.azRowHTML[e.FirstLetter]; !seen {
+ if h, ok := exec("category-az-row", e.FirstLetter); ok {
+ co.azRowHTML[e.FirstLetter] = h
+ }
+ }
+ }
+ co.categoryRowSize[key] += 26 * 64
+ co.categoryRowHTML[key] = rows
+ }
+ }
+}
+
+func (co *Corpus) prerenderDocRows() {
+ t := tmplSet["browse.html"]
+ if t == nil {
+ return
+ }
+ rows := make(map[string]template.HTML, len(co.archive.Docs))
+ marked := map[string]template.HTML{}
+ var buf bytes.Buffer
+ render := func(row docTableRow) (template.HTML, bool) {
+ buf.Reset()
+ if err := t.ExecuteTemplate(&buf, "doc-table-row", row); err != nil {
+ warnf("prerender doc-table-row: %v - rendering table rows per request", err)
+ return "", false
+ }
+ return template.HTML(buf.String()), true
+ }
+ for i := range co.archive.Docs {
+ d := &co.archive.Docs[i]
+ row, ok := render(docTableRow{Document: d})
+ if !ok {
+ return
+ }
+ rows[d.ID] = row
+ if d.ReportedIssue() {
+ if marked[d.ID], ok = render(docTableRow{d, true}); !ok {
+ return
+ }
+ }
+ }
+ co.docRowHTML, co.docRowMarkedHTML = rows, marked
+}
+
+// buildDocRows joins the documents' pre-rendered table rows; markReported
+// takes the rows that badge an individually reported SIDtoday issue.
+func (co *Corpus) buildDocRows(docs []*Document, markReported bool) template.HTML {
+ rows := make([]template.HTML, len(docs))
+ n := 0
+ for i, d := range docs {
+ row, ok := co.docRowMarkedHTML[d.ID]
+ if !markReported || !ok {
+ row, ok = co.docRowHTML[d.ID]
+ }
+ if !ok {
+ return ""
+ }
+ rows[i] = row
+ n += len(row)
+ }
+ var b strings.Builder
+ b.Grow(n)
+ for _, row := range rows {
+ b.WriteString(string(row))
+ }
+ return template.HTML(b.String())
+}
+
+func azID(letter string) string {
+ if letter == "#" {
+ return "other"
+ }
+ return letter
+}
+
+func (co *Corpus) buildCategoryRows(key string, t *categoryTable, order []int32, showAZ bool) template.HTML {
+ rows := co.categoryRowHTML[key]
+ if len(rows) != len(t.Entries) {
+ return ""
+ }
+ var b strings.Builder
+ b.Grow(co.categoryRowSize[key])
+ prevL := ""
+ for _, n := range order {
+ if l := t.Entries[n].FirstLetter; showAZ && l != prevL {
+ b.WriteString(string(co.azRowHTML[l]))
+ prevL = l
+ }
+ b.WriteString(string(rows[n]))
+ }
+ return template.HTML(b.String())
+}
+
+func (co *Corpus) handleCategoryList(w http.ResponseWriter, r *http.Request) {
+ cat := r.PathValue("category")
+ q := r.URL.Query()
+ sortBy := sortToken(q.Get("sort"))
+ order := sortToken(q.Get("order"))
+
+ c := categoryBySlug(cat)
+ if c == nil {
+ notFound(w, r)
+ return
+ }
+
+ prefs := readPrefs(r)
+ collCols := cat != "series"
+ listSort := "name"
+ switch {
+ case sortBy == "count":
+ listSort = sortBy
+ case !collCols:
+ case sortBy == "reported", sortBy == "sidtoday":
+ listSort = sortBy
+ case sortBy == "foia":
+ if prefs.IncludeFOIA {
+ listSort = sortBy
+ }
+ }
+ desc := order == "desc"
+ if listSort != "name" {
+ desc = order != "asc"
+ }
+ key := readerListKey(cat, prefs)
+ table := co.categoryTables[key]
+ rowOrder := table.order(listSort, desc)
+
+ showAZ := listSort == "name" && len(rowOrder) >= azMinEntries
+ var letters []string
+ if showAZ {
+ letters = table.letters[desc]
+ }
+
+ collSort := listSort == "reported" || listSort == "sidtoday" || listSort == "foia"
+
+ nameCol := "NAME"
+ if c.isDate() {
+ nameCol = "YEAR"
+ }
+
+ cols := []sortCol{
+ {Key: "name", Label: nameCol},
+ {Key: "count", Label: "TOTAL", Class: "num-col", DescFirst: true},
+ }
+ if collCols {
+ cols = append(cols,
+ sortCol{Key: "reported", Label: "REPORTED", Class: "num-col coll-col", DescFirst: true},
+ sortCol{Key: "sidtoday", Label: "SIDTODAY", Class: "num-col coll-col", DescFirst: true})
+ }
+ if collCols && prefs.IncludeFOIA {
+ cols = append(cols, sortCol{Key: "foia", Label: "FOIA", Class: "num-col coll-col", DescFirst: true})
+ }
+
+ data := &categoryPage{
+ Base: pageWith(prefs),
+ Title: c.Title,
+ Blurb: c.Blurb,
+ Heads: sortHeads("?", cols, listSort, desc),
+ RowsHTML: co.buildCategoryRows(key, table, rowOrder, showAZ),
+ CollCols: collCols,
+ ShowCollCols: collCols && (prefs.ShowCollCols || collSort),
+ ShowAZ: showAZ,
+ Letters: letters,
+ }
+ data.ActiveCategory = cat
+ if data.RowsHTML == "" {
+ data.Entries = table.ordered(rowOrder)
+ }
+ co.addFacetNav(&data.Base, prefs)
+
+ switch cat {
+ case "classification", "dissemination":
+ data.SeeAlso = []seeAlsoLink{{"/browse/banner", "full classification banners (verbatim marking lines)"}}
+ case "banner":
+ data.SeeAlso = []seeAlsoLink{
+ {"/browse/classification", "classification components"},
+ {"/browse/dissemination", "dissemination controls"},
+ }
+ }
+
+ render(w, r, data)
+}
+
+// categoryEntry = one row of a /browse/{category} list page
+type categoryEntry struct {
+ Name string
+ URL template.URL // /browse/{category}/{name}, path-escaped
+ Note string // the agency expansion on the agency list, else ""
+ Count int // total, all collections
+ News int // type REPORTED (categoryTypeCounts), any collection
+ Archive int // type SIDTODAY, incl. individually reported issues
+ FOIA int // type FOIA
+ NonFOIA int // in the Reported and SIDtoday collections: the total when FOIA releases are left out
+ NoFOIA bool // a row of the FOIA-less table: no FOIA column
+ FirstLetter string
+ fold string // accent-folded lowercase name: the name sort key
+}
+
+// categoryTable = one category's list page
+type categoryTable struct {
+ Entries []categoryEntry // by name, ascending
+ orders map[string][]int32 // categoryListKey -> indices into Entries, in that order
+ letters map[bool][]string // the two name orderings' A-Z letters as they come, by desc
+}
+
+func (t *categoryTable) order(sortBy string, desc bool) []int32 {
+ return t.orders[categoryListKey(sortBy, desc)]
+}
+
+func (t *categoryTable) ordered(order []int32) []categoryEntry {
+ out := make([]categoryEntry, len(order))
+ for i, n := range order {
+ out[i] = t.Entries[n]
+ }
+ return out
+}
+
+// noFOIAKey is where a category's FOIA-less table is kept in categoryLists
+// and categoryRowHTML.
+func noFOIAKey(slug string) string { return slug + "|nofoia" }
+
+// readerListKey is the table a reader's category list comes from: the
+// FOIA-less one unless they include FOIA releases.
+func readerListKey(slug string, p Prefs) string {
+ if p.IncludeFOIA {
+ return slug
+ }
+ return noFOIAKey(slug)
+}
+
+// newCategoryTable sorts entries by name and works out the orderings the
+// sorts ask for, ascending and descending.
+func newCategoryTable(entries []categoryEntry, sorts ...string) *categoryTable {
+ sortCategoryEntries(entries, "name", false)
+ t := &categoryTable{Entries: entries, orders: map[string][]int32{}, letters: map[bool][]string{}}
+ for _, sortBy := range sorts {
+ for _, desc := range []bool{false, true} {
+ order := make([]int32, len(entries))
+ for i := range order {
+ order[i] = int32(i)
+ }
+ sort.Slice(order, func(i, j int) bool {
+ return categoryLess(&entries[order[i]], &entries[order[j]], sortBy, desc)
+ })
+ t.orders[categoryListKey(sortBy, desc)] = order
+ if sortBy != "name" {
+ continue
+ }
+ seen := map[string]bool{}
+ for _, n := range order {
+ if l := entries[n].FirstLetter; !seen[l] {
+ seen[l] = true
+ t.letters[desc] = append(t.letters[desc], l)
+ }
+ }
+ }
+ }
+ return t
+}
+
+func categoryListKey(sortBy string, desc bool) string {
+ if desc {
+ return sortBy + "/desc"
+ }
+ return sortBy + "/asc"
+}
+
+func (co *Corpus) buildCategoryLists() {
+ co.categoryTables = make(map[string]*categoryTable, 2*len(categories))
+ co.categoryValueCounts = map[bool]map[string]int{false: {}, true: {}}
+ for ci := range categories {
+ c := &categories[ci]
+ byType := co.categoryTypeCounts[c.Slug]
+ byColl := co.archive.FacetsByColl[c.Slug]
+ facet := c.Facet(co)
+ base := make([]categoryEntry, 0, len(facet))
+ for name, count := range facet {
+ e := categoryEntry{
+ Name: name,
+ URL: template.URL(categoryValueURL(c.Slug, name)),
+ Count: count,
+ FirstLetter: firstLetter(name),
+ fold: normFoldString(name),
+ }
+ if c.Slug == "agency" {
+ e.Note = agencyName(name)
+ }
+ if tc := byType[name]; tc != nil {
+ e.News, e.Archive, e.FOIA = tc.News, tc.Archive, tc.FOIA
+ }
+ if cc := byColl[name]; cc != nil {
+ e.NonFOIA = cc.News + cc.Archive
+ }
+ base = append(base, e)
+ }
+ noFOIA := make([]categoryEntry, 0, len(base))
+ for i := range base {
+ e := base[i]
+ if e.NonFOIA > 0 {
+ e.Count = e.NonFOIA
+ e.NoFOIA = true
+ noFOIA = append(noFOIA, e)
+ }
+ }
+ co.categoryTables[c.Slug] = newCategoryTable(base, "name", "count", "reported", "sidtoday", "foia")
+ co.categoryTables[noFOIAKey(c.Slug)] = newCategoryTable(noFOIA, "name", "count", "reported", "sidtoday")
+ co.categoryValueCounts[true][c.Slug] = len(base)
+ co.categoryValueCounts[false][c.Slug] = len(noFOIA)
+ }
+ co.seriesBySize = map[bool][]categoryEntry{}
+ for _, foia := range []bool{false, true} {
+ t := co.categoryTables[readerListKey("series", Prefs{IncludeFOIA: foia})]
+ co.seriesBySize[foia] = t.ordered(t.order("count", true))
+ }
+}
+
+func sortCategoryEntries(entries []categoryEntry, sortBy string, desc bool) {
+ sort.Slice(entries, func(i, j int) bool {
+ return categoryLess(&entries[i], &entries[j], sortBy, desc)
+ })
+}
+
+// categoryLess orders two rows of a category table: by the sorted column in
+// the direction asked, then the larger total first, then by name.
+func categoryLess(a, b *categoryEntry, sortBy string, desc bool) bool {
+ col := func(e *categoryEntry) int {
+ switch sortBy {
+ case "count":
+ return e.Count
+ case "reported":
+ return e.News
+ case "sidtoday":
+ return e.Archive
+ case "foia":
+ return e.FOIA
+ }
+ return 0
+ }
+ byName := sortBy == "name"
+ if !byName {
+ if x, y := col(a), col(b); x != y {
+ if desc {
+ return x > y
+ }
+ return x < y
+ }
+ if a.Count != b.Count {
+ return a.Count > b.Count
+ }
+ }
+ if a.fold != b.fold {
+ if byName && desc {
+ return a.fold > b.fold
+ }
+ return a.fold < b.fold
+ }
+ return a.Name < b.Name
+}
+
+type seeAlsoLink struct {
+ URL string
+ Text string
+}
+
+type timelineBar struct {
+ Year string
+ Count int
+ Pct int
+ URL string
+}
+
+func listSortKey(vals []string) string {
+ best := ""
+ for _, v := range vals {
+ lv := strings.ToLower(strings.TrimSpace(v))
+ if lv != "" && (best == "" || lv < best) {
+ best = lv
+ }
+ }
+ return best
+}
+
+var classAlias = map[string]string{
+ "ts": "top secret", "s": "secret", "c": "confidential", "u": "unclassified",
+ "strap1": "strap 1", "strap2": "strap 2", "talent keyhole": "tk",
+}
+
+func foldClass(v string) string {
+ lv := strings.ToLower(strings.TrimSpace(v))
+ if a, ok := classAlias[lv]; ok {
+ return a
+ }
+ return lv
+}
+
+// classSortKey ranks a marking set per the curators' order: level first
+// (TOP SECRET > SECRET > CONFIDENTIAL > UNCLASSIFIED), a level WITH SI
+// above the bare level, and the restrictive compartments raising rank
+// cumulatively - STRAP 2 (weight 4) > STRAP 1 (2), TK and MR (1 each) -
+// so TOP SECRET SI TK outranks TOP SECRET SI.
+func classSortKey(vals []string) string {
+ if len(vals) == 0 {
+ return ""
+ }
+ level, si, raise := 0, 0, 0
+ for _, v := range vals {
+ switch foldClass(v) {
+ case "top secret":
+ if level < 4 {
+ level = 4
+ }
+ case "secret":
+ if level < 3 {
+ level = 3
+ }
+ case "confidential":
+ if level < 2 {
+ level = 2
+ }
+ case "unclassified":
+ if level < 1 {
+ level = 1
+ }
+ case "si", "comint":
+ si = 1
+ case "strap 2":
+ raise += 4
+ case "strap 1":
+ raise += 2
+ case "tk", "mr":
+ raise++
+ }
+ }
+ if level == 0 && si == 0 && raise == 0 {
+ return ""
+ }
+ r := strconv.Itoa(raise)
+ if len(r) < 2 {
+ r = "0" + r
+ }
+ return strconv.Itoa(level) + strconv.Itoa(si) + r
+}
+
+// TODO this file really needs the split.
+
+// searchClause is one row of the search form: the field code, the value as
+// typed into it and how the row joins the ones before it.
+type searchClause struct{ Conj, Field, Value string }
+
+// searchURL builds an executed /search link
+func searchURL(clauses []searchClause, ranges ...[2]string) string {
+ q := url.Values{}
+ q.Set("do_search", "1")
+ q.Set("action", "search")
+ for _, c := range clauses {
+ conj := c.Conj
+ if conj == "" {
+ conj = "and"
+ }
+ q.Add("fqc", conj)
+ q.Add("fqv", c.Value)
+ q.Add("fqf", c.Field)
+ }
+ for _, r := range ranges {
+ q.Set(r[0]+"_start", r[1])
+ q.Set(r[0]+"_end", r[1])
+ }
+ return "/search?" + q.Encode()
+}
+
+func dateRangeSearchURL(fqf, fqv string, ranges ...[2]string) string {
+ return searchURL([]searchClause{{Field: fqf, Value: fqv}}, ranges...)
+}
+
+func appendClause(base, field, value string) string {
+ return base + "fqf=" + url.QueryEscape(field) +
+ "&fqv=" + url.QueryEscape(value) + "&fqc=and"
+}
+
+// entityTimelineBy is the per-year histogram behind both value-page
+// timelines. docs are the page's documents, already narrowed to its
+// collection scope, so every bar link carries that scope too.
+// yearsOf returns a document's distinct years on the bar axis.
+func entityTimelineBy(docs []*Document, c *category, val string, scope collScope, dateParam string, yearsOf func(*Document) []string) []timelineBar {
+ counts := map[string]int{}
+ for _, d := range docs {
+ for _, y := range yearsOf(d) {
+ counts[y]++
+ }
+ }
+ if len(counts) < 2 {
+ return nil
+ }
+ pageParam := ""
+ if c.isDate() && validDatePrefix(val) {
+ pageParam = c.DateParam
+ }
+ return yearBars(counts, func(y string) string {
+ switch {
+ case c.Field != "":
+ return scope.link(dateRangeSearchURL(c.Field, exactClause(val), [2]string{dateParam, y}))
+ case pageParam == "date_doc" && dateParam == "date_pub":
+ return scope.link(dateRangeSearchURL("TE", "", [2]string{pageParam, val}, [2]string{dateParam, y}))
+ }
+ return ""
+ })
+}
+
+func entityTimeline(docs []*Document, c *category, val string, scope collScope) []timelineBar {
+ return entityTimelineBy(docs, c, val, scope, "date_doc", func(d *Document) []string {
+ if y := docYear(d); y != "" {
+ return []string{y}
+ }
+ return nil
+ })
+}
+
+func entityPubTimeline(docs []*Document, c *category, val string, scope collScope) []timelineBar {
+ switch c.Slug {
+ case "journalist", "publisher":
+ return entityTimelineBy(docs, c, val, scope, "date_pub", func(d *Document) []string {
+ return pubYearsMatching(d, c.Field, val)
+ })
+ }
+ return entityTimelineBy(docs, c, val, scope, "date_pub", pubYearsOf)
+}
+
+type monthNav struct {
+ Months []monthNavEntry
+ Year string
+ YearURL string
+ YearActive bool
+ Monthless int
+}
+
+func (co *Corpus) buildMonthNav(c *category, val string, allMatching []*Document, scope collScope) *monthNav {
+ cat := c.Slug
+ if !c.isDate() || !validDatePrefix(val) {
+ return nil
+ }
+ year := val[:4]
+ yearDocs := allMatching
+ if len(val) == 7 {
+ yearDocs = scope.docs(co.categoryMatches(cat, year))
+ }
+ counts := map[string]int{}
+ for _, d := range yearDocs {
+ ds := d.DocMeta.DateCreated
+ if cat == "published" {
+ ds = oldestPubDate(d)
+ }
+ if len(ds) >= 7 {
+ counts[ds[5:7]]++
+ }
+ }
+ var months []monthNavEntry
+ shown := 0
+ for m := 1; m <= 12; m++ {
+ mm := strconv.Itoa(m)
+ if m < 10 {
+ mm = "0" + mm
+ }
+ c := counts[mm]
+ if c == 0 {
+ continue
+ }
+ shown += c
+ months = append(months, monthNavEntry{
+ Label: monthNames[m][:3],
+ URL: scope.link(categoryValueURL(cat, year+"-"+mm)),
+ Count: c,
+ Active: len(val) == 7 && val[5:7] == mm,
+ })
+ }
+ if len(months) == 0 {
+ return nil
+ }
+ return &monthNav{
+ Months: months,
+ Year: year,
+ YearURL: scope.link(categoryValueURL(cat, year)),
+ YearActive: len(val) == 4,
+ Monthless: len(yearDocs) - shown,
+ }
+}
+
+// bdGroup = one of a value page's breakdown charts (its banners, its
+// classification levels), each value a bar.
+type bdGroup struct {
+ Label string
+ Values []bdVal
+}
+
+type bdVal struct {
+ Value string
+ Count int
+ URL string
+ Pct int
+}
+
+func (co *Corpus) handleCategoryFilter(w http.ResponseWriter, r *http.Request) {
+ cat := r.PathValue("category")
+ val := r.PathValue("value")
+ c := categoryBySlug(cat)
+ if c == nil {
+ notFound(w, r)
+ return
+ }
+ if name, ok := co.categoryNames[cat][valueKey(val)]; ok && name != val {
+ u := categoryValueURL(cat, name)
+ if r.URL.RawQuery != "" {
+ u += "?" + r.URL.RawQuery
+ }
+ http.Redirect(w, r, u, http.StatusMovedPermanently)
+ return
+ }
+ prefs := readPrefs(r)
+ q := r.URL.Query()
+
+ selectedTypeSet, typeFiltered := selectedTypes(q)
+
+ scope := pageScope(q, prefs, false)
+
+ everything := co.categoryMatches(cat, val)
+ allMatching := scope.docs(everything)
+ pubYear := ""
+ if y := q.Get("year"); cat == "publisher" && len(y) == 4 && allDigits(y) {
+ pubYear = y
+ allMatching = publishedBy(allMatching, val, y)
+ }
+ viewTypeCounts := typeCounts(allMatching)
+ if !typeFiltered {
+ selectedTypeSet = allTypesSelected(co.archive.Types)
+ }
+ results := append([]*Document(nil), allMatching...)
+ results = filterByType(results, selectedTypeSet)
+
+ basePath := categoryValueURL(cat, val)
+ params := url.Values{}
+ if typeFiltered {
+ params["type"] = q["type"]
+ }
+ if scope.colls != nil {
+ params["coll"] = scope.colls
+ }
+ if pubYear != "" {
+ params.Set("year", pubYear)
+ }
+ defSort, boost := "title", val
+ if cat == "series" {
+ defSort = "series"
+ }
+ if cat == "series" || c.isDate() {
+ boost = ""
+ }
+ data := &listPage{
+ Base: pageWith(prefs),
+ listView: co.buildListViewBoosted(q, prefs, results, basePath, params, defSort, false, boost),
+ }
+
+ searchLink := ""
+ if c.Field != "" {
+ searchLink = searchURL([]searchClause{
+ {Field: "TE"},
+ {Field: c.Field, Value: exactClause(val)},
+ })
+ } else if c.isDate() && validDatePrefix(val) {
+ searchLink = dateRangeSearchURL("TE", "", [2]string{c.DateParam, val})
+ }
+
+ docScope := scope
+ if pubYear != "" {
+ docScope = scope.and(url.Values{"date_pub_start": {pubYear}, "date_pub_end": {pubYear}}.Encode())
+ }
+ searchLink = docScope.link(searchLink)
+
+ if mn := co.buildMonthNav(c, val, allMatching, scope); mn != nil {
+ data.MonthNav = mn.Months
+ data.MonthNavYear = mn.Year
+ data.MonthNavYearURL = mn.YearURL
+ data.MonthNavYearActive = mn.YearActive
+ if mn.Monthless > 0 {
+ data.MonthlessCount = mn.Monthless
+ }
+ }
+
+ var gloss *GlossaryEntry
+ if cat == "program" {
+ gloss = co.glossaryLookup(val)
+ }
+
+ data.GlossaryEntry = gloss
+ if pubYear != "" {
+ data.YearScope = pubYear
+ data.YearScopeAllURL = scope.link(basePath)
+ }
+ refs := categoryRefinements(allMatching, c, val, docScope, pubYear)
+ data.Refinements = refs
+
+ var breakdowns []bdGroup
+ for _, g := range refs {
+ if g.Label != "Banner" && g.Label != "Classification" {
+ continue
+ }
+ top := 0
+ for _, v := range g.Values {
+ if v.Count > top {
+ top = v.Count
+ }
+ }
+ bg := bdGroup{Label: g.Label}
+ for _, v := range g.Values {
+ bg.Values = append(bg.Values, bdVal{v.Name, v.Count, v.URL, barPct(v.Count, top, 2)})
+ }
+ breakdowns = append(breakdowns, bg)
+ }
+ data.Breakdowns = breakdowns
+ data.Timeline = entityTimeline(allMatching, c, val, docScope)
+ if pubYear == "" {
+ // a year-scoped page has one publication year: no timeline
+ data.PubTimeline = entityPubTimeline(allMatching, c, val, scope)
+ }
+ co.addFacetNav(&data.Base, prefs)
+ data.Title = val
+ data.ActiveCategory = cat
+ data.Cards.ActiveCategory, data.Cards.ActiveValue = cat, val
+ data.SearchLink = searchLink
+ data.SelectedTypes = selectedTypeSet
+ data.AllTypes = viewTypeCounts
+ data.Params = q
+ data.CatLabel = c.Label
+ data.Blurb = c.Blurb
+ render(w, r, data)
+}
+
+// categoryRefinements builds the "combine with another filter" facet groups
+// for a category value page. For the docs matching this page's value, tally
+// the other browsable dimensions and link each value to a TWO-clause search
+// (this page's value AND that value). On a publisher page scoped to pubYear
+// the links carry that publication year, and the tallies read the
+// journalists the way that search will (refineValues).
+func categoryRefinements(docs []*Document, c *category, val string, scope collScope, pubYear string) []searchFacet {
+ if c.Field == "" || len(docs) < 2 {
+ return nil
+ }
+ base := scope.link(searchURL([]searchClause{{Field: c.Field, Value: exactClause(val)}})) + "&"
+
+ groups := []facetGroup{
+ {"Banner", "BA"}, {"Classification", "CL"}, {"Dissemination", "DI"}, {"Genre", "GE"},
+ {"Programs", "PR"}, {"Agencies", "CR"}, {"Countries", "CO"}, {"Journalists", "RE"},
+ }
+ kept := groups[:0]
+ for _, g := range groups {
+ if g.Code != c.Field {
+ kept = append(kept, g)
+ }
+ }
+ return facetGroups(docs, kept, base, refineValues(pubYear, pubYear))
+}
+
+func (co *Corpus) buildCategoryDocs() {
+ co.categoryDocs = make(map[string]map[string][]int32, len(categories))
+ co.categoryNames = make(map[string]map[string]string, len(categories))
+ co.archive.Facets = make(map[string]map[string]int, len(categories))
+ co.archive.FacetsByColl = make(map[string]map[string]*collCounts, len(categories))
+ co.categoryTypeCounts = make(map[string]map[string]*collCounts, len(categories))
+ for ci := range categories {
+ c := &categories[ci]
+ byValue := make(map[string][]int32)
+ name := make(map[string]string) // index key -> display value
+ add := func(key, value string, i int) {
+ if _, ok := name[key]; !ok {
+ name[key] = value
+ }
+ byValue[key] = append(byValue[key], int32(i))
+ }
+ for i := range co.archive.Docs {
+ d := &co.archive.Docs[i]
+ if c.isDate() {
+ date := d.DocMeta.DateCreated
+ if c.DateParam == "date_pub" {
+ date = oldestPubDate(d)
+ }
+ if len(date) >= 4 {
+ add(date[:4], date[:4], i)
+ }
+ if len(date) >= 7 {
+ add(date[:7], date[:7], i)
+ }
+ continue
+ }
+ for _, v := range facetValues(d, c.Field) {
+ add(valueKey(v), v, i)
+ }
+ }
+ counts := make(map[string]int)
+ byColl := make(map[string]*collCounts)
+ byType := make(map[string]*collCounts)
+ for key, docs := range byValue {
+ if c.isDate() && len(key) != 4 {
+ continue
+ }
+ cc, tc := &collCounts{}, &collCounts{}
+ for _, n := range docs {
+ cc.add(co.archive.Docs[n].Collection)
+ tc.addType(co.archive.Docs[n].Type)
+ }
+ counts[name[key]] = len(docs)
+ byColl[name[key]] = cc
+ byType[name[key]] = tc
+ }
+ co.categoryDocs[c.Slug] = byValue
+ co.categoryNames[c.Slug] = name
+ co.archive.Facets[c.Slug] = counts
+ co.archive.FacetsByColl[c.Slug] = byColl
+ co.categoryTypeCounts[c.Slug] = byType
+ }
+}
+
+// categoryMatches returns the documents on a /browse/{category}/{value}
+// page, in corpus order, as a fresh slice of pointers the caller owns.
+func (co *Corpus) categoryMatches(cat, val string) []*Document {
+ c := categoryBySlug(cat)
+ if c == nil {
+ return nil
+ }
+ if c.isDate() && !validDatePrefix(val) {
+ return nil
+ }
+ idx := co.categoryDocs[cat][valueKey(val)]
+ out := make([]*Document, len(idx))
+ for i, n := range idx {
+ out[i] = &co.archive.Docs[n]
+ }
+ return out
+}
+
+func validDatePrefix(val string) bool {
+ switch len(val) {
+ case 4:
+ return allDigits(val)
+ case 7:
+ return allDigits(val[:4]) && val[4] == '-' && allDigits(val[5:])
+ }
+ return false
+}
+
+func validDate(val string) bool {
+ if len(val) == 10 {
+ return validDatePrefix(val[:7]) && val[7] == '-' && allDigits(val[8:])
+ }
+ return validDatePrefix(val)
+}
+
+func allDigits(s string) bool {
+ if s == "" {
+ return false
+ }
+ for i := 0; i < len(s); i++ {
+ if s[i] < '0' || s[i] > '9' {
+ return false
+ }
+ }
+ return true
+}
+
+func publishedBy(docs []*Document, publisher, year string) []*Document {
+ var out []*Document
+ for _, d := range docs {
+ for _, p := range d.Publication {
+ if !p.ReportingOnly && sameValue(p.Publisher, publisher) &&
+ strings.HasPrefix(p.DatePublished, year) {
+ out = append(out, d)
+ break
+ }
+ }
+ }
+ return out
+}
+
+func publisherYearURL(publisher, year string) string {
+ return categoryValueURL("publisher", publisher) + "?year=" + url.QueryEscape(year)
+}
+
+func oldestPubDate(d *Document) string {
+ oldest := ""
+ for _, p := range d.Publication {
+ if len(p.DatePublished) >= 4 && (oldest == "" || p.DatePublished < oldest) {
+ oldest = p.DatePublished
+ }
+ }
+ return oldest
+}
+
+func (d *Document) FirstPubYear() string {
+ if date := oldestPubDate(d); len(date) >= 4 {
+ return date[:4]
+ }
+ return ""
+}
+
+func docYear(d *Document) string {
+ if len(d.DocMeta.DateCreated) >= 4 {
+ return d.DocMeta.DateCreated[:4]
+ }
+ return ""
+}
+
+// pubYearsOf lists the years the document was published in, any entry's,
+// in the entries' order.
+func pubYearsOf(d *Document) []string {
+ if d.fields != nil {
+ return d.fields.pubYears
+ }
+ var out []string
+ for _, p := range d.Publication {
+ if len(p.DatePublished) < 4 {
+ continue
+ }
+ if y := p.DatePublished[:4]; !slices.Contains(out, y) {
+ out = append(out, y)
+ }
+ }
+ return out
+}
+
+func pubYearsMatching(d *Document, field, val string) []string {
+ var out []string
+ for i := range d.Publication {
+ date := d.Publication[i].DatePublished
+ if len(date) < 4 || slices.Contains(out, date[:4]) {
+ continue
+ }
+ for _, v := range entryValues(d, i, field) {
+ if sameValue(v, val) {
+ out = append(out, date[:4])
+ break
+ }
+ }
+ }
+ return out
+}
+
+type monthNavEntry struct {
+ Label string
+ URL string
+ Count int
+ Active bool
+}
+
+// docSort is one way to order a list of documents. The sort bar, the
+// doc-table's column headers, the list pages, search and /api/docs all read
+// this table, so a sort means the same thing everywhere.
+type docSort struct {
+ Key string
+ Label string
+ Value func(d *Document) string
+ EmptiesLast bool
+ DescFirst bool
+ InBar bool // offered in the sort bar
+}
+
+// titleSortKey sorts a title by its first letter or digit
+func titleSortKey(d *Document) string {
+ return strings.TrimLeftFunc(strings.ToLower(d.Title), func(r rune) bool {
+ return !unicode.IsLetter(r) && !unicode.IsDigit(r)
+ })
+}
+
+var docSorts = []docSort{
+ {"title", "Title", titleSortKey, false, false, true},
+ {"agency", "Agency", func(d *Document) string { return strings.ToLower(d.DocMeta.Creator) }, false, false, true},
+ {"date", "Doc Date", func(d *Document) string { return d.DocMeta.DateCreated }, true, false, true},
+ {"pubdate", "Pub Date", oldestPubDate, true, false, true},
+ {"classification", "Classification", func(d *Document) string { return classSortKey(d.DocMeta.Classification) }, true, true, false},
+ {"dissemination", "Dissemination", func(d *Document) string { return listSortKey(d.DocMeta.Dissemination) }, true, false, false},
+ {"program", "Programs", func(d *Document) string { return listSortKey(d.DocMeta.Programs) }, true, false, false},
+ {"country", "Countries", func(d *Document) string { return listSortKey(d.DocMeta.Countries) }, true, false, false},
+ {"people", "People", func(d *Document) string { return listSortKey(d.DocMeta.People) }, true, false, false},
+ {"journalist", "Journalists", func(d *Document) string { return listSortKey(d.JournalistsDeduped()) }, true, false, false},
+ {"type", "Type", func(d *Document) string { return strings.ToLower(d.Type) }, false, false, false},
+ {"series", "Series order", func(d *Document) string {
+ if p := d.DocMeta.Series.Part; p > 0 {
+ return fmt.Sprintf("%06d", p)
+ }
+ return ""
+ }, true, false, false},
+}
+
+var docSortByKey = func() map[string]*docSort {
+ m := make(map[string]*docSort, len(docSorts))
+ for i := range docSorts {
+ m[docSorts[i].Key] = &docSorts[i]
+ }
+ return m
+}()
+
+var docTableCols = func() []sortCol {
+ col := func(key string) sortCol {
+ s := docSortByKey[key]
+ return sortCol{Key: s.Key, Label: s.Label, DescFirst: s.DescFirst}
+ }
+ return []sortCol{
+ col("title"), col("agency"), col("date"), col("pubdate"),
+ col("classification"), col("dissemination"), col("program"),
+ col("country"), col("people"), col("journalist"),
+ {Label: "Published"},
+ col("type"),
+ }
+}()
+
+type sortCol struct {
+ Key, Label, Class string
+ DescFirst, Fixed bool
+}
+
+type sortHead struct {
+ sortCol
+ URL string
+ Active, Desc bool
+}
+
+func sortHeads(base string, cols []sortCol, cur string, desc bool) []sortHead {
+ heads := make([]sortHead, len(cols))
+ for i, c := range cols {
+ h := sortHead{sortCol: c}
+ if c.Key != "" {
+ next := c.DescFirst
+ if h.Active = c.Key == cur; h.Active {
+ h.Desc = desc
+ if !c.Fixed {
+ next = !desc
+ }
+ }
+ h.URL = base + "sort=" + c.Key
+ if !c.Fixed {
+ h.URL += "&order=" + orderParam(next)
+ }
+ }
+ heads[i] = h
+ }
+ return heads
+}
+
+func orderParam(desc bool) string {
+ if desc {
+ return "desc"
+ }
+ return "asc"
+}
+
+// sortCmp orders two sort keys: in the direction asked, an empty key after
+// the others either way where the sort puts its empties last.
+func sortCmp(a, b string, desc, emptiesLast bool) int {
+ if emptiesLast && (a == "") != (b == "") {
+ if b == "" {
+ return -1
+ }
+ return 1
+ }
+ if desc {
+ return cmp.Compare(b, a)
+ }
+ return cmp.Compare(a, b)
+}
+
+// boostTitleMatches moves the documents whose title has word as a whole word,
+// in any case, to the front, keeping each group's order e.g. on
+// /browse/tag/Tor the documents about Tor come before the ones that only carry
+// the tag.
+func boostTitleMatches(docs []*Document, word string) {
+ re, err := regexp.Compile(`(?i)\b` + regexp.QuoteMeta(word) + `\b`)
+ if err != nil {
+ return
+ }
+ var hit, rest []*Document
+ for _, d := range docs {
+ if re.MatchString(d.Title) {
+ hit = append(hit, d)
+ } else {
+ rest = append(rest, d)
+ }
+ }
+ copy(docs, append(hit, rest...))
+}
+
+func sortDocs(docs []*Document, sortBy, order string) {
+ desc := order == "desc"
+ if docSortByKey[sortBy] == nil {
+ sortBy = "title"
+ }
+
+ copy(docs, sortByField(docs, func(d *Document) *Document { return d }, sortBy, desc))
+}
+
+func sortScored(scored []ScoredDoc, sortBy, order string) []ScoredDoc {
+ if docSortByKey[sortBy] == nil {
+ return scored
+ }
+ return sortByField(scored, func(s ScoredDoc) *Document { return s.Doc }, sortBy, order == "desc")
+}
+
+func sortByField[T any](items []T, doc func(T) *Document, sortBy string, desc bool) []T {
+ ds := docSortByKey[sortBy]
+ keys := make([]string, len(items))
+ for i := range items {
+ keys[i] = ds.Value(doc(items[i]))
+ }
+ idx := make([]int, len(items))
+ for i := range idx {
+ idx[i] = i
+ }
+ slices.SortStableFunc(idx, func(a, b int) int {
+ return sortCmp(keys[a], keys[b], desc, ds.EmptiesLast)
+ })
+ out := make([]T, len(items))
+ for i, o := range idx {
+ out[i] = items[o]
+ }
+ return out
+}
+
+func (co *Corpus) docVersion(r *http.Request) (*Document, *DocVersion) {
+ doc := co.docIndex[r.PathValue("id")]
+ if doc == nil {
+ return nil, nil
+ }
+ return doc, doc.VersionFor(r.PathValue("slug"))
+}
+
+func (co *Corpus) handleVersionView(w http.ResponseWriter, r *http.Request) {
+ doc, v := co.docVersion(r)
+ if v == nil {
+ notFound(w, r)
+ return
+ }
+ pubs := doc.Publication[:0:0]
+ for _, i := range v.PubIndex {
+ pubs = append(pubs, doc.Publication[i])
+ }
+ prefs := readPrefs(r)
+ render(w, r, &versionPage{
+ Base: pageWith(prefs),
+ viewer: viewer{
+ Images: v.Images,
+ Sizes: v.ImageSizes,
+ ImageBase: imageBase(doc.ID, v.Slug),
+ Content: v.HTMLContent,
+ Version: v,
+ },
+ Doc: doc,
+ Publications: pubs,
+ PDFURL: versionFilePath(doc.ID, v.Slug, v.PDFFile),
+ })
+}
+
+func (co *Corpus) handleVersionPages(w http.ResponseWriter, r *http.Request) {
+ doc, v := co.docVersion(r)
+ if v == nil {
+ notFound(w, r)
+ return
+ }
+ prefs := readPrefs(r)
+ render(w, r, &pagesPage{
+ Base: pageWith(prefs),
+ viewer: viewer{Images: v.Images, Sizes: v.ImageSizes, ImageBase: imageBase(doc.ID, v.Slug), Version: v},
+ Doc: doc,
+ TextOnly: prefs.TextOnly,
+ })
+}
diff --git a/changelog.go b/changelog.go
new file mode 100644
index 0000000..9dd4f45
--- /dev/null
+++ b/changelog.go
@@ -0,0 +1,102 @@
+// changelog.go - the archive changelog (/changelog).
+//
+// The changelog is a HAND-MAINTAINED file: data/changelog.json. Shape:
+//
+// [
+// {
+// "date": "2026-06-11", // YYYY-MM-DD (or full RFC3339 if you care)
+// "type": "added", // added | removed | updated | note
+// "id": "some-doc-id", // which document (omit for type=note)
+// "title": "Document title", // shown in the headline (omit for note)
+// "components": ["pdf"], // optional, for updated: what changed
+// "note": "free text" // the body for type=note, optional otherwise
+// }
+// ]
+package main
+
+import (
+ "encoding/json"
+ "log"
+ "net/http"
+ "os"
+ "sort"
+ "strings"
+ "time"
+)
+
+type ChangeEntry struct {
+ Date string `json:"date"`
+ Type string `json:"type"`
+ ID string `json:"id,omitempty"`
+ Title string `json:"title,omitempty"`
+ Components []string `json:"components,omitempty"`
+ Note string `json:"note,omitempty"`
+}
+
+func (co *Corpus) loadChangelog() {
+ p := projectPath("changelog.json")
+ b, err := os.ReadFile(p)
+ if err != nil {
+ log.Printf("Note: %s not found - the changelog will be empty (see the format at the top of changelog.go).", p)
+ return
+ }
+ if err := json.Unmarshal(b, &co.changelog); err != nil {
+ warnf("%s is malformed: %v - changelog disabled.", p, err)
+ co.changelog = nil
+ return
+ }
+ sort.SliceStable(co.changelog, func(i, j int) bool {
+ return co.changelog[i].Date > co.changelog[j].Date
+ })
+ for _, e := range co.changelog {
+ if !validChangelogDate(e.Date) {
+ warnf("changelog entry %q has date %q - must be YYYY-MM-DD or RFC3339", e.Headline(), e.Date)
+ }
+ if e.Type != "removed" && e.ID != "" && co.docIndex[e.ID] == nil {
+ warnf("changelog entry %q names document %q, which is not in the corpus", e.Headline(), e.ID)
+ }
+ }
+ log.Printf("Changelog loaded: %d entries", len(co.changelog))
+}
+
+func validChangelogDate(d string) bool {
+ if _, err := time.Parse("2006-01-02", d); err == nil {
+ return true
+ }
+ _, err := time.Parse(time.RFC3339, d)
+ return err == nil
+}
+
+func (e ChangeEntry) Headline() string {
+ switch e.Type {
+ case "added":
+ return "Added: " + e.Title
+ case "removed":
+ return "Removed: " + e.Title
+ case "updated":
+ s := "Updated: " + e.Title
+ if len(e.Components) > 0 {
+ s += " (" + strings.Join(e.Components, ", ") + ")"
+ }
+ return s
+ default: // note
+ if e.Note != "" {
+ return e.Note
+ }
+ return "Archive note"
+ }
+}
+
+func (e ChangeEntry) DocURL() string {
+ if e.ID != "" && e.Type != "removed" {
+ return docPath(e.ID)
+ }
+ return "/changelog"
+}
+
+func (co *Corpus) handleChangelog(w http.ResponseWriter, r *http.Request) {
+ render(w, r, &changelogPage{
+ Entries: co.changelog,
+ Total: len(co.changelog),
+ })
+}
diff --git a/codewords.go b/codewords.go
new file mode 100644
index 0000000..1c19f7d
--- /dev/null
+++ b/codewords.go
@@ -0,0 +1,298 @@
+package main
+
+import (
+ "html/template"
+ "net/http"
+ "net/url"
+ "sort"
+ "strings"
+)
+
+// The codeword concordance: an automatically extracted index of ALL-CAPS
+// tokens across the corpus - program names, compartments, org designators.
+
+var codewordStoplist = map[string]bool{
+ "REDACTED": true,
+ "DYNAMIC": true,
+ "PAGE": true,
+ "HIGHEST": true,
+ "POSSIBLE": true,
+ "DERIVED": true,
+ "DECLASSIFY": true,
+ "JAN": true,
+ "ADDED": true,
+ "CLASSIFICATION": true,
+}
+
+type codewordEntry struct {
+ Token string
+ Count int // total occurrences across the corpus
+ Docs int // number of documents containing it
+ First string // earliest document year containing it ("" if only undated docs)
+ Last string // latest such year
+}
+
+func isCodewordToken(w string) bool {
+ if len(w) < 3 || w[0] < 'A' || w[0] > 'Z' {
+ return false
+ }
+ letters := 0
+ var seen [26]bool
+ distinct := 0
+ for i := 0; i < len(w); i++ {
+ c := w[i]
+ switch {
+ case c >= 'A' && c <= 'Z':
+ letters++
+ if !seen[c-'A'] {
+ seen[c-'A'] = true
+ distinct++
+ }
+ case c >= '0' && c <= '9':
+ // digits allowed
+ default:
+ return false
+ }
+ }
+ return letters >= 3 && distinct >= 2
+}
+
+func scanRuns(text string, mention func(w string)) {
+ start := -1
+ for j := 0; j <= len(text); j++ {
+ alnum := false
+ if j < len(text) {
+ ch := text[j]
+ alnum = (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9')
+ }
+ if alnum {
+ if start < 0 {
+ start = j
+ }
+ continue
+ }
+ if start >= 0 {
+ mention(text[start:j])
+ start = -1
+ }
+ }
+}
+
+func (co *Corpus) buildCodewords() {
+ co.codewords, co.codewordTable = nil, nil
+ caps := map[string]int{}
+ other := map[string]int{}
+ docCaps := make([][]string, len(co.archive.Docs))
+ for i := range co.archive.Docs {
+ d := &co.archive.Docs[i]
+ here := map[string]bool{}
+ count := func(w string) {
+ switch {
+ case codewordStoplist[w]:
+ // boilerplate or archive markup (codewordStoplist)
+ case isCodewordToken(w):
+ caps[w]++
+ here[w] = true
+ default:
+ other[strings.ToLower(w)]++
+ }
+ }
+ scanRuns(d.Title, count)
+ scanRuns(d.FullText, count)
+ for w := range here {
+ docCaps[i] = append(docCaps[i], w)
+ }
+ }
+ for w, cnt := range caps {
+ if cnt < 2 || cnt <= other[strings.ToLower(w)] {
+ continue
+ }
+ co.codewords = append(co.codewords, codewordEntry{Token: w, Count: cnt})
+ }
+ kept := make(map[string]*codewordEntry, len(co.codewords))
+ for i := range co.codewords {
+ kept[co.codewords[i].Token] = &co.codewords[i]
+ }
+
+ for i := range co.archive.Docs {
+ d := &co.archive.Docs[i]
+ year := docYear(d)
+ inDoc := map[string]bool{}
+ exact := func(w string) {
+ if e := kept[w]; e != nil {
+ inDoc[e.Token] = true
+ }
+ }
+ folded := func(w string) {
+ if e := kept[strings.ToUpper(w)]; e != nil {
+ inDoc[e.Token] = true
+ }
+ }
+ for _, w := range docCaps[i] {
+ exact(w)
+ }
+ scanRuns(d.Description, exact)
+ scanRuns(d.LongDescription, exact)
+ for _, p := range d.DocMeta.Programs {
+ scanRuns(p, folded)
+ }
+ for _, t := range d.Tags {
+ scanRuns(t, folded)
+ }
+ for w := range inDoc {
+ e := kept[w]
+ e.Docs++
+ if year == "" {
+ continue
+ }
+ if e.First == "" || year < e.First {
+ e.First = year
+ }
+ if year > e.Last {
+ e.Last = year
+ }
+ }
+ }
+ sort.Slice(co.codewords, func(i, j int) bool {
+ if co.codewords[i].Count != co.codewords[j].Count {
+ return co.codewords[i].Count > co.codewords[j].Count
+ }
+ return co.codewords[i].Token < co.codewords[j].Token
+ })
+
+ lowerProg := make(map[string]string, len(co.archive.Facets["program"]))
+ for p := range co.archive.Facets["program"] {
+ lowerProg[valueKey(p)] = p
+ }
+ shown := co.codewords
+ if len(shown) > codewordRows {
+ shown = shown[:codewordRows]
+ }
+ co.codewordTable = make([]codewordRow, 0, len(shown))
+ for _, e := range shown {
+ row := codewordRow{codewordEntry: e, Gloss: co.glossaryLookup(e.Token), SearchURL: tokenSearchURL(e.Token)}
+ if p, ok := lowerProg[valueKey(e.Token)]; ok {
+ row.ProgURL = wholeArchive.link(categoryValueURL("program", p))
+ }
+ if e.First != "" {
+ row.FirstURL = yearTokenURL(e.Token, e.First)
+ }
+ if e.Last != "" {
+ row.LastURL = yearTokenURL(e.Token, e.Last)
+ }
+ if row.Gloss != nil {
+ row.GlossURL = template.URL("/glossary#term-" + url.PathEscape(row.Gloss.Term))
+ }
+ co.codewordTable = append(co.codewordTable, row)
+ }
+ co.buildCodewordViews()
+}
+
+func yearTokenURL(token, year string) template.URL {
+ return template.URL(wholeArchive.link(dateRangeSearchURL("TE", exactClause(token), [2]string{"date_doc", year})))
+}
+
+func tokenSearchURL(token string) template.URL {
+ return template.URL(wholeArchive.link("/search?q=" + url.QueryEscape(exactClause(token))))
+}
+
+type codewordRow struct {
+ codewordEntry
+ Gloss *GlossaryEntry
+ ProgURL string
+ SearchURL template.URL
+ FirstURL, LastURL template.URL
+ GlossURL template.URL
+}
+
+// codewordRows caps the page
+const codewordRows = 500
+
+var codewordCols = []sortCol{
+ {Key: "token", Label: "Token"},
+ {Key: "count", Label: "Occurrences", DescFirst: true},
+ {Key: "docs", Label: "Documents", DescFirst: true},
+ {Key: "first", Label: "First seen"},
+ {Key: "last", Label: "Last seen", DescFirst: true},
+ {Key: "gloss", Label: "Glossary"},
+}
+
+func (co *Corpus) handleCodewords(w http.ResponseWriter, r *http.Request) {
+ q := r.URL.Query()
+ sortBy := sortToken(q.Get("sort"))
+ order := sortToken(q.Get("order"))
+ view, ok := co.codewordViews[sortBy+"/"+order]
+ if !ok {
+ view, ok = co.codewordViews[sortBy+"/"+codewordDefaultOrder(sortBy)]
+ }
+ if !ok {
+ view = co.codewordViews["count/desc"]
+ }
+ render(w, r, &codewordsPage{
+ Rows: view.Rows,
+ RowsHTML: co.preHTML("codewords:" + view.Key),
+ Total: len(co.codewords),
+ Shown: len(view.Rows),
+ Heads: view.Heads,
+ })
+}
+
+type codewordView struct {
+ Key string
+ Rows []codewordRow
+ Heads []sortHead
+}
+
+func codewordDefaultOrder(sortBy string) string {
+ for _, col := range codewordCols {
+ if col.Key == sortBy {
+ return orderParam(col.DescFirst)
+ }
+ }
+ return ""
+}
+
+func (co *Corpus) buildCodewordViews() {
+ co.codewordViews = make(map[string]codewordView, 2*len(codewordCols))
+ for _, sc := range codewordCols {
+ for _, order := range []string{"asc", "desc"} {
+ rows := append([]codewordRow(nil), co.codewordTable...)
+ sortCodewordRows(rows, sc.Key, order == "desc")
+ heads := sortHeads("?", codewordCols, sc.Key, order == "desc")
+ co.codewordViews[sc.Key+"/"+order] = codewordView{sc.Key + "/" + order, rows, heads}
+ }
+ }
+}
+
+func sortCodewordRows(rows []codewordRow, sortBy string, desc bool) {
+ sort.SliceStable(rows, func(i, j int) bool {
+ var less, eq bool
+ switch sortBy {
+ case "token":
+ less, eq = rows[i].Token < rows[j].Token, rows[i].Token == rows[j].Token
+ case "docs":
+ less, eq = rows[i].Docs < rows[j].Docs, rows[i].Docs == rows[j].Docs
+ case "first", "last":
+ a, b := rows[i].First, rows[j].First
+ if sortBy == "last" {
+ a, b = rows[i].Last, rows[j].Last
+ }
+ if (a == "") != (b == "") {
+ return b == ""
+ }
+ less, eq = a < b, a == b
+ case "gloss":
+ a, b := rows[i].Gloss != nil, rows[j].Gloss != nil
+ less, eq = !a && b, a == b // missing before defined, ascending
+ default:
+ less, eq = rows[i].Count < rows[j].Count, rows[i].Count == rows[j].Count
+ }
+ if eq {
+ return rows[i].Token < rows[j].Token
+ }
+ if desc {
+ return !less
+ }
+ return less
+ })
+}
diff --git a/corpus.go b/corpus.go
new file mode 100644
index 0000000..59783f5
--- /dev/null
+++ b/corpus.go
@@ -0,0 +1,162 @@
+package main
+
+import (
+ "html/template"
+ "net/http"
+ "sync/atomic"
+)
+
+// Corpus is everything a load derives from the data and project directories: the
+// documents, every index and table built from them, and the page bodies
+// rendered ahead of time. One is built per load and never changed after it
+// goes live, so handlers read it without locks.
+type Corpus struct {
+ // api.go
+ facetsJSON []byte
+ facetsETag string
+
+ // articles.go
+ articles []Article
+ articleIndex map[string]*Article // key -> article
+ articleByLink map[string]*Article // link -> article
+ articleGroups []articleYearGroup
+ articlesTitled int
+ articlesBooks int // of articlesTitled
+
+ // assemblies.go
+ assemblies []*Assembly
+ assemblyIndex map[string]*Assembly
+ assembliesByDoc map[string][]*Assembly
+
+ // browse.go
+ // homeStats holds the corpus-derived widgets on the home page - the year
+ // histograms and top-N facet lists.
+ //
+ // By default every figure on the home page describes the REPORTED
+ // collection; homeStatsSIDtoday (/?sidtoday=1) adds the SIDtoday archive. FOIA
+ // material can postdate the June 2013 disclosures (a 2014 "year written"
+ // bar on a Snowden archive reads as impossible at a glance), and the
+ // SIDtoday archive is +2,100 issues of an internal newsletter that swamps
+ // the lists and the leak-era shape the charts exist to show.
+ // Every bar and list entry links to its /browse/{category}/{value} page
+ // scoped to that collection, so the number shown is the number of
+ // documents the page lists.
+ homeStats, homeStatsSIDtoday homeFigures
+
+ randomPools map[string][]int
+ collectionDocs map[string][]*Document
+ sidtodayDocs []*Document
+
+ // everyDoc is every document as pointers, built once.
+ everyDoc []*Document
+
+ archiveTimeline []archiveYear
+ categoryRowHTML map[string][]template.HTML // table key -> each entry's row, as Entries has them
+ azRowHTML map[string]template.HTML
+ categoryRowSize map[string]int
+ docRowHTML map[string]template.HTML
+ docRowMarkedHTML map[string]template.HTML // rows with the "reported" badge, for ReportedIssue documents
+
+ // categoryTables holds every category's table under its slug.
+ categoryTables map[string]*categoryTable
+
+ // seriesBySize is the series index of the SIDtoday tab.
+ seriesBySize map[bool][]categoryEntry
+
+ // categoryValueCounts is how many values each category's list holds.
+ categoryValueCounts map[bool]map[string]int
+
+ // categoryDocs is the posting list behind every /browse/{category}/{value}
+ // page: slug -> folded value -> the documents holding it, in corpus order.
+ categoryDocs map[string]map[string][]int32
+
+ // categoryNames maps each category's index keys to the spelling the lists
+ // and counts show: the first one in corpus order.
+ categoryNames map[string]map[string]string
+
+ // categoryTypeCounts splits each category value's count by document type
+ // (REPORTED / SIDTODAY / FOIA): the list pages' breakdown columns. They
+ // count type, not collection.
+ categoryTypeCounts map[string]map[string]*collCounts
+
+ // changelog.go
+ changelog []ChangeEntry
+
+ // codewords.go
+ codewords []codewordEntry
+ codewordTable []codewordRow
+ codewordViews map[string]codewordView
+
+ // data.go
+ archive Archive
+ serialPrev map[string]string
+ serialNext map[string]string
+ seriesIndex map[string]*seriesGroup
+
+ // explore.go
+ exploreTables map[string][]exploreTable
+ classPatterns struct {
+ Strips []clsYearRow
+ Sparse []clsYearRow
+ Cols []facetEntry // crossCols, each linked to its page
+ HeatRows []heatRow
+ }
+
+ // explore_ids.go
+ idRows []idRow
+ idChecksums string
+
+ // glossary.go
+ glossary []GlossaryEntry
+ glossaryIndex map[string]*GlossaryEntry
+ glossaryGroups []glossaryGroup
+
+ // main.go
+ prerendered map[string]template.HTML
+
+ // outlets.go
+ outletTables map[string]outletTable
+
+ // portions.go
+ portionStats portionSummary
+
+ // search.go
+ tfidf TFIDFIndex
+
+ // docIndex is populated by loadData() in data.go after archive.Docs is fully
+ // built. Provides O(1) doc lookup by ID.
+ docIndex map[string]*Document
+
+ // seo.go
+ sitemapPaths []string
+ canonicalSitemap []byte
+
+ // sitemap.go
+ sitemapParts [][]byte
+}
+
+// live is the corpus requests are served from. It is replaced whole, never
+// modified: a load builds a new Corpus and stores it here when it is done.
+var live atomic.Pointer[Corpus]
+
+func init() { live.Store(&Corpus{}) }
+
+// corpus returns the live corpus. Handlers do not call it. bind hands each
+// request the corpus that was live when it arrived, and everything the
+// request touches is reached through that one value.
+func corpus() *Corpus { return live.Load() }
+
+// loadData reads the corpus under dataDir into a new Corpus, pre-renders its
+// big page bodies (from tmplSet, so parse the templates first).
+func loadData() {
+ co := &Corpus{}
+ co.load()
+ co.prerenderPages()
+ live.Store(co)
+}
+
+// bind adapts a handler to the mux. the request is served from the corpus
+// that is live when it arrives, and from that one throughout.
+func bind(h func(*Corpus, http.ResponseWriter, *http.Request)) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) { h(live.Load(), w, r) }
+}
diff --git a/data.go b/data.go
new file mode 100644
index 0000000..697fad3
--- /dev/null
+++ b/data.go
@@ -0,0 +1,1470 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "html/template"
+ "image"
+ _ "image/gif"
+ _ "image/jpeg"
+ _ "image/png"
+ "log"
+ "math"
+ "os"
+ "path/filepath"
+ "slices"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+)
+
+// Publication is one entry of a document's publication list.
+type Publication struct {
+ Title string `json:"title,omitempty"`
+ Publisher string `json:"publisher"`
+ Journalists []string `json:"reporters"`
+ Contributors []string `json:"contributors"`
+ DatePublished string `json:"date_published"`
+ Link string `json:"link"`
+ Note string `json:"note,omitempty"`
+ Version string `json:"version,omitempty"`
+ DocSource string `json:"doc_source,omitempty"`
+ DocURL string `json:"doc_url,omitempty"`
+ ReportingOnly bool `json:"reporting_only,omitempty"`
+}
+
+type Document struct {
+ ID string `json:"id"`
+ AkaID []AkaEntry `json:"aka_id,omitempty"`
+ Title string `json:"title"`
+ Type string `json:"type"`
+ Collection string `json:"collection"`
+ Description string `json:"description"`
+ LongDescription string `json:"long_description"`
+ Tags []string `json:"tags"`
+ Pages int `json:"pages,omitempty"`
+ PDFSha256 string `json:"pdf_sha256,omitempty"`
+ ArchiveComment string `json:"archive_comment,omitempty"`
+ VersionMeta map[string]VersionInfo `json:"versions,omitempty"`
+ DocMeta struct {
+ Creator string `json:"creator"`
+ DateCreated string `json:"date_created"`
+ Programs []string `json:"programs"`
+ Classification []string `json:"classification"`
+ Dissemination []string `json:"dissemination"`
+ Banner string `json:"banner"`
+ Genre string `json:"genre"`
+ Series struct {
+ Name string `json:"name"`
+ Part int `json:"part"`
+ } `json:"series"`
+ Countries []string `json:"countries"`
+ People []string `json:"people"`
+ } `json:"document"`
+ Publication []Publication `json:"publication"`
+
+ Dir string `json:"-"`
+ FullText string `json:"-"`
+ SearchText string `json:"-"`
+ StemmedText string `json:"-"`
+ StemmedTitle string `json:"-"`
+ StemmedDesc string `json:"-"`
+ StemmedFull string `json:"-"` // OCR body only, for "Content" field searches
+ FoldedFull string `json:"-"` // accent-folded + lowercased OCR body, for fast quoted "Content" matching
+ bodyFoldLen int // how much of FoldedFull is the fold of FullText: the versions' lines follow it
+ foldedTitle string // normFoldString(Title)
+ foldedDesc string // normFoldString(Description + valueSep + LongDescription)
+ // terms is the document's entry in the TF-IDF index: every stemmed token
+ // it holds, in any field. fields is its metadata as the matchers and the
+ // tallies read it (buildDocFields). Both are nil until loadData sets them.
+ terms map[string]float64
+ fields *docFields
+
+ Images []string `json:"-"`
+ ImageSizes []imageSize `json:"-"`
+ PDFFile string `json:"-"`
+ PDFSize int64 `json:"-"`
+ TextFile string `json:"-"` // the .md OCR/transcript's file name, "" without one
+ Versions []DocVersion `json:"-"` // differing published copies from versions// (loadVersions)
+ PortionLevels []string `json:"-"`
+ PortionMarkings []string `json:"-"`
+ PortionControls []string `json:"-"`
+ HasThumb bool `json:"-"`
+ HasGifThumb bool `json:"-"`
+ HTMLContent template.HTML `json:"-"`
+}
+
+func (d *Document) foldedBody() string {
+ if d.bodyFoldLen > len(d.FoldedFull) || (d.bodyFoldLen == 0 && d.FullText != "") {
+ return normFoldString(d.FullText) // a document setBodySearchText never saw
+ }
+ return d.FoldedFull[:d.bodyFoldLen]
+}
+
+// ReportedIssue reports whether the document is a SIDtoday issue that was
+// individually reported, and so is in the Reported collection as well.
+func (d *Document) ReportedIssue() bool {
+ return d.Type == "SIDTODAY" && d.Collection == collectionNews
+}
+
+func (d *Document) JournalistsDeduped() []string { return facetValues(d, "RE") }
+
+func (d *Document) Publishers() []string { return facetValues(d, "PU") }
+
+func journalistsOf(d *Document) []string {
+ var all []string
+ for _, p := range d.Publication {
+ all = append(all, p.Journalists...)
+ }
+ return normalizeList(all)
+}
+
+func publishersOf(d *Document) []string {
+ var all []string
+ for _, p := range d.Publication {
+ if !p.ReportingOnly {
+ all = append(all, p.Publisher)
+ }
+ }
+ return normalizeList(all)
+}
+
+func (d *Document) titleFold() string {
+ if d.foldedTitle == "" && d.Title != "" {
+ return normFoldString(d.Title)
+ }
+ return d.foldedTitle
+}
+
+func (d *Document) descFold() string {
+ if d.foldedDesc == "" {
+ return normFoldString(d.Description + valueSep + d.LongDescription)
+ }
+ return d.foldedDesc
+}
+
+// hasTerms reports whether the document holds every one of the stemmed
+// tokens, in any field.
+func (d *Document) hasTerms(tokens []string) bool {
+ if d.terms == nil {
+ return true
+ }
+ for _, t := range tokens {
+ if _, ok := d.terms[t]; !ok {
+ return false
+ }
+ }
+ return true
+}
+
+// AkaEntry is another entry this document was published under.
+type AkaEntry struct {
+ ID string `json:"id"` // usually a alternative PDF filename
+ DocSource string `json:"doc_source,omitempty"`
+ DocURL string `json:"doc_url,omitempty"`
+}
+
+type DocVersion struct {
+ Slug string
+ PDFFile string
+ PDFSize int64
+ TextFile string
+ FullText string
+ HTMLContent template.HTML
+ Images []string
+ ImageSizes []imageSize
+ PubIndex []int
+ SHA string
+ NovelText string
+ Label string
+ DocID string
+ ChangedPages []ChangedPage
+}
+
+type VersionInfo struct {
+ ChangedPages []int `json:"changed_pages,omitempty"`
+ PDFSha256 string `json:"pdf_sha256,omitempty"`
+}
+
+type ChangedPage struct {
+ Page int
+ Pos int
+}
+
+func (v *DocVersion) PageChanged(pos int) bool {
+ if pos < 1 {
+ return false
+ }
+ for _, c := range v.ChangedPages {
+ if c.Pos == pos {
+ return true
+ }
+ }
+ return false
+}
+
+func (d *Document) VersionFor(slug string) *DocVersion {
+ if slug == "" {
+ return nil
+ }
+ for i := range d.Versions {
+ if d.Versions[i].Slug == slug {
+ return &d.Versions[i]
+ }
+ }
+ return nil
+}
+
+// servedFile reports whether rel, a slash-separated path inside the
+// document's directory, is one of the files the loader indexed: the
+// metadata.json, the PDF, the transcript, the thumbnails and the page images,
+// of the document or of one of its versions. /files/ serves these and nothing else a directory
+// has.
+func (d *Document) servedFile(rel string) bool {
+ if rest, ok := strings.CutPrefix(rel, "versions/"); ok {
+ slug, name, _ := strings.Cut(rest, "/")
+ v := d.VersionFor(slug)
+ return v != nil && contentFile(name, v.PDFFile, v.TextFile, v.Images)
+ }
+ switch rel {
+ case "metadata.json":
+ return true
+ case "thumb.jpg":
+ return d.HasThumb
+ case "thumb.gif":
+ return d.HasGifThumb
+ }
+ return contentFile(rel, d.PDFFile, d.TextFile, d.Images)
+}
+
+func contentFile(name, pdf, text string, images []string) bool {
+ if name == "" {
+ return false
+ }
+ if name == pdf || name == text {
+ return true
+ }
+ image, ok := strings.CutPrefix(name, "images/")
+ return ok && slices.Contains(images, image)
+}
+
+func (d *Document) UnlinkedVersions() []DocVersion {
+ var out []DocVersion
+ for _, v := range d.Versions {
+ if len(v.PubIndex) == 0 {
+ out = append(out, v)
+ }
+ }
+ return out
+}
+
+type collCounts struct {
+ News int `json:"reported"`
+ Archive int `json:"sidtoday_archive"`
+ FOIA int `json:"foia"`
+}
+
+func (c *collCounts) add(coll string) {
+ switch coll {
+ case collectionNews:
+ c.News++
+ case collectionArchive:
+ c.Archive++
+ case collectionFOIA:
+ c.FOIA++
+ }
+}
+
+func (c *collCounts) addType(typ string) {
+ switch typ {
+ case "SIDTODAY":
+ c.Archive++
+ case "FOIA":
+ c.FOIA++
+ default:
+ c.News++
+ }
+}
+
+var genreVocabList = []string{"slides", "report", "wiki", "manual", "guide", "newsletter", "other"}
+
+var genreVocab = func() map[string]bool {
+ m := make(map[string]bool, len(genreVocabList))
+ for _, g := range genreVocabList {
+ m[g] = true
+ }
+ return m
+}()
+
+type Archive struct {
+ Docs []Document
+ Types map[string]int
+ NewsTypes map[string]int
+ CollectionCounts map[string]int
+ Facets map[string]map[string]int
+ FacetsByColl map[string]map[string]*collCounts
+ SIDTodayCount int
+ RelatedDocs map[string][]string
+}
+
+const (
+ collectionNews = "news"
+ collectionArchive = "sidtoday-archive"
+ collectionFOIA = "foia"
+)
+
+// dataDir is the corpus, the data/ directory of the archive root (-root).
+var dataDir = "data"
+
+// projectDir, the root's project/ directory, holds the files that describe
+// the archive as a whole and are the same on every mirror: glossary.json,
+// assemblies.json, changelog.json and related.json.
+var projectDir = "project"
+
+func setRoot(root string) {
+ dataDir, projectDir = filepath.Join(root, "data"), filepath.Join(root, "project")
+}
+
+var loadWarnings, loadErrors int
+
+func warnf(format string, args ...interface{}) {
+ loadWarnings++
+ log.Printf("Warning: "+format, args...)
+}
+
+func errorf(format string, args ...interface{}) {
+ loadErrors++
+ log.Printf("Error: "+format, args...)
+}
+
+func projectPath(name string) string { return filepath.Join(projectDir, name) }
+
+func relatedDocsPath() string { return projectPath("related.json") }
+
+const defaultDocType = "REPORTED"
+
+var pageImageExts = map[string]bool{
+ ".jpg": true, ".jpeg": true, ".png": true, ".gif": true, ".webp": true,
+}
+
+func (co *Corpus) buildSerialNav() {
+ co.serialPrev = map[string]string{}
+ co.serialNext = map[string]string{}
+ var prev string
+ for _, yr := range co.archiveTimeline {
+ for _, mo := range yr.Months {
+ for _, d := range mo.Docs {
+ if prev != "" {
+ co.serialPrev[d.ID] = prev
+ co.serialNext[prev] = d.ID
+ }
+ prev = d.ID
+ }
+ }
+ }
+}
+
+var collectionDirs = map[string]string{
+ "foia": collectionFOIA,
+ "sidtoday-archive": collectionArchive,
+}
+
+type docLocation struct {
+ id string
+ dir string
+ collection string
+}
+
+func docLocations(dataDir string) ([]docLocation, error) {
+ entries, err := os.ReadDir(dataDir)
+ if err != nil {
+ return nil, err
+ }
+ var out []docLocation
+ for _, e := range entries {
+ if !e.IsDir() {
+ continue
+ }
+ if coll, ok := collectionDirs[strings.ToLower(e.Name())]; ok {
+ subs, err := os.ReadDir(filepath.Join(dataDir, e.Name()))
+ if err != nil {
+ warnf("cannot read %s: %v", filepath.Join(dataDir, e.Name()), err)
+ continue
+ }
+ for _, s := range subs {
+ if s.IsDir() {
+ out = append(out, docLocation{
+ id: s.Name(),
+ dir: filepath.Join(dataDir, e.Name(), s.Name()),
+ collection: coll,
+ })
+ }
+ }
+ continue
+ }
+ out = append(out, docLocation{
+ id: e.Name(),
+ dir: filepath.Join(dataDir, e.Name()),
+ })
+ }
+ return out, nil
+}
+
+func dirHasPDF(dir string) bool {
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return false
+ }
+ for _, e := range entries {
+ if !e.IsDir() && isPDF(e.Name()) {
+ return true
+ }
+ }
+ return false
+}
+
+func isPDF(name string) bool { return strings.EqualFold(filepath.Ext(name), ".pdf") }
+
+func isTranscript(name string) bool { return strings.EqualFold(filepath.Ext(name), ".md") }
+
+func firstN(vals []string, n int) []string {
+ if len(vals) <= n {
+ return vals
+ }
+ return append(append([]string{}, vals[:n]...), fmt.Sprintf("and %d more", len(vals)-n))
+}
+
+func plural(n int, one, many string) string {
+ if n == 1 {
+ return one
+ }
+ return many
+}
+
+func normalizeList(vals []string) []string {
+ if len(vals) == 0 {
+ return vals
+ }
+ out := make([]string, 0, len(vals))
+ seen := make(map[string]bool)
+ for _, v := range vals {
+ v = strings.TrimSpace(v)
+ if v == "" {
+ continue
+ }
+ k := valueKey(v)
+ if seen[k] {
+ continue
+ }
+ seen[k] = true
+ out = append(out, v)
+ }
+ return out
+}
+
+type seriesGroup struct {
+ Name string
+ Docs []*Document
+}
+
+func (co *Corpus) buildSeriesIndex() {
+ co.seriesIndex = make(map[string]*seriesGroup)
+ for i := range co.archive.Docs {
+ dd := &co.archive.Docs[i]
+ name := dd.DocMeta.Series.Name
+ if name == "" {
+ continue
+ }
+ if dd.DocMeta.Series.Part <= 0 {
+ warnf("%s: series %q without a part number - order matters; sorting it last", dd.ID, name)
+ }
+ key := valueKey(name)
+ g := co.seriesIndex[key]
+ if g == nil {
+ g = &seriesGroup{Name: name}
+ co.seriesIndex[key] = g
+ } else if g.Name != name {
+ warnf("%s: series name casing %q differs from first-seen %q - merged, but copy-paste the exact name", dd.ID, name, g.Name)
+ }
+ g.Docs = append(g.Docs, dd)
+ }
+ for _, g := range co.seriesIndex {
+ sort.SliceStable(g.Docs, func(i, j int) bool {
+ a, b := g.Docs[i].DocMeta.Series.Part, g.Docs[j].DocMeta.Series.Part
+ if (a <= 0) != (b <= 0) {
+ return b <= 0 // numbered parts before unnumbered ones
+ }
+ if a != b {
+ return a < b
+ }
+ return g.Docs[i].Title < g.Docs[j].Title
+ })
+ seen := map[int]string{}
+ for _, m := range g.Docs {
+ p := m.DocMeta.Series.Part
+ if p <= 0 {
+ continue
+ }
+ if prev, dup := seen[p]; dup {
+ warnf("series %q: part %d claimed by both %s and %s", g.Name, p, prev, m.ID)
+ }
+ seen[p] = m.ID
+ }
+ }
+}
+
+func wrapRedactions(html string) string {
+ return strings.ReplaceAll(html, "[REDACTED]", `[REDACTED]`)
+}
+
+// load reads the corpus under dataDir into co and builds every index the
+// handlers serve from. Written here once, before the corpus goes live
+// (loadData), and read-only from then on.
+func (co *Corpus) load() {
+ loadWarnings, loadErrors = 0, 0
+
+ co.loadGlossary()
+
+ co.archive = Archive{
+ Docs: []Document{},
+ Types: make(map[string]int),
+ NewsTypes: make(map[string]int),
+ CollectionCounts: make(map[string]int),
+ RelatedDocs: make(map[string][]string),
+ }
+
+ locations, err := docLocations(dataDir)
+ if err != nil {
+ log.Fatalf("Critical: Could not read data directory: %v", err)
+ }
+
+ seen := make(map[string]string)
+ var skipped, skippedWithPDF []string
+ for _, loc := range locations {
+ metaPath := filepath.Join(loc.dir, "metadata.json")
+ b, err := os.ReadFile(metaPath)
+ if err != nil {
+ skipped = append(skipped, loc.id)
+ if dirHasPDF(loc.dir) {
+ skippedWithPDF = append(skippedWithPDF, loc.id)
+ }
+ continue
+ }
+
+ var d Document
+ if err := json.Unmarshal(b, &d); err != nil {
+ errorf("invalid JSON in %s: %v - skipping document", metaPath, err)
+ continue
+ }
+
+ d.Type = strings.ToUpper(strings.TrimSpace(d.Type))
+ if d.Type == "" {
+ switch loc.collection {
+ case collectionArchive:
+ d.Type = "SIDTODAY"
+ case collectionFOIA:
+ d.Type = "FOIA"
+ default:
+ d.Type = defaultDocType
+ }
+ }
+ d.ID = loc.id
+ d.Dir = loc.dir
+
+ if prev, dup := seen[d.ID]; dup {
+ errorf("duplicate document id %q in %s (already loaded from %s) - skipping",
+ d.ID, loc.dir, prev)
+ continue
+ }
+
+ if !docIDSafe(d.ID) {
+ warnf("document id %q contains characters outside [A-Za-z0-9._~-] - links, sitemap entries or pin/history cookies may misbehave; consider renaming the directory", d.ID)
+ }
+
+ d.Tags = normalizeList(d.Tags)
+ d.AkaID = normalizeAka(&d)
+
+ files, derr := scanContentDir(d.Dir)
+ if derr != nil {
+ errorf("cannot read %s: %v - skipping document", d.Dir, derr)
+ continue
+ }
+ if files.PDFFile == "" {
+ errorf("%s: no PDF in %s - skipping document", d.ID, d.Dir)
+ continue
+ }
+ seen[d.ID] = loc.dir
+ d.PDFFile, d.PDFSize, d.TextFile = files.PDFFile, files.PDFSize, files.TextFile
+ d.HasThumb, d.HasGifThumb = files.HasThumb, files.HasGifThumb
+ if files.Text != "" {
+ d.FullText = files.Text
+ d.HTMLContent = template.HTML(wrapRedactions(Render(d.FullText)))
+ }
+
+ d.Collection = strings.ToLower(strings.TrimSpace(d.Collection))
+ if loc.collection != "" {
+ if d.Collection != "" && d.Collection != loc.collection {
+ warnf("%s declares collection %q but lives in the %s directory - using %q",
+ d.ID, d.Collection, loc.collection, loc.collection)
+ }
+ d.Collection = loc.collection
+ } else {
+ switch d.Collection {
+ case collectionNews, collectionArchive, collectionFOIA:
+ // explicit, keep
+ case "":
+ if d.Type == "FOIA" {
+ d.Collection = collectionFOIA
+ } else {
+ d.Collection = collectionNews
+ }
+ default:
+ warnf("%s has unknown collection %q - treating as %q",
+ d.ID, d.Collection, collectionNews)
+ d.Collection = collectionNews
+ }
+ }
+ if d.Collection == collectionArchive && d.Type != "SIDTODAY" {
+ warnf("%s is in the SIDtoday archive but typed %q - the SIDtoday tab lists the type SIDTODAY, so no browse tab shows it", d.ID, d.Type)
+ }
+ co.archive.CollectionCounts[d.Collection]++
+ if d.Type == "SIDTODAY" {
+ co.archive.SIDTodayCount++
+ }
+
+ newsDoc := d.Collection == collectionNews
+ co.archive.Types[d.Type]++
+ if newsDoc {
+ co.archive.NewsTypes[d.Type]++
+ }
+
+ d.Images = loadPageImages(d.Dir, d.ID)
+
+ if d.Pages <= 0 {
+ d.Pages = len(d.Images)
+ }
+
+ d.DocMeta.Programs = normalizeList(d.DocMeta.Programs)
+ d.DocMeta.Countries = normalizeList(d.DocMeta.Countries)
+ d.DocMeta.People = normalizeList(d.DocMeta.People)
+ d.DocMeta.Classification = normalizeList(d.DocMeta.Classification)
+ for _, c := range d.DocMeta.Classification {
+ if full, ok := classAlias[strings.ToLower(c)]; ok {
+ warnf("%s: classification %q is an abbreviation - write %q; searches and counts treat the two as different values", d.ID, c, strings.ToUpper(full))
+ }
+ }
+ d.DocMeta.Dissemination = normalizeList(d.DocMeta.Dissemination)
+ for i := range d.Publication {
+ p := &d.Publication[i]
+ p.Publisher = strings.TrimSpace(p.Publisher)
+ p.Journalists = normalizeList(p.Journalists)
+ p.Contributors = normalizeList(p.Contributors)
+ p.Note = strings.TrimSpace(p.Note)
+ p.DocSource = strings.TrimSpace(p.DocSource)
+ p.DocURL = strings.TrimSpace(p.DocURL)
+ }
+
+ if v := d.DocMeta.DateCreated; v != "" && !validDate(v) {
+ warnf("%s: date_created %q is not YYYY, YYYY-MM or YYYY-MM-DD - treating the document as undated", d.ID, v)
+ d.DocMeta.DateCreated = ""
+ }
+ for i := range d.Publication {
+ if v := d.Publication[i].DatePublished; v != "" && !validDate(v) {
+ warnf("%s: publication %d date_published %q is not YYYY, YYYY-MM or YYYY-MM-DD - treating it as undated", d.ID, i+1, v)
+ d.Publication[i].DatePublished = ""
+ }
+ }
+
+ co.loadVersions(&d)
+
+ d.DocMeta.Creator = strings.TrimSpace(d.DocMeta.Creator)
+ d.DocMeta.Series.Name = strings.TrimSpace(d.DocMeta.Series.Name)
+ d.DocMeta.Banner = strings.TrimSpace(d.DocMeta.Banner)
+ d.DocMeta.Genre = strings.ToLower(strings.TrimSpace(d.DocMeta.Genre))
+ if d.DocMeta.Genre != "" && !genreVocab[d.DocMeta.Genre] {
+ warnf("%s: genre %q is outside the vocabulary (%s) - serving as-is", d.ID, d.DocMeta.Genre, strings.Join(genreVocabList, ", "))
+ }
+
+ for _, f := range searchFields {
+ if f.Values == nil {
+ continue
+ }
+ for _, v := range f.Values(&d) {
+ if strings.Contains(v, `"`) {
+ warnf("%s: %s %q contains a double quote - its search links cannot quote it; use ' or “ ” instead", d.ID, strings.ToLower(f.Label), v)
+ }
+ }
+ }
+
+ // The all-fields forms are the title, the descriptions and the rest
+ // of the metadata, then the body: each part is tokenized once, and
+ // tokenize of the parts joined is the parts' tokens in a row.
+ meta := searchMeta(&d)
+ metaText := strings.Join([]string{d.Title, d.Description, d.LongDescription, meta}, valueSep)
+ titleToks := tokenize(d.Title)
+ descToks := tokenize(d.Description + " " + d.LongDescription)
+ metaToks := tokenize(meta)
+ stems := make([]string, 0, len(titleToks)+len(descToks)+len(metaToks))
+ stems = append(append(append(stems, titleToks...), descToks...), metaToks...)
+ d.StemmedTitle = strings.Join(titleToks, " ")
+ d.StemmedDesc = strings.Join(descToks, " ")
+ d.foldedTitle = normFoldString(d.Title)
+ d.foldedDesc = normFoldString(d.Description + valueSep + d.LongDescription)
+ setBodySearchText(&d)
+ d.StemmedText = strings.TrimSpace(strings.Join(stems, " ") + " " + d.StemmedFull)
+ d.SearchText = normFoldString(metaText) + valueSep + d.FoldedFull
+ d.FoldedFull = d.SearchText[len(d.SearchText)-len(d.FoldedFull):]
+ d.StemmedFull = d.StemmedText[len(d.StemmedText)-len(d.StemmedFull):]
+
+ co.archive.Docs = append(co.archive.Docs, d)
+ }
+
+ if len(skipped) > 0 {
+ log.Printf("Note: %d directories have no metadata.json and were skipped: %s",
+ len(skipped), strings.Join(firstN(skipped, 8), ", "))
+ if len(skippedWithPDF) > 0 {
+ warnf("%d skipped %s a PDF but no metadata.json - a document nobody described: %s",
+ len(skippedWithPDF), plural(len(skippedWithPDF), "directory holds", "directories hold"),
+ strings.Join(firstN(skippedWithPDF, 8), ", "))
+ }
+ }
+
+ sizesRead := co.readImageSizes()
+ log.Printf("Building TF-IDF index...")
+ co.tfidf = buildTFIDF(co.archive.Docs)
+ log.Printf("TF-IDF done")
+ sizesRead()
+
+ co.docIndex = make(map[string]*Document, len(co.archive.Docs))
+ for i := range co.archive.Docs {
+ d := &co.archive.Docs[i]
+ co.docIndex[d.ID] = d
+ d.terms = co.tfidf.DocStats[d.ID]
+ }
+
+ co.buildSeriesIndex()
+ co.loadAssemblies()
+
+ log.Printf("Loading related docs index...")
+ co.archive.RelatedDocs = co.loadRelatedDocs(co.archive.Docs)
+ log.Printf("Related docs done")
+
+ co.buildArticles()
+ co.loadChangelog()
+
+ co.buildCollectionDocs()
+ co.buildCategoryDocs()
+ co.buildHomeStats()
+ co.buildCodewords()
+ co.buildCategoryLists()
+ co.buildExploreTables()
+ co.buildIDRows()
+ co.buildClassPatterns()
+ co.buildPortions()
+ co.buildDocFields()
+ co.buildOutletTables()
+ co.buildArchiveTimeline()
+ co.buildSerialNav()
+ co.buildFacetsJSON()
+ co.buildSitemapPaths()
+ co.buildCanonicalSitemap()
+
+ log.Printf("Loaded: %d documents (%d error%s, %d warning%s)",
+ len(co.archive.Docs), loadErrors, plural(loadErrors, "", "s"), loadWarnings, plural(loadWarnings, "", "s"))
+}
+
+// loadRelatedDocs reads related.json if it exists - run with --rebuild-related
+// to regenerate the file.
+func (co *Corpus) loadRelatedDocs(docs []Document) map[string][]string {
+ b, err := os.ReadFile(relatedDocsPath())
+ if err != nil {
+ warnf("related.json not found - related docs unavailable. Run with --rebuild-related to generate it.")
+ return make(map[string][]string)
+ }
+
+ var related map[string][]string
+ if err := json.Unmarshal(b, &related); err != nil {
+ warnf("related.json is malformed: %v - related docs unavailable.", err)
+ return make(map[string][]string)
+ }
+
+ staleKeys, staleLinks, staleLists := 0, 0, 0
+ for id, ids := range related {
+ if co.docIndex[id] == nil {
+ delete(related, id)
+ staleKeys++
+ continue
+ }
+ kept := slices.DeleteFunc(ids, co.unknownDoc)
+ if n := len(ids) - len(kept); n > 0 {
+ staleLinks += n
+ staleLists++
+ }
+ related[id] = kept
+ }
+ if staleKeys > 0 || staleLinks > 0 {
+ warnf("related.json: %d entries for documents no longer in the corpus, and %d links to such documents in %d lists - ignored; run --rebuild-related to recompute those lists",
+ staleKeys, staleLinks, staleLists)
+ }
+
+ missing := 0
+ for _, d := range docs {
+ if _, ok := related[d.ID]; !ok {
+ missing++
+ }
+ }
+ if missing > 0 {
+ warnf("%d/%d documents missing from related.json - run --rebuild-related to update.", missing, len(docs))
+ } else {
+ log.Printf("Related docs loaded: %d entries", len(related))
+ }
+
+ return related
+}
+
+func (co *Corpus) unknownDoc(id string) bool { return co.docIndex[id] == nil }
+
+func saveRelatedDocs(related map[string][]string) error {
+ b, err := json.MarshalIndent(related, "", " ")
+ if err != nil {
+ return err
+ }
+ return os.WriteFile(relatedDocsPath(), b, 0644)
+}
+
+func (co *Corpus) buildRelatedDocs(targets, all []Document) map[string][]string {
+ type scored struct {
+ id string
+ score float64
+ collection string
+ }
+
+ programDocs := make(map[string][]int)
+ tagDocs := make(map[string][]int)
+ titleDocs := make(map[string][]int)
+
+ for i, d := range all {
+ for _, p := range d.DocMeta.Programs {
+ key := strings.ToLower(p)
+ programDocs[key] = append(programDocs[key], i)
+ }
+ for _, t := range d.Tags {
+ key := strings.ToLower(t)
+ tagDocs[key] = append(tagDocs[key], i)
+ }
+ for _, w := range uniqueStrings(tokenize(d.Title)) {
+ titleDocs[w] = append(titleDocs[w], i)
+ }
+ }
+
+ tw := titleWeights{df: make(map[string]int, len(titleDocs)), n: len(all)}
+ for w, docs := range titleDocs {
+ tw.df[w] = len(docs)
+ }
+ maxTitleDF := len(all) / 10
+ for w, docs := range titleDocs {
+ if len(docs) > maxTitleDF {
+ delete(titleDocs, w)
+ }
+ }
+
+ result := make(map[string][]string, len(targets))
+
+ for i := range targets {
+ d := &targets[i]
+
+ seen := make(map[int]bool)
+ for _, p := range d.DocMeta.Programs {
+ for _, j := range programDocs[strings.ToLower(p)] {
+ seen[j] = true
+ }
+ }
+ for _, t := range d.Tags {
+ for _, j := range tagDocs[strings.ToLower(t)] {
+ seen[j] = true
+ }
+ }
+ for _, w := range tokenize(d.Title) {
+ for _, j := range titleDocs[w] {
+ seen[j] = true
+ }
+ }
+
+ var candidates []scored
+ for j := range seen {
+ if all[j].ID == d.ID {
+ continue
+ }
+ s := co.relatedScore(d, &all[j], tw)
+ if s >= minRelatedScore {
+ candidates = append(candidates, scored{all[j].ID, s, all[j].Collection})
+ }
+ }
+
+ sort.Slice(candidates, func(a, b int) bool {
+ if candidates[a].score != candidates[b].score {
+ return candidates[a].score > candidates[b].score
+ }
+ return candidates[a].id < candidates[b].id
+ })
+
+ const topN = 5
+ maxBulk := topN
+ if d.Collection == collectionNews {
+ maxBulk = 2
+ }
+ var ids []string
+ var skipped []string
+ bulk := 0
+ for _, c := range candidates {
+ if len(ids) == topN {
+ break
+ }
+ if c.collection != collectionNews && bulk >= maxBulk {
+ skipped = append(skipped, c.id)
+ continue
+ }
+ if c.collection != collectionNews {
+ bulk++
+ }
+ ids = append(ids, c.id)
+ }
+ for _, id := range skipped {
+ if len(ids) == topN {
+ break
+ }
+ ids = append(ids, id)
+ }
+ result[d.ID] = ids
+ }
+
+ log.Printf("Related docs built: %d entries", len(result))
+ return result
+}
+
+const minRelatedScore = 1.5
+
+type titleWeights struct {
+ df map[string]int
+ n int
+}
+
+func (t titleWeights) weight(w string) float64 {
+ if t.n < 2 {
+ return 1
+ }
+ return math.Log(float64(t.n+1)/float64(t.df[w]+1)) / math.Log(float64(t.n+1))
+}
+
+func uniqueStrings(in []string) []string {
+ seen := make(map[string]bool, len(in))
+ out := in[:0:0]
+ for _, s := range in {
+ if key := strings.ToLower(s); !seen[key] {
+ seen[key] = true
+ out = append(out, s)
+ }
+ }
+ return out
+}
+
+func sharedCount(a, b []string) int {
+ inA := make(map[string]bool, len(a))
+ for _, s := range a {
+ inA[strings.ToLower(s)] = true
+ }
+ n := 0
+ for _, s := range uniqueStrings(b) {
+ if inA[strings.ToLower(s)] {
+ n++
+ }
+ }
+ return n
+}
+
+func (co *Corpus) relatedScore(a, b *Document, tw titleWeights) float64 {
+ var score float64
+
+ aTitleWords := make(map[string]bool)
+ for _, w := range tokenize(a.Title) {
+ aTitleWords[w] = true
+ }
+ for _, w := range uniqueStrings(tokenize(b.Title)) {
+ if aTitleWords[w] {
+ score += 3.0 * tw.weight(w)
+ }
+ }
+
+ score += 4.0 * float64(sharedCount(a.DocMeta.Programs, b.DocMeta.Programs))
+ score += 2.0 * float64(sharedCount(a.Tags, b.Tags))
+
+ var ajournos, bjournos []string
+ for _, p := range a.Publication {
+ ajournos = append(ajournos, p.Journalists...)
+ }
+ for _, p := range b.Publication {
+ bjournos = append(bjournos, p.Journalists...)
+ }
+ score += 2.0 * float64(sharedCount(ajournos, bjournos))
+
+ if strings.EqualFold(a.DocMeta.Creator, b.DocMeta.Creator) && a.DocMeta.Creator != "" {
+ score += 1.0
+ }
+
+ score += 0.5 * float64(sharedCount(a.DocMeta.Countries, b.DocMeta.Countries))
+ score += co.tfidf.similarity(a.ID, b.ID) * 2.0
+
+ return score
+}
+
+func pageNumber(s string) (int, error) {
+ parts := strings.SplitN(s, "-", 2)
+ if len(parts) < 2 {
+ return 0, fmt.Errorf("unexpected image filename format: %q", s)
+ }
+ numPart := strings.SplitN(parts[1], ".", 2)[0]
+ n, err := strconv.Atoi(numPart)
+ if err != nil {
+ return 0, fmt.Errorf("could not parse page number from %q: %v", s, err)
+ }
+ return n, nil
+}
+
+func parseImageNum(s string) int {
+ n, err := pageNumber(s)
+ if err != nil {
+ warnf("%v", err)
+ return 0
+ }
+ return n
+}
+
+func docIDSafe(id string) bool {
+ for i := 0; i < len(id); i++ {
+ c := id[i]
+ switch {
+ case c >= 'a' && c <= 'z',
+ c >= 'A' && c <= 'Z',
+ c >= '0' && c <= '9',
+ c == '-', c == '.', c == '_', c == '~':
+ default:
+ return false
+ }
+ }
+ return len(id) > 0
+}
+
+func normalizeAka(d *Document) []AkaEntry {
+ var out []AkaEntry
+ seen := make(map[string]bool)
+ for i, a := range d.AkaID {
+ a.ID = strings.TrimSpace(a.ID)
+ a.DocSource = strings.TrimSpace(a.DocSource)
+ a.DocURL = strings.TrimSpace(a.DocURL)
+ if a.ID == "" {
+ warnf("%s: aka_id[%d] has no id - ignored", d.ID, i)
+ continue
+ }
+ key := strings.ToLower(a.ID)
+ if seen[key] {
+ warnf("%s: aka_id %q is listed twice - keeping the first", d.ID, a.ID)
+ continue
+ }
+ seen[key] = true
+ out = append(out, a)
+ }
+ return out
+}
+
+func searchMeta(d *Document) string {
+ var reporters, publishers, pubTitles, pubNotes, contributors []string
+ for _, p := range d.Publication {
+ if p.Title != "" {
+ pubTitles = append(pubTitles, p.Title)
+ }
+ if p.Note != "" {
+ pubNotes = append(pubNotes, p.Note)
+ }
+ if p.Publisher != "" {
+ publishers = append(publishers, p.Publisher)
+ }
+ contributors = append(contributors, p.Contributors...)
+ reporters = append(reporters, p.Journalists...)
+ }
+ return strings.Join([]string{
+ akaIDs(d.AkaID),
+ d.DocMeta.Creator,
+ strings.Join(reporters, valueSep),
+ strings.Join(contributors, valueSep),
+ strings.Join(publishers, valueSep),
+ strings.Join(pubTitles, valueSep),
+ strings.Join(pubNotes, valueSep),
+ d.ArchiveComment,
+ strings.Join(d.DocMeta.People, valueSep),
+ strings.Join(d.DocMeta.Countries, valueSep),
+ strings.Join(d.DocMeta.Programs, valueSep),
+ strings.Join(d.DocMeta.Classification, valueSep),
+ strings.Join(d.DocMeta.Dissemination, valueSep),
+ d.DocMeta.Banner,
+ d.DocMeta.Genre,
+ d.DocMeta.Series.Name,
+ d.ID,
+ strings.Join(d.Tags, valueSep),
+ }, valueSep)
+}
+
+func akaIDs(akas []AkaEntry) string {
+ ids := make([]string, len(akas))
+ for i, a := range akas {
+ ids[i] = a.ID
+ }
+ return strings.Join(ids, valueSep)
+}
+
+func loadPageImages(dir, label string) []string {
+ files, err := os.ReadDir(filepath.Join(dir, "images"))
+ if err != nil && !os.IsNotExist(err) {
+ warnf("%s: images directory unreadable: %v", label, err)
+ }
+ type pageImage struct {
+ name string
+ num int
+ }
+ var imgs []pageImage
+ for _, f := range files {
+ if !f.IsDir() && pageImageExts[strings.ToLower(filepath.Ext(f.Name()))] {
+ imgs = append(imgs, pageImage{f.Name(), parseImageNum(f.Name())})
+ }
+ }
+ sort.SliceStable(imgs, func(i, j int) bool { return imgs[i].num < imgs[j].num })
+ var out []string
+ for i, im := range imgs {
+ if i > 0 && im.num > 0 && im.num == imgs[i-1].num {
+ warnf("%s: images/%s and images/%s are both page %d - keep one", label, imgs[i-1].name, im.name, im.num)
+ }
+ out = append(out, im.name)
+ }
+ return out
+}
+
+type imageSize struct{ W, H int }
+
+func (s imageSize) Attr() template.HTMLAttr {
+ if s.W <= 0 || s.H <= 0 {
+ return ""
+ }
+ return template.HTMLAttr(fmt.Sprintf(` width="%d" height="%d"`, s.W, s.H))
+}
+
+// readImageSizes reads every page image's pixel size from its header, so a
+// page can reserve each image's height before it loads. It runs alongside the
+// rest of the load; the returned func waits for it and reports the failures.
+func (co *Corpus) readImageSizes() func() {
+ type job struct {
+ path string
+ size *imageSize
+ }
+ var jobs []job
+ add := func(dir string, names []string, sizes *[]imageSize) {
+ *sizes = make([]imageSize, len(names))
+ for i, name := range names {
+ if !strings.EqualFold(filepath.Ext(name), ".webp") {
+ jobs = append(jobs, job{filepath.Join(dir, "images", name), &(*sizes)[i]})
+ }
+ }
+ }
+ for i := range co.archive.Docs {
+ d := &co.archive.Docs[i]
+ add(d.Dir, d.Images, &d.ImageSizes)
+ for j := range d.Versions {
+ v := &d.Versions[j]
+ add(filepath.Join(d.Dir, "versions", v.Slug), v.Images, &v.ImageSizes)
+ }
+ }
+
+ ch := make(chan job)
+ var mu sync.Mutex
+ var wg sync.WaitGroup
+ var failed []string
+ for range 16 {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for j := range ch {
+ f, err := os.Open(j.path)
+ if err == nil {
+ var c image.Config
+ c, _, err = image.DecodeConfig(f)
+ f.Close()
+ *j.size = imageSize{c.Width, c.Height}
+ }
+ if err != nil {
+ mu.Lock()
+ failed = append(failed, fmt.Sprintf("%s: %v", strings.TrimPrefix(j.path, dataDir+string(filepath.Separator)), err))
+ mu.Unlock()
+ }
+ }
+ }()
+ }
+ go func() {
+ for _, j := range jobs {
+ ch <- j
+ }
+ close(ch)
+ }()
+ return func() {
+ wg.Wait()
+ sort.Strings(failed)
+ for _, f := range failed {
+ warnf("page image unreadable, served without its size: %s", f)
+ }
+ }
+}
+
+func versionSlugSafe(s string) bool {
+ if s == "" || strings.Contains(s, "..") || strings.Contains(s, "~") || !docIDSafe(s) {
+ return false
+ }
+ c := s[0]
+ return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9'
+}
+
+func (co *Corpus) loadVersions(d *Document) {
+ vdir := filepath.Join(d.Dir, "versions")
+ entries, err := os.ReadDir(vdir)
+ if err != nil && !os.IsNotExist(err) {
+ warnf("%s: versions directory unreadable: %v", d.ID, err)
+ }
+ for _, e := range entries {
+ name := e.Name()
+ if !e.IsDir() {
+ warnf("%s: versions/%s is not a folder - each version is a folder holding that copy's PDF, .md and images/", d.ID, name)
+ continue
+ }
+ if !versionSlugSafe(name) {
+ warnf("%s: versions/%s: version names may only contain letters, digits, \".\", \"_\" and \"-\", starting with a letter or digit - skipping", d.ID, name)
+ continue
+ }
+ v, err := loadVersion(filepath.Join(vdir, name), d.ID+"/versions/"+name)
+ if err != nil {
+ errorf("%s: versions/%s: %v - skipping version", d.ID, name, err)
+ continue
+ }
+ v.Slug = name
+ v.DocID = d.ID
+ d.Versions = append(d.Versions, v)
+ }
+
+ for i := range d.Publication {
+ p := &d.Publication[i]
+ p.Version = strings.TrimSpace(p.Version)
+ if p.Version == "" {
+ continue
+ }
+ v := d.VersionFor(p.Version)
+ if v == nil {
+ warnf("%s: publication %d (%s) points at versions/%s, which is missing or failed to load - its copy isn't linked", d.ID, i+1, p.Publisher, p.Version)
+ continue
+ }
+ v.PubIndex = append(v.PubIndex, i)
+ }
+ for i := range d.Versions {
+ v := &d.Versions[i]
+ v.Label = versionLabel(d, v)
+ }
+
+ for slug, info := range d.VersionMeta {
+ v := d.VersionFor(slug)
+ if v == nil {
+ warnf("%s: \"versions\" has an entry for versions/%s, which is missing or failed to load", d.ID, slug)
+ continue
+ }
+ v.ChangedPages = changedPages(d.ID, v, info.ChangedPages)
+ v.SHA = strings.ToLower(strings.TrimSpace(info.PDFSha256))
+ }
+
+ for _, v := range d.Versions {
+ if v.SHA == "" {
+ warnf("%s: versions/%s has no pdf_sha256 in metadata.json - its checksum isn't listed", d.ID, v.Slug)
+ }
+ }
+}
+
+func changedPages(docID string, v *DocVersion, pages []int) []ChangedPage {
+ pos := make(map[int]int, len(v.Images))
+ for i, name := range v.Images {
+ if n, err := pageNumber(name); err == nil {
+ if _, seen := pos[n]; !seen {
+ pos[n] = i + 1
+ }
+ }
+ }
+ nums := append([]int(nil), pages...)
+ sort.Ints(nums)
+ var out []ChangedPage
+ for i, n := range nums {
+ if i > 0 && n == nums[i-1] {
+ continue
+ }
+ if n < 1 {
+ warnf("%s: versions/%s: changed page %d is not a page number - ignored", docID, v.Slug, n)
+ continue
+ }
+ p := pos[n]
+ if p == 0 {
+ warnf("%s: versions/%s: changed page %d has no page image with that number - listed without a link", docID, v.Slug, n)
+ }
+ out = append(out, ChangedPage{Page: n, Pos: p})
+ }
+ return out
+}
+
+type contentFiles struct {
+ Text string
+ TextFile string
+ PDFFile string
+ PDFSize int64
+ HasThumb bool
+ HasGifThumb bool
+}
+
+func scanContentDir(dir string) (contentFiles, error) {
+ var c contentFiles
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return c, err
+ }
+ textDone := false
+ for _, f := range entries {
+ if f.IsDir() {
+ continue
+ }
+ name := f.Name()
+ if !textDone && isTranscript(name) {
+ textDone = true
+ c.TextFile = name
+ b, rerr := os.ReadFile(filepath.Join(dir, name))
+ if rerr != nil {
+ warnf("cannot read %s: %v", filepath.Join(dir, name), rerr)
+ } else {
+ c.Text = string(b)
+ }
+ }
+ if c.PDFFile == "" && isPDF(name) {
+ c.PDFFile = name
+ if info, ierr := f.Info(); ierr == nil {
+ c.PDFSize = info.Size()
+ }
+ }
+ if name == "thumb.jpg" {
+ c.HasThumb = true
+ }
+ if name == "thumb.gif" {
+ c.HasGifThumb = true
+ }
+ }
+ return c, nil
+}
+
+func loadVersion(dir, label string) (DocVersion, error) {
+ var v DocVersion
+ files, err := scanContentDir(dir)
+ if err != nil {
+ return v, err
+ }
+ if files.PDFFile == "" {
+ return v, fmt.Errorf("no PDF")
+ }
+ v.PDFFile, v.PDFSize, v.TextFile, v.FullText = files.PDFFile, files.PDFSize, files.TextFile, files.Text
+ if v.FullText != "" {
+ v.HTMLContent = template.HTML(wrapRedactions(Render(v.FullText)))
+ }
+ v.Images = loadPageImages(dir, label)
+ return v, nil
+}
+
+// VersionSnippet is a search-result snippet taken from one of the document's
+// versions, with the copy it came from.
+type VersionSnippet struct {
+ Slug string
+ Label string
+ HTML template.HTML
+}
+
+func versionLabel(d *Document, v *DocVersion) string {
+ var names []string
+ seen := make(map[string]bool)
+ date := ""
+ for _, i := range v.PubIndex {
+ p := d.Publication[i]
+ if key := valueKey(p.Publisher); key != "" && !seen[key] {
+ seen[key] = true
+ names = append(names, p.Publisher)
+ }
+ if p.DatePublished != "" && (date == "" || p.DatePublished < date) {
+ date = p.DatePublished
+ }
+ }
+ if len(names) == 0 {
+ return v.Slug
+ }
+ label := strings.Join(names, " / ")
+ if date != "" {
+ label += ", " + date
+ }
+ return label
+}
+
+// lineKey is the form transcript lines are compared in: folded, lowercased,
+// whitespace collapsed. Lines without a letter or digit ("---", "|") key to
+// "" and are never new.
+func lineKey(line string) string {
+ k := strings.Join(strings.Fields(normFoldString(line)), " ")
+ if strings.IndexFunc(k, isWordRune) < 0 {
+ return ""
+ }
+ return k
+}
+
+func versionNovelText(d *Document) string {
+ if len(d.Versions) == 0 {
+ return ""
+ }
+ inMain := make(map[string]bool)
+ for _, line := range strings.Split(d.FullText, "\n") {
+ if k := lineKey(line); k != "" {
+ inMain[k] = true
+ }
+ }
+ added := make(map[string]bool)
+ var all []string
+ for i := range d.Versions {
+ v := &d.Versions[i]
+ var novel []string
+ for _, line := range strings.Split(v.FullText, "\n") {
+ k := lineKey(line)
+ if k == "" || inMain[k] {
+ continue
+ }
+ line = strings.TrimSpace(line)
+ novel = append(novel, line)
+ if !added[k] {
+ added[k] = true
+ all = append(all, line)
+ }
+ }
+ v.NovelText = strings.Join(novel, "\n")
+ }
+ return strings.Join(all, "\n")
+}
+
+func setBodySearchText(d *Document) {
+ d.StemmedFull = strings.Join(tokenize(d.FullText), " ")
+ d.FoldedFull = normFoldString(d.FullText)
+ d.bodyFoldLen = len(d.FoldedFull)
+ if novel := versionNovelText(d); novel != "" {
+ d.StemmedFull = strings.TrimSpace(d.StemmedFull + " " + valueSep + " " + strings.Join(tokenize(novel), " "))
+ d.FoldedFull += valueSep + normFoldString(novel)
+ }
+}
diff --git a/explore.go b/explore.go
new file mode 100644
index 0000000..78ea5ec
--- /dev/null
+++ b/explore.go
@@ -0,0 +1,377 @@
+package main
+
+import (
+ "log"
+ "net/http"
+ "sort"
+ "strconv"
+ "strings"
+)
+
+// The Explore section (/explore).
+
+type exploreCol struct {
+ Header string
+ Key string
+ Ratio bool
+}
+
+type exploreMetric struct {
+ Slug string
+ Title string
+ Blurb string
+ Cols []exploreCol
+ Vals func(d *Document) ([]float64, bool)
+}
+
+const exploreRows = 200 // rows shown per page ("top 200 of N")
+
+var exploreMetrics = []exploreMetric{
+ {
+ Slug: "programs",
+ Title: "Most programs mentioned",
+ Blurb: "Documents referencing the most surveillance programs.",
+ Cols: []exploreCol{
+ {Header: "Programs", Key: "programs"},
+ {Header: "Pages", Key: "pages"},
+ {Header: "Programs / page", Key: "ratio", Ratio: true},
+ },
+ Vals: func(d *Document) ([]float64, bool) {
+ n := len(d.DocMeta.Programs)
+ if n == 0 {
+ return nil, false
+ }
+ pages, ratio := float64(d.Pages), -1.0
+ if d.Pages > 0 {
+ ratio = float64(n) / float64(d.Pages)
+ }
+ return []float64{float64(n), pages, ratio}, true
+ },
+ },
+ {
+ Slug: "people",
+ Title: "Most people mentioned",
+ Blurb: "Documents whose metadata names the most individuals.",
+ Cols: []exploreCol{{Header: "People", Key: "people"}},
+ Vals: func(d *Document) ([]float64, bool) {
+ n := len(d.DocMeta.People)
+ return []float64{float64(n)}, n > 0
+ },
+ },
+ {
+ Slug: "countries",
+ Title: "Most countries mentioned",
+ Blurb: "Documents spanning the most countries.",
+ Cols: []exploreCol{{Header: "Countries", Key: "countries"}},
+ Vals: func(d *Document) ([]float64, bool) {
+ n := len(d.DocMeta.Countries)
+ return []float64{float64(n)}, n > 0
+ },
+ },
+ {
+ Slug: "pages",
+ Title: "Longest documents",
+ Blurb: "Documents with the most pages.",
+ Cols: []exploreCol{{Header: "Pages", Key: "pages"}},
+ Vals: func(d *Document) ([]float64, bool) {
+ return []float64{float64(d.Pages)}, d.Pages > 0
+ },
+ },
+}
+
+func exploreMetricBySlug(slug string) *exploreMetric {
+ for i := range exploreMetrics {
+ if exploreMetrics[i].Slug == slug {
+ return &exploreMetrics[i]
+ }
+ }
+ return nil
+}
+
+func (co *Corpus) handleExploreIndex(w http.ResponseWriter, r *http.Request) {
+ render(w, r, &explorePage{
+ Metrics: exploreMetrics,
+ HasAssemblies: len(co.assemblies) > 0,
+ })
+}
+
+type exploreRow struct {
+ Rank int
+ Doc Document
+ Vals []string
+}
+
+type exploreTable struct {
+ Rows []exploreRow
+ Total int
+}
+
+func exploreSet(slug string, foia bool) string {
+ if foia {
+ return slug
+ }
+ return slug + ":nofoia"
+}
+
+func (co *Corpus) buildExploreTables() {
+ co.exploreTables = make(map[string][]exploreTable, 2*len(exploreMetrics))
+ for mi := range exploreMetrics {
+ m := &exploreMetrics[mi]
+ if _, dup := co.exploreTables[m.Slug]; dup {
+ log.Fatalf("explore: duplicate metric slug %q", m.Slug)
+ }
+ for _, foia := range []bool{true, false} {
+ co.exploreTables[exploreSet(m.Slug, foia)] = co.buildExploreTable(m, foia)
+ }
+ }
+}
+
+func (co *Corpus) buildExploreTable(m *exploreMetric, foia bool) []exploreTable {
+ type scored struct {
+ doc *Document
+ vals []float64
+ }
+ var rows []scored
+ for i := range co.archive.Docs {
+ d := &co.archive.Docs[i]
+ if !foia && d.Collection == collectionFOIA {
+ continue
+ }
+ if vals, ok := m.Vals(d); ok {
+ rows = append(rows, scored{d, vals})
+ }
+ }
+
+ tables := make([]exploreTable, len(m.Cols))
+ for sortIdx := range m.Cols {
+ sorted := append([]scored(nil), rows...)
+ sort.SliceStable(sorted, func(i, j int) bool {
+ if a, b := sorted[i].vals[sortIdx], sorted[j].vals[sortIdx]; a != b {
+ return a > b
+ }
+ if a, b := sorted[i].vals[0], sorted[j].vals[0]; a != b {
+ return a > b
+ }
+ return sorted[i].doc.Title < sorted[j].doc.Title
+ })
+ if len(sorted) > exploreRows {
+ sorted = sorted[:exploreRows]
+ }
+ out := make([]exploreRow, len(sorted))
+ for i, rw := range sorted {
+ vals := make([]string, len(rw.vals))
+ for k, v := range rw.vals {
+ switch {
+ case v < 0:
+ vals[k] = "-"
+ case m.Cols[k].Ratio:
+ vals[k] = trimRatio(v)
+ default:
+ vals[k] = strconv.Itoa(int(v))
+ }
+ }
+ out[i] = exploreRow{Rank: i + 1, Doc: *rw.doc, Vals: vals}
+ }
+ tables[sortIdx] = exploreTable{Rows: out, Total: len(rows)}
+ }
+ return tables
+}
+
+func exploreKey(set string, sortIdx int) string {
+ return "explore:" + set + ":" + strconv.Itoa(sortIdx)
+}
+
+func (co *Corpus) handleExploreMetric(w http.ResponseWriter, r *http.Request) {
+ m := exploreMetricBySlug(r.PathValue("metric"))
+ if m == nil {
+ notFound(w, r)
+ return
+ }
+
+ sortIdx := 0
+ sortBy := r.URL.Query().Get("sort")
+ for i, c := range m.Cols {
+ if c.Key == sortBy {
+ sortIdx = i
+ break
+ }
+ }
+ set := exploreSet(m.Slug, readPrefs(r).IncludeFOIA)
+ t := co.exploreTables[set][sortIdx]
+
+ cols := []sortCol{{Label: "#", Class: "num-col"}, {Label: "Document"}, {Label: "Date"}}
+ for _, c := range m.Cols {
+ cols = append(cols, sortCol{Key: c.Key, Label: c.Header, Class: "num-col", DescFirst: true, Fixed: true})
+ }
+
+ render(w, r, &exploreTablePage{
+ Metric: m,
+ Rows: t.Rows,
+ RowsHTML: co.preHTML(exploreKey(set, sortIdx)),
+ Total: t.Total,
+ Shown: len(t.Rows),
+ Heads: sortHeads("?", cols, m.Cols[sortIdx].Key, true),
+ })
+}
+
+func trimRatio(v float64) string {
+ s := strconv.FormatFloat(v, 'f', 2, 64)
+ s = strings.TrimRight(s, "0")
+ return strings.TrimRight(s, ".")
+}
+
+// ---------------------------------------------------------------------------
+// /explore/classification
+
+type clsSeg struct {
+ Value string
+ Count int
+ Pct int
+ URL string
+ Class string // seg-0..seg-3, one per stripLevels entry
+}
+
+type clsYearRow struct {
+ Year string
+ URL string // the year's page, over every collection as the counts are
+ Total int
+ Segs []clsSeg
+}
+
+type heatCell struct {
+ Count int
+ URL string
+}
+
+type heatRow struct {
+ Name string
+ URL string
+ Cells []heatCell
+}
+
+var crossCols = []string{"TOP SECRET", "SECRET", "CONFIDENTIAL", "UNCLASSIFIED", "SI", "COMINT", "STRAP 1", "STRAP 2", "TK"}
+var stripLevels = []string{"TOP SECRET", "SECRET", "CONFIDENTIAL", "UNCLASSIFIED"}
+
+const stripMinMarks = 5
+
+func (co *Corpus) buildClassPatterns() {
+ top := crossCols
+ rank := make(map[string]int, len(top))
+ for i, v := range top {
+ rank[valueKey(v)] = i
+ }
+ levelRank := make(map[string]int, len(stripLevels))
+ for i, v := range stripLevels {
+ levelRank[valueKey(v)] = i
+ }
+
+ yearCounts := map[string][]int{}
+ agCounts := map[string][]int{}
+ agName := map[string]string{}
+ for i := range co.archive.Docs {
+ d := &co.archive.Docs[i]
+ year := docYear(d)
+ agency := d.DocMeta.Creator
+ if n, ok := agName[valueKey(agency)]; ok {
+ agency = n
+ } else {
+ agName[valueKey(agency)] = agency
+ }
+ for _, cv := range d.DocMeta.Classification {
+ if lvl, isLevel := levelRank[valueKey(cv)]; isLevel && year != "" {
+ yc := yearCounts[year]
+ if yc == nil {
+ yc = make([]int, len(stripLevels))
+ yearCounts[year] = yc
+ }
+ yc[lvl]++
+ }
+ idx, known := rank[valueKey(cv)]
+ if known && agency != "" {
+ ac := agCounts[agency]
+ if ac == nil {
+ ac = make([]int, len(top))
+ agCounts[agency] = ac
+ }
+ ac[idx]++
+ }
+ }
+ }
+
+ years := make([]string, 0, len(yearCounts))
+ for y := range yearCounts {
+ years = append(years, y)
+ }
+ sort.Strings(years)
+ strips := make([]clsYearRow, 0, len(years))
+ var sparse []clsYearRow
+ for _, y := range years {
+ yc := yearCounts[y]
+ total := 0
+ for _, n := range yc {
+ total += n
+ }
+ row := clsYearRow{Year: y, URL: wholeArchive.link(categoryValueURL("docdate", y)), Total: total}
+ if total < stripMinMarks {
+ sparse = append(sparse, row)
+ continue
+ }
+ for i, n := range yc {
+ if n == 0 {
+ continue
+ }
+ row.Segs = append(row.Segs, clsSeg{
+ Value: stripLevels[i],
+ Count: n,
+ Pct: barPct(n, total, 1),
+ URL: wholeArchive.link(dateRangeSearchURL("CL", exactClause(stripLevels[i]), [2]string{"date_doc", y})),
+ Class: "seg-" + strconv.Itoa(i),
+ })
+ }
+ strips = append(strips, row)
+ }
+
+ mass := make(map[string]int, len(agCounts))
+ for a, cs := range agCounts {
+ for _, n := range cs {
+ mass[a] += n
+ }
+ }
+ ags := rankCounts(mass, -1)
+ rows := make([]heatRow, 0, len(ags))
+ for _, a := range ags {
+ hr := heatRow{Name: a, URL: wholeArchive.link(categoryValueURL("agency", a))}
+ for i, n := range agCounts[a] {
+ cell := heatCell{Count: n}
+ if n > 0 {
+ cell.URL = wholeArchive.link(searchURL([]searchClause{
+ {Field: "CR", Value: exactClause(a)},
+ {Field: "CL", Value: exactClause(top[i])},
+ }))
+ }
+ hr.Cells = append(hr.Cells, cell)
+ }
+ rows = append(rows, hr)
+ }
+
+ cols := make([]facetEntry, len(top))
+ for i, v := range top {
+ cols[i] = facetEntry{Name: v, URL: wholeArchive.link(categoryValueURL("classification", v))}
+ }
+
+ co.classPatterns.Strips = strips
+ co.classPatterns.Sparse = sparse
+ co.classPatterns.Cols = cols
+ co.classPatterns.HeatRows = rows
+}
+
+func (co *Corpus) handleClassPatterns(w http.ResponseWriter, r *http.Request) {
+ render(w, r, &classPatternsPage{
+ Cols: co.classPatterns.Cols,
+ Levels: stripLevels,
+ Strips: co.classPatterns.Strips,
+ Sparse: co.classPatterns.Sparse,
+ MinMarks: stripMinMarks,
+ HeatRows: co.classPatterns.HeatRows,
+ })
+}
diff --git a/explore_ids.go b/explore_ids.go
new file mode 100644
index 0000000..58a873f
--- /dev/null
+++ b/explore_ids.go
@@ -0,0 +1,114 @@
+// explore_ids.go - /explore/ids: every document's identifiers in one table.
+//
+// /explore/ids.txt is the same checksums in sha256sum(1) format, so a mirror
+// or a download verifies with "sha256sum -c" in the corpus root.
+package main
+
+import (
+ "fmt"
+ "net/http"
+ "path/filepath"
+ "sort"
+ "strings"
+)
+
+type idVersionRow struct {
+ Slug string
+ PDFFile string
+ PDFURL string
+ Size string
+ Pages int
+ SHA string
+}
+
+type idRow struct {
+ ID string
+ Collection string
+ Pages int
+ PDFFile string
+ PDFURL string
+ Size string
+ SHA string
+ Aka []AkaEntry
+ Versions []idVersionRow
+}
+
+func humanSize(n int64) string {
+ switch {
+ case n <= 0:
+ return ""
+ case n < 1<<10:
+ return fmt.Sprintf("%d B", n)
+ case n < 1<<20:
+ return fmt.Sprintf("%.0f KB", float64(n)/(1<<10))
+ case n < 1<<30:
+ return fmt.Sprintf("%.1f MB", float64(n)/(1<<20))
+ }
+ return fmt.Sprintf("%.2f GB", float64(n)/(1<<30))
+}
+
+func (co *Corpus) buildIDRows() {
+ co.idRows = make([]idRow, 0, len(co.archive.Docs))
+ var sums strings.Builder
+
+ for i := range co.archive.Docs {
+ d := &co.archive.Docs[i]
+ row := idRow{
+ ID: d.ID,
+ Collection: d.Collection,
+ Pages: d.Pages,
+ PDFFile: d.PDFFile,
+ PDFURL: filePath(d.ID, d.PDFFile),
+ Size: humanSize(d.PDFSize),
+ SHA: strings.TrimSpace(d.PDFSha256),
+ Aka: d.AkaID,
+ }
+ for _, v := range d.Versions {
+ row.Versions = append(row.Versions, idVersionRow{
+ Slug: v.Slug,
+ PDFFile: v.PDFFile,
+ PDFURL: versionFilePath(d.ID, v.Slug, v.PDFFile),
+ Size: humanSize(v.PDFSize),
+ Pages: len(v.Images),
+ SHA: v.SHA,
+ })
+ }
+ co.idRows = append(co.idRows, row)
+
+ rel, err := filepath.Rel(dataDir, d.Dir)
+ if err != nil {
+ rel = d.ID
+ }
+ if row.SHA != "" {
+ sums.WriteString(row.SHA + " " + filepath.ToSlash(filepath.Join(rel, d.PDFFile)) + "\n")
+ }
+ for _, v := range d.Versions {
+ if v.SHA != "" {
+ sums.WriteString(v.SHA + " " + filepath.ToSlash(filepath.Join(rel, "versions", v.Slug, v.PDFFile)) + "\n")
+ }
+ }
+ }
+
+ sort.Slice(co.idRows, func(i, j int) bool {
+ a, b := strings.ToLower(co.idRows[i].ID), strings.ToLower(co.idRows[j].ID)
+ if a != b {
+ return a < b
+ }
+ return co.idRows[i].ID < co.idRows[j].ID
+ })
+ co.idChecksums = sums.String()
+}
+
+func (co *Corpus) handleExploreIDs(w http.ResponseWriter, r *http.Request) {
+ render(w, r, &idsPage{
+ Rows: co.idRows,
+ RowsHTML: co.preHTML("ids"),
+ Shown: len(co.idRows),
+ })
+}
+
+func (co *Corpus) handleExploreIDsText(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+ w.Header().Set("Cache-Control", "public, max-age=3600")
+ w.Write([]byte(co.idChecksums))
+}
diff --git a/glossary.go b/glossary.go
new file mode 100644
index 0000000..2c89d63
--- /dev/null
+++ b/glossary.go
@@ -0,0 +1,135 @@
+// glossary.go - codename/term glossary loaded from data/glossary.json.
+
+package main
+
+import (
+ "encoding/json"
+ "html/template"
+ "log"
+ "net/http"
+ "os"
+ "sort"
+ "strings"
+)
+
+type GlossaryEntry struct {
+ Term string `json:"term"`
+ Aka []string `json:"aka,omitempty"`
+ Definition string `json:"definition"`
+ See []string `json:"see,omitempty"`
+ SeeLinks []seeLink `json:"-"`
+
+ searchLink string
+}
+
+type seeLink struct{ Name, Term string }
+
+func (e GlossaryEntry) DefinitionHTML() template.HTML {
+ return template.HTML(e.Definition)
+}
+
+func (e GlossaryEntry) SearchURL() string {
+ if e.searchLink != "" {
+ return e.searchLink
+ }
+ return e.buildSearchURL()
+}
+
+func (e GlossaryEntry) buildSearchURL() string {
+ clauses := []searchClause{{Conj: "and", Field: "TE", Value: searchClauseValue(e.Term)}}
+ for _, a := range e.Aka {
+ if strings.TrimSpace(a) != "" {
+ clauses = append(clauses, searchClause{Conj: "or", Field: "TE", Value: searchClauseValue(a)})
+ }
+ }
+ return searchURL(clauses)
+}
+
+func searchClauseValue(v string) string {
+ v = strings.TrimSpace(v)
+ if len(splitWords(v)) == 1 && len(splitWords(v)[0]) == len(v) {
+ return v
+ }
+ return `"` + v + `"`
+}
+
+func (co *Corpus) glossaryLookup(term string) *GlossaryEntry {
+ return co.glossaryIndex[strings.ToLower(term)]
+}
+
+func (co *Corpus) loadGlossary() {
+ p := projectPath("glossary.json")
+ b, err := os.ReadFile(p)
+ if err != nil {
+ log.Printf("Note: %s not found - glossary page will be empty.", p)
+ return
+ }
+ if err := json.Unmarshal(b, &co.glossary); err != nil {
+ warnf("%s is malformed: %v - glossary disabled.", p, err)
+ co.glossary = nil
+ return
+ }
+ co.glossaryIndex = map[string]*GlossaryEntry{}
+ sort.SliceStable(co.glossary, func(i, j int) bool {
+ return normFoldString(co.glossary[i].Term) < normFoldString(co.glossary[j].Term)
+ })
+ for i := range co.glossary {
+ e := &co.glossary[i]
+ e.searchLink = e.buildSearchURL()
+ addIndex := func(key string) {
+ low := strings.ToLower(key)
+ if prev := co.glossaryIndex[low]; prev != nil && prev != e {
+ warnf("glossary term/alias %q appears under both %q and %q - the later entry wins", key, prev.Term, e.Term)
+ }
+ co.glossaryIndex[low] = e
+ }
+ addIndex(e.Term)
+ for _, a := range e.Aka {
+ addIndex(a)
+ }
+ }
+ var unknown []string
+ for i := range co.glossary {
+ e := &co.glossary[i]
+ e.SeeLinks = nil
+ for _, name := range e.See {
+ l := seeLink{Name: name}
+ if t := co.glossaryLookup(name); t != nil {
+ l.Term = t.Term
+ } else {
+ unknown = append(unknown, name)
+ }
+ e.SeeLinks = append(e.SeeLinks, l)
+ }
+ }
+ if len(unknown) > 0 {
+ warnf("glossary: %d see-also names match no term or alias, shown unlinked: %s",
+ len(unknown), strings.Join(firstN(unknown, 8), ", "))
+ }
+ co.buildGlossaryGroups()
+ log.Printf("Glossary loaded: %d terms", len(co.glossary))
+}
+
+type glossaryGroup struct {
+ Letter string
+ Entries []GlossaryEntry
+}
+
+func (co *Corpus) buildGlossaryGroups() {
+ co.glossaryGroups = nil
+ for _, e := range co.glossary {
+ letter := firstLetter(e.Term)
+ if len(co.glossaryGroups) == 0 || co.glossaryGroups[len(co.glossaryGroups)-1].Letter != letter {
+ co.glossaryGroups = append(co.glossaryGroups, glossaryGroup{Letter: letter})
+ }
+ co.glossaryGroups[len(co.glossaryGroups)-1].Entries = append(co.glossaryGroups[len(co.glossaryGroups)-1].Entries, e)
+ }
+}
+
+func (co *Corpus) handleGlossary(w http.ResponseWriter, r *http.Request) {
+ render(w, r, &glossaryPage{
+ Groups: co.glossaryGroups,
+ GroupsHTML: co.preHTML("glossary"),
+ Total: len(co.glossary),
+ })
+}
diff --git a/go.mod b/go.mod
new file mode 100644
index 0000000..8578a29
--- /dev/null
+++ b/go.mod
@@ -0,0 +1,3 @@
+module snowchive
+
+go 1.22
diff --git a/history.go b/history.go
new file mode 100644
index 0000000..8b6c03a
--- /dev/null
+++ b/history.go
@@ -0,0 +1,194 @@
+// history.go - opt-in, cookie-local browsing history: recently viewed
+// documents and recent searches. Both are OFF by default (see Prefs).
+
+package main
+
+import (
+ "net/http"
+ "net/url"
+ "strings"
+)
+
+const (
+ recentDocsCookie = "recent_docs"
+ recentSearchesCookie = "recent_searches"
+ recentDocsMax = 20
+ recentSearchesMax = 15
+)
+
+func readIDListCookie(r *http.Request, name string, max int) []string {
+ c, err := r.Cookie(name)
+ if err != nil || c.Value == "" {
+ return nil
+ }
+ var out []string
+ for _, id := range strings.Split(c.Value, "|") {
+ if id != "" {
+ out = append(out, id)
+ if len(out) == max {
+ break
+ }
+ }
+ }
+ return out
+}
+
+func pushFront(list []string, id string, max int) []string {
+ out := make([]string, 0, max)
+ out = append(out, id)
+ for _, x := range list {
+ if len(out) == max {
+ break
+ }
+ if x != id {
+ out = append(out, x)
+ }
+ }
+ return out
+}
+
+func pushRecentDoc(w http.ResponseWriter, r *http.Request, id string) {
+ ids := pushFront(readIDListCookie(r, recentDocsCookie, recentDocsMax), id, recentDocsMax)
+ writeListCookie(w, recentDocsCookie, ids)
+}
+
+type recentSearch struct {
+ Label string
+ URL string
+ QS string
+}
+
+func recordRecentSearch(w http.ResponseWriter, r *http.Request, qs string) []string {
+ if len(qs) > searchQSMax {
+ return nil
+ }
+ entry := url.QueryEscape(qs)
+
+ entries := readIDListCookie(r, recentSearchesCookie, recentSearchesMax)
+ entries = pushFront(entries, entry, recentSearchesMax)
+ return writeListCookie(w, recentSearchesCookie, entries)
+}
+
+func (co *Corpus) readRecentSearches(r *http.Request) []recentSearch {
+ return co.parseSearchEntries(readIDListCookie(r, recentSearchesCookie, recentSearchesMax))
+}
+
+func (co *Corpus) parseSearchEntries(entries []string) []recentSearch {
+ var out []recentSearch
+ for _, e := range entries {
+ qs, err := url.QueryUnescape(e)
+ if err != nil {
+ continue
+ }
+ v, err := url.ParseQuery(qs)
+ if err != nil {
+ continue
+ }
+ if label, ok := co.searchLabel(v); ok {
+ out = append(out, recentSearch{Label: label, URL: "/search?" + qs, QS: qs})
+ }
+ }
+ return out
+}
+
+const searchQSMax = 500
+
+func (co *Corpus) savableSearch(v url.Values) bool {
+ if len(v.Encode()) > searchQSMax {
+ return false
+ }
+ _, ok := co.searchLabel(v)
+ return ok
+}
+
+func (co *Corpus) searchLabel(v url.Values) (label string, ok bool) {
+ var parts []string
+ fields := v["fqf"]
+ conjs := v["fqc"]
+ for i, term := range v["fqv"] {
+ if term == "" {
+ continue
+ }
+ label := term
+ if i < len(fields) && fields[i] != "" && fields[i] != "TE" {
+ code := fields[i]
+ if f, ok := fieldByCode[code]; ok {
+ label = term + " [" + f.Label + "]"
+ } else {
+ label = term + " [" + code + "]"
+ }
+ }
+ if i < len(conjs) && conjs[i] == "not" {
+ label = "-" + label
+ }
+ parts = append(parts, label)
+ }
+ var notes, filters []string
+ if colls := v["coll"]; len(colls) > 0 && len(colls) < len(validCollections) {
+ filters = append(filters, colls...)
+ }
+ if types := v["type"]; len(types) > 0 && (len(co.archive.Types) == 0 || len(types) < len(co.archive.Types)) {
+ filters = append(filters, types...)
+ }
+ if len(filters) > 0 {
+ notes = append(notes, "("+strings.Join(filters, ", ")+")")
+ }
+ if n := dateRangeNote("doc", v.Get("date_doc_start"), v.Get("date_doc_end")); n != "" {
+ notes = append(notes, n)
+ }
+ if n := dateRangeNote("pub", v.Get("date_pub_start"), v.Get("date_pub_end")); n != "" {
+ notes = append(notes, n)
+ }
+ switch {
+ case len(parts) == 0 && len(notes) == 0:
+ return "", false
+ case len(parts) == 0:
+ parts = []string{"all documents"}
+ }
+ return strings.Join(append(parts, notes...), " · "), true
+}
+
+func dateRangeNote(kind, start, end string) string {
+ switch {
+ case start == "" && end == "":
+ return ""
+ case start == end || (start != "" && end == padDateEnd(start)):
+ // date:2013 arrives as start "2013", end "2013-12-31": one year
+ return "(" + kind + " " + start + ")"
+ case start == "":
+ return "(" + kind + " to " + end + ")"
+ case end == "":
+ return "(" + kind + " from " + start + ")"
+ default:
+ return "(" + kind + " " + start + " to " + end + ")"
+ }
+}
+
+func handleHistoryClear(w http.ResponseWriter, r *http.Request) {
+ if !readPOSTForm(w, r) {
+ return
+ }
+ switch r.FormValue("which") {
+ case "docs":
+ delCookie(w, recentDocsCookie)
+ case "searches":
+ delCookie(w, recentSearchesCookie)
+ default:
+ delCookie(w, recentDocsCookie)
+ delCookie(w, recentSearchesCookie)
+ }
+ http.Redirect(w, r, safeBackPath(r.FormValue("back")), http.StatusSeeOther)
+}
+
+// safeBackPath only allows local absolute paths as redirect targets.
+func safeBackPath(p string) string {
+ if !strings.HasPrefix(p, "/") || strings.HasPrefix(p, "//") {
+ return "/"
+ }
+ for i := 0; i < len(p); i++ {
+ if p[i] == '\\' || p[i] < 0x20 {
+ return "/"
+ }
+ }
+ return p
+}
diff --git a/main.go b/main.go
new file mode 100644
index 0000000..cc97920
--- /dev/null
+++ b/main.go
@@ -0,0 +1,773 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "embed"
+ "encoding/hex"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "html/template"
+ "io/fs"
+ "log"
+ "maps"
+ "net"
+ "net/http"
+ "net/url"
+ "os"
+ "os/signal"
+ "path"
+ "path/filepath"
+ "slices"
+ "strconv"
+ "strings"
+ "syscall"
+ "time"
+)
+
+// devMode is set by the -dev flag. In dev mode templates and static files
+// are read from ./templates and ./static on every request (so edits show
+// without a rebuild), caching is disabled and error details are shown in
+// responses. Never run production with -dev: the error detail leaks
+// filesystem paths and template internals.
+var devMode bool
+
+// The templates and static assets are compiled into the binary, so a
+// deployment is one executable plus the data and project directories - nothing depends
+// on the working directory the process happens to start in. -dev swaps in
+// the on-disk copies (see initAssets).
+//
+//go:embed templates static
+var embeddedAssets embed.FS
+
+var (
+ templateFS fs.FS // templates/*.html
+ staticFS fs.FS // static/*
+)
+
+// staticHashes holds a content hash per embedded asset, computed once at
+// startup. It is both the assets' ETag (embed.FS files have a zero modtime,
+// so net/http sends no Last-Modified and an unvalidated asset would be
+// re-downloaded on every view) and their version stamp: base.html links
+// them as ?v= (the asset func), and handleStatic caches exactly that
+// URL forever. A changed file gets a new hash and so a new URL, whatever
+// the build was stamped with.
+var staticHashes map[string]string
+
+func buildStaticHashes() {
+ staticHashes = map[string]string{}
+ err := fs.WalkDir(staticFS, ".", func(p string, d fs.DirEntry, err error) error {
+ if err != nil || d.IsDir() {
+ return err
+ }
+ b, err := fs.ReadFile(staticFS, p)
+ if err != nil {
+ return err
+ }
+ sum := sha256.Sum256(b)
+ staticHashes[p] = hex.EncodeToString(sum[:8])
+ return nil
+ })
+ if err != nil {
+ log.Fatalf("embedded static: %v", err)
+ }
+}
+
+func initAssets() {
+ if devMode {
+ templateFS = os.DirFS("templates")
+ staticFS = os.DirFS("static")
+ staticHashes = nil
+ return
+ }
+ var err error
+ if templateFS, err = fs.Sub(embeddedAssets, "templates"); err != nil {
+ log.Fatalf("embedded templates: %v", err)
+ }
+ if staticFS, err = fs.Sub(embeddedAssets, "static"); err != nil {
+ log.Fatalf("embedded static: %v", err)
+ }
+ buildStaticHashes()
+}
+
+type sortOption struct{ Key, Label string }
+
+var sortOptions = func() []sortOption {
+ opts := []sortOption{{"relevance", "Relevance"}}
+ for _, s := range docSorts {
+ if s.InBar {
+ opts = append(opts, sortOption{s.Key, s.Label})
+ }
+ }
+ return opts
+}()
+
+func sortLabel(key string) string {
+ switch key {
+ case "relevance":
+ return "Relevance"
+ case "":
+ return "Title"
+ }
+ if s := docSortByKey[key]; s != nil {
+ return s.Label
+ }
+ return strings.ToUpper(key[:1]) + key[1:]
+}
+
+var (
+ // tmplSet holds one parsed template set PER PAGE (base + partials +
+ // page).
+ tmplSet map[string]*template.Template
+
+ funcMap = template.FuncMap{
+ "buildInfo": func() map[string]string {
+ return map[string]string{"Commit": buildCommit}
+ },
+ // articleKey is the key of the article page for a publication's
+ // link. Every linked publication of a loaded document has one
+ // (buildArticles), so the key needs no lookup - and no corpus.
+ "articleKey": func(link string) string {
+ if link == "" {
+ return ""
+ }
+ return articleKeyFor(link)
+ },
+ "asset": func(name string) string {
+ if h, ok := staticHashes[name]; ok {
+ return "/static/" + name + "?v=" + h
+ }
+ return "/static/" + name
+ },
+ // wholeArchiveQS is the scope that names every collection, for the
+ // links of a FOIA release on a page rendered once for all readers.
+ "wholeArchiveQS": func() string { return wholeArchive.suffix() },
+ "facetLinks": facetLinks,
+ "add": func(a, b int) int { return a + b },
+ "eqFold": sameValue,
+ "sortOptions": func() []sortOption { return sortOptions },
+ "sortLabel": sortLabel,
+ "agencyName": agencyName,
+ "dict": func(values ...interface{}) (map[string]interface{}, error) {
+ if len(values)%2 != 0 {
+ return nil, fmt.Errorf("dict: odd number of arguments (%d)", len(values))
+ }
+ dict := make(map[string]interface{}, len(values)/2)
+ for i := 0; i < len(values); i += 2 {
+ key, ok := values[i].(string)
+ if !ok {
+ return nil, fmt.Errorf("dict: argument %d is not a string key (%T)", i, values[i])
+ }
+ dict[key] = values[i+1]
+ }
+ return dict, nil
+ },
+ // pathEscape percent-encodes a value for use as a single URL path
+ // segment (so values like "TOP SECRET//SI" don't split the path).
+ "pathEscape": func(s string) template.URL {
+ return template.URL(url.PathEscape(s))
+ },
+ "azID": azID,
+ "pages": func(current, total int) []int {
+ var out []int
+ last := 0
+ for i := 1; i <= total; i++ {
+ if i == 1 || i == total || (i >= current-2 && i <= current+2) {
+ if last != 0 && i-last > 1 {
+ out = append(out, 0)
+ }
+ out = append(out, i)
+ last = i
+ }
+ }
+ return out
+ },
+ }
+)
+
+// Build identity, stamped at build time so mirrors are comparable at a
+// glance. Shown in the footer next to the data load time:
+//
+// go build -ldflags "-X main.buildCommit=$(git rev-parse --short HEAD)"
+var buildCommit = "dev"
+
+// accelPrefix is set by -accel-files: the internal nginx location that maps
+// to the data directory. With it, handleFiles still decides whether a file is
+// served, but answers with an X-Accel-Redirect and nginx sends the bytes
+// (sendfile, ranges, slow readers) without them passing through this process.
+// Only set it behind nginx: any other client gets an empty 200.
+var accelPrefix string
+
+// accelExpires is set by -accel-expires: how long the proxy may keep a page
+// this server marks public, sent as X-Accel-Expires (which nginx obeys ahead
+// of Cache-Control and strips from the response). Browsers still get the
+// short max-age. The corpus is read once at startup, so such a page cannot
+// change before the next restart; clear the proxy's cache after one.
+var accelExpires string
+
+// apiEnabled is cleared by -no-api: public instances that don't want bulk
+// scraping can turn the JSON API off.
+var apiEnabled = true
+
+func main() {
+ rebuildRelated := flag.Bool("rebuild-related", false, "Rebuild related docs index and exit")
+ flag.BoolVar(&devMode, "dev", false, "Dev mode: serve templates/ and static/ from the working directory and re-parse templates per request, no immutable static cache, verbose errors")
+ baseFlag := flag.String("base", os.Getenv("SNOWCHIVE_BASE"),
+ "Canonical base URL (e.g. https://archive.example.org) for the absolute URLs in robots.txt and the sitemap. "+
+ "Without it those URLs are built from each request's Host header, which is fine only behind a proxy that pins Host. "+
+ "Set this on anything internet-facing. Defaults to $SNOWCHIVE_BASE.")
+ listen := flag.String("listen", envOr("SNOWCHIVE_LISTEN", "127.0.0.1:8080"),
+ "TCP address to listen on. Defaults to $SNOWCHIVE_LISTEN, then 127.0.0.1:8080.")
+ root := flag.String("root", envOr("SNOWCHIVE_ROOT", "."),
+ "Archive directory holding data/ (the document corpus, one directory per document) and project/ "+
+ "(the files about the archive as a whole: glossary.json, assemblies.json, changelog.json and related.json; only -rebuild-related writes there). "+
+ "Defaults to $SNOWCHIVE_ROOT, then the working directory.")
+ check := flag.Bool("check", false,
+ "Load the corpus, print the warnings and the tally, then exit - non-zero when anything was reported. ")
+ noAPI := flag.Bool("no-api", envBool("SNOWCHIVE_NO_API"),
+ "Disable the read-only JSON API: every /api/ path answers a JSON 403 and the UI stops linking to it. Defaults on when $SNOWCHIVE_NO_API is true (1, t, true).")
+ flag.StringVar(&accelPrefix, "accel-files", os.Getenv("SNOWCHIVE_ACCEL_FILES"),
+ "Internal nginx location that aliases the data directory (e.g. /_files/). Document files are then answered with an X-Accel-Redirect into it and sent by nginx. "+
+ "Only behind nginx. Defaults to $SNOWCHIVE_ACCEL_FILES.")
+ accelTTL := flag.Duration("accel-expires", envDuration("SNOWCHIVE_ACCEL_EXPIRES"),
+ "How long the proxy may cache a page served to a reader without cookies (e.g. 10m), sent as X-Accel-Expires; browsers keep the short max-age. "+
+ "Clear the proxy cache after a restart with new data. Defaults to $SNOWCHIVE_ACCEL_EXPIRES.")
+ flag.Parse()
+ setRoot(*root)
+ if accelPrefix != "" {
+ if !strings.HasPrefix(accelPrefix, "/") || !strings.HasSuffix(accelPrefix, "/") {
+ log.Fatalf("-accel-files: %q must begin and end with a slash", accelPrefix)
+ }
+ }
+ if *accelTTL >= time.Second {
+ accelExpires = strconv.Itoa(int(*accelTTL / time.Second))
+ }
+ if *baseFlag != "" {
+ b, err := parseCanonicalBase(*baseFlag)
+ if err != nil {
+ log.Fatalf("-base: %v", err)
+ }
+ canonicalBase = b
+ }
+ apiEnabled = !*noAPI
+ initAssets()
+ // Before the load: it pre-renders the big page bodies from these.
+ if !devMode {
+ if err := parseTemplates(); err != nil {
+ log.Fatalf("Fatal: Could not parse templates at startup: %v", err)
+ }
+ }
+
+ loadData()
+ co := corpus()
+
+ if *check {
+ if loadErrors > 0 || loadWarnings > 0 {
+ log.Printf("check failed: %d errors, %d warnings", loadErrors, loadWarnings)
+ os.Exit(1)
+ }
+ log.Printf("check passed: %d documents, nothing to report", len(co.archive.Docs))
+ return
+ }
+
+ if *rebuildRelated {
+ log.Println("Rebuilding related docs index...")
+
+ existing := make(map[string][]string)
+ if b, err := os.ReadFile(relatedDocsPath()); err == nil {
+ if err := json.Unmarshal(b, &existing); err != nil {
+ warnf("could not parse existing related.json: %v - starting fresh", err)
+ existing = make(map[string][]string)
+ } else {
+ log.Printf("Loaded existing related.json with %d entries", len(existing))
+ }
+ }
+
+ stale := 0
+ for id := range existing {
+ if co.unknownDoc(id) {
+ delete(existing, id)
+ stale++
+ }
+ }
+ var missing []Document
+ for _, d := range co.archive.Docs {
+ if ids, ok := existing[d.ID]; !ok || slices.ContainsFunc(ids, co.unknownDoc) {
+ missing = append(missing, d)
+ }
+ }
+
+ if len(missing) == 0 && stale == 0 {
+ log.Println("No new documents - related.json is up to date.")
+ return
+ }
+
+ log.Printf("Computing related docs for %d documents...", len(missing))
+ maps.Copy(existing, co.buildRelatedDocs(missing, co.archive.Docs))
+
+ if err := saveRelatedDocs(existing); err != nil {
+ log.Fatalf("Failed to save related docs: %v", err)
+ }
+ log.Printf("Saved related.json with %d entries (%d computed, %d stale dropped, other entries and manual edits preserved)",
+ len(existing), len(missing), stale)
+ return
+ }
+
+ handler := withBaseHeaders(newMux())
+
+ srv := &http.Server{
+ Addr: *listen,
+ Handler: handler,
+ ReadHeaderTimeout: 5 * time.Second,
+ ReadTimeout: 15 * time.Second,
+ WriteTimeout: writeTimeout, // downloads set their own: fileWriteBudget
+ IdleTimeout: 60 * time.Second,
+ MaxHeaderBytes: 64 << 10,
+ }
+
+ warnNonLoopback(*listen)
+ log.Printf("Archive Server listening on %s (data: %s, project: %s)", srv.Addr, dataDir, projectDir)
+
+ errCh := make(chan error, 1)
+ go func() { errCh <- srv.ListenAndServe() }()
+ sig := make(chan os.Signal, 1)
+ signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
+ select {
+ case err := <-errCh:
+ log.Fatal(err)
+ case s := <-sig:
+ signal.Reset(os.Interrupt, syscall.SIGTERM)
+ log.Printf("%s received - draining connections (again to stop now)", s)
+ ctx, cancel := context.WithTimeout(context.Background(), writeTimeout)
+ defer cancel()
+ if err := srv.Shutdown(ctx); err != nil {
+ log.Printf("Shutdown: %v", err)
+ }
+ }
+}
+
+func (co *Corpus) prerenderPages() {
+ co.prerendered = map[string]template.HTML{}
+ co.docRowHTML = nil
+ if devMode {
+ return
+ }
+ exec := func(key, page, block string, data interface{}) {
+ t := tmplSet[page]
+ if t == nil {
+ return
+ }
+ var buf bytes.Buffer
+ if err := t.ExecuteTemplate(&buf, block, data); err != nil {
+ warnf("prerender %s/%s: %v - rendering it per request", page, block, err)
+ return
+ }
+ co.prerendered[key] = template.HTML(buf.String())
+ }
+
+ exec("ids", "ids.html", "id-rows", &idsPage{Rows: co.idRows})
+ exec("archive-timeline", "browse.html", "archive-timeline",
+ &browsePage{ArchiveYears: co.archiveTimeline})
+ exec("glossary", "glossary.html", "glossary-groups",
+ &glossaryPage{Groups: co.glossaryGroups})
+ exec("articles", "articles.html", "article-groups",
+ &articlesPage{Groups: co.articleGroups})
+ for key, v := range co.codewordViews {
+ exec("codewords:"+key, "codewords.html", "codeword-rows",
+ &codewordsPage{Rows: v.Rows})
+ }
+ for set, tables := range co.exploreTables {
+ for i, t := range tables {
+ exec(exploreKey(set, i), "explore_table.html", "explore-rows",
+ &exploreTablePage{Rows: t.Rows})
+ }
+ }
+ co.prerenderCategoryRows()
+ co.prerenderDocRows()
+ total := 0
+ for _, h := range co.prerendered {
+ total += len(h)
+ }
+ log.Printf("Pre-rendered %d page bodies (%d KB)", len(co.prerendered), total/1024)
+}
+
+func (co *Corpus) preHTML(key string) template.HTML { return co.prerendered[key] }
+
+func newMux() *http.ServeMux {
+ mux := http.NewServeMux()
+ mux.HandleFunc("GET /{$}", bind((*Corpus).handleIndex))
+ mux.HandleFunc("GET /random", bind((*Corpus).handleRandom))
+ mux.HandleFunc("GET /glossary", bind((*Corpus).handleGlossary))
+ mux.HandleFunc("GET /preferences", handlePreferences)
+ mux.HandleFunc("POST /preferences/save", handlePreferencesSave)
+ mux.HandleFunc("GET /pinned", bind((*Corpus).handlePinnedPage))
+ mux.HandleFunc("GET /pinned/export", bind((*Corpus).handlePinnedExport))
+ mux.HandleFunc("POST /pin/{id}", bind((*Corpus).handlePinToggle))
+ mux.HandleFunc("GET /articles", bind((*Corpus).handleArticles))
+ mux.HandleFunc("GET /article/{key}", bind((*Corpus).handleArticleView))
+ mux.HandleFunc("GET /changelog", bind((*Corpus).handleChangelog))
+ mux.HandleFunc("POST /history/clear", handleHistoryClear)
+ mux.HandleFunc("POST /pinsearch", bind((*Corpus).handleSearchPinToggle))
+ mux.HandleFunc("GET /robots.txt", handleRobots)
+ mux.HandleFunc("GET /sitemap.xml", bind((*Corpus).handleSitemap))
+ mux.HandleFunc("/", notFound)
+ mux.HandleFunc("GET /browse", bind((*Corpus).handleBrowseRoot))
+ mux.HandleFunc("GET /browse/{category}", bind((*Corpus).handleCategoryList))
+ mux.HandleFunc("GET /explore", bind((*Corpus).handleExploreIndex))
+ mux.HandleFunc("GET /explore/{metric}", bind((*Corpus).handleExploreMetric))
+ mux.HandleFunc("GET /explore/terms", bind((*Corpus).handleCodewords))
+ mux.HandleFunc("GET /explore/ids", bind((*Corpus).handleExploreIDs))
+ mux.HandleFunc("GET /explore/ids.txt", bind((*Corpus).handleExploreIDsText))
+ mux.HandleFunc("GET /explore/classification", bind((*Corpus).handleClassPatterns))
+ mux.HandleFunc("GET /explore/portions", bind((*Corpus).handlePortions))
+ mux.HandleFunc("GET /explore/outlets", bind((*Corpus).handleOutlets))
+ mux.HandleFunc("GET /explore/assemblies", bind((*Corpus).handleAssemblies))
+ mux.HandleFunc("GET /explore/assemblies/{id}", bind((*Corpus).handleAssemblyView))
+ mux.HandleFunc("GET /browse/{category}/{value}", bind((*Corpus).handleCategoryFilter))
+ mux.HandleFunc("GET /doc/{id}", bind((*Corpus).handleDocView))
+ mux.HandleFunc("GET /doc/{id}/pages", bind((*Corpus).handleDocPages))
+ mux.HandleFunc("GET /doc/{id}/versions/{slug}", bind((*Corpus).handleVersionView))
+ mux.HandleFunc("GET /doc/{id}/versions/{slug}/pages", bind((*Corpus).handleVersionPages))
+ mux.HandleFunc("GET /search", bind((*Corpus).handleSearch))
+ mux.HandleFunc("GET /about", handleAbout)
+ mux.HandleFunc("GET /files/", bind((*Corpus).handleFiles))
+ mux.HandleFunc("GET /static/", handleStatic)
+
+ // JSON API (read-only) - see api.go for endpoint documentation.
+ if apiEnabled {
+ mux.HandleFunc("GET /api/docs", bind((*Corpus).handleAPIDocs))
+ mux.HandleFunc("GET /api/doc/{id}", bind((*Corpus).handleAPIDoc))
+ mux.HandleFunc("GET /api/search", bind((*Corpus).handleAPISearch))
+ mux.HandleFunc("GET /api/facets", bind((*Corpus).handleAPIFacets))
+ mux.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) {
+ apiError(w, http.StatusNotFound, "unknown API endpoint")
+ })
+ } else {
+ mux.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) {
+ apiError(w, http.StatusForbidden, "the JSON API is disabled on this instance")
+ })
+ }
+
+ return mux
+}
+
+// withBaseHeaders wraps a handler with the headers every response carries.
+func withBaseHeaders(next http.Handler) http.Handler {
+ const csp = "default-src 'self'; img-src 'self'; script-src 'self'; " +
+ "style-src 'self' 'unsafe-inline'; object-src 'none'; base-uri 'none'; " +
+ "form-action 'self'; frame-ancestors 'self'"
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ h := w.Header()
+ h.Set("Content-Security-Policy", csp)
+ h.Set("X-Frame-Options", "SAMEORIGIN")
+ h.Set("X-Content-Type-Options", "nosniff")
+ h.Set("Referrer-Policy", "no-referrer")
+ switch r.Method {
+ case http.MethodGet, http.MethodHead, http.MethodPost:
+ default:
+ h.Set("Allow", "GET, HEAD, POST")
+ http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+ next.ServeHTTP(w, r)
+ })
+}
+
+func envOr(key, fallback string) string {
+ if v := os.Getenv(key); v != "" {
+ return v
+ }
+ return fallback
+}
+
+func envBool(key string) bool {
+ v := os.Getenv(key)
+ if v == "" {
+ return false
+ }
+ b, err := strconv.ParseBool(v)
+ if err != nil {
+ log.Fatalf("$%s=%q is not a boolean (use 1/0 or true/false)", key, v)
+ }
+ return b
+}
+
+func envDuration(key string) time.Duration {
+ v := os.Getenv(key)
+ if v == "" {
+ return 0
+ }
+ d, err := time.ParseDuration(v)
+ if err != nil {
+ log.Fatalf("$%s=%q is not a duration (use e.g. 90s, 10m)", key, v)
+ }
+ return d
+}
+
+// accelPath is the X-Accel-Redirect target for a file: its path relative to
+// the data directory under accelPrefix, percent-encoded because nginx decodes
+// the header as a URI.
+func accelPath(file string) (string, bool) {
+ rel, err := filepath.Rel(dataDir, file)
+ if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
+ return "", false
+ }
+ u := url.URL{Path: accelPrefix + filepath.ToSlash(rel)}
+ return u.EscapedPath(), true
+}
+
+// handleFiles serves the files the loader indexed for a document, and no
+// other file its directory may hold (Document.servedFile).
+func (co *Corpus) handleFiles(w http.ResponseWriter, r *http.Request) {
+ rest := strings.TrimPrefix(r.URL.Path, "/files/")
+ id, sub, ok := strings.Cut(rest, "/")
+ if !ok || sub == "" {
+ http.NotFound(w, r)
+ return
+ }
+ doc := co.docIndex[id]
+ if doc == nil {
+ http.NotFound(w, r)
+ return
+ }
+
+ sub = path.Clean("/" + sub)
+ rel := strings.TrimPrefix(sub, "/")
+ if strings.Contains(sub, "..") || !doc.servedFile(rel) {
+ http.NotFound(w, r)
+ return
+ }
+
+ cacheIfFile(w, os.DirFS(doc.Dir), rel, "public, max-age=86400")
+ if isTranscript(rel) {
+ w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+ }
+ file := filepath.Join(doc.Dir, filepath.FromSlash(sub))
+ if accelPrefix != "" {
+ if target, ok := accelPath(file); ok {
+ w.Header().Set("X-Accel-Redirect", target)
+ return
+ }
+ }
+ _ = http.NewResponseController(w).SetWriteDeadline(time.Now().Add(fileWriteBudget))
+ http.ServeFile(w, r, file)
+}
+
+const writeTimeout = 30 * time.Second
+
+// fileWriteBudget is how long a single download may take in total. A 123 MB
+// PDF over a bad mobile link (~35 KB/s) fits in an hour, so in practice this
+// bound is only reached by a reader that has gone away.
+var fileWriteBudget = time.Hour
+
+func cacheIfFile(w http.ResponseWriter, fsys fs.FS, name, policy string) {
+ if fi, err := fs.Stat(fsys, name); err == nil && fi.Mode().IsRegular() {
+ w.Header().Set("Cache-Control", policy)
+ }
+}
+
+func handleStatic(w http.ResponseWriter, r *http.Request) {
+ if path.Ext(r.URL.Path) == "" {
+ http.NotFound(w, r)
+ return
+ }
+ rel := strings.TrimPrefix(path.Clean("/"+strings.TrimPrefix(r.URL.Path, "/static/")), "/")
+ if h, ok := staticHashes[rel]; ok {
+ if r.URL.Query().Get("v") == h {
+ w.Header().Set("Cache-Control", "public, max-age=31536000, immutable")
+ }
+ w.Header().Set("ETag", `"`+h+`"`)
+ }
+ http.StripPrefix("/static/", http.FileServerFS(staticFS)).ServeHTTP(w, r)
+}
+
+func navSection(path string) string {
+ switch {
+ case path == "/":
+ return "home"
+ case path == "/browse" || strings.HasPrefix(path, "/browse/"):
+ return "browse"
+ case path == "/search":
+ return "search"
+ case path == "/articles" || strings.HasPrefix(path, "/article/"):
+ return "articles"
+ case path == "/glossary":
+ return "glossary"
+ case path == "/about":
+ return "about"
+ case path == "/pinned":
+ return "pinned"
+ case path == "/preferences":
+ return "preferences"
+ }
+ return ""
+}
+
+func render(w http.ResponseWriter, r *http.Request, data pageData) {
+ renderCode(w, r, data, http.StatusOK)
+}
+
+func crossSitePOST(r *http.Request) bool {
+ switch r.Header.Get("Sec-Fetch-Site") {
+ case "same-origin", "none":
+ return false
+ case "cross-site", "same-site":
+ return true
+ }
+
+ if o := r.Header.Get("Origin"); o != "" && o != "null" {
+ if u, err := url.Parse(o); err == nil && !originIsHost(u, r.Host) {
+ return true
+ }
+ }
+ return false
+}
+
+func originIsHost(origin *url.URL, host string) bool {
+ def := map[string]string{"http": "80", "https": "443"}[strings.ToLower(origin.Scheme)]
+ op := origin.Port()
+ if op == "" {
+ op = def
+ }
+ h, hp, err := net.SplitHostPort(host)
+ if err != nil { // no port
+ h, hp = strings.Trim(host, "[]"), def
+ }
+ return strings.EqualFold(origin.Hostname(), h) && op == hp
+}
+
+func noStore(w http.ResponseWriter) {
+ w.Header().Set("Cache-Control", "private, no-store")
+}
+
+func readPOSTForm(w http.ResponseWriter, r *http.Request) bool {
+ if crossSitePOST(r) {
+ http.Error(w, "cross-site request rejected", http.StatusForbidden)
+ return false
+ }
+ r.Body = http.MaxBytesReader(w, r.Body, 1<<16)
+ if err := r.ParseForm(); err != nil {
+ http.Error(w, "bad form", http.StatusBadRequest)
+ return false
+ }
+ return true
+}
+
+func crossSiteFetch(r *http.Request) bool {
+ switch r.Header.Get("Sec-Fetch-Site") {
+ case "cross-site", "same-site":
+ return true
+ }
+ return false
+}
+
+func renderCode(w http.ResponseWriter, r *http.Request, data pageData, code int) {
+ name := data.templateName()
+ if w.Header().Get("Cache-Control") == "" {
+ if (r.Method == http.MethodGet || r.Method == http.MethodHead) &&
+ (code == http.StatusOK || code == http.StatusNotFound) &&
+ r.Header.Get("Cookie") == "" {
+ w.Header().Set("Cache-Control", "public, max-age=10")
+ w.Header().Set("Vary", "Cookie")
+ if accelExpires != "" {
+ w.Header().Set("X-Accel-Expires", accelExpires)
+ }
+ } else {
+ w.Header().Set("Cache-Control", "private, no-cache")
+ }
+ }
+
+ b := data.base()
+ b.Nav = navSection(r.URL.Path)
+ b.APIEnabled = apiEnabled
+ // A handler that read the reader's preferences passes them on
+ // (pageWith); they are only read here for one that had no use for them.
+ if !b.hasPrefs {
+ b.Prefs = readPrefs(r)
+ }
+ b.Theme = b.Prefs.ThemeAttr()
+ b.CardPagePreview = b.Prefs.CardPagePreview
+
+ var t *template.Template
+ var err error
+
+ if devMode {
+ t, err = parsePage(name)
+ if err != nil {
+ log.Printf("Template Parse Error (%s): %v", name, err)
+ serverError(w, err)
+ return
+ }
+ } else {
+ t = tmplSet[name]
+ if t == nil {
+ log.Printf("Template not found: %s", name)
+ serverError(w, fmt.Errorf("template not found: %s", name))
+ return
+ }
+ }
+
+ var buf bytes.Buffer
+ if err := t.ExecuteTemplate(&buf, name, data); err != nil {
+ log.Printf("Template Exec Error (%s): %v", name, err)
+ serverError(w, err)
+ return
+ }
+ if code != http.StatusOK {
+ w.WriteHeader(code)
+ }
+ w.Write(bytes.TrimSpace(buf.Bytes()))
+ w.Write([]byte("\n"))
+}
+
+func notFound(w http.ResponseWriter, r *http.Request) {
+ renderCode(w, r, ¬FoundPage{Path: r.URL.Path}, http.StatusNotFound)
+}
+
+func parsePage(name string) (*template.Template, error) {
+ return template.New(name).Funcs(funcMap).ParseFS(templateFS, "base.html", "partials.html", name)
+}
+
+func parseTemplates() error {
+ pages, err := fs.Glob(templateFS, "*.html")
+ if err != nil {
+ return err
+ }
+ tmplSet = make(map[string]*template.Template)
+ for _, name := range pages {
+ if name == "base.html" || name == "partials.html" {
+ continue
+ }
+ t, err := parsePage(name)
+ if err != nil {
+ return fmt.Errorf("%s: %w", name, err)
+ }
+ tmplSet[name] = t
+ }
+ return nil
+}
+
+func serverError(w http.ResponseWriter, err error) {
+ w.Header().Set("Cache-Control", "no-store")
+ if devMode {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+}
+
+func warnNonLoopback(addr string) {
+ host, _, err := net.SplitHostPort(addr)
+ if err != nil {
+ return
+ }
+ if host == "localhost" {
+ return
+ }
+ if ip := net.ParseIP(host); ip != nil && ip.IsLoopback() {
+ return
+ }
+ warnf("-listen %s is not loopback - anything that can route to this host reaches the archive directly, bypassing your proxy's caching, rate limits and logs.", addr)
+}
diff --git a/mdparser.go b/mdparser.go
new file mode 100644
index 0000000..a7b82c4
--- /dev/null
+++ b/mdparser.go
@@ -0,0 +1,591 @@
+// mdparser.go - minimal dependency-free Markdown parser.
+//
+// Input trust: everything rendered through this parser is the archive's own
+// curator-produced OCR never user-supplied input.
+
+package main
+
+import (
+ "fmt"
+ "html"
+ "regexp"
+ "strconv"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+)
+
+// -- Inline patterns ----------------------------------------------------
+
+var (
+ reInlineCode = regexp.MustCompile("`(.+?)`")
+ reStrikethrough = regexp.MustCompile(`~~(.+?)~~`)
+ reHardBreak = regexp.MustCompile(`(?i)<br\s*/?>`)
+ reEscape = regexp.MustCompile("\\\\([!-/:-@\\[-`|{}~])")
+ reOrderedItem = regexp.MustCompile(`^(\s*)(\d{1,9})\.(?:\s+(.*))?\s*$`)
+ reUnorderedItem = regexp.MustCompile(`^(\s*)[-*+](?:\s+(.*))?\s*$`)
+ reTableSep = regexp.MustCompile(`^\|[-| :]+\|$`)
+)
+
+// parseInline converts inline Markdown to HTML.
+func parseInline(s string) string {
+ s = strings.ReplaceAll(s, "\x00", "")
+ s = strings.ReplaceAll(s, "\x01", "")
+ s = html.EscapeString(s)
+
+ // Backslash escapes (CommonMark): "\*" is a literal "*", not emphasis
+ // syntax. Pull escaped ASCII punctuation out to placeholders BEFORE the
+ // emphasis passes so the character can't be interpreted as a marker,
+ // then restore it at the end.
+ var escaped []string
+ if strings.IndexByte(s, '\\') >= 0 {
+ s = reEscape.ReplaceAllStringFunc(s, func(m string) string {
+ // m is "\X": EscapeString leaves the backslash alone. Keep
+ // the char after it.
+ ch := m[len(m)-1:]
+ escaped = append(escaped, ch)
+ return "\x01" + strconv.Itoa(len(escaped)-1) + "\x01"
+ })
+ }
+
+ // Extract inline code spans next and replace them with placeholders so
+ // the strikethrough/bold/italic passes can't reformat code content
+ // (e.g. `*x*` must render as literal *x*, not x). Escape
+ // placeholders that landed INSIDE a code span are put back as their
+ // original "\X" source text first: backslash escapes are not escapes
+ // within code spans (CommonMark), so `\*` renders as \*, not *. The
+ // escape pass still runs before this one on purpose - that ordering is
+ // what lets \` be a literal backtick that cannot OPEN a code span.
+ var codes []string
+ if strings.IndexByte(s, '`') >= 0 {
+ s = reInlineCode.ReplaceAllStringFunc(s, func(m string) string {
+ inner := reInlineCode.FindStringSubmatch(m)[1]
+ for i, ch := range escaped {
+ inner = strings.ReplaceAll(inner, "\x01"+strconv.Itoa(i)+"\x01", "\\"+ch)
+ }
+ codes = append(codes, ""+inner+"")
+ return "\x00" + strconv.Itoa(len(codes)-1) + "\x00"
+ })
+ }
+
+ // After the code-span pass, so ` ` inside backticks stays literal.
+ if strings.Contains(s, "<") {
+ s = reHardBreak.ReplaceAllString(s, " ")
+ }
+ if strings.Contains(s, "~~") {
+ s = reStrikethrough.ReplaceAllString(s, `$1`)
+ }
+ // ***x*** first, so it nests as rather than crossing.
+ s = emphasize(s, '*', 3, "", "")
+ s = emphasize(s, '*', 2, "", "")
+ s = emphasize(s, '_', 2, "", "")
+ s = emphasize(s, '*', 1, "", "")
+ s = emphasize(s, '_', 1, "", "")
+
+ // Restore code spans.
+ for i, code := range codes {
+ s = strings.Replace(s, "\x00"+strconv.Itoa(i)+"\x00", code, 1)
+ }
+ // Restore backslash-escaped characters as their literal selves.
+ // (Placeholders that sat inside a code span were already restored to
+ // their "\X" source form up in the code-extraction pass and no longer
+ // occur here; Replace on an absent placeholder is a no-op.)
+ for i, ch := range escaped {
+ s = strings.Replace(s, "\x01"+strconv.Itoa(i)+"\x01", ch, 1)
+ }
+ return s
+}
+
+// emphasize wraps the text between two runs of k delimiters c ('*' or '_')
+// in open/close. It pairs them as CommonMark does: left to right, a run
+// that can close gives its first k delimiters to the nearest unpaired
+// opener before it, and a run that can open offers its last k to a later
+// closer. Which runs can open or close follows CommonMark's flanking rules
+// (delimiterRun).
+func emphasize(s string, c byte, k int, open, close string) string {
+ var openers []int // offsets of unpaired openers, nearest last
+ tags := map[int]string{} // offset of k paired delimiters -> their tag
+ for i := 0; i < len(s); {
+ if s[i] != c {
+ i++
+ continue
+ }
+ end := i
+ for end < len(s) && s[end] == c {
+ end++
+ }
+ canOpen, canClose := delimiterRun(s, i, end, c)
+ n := end - i
+ if n >= k && canClose && len(openers) > 0 {
+ tags[openers[len(openers)-1]], tags[i] = open, close
+ openers = openers[:len(openers)-1]
+ n -= k
+ }
+ if n >= k && canOpen {
+ openers = append(openers, end-k)
+ }
+ i = end
+ }
+ if len(tags) == 0 {
+ return s
+ }
+ var b strings.Builder
+ for i := 0; i < len(s); i++ {
+ if t, ok := tags[i]; ok {
+ b.WriteString(t)
+ i += k - 1
+ } else {
+ b.WriteByte(s[i])
+ }
+ }
+ return b.String()
+}
+
+// delimiterRun reports whether the run of delimiters c at s[start:end]
+// can open and can close emphasis, by CommonMark's flanking rules, which
+// look at the whole run.
+func delimiterRun(s string, start, end int, c byte) (canOpen, canClose bool) {
+ prev, next := ' ', ' '
+ if start > 0 {
+ prev, _ = utf8.DecodeLastRuneInString(s[:start])
+ }
+ if end < len(s) {
+ next, _ = utf8.DecodeRuneInString(s[end:])
+ }
+ left := !unicode.IsSpace(next) &&
+ (!isPunct(next) || unicode.IsSpace(prev) || isPunct(prev))
+ right := !unicode.IsSpace(prev) &&
+ (!isPunct(prev) || unicode.IsSpace(next) || isPunct(next))
+ if c == '_' {
+ return left && (!right || isPunct(prev)), right && (!left || isPunct(next))
+ }
+ return left, right
+}
+
+// isPunct is CommonMark's punctuation (Unicode P and S). The \x00/\x01
+// placeholders count too: they stand for a code span (opened by a
+// backtick) or a backslash-escaped punctuation character.
+func isPunct(r rune) bool {
+ return r == 0 || r == 1 || unicode.IsPunct(r) || unicode.IsSymbol(r)
+}
+
+// -- Block-level parser --------------------------------------------
+
+type parser struct {
+ lines []string
+ pos int
+ out strings.Builder
+ open []listLevel // the lists parseList is inside, outermost first
+ // numbered holds the last number of the ordered list at each indent,
+ // for as long as list lines keep coming (see endsItem).
+ numbered map[int]int
+ // top is the source's highest heading level and last the level of the
+ // heading emitted last (see parseHeading).
+ top, last int
+}
+
+// listLevel is one open list: the indent of its items and their kind.
+type listLevel struct {
+ indent int
+ ordered bool
+}
+
+func newParser(src string) *parser {
+ src = strings.ReplaceAll(src, "\r\n", "\n")
+ src = strings.ReplaceAll(src, "\r", "\n")
+ p := &parser{lines: strings.Split(src, "\n"), top: 7, last: 1}
+ fence := ""
+ for _, line := range p.lines {
+ switch {
+ case fence != "":
+ if strings.HasPrefix(line, fence) {
+ fence = ""
+ }
+ case isFencedCodeStart(line):
+ fence = line[:3]
+ case isHeading(line):
+ p.top = min(p.top, headingLevel(line))
+ }
+ }
+ return p
+}
+
+func (p *parser) peek() (string, bool) {
+ if p.pos >= len(p.lines) {
+ return "", false
+ }
+ return p.lines[p.pos], true
+}
+
+func (p *parser) next() (string, bool) {
+ l, ok := p.peek()
+ if ok {
+ p.pos++
+ }
+ return l, ok
+}
+
+func (p *parser) emit(s string) { p.out.WriteString(s) }
+
+// parse processes all lines and returns HTML.
+func (p *parser) parse() string {
+ for p.pos < len(p.lines) {
+ line, _ := p.peek()
+ if !isListItem(line) {
+ p.numbered = nil // the run of list lines is over
+ }
+
+ switch {
+ case isFencedCodeStart(line):
+ p.parseFencedCode()
+ case isHR(line):
+ p.next()
+ p.emit("\n")
+ case isHeading(line):
+ p.parseHeading()
+ case isTableRow(line):
+ p.parseTable()
+ case isListItem(line):
+ p.parseList(reOrderedItem.MatchString(line), indentOf(line))
+ case strings.TrimSpace(line) == "":
+ p.next()
+ default:
+ p.parseParagraph()
+ }
+ }
+ return p.out.String()
+}
+
+// -- Heading --------------------------------------------------
+
+// headingLevel returns the heading level of an ATX heading line - one to
+// six '#' followed by whitespace or the end of the line - or 0 when the
+// line is not a heading.
+func headingLevel(s string) int {
+ n := 0
+ for n < len(s) && s[n] == '#' {
+ n++
+ }
+ if n == 0 || n > 6 || (n < len(s) && s[n] != ' ' && s[n] != '\t') {
+ return 0
+ }
+ return n
+}
+
+func isHeading(s string) bool { return headingLevel(s) > 0 }
+
+// parseHeading emits a heading under the page's own
: the source's top
+// level becomes
and no heading skips a level. The md-hN class keeps the
+// size of the level the source wrote.
+func (p *parser) parseHeading() {
+ line, _ := p.next()
+ level := headingLevel(line)
+ content := strings.TrimSpace(line[level:])
+ tag := min(level-p.top+2, p.last+1, 6)
+ p.last = tag
+ p.emit(fmt.Sprintf("%s\n", tag, level, parseInline(content), tag))
+}
+
+// -- Horizontal rule -------------------------------------------------
+
+// isHR reports whether s is a thematic break: three or more of the same
+// '-', '*' or '_', optionally with spaces or tabs between them.
+func isHR(s string) bool {
+ t := strings.TrimSpace(s)
+ if t == "" || (t[0] != '-' && t[0] != '*' && t[0] != '_') {
+ return false
+ }
+ n := 0
+ for i := 0; i < len(t); i++ {
+ switch t[i] {
+ case t[0]:
+ n++
+ case ' ', '\t':
+ default:
+ return false
+ }
+ }
+ return n >= 3
+}
+
+// Fenced code block ---------------------------------------------------------
+
+func isFencedCodeStart(s string) bool {
+ return strings.HasPrefix(s, "```") || strings.HasPrefix(s, "~~~")
+}
+
+func (p *parser) parseFencedCode() {
+ opener, _ := p.next()
+ fence := opener[:3]
+
+ var body strings.Builder
+ for {
+ line, ok := p.next()
+ if !ok || strings.HasPrefix(line, fence) {
+ break
+ }
+ body.WriteString(html.EscapeString(line))
+ body.WriteByte('\n')
+ }
+ p.emit("
" + body.String() + "
\n")
+}
+
+// -- Lists (unified, handles nesting) -----------------------------------------
+
+// indentOf returns the number of leading whitespace characters in s.
+func indentOf(s string) int {
+ return len(s) - len(strings.TrimLeft(s, " \t"))
+}
+
+// isListItem returns true if the line is a ul or ol item at any indent.
+// A thematic break wins over a bullet ("* * *" is a rule, not an item).
+func isListItem(s string) bool {
+ // A list marker is the first non-space byte: rule out the rest (most
+ // lines) before any regex runs.
+ t := strings.TrimLeft(s, "\t\n\f\r ")
+ if t == "" || (t[0] != '-' && t[0] != '*' && t[0] != '+' && (t[0] < '0' || t[0] > '9')) {
+ return false
+ }
+ return !isHR(s) && (reUnorderedItem.MatchString(s) || reOrderedItem.MatchString(s))
+}
+
+// itemContent strips the list marker and returns the item text (possibly
+// empty for a bare marker like "6.").
+func itemContent(s string) string {
+ if sub := reUnorderedItem.FindStringSubmatch(s); sub != nil {
+ return sub[2]
+ }
+ if sub := reOrderedItem.FindStringSubmatch(s); sub != nil {
+ return sub[3]
+ }
+ return strings.TrimSpace(s)
+}
+
+// orderedNum returns the source number of an ordered-list line ("7." -> 7)
+// and true, or false if the line isn't an ordered item. "0." is a real
+// number, so it cannot double as the "not ordered" answer.
+func orderedNum(s string) (int, bool) {
+ if sub := reOrderedItem.FindStringSubmatch(s); sub != nil {
+ n, _ := strconv.Atoi(sub[2]) // at most nine digits: cannot fail
+ return n, true
+ }
+ return 0, false
+}
+
+// startsList reports whether list-item line s may open a new list on the
+// line after text, where it would otherwise continue that text.
+func startsList(s string) bool {
+ if itemContent(s) == "" {
+ return false
+ }
+ n, ok := orderedNum(s)
+ return !ok || n == 1
+}
+
+// endsItem reports whether line s, met while collecting the lazily
+// continued text of a list item, ends that item rather than continuing it.
+func (p *parser) endsItem(s string) bool {
+ if isHR(s) || isHeading(s) || isFencedCodeStart(s) || isTableRow(s) {
+ return true
+ }
+ if !isListItem(s) {
+ return false
+ }
+ if startsList(s) {
+ return true
+ }
+ indent, ordered := indentOf(s), reOrderedItem.MatchString(s)
+ for _, l := range p.open {
+ if l.indent == indent && l.ordered == ordered {
+ return true
+ }
+ }
+ if n, ok := orderedNum(s); ok {
+ for _, last := range p.numbered {
+ if n > last && n <= last+2 {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+// parseList recursively parses a ul or ol at the given indent level.
+func (p *parser) parseList(ordered bool, minIndent int) {
+ tag := "ul"
+ if ordered {
+ tag = "ol"
+ }
+ p.emit("<" + tag + ">\n")
+ p.open = append(p.open, listLevel{minIndent, ordered})
+ defer func() { p.open = p.open[:len(p.open)-1] }()
+
+ expected := 0 // running ordered-list counter, for value="N" decisions
+
+ for {
+ line, ok := p.peek()
+ if !ok || strings.TrimSpace(line) == "" || !isListItem(line) {
+ break
+ }
+ indent := indentOf(line)
+ if indent < minIndent {
+ // Belongs to a parent list - stop.
+ break
+ }
+ if indent == minIndent && reOrderedItem.MatchString(line) != ordered {
+ // Bullets after numbers, or numbers after bullets: a new
+ // list (as in CommonMark). Kept in this one, a number would
+ // lose itself to a bullet and a bullet would gain a number.
+ break
+ }
+ if indent > minIndent {
+ // Deeper indent mid-list without a parent
- treat as nested.
+ p.parseList(reOrderedItem.MatchString(line), indent)
+ continue
+ }
+
+ p.next()
+
+ // Item body: the marker line plus any lazily-continued lines after it.
+ var body []string
+ if c := itemContent(line); c != "" {
+ body = append(body, c)
+ }
+ for {
+ cont, ok := p.peek()
+ if !ok || strings.TrimSpace(cont) == "" || p.endsItem(cont) {
+ break
+ }
+ p.next()
+ body = append(body, strings.TrimSpace(cont))
+ }
+
+ liOpen := "
"
+ if ordered {
+ n, _ := orderedNum(line)
+ expected++
+ if n != expected {
+ liOpen = "
"
+ expected = n
+ }
+ if p.numbered == nil {
+ p.numbered = map[int]int{}
+ }
+ p.numbered[minIndent] = n
+ }
+
+ var parts []string
+ for _, l := range body {
+ parts = append(parts, parseInline(l))
+ }
+ text := strings.Join(parts, " \n")
+
+ // Peek: if next line is deeper, nest a list inside this
-
+ // as many as follow, since bullets turning into numbers down
+ // there start a second nested list, which belongs here too.
+ next, hasNext := p.peek()
+ if hasNext && isListItem(next) && indentOf(next) > indent {
+ p.emit(liOpen + text + "\n")
+ for hasNext && isListItem(next) && indentOf(next) > indent {
+ p.parseList(reOrderedItem.MatchString(next), indentOf(next))
+ next, hasNext = p.peek()
+ }
+ p.emit("
+{{ end }}
+{{ template "base.html" . }}
diff --git a/templates/about.html b/templates/about.html
new file mode 100644
index 0000000..4e6b8c4
--- /dev/null
+++ b/templates/about.html
@@ -0,0 +1,544 @@
+{{ define "title" }}About - Snowden Archive{{ end }}
+{{ define "content" }}
+
+
About this archive
+
+
This is a searchable archive of all published documents from the 2013
+ Snowden disclosures. The archive includes documents published alongside
+ news reporting, The Intercept's bulk SIDtoday release, and FOIA material
+ (official releases from FOIA litigation and FISC and ODNI
+ declassifications).
+
Every document carries its original PDF, page images, extracted text,
+ and curated metadata - agencies, programs, countries, classification
+ markings, people, and the journalists and articles that published it.
+
Use Browse to explore by category,
+ Search for fielded boolean queries across text and
+ metadata, Articles to follow the disclosures story
+ by story, the Glossary to decode codenames and
+ acronyms, and Explore to dig deeper into ranked
+ views and lists.
+
+
+
+
+
Privacy
+
By default, the server keeps no state about its visitors. Pins and
+ history live only in your own browser's cookies, and history features
+ are off unless you opt in via Preferences.
+ Anonymous page views set no cookies at all. All core features on this
+ website work without JavaScript.
+
Note that you can't verify that this server doesn't log and analyze
+ your visits. Use Tor to hide your real IP address, or
+ set up your own instance on your machine to
+ browse fully offline.
+
+
+
+
What's in this archive?
+
By default, two collections are shown as tabs on the
+ Browse page: Reported documents
+ (leaked files as the outlets that broke the stories presented them) and
+ the SIDtoday Archive (The Intercept's bulk release of
+ the NSA's internal SIDtoday newsletters). The third collection,
+ FOIA (official releases from FOIA litigation and FISC
+ and ODNI declassifications), can be included via
+ Preferences.
+
The metadata and many of the extracted texts are curated by hand and
+ may contain errors or omissions, so treat them as a guide to the
+ documents rather than as established fact, and check the original
+ documents and reporting before relying on them.
+
+
+
+
How to search
+
The Search page supports fielded boolean
+ queries. Restrict a term to a field (title, full text, programs,
+ countries, journalists, and so on), combine clauses with AND/OR, quote
+ "text" to find it anywhere, match exact whole words with
+ =... (publisher:="The Guardian" matches the
+ whole value), and use * wildcards (tap*,
+ *apping).
+
+
Ways to write a value, in any row or after any operator:
+
+
cable tapping finds both words anywhere, in their other forms too (tapping also finds tapped).
+
"apping" finds that text anywhere, even inside a word, like grep -i (it finds, for example, mapping and tapping).
+
=tap or ="cable tapping" finds exactly those whole words, so =tap skips taps and strap. On a metadata field it matches one whole value, so agency:=NSA finds documents whose agency is NSA but not NSA, GCHQ.
+
The * wildcard finds part of a word: tap* finds words starting with tap, *land words ending in land, and *keys* words containing keys. A leading * needs at least three letters after it.
+
"upstream collection"~10 finds the words within 10 words of each other (in any order, common stopwords ignored).
+
A leading - excludes within its row: xkeyscore -tempora finds xkeyscore without tempora, and the row keeps that meaning under OR and NOT.
+
+
+
The simple search box (on the home page, on the 404 page, and via
+ the URL parameter ?q=) understands field operators:
+
+
program:XKEYSCORE country:Germany cable or journalist:="Poitras, Laura".
+
A leading - negates (-tag:=junk, -draft).
+
date:2013, date:2013..2015 and published:2014-06 filter by date.
Search results for term xkeyscore. It accepts the same parameters as the Search page, so you can also pass fqf/fqv/fqc clause parameters for fielded queries.
Important if you are a crawler, robot or AI trying to
+ download large amounts of data from this website: please use
+ the downloadable datasets from the
+ Downloading the archive section. The
+ information on this website is identical to the datasets, so you gain
+ nothing by crawling this website.
+ {{ end }}
+
+
+ {{ if .APIEnabled }}
Metadata structure
{{ else }}
Metadata structure
{{ end }}
+
Every document ships a curated metadata.json:
+
+
The document record
+
+
id
+
The document's stable identifier: the name of its directory
+ and the last part of its URL (/doc/{id}). It is not
+ written inside metadata.json - the directory name is
+ the id.
+
+
aka_id
+
Other names the same document was released under, most often
+ the file name another publisher or a mirror gave its PDF. A list
+ of entries, each described below.
+
+
aka_id[].id
+
The other name itself. Searchable, so a document can be found
+ by a file name it circulated under elsewhere.
+
+
aka_id[].doc_source
+
The web page where the document was released under that name,
+ linked as “page” next to the alias.
+
+
aka_id[].doc_url
+
The direct link to the PDF released under that name, linked
+ as “PDF” next to the alias.
+
+
title
+
The document's title, written by the curators - usually the
+ title the document gives itself or the one its publisher used.
+
+
description
+
A short summary of what the document is and says, shown under
+ the publication list and in search results and cards.
+ Descriptions are the work of archivists without inside knowledge -
+ corrections welcome.
+
+
long_description
+
A longer account of the contents where one has been written,
+ behind “Show more info” on the document page; often
+ empty. A description: search looks in both
+ descriptions.
+
+
tags
+
Free-text subjects added by the curators. Unlike the other
+ lists they are not necessarily taken from the document's own
+ vocabulary, so they are the loosest thing to browse or search by.
+
+
+
type
+
The kind of release: REPORTED (a leaked document
+ as an outlet published it), SIDTODAY (an article of
+ the NSA's internal SIDtoday newsletter from The Intercept's bulk
+ release) or FOIA (obtained through a public-records
+ request). When it is empty, a document filed in the SIDtoday or FOIA
+ folder takes that type and any other is REPORTED.
+
+
collection
+
Which of the three collections - the tabs on
+ Browse - the document belongs to:
+ news (Reported), sidtoday-archive or
+ foia. Documents filed under the SIDtoday and FOIA
+ folders belong to those collections whatever this says; elsewhere
+ an empty value means Reported, or FOIA for a document of type
+ FOIA. FOIA releases are left out of
+ browsing and search unless you include them in
+ Preferences.
+
+
pages
+
The page count of the PDF. When it is missing, the number of
+ page images is used instead.
+
+
pdf_sha256
+
The SHA-256 of the PDF as archived.
+
+
archive_comment
+
A note from the curators about this archive's copy - how it
+ was assembled, where it was found, what differs between releases,
+ or a known problem with it.
+
+
+
The document itself: document
+
+
document.creator
+
The agency that wrote the document, in its short form
+ (NSA, GCHQ, FISC,
+ DOJ, CSE ...) - not the agency it is
+ about and not the one that published it. Empty when the author is
+ not known.
+
+
document.date_created
+
When the document was written, as precise as the document
+ allows: 2013-03-20, 2013-03,
+ 2013, or empty when unknown. Nothing is invented to
+ fill the gaps, and nothing is guessed from the publication date.
+
+
+
document.banner
+
The marking line verbatim - what the
+ document says about itself, qualifiers and all. Intranet pages
+ often carry a ceiling banner (“DYNAMIC PAGE - HIGHEST
+ POSSIBLE CLASSIFICATION IS ...”) while their content is
+ portion-marked unclassified; the banner records the claim, not an
+ assessment. Empty means the artifact has no marking line - nothing
+ is reconstructed from portion marks.
+
+
document.classification
+
The curator's assessment of the content's
+ classification, as component values: the level
+ (TOP SECRET, SECRET,
+ CONFIDENTIAL, UNCLASSIFIED) and any
+ control systems (COMINT, SI,
+ TK, STRAP 1 ...), judged from portion
+ marks. If a document contains multiple classifications, highest is
+ used here.
+
+
document.dissemination
+
The curator's assessment of the handling controls, as
+ component values (FOUO, NOFORN,
+ ORCON, REL TO ...). Assessed the same
+ way as the classification.
+
+
document.genre
+
The document's form, from a fixed vocabulary:
+ slides, report, wiki,
+ manual, guide,
+ newsletter, other.
+
+
document.series.name
+
For a document that prints a series roster (currently only in
+ SIDtoday): the series name verbatim, as the document prints it.
+ Documents with the same name are listed together on the document
+ page and under Series.
+
+
document.series.part
+
This document's 1-based position in the printed roster. Order
+ comes from the artifact; parts of a series not yet in the archive
+ show as gaps in the numbering, never renumbered away.
+
+
document.programs
+
The codenames and programs the document names
+ (XKEYSCORE, PRISM, ...), as curated
+ values for browsing and search; the
+ Glossary explains many of them.
+
+
document.countries
+
The countries the document mentions, by their English names.
+ A mention, not a claim that the country was a target or a partner.
+
+
+
document.people
+
The people who appear in the document: named in its text, or
+ recognised in its images - for example
+ Mobile Networks
+ in My NOC World lists King Charles and Queen Camilla,
+ who appear only in a photograph on one of its slides. Usually
+ written “Surname, Given name”. A person appearing in
+ these documents implies nothing about them beyond that appearance.
+ The journalists who published the document are under
+ publication[].reporters, not here.
+
+
+
Where it was published: publication[]
+
One entry per story or book that published (or reported) the
+ document. Browse-by-publication-date counts a document under its
+ oldest dated entry; a publication-date search matches
+ any entry's date - or, combined with a journalist or
+ publisher, that entry's date.
+
+
publication[].publisher
+
Who published it: a news outlet, a book's publisher, or for a
+ FOIA release the organization that obtained and released it.
+ Browsable under Publishers - except
+ entries marked reporting_only.
+
+
publication[].title
+
The headline of the article, or the book's title. Shown on the
+ document page and names the story on
+ Articles.
+
+
publication[].link
+
The URL of the article. It is the story's identity: documents
+ whose entries share a link were published in the same report and
+ are listed together on its article page.
+
+
publication[].date_published
+
When this publication appeared, in the same
+ YYYY-MM-DD / YYYY-MM /
+ YYYY forms as the document date.
+
+
publication[].reporters
+
The journalists bylined on the story, written “Surname,
+ Given name”. Browsable under
+ Journalists and searchable with
+ journalist:.
+
+
publication[].contributors
+
People credited on the story without a byline - additional
+ reporting, research, technical analysis. Shown on the document
+ page and found by an all-fields search, but not listed among the
+ journalists.
+
+
publication[].note
+
Anything particular to this publication: that it showed only
+ some of the pages, published them in a different order, redacted
+ differently, or printed them in a book appendix.
+
+
publication[].version
+
When this publication released a different copy from the one
+ shown on the document page, the name of that copy in the
+ document's versions/ folder; its entry then links to
+ “view this copy”. Without it, the entry published the
+ copy shown on the page.
+
+
publication[].doc_source
+
The web page where this publication presented the document
+ (the outlet's own document page, a DocumentCloud page), linked as
+ “document page”. Where a copy came from is recorded
+ here, on the publication that released it, not on the
+ document.
+
+
publication[].doc_url
+
The direct link to the PDF file this publication released,
+ linked as “original PDF”.
+
+
publication[].reporting_only
+
true when the outlet reported the story without
+ publishing the document - a reporting partner. The entry keeps its
+ byline and its article, but the outlet is not counted as one of
+ the document's publishers and no copy is linked from it.
+
+
+
Other copies: versions
+
Other published copies of a document (e.g. less redacted, with extra
+ pages, or a different scan) are kept in its versions/
+ folder, one sub-folder per copy, each with its own PDF, text and page
+ images. The versions object adds details about them,
+ keyed by the copy's folder name.
+
+
versions.{name}.changed_pages
+
The pages of that copy, by its own page numbers, that differ
+ from the copy shown on the document page. They are linked as
+ “changed pages” next to the copy.
+
+
versions.{name}.pdf_sha256
+
The SHA-256 of that copy's PDF.
+
+
+
+
+
+
Downloading the archive
+
Each release comes in two packages:
+
+
full, ~18 GB: all the PDFs, page images (the
+ PDF pages as JPEG files), extracted texts, and metadata,
+
pdfs, ~3.5 GB: the same without the page
+ images.
+
+
Both extract to data/ and project/, ready
+ to run your own copy. Each release also
+ includes the source of the server version it was published with, and
+ the last six releases stay online. For only the extracted texts and
+ metadata (~58 MB), see below.
+git clone https://git.disroot.org/libroot/snowchive-meta
+cd snowchive-meta/data
+
+# grep only the extracted texts:
+grep "fiber optic" -ri --include=*.md
+
+# grep only the metadata:
+grep "fiber optic" -ri --include=*.json
+
+
+
+
+
Can I run my own copy or mirror this?
+
Yes. The server software behind this archive is free software with
+ no database and no external dependencies, designed to run offline or,
+ for example, as a Tor hidden service.
+
First, get a release package
+ and extract it: full for everything, or
+ pdfs for a site that works the same but shows no page
+ images.
+
Then get the server (Go required). Each release folder has the
+ source of the server version it was published with,
+ snowchive-server-<commit>.tar.gz, covered by the same
+ SHA256SUMS - the version known to work with that data:
System requirements: at least 300 MB of RAM (1 GB or more
+ recommended for production).
+
The maintainers of this project do not endorse any public Snowchive
+ instance other than the official snowchive.org instance (unless you
+ are a reputable organization that wants to host a public
+ instance).
We welcome anyone who wants to contribute to the project, for
+ example by fixing errors in the metadata or in the extracted texts.
+ Please see the Contact section above.
+
Running this project doesn't cost much, but if you wish, you
+ can send a donation to:
No documents found for this {{ if .Article.Book }}book{{ else }}article{{ end }}.
+{{ end }}
+{{ end }}
+{{ template "base.html" . }}
diff --git a/templates/articles.html b/templates/articles.html
new file mode 100644
index 0000000..b79929d
--- /dev/null
+++ b/templates/articles.html
@@ -0,0 +1,39 @@
+{{ define "title" }}Articles - Snowden Archive{{ end }}
+{{ define "content" }}
+
+
Articles
+ {{ .Total }} articles{{ with .Books }} and {{ . }} book{{ if ne . 1 }}s{{ end }}{{ end }}
+
+
+
+ The news stories, and the few books, these documents were published with, newest first.
+ A document covered by several stories appears under each.
+
+
+{{ if .Groups }}
+{{ if .GroupsHTML }}{{ .GroupsHTML }}{{ else }}{{ template "article-groups" . }}{{ end }}{{ else }}
+
No articles.
+{{ end }}
+{{ end }}
+{{ template "base.html" . }}
+
+{{ define "article-groups" }}{{ range .Groups }}
+
+
{{ .Year }} {{ len .Articles }}
+
+ {{ range .Articles }}
+
+ {{ if .Date }}{{ .Date }}{{ else }}-{{ end }}
+
+ {{ if .Title }}{{ .Title }}{{ else }}[untitled] {{ .Host }}{{ end }}
+
+ {{ if .Book }}Book · {{ end }}{{ with .Publisher }}{{ . }}{{ end }}{{ if and .Publisher .Journalists }} · {{ end }}{{ range $i, $j := .Journalists }}{{ if $i }}, {{ end }}{{ $j }}{{ end }}
+ · {{ len .DocIDs }} doc{{ if ne (len .DocIDs) 1 }}s{{ end }}
+
+
+
+ {{ end }}
+
+
+{{ end }}
+{{ end }}
diff --git a/templates/assemblies.html b/templates/assemblies.html
new file mode 100644
index 0000000..5374430
--- /dev/null
+++ b/templates/assemblies.html
@@ -0,0 +1,32 @@
+{{ define "title" }}Assemblies - Snowden Archive{{ end }}
+{{ define "content" }}
+
+
+ Sets of pages this archive put together from documents and copies published
+ separately, so they can be read in one place. An assembly is our own
+ arrangement, not a document that was leaked or published in this form.
+
{{ $a.Title }} {{ $n }} slide{{ if ne $n 1 }}s{{ end }}
+
Put together by this archive from {{ $a.Copies }} separately published cop{{ if eq $a.Copies 1 }}y{{ else }}ies{{ end }}. This is our arrangement, not a document in its own right.
+ {{ with $a.Note }}
{{ . }}
{{ end }}
+
+
+{{ if .TextOnly }}
+
Text-only mode is on - page images are not loaded. Preferences
+
+ {{ range $a.Slides }}
+
Slide {{ .Pos }}{{ template "assembly-credit" . }}
+ {{- with .Note }} - {{ . }}{{ end }}
+ {{- range .Alts }} Other copy{{ template "assembly-credit" . }}{{ end }}
+
+
+ ×
+ {{ if $i }}‹{{ end }}
+ {{ if lt $s.Pos $n }}›{{ end }}
+
+ {{ $s.Pos }} / {{ $n }} · {{ template "assembly-credit" $s }}{{ with $s.Note }} · {{ . }}{{ end }}{{ range $k, $alt := $s.Alts }} · Other copy{{ with $alt.Source }}: {{ . }}{{ end }}{{ end }}
+
+{{ range $k, $alt := $s.Alts }}
+
+
+ ×
+
+ Other copy of slide {{ $s.Pos }} · {{ template "assembly-credit" $alt }} · Back to slide {{ $s.Pos }}
+
+{{ end }}
+{{ end }}
+{{ end }}
+{{ end }}
+
+{{ template "base.html" . }}
diff --git a/templates/base.html b/templates/base.html
new file mode 100644
index 0000000..fa87396
--- /dev/null
+++ b/templates/base.html
@@ -0,0 +1,35 @@
+
+
+
+
+
+ {{ block "title" . }}Snowden Archive{{ end }}
+
+
+ {{ block "head" . }}{{ end }}
+
+
+ Skip to main content
+
+ {{ if .ShowCatBoxes }}{{ template "cat-boxes" . }}{{ end }}
+
+ {{ block "content" . }}{{ end }}
+
+
+
+
diff --git a/templates/browse.html b/templates/browse.html
new file mode 100644
index 0000000..0f4d09a
--- /dev/null
+++ b/templates/browse.html
@@ -0,0 +1,155 @@
+{{ define "title" }}{{ if eq .Collection "foia" }}FOIA{{ else if eq .Collection "sidtoday-archive" }}SIDtoday Archive{{ else }}Reported{{ end }} documents - Snowden Archive{{ end }}
+{{ define "content" }}
+
+
+
+
+
+{{ if eq .Collection "sidtoday-archive" }}
+
+
+
SIDtoday Archive ({{ .Total }} issues)
+
+ All NSA SIDtoday newsletters in chronological order. This section
+ includes The Intercept's bulk release plus issues that were individually
+ reported by media (marked reported; those
+ also appear in the Reported collection).
+
+ {{ with .SeriesList }}
+
+ Series {{ len . }}- recurring columns and multi-part runs
+
+ {{ if eq .Collection "foia" }}
+ Documents released by government agencies under Freedom of Information Act
+ requests, rather than through the Snowden disclosures - obtained by
+ journalists, researchers, and transparency organizations and added here
+ for context.
+ {{ else }}
+ Documents published alongside news reporting on the Snowden disclosures.
+ {{ end }}
+
Two views of the marking metadata. Every number counts the
+documents carrying that exact classification component. A document carrying
+several markings counts once per marking.
+
+
Mix by document year
+
+ {{ range $i, $v := .Levels }}{{ $v }}{{ end }}
+
ALL-CAPS tokens (3+ letters) extracted automatically from titles and
+transcripts - acronyms, program names, compartments, org designators. This is a
+text statistic, not curated metadata. A token links to a search for the word in any
+letter case, across the metadata too, so the search can find more documents than the
+capitals counted here.
+
+
+
+
+
{{ range .Heads }}{{ template "sort-th" . }}{{ end }}
+
+
+{{ if .RowsHTML }}{{ .RowsHTML }}{{ else }}{{ template "codeword-rows" . }}{{ end }}
+
+
+{{ end }}
+
+{{ template "base.html" . }}
+
+{{ define "codeword-rows" }} {{ range .Rows }}
+
+ {{- end -}}
+ {{- if or $pub.Journalists $pub.Contributors }}
+
+ {{- range $j, $name := $pub.Journalists -}}
+ {{- if $j }} · {{ end -}}
+ {{ $name }}
+ {{- end -}}
+ {{- range $j, $name := $pub.Contributors -}}
+ {{- if or $j $pub.Journalists }} · {{ end -}}
+ {{ $name }}
+ {{- end -}}
+
+ {{- end -}}
+ {{- if $pub.ReportingOnly }}
+
Reporting partner - published this story, not the document
+ {{- end -}}
+ {{- with $pub.Note }}
+
{{ . }}
+ {{- end -}}
+ {{- $copy := $.Doc.VersionFor $pub.Version -}}
+ {{- $shown := and (not $pub.ReportingOnly) (not $pub.Version) $.Doc.Versions -}}
+ {{- if and (not $pub.ReportingOnly) (or $copy $pub.DocSource $pub.DocURL $shown) }}
+
+ {{- if $shown }}the copy shown on this page{{ end -}}
+ {{- with $copy }}view this copy →{{ if .ChangedPages }}changed pages: {{ template "changed-page-links" . }}{{ end }}{{ end -}}
+ {{- with $pub.DocSource }}document page ↗{{ end -}}
+ {{- with $pub.DocURL }}original PDF ↗{{ end -}}
+
+ {{- end -}}
+
+ {{- end }}
+ {{- with .Doc.UnlinkedVersions }}
+
+
Other copies
+
+ {{- range . }}{{ .Slug }} →{{ if .ChangedPages }} (changed pages: {{ template "changed-page-links" . }}){{ end }}{{ end -}}
+
+
+ {{- end }}
+
+
+
+
{{ .Doc.Description }}
+
+
+
+
+
+
+
+
+ Long Description:
+
+ {{- with .Doc.LongDescription }}{{ . }}{{ else }}No extended description available.{{ end -}}
+
+ Identifier:
+ {{ .Doc.ID }}
+ {{ with .Doc.AkaID }}
+ Also known as:
+ {{ range $i, $aka := . }}{{ if $i }} {{ end }}{{ $aka.ID }}{{ with $aka.DocSource }} page ↗{{ end }}{{ with $aka.DocURL }} PDF ↗{{ end }}{{ end }}
+ {{ end }}
+ {{ with .Doc.PDFSha256 }}
+
+ PDF SHA-256:
+ {{ . }}
+ {{ end }}
+
+
+
+
+
+
+ My notes
+
+
+
+
+
+
+
+{{ if $sidebar }}
+
+{{ end }}
+
+
+
+ {{ template "doc-viewport" . }}
+
+
{{/* /.doc-layout */}}
+
+{{ if and .Images (not .Prefs.TextOnly) }}
+{{ template "lightbox-overlays" . }}
+
+{{ end }}
+
+
+ {{ with .Doc.ArchiveComment }}
+
+ Archive note: {{ . }}
+
+ {{ end }}
+
+
+
+
+{{ if or .PublishedTogether .RelatedDocs }}
+
+
+ {{ if .PublishedTogether }}
+
+
+
+
Published in the Same Report
+ {{ if .PublishedMore }}({{ .PublishedShown }} of {{ len .PublishedTogether }} documents){{ else }}({{ len .PublishedTogether }} documents){{ end }}
+
+ {{ end }}{{ end }}
diff --git a/templates/index.html b/templates/index.html
new file mode 100644
index 0000000..105d940
--- /dev/null
+++ b/templates/index.html
@@ -0,0 +1,150 @@
+{{ define "content" }}
+
Snowden Archive
+
+
+ This archive holds {{ .SnowdenTotal }} published Snowden documents. {{ .ReportedTotal }} documents are
+ published with news reporting, and {{ .ArchiveTotal }} documents are the NSA newsletter SIDtoday
+ issues, from The Intercept's bulk release.{{ if .FOIATotal }} This archive also holds {{ .FOIATotal }}
+ FOIA documents related to the Snowden disclosures (FOIA litigation, FISC and ODNI declassifications),
+ which {{ if .IncludeFOIA }}you have included via Preferences{{ else }}are left out by default but can be included via Preferences{{ end }}.{{ end }}
+
Digging deeper? Explore the archive for ranked views and lists.
+{{ end }}
+{{ template "base.html" . }}
diff --git a/templates/list.html b/templates/list.html
new file mode 100644
index 0000000..ffa91ad
--- /dev/null
+++ b/templates/list.html
@@ -0,0 +1,95 @@
+{{ define "title" }}{{ with .CatLabel }}{{ . }}{{ else }}Category{{ end }}: {{ .Title }} - Snowden Archive{{ end }}
+{{ define "content" }}
+
+
+
{{ with .CatLabel }}{{ . }}{{ else }}Category{{ end }}: {{ .Title }}
+ {{- if eq .ActiveCategory "agency" }}{{ with agencyName .Title }} {{ . }}{{ end }}{{ end }}
+ {{ .Total }} document{{ if ne .Total 1 }}s{{ end }}
+
+{{ end }}
+{{ end }}
+
+{{ template "base.html" . }}
diff --git a/templates/partials.html b/templates/partials.html
new file mode 100644
index 0000000..225098e
--- /dev/null
+++ b/templates/partials.html
@@ -0,0 +1,412 @@
+{{ define "cat-boxes" }}
+{{ $a := .ActiveCategory }}{{ if not $a }}{{ $a = "" }}{{ end }}
+
+{{ end }}
+
+{{/* pub-publisher is a publication entry's outlet: a link to the story where
+ the entry has one, else to the outlet's page - which a partner that only
+ reported has none of. */}}
+{{ define "pub-publisher" -}}
+{{ if .Pub.Link }}{{ .Pub.Publisher }} ↗
+{{- else if .Pub.ReportingOnly }}{{ .Pub.Publisher }}
+{{- else }}{{ .Pub.Publisher }}{{ end }}
+{{- end }}
+
+{{ define "sort-th" }}{{ if .URL }}
{{ range .TableHeads }}{{ template "sort-th" . }}{{ end }}
+
+
+{{ if .RowsHTML }}{{ .RowsHTML }}{{ else }}{{ range .TableRows }}{{ template "doc-table-row" . }}{{ end }}{{ end }}
+
+
+{{ end }}
+
+{{/* The rows are rendered once for every reader, so a FOIA release's links
+ always name every collection: they must list the release itself. */}}
+{{ define "doc-table-row" }}{{ $qs := "" }}{{ if eq .Collection "foia" }}{{ $qs = wholeArchiveQS }}{{ end }}
+
+ {{ $cat := $.ActiveCategory }}{{ $val := $.ActiveValue }}
+ {{/* A FOIA release's links name every collection for a reader who
+ leaves FOIA out: the pages they open must list the release. */}}
+ {{ $qs := "" }}{{ if eq .Collection "foia" }}{{ $qs = $.FOIAQS }}{{ end }}
+
+
+
+{{ end }}
+
+{{ define "type-badge" }}
+{{ if eq . "FOIA" }}
+FOIA
+{{ else if eq . "SIDTODAY" }}
+SIDtoday
+{{ end }}
+{{ end }}
+
+{{ define "lightbox-overlays" }}
+{{ $lbBase := .ImageBase }}{{ $lbN := len .Images }}
+{{ range $i, $img := .Images }}
+
+
+ ×
+ {{ if $i }}‹{{ end }}
+ {{ if lt (add $i 1) $lbN }}›{{ end }}
+
+ {{ add $i 1 }} / {{ $lbN }}{{ if and $.Version ($.Version.PageChanged (add $i 1)) }} · differs from the main document{{ end }}
+
+{{ end }}
+{{ end }}
+
+{{ define "changed-page-links" }}
+{{- $v := . -}}
+{{- range $k, $p := .ChangedPages -}}
+{{- if $k }}, {{ end -}}
+{{- if $p.Pos }}{{ $p.Page }}{{ else }}{{ $p.Page }}{{ end -}}
+{{- end -}}
+{{ end }}
+
+{{ define "doc-viewport" }}
+
+
+
+
+
+
+
+
+
+
+
+
+ {{ if gt (len .Images) 1 }}
+
+ {{ end }}
+
+ {{ if .Prefs.TextOnly }}
+
Text-only mode is on - page images are not loaded. Preferences
+
+ {{ range $i, $img := .Images }}
+
Page {{ add $i 1 }}{{ if and $.Version ($.Version.PageChanged (add $i 1)) }} (differs from the main document){{ end }}
Stored only in the localStorage and applied on every page. To turn it off, empty the box and save. If a bad rule ever makes the site unusable, open any page with ?nocss=1 (e.g. /preferences?nocss=1).
+
+
+
+
+
+
+
+{{ end }}
+{{ template "base.html" . }}
diff --git a/templates/search.html b/templates/search.html
new file mode 100644
index 0000000..c8711ab
--- /dev/null
+++ b/templates/search.html
@@ -0,0 +1,280 @@
+{{ define "title" }}{{ with .QueryLabel }}{{ . }} - {{ end }}Search - Snowden Archive{{ end }}
+{{ define "search-result-item" }}
+{{ $r := .Result }}{{ $d := $r.Doc }}
+{{ $qs := "" }}{{ if eq $d.Collection "foia" }}{{ $qs = .FOIAQS }}{{ end }}
+
+ {{ template "type-badge" $d.Type }}
+
+ {{ if $d.Publication }}
+ {{/* A year links to its publication-year page only when that page
+ lists the document: the year of its FIRST publication. */}}
+
+ {{- $shown := false -}}
+ {{- range $pub := $d.Publication -}}
+ {{- if $pub.Publisher -}}
+ {{- if $shown }} · {{ end }}{{ $shown = true -}}
+ {{ template "pub-publisher" (dict "Pub" $pub "QS" $qs) }}
+ {{- if ge (len $pub.DatePublished) 4 }}{{ $y := slice $pub.DatePublished 0 4 }} {{ if eq $y $d.FirstPubYear }}{{ $y }}{{ else }}{{ $y }}{{ end }}{{ end -}}
+ {{- range $j, $r := $pub.Journalists }}{{ if $j }}, {{ else }} - {{ end }}{{ $r }}{{ end -}}
+ {{- end -}}
+ {{- end }}
+
+ {{ end }}
+
+
+ {{ if $d.DocMeta.Creator }}
+ {{ $d.DocMeta.Creator }}
+ {{ end }}
+ {{ if and $d.DocMeta.Creator $d.DocMeta.DateCreated }}
+ ·
+ {{ end }}
+ {{ if $d.DocMeta.DateCreated }}
+ {{ if ge (len $d.DocMeta.DateCreated) 4 }}
+ {{ $d.DocMeta.DateCreated }}
+ {{ else }}
+ {{ $d.DocMeta.DateCreated }}
+ {{ end }}
+ {{ end }}
+
+ {{ if $d.DocMeta.Programs }}
+ ·
+ {{ $progs := $d.DocMeta.Programs }}
+ {{ $shown := $progs }}{{ if gt (len $progs) 8 }}{{ $shown = slice $progs 0 8 }}{{ end }}
+ {{ facetLinks (dict "Vals" $shown "Cat" "program" "Sep" "dot" "Class" "meta-link-soft" "QS" $qs) }}
+ {{ if gt (len $progs) 8 }}+{{ add (len $progs) -8 }} more{{ end }}
+ {{ end }}
+
Matches in:
+ {{- range $i, $v := . }}{{ if $i }},{{ end }} {{ $v.Label }}{{ end }}
+
+ {{ end }}
+
+
+{{ end }}
+
+{{ define "content" }}
+
Search
+
+
+ Search syntax help
+
+ Ways to write a value, in any row or after any operator:
+
+
cable tapping finds both words anywhere, in their other forms too (tapping also finds tapped);
+
"apping" finds that text anywhere, even inside a word, like grep -i (it finds for example mapping and tapping);
+
=tap or ="cable tapping" finds exactly those whole words, so =tap skips taps and strap, and on a metadata field it matches one whole value, so agency:=NSA finds documents whose agency is NSA but not NSA, GCHQ;
+
* wildcard finds part of a word: tap* finds words starting with tap, *land words ending in land and *keys* words containing keys. A leading * needs at least three letters after it.
+
"upstream collection"~10 finds the words within 10 words of each other (in any order, common stopwords ignored).
+
A leading - excludes within its row: xkeyscore -tempora finds xkeyscore without tempora, and the row keeps that meaning under OR and NOT.
+
+
+
+ The simple search box (on home page, on 404 page, and via URL ?q=) understands field operators:
+
+
program:XKEYSCORE country:Germany cable or journalist:="Poitras, Laura".
+
A leading - negates (-tag:=junk, -draft);
+
date:2013, date:2013..2015, and published:2014-06 filter by date.
+
+
+
+
+
+{{ if .RecentSearches }}
+
+ Recent searches ({{ len .RecentSearches }})
+
+ {{ range .RecentSearches }}{{ .Label }}{{ end }}
+
+
+
+{{ end }}
+
+
+
+{{ if .OverClauses }}
+
Search not run: it has {{ .OverClauses }} parts - rows, or the
+field:, “quoted” and -excluded terms of a typed query -
+and a search can have at most {{ .ClauseLimit }}. The form above keeps the first {{ .ClauseLimit }};
+the rest, including any exclusions at the end, were left out. Check the rows and search again.
+{{ else if .OverWords }}
+
Search not run: it has {{ .OverWords }} words, counting the words of
+“quoted” phrases and -excluded terms, and a search can have at most
+{{ .WordLimit }}. Shorten it and search again.
+{{ end }}
+
+{{ if .Searched }}
+{{ if gt (len .SearchTypeCounts) 1 }}{{ template "type-filter" (dict "Action" "/search" "Params" .FormParams "AllTypes" .SearchTypeCounts "SelectedTypes" .SelTypeSet) }}{{ end }}
+
Text snippets are shown for the first {{ .SnippetLimit }} results on this page -
+choose a page size in preferences, or narrow the search, to see
+snippets for the rest.
Another published copy of this document that differs from the one on the
+ document page - for example less redacted, with extra pages, or a different scan.
This copy has no transcript and no page images - only the PDF above.
+{{ end }}
+
+{{ if and .Images (not .Prefs.TextOnly) }}
+{{ template "lightbox-overlays" . }}
+{{ end }}
+{{ end }}
+
+{{ template "base.html" . }}
diff --git a/tests/api_test.go b/tests/api_test.go
new file mode 100644
index 0000000..8ed76a6
--- /dev/null
+++ b/tests/api_test.go
@@ -0,0 +1,122 @@
+package main
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "path/filepath"
+ "testing"
+)
+
+func TestAPIPageSize(t *testing.T) {
+ for raw, want := range map[string]int{
+ "": apiPerPageDefault,
+ "25": 25,
+ "500": 500,
+ "1000": apiMaxPerPage,
+ "99999999": apiMaxPerPage,
+ "0": apiPerPageDefault, // not a page size; fall back
+ "-5": apiPerPageDefault,
+ "lots": apiPerPageDefault,
+ "all": 0,
+ } {
+ v := url.Values{}
+ if raw != "" {
+ v.Set("per_page", raw)
+ }
+ if got := apiPageSize(v); got != want {
+ t.Errorf("apiPageSize(per_page=%q) = %d, want %d", raw, got, want)
+ }
+ }
+}
+
+func TestAPIRespPerPage(t *testing.T) {
+ for _, c := range []struct{ perPage, total, want int }{
+ {50, 3106, 50},
+ {0, 3106, 3106},
+ {0, 0, 1},
+ } {
+ if got := apiRespPerPage(c.perPage, c.total); got != c.want {
+ t.Errorf("apiRespPerPage(%d, %d) = %d, want %d", c.perPage, c.total, got, c.want)
+ }
+ }
+}
+
+func TestAPIVersionsShape(t *testing.T) {
+ root := t.TempDir()
+ writeTestFile(t, filepath.Join(root, "doc-a", "metadata.json"), `{
+ "title": "A", "type": "REPORTED",
+ "publication": [{"publisher": "Le Monde", "version": "entier"}],
+ "versions": {"entier": {"changed_pages": [1]}}
+ }`)
+ writeTestFile(t, filepath.Join(root, "doc-a", "a.pdf"), "%PDF-1.4")
+ writeTestFile(t, filepath.Join(root, "doc-a", "versions", "entier", "entier.pdf"), "%PDF-1.4")
+ withCorpus(t, root)
+
+ type versionsObj struct {
+ Versions map[string]VersionInfo `json:"versions"`
+ }
+ var detail struct {
+ versionsObj
+ VersionCopies []apiVersion `json:"version_copies"`
+ }
+ decode := func(path string, v interface{}) {
+ t.Helper()
+ w := get(t, path)
+ if w.Code != 200 {
+ t.Fatalf("%s: %d", path, w.Code)
+ }
+ if err := json.Unmarshal(w.Body.Bytes(), v); err != nil {
+ t.Fatalf("%s: %v", path, err)
+ }
+ }
+ decode("/api/doc/doc-a", &detail)
+ if len(detail.Versions["entier"].ChangedPages) != 1 {
+ t.Errorf("/api/doc versions = %+v, want the metadata object", detail.Versions)
+ }
+ if len(detail.VersionCopies) != 1 || detail.VersionCopies[0].Slug != "entier" {
+ t.Errorf("/api/doc version_copies = %+v, want the entier copy", detail.VersionCopies)
+ }
+ var list struct{ Docs []versionsObj }
+ decode("/api/docs", &list)
+ var search struct{ Results []versionsObj }
+ decode("/api/search?q=A", &search)
+ for name, docs := range map[string][]versionsObj{"/api/docs": list.Docs, "/api/search": search.Results} {
+ if len(docs) != 1 || len(docs[0].Versions["entier"].ChangedPages) != 1 {
+ t.Errorf("%s versions = %+v, want the metadata object", name, docs)
+ }
+ }
+}
+
+func TestAPIFacetsRevalidates(t *testing.T) {
+ w := get(t, "/api/facets")
+ tag := w.Header().Get("ETag")
+ if w.Code != 200 || tag == "" || w.Header().Get("Access-Control-Allow-Origin") != "*" {
+ t.Fatalf("/api/facets: %d, ETag %q, CORS %q", w.Code, tag, w.Header().Get("Access-Control-Allow-Origin"))
+ }
+ mux, _ := buildFixture(t)
+ r := httptest.NewRequest("GET", "/api/facets", nil)
+ r.Header.Set("If-None-Match", tag)
+ if w := record(mux, r); w.Code != http.StatusNotModified || w.Body.Len() != 0 {
+ t.Errorf("revalidation: %d with %d bytes, want 304 and none", w.Code, w.Body.Len())
+ }
+}
+
+// /api/docs takes coll= as /api/search and the pages do.
+func TestAPIDocsCollections(t *testing.T) {
+ total := func(query string) int {
+ var body struct{ Total int }
+ if err := json.Unmarshal(get(t, "/api/docs?"+query).Body.Bytes(), &body); err != nil {
+ t.Fatalf("%s: %v", query, err)
+ }
+ return body.Total
+ }
+ all, news, foia := total(""), total("coll=news"), total("coll=foia")
+ if news == 0 || foia == 0 || news+foia >= all {
+ t.Fatalf("fixture totals: %d documents, %d news, %d foia", all, news, foia)
+ }
+ if got := total("coll=news&coll=foia"); got != news+foia {
+ t.Errorf("coll=news&coll=foia lists %d, want %d", got, news+foia)
+ }
+}
diff --git a/tests/assemblies_test.go b/tests/assemblies_test.go
new file mode 100644
index 0000000..58f37b9
--- /dev/null
+++ b/tests/assemblies_test.go
@@ -0,0 +1,211 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "path/filepath"
+ "reflect"
+ "slices"
+ "strings"
+ "testing"
+)
+
+func writeAssemblyDoc(t *testing.T, root, id string, meta map[string]interface{}, pages []string, versions map[string][]string) {
+ t.Helper()
+ dir := filepath.Join(root, id)
+ b, err := json.Marshal(meta)
+ if err != nil {
+ t.Fatal(err)
+ }
+ writeFixtureFile(t, filepath.Join(dir, "metadata.json"), string(b))
+ writeFixtureFile(t, filepath.Join(dir, id+".pdf"), "%PDF-1.4\n")
+ for _, p := range pages {
+ writeFixtureFile(t, filepath.Join(dir, "images", p), fixtureJPEG)
+ }
+ for slug, imgs := range versions {
+ v := filepath.Join(dir, "versions", slug)
+ writeFixtureFile(t, filepath.Join(v, slug+".pdf"), "%PDF-1.4\n")
+ for _, p := range imgs {
+ writeFixtureFile(t, filepath.Join(v, "images", p), fixtureJPEG)
+ }
+ }
+}
+
+func assemblyCorpus(t *testing.T, manifest string) string {
+ t.Helper()
+ root := t.TempDir()
+ writeAssemblyDoc(t, root, "deck", map[string]interface{}{
+ "title": "The Deck",
+ "publication": []map[string]interface{}{
+ {"publisher": "Outlet A"},
+ {"publisher": "Partner", "reporting_only": true},
+ {"publisher": "Outlet B", "version": "fuller"},
+ },
+ }, []string{"page-1.jpg", "page-2.jpg", "page-3.jpg"}, map[string][]string{"fuller": {"page-1.jpg", "page-2.jpg"}})
+ writeAssemblyDoc(t, root, "single", map[string]interface{}{
+ "title": "One Slide",
+ "publication": []map[string]interface{}{{"publisher": "Outlet C"}},
+ }, []string{"page-1.jpg"}, nil)
+ writeAssemblyDoc(t, root, "bystander", map[string]interface{}{"title": "Bystander"}, []string{"page-1.jpg"}, nil)
+ if manifest != "" {
+ writeFixtureFile(t, filepath.Join(root, "assemblies.json"), manifest)
+ }
+ return root
+}
+
+const assemblyManifest = `[{
+ "id": "set", "title": "A Set", "note": "Order is a guess.",
+ "slides": [
+ {"doc": "deck", "pages": "1-2", "alt": [{"doc": "deck", "version": "fuller", "pages": "1-2"}]},
+ {"doc": "deck", "version": "fuller", "pages": "2", "note": "Only here."},
+ {"doc": "single", "pages": "1"}
+ ]
+}]`
+
+func TestParsePageList(t *testing.T) {
+ for in, want := range map[string][]int{
+ "8": {8}, "1-3": {1, 2, 3}, "1,3,5-6": {1, 3, 5, 6}, " 2 , 4 ": {2, 4}, "9,1": {9, 1},
+ } {
+ got, err := parsePageList(in)
+ if err != nil || !reflect.DeepEqual(got, want) {
+ t.Errorf("parsePageList(%q) = %v, %v; want %v", in, got, err, want)
+ }
+ }
+ for _, in := range []string{"", "a", "0", "3-1", "1-", "1,,2"} {
+ if got, err := parsePageList(in); err == nil {
+ t.Errorf("parsePageList(%q) = %v, want an error", in, got)
+ }
+ }
+}
+
+func TestAssemblyResolves(t *testing.T) {
+ withCorpus(t, assemblyCorpus(t, assemblyManifest))
+ a := corpus().assemblyIndex["set"]
+ if a == nil {
+ t.Fatal("assembly not loaded")
+ }
+ if len(a.Slides) != 4 {
+ t.Fatalf("%d slides, want 4", len(a.Slides))
+ }
+ for i, s := range a.Slides {
+ if s.Pos != i+1 {
+ t.Errorf("slide %d has Pos %d", i+1, s.Pos)
+ }
+ }
+ first := a.Slides[0]
+ if first.Image != "/files/deck/images/page-1.jpg" || first.Source != "Outlet A" {
+ t.Errorf("first slide %+v", first)
+ }
+ if len(first.Alts) != 1 || first.Alts[0].Image != "/files/deck/versions/fuller/images/page-1.jpg" || first.Alts[0].Source != "Outlet B" {
+ t.Errorf("first slide alts %+v", first.Alts)
+ }
+ if a.Slides[2].Note != "Only here." || a.Slides[2].Path != "/doc/deck/versions/fuller" {
+ t.Errorf("third slide %+v", a.Slides[2])
+ }
+ if fmt.Sprint(a.DocList) != "[deck single]" {
+ t.Errorf("DocList %v", a.DocList)
+ }
+ if a.Copies != 3 {
+ t.Errorf("Copies %d, want 3", a.Copies)
+ }
+ if len(corpus().assembliesByDoc["single"]) != 1 || len(corpus().assembliesByDoc["bystander"]) != 0 {
+ t.Errorf("assembliesByDoc %v", corpus().assembliesByDoc)
+ }
+}
+
+func TestAssemblyPages(t *testing.T) {
+ withCorpus(t, assemblyCorpus(t, assemblyManifest))
+ h := withBaseHeaders(newMux())
+ body := func(path string, code int) string {
+ t.Helper()
+ w := record(h, httptest_NewGet(path))
+ if w.Code != code {
+ t.Fatalf("%s: %d, want %d", path, w.Code, code)
+ }
+ return w.Body.String()
+ }
+ list := body("/explore/assemblies", 200)
+ if !strings.Contains(list, `href="/explore/assemblies/set"`) || !strings.Contains(list, "4 slides from 3 copies") {
+ t.Errorf("/explore/assemblies:\n%s", list)
+ }
+ page := body("/explore/assemblies/set", 200)
+ for _, want := range []string{
+ "A Set - Snowden Archive", "from 3 separately published copies.", "Order is a guess.", "not a document in its own right",
+ `id="lb-4"`, `href="#lb-a1-1"`, `id="lb-a1-1"`, "Other copy: Outlet B", "Back to slide 1", "(copy: fuller)", "Outlet C", "· Only here.",
+ } {
+ if !strings.Contains(page, want) {
+ t.Errorf("/explore/assemblies/set lacks %q", want)
+ }
+ }
+ if strings.Contains(page, `id="lb-5"`) || strings.Contains(page, "Partner") {
+ t.Errorf("/explore/assemblies/set has a fifth overlay or a reporting-only credit")
+ }
+ if !strings.Contains(body("/doc/single", 200), `href="/explore/assemblies/set"`) {
+ t.Errorf("/doc/single has no assembly link")
+ }
+ if strings.Contains(body("/doc/bystander", 200), "/explore/assemblies/") {
+ t.Errorf("/doc/bystander links an assembly")
+ }
+ // Assemblies live under /explore only: listed there, not on /browse.
+ if !strings.Contains(body("/explore", 200), `href="/explore/assemblies"`) {
+ t.Errorf("/explore has no assemblies link")
+ }
+ if strings.Contains(body("/browse", 200), "assemblies") {
+ t.Errorf("/browse still links assemblies")
+ }
+ body("/explore/assemblies/nope", 404)
+ if !slicesContains(corpus().sitemapPaths, "/explore/assemblies/set") {
+ t.Errorf("sitemap lacks the assembly page")
+ }
+}
+
+func TestAssemblyProblemsWarn(t *testing.T) {
+ cases := map[string]struct {
+ manifest string
+ slides int
+ loaded bool
+ }{
+ "missing doc": {`[{"id":"x","title":"X","slides":[{"doc":"ghost","pages":"1"},{"doc":"single","pages":"1"}]}]`, 1, true},
+ "missing page": {`[{"id":"x","title":"X","slides":[{"doc":"deck","pages":"3-4"}]}]`, 1, true},
+ "bad version": {`[{"id":"x","title":"X","slides":[{"doc":"deck","version":"no","pages":"1"},{"doc":"single","pages":"1"}]}]`, 1, true},
+ "bad pages": {`[{"id":"x","title":"X","slides":[{"doc":"deck","pages":"two"},{"doc":"single","pages":"1"}]}]`, 1, true},
+ "alt mismatch": {`[{"id":"x","title":"X","slides":[{"doc":"deck","pages":"1-2","alt":[{"doc":"single","pages":"1"}]}]}]`, 2, true},
+ "nothing left": {`[{"id":"x","title":"X","slides":[{"doc":"ghost","pages":"1"}]}]`, 0, false},
+ "unsafe id": {`[{"id":"a/b","title":"X","slides":[{"doc":"single","pages":"1"}]}]`, 0, false},
+ "malformed": {`[{`, 0, false},
+ }
+ for name, c := range cases {
+ t.Run(name, func(t *testing.T) {
+ withCorpus(t, assemblyCorpus(t, c.manifest))
+ if loadWarnings == 0 {
+ t.Errorf("no warning")
+ }
+ a := corpus().assemblyIndex["x"]
+ if (a != nil) != c.loaded {
+ t.Fatalf("loaded = %v, want %v", a != nil, c.loaded)
+ }
+ if a != nil && len(a.Slides) != c.slides {
+ t.Errorf("%d slides, want %d", len(a.Slides), c.slides)
+ }
+ })
+ }
+}
+
+func TestNoAssembliesFile(t *testing.T) {
+ withCorpus(t, assemblyCorpus(t, ""))
+ if len(corpus().assemblies) != 0 {
+ t.Fatalf("assemblies %v", corpus().assemblies)
+ }
+ h := withBaseHeaders(newMux())
+ if w := record(h, httptest_NewGet("/explore/assemblies")); w.Code != 200 {
+ t.Errorf("/explore/assemblies: %d", w.Code)
+ }
+ if w := record(h, httptest_NewGet("/explore")); strings.Contains(w.Body.String(), `href="/explore/assemblies"`) {
+ t.Errorf("/explore links assemblies with none loaded")
+ }
+ if slicesContains(corpus().sitemapPaths, "/explore/assemblies") {
+ t.Errorf("sitemap lists assemblies with none loaded")
+ }
+}
+
+func slicesContains(s []string, v string) bool { return slices.Contains(s, v) }
diff --git a/tests/bench_test.go b/tests/bench_test.go
new file mode 100644
index 0000000..6e47db5
--- /dev/null
+++ b/tests/bench_test.go
@@ -0,0 +1,312 @@
+package main
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "runtime"
+ "sort"
+ "strconv"
+ "sync"
+ "testing"
+ "time"
+)
+
+// Page benchmarks against a REAL corpus - the fixture is four documents, so
+// it cannot show what 3,000 do. Point $SNOWCHIVE_BENCH_ROOT at an archive
+// root (the directory holding data/ and project/) and run:
+// tests/run.sh -run XXX -bench . -benchtime 40x
+//
+// What they are for: every page but /search is built from data indexed at
+// load, so a request should cost what its OUTPUT costs (template execution)
+// and nothing more. A page whose time or allocations grow with the CORPUS
+// rather than with the page has picked up a scan.
+var (
+ benchOnce sync.Once
+ benchMux http.Handler
+)
+
+func benchCorpus(tb testing.TB) http.Handler {
+ root := os.Getenv("SNOWCHIVE_BENCH_ROOT")
+ if root == "" {
+ tb.Skip("set SNOWCHIVE_BENCH_ROOT to an archive root to run the page benchmarks")
+ }
+ benchOnce.Do(func() {
+ setRoot(root)
+ devMode = false
+ apiEnabled = true
+ initAssets()
+ if err := parseTemplates(); err != nil {
+ tb.Fatal(err)
+ }
+ loadData()
+ benchMux = withBaseHeaders(newMux())
+ })
+ return benchMux
+}
+
+func benchPage(b *testing.B, path string) {
+ h := benchCorpus(b)
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ r := httptest.NewRequest("GET", path, nil)
+ w := httptest.NewRecorder()
+ h.ServeHTTP(w, r)
+ if w.Code != 200 {
+ b.Fatalf("%s: %d", path, w.Code)
+ }
+ }
+}
+
+func BenchmarkPageHome(b *testing.B) { benchPage(b, "/") }
+func BenchmarkPageBrowse(b *testing.B) { benchPage(b, "/browse") }
+func BenchmarkPageArchiveTab(b *testing.B) { benchPage(b, "/browse?collection=sidtoday-archive") }
+func BenchmarkPageCategoryList(b *testing.B) { benchPage(b, "/browse/program") }
+func BenchmarkPageCategoryValue(b *testing.B) { benchPage(b, "/browse/agency/NSA") }
+func BenchmarkPageDoc(b *testing.B) {
+ h := benchCorpus(b)
+ if len(corpus().archive.Docs) == 0 {
+ b.Skip("empty corpus")
+ }
+ benchPageWith(b, h, "/doc/"+corpus().archive.Docs[0].ID)
+}
+func BenchmarkPageExploreIDs(b *testing.B) { benchPage(b, "/explore/ids") }
+func BenchmarkPageGlossary(b *testing.B) { benchPage(b, "/glossary") }
+func BenchmarkPageSitemap(b *testing.B) { benchPage(b, "/sitemap.xml") }
+func BenchmarkPageFacetsAPI(b *testing.B) { benchPage(b, "/api/facets") }
+func BenchmarkPageSearch(b *testing.B) { benchPage(b, "/search?q=nsa&do_search=1") }
+
+func benchPageWith(b *testing.B, h http.Handler, path string) {
+ b.ReportAllocs()
+ b.ResetTimer()
+ for i := 0; i < b.N; i++ {
+ r := httptest.NewRequest("GET", path, nil)
+ w := httptest.NewRecorder()
+ h.ServeHTTP(w, r)
+ if w.Code != 200 {
+ b.Fatalf("%s: %d", path, w.Code)
+ }
+ }
+}
+
+func BenchmarkPageCategoryTag(b *testing.B) { benchPage(b, "/browse/tag") }
+func BenchmarkPageCategoryPeople(b *testing.B) { benchPage(b, "/browse/people") }
+func BenchmarkPageCategoryTagSorted(b *testing.B) { benchPage(b, "/browse/tag?sort=count") }
+func BenchmarkPageExploreMetric(b *testing.B) { benchPage(b, "/explore/pages") }
+func BenchmarkPageCodewords(b *testing.B) { benchPage(b, "/explore/terms") }
+func BenchmarkPageOutlets(b *testing.B) { benchPage(b, "/explore/outlets") }
+func BenchmarkPageClassPatterns(b *testing.B) { benchPage(b, "/explore/classification") }
+func BenchmarkPageArticles(b *testing.B) { benchPage(b, "/articles") }
+func BenchmarkPageChangelog(b *testing.B) { benchPage(b, "/changelog") }
+func BenchmarkPageAbout(b *testing.B) { benchPage(b, "/about") }
+func BenchmarkPageAPIDocs(b *testing.B) { benchPage(b, "/api/docs") }
+func BenchmarkPageExploreIndex(b *testing.B) { benchPage(b, "/explore") }
+func BenchmarkPageContactSheet(b *testing.B) {
+ h := benchCorpus(b)
+ id := ""
+ for i := range corpus().archive.Docs {
+ if len(corpus().archive.Docs[i].Images) > 40 {
+ id = corpus().archive.Docs[i].ID
+ break
+ }
+ }
+ if id == "" {
+ b.Skip("no document with many pages")
+ }
+ benchPageWith(b, h, "/doc/"+id+"/pages")
+}
+
+// Pathological documents: the biggest thing in the corpus is what breaks
+// first. What IS the biggest thing?
+func TestProbeExtremes(t *testing.T) {
+ if os.Getenv("SNOWCHIVE_BENCH_ROOT") == "" {
+ t.Skip("needs a real corpus")
+ }
+ benchCorpus(t)
+ var maxImgs, maxText, maxPubs, maxVersions *Document
+ for i := range corpus().archive.Docs {
+ d := &corpus().archive.Docs[i]
+ if maxImgs == nil || len(d.Images) > len(maxImgs.Images) {
+ maxImgs = d
+ }
+ if maxText == nil || len(d.FullText) > len(maxText.FullText) {
+ maxText = d
+ }
+ if maxPubs == nil || len(d.Publication) > len(maxPubs.Publication) {
+ maxPubs = d
+ }
+ if maxVersions == nil || len(d.Versions) > len(maxVersions.Versions) {
+ maxVersions = d
+ }
+ }
+ h := benchCorpus(t)
+ timeIt := func(p string) time.Duration {
+ start := time.Now()
+ r := httptest.NewRequest("GET", p, nil)
+ w := httptest.NewRecorder()
+ h.ServeHTTP(w, r)
+ return time.Since(start)
+ }
+ fmt.Printf("most pages: %d %s doc %s, contact sheet %s\n", len(maxImgs.Images), trunc30(maxImgs.ID),
+ timeIt("/doc/"+maxImgs.ID).Round(time.Millisecond), timeIt("/doc/"+maxImgs.ID+"/pages").Round(time.Millisecond))
+ fmt.Printf("most text: %d KB %s doc page %s\n", len(maxText.FullText)/1024, trunc30(maxText.ID),
+ timeIt("/doc/"+maxText.ID).Round(time.Millisecond))
+ fmt.Printf("most pubs: %d %s\n", len(maxPubs.Publication), trunc30(maxPubs.ID))
+ fmt.Printf("most copies: %d %s\n", len(maxVersions.Versions), trunc30(maxVersions.ID))
+}
+
+// Distribution, not the median: which document pages are slow, and why?
+func TestProbeDocPageDistribution(t *testing.T) {
+ if os.Getenv("SNOWCHIVE_BENCH_ROOT") == "" {
+ t.Skip("needs a real corpus")
+ }
+ h := benchCorpus(t)
+ totalImgs := 0
+ for i := range corpus().archive.Docs {
+ totalImgs += len(corpus().archive.Docs[i].Images)
+ }
+ var ds []time.Duration
+ for i := 0; i < len(corpus().archive.Docs); i += 7 { // a sample, every 7th
+ d := &corpus().archive.Docs[i]
+ start := time.Now()
+ r := httptest.NewRequest("GET", "/doc/"+d.ID, nil)
+ w := httptest.NewRecorder()
+ h.ServeHTTP(w, r)
+ ds = append(ds, time.Since(start))
+ }
+ sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] })
+ p := func(q float64) time.Duration { return ds[int(float64(len(ds)-1)*q)] }
+ fmt.Printf("%d page images in the corpus (%d docs sampled)\n", totalImgs, len(ds))
+ fmt.Printf("doc page: p50 %s p90 %s p99 %s max %s\n",
+ p(0.5).Round(time.Microsecond), p(0.9).Round(time.Microsecond),
+ p(0.99).Round(time.Millisecond), ds[len(ds)-1].Round(time.Millisecond))
+}
+
+func TestProbeWorstDocSplit(t *testing.T) {
+ if os.Getenv("SNOWCHIVE_BENCH_ROOT") == "" {
+ t.Skip("needs a real corpus")
+ }
+ h := benchCorpus(t)
+ var worst *Document
+ for i := range corpus().archive.Docs {
+ if worst == nil || len(corpus().archive.Docs[i].Images) > len(worst.Images) {
+ worst = &corpus().archive.Docs[i]
+ }
+ }
+ run := func(path string, cookies ...*http.Cookie) (time.Duration, int) {
+ var best time.Duration
+ var size int
+ for i := 0; i < 5; i++ {
+ r := httptest.NewRequest("GET", path, nil)
+ for _, c := range cookies {
+ r.AddCookie(c)
+ }
+ w := httptest.NewRecorder()
+ start := time.Now()
+ h.ServeHTTP(w, r)
+ d := time.Since(start)
+ if i == 0 || d < best {
+ best = d
+ }
+ size = w.Body.Len()
+ }
+ return best, size
+ }
+ full, fullSize := run("/doc/" + worst.ID)
+ textOnly, textSize := run("/doc/"+worst.ID, &http.Cookie{Name: "pref_text_only", Value: "1"})
+ sheet, sheetSize := run("/doc/" + worst.ID + "/pages")
+ fmt.Printf("worst doc: %d images, %d KB transcript\n", len(worst.Images), len(worst.FullText)/1024)
+ fmt.Printf(" full page %6s %d KB\n", full.Round(time.Microsecond), fullSize/1024)
+ fmt.Printf(" text-only %6s %d KB (no image grid, no lightbox)\n", textOnly.Round(time.Microsecond), textSize/1024)
+ fmt.Printf(" contact sheet %6s %d KB\n", sheet.Round(time.Microsecond), sheetSize/1024)
+}
+
+func trunc30(s string) string {
+ if len(s) > 30 {
+ return s[:30]
+ }
+ return s
+}
+
+// TestSoak hammers the server for a while and watches its memory. The claim
+// it checks: nothing accumulates between requests - no cache, no growing
+// map, no goroutine left behind. Run it against a real corpus:
+//
+// SNOWCHIVE_BENCH_ROOT=/path/to/archive tests/run.sh -run TestSoak -timeout 30m
+//
+// SNOWCHIVE_SOAK_REQUESTS sets the count (default 20,000).
+func TestSoak(t *testing.T) {
+ h := benchCorpus(t)
+ n := 20000
+ if v := os.Getenv("SNOWCHIVE_SOAK_REQUESTS"); v != "" {
+ if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 {
+ n = parsed
+ }
+ }
+ paths := []string{"/", "/browse", "/browse?collection=foia", "/browse/program",
+ "/browse/tag?sort=count", "/browse/agency/NSA", "/search?q=nsa&do_search=1",
+ "/search?q=%22upstream+collection%22&do_search=1", "/explore/ids", "/glossary",
+ "/articles", "/api/facets", "/api/search?q=prism", "/sitemap.xml"}
+ cookies := []*http.Cookie{
+ {Name: "pref_theme", Value: "dark"},
+ {Name: "pref_per_page", Value: "100"},
+ {Name: "pref_text_only", Value: "1"},
+ }
+
+ read := func() (heap, sys uint64, goroutines int) {
+ runtime.GC()
+ var m runtime.MemStats
+ runtime.ReadMemStats(&m)
+ return m.HeapAlloc, m.Sys, runtime.NumGoroutine()
+ }
+ // Warm up first: the pre-rendered bodies and the first template
+ // executions allocate once and stay.
+ for i := 0; i < 200; i++ {
+ r := httptest.NewRequest("GET", paths[i%len(paths)], nil)
+ h.ServeHTTP(httptest.NewRecorder(), r)
+ }
+ heap0, sys0, gor0 := read()
+
+ start := time.Now()
+ var wg sync.WaitGroup
+ const workers = 8
+ for w := 0; w < workers; w++ {
+ wg.Add(1)
+ go func(id int) {
+ defer wg.Done()
+ for i := id; i < n; i += workers {
+ r := httptest.NewRequest("GET", paths[i%len(paths)], nil)
+ if i%4 == 0 {
+ r.AddCookie(cookies[i%len(cookies)])
+ }
+ rec := httptest.NewRecorder()
+ h.ServeHTTP(rec, r)
+ if rec.Code != 200 {
+ t.Errorf("%s: %d", paths[i%len(paths)], rec.Code)
+ return
+ }
+ }
+ }(w)
+ }
+ wg.Wait()
+ elapsed := time.Since(start)
+ heap1, sys1, gor1 := read()
+
+ mb := func(b uint64) float64 { return float64(b) / (1 << 20) }
+ fmt.Printf("%d requests in %s (%.0f req/s, %d workers)\n",
+ n, elapsed.Round(time.Millisecond), float64(n)/elapsed.Seconds(), workers)
+ fmt.Printf("heap %.1f MB -> %.1f MB (%+.1f)\n", mb(heap0), mb(heap1), mb(heap1)-mb(heap0))
+ fmt.Printf("sys %.1f MB -> %.1f MB (%+.1f)\n", mb(sys0), mb(sys1), mb(sys1)-mb(sys0))
+ fmt.Printf("goroutines %d -> %d\n", gor0, gor1)
+
+ if gor1 > gor0+2 {
+ t.Errorf("goroutines grew from %d to %d", gor0, gor1)
+ }
+ // The heap after a GC should be the corpus and the pre-rendered pages,
+ // not a function of how many requests have been served.
+ if grown := mb(heap1) - mb(heap0); grown > 32 {
+ t.Errorf("heap grew by %.1f MB over %d requests - something is accumulating", grown, n)
+ }
+}
diff --git a/tests/bugs_test.go b/tests/bugs_test.go
new file mode 100644
index 0000000..607025f
--- /dev/null
+++ b/tests/bugs_test.go
@@ -0,0 +1,78 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/url"
+ "strings"
+ "testing"
+)
+
+// A hyphenated term must match its words in sequence, not just the first one.
+func TestHyphenatedTermMatchesInSequence(t *testing.T) {
+ hit := Document{ID: "hit", FullText: "A cyber attack on the power grid."}
+ miss := Document{ID: "miss", FullText: "Cyber policy, and an attack helicopter."}
+ for _, d := range []*Document{&hit, &miss} {
+ setBodySearchText(d)
+ d.StemmedText, d.SearchText = d.StemmedFull, d.FoldedFull
+ }
+
+ for _, field := range []string{"FT", "TE"} {
+ if !compileQuery([]string{"cyber-attack"}, []string{field}, []string{"and"}).matches(&hit) {
+ t.Errorf("%s: cyber-attack should match text containing the words in sequence", field)
+ }
+ if compileQuery([]string{"cyber-attack"}, []string{field}, []string{"and"}).matches(&miss) {
+ t.Errorf("%s: cyber-attack matched a document where the words are apart", field)
+ }
+ }
+ // The words on their own still match both, as before.
+ if !compileQuery([]string{"cyber"}, []string{"FT"}, []string{"and"}).matches(&miss) {
+ t.Error("a plain word should still match")
+ }
+}
+
+// A value page's publication-year bars count every year a document was
+// published in, because each bar links to a search that matches any of them.
+func TestPubYearsOf(t *testing.T) {
+ var d Document
+ if err := json.Unmarshal([]byte(`{"publication": [
+ {"publisher": "The Guardian", "date_published": "2013-06-06"},
+ {"publisher": "Le Monde", "date_published": "2014-10-21"},
+ {"publisher": "Der Spiegel", "date_published": "2013-12-30"},
+ {"publisher": "No date"},
+ {"publisher": "Bad date", "date_published": "201"}
+ ]}`), &d); err != nil {
+ t.Fatal(err)
+ }
+ if got := fmt.Sprint(pubYearsOf(&d)); got != "[2013 2014]" {
+ t.Errorf("pubYearsOf = %s, want [2013 2014]", got)
+ }
+ if got := fmt.Sprint(pubYearsOf(&Document{})); got != "[]" {
+ t.Errorf("pubYearsOf(no publications) = %s, want []", got)
+ }
+}
+
+// A saved search's label must name the type filter, or two saved searches
+// differing only by Reported/SIDtoday are indistinguishable.
+func TestSearchLabelIncludesTypeFilter(t *testing.T) {
+ entry := func(v url.Values) string { return url.QueryEscape(v.Encode()) }
+ base := url.Values{"fqv": {"xkeyscore"}, "fqf": {"TE"}, "fqc": {"and"}}
+ typed := url.Values{"fqv": {"xkeyscore"}, "fqf": {"TE"}, "fqc": {"and"}, "type": {"REPORTED"}}
+ both := url.Values{"fqv": {"xkeyscore"}, "fqf": {"TE"}, "fqc": {"and"},
+ "type": {"SIDTODAY"}, "coll": {"news"}}
+
+ got := corpus().parseSearchEntries([]string{entry(typed), entry(base), entry(both)})
+ if len(got) != 3 {
+ t.Fatalf("parsed %d entries, want 3", len(got))
+ }
+ if !strings.Contains(got[0].Label, "REPORTED") {
+ t.Errorf("label %q should name the type filter", got[0].Label)
+ }
+ if got[0].Label == got[1].Label {
+ t.Errorf("searches differing only by type share the label %q", got[0].Label)
+ }
+ // Collection and type share one bracket rather than two.
+ if !strings.Contains(got[2].Label, "(news, SIDTODAY)") {
+ t.Errorf("label %q should list collection and type together", got[2].Label)
+ }
+}
diff --git a/tests/consistency_test.go b/tests/consistency_test.go
new file mode 100644
index 0000000..e001fc3
--- /dev/null
+++ b/tests/consistency_test.go
@@ -0,0 +1,538 @@
+package main
+
+import (
+ "encoding/json"
+ "html"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "path/filepath"
+ "reflect"
+ "regexp"
+ "slices"
+ "sort"
+ "strings"
+ "testing"
+)
+
+// A corpus built to exercise the places where one fact used to be computed
+// twice: search rows and their negations, the page and the API, counts and
+// the pages they link to. Each document's words are chosen so the expected
+// result sets can be written down by hand.
+//
+// n1 news alpha gamma ZEBRA 2010 The Intercept 2014 + the intercept 2016
+// n2 news alpha ZEBRA 2011 The Intercept 2014 + Der Spiegel 2014
+// n3 news beta gamma 2012 Der Spiegel 2015
+// n4 news beta 2012 Der Spiegel 2015
+// f1 foia alpha ZEBRA 2010 (court filing: "(c)" list labels)
+// s1 sidtoday ZEBRA 2011
+// a1 news "aka_id": null
+// d1 news date_created "ODNI", date_published "15 May 2015"
+func writeConsistencyCorpus(t *testing.T) string {
+ t.Helper()
+ root := t.TempDir()
+ doc := func(dir, id string, meta map[string]interface{}, text string) {
+ b, err := json.Marshal(meta)
+ if err != nil {
+ t.Fatal(err)
+ }
+ d := filepath.Join(root, dir, id)
+ writeFixtureFile(t, filepath.Join(d, "metadata.json"), string(b))
+ writeFixtureFile(t, filepath.Join(d, id+".pdf"), "%PDF-1.4\n")
+ if text != "" {
+ writeFixtureFile(t, filepath.Join(d, id+".md"), text)
+ }
+ }
+ pub := func(publisher, date, link string, reporters ...string) map[string]interface{} {
+ return map[string]interface{}{"publisher": publisher, "date_published": date,
+ "link": link, "reporters": reporters}
+ }
+ meta := func(title, typ, created string, programs []string, pubs ...map[string]interface{}) map[string]interface{} {
+ return map[string]interface{}{"title": title, "type": typ,
+ "document": map[string]interface{}{"creator": "NSA", "date_created": created, "programs": programs},
+ "publication": pubs}
+ }
+ zebra := []string{"ZEBRA"}
+ doc("", "n1", meta("Report one", "REPORTED", "2010-03-01", zebra,
+ pub("The Intercept", "2014-01-01", "https://example.org/a", "Glenn Greenwald"),
+ pub("the intercept", "2016-01-01", "https://example.org/b", "glenn greenwald")),
+ "alpha gamma\n\n(S//SI) a secret line\n(S/SI) the same marking, spelled by OCR\n(U) plain\n")
+ doc("", "n2", meta("Report two", "REPORTED", "2011-03-01", zebra,
+ pub("The Intercept", "2014-02-01", "https://example.org/c"),
+ pub("Der Spiegel", "2014-02-02", "https://example.org/f")),
+ "alpha\n\n(TS//SI//NF) top line\n")
+ doc("", "n3", meta("Report three", "REPORTED", "2012-03-01", nil,
+ pub("Der Spiegel", "2015-01-01", "https://example.org/d")), "beta gamma\n")
+ doc("", "n4", meta("Report four", "REPORTED", "2012-04-01", nil,
+ pub("Der Spiegel", "2015-01-01", "https://example.org/d")), "beta\n")
+ doc("FOIA", "f1", meta("Court filing", "FOIA", "2010-05-01", zebra),
+ "alpha\n\n(c) The Court shall issue an order.\n(s) another subsection\n")
+ doc("SIDtoday-archive", "s1", meta("Issue", "SIDTODAY", "2011-05-01", zebra), "(S) issue line\n")
+ a1 := meta("Null aka", "REPORTED", "", nil)
+ a1["aka_id"] = nil
+ doc("", "a1", a1, "")
+ // A date of no shape the archive reads: served, undated.
+ doc("", "d1", meta("Odd date", "REPORTED", "ODNI", nil, pub("Der Spiegel", "15 May 2015", "https://example.org/e")), "")
+ writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[]`)
+ return root
+}
+
+// matchingIDs runs the search engine and returns the matching ids, sorted.
+func matchingIDs(queries, fields, conjs []string) []string {
+ q, f, c := normalizeClauses(queries, fields, conjs)
+ var ids []string
+ scored, _ := corpus().executeSearch(searchParams{Queries: q, Fields: f, Conjs: c})
+ for _, s := range scored {
+ ids = append(ids, s.Doc.ID)
+ }
+ sort.Strings(ids)
+ return ids
+}
+
+// A row is one unit: its -terms are exclusions within it, whatever joins
+// it to the rows before. And an empty row starts nothing - a blank first
+// row with "OR x" under it is a search for x.
+func TestSearchRowsAndNegations(t *testing.T) {
+ withCorpus(t, writeConsistencyCorpus(t))
+ te2 := []string{"TE", "TE"}
+ for _, c := range []struct {
+ name string
+ queries, conjs []string
+ want []string
+ }{
+ {"inline negation", []string{"alpha -gamma"}, []string{"and"}, []string{"f1", "n2"}},
+ // alpha AND NOT (beta AND NOT gamma): no alpha document is lost
+ // for mentioning gamma.
+ {"negation inside NOT", []string{"alpha", "beta -gamma"}, []string{"and", "not"}, []string{"f1", "n1", "n2"}},
+ // alpha OR (beta AND NOT gamma): the exclusion stays inside its row.
+ {"negation inside OR", []string{"alpha", "beta -gamma"}, []string{"and", "or"}, []string{"f1", "n1", "n2", "n4"}},
+ {"blank first row, OR", []string{"", "alpha"}, []string{"and", "or"}, []string{"f1", "n1", "n2"}},
+ {"spaces first row, OR", []string{" ", "alpha"}, []string{"and", "or"}, []string{"f1", "n1", "n2"}},
+ {"blank first row, OR nothing", []string{"", "zzzzunmatched"}, []string{"and", "or"}, nil},
+ {"blank first row, NOT", []string{"", "gamma"}, []string{"and", "not"}, []string{"a1", "d1", "f1", "n2", "n4", "s1"}},
+ {"only exclusions", []string{"-gamma -alpha"}, []string{"and"}, []string{"a1", "d1", "n4", "s1"}},
+ } {
+ if got := matchingIDs(c.queries, te2[:len(c.queries)], c.conjs); !reflect.DeepEqual(got, c.want) {
+ t.Errorf("%s: %q %v matched %v, want %v", c.name, c.queries, c.conjs, got, c.want)
+ }
+ }
+ // The scorer and the highlighter read the same rows: an excluded term
+ // is neither rewarded nor marked.
+ if got := positiveTerms([]string{"alpha -gamma", "", "beta"}, []string{"and", "and", "not"}); !reflect.DeepEqual(got, []string{"alpha"}) {
+ t.Errorf("positiveTerms = %q, want [alpha]", got)
+ }
+ if term, _ := snippetAnchor([]string{"-gamma", "alpha -beta"}, te2, []string{"and", "and"}); term != "alpha" {
+ t.Errorf("snippetAnchor = %q, want alpha", term)
+ }
+}
+
+// "+ Add row" and "remove" re-run the search: on the same rows, meaning the
+// same thing, as the Search button would.
+func TestRowEditsSearchTheSameRows(t *testing.T) {
+ withCorpus(t, writeConsistencyCorpus(t))
+ rows := "do_search=1&fqv=alpha+-gamma&fqf=TE&fqc=and"
+ want := resultTotal(t, "/search?action=search&"+rows, foiaOn)
+ if want != 2 {
+ t.Fatalf("search for alpha -gamma found %d, want 2", want)
+ }
+ for _, action := range []string{"add", "remove-1"} {
+ path := "/search?action=" + action + "&" + rows + "&fqv=beta&fqf=TE&fqc=or"
+ if action == "add" {
+ path = "/search?action=add&" + rows
+ }
+ if got := resultTotal(t, path, foiaOn); got != want {
+ t.Errorf("action=%s: %d results, the same rows searched find %d", action, got, want)
+ }
+ }
+}
+
+// The type filter resubmits the search the page shows, not the request: a
+// removed row stays removed, and the chosen sort is kept.
+func TestTypeFilterFormCarriesTheEffectiveSearch(t *testing.T) {
+ withCorpus(t, writeConsistencyCorpus(t))
+ body := get(t, "/search?do_search=1&action=remove-1&sort=title"+
+ "&fqv=alpha&fqf=TE&fqc=and&fqv=removedrow&fqf=TE&fqc=or", foiaOn).Body.String()
+ if !strings.Contains(body, `class="type-filter"`) {
+ t.Fatal("no type filter on a result set holding two types")
+ }
+ if strings.Contains(body, "removedrow") {
+ t.Error("the removed row is still in the page - the type filter would bring it back")
+ }
+ if !strings.Contains(body, `name="sort" value="title"`) {
+ t.Error("the type filter drops the chosen sort")
+ }
+}
+
+// /api/search runs the search page's pipeline, type filter included. The
+// API has no preference: it matches the page of a reader who includes FOIA
+// releases.
+func TestAPISearchMatchesThePage(t *testing.T) {
+ withCorpus(t, writeConsistencyCorpus(t))
+ for qs, want := range map[string]int{
+ "q=alpha": 3,
+ "q=alpha&type=REPORTED": 2,
+ "q=alpha&type=FOIA&type=REPORTED": 3,
+ "q=alpha&type=reported": 2,
+ "q=zebra&coll=news": 2,
+ "q=zebra&coll=foia&type=FOIA": 1,
+ "do_search=1&fqv=alpha&fqf=TE&fqc=and&fqv=beta+-gamma&fqf=TE&fqc=or": 4,
+ } {
+ page := resultTotal(t, "/search?"+qs, foiaOn)
+ var api struct{ Total int }
+ if err := json.Unmarshal(get(t, "/api/search?"+qs).Body.Bytes(), &api); err != nil {
+ t.Fatalf("%s: %v", qs, err)
+ }
+ if page != want || api.Total != want {
+ t.Errorf("%s: the page finds %d, the API %d, want %d", qs, page, api.Total, want)
+ }
+ }
+}
+
+// Every count in this corpus is the number its page or search returns.
+func TestConsistencyCorpusCounts(t *testing.T) {
+ withCorpus(t, writeConsistencyCorpus(t))
+
+ // Facets, with journalist and publisher spellings that differ by case
+ // across one document's publications.
+ for ci := range categories {
+ c := &categories[ci]
+ for val, want := range c.Facet(corpus()) {
+ if got := len(corpus().categoryMatches(c.Slug, val)); got != want {
+ t.Errorf("%s=%q: facet says %d, the page lists %d", c.Slug, val, want, got)
+ }
+ }
+ }
+ if js := corpus().archive.Facets["journalist"]; len(js) != 1 || js["Glenn Greenwald"] != 1 {
+ t.Errorf("journalists = %v, want Glenn Greenwald once", js)
+ }
+ if n := corpus().archive.Facets["publisher"]["The Intercept"]; n != 2 {
+ t.Errorf("The Intercept: %d documents, want 2 (%v)", n, corpus().archive.Facets["publisher"])
+ }
+
+ // The value-page timelines and refine links, scoped to a collection:
+ // f1 (FOIA) shares 2010 with n1 and s1 (SIDtoday) shares 2011 with n2,
+ // so an unscoped bar link finds more than its bar counts.
+ for _, p := range []string{"/browse/program/ZEBRA?coll=news", "/browse/program/ZEBRA"} {
+ if n := followLinks(t, p, reBarLink, "timeline bar"); n < 2 {
+ t.Errorf("%s: %d timeline bars checked, want the doc and pub timelines", p, n)
+ }
+ }
+
+ // /explore/outlets: rows, cells and the per-year footer.
+ for coll, table := range corpus().outletTables {
+ for _, row := range table.Rows {
+ if got := resultTotal(t, row.URL); got != row.Total {
+ t.Errorf("outlets[%q] %s: total %d, its page lists %d", coll, row.Name, row.Total, got)
+ }
+ for _, cell := range row.Cells {
+ if cell.URL != "" && resultTotal(t, cell.URL) != cell.Docs {
+ t.Errorf("outlets[%q] %s: a cell says %d, its search finds %d", coll, row.Name, cell.Docs, resultTotal(t, cell.URL))
+ }
+ }
+ }
+ // The footer adds up its column: a document several outlets
+ // published counts once for each of them.
+ for i, y := range table.Years {
+ sum := 0
+ for _, row := range table.Rows {
+ sum += row.Cells[i].Docs
+ }
+ if table.Totals[i] != sum {
+ t.Errorf("outlets[%q] %s: footer %d, its column adds up to %d", coll, y, table.Totals[i], sum)
+ }
+ }
+ }
+ cell := func(outlet, year string) (docs, footer int) {
+ table := corpus().outletTables[""]
+ for i, y := range table.Years {
+ if y != year {
+ continue
+ }
+ footer = table.Totals[i]
+ for _, row := range table.Rows {
+ if row.Name == outlet {
+ docs = row.Cells[i].Docs
+ }
+ }
+ }
+ return docs, footer
+ }
+ // n2: The Intercept and Der Spiegel both published it in 2014, so it
+ // counts for both outlets and twice in the year's footer (n1 is the
+ // third: The Intercept, 2014).
+ if spiegel, footer := cell("Der Spiegel", "2014"); spiegel != 1 || footer != 3 {
+ t.Errorf("2014: Der Spiegel cell %d, footer %d - want 1 and 3 (n2 once per outlet)", spiegel, footer)
+ }
+ if intercept, _ := cell("The Intercept", "2014"); intercept != 2 {
+ t.Errorf("2014: The Intercept cell %d, want 2", intercept)
+ }
+ intercept := 0
+ for _, row := range corpus().outletTables[""].Rows {
+ if row.Name == "The Intercept" {
+ intercept = row.Total
+ }
+ }
+ if intercept != 2 {
+ t.Errorf("The Intercept's outlet total = %d, want 2 documents (n1 counts once for its two years)", intercept)
+ }
+
+ // /explore/portions: lower-case list labels are not markings, and every
+ // row's document count is what its search finds.
+ for _, l := range corpus().portionStats.Levels {
+ if l.Name == "C" {
+ t.Errorf("a CONFIDENTIAL row from the court filing's (c) labels: %+v", l)
+ }
+ }
+ rows := append(append(append([]portionRow(nil), corpus().portionStats.Levels...), corpus().portionStats.Markings...), corpus().portionStats.Controls...)
+ if len(rows) < 6 {
+ t.Fatalf("only %d portion rows", len(rows))
+ }
+ for _, r := range rows {
+ if got := resultTotal(t, string(r.SearchURL)); got != r.Docs {
+ t.Errorf("portion row %s: %d documents, its search finds %d (%s)", r.Name, r.Docs, got, r.SearchURL)
+ }
+ }
+ if got := corpus().docIndex["n1"].PortionMarkings; !reflect.DeepEqual(got, []string{"S//SI", "U"}) {
+ t.Errorf("n1 markings = %v, want [S//SI U]", got)
+ }
+
+ // A malformed date is no date: no "ODNI" year that no year page opens.
+ if d := corpus().docIndex["d1"]; d == nil || d.DocMeta.DateCreated != "" || d.Publication[0].DatePublished != "" {
+ t.Errorf("d1's malformed dates were kept: %+v", d)
+ }
+ if _, ok := corpus().archive.Facets["docdate"]["ODNI"]; ok {
+ t.Error(`"ODNI" counted as a document year`)
+ }
+
+ // aka_id: null is no akas, not a reason to drop the document.
+ if corpus().docIndex["a1"] == nil {
+ t.Error(`a document with "aka_id": null was not loaded`)
+ }
+}
+
+// A corpus for the counts that turn on WHICH publication entry carries a
+// date or a byline, and on the reader's collection scope. Every document
+// says "alpha".
+//
+// p1 news 2010-03 Outlet X 2013 (Able), Outlet Y 2014 (Baker)
+// p2 news 2010-06 Outlet X 2014 (Able) people: Clerk
+// p3 news 2011-01 Outlet X 2014 (Baker)
+// p4 news 2012-03 Outlet Y 2014
+// p5 news 2012-09 Outlet Y 2014
+// p6 news 2013-02 Outlet Y 2014, Partner Z 2014 (its own story; reporting only)
+// f1 foia 2009-07 Outlet Y 2014 (a story of its own) people: Docket
+func writeEntryCorpus(t *testing.T) string {
+ t.Helper()
+ root := t.TempDir()
+ pub := func(publisher, date string, reporters ...string) map[string]interface{} {
+ return map[string]interface{}{"publisher": publisher, "date_published": date, "reporters": reporters}
+ }
+ doc := func(dir, id, typ, created string, people []string, pubs ...map[string]interface{}) {
+ b, err := json.Marshal(map[string]interface{}{"title": "Document " + id, "type": typ,
+ "document": map[string]interface{}{"creator": "NSA", "date_created": created, "people": people},
+ "publication": pubs})
+ if err != nil {
+ t.Fatal(err)
+ }
+ d := filepath.Join(root, dir, id)
+ writeFixtureFile(t, filepath.Join(d, "metadata.json"), string(b))
+ writeFixtureFile(t, filepath.Join(d, id+".pdf"), "%PDF-1.4\n")
+ writeFixtureFile(t, filepath.Join(d, id+".md"), "alpha\n")
+ }
+ doc("", "p1", "REPORTED", "2010-03-01", nil,
+ pub("Outlet X", "2013-05-01", "Able, Ann"), pub("Outlet Y", "2014-02-01", "Baker, Bob"))
+ doc("", "p2", "REPORTED", "2010-06-01", []string{"Clerk, Carol"}, pub("Outlet X", "2014-03-01", "Able, Ann"))
+ doc("", "p3", "REPORTED", "2011-01-01", nil, pub("Outlet X", "2014-04-01", "Baker, Bob"))
+ doc("", "p4", "REPORTED", "2012-03-01", nil, pub("Outlet Y", "2014-05-01"))
+ doc("", "p5", "REPORTED", "2012-09-01", nil, pub("Outlet Y", "2014-06-01"))
+ story := func(p map[string]interface{}, title, link string) map[string]interface{} {
+ p["title"], p["link"] = title, link
+ return p
+ }
+ partner := story(pub("Partner Z", "2014-07-02"), "Partner story", "https://example.org/partner")
+ partner["reporting_only"] = true
+ doc("", "p6", "REPORTED", "2013-02-01", nil, pub("Outlet Y", "2014-07-01"), partner)
+ doc("FOIA", "f1", "FOIA", "2009-07-01", []string{"Docket, Dan"},
+ story(pub("Outlet Y", "2014-08-01"), "FOIA story", "https://example.org/foia-story"))
+ writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[]`)
+ return root
+}
+
+// Counts that turn on a publication entry's date or byline, or on the
+// reader's scope, are still the numbers their links return.
+func TestCountsTiedToEntriesAndScope(t *testing.T) {
+ withCorpus(t, writeEntryCorpus(t))
+
+ // A publication-year page files a document under its first publication,
+ // which no search asks for: p1, first published in 2013, would join the
+ // 2010 bar's search for 2014. So those bars carry no link. The
+ // document-year page's publication bars do, and they are exact.
+ body := get(t, "/browse/published/2014").Body.String()
+ if !strings.Contains(body, "By document date") || strings.Contains(body, `[^<]*(\d+)`)
+
+// checkCategoryBoxes holds the number on each category box in the browse
+// nav to the rows of the list its click opens, for the reader with these
+// cookies.
+func checkCategoryBoxes(t *testing.T, cookies ...*http.Cookie) int {
+ t.Helper()
+ boxes := reCategoryBox.FindAllStringSubmatch(get(t, "/browse", cookies...).Body.String(), -1)
+ for _, m := range boxes {
+ slug, shown := m[1], mustAtoi(t, m[2])
+ list := get(t, "/browse/"+slug, cookies...).Body.String()
+ if rows := strings.Count(list, `href="/browse/`+slug+`/`); rows != shown {
+ t.Errorf("the %s box says %d, its list has %d rows (%d cookies)", slug, shown, rows, len(cookies))
+ }
+ }
+ return len(boxes)
+}
+
+// -check reads the load counters, so the glossary's problems must still be
+// in them when it does.
+func TestGlossaryWarningsReachCheck(t *testing.T) {
+ root := writeConsistencyCorpus(t)
+ withCorpus(t, root)
+ clean := loadWarnings
+ writeFixtureFile(t, filepath.Join(root, "glossary.json"), `{not json`)
+ loadData()
+ if loadWarnings != clean+1 {
+ t.Errorf("malformed glossary.json: %d warnings, want %d", loadWarnings, clean+1)
+ }
+}
+
+// A 404 under /files must not carry the files' public cache policy.
+func TestMissingFileIsNotCached(t *testing.T) {
+ w := get(t, "/files/prism-slides/no-such-file.pdf")
+ if w.Code != http.StatusNotFound {
+ t.Fatalf("status %d, want 404", w.Code)
+ }
+ if cc := w.Header().Get("Cache-Control"); strings.Contains(cc, "public") {
+ t.Errorf("404 carries Cache-Control %q", cc)
+ }
+}
+
+// -dev serves the on-disk assets so edits show without a restart: no
+// startup ETag may pin the old content.
+func TestDevModeStaticRevalidatesEdits(t *testing.T) {
+ buildFixture(t)
+ devMode = true
+ initAssets()
+ t.Cleanup(func() {
+ devMode = false
+ initAssets()
+ })
+ if staticHashes != nil {
+ t.Errorf("dev mode built %d static hashes", len(staticHashes))
+ }
+ w := httptest.NewRecorder()
+ handleStatic(w, httptest.NewRequest("GET", "/static/style.css", nil))
+ if w.Code != http.StatusOK || w.Header().Get("ETag") != "" || w.Header().Get("Last-Modified") == "" {
+ t.Errorf("dev static: status %d, ETag %q, Last-Modified %q - want 200, none, the file's",
+ w.Code, w.Header().Get("ETag"), w.Header().Get("Last-Modified"))
+ }
+}
+
+// What the search page offers to pin is what the pinned page can list.
+func TestPinnableSearchesAreListed(t *testing.T) {
+ h, _ := buildFixture(t)
+ if label, ok := corpus().searchLabel(mustQuery(t, "/search?do_search=1&fqv=&fqf=TE&fqc=and&date_doc_start=2013&date_doc_end=2013")); !ok || label != "all documents · (doc 2013)" {
+ t.Errorf("date-only search label = %q, %v", label, ok)
+ }
+ if strings.Contains(get(t, "/search?do_search=1&fqv=&fqf=TE&fqc=and").Body.String(), "pin-search-btn") {
+ t.Error("a search with no terms and no filters offers a pin")
+ }
+ if !strings.Contains(get(t, "/search?q=date:2013").Body.String(), "pin-search-btn") {
+ t.Error("a date-only search offers no pin")
+ }
+
+ pin := func(qs string) *httptest.ResponseRecorder {
+ r := httptest.NewRequest("POST", "/pinsearch", strings.NewReader(url.Values{"qs": {qs}, "back": {"/pinned"}}.Encode()))
+ r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ return record(h, r)
+ }
+ if w := pin("do_search=1&fqc=and&fqf=TE&fqv="); w.Code != http.StatusBadRequest {
+ t.Errorf("pinning an empty search: status %d, want 400", w.Code)
+ }
+ w := pin("date_doc_end=2013&date_doc_start=2013&do_search=1&fqc=and&fqf=TE&fqv=")
+ if w.Code != http.StatusSeeOther {
+ t.Fatalf("pinning a date-only search: status %d", w.Code)
+ }
+ body := get(t, "/pinned", w.Result().Cookies()...).Body.String()
+ if !strings.Contains(html.UnescapeString(body), "all documents · (doc 2013)") {
+ t.Error("the pinned date-only search is not listed on /pinned")
+ }
+}
diff --git a/tests/corpus_test.go b/tests/corpus_test.go
new file mode 100644
index 0000000..6bd36d0
--- /dev/null
+++ b/tests/corpus_test.go
@@ -0,0 +1,199 @@
+package main
+
+import (
+ "html"
+ "net/http"
+ "os"
+ "reflect"
+ "regexp"
+ "slices"
+ "strings"
+ "testing"
+)
+
+// Checks against a REAL corpus. Point $SNOWCHIVE_BENCH_DATA at one, as for
+// the page benchmarks, and run: go test -run TestCorpus -v
+//
+// The fixtures are a handful of documents, each built to show one thing. A
+// count and its link are computed by different code, and whether the two
+// agree turns on what the documents happen to hold: a second publication in
+// a later year, a byline on only one of them, a value found only in FOIA
+// releases. The disagreements these tests walk the archive for were all
+// found on the full corpus and on no fixture.
+func TestCorpus(t *testing.T) {
+ root := os.Getenv("SNOWCHIVE_BENCH_DATA")
+ if root == "" {
+ t.Skip("set SNOWCHIVE_BENCH_DATA to a corpus directory to check a real corpus")
+ }
+ withCorpus(t, root)
+ t.Run("counts", corpusCounts)
+ t.Run("FOIA links", corpusFOIALinks)
+ t.Run("inflections", corpusInflections)
+ t.Run("snippets", corpusSnippets)
+ t.Run("index", corpusIndex)
+}
+
+// corpusCounts follows the counted links of the loaded corpus, as a reader
+// who leaves FOIA releases out and as one who includes them.
+func corpusCounts(t *testing.T) {
+ // The pages whose bars and refine links are followed: the year pages of
+ // both date categories, the three commonest values of every other
+ // category, each cell of the outlets table (a publisher's page for one
+ // publication year) and searches under each kind of date range.
+ var pages []string
+ for ci := range categories {
+ c := &categories[ci]
+ n := 3
+ if c.isDate() {
+ n = -1
+ }
+ for _, v := range rankCounts(c.Facet(corpus()), n) {
+ pages = append(pages, categoryValueURL(c.Slug, v))
+ }
+ }
+ for _, row := range corpus().outletTables[""].Rows {
+ for _, cell := range row.Cells {
+ if cell.URL != "" {
+ pages = append(pages, cell.URL)
+ }
+ }
+ }
+ pages = append(pages,
+ "/search?q=nsa",
+ "/search?q=nsa+date%3A2012-06..2013-06",
+ "/search?q=nsa+published%3A2016",
+ "/search?q=published%3A2014",
+ "/search?do_search=1&fqv=nsa&fqf=TE&fqc=and&date_pub_start=2015-01-01&date_pub_end=2016-06-30",
+ "/search?do_search=1&fqv=nsa&fqf=TE&fqc=and&fqv=gchq&fqf=TE&fqc=or&date_pub_start=2016&date_pub_end=2016",
+ )
+
+ for name, reader := range map[string][]*http.Cookie{"FOIA left out": nil, "FOIA included": {foiaOn}} {
+ t.Run(name, func(t *testing.T) {
+ bars, refines := 0, 0
+ for _, p := range pages {
+ bars += followLinks(t, p, reBarLink, "bar", reader...)
+ refines += followLinks(t, p, reFacetLink, "refine link", reader...)
+ }
+ if bars == 0 || refines == 0 {
+ t.Errorf("nothing exercised: %d bars, %d refine links", bars, refines)
+ }
+ boxes := checkCategoryBoxes(t, reader...)
+ t.Logf("%d pages: %d bars, %d refine links and %d category boxes checked", len(pages), bars, refines, boxes)
+ })
+ }
+}
+
+// corpusFOIALinks opens the page of every tenth FOIA release as a reader
+// who leaves FOIA out, and follows each category value it links: the page
+// behind the link has to list the release.
+func corpusFOIALinks(t *testing.T) {
+ reValueLink := regexp.MustCompile(`href="(/browse/[a-z]+/[^"]+)"`)
+ links := 0
+ for i, d := range corpus().collectionDocs[collectionFOIA] {
+ if i%10 != 0 {
+ continue
+ }
+ // The release's own metadata: the cards under it are other documents.
+ page, _, _ := strings.Cut(get(t, docPath(d.ID)).Body.String(), `class="related-section"`)
+ for _, m := range reValueLink.FindAllStringSubmatch(page, -1) {
+ link := html.UnescapeString(m[1])
+ links++
+ if !strings.Contains(get(t, withQuery(link, "view=table")).Body.String(), `href="`+docPath(d.ID)+`"`) {
+ t.Errorf("%s links %s, which does not list it", d.ID, link)
+ }
+ }
+ }
+ if links == 0 {
+ t.Error("no FOIA release with a value link to follow")
+ }
+ t.Logf("%d value links followed", links)
+}
+
+// A word in -ies or -ied stems as the word in -y it is a form of, whatever
+// rules that word goes through - checked over every such pair the corpus
+// holds, since a suffix rule that reads one of the two forms and not the
+// other splits them silently ("capability" from "capabilities").
+func corpusInflections(t *testing.T) {
+ vocab := map[string]bool{}
+ for i := range corpus().archive.Docs {
+ for _, w := range splitWords(corpus().archive.Docs[i].SearchText) {
+ vocab[w] = true
+ }
+ }
+ pairs := 0
+ for w := range vocab {
+ if len(w) < 5 || !(strings.HasSuffix(w, "ies") || strings.HasSuffix(w, "ied")) {
+ continue
+ }
+ base := w[:len(w)-3] + "y"
+ if !vocab[base] {
+ continue
+ }
+ pairs++
+ if stem(w) != stem(base) {
+ t.Errorf("stem(%q) = %q, stem(%q) = %q; want equal", w, stem(w), base, stem(base))
+ }
+ }
+ if pairs == 0 {
+ t.Error("the corpus holds no -ies/-ied word next to its -y form")
+ }
+ t.Logf("%d pairs checked in a vocabulary of %d words", pairs, len(vocab))
+}
+
+// corpusSnippets holds every document's snippets off the fold kept at load
+// to the ones folding its transcript again gives - the versions' lines sit
+// behind that fold in the same string - and a page of "all" cards to its
+// bound.
+func corpusSnippets(t *testing.T) {
+ for _, q := range []string{"nsa", "cable tapping", `"upstream collection"`, "tap*", "the"} {
+ for i := range corpus().archive.Docs {
+ d := &corpus().archive.Docs[i]
+ if got, want := d.bodySnippets(q, 3), getSnippets(d.FullText, q, 3); !reflect.DeepEqual(got, want) {
+ t.Fatalf("%s, %q: %q from the loaded fold, %q from the text", d.ID, q, got, want)
+ }
+ }
+ }
+ all := &http.Cookie{Name: "pref_per_page", Value: "all"}
+ for _, c := range categories {
+ for _, v := range rankCounts(c.Facet(corpus()), 1) {
+ if c.Facet(corpus())[v] <= maxCardsPerPage {
+ continue
+ }
+ body := get(t, categoryValueURL(c.Slug, v), all, foiaOn).Body.String()
+ if n := strings.Count(body, `class="doc-card"`); n != maxCardsPerPage {
+ t.Errorf("%s=%s: %d cards on a page of all, want %d", c.Slug, v, n, maxCardsPerPage)
+ }
+ }
+ }
+}
+
+// corpusIndex holds the shortcut a one-word all-fields clause takes - it
+// asks the document's entry in the index instead of walking its text - to
+// the walk, for a spread of the vocabulary over every document.
+func corpusIndex(t *testing.T) {
+ var vocab []string
+ for w := range corpus().tfidf.DocFreq {
+ vocab = append(vocab, w)
+ }
+ slices.Sort(vocab)
+ checked := 0
+ for i := 0; i < len(vocab); i += 131 {
+ w := vocab[i]
+ docs := 0
+ for di := range corpus().archive.Docs {
+ d := &corpus().archive.Docs[di]
+ _, indexed := d.terms[w]
+ if indexed != containsTokens(d.StemmedText, w) {
+ t.Fatalf("%q in %s: the index says %v, the all-fields stream the opposite", w, d.ID, indexed)
+ }
+ if indexed {
+ docs++
+ }
+ }
+ if docs != corpus().tfidf.DocFreq[w] {
+ t.Errorf("%q: in %d documents' entries, with a document frequency of %d", w, docs, corpus().tfidf.DocFreq[w])
+ }
+ checked++
+ }
+ t.Logf("%d terms checked over %d documents", checked, len(corpus().archive.Docs))
+}
diff --git a/tests/data_test.go b/tests/data_test.go
new file mode 100644
index 0000000..5b09ddf
--- /dev/null
+++ b/tests/data_test.go
@@ -0,0 +1,123 @@
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestAkaIDShape(t *testing.T) {
+ for _, s := range []string{
+ `{"title": "no aka_id key"}`,
+ `{"aka_id": []}`,
+ // null is "no akas": not a reason to drop the document.
+ `{"aka_id": null}`,
+ `{"aka_id": [{"id": "a"}, {"id": "b", "doc_source": "https://e.org/d", "doc_url": "https://e.org/b.pdf"}]}`,
+ } {
+ var d Document
+ if err := json.Unmarshal([]byte(s), &d); err != nil {
+ t.Errorf("%s: unexpected error: %v", s, err)
+ }
+ }
+
+ for _, s := range []string{`{"aka_id": "a"}`, `{"aka_id": {"id": "a"}}`, `{"aka_id": ["a", "b"]}`, `{"aka_id": [{"id": "a"}, 3]}`} {
+ var d Document
+ if err := json.Unmarshal([]byte(s), &d); err == nil {
+ t.Errorf("%s: decoded without an error", s)
+ }
+ }
+}
+
+func TestNormalizeAka(t *testing.T) {
+ d := Document{ID: "x", AkaID: []AkaEntry{
+ {ID: " first ", DocSource: " https://e.org/p "},
+ {ID: " "},
+ {ID: "FIRST"},
+ {ID: "second", DocURL: "https://e.org/s.pdf"},
+ }}
+ got := normalizeAka(&d)
+ if len(got) != 2 || got[0].ID != "first" || got[0].DocSource != "https://e.org/p" || got[1].ID != "second" {
+ t.Errorf("normalizeAka = %+v, want first (trimmed) and second, without the blank id and the repeat", got)
+ }
+}
+
+func writeTestFile(t *testing.T, path, content string) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestLoadVersions(t *testing.T) {
+ dir := t.TempDir()
+ v := filepath.Join(dir, "versions")
+ writeTestFile(t, filepath.Join(v, "lemonde-2013", "copy.pdf"), "%PDF-1.4")
+ writeTestFile(t, filepath.Join(v, "lemonde-2013", "copy.md"), "# Slide 12\n\nOnly in Le Monde's copy.")
+ writeTestFile(t, filepath.Join(v, "lemonde-2013", "images", "page-10.jpg"), "")
+ writeTestFile(t, filepath.Join(v, "lemonde-2013", "images", "page-2.jpg"), "")
+ writeTestFile(t, filepath.Join(v, "guardian", "copy.pdf"), "%PDF-1.4") // no transcript or images: still a version
+ writeTestFile(t, filepath.Join(v, "no-pdf", "copy.md"), "text only") // skipped: no PDF
+ writeTestFile(t, filepath.Join(v, "bad name", "copy.pdf"), "%PDF-1.4") // skipped: name
+ writeTestFile(t, filepath.Join(v, "loose.pdf"), "%PDF-1.4") // skipped: not a folder
+
+ var d Document
+ if err := json.Unmarshal([]byte(`{"publication": [
+ {"publisher": "Le Monde", "version": "lemonde-2013"},
+ {"publisher": "The Guardian"},
+ {"publisher": "Dagbladet", "version": " lemonde-2013 "},
+ {"publisher": "Nobody", "version": "missing"}
+ ]}`), &d); err != nil {
+ t.Fatal(err)
+ }
+ d.ID, d.Dir = "prism", dir
+ corpus().loadVersions(&d)
+
+ var slugs []string
+ for _, v := range d.Versions {
+ slugs = append(slugs, v.Slug)
+ }
+ if got := strings.Join(slugs, ","); got != "guardian,lemonde-2013" {
+ t.Fatalf("versions %q, want guardian,lemonde-2013", got)
+ }
+
+ lm := d.VersionFor("lemonde-2013")
+ if lm.PDFFile != "copy.pdf" || !strings.Contains(lm.FullText, "Only in Le Monde") || lm.HTMLContent == "" {
+ t.Errorf("lemonde-2013 not fully loaded: pdf %q, text %q, html %q", lm.PDFFile, lm.FullText, lm.HTMLContent)
+ }
+ if got := strings.Join(lm.Images, ","); got != "page-2.jpg,page-10.jpg" {
+ t.Errorf("images %q, want page-2.jpg,page-10.jpg (page order)", got)
+ }
+ if got := fmt.Sprint(lm.PubIndex); got != "[0 2]" {
+ t.Errorf("PubIndex %s, want [0 2]", got)
+ }
+ if got := d.Publication[2].Version; got != "lemonde-2013" {
+ t.Errorf("publication version not trimmed: %q", got)
+ }
+ if u := d.UnlinkedVersions(); len(u) != 1 || u[0].Slug != "guardian" {
+ t.Errorf("unlinked versions %+v, want only guardian", u)
+ }
+ if d.VersionFor("missing") != nil || d.VersionFor("") != nil {
+ t.Error("VersionFor returned a version that doesn't exist")
+ }
+}
+
+// TestBodyFormsAreTheCombinedTails: loadData slices FoldedFull and
+// StemmedFull out of the ends of SearchText and StemmedText to save memory;
+// they must still be exactly what setBodySearchText builds.
+func TestBodyFormsAreTheCombinedTails(t *testing.T) {
+ buildFixture(t)
+ for i := range corpus().archive.Docs {
+ d := &corpus().archive.Docs[i]
+ c := *d
+ setBodySearchText(&c)
+ if c.FoldedFull != d.FoldedFull || c.StemmedFull != d.StemmedFull {
+ t.Errorf("%s: body forms differ from a fresh build", d.ID)
+ }
+ }
+}
diff --git a/tests/fixture_test.go b/tests/fixture_test.go
new file mode 100644
index 0000000..1b39265
--- /dev/null
+++ b/tests/fixture_test.go
@@ -0,0 +1,295 @@
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "image"
+ "image/jpeg"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+)
+
+// A small corpus on disk plus a running mux: the harness every handler test
+// uses. The fixture covers the shapes the handlers branch on - one document
+// per collection, a SIDtoday issue with a series, another published copy in
+// versions/, a joint investigation with a reporting-only partner, and a
+// directory with no PDF (which the loader must skip).
+//
+// Built once for the package: loadData writes package globals, so tests
+// share one corpus rather than racing to replace it.
+
+var (
+ fixtureOnce sync.Once
+ fixtureMux http.Handler
+ fixtureDir string
+)
+
+type fixtureDoc struct {
+ id, collection string
+ meta map[string]interface{}
+ text string
+ pages []string
+ versions map[string][]string // slug -> page images
+ noPDF bool
+}
+
+// fixtureJPEG is a real 4x3 JPEG, so the loader can read each page image's size.
+var fixtureJPEG = func() string {
+ var b bytes.Buffer
+ if err := jpeg.Encode(&b, image.NewGray(image.Rect(0, 0, 4, 3)), nil); err != nil {
+ panic(err)
+ }
+ return b.String()
+}()
+
+func writeFixtureFile(t *testing.T, path, content string) {
+ t.Helper()
+ if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path, []byte(content), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestMain removes the shared fixture corpus once every test has run.
+func TestMain(m *testing.M) {
+ code := m.Run()
+ if fixtureDir != "" {
+ os.RemoveAll(fixtureDir)
+ }
+ os.Exit(code)
+}
+
+func buildFixture(t *testing.T) (http.Handler, string) {
+ t.Helper()
+ fixtureOnce.Do(func() {
+ root, err := os.MkdirTemp("", "snowchive-fixture")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, d := range fixtureDocs() {
+ dir := root
+ switch d.collection {
+ case collectionFOIA:
+ dir = filepath.Join(root, "FOIA")
+ case collectionArchive:
+ dir = filepath.Join(root, "SIDtoday-archive")
+ }
+ dir = filepath.Join(dir, d.id)
+ b, err := json.MarshalIndent(d.meta, "", " ")
+ if err != nil {
+ t.Fatal(err)
+ }
+ writeFixtureFile(t, filepath.Join(dir, "metadata.json"), string(b)+"\n")
+ if !d.noPDF {
+ writeFixtureFile(t, filepath.Join(dir, d.id+".pdf"), "%PDF-1.4\nfixture\n")
+ }
+ if d.text != "" {
+ writeFixtureFile(t, filepath.Join(dir, d.id+".md"), d.text)
+ }
+ for _, p := range d.pages {
+ writeFixtureFile(t, filepath.Join(dir, "images", p), fixtureJPEG)
+ }
+ for slug, imgs := range d.versions {
+ vdir := filepath.Join(dir, "versions", slug)
+ writeFixtureFile(t, filepath.Join(vdir, slug+".pdf"), "%PDF-1.4\n")
+ writeFixtureFile(t, filepath.Join(vdir, slug+".md"),
+ "# Copy\n\nA line only this copy has: unredacted FAIRVIEW detail.\n")
+ for _, p := range imgs {
+ writeFixtureFile(t, filepath.Join(vdir, "images", p), fixtureJPEG)
+ }
+ }
+ }
+ writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[
+ {"term": "XKEYSCORE", "definition": "A search system.", "aka": ["XKS"]}
+ ]`)
+
+ dataDir, projectDir = root, root
+ devMode = false
+ apiEnabled = true
+ initAssets()
+ if err := parseTemplates(); err != nil {
+ t.Fatalf("templates: %v", err)
+ }
+ loadData()
+ fixtureDir = root
+ fixtureMux = withBaseHeaders(newMux())
+ })
+ if fixtureMux == nil {
+ t.Fatal("fixture not built")
+ }
+ return fixtureMux, fixtureDir
+}
+
+func fixtureDocs() []fixtureDoc {
+ return []fixtureDoc{
+ {
+ id: "prism-slides",
+ collection: collectionNews,
+ text: "# PRISM\n\nCollection directly from the servers. XKEYSCORE ingests it.\n" +
+ "Upstream collection feeds the corpus, and analysts were collecting metadata\n" +
+ "from RT-RG dashboards. Müller-Maguhn reviewed the São Paulo cable tapping.\n",
+ pages: []string{"page-1.jpg", "page-2.jpg"},
+ versions: map[string][]string{"lemonde": {"page-1.jpg"}},
+ meta: map[string]interface{}{
+ "title": "PRISM Collection Slides",
+ "type": "REPORTED",
+ "description": "Slides describing PRISM collection.",
+ "tags": []string{"PRISM", "collection", "cable tapping"},
+ "pdf_sha256": strings.Repeat("a", 64),
+ "aka_id": []map[string]string{{"id": "prism-slides-guardian"}},
+ "document": map[string]interface{}{
+ "creator": "NSA", "date_created": "2013-04-01",
+ "programs": []string{"PRISM", "XKEYSCORE"},
+ "classification": []string{"TOP SECRET"}, "dissemination": []string{"NOFORN"},
+ "countries": []string{"United States", "Brazil"},
+ "people": []string{"Snowden, Edward", "Müller-Maguhn, Andy"},
+ "banner": "TOP SECRET//SI//NOFORN", "genre": "slides",
+ },
+ "publication": []map[string]interface{}{
+ {"publisher": "The Guardian", "title": "NSA Prism program",
+ "link": "https://example.org/prism", "date_published": "2013-06-06",
+ "reporters": []string{"Greenwald, Glenn"}},
+ {"publisher": "Le Monde", "link": "https://example.org/prism-fr",
+ "date_published": "2013-06-08", "version": "lemonde"},
+ {"publisher": "NRK", "link": "https://example.org/prism",
+ "date_published": "2013-06-06", "reporting_only": true},
+ },
+ },
+ },
+ {
+ // Shares the Guardian link with prism-slides, which is what
+ // makes the "published in the same report" section exist.
+ id: "prism-companion", collection: collectionNews,
+ text: "# Companion\n\nA second document from the same story.\n",
+ pages: []string{"page-1.jpg"},
+ meta: map[string]interface{}{
+ "title": "PRISM Companion Slide", "type": "REPORTED",
+ "description": "A second slide from the same story.",
+ "tags": []string{"PRISM"},
+ "document": map[string]interface{}{
+ "creator": "NSA", "date_created": "2013-04-02",
+ "programs": []string{"PRISM"},
+ },
+ "publication": []map[string]interface{}{
+ {"publisher": "The Guardian", "title": "NSA Prism program",
+ "link": "https://example.org/prism", "date_published": "2013-06-06",
+ "reporters": []string{"Greenwald, Glenn"}},
+ },
+ },
+ },
+ {
+ id: "2003-05-29_SIDToday_-_Fixture_Issue",
+ collection: collectionArchive,
+ text: "# SIDtoday\n\nThe analyst cockpit and other matters. Upstream tapping was\n" +
+ "discussed at length by the directorate, and the collection of that traffic\n" +
+ "began only the following year.\n",
+ pages: []string{"page-1.jpg"},
+ meta: map[string]interface{}{
+ "title": "A Day in the Life of a Fixture", "type": "SIDTODAY",
+ "description": "An issue of SIDtoday.",
+ "tags": []string{"analysis"},
+ "document": map[string]interface{}{
+ "creator": "NSA", "date_created": "2003-05-29",
+ "classification": []string{"UNCLASSIFIED"}, "distribution": []string{},
+ "series": map[string]interface{}{"name": "(U) A Day in the Life of...", "part": 1},
+ },
+ "publication": []map[string]interface{}{
+ {"publisher": "The Intercept", "link": "https://example.org/sidtoday/1",
+ "date_published": "2016-05-16"},
+ },
+ },
+ },
+ {
+ id: "FISC_-_fixture_order",
+ collection: collectionFOIA,
+ text: "# Order\n\nThe Court finds the application satisfies the statute.\n",
+ meta: map[string]interface{}{
+ "title": "Fixture FISC Order", "type": "FOIA",
+ "description": "A court order released under FOIA.",
+ "document": map[string]interface{}{
+ "creator": "FISC", "date_created": "2013-10-11",
+ "classification": []string{"SECRET"},
+ },
+ "publication": []map[string]interface{}{{"publisher": "ODNI", "link": "https://example.org/foia/1"}},
+ },
+ },
+ {
+ id: "undated-fixture",
+ collection: collectionNews,
+ text: "# Undated\n\nNo document date at all.\n",
+ meta: map[string]interface{}{
+ "title": "Undated Fixture", "type": "REPORTED",
+ "description": "A document with no date.",
+ "document": map[string]interface{}{"creator": "GCHQ"},
+ "publication": []map[string]interface{}{{"publisher": "The Intercept", "link": "https://example.org/undated"}},
+ },
+ },
+ {
+ id: "broken-no-pdf",
+ collection: collectionNews,
+ noPDF: true,
+ meta: map[string]interface{}{"title": "Broken", "type": "REPORTED"},
+ },
+ }
+}
+
+// get runs one request through the real routing table and header wrapper.
+func get(t *testing.T, path string, cookies ...*http.Cookie) *httptest.ResponseRecorder {
+ t.Helper()
+ mux, _ := buildFixture(t)
+ r := httptest.NewRequest("GET", path, nil)
+ for _, c := range cookies {
+ r.AddCookie(c)
+ }
+ w := httptest.NewRecorder()
+ mux.ServeHTTP(w, r)
+ return w
+}
+
+// newPost builds a form POST for the state-changing endpoints.
+func newPost(path string) *http.Request {
+ r := httptest.NewRequest("POST", path, strings.NewReader("back=/browse"))
+ r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ return r
+}
+
+func record(h http.Handler, r *http.Request) *httptest.ResponseRecorder {
+ w := httptest.NewRecorder()
+ h.ServeHTTP(w, r)
+ return w
+}
+
+// withCorpus loads another corpus for one test and puts the fixture back
+// afterwards. Only possible because loadData is idempotent - every build*
+// starts from empty (TestLoadDataIsIdempotent).
+func withCorpus(t *testing.T, root string) {
+ t.Helper()
+ withDirs(t, root, root)
+}
+
+// withDirs is withCorpus with the corpus and the project files apart, as
+// -root lays them out.
+func withDirs(t *testing.T, data, project string) {
+ t.Helper()
+ buildFixture(t) // make sure templates and the fixture path exist first
+ prev := dataDir
+ t.Cleanup(func() {
+ dataDir, projectDir = prev, prev
+ loadData()
+ })
+ dataDir, projectDir = data, project
+ loadData()
+}
+
+// httptest_NewGet is a plain GET request for the tests that drive a mux
+// directly rather than through get().
+func httptest_NewGet(path string) *http.Request {
+ return httptest.NewRequest("GET", path, nil)
+}
diff --git a/tests/flags_test.go b/tests/flags_test.go
new file mode 100644
index 0000000..647fcbd
--- /dev/null
+++ b/tests/flags_test.go
@@ -0,0 +1,167 @@
+package main
+
+// flags_test.go - what each command-line flag actually changes. The flags
+// are parsed in main(), which no test can call, so these exercise the state
+// the flags set and the behaviour that depends on it: the routes an
+// instance serves, the URLs it writes, the warnings it prints.
+
+import (
+ "encoding/json"
+ "net/http"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// -no-api: every /api/ path answers a JSON 403, and the UI stops offering
+// the API. The kill switch is one registration, so a future endpoint
+// inherits it - that is the property worth testing.
+func TestFlagNoAPI(t *testing.T) {
+ buildFixture(t)
+ prev := apiEnabled
+ t.Cleanup(func() { apiEnabled = prev; fixtureMux = withBaseHeaders(newMux()) })
+
+ apiEnabled = false
+ mux := withBaseHeaders(newMux())
+ for _, p := range []string{"/api/docs", "/api/doc/prism-slides", "/api/search?q=x",
+ "/api/facets", "/api/something-added-next-year"} {
+ w := record(mux, httptest_NewGet(p))
+ if w.Code != http.StatusForbidden {
+ t.Errorf("%s with -no-api: status %d, want 403", p, w.Code)
+ }
+ var body map[string]string
+ if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil || body["error"] == "" {
+ t.Errorf("%s: body %q is not a JSON error", p, w.Body.String())
+ }
+ if ct := w.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/json") {
+ t.Errorf("%s: Content-Type %q", p, ct)
+ }
+ }
+ // The visitor's own export is NOT part of the API and stays available.
+ if w := record(mux, httptest_NewGet("/pinned/export")); w.Code != 200 {
+ t.Errorf("/pinned/export with -no-api: %d", w.Code)
+ }
+}
+
+// -base: the absolute URLs in robots.txt and the sitemap follow
+// the canonical host instead of the request's Host header.
+func TestFlagBase(t *testing.T) {
+ buildFixture(t)
+ prev, prevMap := canonicalBase, corpus().canonicalSitemap
+ t.Cleanup(func() {
+ canonicalBase, corpus().canonicalSitemap = prev, prevMap
+ corpus().buildCanonicalSitemap()
+ })
+ canonicalBase = "https://archive.example.org"
+ corpus().buildCanonicalSitemap()
+
+ for _, c := range []struct{ path, want string }{
+ {"/robots.txt", "Sitemap: https://archive.example.org/sitemap.xml"},
+ {"/sitemap.xml", "https://archive.example.org/doc/prism-slides"},
+ } {
+ if body := get(t, c.path).Body.String(); !strings.Contains(body, c.want) {
+ t.Errorf("%s does not carry %q", c.path, c.want)
+ }
+ }
+ // A hostile Host header cannot reach a response once -base is set.
+ r := httptest_NewGet("/robots.txt")
+ r.Host = "evil.example.net"
+ if body := record(fixtureMux, r).Body.String(); strings.Contains(body, "evil") {
+ t.Errorf("the Host header leaked into robots.txt: %s", body)
+ }
+}
+
+// -base is validated, not trusted: anything but scheme://host is refused,
+// so the flag cannot produce URLs that do not resolve.
+func TestFlagBaseValidation(t *testing.T) {
+ for _, ok := range []string{"https://a.org", "http://a.org", "https://a.org:8443", "https://a.org/"} {
+ if _, err := parseCanonicalBase(ok); err != nil {
+ t.Errorf("%s rejected: %v", ok, err)
+ }
+ }
+ for _, bad := range []string{"", "a.org", "ftp://a.org", "https://", "https://a.org/path",
+ "https://a.org?q=1", "https://a.org#f", "https://user@a.org", "://x"} {
+ if _, err := parseCanonicalBase(bad); err == nil {
+ t.Errorf("%q accepted", bad)
+ }
+ }
+}
+
+// -check counts what it reports: the fixture holds one document the loader
+// refuses (no PDF) and one directory that looks like a document.
+func TestFlagCheckCounts(t *testing.T) {
+ buildFixture(t)
+ if loadErrors == 0 {
+ t.Error("the fixture's PDF-less document should be an error")
+ }
+ if loadErrors+loadWarnings == 0 {
+ t.Error("-check would exit 0 on a corpus with problems")
+ }
+}
+
+// -listen warns for anything but loopback, because binding elsewhere puts
+// the archive in front of the proxy instead of behind it.
+func TestFlagListenWarning(t *testing.T) {
+ before := loadWarnings
+ for _, quiet := range []string{"127.0.0.1:8080", "localhost:8080", "[::1]:8080", "127.0.0.53:80"} {
+ warnNonLoopback(quiet)
+ if loadWarnings != before {
+ t.Errorf("%s warned", quiet)
+ before = loadWarnings
+ }
+ }
+ for _, loud := range []string{"0.0.0.0:8080", ":8080", "192.168.1.10:8080", "[::]:8080"} {
+ warnNonLoopback(loud)
+ if loadWarnings == before {
+ t.Errorf("%s did not warn", loud)
+ }
+ before = loadWarnings
+ }
+}
+
+// -dev: templates come from disk and nothing is pre-rendered, so every page
+// must still render - the fallback path the pre-rendered blocks rely on.
+func TestFlagDevMode(t *testing.T) {
+ buildFixture(t)
+ prevDev, prevPre, prevTmpl := devMode, corpus().prerendered, tmplSet
+ t.Cleanup(func() {
+ devMode, corpus().prerendered, tmplSet = prevDev, prevPre, prevTmpl
+ initAssets()
+ })
+ devMode = true
+ initAssets() // templates/ and static/ from the working directory
+ corpus().prerenderPages()
+ if len(corpus().prerendered) != 0 {
+ t.Error("dev mode pre-rendered pages; it must render them live")
+ }
+ for _, p := range []string{"/", "/browse", "/browse/program", "/explore/ids", "/glossary",
+ "/articles", "/explore/terms", "/doc/prism-slides"} {
+ if w := get(t, p); w.Code != 200 {
+ t.Errorf("%s in dev mode: %d", p, w.Code)
+ }
+ }
+}
+
+// -root: the documents are read from its data/ and the files about the
+// archive from its project/.
+func TestRootDir(t *testing.T) {
+ buildFixture(t)
+ prevData, prevProject := dataDir, projectDir
+ t.Cleanup(func() {
+ dataDir, projectDir = prevData, prevProject
+ loadData()
+ })
+ root := t.TempDir()
+ writeFixtureFile(t, filepath.Join(root, "data", "doc-in-root", "metadata.json"), `{"title": "In the root"}`)
+ writeFixtureFile(t, filepath.Join(root, "data", "doc-in-root", "doc-in-root.pdf"), "%PDF-1.4\n")
+ writeFixtureFile(t, filepath.Join(root, "project", "glossary.json"), `[{"term": "INPROJECT", "definition": "Read from project/."}]`)
+ setRoot(root)
+ loadData()
+
+ if corpus().docIndex["doc-in-root"] == nil {
+ t.Error("the document in data/ was not loaded")
+ }
+ if corpus().glossaryLookup("INPROJECT") == nil {
+ t.Error("glossary.json in project/ was not loaded")
+ }
+}
diff --git a/tests/foia_test.go b/tests/foia_test.go
new file mode 100644
index 0000000..7e35aab
--- /dev/null
+++ b/tests/foia_test.go
@@ -0,0 +1,272 @@
+package main
+
+import (
+ "html"
+ "net/http"
+ "net/url"
+ "regexp"
+ "strconv"
+ "strings"
+ "testing"
+)
+
+// foiaOn is the cookie of a reader who opted in to FOIA releases.
+var foiaOn = &http.Cookie{Name: "pref_include_foia", Value: "1"}
+
+// In the consistency corpus "alpha" is in n1, n2 and the FOIA release f1.
+func TestSearchLeavesOutFOIAByDefault(t *testing.T) {
+ withCorpus(t, writeConsistencyCorpus(t))
+
+ if got := resultTotal(t, "/search?q=alpha"); got != 2 {
+ t.Errorf("q=alpha, no preference: %d results, want 2 (FOIA left out)", got)
+ }
+ body := get(t, "/search?q=alpha").Body.String()
+ if strings.Contains(body, "left out") {
+ t.Error("the page counts the FOIA releases it left out")
+ }
+ // The page's own links name the collections it searched, so a copied
+ // link shows the same results to a reader with the other preference.
+ if !strings.Contains(body, "coll=news&coll=sidtoday-archive") {
+ t.Error("the page's links do not carry the default collections")
+ }
+ if got := resultTotal(t, "/search?q=alpha&coll=news&coll=sidtoday-archive&coll=foia"); got != 3 {
+ t.Errorf("q=alpha in every collection: %d results, want 3", got)
+ }
+
+ if got := resultTotal(t, "/search?q=alpha", foiaOn); got != 3 {
+ t.Errorf("q=alpha with FOIA on: %d results, want 3", got)
+ }
+ if strings.Contains(get(t, "/search?q=alpha", foiaOn).Body.String(), "left out") {
+ t.Error("FOIA on, yet the page says something was left out")
+ }
+ // An explicit collection choice wins over the preference.
+ if got := resultTotal(t, "/search?q=alpha&coll=foia"); got != 1 {
+ t.Errorf("q=alpha&coll=foia, no preference: %d results, want 1", got)
+ }
+ if got := resultTotal(t, "/search?q=alpha&coll=news", foiaOn); got != 2 {
+ t.Errorf("q=alpha&coll=news with FOIA on: %d results, want 2", got)
+ }
+}
+
+// ZEBRA is on n1, n2, f1 (FOIA) and s1 (SIDtoday).
+func TestCategoryPagesLeaveOutFOIAByDefault(t *testing.T) {
+ withCorpus(t, writeConsistencyCorpus(t))
+
+ if got := resultTotal(t, "/browse/program/ZEBRA"); got != 3 {
+ t.Errorf("ZEBRA, no preference: %d documents, want 3", got)
+ }
+ if strings.Contains(get(t, "/browse/program/ZEBRA").Body.String(), "left out") {
+ t.Error("the page still carries a FOIA notice")
+ }
+ if got := resultTotal(t, "/browse/program/ZEBRA", foiaOn); got != 4 {
+ t.Errorf("ZEBRA with FOIA on: %d documents, want 4", got)
+ }
+ if strings.Contains(get(t, "/browse/program/ZEBRA", foiaOn).Body.String(), "left out") {
+ t.Error("FOIA on, yet the page says FOIA releases are left out")
+ }
+}
+
+// Every total in a category table is the number its row's link lists, for
+// either preference.
+func TestCategoryTotalsMatchTheirPages(t *testing.T) {
+ withCorpus(t, writeConsistencyCorpus(t))
+ reRow := regexp.MustCompile(`(?s)\s*([^<]+?)\s*.*?\s*
(\d+)
`)
+ for _, cookies := range [][]*http.Cookie{nil, {foiaOn}} {
+ for _, slug := range []string{"program", "agency", "docdate", "published", "publisher"} {
+ body := get(t, "/browse/"+slug, cookies...).Body.String()
+ _, body, _ = strings.Cut(body, "")
+ body, _, _ = strings.Cut(body, "")
+ rows := reRow.FindAllStringSubmatch(body, -1)
+ if len(rows) == 0 {
+ t.Fatalf("/browse/%s: no rows", slug)
+ }
+ for _, row := range rows {
+ want, _ := strconv.Atoi(row[3])
+ if strings.Contains(row[1], "coll=") {
+ t.Errorf("/browse/%s %s: row links with a collection scope: %s", slug, row[2], row[1])
+ }
+ if got := resultTotal(t, html.UnescapeString(row[1]), cookies...); got != want {
+ t.Errorf("/browse/%s %s (FOIA on: %v): total %d, its page lists %d", slug, row[2], cookies != nil, want, got)
+ }
+ }
+ }
+ }
+ off, on := get(t, "/browse/program").Body.String(), get(t, "/browse/program", foiaOn).Body.String()
+ if strings.Contains(off, "Totals leave out") || strings.Contains(off, "sort=foia") {
+ t.Error("FOIA off: the table still has a FOIA notice or column")
+ }
+ if !strings.Contains(on, "sort=foia") {
+ t.Error("FOIA on: the table has no FOIA column")
+ }
+}
+
+func TestRandomAndRelatedLeaveOutFOIA(t *testing.T) {
+ withCorpus(t, writeConsistencyCorpus(t))
+
+ sawFOIA := false
+ for i := 0; i < 300; i++ {
+ if d := corpus().randomDoc(false); d == nil || d.Collection == collectionFOIA {
+ t.Fatalf("randomDoc(false) = %v", d)
+ }
+ if corpus().randomDoc(true).Collection == collectionFOIA {
+ sawFOIA = true
+ }
+ }
+ if !sawFOIA {
+ t.Error("randomDoc(true) never picked the FOIA release in 300 tries")
+ }
+
+ corpus().archive.RelatedDocs["n2"] = []string{"f1", "n1"}
+ corpus().archive.RelatedDocs["f1"] = []string{"n1"}
+ body := get(t, "/doc/n2").Body.String()
+ if strings.Contains(body, `href="/doc/f1"`) || !strings.Contains(body, `href="/doc/n1"`) {
+ t.Error("/doc/n2, no preference: want n1 related and f1 left out")
+ }
+ if !strings.Contains(get(t, "/doc/n2", foiaOn).Body.String(), `href="/doc/f1"`) {
+ t.Error("/doc/n2 with FOIA on: f1 is not related")
+ }
+ if !strings.Contains(get(t, "/doc/f1").Body.String(), `href="/doc/n1"`) {
+ t.Error("/doc/f1: its related documents are gone")
+ }
+}
+
+// The home page counts the Snowden documents apart from the FOIA releases and
+// says how the preference treats them. The corpus: n1-n4, a1, d1 (news), s1
+// (SIDtoday archive), f1 (FOIA).
+func TestHomeStatesTheFOIAScope(t *testing.T) {
+ withCorpus(t, writeConsistencyCorpus(t))
+ body := get(t, "/").Body.String()
+ for _, want := range []string{"holds 7 published Snowden documents. 6 documents are", "1 documents are the NSA newsletter",
+ "also holds 1", "are left out by default but can be included via"} {
+ if !strings.Contains(body, want) {
+ t.Errorf("home page lacks %q", want)
+ }
+ }
+ if !strings.Contains(get(t, "/", foiaOn).Body.String(), "you have included via") {
+ t.Error("FOIA on: the home page does not say so")
+ }
+}
+
+func TestFOIAPreferenceIsSaved(t *testing.T) {
+ withCorpus(t, writeConsistencyCorpus(t))
+ if !strings.Contains(get(t, "/preferences").Body.String(), `name="include_foia"`) {
+ t.Error("/preferences has no FOIA checkbox")
+ }
+ if readPrefs(httptest_NewGet("/")).IncludeFOIA {
+ t.Error("FOIA releases are included by default")
+ }
+}
+
+// The form shows the scope the search ran with: FOIA left out by default
+// is an unchecked box, so submitting the form keeps it out.
+func TestSearchFormKeepsTheDefaultScope(t *testing.T) {
+ withCorpus(t, writeConsistencyCorpus(t))
+
+ checked := regexp.MustCompile(`value="foia" checked`)
+ if checked.MatchString(get(t, "/search?q=alpha").Body.String()) {
+ t.Error("FOIA left out, yet its box is checked")
+ }
+ if !checked.MatchString(get(t, "/search?q=alpha", foiaOn).Body.String()) {
+ t.Error("FOIA on, yet its box is not checked")
+ }
+ if got := resultTotal(t, "/search?q=alpha&coll=news&coll=sidtoday-archive&do_search=1"); got != 2 {
+ t.Errorf("the form's resubmit finds %d, want 2", got)
+ }
+}
+
+// A FOIA release's metadata links name every collection, so its value
+// pages list it under the default scope.
+func TestFOIADocLinksIncludeFOIA(t *testing.T) {
+ buildFixture(t)
+ // Every value link printed for the release - on its own page, on its
+ // card and its table row in the FOIA tab, in a search result - opens a
+ // page that lists the release.
+ const id = "FISC_-_fixture_order"
+ reValueLink := regexp.MustCompile(`href="(/browse/[a-z]+/[^"]+)"`)
+ for _, page := range []string{"/doc/" + id, "/browse?collection=foia", "/browse?collection=foia&view=table", "/search?q=court&coll=foia"} {
+ links := reValueLink.FindAllStringSubmatch(get(t, page).Body.String(), -1)
+ if len(links) < 2 {
+ t.Fatalf("%s: %d value links, want the release's agency and classification at least", page, len(links))
+ }
+ for _, m := range links {
+ link := html.UnescapeString(m[1])
+ if !strings.Contains(get(t, withQuery(link, "view=table")).Body.String(), `href="/doc/`+id+`"`) {
+ t.Errorf("%s links %s, which does not list the release", page, link)
+ }
+ }
+ }
+ // /explore/classification counts every collection, and so do its links.
+ for _, m := range regexp.MustCompile(`href="(/browse/[^"]+)"`).FindAllStringSubmatch(get(t, "/explore/classification").Body.String(), -1) {
+ if !strings.Contains(m[1], "coll=foia") {
+ t.Errorf("/explore/classification links %s without its scope", m[1])
+ }
+ }
+
+ body := get(t, "/doc/FISC_-_fixture_order").Body.String()
+ m := regexp.MustCompile(`href="(/browse/agency/FISC[^"]*)"`).FindStringSubmatch(body)
+ if m == nil || !strings.Contains(m[1], "coll=foia") {
+ t.Fatalf("agency link %v does not include FOIA", m)
+ }
+ if got := resultTotal(t, html.UnescapeString(m[1])); got != 1 {
+ t.Errorf("the agency link lists %d documents, want 1", got)
+ }
+ if strings.Contains(get(t, "/doc/FISC_-_fixture_order", foiaOn).Body.String(), "/browse/agency/FISC?") {
+ t.Error("FOIA on: the links need no scope")
+ }
+}
+
+// On a publisher page scoped to a publication year, the links built from
+// its documents carry that year.
+func TestPublisherYearLinksCarryTheYear(t *testing.T) {
+ buildFixture(t)
+ body := get(t, "/browse/publisher/The%20Intercept?year=2016").Body.String()
+ if !strings.Contains(body, "date_pub_start=2016") {
+ t.Error("no link on the year-scoped page carries the publication year")
+ }
+}
+
+// collScope is the one place a page's collection scope is decided: FOIA
+// releases are opt-in when the URL names no collection, an explicit coll=
+// wins, and only the search pins a defaulted scope into its links.
+func TestCollScope(t *testing.T) {
+ off, on := Prefs{}, Prefs{IncludeFOIA: true}
+ none := url.Values{}
+ for _, c := range []struct {
+ name string
+ v url.Values
+ p Prefs
+ pin bool
+ foia, def bool
+ qs string
+ }{
+ {"default, browse", none, off, false, false, true, ""},
+ {"default, search", none, off, true, false, true, "coll=news&coll=sidtoday-archive"},
+ {"opted in, browse", none, on, false, true, true, ""},
+ {"opted in, search", none, on, true, true, true, "coll=news&coll=sidtoday-archive&coll=foia"},
+ {"explicit foia", url.Values{"coll": {"foia"}}, off, false, true, false, "coll=foia"},
+ {"explicit news over opt-in", url.Values{"coll": {"news"}}, on, false, false, false, "coll=news"},
+ {"explicit everything", url.Values{"coll": {"news", "foia", "sidtoday-archive"}}, off, false, true, false,
+ "coll=news&coll=sidtoday-archive&coll=foia"},
+ } {
+ s := pageScope(c.v, c.p, c.pin)
+ foia := s.shows(&Document{Collection: collectionFOIA})
+ if foia != c.foia || s.defaulted != c.def || s.qs() != c.qs || s.shown[collectionFOIA] != c.foia {
+ t.Errorf("%s: shows FOIA %v, defaulted %v, qs %q; want %v, %v, %q",
+ c.name, foia, s.defaulted, s.qs(), c.foia, c.def, c.qs)
+ }
+ }
+
+ // A FOIA release's links widen the default to the whole archive and
+ // say so; a reader who opted in needs no widening.
+ if got := pageScope(none, off, false).including(collectionFOIA).suffix(); got != "?coll=news&coll=sidtoday-archive&coll=foia" {
+ t.Errorf("widened default: %q", got)
+ }
+ if got := pageScope(none, on, false).including(collectionFOIA).suffix(); got != "" {
+ t.Errorf("opted in, widened: %q, want no scope", got)
+ }
+ // and adds to the links, not to the documents.
+ s := pageScope(none, off, false).and("date_pub_start=2014")
+ if s.link("/search?q=x") != "/search?q=x&date_pub_start=2014" || s.shows(&Document{Collection: collectionFOIA}) {
+ t.Errorf("and: %q", s.link("/search?q=x"))
+ }
+}
diff --git a/tests/go.mod b/tests/go.mod
new file mode 100644
index 0000000..712d687
--- /dev/null
+++ b/tests/go.mod
@@ -0,0 +1,5 @@
+// Keeps this directory out of the snowchive module's ./... patterns. The
+// files here are package main's tests; tests/run.sh compiles them with it.
+module snowchive/tests
+
+go 1.22
diff --git a/tests/headings_test.go b/tests/headings_test.go
new file mode 100644
index 0000000..325b241
--- /dev/null
+++ b/tests/headings_test.go
@@ -0,0 +1,53 @@
+package main
+
+import (
+ "regexp"
+ "strconv"
+ "strings"
+ "testing"
+)
+
+var reHeadingTag = regexp.MustCompile(`A
B
C
D
`,
+ "### A\n\n### B\n\n#### C": `
A
B
C
`,
+ "```\n# not a heading\n```\n\n## A": `
A
`,
+ } {
+ var got []string
+ for _, line := range strings.Split(Render(src), "\n") {
+ if strings.HasPrefix(line, " prev+1 {
+ t.Errorf("%s: follows ", path, level, prev)
+ }
+ prev = level
+ }
+ if h1 != 1 {
+ t.Errorf("%s: %d
, want 1", path, h1)
+ }
+ }
+}
diff --git a/tests/http_test.go b/tests/http_test.go
new file mode 100644
index 0000000..b255c6a
--- /dev/null
+++ b/tests/http_test.go
@@ -0,0 +1,231 @@
+package main
+
+// http_test.go - protocol behaviour the browser and the proxy rely on:
+// HEAD, byte ranges, conditional requests, and method handling. net/http
+// implements most of it, but only if the handlers hand it the right thing -
+// and a download path that loses Range support breaks resuming a 120 MB PDF
+// and the page-by-page fetches a PDF viewer makes.
+
+import (
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestHTTPHead(t *testing.T) {
+ buildFixture(t)
+ for _, p := range []string{"/", "/browse", "/doc/prism-slides",
+ "/files/prism-slides/prism-slides.pdf", "/api/facets", "/static/style.css"} {
+ r := httptest.NewRequest("HEAD", p, nil)
+ w := httptest.NewRecorder()
+ fixtureMux.ServeHTTP(w, r)
+ if w.Code != 200 {
+ t.Errorf("HEAD %s: %d", p, w.Code)
+ continue
+ }
+ // net/http drops the body for HEAD at the transport layer, so the
+ // recorder still sees one; the headers are what matters here.
+ if w.Header().Get("Content-Type") == "" {
+ t.Errorf("HEAD %s: no Content-Type", p)
+ }
+ }
+}
+
+// Range on a document file. http.ServeFile implements it - this test is
+// here so a future change to the download path (a buffered writer, a
+// compressing wrapper, a manual io.Copy) cannot quietly remove it.
+func TestHTTPRangeOnFiles(t *testing.T) {
+ buildFixture(t)
+ const path = "/files/prism-slides/prism-slides.pdf"
+ full := get(t, path)
+ body := full.Body.String()
+ if full.Header().Get("Accept-Ranges") != "bytes" {
+ t.Fatalf("no Accept-Ranges on a document file: %v", full.Header())
+ }
+
+ req := func(rangeHdr string) *httptest.ResponseRecorder {
+ r := httptest.NewRequest("GET", path, nil)
+ r.Header.Set("Range", rangeHdr)
+ w := httptest.NewRecorder()
+ fixtureMux.ServeHTTP(w, r)
+ return w
+ }
+
+ w := req("bytes=0-3")
+ if w.Code != http.StatusPartialContent {
+ t.Fatalf("Range bytes=0-3: status %d, want 206", w.Code)
+ }
+ if got, want := w.Body.String(), body[:4]; got != want {
+ t.Errorf("range body %q, want %q", got, want)
+ }
+ if cr := w.Header().Get("Content-Range"); cr != fmt.Sprintf("bytes 0-3/%d", len(body)) {
+ t.Errorf("Content-Range %q", cr)
+ }
+
+ // A suffix range: what a resumed download asks for.
+ w = req(fmt.Sprintf("bytes=%d-", len(body)-4))
+ if w.Code != http.StatusPartialContent || w.Body.String() != body[len(body)-4:] {
+ t.Errorf("suffix range: %d %q", w.Code, w.Body.String())
+ }
+
+ // Past the end is a 416 with the length, not a 200 of the whole file.
+ w = req(fmt.Sprintf("bytes=%d-%d", len(body)+10, len(body)+20))
+ if w.Code != http.StatusRequestedRangeNotSatisfiable {
+ t.Errorf("unsatisfiable range: status %d, want 416", w.Code)
+ }
+
+ // A malformed Range is refused with 416 - net/http's choice, not this
+ // server's; it is recorded here so a change in behaviour is noticed.
+ if w = req("bytes=abc"); w.Code != http.StatusRequestedRangeNotSatisfiable {
+ t.Errorf("malformed Range: status %d, want 416", w.Code)
+ }
+ // An unknown range unit is refused the same way. RFC 7233 says to
+ // ignore it and serve the whole representation; net/http does not, and
+ // nothing this archive serves sends one. Recorded, not worked around.
+ if w = req("pages=1-2"); w.Code != http.StatusRequestedRangeNotSatisfiable {
+ t.Errorf("unknown range unit: status %d, want net/http's 416", w.Code)
+ }
+}
+
+// Conditional requests: a document file and a static asset both carry
+// validators, so a reader's second visit is a 304 instead of a re-download.
+func TestHTTPConditional(t *testing.T) {
+ buildFixture(t)
+ // A document file comes off disk and carries a modification time.
+ const doc = "/files/prism-slides/prism-slides.pdf"
+ lm := get(t, doc).Header().Get("Last-Modified")
+ if lm == "" {
+ t.Fatalf("%s: no Last-Modified", doc)
+ }
+ r := httptest.NewRequest("GET", doc, nil)
+ r.Header.Set("If-Modified-Since", lm)
+ w := httptest.NewRecorder()
+ fixtureMux.ServeHTTP(w, r)
+ if w.Code != http.StatusNotModified || w.Body.Len() != 0 {
+ t.Errorf("%s with If-Modified-Since: status %d, %d bytes", doc, w.Code, w.Body.Len())
+ }
+
+ // An embedded asset has no modification time - embed.FS zeroes it - so
+ // it carries a content ETag instead. Without one, every page view
+ // re-downloads the stylesheet on an unstamped build.
+ for _, asset := range []string{"/static/style.css", "/static/main.js"} {
+ first := get(t, asset)
+ tag := first.Header().Get("ETag")
+ if tag == "" {
+ t.Errorf("%s: no ETag", asset)
+ continue
+ }
+ r := httptest.NewRequest("GET", asset, nil)
+ r.Header.Set("If-None-Match", tag)
+ w := httptest.NewRecorder()
+ fixtureMux.ServeHTTP(w, r)
+ if w.Code != http.StatusNotModified || w.Body.Len() != 0 {
+ t.Errorf("%s with If-None-Match: status %d, %d bytes (want an empty 304)",
+ asset, w.Code, w.Body.Len())
+ }
+ // A different asset's tag must not satisfy it.
+ r = httptest.NewRequest("GET", asset, nil)
+ r.Header.Set("If-None-Match", `"0000000000000000"`)
+ w = httptest.NewRecorder()
+ fixtureMux.ServeHTTP(w, r)
+ if w.Code != 200 {
+ t.Errorf("%s with a stale ETag: status %d, want the asset", asset, w.Code)
+ }
+ }
+}
+
+// Method handling: the routes declare GET or POST, so anything else is a
+// 405 from the mux rather than a handler running on the wrong verb.
+func TestHTTPMethods(t *testing.T) {
+ buildFixture(t)
+ for _, c := range []struct {
+ method, path string
+ want int
+ }{
+ // Methods the archive never answers stop at the wrapper.
+ {"PUT", "/api/docs", http.StatusMethodNotAllowed},
+ {"DELETE", "/doc/prism-slides", http.StatusMethodNotAllowed},
+ {"OPTIONS", "/", http.StatusMethodNotAllowed},
+ {"TRACE", "/browse", http.StatusMethodNotAllowed},
+ // A supported method on the wrong route is a 404: the catch-all
+ // route claims every path, which is what keeps unknown paths on the
+ // archive's own 404 page, and it claims them for POST too.
+ {"POST", "/browse", http.StatusNotFound},
+ {"GET", "/preferences/save", http.StatusNotFound},
+ {"GET", "/pin/prism-slides", http.StatusNotFound},
+ } {
+ r := httptest.NewRequest(c.method, c.path, nil)
+ w := httptest.NewRecorder()
+ fixtureMux.ServeHTTP(w, r)
+ if w.Code != c.want {
+ t.Errorf("%s %s: status %d, want %d", c.method, c.path, w.Code, c.want)
+ }
+ }
+}
+
+// Odd but legal requests must not break anything: repeated parameters,
+// unknown parameters, a very long path, an empty query.
+func TestHTTPOddRequests(t *testing.T) {
+ buildFixture(t)
+ for _, c := range []struct {
+ path string
+ want int
+ }{
+ {"/browse?sort=title&sort=date", 200},
+ {"/browse?unknown=1&another=2", 200},
+ {"/search?q=a&q=b&do_search=1", 200},
+ {"/browse/program/" + strings.Repeat("x", 2000), 200}, // an empty page, not a crash
+ {"/doc/" + strings.Repeat("x", 2000), 404},
+ {"/browse?", 200},
+ {"/search?do_search=1&fqv=&fqv=&fqv=", 200},
+ } {
+ if w := get(t, c.path); w.Code != c.want {
+ t.Errorf("%.60s: status %d, want %d", c.path, w.Code, c.want)
+ }
+ }
+}
+
+// With -accel-files the file is nginx's to send: the handler answers only
+// the files it would have served itself, with the redirect and no body.
+func TestAccelFiles(t *testing.T) {
+ buildFixture(t)
+ accelPrefix = "/_files/"
+ defer func() { accelPrefix = "" }()
+
+ w := get(t, "/files/prism-slides/prism-slides.pdf")
+ rel, err := filepath.Rel(dataDir, filepath.Join(corpus().docIndex["prism-slides"].Dir, "prism-slides.pdf"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got, want := w.Header().Get("X-Accel-Redirect"), "/_files/"+filepath.ToSlash(rel); got != want {
+ t.Errorf("X-Accel-Redirect %q, want %q", got, want)
+ }
+ if w.Code != http.StatusOK || w.Body.Len() != 0 {
+ t.Errorf("accel answer: status %d, %d body bytes, want an empty 200", w.Code, w.Body.Len())
+ }
+ if w.Header().Get("Cache-Control") == "" {
+ t.Error("accel answer carries no Cache-Control for nginx to pass on")
+ }
+
+ for _, p := range []string{"/files/prism-slides/zz-stray.pdf", "/files/prism-slides/../glossary.json", "/files/no-such-doc/x.pdf"} {
+ w := get(t, p)
+ if w.Header().Get("X-Accel-Redirect") != "" || w.Code == http.StatusOK {
+ t.Errorf("%s: status %d, X-Accel-Redirect %q - a file the server does not serve was handed to nginx", p, w.Code, w.Header().Get("X-Accel-Redirect"))
+ }
+ }
+}
+
+func TestAccelPathEscapes(t *testing.T) {
+ old, oldPrefix := dataDir, accelPrefix
+ dataDir, accelPrefix = "/srv/data", "/_files/"
+ defer func() { dataDir, accelPrefix = old, oldPrefix }()
+ if got, ok := accelPath("/srv/data/FOIA/a doc/50%?.pdf"); !ok || got != "/_files/FOIA/a%20doc/50%25%3F.pdf" {
+ t.Errorf("accelPath = %q, %v", got, ok)
+ }
+ if got, ok := accelPath("/srv/other/x.pdf"); ok {
+ t.Errorf("a file outside the data directory mapped to %q", got)
+ }
+}
diff --git a/tests/ids_test.go b/tests/ids_test.go
new file mode 100644
index 0000000..f00a615
--- /dev/null
+++ b/tests/ids_test.go
@@ -0,0 +1,99 @@
+package main
+
+import (
+ "bytes"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// The /explore/ids table: one row per document in id order, versions nested
+// under their document, and a checksum file that sha256sum -c can read from
+// the corpus root.
+func TestBuildIDRows(t *testing.T) {
+ oldDocs, oldDir := corpus().archive.Docs, dataDir
+ t.Cleanup(func() {
+ // The id tables are package state the routes serve: rebuild them
+ // from the restored corpus, or the fake rows below leak into
+ // every later test.
+ corpus().archive.Docs, dataDir = oldDocs, oldDir
+ corpus().buildIDRows()
+ })
+
+ dataDir = "/corpus"
+ corpus().archive.Docs = []Document{
+ {
+ ID: "zebra", Dir: filepath.Join(dataDir, "FOIA", "zebra"), Collection: "foia",
+ PDFFile: "zebra.pdf", PDFSize: 2 << 20, Pages: 3,
+ },
+ {
+ ID: "alpha", Dir: filepath.Join(dataDir, "alpha"), Collection: "news",
+ PDFFile: "alpha.pdf", PDFSize: 1500, Pages: 1,
+ PDFSha256: "abc123", AkaID: []AkaEntry{{ID: "alpha-lemonde"}},
+ Versions: []DocVersion{{Slug: "entier", PDFFile: "copy.pdf", PDFSize: 4096, Images: []string{"a", "b"}, SHA: "def456"}},
+ },
+ }
+ corpus().buildIDRows()
+
+ if len(corpus().idRows) != 2 || corpus().idRows[0].ID != "alpha" || corpus().idRows[1].ID != "zebra" {
+ t.Fatalf("rows out of order: %v", corpus().idRows)
+ }
+ if got := corpus().idRows[0].Size; got != "1 KB" {
+ t.Errorf("size %q, want 1 KB", got)
+ }
+ if v := corpus().idRows[0].Versions; len(v) != 1 || v[0].Pages != 2 || v[0].Size != "4 KB" || v[0].SHA != "def456" {
+ t.Errorf("version row %+v", v)
+ }
+ if len(corpus().idRows[1].Versions) != 0 {
+ t.Errorf("zebra has no versions, got %+v", corpus().idRows[1].Versions)
+ }
+
+ // sha256sum -c format: checksum, two spaces, path from the corpus root,
+ // a document's versions after it. Files without a checksum are absent.
+ if corpus().idChecksums != "abc123 alpha/alpha.pdf\ndef456 alpha/versions/entier/copy.pdf\n" {
+ t.Errorf("checksums %q", corpus().idChecksums)
+ }
+}
+
+func TestHumanSize(t *testing.T) {
+ for n, want := range map[int64]string{0: "", -1: "", 900: "900 B", 2048: "2 KB",
+ 3 << 20: "3.0 MB", 5 << 30: "5.00 GB"} {
+ if got := humanSize(n); got != want {
+ t.Errorf("humanSize(%d) = %q, want %q", n, got, want)
+ }
+ }
+ if !strings.HasSuffix(humanSize(1<<20+1<<19), " MB") {
+ t.Errorf("1.5 MiB should render in MB")
+ }
+}
+
+// A version's PDF checksum comes from the metadata's
+// "versions"//pdf_sha256 only; a version without one is warned about
+// and has no line in ids.txt.
+func TestVersionChecksums(t *testing.T) {
+ root := assemblyCorpus(t, "")
+ writeAssemblyDoc(t, root, "listed", map[string]interface{}{
+ "title": "Listed",
+ "versions": map[string]interface{}{"copy": map[string]interface{}{"pdf_sha256": " ABC999 "}},
+ }, []string{"page-1.jpg"}, map[string][]string{"copy": {"page-1.jpg"}})
+ var logged bytes.Buffer
+ log.SetOutput(&logged)
+ withCorpus(t, root)
+ log.SetOutput(os.Stderr)
+
+ if v := corpus().docIndex["listed"].VersionFor("copy"); v == nil || v.SHA != "abc999" {
+ t.Errorf("metadata version checksum: %+v, want abc999 from the metadata", v)
+ }
+ if v := corpus().docIndex["deck"].VersionFor("fuller"); v == nil || v.SHA != "" {
+ t.Errorf("a version without pdf_sha256 in the metadata got a checksum: %+v", v)
+ }
+ if !strings.Contains(logged.String(), "deck: versions/fuller has no pdf_sha256") {
+ t.Errorf("no warning for the version without a checksum:\n%s", logged.String())
+ }
+ if !strings.Contains(corpus().idChecksums, "abc999 listed/versions/copy/copy.pdf\n") ||
+ strings.Contains(corpus().idChecksums, "deck/versions/fuller/") {
+ t.Errorf("ids.txt version lines:\n%s", corpus().idChecksums)
+ }
+}
diff --git a/tests/image_sizes_test.go b/tests/image_sizes_test.go
new file mode 100644
index 0000000..a10abe5
--- /dev/null
+++ b/tests/image_sizes_test.go
@@ -0,0 +1,53 @@
+package main
+
+import (
+ "bytes"
+ "log"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestPageImagesCarryTheirSize(t *testing.T) {
+ for _, path := range []string{
+ "/doc/prism-slides",
+ "/doc/prism-slides/pages",
+ "/doc/prism-slides/versions/lemonde",
+ "/doc/prism-slides/versions/lemonde/pages",
+ } {
+ body := get(t, path).Body.String()
+ if n := strings.Count(body, `page-1.jpg" width="4" height="3"`); n < 2 {
+ t.Errorf("%s: %d page-1 images with their size, want the page and its full-screen view", path, n)
+ }
+ }
+}
+
+func TestAssemblySlidesCarryTheirSize(t *testing.T) {
+ root := assemblyCorpus(t, `[{"id": "sized", "title": "Sized", "slides": [{"doc": "deck", "pages": "1"}]}]`)
+ withCorpus(t, root)
+ for _, path := range []string{"/explore/assemblies", "/explore/assemblies/sized"} {
+ if body := get(t, path).Body.String(); !strings.Contains(body, `page-1.jpg" width="4" height="3"`) {
+ t.Errorf("%s: slide image without its size", path)
+ }
+ }
+}
+
+func TestUnreadablePageImageWarns(t *testing.T) {
+ root := assemblyCorpus(t, "")
+ writeAssemblyDoc(t, root, "broken", map[string]interface{}{"title": "Broken"}, []string{"page-1.jpg"}, nil)
+ writeFixtureFile(t, filepath.Join(root, "broken", "images", "page-1.jpg"), "not an image")
+ var logged bytes.Buffer
+ log.SetOutput(&logged)
+ withCorpus(t, root)
+ log.SetOutput(os.Stderr)
+
+ if !strings.Contains(logged.String(), "page image unreadable, served without its size: broken/images/page-1.jpg") {
+ t.Errorf("no warning for the unreadable image:\n%s", logged.String())
+ }
+ body := get(t, "/doc/broken").Body.String()
+ if !strings.Contains(body, `.*?(\d+)`)
+ reBarLink = regexp.MustCompile(`]*>\s*(\d+)`)
+ reTotal = regexp.MustCompile(`([\d,]+)\s+results?`)
+ reListTotal = regexp.MustCompile(`([\d,]+)\s+documents?`)
+)
+
+// resultTotal reads the count a /search or a list page reports.
+func resultTotal(t *testing.T, path string, cookies ...*http.Cookie) int {
+ t.Helper()
+ body := get(t, path, cookies...).Body.String()
+ re := reTotal
+ if !strings.HasPrefix(path, "/search") {
+ re = reListTotal
+ }
+ m := re.FindStringSubmatch(body)
+ if m == nil {
+ t.Fatalf("%s: no result count in the page", path)
+ }
+ n, err := strconv.Atoi(strings.ReplaceAll(m[1], ",", ""))
+ if err != nil {
+ t.Fatalf("%s: %v", path, err)
+ }
+ return n
+}
+
+// followLinks checks every (link, count) pair a page prints, as the reader
+// with these cookies sees it.
+func followLinks(t *testing.T, page string, re *regexp.Regexp, what string, cookies ...*http.Cookie) int {
+ t.Helper()
+ body := get(t, page, cookies...).Body.String()
+ pairs := re.FindAllStringSubmatch(body, -1)
+ for _, p := range pairs {
+ link := html.UnescapeString(p[1])
+ shown, _ := strconv.Atoi(p[2])
+ if got := resultTotal(t, link, cookies...); got != shown {
+ t.Errorf("%s: %s says %d, its link returns %d\n %s", page, what, shown, got, link)
+ }
+ }
+ return len(pairs)
+}
+
+// A category page lists exactly as many documents as its facet counts.
+func TestInvariantFacetCountsMatchPages(t *testing.T) {
+ buildFixture(t)
+ checked := 0
+ for ci := range categories {
+ c := &categories[ci]
+ for val, want := range c.Facet(corpus()) {
+ if got := len(corpus().categoryMatches(c.Slug, val)); got != want {
+ t.Errorf("%s=%q: facet says %d, the page lists %d", c.Slug, val, want, got)
+ }
+ checked++
+ }
+ }
+ if checked == 0 {
+ t.Fatal("no facet values in the fixture")
+ }
+}
+
+// Refine links and histogram bars open searches that must return what they
+// promise - the two places where a count and its click are computed by
+// different code paths.
+func TestInvariantRefineAndBarCounts(t *testing.T) {
+ pages := []string{
+ "/browse/agency/NSA",
+ "/browse/program/PRISM",
+ "/browse/classification/TOP%20SECRET",
+ "/search?q=prism&do_search=1",
+ "/search?do_search=1&fqv=NSA&fqf=CR&fqc=and",
+ }
+ facets, bars := 0, 0
+ for _, p := range pages {
+ facets += followLinks(t, p, reFacetLink, "refine link")
+ bars += followLinks(t, p, reBarLink, "histogram bar")
+ }
+ if facets == 0 || bars == 0 {
+ t.Fatalf("nothing exercised: %d refine links, %d bars", facets, bars)
+ }
+ t.Logf("checked %d refine links and %d histogram bars", facets, bars)
+}
+
+// The home page is scoped to the reported collection while the category
+// pages follow the reader's scope, so its bars and lists link to those
+// pages with the collection named. Every one of them has to be the number
+// its click returns.
+func TestInvariantHomeCounts(t *testing.T) {
+ body := get(t, "/").Body.String()
+ pairs := regexp.MustCompile(`[^<]*(\d+)`).FindAllStringSubmatch(body, -1)
+ lists := regexp.MustCompile(`
`) {
+ t.Error("/browse/program/PRISM does not mark the program links")
+ }
+}
+
+// TestUnknownSortIsNotEchoed: a sort key no list has is the default sort,
+// and stays out of the page's links and labels.
+func TestUnknownSortIsNotEchoed(t *testing.T) {
+ for _, path := range []string{"/browse?sort=bogus", "/browse/program/PRISM?sort=bogus", "/search?q=prism&do_search=1&sort=bogus"} {
+ body := get(t, path).Body.String()
+ if strings.Contains(body, "sort=bogus") || strings.Contains(body, "Bogus") {
+ t.Errorf("%s: the unknown sort is echoed", path)
+ }
+ }
+ if body := get(t, "/search?q=prism&do_search=1").Body.String(); strings.Contains(body, "sort=relevance&order=desc") {
+ t.Error("relevance offers a direction it does not have")
+ }
+}
+
+// TestOutletYearLinksKeepTheScope: on the reported-only outlets table the
+// year headings open the reported-only year, like the rows and cells do.
+func TestOutletYearLinksKeepTheScope(t *testing.T) {
+ body := get(t, "/explore/outlets?coll=news").Body.String()
+ if !strings.Contains(body, `href="/browse/published/2013?coll=news"`) {
+ t.Error("the 2013 heading drops coll=news")
+ }
+}
+
+// pinSearch posts qs to /pinsearch from this site, with the given pinned
+// searches already in the cookie.
+func pinSearch(t *testing.T, qs string, pinned []string) *httptest.ResponseRecorder {
+ t.Helper()
+ mux, _ := buildFixture(t)
+ r := httptest.NewRequest("POST", "/pinsearch", strings.NewReader(url.Values{"qs": {qs}, "back": {"/search"}}.Encode()))
+ r.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ r.Header.Set("Sec-Fetch-Site", "same-origin")
+ if len(pinned) > 0 {
+ r.AddCookie(&http.Cookie{Name: pinnedSearchesCookie, Value: strings.Join(pinned, "|")})
+ }
+ return record(mux, r)
+}
+
+// TestPinListFull: a pin the cookie has no room for is refused, and the
+// reader lands on /pinned, which says so, rather than back on a page whose
+// button still reads "Pin" as though nothing happened.
+func TestPinListFull(t *testing.T) {
+ qs := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {"prism"}}.Encode()
+ if w := pinSearch(t, qs, nil); w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/search" {
+ t.Fatalf("pinning into an empty list: %d to %q", w.Code, w.Header().Get("Location"))
+ }
+ // Filler searches up to the byte budget: stored entries are not
+ // checked on read, so any escaped query string stands in for one.
+ var fill []string
+ for i := 0; ; i++ {
+ e := url.QueryEscape(fmt.Sprintf("fqv=filler%02d", i) + strings.Repeat("x", 180))
+ if len(strings.Join(append(fill, e), "|")) > cookieValueMax {
+ break
+ }
+ fill = append(fill, e)
+ }
+ // The last filler takes the cookie to exactly its budget.
+ fill[len(fill)-1] += strings.Repeat("x", cookieValueMax-len(strings.Join(fill, "|")))
+ w := pinSearch(t, qs, fill)
+ if w.Code != http.StatusSeeOther || w.Header().Get("Location") != "/pinned?full=1" {
+ t.Errorf("pinning into a full list: %d to %q, want 303 to /pinned?full=1", w.Code, w.Header().Get("Location"))
+ }
+ if body := get(t, "/pinned?full=1").Body.String(); !strings.Contains(body, "Your pinned list is full") {
+ t.Error("/pinned?full=1 does not say the list is full")
+ }
+}
+
+// TestSavableSearchMatchesPinSearch: the search page offers the pin button
+// for exactly the searches /pinsearch accepts - one size limit for both.
+func TestSavableSearchMatchesPinSearch(t *testing.T) {
+ long := strings.TrimSpace(strings.Repeat("abcdefgh ", 58)) // 58 words, under the word cap
+ q := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {long}}
+ if len(q.Encode()) <= searchQSMax {
+ t.Fatalf("test query is %d bytes, not over the %d limit", len(q.Encode()), searchQSMax)
+ }
+ if body := get(t, "/search?"+q.Encode()).Body.String(); strings.Contains(body, `action="/pinsearch"`) {
+ t.Error("the search page offers to pin a search /pinsearch refuses")
+ }
+ if w := pinSearch(t, q.Encode(), nil); w.Code != http.StatusBadRequest {
+ t.Errorf("/pinsearch with an over-long search: %d, want 400", w.Code)
+ }
+ short := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {"prism"}}
+ if body := get(t, "/search?"+short.Encode()).Body.String(); !strings.Contains(body, `action="/pinsearch"`) {
+ t.Error("the search page does not offer to pin a short search")
+ }
+}
+
+// TestAssetStamp: pages link each asset by its content hash; that URL is
+// cached for a year, and any other stamp only revalidates.
+func TestAssetStamp(t *testing.T) {
+ body := get(t, "/").Body.String()
+ want := "/static/style.css?v=" + staticHashes["style.css"]
+ if !strings.Contains(body, `href="`+want+`"`) {
+ t.Fatalf("home page does not link %s", want)
+ }
+ if cc := get(t, want).Header().Get("Cache-Control"); !strings.Contains(cc, "immutable") {
+ t.Errorf("%s: Cache-Control %q, want immutable", want, cc)
+ }
+ w := get(t, "/static/style.css?v=0000000000000000")
+ if cc := w.Header().Get("Cache-Control"); strings.Contains(cc, "immutable") || w.Header().Get("ETag") == "" {
+ t.Errorf("a stale stamp: Cache-Control %q, ETag %q; want revalidation", cc, w.Header().Get("ETag"))
+ }
+}
+
+// TestSitemapIgnoresFile: the sitemap is always the one built from the
+// loaded corpus - a sitemap.xml lying in the data or project directory is
+// never served, with -base or without it.
+func TestSitemapIgnoresFile(t *testing.T) {
+ root := t.TempDir()
+ writeFixtureFile(t, filepath.Join(root, "sitemap.xml"), xmlHeaderForTest+"https://stale.example/\n")
+ withCorpus(t, root)
+ prev := canonicalBase
+ t.Cleanup(func() { canonicalBase = prev })
+ for _, base := range []string{"", "https://archive.example.org"} {
+ canonicalBase = base
+ loadData()
+ if body := get(t, "/sitemap.xml").Body.String(); strings.Contains(body, "stale.example") {
+ t.Errorf("base %q: a sitemap.xml file was served", base)
+ }
+ }
+}
+
+const xmlHeaderForTest = `` + "\n"
+
+// TestPinnedSearchListed: /pinned lists a pinned search with its label,
+// its link and the unpin form carrying the canonical query string.
+func TestPinnedSearchListed(t *testing.T) {
+ qs := url.Values{"do_search": {"1"}, "fqc": {"and"}, "fqf": {"TE"}, "fqv": {"prism"}}.Encode()
+ body := get(t, "/pinned", &http.Cookie{Name: pinnedSearchesCookie, Value: url.QueryEscape(qs)}).Body.String()
+ for _, want := range []string{`href="/search?` + template.HTMLEscapeString(qs) + `"`, `name="qs" value="` + template.HTMLEscapeString(qs) + `"`, ">prism<"} {
+ if !strings.Contains(body, want) {
+ t.Errorf("/pinned lacks %s", want)
+ }
+ }
+}
+
+// TestPageTitles: every page names itself in the tab, not just the site.
+func TestPageTitles(t *testing.T) {
+ buildFixture(t) // the article key comes from the loaded corpus
+ for path, want := range map[string]string{
+ "/browse/program/PRISM": "Program: PRISM - Snowden Archive",
+ "/browse/program": "Browse by Programs - Snowden Archive",
+ "/search?q=prism&do_search=1": "prism - Search - Snowden Archive",
+ "/search": "Search - Snowden Archive",
+ "/browse?collection=foia": "FOIA documents - Snowden Archive",
+ "/pinned": "Pinned - Snowden Archive",
+ "/nosuchpage": "Page not found - Snowden Archive",
+ "/explore/pages": "- Snowden Archive",
+ "/article/" + corpus().articleByLink["https://example.org/prism"].Key: "NSA Prism program - Snowden Archive",
+ } {
+ if body := get(t, path).Body.String(); !strings.Contains(body, want) {
+ t.Errorf("%s: no %q", path, want)
+ }
+ }
+}
+
+// TestNoBlockContentInHeadings: a heading holds text and inline controls
+// only - a form or a disclosure inside one is invalid HTML and lands in
+// the heading's accessible name. Forms tie their buttons from outside.
+func TestNoBlockContentInHeadings(t *testing.T) {
+ heading := regexp.MustCompile(`(?s)]*>(.*?)`)
+ block := regexp.MustCompile(`<(form|details|div|p|ul|ol|table)\b`)
+ cookie := &http.Cookie{Name: pinnedCookie, Value: "prism-slides"}
+ recent := []*http.Cookie{{Name: "pref_track_recent", Value: "1"}, {Name: recentDocsCookie, Value: "prism-slides"}}
+ pages := map[string][]*http.Cookie{
+ "/pinned": {cookie},
+ "/": recent,
+ "/search?q=prism&do_search=1": nil,
+ "/doc/prism-slides": nil,
+ }
+ for path, cookies := range pages {
+ body := get(t, path, cookies...).Body.String()
+ for _, m := range heading.FindAllStringSubmatch(body, -1) {
+ if b := block.FindString(m[2]); b != "" {
+ t.Errorf("%s: %s inside ", path, b, m[1])
+ }
+ }
+ }
+ if body := get(t, "/pinned", cookie).Body.String(); !strings.Contains(body, `form="unpin-prism-slides"`) || !strings.Contains(body, `id="unpin-prism-slides"`) {
+ t.Error("/pinned: the unpin button is not tied to its form")
+ }
+}
+
+// TestGlossarySeeAlsoResolves: a see-also name links to the entry it
+// names - through an alias or another case too - and a name no entry has
+// is plain text rather than a link to nowhere.
+func TestGlossarySeeAlsoResolves(t *testing.T) {
+ root := t.TempDir()
+ writeFixtureFile(t, filepath.Join(root, "glossary.json"), `[
+ {"term": "XKEYSCORE", "definition": "A search system.", "aka": ["XKS"], "see": ["rt-rg", "TURMOIL", "NOSUCHTERM"]},
+ {"term": "RT-RG", "definition": "Real time regional gateway."},
+ {"term": "TURMOIL", "definition": "Passive collection.", "see": ["xks"]}
+ ]`)
+ withCorpus(t, root)
+ body := get(t, "/glossary").Body.String()
+ for _, want := range []string{`rt-rg`, `TURMOIL`, `xks`} {
+ if !strings.Contains(body, want) {
+ t.Errorf("/glossary lacks %s", want)
+ }
+ }
+ if strings.Contains(body, `href="#term-NOSUCHTERM"`) {
+ t.Error("/glossary links a see-also name no entry has")
+ }
+}
+
+// A pager or sort link appends its parameter to queryBase, with nothing
+// left over when the page has no query of its own.
+func TestQueryBase(t *testing.T) {
+ for want, q := range map[string]url.Values{
+ "/browse?": {},
+ "/browse?view=table&": {"view": {"table"}},
+ "/browse?coll=foia&view=card&": {"view": {"card"}, "coll": {"foia"}},
+ } {
+ if got := queryBase("/browse", q); got != want {
+ t.Errorf("queryBase(%v) = %q, want %q", q, got, want)
+ }
+ }
+}
diff --git a/tests/run.sh b/tests/run.sh
new file mode 100755
index 0000000..258e462
--- /dev/null
+++ b/tests/run.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+# Runs the tests in this directory. They are package main's own tests, and Go
+# only compiles those from the package's directory, so an overlay presents
+# them to the go tool as if they were there. Arguments go to "go test":
+#
+# tests/run.sh
+# tests/run.sh -run TestCorpus -v
+set -e
+cd "$(dirname "$0")/.."
+root=$(pwd)
+overlay=$(mktemp)
+trap 'rm -f "$overlay"' EXIT
+{
+ printf '{"Replace":{'
+ sep=
+ for f in tests/*_test.go; do
+ printf '%s"%s/%s":"%s/%s"' "$sep" "$root" "${f#tests/}" "$root" "$f"
+ sep=,
+ done
+ printf '}}\n'
+} >"$overlay"
+go test -overlay "$overlay" "$@" .
diff --git a/tests/search_test.go b/tests/search_test.go
new file mode 100644
index 0000000..abb7a20
--- /dev/null
+++ b/tests/search_test.go
@@ -0,0 +1,1000 @@
+package main
+
+import (
+ "encoding/json"
+ "net/url"
+ "reflect"
+ "strconv"
+ "strings"
+ "testing"
+)
+
+func mustQuery(t *testing.T, u string) url.Values {
+ t.Helper()
+ parsed, err := url.Parse(u)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return parsed.Query()
+}
+
+// TestStemUnifiesForms pins the stemmer's contract: the inflections a
+// reader is likely to type must land on the same token as the base word,
+// because the index and the query both go through stem().
+func TestStemUnifiesForms(t *testing.T) {
+ groups := [][]string{
+ {"operation", "operations", "operational"},
+ {"communication", "communications"},
+ {"collection", "collections"},
+ {"collecting", "collected"},
+ {"agency", "agencies"},
+ {"surveillance", "surveilling"},
+ {"target", "targets", "targeting", "targeted"},
+ {"meeting", "meetings"},
+ {"computer", "computers"},
+ {"process", "processes"},
+ {"study", "studies"},
+ {"tie", "ties", "tied"},
+ {"lie", "lies", "lied"},
+ {"spy", "spies", "spied", "spying"},
+ {"fly", "flies", "flying", "flyer"},
+ {"study", "studying", "studied"},
+ {"control", "controlled", "controlling", "controls"},
+ {"install", "installed", "installing"},
+ {"cancel", "cancelled", "canceled"},
+ {"fall", "falling", "falls"},
+ {"kill", "killed", "killing"},
+ // A plural in -ies takes its singular's path through the -ity and
+ // -ly rules, and a word in -ie meets its plural.
+ {"capability", "capabilities"},
+ {"vulnerability", "vulnerabilities"},
+ {"activity", "activities"},
+ {"family", "families"},
+ {"anomaly", "anomalies"},
+ {"cookie", "cookies"},
+ // Verbs in -ply are not adverbs.
+ {"apply", "applies", "applied", "applying"},
+ {"supply", "supplies", "supplied", "supplying", "supplier"},
+ {"comply", "complies", "complied", "complying", "compliance"},
+ {"reply", "replies", "replied"},
+ }
+ for _, g := range groups {
+ want := stem(g[0])
+ for _, w := range g[1:] {
+ if got := stem(w); got != want {
+ t.Errorf("stem(%q) = %q, stem(%q) = %q; want equal", w, got, g[0], want)
+ }
+ }
+ }
+ // Words a stripped ending must not turn into each other.
+ for _, p := range [][2]string{
+ {"apply", "app"}, {"reply", "rep"}, {"supply", "supp"}, {"early", "ear"},
+ {"italy", "ita"}, {"july", "julie"},
+ } {
+ if stem(p[0]) == stem(p[1]) {
+ t.Errorf("stem(%q) = stem(%q) = %q; want them apart", p[0], p[1], stem(p[0]))
+ }
+ }
+ // Rules that only exist for suffixes ending in "s"/"ss" must be
+ // reachable: the plural pass no longer returns early.
+ if stem("happiness") == "happiness" || stem("useless") == "useless" {
+ t.Errorf("-ness/-less rules unreachable: %q %q", stem("happiness"), stem("useless"))
+ }
+ // Domain terms the conservative stemmer must leave recognizable.
+ for w, want := range map[string]string{"sigint": "sigint", "nsa": "nsa", "class": "class", "yes": "yes", "tdy": "tdy", "tdi": "tdi"} {
+ if got := stem(w); got != want {
+ t.Errorf("stem(%q) = %q, want %q", w, got, want)
+ }
+ }
+}
+
+func TestTokenizeSplitsLikeConcatenation(t *testing.T) {
+ // loadData relies on tokenize(a+" "+b) == tokenize(a) ++ tokenize(b).
+ a, b := "Müller-Maguhn's XKEYSCORE deck, 2013", "collecting the metadata of targets"
+ joined := tokenize(a + " " + b)
+ parts := append(tokenize(a), tokenize(b)...)
+ if !reflect.DeepEqual(joined, parts) {
+ t.Errorf("tokenize(a+b) = %v, parts = %v", joined, parts)
+ }
+}
+
+func TestParseSimpleQuery(t *testing.T) {
+ q, f, c, d := parseSimpleQuery(`program:XKEYSCORE -tempora "cable tapping"~5 date:2013..2015 upstream`)
+ wantQ := []string{"XKEYSCORE", `"cable tapping"~5`, "upstream", "tempora"}
+ wantF := []string{"PR", "TE", "TE", "TE"}
+ wantC := []string{"and", "and", "and", "not"}
+ if !reflect.DeepEqual(q, wantQ) || !reflect.DeepEqual(f, wantF) || !reflect.DeepEqual(c, wantC) {
+ t.Errorf("got %v %v %v, want %v %v %v", q, f, c, wantQ, wantF, wantC)
+ }
+ if d.DocStart != "2013" || d.DocEnd != "2015-12-31" {
+ t.Errorf("dates = %+v", d)
+ }
+ // Pure-negative queries get a match-all anchor.
+ q, _, c, _ = parseSimpleQuery("-draft")
+ if !reflect.DeepEqual(q, []string{"", "draft"}) || !reflect.DeepEqual(c, []string{"and", "not"}) {
+ t.Errorf("pure negative: %v %v", q, c)
+ }
+}
+
+func TestNormalizeClauses(t *testing.T) {
+ q, f, c := normalizeClauses([]string{"a", "b"}, []string{"TI"}, nil)
+ if !reflect.DeepEqual(f, []string{"TI", "TE"}) || !reflect.DeepEqual(c, []string{"and", "and"}) || len(q) != 2 {
+ t.Errorf("got %v %v %v", q, f, c)
+ }
+ many := make([]string, maxSearchClauses+10)
+ q, f, c = normalizeClauses(many, nil, nil)
+ if len(q) != maxSearchClauses || len(f) != maxSearchClauses || len(c) != maxSearchClauses {
+ t.Errorf("cap not applied: %d %d %d", len(q), len(f), len(c))
+ }
+}
+
+func TestPaginate(t *testing.T) {
+ cases := []struct {
+ total, page, perPage int
+ start, end, wantPage, wantTotalPages int
+ }{
+ {45, 1, 20, 0, 20, 1, 3},
+ {45, 3, 20, 40, 45, 3, 3},
+ {45, 99, 20, 40, 45, 3, 3},
+ {45, -5, 20, 0, 20, 1, 3},
+ {0, 7, 20, 0, 0, 1, 0},
+ {45, 2, 0, 0, 45, 1, 1},
+ }
+ for _, c := range cases {
+ s, e, p, tp := paginate(c.total, c.page, c.perPage)
+ if s != c.start || e != c.end || p != c.wantPage || tp != c.wantTotalPages {
+ t.Errorf("paginate(%d,%d,%d) = %d,%d,%d,%d; want %d,%d,%d,%d",
+ c.total, c.page, c.perPage, s, e, p, tp, c.start, c.end, c.wantPage, c.wantTotalPages)
+ }
+ }
+}
+
+func TestSafeBackPath(t *testing.T) {
+ for in, want := range map[string]string{
+ "/doc/x": "/doc/x",
+ "//evil.com": "/",
+ "/\\evil.com": "/",
+ "https://evil": "/",
+ "": "/",
+ "/search?q=a\r\n": "/",
+ } {
+ if got := safeBackPath(in); got != want {
+ t.Errorf("safeBackPath(%q) = %q, want %q", in, got, want)
+ }
+ }
+}
+
+// Tags are curated in the metadata itself, so the
+// server does nothing to them beyond the normalization every list field
+// gets: trim, drop empties, fold case-duplicates keeping the first casing.
+func TestNormalizeListOnTags(t *testing.T) {
+ got := normalizeList([]string{" XKS ", "xks", "", "PRISM", "prism"})
+ want := []string{"XKS", "PRISM"}
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("normalizeList = %v, want %v", got, want)
+ }
+ if normalizeList(nil) != nil {
+ t.Error("nil in should stay nil (JSON null, not [])")
+ }
+}
+
+func TestClassSortKeyFoldsAliases(t *testing.T) {
+ if classSortKey([]string{"TS", "SI", "TK"}) != classSortKey([]string{"TOP SECRET", "COMINT", "Talent Keyhole"}) {
+ t.Error("abbreviations and full names must rank identically")
+ }
+ if classSortKey([]string{"TOP SECRET", "SI"}) <= classSortKey([]string{"TOP SECRET"}) {
+ t.Error("a level with SI must outrank the bare level")
+ }
+ if classSortKey([]string{"NOFORN"}) != "" {
+ t.Error("only unranked components must sort with the empties")
+ }
+}
+
+func TestMarkTermsEscapes(t *testing.T) {
+ hl := buildHighlightTerms([]string{"müller says"}, []string{"and"})
+ out := string(markTerms(`Müller says hi`, hl))
+ if out != `Müller says <b>hi</b>` {
+ t.Errorf("markTerms = %s", out)
+ }
+}
+
+func TestHeadingLevel(t *testing.T) {
+ for in, want := range map[string]int{
+ "# Title": 1, "### Deep": 3, "######\tSix": 6, "#": 1,
+ "#1": 0, "#hashtag": 0, "####### seven": 0, "no": 0,
+ "##########################": 0,
+ } {
+ if got := headingLevel(in); got != want {
+ t.Errorf("headingLevel(%q) = %d, want %d", in, got, want)
+ }
+ }
+ if got := Render("#1 priority\n#2 next"); got != "
#1 priority \n#2 next
\n" {
+ t.Errorf("Render = %q", got)
+ }
+}
+
+// is the transcription convention for a line break inside a table
+// cell - the one place the table syntax cannot express one. It survives as
+// a break, everything else in the tag vocabulary stays escaped text, and a
+// code span keeps it literal.
+func TestRenderHardBreak(t *testing.T) {
+ got := Render("| a | b |\n| --- | --- |\n| one two | x y |\n")
+ for _, want := range []string{"
one two
", "
x y
"} {
+ if !strings.Contains(got, want) {
+ t.Errorf("Render table = %q, want %q in it", got, want)
+ }
+ }
+ if got := Render("text bold and "); strings.Contains(got, "") || strings.Contains(got, "