snowchive-server/api.go
2026-11-19 04:40:59 +02:00

335 lines
10 KiB
Go

// api.go - read-only JSON API mirroring the UI's features.
//
// Endpoints (all GET):
//
// /api/docs list documents; params: coll (repeatable:
// news, sidtoday-archive, foia), type
// (repeatable), sort (any list-page sort: title,
// agency, date, pubdate, classification, ...; see
// docSorts), order (asc|desc), p, per_page (max
// 500, or "all" for exports)
// /api/doc/{id} full document: metadata, full text, images,
// related IDs, published-together IDs,
// version_copies (other published copies: PDF,
// page images, transcript, publication indices)
// /api/search params: q (simple) OR fqv/fqf/fqc clause
// triples (same field codes and value syntax as
// the UI: word, "text", =exact, *wildcard*),
// plus coll and type (both repeatable, as on
// /api/docs; no coll means every collection),
// date_doc_start/_end, date_pub_start/_end
// (partial END dates are inclusive: 2013 means
// through 2013-12-31), sort, order, and the same
// p / per_page pagination as /api/docs (default
// 50, max 500, "all" for every match without
// snippets see apiPageSize). results come
// as one flat list, mirroring the search page:
// all collections rank equally. a result whose match
// is in another published copy of the document
// lists that copy in matched_versions (slug,
// label, url, snippet)
// /api/facets facet maps (whole corpus, like the UI; the
// one news-scoped map is news_types) plus
// collection counts
package main
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"log"
"net/http"
"net/url"
"strconv"
"strings"
"time"
)
const (
apiPerPageDefault = 50
apiMaxPerPage = 500
)
func writeJSON(w http.ResponseWriter, status int, v interface{}) {
b, err := json.Marshal(v)
if err != nil {
log.Printf("API: marshal error: %v", err)
w.Header().Set("Cache-Control", "no-store")
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(`{"error":"internal error"}`))
return
}
writeJSONBytes(w, status, b)
}
func writeJSONBytes(w http.ResponseWriter, status int, b []byte) {
setJSONHeaders(w)
w.WriteHeader(status)
w.Write(b)
}
func setJSONHeaders(w http.ResponseWriter) {
cc := w.Header().Get("Cache-Control")
if cc == "" {
cc = "public, max-age=60"
w.Header().Set("Cache-Control", cc)
}
w.Header().Set("Content-Type", "application/json; charset=utf-8")
if !strings.Contains(cc, "private") {
w.Header().Set("Access-Control-Allow-Origin", "*")
}
}
func apiError(w http.ResponseWriter, status int, msg string) {
writeJSON(w, status, map[string]string{"error": msg})
}
type apiDocSummary struct {
Document
URL string `json:"url"`
PDF string `json:"pdf"`
}
func docSummary(d Document) apiDocSummary {
return apiDocSummary{
Document: d,
URL: docPath(d.ID),
PDF: filePath(d.ID, d.PDFFile),
}
}
type apiDocDetail struct {
apiDocSummary
FullText string `json:"full_text,omitempty"`
Images []string `json:"images,omitempty"`
Related []string `json:"related,omitempty"`
PublishedTogether []string `json:"published_together,omitempty"`
VersionCopies []apiVersion `json:"version_copies,omitempty"`
}
type apiSearchResult struct {
apiDocSummary
Score float64 `json:"score"`
Snippet string `json:"snippet,omitempty"`
MatchedVersions []apiMatchedVersion `json:"matched_versions,omitempty"`
}
func (co *Corpus) handleAPIDocs(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
typeFilter, _ := selectedTypes(q)
// coll=, repeatable, as /api/search and the pages take it.
pool := filterByCollection(co.everyDoc, collectionFilter(q))
docs := filterByType(append([]*Document(nil), pool...), typeFilter)
sortDocs(docs, q.Get("sort"), q.Get("order"))
perPage := apiPageSize(q)
page, _ := strconv.Atoi(q.Get("p"))
start, end, page, totalPages := paginate(len(docs), page, perPage)
out := make([]apiDocSummary, 0, end-start)
for _, d := range docs[start:end] {
out = append(out, docSummary(*d))
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"total": len(docs),
"page": page,
"per_page": apiRespPerPage(perPage, len(docs)),
"total_pages": totalPages,
"docs": out,
})
}
func apiPageSize(q url.Values) int {
if q.Get("per_page") == "all" {
return 0
}
if v, err := strconv.Atoi(q.Get("per_page")); err == nil && v > 0 {
if v > apiMaxPerPage {
return apiMaxPerPage
}
return v
}
return apiPerPageDefault
}
func apiRespPerPage(perPage, total int) int {
if perPage > 0 {
return perPage
}
if total == 0 {
return 1
}
return total
}
func (co *Corpus) handleAPIDoc(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
doc := co.docIndex[id]
if doc == nil {
apiError(w, http.StatusNotFound, "document not found")
return
}
var pubIDs []string
for _, d := range co.publishedTogether(doc) {
pubIDs = append(pubIDs, d.ID)
}
writeJSON(w, http.StatusOK, apiDocDetail{
apiDocSummary: docSummary(*doc),
FullText: doc.FullText,
Images: doc.Images,
Related: co.archive.RelatedDocs[id],
PublishedTogether: pubIDs,
VersionCopies: apiVersions(doc),
})
}
func (co *Corpus) handleAPISearch(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
sp := readSearchParams(q)
if len(sp.Queries) == 0 {
apiError(w, http.StatusBadRequest,
"missing query: pass ?q=term or fqv/fqf/fqc clause parameters")
return
}
if sp.OverClauses > 0 {
apiError(w, http.StatusBadRequest, fmt.Sprintf(
"too many clauses: %d (the limit is %d)", sp.OverClauses, maxSearchClauses))
return
}
if sp.OverWords > 0 {
apiError(w, http.StatusBadRequest, fmt.Sprintf(
"too many words: %d (the limit is %d)", sp.OverWords, maxSearchWords))
return
}
scored, _ := co.executeSearch(sp)
snippetTerm, _ := snippetAnchor(sp.Queries, sp.Fields, sp.Conjs)
perPage := apiPageSize(q)
page, _ := strconv.Atoi(q.Get("p"))
start, end, page, totalPages := paginate(len(scored), page, perPage)
withSnippets := snippetTerm != "" && perPage > 0
results := make([]apiSearchResult, 0, end-start)
for _, s := range scored[start:end] {
res := apiSearchResult{apiDocSummary: docSummary(*s.Doc), Score: s.Score}
if withSnippets {
if body := s.Doc.bodySnippets(snippetTerm, 1); len(body) > 0 {
res.Snippet = body[0]
}
for _, h := range versionHits(s.Doc, snippetTerm, 1) {
res.MatchedVersions = append(res.MatchedVersions, apiMatchedVersion{
Slug: h.Version.Slug,
Label: h.Version.Label,
URL: versionPath(s.Doc.ID, h.Version.Slug),
Snippet: h.Snippets[0],
})
}
}
results = append(results, res)
}
body := map[string]interface{}{
"total": len(scored),
"page": page,
"per_page": apiRespPerPage(perPage, len(scored)),
"total_pages": totalPages,
"results": results,
}
if snippetTerm != "" && !withSnippets {
body["snippets_omitted"] = true
}
writeJSON(w, http.StatusOK, body)
}
func (co *Corpus) buildFacetsJSON() {
b, err := json.Marshal(map[string]interface{}{
"note": "facet counts cover the whole corpus; news_types covers the news collection only",
"collections": co.archive.CollectionCounts,
"sidtoday_total": co.archive.SIDTodayCount,
"types": co.archive.Types,
"news_types": co.archive.NewsTypes,
"agencies": co.archive.Facets["agency"],
"programs": co.archive.Facets["program"],
"journalists": co.archive.Facets["journalist"],
"publishers": co.archive.Facets["publisher"],
"countries": co.archive.Facets["country"],
"people": co.archive.Facets["people"],
"classification": co.archive.Facets["classification"],
"dissemination": co.archive.Facets["dissemination"],
"banners": co.archive.Facets["banner"],
"genres": co.archive.Facets["genre"],
"series": co.archive.Facets["series"],
"tags": co.archive.Facets["tag"],
"pub_years": co.archive.Facets["published"],
"doc_years": co.archive.Facets["docdate"],
"by_collection": co.archive.FacetsByColl,
})
if err != nil {
log.Fatalf("facets: marshal failed: %v", err)
}
co.facetsJSON = b
sum := sha256.Sum256(b)
co.facetsETag = `"` + hex.EncodeToString(sum[:8]) + `"`
log.Printf("Facets JSON pre-built: %d bytes", len(co.facetsJSON))
}
func (co *Corpus) handleAPIFacets(w http.ResponseWriter, r *http.Request) {
setJSONHeaders(w)
w.Header().Set("ETag", co.facetsETag)
http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(co.facetsJSON))
}
// "version" as in document versions (metadata.json can have "versions" object)
type apiVersion struct {
Slug string `json:"slug"`
URL string `json:"url"`
PDF string `json:"pdf"`
PDFSha256 string `json:"pdf_sha256,omitempty"`
Images []string `json:"images,omitempty"`
FullText string `json:"full_text,omitempty"`
Publications []int `json:"publications,omitempty"`
ChangedPages []int `json:"changed_pages,omitempty"`
}
func apiVersions(doc *Document) []apiVersion {
var out []apiVersion
for _, v := range doc.Versions {
out = append(out, apiVersion{
Slug: v.Slug,
URL: versionPath(doc.ID, v.Slug),
PDF: versionFilePath(doc.ID, v.Slug, v.PDFFile),
PDFSha256: v.SHA,
Images: v.Images,
FullText: v.FullText,
Publications: v.PubIndex,
ChangedPages: changedPageNumbers(v.ChangedPages),
})
}
return out
}
type apiMatchedVersion struct {
Slug string `json:"slug"`
Label string `json:"label"`
URL string `json:"url"`
Snippet string `json:"snippet"`
}
func changedPageNumbers(pages []ChangedPage) []int {
var out []int
for _, p := range pages {
out = append(out, p.Page)
}
return out
}