// api.go - read-only JSON API mirroring the UI's features. // // Endpoints (all GET): // // /api/docs list documents; params: coll (repeatable: // news, sidtoday-archive, foia), type // (repeatable), sort (any list-page sort: title, // agency, date, pubdate, classification, ...; see // docSorts), order (asc|desc), p, per_page (max // 500, or "all" for exports) // /api/doc/{id} full document: metadata, full text, images, // related IDs, published-together IDs, // version_copies (other published copies: PDF, // page images, transcript, publication indices) // /api/search params: q (simple) OR fqv/fqf/fqc clause // triples (same field codes and value syntax as // the UI: word, "text", =exact, *wildcard*), // plus coll and type (both repeatable, as on // /api/docs; no coll means every collection), // date_doc_start/_end, date_pub_start/_end // (partial END dates are inclusive: 2013 means // through 2013-12-31), sort, order, and the same // p / per_page pagination as /api/docs (default // 50, max 500, "all" for every match without // snippets see apiPageSize). results come // as one flat list, mirroring the search page: // all collections rank equally. a result whose match // is in another published copy of the document // lists that copy in matched_versions (slug, // label, url, snippet) // /api/facets facet maps (whole corpus, like the UI; the // one news-scoped map is news_types) plus // collection counts package main import ( "bytes" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "log" "net/http" "net/url" "strconv" "strings" "time" ) const ( apiPerPageDefault = 50 apiMaxPerPage = 500 ) func writeJSON(w http.ResponseWriter, status int, v interface{}) { b, err := json.Marshal(v) if err != nil { log.Printf("API: marshal error: %v", err) w.Header().Set("Cache-Control", "no-store") w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(http.StatusInternalServerError) w.Write([]byte(`{"error":"internal error"}`)) return } writeJSONBytes(w, status, b) } func writeJSONBytes(w http.ResponseWriter, status int, b []byte) { setJSONHeaders(w) w.WriteHeader(status) w.Write(b) } func setJSONHeaders(w http.ResponseWriter) { cc := w.Header().Get("Cache-Control") if cc == "" { cc = "public, max-age=60" w.Header().Set("Cache-Control", cc) } w.Header().Set("Content-Type", "application/json; charset=utf-8") if !strings.Contains(cc, "private") { w.Header().Set("Access-Control-Allow-Origin", "*") } } func apiError(w http.ResponseWriter, status int, msg string) { writeJSON(w, status, map[string]string{"error": msg}) } type apiDocSummary struct { Document URL string `json:"url"` PDF string `json:"pdf"` } func docSummary(d Document) apiDocSummary { return apiDocSummary{ Document: d, URL: docPath(d.ID), PDF: filePath(d.ID, d.PDFFile), } } type apiDocDetail struct { apiDocSummary FullText string `json:"full_text,omitempty"` Images []string `json:"images,omitempty"` Related []string `json:"related,omitempty"` PublishedTogether []string `json:"published_together,omitempty"` VersionCopies []apiVersion `json:"version_copies,omitempty"` } type apiSearchResult struct { apiDocSummary Score float64 `json:"score"` Snippet string `json:"snippet,omitempty"` MatchedVersions []apiMatchedVersion `json:"matched_versions,omitempty"` } func (co *Corpus) handleAPIDocs(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() typeFilter, _ := selectedTypes(q) // coll=, repeatable, as /api/search and the pages take it. pool := filterByCollection(co.everyDoc, collectionFilter(q)) docs := filterByType(append([]*Document(nil), pool...), typeFilter) sortDocs(docs, q.Get("sort"), q.Get("order")) perPage := apiPageSize(q) page, _ := strconv.Atoi(q.Get("p")) start, end, page, totalPages := paginate(len(docs), page, perPage) out := make([]apiDocSummary, 0, end-start) for _, d := range docs[start:end] { out = append(out, docSummary(*d)) } writeJSON(w, http.StatusOK, map[string]interface{}{ "total": len(docs), "page": page, "per_page": apiRespPerPage(perPage, len(docs)), "total_pages": totalPages, "docs": out, }) } func apiPageSize(q url.Values) int { if q.Get("per_page") == "all" { return 0 } if v, err := strconv.Atoi(q.Get("per_page")); err == nil && v > 0 { if v > apiMaxPerPage { return apiMaxPerPage } return v } return apiPerPageDefault } func apiRespPerPage(perPage, total int) int { if perPage > 0 { return perPage } if total == 0 { return 1 } return total } func (co *Corpus) handleAPIDoc(w http.ResponseWriter, r *http.Request) { id := r.PathValue("id") doc := co.docIndex[id] if doc == nil { apiError(w, http.StatusNotFound, "document not found") return } var pubIDs []string for _, d := range co.publishedTogether(doc) { pubIDs = append(pubIDs, d.ID) } writeJSON(w, http.StatusOK, apiDocDetail{ apiDocSummary: docSummary(*doc), FullText: doc.FullText, Images: doc.Images, Related: co.archive.RelatedDocs[id], PublishedTogether: pubIDs, VersionCopies: apiVersions(doc), }) } func (co *Corpus) handleAPISearch(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() sp := readSearchParams(q) if len(sp.Queries) == 0 { apiError(w, http.StatusBadRequest, "missing query: pass ?q=term or fqv/fqf/fqc clause parameters") return } if sp.OverClauses > 0 { apiError(w, http.StatusBadRequest, fmt.Sprintf( "too many clauses: %d (the limit is %d)", sp.OverClauses, maxSearchClauses)) return } if sp.OverWords > 0 { apiError(w, http.StatusBadRequest, fmt.Sprintf( "too many words: %d (the limit is %d)", sp.OverWords, maxSearchWords)) return } scored, _ := co.executeSearch(sp) snippetTerm, _ := snippetAnchor(sp.Queries, sp.Fields, sp.Conjs) perPage := apiPageSize(q) page, _ := strconv.Atoi(q.Get("p")) start, end, page, totalPages := paginate(len(scored), page, perPage) withSnippets := snippetTerm != "" && perPage > 0 results := make([]apiSearchResult, 0, end-start) for _, s := range scored[start:end] { res := apiSearchResult{apiDocSummary: docSummary(*s.Doc), Score: s.Score} if withSnippets { if body := s.Doc.bodySnippets(snippetTerm, 1); len(body) > 0 { res.Snippet = body[0] } for _, h := range versionHits(s.Doc, snippetTerm, 1) { res.MatchedVersions = append(res.MatchedVersions, apiMatchedVersion{ Slug: h.Version.Slug, Label: h.Version.Label, URL: versionPath(s.Doc.ID, h.Version.Slug), Snippet: h.Snippets[0], }) } } results = append(results, res) } body := map[string]interface{}{ "total": len(scored), "page": page, "per_page": apiRespPerPage(perPage, len(scored)), "total_pages": totalPages, "results": results, } if snippetTerm != "" && !withSnippets { body["snippets_omitted"] = true } writeJSON(w, http.StatusOK, body) } func (co *Corpus) buildFacetsJSON() { b, err := json.Marshal(map[string]interface{}{ "note": "facet counts cover the whole corpus; news_types covers the news collection only", "collections": co.archive.CollectionCounts, "sidtoday_total": co.archive.SIDTodayCount, "types": co.archive.Types, "news_types": co.archive.NewsTypes, "agencies": co.archive.Facets["agency"], "programs": co.archive.Facets["program"], "journalists": co.archive.Facets["journalist"], "publishers": co.archive.Facets["publisher"], "countries": co.archive.Facets["country"], "people": co.archive.Facets["people"], "classification": co.archive.Facets["classification"], "dissemination": co.archive.Facets["dissemination"], "banners": co.archive.Facets["banner"], "genres": co.archive.Facets["genre"], "series": co.archive.Facets["series"], "tags": co.archive.Facets["tag"], "pub_years": co.archive.Facets["published"], "doc_years": co.archive.Facets["docdate"], "by_collection": co.archive.FacetsByColl, }) if err != nil { log.Fatalf("facets: marshal failed: %v", err) } co.facetsJSON = b sum := sha256.Sum256(b) co.facetsETag = `"` + hex.EncodeToString(sum[:8]) + `"` log.Printf("Facets JSON pre-built: %d bytes", len(co.facetsJSON)) } func (co *Corpus) handleAPIFacets(w http.ResponseWriter, r *http.Request) { setJSONHeaders(w) w.Header().Set("ETag", co.facetsETag) http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(co.facetsJSON)) } // "version" as in document versions (metadata.json can have "versions" object) type apiVersion struct { Slug string `json:"slug"` URL string `json:"url"` PDF string `json:"pdf"` PDFSha256 string `json:"pdf_sha256,omitempty"` Images []string `json:"images,omitempty"` FullText string `json:"full_text,omitempty"` Publications []int `json:"publications,omitempty"` ChangedPages []int `json:"changed_pages,omitempty"` } func apiVersions(doc *Document) []apiVersion { var out []apiVersion for _, v := range doc.Versions { out = append(out, apiVersion{ Slug: v.Slug, URL: versionPath(doc.ID, v.Slug), PDF: versionFilePath(doc.ID, v.Slug, v.PDFFile), PDFSha256: v.SHA, Images: v.Images, FullText: v.FullText, Publications: v.PubIndex, ChangedPages: changedPageNumbers(v.ChangedPages), }) } return out } type apiMatchedVersion struct { Slug string `json:"slug"` Label string `json:"label"` URL string `json:"url"` Snippet string `json:"snippet"` } func changedPageNumbers(pages []ChangedPage) []int { var out []int for _, p := range pages { out = append(out, p.Page) } return out }