335 lines
10 KiB
Go
335 lines
10 KiB
Go
// api.go - read-only JSON API mirroring the UI's features.
|
|
//
|
|
// Endpoints (all GET):
|
|
//
|
|
// /api/docs list documents; params: coll (repeatable:
|
|
// news, sidtoday-archive, foia), type
|
|
// (repeatable), sort (any list-page sort: title,
|
|
// agency, date, pubdate, classification, ...; see
|
|
// docSorts), order (asc|desc), p, per_page (max
|
|
// 500, or "all" for exports)
|
|
// /api/doc/{id} full document: metadata, full text, images,
|
|
// related IDs, published-together IDs,
|
|
// version_copies (other published copies: PDF,
|
|
// page images, transcript, publication indices)
|
|
// /api/search params: q (simple) OR fqv/fqf/fqc clause
|
|
// triples (same field codes and value syntax as
|
|
// the UI: word, "text", =exact, *wildcard*),
|
|
// plus coll and type (both repeatable, as on
|
|
// /api/docs; no coll means every collection),
|
|
// date_doc_start/_end, date_pub_start/_end
|
|
// (partial END dates are inclusive: 2013 means
|
|
// through 2013-12-31), sort, order, and the same
|
|
// p / per_page pagination as /api/docs (default
|
|
// 50, max 500, "all" for every match without
|
|
// snippets see apiPageSize). results come
|
|
// as one flat list, mirroring the search page:
|
|
// all collections rank equally. a result whose match
|
|
// is in another published copy of the document
|
|
// lists that copy in matched_versions (slug,
|
|
// label, url, snippet)
|
|
// /api/facets facet maps (whole corpus, like the UI; the
|
|
// one news-scoped map is news_types) plus
|
|
// collection counts
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"net/http"
|
|
"net/url"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
apiPerPageDefault = 50
|
|
apiMaxPerPage = 500
|
|
)
|
|
|
|
func writeJSON(w http.ResponseWriter, status int, v interface{}) {
|
|
b, err := json.Marshal(v)
|
|
if err != nil {
|
|
log.Printf("API: marshal error: %v", err)
|
|
w.Header().Set("Cache-Control", "no-store")
|
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
w.Write([]byte(`{"error":"internal error"}`))
|
|
return
|
|
}
|
|
writeJSONBytes(w, status, b)
|
|
}
|
|
|
|
func writeJSONBytes(w http.ResponseWriter, status int, b []byte) {
|
|
setJSONHeaders(w)
|
|
w.WriteHeader(status)
|
|
w.Write(b)
|
|
}
|
|
|
|
func setJSONHeaders(w http.ResponseWriter) {
|
|
cc := w.Header().Get("Cache-Control")
|
|
if cc == "" {
|
|
cc = "public, max-age=60"
|
|
w.Header().Set("Cache-Control", cc)
|
|
}
|
|
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
|
|
|
if !strings.Contains(cc, "private") {
|
|
w.Header().Set("Access-Control-Allow-Origin", "*")
|
|
}
|
|
}
|
|
|
|
func apiError(w http.ResponseWriter, status int, msg string) {
|
|
writeJSON(w, status, map[string]string{"error": msg})
|
|
}
|
|
|
|
type apiDocSummary struct {
|
|
Document
|
|
URL string `json:"url"`
|
|
PDF string `json:"pdf"`
|
|
}
|
|
|
|
func docSummary(d Document) apiDocSummary {
|
|
return apiDocSummary{
|
|
Document: d,
|
|
URL: docPath(d.ID),
|
|
PDF: filePath(d.ID, d.PDFFile),
|
|
}
|
|
}
|
|
|
|
type apiDocDetail struct {
|
|
apiDocSummary
|
|
FullText string `json:"full_text,omitempty"`
|
|
Images []string `json:"images,omitempty"`
|
|
Related []string `json:"related,omitempty"`
|
|
PublishedTogether []string `json:"published_together,omitempty"`
|
|
VersionCopies []apiVersion `json:"version_copies,omitempty"`
|
|
}
|
|
|
|
type apiSearchResult struct {
|
|
apiDocSummary
|
|
Score float64 `json:"score"`
|
|
Snippet string `json:"snippet,omitempty"`
|
|
MatchedVersions []apiMatchedVersion `json:"matched_versions,omitempty"`
|
|
}
|
|
|
|
func (co *Corpus) handleAPIDocs(w http.ResponseWriter, r *http.Request) {
|
|
q := r.URL.Query()
|
|
|
|
typeFilter, _ := selectedTypes(q)
|
|
|
|
// coll=, repeatable, as /api/search and the pages take it.
|
|
pool := filterByCollection(co.everyDoc, collectionFilter(q))
|
|
docs := filterByType(append([]*Document(nil), pool...), typeFilter)
|
|
|
|
sortDocs(docs, q.Get("sort"), q.Get("order"))
|
|
|
|
perPage := apiPageSize(q)
|
|
page, _ := strconv.Atoi(q.Get("p"))
|
|
start, end, page, totalPages := paginate(len(docs), page, perPage)
|
|
|
|
out := make([]apiDocSummary, 0, end-start)
|
|
for _, d := range docs[start:end] {
|
|
out = append(out, docSummary(*d))
|
|
}
|
|
|
|
writeJSON(w, http.StatusOK, map[string]interface{}{
|
|
"total": len(docs),
|
|
"page": page,
|
|
"per_page": apiRespPerPage(perPage, len(docs)),
|
|
"total_pages": totalPages,
|
|
"docs": out,
|
|
})
|
|
}
|
|
|
|
func apiPageSize(q url.Values) int {
|
|
if q.Get("per_page") == "all" {
|
|
return 0
|
|
}
|
|
if v, err := strconv.Atoi(q.Get("per_page")); err == nil && v > 0 {
|
|
if v > apiMaxPerPage {
|
|
return apiMaxPerPage
|
|
}
|
|
return v
|
|
}
|
|
return apiPerPageDefault
|
|
}
|
|
|
|
func apiRespPerPage(perPage, total int) int {
|
|
if perPage > 0 {
|
|
return perPage
|
|
}
|
|
if total == 0 {
|
|
return 1
|
|
}
|
|
return total
|
|
}
|
|
|
|
func (co *Corpus) handleAPIDoc(w http.ResponseWriter, r *http.Request) {
|
|
id := r.PathValue("id")
|
|
doc := co.docIndex[id]
|
|
if doc == nil {
|
|
apiError(w, http.StatusNotFound, "document not found")
|
|
return
|
|
}
|
|
|
|
var pubIDs []string
|
|
for _, d := range co.publishedTogether(doc) {
|
|
pubIDs = append(pubIDs, d.ID)
|
|
}
|
|
|
|
writeJSON(w, http.StatusOK, apiDocDetail{
|
|
apiDocSummary: docSummary(*doc),
|
|
FullText: doc.FullText,
|
|
Images: doc.Images,
|
|
Related: co.archive.RelatedDocs[id],
|
|
PublishedTogether: pubIDs,
|
|
VersionCopies: apiVersions(doc),
|
|
})
|
|
}
|
|
|
|
func (co *Corpus) handleAPISearch(w http.ResponseWriter, r *http.Request) {
|
|
q := r.URL.Query()
|
|
|
|
sp := readSearchParams(q)
|
|
if len(sp.Queries) == 0 {
|
|
apiError(w, http.StatusBadRequest,
|
|
"missing query: pass ?q=term or fqv/fqf/fqc clause parameters")
|
|
return
|
|
}
|
|
if sp.OverClauses > 0 {
|
|
apiError(w, http.StatusBadRequest, fmt.Sprintf(
|
|
"too many clauses: %d (the limit is %d)", sp.OverClauses, maxSearchClauses))
|
|
return
|
|
}
|
|
if sp.OverWords > 0 {
|
|
apiError(w, http.StatusBadRequest, fmt.Sprintf(
|
|
"too many words: %d (the limit is %d)", sp.OverWords, maxSearchWords))
|
|
return
|
|
}
|
|
scored, _ := co.executeSearch(sp)
|
|
snippetTerm, _ := snippetAnchor(sp.Queries, sp.Fields, sp.Conjs)
|
|
|
|
perPage := apiPageSize(q)
|
|
page, _ := strconv.Atoi(q.Get("p"))
|
|
start, end, page, totalPages := paginate(len(scored), page, perPage)
|
|
withSnippets := snippetTerm != "" && perPage > 0
|
|
|
|
results := make([]apiSearchResult, 0, end-start)
|
|
for _, s := range scored[start:end] {
|
|
res := apiSearchResult{apiDocSummary: docSummary(*s.Doc), Score: s.Score}
|
|
if withSnippets {
|
|
if body := s.Doc.bodySnippets(snippetTerm, 1); len(body) > 0 {
|
|
res.Snippet = body[0]
|
|
}
|
|
for _, h := range versionHits(s.Doc, snippetTerm, 1) {
|
|
res.MatchedVersions = append(res.MatchedVersions, apiMatchedVersion{
|
|
Slug: h.Version.Slug,
|
|
Label: h.Version.Label,
|
|
URL: versionPath(s.Doc.ID, h.Version.Slug),
|
|
Snippet: h.Snippets[0],
|
|
})
|
|
}
|
|
}
|
|
results = append(results, res)
|
|
}
|
|
|
|
body := map[string]interface{}{
|
|
"total": len(scored),
|
|
"page": page,
|
|
"per_page": apiRespPerPage(perPage, len(scored)),
|
|
"total_pages": totalPages,
|
|
"results": results,
|
|
}
|
|
if snippetTerm != "" && !withSnippets {
|
|
body["snippets_omitted"] = true
|
|
}
|
|
writeJSON(w, http.StatusOK, body)
|
|
}
|
|
|
|
func (co *Corpus) buildFacetsJSON() {
|
|
b, err := json.Marshal(map[string]interface{}{
|
|
"note": "facet counts cover the whole corpus; news_types covers the news collection only",
|
|
"collections": co.archive.CollectionCounts,
|
|
"sidtoday_total": co.archive.SIDTodayCount,
|
|
"types": co.archive.Types,
|
|
"news_types": co.archive.NewsTypes,
|
|
"agencies": co.archive.Facets["agency"],
|
|
"programs": co.archive.Facets["program"],
|
|
"journalists": co.archive.Facets["journalist"],
|
|
"publishers": co.archive.Facets["publisher"],
|
|
"countries": co.archive.Facets["country"],
|
|
"people": co.archive.Facets["people"],
|
|
"classification": co.archive.Facets["classification"],
|
|
"dissemination": co.archive.Facets["dissemination"],
|
|
"banners": co.archive.Facets["banner"],
|
|
"genres": co.archive.Facets["genre"],
|
|
"series": co.archive.Facets["series"],
|
|
"tags": co.archive.Facets["tag"],
|
|
"pub_years": co.archive.Facets["published"],
|
|
"doc_years": co.archive.Facets["docdate"],
|
|
"by_collection": co.archive.FacetsByColl,
|
|
})
|
|
if err != nil {
|
|
log.Fatalf("facets: marshal failed: %v", err)
|
|
}
|
|
co.facetsJSON = b
|
|
sum := sha256.Sum256(b)
|
|
co.facetsETag = `"` + hex.EncodeToString(sum[:8]) + `"`
|
|
log.Printf("Facets JSON pre-built: %d bytes", len(co.facetsJSON))
|
|
}
|
|
|
|
func (co *Corpus) handleAPIFacets(w http.ResponseWriter, r *http.Request) {
|
|
setJSONHeaders(w)
|
|
w.Header().Set("ETag", co.facetsETag)
|
|
http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(co.facetsJSON))
|
|
}
|
|
|
|
// "version" as in document versions (metadata.json can have "versions" object)
|
|
type apiVersion struct {
|
|
Slug string `json:"slug"`
|
|
URL string `json:"url"`
|
|
PDF string `json:"pdf"`
|
|
PDFSha256 string `json:"pdf_sha256,omitempty"`
|
|
Images []string `json:"images,omitempty"`
|
|
FullText string `json:"full_text,omitempty"`
|
|
Publications []int `json:"publications,omitempty"`
|
|
ChangedPages []int `json:"changed_pages,omitempty"`
|
|
}
|
|
|
|
func apiVersions(doc *Document) []apiVersion {
|
|
var out []apiVersion
|
|
for _, v := range doc.Versions {
|
|
out = append(out, apiVersion{
|
|
Slug: v.Slug,
|
|
URL: versionPath(doc.ID, v.Slug),
|
|
PDF: versionFilePath(doc.ID, v.Slug, v.PDFFile),
|
|
PDFSha256: v.SHA,
|
|
Images: v.Images,
|
|
FullText: v.FullText,
|
|
Publications: v.PubIndex,
|
|
ChangedPages: changedPageNumbers(v.ChangedPages),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
type apiMatchedVersion struct {
|
|
Slug string `json:"slug"`
|
|
Label string `json:"label"`
|
|
URL string `json:"url"`
|
|
Snippet string `json:"snippet"`
|
|
}
|
|
|
|
func changedPageNumbers(pages []ChangedPage) []int {
|
|
var out []int
|
|
for _, p := range pages {
|
|
out = append(out, p.Page)
|
|
}
|
|
return out
|
|
}
|