// explore_ids.go - /explore/ids: every document's identifiers in one table. // // /explore/ids.txt is the same checksums in sha256sum(1) format, so a mirror // or a download verifies with "sha256sum -c" in the corpus root. package main import ( "fmt" "net/http" "path/filepath" "sort" "strings" ) type idVersionRow struct { Slug string PDFFile string PDFURL string Size string Pages int SHA string } type idRow struct { ID string Collection string Pages int PDFFile string PDFURL string Size string SHA string Aka []AkaEntry Versions []idVersionRow } func humanSize(n int64) string { switch { case n <= 0: return "" case n < 1<<10: return fmt.Sprintf("%d B", n) case n < 1<<20: return fmt.Sprintf("%.0f KB", float64(n)/(1<<10)) case n < 1<<30: return fmt.Sprintf("%.1f MB", float64(n)/(1<<20)) } return fmt.Sprintf("%.2f GB", float64(n)/(1<<30)) } func (co *Corpus) buildIDRows() { co.idRows = make([]idRow, 0, len(co.archive.Docs)) var sums strings.Builder for i := range co.archive.Docs { d := &co.archive.Docs[i] row := idRow{ ID: d.ID, Collection: d.Collection, Pages: d.Pages, PDFFile: d.PDFFile, PDFURL: filePath(d.ID, d.PDFFile), Size: humanSize(d.PDFSize), SHA: strings.TrimSpace(d.PDFSha256), Aka: d.AkaID, } for _, v := range d.Versions { row.Versions = append(row.Versions, idVersionRow{ Slug: v.Slug, PDFFile: v.PDFFile, PDFURL: versionFilePath(d.ID, v.Slug, v.PDFFile), Size: humanSize(v.PDFSize), Pages: len(v.Images), SHA: v.SHA, }) } co.idRows = append(co.idRows, row) rel, err := filepath.Rel(dataDir, d.Dir) if err != nil { rel = d.ID } if row.SHA != "" { sums.WriteString(row.SHA + " " + filepath.ToSlash(filepath.Join(rel, d.PDFFile)) + "\n") } for _, v := range d.Versions { if v.SHA != "" { sums.WriteString(v.SHA + " " + filepath.ToSlash(filepath.Join(rel, "versions", v.Slug, v.PDFFile)) + "\n") } } } sort.Slice(co.idRows, func(i, j int) bool { a, b := strings.ToLower(co.idRows[i].ID), strings.ToLower(co.idRows[j].ID) if a != b { return a < b } return co.idRows[i].ID < co.idRows[j].ID }) co.idChecksums = sums.String() } func (co *Corpus) handleExploreIDs(w http.ResponseWriter, r *http.Request) { render(w, r, &idsPage{ Rows: co.idRows, RowsHTML: co.preHTML("ids"), Shown: len(co.idRows), }) } func (co *Corpus) handleExploreIDsText(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") w.Header().Set("Cache-Control", "public, max-age=3600") w.Write([]byte(co.idChecksums)) }