1191 lines
46 KiB
Odin
1191 lines
46 KiB
Odin
package pdf
|
|
|
|
import "core:fmt"
|
|
import "core:log"
|
|
import "core:mem"
|
|
import "core:os"
|
|
import "core:path/filepath"
|
|
import "core:slice"
|
|
import "core:strings"
|
|
import "core:testing"
|
|
import "core:time"
|
|
import "core:unicode/utf8"
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Corpus tests
|
|
//
|
|
// These run over whatever PDFs are in the directory named by PDF_ODIN_CORPUS
|
|
// and skip silently when it is unset, so the suite never depends on files that
|
|
// are not in the repository.
|
|
//
|
|
// They assert invariants, not results. There is no correct answer to check a
|
|
// real-world file against -- that is what the golden files in testdata/ are for
|
|
// -- so what is checked here is the contract every file must satisfy: opening
|
|
// terminates, whatever comes back is self-consistent, and nothing crashes,
|
|
// hangs or leaks. That is precisely the class of bug a corpus finds and a unit
|
|
// test cannot.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// A file that takes longer than this is logged. It is not a hard timeout --
|
|
// the test runner cannot interrupt a running test -- but a file that trips it
|
|
// has found a loop bound that is too generous, and that is worth seeing.
|
|
@(private = "file")
|
|
CORPUS_SLOW_MS :: 5000
|
|
|
|
// Files larger than this are skipped. The corpus directory is whatever the
|
|
// developer happened to point at, and a 500 MB scanned document exercises
|
|
// nothing a 5 MB one does not.
|
|
@(private = "file")
|
|
CORPUS_MAX_FILE :: 64 << 20
|
|
|
|
@(private = "file")
|
|
CORPUS_MAX_DEPTH :: 8
|
|
|
|
@(private = "file")
|
|
Corpus_Stats :: struct {
|
|
files: int,
|
|
opened: int,
|
|
failed: int,
|
|
damaged: int,
|
|
pages: int,
|
|
skipped: int,
|
|
}
|
|
|
|
// corpus_dir returns the corpus directory, or "" when the variable is unset.
|
|
@(private = "file")
|
|
corpus_dir :: proc(allocator := context.allocator) -> string {
|
|
raw := os.get_env("PDF_ODIN_CORPUS", allocator)
|
|
trimmed := strings.trim_space(raw)
|
|
// trim_space returns a slice of `raw`, so the original allocation is still
|
|
// the one to free; hand back a clone and release it.
|
|
out := strings.clone(trimmed, allocator)
|
|
delete(raw, allocator)
|
|
return out
|
|
}
|
|
|
|
// corpus_files lists the PDFs under `dir`, recursively and in a stable order,
|
|
// so that two runs report the same files in the same sequence.
|
|
@(private = "file")
|
|
corpus_files :: proc(dir: string, out: ^[dynamic]string, depth := 0) {
|
|
if depth > CORPUS_MAX_DEPTH { return }
|
|
|
|
entries, err := os.read_directory_by_path(dir, -1, context.allocator)
|
|
if err != nil { return }
|
|
defer os.file_info_slice_delete(entries, context.allocator)
|
|
|
|
slice.sort_by(entries, proc(a, b: os.File_Info) -> bool { return a.name < b.name })
|
|
|
|
for e in entries {
|
|
if e.type == .Directory {
|
|
corpus_files(e.fullpath, out, depth + 1)
|
|
continue
|
|
}
|
|
if e.type != .Regular { continue }
|
|
|
|
lower := strings.to_lower(e.name, context.temp_allocator)
|
|
if !strings.has_suffix(lower, ".pdf") { continue }
|
|
append(out, strings.clone(e.fullpath))
|
|
}
|
|
}
|
|
|
|
@(private = "file")
|
|
corpus_collect :: proc(dir: string) -> [dynamic]string {
|
|
paths := make([dynamic]string)
|
|
corpus_files(dir, &paths)
|
|
return paths
|
|
}
|
|
|
|
@(private = "file")
|
|
corpus_release :: proc(paths: ^[dynamic]string) {
|
|
for p in paths { delete(p) }
|
|
delete(paths^)
|
|
}
|
|
|
|
@(private = "file")
|
|
corpus_size :: proc(path: string) -> i64 {
|
|
info, err := os.stat(path, context.allocator)
|
|
if err != nil { return 0 }
|
|
defer os.file_info_delete(info, context.allocator)
|
|
return info.size
|
|
}
|
|
|
|
// corpus_check_one opens one file and asserts the invariants on it.
|
|
//
|
|
// Every failure names the file: a corpus run that says only "3 files failed"
|
|
// costs more time than it saves.
|
|
@(private = "file")
|
|
corpus_check_one :: proc(t: ^testing.T, path: string, stats: ^Corpus_Stats) {
|
|
if corpus_size(path) > CORPUS_MAX_FILE {
|
|
stats.skipped += 1
|
|
return
|
|
}
|
|
|
|
name := filepath.base(path)
|
|
stats.files += 1
|
|
started := time.now()
|
|
|
|
doc, err := open_file(path)
|
|
defer close(doc)
|
|
|
|
if err != .None {
|
|
// A file that will not open is not automatically a bug: a corpus
|
|
// collected off a disk contains genuinely broken files, and .No_Pages
|
|
// on a zero-page document is the right answer. What must never happen
|
|
// is a crash or a hang, and reaching this line proves neither did.
|
|
stats.failed += 1
|
|
log.infof("corpus: %s: open failed: %s", name, error_string(err))
|
|
return
|
|
}
|
|
if !testing.expectf(t, doc != nil, "%s: open reported success but gave no document", name) {
|
|
return
|
|
}
|
|
|
|
stats.opened += 1
|
|
if doc.is_damaged { stats.damaged += 1 }
|
|
|
|
n := page_count(doc)
|
|
testing.expectf(t, n > 0, "%s: opened with %v pages", name, n)
|
|
stats.pages += n
|
|
|
|
// page_get reads doc.pages and doc.page_refs in lockstep, so a length
|
|
// mismatch is an out-of-range read waiting to happen.
|
|
testing.expectf(t, len(doc.pages) == len(doc.page_refs),
|
|
"%s: %v page dicts against %v page refs", name, len(doc.pages), len(doc.page_refs))
|
|
|
|
for i in 0 ..< n {
|
|
page, perr := page_get(doc, i)
|
|
if !testing.expectf(t, perr == .None, "%s: page %v: %s", name, i, error_string(perr)) {
|
|
continue
|
|
}
|
|
testing.expectf(t, page.dict != nil, "%s: page %v has no dictionary", name, i)
|
|
|
|
// No NaN or infinity may reach the device layer, and a zero-area page
|
|
// divides to one. page_get is the boundary that has to guarantee it.
|
|
testing.expectf(t, rect_is_finite(page.media_box),
|
|
"%s: page %v media box is not finite: %v", name, i, page.media_box)
|
|
testing.expectf(t, !rect_is_empty(page.media_box),
|
|
"%s: page %v media box is empty: %v", name, i, page.media_box)
|
|
testing.expectf(t, rect_is_finite(page.crop_box),
|
|
"%s: page %v crop box is not finite: %v", name, i, page.crop_box)
|
|
testing.expectf(t, !rect_is_empty(page.crop_box),
|
|
"%s: page %v crop box is empty: %v", name, i, page.crop_box)
|
|
testing.expectf(t, page.rotate >= 0 && page.rotate < 360 && page.rotate % 90 == 0,
|
|
"%s: page %v rotate is %v", name, i, page.rotate)
|
|
|
|
corpus_check_contents(t, doc, page, name)
|
|
}
|
|
|
|
testing.expectf(t, len(doc.repairs) <= MAX_REPAIRS,
|
|
"%s: repair list grew past its cap: %v", name, len(doc.repairs))
|
|
|
|
ms := time.duration_milliseconds(time.since(started))
|
|
if ms > CORPUS_SLOW_MS {
|
|
log.infof("corpus: %s: slow: %.0f ms for %v pages", name, ms, n)
|
|
}
|
|
}
|
|
|
|
// corpus_check_contents decodes a page's content streams. A stream that will
|
|
// not decode is not a test failure -- plenty of real files hold one, and the
|
|
// reader is required to degrade rather than refuse -- so what is checked is
|
|
// the contract `stream_data` states, which no file is allowed to break.
|
|
@(private = "file")
|
|
corpus_check_contents :: proc(t: ^testing.T, doc: ^Document, page: Page, name: string) {
|
|
contents := dict_get(doc, page.dict, "Contents")
|
|
|
|
#partial switch v in contents {
|
|
case ^Stream:
|
|
corpus_check_stream(t, doc, v, name)
|
|
case ^Array:
|
|
// A /Contents array in a real file has a handful of entries; one with
|
|
// thousands is a file trying to make this loop expensive.
|
|
limit := min(array_len(v), 4096)
|
|
for i in 0 ..< limit {
|
|
s, ok := to_stream(array_get(doc, v, i))
|
|
if !ok { continue }
|
|
corpus_check_stream(t, doc, s, name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// corpus_check_stream holds `stream_data` to its documented contract over real
|
|
// data.
|
|
//
|
|
// Three properties, all of which a malformed file could plausibly break:
|
|
// decoding a stream never reports an error (the error return is reserved for a
|
|
// caller mistake -- a broken filter chain degrades to empty plus a repair); a
|
|
// call always leaves the stream decided, never still .Pending; and the result
|
|
// is cached, so the second call hands back the identical bytes rather than
|
|
// decoding again. The last one is the reason nothing in the library frees what
|
|
// this returns, and it is checked by pointer, not by length.
|
|
@(private = "file")
|
|
corpus_check_stream :: proc(t: ^testing.T, doc: ^Document, s: ^Stream, name: string) {
|
|
if s == nil { return }
|
|
|
|
data, err := stream_data(doc, s)
|
|
testing.expectf(t, err == .None, "%s: stream_data returned %s", name, error_string(err))
|
|
testing.expectf(t, s.state != .Pending, "%s: stream_data left the stream undecoded", name)
|
|
if s.state == .Failed {
|
|
testing.expectf(t, data == nil,
|
|
"%s: a stream that failed to decode returned %v bytes", name, len(data))
|
|
testing.expectf(t, doc.is_damaged,
|
|
"%s: a stream failed to decode without recording a repair", name)
|
|
}
|
|
|
|
again, aerr := stream_data(doc, s)
|
|
testing.expectf(t, aerr == .None, "%s: second stream_data returned %s", name, error_string(aerr))
|
|
testing.expectf(t, raw_data(again) == raw_data(data) && len(again) == len(data),
|
|
"%s: stream_data decoded the same stream twice: %v bytes, then %v bytes",
|
|
name, len(data), len(again))
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
|
|
@(test)
|
|
test_corpus :: proc(t: ^testing.T) {
|
|
dir := corpus_dir()
|
|
defer delete(dir)
|
|
if dir == "" {
|
|
log.info("PDF_ODIN_CORPUS is unset; skipping the corpus test")
|
|
return
|
|
}
|
|
if !os.is_dir(dir) {
|
|
log.infof("PDF_ODIN_CORPUS is %q, which is not a directory; skipping", dir)
|
|
return
|
|
}
|
|
|
|
paths := corpus_collect(dir)
|
|
defer corpus_release(&paths)
|
|
|
|
if len(paths) == 0 {
|
|
log.infof("no PDFs under %q; skipping", dir)
|
|
return
|
|
}
|
|
|
|
stats: Corpus_Stats
|
|
for p in paths {
|
|
corpus_check_one(t, p, &stats)
|
|
free_all(context.temp_allocator)
|
|
}
|
|
|
|
log.infof("corpus %q: %v files, %v opened, %v failed, %v damaged, %v skipped, %v pages",
|
|
dir, stats.files, stats.opened, stats.failed, stats.damaged, stats.skipped, stats.pages)
|
|
|
|
// The corpus is allowed to contain broken files, but a run in which most of
|
|
// them fail to open means the library regressed, not that the disk filled
|
|
// up with rubbish overnight.
|
|
if stats.files > 0 {
|
|
ratio := f64(stats.opened) / f64(stats.files)
|
|
testing.expectf(t, ratio >= 0.5, "only %v of %v corpus files opened (%.0f%%)",
|
|
stats.opened, stats.files, ratio * 100)
|
|
}
|
|
}
|
|
|
|
// A second pass that re-opens every file with the page tree skipped and
|
|
// resolves every object in it. This is the broadest reachability test
|
|
// available: it drives the parser, the object streams, the decryption and the
|
|
// codecs over data no unit test invents.
|
|
@(test)
|
|
test_corpus_objects :: proc(t: ^testing.T) {
|
|
dir := corpus_dir()
|
|
defer delete(dir)
|
|
if dir == "" || !os.is_dir(dir) { return }
|
|
|
|
paths := corpus_collect(dir)
|
|
defer corpus_release(&paths)
|
|
|
|
checked, streams := 0, 0
|
|
for p in paths {
|
|
if corpus_size(p) > CORPUS_MAX_FILE { continue }
|
|
|
|
doc, err := open_file(p, Open_Options{skip_page_tree = true})
|
|
defer close(doc)
|
|
if err != .None || doc == nil { continue }
|
|
checked += 1
|
|
|
|
// The cross-reference table is what `get_object` indexes and what
|
|
// `objstm_load` trusts, so its shape is a memory-safety property
|
|
// rather than a cosmetic one, and every field in it came out of the
|
|
// file. `count >= 0` stood here and could not fail; these are the
|
|
// bounds the loader actually enforces against an attacker-controlled
|
|
// /Size and attacker-controlled offsets.
|
|
name := filepath.base(p)
|
|
count := object_count(doc)
|
|
testing.expectf(t, count <= xref_max_objects(doc),
|
|
"%s: %v xref entries for a %v byte file", name, count, len(doc.data))
|
|
|
|
for num in 1 ..< count {
|
|
e := doc.xref[num]
|
|
#partial switch e.kind {
|
|
case .Normal:
|
|
testing.expectf(t, e.offset > 0 && e.offset < i64(len(doc.data)),
|
|
"%s: object %v is at offset %v of a %v byte file",
|
|
name, num, e.offset, len(doc.data))
|
|
case .Compressed:
|
|
testing.expectf(t, e.container != 0 && e.container <= MAX_OBJECT_NUMBER,
|
|
"%s: object %v is compressed into container %v", name, num, e.container)
|
|
}
|
|
|
|
obj := get_object(doc, Ref{u32(num), e.gen})
|
|
if s, ok := to_stream(obj); ok {
|
|
corpus_check_stream(t, doc, s, name)
|
|
streams += 1
|
|
}
|
|
}
|
|
free_all(context.temp_allocator)
|
|
}
|
|
|
|
if checked > 0 {
|
|
log.infof("corpus objects: walked %v files, decoded %v streams", checked, streams)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Fuzz-shaped tests over the corpus
|
|
//
|
|
// Truncating or corrupting a real file is the cheapest way to produce a
|
|
// malformed one that is still realistic enough to reach code a hand-written
|
|
// case never does.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
@(test)
|
|
test_corpus_truncated :: proc(t: ^testing.T) {
|
|
dir := corpus_dir()
|
|
defer delete(dir)
|
|
if dir == "" || !os.is_dir(dir) { return }
|
|
|
|
paths := corpus_collect(dir)
|
|
defer corpus_release(&paths)
|
|
|
|
// A handful of files is enough: this looks for crashes in the recovery
|
|
// path, and every file exercises the same path.
|
|
limit := min(len(paths), 12)
|
|
for i in 0 ..< limit {
|
|
data, rerr := os.read_entire_file(paths[i], context.allocator)
|
|
if rerr != nil { continue }
|
|
defer delete(data)
|
|
if len(data) > 8 << 20 { continue }
|
|
|
|
// Cuts at a spread of fractions, so the file ends inside the trailer,
|
|
// inside an object, and inside a stream in turn.
|
|
for num in 1 ..< 8 {
|
|
cut := len(data) * num / 8
|
|
if cut <= 0 { continue }
|
|
doc, _ := open_bytes(data[:cut])
|
|
// No assertion on the outcome: opening a truncated file may
|
|
// legitimately succeed -- the xref rebuild is good at it -- or
|
|
// fail. The test is that control reaches the next line at all.
|
|
if doc != nil {
|
|
for pi in 0 ..< min(page_count(doc), 4) {
|
|
page, perr := page_get(doc, pi)
|
|
if perr != .None { continue }
|
|
testing.expectf(t, rect_is_finite(page.media_box),
|
|
"%s: truncated at %v gave a non-finite media box %v",
|
|
filepath.base(paths[i]), cut, page.media_box)
|
|
}
|
|
}
|
|
close(doc)
|
|
}
|
|
free_all(context.temp_allocator)
|
|
}
|
|
}
|
|
|
|
// A byte flipped in a real file reaches paths truncation does not: a corrupted
|
|
// /Length, a broken filter header, a mangled xref offset.
|
|
@(test)
|
|
test_corpus_bitflips :: proc(t: ^testing.T) {
|
|
dir := corpus_dir()
|
|
defer delete(dir)
|
|
if dir == "" || !os.is_dir(dir) { return }
|
|
|
|
paths := corpus_collect(dir)
|
|
defer corpus_release(&paths)
|
|
|
|
limit := min(len(paths), 6)
|
|
for i in 0 ..< limit {
|
|
data, rerr := os.read_entire_file(paths[i], context.allocator)
|
|
if rerr != nil { continue }
|
|
defer delete(data)
|
|
if len(data) > 4 << 20 || len(data) < 64 { continue }
|
|
|
|
scratch := make([]byte, len(data))
|
|
defer delete(scratch)
|
|
|
|
FLIPS :: 32
|
|
for k in 0 ..< FLIPS {
|
|
copy(scratch, data)
|
|
// The flips are spread evenly rather than randomly: a reproducible
|
|
// failure is worth more here than a broader search.
|
|
at := len(scratch) * k / FLIPS
|
|
scratch[at] ~= 1 << u8(k % 8)
|
|
|
|
doc, _ := open_bytes(scratch)
|
|
if doc != nil {
|
|
for pi in 0 ..< min(page_count(doc), 8) {
|
|
page, perr := page_get(doc, pi)
|
|
if perr != .None { continue }
|
|
testing.expectf(t, rect_is_finite(page.media_box),
|
|
"%s: flip at %v gave a non-finite media box %v",
|
|
filepath.base(paths[i]), at, page.media_box)
|
|
}
|
|
}
|
|
close(doc)
|
|
}
|
|
free_all(context.temp_allocator)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// The resource sweep
|
|
//
|
|
// Every font, image, shading and pattern named by every page of every corpus
|
|
// file, loaded and driven. This is the first thing in the library that touches
|
|
// real *subsetted* font programs at scale, and a subsetter mangles a font more
|
|
// than anything else in the format mangles anything: it keeps `maxp.numGlyphs`
|
|
// at the original count while emptying `loca`, it strips a Type1C down to
|
|
// .notdef and space, it leaves a `cmap` naming glyphs that are no longer there.
|
|
// None of that is invalid and none of it may crash, hang or produce garbage.
|
|
//
|
|
// What is asserted is what the layer promises and no more:
|
|
//
|
|
// * `font_load` always hands back a Font. A font whose program cannot be
|
|
// read still has to lay text out, so failure is a flag on the result, not
|
|
// an error return.
|
|
// * A font with an EMBEDDED program either has glyphs or has `program_failed`
|
|
// set. A font with no embedded program has neither, and that is not a
|
|
// defect -- it is a standard-14 or a system font, and the widths are the
|
|
// whole of what the file gave.
|
|
// * Every outline that comes back is finite and inside a sane box. This is
|
|
// the "no NaN below the Device line" rule at the one boundary where the
|
|
// numbers come from a charstring interpreter rather than from the file.
|
|
// * `font_decode` consumes exactly the bytes it is given, one code at a
|
|
// time, never zero. The interpreter advances the pen by `nbytes`, so a
|
|
// zero there is an infinite loop on a real page.
|
|
// * An image either decodes to a bitmap of exactly its own dimensions -- one
|
|
// channel for a stencil, four otherwise -- or degrades to an error and no
|
|
// bitmap. A short buffer is the failure that matters: it is an
|
|
// out-of-bounds read in whatever composites it.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// Glyphs probed per font. A CJK font declares 20000 glyphs of which a subset
|
|
// keeps twenty, so the probe is spread across the whole range rather than
|
|
// taken from the front; the cap is what keeps the sweep to seconds.
|
|
@(private = "file")
|
|
CORPUS_GLYPH_PROBES :: 256
|
|
|
|
// A glyph-space coordinate larger than this is not a glyph. Real outlines live
|
|
// within a few thousand units of the origin whatever the em square is.
|
|
@(private = "file")
|
|
CORPUS_MAX_GLYPH_COORD :: 1e6
|
|
|
|
@(private = "file")
|
|
CORPUS_RES_DEPTH :: 6
|
|
|
|
// Decoding every image in a corpus of scanned documents is where this test
|
|
// spends its time -- a full-page scan is nine megapixels and there are
|
|
// thousands of them -- and the corpus directory is whatever the developer
|
|
// happened to point at. Past this many decoded pixels images are still loaded
|
|
// but no longer decoded, so a disk full of 600 dpi TIFFs cannot turn the
|
|
// suite into a coffee break. Neither corpus this was written against comes
|
|
// close to it.
|
|
@(private = "file")
|
|
CORPUS_PIXEL_BUDGET :: 1 << 30
|
|
|
|
@(private = "file")
|
|
Corpus_Res_Stats :: struct {
|
|
fonts, fonts_with_glyphs, fonts_failed, fonts_unembedded: int,
|
|
glyphs_probed, glyphs_drawn: int,
|
|
codes_decoded: int,
|
|
images, images_decoded, images_degraded, images_skipped: int,
|
|
pixels: i64,
|
|
shadings, patterns: int,
|
|
}
|
|
|
|
// corpus_glyph_count asks whichever program the Font ended up with. There is
|
|
// no `font_glyph_count`: nothing above this line needs one, because the
|
|
// interpreter asks for a glyph by id and takes "nothing to draw" for an answer.
|
|
@(private = "file")
|
|
corpus_glyph_count :: proc(f: ^Font) -> int {
|
|
if f == nil { return 0 }
|
|
// The order matters for an OpenType font whose outlines are CFF: both
|
|
// readers parsed it and only the CFF can draw, which is the same order
|
|
// `font_glyph_path` dispatches in.
|
|
if f.ttf != nil && tt_has_glyf(f.ttf) { return tt_glyph_count(f.ttf) }
|
|
if f.cff != nil { return cff_glyph_count(f.cff) }
|
|
if f.t1 != nil { return t1_glyph_count(f.t1) }
|
|
if f.ttf != nil { return tt_glyph_count(f.ttf) }
|
|
return 0
|
|
}
|
|
|
|
@(private = "file")
|
|
corpus_check_font :: proc(t: ^testing.T, doc: ^Document, fd: ^Dict, name, fname: string,
|
|
st: ^Corpus_Res_Stats) {
|
|
st.fonts += 1
|
|
|
|
f, err := font_load(doc, fd)
|
|
// Not an expectf on the error alone: a Font is what the caller is promised,
|
|
// and a nil one with .None would pass that check and crash the interpreter.
|
|
if !testing.expectf(t, err == .None && f != nil, "%s: font /%s: font_load -> %s",
|
|
name, fname, error_string(err)) {
|
|
return
|
|
}
|
|
|
|
n := corpus_glyph_count(f)
|
|
switch {
|
|
case f.kind == .Type3:
|
|
// Type3 glyphs are content streams, so there is no program to count.
|
|
// What must hold is the matrix: it is the font's own declaration of
|
|
// its glyph space, every glyph in it is drawn through it, and a
|
|
// singular or non-finite one makes every one of them infinite.
|
|
m := font_matrix(f)
|
|
testing.expectf(t, matrix_is_finite(m) && matrix_determinant(m) != 0,
|
|
"%s: font /%s: Type3 /FontMatrix is %v", name, fname, m)
|
|
case n > 0:
|
|
st.fonts_with_glyphs += 1
|
|
case f.embedded:
|
|
st.fonts_failed += 1
|
|
testing.expectf(t, f.program_failed,
|
|
"%s: font /%s (%s, %v): an embedded program yielded no glyphs "+
|
|
"and program_failed is not set", name, fname, f.base_font, f.kind)
|
|
case:
|
|
st.fonts_unembedded += 1
|
|
}
|
|
|
|
// Widths must work whether or not the program does: that is the whole
|
|
// reason a failed font still comes back.
|
|
for c in 0 ..< FONT_CODES {
|
|
w := f.code_width[c]
|
|
if !is_finite(w) || abs(w) > MAX_GLYPH_WIDTH {
|
|
testing.expectf(t, false, "%s: font /%s: code %v is %v wide", name, fname, c, w)
|
|
break
|
|
}
|
|
}
|
|
|
|
// Outlines. A glyph the subsetter dropped reports .No_Glyph or comes back
|
|
// empty, both of which are correct; anything else is not.
|
|
if n > 0 {
|
|
step := max(n / CORPUS_GLYPH_PROBES, 1)
|
|
p := path_make(context.temp_allocator)
|
|
for g := 0; g < n; g += step {
|
|
clear(&p.verbs)
|
|
clear(&p.points)
|
|
st.glyphs_probed += 1
|
|
gerr := font_glyph_path(f, u32(g), &p)
|
|
if gerr != .None && gerr != .No_Glyph {
|
|
testing.expectf(t, false, "%s: font /%s (%s): glyph %v -> %s",
|
|
name, fname, f.base_font, g, error_string(gerr))
|
|
break
|
|
}
|
|
if len(p.verbs) == 0 { continue }
|
|
st.glyphs_drawn += 1
|
|
bad := false
|
|
for pt in p.points {
|
|
if !is_finite(pt.x) || !is_finite(pt.y) ||
|
|
abs(pt.x) > CORPUS_MAX_GLYPH_COORD || abs(pt.y) > CORPUS_MAX_GLYPH_COORD {
|
|
testing.expectf(t, false, "%s: font /%s (%s): glyph %v has point %v",
|
|
name, fname, f.base_font, g, pt)
|
|
bad = true
|
|
break
|
|
}
|
|
}
|
|
if bad { break }
|
|
}
|
|
}
|
|
|
|
// Decoding. The probe is every byte value twice over, so a two-byte CMap
|
|
// sees each of 256 leading bytes and a one-byte font sees each code.
|
|
probe: [512]byte
|
|
for i in 0 ..< len(probe) { probe[i] = u8(i) }
|
|
codes := make([dynamic]Font_Code, 0, 64, context.temp_allocator)
|
|
font_decode(f, probe[:], &codes)
|
|
consumed := 0
|
|
for c in codes {
|
|
if c.nbytes <= 0 {
|
|
testing.expectf(t, false, "%s: font /%s: code %v consumed %v bytes",
|
|
name, fname, c.code, c.nbytes)
|
|
return
|
|
}
|
|
consumed += c.nbytes
|
|
if !is_finite(c.width) || abs(c.width) > MAX_GLYPH_WIDTH {
|
|
testing.expectf(t, false, "%s: font /%s: code %v is %v wide",
|
|
name, fname, c.code, c.width)
|
|
return
|
|
}
|
|
}
|
|
testing.expectf(t, consumed == len(probe),
|
|
"%s: font /%s (%s, %v): %v bytes in, %v consumed by %v codes",
|
|
name, fname, f.base_font, f.kind, len(probe), consumed, len(codes))
|
|
st.codes_decoded += len(codes)
|
|
}
|
|
|
|
@(private = "file")
|
|
corpus_check_image :: proc(t: ^testing.T, doc: ^Document, stm: ^Stream, res: ^Dict,
|
|
name, xname: string, st: ^Corpus_Res_Stats) {
|
|
st.images += 1
|
|
|
|
img, err := image_load(doc, stm, res)
|
|
if err != .None || img == nil {
|
|
// An image the loader refuses is skipped by the interpreter; what
|
|
// matters is that it said so instead of handing back a half-built one.
|
|
st.images_degraded += 1
|
|
testing.expectf(t, img == nil, "%s: image /%s: %s with an image anyway",
|
|
name, xname, error_string(err))
|
|
return
|
|
}
|
|
|
|
// Loading is cheap and is checked above for every image; decoding is not.
|
|
if st.pixels > CORPUS_PIXEL_BUDGET {
|
|
st.images_skipped += 1
|
|
return
|
|
}
|
|
st.pixels += i64(img.width) * i64(img.height)
|
|
|
|
bmp, derr := image_decode(doc, img)
|
|
if derr != .None || bmp == nil {
|
|
st.images_degraded += 1
|
|
testing.expectf(t, bmp == nil, "%s: image /%s: %s with a bitmap anyway",
|
|
name, xname, error_string(derr))
|
|
return
|
|
}
|
|
st.images_decoded += 1
|
|
|
|
want_n := img.is_mask ? 1 : 4
|
|
testing.expectf(t, bmp.width == img.width && bmp.height == img.height && bmp.n == want_n,
|
|
"%s: image /%s: decoded %vx%vx%v for a %vx%v image (mask=%v)",
|
|
name, xname, bmp.width, bmp.height, bmp.n, img.width, img.height, img.is_mask)
|
|
// The one failure that is an out-of-bounds read rather than a wrong pixel.
|
|
testing.expectf(t, len(bmp.pixels) >= bmp.height * bmp.stride && bmp.stride >= bmp.width * bmp.n,
|
|
"%s: image /%s: %v bytes for %v rows of %v", name, xname,
|
|
len(bmp.pixels), bmp.height, bmp.stride)
|
|
testing.expectf(t, !bmp.premultiplied,
|
|
"%s: image /%s: image_decode must hand back straight alpha", name, xname)
|
|
}
|
|
|
|
// corpus_check_resources walks one /Resources dictionary, recursing into form
|
|
// XObjects and tiling patterns, which is where most of a real file's fonts
|
|
// actually live. `seen` keys on the dictionary pointer, so a resource shared
|
|
// by 900 pages is done once and a cycle terminates.
|
|
@(private = "file")
|
|
corpus_check_resources :: proc(t: ^testing.T, doc: ^Document, res: ^Dict, name: string,
|
|
st: ^Corpus_Res_Stats, seen: ^map[rawptr]bool, depth: int) {
|
|
if res == nil || depth > CORPUS_RES_DEPTH { return }
|
|
if seen[rawptr(res)] { return }
|
|
seen[rawptr(res)] = true
|
|
|
|
if fonts := dict_dict(doc, res, "Font"); fonts != nil {
|
|
for fname, fobj in fonts {
|
|
fd, ok := to_dict(resolve(doc, fobj))
|
|
if !ok { continue }
|
|
if seen[rawptr(fd)] { continue }
|
|
seen[rawptr(fd)] = true
|
|
corpus_check_font(t, doc, fd, name, fname, st)
|
|
}
|
|
}
|
|
|
|
if xo := dict_dict(doc, res, "XObject"); xo != nil {
|
|
for xname, xobj in xo {
|
|
stm, ok := to_stream(resolve(doc, xobj))
|
|
if !ok { continue }
|
|
if seen[rawptr(stm)] { continue }
|
|
seen[rawptr(stm)] = true
|
|
if dict_name(doc, stm.dict, "Subtype") == "Form" {
|
|
corpus_check_resources(t, doc, dict_dict(doc, stm.dict, "Resources"),
|
|
name, st, seen, depth + 1)
|
|
continue
|
|
}
|
|
corpus_check_image(t, doc, stm, res, name, xname, st)
|
|
}
|
|
}
|
|
|
|
if shs := dict_dict(doc, res, "Shading"); shs != nil {
|
|
for sname, sobj in shs {
|
|
sh, serr := shading_load(doc, sobj, res)
|
|
if serr != .None || sh == nil { continue }
|
|
st.shadings += 1
|
|
bmp, berr := bitmap_make(16, 16, 4, context.temp_allocator)
|
|
if berr != .None { continue }
|
|
perr := shading_paint(doc, sh, &bmp, Matrix{1, 0, 0, 1, 8, 8},
|
|
Raster_Clip{0, 0, 16, 16}, 1, .Normal, nil)
|
|
testing.expectf(t, perr == .None, "%s: shading /%s (%v): %s",
|
|
name, sname, sh.kind, error_string(perr))
|
|
}
|
|
}
|
|
|
|
if pats := dict_dict(doc, res, "Pattern"); pats != nil {
|
|
for pname, pobj in pats {
|
|
pat, perr := pattern_load(doc, pobj, res)
|
|
if perr != .None || pat == nil { continue }
|
|
st.patterns += 1
|
|
if pat.kind == .Shading && pat.shading != nil {
|
|
bmp, berr := bitmap_make(16, 16, 4, context.temp_allocator)
|
|
if berr != .None { continue }
|
|
e := pattern_paint_shading(doc, pat, &bmp, MATRIX_IDENTITY,
|
|
Raster_Clip{0, 0, 16, 16}, 1, .Normal, nil)
|
|
testing.expectf(t, e == .None, "%s: shading pattern /%s: %s",
|
|
name, pname, error_string(e))
|
|
}
|
|
if pat.kind == .Tiling {
|
|
corpus_check_resources(t, doc, pat.resources, name, st, seen, depth + 1)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
@(test)
|
|
test_corpus_resources :: proc(t: ^testing.T) {
|
|
dir := corpus_dir()
|
|
defer delete(dir)
|
|
if dir == "" || !os.is_dir(dir) { return }
|
|
|
|
paths := corpus_collect(dir)
|
|
defer corpus_release(&paths)
|
|
|
|
st: Corpus_Res_Stats
|
|
files := 0
|
|
for p in paths {
|
|
if corpus_size(p) > CORPUS_MAX_FILE { continue }
|
|
|
|
doc, err := open_file(p)
|
|
defer close(doc)
|
|
if err != .None || doc == nil { continue }
|
|
files += 1
|
|
|
|
name := filepath.base(p)
|
|
seen := make(map[rawptr]bool)
|
|
defer delete(seen)
|
|
|
|
for i in 0 ..< page_count(doc) {
|
|
page, perr := page_get(doc, i)
|
|
if perr != .None { continue }
|
|
corpus_check_resources(t, doc, page.resources, name, &st, &seen, 0)
|
|
free_all(context.temp_allocator)
|
|
}
|
|
}
|
|
|
|
if files == 0 { return }
|
|
log.infof("corpus resources: %v files | fonts %v (%v with glyphs, %v failed, %v unembedded), "+
|
|
"%v glyphs probed, %v drawn, %v codes decoded | images %v (%v decoded, %v degraded, "+
|
|
"%v not decoded, %v megapixels) | "+
|
|
"%v shadings, %v patterns",
|
|
files, st.fonts, st.fonts_with_glyphs, st.fonts_failed, st.fonts_unembedded,
|
|
st.glyphs_probed, st.glyphs_drawn, st.codes_decoded,
|
|
st.images, st.images_decoded, st.images_degraded, st.images_skipped,
|
|
st.pixels / (1 << 20), st.shadings, st.patterns)
|
|
|
|
// A corpus that reached no font at all means the walk broke, not that
|
|
// seventy documents contain no text.
|
|
testing.expectf(t, st.fonts > 0, "the sweep found no fonts in %v files", files)
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// The rendering sweep
|
|
//
|
|
// Everything above this point checks that the library survives the corpus.
|
|
// This checks that it *draws* it, which is a different and much stronger
|
|
// claim: a page can parse, load every font, decode every image and still
|
|
// rasterise to nothing at all, because the text never drew, a clip swallowed
|
|
// the page, or the base matrix put the content off the canvas. None of those
|
|
// show up as an error -- they show up as a white rectangle.
|
|
//
|
|
// So the invariant here is measured in ink. Every page is rendered and the
|
|
// fraction of pixels that are not the untouched background is recorded; the
|
|
// distribution is logged, and a page that comes back blank is named. That
|
|
// number is not a correctness proof -- a page can be inked and wrong -- but
|
|
// blankness is a one-sided test that is never a false alarm on a page with
|
|
// content on it, and it is the only automatic check there is that the
|
|
// interpreter and the devices are connected to each other at all.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// 72 dpi. Big enough that a 6-point glyph still deposits ink, small enough
|
|
// that three thousand pages finish in a minute.
|
|
@(private = "file")
|
|
CORPUS_RENDER_SCALE :: 1.0
|
|
|
|
// A document with fewer pages than this renders all of them; a longer one
|
|
// renders only its first. A 900-page standard would otherwise dominate the
|
|
// sweep while telling it nothing its first fifty pages did not.
|
|
@(private = "file")
|
|
CORPUS_RENDER_ALL_UNDER :: 20
|
|
|
|
// A page that takes longer than this has found a loop bound that is too
|
|
// generous. It is checked after the fact -- the runner cannot interrupt a
|
|
// running test -- which is enough to catch a regression from milliseconds to
|
|
// minutes.
|
|
@(private = "file")
|
|
CORPUS_RENDER_MAX_MS :: 30_000
|
|
|
|
// Under one pixel in ten thousand is blank: an accidental hairline or a single
|
|
// anti-aliased dot is not a rendered page.
|
|
@(private = "file")
|
|
CORPUS_BLANK :: 0.0001
|
|
|
|
// Under a tenth of a percent is near-blank -- a page number and nothing else.
|
|
@(private = "file")
|
|
CORPUS_NEAR_BLANK :: 0.001
|
|
|
|
// A page carrying this much text has a paragraph or more on it, and a page
|
|
// with a paragraph on it covers well over a percent of itself in ink. Below
|
|
// that the page drew a fraction of what the interpreter read, which is the
|
|
// partial version of the blank-page bug and just as much a defect.
|
|
@(private = "file")
|
|
CORPUS_DENSE_CHARS :: 1000
|
|
@(private = "file")
|
|
CORPUS_STARVED_INK :: 0.01
|
|
|
|
@(private = "file")
|
|
Corpus_Render_Stats :: struct {
|
|
files: int,
|
|
pages: int,
|
|
errors: int,
|
|
blank: int,
|
|
near_blank: int,
|
|
first_blank: int, // files whose FIRST page is blank -- the loud case
|
|
// A blank page that extracts text is the signature this sweep exists to
|
|
// find: the interpreter reached the glyphs, so the page is not empty --
|
|
// something between there and the pixels dropped them.
|
|
blank_with_text: int,
|
|
empty: int, // blank and extracts nothing: genuinely a blank sheet
|
|
starved: int, // a paragraph of text and almost no ink
|
|
ink_sum: f64,
|
|
hist: [10]int,
|
|
slowest_ms: f64,
|
|
slowest_name: string,
|
|
total_ms: f64,
|
|
max_pixels: int,
|
|
}
|
|
|
|
// The bucket boundaries the histogram reports, as fractions of the page.
|
|
@(private = "file")
|
|
CORPUS_INK_BUCKETS := [9]f64{0.0001, 0.001, 0.005, 0.01, 0.02, 0.05, 0.10, 0.25, 0.50}
|
|
|
|
@(private = "file")
|
|
corpus_ink_bucket :: proc(f: f64) -> int {
|
|
for edge, i in CORPUS_INK_BUCKETS {
|
|
if f < edge { return i }
|
|
}
|
|
return len(CORPUS_INK_BUCKETS)
|
|
}
|
|
|
|
// corpus_ink is the share of pixels that are not the background the render
|
|
// started from. `render_page` leaves straight alpha and an opaque white page,
|
|
// so anything that is not pure white was painted.
|
|
@(private = "file")
|
|
corpus_ink :: proc(bmp: ^Bitmap) -> f64 {
|
|
if bmp.width <= 0 || bmp.height <= 0 || bmp.n < 3 { return 0 }
|
|
inked := 0
|
|
for y in 0 ..< bmp.height {
|
|
row := bitmap_row(bmp, y)
|
|
for x in 0 ..< bmp.width {
|
|
p := row[x * bmp.n:]
|
|
if p[0] != 255 || p[1] != 255 || p[2] != 255 {
|
|
inked += 1
|
|
}
|
|
}
|
|
}
|
|
return f64(inked) / f64(bmp.width * bmp.height)
|
|
}
|
|
|
|
// corpus_render_page renders one page and holds it to the contract: a bitmap
|
|
// of exactly the size `page_size` promised, no error, and no runaway time.
|
|
@(private = "file")
|
|
corpus_render_page :: proc(t: ^testing.T, page: ^Page, name: string, index: int,
|
|
st: ^Corpus_Render_Stats) -> (ink: f64, ok: bool) {
|
|
want_w, want_h := page_size(page, CORPUS_RENDER_SCALE)
|
|
|
|
start := time.now()
|
|
bmp, err := render_page(page, Render_Options{
|
|
ctm = scale_matrix(CORPUS_RENDER_SCALE, CORPUS_RENDER_SCALE),
|
|
})
|
|
ms := time.duration_milliseconds(time.since(start))
|
|
defer bitmap_destroy(&bmp)
|
|
|
|
st.total_ms += ms
|
|
if ms > st.slowest_ms {
|
|
st.slowest_ms = ms
|
|
st.slowest_name = name
|
|
}
|
|
testing.expectf(t, ms < CORPUS_RENDER_MAX_MS, "%s page %d took %.0f ms to render",
|
|
name, index + 1, ms)
|
|
|
|
if err != .None {
|
|
st.errors += 1
|
|
testing.expectf(t, false, "%s page %d: render_page returned %s",
|
|
name, index + 1, error_string(err))
|
|
return 0, false
|
|
}
|
|
|
|
// The size is part of the contract: `page_size` is what a caller allocates
|
|
// a window for before it ever calls render_page, and a bitmap that does
|
|
// not match it means one of the two derived the base matrix differently.
|
|
testing.expectf(t, bmp.width == want_w && bmp.height == want_h,
|
|
"%s page %d: rendered %dx%d, page_size said %dx%d",
|
|
name, index + 1, bmp.width, bmp.height, want_w, want_h)
|
|
testing.expectf(t, bmp.n == 4 && len(bmp.pixels) >= bmp.stride * bmp.height,
|
|
"%s page %d: %d components, %d bytes for %d rows of %d",
|
|
name, index + 1, bmp.n, len(bmp.pixels), bmp.height, bmp.stride)
|
|
testing.expectf(t, !bmp.premultiplied,
|
|
"%s page %d: the bitmap left the library premultiplied", name, index + 1)
|
|
|
|
st.max_pixels = max(st.max_pixels, bmp.width * bmp.height)
|
|
return corpus_ink(&bmp), true
|
|
}
|
|
|
|
@(test)
|
|
test_corpus_render :: proc(t: ^testing.T) {
|
|
dir := corpus_dir()
|
|
defer delete(dir)
|
|
if dir == "" {
|
|
log.info("PDF_ODIN_CORPUS is unset; skipping the render sweep")
|
|
return
|
|
}
|
|
if !os.is_dir(dir) { return }
|
|
|
|
paths := corpus_collect(dir)
|
|
defer corpus_release(&paths)
|
|
if len(paths) == 0 { return }
|
|
|
|
// Nothing a render allocates may outlive it. The Document arena is
|
|
// excluded on purpose -- that is freed by `close` and is not what this is
|
|
// watching -- so what is left is the bitmap, the devices and the scratch,
|
|
// and all three are supposed to be gone by the time the call returns.
|
|
track: mem.Tracking_Allocator
|
|
mem.tracking_allocator_init(&track, context.allocator)
|
|
defer mem.tracking_allocator_destroy(&track)
|
|
|
|
st: Corpus_Render_Stats
|
|
blanks := make([dynamic]string)
|
|
defer {
|
|
for b in blanks { delete(b) }
|
|
delete(blanks)
|
|
}
|
|
|
|
for p in paths {
|
|
if corpus_size(p) > CORPUS_MAX_FILE { continue }
|
|
|
|
doc, err := open_file(p)
|
|
defer close(doc)
|
|
if err != .None || doc == nil { continue }
|
|
st.files += 1
|
|
|
|
name := filepath.base(p)
|
|
n := page_count(doc)
|
|
last := n <= CORPUS_RENDER_ALL_UNDER ? n : 1
|
|
|
|
for i in 0 ..< last {
|
|
page, perr := page_get(doc, i)
|
|
if perr != .None { continue }
|
|
|
|
ink, ok := 0.0, false
|
|
{
|
|
context.allocator = mem.tracking_allocator(&track)
|
|
ink, ok = corpus_render_page(t, &page, name, i, &st)
|
|
}
|
|
if !ok { continue }
|
|
|
|
st.pages += 1
|
|
st.ink_sum += ink
|
|
st.hist[corpus_ink_bucket(ink)] += 1
|
|
|
|
// Ask the text device what the interpreter saw on any page that
|
|
// came out light. A page that draws nothing and extracts nothing
|
|
// is a blank sheet, which every corpus has; a page that draws
|
|
// nothing and extracts four hundred characters is a defect, and
|
|
// the two must never be reported as the same thing.
|
|
if ink < CORPUS_STARVED_INK {
|
|
text, _ := extract_text(&page)
|
|
defer delete(text)
|
|
chars := len(text)
|
|
|
|
if ink < CORPUS_BLANK {
|
|
st.blank += 1
|
|
if i == 0 { st.first_blank += 1 }
|
|
if chars > 0 { st.blank_with_text += 1 } else { st.empty += 1 }
|
|
append(&blanks, corpus_blank_line(name, i, ink, chars))
|
|
} else if ink < CORPUS_NEAR_BLANK {
|
|
st.near_blank += 1
|
|
if chars > 0 { st.blank_with_text += 1 } else { st.empty += 1 }
|
|
append(&blanks, corpus_blank_line(name, i, ink, chars))
|
|
} else if chars >= CORPUS_DENSE_CHARS {
|
|
st.starved += 1
|
|
append(&blanks, corpus_blank_line(name, i, ink, chars))
|
|
}
|
|
}
|
|
}
|
|
free_all(context.temp_allocator)
|
|
}
|
|
|
|
if st.files == 0 { return }
|
|
|
|
log.infof("render sweep: %d files, %d pages rendered at %gx, %d errors, "+
|
|
"%.1f%% mean ink, largest %d px, %.0f ms total, slowest %.0f ms (%s)",
|
|
st.files, st.pages, f64(CORPUS_RENDER_SCALE), st.errors,
|
|
st.pages > 0 ? st.ink_sum / f64(st.pages) * 100 : 0,
|
|
st.max_pixels, st.total_ms, st.slowest_ms, st.slowest_name)
|
|
|
|
labels := [10]string{
|
|
" 0 .. 0.01%", " 0.01 .. 0.1%", " 0.1 .. 0.5%",
|
|
" 0.5 .. 1%", " 1 .. 2%", " 2 .. 5%",
|
|
" 5 .. 10%", " 10 .. 25%", " 25 .. 50%", " 50 .. 100%",
|
|
}
|
|
for count, i in st.hist {
|
|
if count == 0 { continue }
|
|
log.infof(" ink %s : %5d pages", labels[i], count)
|
|
}
|
|
|
|
for b in blanks {
|
|
log.infof(" %s", b)
|
|
}
|
|
log.infof("render sweep: %d blank pages, %d near-blank, %d starved (a paragraph of text "+
|
|
"under %g%% ink), %d of them a file's first page; %d drew nothing but extract "+
|
|
"text, %d are genuinely empty sheets",
|
|
st.blank, st.near_blank, st.starved, f64(CORPUS_STARVED_INK) * 100, st.first_blank,
|
|
st.blank_with_text, st.empty)
|
|
|
|
// A sweep that rendered nothing would satisfy every assertion above.
|
|
testing.expectf(t, st.pages > 0, "the sweep rendered no pages from %d files", st.files)
|
|
|
|
// Blankness is the property this whole test exists for. A handful of
|
|
// genuinely empty pages is normal in any corpus -- a separator sheet, the
|
|
// back of a cover -- but a tenth of the corpus coming back white is a bug
|
|
// in the interpreter, not seventy documents full of empty paper.
|
|
if st.pages > 0 {
|
|
ratio := f64(st.blank) / f64(st.pages)
|
|
testing.expectf(t, ratio < 0.10, "%d of %d rendered pages (%.0f%%) came out blank",
|
|
st.blank, st.pages, ratio * 100)
|
|
}
|
|
|
|
// The allocations a render makes are its own to free. Assert the tracker
|
|
// saw traffic first: a leak check over an allocator nothing ever reached
|
|
// passes for the wrong reason, and rewiring `context.allocator` is exactly
|
|
// the sort of thing a later edit breaks silently.
|
|
testing.expectf(t, track.total_allocation_count > 0,
|
|
"the render sweep tracked no allocations at all over %d pages", st.pages)
|
|
log.infof("render sweep: %d allocations, %d MiB peak, %d MiB total through the tracker",
|
|
track.total_allocation_count, track.peak_memory_allocated / (1 << 20),
|
|
track.total_memory_allocated / (1 << 20))
|
|
|
|
leaked := 0
|
|
for _, entry in track.allocation_map {
|
|
leaked += entry.size
|
|
}
|
|
testing.expectf(t, len(track.allocation_map) == 0,
|
|
"%d allocations (%d bytes) survived the render sweep", len(track.allocation_map), leaked)
|
|
testing.expectf(t, len(track.bad_free_array) == 0,
|
|
"%d bad frees during the render sweep", len(track.bad_free_array))
|
|
}
|
|
|
|
@(private = "file")
|
|
corpus_blank_line :: proc(name: string, index: int, ink: f64, chars: int) -> string {
|
|
what := "empty sheet "
|
|
if chars > 0 {
|
|
what = ink < CORPUS_NEAR_BLANK ? "DREW NOTHING" : "STARVED "
|
|
}
|
|
return fmt.aprintf("%s %s page %d: %.4f%% ink, %d characters of text",
|
|
what, name, index + 1, ink * 100, chars)
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Text extraction over the corpus
|
|
//
|
|
// The same interpreter, into the text device instead of the rasteriser. What
|
|
// it measures is coverage of the unicode chain -- /ToUnicode, the encoding
|
|
// differences, the predefined CMaps and the AGL -- because a code that
|
|
// resolves to nothing extracts as U+FFFD, and a page that is mostly U+FFFD is
|
|
// text nobody can search or copy even though it draws correctly.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
@(test)
|
|
test_corpus_text :: proc(t: ^testing.T) {
|
|
dir := corpus_dir()
|
|
defer delete(dir)
|
|
if dir == "" {
|
|
log.info("PDF_ODIN_CORPUS is unset; skipping the text sweep")
|
|
return
|
|
}
|
|
if !os.is_dir(dir) { return }
|
|
|
|
paths := corpus_collect(dir)
|
|
defer corpus_release(&paths)
|
|
if len(paths) == 0 { return }
|
|
|
|
files, pages, with_text, chars, runes_, bad := 0, 0, 0, 0, 0, 0
|
|
|
|
// Every page that is more than a tenth replacement characters, by name:
|
|
// a ratio alone says the chain is failing somewhere and not where, and
|
|
// "somewhere" is not something anyone can act on.
|
|
suspect := make([dynamic]string)
|
|
defer {
|
|
for s in suspect { delete(s) }
|
|
delete(suspect)
|
|
}
|
|
|
|
for p in paths {
|
|
if corpus_size(p) > CORPUS_MAX_FILE { continue }
|
|
|
|
doc, err := open_file(p)
|
|
defer close(doc)
|
|
if err != .None || doc == nil { continue }
|
|
files += 1
|
|
name := filepath.base(p)
|
|
|
|
for i in 0 ..< page_count(doc) {
|
|
page, perr := page_get(doc, i)
|
|
if perr != .None { continue }
|
|
|
|
text, terr := extract_text(&page)
|
|
defer delete(text)
|
|
testing.expectf(t, terr == .None, "%s page %d: extract_text returned %s",
|
|
name, i + 1, error_string(terr))
|
|
pages += 1
|
|
|
|
// Whatever comes out must be valid UTF-8: it is a Go-style string
|
|
// that goes straight into a caller's search index.
|
|
testing.expectf(t, utf8.valid_string(text),
|
|
"%s page %d: extracted text is not valid UTF-8", name, i + 1)
|
|
|
|
nr, nbad := 0, 0
|
|
for r in text {
|
|
nr += 1
|
|
if r == utf8.RUNE_ERROR { nbad += 1 }
|
|
}
|
|
chars += len(text)
|
|
runes_ += nr
|
|
bad += nbad
|
|
if len(text) > 0 { with_text += 1 }
|
|
|
|
// Short strings make wild ratios; a page with a handful of glyphs
|
|
// that all failed is not the signal this is looking for.
|
|
if nr >= 32 {
|
|
frac := f64(nbad) / f64(nr)
|
|
if frac > 0.10 {
|
|
append(&suspect, fmt.aprintf("%s page %d: %.0f%% U+FFFD (%d of %d runes)",
|
|
name, i + 1, frac * 100, nbad, nr))
|
|
}
|
|
}
|
|
free_all(context.temp_allocator)
|
|
}
|
|
}
|
|
|
|
if files == 0 { return }
|
|
log.infof("text sweep: %d files, %d pages, %d with text (%.0f%%), %d bytes, %d runes, "+
|
|
"%d U+FFFD (%.3f%%)",
|
|
files, pages, with_text, pages > 0 ? f64(with_text) / f64(pages) * 100 : 0,
|
|
chars, runes_, bad, runes_ > 0 ? f64(bad) / f64(runes_) * 100 : 0)
|
|
for s in suspect {
|
|
log.infof(" UNRESOLVED %s", s)
|
|
}
|
|
if len(suspect) > 0 {
|
|
log.infof("text sweep: %d pages are over 10%% U+FFFD", len(suspect))
|
|
}
|
|
|
|
testing.expectf(t, with_text > 0, "no page in %d files yielded any text", files)
|
|
|
|
// A corpus this size is mostly ordinary prose in ordinary fonts. If one
|
|
// rune in twenty comes out as U+FFFD the unicode chain is broken, not the
|
|
// files.
|
|
if runes_ > 0 {
|
|
ratio := f64(bad) / f64(runes_)
|
|
testing.expectf(t, ratio < 0.05, "%d of %d extracted runes (%.1f%%) are U+FFFD",
|
|
bad, runes_, ratio * 100)
|
|
}
|
|
}
|