package pdf import "core:fmt" import "core:log" import "core:mem" import "core:os" import "core:path/filepath" import "core:slice" import "core:strings" import "core:testing" import "core:time" import "core:unicode/utf8" // --------------------------------------------------------------------------- // Corpus tests // // These run over whatever PDFs are in the directory named by PDF_ODIN_CORPUS // and skip silently when it is unset, so the suite never depends on files that // are not in the repository. // // They assert invariants, not results. There is no correct answer to check a // real-world file against -- that is what the golden files in testdata/ are for // -- so what is checked here is the contract every file must satisfy: opening // terminates, whatever comes back is self-consistent, and nothing crashes, // hangs or leaks. That is precisely the class of bug a corpus finds and a unit // test cannot. // --------------------------------------------------------------------------- // A file that takes longer than this is logged. It is not a hard timeout -- // the test runner cannot interrupt a running test -- but a file that trips it // has found a loop bound that is too generous, and that is worth seeing. @(private = "file") CORPUS_SLOW_MS :: 5000 // Files larger than this are skipped. The corpus directory is whatever the // developer happened to point at, and a 500 MB scanned document exercises // nothing a 5 MB one does not. @(private = "file") CORPUS_MAX_FILE :: 64 << 20 @(private = "file") CORPUS_MAX_DEPTH :: 8 @(private = "file") Corpus_Stats :: struct { files: int, opened: int, failed: int, damaged: int, pages: int, skipped: int, } // corpus_dir returns the corpus directory, or "" when the variable is unset. @(private = "file") corpus_dir :: proc(allocator := context.allocator) -> string { raw := os.get_env("PDF_ODIN_CORPUS", allocator) trimmed := strings.trim_space(raw) // trim_space returns a slice of `raw`, so the original allocation is still // the one to free; hand back a clone and release it. out := strings.clone(trimmed, allocator) delete(raw, allocator) return out } // corpus_files lists the PDFs under `dir`, recursively and in a stable order, // so that two runs report the same files in the same sequence. @(private = "file") corpus_files :: proc(dir: string, out: ^[dynamic]string, depth := 0) { if depth > CORPUS_MAX_DEPTH { return } entries, err := os.read_directory_by_path(dir, -1, context.allocator) if err != nil { return } defer os.file_info_slice_delete(entries, context.allocator) slice.sort_by(entries, proc(a, b: os.File_Info) -> bool { return a.name < b.name }) for e in entries { if e.type == .Directory { corpus_files(e.fullpath, out, depth + 1) continue } if e.type != .Regular { continue } lower := strings.to_lower(e.name, context.temp_allocator) if !strings.has_suffix(lower, ".pdf") { continue } append(out, strings.clone(e.fullpath)) } } @(private = "file") corpus_collect :: proc(dir: string) -> [dynamic]string { paths := make([dynamic]string) corpus_files(dir, &paths) return paths } @(private = "file") corpus_release :: proc(paths: ^[dynamic]string) { for p in paths { delete(p) } delete(paths^) } @(private = "file") corpus_size :: proc(path: string) -> i64 { info, err := os.stat(path, context.allocator) if err != nil { return 0 } defer os.file_info_delete(info, context.allocator) return info.size } // corpus_check_one opens one file and asserts the invariants on it. // // Every failure names the file: a corpus run that says only "3 files failed" // costs more time than it saves. @(private = "file") corpus_check_one :: proc(t: ^testing.T, path: string, stats: ^Corpus_Stats) { if corpus_size(path) > CORPUS_MAX_FILE { stats.skipped += 1 return } name := filepath.base(path) stats.files += 1 started := time.now() doc, err := open_file(path) defer close(doc) if err != .None { // A file that will not open is not automatically a bug: a corpus // collected off a disk contains genuinely broken files, and .No_Pages // on a zero-page document is the right answer. What must never happen // is a crash or a hang, and reaching this line proves neither did. stats.failed += 1 log.infof("corpus: %s: open failed: %s", name, error_string(err)) return } if !testing.expectf(t, doc != nil, "%s: open reported success but gave no document", name) { return } stats.opened += 1 if doc.is_damaged { stats.damaged += 1 } n := page_count(doc) testing.expectf(t, n > 0, "%s: opened with %v pages", name, n) stats.pages += n // page_get reads doc.pages and doc.page_refs in lockstep, so a length // mismatch is an out-of-range read waiting to happen. testing.expectf(t, len(doc.pages) == len(doc.page_refs), "%s: %v page dicts against %v page refs", name, len(doc.pages), len(doc.page_refs)) for i in 0 ..< n { page, perr := page_get(doc, i) if !testing.expectf(t, perr == .None, "%s: page %v: %s", name, i, error_string(perr)) { continue } testing.expectf(t, page.dict != nil, "%s: page %v has no dictionary", name, i) // No NaN or infinity may reach the device layer, and a zero-area page // divides to one. page_get is the boundary that has to guarantee it. testing.expectf(t, rect_is_finite(page.media_box), "%s: page %v media box is not finite: %v", name, i, page.media_box) testing.expectf(t, !rect_is_empty(page.media_box), "%s: page %v media box is empty: %v", name, i, page.media_box) testing.expectf(t, rect_is_finite(page.crop_box), "%s: page %v crop box is not finite: %v", name, i, page.crop_box) testing.expectf(t, !rect_is_empty(page.crop_box), "%s: page %v crop box is empty: %v", name, i, page.crop_box) testing.expectf(t, page.rotate >= 0 && page.rotate < 360 && page.rotate % 90 == 0, "%s: page %v rotate is %v", name, i, page.rotate) corpus_check_contents(t, doc, page, name) } testing.expectf(t, len(doc.repairs) <= MAX_REPAIRS, "%s: repair list grew past its cap: %v", name, len(doc.repairs)) ms := time.duration_milliseconds(time.since(started)) if ms > CORPUS_SLOW_MS { log.infof("corpus: %s: slow: %.0f ms for %v pages", name, ms, n) } } // corpus_check_contents decodes a page's content streams. A stream that will // not decode is not a test failure -- plenty of real files hold one, and the // reader is required to degrade rather than refuse -- so what is checked is // the contract `stream_data` states, which no file is allowed to break. @(private = "file") corpus_check_contents :: proc(t: ^testing.T, doc: ^Document, page: Page, name: string) { contents := dict_get(doc, page.dict, "Contents") #partial switch v in contents { case ^Stream: corpus_check_stream(t, doc, v, name) case ^Array: // A /Contents array in a real file has a handful of entries; one with // thousands is a file trying to make this loop expensive. limit := min(array_len(v), 4096) for i in 0 ..< limit { s, ok := to_stream(array_get(doc, v, i)) if !ok { continue } corpus_check_stream(t, doc, s, name) } } } // corpus_check_stream holds `stream_data` to its documented contract over real // data. // // Three properties, all of which a malformed file could plausibly break: // decoding a stream never reports an error (the error return is reserved for a // caller mistake -- a broken filter chain degrades to empty plus a repair); a // call always leaves the stream decided, never still .Pending; and the result // is cached, so the second call hands back the identical bytes rather than // decoding again. The last one is the reason nothing in the library frees what // this returns, and it is checked by pointer, not by length. @(private = "file") corpus_check_stream :: proc(t: ^testing.T, doc: ^Document, s: ^Stream, name: string) { if s == nil { return } data, err := stream_data(doc, s) testing.expectf(t, err == .None, "%s: stream_data returned %s", name, error_string(err)) testing.expectf(t, s.state != .Pending, "%s: stream_data left the stream undecoded", name) if s.state == .Failed { testing.expectf(t, data == nil, "%s: a stream that failed to decode returned %v bytes", name, len(data)) testing.expectf(t, doc.is_damaged, "%s: a stream failed to decode without recording a repair", name) } again, aerr := stream_data(doc, s) testing.expectf(t, aerr == .None, "%s: second stream_data returned %s", name, error_string(aerr)) testing.expectf(t, raw_data(again) == raw_data(data) && len(again) == len(data), "%s: stream_data decoded the same stream twice: %v bytes, then %v bytes", name, len(data), len(again)) } // --------------------------------------------------------------------------- @(test) test_corpus :: proc(t: ^testing.T) { dir := corpus_dir() defer delete(dir) if dir == "" { log.info("PDF_ODIN_CORPUS is unset; skipping the corpus test") return } if !os.is_dir(dir) { log.infof("PDF_ODIN_CORPUS is %q, which is not a directory; skipping", dir) return } paths := corpus_collect(dir) defer corpus_release(&paths) if len(paths) == 0 { log.infof("no PDFs under %q; skipping", dir) return } stats: Corpus_Stats for p in paths { corpus_check_one(t, p, &stats) free_all(context.temp_allocator) } log.infof("corpus %q: %v files, %v opened, %v failed, %v damaged, %v skipped, %v pages", dir, stats.files, stats.opened, stats.failed, stats.damaged, stats.skipped, stats.pages) // The corpus is allowed to contain broken files, but a run in which most of // them fail to open means the library regressed, not that the disk filled // up with rubbish overnight. if stats.files > 0 { ratio := f64(stats.opened) / f64(stats.files) testing.expectf(t, ratio >= 0.5, "only %v of %v corpus files opened (%.0f%%)", stats.opened, stats.files, ratio * 100) } } // A second pass that re-opens every file with the page tree skipped and // resolves every object in it. This is the broadest reachability test // available: it drives the parser, the object streams, the decryption and the // codecs over data no unit test invents. @(test) test_corpus_objects :: proc(t: ^testing.T) { dir := corpus_dir() defer delete(dir) if dir == "" || !os.is_dir(dir) { return } paths := corpus_collect(dir) defer corpus_release(&paths) checked, streams := 0, 0 for p in paths { if corpus_size(p) > CORPUS_MAX_FILE { continue } doc, err := open_file(p, Open_Options{skip_page_tree = true}) defer close(doc) if err != .None || doc == nil { continue } checked += 1 // The cross-reference table is what `get_object` indexes and what // `objstm_load` trusts, so its shape is a memory-safety property // rather than a cosmetic one, and every field in it came out of the // file. `count >= 0` stood here and could not fail; these are the // bounds the loader actually enforces against an attacker-controlled // /Size and attacker-controlled offsets. name := filepath.base(p) count := object_count(doc) testing.expectf(t, count <= xref_max_objects(doc), "%s: %v xref entries for a %v byte file", name, count, len(doc.data)) for num in 1 ..< count { e := doc.xref[num] #partial switch e.kind { case .Normal: testing.expectf(t, e.offset > 0 && e.offset < i64(len(doc.data)), "%s: object %v is at offset %v of a %v byte file", name, num, e.offset, len(doc.data)) case .Compressed: testing.expectf(t, e.container != 0 && e.container <= MAX_OBJECT_NUMBER, "%s: object %v is compressed into container %v", name, num, e.container) } obj := get_object(doc, Ref{u32(num), e.gen}) if s, ok := to_stream(obj); ok { corpus_check_stream(t, doc, s, name) streams += 1 } } free_all(context.temp_allocator) } if checked > 0 { log.infof("corpus objects: walked %v files, decoded %v streams", checked, streams) } } // --------------------------------------------------------------------------- // Fuzz-shaped tests over the corpus // // Truncating or corrupting a real file is the cheapest way to produce a // malformed one that is still realistic enough to reach code a hand-written // case never does. // --------------------------------------------------------------------------- @(test) test_corpus_truncated :: proc(t: ^testing.T) { dir := corpus_dir() defer delete(dir) if dir == "" || !os.is_dir(dir) { return } paths := corpus_collect(dir) defer corpus_release(&paths) // A handful of files is enough: this looks for crashes in the recovery // path, and every file exercises the same path. limit := min(len(paths), 12) for i in 0 ..< limit { data, rerr := os.read_entire_file(paths[i], context.allocator) if rerr != nil { continue } defer delete(data) if len(data) > 8 << 20 { continue } // Cuts at a spread of fractions, so the file ends inside the trailer, // inside an object, and inside a stream in turn. for num in 1 ..< 8 { cut := len(data) * num / 8 if cut <= 0 { continue } doc, _ := open_bytes(data[:cut]) // No assertion on the outcome: opening a truncated file may // legitimately succeed -- the xref rebuild is good at it -- or // fail. The test is that control reaches the next line at all. if doc != nil { for pi in 0 ..< min(page_count(doc), 4) { page, perr := page_get(doc, pi) if perr != .None { continue } testing.expectf(t, rect_is_finite(page.media_box), "%s: truncated at %v gave a non-finite media box %v", filepath.base(paths[i]), cut, page.media_box) } } close(doc) } free_all(context.temp_allocator) } } // A byte flipped in a real file reaches paths truncation does not: a corrupted // /Length, a broken filter header, a mangled xref offset. @(test) test_corpus_bitflips :: proc(t: ^testing.T) { dir := corpus_dir() defer delete(dir) if dir == "" || !os.is_dir(dir) { return } paths := corpus_collect(dir) defer corpus_release(&paths) limit := min(len(paths), 6) for i in 0 ..< limit { data, rerr := os.read_entire_file(paths[i], context.allocator) if rerr != nil { continue } defer delete(data) if len(data) > 4 << 20 || len(data) < 64 { continue } scratch := make([]byte, len(data)) defer delete(scratch) FLIPS :: 32 for k in 0 ..< FLIPS { copy(scratch, data) // The flips are spread evenly rather than randomly: a reproducible // failure is worth more here than a broader search. at := len(scratch) * k / FLIPS scratch[at] ~= 1 << u8(k % 8) doc, _ := open_bytes(scratch) if doc != nil { for pi in 0 ..< min(page_count(doc), 8) { page, perr := page_get(doc, pi) if perr != .None { continue } testing.expectf(t, rect_is_finite(page.media_box), "%s: flip at %v gave a non-finite media box %v", filepath.base(paths[i]), at, page.media_box) } } close(doc) } free_all(context.temp_allocator) } } // --------------------------------------------------------------------------- // The resource sweep // // Every font, image, shading and pattern named by every page of every corpus // file, loaded and driven. This is the first thing in the library that touches // real *subsetted* font programs at scale, and a subsetter mangles a font more // than anything else in the format mangles anything: it keeps `maxp.numGlyphs` // at the original count while emptying `loca`, it strips a Type1C down to // .notdef and space, it leaves a `cmap` naming glyphs that are no longer there. // None of that is invalid and none of it may crash, hang or produce garbage. // // What is asserted is what the layer promises and no more: // // * `font_load` always hands back a Font. A font whose program cannot be // read still has to lay text out, so failure is a flag on the result, not // an error return. // * A font with an EMBEDDED program either has glyphs or has `program_failed` // set. A font with no embedded program has neither, and that is not a // defect -- it is a standard-14 or a system font, and the widths are the // whole of what the file gave. // * Every outline that comes back is finite and inside a sane box. This is // the "no NaN below the Device line" rule at the one boundary where the // numbers come from a charstring interpreter rather than from the file. // * `font_decode` consumes exactly the bytes it is given, one code at a // time, never zero. The interpreter advances the pen by `nbytes`, so a // zero there is an infinite loop on a real page. // * An image either decodes to a bitmap of exactly its own dimensions -- one // channel for a stencil, four otherwise -- or degrades to an error and no // bitmap. A short buffer is the failure that matters: it is an // out-of-bounds read in whatever composites it. // --------------------------------------------------------------------------- // Glyphs probed per font. A CJK font declares 20000 glyphs of which a subset // keeps twenty, so the probe is spread across the whole range rather than // taken from the front; the cap is what keeps the sweep to seconds. @(private = "file") CORPUS_GLYPH_PROBES :: 256 // A glyph-space coordinate larger than this is not a glyph. Real outlines live // within a few thousand units of the origin whatever the em square is. @(private = "file") CORPUS_MAX_GLYPH_COORD :: 1e6 @(private = "file") CORPUS_RES_DEPTH :: 6 // Decoding every image in a corpus of scanned documents is where this test // spends its time -- a full-page scan is nine megapixels and there are // thousands of them -- and the corpus directory is whatever the developer // happened to point at. Past this many decoded pixels images are still loaded // but no longer decoded, so a disk full of 600 dpi TIFFs cannot turn the // suite into a coffee break. Neither corpus this was written against comes // close to it. @(private = "file") CORPUS_PIXEL_BUDGET :: 1 << 30 @(private = "file") Corpus_Res_Stats :: struct { fonts, fonts_with_glyphs, fonts_failed, fonts_unembedded: int, glyphs_probed, glyphs_drawn: int, codes_decoded: int, images, images_decoded, images_degraded, images_skipped: int, pixels: i64, shadings, patterns: int, } // corpus_glyph_count asks whichever program the Font ended up with. There is // no `font_glyph_count`: nothing above this line needs one, because the // interpreter asks for a glyph by id and takes "nothing to draw" for an answer. @(private = "file") corpus_glyph_count :: proc(f: ^Font) -> int { if f == nil { return 0 } // The order matters for an OpenType font whose outlines are CFF: both // readers parsed it and only the CFF can draw, which is the same order // `font_glyph_path` dispatches in. if f.ttf != nil && tt_has_glyf(f.ttf) { return tt_glyph_count(f.ttf) } if f.cff != nil { return cff_glyph_count(f.cff) } if f.t1 != nil { return t1_glyph_count(f.t1) } if f.ttf != nil { return tt_glyph_count(f.ttf) } return 0 } @(private = "file") corpus_check_font :: proc(t: ^testing.T, doc: ^Document, fd: ^Dict, name, fname: string, st: ^Corpus_Res_Stats) { st.fonts += 1 f, err := font_load(doc, fd) // Not an expectf on the error alone: a Font is what the caller is promised, // and a nil one with .None would pass that check and crash the interpreter. if !testing.expectf(t, err == .None && f != nil, "%s: font /%s: font_load -> %s", name, fname, error_string(err)) { return } n := corpus_glyph_count(f) switch { case f.kind == .Type3: // Type3 glyphs are content streams, so there is no program to count. // What must hold is the matrix: it is the font's own declaration of // its glyph space, every glyph in it is drawn through it, and a // singular or non-finite one makes every one of them infinite. m := font_matrix(f) testing.expectf(t, matrix_is_finite(m) && matrix_determinant(m) != 0, "%s: font /%s: Type3 /FontMatrix is %v", name, fname, m) case n > 0: st.fonts_with_glyphs += 1 case f.embedded: st.fonts_failed += 1 testing.expectf(t, f.program_failed, "%s: font /%s (%s, %v): an embedded program yielded no glyphs "+ "and program_failed is not set", name, fname, f.base_font, f.kind) case: st.fonts_unembedded += 1 } // Widths must work whether or not the program does: that is the whole // reason a failed font still comes back. for c in 0 ..< FONT_CODES { w := f.code_width[c] if !is_finite(w) || abs(w) > MAX_GLYPH_WIDTH { testing.expectf(t, false, "%s: font /%s: code %v is %v wide", name, fname, c, w) break } } // Outlines. A glyph the subsetter dropped reports .No_Glyph or comes back // empty, both of which are correct; anything else is not. if n > 0 { step := max(n / CORPUS_GLYPH_PROBES, 1) p := path_make(context.temp_allocator) for g := 0; g < n; g += step { clear(&p.verbs) clear(&p.points) st.glyphs_probed += 1 gerr := font_glyph_path(f, u32(g), &p) if gerr != .None && gerr != .No_Glyph { testing.expectf(t, false, "%s: font /%s (%s): glyph %v -> %s", name, fname, f.base_font, g, error_string(gerr)) break } if len(p.verbs) == 0 { continue } st.glyphs_drawn += 1 bad := false for pt in p.points { if !is_finite(pt.x) || !is_finite(pt.y) || abs(pt.x) > CORPUS_MAX_GLYPH_COORD || abs(pt.y) > CORPUS_MAX_GLYPH_COORD { testing.expectf(t, false, "%s: font /%s (%s): glyph %v has point %v", name, fname, f.base_font, g, pt) bad = true break } } if bad { break } } } // Decoding. The probe is every byte value twice over, so a two-byte CMap // sees each of 256 leading bytes and a one-byte font sees each code. probe: [512]byte for i in 0 ..< len(probe) { probe[i] = u8(i) } codes := make([dynamic]Font_Code, 0, 64, context.temp_allocator) font_decode(f, probe[:], &codes) consumed := 0 for c in codes { if c.nbytes <= 0 { testing.expectf(t, false, "%s: font /%s: code %v consumed %v bytes", name, fname, c.code, c.nbytes) return } consumed += c.nbytes if !is_finite(c.width) || abs(c.width) > MAX_GLYPH_WIDTH { testing.expectf(t, false, "%s: font /%s: code %v is %v wide", name, fname, c.code, c.width) return } } testing.expectf(t, consumed == len(probe), "%s: font /%s (%s, %v): %v bytes in, %v consumed by %v codes", name, fname, f.base_font, f.kind, len(probe), consumed, len(codes)) st.codes_decoded += len(codes) } @(private = "file") corpus_check_image :: proc(t: ^testing.T, doc: ^Document, stm: ^Stream, res: ^Dict, name, xname: string, st: ^Corpus_Res_Stats) { st.images += 1 img, err := image_load(doc, stm, res) if err != .None || img == nil { // An image the loader refuses is skipped by the interpreter; what // matters is that it said so instead of handing back a half-built one. st.images_degraded += 1 testing.expectf(t, img == nil, "%s: image /%s: %s with an image anyway", name, xname, error_string(err)) return } // Loading is cheap and is checked above for every image; decoding is not. if st.pixels > CORPUS_PIXEL_BUDGET { st.images_skipped += 1 return } st.pixels += i64(img.width) * i64(img.height) bmp, derr := image_decode(doc, img) if derr != .None || bmp == nil { st.images_degraded += 1 testing.expectf(t, bmp == nil, "%s: image /%s: %s with a bitmap anyway", name, xname, error_string(derr)) return } st.images_decoded += 1 want_n := img.is_mask ? 1 : 4 testing.expectf(t, bmp.width == img.width && bmp.height == img.height && bmp.n == want_n, "%s: image /%s: decoded %vx%vx%v for a %vx%v image (mask=%v)", name, xname, bmp.width, bmp.height, bmp.n, img.width, img.height, img.is_mask) // The one failure that is an out-of-bounds read rather than a wrong pixel. testing.expectf(t, len(bmp.pixels) >= bmp.height * bmp.stride && bmp.stride >= bmp.width * bmp.n, "%s: image /%s: %v bytes for %v rows of %v", name, xname, len(bmp.pixels), bmp.height, bmp.stride) testing.expectf(t, !bmp.premultiplied, "%s: image /%s: image_decode must hand back straight alpha", name, xname) } // corpus_check_resources walks one /Resources dictionary, recursing into form // XObjects and tiling patterns, which is where most of a real file's fonts // actually live. `seen` keys on the dictionary pointer, so a resource shared // by 900 pages is done once and a cycle terminates. @(private = "file") corpus_check_resources :: proc(t: ^testing.T, doc: ^Document, res: ^Dict, name: string, st: ^Corpus_Res_Stats, seen: ^map[rawptr]bool, depth: int) { if res == nil || depth > CORPUS_RES_DEPTH { return } if seen[rawptr(res)] { return } seen[rawptr(res)] = true if fonts := dict_dict(doc, res, "Font"); fonts != nil { for fname, fobj in fonts { fd, ok := to_dict(resolve(doc, fobj)) if !ok { continue } if seen[rawptr(fd)] { continue } seen[rawptr(fd)] = true corpus_check_font(t, doc, fd, name, fname, st) } } if xo := dict_dict(doc, res, "XObject"); xo != nil { for xname, xobj in xo { stm, ok := to_stream(resolve(doc, xobj)) if !ok { continue } if seen[rawptr(stm)] { continue } seen[rawptr(stm)] = true if dict_name(doc, stm.dict, "Subtype") == "Form" { corpus_check_resources(t, doc, dict_dict(doc, stm.dict, "Resources"), name, st, seen, depth + 1) continue } corpus_check_image(t, doc, stm, res, name, xname, st) } } if shs := dict_dict(doc, res, "Shading"); shs != nil { for sname, sobj in shs { sh, serr := shading_load(doc, sobj, res) if serr != .None || sh == nil { continue } st.shadings += 1 bmp, berr := bitmap_make(16, 16, 4, context.temp_allocator) if berr != .None { continue } perr := shading_paint(doc, sh, &bmp, Matrix{1, 0, 0, 1, 8, 8}, Raster_Clip{0, 0, 16, 16}, 1, .Normal, nil) testing.expectf(t, perr == .None, "%s: shading /%s (%v): %s", name, sname, sh.kind, error_string(perr)) } } if pats := dict_dict(doc, res, "Pattern"); pats != nil { for pname, pobj in pats { pat, perr := pattern_load(doc, pobj, res) if perr != .None || pat == nil { continue } st.patterns += 1 if pat.kind == .Shading && pat.shading != nil { bmp, berr := bitmap_make(16, 16, 4, context.temp_allocator) if berr != .None { continue } e := pattern_paint_shading(doc, pat, &bmp, MATRIX_IDENTITY, Raster_Clip{0, 0, 16, 16}, 1, .Normal, nil) testing.expectf(t, e == .None, "%s: shading pattern /%s: %s", name, pname, error_string(e)) } if pat.kind == .Tiling { corpus_check_resources(t, doc, pat.resources, name, st, seen, depth + 1) } } } } @(test) test_corpus_resources :: proc(t: ^testing.T) { dir := corpus_dir() defer delete(dir) if dir == "" || !os.is_dir(dir) { return } paths := corpus_collect(dir) defer corpus_release(&paths) st: Corpus_Res_Stats files := 0 for p in paths { if corpus_size(p) > CORPUS_MAX_FILE { continue } doc, err := open_file(p) defer close(doc) if err != .None || doc == nil { continue } files += 1 name := filepath.base(p) seen := make(map[rawptr]bool) defer delete(seen) for i in 0 ..< page_count(doc) { page, perr := page_get(doc, i) if perr != .None { continue } corpus_check_resources(t, doc, page.resources, name, &st, &seen, 0) free_all(context.temp_allocator) } } if files == 0 { return } log.infof("corpus resources: %v files | fonts %v (%v with glyphs, %v failed, %v unembedded), "+ "%v glyphs probed, %v drawn, %v codes decoded | images %v (%v decoded, %v degraded, "+ "%v not decoded, %v megapixels) | "+ "%v shadings, %v patterns", files, st.fonts, st.fonts_with_glyphs, st.fonts_failed, st.fonts_unembedded, st.glyphs_probed, st.glyphs_drawn, st.codes_decoded, st.images, st.images_decoded, st.images_degraded, st.images_skipped, st.pixels / (1 << 20), st.shadings, st.patterns) // A corpus that reached no font at all means the walk broke, not that // seventy documents contain no text. testing.expectf(t, st.fonts > 0, "the sweep found no fonts in %v files", files) } // --------------------------------------------------------------------------- // The rendering sweep // // Everything above this point checks that the library survives the corpus. // This checks that it *draws* it, which is a different and much stronger // claim: a page can parse, load every font, decode every image and still // rasterise to nothing at all, because the text never drew, a clip swallowed // the page, or the base matrix put the content off the canvas. None of those // show up as an error -- they show up as a white rectangle. // // So the invariant here is measured in ink. Every page is rendered and the // fraction of pixels that are not the untouched background is recorded; the // distribution is logged, and a page that comes back blank is named. That // number is not a correctness proof -- a page can be inked and wrong -- but // blankness is a one-sided test that is never a false alarm on a page with // content on it, and it is the only automatic check there is that the // interpreter and the devices are connected to each other at all. // --------------------------------------------------------------------------- // 72 dpi. Big enough that a 6-point glyph still deposits ink, small enough // that three thousand pages finish in a minute. @(private = "file") CORPUS_RENDER_SCALE :: 1.0 // A document with fewer pages than this renders all of them; a longer one // renders only its first. A 900-page standard would otherwise dominate the // sweep while telling it nothing its first fifty pages did not. @(private = "file") CORPUS_RENDER_ALL_UNDER :: 20 // A page that takes longer than this has found a loop bound that is too // generous. It is checked after the fact -- the runner cannot interrupt a // running test -- which is enough to catch a regression from milliseconds to // minutes. @(private = "file") CORPUS_RENDER_MAX_MS :: 30_000 // Under one pixel in ten thousand is blank: an accidental hairline or a single // anti-aliased dot is not a rendered page. @(private = "file") CORPUS_BLANK :: 0.0001 // Under a tenth of a percent is near-blank -- a page number and nothing else. @(private = "file") CORPUS_NEAR_BLANK :: 0.001 // A page carrying this much text has a paragraph or more on it, and a page // with a paragraph on it covers well over a percent of itself in ink. Below // that the page drew a fraction of what the interpreter read, which is the // partial version of the blank-page bug and just as much a defect. @(private = "file") CORPUS_DENSE_CHARS :: 1000 @(private = "file") CORPUS_STARVED_INK :: 0.01 @(private = "file") Corpus_Render_Stats :: struct { files: int, pages: int, errors: int, blank: int, near_blank: int, first_blank: int, // files whose FIRST page is blank -- the loud case // A blank page that extracts text is the signature this sweep exists to // find: the interpreter reached the glyphs, so the page is not empty -- // something between there and the pixels dropped them. blank_with_text: int, empty: int, // blank and extracts nothing: genuinely a blank sheet starved: int, // a paragraph of text and almost no ink ink_sum: f64, hist: [10]int, slowest_ms: f64, slowest_name: string, total_ms: f64, max_pixels: int, } // The bucket boundaries the histogram reports, as fractions of the page. @(private = "file") CORPUS_INK_BUCKETS := [9]f64{0.0001, 0.001, 0.005, 0.01, 0.02, 0.05, 0.10, 0.25, 0.50} @(private = "file") corpus_ink_bucket :: proc(f: f64) -> int { for edge, i in CORPUS_INK_BUCKETS { if f < edge { return i } } return len(CORPUS_INK_BUCKETS) } // corpus_ink is the share of pixels that are not the background the render // started from. `render_page` leaves straight alpha and an opaque white page, // so anything that is not pure white was painted. @(private = "file") corpus_ink :: proc(bmp: ^Bitmap) -> f64 { if bmp.width <= 0 || bmp.height <= 0 || bmp.n < 3 { return 0 } inked := 0 for y in 0 ..< bmp.height { row := bitmap_row(bmp, y) for x in 0 ..< bmp.width { p := row[x * bmp.n:] if p[0] != 255 || p[1] != 255 || p[2] != 255 { inked += 1 } } } return f64(inked) / f64(bmp.width * bmp.height) } // corpus_render_page renders one page and holds it to the contract: a bitmap // of exactly the size `page_size` promised, no error, and no runaway time. @(private = "file") corpus_render_page :: proc(t: ^testing.T, page: ^Page, name: string, index: int, st: ^Corpus_Render_Stats) -> (ink: f64, ok: bool) { want_w, want_h := page_size(page, CORPUS_RENDER_SCALE) start := time.now() bmp, err := render_page(page, Render_Options{ ctm = scale_matrix(CORPUS_RENDER_SCALE, CORPUS_RENDER_SCALE), }) ms := time.duration_milliseconds(time.since(start)) defer bitmap_destroy(&bmp) st.total_ms += ms if ms > st.slowest_ms { st.slowest_ms = ms st.slowest_name = name } testing.expectf(t, ms < CORPUS_RENDER_MAX_MS, "%s page %d took %.0f ms to render", name, index + 1, ms) if err != .None { st.errors += 1 testing.expectf(t, false, "%s page %d: render_page returned %s", name, index + 1, error_string(err)) return 0, false } // The size is part of the contract: `page_size` is what a caller allocates // a window for before it ever calls render_page, and a bitmap that does // not match it means one of the two derived the base matrix differently. testing.expectf(t, bmp.width == want_w && bmp.height == want_h, "%s page %d: rendered %dx%d, page_size said %dx%d", name, index + 1, bmp.width, bmp.height, want_w, want_h) testing.expectf(t, bmp.n == 4 && len(bmp.pixels) >= bmp.stride * bmp.height, "%s page %d: %d components, %d bytes for %d rows of %d", name, index + 1, bmp.n, len(bmp.pixels), bmp.height, bmp.stride) testing.expectf(t, !bmp.premultiplied, "%s page %d: the bitmap left the library premultiplied", name, index + 1) st.max_pixels = max(st.max_pixels, bmp.width * bmp.height) return corpus_ink(&bmp), true } @(test) test_corpus_render :: proc(t: ^testing.T) { dir := corpus_dir() defer delete(dir) if dir == "" { log.info("PDF_ODIN_CORPUS is unset; skipping the render sweep") return } if !os.is_dir(dir) { return } paths := corpus_collect(dir) defer corpus_release(&paths) if len(paths) == 0 { return } // Nothing a render allocates may outlive it. The Document arena is // excluded on purpose -- that is freed by `close` and is not what this is // watching -- so what is left is the bitmap, the devices and the scratch, // and all three are supposed to be gone by the time the call returns. track: mem.Tracking_Allocator mem.tracking_allocator_init(&track, context.allocator) defer mem.tracking_allocator_destroy(&track) st: Corpus_Render_Stats blanks := make([dynamic]string) defer { for b in blanks { delete(b) } delete(blanks) } for p in paths { if corpus_size(p) > CORPUS_MAX_FILE { continue } doc, err := open_file(p) defer close(doc) if err != .None || doc == nil { continue } st.files += 1 name := filepath.base(p) n := page_count(doc) last := n <= CORPUS_RENDER_ALL_UNDER ? n : 1 for i in 0 ..< last { page, perr := page_get(doc, i) if perr != .None { continue } ink, ok := 0.0, false { context.allocator = mem.tracking_allocator(&track) ink, ok = corpus_render_page(t, &page, name, i, &st) } if !ok { continue } st.pages += 1 st.ink_sum += ink st.hist[corpus_ink_bucket(ink)] += 1 // Ask the text device what the interpreter saw on any page that // came out light. A page that draws nothing and extracts nothing // is a blank sheet, which every corpus has; a page that draws // nothing and extracts four hundred characters is a defect, and // the two must never be reported as the same thing. if ink < CORPUS_STARVED_INK { text, _ := extract_text(&page) defer delete(text) chars := len(text) if ink < CORPUS_BLANK { st.blank += 1 if i == 0 { st.first_blank += 1 } if chars > 0 { st.blank_with_text += 1 } else { st.empty += 1 } append(&blanks, corpus_blank_line(name, i, ink, chars)) } else if ink < CORPUS_NEAR_BLANK { st.near_blank += 1 if chars > 0 { st.blank_with_text += 1 } else { st.empty += 1 } append(&blanks, corpus_blank_line(name, i, ink, chars)) } else if chars >= CORPUS_DENSE_CHARS { st.starved += 1 append(&blanks, corpus_blank_line(name, i, ink, chars)) } } } free_all(context.temp_allocator) } if st.files == 0 { return } log.infof("render sweep: %d files, %d pages rendered at %gx, %d errors, "+ "%.1f%% mean ink, largest %d px, %.0f ms total, slowest %.0f ms (%s)", st.files, st.pages, f64(CORPUS_RENDER_SCALE), st.errors, st.pages > 0 ? st.ink_sum / f64(st.pages) * 100 : 0, st.max_pixels, st.total_ms, st.slowest_ms, st.slowest_name) labels := [10]string{ " 0 .. 0.01%", " 0.01 .. 0.1%", " 0.1 .. 0.5%", " 0.5 .. 1%", " 1 .. 2%", " 2 .. 5%", " 5 .. 10%", " 10 .. 25%", " 25 .. 50%", " 50 .. 100%", } for count, i in st.hist { if count == 0 { continue } log.infof(" ink %s : %5d pages", labels[i], count) } for b in blanks { log.infof(" %s", b) } log.infof("render sweep: %d blank pages, %d near-blank, %d starved (a paragraph of text "+ "under %g%% ink), %d of them a file's first page; %d drew nothing but extract "+ "text, %d are genuinely empty sheets", st.blank, st.near_blank, st.starved, f64(CORPUS_STARVED_INK) * 100, st.first_blank, st.blank_with_text, st.empty) // A sweep that rendered nothing would satisfy every assertion above. testing.expectf(t, st.pages > 0, "the sweep rendered no pages from %d files", st.files) // Blankness is the property this whole test exists for. A handful of // genuinely empty pages is normal in any corpus -- a separator sheet, the // back of a cover -- but a tenth of the corpus coming back white is a bug // in the interpreter, not seventy documents full of empty paper. if st.pages > 0 { ratio := f64(st.blank) / f64(st.pages) testing.expectf(t, ratio < 0.10, "%d of %d rendered pages (%.0f%%) came out blank", st.blank, st.pages, ratio * 100) } // The allocations a render makes are its own to free. Assert the tracker // saw traffic first: a leak check over an allocator nothing ever reached // passes for the wrong reason, and rewiring `context.allocator` is exactly // the sort of thing a later edit breaks silently. testing.expectf(t, track.total_allocation_count > 0, "the render sweep tracked no allocations at all over %d pages", st.pages) log.infof("render sweep: %d allocations, %d MiB peak, %d MiB total through the tracker", track.total_allocation_count, track.peak_memory_allocated / (1 << 20), track.total_memory_allocated / (1 << 20)) leaked := 0 for _, entry in track.allocation_map { leaked += entry.size } testing.expectf(t, len(track.allocation_map) == 0, "%d allocations (%d bytes) survived the render sweep", len(track.allocation_map), leaked) testing.expectf(t, len(track.bad_free_array) == 0, "%d bad frees during the render sweep", len(track.bad_free_array)) } @(private = "file") corpus_blank_line :: proc(name: string, index: int, ink: f64, chars: int) -> string { what := "empty sheet " if chars > 0 { what = ink < CORPUS_NEAR_BLANK ? "DREW NOTHING" : "STARVED " } return fmt.aprintf("%s %s page %d: %.4f%% ink, %d characters of text", what, name, index + 1, ink * 100, chars) } // --------------------------------------------------------------------------- // Text extraction over the corpus // // The same interpreter, into the text device instead of the rasteriser. What // it measures is coverage of the unicode chain -- /ToUnicode, the encoding // differences, the predefined CMaps and the AGL -- because a code that // resolves to nothing extracts as U+FFFD, and a page that is mostly U+FFFD is // text nobody can search or copy even though it draws correctly. // --------------------------------------------------------------------------- @(test) test_corpus_text :: proc(t: ^testing.T) { dir := corpus_dir() defer delete(dir) if dir == "" { log.info("PDF_ODIN_CORPUS is unset; skipping the text sweep") return } if !os.is_dir(dir) { return } paths := corpus_collect(dir) defer corpus_release(&paths) if len(paths) == 0 { return } files, pages, with_text, chars, runes_, bad := 0, 0, 0, 0, 0, 0 // Every page that is more than a tenth replacement characters, by name: // a ratio alone says the chain is failing somewhere and not where, and // "somewhere" is not something anyone can act on. suspect := make([dynamic]string) defer { for s in suspect { delete(s) } delete(suspect) } for p in paths { if corpus_size(p) > CORPUS_MAX_FILE { continue } doc, err := open_file(p) defer close(doc) if err != .None || doc == nil { continue } files += 1 name := filepath.base(p) for i in 0 ..< page_count(doc) { page, perr := page_get(doc, i) if perr != .None { continue } text, terr := extract_text(&page) defer delete(text) testing.expectf(t, terr == .None, "%s page %d: extract_text returned %s", name, i + 1, error_string(terr)) pages += 1 // Whatever comes out must be valid UTF-8: it is a Go-style string // that goes straight into a caller's search index. testing.expectf(t, utf8.valid_string(text), "%s page %d: extracted text is not valid UTF-8", name, i + 1) nr, nbad := 0, 0 for r in text { nr += 1 if r == utf8.RUNE_ERROR { nbad += 1 } } chars += len(text) runes_ += nr bad += nbad if len(text) > 0 { with_text += 1 } // Short strings make wild ratios; a page with a handful of glyphs // that all failed is not the signal this is looking for. if nr >= 32 { frac := f64(nbad) / f64(nr) if frac > 0.10 { append(&suspect, fmt.aprintf("%s page %d: %.0f%% U+FFFD (%d of %d runes)", name, i + 1, frac * 100, nbad, nr)) } } free_all(context.temp_allocator) } } if files == 0 { return } log.infof("text sweep: %d files, %d pages, %d with text (%.0f%%), %d bytes, %d runes, "+ "%d U+FFFD (%.3f%%)", files, pages, with_text, pages > 0 ? f64(with_text) / f64(pages) * 100 : 0, chars, runes_, bad, runes_ > 0 ? f64(bad) / f64(runes_) * 100 : 0) for s in suspect { log.infof(" UNRESOLVED %s", s) } if len(suspect) > 0 { log.infof("text sweep: %d pages are over 10%% U+FFFD", len(suspect)) } testing.expectf(t, with_text > 0, "no page in %d files yielded any text", files) // A corpus this size is mostly ordinary prose in ordinary fonts. If one // rune in twenty comes out as U+FFFD the unicode chain is broken, not the // files. if runes_ > 0 { ratio := f64(bad) / f64(runes_) testing.expectf(t, ratio < 0.05, "%d of %d extracted runes (%.1f%%) are U+FFFD", bad, runes_, ratio * 100) } }