pdf-odin-llm/xref_test.odin

package pdf

import "core:testing"

// ---------------------------------------------------------------------------
// Sample-PDF builders
//
// Every structural test in the library needs a file whose byte offsets are
// right, and hand-counting them is how a test ends up asserting the wrong
// thing. These assemble a document and record the offsets as they go, so a
// test says what shape it wants and never a number.
//
// They are `@(private)` rather than file-private on purpose: the page tree,
// annotation and encryption layers all want the same three-object skeleton.
// ---------------------------------------------------------------------------

@(private)
Pdf_Builder :: struct {
    buf:  [dynamic]byte,
    // Byte offset of each object header, indexed by object number; -1 for an
    // object number that was never written.
    offs: [dynamic]int,
}

@(private)
pb_make :: proc(header := "%PDF-1.7\n") -> Pdf_Builder {
    pb: Pdf_Builder
    pb.buf  = make([dynamic]byte)
    pb.offs = make([dynamic]int)
    pb_str(&pb, header)
    return pb
}

@(private)
pb_destroy :: proc(pb: ^Pdf_Builder) {
    delete(pb.buf)
    delete(pb.offs)
}

@(private)
pb_str :: proc(pb: ^Pdf_Builder, s: string) {
    append(&pb.buf, ..transmute([]byte)s)
}

@(private)
pb_raw :: proc(pb: ^Pdf_Builder, s: []byte) {
    if len(s) > 0 {
        append(&pb.buf, ..s)
    }
}

@(private)
pb_num :: proc(pb: ^Pdf_Builder, v: int) {
    digits: [24]byte
    n := 0
    x := v
    if x < 0 {
        append(&pb.buf, '-')
        x = -x
    }
    if x == 0 {
        digits[0] = '0'
        n = 1
    }
    for x > 0 {
        digits[n] = byte('0' + x % 10)
        x /= 10
        n += 1
    }
    for i := n - 1; i >= 0; i -= 1 {
        append(&pb.buf, digits[i])
    }
}

@(private)
pb_pad :: proc(pb: ^Pdf_Builder, v, width: int) {
    digits: [24]byte
    n := 0
    x := max(v, 0)
    if x == 0 {
        digits[0] = '0'
        n = 1
    }
    for x > 0 {
        digits[n] = byte('0' + x % 10)
        x /= 10
        n += 1
    }
    for _ in n ..< width {
        append(&pb.buf, '0')
    }
    for i := n - 1; i >= 0; i -= 1 {
        append(&pb.buf, digits[i])
    }
}

@(private)
pb_here :: proc(pb: ^Pdf_Builder) -> int {
    return len(pb.buf)
}

@(private)
pb_mark :: proc(pb: ^Pdf_Builder, num: u32) {
    for len(pb.offs) <= int(num) {
        append(&pb.offs, -1)
    }
    pb.offs[num] = len(pb.buf)
}

// pb_obj writes "N 0 obj <body> endobj".
@(private)
pb_obj :: proc(pb: ^Pdf_Builder, num: u32, body: string) {
    pb_mark(pb, num)
    pb_num(pb, int(num))
    pb_str(pb, " 0 obj\n")
    pb_str(pb, body)
    pb_str(pb, "\nendobj\n")
}

// pb_stream_obj writes a stream object, appending the /Length the payload
// actually has. `dict_head` is the dictionary without its << >>.
@(private)
pb_stream_obj :: proc(pb: ^Pdf_Builder, num: u32, dict_head: string, payload: []byte) {
    pb_mark(pb, num)
    pb_num(pb, int(num))
    pb_str(pb, " 0 obj\n<< ")
    pb_str(pb, dict_head)
    pb_str(pb, " /Length ")
    pb_num(pb, len(payload))
    pb_str(pb, " >>\nstream\n")
    pb_raw(pb, payload)
    pb_str(pb, "\nendstream\nendobj\n")
}

@(private)
pb_xref_begin :: proc(pb: ^Pdf_Builder) -> int {
    at := len(pb.buf)
    pb_str(pb, "xref\n")
    return at
}

@(private)
pb_xref_sub :: proc(pb: ^Pdf_Builder, start, count: int) {
    pb_num(pb, start)
    pb_str(pb, " ")
    pb_num(pb, count)
    pb_str(pb, "\n")
}

// pb_entry writes one table entry. The default `eol` gives the 20-byte form
// the spec requires; passing "\n" gives the 19-byte form most producers write.
@(private)
pb_entry :: proc(pb: ^Pdf_Builder, offset, gen: int, used: bool, eol := " \n") {
    pb_pad(pb, offset, 10)
    pb_str(pb, " ")
    pb_pad(pb, gen, 5)
    pb_str(pb, used ? " n" : " f")
    pb_str(pb, eol)
}

// pb_classic_xref_ex writes one subsection covering objects `real_start`
// upwards, but declaring it starts at `declared_start`. The two differ only in
// the tests for the off-by-one repair.
@(private)
pb_classic_xref_ex :: proc(pb: ^Pdf_Builder, declared_start, real_start: int, eol := " \n") -> int {
    at := pb_xref_begin(pb)
    n := len(pb.offs)
    pb_xref_sub(pb, declared_start, n - real_start)
    for i in real_start ..< n {
        if i == 0 || pb.offs[i] < 0 {
            pb_entry(pb, 0, 65535, false, eol)
        } else {
            pb_entry(pb, pb.offs[i], 0, true, eol)
        }
    }
    return at
}

@(private)
pb_classic_xref :: proc(pb: ^Pdf_Builder) -> int {
    return pb_classic_xref_ex(pb, 0, 0)
}

@(private)
pb_trailer :: proc(pb: ^Pdf_Builder, body: string, xref_at: int) {
    pb_str(pb, "trailer\n")
    pb_str(pb, body)
    pb_str(pb, "\n")
    pb_startxref(pb, xref_at)
}

@(private)
pb_startxref :: proc(pb: ^Pdf_Builder, at: int) {
    pb_str(pb, "startxref\n")
    pb_num(pb, at)
    pb_str(pb, "\n%%EOF\n")
}

// pb_placeholder reserves ten digits that pb_patch fills in later, for the
// forward references a /Prev loop needs.
@(private)
pb_placeholder :: proc(pb: ^Pdf_Builder) -> int {
    at := len(pb.buf)
    pb_str(pb, "0000000000")
    return at
}

@(private)
pb_patch :: proc(pb: ^Pdf_Builder, at, value: int) {
    x := max(value, 0)
    for i := 9; i >= 0; i -= 1 {
        if at + i < len(pb.buf) {
            pb.buf[at + i] = byte('0' + x % 10)
        }
        x /= 10
    }
}

// pb_skeleton writes the three objects every document needs: a catalog, a page
// tree root and one page.
@(private)
pb_skeleton :: proc(pb: ^Pdf_Builder) {
    pb_obj(pb, 1, "<< /Type /Catalog /Pages 2 0 R >>")
    pb_obj(pb, 2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")
    pb_obj(pb, 3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>")
}

// pb_sample_classic is the smallest well-formed file: three objects and a
// classic single-section table.
@(private)
pb_sample_classic :: proc() -> Pdf_Builder {
    pb := pb_make()
    pb_skeleton(&pb)
    at := pb_classic_xref(&pb)
    pb_trailer(&pb, "<< /Size 4 /Root 1 0 R >>", at)
    return pb
}

// ---------------------------------------------------------------------------
// Cross-reference stream payloads
// ---------------------------------------------------------------------------

@(private)
Xref_Row :: struct {
    f0, f1, f2: u64,
}

// xref_rows_encode lays out rows in the /W field widths given.
@(private)
xref_rows_encode :: proc(rows: []Xref_Row, w0, w1, w2: int, allocator := context.allocator) -> []byte {
    row := w0 + w1 + w2
    out := make([]byte, len(rows) * row, allocator)
    for r, i in rows {
        base := i * row
        xref_put_be(out[base:base + w0], r.f0)
        xref_put_be(out[base + w0:base + w0 + w1], r.f1)
        xref_put_be(out[base + w0 + w1:base + row], r.f2)
    }
    return out
}

@(private)
xref_put_be :: proc(dst: []byte, v: u64) {
    x := v
    for i := len(dst) - 1; i >= 0; i -= 1 {
        dst[i] = byte(x & 0xff)
        x >>= 8
    }
}

// png_up_encode applies the PNG "Up" filter, which is what a producer writing
// /Predictor 12 does. Every row gains a leading tag byte.
@(private)
png_up_encode :: proc(src: []byte, row_len: int, allocator := context.allocator) -> []byte {
    if row_len <= 0 { return nil }
    rows := len(src) / row_len
    out := make([]byte, rows * (row_len + 1), allocator)
    prev := make([]byte, row_len, allocator)
    defer delete(prev, allocator)
    for r in 0 ..< rows {
        o := r * (row_len + 1)
        out[o] = 2
        for c in 0 ..< row_len {
            out[o + 1 + c] = byte((int(src[r * row_len + c]) - int(prev[c])) & 0xff)
        }
        copy(prev, src[r * row_len:(r + 1) * row_len])
    }
    return out
}

// zlib_stored wraps bytes in an RFC 1950 stream of stored (uncompressed)
// DEFLATE blocks. A test does not need compression, only a payload the
// library's inflate must accept.
@(private)
zlib_stored :: proc(src: []byte, allocator := context.allocator) -> []byte {
    n_blocks := max(1, (len(src) + 65534) / 65535)
    out := make([dynamic]byte, 0, len(src) + n_blocks * 5 + 6, allocator)
    append(&out, 0x78, 0x01)
    i := 0
    for b in 0 ..< n_blocks {
        n := min(65535, len(src) - i)
        append(&out, b == n_blocks - 1 ? 0x01 : 0x00)
        append(&out, byte(n & 0xff), byte((n >> 8) & 0xff))
        m := 0xffff - n
        append(&out, byte(m & 0xff), byte((m >> 8) & 0xff))
        if n > 0 {
            append(&out, ..src[i:i + n])
        }
        i += n
    }
    s1, s2 := u32(1), u32(0)
    for c in src {
        s1 = (s1 + u32(c)) % 65521
        s2 = (s2 + s1) % 65521
    }
    adler := s2 << 16 | s1
    append(&out, byte(adler >> 24), byte(adler >> 16), byte(adler >> 8), byte(adler))
    return out[:]
}

// objstm_payload builds the body of a /Type /ObjStm: the pair table followed
// by the objects. `first` is the /First the dictionary must declare.
@(private)
objstm_payload :: proc(nums: []u32, bodies: []string, allocator := context.allocator) ->
                       (payload: []byte, first: int) {
    pb := Pdf_Builder{buf = make([dynamic]byte, allocator), offs = make([dynamic]int, allocator)}
    defer delete(pb.offs)

    rel := make([]int, len(bodies), allocator)
    defer delete(rel, allocator)
    total := 0
    for body, i in bodies {
        rel[i] = total
        total += len(body) + 1
    }
    for num, i in nums {
        if i >= len(rel) { break }
        pb_num(&pb, int(num))
        pb_str(&pb, " ")
        pb_num(&pb, rel[i])
        pb_str(&pb, " ")
    }
    first = len(pb.buf)
    for body in bodies {
        pb_str(&pb, body)
        pb_str(&pb, " ")
    }
    return pb.buf[:], first
}

// ---------------------------------------------------------------------------
// Test helpers
// ---------------------------------------------------------------------------

@(private)
pb_open :: proc(pb: ^Pdf_Builder) -> (^Document, Error) {
    // The page tree is a layer above this one; xref tests must not depend on
    // it being able to walk what they built.
    return open_bytes(pb.buf[:], Open_Options{skip_page_tree = true})
}

@(private)
doc_has_repair :: proc(doc: ^Document, kind: Repair_Kind) -> bool {
    if doc == nil { return false }
    for r in doc.repairs {
        if r.kind == kind { return true }
    }
    return false
}

@(private = "file")
expect_page :: proc(t: ^testing.T, doc: ^Document, num: u32, width: f64) {
    d := get_dict(doc, Ref{num, 0})
    if !testing.expect(t, d != nil, "page object did not resolve to a dictionary") {
        return
    }
    testing.expect(t, dict_name(doc, d, "Type") == "Page", "object is not a /Page")
    r := dict_rect(doc, d, "MediaBox")
    testing.expectf(t, r.x1 == width, "MediaBox width %v, wanted %v", r.x1, width)
}

// ---------------------------------------------------------------------------
// Classic tables
// ---------------------------------------------------------------------------

@(test)
test_xref_classic_single_section :: proc(t: ^testing.T) {
    pb := pb_sample_classic()
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    testing.expect(t, doc != nil, "no document")
    if doc == nil { return }

    testing.expect(t, !doc.is_damaged, "a well-formed file recorded a repair")
    testing.expectf(t, object_count(doc) == 4, "object_count %v, wanted 4", object_count(doc))
    testing.expect(t, doc.catalog != nil, "no catalog")
    testing.expect(t, doc.xref[1].kind == .Normal, "object 1 is not a normal entry")
    testing.expect(t, doc.xref[0].kind == .Free, "object 0 must be free")
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_classic_19_byte_entries :: proc(t: ^testing.T) {
    // The spec fixes the entry at 20 bytes; a large fraction of producers
    // write 19 by using a bare LF. Both must read the same.
    pb := pb_make()
    pb_skeleton(&pb)
    at := pb_classic_xref_ex(&pb, 0, 0, "\n")
    pb_trailer(&pb, "<< /Size 4 /Root 1 0 R >>", at)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    testing.expect(t, !doc.is_damaged, "19-byte entries should not be a repair")
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_incremental_update :: proc(t: ^testing.T) {
    pb := pb_make()
    pb_skeleton(&pb)
    x1 := pb_classic_xref(&pb)
    pb_trailer(&pb, "<< /Size 4 /Root 1 0 R >>", x1)

    // Generation two: object 3 is rewritten with a different page size, and
    // object 4 appears for the first time.
    pb_obj(&pb, 3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 400 400] >>")
    pb_obj(&pb, 4, "<< /Type /ExtGState /LW 3 >>")
    x2 := pb_xref_begin(&pb)
    pb_xref_sub(&pb, 3, 2)
    pb_entry(&pb, pb.offs[3], 0, true)
    pb_entry(&pb, pb.offs[4], 0, true)
    pb_str(&pb, "trailer\n<< /Size 5 /Root 1 0 R /Prev ")
    pb_num(&pb, x1)
    pb_str(&pb, " >>\n")
    pb_startxref(&pb, x2)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }

    // The newest definition of object 3 wins ...
    expect_page(t, doc, 3, 400)
    // ... an object only the newest section has is there ...
    testing.expect(t, dict_name(doc, get_dict(doc, Ref{4, 0}), "Type") == "ExtGState",
        "object added by the update is missing")
    // ... and objects only the first generation defined are still reachable.
    testing.expect(t, doc.catalog != nil, "catalog from the older section was lost")
    testing.expect(t, dict_name(doc, get_dict(doc, Ref{2, 0}), "Type") == "Pages",
        "object from the older section was lost")
}

@(test)
test_xref_trailer_merge_keeps_older_keys :: proc(t: ^testing.T) {
    pb := pb_make()
    pb_skeleton(&pb)
    pb_obj(&pb, 4, "<< /Producer (first) >>")
    x1 := pb_classic_xref(&pb)
    pb_trailer(&pb, "<< /Size 5 /Root 1 0 R /Info 4 0 R /ID [<01> <02>] >>", x1)

    pb_obj(&pb, 5, "<< /Type /ExtGState >>")
    x2 := pb_xref_begin(&pb)
    pb_xref_sub(&pb, 5, 1)
    pb_entry(&pb, pb.offs[5], 0, true)
    // The newest trailer omits /Info entirely, which is legal and common.
    pb_str(&pb, "trailer\n<< /Size 6 /Root 1 0 R /ID [<03> <04>] /Prev ")
    pb_num(&pb, x1)
    pb_str(&pb, " >>\n")
    pb_startxref(&pb, x2)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }

    info := dict_dict(doc, doc.trailer, "Info")
    testing.expect(t, info != nil, "/Info from the older trailer was dropped")

    id := dict_array(doc, doc.trailer, "ID")
    first, ok := to_bytes(array_get(doc, id, 0))
    testing.expect(t, ok && len(first) == 1 && first[0] == 0x03,
        "the newest /ID did not win the merge")
    testing.expectf(t, dict_int(doc, doc.trailer, "Size", 0) == 6,
        "merged /Size %v, wanted 6", dict_int(doc, doc.trailer, "Size", 0))
}

@(test)
test_xref_free_entry_shadows_older :: proc(t: ^testing.T) {
    // An incremental update that deletes an object marks it free. The older
    // section still has an in-use entry, and it must not win.
    pb := pb_make()
    pb_skeleton(&pb)
    pb_obj(&pb, 4, "<< /Type /ExtGState >>")
    x1 := pb_classic_xref(&pb)
    pb_trailer(&pb, "<< /Size 5 /Root 1 0 R >>", x1)

    x2 := pb_xref_begin(&pb)
    pb_xref_sub(&pb, 4, 1)
    pb_entry(&pb, 0, 1, false)
    pb_str(&pb, "trailer\n<< /Size 5 /Root 1 0 R /Prev ")
    pb_num(&pb, x1)
    pb_str(&pb, " >>\n")
    pb_startxref(&pb, x2)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }

    testing.expect(t, doc.xref[4].kind == .Free, "deleted object was resurrected")
    testing.expect(t, get_object(doc, Ref{4, 0}) == nil, "deleted object still resolves")
}

// ---------------------------------------------------------------------------
// The off-by-one subsection header
// ---------------------------------------------------------------------------

@(test)
test_xref_subsection_declared_one_starts_at_zero :: proc(t: ^testing.T) {
    // "1 4" over entries that really begin with the free-list head. Trusting
    // the header shifts every object number by one.
    pb := pb_make()
    pb_skeleton(&pb)
    at := pb_classic_xref_ex(&pb, 1, 0)
    pb_trailer(&pb, "<< /Size 4 /Root 1 0 R >>", at)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }

    testing.expect(t, doc.catalog != nil, "catalog not found after the shift was repaired")
    testing.expect(t, dict_name(doc, get_dict(doc, Ref{1, 0}), "Type") == "Catalog",
        "object 1 is not the catalog")
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_subsection_declared_zero_starts_at_one :: proc(t: ^testing.T) {
    // The mirror image: "0 3" over entries that begin at object 1. Object 0
    // can never be in use, which is what gives it away.
    pb := pb_make()
    pb_skeleton(&pb)
    at := pb_classic_xref_ex(&pb, 0, 1)
    pb_trailer(&pb, "<< /Size 4 /Root 1 0 R >>", at)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }

    testing.expect(t, dict_name(doc, get_dict(doc, Ref{1, 0}), "Type") == "Catalog",
        "object 1 is not the catalog")
    expect_page(t, doc, 3, 200)
}

// ---------------------------------------------------------------------------
// Cross-reference streams
// ---------------------------------------------------------------------------

@(private = "file")
build_xref_stream_file :: proc(predictor: bool) -> Pdf_Builder {
    pb := pb_make("%PDF-1.5\n")
    pb_skeleton(&pb)

    at := pb_here(&pb)
    rows := [5]Xref_Row{
        {0, 0, 65535},
        {1, u64(pb.offs[1]), 0},
        {1, u64(pb.offs[2]), 0},
        {1, u64(pb.offs[3]), 0},
        {1, u64(at), 0},
    }
    raw := xref_rows_encode(rows[:], 1, 4, 2)
    defer delete(raw)

    if predictor {
        pred := png_up_encode(raw, 7)
        defer delete(pred)
        payload := zlib_stored(pred)
        defer delete(payload)
        pb_stream_obj(&pb, 4,
            "/Type /XRef /Size 5 /W [1 4 2] /Root 1 0 R /Filter /FlateDecode " +
            "/DecodeParms << /Predictor 12 /Columns 7 >>", payload)
    } else {
        pb_stream_obj(&pb, 4, "/Type /XRef /Size 5 /W [1 4 2] /Root 1 0 R", raw)
    }
    pb_startxref(&pb, at)
    return pb
}

@(test)
test_xref_stream_plain :: proc(t: ^testing.T) {
    pb := build_xref_stream_file(false)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    testing.expect(t, !doc.is_damaged, "a well-formed xref stream recorded a repair")
    testing.expect(t, doc.catalog != nil, "no catalog")
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_stream_png_predictor :: proc(t: ^testing.T) {
    pb := build_xref_stream_file(true)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    testing.expect(t, !doc.is_damaged, "a well-formed xref stream recorded a repair")
    expect_page(t, doc, 3, 200)
    testing.expect(t, doc.xref[4].kind == .Normal, "the xref stream's own entry is missing")
}

@(test)
test_xref_stream_zero_width_type_field :: proc(t: ^testing.T) {
    // "/W [0 2 1]": the type field is not stored and defaults to 1, which is
    // how a file with no free and no compressed objects saves a byte a row.
    pb := pb_make("%PDF-1.5\n")
    pb_skeleton(&pb)

    at := pb_here(&pb)
    rows := [5]Xref_Row{
        {0, 0, 0},
        {0, u64(pb.offs[1]), 0},
        {0, u64(pb.offs[2]), 0},
        {0, u64(pb.offs[3]), 0},
        {0, u64(at), 0},
    }
    raw := xref_rows_encode(rows[:], 0, 2, 1)
    defer delete(raw)
    pb_stream_obj(&pb, 4, "/Type /XRef /Size 5 /W [0 2 1] /Root 1 0 R /Index [1 4]", raw[3:])
    pb_startxref(&pb, at)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    testing.expect(t, doc.xref[1].kind == .Normal, "type defaulted wrong")
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_stream_index_clamped_to_data :: proc(t: ^testing.T) {
    // /Index promises far more entries than the stream holds. The extra rows
    // must be dropped, not read off the end of the buffer.
    pb := pb_make("%PDF-1.5\n")
    pb_skeleton(&pb)

    at := pb_here(&pb)
    rows := [5]Xref_Row{
        {0, 0, 65535},
        {1, u64(pb.offs[1]), 0},
        {1, u64(pb.offs[2]), 0},
        {1, u64(pb.offs[3]), 0},
        {1, u64(at), 0},
    }
    raw := xref_rows_encode(rows[:], 1, 4, 2)
    defer delete(raw)
    pb_stream_obj(&pb, 4, "/Type /XRef /Size 5 /W [1 4 2] /Root 1 0 R /Index [0 900000]", raw)
    pb_startxref(&pb, at)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    expect_page(t, doc, 3, 200)
    testing.expect(t, object_count(doc) < 900000, "the table grew to a fabricated /Index count")
}

@(test)
test_xref_absurd_size_is_clamped :: proc(t: ^testing.T) {
    pb := pb_make()
    pb_skeleton(&pb)
    at := pb_classic_xref(&pb)
    pb_trailer(&pb, "<< /Size 2000000000 /Root 1 0 R >>", at)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    // The floor in xref_max_objects is 1024 entries, so a tiny file with a
    // preposterous /Size costs 24 KiB rather than 48 GB.
    testing.expectf(t, object_count(doc) <= 1024,
        "table grew to %v for a %v byte file", object_count(doc), len(pb.buf))
    expect_page(t, doc, 3, 200)
}

// ---------------------------------------------------------------------------
// Hybrid-reference files
// ---------------------------------------------------------------------------

@(test)
test_xref_hybrid_xrefstm :: proc(t: ^testing.T) {
    // The classic table hides object 3 by marking it free; the stream named by
    // /XRefStm says it lives inside object stream 4. A reader that lets the
    // classic entry win loses the page.
    pb := pb_make("%PDF-1.5\n")
    pb_obj(&pb, 1, "<< /Type /Catalog /Pages 2 0 R >>")
    pb_obj(&pb, 2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")

    bodies := [1]string{"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] >>"}
    nums := [1]u32{3}
    payload, first := objstm_payload(nums[:], bodies[:])
    defer delete(payload)
    pb_mark(&pb, 3)                 // object 3 has no header of its own
    pb.offs[3] = -1
    pb_mark(&pb, 4)
    pb_str(&pb, "4 0 obj\n<< /Type /ObjStm /N 1 /First ")
    pb_num(&pb, first)
    pb_str(&pb, " /Length ")
    pb_num(&pb, len(payload))
    pb_str(&pb, " >>\nstream\n")
    pb_raw(&pb, payload)
    pb_str(&pb, "\nendstream\nendobj\n")

    stm_at := pb_here(&pb)
    rows := [6]Xref_Row{
        {0, 0, 65535},
        {1, u64(pb.offs[1]), 0},
        {1, u64(pb.offs[2]), 0},
        {2, 4, 0},
        {1, u64(pb.offs[4]), 0},
        {1, u64(stm_at), 0},
    }
    raw := xref_rows_encode(rows[:], 1, 4, 2)
    defer delete(raw)
    pb_stream_obj(&pb, 5, "/Type /XRef /Size 6 /W [1 4 2] /Root 1 0 R", raw)

    at := pb_xref_begin(&pb)
    pb_xref_sub(&pb, 0, 6)
    pb_entry(&pb, 0, 65535, false)
    pb_entry(&pb, pb.offs[1], 0, true)
    pb_entry(&pb, pb.offs[2], 0, true)
    pb_entry(&pb, 0, 0, false)                 // object 3: hidden from old readers
    pb_entry(&pb, pb.offs[4], 0, true)
    pb_entry(&pb, pb.offs[5], 0, true)
    pb_str(&pb, "trailer\n<< /Size 6 /Root 1 0 R /XRefStm ")
    pb_num(&pb, stm_at)
    pb_str(&pb, " >>\n")
    pb_startxref(&pb, at)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }

    testing.expect(t, doc.xref[3].kind == .Compressed,
        "the /XRefStm entry did not replace the classic free entry")
    expect_page(t, doc, 3, 300)
}

// ---------------------------------------------------------------------------
// Loops and bad pointers
// ---------------------------------------------------------------------------

@(test)
test_xref_prev_loop_terminates :: proc(t: ^testing.T) {
    pb := pb_make()
    pb_skeleton(&pb)

    xa := pb_xref_begin(&pb)
    pb_xref_sub(&pb, 0, 4)
    pb_entry(&pb, 0, 65535, false)
    pb_entry(&pb, pb.offs[1], 0, true)
    pb_entry(&pb, pb.offs[2], 0, true)
    pb_entry(&pb, pb.offs[3], 0, true)
    pb_str(&pb, "trailer\n<< /Size 4 /Root 1 0 R /Prev ")
    hole := pb_placeholder(&pb)
    pb_str(&pb, " >>\n")

    xb := pb_xref_begin(&pb)
    pb_xref_sub(&pb, 0, 1)
    pb_entry(&pb, 0, 65535, false)
    pb_str(&pb, "trailer\n<< /Size 4 /Root 1 0 R /Prev ")
    pb_num(&pb, xa)
    pb_str(&pb, " >>\n")
    pb_startxref(&pb, xb)
    pb_patch(&pb, hole, xb)          // A points back at B: a two-node loop
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_startxref_past_eof_rebuilds :: proc(t: ^testing.T) {
    pb := pb_make()
    pb_skeleton(&pb)
    at := pb_classic_xref(&pb)
    _ = at
    pb_trailer(&pb, "<< /Size 4 /Root 1 0 R >>", 99_000_000)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "a rebuild should have opened this file")
    if doc == nil { return }
    testing.expect(t, doc_has_repair(doc, .Rebuilt_Xref), "no rebuild was recorded")
    testing.expect(t, doc.catalog != nil, "rebuild did not find the catalog")
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_startxref_into_junk_rebuilds :: proc(t: ^testing.T) {
    // startxref points into the middle of a stream's payload, which is what a
    // file that was concatenated or partly re-saved looks like.
    pb := pb_make()
    pb_skeleton(&pb)
    junk := make([]byte, 512)
    defer delete(junk)
    for i in 0 ..< len(junk) {
        junk[i] = byte('a' + i % 26)
    }
    pb_stream_obj(&pb, 4, "/Type /Junk", junk)
    at := pb_classic_xref(&pb)
    _ = at
    pb_trailer(&pb, "<< /Size 5 /Root 1 0 R >>", pb.offs[4] + 200)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "a rebuild should have opened this file")
    if doc == nil { return }
    testing.expect(t, doc_has_repair(doc, .Rebuilt_Xref), "no rebuild was recorded")
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_missing_entirely_rebuilds :: proc(t: ^testing.T) {
    pb := pb_make()
    pb_skeleton(&pb)
    pb_str(&pb, "trailer\n<< /Size 4 /Root 1 0 R >>\n%%EOF\n")
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "a rebuild should have opened this file")
    if doc == nil { return }
    testing.expect(t, doc_has_repair(doc, .Rebuilt_Xref), "no rebuild was recorded")
    testing.expect(t, doc.catalog != nil, "rebuild did not find the catalog")
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_rebuild_without_any_trailer :: proc(t: ^testing.T) {
    // No xref, no trailer: the catalog has to be found by looking for
    // /Type /Catalog.
    pb := pb_make()
    pb_skeleton(&pb)
    pb_str(&pb, "%%EOF\n")
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "a rebuild should have opened this file")
    if doc == nil { return }
    testing.expect(t, doc.catalog != nil, "catalog was not found by scanning")
    testing.expect(t, dict_name(doc, doc.catalog, "Type") == "Catalog", "wrong catalog")
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_rebuild_prefers_later_definition :: proc(t: ^testing.T) {
    // Two definitions of object 3 and no usable table. The one further down
    // the file is the newer one.
    pb := pb_make()
    pb_skeleton(&pb)
    pb_obj(&pb, 3, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 500] >>")
    pb_str(&pb, "trailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n7\n%%EOF\n")
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    expect_page(t, doc, 3, 500)
}

@(test)
test_xref_rebuild_skips_headers_inside_streams :: proc(t: ^testing.T) {
    // A stream payload that spells out an object header must not be mistaken
    // for one: doing so would point object 3 at the middle of a stream.
    pb := pb_make()
    pb_skeleton(&pb)
    decoy := "3 0 obj << /Type /Page /MediaBox [0 0 999 999] >> endobj"
    pb_stream_obj(&pb, 4, "/Type /Junk", transmute([]byte)decoy)
    pb_str(&pb, "trailer\n<< /Size 5 /Root 1 0 R >>\n%%EOF\n")
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    expect_page(t, doc, 3, 200)
}

// ---------------------------------------------------------------------------
// Files with junk before %PDF-
// ---------------------------------------------------------------------------

@(test)
test_xref_header_offset :: proc(t: ^testing.T) {
    // A mail header, an HTTP response or a self-extracting wrapper in front of
    // the file. Every offset in it is measured from "%PDF-", not from zero.
    inner := pb_sample_classic()
    defer pb_destroy(&inner)

    junk := "Received: by 10.0.0.1 with SMTP id x1234567890abcdefghijklmnop;\r\n" +
            "Content-Type: application/pdf\r\n\r\n"

    pb := pb_make(junk)
    defer pb_destroy(&pb)
    pb_raw(&pb, inner.buf[:])

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    testing.expectf(t, doc.header_offset == i64(len(junk)),
        "header_offset %v, wanted %v", doc.header_offset, len(junk))
    testing.expect(t, doc.catalog != nil, "no catalog")
    expect_page(t, doc, 3, 200)
}

// ---------------------------------------------------------------------------
// Malformed input must not crash or hang
// ---------------------------------------------------------------------------

@(test)
test_xref_truncation_survives :: proc(t: ^testing.T) {
    pb := build_xref_stream_file(true)
    defer pb_destroy(&pb)

    for cut := 1; cut <= len(pb.buf); cut += 7 {
        doc, _ := open_bytes(pb.buf[:cut], Open_Options{skip_page_tree = true})
        if doc != nil {
            // Touching every object is what a caller would do next.
            for num in 1 ..< object_count(doc) {
                _ = get_object(doc, Ref{u32(num), 0})
            }
            close(doc)
        }
    }
    testing.expect(t, true, "truncation must never crash or hang")
}

@(test)
test_xref_byte_flips_survive :: proc(t: ^testing.T) {
    base := pb_sample_classic()
    defer pb_destroy(&base)

    scratch := make([]byte, len(base.buf))
    defer delete(scratch)

    for at := 0; at < len(base.buf); at += 5 {
        copy(scratch, base.buf[:])
        scratch[at] ~= 0xff
        doc, _ := open_bytes(scratch, Open_Options{skip_page_tree = true})
        if doc != nil {
            for num in 1 ..< object_count(doc) {
                _ = get_object(doc, Ref{u32(num), 0})
            }
            close(doc)
        }
    }
    testing.expect(t, true, "a flipped byte must never crash or hang")
}

@(test)
test_xref_empty_and_tiny_inputs :: proc(t: ^testing.T) {
    cases := [?]string{
        "",
        "%",
        "%PDF-1.7\n",
        "%PDF-1.7\nstartxref\n",
        "%PDF-1.7\nstartxref\n0\n%%EOF\n",
        "%PDF-1.7\nxref\ntrailer\n",
        "%PDF-1.7\n1 0 obj\nendobj\n",
        "startxref\n4\n%%EOF",
        "xref\n0 4294967295\n",
    }
    for src in cases {
        doc, err := open_bytes(transmute([]byte)src, Open_Options{skip_page_tree = true})
        if doc != nil {
            close(doc)
        } else {
            testing.expect(t, err != .None, "a nil document must come with an error")
        }
    }
}

// ---------------------------------------------------------------------------
// Unit tests for the scanning primitives
// ---------------------------------------------------------------------------

@(test)
test_xref_find_token_is_whole_word :: proc(t: ^testing.T) {
    data := transmute([]byte)string("endobj obj objx 1 0 obj")
    // "obj" inside "endobj" and inside "objx" must not match.
    at := xref_find_token(data, 0, "obj")
    testing.expectf(t, at == 7, "found obj at %v, wanted 7", at)
    at2 := xref_find_token(data, 8, "obj")
    testing.expectf(t, at2 == 20, "found obj at %v, wanted 20", at2)
    testing.expect(t, xref_find_token(data, 0, "stream") == -1, "matched a keyword that is absent")

    stream := transmute([]byte)string("endstream stream")
    testing.expectf(t, xref_find_token(stream, 0, "stream") == 10,
        "\"stream\" matched inside \"endstream\"")
}

@(test)
test_xref_header_before :: proc(t: ^testing.T) {
    Case :: struct {
        src:  string,
        num:  u32,
        gen:  u16,
        ok:   bool,
    }
    cases := [?]Case{
        {"12 0 obj",        12, 0, true},
        {"  7 65535 obj",   7, 65535, true},
        {"3\t2\nobj",       3, 2, true},
        {"x12 0 obj",       0, 0, false},   // glued to a regular character
        {"0 0 obj",         0, 0, false},   // object 0 cannot be defined
        {"obj",             0, 0, false},
        {"12 obj",          0, 0, false},   // handled elsewhere, not here
        {"12 0 999999 obj", 0, 0, false},   // generation out of range
    }
    for c in cases {
        data := transmute([]byte)c.src
        at := xref_find_token(data, 0, "obj")
        if at < 0 {
            testing.expect(t, !c.ok, "no obj keyword in a case that expected one")
            continue
        }
        num, gen, _, ok := xref_header_before(data, at)
        testing.expectf(t, ok == c.ok, "%q: ok %v, wanted %v", c.src, ok, c.ok)
        if ok && c.ok {
            testing.expectf(t, num == c.num && gen == c.gen,
                "%q: got %v %v, wanted %v %v", c.src, num, gen, c.num, c.gen)
        }
    }
}

@(test)
test_xref_read_field_big_endian :: proc(t: ^testing.T) {
    data := [?]byte{0x01, 0x02, 0x03, 0x04, 0x05}
    testing.expect(t, xref_read_field(data[:], 0, 0) == 0, "a zero-width field is zero")
    testing.expect(t, xref_read_field(data[:], 0, 1) == 0x01, "one byte")
    testing.expect(t, xref_read_field(data[:], 1, 2) == 0x0203, "two bytes")
    testing.expect(t, xref_read_field(data[:], 1, 4) == 0x02030405, "four bytes")
    // Reading past the end stops rather than wandering off the buffer.
    testing.expect(t, xref_read_field(data[:], 3, 8) == 0x0405, "clamped at the end")
}

@(test)
test_xref_digits :: proc(t: ^testing.T) {
    v, ok := xref_digits(transmute([]byte)string("00123"))
    testing.expect(t, ok && v == 123, "leading zeros")
    _, long_ok := xref_digits(transmute([]byte)string("1234567890123456"))
    testing.expect(t, !long_ok, "an absurdly long run is not an object number")
    _, empty_ok := xref_digits(nil)
    testing.expect(t, !empty_ok, "an empty run is not a number")
}

@(test)
test_xref_previous_startxref_rescues :: proc(t: ^testing.T) {
    // A tool that crashed part way through an incremental update leaves a
    // startxref pointing at nothing, with the previous one still intact. That
    // is a far better table than anything a rebuild can reconstruct.
    pb := pb_sample_classic()
    defer pb_destroy(&pb)
    pb_str(&pb, "startxref\n99000000\n%%EOF\n")

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }

    testing.expect(t, doc_has_repair(doc, .Bad_Startxref), "the dead startxref was not noticed")
    testing.expect(t, !doc_has_repair(doc, .Rebuilt_Xref),
        "the older startxref should have been used instead of a rebuild")
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_trailer_keyword_missing :: proc(t: ^testing.T) {
    // Some producers write the dictionary straight after the table.
    pb := pb_make()
    pb_skeleton(&pb)
    at := pb_classic_xref(&pb)
    pb_str(&pb, "<< /Size 4 /Root 1 0 R >>\n")
    pb_startxref(&pb, at)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    testing.expect(t, doc.catalog != nil, "no catalog")
    testing.expect(t, !doc_has_repair(doc, .Rebuilt_Xref), "should not have needed a rebuild")
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_subsection_count_absurd :: proc(t: ^testing.T) {
    // "0 900000000" over four entries. The count has to be clamped to what the
    // file can hold, and the entries after the table must not be eaten.
    pb := pb_make()
    pb_skeleton(&pb)
    at := pb_xref_begin(&pb)
    pb_xref_sub(&pb, 0, 900_000_000)
    pb_entry(&pb, 0, 65535, false)
    pb_entry(&pb, pb.offs[1], 0, true)
    pb_entry(&pb, pb.offs[2], 0, true)
    pb_entry(&pb, pb.offs[3], 0, true)
    pb_str(&pb, "trailer\n<< /Size 4 /Root 1 0 R >>\n")
    pb_startxref(&pb, at)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    expect_page(t, doc, 3, 200)
    testing.expectf(t, object_count(doc) <= 1024, "table grew to %v", object_count(doc))
}

@(test)
test_xref_stream_without_w_falls_back :: proc(t: ^testing.T) {
    // No /W at all: the rows cannot be decoded, so the whole section is
    // unusable and the file has to be rebuilt rather than half-read.
    pb := pb_make("%PDF-1.5\n")
    pb_skeleton(&pb)
    at := pb_here(&pb)
    payload := [?]byte{0, 0, 0, 0, 0, 0, 0}
    pb_stream_obj(&pb, 4, "/Type /XRef /Size 5 /Root 1 0 R", payload[:])
    pb_startxref(&pb, at)
    defer pb_destroy(&pb)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "a rebuild should have opened this file")
    if doc == nil { return }
    testing.expect(t, doc_has_repair(doc, .Rebuilt_Xref), "no rebuild was recorded")
    expect_page(t, doc, 3, 200)
}

@(test)
test_xref_many_objects :: proc(t: ^testing.T) {
    // Enough objects that the table is regrown many times over.
    N :: 1500
    pb := pb_make()
    defer pb_destroy(&pb)
    pb_skeleton(&pb)
    for i in 4 ..= u32(N) {
        pb_obj(&pb, i, "<< /Type /Junk >>")
    }
    at := pb_classic_xref(&pb)
    pb_trailer(&pb, "<< /Size 1501 /Root 1 0 R >>", at)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }
    testing.expectf(t, object_count(doc) == N + 1, "object_count %v, wanted %v",
        object_count(doc), N + 1)
    n_found := 0
    for i in 4 ..= u32(N) {
        if dict_name(doc, get_dict(doc, Ref{i, 0}), "Type") == "Junk" {
            n_found += 1
        }
    }
    testing.expectf(t, n_found == N - 3, "resolved %v of %v objects", n_found, N - 3)
}

@(test)
test_xref_object_read_during_load_is_not_cached :: proc(t: ^testing.T) {
    // An older section's cross-reference stream gives its /Length indirectly.
    // Resolving that reference while the table is still half built reads the
    // object as null -- correctly, there is nothing there yet -- and caching
    // that answer would make it null for the life of the Document, long after
    // the section that defines it has been read.
    pb := pb_make("%PDF-1.5\n")
    defer pb_destroy(&pb)
    pb_skeleton(&pb)

    // The section below is one row wide, so this is also the honest /Length.
    pb_obj(&pb, 7, "7")

    // The older section, covering object 7 alone.
    xb := pb_here(&pb)
    row_b := [1]Xref_Row{{1, u64(pb.offs[7]), 0}}
    raw_b := xref_rows_encode(row_b[:], 1, 4, 2)
    defer delete(raw_b)
    pb_mark(&pb, 6)
    pb_str(&pb, "6 0 obj\n<< /Type /XRef /Size 10 /W [1 4 2] /Index [7 1] /Root 1 0 R /Length 7 0 R >>\nstream\n")
    pb_raw(&pb, raw_b)
    pb_str(&pb, "\nendstream\nendobj\n")

    pb_obj(&pb, 9, "<< /Type /Junk >>")

    // The newest section covers 0..6 and 8..9, deliberately skipping object 7
    // so that the older section is the only place it is defined.
    xa := pb_here(&pb)
    rows_a := [9]Xref_Row{
        {0, 0, 65535},
        {1, u64(pb.offs[1]), 0},
        {1, u64(pb.offs[2]), 0},
        {1, u64(pb.offs[3]), 0},
        {0, 0, 0},
        {0, 0, 0},
        {1, u64(xb), 0},
        {1, u64(pb.offs[9]), 0},
        {1, u64(xa), 0},
    }
    raw_a := xref_rows_encode(rows_a[:], 1, 4, 2)
    defer delete(raw_a)
    pb_mark(&pb, 8)
    pb_str(&pb, "8 0 obj\n<< /Type /XRef /Size 10 /W [1 4 2] /Index [0 7 8 2] /Root 1 0 R /Prev ")
    pb_num(&pb, xb)
    pb_str(&pb, " /Length ")
    pb_num(&pb, len(raw_a))
    pb_str(&pb, " >>\nstream\n")
    pb_raw(&pb, raw_a)
    pb_str(&pb, "\nendstream\nendobj\n")
    pb_startxref(&pb, xa)

    doc, err := pb_open(&pb)
    defer close(doc)
    testing.expect(t, err == .None, "open failed")
    if doc == nil { return }

    testing.expect(t, doc.xref[7].kind == .Normal,
        "object 7 should come from the older section")
    v, ok := to_int(get_object(doc, Ref{7, 0}))
    testing.expectf(t, ok && v == i64(len(raw_b)),
        "object 7 read as %v (ok %v), wanted %v", v, ok, len(raw_b))
    expect_page(t, doc, 3, 200)
}