pdf-odin-llm/parser_test.odin

package pdf

import "core:mem"
import "core:mem/virtual"
import "core:slice"
import "core:strings"
import "core:testing"

@(private = "file")
scratch_begin :: proc(t: ^testing.T, a: ^virtual.Arena) -> mem.Allocator {
    err := virtual.arena_init_growing(a)
    testing.expect(t, err == nil, "arena init failed")
    return virtual.arena_allocator(a)
}

@(private = "file")
bytes_of :: proc(s: string) -> []byte {
    return transmute([]byte)s
}

// test_doc builds a Document by hand. `open_bytes` cannot be used here: it
// needs an xref loader, and the point of these tests is the parser on its own.
// Nothing calls `close`, so the Document's own arena is never initialised --
// everything lives in the caller's scratch arena instead.
@(private = "file")
test_doc :: proc(src: string, alloc: mem.Allocator) -> ^Document {
    doc := new(Document, alloc)
    doc.data         = bytes_of(src)
    doc.allocator    = alloc
    doc.backing      = alloc
    doc.xref         = make([dynamic]Xref_Entry, alloc)
    doc.cache        = make(map[u32]Object, 16, alloc)
    doc.loading      = make(map[u32]bool, 8, alloc)
    doc.objstm_cache = make(map[u32]^Objstm, 4, alloc)
    doc.pages        = make([dynamic]^Dict, alloc)
    doc.page_refs    = make([dynamic]Ref, alloc)
    doc.repairs      = make([dynamic]Repair, alloc)
    return doc
}

// test_xref_at registers object `num` as living at the offset of `header` in
// the file, so `get_object` can find it.
@(private = "file")
test_xref_at :: proc(doc: ^Document, num: u32, header: string) {
    for len(doc.xref) <= int(num) {
        append(&doc.xref, Xref_Entry{})
    }
    doc.xref[num] = Xref_Entry{
        offset = i64(strings.index(string(doc.data), header)),
        kind   = .Normal,
    }
}

// Object holds a slice (String) and pointers, so it is not comparable with
// '=='. Direct objects are compared by value here; containers by identity.
@(private = "file")
object_same :: proc(a, b: Object) -> bool {
    sa, a_is_str := a.(String)
    sb, b_is_str := b.(String)
    if a_is_str || b_is_str {
        return a_is_str && b_is_str && slice.equal(transmute([]byte)sa, transmute([]byte)sb)
    }
    switch va in a {
    case bool:    vb, ok := b.(bool);    return ok && va == vb
    case i64:     vb, ok := b.(i64);     return ok && va == vb
    case f64:     vb, ok := b.(f64);     return ok && va == vb
    case Name:    vb, ok := b.(Name);    return ok && va == vb
    case Ref:     vb, ok := b.(Ref);     return ok && va == vb
    case String:  return false
    case ^Array:  vb, ok := b.(^Array);  return ok && va == vb
    case ^Dict:   vb, ok := b.(^Dict);   return ok && va == vb
    case ^Stream: vb, ok := b.(^Stream); return ok && va == vb
    }
    return b == nil
}

@(private = "file")
parse_one :: proc(src: string, alloc: mem.Allocator) -> (Object, Error) {
    p := parser_make(nil, bytes_of(src), alloc)
    return parse_object(&p)
}

@(private = "file")
has_repair :: proc(doc: ^Document, kind: Repair_Kind) -> bool {
    for r in doc.repairs {
        if r.kind == kind { return true }
    }
    return false
}

// ---------------------------------------------------------------------------
// Scalars
// ---------------------------------------------------------------------------

@(test)
test_parser_scalars :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    Case :: struct {
        src:  string,
        want: Object,
    }
    cases := []Case{
        {"true",    true},
        {"false",   false},
        {"null",    nil},
        {"42",      i64(42)},
        {"-7",      i64(-7)},
        {"3.5",     f64(3.5)},
        {"-.25",    f64(-0.25)},
        {"/Name",   Name("Name")},
        {"/",       Name("")},
        {"/a#20b",  Name("a b")},
        // A number a viewer would read as 0 must still be an object.
        {"--3",     i64(0)},
    }

    for c in cases {
        obj, err := parse_one(c.src, alloc)
        testing.expectf(t, err == .None, "%q: error %v", c.src, err)
        testing.expectf(t, object_same(obj, c.want), "%q: got %v, want %v", c.src, obj, c.want)
    }

    // Strings compare by content, not by pointer.
    String_Case :: struct {
        src:  string,
        want: string,
    }
    for c in ([]String_Case{
        {"(hi)", "hi"},
        {"<4869>", "Hi"},
        {"()", ""},
        {"(a\\061b)", "a1b"},
    }) {
        obj, err := parse_one(c.src, alloc)
        testing.expectf(t, err == .None, "%q: error %v", c.src, err)
        b, ok := to_bytes(obj)
        testing.expectf(t, ok, "%q: not a string", c.src)
        testing.expectf(t, slice.equal(b, bytes_of(c.want)), "%q: got %q", c.src, string(b))
    }
}

@(test)
test_parser_empty_and_junk :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    _, err := parse_one("", alloc)
    testing.expect_value(t, err, Error.Truncated)

    _, err2 := parse_one("   % nothing but a comment\n", alloc)
    testing.expect_value(t, err2, Error.Truncated)

    _, err3 := parse_one("endobj", alloc)
    testing.expect_value(t, err3, Error.Unexpected_Token)

    _, err4 := parse_one(">", alloc)
    testing.expect_value(t, err4, Error.Syntax_Error)

    _, err5 := parse_one("]", alloc)
    testing.expect_value(t, err5, Error.Unexpected_Token)
}

// ---------------------------------------------------------------------------
// References
// ---------------------------------------------------------------------------

@(test)
test_parser_reference_vs_header :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    Case :: struct {
        src:  string,
        want: Object,
    }
    cases := []Case{
        {"1 0 R",      Ref{1, 0}},
        {"12 65535 R", Ref{12, 65535}},
        {"1 0 R ",     Ref{1, 0}},
        // "12 0 obj" is a header, not a reference: the first integer is the
        // object here, and the caller sees the rest.
        {"12 0 obj",   i64(12)},
        {"12 0",       i64(12)},
        {"12",         i64(12)},
        // "1 0 0 RG" in a content stream is three numbers and an operator.
        {"1 0 0 RG",   i64(1)},
        {"1 0 RG",     i64(1)},
        // Object 0 is the free-list head and can never be referenced.
        {"0 0 R",      nil},
        {"-1 0 R",     nil},
        // Past the format's object-number ceiling.
        {"8388608 0 R", nil},
        // A generation that cannot fit is not a reference at all.
        {"1 70000 R",  i64(1)},
    }

    for c in cases {
        obj, err := parse_one(c.src, alloc)
        testing.expectf(t, err == .None, "%q: error %v", c.src, err)
        testing.expectf(t, object_same(obj, c.want), "%q: got %v, want %v", c.src, obj, c.want)
    }
}

@(test)
test_parser_references_in_containers :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    obj, err := parse_one("[1 0 R 2 0 R 3 4 R]", alloc)
    testing.expect_value(t, err, Error.None)
    a, ok := to_array(obj)
    testing.expect(t, ok, "not an array")
    testing.expect_value(t, array_len(a), 3)
    testing.expect(t, object_same(array_get_raw(a, 0), Ref{1, 0}), "element 0 is not 1 0 R")
    testing.expect(t, object_same(array_get_raw(a, 2), Ref{3, 4}), "element 2 is not 3 4 R")

    dobj, derr := parse_one("<< /Kids [4 0 R] /Parent 9 0 R /Count 1 >>", alloc)
    testing.expect_value(t, derr, Error.None)
    d, dok := to_dict(dobj)
    testing.expect(t, dok, "not a dictionary")
    ref, rok := dict_ref(d, "Parent")
    testing.expect(t, rok, "/Parent is not a reference")
    testing.expect_value(t, ref, Ref{9, 0})
}

// ---------------------------------------------------------------------------
// Arrays
// ---------------------------------------------------------------------------

@(test)
test_parser_nested_arrays :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    obj, err := parse_one("[1 [2 [3 [4 5]]] 6]", alloc)
    testing.expect_value(t, err, Error.None)
    a, _ := to_array(obj)
    testing.expect_value(t, array_len(a), 3)
    testing.expect(t, object_same(array_get_raw(a, 0), i64(1)), "element 0 is not 1")
    testing.expect(t, object_same(array_get_raw(a, 2), i64(6)), "element 2 is not 6")

    b, _ := to_array(array_get_raw(a, 1))
    testing.expect_value(t, array_len(b), 2)
    c, _ := to_array(array_get_raw(b, 1))
    testing.expect_value(t, array_len(c), 2)
    d, _ := to_array(array_get_raw(c, 1))
    testing.expect_value(t, array_len(d), 2)
    testing.expect(t, object_same(array_get_raw(d, 1), i64(5)), "innermost element is not 5")

    // Nulls hold their position: a /Widths or /Decode array is read by index.
    nobj, nerr := parse_one("[1 null 2]", alloc)
    testing.expect_value(t, nerr, Error.None)
    na, _ := to_array(nobj)
    testing.expect_value(t, array_len(na), 3)
    testing.expect(t, is_null(array_get_raw(na, 1)), "the null element was dropped")

    // Mixed types, no separators needed between delimited objects.
    mobj, merr := parse_one("[/A(b)<43>[]<< >>true]", alloc)
    testing.expect_value(t, merr, Error.None)
    ma, _ := to_array(mobj)
    testing.expect_value(t, array_len(ma), 6)
}

@(test)
test_parser_depth_limit :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    build :: proc(n: int, alloc: mem.Allocator) -> string {
        b := strings.builder_make(0, 2 * n, alloc)
        for _ in 0 ..< n { strings.write_byte(&b, '[') }
        for _ in 0 ..< n { strings.write_byte(&b, ']') }
        return strings.to_string(b)
    }

    ok_src := build(MAX_DEPTH, alloc)
    _, err := parse_one(ok_src, alloc)
    testing.expectf(t, err == .None, "%d levels should parse, got %v", MAX_DEPTH, err)

    deep := build(MAX_DEPTH + 1, alloc)
    _, derr := parse_one(deep, alloc)
    testing.expect_value(t, derr, Error.Recursion_Limit)

    // Dictionaries are bounded by the same counter, and a mixed nest too.
    dict_src := strings.builder_make(0, 8 * MAX_DEPTH, alloc)
    for _ in 0 ..< MAX_DEPTH + 1 { strings.write_string(&dict_src, "<< /A ") }
    for _ in 0 ..< MAX_DEPTH + 1 { strings.write_string(&dict_src, ">>") }
    _, dicterr := parse_one(strings.to_string(dict_src), alloc)
    testing.expect_value(t, dicterr, Error.Recursion_Limit)

    // A pathological file is not allowed to recurse the stack away either.
    runaway := strings.builder_make(0, 4096, alloc)
    for _ in 0 ..< 4000 { strings.write_byte(&runaway, '[') }
    _, rerr := parse_one(strings.to_string(runaway), alloc)
    testing.expect_value(t, rerr, Error.Recursion_Limit)
}

@(test)
test_parser_unterminated_array :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    doc := test_doc("[1 2 3", alloc)
    p := parser_make(doc, doc.data, alloc)
    obj, err := parse_object(&p)
    testing.expect_value(t, err, Error.None)
    a, _ := to_array(obj)
    testing.expect_value(t, array_len(a), 3)
    testing.expect(t, has_repair(doc, .Unterminated_Array), "no Unterminated_Array repair")
    testing.expect(t, doc.is_damaged, "document should be marked damaged")

    // A '>>' inside an array closes the array without being eaten: the
    // enclosing dictionary still needs it.
    doc2 := test_doc("<< /A [1 2 >> /B 3", alloc)
    p2 := parser_make(doc2, doc2.data, alloc)
    obj2, err2 := parse_object(&p2)
    testing.expect_value(t, err2, Error.None)
    d2, _ := to_dict(obj2)
    a2 := dict_array(nil, d2, "A")
    testing.expect_value(t, array_len(a2), 2)

    // Likewise a keyword that can only appear outside a container.
    doc3 := test_doc("[1 2 endobj", alloc)
    p3 := parser_make(doc3, doc3.data, alloc)
    obj3, _ := parse_object(&p3)
    a3, _ := to_array(obj3)
    testing.expect_value(t, array_len(a3), 2)

    // Junk between elements is stepped over, not fatal.
    doc4 := test_doc("[1 ) 2 endobj]", alloc)
    p4 := parser_make(doc4, doc4.data, alloc)
    obj4, err4 := parse_object(&p4)
    testing.expect_value(t, err4, Error.None)
    a4, _ := to_array(obj4)
    testing.expect_value(t, array_len(a4), 2)
}

// ---------------------------------------------------------------------------
// Dictionaries
// ---------------------------------------------------------------------------

@(test)
test_parser_dict_basics :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    obj, err := parse_one("<< /Type /Page /Count 3 /Box [0 0 612 792] /S (x) /Null null >>", alloc)
    testing.expect_value(t, err, Error.None)
    d, ok := to_dict(obj)
    testing.expect(t, ok, "not a dictionary")
    testing.expect_value(t, dict_name(nil, d, "Type"), "Page")
    testing.expect_value(t, dict_int(nil, d, "Count"), i64(3))
    testing.expect_value(t, array_len(dict_array(nil, d, "Box")), 4)
    // A key with a null value is indistinguishable from an absent key, so it
    // is never stored.
    testing.expect(t, !dict_has(d, "Null"), "a null value must be dropped")
    testing.expect_value(t, len(d^), 4)

    // The last of a repeated key wins, which is what every viewer does.
    dup, _ := parse_one("<< /A 1 /A 2 >>", alloc)
    dd, _ := to_dict(dup)
    testing.expect_value(t, dict_int(nil, dd, "A"), i64(2))

    // Nesting.
    nest, _ := parse_one("<< /R << /Font << /F1 5 0 R >> >> >>", alloc)
    nd, _ := to_dict(nest)
    fonts := dict_dict(nil, dict_dict(nil, nd, "R"), "Font")
    ref, rok := dict_ref(fonts, "F1")
    testing.expect(t, rok, "/F1 is not a reference")
    testing.expect_value(t, ref, Ref{5, 0})

    empty, eerr := parse_one("<<>>", alloc)
    testing.expect_value(t, eerr, Error.None)
    ed, eok := to_dict(empty)
    testing.expect(t, eok, "<<>> is a dictionary")
    testing.expect_value(t, len(ed^), 0)
}

@(test)
test_parser_dict_bad_key :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    // A key that is not a name is dropped with its value; the rest of the
    // dictionary is still read.
    doc := test_doc("<< 5 6 /A 1 (k) (v) /B 2 >>", alloc)
    p := parser_make(doc, doc.data, alloc)
    obj, err := parse_object(&p)
    testing.expect_value(t, err, Error.None)
    d, _ := to_dict(obj)
    testing.expect_value(t, dict_int(nil, d, "A"), i64(1))
    testing.expect_value(t, dict_int(nil, d, "B"), i64(2))
    testing.expect_value(t, len(d^), 2)
    testing.expect(t, has_repair(doc, .Unterminated_Dict), "no repair recorded for the bad keys")
}

@(test)
test_parser_dict_unterminated :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    doc := test_doc("<< /A 1 /B 2", alloc)
    p := parser_make(doc, doc.data, alloc)
    obj, err := parse_object(&p)
    testing.expect_value(t, err, Error.None)
    d, _ := to_dict(obj)
    testing.expect_value(t, dict_int(nil, d, "A"), i64(1))
    testing.expect_value(t, dict_int(nil, d, "B"), i64(2))
    testing.expect(t, has_repair(doc, .Unterminated_Dict), "no Unterminated_Dict repair")

    // A key whose value is missing must not swallow the closing '>>'.
    doc2 := test_doc("<< /A 1 /B >>", alloc)
    p2 := parser_make(doc2, doc2.data, alloc)
    obj2, _ := parse_object(&p2)
    d2, _ := to_dict(obj2)
    testing.expect_value(t, dict_int(nil, d2, "A"), i64(1))
    testing.expect(t, !dict_has(d2, "B"), "/B has no value to store")
    tail := lexer_next(&p2.lex, alloc)
    testing.expect_value(t, tail.kind, Token_Kind.EOF)

    // Truncated in the middle of a key.
    doc3 := test_doc("<< /A 1 /B", alloc)
    p3 := parser_make(doc3, doc3.data, alloc)
    obj3, err3 := parse_object(&p3)
    testing.expect_value(t, err3, Error.None)
    d3, _ := to_dict(obj3)
    testing.expect_value(t, len(d3^), 1)
}

@(test)
test_parser_string_repairs :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    doc := test_doc("<< /T (unterminated", alloc)
    p := parser_make(doc, doc.data, alloc)
    obj, err := parse_object(&p)
    testing.expect_value(t, err, Error.None)
    d, _ := to_dict(obj)
    testing.expect(t, slice.equal(dict_bytes(nil, d, "T"), bytes_of("unterminated")),
        "the value of an unterminated string is still usable")
    testing.expect(t, has_repair(doc, .Unterminated_String), "no Unterminated_String repair")
}

// ---------------------------------------------------------------------------
// Streams
// ---------------------------------------------------------------------------

@(private = "file")
STREAM_SRC :: "%PDF-1.7\n1 0 obj\n<< /Length 5 >>\nstream\r\nHello\nendstream\nendobj\n"

@(private = "file")
parse_first_object :: proc(doc: ^Document, alloc: mem.Allocator) -> (Object, Ref, Error) {
    return parse_indirect_object_at(doc, i64(strings.index(string(doc.data), "1 0 obj")), Ref{1, 0})
}

@(test)
test_parser_stream_exact_length :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    doc := test_doc(STREAM_SRC, alloc)
    obj, got, err := parse_first_object(doc, alloc)
    testing.expect_value(t, err, Error.None)
    testing.expect_value(t, got, Ref{1, 0})

    s, ok := to_stream(obj)
    testing.expect(t, ok, "not a stream")
    testing.expect(t, slice.equal(s.raw, bytes_of("Hello")), "wrong payload")
    testing.expect_value(t, s.ref, Ref{1, 0})
    testing.expect_value(t, dict_int(doc, s.dict, "Length"), i64(5))
    testing.expect(t, len(doc.repairs) == 0, "a well-formed stream needs no repair")

    // The payload is a view into the file, not a copy: a 200 MB document must
    // not be doubled in memory by parsing it.
    base := uintptr(raw_data(doc.data))
    at   := uintptr(raw_data(s.raw))
    testing.expect(t, at >= base && at < base + uintptr(len(doc.data)),
        "the stream payload was copied out of doc.data")
}

@(test)
test_parser_stream_bad_length :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    Case :: struct {
        src:  string,
        want: string,
    }
    cases := []Case{
        // Too long, too short, absurd, negative, missing, not a number.
        {"%PDF-1.7\n1 0 obj\n<< /Length 999 >>\nstream\r\nHello\nendstream\nendobj\n", "Hello"},
        {"%PDF-1.7\n1 0 obj\n<< /Length 2 >>\nstream\r\nHello\nendstream\nendobj\n", "Hello"},
        {"%PDF-1.7\n1 0 obj\n<< /Length 99999999999999 >>\nstream\r\nHello\nendstream\nendobj\n", "Hello"},
        {"%PDF-1.7\n1 0 obj\n<< /Length -5 >>\nstream\r\nHello\nendstream\nendobj\n", "Hello"},
        {"%PDF-1.7\n1 0 obj\n<< >>\nstream\r\nHello\nendstream\nendobj\n", "Hello"},
        {"%PDF-1.7\n1 0 obj\n<< /Length /Bogus >>\nstream\r\nHello\nendstream\nendobj\n", "Hello"},
        // The EOL before `endstream` is a separator, not payload -- one
        // marker's worth, whether it is LF, CR or CRLF.
        {"%PDF-1.7\n1 0 obj\n<< /Length 0 >>\nstream\r\nHello\r\nendstream\nendobj\n", "Hello"},
        {"%PDF-1.7\n1 0 obj\n<< /Length 0 >>\nstream\r\nHello\rendstream\nendobj\n", "Hello"},
        {"%PDF-1.7\n1 0 obj\n<< /Length 0 >>\nstream\r\nHelloendstream\nendobj\n", "Hello"},
        // Binary payload with an embedded "endstream"-looking prefix.
        {"%PDF-1.7\n1 0 obj\n<< /Length 7 >>\nstream\r\nend\x00\x01\x02x\nendstream\nendobj\n",
         "end\x00\x01\x02x"},
    }

    for c in cases {
        doc := test_doc(c.src, alloc)
        obj, _, err := parse_first_object(doc, alloc)
        testing.expectf(t, err == .None, "%q: error %v", c.src, err)
        s, ok := to_stream(obj)
        testing.expectf(t, ok, "%q: not a stream", c.src)
        if !ok { continue }
        testing.expectf(t, slice.equal(s.raw, bytes_of(c.want)),
            "%q: payload %q, want %q", c.src, string(s.raw), c.want)
    }

    // The first case above is the canonical one: it must be reported.
    doc := test_doc(cases[0].src, alloc)
    parse_first_object(doc, alloc)
    testing.expect(t, has_repair(doc, .Bad_Stream_Length), "no Bad_Stream_Length repair")
}

@(test)
test_parser_stream_indirect_length :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    src := "%PDF-1.7\n" +
           "1 0 obj\n<< /Length 5 0 R >>\nstream\r\nHello\nendstream\nendobj\n" +
           "5 0 obj\n5\nendobj\n"
    doc := test_doc(src, alloc)
    test_xref_at(doc, 5, "5 0 obj")

    obj, _, err := parse_first_object(doc, alloc)
    testing.expect_value(t, err, Error.None)
    s, ok := to_stream(obj)
    testing.expect(t, ok, "not a stream")
    testing.expect(t, slice.equal(s.raw, bytes_of("Hello")), "wrong payload")
    testing.expect(t, !has_repair(doc, .Bad_Stream_Length),
        "an indirect /Length that is right needs no repair")

    // An indirect /Length pointing at nothing falls back to the scan.
    src2 := "%PDF-1.7\n1 0 obj\n<< /Length 9 0 R >>\nstream\r\nHello\nendstream\nendobj\n"
    doc2 := test_doc(src2, alloc)
    obj2, _, err2 := parse_first_object(doc2, alloc)
    testing.expect_value(t, err2, Error.None)
    s2, ok2 := to_stream(obj2)
    testing.expect(t, ok2, "not a stream")
    testing.expect(t, slice.equal(s2.raw, bytes_of("Hello")), "wrong payload")
    testing.expect(t, has_repair(doc2, .Bad_Stream_Length), "no Bad_Stream_Length repair")

    // An indirect /Length that points back at the stream's own object must not
    // recurse: the cycle guard in get_object turns it into null.
    src3 := "%PDF-1.7\n1 0 obj\n<< /Length 1 0 R >>\nstream\r\nHello\nendstream\nendobj\n"
    doc3 := test_doc(src3, alloc)
    test_xref_at(doc3, 1, "1 0 obj")
    obj3, _, err3 := parse_first_object(doc3, alloc)
    testing.expect_value(t, err3, Error.None)
    s3, ok3 := to_stream(obj3)
    testing.expect(t, ok3, "not a stream")
    testing.expect(t, slice.equal(s3.raw, bytes_of("Hello")), "wrong payload")
}

@(test)
test_parser_stream_eol_tolerance :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    Case :: struct {
        src:  string,
        want: string,
    }
    cases := []Case{
        // CRLF (the only spelling the spec allows), LF, a bare CR, and a
        // trailing space before the EOL. All four are in the wild.
        {"%PDF\n1 0 obj\n<< /Length 5 >>\nstream\r\nHello\nendstream\nendobj\n", "Hello"},
        {"%PDF\n1 0 obj\n<< /Length 5 >>\nstream\nHello\nendstream\nendobj\n", "Hello"},
        {"%PDF\n1 0 obj\n<< /Length 5 >>\nstream\rHello\nendstream\nendobj\n", "Hello"},
        {"%PDF\n1 0 obj\n<< /Length 5 >>\nstream  \r\nHello\nendstream\nendobj\n", "Hello"},
        // A payload that starts with what looks like an EOL: only one marker
        // is consumed, so the leading LF here is data.
        {"%PDF\n1 0 obj\n<< /Length 6 >>\nstream\r\n\nHello\nendstream\nendobj\n", "\nHello"},
        // Zero-length streams are legal and common (empty appearance streams).
        {"%PDF\n1 0 obj\n<< /Length 0 >>\nstream\r\n\nendstream\nendobj\n", ""},
    }

    for c in cases {
        doc := test_doc(c.src, alloc)
        obj, _, err := parse_first_object(doc, alloc)
        testing.expectf(t, err == .None, "%q: error %v", c.src, err)
        s, ok := to_stream(obj)
        testing.expectf(t, ok, "%q: not a stream", c.src)
        if !ok { continue }
        testing.expectf(t, slice.equal(s.raw, bytes_of(c.want)),
            "%q: payload %q, want %q", c.src, string(s.raw), c.want)
    }
}

@(test)
test_parser_stream_missing_endstream :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    src := "%PDF-1.7\n1 0 obj\n<< /Length 9999 >>\nstream\r\nHello, truncated"
    doc := test_doc(src, alloc)
    obj, _, err := parse_first_object(doc, alloc)
    testing.expect_value(t, err, Error.None)
    s, ok := to_stream(obj)
    testing.expect(t, ok, "not a stream")
    testing.expect(t, slice.equal(s.raw, bytes_of("Hello, truncated")),
        "a truncated stream keeps what is there")
    testing.expect(t, has_repair(doc, .Missing_Endstream), "no Missing_Endstream repair")
}

@(test)
test_parser_stream_only_at_top_level :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    // A `stream` keyword after a *nested* dictionary is not a stream: only the
    // body of an indirect object can be one.
    obj, err := parse_one("[ << /A 1 >> stream ]", alloc)
    testing.expect_value(t, err, Error.None)
    a, _ := to_array(obj)
    testing.expect_value(t, array_len(a), 1)
    _, is_dict := to_dict(array_get_raw(a, 0))
    testing.expect(t, is_dict, "the inner object is a plain dictionary")
    _, is_stream := to_stream(array_get_raw(a, 0))
    testing.expect(t, !is_stream, "a nested dictionary must not become a stream")
}

// ---------------------------------------------------------------------------
// Indirect objects
// ---------------------------------------------------------------------------

@(test)
test_parse_indirect_object_at :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    // The trailing padding puts the junk further from object 8 than the
    // window search reaches.
    src :: "%PDF-1.7\n7 0 obj\n<< /Type /Page >>\nendobj\n8 2 obj\n42\nendobj\n" +
           "                                                                         " +
           "                                                                         " +
           "not an object at all\n"
    doc := test_doc(src, alloc)
    at7 := i64(strings.index(src, "7 0 obj"))
    at8 := i64(strings.index(src, "8 2 obj"))

    obj, got, err := parse_indirect_object_at(doc, at7, Ref{7, 0})
    testing.expect_value(t, err, Error.None)
    testing.expect_value(t, got, Ref{7, 0})
    testing.expect_value(t, dict_name(doc, to_dict(obj) or_else nil, "Type"), "Page")

    obj8, got8, err8 := parse_indirect_object_at(doc, at8, Ref{8, 2})
    testing.expect_value(t, err8, Error.None)
    testing.expect_value(t, got8, Ref{8, 2})
    testing.expect(t, object_same(obj8, i64(42)), "object 8 is not 42")

    // An offset a few bytes off is searched around, and the reference that was
    // actually found comes back so a stale table can be detected.
    for delta in ([]i64{-3, -1, 1, 4, 12}) {
        o, g, e := parse_indirect_object_at(doc, at7 + delta, Ref{7, 0})
        testing.expectf(t, e == .None, "delta %d: error %v", delta, e)
        testing.expectf(t, g == Ref{7, 0}, "delta %d: got %v", delta, g)
        testing.expectf(t, o != nil, "delta %d: no object", delta)
    }

    // The window must not match the digits inside a longer number: "8 2 obj"
    // found from an offset near object 7 is still object 8, never object 2.
    _, gnear, _ := parse_indirect_object_at(doc, at8 - 2, Ref{8, 0})
    testing.expect_value(t, gnear, Ref{8, 2})

    // Nowhere near an object header.
    _, _, enone := parse_indirect_object_at(doc, i64(strings.index(src, "not an object")), Ref{9, 0})
    testing.expect_value(t, enone, Error.Object_Not_Found)

    // Out of range offsets, including the ones a broken xref produces.
    for bad in ([]i64{-1, 0x7fff_ffff, i64(len(src)), i64(len(src)) + 1000}) {
        _, _, ebad := parse_indirect_object_at(doc, bad, Ref{7, 0})
        testing.expectf(t, ebad == .Object_Not_Found, "offset %d: error %v", bad, ebad)
    }

    // A nil Document is a caller error, not a crash.
    _, _, enil := parse_indirect_object_at(nil, 0, Ref{1, 0})
    testing.expect_value(t, enil, Error.Invalid_Argument)
}

@(test)
test_parse_indirect_object_stale_xref :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    // The offset points at object 4 while the table promised object 3. The
    // caller gets the truth and decides; here it is close enough that the
    // window search must not "helpfully" find something else.
    src := "%PDF-1.7\n4 0 obj\n(four)\nendobj\n"
    doc := test_doc(src, alloc)
    _, got, err := parse_indirect_object_at(doc, i64(strings.index(src, "4 0 obj")), Ref{3, 0})
    testing.expect_value(t, err, Error.None)
    testing.expect_value(t, got, Ref{4, 0})
}

@(test)
test_parse_indirect_object_empty_body :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    // "1 0 obj endobj" is an object with no body, which is null.
    src := "%PDF-1.7\n1 0 obj\nendobj\n"
    doc := test_doc(src, alloc)
    obj, got, err := parse_indirect_object_at(doc, i64(strings.index(src, "1 0 obj")), Ref{1, 0})
    testing.expect_value(t, err, Error.None)
    testing.expect_value(t, got, Ref{1, 0})
    testing.expect(t, is_null(obj), "an empty object body is null")

    // A missing generation number is tolerated: "1 obj" is generation 0.
    src2 := "%PDF-1.7\n1 obj\n(x)\nendobj\n"
    doc2 := test_doc(src2, alloc)
    obj2, got2, err2 := parse_indirect_object_at(doc2, i64(strings.index(src2, "1 obj")), Ref{1, 0})
    testing.expect_value(t, err2, Error.None)
    testing.expect_value(t, got2, Ref{1, 0})
    testing.expect(t, obj2 != nil, "the body should still be read")
}

@(test)
test_parse_indirect_object_header_offset :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    // Junk before "%PDF-" is common (mail headers, HTTP responses), and some
    // of those files measure their offsets from the header instead of byte 0.
    junk :: "MIME-Version: 1.0\r\nContent-Type: application/pdf\r\n\r\n"
    src :: junk + "%PDF-1.4\n3 0 obj\n(payload)\nendobj\n"
    doc := test_doc(src, alloc)
    doc.header_offset = i64(len(junk))

    relative := i64(strings.index(src, "3 0 obj")) - doc.header_offset
    obj, got, err := parse_indirect_object_at(doc, relative, Ref{3, 0})
    testing.expect_value(t, err, Error.None)
    testing.expect_value(t, got, Ref{3, 0})
    testing.expect(t, slice.equal(to_bytes(obj) or_else nil, bytes_of("payload")), "wrong payload")
}

// ---------------------------------------------------------------------------
// Fuzz-shaped
// ---------------------------------------------------------------------------

@(private = "file")
FUZZ_SRC :: "%PDF-1.7\n12 0 obj\n<< /Type /Page /Parent 3 0 R /Kids [4 0 R 5 0 R] " +
            "/S (a\\(b\\) c) /H <41 42 43> /N /A#20B /D [1 [2 [3 [4]]]] /L 1e3 /Length 4 >>\n" +
            "stream\r\nBODY\nendstream\nendobj\n"

@(test)
test_parser_survives_truncation :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    // Every prefix of a real object must parse or fail, never hang or crash.
    whole := FUZZ_SRC
    for n in 0 ..= len(whole) {
        src := whole[:n]
        doc := test_doc(src, alloc)
        at := strings.index(src, "12 0 obj")
        if at >= 0 {
            _, _, _ = parse_indirect_object_at(doc, i64(at), Ref{12, 0})
        }
        p := parser_make(doc, doc.data, alloc)
        _, _ = parse_object(&p)
        testing.expectf(t, p.lex.pos <= len(src), "truncation %d ran past the end", n)
    }
}

@(test)
test_parser_survives_corruption :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    // Single-byte substitutions over the same object, cycling through the
    // bytes that carry syntactic weight.
    poison := "()<>[]{}/%\\ \r\n0123456789.+-eRobj\x00\xff"
    buf := make([]byte, len(FUZZ_SRC), alloc)
    for i in 0 ..< len(FUZZ_SRC) {
        copy(buf, FUZZ_SRC)
        buf[i] = poison[i % len(poison)]
        doc := test_doc(string(buf), alloc)
        at := strings.index(string(buf), "12 0 obj")
        if at >= 0 {
            _, _, _ = parse_indirect_object_at(doc, i64(at), Ref{12, 0})
        }
        p := parser_make(doc, doc.data, alloc)
        _, _ = parse_object(&p)
        testing.expectf(t, p.lex.pos <= len(buf), "corruption at %d ran past the end", i)
    }
}

@(test)
test_parser_cyclic_and_absurd :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    // A reference chain that loops. `resolve` bounds it; nothing here should
    // recurse.
    src := "%PDF\n1 0 obj\n2 0 R\nendobj\n2 0 obj\n1 0 R\nendobj\n"
    doc := test_doc(src, alloc)
    test_xref_at(doc, 1, "1 0 obj")
    test_xref_at(doc, 2, "2 0 obj")
    testing.expect(t, is_null(resolve(doc, Object(Ref{1, 0}))), "a reference loop resolves to null")

    // An object that refers to itself. Resolving /Self is not an error -- it
    // is the same object -- but it must terminate, and the parse that had to
    // read /Length out of it must not have recursed.
    src2 := "%PDF\n1 0 obj\n<< /Self 1 0 R >>\nendobj\n"
    doc2 := test_doc(src2, alloc)
    test_xref_at(doc2, 1, "1 0 obj")
    d, ok := to_dict(get_object(doc2, Ref{1, 0}))
    testing.expect(t, ok, "the object itself is still readable")
    self, self_ok := to_dict(dict_get(doc2, d, "Self"))
    testing.expect(t, self_ok && self == d, "the self-reference should resolve to the object")

    // A dictionary whose value is twice as deep as the limit: the limit is
    // reported, and the recursion stops at MAX_DEPTH frames rather than at the
    // end of the stack.
    deep := strings.builder_make(0, 256, alloc)
    strings.write_string(&deep, "<< /A ")
    for _ in 0 ..< 2 * MAX_DEPTH { strings.write_string(&deep, "[") }
    _, err := parse_one(strings.to_string(deep), alloc)
    testing.expect_value(t, err, Error.Recursion_Limit)
}

// ---------------------------------------------------------------------------
// Decryption wiring
//
// The cipher itself lives in crypt.odin; what belongs to the parser is which
// Ref a string or payload is decrypted with, and the rule that objects out of
// an object stream are not decrypted individually.
// ---------------------------------------------------------------------------

@(test)
test_parser_obj_ref_defaults_to_zero :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    // A fresh parser carries the zero Ref, which is the signal not to decrypt.
    // Object-stream contents and inline images are parsed exactly this way.
    doc := test_doc("<< /Length 4 >>\nstream\r\nBODY\nendstream\n", alloc)
    p := parser_make(doc, doc.data, alloc)
    testing.expect_value(t, p.obj_ref, Ref{})

    obj, err := parse_object(&p)
    testing.expect_value(t, err, Error.None)
    s, ok := to_stream(obj)
    testing.expect(t, ok, "not a stream")
    testing.expect_value(t, s.ref, Ref{})

    // Parsed as an indirect object instead, the stream carries the identity
    // its key would be derived from.
    doc2 := test_doc(STREAM_SRC, alloc)
    obj2, _, err2 := parse_first_object(doc2, alloc)
    testing.expect_value(t, err2, Error.None)
    s2, ok2 := to_stream(obj2)
    testing.expect(t, ok2, "not a stream")
    testing.expect_value(t, s2.ref, Ref{1, 0})
}