pdf-odin-llm/xref.odin

package pdf

// Cross-reference reading: classic tables (ISO 32000-2, 7.5.4), cross-reference
// streams (7.5.8), the trailer chain, and rebuild-by-scanning.
//
// The layout of this file follows the order a reader meets the file's
// structure in: find `startxref`, walk the /Prev chain from the newest section
// to the oldest, merge the trailers, and -- when any of that does not add up --
// throw the lot away and find the objects by scanning.
//
// Two rules run through everything here.
//
// The first is that a *newer* section always wins. The chain is walked newest
// to oldest and an entry that has already been established is never
// overwritten, because that is precisely what an incremental update means: the
// bytes at the end of the file supersede the ones before them.
//
// The second is that nothing in this file is allowed to fail the open while a
// rebuild is still possible. A cross-reference table is an index, not content;
// every object it describes is still in the file whether or not the index
// agrees, and a reader that refuses a document because its index is wrong is
// refusing a document that every other viewer opens.

// Bytes scanned backwards from the end of the file for `startxref` before
// falling back to searching the whole file. The format requires it to be in
// the last few dozen bytes; 2 KiB covers the files that append a signature,
// a trailing comment banner or a stray page of junk after it.
@(private)
STARTXREF_WINDOW :: 2048

// How many `startxref` keywords are tried, newest first. A file whose last
// startxref was written by a tool that crashed part way through still has the
// previous one, and following that is far better than a full rebuild.
@(private)
MAX_STARTXREF_CANDIDATES :: 4

// How far either side of a claimed section offset the section is looked for.
// Producers that miscount by a byte or two are common; a window this small
// cannot wander into an unrelated part of the file.
@(private)
XREF_LOCATE_WINDOW :: 64

// Smallest a classic cross-reference entry can be written in. The format says
// 20 bytes exactly, real files use 19, and the tolerant reader below will take
// "0 0 n" -- 5 -- so that is what allocations are bounded against.
@(private)
XREF_ENTRY_MIN_BYTES :: 5

// Bytes searched past the end of the subsections for the `trailer` keyword
// when it is not where it should be.
@(private)
XREF_TRAILER_WINDOW :: 4096

// Trailer dictionaries kept while rebuilding. Files with more `trailer`
// keywords than this are not going to be diagnosed by reading more of them.
@(private)
MAX_REBUILD_TRAILERS :: 64

// Bytes of an object's body looked at before deciding whether it is worth
// parsing during a rebuild. An /ObjStm, /XRef or /Catalog dictionary names its
// own type long before this.
@(private)
REBUILD_PEEK_BYTES :: 4096

@(private)
Xref_Section_Kind :: enum u8 {
    None = 0,
    Classic,     // "xref" followed by subsections and a trailer
    Stream,      // an indirect object holding a /Type /XRef stream
}

// Xref_Loader is the state one call to `xref_load` (or `xref_rebuild`) needs
// beyond what the Document already holds.
//
// `defined` is parallel to `doc.xref` and is what makes "newer wins" work: a
// free entry written by a newer section has to shadow an in-use entry from an
// older one, and `Xref_Entry` on its own cannot tell "explicitly free" from
// "never mentioned".
@(private)
Xref_Loader :: struct {
    doc:         ^Document,
    defined:     [dynamic]bool,
    visited:     map[i64]bool,
    // Newest first; merged oldest-to-newest so the newest entry wins.
    trailers:    [dynamic]^Dict,
    max_objects: int,
    // Entries established that are not free. A table that describes no live
    // object at all is not a table.
    n_live:      int,
    // Objects a rebuild's scan flagged as worth parsing: the ones whose bytes
    // mention /ObjStm, /XRef or /Catalog.
    candidates:  [dynamic]u32,
}

// ---------------------------------------------------------------------------
// Entry points
// ---------------------------------------------------------------------------

// xref_load fills in `doc.xref` and `doc.trailer`.
//
// It returns an error only when the file has no recoverable structure at all:
// anything less than that -- a missing startxref, an offset pointing at
// nothing, a table that stops parsing half way, a /Root that is not a
// dictionary -- falls back to `xref_rebuild` and is recorded as a repair.
xref_load :: proc(doc: ^Document) -> Error {
    if doc == nil { return .Invalid_Argument }

    ld := xref_loader_make(doc)
    ok := xref_load_chain(&ld) && xref_finish(&ld)

    err := Error.None
    if !ok {
        err = xref_rebuild(doc)
    }

    // Everything resolved above was resolved against a table that was still
    // being built, and -- in an encrypted document -- before the file key
    // existed. Neither is a state to leave a cache in.
    clear(&doc.cache)
    if dict_has(doc.trailer, "Encrypt") {
        clear(&doc.objstm_cache)
    }
    return err
}

// xref_rebuild discards the cross-reference table and finds the objects by
// scanning the whole file for "N G obj" headers.
//
// Later definitions of the same object number win, which is what an
// incremental update means when read this way. Objects that exist only inside
// an object stream are picked up afterwards, by reading the header of every
// /ObjStm the scan found.
xref_rebuild :: proc(doc: ^Document) -> Error {
    if doc == nil { return .Invalid_Argument }

    add_repair(doc, .Rebuilt_Xref, 0)
    clear(&doc.xref)
    clear(&doc.cache)
    clear(&doc.objstm_cache)
    doc.trailer = nil

    ld := xref_loader_make(doc)
    catalog: Ref
    xref_scan_objects(&ld)
    xref_scan_objstms(&ld, &catalog)
    xref_rebuild_trailer(&ld, catalog)

    if ld.n_live == 0 {
        // No object header anywhere in the file. There is nothing to read.
        return .No_Xref
    }
    return .None
}

@(private)
xref_loader_make :: proc(doc: ^Document) -> Xref_Loader {
    ld := Xref_Loader{
        doc         = doc,
        max_objects = xref_max_objects(doc),
    }
    ld.defined  = make([dynamic]bool, 0, 64, doc.allocator)
    ld.visited  = make(map[i64]bool, 8, doc.allocator)
    ld.trailers = make([dynamic]^Dict, 0, 8, doc.allocator)
    ld.candidates = make([dynamic]u32, 0, 16, doc.allocator)
    return ld
}

// xref_max_objects bounds the table by what the file could possibly describe.
// Every object costs bytes wherever it is written -- a header and an `endobj`
// at the top level, a "number offset" pair inside an object stream -- so the
// file's own length is a ceiling on how many there can be. The floor keeps a
// tiny hand-written file from being clamped below its own /Size.
@(private)
xref_max_objects :: proc(doc: ^Document) -> int {
    n := max(len(doc.data), 1024)
    return min(n, MAX_OBJECT_NUMBER + 1)
}

// ---------------------------------------------------------------------------
// Table storage
// ---------------------------------------------------------------------------

@(private)
xref_grow :: proc(ld: ^Xref_Loader, count: int) {
    n := min(count, ld.max_objects)
    old := len(ld.doc.xref)
    if n <= old { return }
    // `resize` reallocates to exactly the length asked for, and the table is
    // grown one object at a time while a file is scanned. Reserving
    // geometrically first is the difference between a linear rebuild and a
    // quadratic one -- and, on an arena, between one copy of the table and
    // thousands of abandoned ones.
    if cap(ld.doc.xref) < n {
        want := min(max(n, cap(ld.doc.xref) * 2, 64), ld.max_objects)
        reserve(&ld.doc.xref, max(want, n))
        reserve(&ld.defined, max(want, n))
    }
    resize(&ld.doc.xref, n)
    resize(&ld.defined, n)
    // Not every allocator hands back zeroed memory when a dynamic array is
    // regrown into capacity it already held, and a stale entry here is an
    // offset into the middle of the file.
    for i in old ..< len(ld.doc.xref) {
        ld.doc.xref[i] = {}
    }
    for i in old ..< len(ld.defined) {
        ld.defined[i] = false
    }
}

// xref_set records an entry unless a newer section already spoke for it.
//
// `over_free` is the one exception, and it exists for hybrid-reference files:
// there the classic table deliberately marks every compressed object free so
// that a reader which cannot follow /XRefStm ignores them, and the stream that
// follows carries their real entries (ISO 32000-2, 7.5.8.4).
@(private)
xref_set :: proc(ld: ^Xref_Loader, num: u32, entry: Xref_Entry, over_free := false) {
    if num == 0 || int(num) >= ld.max_objects { return }
    xref_grow(ld, int(num) + 1)
    if int(num) >= len(ld.doc.xref) { return }

    if ld.defined[num] {
        replaces_free := over_free &&
            ld.doc.xref[num].kind == .Free && entry.kind != .Free
        // The entry being replaced was free, so it was never counted as live.
        if !replaces_free { return }
    }
    ld.doc.xref[num] = entry
    ld.defined[num] = true
    if entry.kind != .Free {
        ld.n_live += 1
    }
}

// xref_put overwrites unconditionally. Only the rebuild uses it: there the
// scan runs from the front of the file to the back, so a later definition is a
// newer one and must replace what came before it.
@(private)
xref_put :: proc(ld: ^Xref_Loader, num: u32, entry: Xref_Entry) {
    if num == 0 || int(num) >= ld.max_objects { return }
    xref_grow(ld, int(num) + 1)
    if int(num) >= len(ld.doc.xref) { return }

    if ld.defined[num] && ld.doc.xref[num].kind != .Free {
        ld.n_live -= 1
    }
    ld.doc.xref[num] = entry
    ld.defined[num] = true
    if entry.kind != .Free {
        ld.n_live += 1
    }
}

@(private)
xref_entry_at :: proc(ld: ^Xref_Loader, num: u32) -> Xref_Entry {
    if int(num) >= len(ld.doc.xref) { return {} }
    return ld.doc.xref[num]
}

// ---------------------------------------------------------------------------
// startxref
// ---------------------------------------------------------------------------

// xref_last_startxref finds the last `startxref` before `before` whose keyword
// is followed by a number, and reports the number and where the keyword was.
//
// The search runs backwards because that is where the newest one is, and it is
// only ever called again -- for the one before it -- when the newest turned out
// to point at nothing.
@(private)
xref_last_startxref :: proc(doc: ^Document, before: int) -> (value: i64, at: int, ok: bool) {
    needle := transmute([]byte)string("startxref")
    limit := min(before, len(doc.data))
    for limit > 0 {
        idx := last_index_bytes(doc.data[:limit], needle)
        if idx < 0 { break }
        limit = idx
        if v, vok := xref_int_after(doc, idx + len(needle)); vok {
            return v, idx, true
        }
    }
    return 0, 0, false
}

// xref_first_startxref looks in the last couple of kilobytes, where the format
// puts the keyword, before paying for a scan of the whole file.
@(private)
xref_first_startxref :: proc(doc: ^Document) -> (value: i64, at: int, ok: bool) {
    tail := max(0, len(doc.data) - STARTXREF_WINDOW)
    if tail > 0 {
        needle := transmute([]byte)string("startxref")
        if idx := last_index_bytes(doc.data[tail:], needle); idx >= 0 {
            if v, vok := xref_int_after(doc, tail + idx + len(needle)); vok {
                return v, tail + idx, true
            }
        }
    }
    return xref_last_startxref(doc, len(doc.data))
}

@(private)
xref_int_after :: proc(doc: ^Document, pos: int) -> (value: i64, ok: bool) {
    if pos < 0 || pos > len(doc.data) { return 0, false }
    lex := Lexer{data = doc.data, pos = pos}
    tok := lexer_next_structural(&lex)
    if tok.kind != .Integer || tok.int_value < 0 { return 0, false }
    return tok.int_value, true
}

// ---------------------------------------------------------------------------
// Locating a section
// ---------------------------------------------------------------------------

// xref_probe reports what kind of cross-reference section starts at `off`, if
// any. It is the only thing that decides an offset is worth following, so it
// insists the keyword or object header begins on a token boundary: "xref" also
// occurs as the tail of "startxref".
@(private)
xref_probe :: proc(doc: ^Document, off: i64) -> Xref_Section_Kind {
    if off < 0 || off >= i64(len(doc.data)) { return .None }

    lex := Lexer{data = doc.data, pos = int(off)}
    t1 := lexer_next_structural(&lex)
    if t1.start > 0 && is_pdf_regular(doc.data[t1.start - 1]) { return .None }

    if t1.kind == .Keyword && t1.text == "xref" {
        return .Classic
    }
    if t1.kind != .Integer || t1.int_value < 1 || t1.int_value > MAX_OBJECT_NUMBER {
        return .None
    }
    t2 := lexer_next_structural(&lex)
    // "12 obj" -- a header whose generation number was dropped.
    if t2.kind == .Keyword && t2.text == "obj" { return .Stream }
    if t2.kind != .Integer || t2.int_value < 0 || t2.int_value > 65535 { return .None }
    t3 := lexer_next_structural(&lex)
    if t3.kind == .Keyword && t3.text == "obj" { return .Stream }
    return .None
}

// xref_locate turns an offset read out of the file into one that actually
// points at a section.
//
// Two things can be wrong with it. A file with junk before "%PDF-" measures
// its offsets from the header rather than from byte zero (and the ones written
// by a tool that did not notice the junk measure them from byte zero anyway,
// so both are tried). And a producer can simply be off by a few bytes, which
// is worth a small search rather than a whole-file rebuild.
@(private)
xref_locate :: proc(doc: ^Document, raw: i64) -> (off: i64, kind: Xref_Section_Kind) {
    bases := [2]i64{raw + doc.header_offset, raw}
    n_bases := doc.header_offset != 0 ? 2 : 1

    for i in 0 ..< n_bases {
        if k := xref_probe(doc, bases[i]); k != .None {
            return bases[i], k
        }
    }
    // Nearest first, so an exact-but-late match beats a spurious object header
    // found behind us.
    for d in 1 ..= i64(XREF_LOCATE_WINDOW) {
        for i in 0 ..< n_bases {
            for cand in ([2]i64{bases[i] + d, bases[i] - d}) {
                if cand < 0 || cand >= i64(len(doc.data)) { continue }
                if is_pdf_regular(doc.data[cand]) && cand > 0 && is_pdf_regular(doc.data[cand - 1]) {
                    continue
                }
                if k := xref_probe(doc, cand); k != .None {
                    return cand, k
                }
            }
        }
    }
    return 0, .None
}

// ---------------------------------------------------------------------------
// The chain
// ---------------------------------------------------------------------------

@(private)
xref_load_chain :: proc(ld: ^Xref_Loader) -> bool {
    doc := ld.doc

    raw, at, found := xref_first_startxref(doc)
    if !found {
        add_repair(doc, .Bad_Startxref, 0)
        return false
    }

    off, kind := xref_locate(doc, raw)
    // A tool that crashed part way through an incremental update leaves a
    // startxref pointing at nothing and the previous one still intact.
    for _ in 1 ..< MAX_STARTXREF_CANDIDATES {
        if kind != .None { break }
        add_repair(doc, .Bad_Startxref, raw)
        v, a, more := xref_last_startxref(doc, at)
        if !more { break }
        raw, at = v, a
        off, kind = xref_locate(doc, raw)
    }
    if kind == .None {
        add_repair(doc, .Bad_Startxref, raw)
        return false
    }

    got_any := false
    for _ in 0 ..< MAX_XREF_CHAIN {
        if ld.visited[off] or_else false {
            // /Prev loops are a real shape, not a hypothetical one: a file
            // whose last two sections point at each other opens in every
            // viewer because every viewer keeps this set.
            add_repair(doc, .Bad_Startxref, off)
            break
        }
        ld.visited[off] = true

        trailer, ok := xref_read_section(ld, off, false)
        if !ok { break }
        got_any = true
        append(&ld.trailers, trailer)

        xref_read_hybrid(ld, trailer)

        prev, has_prev := to_int(dict_get_raw(trailer, "Prev"))
        if !has_prev { break }
        poff, pkind := xref_locate(doc, prev)
        if pkind == .None {
            add_repair(doc, .Bad_Startxref, prev)
            break
        }
        off = poff
    }
    return got_any
}

// xref_read_hybrid reads the cross-reference stream a classic trailer points
// at with /XRefStm (ISO 32000-2, 7.5.8.4).
//
// It is read after the classic section it belongs to, so the classic entries
// win -- except where the classic table marked an object free, which in a
// hybrid file is how it hides compressed objects from older readers.
@(private)
xref_read_hybrid :: proc(ld: ^Xref_Loader, trailer: ^Dict) {
    if trailer == nil { return }
    raw, ok := to_int(dict_get_raw(trailer, "XRefStm"))
    if !ok { return }

    off, kind := xref_locate(ld.doc, raw)
    if kind != .Stream { return }
    if ld.visited[off] or_else false { return }
    ld.visited[off] = true

    // Its own /Prev is deliberately not followed: the classic chain drives.
    stm_dict, sok := xref_read_stream(ld, off, true)
    if sok && stm_dict != nil {
        append(&ld.trailers, stm_dict)
    }
}

@(private)
xref_read_section :: proc(ld: ^Xref_Loader, off: i64, over_free: bool) -> (trailer: ^Dict, ok: bool) {
    switch xref_probe(ld.doc, off) {
    case .Classic:
        return xref_read_table(ld, off)
    case .Stream:
        return xref_read_stream(ld, off, over_free)
    case .None:
        return nil, false
    }
    return nil, false
}

// ---------------------------------------------------------------------------
// Classic cross-reference tables (ISO 32000-2, 7.5.4)
// ---------------------------------------------------------------------------

// Xref_Raw_Entry is one line of a classic table, before it is decided which
// object number it belongs to.
@(private)
Xref_Raw_Entry :: struct {
    offset: i64,
    gen:    i64,
    used:   bool,    // 'n' rather than 'f'
    ok:     bool,
}

// xref_next_entry reads one entry by tokens rather than by fixed offsets.
//
// The spec fixes the entry at 20 bytes, but 19-byte entries (a single LF
// instead of CRLF) are written by a large fraction of producers, and entries
// with a single space before the EOL, with no leading zeros, or with the
// fields separated by a tab all occur. None of that changes the tokens.
@(private)
xref_next_entry :: proc(doc: ^Document, lex: ^Lexer) -> (e: Xref_Raw_Entry) {
    save := lex.pos
    t1 := lexer_next_structural(lex)
    if t1.kind != .Integer || t1.int_value < 0 {
        lex.pos = save
        return
    }
    t2 := lexer_next_structural(lex)
    if t2.kind != .Integer || t2.int_value < 0 {
        lex.pos = save
        return
    }
    t3 := lexer_next_structural(lex)
    if t3.kind != .Keyword || len(t3.text) == 0 {
        lex.pos = save
        return
    }
    c := t3.text[0]
    if c != 'n' && c != 'f' {
        lex.pos = save
        return
    }
    // "...00000 n0000000123 00000 n": no separator at all after the type
    // letter, so the tokeniser swallowed the next entry's offset with it.
    if len(t3.text) > 1 {
        lex.pos = t3.start + 1
    }

    e.offset = t1.int_value
    e.gen    = t2.int_value
    e.used   = c == 'n'
    e.ok     = true
    return
}

@(private)
xref_read_table :: proc(ld: ^Xref_Loader, off: i64) -> (trailer: ^Dict, ok: bool) {
    doc := ld.doc
    lex := Lexer{data = doc.data, pos = int(off)}
    if !lexer_accept_keyword(&lex, "xref") { return nil, false }

    // Each subsection header costs at least four bytes ("0 0"), and the loop
    // only continues while it is consuming tokens, so this bound is never the
    // thing that stops it -- it is here because a table is file data.
    fuel := len(doc.data) / 4 + 16
    first_sub := true

    for fuel > 0 {
        fuel -= 1
        lexer_skip_whitespace(&lex)
        if lex.pos >= len(doc.data) { break }
        if lexer_at_keyword(&lex, "trailer") { break }

        save := lex.pos
        t1 := lexer_next_structural(&lex)
        t2 := lexer_next_structural(&lex)
        if t1.kind != .Integer || t2.kind != .Integer {
            lex.pos = save
            break
        }
        xref_read_subsection(ld, &lex, t1.int_value, t2.int_value, first_sub)
        first_sub = false
    }

    return xref_read_trailer(ld, &lex), true
}

@(private)
xref_read_subsection :: proc(ld: ^Xref_Loader, lex: ^Lexer, start_in, count_in: i64, first_sub: bool) {
    doc := ld.doc

    if start_in < 0 || start_in > MAX_OBJECT_NUMBER { return }

    count := count_in
    if count <= 0 { return }
    // A subsection cannot describe more objects than the rest of the file has
    // room to spell out, however large a number the header claims.
    room := i64(len(doc.data) - lex.pos) / XREF_ENTRY_MIN_BYTES + 1
    if count > room {
        add_repair(doc, .Xref_Entry_Mismatch, i64(lex.pos))
        count = room
    }
    count = min(count, i64(ld.max_objects))

    // Look at the first few entries without consuming them: whether the
    // declared starting object number is right is decided from what the file
    // actually has at the offsets they name.
    peek: [8]Xref_Raw_Entry
    n_peek := 0
    {
        probe := lex^
        for n_peek < len(peek) {
            e := xref_next_entry(doc, &probe)
            if !e.ok { break }
            peek[n_peek] = e
            n_peek += 1
        }
    }
    start := xref_correct_start(doc, peek[:n_peek], start_in, first_sub)
    if start != start_in {
        add_repair(doc, .Xref_Entry_Mismatch, i64(lex.pos), Ref{u32(min(start_in, MAX_OBJECT_NUMBER)), 0})
    }
    if start + count > MAX_OBJECT_NUMBER + 1 {
        count = MAX_OBJECT_NUMBER + 1 - start
    }

    for i in 0 ..< count {
        e := xref_next_entry(doc, lex)
        if !e.ok {
            // The table stops making sense here. Everything read so far is
            // still good, and the entries after the damage are picked up from
            // an older section or, failing that, by a rebuild.
            add_repair(doc, .Xref_Entry_Mismatch, i64(lex.pos))
            break
        }
        num := start + i
        if num < 1 { continue }

        gen := u16(clamp(e.gen, 0, 65535))
        if !e.used {
            // A free entry's first field is the next free object number, not
            // an offset; nothing here needs the free list.
            xref_set(ld, u32(num), Xref_Entry{kind = .Free, gen = gen})
            continue
        }
        // Byte 0 is the header, so an in-use object can never be there. An
        // offset past the end of the file is nonsense too. Leaving such an
        // entry undefined lets an older section supply the object instead.
        if e.offset <= 0 || e.offset >= i64(len(doc.data)) { continue }
        at := xref_normalize_offset(doc, e.offset, num)
        xref_set(ld, u32(num), Xref_Entry{kind = .Normal, offset = at, gen = gen})
    }
}

// xref_object_at reports whether the header of object `num` is exactly at
// `off`.
@(private)
xref_object_at :: proc(doc: ^Document, off: i64, num: i64) -> bool {
    if num < 1 || num > MAX_OBJECT_NUMBER { return false }
    if off < 0 || off >= i64(len(doc.data)) { return false }
    lex := Lexer{data = doc.data, pos = int(off)}
    t1 := lexer_next_structural(&lex)
    if t1.kind != .Integer || t1.int_value != num { return false }
    t2 := lexer_next_structural(&lex)
    // "12 obj": a header whose generation number was dropped.
    if t2.kind == .Keyword && t2.text == "obj" { return true }
    if t2.kind != .Integer { return false }
    t3 := lexer_next_structural(&lex)
    return t3.kind == .Keyword && t3.text == "obj"
}

// xref_offset_is_object reports whether "num G obj" really is at `off`, from
// either origin, for the same reason `xref_locate` tries both.
@(private)
xref_offset_is_object :: proc(doc: ^Document, off: i64, num: i64) -> bool {
    return xref_object_at(doc, off, num) ||
           (doc.header_offset != 0 && xref_object_at(doc, off + doc.header_offset, num))
}

// xref_normalize_offset resolves which origin an entry's offset was measured
// from, and returns one that indexes `doc.data`.
//
// A file with junk before "%PDF-" has two plausible readings of every offset in
// its table: from byte zero, and from the header. Producers do both -- the
// wrapper is usually added by a tool that never looked inside -- and the
// difference is every object in the file, so it is settled here, against the
// bytes, rather than left for the object reader to guess at one reference at a
// time.
@(private)
xref_normalize_offset :: proc(doc: ^Document, off: i64, num: i64) -> i64 {
    if doc.header_offset == 0 { return off }
    if xref_object_at(doc, off, num) { return off }
    shifted := off + doc.header_offset
    if xref_object_at(doc, shifted, num) { return shifted }
    return off
}

@(private)
xref_score_start :: proc(doc: ^Document, ents: []Xref_Raw_Entry, start: i64) -> int {
    score := 0
    for e, i in ents {
        if !e.used { continue }
        if xref_offset_is_object(doc, e.offset, start + i64(i)) {
            score += 1
        }
    }
    return score
}

// xref_correct_start repairs the off-by-one in a subsection header.
//
// A table whose first subsection is written "1 N" when it really starts at 0
// (or "0 N" when it really starts at 1) is one of the most common
// malformations there is -- it shifts every object number in the file by one,
// so a reader that trusts the header reads the wrong object for every
// reference. The header is checked against the file: whichever starting number
// makes the most of the entries land on the object they claim wins.
@(private)
xref_correct_start :: proc(doc: ^Document, ents: []Xref_Raw_Entry, start: i64, first_sub: bool) -> i64 {
    if len(ents) == 0 { return start }

    base := xref_score_start(doc, ents, start)
    best := start
    best_score := base
    for delta in ([2]i64{-1, 1}) {
        cand := start + delta
        if cand < 0 || cand > MAX_OBJECT_NUMBER { continue }
        if s := xref_score_start(doc, ents, cand); s > best_score {
            best, best_score = cand, s
        }
    }
    if best != start {
        return best
    }
    if base > 0 || !first_sub {
        return start
    }

    // Nothing verified either way -- the entries may point into a part of the
    // file this section does not describe. Fall back to what the format
    // guarantees: object 0 is the head of the free list and can never be in
    // use, and the head is always generation 65535 (7.5.4).
    if start == 1 && ents[0].ok && !ents[0].used && ents[0].gen == 65535 {
        return 0
    }
    if start == 0 && ents[0].ok && ents[0].used {
        return 1
    }
    return start
}

@(private)
xref_read_trailer :: proc(ld: ^Xref_Loader, lex: ^Lexer) -> ^Dict {
    doc := ld.doc

    if !lexer_accept_keyword(lex, "trailer") {
        // Some producers drop the keyword and write the dictionary straight
        // after the table; that is unambiguous, so take it.
        probe := lex^
        tok := lexer_next_structural(&probe)
        if tok.kind != .Dict_Open {
            // Junk between the last entry and the keyword. Worth a bounded
            // look: the alternative is losing /Root and rebuilding the file.
            at := xref_find_token(doc.data, lex.pos, "trailer")
            if at < 0 || at - lex.pos > XREF_TRAILER_WINDOW { return nil }
            lex.pos = at + len("trailer")
        }
    }

    p := parser_make(doc, doc.data, doc.allocator)
    p.lex.pos = lex.pos
    // The trailer is never encrypted (ISO 32000-2, 7.6.2), and the zero
    // obj_ref is what says so to the parser.
    obj, err := parse_object(&p)
    if err != .None { return nil }
    d, ok := to_dict(obj)
    if !ok { return nil }
    return d
}

// ---------------------------------------------------------------------------
// Cross-reference streams (ISO 32000-2, 7.5.8)
// ---------------------------------------------------------------------------

@(private)
xref_read_stream :: proc(ld: ^Xref_Loader, off: i64, over_free: bool) -> (trailer: ^Dict, ok: bool) {
    doc := ld.doc

    obj, _, err := parse_indirect_object_at(doc, off, Ref{})
    if err != .None { return nil, false }
    s, is_stream := to_stream(obj)
    if !is_stream || s == nil || s.dict == nil { return nil, false }
    d := s.dict

    // /Type is required here, but a dictionary carrying /W and /Size is
    // unambiguous enough that a missing one is not worth refusing.
    if ty := dict_name(doc, d, "Type"); ty != "" && ty != "XRef" { return nil, false }

    // Field widths. A width of 0 means the field is not stored and takes its
    // default: type 1 for the first field, 0 for the others (7.5.8.2).
    w := dict_array(doc, d, "W")
    n_w := array_len(w)
    if n_w < 1 || n_w > 32 { return nil, false }
    widths: [32]int
    row := 0
    for i in 0 ..< n_w {
        v := array_int(doc, w, i, 0)
        if v < 0 || v > 8 { return nil, false }
        widths[i] = int(v)
        row += widths[i]
    }
    if row <= 0 { return nil, false }

    data, derr := stream_data(doc, s)
    if derr != .None { return nil, false }
    n_rows := len(data) / row
    if n_rows == 0 {
        // A /Type /XRef stream that decoded to nothing still carries the
        // trailer, and the entries may all live in an older section.
        return d, true
    }

    w0 := widths[0]
    w1 := n_w > 1 ? widths[1] : 0
    w2 := n_w > 2 ? widths[2] : 0
    off0 := 0
    off1 := w0
    off2 := w0 + w1

    size := dict_int(doc, d, "Size", 0)
    index := dict_array(doc, d, "Index")
    n_pairs := array_len(index) / 2
    if n_pairs == 0 {
        n_pairs = 1        // the default /Index is [0 /Size]
    }

    consumed := 0
    for p in 0 ..< n_pairs {
        start := i64(0)
        count := size
        if array_len(index) >= 2 {
            start = array_int(doc, index, p * 2, 0)
            count = array_int(doc, index, p * 2 + 1, 0)
        }
        if start < 0 || start > MAX_OBJECT_NUMBER { continue }
        if count <= 0 { continue }
        if count > i64(n_rows - consumed) {
            add_repair(doc, .Xref_Entry_Mismatch, off)
            count = i64(n_rows - consumed)
        }
        if start + count > MAX_OBJECT_NUMBER + 1 {
            count = MAX_OBJECT_NUMBER + 1 - start
        }

        for k in 0 ..< count {
            base := (consumed + int(k)) * row
            f0 := w0 == 0 ? u64(1) : xref_read_field(data, base + off0, w0)
            f1 := xref_read_field(data, base + off1, w1)
            f2 := xref_read_field(data, base + off2, w2)
            xref_stream_entry(ld, u32(start + k), f0, f1, f2, over_free)
        }
        consumed += int(count)
        if consumed >= n_rows { break }
    }
    return d, true
}

@(private)
xref_stream_entry :: proc(ld: ^Xref_Loader, num: u32, f0, f1, f2: u64, over_free: bool) {
    doc := ld.doc
    switch f0 {
    case 0:
        xref_set(ld, num, Xref_Entry{kind = .Free, gen = u16(min(f2, 65535))}, over_free)
    case 1:
        if f1 == 0 || f1 >= u64(len(doc.data)) { return }
        xref_set(ld, num, Xref_Entry{
            kind   = .Normal,
            offset = xref_normalize_offset(doc, i64(f1), i64(num)),
            gen    = u16(min(f2, 65535)),
        }, over_free)
    case 2:
        if f1 == 0 || f1 > MAX_OBJECT_NUMBER { return }
        if f2 > u64(max(u32)) { return }
        xref_set(ld, num, Xref_Entry{
            kind      = .Compressed,
            container = u32(f1),
            index     = u32(f2),
        }, over_free)
    case:
        // "Any other value shall be interpreted as a reference to the null
        // object" (7.5.8.3). It still shadows older sections.
        xref_set(ld, num, Xref_Entry{kind = .Free}, over_free)
    }
}

@(private)
xref_read_field :: proc(data: []byte, pos, width: int) -> u64 {
    v := u64(0)
    for i in 0 ..< width {
        j := pos + i
        if j < 0 || j >= len(data) { break }
        v = v << 8 | u64(data[j])
    }
    return v
}

// ---------------------------------------------------------------------------
// Trailers
// ---------------------------------------------------------------------------

// xref_merge_trailers folds the chain into one dictionary.
//
// The list runs newest first and is walked backwards, so a key present in
// several trailers takes its newest value while a key that only an older
// trailer had -- an /Info or an /Encrypt that a later incremental update did
// not repeat -- survives.
@(private)
xref_merge_trailers :: proc(ld: ^Xref_Loader) -> (merged: ^Dict, size: i64) {
    doc := ld.doc
    merged = new_dict(doc.allocator, 16)
    for i := len(ld.trailers) - 1; i >= 0; i -= 1 {
        t := ld.trailers[i]
        if t == nil { continue }
        for k, v in t^ {
            // Both are properties of one section, not of the document, and
            // keeping them in the merged trailer would invite a second walk.
            if k == "Prev" || k == "XRefStm" { continue }
            merged[k] = v
        }
        if v, ok := to_int(dict_get_raw(t, "Size")); ok && v > size {
            size = v
        }
    }
    if size > 0 {
        merged["Size"] = size
    }
    return
}

@(private)
xref_finish :: proc(ld: ^Xref_Loader) -> bool {
    doc := ld.doc
    if len(ld.trailers) == 0 { return false }

    merged, size := xref_merge_trailers(ld)
    doc.trailer = merged

    if size > 0 {
        xref_grow(ld, int(min(size, i64(ld.max_objects))))
    }
    if ld.n_live == 0 { return false }

    if dict_has(merged, "Encrypt") {
        // The catalog of an encrypted document may sit inside an object stream
        // that cannot be decoded until crypt_init has run, so there is nothing
        // to check yet beyond /Root being present at all.
        return dict_get_raw(merged, "Root") != nil
    }
    root, ok := to_dict(dict_get(doc, merged, "Root"))
    return ok && root != nil
}

// ---------------------------------------------------------------------------
// Rebuilding by scanning
// ---------------------------------------------------------------------------

// Kw_Cursor remembers where a keyword was last found so the scan below stays
// linear: every search resumes from the previous hit instead of from the
// caller's position.
@(private)
Kw_Cursor :: struct {
    kw:   string,
    at:   int,
    done: bool,
}

@(private)
kw_cursor :: proc(kw: string) -> Kw_Cursor {
    return Kw_Cursor{kw = kw, at = -1}
}

@(private)
kw_at_or_after :: proc(c: ^Kw_Cursor, data: []byte, from: int) -> int {
    if c.done { return -1 }
    if c.at >= from { return c.at }
    c.at = xref_find_token(data, from, c.kw)
    if c.at < 0 { c.done = true }
    return c.at
}

// xref_find_token finds `kw` as a whole token: "obj" must not match inside
// "endobj", and "stream" must not match inside "endstream".
@(private)
xref_find_token :: proc(data: []byte, from: int, kw: string) -> int {
    if from < 0 || len(kw) == 0 { return -1 }
    needle := transmute([]byte)kw
    i := max(from, 0)
    for i < len(data) {
        idx := index_bytes(data[i:], needle)
        if idx < 0 { return -1 }
        p := i + idx
        after := p + len(kw)
        before_ok := p == 0 || !is_pdf_regular(data[p - 1])
        after_ok  := after >= len(data) || !is_pdf_regular(data[after])
        if before_ok && after_ok { return p }
        i = p + 1
    }
    return -1
}

// xref_header_before reads the "N G" that precedes the `obj` keyword at `at`.
@(private)
xref_header_before :: proc(data: []byte, at: int) -> (num: u32, gen: u16, start: int, ok: bool) {
    k := at - 1
    for k >= 0 && is_pdf_whitespace(data[k]) { k -= 1 }
    gen_end := k + 1
    for k >= 0 && data[k] >= '0' && data[k] <= '9' { k -= 1 }
    gen_start := k + 1
    if gen_start == gen_end { return }

    if k < 0 || !is_pdf_whitespace(data[k]) { return }
    for k >= 0 && is_pdf_whitespace(data[k]) { k -= 1 }
    num_end := k + 1
    for k >= 0 && data[k] >= '0' && data[k] <= '9' { k -= 1 }
    num_start := k + 1
    if num_start == num_end { return }
    if k >= 0 && is_pdf_regular(data[k]) { return }

    n, nok := xref_digits(data[num_start:num_end])
    g, gok := xref_digits(data[gen_start:gen_end])
    if !nok || !gok { return }
    if n < 1 || n > MAX_OBJECT_NUMBER || g > 65535 { return }
    return u32(n), u16(g), num_start, true
}

@(private)
xref_digits :: proc(s: []byte) -> (value: i64, ok: bool) {
    // A run this long is not an object number whatever it is.
    if len(s) == 0 || len(s) > 12 { return 0, false }
    for c in s {
        if c < '0' || c > '9' { return 0, false }
        value = value * 10 + i64(c - '0')
    }
    return value, true
}

// xref_skip_object_body returns where scanning should resume after an object
// header, so that "N G obj" sequences inside a stream's binary payload are not
// mistaken for object headers of their own.
@(private)
xref_skip_object_body :: proc(data: []byte, from: int, c_stream, c_endobj, c_endstream: ^Kw_Cursor) -> int {
    s := kw_at_or_after(c_stream, data, from)
    e := kw_at_or_after(c_endobj, data, from)
    if s >= 0 && (e < 0 || s < e) {
        if es := kw_at_or_after(c_endstream, data, s + len("stream")); es >= 0 {
            return es + len("endstream")
        }
        // Truncated inside a stream: nothing after this is readable.
        return len(data)
    }
    if e >= 0 { return e + len("endobj") }
    return from
}

// xref_scan_objects walks the whole file recording every object header and
// every trailer dictionary it meets, in file order, so that a later definition
// of an object number replaces an earlier one.
@(private)
xref_scan_objects :: proc(ld: ^Xref_Loader) {
    doc := ld.doc
    data := doc.data

    c_obj       := kw_cursor("obj")
    c_trailer   := kw_cursor("trailer")
    c_stream    := kw_cursor("stream")
    c_endobj    := kw_cursor("endobj")
    c_endstream := kw_cursor("endstream")

    pos := 0
    fuel := len(data) + 16
    for fuel > 0 {
        fuel -= 1
        o := kw_at_or_after(&c_obj, data, pos)
        t := kw_at_or_after(&c_trailer, data, pos)
        if o < 0 && t < 0 { break }

        if t >= 0 && (o < 0 || t < o) {
            p := parser_make(doc, data, doc.allocator)
            p.lex.pos = t + len("trailer")
            if obj, err := parse_object(&p); err == .None {
                if d, dok := to_dict(obj); dok && d != nil {
                    // At the cap the oldest goes, not the newest: the trailers
                    // worth keeping are the ones nearest the end of the file.
                    if len(ld.trailers) >= MAX_REBUILD_TRAILERS {
                        ordered_remove(&ld.trailers, 0)
                    }
                    append(&ld.trailers, d)
                }
            }
            pos = t + len("trailer")
            continue
        }

        num, gen, start, hok := xref_header_before(data, o)
        if hok {
            xref_put(ld, num, Xref_Entry{kind = .Normal, offset = i64(start), gen = gen})
        }
        pos = xref_skip_object_body(data, o + len("obj"), &c_stream, &c_endobj, &c_endstream)
        if pos <= o {
            pos = o + len("obj")
        }
        // The /Type test happens here rather than in a second pass because
        // this is the only place the object's extent is known: bounding the
        // window by it keeps the whole scan linear in the file's length
        // instead of quadratic in its object count.
        if hok {
            hi := min(start + REBUILD_PEEK_BYTES, max(pos, start))
            if xref_head_is_interesting(data[start:min(hi, len(data))]) {
                append(&ld.candidates, num)
            }
        }
    }

    // The scan found them oldest first; everything downstream expects newest
    // first.
    for i, j := 0, len(ld.trailers) - 1; i < j; i, j = i + 1, j - 1 {
        ld.trailers[i], ld.trailers[j] = ld.trailers[j], ld.trailers[i]
    }
}

// xref_head_is_interesting reports whether the head of an object mentions one
// of the three /Type values a rebuild has to act on. One pass with a switch on
// the leading byte, because this runs over every object in a damaged file and
// three separate substring searches over the same window do not.
@(private)
xref_head_is_interesting :: proc(head: []byte) -> bool {
    for i in 0 ..< len(head) {
        switch head[i] {
        case 'O': if xref_starts_with(head[i:], "ObjStm")  { return true }
        case 'C': if xref_starts_with(head[i:], "Catalog") { return true }
        case 'X': if xref_starts_with(head[i:], "XRef")    { return true }
        }
    }
    return false
}

@(private)
xref_starts_with :: proc(data: []byte, prefix: string) -> bool {
    if len(data) < len(prefix) { return false }
    return string(data[:len(prefix)]) == prefix
}

// xref_scan_objstms registers the contents of every object stream the scan
// found, and picks up the /Type /XRef dictionaries and the catalog on the way.
//
// Without this pass a file whose cross-reference stream is damaged loses every
// object that lives in an object stream -- which, in a modern file, is most of
// them.
@(private)
xref_scan_objstms :: proc(ld: ^Xref_Loader, catalog: ^Ref) {
    doc := ld.doc

    // Parsing every object in the file to read its /Type would cost the arena
    // a second copy of the document; the scan already narrowed it down to the
    // objects whose bytes name one of the three types that matter here.
    objstms := make([dynamic]u32, 0, 8, doc.allocator)
    for num in ld.candidates {
        e := xref_entry_at(ld, num)
        if e.kind != .Normal { continue }
        obj, got, err := parse_indirect_object_at(doc, e.offset, Ref{num, e.gen})
        if err != .None || got.num != num { continue }
        d, dok := to_dict(obj)
        if !dok || d == nil { continue }

        switch dict_name(doc, d, "Type") {
        case "ObjStm":
            append(&objstms, num)
        case "XRef":
            if len(ld.trailers) < MAX_REBUILD_TRAILERS {
                // Lowest priority: a real `trailer` keyword, if the file has
                // one, is more likely to be the newest word on the subject.
                append(&ld.trailers, d)
            }
        case "Catalog":
            catalog^ = Ref{num, e.gen}
        }
    }

    // Object streams come last, because whether it is safe to read them at all
    // depends on what the trailers just collected say. In an encrypted
    // document the file key does not exist yet -- `crypt_init` runs after this
    // -- so every object stream would decode to ciphertext, and a pair table
    // read out of ciphertext invents object numbers that would then shadow the
    // real, recoverable entries the scan just found. Losing the compressed
    // objects is bad; overwriting the ones that survived is worse.
    if xref_trailers_encrypted(ld) {
        add_repair(doc, .Missing_Object, 0)
        return
    }
    for num in objstms {
        e := xref_entry_at(ld, num)
        if e.kind != .Normal { continue }
        xref_register_objstm(ld, num, e.offset)
    }
}

@(private)
xref_trailers_encrypted :: proc(ld: ^Xref_Loader) -> bool {
    for t in ld.trailers {
        if dict_has(t, "Encrypt") { return true }
    }
    return false
}

@(private)
xref_register_objstm :: proc(ld: ^Xref_Loader, container: u32, container_offset: i64) {
    doc := ld.doc
    stm, err := objstm_load(doc, container)
    if err != .None || stm == nil { return }
    // The dictionary survives whatever went wrong with the payload, so /N
    // disagreeing with the pair table means the bytes were not the pair table.
    // In the normal path a short table is worth keeping; here, where the
    // entries it produces overwrite ones a scan established, it is not.
    if stm.declared_n > 0 && len(stm.nums) != stm.declared_n { return }

    for n, i in stm.nums {
        if n == 0 || n == container { continue }
        cur := xref_entry_at(ld, n)
        // A top-level definition that sits later in the file than the object
        // stream is the newer of the two and keeps its place.
        if cur.kind == .Normal && cur.offset > container_offset { continue }
        xref_put(ld, n, Xref_Entry{kind = .Compressed, container = container, index = u32(i)})
    }
}

@(private)
xref_rebuild_trailer :: proc(ld: ^Xref_Loader, catalog: Ref) {
    doc := ld.doc
    merged, _ := xref_merge_trailers(ld)
    doc.trailer = merged
    merged["Size"] = i64(len(doc.xref))

    if root, ok := to_dict(dict_get(doc, merged, "Root")); ok && root != nil {
        if dict_type(doc, root) == "Catalog" || dict_has(root, "Pages") {
            return
        }
    }
    if catalog.num != 0 {
        merged["Root"] = catalog
        add_repair(doc, .Missing_Root, 0, catalog)
    }
}