pdf-odin-llm/cmap_test.odin

package pdf

import "core:fmt"
import "core:mem"
import "core:mem/virtual"
import "core:testing"
import "core:time"

// ---------------------------------------------------------------------------
// Fixture
//
// Every CMap under test is built in a local arena, so a failing assertion
// cannot leak: the arena is destroyed on the way out whatever happened. The
// context allocator is pointed at it too, because `cmap_load` with a nil
// Document has no arena of its own to use.
// ---------------------------------------------------------------------------

@(private = "file")
Cmap_Fixture :: struct {
    arena: virtual.Arena,
    alloc: mem.Allocator,
}

@(private = "file")
cmf_init :: proc(f: ^Cmap_Fixture) {
    _ = virtual.arena_init_growing(&f.arena)
    f.alloc = virtual.arena_allocator(&f.arena)
}

@(private = "file")
cmf_destroy :: proc(f: ^Cmap_Fixture) {
    virtual.arena_destroy(&f.arena)
}

@(private = "file")
cmf_parse :: proc(f: ^Cmap_Fixture, src: string) -> ^Cmap {
    cm, err := cmap_parse(nil, transmute([]byte)src, f.alloc)
    if err != .None || cm == nil {
        return nil
    }
    return cm
}

// cmf_stream makes a filterless stream, whose payload `stream_data` hands back
// as a sub-slice of the input: no allocation, nothing to own.
@(private = "file")
cmf_stream :: proc(f: ^Cmap_Fixture, src: string) -> ^Stream {
    d := new_dict(f.alloc)
    return new_stream(d, transmute([]byte)src, f.alloc)
}

// cmf_codes decodes a whole string the way a text operator would, returning
// (code, width) pairs. It also asserts the loop always advances.
@(private = "file")
cmf_codes :: proc(t: ^testing.T, cm: ^Cmap, s: []byte) -> (codes: [dynamic]u32, widths: [dynamic]int) {
    for i := 0; i < len(s); {
        code, n := cmap_next_code(cm, s[i:])
        if !testing.expectf(t, n > 0, "cmap_next_code stalled at offset %d", i) {
            break
        }
        append(&codes, code)
        append(&widths, n)
        i += n
    }
    return
}

@(private = "file")
expect_text :: proc(t: ^testing.T, cm: ^Cmap, code: u32, want: string, loc := #caller_location) {
    got, ok := cmap_lookup_text(cm, code)
    testing.expectf(t, ok, "no text for code 0x%X, wanted %q", code, want, loc = loc)
    testing.expectf(t, got == want, "code 0x%X: got %q, wanted %q", code, got, want, loc = loc)
}

// ---------------------------------------------------------------------------
// Variable-width codes
// ---------------------------------------------------------------------------

@(test)
test_cmap_mixed_width_codespace :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // A Shift-JIS-shaped encoding: single bytes below 0x81, and a two-byte
    // lead/trail pair above it. Both widths in one CMap is the whole reason
    // this file exists.
    cm := cmf_parse(&f, `
/CIDInit /ProcSet findresource begin
12 dict begin
begincmap
/CMapName /Test-H def
/CMapType 1 def
2 begincodespacerange
<00> <80>
<8140> <9ffc>
endcodespacerange
endcmap
CMapName currentdict /CMap defineresource pop
end
end
`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    src := []byte{0x41, 0x81, 0x40, 0x7F, 0x9F, 0xFC, 0x00}
    codes, widths := cmf_codes(t, cm, src)
    defer delete(codes)
    defer delete(widths)

    want_codes  := []u32{0x41, 0x8140, 0x7F, 0x9FFC, 0x00}
    want_widths := []int{1, 2, 1, 2, 1}
    testing.expectf(t, len(codes) == len(want_codes), "segmented into %d codes: %v", len(codes), codes)
    for w, i in want_codes {
        if i >= len(codes) { break }
        testing.expectf(t, codes[i] == w, "code %d: got 0x%X, want 0x%X", i, codes[i], w)
        testing.expectf(t, widths[i] == want_widths[i],
            "code %d: got %d bytes, want %d", i, widths[i], want_widths[i])
    }
}

@(test)
test_cmap_codespace_is_bytewise :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    cm := cmf_parse(&f, `begincmap
2 begincodespacerange
<00> <80>
<8140> <9ffc>
endcodespacerange
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    // 0x8200 is inside the numeric interval [0x8140, 0x9FFC] but its trailing
    // byte is outside [0x40, 0xFC], so it is NOT a code in this range
    // (ISO 32000-2, 9.7.6.2) -- it is an invalid code. Its lead byte is still
    // inside [0x81, 0x9F], which is a partial match, so 9.7.6.3 b) makes the
    // invalid code two bytes wide all the same.
    code, n := cmap_next_code(cm, []byte{0x82, 0x00})
    testing.expectf(t, n == 2, "0x8200 consumed %d bytes; a partial match consumes the range's width", n)
    testing.expectf(t, code == 0x8200, "got code 0x%X, want 0x8200", code)

    // 0x8240 is inside both byte ranges, so it is a two-byte code.
    code2, n2 := cmap_next_code(cm, []byte{0x82, 0x40})
    testing.expectf(t, n2 == 2 && code2 == 0x8240, "0x8240: got 0x%X in %d bytes", code2, n2)

    // Where the two readings really part company is the width. Give the same
    // two-byte range a one-byte range that covers every lead byte: bytewise,
    // 0x82 0x00 is a complete one-byte code and nothing longer matches, so it
    // is one byte. Read as a numeric interval, 0x8200 would fall inside
    // [0x8140, 0x9FFC] and swallow the byte after it.
    cm2 := cmf_parse(&f, `begincmap
2 begincodespacerange
<00> <ff>
<8140> <9ffc>
endcodespacerange
endcmap`)
    if !testing.expect(t, cm2 != nil, "parse failed") { return }
    code3, n3 := cmap_next_code(cm2, []byte{0x82, 0x00})
    testing.expectf(t, n3 == 1 && code3 == 0x82,
        "0x82 0x00 under a one-byte catch-all: got 0x%X in %d bytes, want 0x82 in 1", code3, n3)
    code4, n4 := cmap_next_code(cm2, []byte{0x82, 0x40})
    testing.expectf(t, n4 == 2 && code4 == 0x8240,
        "0x8240 is still the longer complete match: got 0x%X in %d bytes", code4, n4)
}

// ISO 32000-2, 9.7.6.3 splits an unmatched code into two cases. a) A first
// byte that begins no range at all takes the width of the shortest range.
// b) A first byte that DOES begin a longer range takes that range's width,
// even though the bytes after it do not complete the code. Only b) keeps the
// rest of the string in step: one byte too few and every code after the bad
// one is read at the wrong offset, so a single stray byte in a CJK string
// turns the whole of the rest of it into wrong glyphs.
@(test)
test_cmap_next_code_partial_match :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // The codespace of the standard's own example.
    cm := cmf_parse(&f, `begincmap
4 begincodespacerange
<00> <80>
<8140> <9ffc>
<a0> <df>
<e040> <fbec>
endcodespacerange
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    Case :: struct { src: []byte, want_code: u32, want_n: int }
    for c in ([]Case{
        // 0x81 opens the two-byte range but 0x20 does not complete it.
        {[]byte{0x81, 0x20, 0x41}, 0x8120, 2},
        {[]byte{0x82, 0x39},       0x8239, 2},
        {[]byte{0x90, 0x30, 0x41}, 0x9030, 2},
        // The same for the second two-byte range.
        {[]byte{0xE0, 0x00},       0xE000, 2},
        // 0xFC is outside [0xE0, 0xFB] and every other lead range, so this is
        // rule a) and the shortest range decides: one byte.
        {[]byte{0xFC, 0x41},       0xFC,   1},
        // Valid codes are untouched.
        {[]byte{0x41},             0x41,   1},
        {[]byte{0x81, 0x40},       0x8140, 2},
        {[]byte{0xA0, 0x41},       0xA0,   1},
        // A partial match with the string running out still consumes what is
        // left rather than stalling.
        {[]byte{0x81},             0x81,   1},
    }) {
        code, n := cmap_next_code(cm, c.src)
        testing.expectf(t, n == c.want_n && code == c.want_code,
            "%X: got 0x%X in %d bytes, want 0x%X in %d", c.src, code, n, c.want_code, c.want_n)
    }

    // The whole point: the code after a bad one lands where it should.
    codes, widths := cmf_codes(t, cm, []byte{0x81, 0x20, 0x41})
    defer delete(codes)
    defer delete(widths)
    testing.expectf(t, len(codes) == 2, "segmented into %d codes: %v", len(codes), codes)
    if len(codes) == 2 {
        testing.expectf(t, codes[1] == 0x41, "the code after the invalid one is 0x%X, want 0x41", codes[1])
    }
}

@(test)
test_cmap_next_code_always_advances :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // Two-byte codespace only, driven with an odd number of bytes and with a
    // byte outside every range: neither may stall the caller.
    cm := cmf_parse(&f, `begincmap
1 begincodespacerange <2000> <20ff> endcodespacerange
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    _, n := cmap_next_code(cm, []byte{0x99, 0x99})
    testing.expectf(t, n == 2, "unmatched bytes consumed %d, want the shortest range's 2", n)

    // A trailing half code unit still has to be consumed.
    code, n1 := cmap_next_code(cm, []byte{0x20})
    testing.expectf(t, n1 == 1 && code == 0x20, "trailing byte: got 0x%X in %d bytes", code, n1)

    // No CMap and no bytes are both defined.
    c0, n0 := cmap_next_code(cm, nil)
    testing.expectf(t, n0 == 0 && c0 == 0, "empty string: got 0x%X in %d bytes", c0, n0)
    cn, nn := cmap_next_code(nil, []byte{0xAB, 0xCD})
    testing.expectf(t, nn == 1 && cn == 0xAB, "nil CMap: got 0x%X in %d bytes", cn, nn)
}

@(test)
test_cmap_identity :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    cm := cmap_identity(false, f.alloc)
    if !testing.expect(t, cm != nil, "cmap_identity returned nil") { return }
    testing.expect(t, cm.name == "Identity-H", "wrong name")
    testing.expect(t, cm.wmode == 0, "Identity-H is horizontal")
    testing.expect(t, !cm.predefined_fallback, "Identity-H is not a fallback")

    src := []byte{0x00, 0x41, 0xFF, 0xFE}
    codes, widths := cmf_codes(t, cm, src)
    defer delete(codes)
    defer delete(widths)
    testing.expectf(t, len(codes) == 2, "identity segmented into %d codes: %v", len(codes), codes)
    if len(codes) == 2 {
        testing.expectf(t, codes[0] == 0x0041 && widths[0] == 2, "first code 0x%X", codes[0])
        testing.expectf(t, codes[1] == 0xFFFE && widths[1] == 2, "second code 0x%X", codes[1])
    }

    for code in ([]u32{0, 1, 0x41, 0x1234, 0xFFFF}) {
        cid, ok := cmap_lookup(cm, code)
        testing.expectf(t, ok && cid == code, "identity lookup 0x%X gave 0x%X, %v", code, cid, ok)
    }

    v := cmap_identity(true, f.alloc)
    if !testing.expect(t, v != nil, "cmap_identity(vertical) returned nil") { return }
    testing.expect(t, v.name == "Identity-V" && v.wmode == 1, "Identity-V is vertical")
}

@(test)
test_cmap_predefined_fallback :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // The CJK registries are not shipped. A name from one must still yield a
    // usable two-byte CMap, and must say that it is not the real thing.
    cm, err := cmap_load(nil, Name("UniJIS-UCS2-H"))
    if !testing.expectf(t, err == .None && cm != nil, "load: %s", error_string(err)) { return }
    testing.expect(t, cm.predefined_fallback, "a substituted CMap must be flagged")
    testing.expect(t, cm.name == "UniJIS-UCS2-H", "the requested name is kept")
    testing.expect(t, cm.wmode == 0, "-H is horizontal")
    _, n := cmap_next_code(cm, []byte{0x30, 0x42})
    testing.expectf(t, n == 2, "the fallback is two-byte, got %d", n)

    vert, verr := cmap_load(nil, Name("UniJIS-UCS2-V"))
    if !testing.expectf(t, verr == .None && vert != nil, "load: %s", error_string(verr)) { return }
    testing.expect(t, vert.wmode == 1, "-V must be recognised as vertical")

    // Identity-H is real, not a fallback.
    id, ierr := cmap_load(nil, Name("Identity-H"))
    if !testing.expectf(t, ierr == .None && id != nil, "load: %s", error_string(ierr)) { return }
    testing.expect(t, !id.predefined_fallback, "Identity-H must not be flagged as substituted")

    // Neither a name nor a stream is the one case with nothing to hand back.
    bad, berr := cmap_load(nil, i64(7))
    testing.expect(t, bad == nil && berr == .Bad_Cmap, "a non-CMap object is an error")
}

// ---------------------------------------------------------------------------
// CID mappings
// ---------------------------------------------------------------------------

@(test)
test_cmap_cid_ranges_and_chars :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    cm := cmf_parse(&f, `begincmap
1 begincodespacerange <0000> <ffff> endcodespacerange
2 begincidrange
<0000> <00ff> 100
<0300> <03ff> 900
endcidrange
1 begincidchar
<1000> 4242
endcidchar
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    Case :: struct { code: u32, cid: u32, ok: bool }
    for c in ([]Case{
        {0x0000, 100,  true},
        {0x0010, 116,  true},
        {0x00FF, 355,  true},
        {0x0300, 900,  true},
        {0x03FF, 1155, true},
        {0x1000, 4242, true},
        {0x0100, 0,    false},
        {0x2000, 0,    false},
    }) {
        cid, ok := cmap_lookup(cm, c.code)
        testing.expectf(t, ok == c.ok, "code 0x%X: found=%v, want %v", c.code, ok, c.ok)
        if c.ok {
            testing.expectf(t, cid == c.cid, "code 0x%X: cid %d, want %d", c.code, cid, c.cid)
        }
    }
}

@(test)
test_cmap_bad_ranges_dropped :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    cm := cmf_parse(&f, `begincmap
2 begincodespacerange
<0030> <0020>
<0000> <ffff>
endcodespacerange
2 begincidrange
<0030> <0020> 7
<0040> <0041> 9
endcidrange
1 beginbfrange
<0060> <0050> <0041>
endbfrange
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    // A range that runs backwards describes nothing and must not be stored,
    // let alone answered from.
    _, ok30 := cmap_lookup(cm, 0x30)
    testing.expect(t, !ok30, "a lo>hi cidrange must be dropped")
    _, ok20 := cmap_lookup(cm, 0x20)
    testing.expect(t, !ok20, "a lo>hi cidrange must be dropped")
    _, okbf := cmap_lookup_text(cm, 0x60)
    testing.expect(t, !okbf, "a lo>hi bfrange must be dropped")

    cid, ok := cmap_lookup(cm, 0x41)
    testing.expectf(t, ok && cid == 10, "the good range still works: %d, %v", cid, ok)

    // Dropped, not merely never matched: a backwards range that is kept would
    // still be searched on every lookup, and `dst + (code - lo)` under it
    // underflows.
    testing.expectf(t, len(cm.ranges) == 1, "kept %d cid ranges, want 1", len(cm.ranges))
    testing.expectf(t, len(cm.codespace) == 1, "kept %d codespace ranges, want 1", len(cm.codespace))
    testing.expectf(t, len(cm.texts) == 0, "kept %d bf entries, want 0", len(cm.texts))

    // The backwards codespace range is gone, so only the two-byte one is left.
    _, n := cmap_next_code(cm, []byte{0x00, 0x41})
    testing.expectf(t, n == 2, "a lo>hi codespace range must be dropped, got width %d", n)
}

// ---------------------------------------------------------------------------
// /ToUnicode
// ---------------------------------------------------------------------------

@(test)
test_cmap_bfrange_numeric_destination :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    cm := cmf_parse(&f, `begincmap
1 begincodespacerange <0000> <ffff> endcodespacerange
1 beginbfrange
<0020> <0022> <0041>
endbfrange
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    expect_text(t, cm, 0x20, "A")
    expect_text(t, cm, 0x21, "B")
    expect_text(t, cm, 0x22, "C")
    _, ok := cmap_lookup_text(cm, 0x23)
    testing.expect(t, !ok, "one past the end of a bfrange is unmapped")
}

@(test)
test_cmap_bfrange_array_destination :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // The second bfrange form: one destination string per source code, which
    // is how a subset font spells out an irregular run.
    cm := cmf_parse(&f, `begincmap
1 begincodespacerange <0000> <ffff> endcodespacerange
2 beginbfrange
<0005> <0007> [<0041> <00420043> <d840dc3e>]
<0010> <0013> [<0058> <0059>]
endbfrange
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    expect_text(t, cm, 0x05, "A")
    expect_text(t, cm, 0x06, "BC")
    expect_text(t, cm, 0x07, "\U0002003E")

    // An array shorter than the range maps only as far as it reaches.
    expect_text(t, cm, 0x10, "X")
    expect_text(t, cm, 0x11, "Y")
    _, ok := cmap_lookup_text(cm, 0x12)
    testing.expect(t, !ok, "a short destination array must not invent entries")
}

@(test)
test_cmap_bfchar_ligature_and_surrogates :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    cm := cmf_parse(&f, `begincmap
1 begincodespacerange <0000> <ffff> endcodespacerange
5 beginbfchar
<0001> <00660066006c>
<0002> <d840dc3e>
<0003> <d800>
<0004> <dc00>
<0005> <004160>
endbfchar
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    // A single glyph standing for a ligature: three code points, one code.
    expect_text(t, cm, 0x0001, "ffl")
    // A surrogate pair is one character, U+2003E, not two.
    expect_text(t, cm, 0x0002, "\U0002003E")
    // An unpaired surrogate, high or low, is U+FFFD -- never invalid UTF-8.
    expect_text(t, cm, 0x0003, "�")
    expect_text(t, cm, 0x0004, "�")
    // A destination that is not a whole number of UTF-16 code units -- three
    // bytes -- ends in half a unit, which is U+FFFD like any other truncation.
    expect_text(t, cm, 0x0005, "A�")
}

@(test)
test_cmap_later_definition_wins :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // A broad bfrange followed by a bfchar correcting one code out of it is a
    // shape real producers emit; the correction has to be the one that sticks.
    cm := cmf_parse(&f, `begincmap
1 begincodespacerange <0000> <ffff> endcodespacerange
1 beginbfrange
<0041> <005a> <0061>
endbfrange
1 beginbfchar
<0042> <0030>
endbfchar
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    expect_text(t, cm, 0x41, "a")
    expect_text(t, cm, 0x42, "0")
    expect_text(t, cm, 0x43, "c")
}

@(test)
test_cmap_absurd_range_is_clamped :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // Four billion codes in one line is legal syntax. Expanding it is not an
    // option, so the expansion stops -- but what it did produce is correct.
    cm := cmf_parse(&f, `begincmap
1 begincodespacerange <00000000> <ffffffff> endcodespacerange
1 beginbfrange
<00000000> <ffffffff> <0041>
endbfrange
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    expect_text(t, cm, 0, "A")
    testing.expectf(t, len(cm.texts) == MAX_CMAP_RANGE_CODES,
        "expanded %d entries, the cap is %d", len(cm.texts), MAX_CMAP_RANGE_CODES)
    _, ok := cmap_lookup_text(cm, 0x00FF_FFFF)
    testing.expect(t, !ok, "the clamped tail of the range is simply unmapped")
}

// A bfrange whose expansion is thrown away must not be expanded. One
// `<0000> <ffff>` line is half the entry cap and two fill it, so every line
// after that produces nothing at all -- while still costing 65536 iterations
// each if the loop only learns about the cap from the entries it stores being
// dropped. Nothing bounds the number of lines but the token fuel, which allows
// about 1.4 million of them, and a body that repetitive compresses to a few
// tens of KB: a 30 KB file would pin a core for half an hour.
@(test)
test_cmap_repeated_full_bfrange_terminates :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    LINES :: 20_000
    src: [dynamic]u8
    defer delete(src)
    append(&src, ..transmute([]byte)string(
        "begincmap\n1 begincodespacerange <0000> <ffff> endcodespacerange\nbeginbfrange\n"))
    for _ in 0 ..< LINES {
        append(&src, ..transmute([]byte)string("<0000><ffff><0020>\n"))
    }
    append(&src, ..transmute([]byte)string("endbfrange\nendcmap\n"))

    started := time.now()
    cm := cmf_parse(&f, string(src[:]))
    ms := time.duration_milliseconds(time.since(started))
    if !testing.expect(t, cm != nil, "parse failed") { return }

    // What it does produce is still right: the destination is incremented once
    // per code, and the repeated lines are all the same mapping.
    expect_text(t, cm, 0x0000, " ")
    expect_text(t, cm, 0x0021, "A")
    testing.expectf(t, len(cm.texts) == MAX_CMAP_RANGE_CODES,
        "expanded %d distinct codes, want %d", len(cm.texts), MAX_CMAP_RANGE_CODES)

    // Not a performance bound: without the early exit this same body takes
    // about 25 seconds, and the limit is loose enough that a loaded machine
    // cannot trip it.
    testing.expectf(t, ms < 5000, "%d bfrange lines took %.0f ms", LINES, ms)
}

// ---------------------------------------------------------------------------
// Malformed input
// ---------------------------------------------------------------------------

@(test)
test_cmap_count_disagrees_with_contents :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // The integer before `beginbfchar` is a hint, not a contract: files get it
    // wrong in both directions and the contents are what count.
    cm := cmf_parse(&f, `begincmap
9 begincodespacerange <0000> <ffff> endcodespacerange
99 beginbfchar
<0041> <0061>
<0042> <0062>
endbfchar
0 begincidchar
<0043> 7
endcidchar
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    expect_text(t, cm, 0x41, "a")
    expect_text(t, cm, 0x42, "b")
    testing.expectf(t, len(cm.texts) == 2, "got %d text entries, want 2", len(cm.texts))
    cid, ok := cmap_lookup(cm, 0x43)
    testing.expectf(t, ok && cid == 7, "a section declared empty still maps its contents: %d %v", cid, ok)
}

@(test)
test_cmap_truncated :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // Stops in the middle of a bfchar section, with a half-written pair. What
    // was collected before the cut has to survive.
    cm := cmf_parse(&f, `begincmap
1 begincodespacerange <0000> <ffff> endcodespacerange
3 beginbfchar
<0041> <0061>
<0042> `)
    if !testing.expect(t, cm != nil, "a truncated CMap still parses") { return }

    expect_text(t, cm, 0x41, "a")
    _, ok := cmap_lookup_text(cm, 0x42)
    testing.expect(t, !ok, "the half-written pair maps nothing")
    _, n := cmap_next_code(cm, []byte{0x00, 0x41})
    testing.expectf(t, n == 2, "the codespace before the cut survives, got width %d", n)
}

@(test)
test_cmap_unterminated_section :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // No `endbfchar`; the next `begin...` is where the section really ended.
    cm := cmf_parse(&f, `begincmap
1 begincodespacerange <0000> <ffff>
1 beginbfchar
<0041> <0061>
1 begincidchar
<0043> 7
endcidchar
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    expect_text(t, cm, 0x41, "a")
    cid, ok := cmap_lookup(cm, 0x43)
    testing.expectf(t, ok && cid == 7, "the section after the unterminated one still parses: %d %v", cid, ok)
    _, n := cmap_next_code(cm, []byte{0x00, 0x41})
    testing.expectf(t, n == 2, "the unterminated codespace range still counts, got width %d", n)
}

@(test)
test_cmap_unknown_section_is_skipped :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // `notdefrange` and `usematrix` are sections this reader does not
    // interpret. Stepping over their contents is not the same as ignoring the
    // section header: what is inside must not reach the enclosing CMap, or a
    // stray `/WMode 1 def` in a section that has nothing to do with the
    // writing mode would turn the whole font vertical.
    cm := cmf_parse(&f, `begincmap
1 begincodespacerange <0000> <ffff> endcodespacerange
1 beginbfchar
<0041> <0061>
endbfchar
1 beginnotdefrange
<0041> <0062>
endnotdefrange
1 beginusematrix
/WMode 1 def
/CMapName /Wrong def
<0041> <0063>
endusematrix
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    expect_text(t, cm, 0x41, "a")
    testing.expectf(t, len(cm.texts) == 1, "got %d text entries, want 1", len(cm.texts))
    testing.expectf(t, cm.wmode == 0, "a /WMode inside a skipped section must not apply, got %d", cm.wmode)
    testing.expectf(t, cm.name == "", "a /CMapName inside a skipped section must not apply, got %q", cm.name)
}

@(test)
test_cmap_longest_codespace_wins :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // Overlapping widths: 0x41 is a code in the one-byte range and also the
    // lead byte of the two-byte range. The wider code is the intended one, so
    // declaration order must not decide it.
    cm := cmf_parse(&f, `begincmap
2 begincodespacerange
<00> <ff>
<4100> <41ff>
endcodespacerange
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    code, n := cmap_next_code(cm, []byte{0x41, 0x00})
    testing.expectf(t, n == 2 && code == 0x4100, "longest match: got 0x%X in %d bytes", code, n)
    // Only one byte left: the wide range cannot apply and the narrow one does.
    c1, n1 := cmap_next_code(cm, []byte{0x41})
    testing.expectf(t, n1 == 1 && c1 == 0x41, "one byte left: got 0x%X in %d bytes", c1, n1)
    // A byte the wide range does not lead with is narrow.
    c2, n2 := cmap_next_code(cm, []byte{0x42, 0x00})
    testing.expectf(t, n2 == 1 && c2 == 0x42, "narrow code: got 0x%X in %d bytes", c2, n2)
}

@(test)
test_cmap_junk_never_hangs :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // A well-formed CMap cut at every offset, plus a few shapes chosen to make
    // the tokeniser unhappy. None may crash, hang or fail to return a CMap.
    full := `begincmap
/CMapName /X def
/WMode 0 def
2 begincodespacerange <00> <80> <8140> <9ffc> endcodespacerange
1 begincidrange <8140> <9ffc> 1 endcidrange
1 beginbfrange <0041> <005a> [<0061> <0062>] endbfrange
1 beginbfchar <0041> <d840dc3e> endbfchar
endcmap`
    for cut in 0 ..= len(full) {
        cm, err := cmap_parse(nil, transmute([]byte)full[:cut], f.alloc)
        testing.expectf(t, err == .None && cm != nil, "cut at %d: %s", cut, error_string(err))
        if cm != nil {
            _, _ = cmap_next_code(cm, []byte{0x81, 0x40, 0x41})
            _, _ = cmap_lookup(cm, 0x8140)
            _, _ = cmap_lookup_text(cm, 0x41)
        }
    }

    for junk in ([]string{
        "",
        "endcmap",
        "beginbfchar",
        "beginbfrange <0041",
        "beginbfrange <0041> <0042> [",
        "beginbfrange <0041> <0042> [<0061>",
        "begincodespacerange <> <>",
        "begincidrange /a /b /c endcidrange",
        "begincidchar <41> -5 endcidchar",
        "beginbfchar () () endbfchar",
        "usecmap def begin end pop",
        "/WMode /oops def",
        ") ) ) > > >",
        "begin begincmap begin",
    }) {
        cm, err := cmap_parse(nil, transmute([]byte)junk, f.alloc)
        testing.expectf(t, err == .None && cm != nil, "%q: %s", junk, error_string(err))
        if cm != nil {
            _, n := cmap_next_code(cm, []byte{0x41, 0x42})
            testing.expectf(t, n > 0, "%q: decoding stalled", junk)
        }
    }
}

// ---------------------------------------------------------------------------
// /UseCMap
// ---------------------------------------------------------------------------

@(test)
test_cmap_usecmap_inheritance :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    parent := cmf_stream(&f, `begincmap
1 begincodespacerange <0000> <ffff> endcodespacerange
1 begincidrange <0000> <00ff> 500 endcidrange
/WMode 1 def
endcmap`)
    child := cmf_stream(&f, `begincmap
1 begincidchar <0005> 7 endcidchar
endcmap`)
    dict_set(child.dict, "UseCMap", parent)

    cm, err := cmap_load(nil, child)
    if !testing.expectf(t, err == .None && cm != nil, "load: %s", error_string(err)) { return }

    // The child's own entry answers first.
    own, ok_own := cmap_lookup(cm, 0x05)
    testing.expectf(t, ok_own && own == 7, "own entry: %d %v", own, ok_own)
    // Anything it does not define falls through to the used CMap.
    inh, ok_inh := cmap_lookup(cm, 0x06)
    testing.expectf(t, ok_inh && inh == 506, "inherited entry: %d %v", inh, ok_inh)
    // A CMap with no codespace ranges of its own takes the used CMap's, which
    // is what makes the inherited mapping addressable at all: without the
    // inheritance there would be no ranges here and codes would be one byte.
    code, n := cmap_next_code(cm, []byte{0x00, 0x06, 0x00, 0x07})
    testing.expectf(t, n == 2 && code == 0x0006, "inherited codespace: 0x%X in %d bytes", code, n)
    testing.expectf(t, cm.wmode == 1, "writing mode is inherited too, got %d", cm.wmode)

    _, miss := cmap_lookup(cm, 0x1234)
    testing.expect(t, !miss, "a code neither CMap defines is unmapped")
}

@(test)
test_cmap_usecmap_own_codespace_wins :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    parent := cmf_stream(&f, `begincmap
1 begincodespacerange <00> <ff> endcodespacerange
endcmap`)
    child := cmf_stream(&f, `begincmap
1 begincodespacerange <0000> <ffff> endcodespacerange
endcmap`)
    dict_set(child.dict, "UseCMap", parent)

    cm, err := cmap_load(nil, child)
    if !testing.expectf(t, err == .None && cm != nil, "load: %s", error_string(err)) { return }

    _, n := cmap_next_code(cm, []byte{0x00, 0x41})
    testing.expectf(t, n == 2, "the CMap's own codespace must win, got width %d", n)
}

@(test)
test_cmap_usecmap_cycle :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // Two CMaps naming each other, and one naming itself. Both must terminate
    // and still produce a usable CMap.
    a := cmf_stream(&f, `begincmap
1 begincodespacerange <00> <ff> endcodespacerange
1 begincidchar <01> 11 endcidchar
endcmap`)
    b := cmf_stream(&f, `begincmap
1 begincidchar <02> 22 endcidchar
endcmap`)
    dict_set(a.dict, "UseCMap", b)
    dict_set(b.dict, "UseCMap", a)

    cm, err := cmap_load(nil, a)
    if !testing.expectf(t, err == .None && cm != nil, "load: %s", error_string(err)) { return }
    v1, ok1 := cmap_lookup(cm, 0x01)
    testing.expectf(t, ok1 && v1 == 11, "own entry survives the cycle: %d %v", v1, ok1)
    v2, ok2 := cmap_lookup(cm, 0x02)
    testing.expectf(t, ok2 && v2 == 22, "one hop through the cycle still resolves: %d %v", v2, ok2)

    self := cmf_stream(&f, `begincmap
1 begincidchar <03> 33 endcidchar
endcmap`)
    dict_set(self.dict, "UseCMap", self)
    scm, serr := cmap_load(nil, self)
    if !testing.expectf(t, serr == .None && scm != nil, "self-reference: %s", error_string(serr)) { return }
    v3, ok3 := cmap_lookup(scm, 0x03)
    testing.expectf(t, ok3 && v3 == 33, "self-referencing CMap: %d %v", v3, ok3)
}

@(test)
test_cmap_usecmap_operator :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // The `usecmap` operator names a predefined CMap. Identity-H is the only
    // one worth naming that this library actually has.
    cm := cmf_parse(&f, `begincmap
/Identity-H usecmap
1 begincidchar <0005> 77 endcidchar
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }

    own, ok_own := cmap_lookup(cm, 0x0005)
    testing.expectf(t, ok_own && own == 77, "own entry: %d %v", own, ok_own)
    // Identity answers for everything else, and lends its codespace.
    inh, ok_inh := cmap_lookup(cm, 0x1234)
    testing.expectf(t, ok_inh && inh == 0x1234, "identity parent: 0x%X %v", inh, ok_inh)
    code, n := cmap_next_code(cm, []byte{0x12, 0x34})
    testing.expectf(t, n == 2 && code == 0x1234, "inherited codespace: 0x%X in %d bytes", code, n)
}

@(test)
test_cmap_wmode_and_name :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    cm := cmf_parse(&f, `begincmap
/CIDSystemInfo << /Registry (Adobe) /Ordering (Japan1) /Supplement 6 >> def
/CMapName /Test-V def
/CMapType 1 def
/WMode 1 def
1 begincodespacerange <0000> <ffff> endcodespacerange
endcmap`)
    if !testing.expect(t, cm != nil, "parse failed") { return }
    testing.expectf(t, cm.wmode == 1, "/WMode 1 def, got %d", cm.wmode)
    testing.expectf(t, cm.name == "Test-V", "/CMapName, got %q", cm.name)

    // The dictionary entries a stream may carry instead.
    s := cmf_stream(&f, "begincmap\n1 begincodespacerange <0000> <ffff> endcodespacerange\nendcmap")
    dict_set(s.dict, "WMode", i64(1))
    dict_set(s.dict, "CMapName", Name("Dict-V"))
    scm, err := cmap_load(nil, s)
    if !testing.expectf(t, err == .None && scm != nil, "load: %s", error_string(err)) { return }
    testing.expectf(t, scm.wmode == 1, "/WMode from the stream dictionary, got %d", scm.wmode)
    testing.expectf(t, scm.name == "Dict-V", "/CMapName from the stream dictionary, got %q", scm.name)
}

// ---------------------------------------------------------------------------
// A CMap shaped like the ones real files carry
// ---------------------------------------------------------------------------

@(test)
test_cmap_realistic_tounicode :: proc(t: ^testing.T) {
    f: Cmap_Fixture
    cmf_init(&f)
    defer cmf_destroy(&f)
    context.allocator = f.alloc

    // Byte-for-byte the shape a Word or InDesign export writes for a subset
    // TrueType font, which is 63% of the corpus.
    src := `/CIDInit /ProcSet findresource begin
12 dict begin
begincmap
/CIDSystemInfo
<< /Registry (Adobe)
/Ordering (UCS)
/Supplement 0
>> def
/CMapName /Adobe-Identity-UCS def
/CMapType 2 def
1 begincodespacerange
<0000> <FFFF>
endcodespacerange
3 beginbfrange
<0003> <0003> <0020>
<000F> <0018> <002C>
<0024> <003D> <0041>
endbfrange
2 beginbfchar
<00AB> <00660066>
<00B6> <FB03>
endbfchar
endcmap
CMapName currentdict /CMap defineresource pop
end
end`
    cm, err := cmap_parse(nil, transmute([]byte)src, f.alloc)
    if !testing.expectf(t, err == .None && cm != nil, "parse: %s", error_string(err)) { return }

    testing.expectf(t, cm.name == "Adobe-Identity-UCS", "name %q", cm.name)
    expect_text(t, cm, 0x0003, " ")
    expect_text(t, cm, 0x000F, ",")
    expect_text(t, cm, 0x0018, "5")
    expect_text(t, cm, 0x0024, "A")
    expect_text(t, cm, 0x003D, "Z")
    expect_text(t, cm, 0x00AB, "ff")
    expect_text(t, cm, 0x00B6, "ffi")

    // Decoding a two-byte string end to end, the way a Tj operand arrives.
    s := []byte{0x00, 0x24, 0x00, 0x25, 0x00, 0xAB}
    b: [64]byte
    k := 0
    for i := 0; i < len(s); {
        code, n := cmap_next_code(cm, s[i:])
        if n <= 0 { break }
        i += n
        if txt, ok := cmap_lookup_text(cm, code); ok {
            k += copy(b[k:], txt)
        }
    }
    got := string(b[:k])
    testing.expectf(t, got == "ABff", "decoded %q, want %q", got, fmt.tprint("ABff"))
}