package pdf import "core:fmt" import "core:mem" import "core:mem/virtual" import "core:testing" import "core:time" // --------------------------------------------------------------------------- // Fixture // // Every CMap under test is built in a local arena, so a failing assertion // cannot leak: the arena is destroyed on the way out whatever happened. The // context allocator is pointed at it too, because `cmap_load` with a nil // Document has no arena of its own to use. // --------------------------------------------------------------------------- @(private = "file") Cmap_Fixture :: struct { arena: virtual.Arena, alloc: mem.Allocator, } @(private = "file") cmf_init :: proc(f: ^Cmap_Fixture) { _ = virtual.arena_init_growing(&f.arena) f.alloc = virtual.arena_allocator(&f.arena) } @(private = "file") cmf_destroy :: proc(f: ^Cmap_Fixture) { virtual.arena_destroy(&f.arena) } @(private = "file") cmf_parse :: proc(f: ^Cmap_Fixture, src: string) -> ^Cmap { cm, err := cmap_parse(nil, transmute([]byte)src, f.alloc) if err != .None || cm == nil { return nil } return cm } // cmf_stream makes a filterless stream, whose payload `stream_data` hands back // as a sub-slice of the input: no allocation, nothing to own. @(private = "file") cmf_stream :: proc(f: ^Cmap_Fixture, src: string) -> ^Stream { d := new_dict(f.alloc) return new_stream(d, transmute([]byte)src, f.alloc) } // cmf_codes decodes a whole string the way a text operator would, returning // (code, width) pairs. It also asserts the loop always advances. @(private = "file") cmf_codes :: proc(t: ^testing.T, cm: ^Cmap, s: []byte) -> (codes: [dynamic]u32, widths: [dynamic]int) { for i := 0; i < len(s); { code, n := cmap_next_code(cm, s[i:]) if !testing.expectf(t, n > 0, "cmap_next_code stalled at offset %d", i) { break } append(&codes, code) append(&widths, n) i += n } return } @(private = "file") expect_text :: proc(t: ^testing.T, cm: ^Cmap, code: u32, want: string, loc := #caller_location) { got, ok := cmap_lookup_text(cm, code) testing.expectf(t, ok, "no text for code 0x%X, wanted %q", code, want, loc = loc) testing.expectf(t, got == want, "code 0x%X: got %q, wanted %q", code, got, want, loc = loc) } // --------------------------------------------------------------------------- // Variable-width codes // --------------------------------------------------------------------------- @(test) test_cmap_mixed_width_codespace :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // A Shift-JIS-shaped encoding: single bytes below 0x81, and a two-byte // lead/trail pair above it. Both widths in one CMap is the whole reason // this file exists. cm := cmf_parse(&f, ` /CIDInit /ProcSet findresource begin 12 dict begin begincmap /CMapName /Test-H def /CMapType 1 def 2 begincodespacerange <00> <80> <8140> <9ffc> endcodespacerange endcmap CMapName currentdict /CMap defineresource pop end end `) if !testing.expect(t, cm != nil, "parse failed") { return } src := []byte{0x41, 0x81, 0x40, 0x7F, 0x9F, 0xFC, 0x00} codes, widths := cmf_codes(t, cm, src) defer delete(codes) defer delete(widths) want_codes := []u32{0x41, 0x8140, 0x7F, 0x9FFC, 0x00} want_widths := []int{1, 2, 1, 2, 1} testing.expectf(t, len(codes) == len(want_codes), "segmented into %d codes: %v", len(codes), codes) for w, i in want_codes { if i >= len(codes) { break } testing.expectf(t, codes[i] == w, "code %d: got 0x%X, want 0x%X", i, codes[i], w) testing.expectf(t, widths[i] == want_widths[i], "code %d: got %d bytes, want %d", i, widths[i], want_widths[i]) } } @(test) test_cmap_codespace_is_bytewise :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc cm := cmf_parse(&f, `begincmap 2 begincodespacerange <00> <80> <8140> <9ffc> endcodespacerange endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } // 0x8200 is inside the numeric interval [0x8140, 0x9FFC] but its trailing // byte is outside [0x40, 0xFC], so it is NOT a code in this range // (ISO 32000-2, 9.7.6.2) -- it is an invalid code. Its lead byte is still // inside [0x81, 0x9F], which is a partial match, so 9.7.6.3 b) makes the // invalid code two bytes wide all the same. code, n := cmap_next_code(cm, []byte{0x82, 0x00}) testing.expectf(t, n == 2, "0x8200 consumed %d bytes; a partial match consumes the range's width", n) testing.expectf(t, code == 0x8200, "got code 0x%X, want 0x8200", code) // 0x8240 is inside both byte ranges, so it is a two-byte code. code2, n2 := cmap_next_code(cm, []byte{0x82, 0x40}) testing.expectf(t, n2 == 2 && code2 == 0x8240, "0x8240: got 0x%X in %d bytes", code2, n2) // Where the two readings really part company is the width. Give the same // two-byte range a one-byte range that covers every lead byte: bytewise, // 0x82 0x00 is a complete one-byte code and nothing longer matches, so it // is one byte. Read as a numeric interval, 0x8200 would fall inside // [0x8140, 0x9FFC] and swallow the byte after it. cm2 := cmf_parse(&f, `begincmap 2 begincodespacerange <00> <8140> <9ffc> endcodespacerange endcmap`) if !testing.expect(t, cm2 != nil, "parse failed") { return } code3, n3 := cmap_next_code(cm2, []byte{0x82, 0x00}) testing.expectf(t, n3 == 1 && code3 == 0x82, "0x82 0x00 under a one-byte catch-all: got 0x%X in %d bytes, want 0x82 in 1", code3, n3) code4, n4 := cmap_next_code(cm2, []byte{0x82, 0x40}) testing.expectf(t, n4 == 2 && code4 == 0x8240, "0x8240 is still the longer complete match: got 0x%X in %d bytes", code4, n4) } // ISO 32000-2, 9.7.6.3 splits an unmatched code into two cases. a) A first // byte that begins no range at all takes the width of the shortest range. // b) A first byte that DOES begin a longer range takes that range's width, // even though the bytes after it do not complete the code. Only b) keeps the // rest of the string in step: one byte too few and every code after the bad // one is read at the wrong offset, so a single stray byte in a CJK string // turns the whole of the rest of it into wrong glyphs. @(test) test_cmap_next_code_partial_match :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // The codespace of the standard's own example. cm := cmf_parse(&f, `begincmap 4 begincodespacerange <00> <80> <8140> <9ffc> endcodespacerange endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } Case :: struct { src: []byte, want_code: u32, want_n: int } for c in ([]Case{ // 0x81 opens the two-byte range but 0x20 does not complete it. {[]byte{0x81, 0x20, 0x41}, 0x8120, 2}, {[]byte{0x82, 0x39}, 0x8239, 2}, {[]byte{0x90, 0x30, 0x41}, 0x9030, 2}, // The same for the second two-byte range. {[]byte{0xE0, 0x00}, 0xE000, 2}, // 0xFC is outside [0xE0, 0xFB] and every other lead range, so this is // rule a) and the shortest range decides: one byte. {[]byte{0xFC, 0x41}, 0xFC, 1}, // Valid codes are untouched. {[]byte{0x41}, 0x41, 1}, {[]byte{0x81, 0x40}, 0x8140, 2}, {[]byte{0xA0, 0x41}, 0xA0, 1}, // A partial match with the string running out still consumes what is // left rather than stalling. {[]byte{0x81}, 0x81, 1}, }) { code, n := cmap_next_code(cm, c.src) testing.expectf(t, n == c.want_n && code == c.want_code, "%X: got 0x%X in %d bytes, want 0x%X in %d", c.src, code, n, c.want_code, c.want_n) } // The whole point: the code after a bad one lands where it should. codes, widths := cmf_codes(t, cm, []byte{0x81, 0x20, 0x41}) defer delete(codes) defer delete(widths) testing.expectf(t, len(codes) == 2, "segmented into %d codes: %v", len(codes), codes) if len(codes) == 2 { testing.expectf(t, codes[1] == 0x41, "the code after the invalid one is 0x%X, want 0x41", codes[1]) } } @(test) test_cmap_next_code_always_advances :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // Two-byte codespace only, driven with an odd number of bytes and with a // byte outside every range: neither may stall the caller. cm := cmf_parse(&f, `begincmap 1 begincodespacerange <2000> <20ff> endcodespacerange endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } _, n := cmap_next_code(cm, []byte{0x99, 0x99}) testing.expectf(t, n == 2, "unmatched bytes consumed %d, want the shortest range's 2", n) // A trailing half code unit still has to be consumed. code, n1 := cmap_next_code(cm, []byte{0x20}) testing.expectf(t, n1 == 1 && code == 0x20, "trailing byte: got 0x%X in %d bytes", code, n1) // No CMap and no bytes are both defined. c0, n0 := cmap_next_code(cm, nil) testing.expectf(t, n0 == 0 && c0 == 0, "empty string: got 0x%X in %d bytes", c0, n0) cn, nn := cmap_next_code(nil, []byte{0xAB, 0xCD}) testing.expectf(t, nn == 1 && cn == 0xAB, "nil CMap: got 0x%X in %d bytes", cn, nn) } @(test) test_cmap_identity :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc cm := cmap_identity(false, f.alloc) if !testing.expect(t, cm != nil, "cmap_identity returned nil") { return } testing.expect(t, cm.name == "Identity-H", "wrong name") testing.expect(t, cm.wmode == 0, "Identity-H is horizontal") testing.expect(t, !cm.predefined_fallback, "Identity-H is not a fallback") src := []byte{0x00, 0x41, 0xFF, 0xFE} codes, widths := cmf_codes(t, cm, src) defer delete(codes) defer delete(widths) testing.expectf(t, len(codes) == 2, "identity segmented into %d codes: %v", len(codes), codes) if len(codes) == 2 { testing.expectf(t, codes[0] == 0x0041 && widths[0] == 2, "first code 0x%X", codes[0]) testing.expectf(t, codes[1] == 0xFFFE && widths[1] == 2, "second code 0x%X", codes[1]) } for code in ([]u32{0, 1, 0x41, 0x1234, 0xFFFF}) { cid, ok := cmap_lookup(cm, code) testing.expectf(t, ok && cid == code, "identity lookup 0x%X gave 0x%X, %v", code, cid, ok) } v := cmap_identity(true, f.alloc) if !testing.expect(t, v != nil, "cmap_identity(vertical) returned nil") { return } testing.expect(t, v.name == "Identity-V" && v.wmode == 1, "Identity-V is vertical") } @(test) test_cmap_predefined_fallback :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // The CJK registries are not shipped. A name from one must still yield a // usable two-byte CMap, and must say that it is not the real thing. cm, err := cmap_load(nil, Name("UniJIS-UCS2-H")) if !testing.expectf(t, err == .None && cm != nil, "load: %s", error_string(err)) { return } testing.expect(t, cm.predefined_fallback, "a substituted CMap must be flagged") testing.expect(t, cm.name == "UniJIS-UCS2-H", "the requested name is kept") testing.expect(t, cm.wmode == 0, "-H is horizontal") _, n := cmap_next_code(cm, []byte{0x30, 0x42}) testing.expectf(t, n == 2, "the fallback is two-byte, got %d", n) vert, verr := cmap_load(nil, Name("UniJIS-UCS2-V")) if !testing.expectf(t, verr == .None && vert != nil, "load: %s", error_string(verr)) { return } testing.expect(t, vert.wmode == 1, "-V must be recognised as vertical") // Identity-H is real, not a fallback. id, ierr := cmap_load(nil, Name("Identity-H")) if !testing.expectf(t, ierr == .None && id != nil, "load: %s", error_string(ierr)) { return } testing.expect(t, !id.predefined_fallback, "Identity-H must not be flagged as substituted") // Neither a name nor a stream is the one case with nothing to hand back. bad, berr := cmap_load(nil, i64(7)) testing.expect(t, bad == nil && berr == .Bad_Cmap, "a non-CMap object is an error") } // --------------------------------------------------------------------------- // CID mappings // --------------------------------------------------------------------------- @(test) test_cmap_cid_ranges_and_chars :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc cm := cmf_parse(&f, `begincmap 1 begincodespacerange <0000> endcodespacerange 2 begincidrange <0000> <00ff> 100 <0300> <03ff> 900 endcidrange 1 begincidchar <1000> 4242 endcidchar endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } Case :: struct { code: u32, cid: u32, ok: bool } for c in ([]Case{ {0x0000, 100, true}, {0x0010, 116, true}, {0x00FF, 355, true}, {0x0300, 900, true}, {0x03FF, 1155, true}, {0x1000, 4242, true}, {0x0100, 0, false}, {0x2000, 0, false}, }) { cid, ok := cmap_lookup(cm, c.code) testing.expectf(t, ok == c.ok, "code 0x%X: found=%v, want %v", c.code, ok, c.ok) if c.ok { testing.expectf(t, cid == c.cid, "code 0x%X: cid %d, want %d", c.code, cid, c.cid) } } } @(test) test_cmap_bad_ranges_dropped :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc cm := cmf_parse(&f, `begincmap 2 begincodespacerange <0030> <0020> <0000> endcodespacerange 2 begincidrange <0030> <0020> 7 <0040> <0041> 9 endcidrange 1 beginbfrange <0060> <0050> <0041> endbfrange endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } // A range that runs backwards describes nothing and must not be stored, // let alone answered from. _, ok30 := cmap_lookup(cm, 0x30) testing.expect(t, !ok30, "a lo>hi cidrange must be dropped") _, ok20 := cmap_lookup(cm, 0x20) testing.expect(t, !ok20, "a lo>hi cidrange must be dropped") _, okbf := cmap_lookup_text(cm, 0x60) testing.expect(t, !okbf, "a lo>hi bfrange must be dropped") cid, ok := cmap_lookup(cm, 0x41) testing.expectf(t, ok && cid == 10, "the good range still works: %d, %v", cid, ok) // Dropped, not merely never matched: a backwards range that is kept would // still be searched on every lookup, and `dst + (code - lo)` under it // underflows. testing.expectf(t, len(cm.ranges) == 1, "kept %d cid ranges, want 1", len(cm.ranges)) testing.expectf(t, len(cm.codespace) == 1, "kept %d codespace ranges, want 1", len(cm.codespace)) testing.expectf(t, len(cm.texts) == 0, "kept %d bf entries, want 0", len(cm.texts)) // The backwards codespace range is gone, so only the two-byte one is left. _, n := cmap_next_code(cm, []byte{0x00, 0x41}) testing.expectf(t, n == 2, "a lo>hi codespace range must be dropped, got width %d", n) } // --------------------------------------------------------------------------- // /ToUnicode // --------------------------------------------------------------------------- @(test) test_cmap_bfrange_numeric_destination :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc cm := cmf_parse(&f, `begincmap 1 begincodespacerange <0000> endcodespacerange 1 beginbfrange <0020> <0022> <0041> endbfrange endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } expect_text(t, cm, 0x20, "A") expect_text(t, cm, 0x21, "B") expect_text(t, cm, 0x22, "C") _, ok := cmap_lookup_text(cm, 0x23) testing.expect(t, !ok, "one past the end of a bfrange is unmapped") } @(test) test_cmap_bfrange_array_destination :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // The second bfrange form: one destination string per source code, which // is how a subset font spells out an irregular run. cm := cmf_parse(&f, `begincmap 1 begincodespacerange <0000> endcodespacerange 2 beginbfrange <0005> <0007> [<0041> <00420043> ] <0010> <0013> [<0058> <0059>] endbfrange endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } expect_text(t, cm, 0x05, "A") expect_text(t, cm, 0x06, "BC") expect_text(t, cm, 0x07, "\U0002003E") // An array shorter than the range maps only as far as it reaches. expect_text(t, cm, 0x10, "X") expect_text(t, cm, 0x11, "Y") _, ok := cmap_lookup_text(cm, 0x12) testing.expect(t, !ok, "a short destination array must not invent entries") } @(test) test_cmap_bfchar_ligature_and_surrogates :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc cm := cmf_parse(&f, `begincmap 1 begincodespacerange <0000> endcodespacerange 5 beginbfchar <0001> <00660066006c> <0002> <0003> <0004> <0005> <004160> endbfchar endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } // A single glyph standing for a ligature: three code points, one code. expect_text(t, cm, 0x0001, "ffl") // A surrogate pair is one character, U+2003E, not two. expect_text(t, cm, 0x0002, "\U0002003E") // An unpaired surrogate, high or low, is U+FFFD -- never invalid UTF-8. expect_text(t, cm, 0x0003, "�") expect_text(t, cm, 0x0004, "�") // A destination that is not a whole number of UTF-16 code units -- three // bytes -- ends in half a unit, which is U+FFFD like any other truncation. expect_text(t, cm, 0x0005, "A�") } @(test) test_cmap_later_definition_wins :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // A broad bfrange followed by a bfchar correcting one code out of it is a // shape real producers emit; the correction has to be the one that sticks. cm := cmf_parse(&f, `begincmap 1 begincodespacerange <0000> endcodespacerange 1 beginbfrange <0041> <005a> <0061> endbfrange 1 beginbfchar <0042> <0030> endbfchar endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } expect_text(t, cm, 0x41, "a") expect_text(t, cm, 0x42, "0") expect_text(t, cm, 0x43, "c") } @(test) test_cmap_absurd_range_is_clamped :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // Four billion codes in one line is legal syntax. Expanding it is not an // option, so the expansion stops -- but what it did produce is correct. cm := cmf_parse(&f, `begincmap 1 begincodespacerange <00000000> endcodespacerange 1 beginbfrange <00000000> <0041> endbfrange endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } expect_text(t, cm, 0, "A") testing.expectf(t, len(cm.texts) == MAX_CMAP_RANGE_CODES, "expanded %d entries, the cap is %d", len(cm.texts), MAX_CMAP_RANGE_CODES) _, ok := cmap_lookup_text(cm, 0x00FF_FFFF) testing.expect(t, !ok, "the clamped tail of the range is simply unmapped") } // A bfrange whose expansion is thrown away must not be expanded. One // `<0000> ` line is half the entry cap and two fill it, so every line // after that produces nothing at all -- while still costing 65536 iterations // each if the loop only learns about the cap from the entries it stores being // dropped. Nothing bounds the number of lines but the token fuel, which allows // about 1.4 million of them, and a body that repetitive compresses to a few // tens of KB: a 30 KB file would pin a core for half an hour. @(test) test_cmap_repeated_full_bfrange_terminates :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc LINES :: 20_000 src: [dynamic]u8 defer delete(src) append(&src, ..transmute([]byte)string( "begincmap\n1 begincodespacerange <0000> endcodespacerange\nbeginbfrange\n")) for _ in 0 ..< LINES { append(&src, ..transmute([]byte)string("<0000><0020>\n")) } append(&src, ..transmute([]byte)string("endbfrange\nendcmap\n")) started := time.now() cm := cmf_parse(&f, string(src[:])) ms := time.duration_milliseconds(time.since(started)) if !testing.expect(t, cm != nil, "parse failed") { return } // What it does produce is still right: the destination is incremented once // per code, and the repeated lines are all the same mapping. expect_text(t, cm, 0x0000, " ") expect_text(t, cm, 0x0021, "A") testing.expectf(t, len(cm.texts) == MAX_CMAP_RANGE_CODES, "expanded %d distinct codes, want %d", len(cm.texts), MAX_CMAP_RANGE_CODES) // Not a performance bound: without the early exit this same body takes // about 25 seconds, and the limit is loose enough that a loaded machine // cannot trip it. testing.expectf(t, ms < 5000, "%d bfrange lines took %.0f ms", LINES, ms) } // --------------------------------------------------------------------------- // Malformed input // --------------------------------------------------------------------------- @(test) test_cmap_count_disagrees_with_contents :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // The integer before `beginbfchar` is a hint, not a contract: files get it // wrong in both directions and the contents are what count. cm := cmf_parse(&f, `begincmap 9 begincodespacerange <0000> endcodespacerange 99 beginbfchar <0041> <0061> <0042> <0062> endbfchar 0 begincidchar <0043> 7 endcidchar endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } expect_text(t, cm, 0x41, "a") expect_text(t, cm, 0x42, "b") testing.expectf(t, len(cm.texts) == 2, "got %d text entries, want 2", len(cm.texts)) cid, ok := cmap_lookup(cm, 0x43) testing.expectf(t, ok && cid == 7, "a section declared empty still maps its contents: %d %v", cid, ok) } @(test) test_cmap_truncated :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // Stops in the middle of a bfchar section, with a half-written pair. What // was collected before the cut has to survive. cm := cmf_parse(&f, `begincmap 1 begincodespacerange <0000> endcodespacerange 3 beginbfchar <0041> <0061> <0042> `) if !testing.expect(t, cm != nil, "a truncated CMap still parses") { return } expect_text(t, cm, 0x41, "a") _, ok := cmap_lookup_text(cm, 0x42) testing.expect(t, !ok, "the half-written pair maps nothing") _, n := cmap_next_code(cm, []byte{0x00, 0x41}) testing.expectf(t, n == 2, "the codespace before the cut survives, got width %d", n) } @(test) test_cmap_unterminated_section :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // No `endbfchar`; the next `begin...` is where the section really ended. cm := cmf_parse(&f, `begincmap 1 begincodespacerange <0000> 1 beginbfchar <0041> <0061> 1 begincidchar <0043> 7 endcidchar endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } expect_text(t, cm, 0x41, "a") cid, ok := cmap_lookup(cm, 0x43) testing.expectf(t, ok && cid == 7, "the section after the unterminated one still parses: %d %v", cid, ok) _, n := cmap_next_code(cm, []byte{0x00, 0x41}) testing.expectf(t, n == 2, "the unterminated codespace range still counts, got width %d", n) } @(test) test_cmap_unknown_section_is_skipped :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // `notdefrange` and `usematrix` are sections this reader does not // interpret. Stepping over their contents is not the same as ignoring the // section header: what is inside must not reach the enclosing CMap, or a // stray `/WMode 1 def` in a section that has nothing to do with the // writing mode would turn the whole font vertical. cm := cmf_parse(&f, `begincmap 1 begincodespacerange <0000> endcodespacerange 1 beginbfchar <0041> <0061> endbfchar 1 beginnotdefrange <0041> <0062> endnotdefrange 1 beginusematrix /WMode 1 def /CMapName /Wrong def <0041> <0063> endusematrix endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } expect_text(t, cm, 0x41, "a") testing.expectf(t, len(cm.texts) == 1, "got %d text entries, want 1", len(cm.texts)) testing.expectf(t, cm.wmode == 0, "a /WMode inside a skipped section must not apply, got %d", cm.wmode) testing.expectf(t, cm.name == "", "a /CMapName inside a skipped section must not apply, got %q", cm.name) } @(test) test_cmap_longest_codespace_wins :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // Overlapping widths: 0x41 is a code in the one-byte range and also the // lead byte of the two-byte range. The wider code is the intended one, so // declaration order must not decide it. cm := cmf_parse(&f, `begincmap 2 begincodespacerange <00> <4100> <41ff> endcodespacerange endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } code, n := cmap_next_code(cm, []byte{0x41, 0x00}) testing.expectf(t, n == 2 && code == 0x4100, "longest match: got 0x%X in %d bytes", code, n) // Only one byte left: the wide range cannot apply and the narrow one does. c1, n1 := cmap_next_code(cm, []byte{0x41}) testing.expectf(t, n1 == 1 && c1 == 0x41, "one byte left: got 0x%X in %d bytes", c1, n1) // A byte the wide range does not lead with is narrow. c2, n2 := cmap_next_code(cm, []byte{0x42, 0x00}) testing.expectf(t, n2 == 1 && c2 == 0x42, "narrow code: got 0x%X in %d bytes", c2, n2) } @(test) test_cmap_junk_never_hangs :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // A well-formed CMap cut at every offset, plus a few shapes chosen to make // the tokeniser unhappy. None may crash, hang or fail to return a CMap. full := `begincmap /CMapName /X def /WMode 0 def 2 begincodespacerange <00> <80> <8140> <9ffc> endcodespacerange 1 begincidrange <8140> <9ffc> 1 endcidrange 1 beginbfrange <0041> <005a> [<0061> <0062>] endbfrange 1 beginbfchar <0041> endbfchar endcmap` for cut in 0 ..= len(full) { cm, err := cmap_parse(nil, transmute([]byte)full[:cut], f.alloc) testing.expectf(t, err == .None && cm != nil, "cut at %d: %s", cut, error_string(err)) if cm != nil { _, _ = cmap_next_code(cm, []byte{0x81, 0x40, 0x41}) _, _ = cmap_lookup(cm, 0x8140) _, _ = cmap_lookup_text(cm, 0x41) } } for junk in ([]string{ "", "endcmap", "beginbfchar", "beginbfrange <0041", "beginbfrange <0041> <0042> [", "beginbfrange <0041> <0042> [<0061>", "begincodespacerange <> <>", "begincidrange /a /b /c endcidrange", "begincidchar <41> -5 endcidchar", "beginbfchar () () endbfchar", "usecmap def begin end pop", "/WMode /oops def", ") ) ) > > >", "begin begincmap begin", }) { cm, err := cmap_parse(nil, transmute([]byte)junk, f.alloc) testing.expectf(t, err == .None && cm != nil, "%q: %s", junk, error_string(err)) if cm != nil { _, n := cmap_next_code(cm, []byte{0x41, 0x42}) testing.expectf(t, n > 0, "%q: decoding stalled", junk) } } } // --------------------------------------------------------------------------- // /UseCMap // --------------------------------------------------------------------------- @(test) test_cmap_usecmap_inheritance :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc parent := cmf_stream(&f, `begincmap 1 begincodespacerange <0000> endcodespacerange 1 begincidrange <0000> <00ff> 500 endcidrange /WMode 1 def endcmap`) child := cmf_stream(&f, `begincmap 1 begincidchar <0005> 7 endcidchar endcmap`) dict_set(child.dict, "UseCMap", parent) cm, err := cmap_load(nil, child) if !testing.expectf(t, err == .None && cm != nil, "load: %s", error_string(err)) { return } // The child's own entry answers first. own, ok_own := cmap_lookup(cm, 0x05) testing.expectf(t, ok_own && own == 7, "own entry: %d %v", own, ok_own) // Anything it does not define falls through to the used CMap. inh, ok_inh := cmap_lookup(cm, 0x06) testing.expectf(t, ok_inh && inh == 506, "inherited entry: %d %v", inh, ok_inh) // A CMap with no codespace ranges of its own takes the used CMap's, which // is what makes the inherited mapping addressable at all: without the // inheritance there would be no ranges here and codes would be one byte. code, n := cmap_next_code(cm, []byte{0x00, 0x06, 0x00, 0x07}) testing.expectf(t, n == 2 && code == 0x0006, "inherited codespace: 0x%X in %d bytes", code, n) testing.expectf(t, cm.wmode == 1, "writing mode is inherited too, got %d", cm.wmode) _, miss := cmap_lookup(cm, 0x1234) testing.expect(t, !miss, "a code neither CMap defines is unmapped") } @(test) test_cmap_usecmap_own_codespace_wins :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc parent := cmf_stream(&f, `begincmap 1 begincodespacerange <00> endcodespacerange endcmap`) child := cmf_stream(&f, `begincmap 1 begincodespacerange <0000> endcodespacerange endcmap`) dict_set(child.dict, "UseCMap", parent) cm, err := cmap_load(nil, child) if !testing.expectf(t, err == .None && cm != nil, "load: %s", error_string(err)) { return } _, n := cmap_next_code(cm, []byte{0x00, 0x41}) testing.expectf(t, n == 2, "the CMap's own codespace must win, got width %d", n) } @(test) test_cmap_usecmap_cycle :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // Two CMaps naming each other, and one naming itself. Both must terminate // and still produce a usable CMap. a := cmf_stream(&f, `begincmap 1 begincodespacerange <00> endcodespacerange 1 begincidchar <01> 11 endcidchar endcmap`) b := cmf_stream(&f, `begincmap 1 begincidchar <02> 22 endcidchar endcmap`) dict_set(a.dict, "UseCMap", b) dict_set(b.dict, "UseCMap", a) cm, err := cmap_load(nil, a) if !testing.expectf(t, err == .None && cm != nil, "load: %s", error_string(err)) { return } v1, ok1 := cmap_lookup(cm, 0x01) testing.expectf(t, ok1 && v1 == 11, "own entry survives the cycle: %d %v", v1, ok1) v2, ok2 := cmap_lookup(cm, 0x02) testing.expectf(t, ok2 && v2 == 22, "one hop through the cycle still resolves: %d %v", v2, ok2) self := cmf_stream(&f, `begincmap 1 begincidchar <03> 33 endcidchar endcmap`) dict_set(self.dict, "UseCMap", self) scm, serr := cmap_load(nil, self) if !testing.expectf(t, serr == .None && scm != nil, "self-reference: %s", error_string(serr)) { return } v3, ok3 := cmap_lookup(scm, 0x03) testing.expectf(t, ok3 && v3 == 33, "self-referencing CMap: %d %v", v3, ok3) } @(test) test_cmap_usecmap_operator :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // The `usecmap` operator names a predefined CMap. Identity-H is the only // one worth naming that this library actually has. cm := cmf_parse(&f, `begincmap /Identity-H usecmap 1 begincidchar <0005> 77 endcidchar endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } own, ok_own := cmap_lookup(cm, 0x0005) testing.expectf(t, ok_own && own == 77, "own entry: %d %v", own, ok_own) // Identity answers for everything else, and lends its codespace. inh, ok_inh := cmap_lookup(cm, 0x1234) testing.expectf(t, ok_inh && inh == 0x1234, "identity parent: 0x%X %v", inh, ok_inh) code, n := cmap_next_code(cm, []byte{0x12, 0x34}) testing.expectf(t, n == 2 && code == 0x1234, "inherited codespace: 0x%X in %d bytes", code, n) } @(test) test_cmap_wmode_and_name :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc cm := cmf_parse(&f, `begincmap /CIDSystemInfo << /Registry (Adobe) /Ordering (Japan1) /Supplement 6 >> def /CMapName /Test-V def /CMapType 1 def /WMode 1 def 1 begincodespacerange <0000> endcodespacerange endcmap`) if !testing.expect(t, cm != nil, "parse failed") { return } testing.expectf(t, cm.wmode == 1, "/WMode 1 def, got %d", cm.wmode) testing.expectf(t, cm.name == "Test-V", "/CMapName, got %q", cm.name) // The dictionary entries a stream may carry instead. s := cmf_stream(&f, "begincmap\n1 begincodespacerange <0000> endcodespacerange\nendcmap") dict_set(s.dict, "WMode", i64(1)) dict_set(s.dict, "CMapName", Name("Dict-V")) scm, err := cmap_load(nil, s) if !testing.expectf(t, err == .None && scm != nil, "load: %s", error_string(err)) { return } testing.expectf(t, scm.wmode == 1, "/WMode from the stream dictionary, got %d", scm.wmode) testing.expectf(t, scm.name == "Dict-V", "/CMapName from the stream dictionary, got %q", scm.name) } // --------------------------------------------------------------------------- // A CMap shaped like the ones real files carry // --------------------------------------------------------------------------- @(test) test_cmap_realistic_tounicode :: proc(t: ^testing.T) { f: Cmap_Fixture cmf_init(&f) defer cmf_destroy(&f) context.allocator = f.alloc // Byte-for-byte the shape a Word or InDesign export writes for a subset // TrueType font, which is 63% of the corpus. src := `/CIDInit /ProcSet findresource begin 12 dict begin begincmap /CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def /CMapName /Adobe-Identity-UCS def /CMapType 2 def 1 begincodespacerange <0000> endcodespacerange 3 beginbfrange <0003> <0003> <0020> <000F> <0018> <002C> <0024> <003D> <0041> endbfrange 2 beginbfchar <00AB> <00660066> <00B6> endbfchar endcmap CMapName currentdict /CMap defineresource pop end end` cm, err := cmap_parse(nil, transmute([]byte)src, f.alloc) if !testing.expectf(t, err == .None && cm != nil, "parse: %s", error_string(err)) { return } testing.expectf(t, cm.name == "Adobe-Identity-UCS", "name %q", cm.name) expect_text(t, cm, 0x0003, " ") expect_text(t, cm, 0x000F, ",") expect_text(t, cm, 0x0018, "5") expect_text(t, cm, 0x0024, "A") expect_text(t, cm, 0x003D, "Z") expect_text(t, cm, 0x00AB, "ff") expect_text(t, cm, 0x00B6, "ffi") // Decoding a two-byte string end to end, the way a Tj operand arrives. s := []byte{0x00, 0x24, 0x00, 0x25, 0x00, 0xAB} b: [64]byte k := 0 for i := 0; i < len(s); { code, n := cmap_next_code(cm, s[i:]) if n <= 0 { break } i += n if txt, ok := cmap_lookup_text(cm, code); ok { k += copy(b[k:], txt) } } got := string(b[:k]) testing.expectf(t, got == "ABff", "decoded %q, want %q", got, fmt.tprint("ABff")) }