526 lines
18 KiB
Odin
526 lines
18 KiB
Odin
package pdf
|
|
|
|
import "core:mem"
|
|
import "core:mem/virtual"
|
|
import "core:slice"
|
|
import "core:testing"
|
|
|
|
// A scratch arena keeps the tokens' allocations together and out of the test
|
|
// runner's leak tracker, which is exactly how the library uses the parser.
|
|
@(private = "file")
|
|
scratch_begin :: proc(t: ^testing.T, a: ^virtual.Arena) -> mem.Allocator {
|
|
err := virtual.arena_init_growing(a)
|
|
testing.expect(t, err == nil, "arena init failed")
|
|
return virtual.arena_allocator(a)
|
|
}
|
|
|
|
// Odin will not transmute an untyped literal, and every test here starts from
|
|
// one.
|
|
@(private = "file")
|
|
bytes_of :: proc(s: string) -> []byte {
|
|
return transmute([]byte)s
|
|
}
|
|
|
|
@(private = "file")
|
|
lex_all :: proc(src: string, allocator: mem.Allocator, out: ^[dynamic]Token) {
|
|
lex := lexer_make(transmute([]byte)src)
|
|
for {
|
|
tok := lexer_next(&lex, allocator)
|
|
if tok.kind == .EOF { return }
|
|
append(out, tok)
|
|
if len(out) > 4096 { return }
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Structure tokens
|
|
// ---------------------------------------------------------------------------
|
|
|
|
@(test)
|
|
test_lexer_structure_tokens :: proc(t: ^testing.T) {
|
|
arena: virtual.Arena
|
|
alloc := scratch_begin(t, &arena)
|
|
defer virtual.arena_destroy(&arena)
|
|
|
|
Case :: struct {
|
|
src: string,
|
|
kinds: []Token_Kind,
|
|
}
|
|
cases := []Case{
|
|
{"", {}},
|
|
{" \r\n\t\x00\f ", {}},
|
|
{"[]", {.Array_Open, .Array_Close}},
|
|
{"<<>>", {.Dict_Open, .Dict_Close}},
|
|
{"{}", {.Brace_Open, .Brace_Close}},
|
|
{"[ 1 2 3 ]", {.Array_Open, .Integer, .Integer, .Integer, .Array_Close}},
|
|
{"<</A/B>>", {.Dict_Open, .Name, .Name, .Dict_Close}},
|
|
// A hex string is not a dictionary, and "<<" is not two hex strings.
|
|
{"<41>", {.String}},
|
|
{"<< <41> >>", {.Dict_Open, .String, .Dict_Close}},
|
|
// Stray closers are Error tokens, and each consumes exactly one byte.
|
|
{">", {.Error}},
|
|
{")", {.Error}},
|
|
{"))", {.Error, .Error}},
|
|
{"1 0 obj", {.Integer, .Integer, .Keyword}},
|
|
{"true false null", {.Keyword, .Keyword, .Keyword}},
|
|
// '{' and '}' only mean anything in a type 4 function, but must
|
|
// tokenise wherever they appear.
|
|
{"{ dup 0.5 mul }", {.Brace_Open, .Keyword, .Real, .Keyword, .Brace_Close}},
|
|
}
|
|
|
|
toks := make([dynamic]Token, alloc)
|
|
for c in cases {
|
|
clear(&toks)
|
|
lex_all(c.src, alloc, &toks)
|
|
testing.expectf(t, len(toks) == len(c.kinds),
|
|
"%q: got %d tokens, want %d", c.src, len(toks), len(c.kinds))
|
|
if len(toks) != len(c.kinds) { continue }
|
|
for k, i in c.kinds {
|
|
testing.expectf(t, toks[i].kind == k,
|
|
"%q: token %d is %v, want %v", c.src, i, toks[i].kind, k)
|
|
}
|
|
}
|
|
}
|
|
|
|
@(test)
|
|
test_lexer_token_spans :: proc(t: ^testing.T) {
|
|
arena: virtual.Arena
|
|
alloc := scratch_begin(t, &arena)
|
|
defer virtual.arena_destroy(&arena)
|
|
|
|
src := " /Name 42 (str) "
|
|
lex := lexer_make(transmute([]byte)src)
|
|
|
|
n := lexer_next(&lex, alloc)
|
|
testing.expect_value(t, n.start, 2)
|
|
testing.expect_value(t, n.end, 7)
|
|
|
|
i := lexer_next(&lex, alloc)
|
|
testing.expect_value(t, i.start, 8)
|
|
testing.expect_value(t, i.end, 10)
|
|
|
|
s := lexer_next(&lex, alloc)
|
|
testing.expect_value(t, s.start, 11)
|
|
testing.expect_value(t, s.end, 16)
|
|
|
|
e := lexer_next(&lex, alloc)
|
|
testing.expect_value(t, e.kind, Token_Kind.EOF)
|
|
testing.expect_value(t, e.start, len(src))
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Comments (ISO 32000-2, 7.2.4)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
@(test)
|
|
test_lexer_comments :: proc(t: ^testing.T) {
|
|
arena: virtual.Arena
|
|
alloc := scratch_begin(t, &arena)
|
|
defer virtual.arena_destroy(&arena)
|
|
|
|
toks := make([dynamic]Token, alloc)
|
|
lex_all("% comment\n1 % another\n2%trailing, no EOL", alloc, &toks)
|
|
testing.expect_value(t, len(toks), 2)
|
|
if len(toks) == 2 {
|
|
testing.expect_value(t, toks[0].int_value, i64(1))
|
|
testing.expect_value(t, toks[1].int_value, i64(2))
|
|
}
|
|
|
|
// A '%' inside a string is data, not a comment.
|
|
clear(&toks)
|
|
lex_all("(50% off)", alloc, &toks)
|
|
testing.expect_value(t, len(toks), 1)
|
|
if len(toks) == 1 {
|
|
testing.expect(t, slice.equal(toks[0].bytes, transmute([]byte)string("50% off")),
|
|
"'%' inside a literal string must not start a comment")
|
|
}
|
|
|
|
// A comment ends the token before it without whitespace in between.
|
|
clear(&toks)
|
|
lex_all("/Name%c\n/Other", alloc, &toks)
|
|
testing.expect_value(t, len(toks), 2)
|
|
if len(toks) == 2 {
|
|
testing.expect_value(t, toks[0].text, "Name")
|
|
testing.expect_value(t, toks[1].text, "Other")
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Numbers (ISO 32000-2, 7.3.3)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
@(test)
|
|
test_lexer_numbers :: proc(t: ^testing.T) {
|
|
arena: virtual.Arena
|
|
alloc := scratch_begin(t, &arena)
|
|
defer virtual.arena_destroy(&arena)
|
|
|
|
Case :: struct {
|
|
src: string,
|
|
kind: Token_Kind,
|
|
i: i64,
|
|
r: f64,
|
|
}
|
|
cases := []Case{
|
|
// The spec's own examples, table 2.
|
|
{"123", .Integer, 123, 123},
|
|
{"43445", .Integer, 43445, 43445},
|
|
{"+17", .Integer, 17, 17},
|
|
{"-98", .Integer, -98, -98},
|
|
{"0", .Integer, 0, 0},
|
|
{"34.5", .Real, 34, 34.5},
|
|
{"-3.62", .Real, -3, -3.62},
|
|
{"+123.6", .Real, 123, 123.6},
|
|
{"4.", .Real, 4, 4},
|
|
{"-.002", .Real, 0, -0.002},
|
|
{"0.0", .Real, 0, 0},
|
|
// Shapes that only real files produce.
|
|
{".5", .Real, 0, 0.5},
|
|
{".", .Real, 0, 0},
|
|
{"1e2", .Real, 100, 100},
|
|
{"-2.5e-2", .Real, 0, -0.025},
|
|
// Nothing here is allowed to fail; an unreadable number is 0.
|
|
{"--5", .Integer, 0, 0},
|
|
{"1.2.3", .Real, 0, 0},
|
|
{"-", .Integer, 0, 0},
|
|
{"+", .Integer, 0, 0},
|
|
// An exponent that overflows must not leak an infinity into geometry.
|
|
{"1e400", .Real, 0, 0},
|
|
{"-1e400", .Real, 0, 0},
|
|
}
|
|
|
|
for c in cases {
|
|
lex := lexer_make(transmute([]byte)c.src)
|
|
tok := lexer_next(&lex, alloc)
|
|
testing.expectf(t, tok.kind == c.kind, "%q: kind %v, want %v", c.src, tok.kind, c.kind)
|
|
testing.expectf(t, tok.int_value == c.i, "%q: int %d, want %d", c.src, tok.int_value, c.i)
|
|
testing.expectf(t, tok.real_value == c.r, "%q: real %f, want %f", c.src, tok.real_value, c.r)
|
|
testing.expectf(t, is_finite(tok.real_value), "%q: real value is not finite", c.src)
|
|
testing.expectf(t, tok.end == len(c.src), "%q: consumed %d of %d bytes", c.src, tok.end, len(c.src))
|
|
}
|
|
}
|
|
|
|
@(test)
|
|
test_lexer_number_stops_at_delimiter :: proc(t: ^testing.T) {
|
|
arena: virtual.Arena
|
|
alloc := scratch_begin(t, &arena)
|
|
defer virtual.arena_destroy(&arena)
|
|
|
|
lex := lexer_make(bytes_of("12/Name"))
|
|
a := lexer_next(&lex, alloc)
|
|
testing.expect_value(t, a.kind, Token_Kind.Integer)
|
|
testing.expect_value(t, a.int_value, i64(12))
|
|
b := lexer_next(&lex, alloc)
|
|
testing.expect_value(t, b.kind, Token_Kind.Name)
|
|
testing.expect_value(t, b.text, "Name")
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Names (ISO 32000-2, 7.3.5, table 4)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
@(test)
|
|
test_lexer_names :: proc(t: ^testing.T) {
|
|
arena: virtual.Arena
|
|
alloc := scratch_begin(t, &arena)
|
|
defer virtual.arena_destroy(&arena)
|
|
|
|
Case :: struct {
|
|
src: string,
|
|
want: string,
|
|
}
|
|
cases := []Case{
|
|
{"/Name1", "Name1"},
|
|
{"/ASomewhatLongerName", "ASomewhatLongerName"},
|
|
{"/A;Name_With-Various***Characters?", "A;Name_With-Various***Characters?"},
|
|
{"/1.2", "1.2"},
|
|
{"/$$", "$$"},
|
|
{"/@pattern", "@pattern"},
|
|
{"/.notdef", ".notdef"},
|
|
{"/lime#20Green", "lime Green"},
|
|
{"/paired#28#29parentheses", "paired()parentheses"},
|
|
{"/The_Key_of_F#23_Minor", "The_Key_of_F#_Minor"},
|
|
{"/A#42", "AB"},
|
|
// A bare '/' is the empty name, which is legal.
|
|
{"/", ""},
|
|
{"/ ", ""},
|
|
{"/[", ""},
|
|
// A '#' that is not followed by two hex digits stays as written.
|
|
{"/Bad#", "Bad#"},
|
|
{"/Bad#zz", "Bad#zz"},
|
|
{"/Bad#4", "Bad#4"},
|
|
}
|
|
|
|
for c in cases {
|
|
lex := lexer_make(transmute([]byte)c.src)
|
|
tok := lexer_next(&lex, alloc)
|
|
testing.expectf(t, tok.kind == .Name, "%q: kind %v", c.src, tok.kind)
|
|
testing.expectf(t, tok.text == c.want, "%q: got %q, want %q", c.src, tok.text, c.want)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Literal strings (ISO 32000-2, 7.3.4.2)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
@(test)
|
|
test_lexer_literal_strings :: proc(t: ^testing.T) {
|
|
arena: virtual.Arena
|
|
alloc := scratch_begin(t, &arena)
|
|
defer virtual.arena_destroy(&arena)
|
|
|
|
Case :: struct {
|
|
src: string,
|
|
want: string,
|
|
truncated: bool,
|
|
}
|
|
cases := []Case{
|
|
{"(This is a string)", "This is a string", false},
|
|
{"()", "", false},
|
|
// Balanced unescaped parentheses nest.
|
|
{"(Strings may contain balanced parentheses ( ) and\nspecial characters (*!&}^% and so on).)",
|
|
"Strings may contain balanced parentheses ( ) and\nspecial characters (*!&}^% and so on).", false},
|
|
{"(((deep)))", "((deep))", false},
|
|
{"(These \\(brackets\\) work)", "These (brackets) work", false},
|
|
// Every escape in table 3.
|
|
{"(a\\nb)", "a\nb", false},
|
|
{"(a\\rb)", "a\rb", false},
|
|
{"(a\\tb)", "a\tb", false},
|
|
{"(a\\bb)", "a\bb", false},
|
|
{"(a\\fb)", "a\fb", false},
|
|
{"(a\\(b)", "a(b", false},
|
|
{"(a\\)b)", "a)b", false},
|
|
{"(a\\\\b)", "a\\b", false},
|
|
// Octal, 1 to 3 digits. \0053 is \005 followed by '3'.
|
|
{"(\\53)", "+", false},
|
|
{"(\\053)", "+", false},
|
|
{"(\\0053)", "\x053", false},
|
|
{"(\\401)", "\x01", false}, // 0401 & 0xFF
|
|
{"(\\0)", "\x00", false},
|
|
// A backslash before anything else is dropped, the byte kept.
|
|
{"(\\8)", "8", false},
|
|
{"(\\q)", "q", false},
|
|
// A backslash before an EOL is a line continuation.
|
|
{"(line1\\\nline2)", "line1line2", false},
|
|
{"(line1\\\r\nline2)", "line1line2", false},
|
|
{"(line1\\\rline2)", "line1line2", false},
|
|
// A raw CR, LF or CRLF in the source is a single LF in the value.
|
|
{"(a\nb)", "a\nb", false},
|
|
{"(a\rb)", "a\nb", false},
|
|
{"(a\r\nb)", "a\nb", false},
|
|
{"(a\n\rb)", "a\n\nb", false},
|
|
// Unterminated: the value survives, the damage is flagged.
|
|
{"(abc", "abc", true},
|
|
{"(abc\\", "abc", true},
|
|
{"((abc)", "(abc)", true},
|
|
{"(", "", true},
|
|
}
|
|
|
|
for c in cases {
|
|
lex := lexer_make(transmute([]byte)c.src)
|
|
tok := lexer_next(&lex, alloc)
|
|
testing.expectf(t, tok.kind == .String, "%q: kind %v", c.src, tok.kind)
|
|
testing.expectf(t, slice.equal(tok.bytes, transmute([]byte)c.want),
|
|
"%q: got %q, want %q", c.src, string(tok.bytes), c.want)
|
|
testing.expectf(t, tok.truncated == c.truncated,
|
|
"%q: truncated %v, want %v", c.src, tok.truncated, c.truncated)
|
|
testing.expectf(t, lex.pos <= len(c.src), "%q: lexer ran past the end", c.src)
|
|
}
|
|
}
|
|
|
|
@(test)
|
|
test_lexer_string_stops_where_it_should :: proc(t: ^testing.T) {
|
|
arena: virtual.Arena
|
|
alloc := scratch_begin(t, &arena)
|
|
defer virtual.arena_destroy(&arena)
|
|
|
|
// The escaped ')' must not close the string, and the token must end
|
|
// exactly at the real closer.
|
|
src := "(a\\)b) 7"
|
|
lex := lexer_make(transmute([]byte)src)
|
|
s := lexer_next(&lex, alloc)
|
|
testing.expect_value(t, s.kind, Token_Kind.String)
|
|
testing.expect_value(t, s.end, 6)
|
|
n := lexer_next(&lex, alloc)
|
|
testing.expect_value(t, n.kind, Token_Kind.Integer)
|
|
testing.expect_value(t, n.int_value, i64(7))
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Hex strings (ISO 32000-2, 7.3.4.3)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
@(test)
|
|
test_lexer_hex_strings :: proc(t: ^testing.T) {
|
|
arena: virtual.Arena
|
|
alloc := scratch_begin(t, &arena)
|
|
defer virtual.arena_destroy(&arena)
|
|
|
|
Case :: struct {
|
|
src: string,
|
|
want: string,
|
|
truncated: bool,
|
|
}
|
|
cases := []Case{
|
|
{"<4E6F762073686D6F7A206B6120706F702E>", "Nov shmoz ka pop.", false},
|
|
{"<901FA3>", "\x90\x1f\xa3", false},
|
|
// "If the final digit is missing it is assumed to be 0."
|
|
{"<901FA>", "\x90\x1f\xa0", false},
|
|
{"<>", "", false},
|
|
{"<a>", "\xa0", false},
|
|
// Case-insensitive, whitespace anywhere.
|
|
{"<4a4B>", "JK", false},
|
|
{"<4 1 4 2>", "AB", false},
|
|
{"<41\n42\r43>", "ABC", false},
|
|
// Junk between digits is skipped rather than failing the string.
|
|
{"<41-42>", "AB", false},
|
|
{"<41zz42>", "AB", false},
|
|
// Unterminated runs to the end of the input.
|
|
{"<4142", "AB", true},
|
|
{"<", "", true},
|
|
}
|
|
|
|
for c in cases {
|
|
lex := lexer_make(transmute([]byte)c.src)
|
|
tok := lexer_next(&lex, alloc)
|
|
testing.expectf(t, tok.kind == .String, "%q: kind %v", c.src, tok.kind)
|
|
testing.expectf(t, slice.equal(tok.bytes, transmute([]byte)c.want),
|
|
"%q: got %x, want %x", c.src, tok.bytes, transmute([]byte)c.want)
|
|
testing.expectf(t, tok.truncated == c.truncated,
|
|
"%q: truncated %v, want %v", c.src, tok.truncated, c.truncated)
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Keyword helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
@(test)
|
|
test_lexer_keyword_helpers :: proc(t: ^testing.T) {
|
|
lex := lexer_make(bytes_of(" stream\r\ndata"))
|
|
testing.expect(t, lexer_at_keyword(&lex, "stream"), "at_keyword should match past whitespace")
|
|
testing.expect_value(t, lex.pos, 0) // and must not consume
|
|
testing.expect(t, !lexer_at_keyword(&lex, "endstream"), "wrong keyword must not match")
|
|
|
|
// A keyword must not match a prefix of a longer regular-character run.
|
|
lex2 := lexer_make(bytes_of("streamable"))
|
|
testing.expect(t, !lexer_at_keyword(&lex2, "stream"), "prefix must not match")
|
|
|
|
testing.expect(t, lexer_accept_keyword(&lex, "stream"), "accept should match")
|
|
testing.expect_value(t, lex.pos, 8)
|
|
lexer_skip_eol(&lex)
|
|
testing.expect_value(t, lex.pos, 10)
|
|
|
|
// A failed accept leaves the position alone.
|
|
save := lex.pos
|
|
testing.expect(t, !lexer_accept_keyword(&lex, "endstream"), "accept should fail")
|
|
testing.expect_value(t, lex.pos, save)
|
|
|
|
lex3 := lexer_make(bytes_of("\rx"))
|
|
lexer_skip_eol(&lex3)
|
|
testing.expect_value(t, lex3.pos, 1) // a bare CR is one EOL
|
|
}
|
|
|
|
@(test)
|
|
test_lexer_peek :: proc(t: ^testing.T) {
|
|
arena: virtual.Arena
|
|
alloc := scratch_begin(t, &arena)
|
|
defer virtual.arena_destroy(&arena)
|
|
|
|
lex := lexer_make(bytes_of("/A /B"))
|
|
p1 := lexer_peek(&lex, alloc)
|
|
p2 := lexer_peek(&lex, alloc)
|
|
testing.expect_value(t, p1.text, "A")
|
|
testing.expect_value(t, p2.text, "A")
|
|
testing.expect_value(t, lex.pos, 0)
|
|
n := lexer_next(&lex, alloc)
|
|
testing.expect_value(t, n.text, "A")
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Totality
|
|
//
|
|
// The one property the whole reader rests on: no input makes the lexer stall.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
@(private = "file")
|
|
Rng :: struct {
|
|
state: u64,
|
|
}
|
|
|
|
// xorshift64*, so the corpus is identical on every run and every platform.
|
|
@(private = "file")
|
|
rng_next :: proc(r: ^Rng) -> u64 {
|
|
x := r.state
|
|
x ~= x >> 12
|
|
x ~= x << 25
|
|
x ~= x >> 27
|
|
r.state = x
|
|
return x * 0x2545F4914F6CDD1D
|
|
}
|
|
|
|
@(private = "file")
|
|
assert_lexer_terminates :: proc(t: ^testing.T, data: []byte, alloc: mem.Allocator, what: string) {
|
|
lex := lexer_make(data)
|
|
// Every token consumes at least one byte, so more iterations than there
|
|
// are bytes means the lexer stalled.
|
|
budget := len(data) + 2
|
|
for _ in 0 ..< budget {
|
|
before := lex.pos
|
|
tok := lexer_next(&lex, alloc)
|
|
if tok.kind == .EOF {
|
|
testing.expectf(t, lex.pos >= before, "%s: EOF moved backwards", what)
|
|
return
|
|
}
|
|
testing.expectf(t, lex.pos > before, "%s: token %v did not advance at %d", what, tok.kind, before)
|
|
testing.expectf(t, tok.end >= tok.start && tok.end <= len(data),
|
|
"%s: token span [%d,%d) outside the input", what, tok.start, tok.end)
|
|
if lex.pos <= before { return }
|
|
}
|
|
testing.expectf(t, false, "%s: lexer did not reach EOF within %d tokens", what, budget)
|
|
}
|
|
|
|
@(test)
|
|
test_lexer_is_total_on_random_bytes :: proc(t: ^testing.T) {
|
|
arena: virtual.Arena
|
|
alloc := scratch_begin(t, &arena)
|
|
defer virtual.arena_destroy(&arena)
|
|
|
|
r := Rng{state = 0x9E3779B97F4A7C15}
|
|
buf := make([]byte, 64, alloc)
|
|
for round in 0 ..< 400 {
|
|
n := int(rng_next(&r) % 64)
|
|
for i in 0 ..< n {
|
|
v := rng_next(&r)
|
|
// Half the corpus is drawn from the bytes that mean something, so
|
|
// the delimiters and escapes are hit far more often than 1/256.
|
|
if v & 1 == 0 {
|
|
syntax := "()<>[]{}/%\\ \t\r\n0123456789.+-eRobj"
|
|
buf[i] = syntax[int(v >> 8) % len(syntax)]
|
|
} else {
|
|
buf[i] = byte(v >> 8)
|
|
}
|
|
}
|
|
assert_lexer_terminates(t, buf[:n], alloc, "random")
|
|
_ = round
|
|
}
|
|
}
|
|
|
|
@(test)
|
|
test_lexer_is_total_on_truncations :: proc(t: ^testing.T) {
|
|
arena: virtual.Arena
|
|
alloc := scratch_begin(t, &arena)
|
|
defer virtual.arena_destroy(&arena)
|
|
|
|
sample := transmute([]byte)string(
|
|
"<< /Type /Page /Parent 3 0 R /Contents [ 4 0 R ] /S (a\\(b) /H <4142> " +
|
|
"/N /A#20B /R [1 2 [3 [4]]] /X 1e3 >> stream\r\nxx\nendstream endobj %c\n")
|
|
for n in 0 ..= len(sample) {
|
|
assert_lexer_terminates(t, sample[:n], alloc, "truncation")
|
|
}
|
|
// And every suffix, which puts the lexer in the middle of every construct.
|
|
for n in 0 ..< len(sample) {
|
|
assert_lexer_terminates(t, sample[n:], alloc, "suffix")
|
|
}
|
|
}
|