package pdf

import "core:mem"
import "core:mem/virtual"
import "core:slice"
import "core:testing"

// A scratch arena keeps the tokens' allocations together and out of the test
// runner's leak tracker, which is exactly how the library uses the parser.
@(private = "file")
scratch_begin :: proc(t: ^testing.T, a: ^virtual.Arena) -> mem.Allocator {
    err := virtual.arena_init_growing(a)
    testing.expect(t, err == nil, "arena init failed")
    return virtual.arena_allocator(a)
}

// Odin will not transmute an untyped literal, and every test here starts from
// one.
@(private = "file")
bytes_of :: proc(s: string) -> []byte {
    return transmute([]byte)s
}

@(private = "file")
lex_all :: proc(src: string, allocator: mem.Allocator, out: ^[dynamic]Token) {
    lex := lexer_make(transmute([]byte)src)
    for {
        tok := lexer_next(&lex, allocator)
        if tok.kind == .EOF { return }
        append(out, tok)
        if len(out) > 4096 { return }
    }
}

// ---------------------------------------------------------------------------
// Structure tokens
// ---------------------------------------------------------------------------

@(test)
test_lexer_structure_tokens :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    Case :: struct {
        src:   string,
        kinds: []Token_Kind,
    }
    cases := []Case{
        {"", {}},
        {"   \r\n\t\x00\f ", {}},
        {"[]", {.Array_Open, .Array_Close}},
        {"<<>>", {.Dict_Open, .Dict_Close}},
        {"{}", {.Brace_Open, .Brace_Close}},
        {"[ 1 2 3 ]", {.Array_Open, .Integer, .Integer, .Integer, .Array_Close}},
        {"<</A/B>>", {.Dict_Open, .Name, .Name, .Dict_Close}},
        // A hex string is not a dictionary, and "<<" is not two hex strings.
        {"<41>", {.String}},
        {"<< <41> >>", {.Dict_Open, .String, .Dict_Close}},
        // Stray closers are Error tokens, and each consumes exactly one byte.
        {">", {.Error}},
        {")", {.Error}},
        {"))", {.Error, .Error}},
        {"1 0 obj", {.Integer, .Integer, .Keyword}},
        {"true false null", {.Keyword, .Keyword, .Keyword}},
        // '{' and '}' only mean anything in a type 4 function, but must
        // tokenise wherever they appear.
        {"{ dup 0.5 mul }", {.Brace_Open, .Keyword, .Real, .Keyword, .Brace_Close}},
    }

    toks := make([dynamic]Token, alloc)
    for c in cases {
        clear(&toks)
        lex_all(c.src, alloc, &toks)
        testing.expectf(t, len(toks) == len(c.kinds),
            "%q: got %d tokens, want %d", c.src, len(toks), len(c.kinds))
        if len(toks) != len(c.kinds) { continue }
        for k, i in c.kinds {
            testing.expectf(t, toks[i].kind == k,
                "%q: token %d is %v, want %v", c.src, i, toks[i].kind, k)
        }
    }
}

@(test)
test_lexer_token_spans :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    src := "  /Name 42 (str) "
    lex := lexer_make(transmute([]byte)src)

    n := lexer_next(&lex, alloc)
    testing.expect_value(t, n.start, 2)
    testing.expect_value(t, n.end, 7)

    i := lexer_next(&lex, alloc)
    testing.expect_value(t, i.start, 8)
    testing.expect_value(t, i.end, 10)

    s := lexer_next(&lex, alloc)
    testing.expect_value(t, s.start, 11)
    testing.expect_value(t, s.end, 16)

    e := lexer_next(&lex, alloc)
    testing.expect_value(t, e.kind, Token_Kind.EOF)
    testing.expect_value(t, e.start, len(src))
}

// ---------------------------------------------------------------------------
// Comments (ISO 32000-2, 7.2.4)
// ---------------------------------------------------------------------------

@(test)
test_lexer_comments :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    toks := make([dynamic]Token, alloc)
    lex_all("% comment\n1 % another\n2%trailing, no EOL", alloc, &toks)
    testing.expect_value(t, len(toks), 2)
    if len(toks) == 2 {
        testing.expect_value(t, toks[0].int_value, i64(1))
        testing.expect_value(t, toks[1].int_value, i64(2))
    }

    // A '%' inside a string is data, not a comment.
    clear(&toks)
    lex_all("(50% off)", alloc, &toks)
    testing.expect_value(t, len(toks), 1)
    if len(toks) == 1 {
        testing.expect(t, slice.equal(toks[0].bytes, transmute([]byte)string("50% off")),
            "'%' inside a literal string must not start a comment")
    }

    // A comment ends the token before it without whitespace in between.
    clear(&toks)
    lex_all("/Name%c\n/Other", alloc, &toks)
    testing.expect_value(t, len(toks), 2)
    if len(toks) == 2 {
        testing.expect_value(t, toks[0].text, "Name")
        testing.expect_value(t, toks[1].text, "Other")
    }
}

// ---------------------------------------------------------------------------
// Numbers (ISO 32000-2, 7.3.3)
// ---------------------------------------------------------------------------

@(test)
test_lexer_numbers :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    Case :: struct {
        src:  string,
        kind: Token_Kind,
        i:    i64,
        r:    f64,
    }
    cases := []Case{
        // The spec's own examples, table 2.
        {"123",     .Integer, 123, 123},
        {"43445",   .Integer, 43445, 43445},
        {"+17",     .Integer, 17, 17},
        {"-98",     .Integer, -98, -98},
        {"0",       .Integer, 0, 0},
        {"34.5",    .Real, 34, 34.5},
        {"-3.62",   .Real, -3, -3.62},
        {"+123.6",  .Real, 123, 123.6},
        {"4.",      .Real, 4, 4},
        {"-.002",   .Real, 0, -0.002},
        {"0.0",     .Real, 0, 0},
        // Shapes that only real files produce.
        {".5",      .Real, 0, 0.5},
        {".",       .Real, 0, 0},
        {"1e2",     .Real, 100, 100},
        {"-2.5e-2", .Real, 0, -0.025},
        // Nothing here is allowed to fail; an unreadable number is 0.
        {"--5",     .Integer, 0, 0},
        {"1.2.3",   .Real, 0, 0},
        {"-",       .Integer, 0, 0},
        {"+",       .Integer, 0, 0},
        // An exponent that overflows must not leak an infinity into geometry.
        {"1e400",   .Real, 0, 0},
        {"-1e400",  .Real, 0, 0},
    }

    for c in cases {
        lex := lexer_make(transmute([]byte)c.src)
        tok := lexer_next(&lex, alloc)
        testing.expectf(t, tok.kind == c.kind, "%q: kind %v, want %v", c.src, tok.kind, c.kind)
        testing.expectf(t, tok.int_value == c.i, "%q: int %d, want %d", c.src, tok.int_value, c.i)
        testing.expectf(t, tok.real_value == c.r, "%q: real %f, want %f", c.src, tok.real_value, c.r)
        testing.expectf(t, is_finite(tok.real_value), "%q: real value is not finite", c.src)
        testing.expectf(t, tok.end == len(c.src), "%q: consumed %d of %d bytes", c.src, tok.end, len(c.src))
    }
}

@(test)
test_lexer_number_stops_at_delimiter :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    lex := lexer_make(bytes_of("12/Name"))
    a := lexer_next(&lex, alloc)
    testing.expect_value(t, a.kind, Token_Kind.Integer)
    testing.expect_value(t, a.int_value, i64(12))
    b := lexer_next(&lex, alloc)
    testing.expect_value(t, b.kind, Token_Kind.Name)
    testing.expect_value(t, b.text, "Name")
}

// ---------------------------------------------------------------------------
// Names (ISO 32000-2, 7.3.5, table 4)
// ---------------------------------------------------------------------------

@(test)
test_lexer_names :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    Case :: struct {
        src:  string,
        want: string,
    }
    cases := []Case{
        {"/Name1", "Name1"},
        {"/ASomewhatLongerName", "ASomewhatLongerName"},
        {"/A;Name_With-Various***Characters?", "A;Name_With-Various***Characters?"},
        {"/1.2", "1.2"},
        {"/$$", "$$"},
        {"/@pattern", "@pattern"},
        {"/.notdef", ".notdef"},
        {"/lime#20Green", "lime Green"},
        {"/paired#28#29parentheses", "paired()parentheses"},
        {"/The_Key_of_F#23_Minor", "The_Key_of_F#_Minor"},
        {"/A#42", "AB"},
        // A bare '/' is the empty name, which is legal.
        {"/", ""},
        {"/ ", ""},
        {"/[", ""},
        // A '#' that is not followed by two hex digits stays as written.
        {"/Bad#", "Bad#"},
        {"/Bad#zz", "Bad#zz"},
        {"/Bad#4", "Bad#4"},
    }

    for c in cases {
        lex := lexer_make(transmute([]byte)c.src)
        tok := lexer_next(&lex, alloc)
        testing.expectf(t, tok.kind == .Name, "%q: kind %v", c.src, tok.kind)
        testing.expectf(t, tok.text == c.want, "%q: got %q, want %q", c.src, tok.text, c.want)
    }
}

// ---------------------------------------------------------------------------
// Literal strings (ISO 32000-2, 7.3.4.2)
// ---------------------------------------------------------------------------

@(test)
test_lexer_literal_strings :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    Case :: struct {
        src:       string,
        want:      string,
        truncated: bool,
    }
    cases := []Case{
        {"(This is a string)", "This is a string", false},
        {"()", "", false},
        // Balanced unescaped parentheses nest.
        {"(Strings may contain balanced parentheses ( ) and\nspecial characters (*!&}^% and so on).)",
         "Strings may contain balanced parentheses ( ) and\nspecial characters (*!&}^% and so on).", false},
        {"(((deep)))", "((deep))", false},
        {"(These \\(brackets\\) work)", "These (brackets) work", false},
        // Every escape in table 3.
        {"(a\\nb)", "a\nb", false},
        {"(a\\rb)", "a\rb", false},
        {"(a\\tb)", "a\tb", false},
        {"(a\\bb)", "a\bb", false},
        {"(a\\fb)", "a\fb", false},
        {"(a\\(b)", "a(b", false},
        {"(a\\)b)", "a)b", false},
        {"(a\\\\b)", "a\\b", false},
        // Octal, 1 to 3 digits. \0053 is \005 followed by '3'.
        {"(\\53)", "+", false},
        {"(\\053)", "+", false},
        {"(\\0053)", "\x053", false},
        {"(\\401)", "\x01", false},           // 0401 & 0xFF
        {"(\\0)", "\x00", false},
        // A backslash before anything else is dropped, the byte kept.
        {"(\\8)", "8", false},
        {"(\\q)", "q", false},
        // A backslash before an EOL is a line continuation.
        {"(line1\\\nline2)", "line1line2", false},
        {"(line1\\\r\nline2)", "line1line2", false},
        {"(line1\\\rline2)", "line1line2", false},
        // A raw CR, LF or CRLF in the source is a single LF in the value.
        {"(a\nb)", "a\nb", false},
        {"(a\rb)", "a\nb", false},
        {"(a\r\nb)", "a\nb", false},
        {"(a\n\rb)", "a\n\nb", false},
        // Unterminated: the value survives, the damage is flagged.
        {"(abc", "abc", true},
        {"(abc\\", "abc", true},
        {"((abc)", "(abc)", true},
        {"(", "", true},
    }

    for c in cases {
        lex := lexer_make(transmute([]byte)c.src)
        tok := lexer_next(&lex, alloc)
        testing.expectf(t, tok.kind == .String, "%q: kind %v", c.src, tok.kind)
        testing.expectf(t, slice.equal(tok.bytes, transmute([]byte)c.want),
            "%q: got %q, want %q", c.src, string(tok.bytes), c.want)
        testing.expectf(t, tok.truncated == c.truncated,
            "%q: truncated %v, want %v", c.src, tok.truncated, c.truncated)
        testing.expectf(t, lex.pos <= len(c.src), "%q: lexer ran past the end", c.src)
    }
}

@(test)
test_lexer_string_stops_where_it_should :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    // The escaped ')' must not close the string, and the token must end
    // exactly at the real closer.
    src := "(a\\)b) 7"
    lex := lexer_make(transmute([]byte)src)
    s := lexer_next(&lex, alloc)
    testing.expect_value(t, s.kind, Token_Kind.String)
    testing.expect_value(t, s.end, 6)
    n := lexer_next(&lex, alloc)
    testing.expect_value(t, n.kind, Token_Kind.Integer)
    testing.expect_value(t, n.int_value, i64(7))
}

// ---------------------------------------------------------------------------
// Hex strings (ISO 32000-2, 7.3.4.3)
// ---------------------------------------------------------------------------

@(test)
test_lexer_hex_strings :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    Case :: struct {
        src:       string,
        want:      string,
        truncated: bool,
    }
    cases := []Case{
        {"<4E6F762073686D6F7A206B6120706F702E>", "Nov shmoz ka pop.", false},
        {"<901FA3>", "\x90\x1f\xa3", false},
        // "If the final digit is missing it is assumed to be 0."
        {"<901FA>", "\x90\x1f\xa0", false},
        {"<>", "", false},
        {"<a>", "\xa0", false},
        // Case-insensitive, whitespace anywhere.
        {"<4a4B>", "JK", false},
        {"<4 1 4 2>", "AB", false},
        {"<41\n42\r43>", "ABC", false},
        // Junk between digits is skipped rather than failing the string.
        {"<41-42>", "AB", false},
        {"<41zz42>", "AB", false},
        // Unterminated runs to the end of the input.
        {"<4142", "AB", true},
        {"<", "", true},
    }

    for c in cases {
        lex := lexer_make(transmute([]byte)c.src)
        tok := lexer_next(&lex, alloc)
        testing.expectf(t, tok.kind == .String, "%q: kind %v", c.src, tok.kind)
        testing.expectf(t, slice.equal(tok.bytes, transmute([]byte)c.want),
            "%q: got %x, want %x", c.src, tok.bytes, transmute([]byte)c.want)
        testing.expectf(t, tok.truncated == c.truncated,
            "%q: truncated %v, want %v", c.src, tok.truncated, c.truncated)
    }
}

// ---------------------------------------------------------------------------
// Keyword helpers
// ---------------------------------------------------------------------------

@(test)
test_lexer_keyword_helpers :: proc(t: ^testing.T) {
    lex := lexer_make(bytes_of("  stream\r\ndata"))
    testing.expect(t, lexer_at_keyword(&lex, "stream"), "at_keyword should match past whitespace")
    testing.expect_value(t, lex.pos, 0)      // and must not consume
    testing.expect(t, !lexer_at_keyword(&lex, "endstream"), "wrong keyword must not match")

    // A keyword must not match a prefix of a longer regular-character run.
    lex2 := lexer_make(bytes_of("streamable"))
    testing.expect(t, !lexer_at_keyword(&lex2, "stream"), "prefix must not match")

    testing.expect(t, lexer_accept_keyword(&lex, "stream"), "accept should match")
    testing.expect_value(t, lex.pos, 8)
    lexer_skip_eol(&lex)
    testing.expect_value(t, lex.pos, 10)

    // A failed accept leaves the position alone.
    save := lex.pos
    testing.expect(t, !lexer_accept_keyword(&lex, "endstream"), "accept should fail")
    testing.expect_value(t, lex.pos, save)

    lex3 := lexer_make(bytes_of("\rx"))
    lexer_skip_eol(&lex3)
    testing.expect_value(t, lex3.pos, 1)     // a bare CR is one EOL
}

@(test)
test_lexer_peek :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    lex := lexer_make(bytes_of("/A /B"))
    p1 := lexer_peek(&lex, alloc)
    p2 := lexer_peek(&lex, alloc)
    testing.expect_value(t, p1.text, "A")
    testing.expect_value(t, p2.text, "A")
    testing.expect_value(t, lex.pos, 0)
    n := lexer_next(&lex, alloc)
    testing.expect_value(t, n.text, "A")
}

// ---------------------------------------------------------------------------
// Totality
//
// The one property the whole reader rests on: no input makes the lexer stall.
// ---------------------------------------------------------------------------

@(private = "file")
Rng :: struct {
    state: u64,
}

// xorshift64*, so the corpus is identical on every run and every platform.
@(private = "file")
rng_next :: proc(r: ^Rng) -> u64 {
    x := r.state
    x ~= x >> 12
    x ~= x << 25
    x ~= x >> 27
    r.state = x
    return x * 0x2545F4914F6CDD1D
}

@(private = "file")
assert_lexer_terminates :: proc(t: ^testing.T, data: []byte, alloc: mem.Allocator, what: string) {
    lex := lexer_make(data)
    // Every token consumes at least one byte, so more iterations than there
    // are bytes means the lexer stalled.
    budget := len(data) + 2
    for _ in 0 ..< budget {
        before := lex.pos
        tok := lexer_next(&lex, alloc)
        if tok.kind == .EOF {
            testing.expectf(t, lex.pos >= before, "%s: EOF moved backwards", what)
            return
        }
        testing.expectf(t, lex.pos > before, "%s: token %v did not advance at %d", what, tok.kind, before)
        testing.expectf(t, tok.end >= tok.start && tok.end <= len(data),
            "%s: token span [%d,%d) outside the input", what, tok.start, tok.end)
        if lex.pos <= before { return }
    }
    testing.expectf(t, false, "%s: lexer did not reach EOF within %d tokens", what, budget)
}

@(test)
test_lexer_is_total_on_random_bytes :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    r := Rng{state = 0x9E3779B97F4A7C15}
    buf := make([]byte, 64, alloc)
    for round in 0 ..< 400 {
        n := int(rng_next(&r) % 64)
        for i in 0 ..< n {
            v := rng_next(&r)
            // Half the corpus is drawn from the bytes that mean something, so
            // the delimiters and escapes are hit far more often than 1/256.
            if v & 1 == 0 {
                syntax := "()<>[]{}/%\\ \t\r\n0123456789.+-eRobj"
                buf[i] = syntax[int(v >> 8) % len(syntax)]
            } else {
                buf[i] = byte(v >> 8)
            }
        }
        assert_lexer_terminates(t, buf[:n], alloc, "random")
        _ = round
    }
}

@(test)
test_lexer_is_total_on_truncations :: proc(t: ^testing.T) {
    arena: virtual.Arena
    alloc := scratch_begin(t, &arena)
    defer virtual.arena_destroy(&arena)

    sample := transmute([]byte)string(
        "<< /Type /Page /Parent 3 0 R /Contents [ 4 0 R ] /S (a\\(b) /H <4142> " +
        "/N /A#20B /R [1 2 [3 [4]]] /X 1e3 >> stream\r\nxx\nendstream endobj %c\n")
    for n in 0 ..= len(sample) {
        assert_lexer_terminates(t, sample[:n], alloc, "truncation")
    }
    // And every suffix, which puts the lexer in the middle of every construct.
    for n in 0 ..< len(sample) {
        assert_lexer_terminates(t, sample[n:], alloc, "suffix")
    }
}