Illusion/pdf-odin-llm
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
Illusion
2026-09-25 23:32:29 +02:00
commit fff8b677f8
114 changed files with 84233 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
filter_ascii_test.odin
+355
View File
@@ -0,0 +1,355 @@
package pdf
import "core:mem/virtual"
import "core:slice"
import "core:testing"
@(private = "file")
Ascii_Case :: struct {
name: string,
in_: string,
want: string,
}
@(private = "file")
check_hex :: proc(t: ^testing.T, c: Ascii_Case) {
out, err := ascii_hex_decode(transmute([]byte)c.in_)
defer delete(out)
testing.expectf(t, err == .None, "%s: err = %v", c.name, err)
testing.expectf(t, slice.equal(out, transmute([]byte)c.want),
"%s: got %v, want %v", c.name, out, transmute([]byte)c.want)
}
@(private = "file")
check_a85 :: proc(t: ^testing.T, c: Ascii_Case) {
out, err := ascii85_decode(transmute([]byte)c.in_)
defer delete(out)
testing.expectf(t, err == .None, "%s: err = %v", c.name, err)
testing.expectf(t, slice.equal(out, transmute([]byte)c.want),
"%s: got %v, want %v", c.name, out, transmute([]byte)c.want)
}
// ---------------------------------------------------------------------------
// ASCIIHexDecode
// ---------------------------------------------------------------------------
@(test)
test_ascii_hex_basic :: proc(t: ^testing.T) {
cases := []Ascii_Case{
{"hello", "48656C6C6F>", "Hello"},
{"lowercase", "48656c6c6f>", "Hello"},
{"mixed case", "48656C6c6F>", "Hello"},
{"zero byte", "00>", "\x00"},
{"all nibbles", "0123456789ABCDEF>", "\x01\x23\x45\x67\x89\xAB\xCD\xEF"},
{"no terminator", "48656C6C6F", "Hello"},
{"data after '>'", "4142>43444546", "AB"},
}
for c in cases {
check_hex(t, c)
}
}
@(test)
test_ascii_hex_odd_digit :: proc(t: ^testing.T) {
// ISO 32000-2, 7.4.2: an odd final digit is treated as if followed by '0'.
cases := []Ascii_Case{
{"single digit", "4>", "\x40"},
{"odd tail", "414>", "A\x40"},
{"odd, unterminated", "41 4", "A\x40"},
{"odd 'f'", "f>", "\xF0"},
}
for c in cases {
check_hex(t, c)
}
}
@(test)
test_ascii_hex_whitespace_and_junk :: proc(t: ^testing.T) {
cases := []Ascii_Case{
{"spaces", "48 65 6C 6C 6F >", "Hello"},
{"newlines", "48\n65\r\n6C\t6C\x0C6F>", "Hello"},
{"nul padding", "\x0048656C6C6F\x00>", "Hello"},
{"leading ws", " \n48>", "H"},
{"junk skipped", "4?8g6/5>", "H\x65"},
{"all junk", "hello world>", "\xed"},
}
for c in cases {
check_hex(t, c)
}
}
@(test)
test_ascii_hex_degenerate :: proc(t: ^testing.T) {
cases := []Ascii_Case{
{"empty", "", ""},
{"eod only", ">", ""},
{"only ws", " \n\t ", ""},
{"eod first", "> 4142", ""},
{"only junk", "zz!!", ""},
}
for c in cases {
check_hex_nothing(t, c)
}
}
// A decode that produces nothing must also allocate nothing, or every empty
// stream in a document leaks a zero-length block.
@(private = "file")
check_hex_nothing :: proc(t: ^testing.T, c: Ascii_Case) {
out, err := ascii_hex_decode(transmute([]byte)c.in_)
defer delete(out)
testing.expectf(t, err == .None, "%s: err = %v", c.name, err)
testing.expectf(t, len(out) == 0, "%s: got %v, want empty", c.name, out)
}
@(test)
test_ascii_hex_long_run :: proc(t: ^testing.T) {
// 512 bytes through the filter, to exercise the measure/fill passes on
// something longer than one cache line of hand-written input.
src := make([]byte, 512)
defer delete(src)
enc := make([dynamic]byte, 0, 1100)
defer delete(enc)
digits := "0123456789ABCDEF"
for i in 0 ..< len(src) {
src[i] = byte(i * 7)
append(&enc, digits[src[i] >> 4], digits[src[i] & 0xF])
if i % 40 == 39 {
append(&enc, '\n')
}
}
append(&enc, '>')
out, err := ascii_hex_decode(enc[:])
defer delete(out)
testing.expect(t, err == .None)
testing.expect(t, slice.equal(out, src))
}
// ---------------------------------------------------------------------------
// ASCII85Decode
// ---------------------------------------------------------------------------
@(test)
test_ascii85_basic :: proc(t: ^testing.T) {
cases := []Ascii_Case{
{"full group", "9jqo^~>", "Man "},
{"three bytes", "9jqo~>", "Man"},
{"two bytes", "9jn~>", "Ma"},
{"one byte", "9`~>", "M"},
{"six bytes", "F*2M7/c~>", "sure."},
{"max u32", "s8W-!~>", "\xFF\xFF\xFF\xFF"},
{"no terminator", "9jqo^", "Man "},
{"partial, no terminator", "9jn", "Ma"},
}
for c in cases {
check_a85(t, c)
}
}
@(test)
test_ascii85_z_shorthand :: proc(t: ^testing.T) {
cases := []Ascii_Case{
{"z alone", "z~>", "\x00\x00\x00\x00"},
{"z twice", "zz~>", "\x00\x00\x00\x00\x00\x00\x00\x00"},
{"z then data", "zBOu!rDZ~>", "\x00\x00\x00\x00hello"},
{"data then z", "9jqo^z~>", "Man \x00\x00\x00\x00"},
// 'z' inside a group has no meaning; dropping it lets the surrounding
// group still decode, which is what a producer that emitted stray
// bytes meant.
{"z mid-group", "9jzqo^~>", "Man "},
}
for c in cases {
check_a85(t, c)
}
}
@(test)
test_ascii85_whitespace_and_prefix :: proc(t: ^testing.T) {
cases := []Ascii_Case{
{"prefix", "<~9jqo^~>", "Man "},
{"prefix and ws", " \n<~9jqo^ ~>", "Man "},
{"ws everywhere", "9 j\nq\ro\t^\x0C~>", "Man "},
{"nul padding", "\x009jqo^\x00~>", "Man "},
{"ws in prefix-less", " 9jqo^~>", "Man "},
}
for c in cases {
check_a85(t, c)
}
}
@(test)
test_ascii85_degenerate :: proc(t: ^testing.T) {
cases := []Ascii_Case{
{"empty", "", ""},
{"eod only", "~>", ""},
{"prefix and eod", "<~~>", ""},
{"only ws", " \n\t", ""},
// A final group of one character encodes nothing at all: it carries
// fewer than 8 bits.
{"single leftover", "9jqo^9~>", "Man "},
{"lone leftover", "9~>", ""},
// A lone '~' is not a legal terminator but is still the end of
// anything decodable.
{"bare tilde", "9jqo^~", "Man "},
{"data after eod", "9jqo^~>9jqo^", "Man "},
}
for c in cases {
check_a85(t, c)
}
}
@(test)
test_ascii85_out_of_range_group :: proc(t: ^testing.T) {
// "uuuuu" is 85^5-1 = 4437053124, past 2^32-1. Clamping keeps the group;
// rejecting it would lose the rest of the stream with it.
out, err := ascii85_decode(transmute([]byte)string("uuuuu~>"))
defer delete(out)
testing.expect(t, err == .None)
testing.expect(t, slice.equal(out, []byte{0xFF, 0xFF, 0xFF, 0xFF}))
// A short group padded with 'u' can overflow the same way.
out2, err2 := ascii85_decode(transmute([]byte)string("uuuu~>"))
defer delete(out2)
testing.expect(t, err2 == .None)
testing.expect(t, slice.equal(out2, []byte{0xFF, 0xFF, 0xFF}))
}
@(test)
test_ascii85_out_of_alphabet :: proc(t: ^testing.T) {
cases := []Ascii_Case{
// 'v'..'y' and DEL are outside '!'..'u' and are dropped.
{"high letters", "9jqvo^~>", "Man "},
{"del", "9jq\x7Fo^~>", "Man "},
{"high bit", "9jq\xFFo^~>", "Man "},
{"below bang", "9jq\x1Fo^~>", "Man "},
}
for c in cases {
check_a85(t, c)
}
}
@(test)
test_ascii85_spec_vector :: proc(t: ^testing.T) {
// The canonical Ascii85 vector: Hamlet, II.ii.
enc := `9jqo^BlbD-BleB1DJ+*+F(f,q/0JhKF<GL>Cj@.4Gp$d7F!,L7@<6@)/0JDEF<G%<+EV:2F!,O<DJ+*.@<*K0@<6L(Df-\0Ec5e;DffZ(EZee.Bl.9pF"AGXBPCsi+DGm>@3BB/F*&OCAfu2/AKYi(DIb9~>`
want := "Man is distinguished, not only by his reason, but by this singular " +
"passion from other animals, which is a lust of the mind,"
out, err := ascii85_decode(transmute([]byte)enc)
defer delete(out)
testing.expect(t, err == .None)
testing.expectf(t, slice.equal(out, transmute([]byte)want), "got %q", string(out))
}
@(test)
test_ascii85_truncated_everywhere :: proc(t: ^testing.T) {
// Truncating the vector at every offset must decode a prefix of the
// plaintext and never crash: a stream cut short mid-group is the normal
// shape of a damaged file.
enc := `9jqo^BlbD-BleB1DJ+*+F(f,q/0JhKF<GL>~>`
full := "Man is distinguished, not only by his reason, "
for cut in 0 ..= len(enc) {
out, err := ascii85_decode(transmute([]byte)enc[:cut])
testing.expectf(t, err == .None, "cut %d: err = %v", cut, err)
testing.expectf(t, len(out) <= len(full), "cut %d: %d bytes", cut, len(out))
if len(out) > 0 {
testing.expectf(t, slice.equal(out, transmute([]byte)full[:len(out)]),
"cut %d: got %q", cut, string(out))
}
delete(out)
}
}
@(test)
test_ascii_hex_truncated_everywhere :: proc(t: ^testing.T) {
enc := "48656C6C6F2C20776F726C64>"
full := "Hello, world"
for cut in 0 ..= len(enc) {
out, err := ascii_hex_decode(transmute([]byte)enc[:cut])
testing.expectf(t, err == .None, "cut %d: err = %v", cut, err)
testing.expectf(t, len(out) <= len(full), "cut %d: %d bytes", cut, len(out))
// An odd cut lands mid-byte, whose value is the high nibble alone.
n := cut / 2
if n > 0 && n <= len(full) {
testing.expectf(t, slice.equal(out[:n], transmute([]byte)full[:n]),
"cut %d: got %q", cut, string(out))
}
delete(out)
}
}
@(test)
test_ascii85_cross_check :: proc(t: ^testing.T) {
// Vectors produced by an independent encoder (Python's base64.a85encode)
// over random bytes, at every length modulo 4, so the partial-group path
// is checked against something other than this file's own arithmetic.
cases := []Ascii_Case{
{"n=1", "i;~>", "\xE1"},
{"n=2", "3s,~>", ";\x03"},
{"n=3", "/eKQ~>", ".\x11*"},
{"n=4", "1:;B5~>", "2\xB5y\x08"},
{"n=5", "%g]H7m/~>", "\x0F\x08\xB1\xF7\xED"},
{"n=7", "9J0U3#Pii~>", "L.]:\x07\xF9\x7F"},
{"n=11", "+mpR,(Q'?(p9f\\~>", "!\xEE#-\x17\x8A \x9A\xF6\xB5\x88"},
{"n=16", "IqImN,Qau;o\"e<F-iR$k~>", "\x7Ff\xE8\x09$\x02\xAAI\xF2\xC1U\x1B'\xFES&"},
{"n=23", "DFk;B3(TS''\\/0W>)Q1E:[Jn9ZMgp~>",
"nI\x0D\xB18H\x9C\xE8\x14\xD5\x8D\x14Z\x8BO\x99O\xED\x15\xC5\xB2\xFD\xAE"},
}
for c in cases {
check_a85(t, c)
}
}
@(test)
test_ascii_filters_honour_allocator :: proc(t: ^testing.T) {
// Decoded stream data usually goes straight into the Document arena, which
// is never freed element by element, so the allocator parameter has to be
// the one that is actually used.
arena: virtual.Arena
testing.expect(t, virtual.arena_init_growing(&arena) == nil)
defer virtual.arena_destroy(&arena)
alloc := virtual.arena_allocator(&arena)
hex, herr := ascii_hex_decode(transmute([]byte)string("48656C6C6F>"), alloc)
testing.expect(t, herr == .None)
testing.expect(t, slice.equal(hex, transmute([]byte)string("Hello")))
a85, aerr := ascii85_decode(transmute([]byte)string("9jqo^~>"), alloc)
testing.expect(t, aerr == .None)
testing.expect(t, slice.equal(a85, transmute([]byte)string("Man ")))
}
// A tiny deterministic PRNG, so the fuzz tests are reproducible and do not
// depend on the state of the context's random generator.
@(private = "file")
ascii_fuzz_next :: proc(state: ^u64) -> u32 {
state^ = state^ * 6364136223846793005 + 1442695040888963407
return u32(state^ >> 33)
}
@(test)
test_ascii_filters_fuzz :: proc(t: ^testing.T) {
// Random bytes are not valid data for either filter, which is the point:
// neither may crash, hang, or claim more output than the input can hold.
state: u64 = 0xC0FFEE
buf: [256]byte
for round in 0 ..< 400 {
n := int(ascii_fuzz_next(&state) % u32(len(buf) + 1))
for i in 0 ..< n {
buf[i] = byte(ascii_fuzz_next(&state))
}
data := buf[:n]
hex, herr := ascii_hex_decode(data)
testing.expectf(t, herr == .None, "round %d: hex err %v", round, herr)
testing.expectf(t, len(hex) <= n / 2 + 1, "round %d: hex grew to %d from %d",
round, len(hex), n)
delete(hex)
a85, aerr := ascii85_decode(data)
testing.expectf(t, aerr == .None, "round %d: a85 err %v", round, aerr)
testing.expectf(t, len(a85) <= 4 * n, "round %d: a85 grew to %d from %d",
round, len(a85), n)
delete(a85)
}
}