diff --git a/TODO.md b/TODO.md index 0cf55ed..a85ec16 100644 --- a/TODO.md +++ b/TODO.md @@ -1,15 +1,35 @@ Files:- +14 -> MD5 mismatch +23 -> MD5 mismatch 27 -> This one uses the old format of specifying a variable block strategy. Look into it. The way to know the blocking strategy for this old format FLACs is to see if min_block_size and max_block_size are different, if they are, then the blocking strategy is VARIABLE, if not, then it's FIXED. -61 -> I suspect the predictor DOES overflow here. check it -62 -> I suspect the predictor DOES overflow here. check it -63 -> I suspect the predictor DOES overflow here. check it + Also MD5 mismatch +28 -> MD5 mismatch +29 -> MD5 mismatch +30 -> MD5 mismatch +31 -> MD5 mismatch +32 -> MD5 mismatch +33 -> MD5 mismatch +34 -> MD5 mismatch +35 -> MD5 mismatch +36 -> MD5 mismatch +38 -> MD5 mismatch +39 -> MD5 mismatch +40 -> MD5 mismatch +44 -> MD5 mismatch + High memory and CPU usage. needs optimization. +--- 40s and 50s have a lot of MD5 mismatches too, too lazy to write them. +60 -> MD5 mismatch +61 -> MD5 mismatch +63 -> MD5 mismatch +64 -> MD5 mismatch uncommon/01 -> This one decodes fine but the expected MD5 is 0x00 * 16 which doesn't match the calculated MD5. not sure what to do about this. -uncommon/02 -> We fail a CRC check -uncommon/03 -> We fail so hard we end up reading an invalid residual coding method +uncommon/02 -> No MD5 +uncommon/03 -> No MD5 uncommon/04 -> We get a bits per sample mismatch +uncommon/05 -> MD5 mismatch +uncommon/06 -> MD5 mismatch uncommon/10 -> We error with Invalid Signature. I assume for these two I should look for the frame sync bits. But maybe my decoder should reject these. idk. uncommon/11 -> We error with Invalid Signature @@ -18,21 +38,21 @@ faulty/02 -> This one works when it shouldn't (Wrong max framesize) faulty/04 -> We fail on Invalid_Coefficient_Precision. This one has a wrong number of channels (whatever that means) faulty/05 -> This one works when it shouldn't. (Wrong number of samples) faulty/08 -> This one works when it shouldn't. (Blocksize 65536, exceeds the allowed max of 65535) -faulty/09 -> This one works when it shouldn't. (Blocksize 1, below the allowed min of 16 (last frame is allowed to have less than 16)) +faulty/09 -> This one works when it shouldn't. (Blocksize 1, below the allowed min of 16 (last frame(subframe?) is allowed to have less than 16)) Things:- -[ ] Stereo Decorrelation [ ] Lots of checks (look at the TODOs in flac.odin) [ ] MD5 checksums (depends on stereo decorrelation) - - [ ] VERBATIM subframe + - [x] VERBATIM subframe (seems to work fine if all frames are verbatim, might not work if mixed with other frames) - [ ] CONSTANT subframe - - [ ] FIXED subframe + - [x] FIXED subframe - [ ] LPC subframe [ ] Seeking [x] RICE2 [x] Add check for mismatched bits per sample [x] Unencoded BPS is zero [x] CONSTANT subframe +[x] Stereo Decorrelation [ ] Encoder (low priority for now) diff --git a/flac.odin b/flac.odin index caf2a0b..170b6fa 100644 --- a/flac.odin +++ b/flac.odin @@ -85,6 +85,13 @@ crc16_lookup := [256]u16 { 0x8213, 0x0216, 0x021C, 0x8219, 0x0208, 0x820D, 0x8207, 0x0202, } +fixed_coefficients := [4][]i32 { + {1}, + {2, -1}, + {3, -3, 1}, + {4, -6, 4, -1}, +} + md5_ctx: md5.Context decoded_samples: [dynamic]byte output_file: os.Handle @@ -189,7 +196,7 @@ SampleSize :: enum { ChannelAssignment :: enum { MONO, - _2CHANNEL, + _2CHANNELS, _3CHANNELS, _4CHANNELS, _5CHANNELS, @@ -274,7 +281,7 @@ CueSheetTrackIndex :: struct #packed { reserved: [3]byte, } -decode_subframe :: proc(r: ^Reader, bps: u8, block_size: u16) -> Error { +decode_subframe :: proc(r: ^Reader, bps: u8, block_size: u16) -> (samples: []i32, err: Error) { subframe_type_str := "" // TODO: temp allocator?? so we copy the slice of data into the _actual_ sample buffer // Maybe even a static arena with a size of 32MB by default (and configurable using #config) or something @@ -282,13 +289,13 @@ decode_subframe :: proc(r: ^Reader, bps: u8, block_size: u16) -> Error { // is probably noticibly slower) subframe_samples := make([dynamic]i32, 0, block_size) - fmt.println("cur offset", r.i) - fmt.println("cur offset bits", r.bits_read_in_byte) + fmt.println("SUBFRAME START AT OFFSET:", r.i) + fmt.println("SUBFRAME offset bits", r.bits_read_in_byte) data := read_byte(r) or_return - fmt.println("First subframe byte:", data) - fmt.printfln("First subframe byte: 0b%b", data) + //fmt.println("First subframe byte:", data) + //fmt.printfln("First subframe byte: 0b%b", data) subframe_type := (data & 0x7E) >> 1 low_3_bits := subframe_type & 7 @@ -306,7 +313,7 @@ decode_subframe :: proc(r: ^Reader, bps: u8, block_size: u16) -> Error { } if subframe_type == 0 { // CONSTANT - fmt.println("SUBFRAME: CONSTANT") + //fmt.println("SUBFRAME: CONSTANT") subframe_type_str = "CONSTANT" bits_to_read := int(bps) - wasted_bits // TODO: should we shift to the left by the number of wasted bits?? @@ -316,31 +323,32 @@ decode_subframe :: proc(r: ^Reader, bps: u8, block_size: u16) -> Error { } append(&subframe_samples, sample) } else if subframe_type == 1 { // VERBATIM - fmt.println("SUBFRAME: VERBATIM") + //fmt.println("SUBFRAME: VERBATIM") subframe_type_str = "VERBATIM" // We loop over the samples in the block here because each sample is BPS bits long. // Which I assume is always 16 but that's wrong because BPS can go as high as 32 bits // TODO: this will break if bits per sample is anything BUT 16 if bps < 16 || bps > 17 { fmt.println("UNSUPPORTED BPS FOR VERBATIM SUBFRAME") - return .Unsupported_BPS + return nil, .Unsupported_BPS } for sample in 0..> 5) & 1 == 1 { // LPC fmt.println("SUBFRAME: LPC") @@ -349,9 +357,9 @@ decode_subframe :: proc(r: ^Reader, bps: u8, block_size: u16) -> Error { predictor_order := (subframe_type & 0x1F) + 1 fmt.println("predictor_order:", predictor_order) + bits_to_read := int(bps) - wasted_bits for sample in 0..> (uint(bits_to_read) - 1)) & 1 == 1 { @@ -359,14 +367,14 @@ decode_subframe :: proc(r: ^Reader, bps: u8, block_size: u16) -> Error { } //fmt.printfln("unencoded warmup sample: 0b%b", unencoded_warmup_sample) - //fmt.println("unencoded warmup sample:", unencoded_warmup_sample) + fmt.println("unencoded warmup sample:", unencoded_warmup_sample) //fmt.printfln("unencoded warmup sample: 0x%X", unencoded_warmup_sample) append(&subframe_samples, unencoded_warmup_sample) } coeff_precision_minus_one := u8(read_bits(r, 4) or_return) if coeff_precision_minus_one == 0xF { - return .Invalid_Coefficient_Precision + return nil, .Invalid_Coefficient_Precision } coeff_precision := coeff_precision_minus_one + 1 @@ -390,26 +398,26 @@ decode_subframe :: proc(r: ^Reader, bps: u8, block_size: u16) -> Error { unencoded_coefficient = -(((~unencoded_coefficient) & i32(math.pow2_f32(coeff_precision) - 1)) + 1) } coefficients[i] = unencoded_coefficient - //fmt.println("unencoded_coefficient", unencoded_coefficient) + fmt.println("unencoded_coefficient", unencoded_coefficient) } residual_coding_method := ResidualCodingMethod(read_bits(r, 2) or_return) if residual_coding_method != .RICE && residual_coding_method != .RICE2 { - return .Invalid_Residual_Coding_Method + return nil, .Invalid_Residual_Coding_Method } fmt.println(residual_coding_method) partition_order := read_bits(r, 4) or_return - //fmt.println("parition order", partition_order) + fmt.println("parition order", partition_order) partitions := int(math.pow2_f32(partition_order)) - //fmt.println("partitions:", partitions) - //fmt.println("block_size:", block_size) + fmt.println("partitions:", partitions) + fmt.println("block_size:", block_size) lpc_partitions_loop: for i in 0.. Error { //fmt.println("shifting by", coeff_bits_to_shift) // NOTE: Apply arithmetic right shift in case the bits to shift are negative - predictor := int(math.floor(f32(predictor_before_shift) / math.pow2_f32(coeff_bits_to_shift))) - //predictor := predictor_before_shift >> u8(coeff_bits_to_shift) + //predictor := int(math.floor(f32(predictor_before_shift) / math.pow2_f32(coeff_bits_to_shift))) + predictor := predictor_before_shift >> u32(coeff_bits_to_shift) //fmt.println("predictor:", predictor) sample := i32(predictor) + residual_sample_value append(&subframe_samples, sample) - //fmt.println("residual sample value:", residual_sample_value) + //fmt.printfln(" residual[%d]=%d", s, residual_sample_value) //fmt.println("sample value:", sample) sample_i += 1 } } else { - //fmt.println("rice parameter:", rice_parameter) + fmt.println("rice parameter:", rice_parameter) num_samples := 0 if partition_order == 0 { @@ -480,7 +488,7 @@ decode_subframe :: proc(r: ^Reader, bps: u8, block_size: u16) -> Error { // Undo the RICE coding // previous_sample_value := 0 - for j in 0.. Error { //fmt.println("shifting by", coeff_bits_to_shift) // NOTE: Apply arithmetic right shift in case the bits to shift are negative - predictor := int(math.floor(f32(predictor_before_shift) / math.pow2_f32(coeff_bits_to_shift))) - //predictor := predictor_before_shift >> u8(coeff_bits_to_shift) + //predictor := int(math.floor(f32(predictor_before_shift) / math.pow2_f32(coeff_bits_to_shift))) + predictor := predictor_before_shift >> u32(coeff_bits_to_shift) //fmt.println("predictor:", predictor) sample := predictor + residual_sample_value append(&subframe_samples, i32(sample)) sample_i += 1 - //fmt.println("residual sample value:", residual_sample_value) + //fmt.printfln(" residual[%d]=%d", s, residual_sample_value) //fmt.println("sample value:", sample) } } @@ -532,6 +540,7 @@ decode_subframe :: proc(r: ^Reader, bps: u8, block_size: u16) -> Error { } else if (subframe_type >> 3) & 1 == 1 && low_3_bits <= 4 { // FIXED fmt.println("SUBFRAME: FIXED") subframe_type_str = "FIXED" + sample_i := 0 predictor_order := low_3_bits fmt.println("predictor_order:", predictor_order) @@ -541,22 +550,24 @@ decode_subframe :: proc(r: ^Reader, bps: u8, block_size: u16) -> Error { // few samples in each subframe cannot be predicted. Therefore, a // number of so-called warm-up samples equal to the predictor order is // stored" - bits_to_read := bps - u8(wasted_bits) - unencoded_samples_n := bits_to_read * predictor_order - //fmt.println("unencoded warmup samples n:", unencoded_samples_n) + bits_to_read := int(bps) - wasted_bits // NOTE: samples can be negative // TODO: should we shift to the left by the number of wasted bits?? - unencoded_warmup_sample := i32(read_bits(r, int(unencoded_samples_n)) or_return) - if (unencoded_warmup_sample >> (uint(bits_to_read) - 1)) & 1 == 1 { - unencoded_warmup_sample = -(((~unencoded_warmup_sample) & i32(math.pow2_f32(bits_to_read) - 1)) + 1) + for sample in 0..> (uint(bits_to_read) - 1)) & 1 == 1 { + unencoded_warmup_sample = -(((~unencoded_warmup_sample) & i32(math.pow2_f32(bits_to_read) - 1)) + 1) + } + + fmt.println("unencoded warmup sample:", unencoded_warmup_sample) + //fmt.printfln("unencoded warmup sample: 0x%X", unencoded_warmup_sample) + append(&subframe_samples, unencoded_warmup_sample) } - fmt.println("unencoded warmup sample:", unencoded_warmup_sample) - fmt.printfln("unencoded warmup sample: 0x%X", unencoded_warmup_sample) residual_coding_method := ResidualCodingMethod(read_bits(r, 2) or_return) if residual_coding_method != .RICE && residual_coding_method != .RICE2 { - return .Invalid_Residual_Coding_Method + return nil, .Invalid_Residual_Coding_Method } fmt.println(residual_coding_method) @@ -564,7 +575,7 @@ decode_subframe :: proc(r: ^Reader, bps: u8, block_size: u16) -> Error { fmt.println("parition order", partition_order) partitions := int(math.pow2_f32(partition_order)) fmt.println("partitions:", partitions) - //fmt.println("block_size:", block_size) + fmt.println("block_size:", block_size) fixed_partitions_loop: for i in 0.. Error { continue fixed_partitions_loop } - fmt.printfln("reading %d samples for residual", num_samples) + //fmt.printfln("reading %d samples for residual", num_samples) - for sample in 0..> (unencoded_bps - 1)) & 1 == 1 { residual_sample_value = -(((~residual_sample_value) & i8(math.pow2_f32(unencoded_bps) - 1)) + 1) } - fmt.println("residual sample value:", residual_sample_value) + //fmt.printfln(" residual[%d]=%d", s + (i * (int(block_size - 1) / partitions)), residual_sample_value) } - return .Unencoded_BPS_For_Fixed_Subframe_Is_Unsupported + return nil, .Unencoded_BPS_For_Fixed_Subframe_Is_Unsupported // TODO: calculate the actual sample value by using the residual AND the previous sample value // TODO: add the samples to something ?? } else { fmt.println("rice parameter:", rice_parameter) - samples := 0 + num_samples := 0 if partition_order == 0 { - samples = int(block_size - u16(predictor_order)) + num_samples = int(block_size - u16(predictor_order)) } else if i != 0 { - samples = int(block_size / u16(math.pow2_f32(partition_order))) + num_samples = int(block_size / u16(math.pow2_f32(partition_order))) } else { - samples = int((block_size / u16(math.pow2_f32(partition_order))) - u16(predictor_order)) + num_samples = int((block_size / u16(math.pow2_f32(partition_order))) - u16(predictor_order)) } - fmt.printfln("reading %d samples for residual", samples) + //fmt.printfln("reading %d samples for residual", samples) // // Undo the RICE coding // - previous_sample_value := 0 - for j in 0.. Error { } remainder := int(read_bits(r, int(rice_parameter)) or_return) - fmt.println("quotient:", quotient) - fmt.println("remainder:", remainder) + //fmt.println("quotient:", quotient) + //fmt.println("remainder:", remainder) zigzag_encoded_value := quotient * int(math.pow2_f32(rice_parameter)) + remainder // Unzigzag the residual sample values residual_sample_value := zigzag_encoded_value % 2 == 0 ? zigzag_encoded_value / 2 : (zigzag_encoded_value + 1) / -2 - fmt.println("residual sample value:", residual_sample_value) + //fmt.printfln(" residual[%d]=%d", sample_i, residual_sample_value) + + sample_value := 0 // Restore sample values using the predictor and the residual values - sample_value := j == 0 ? residual_sample_value + int(unencoded_warmup_sample) : residual_sample_value + previous_sample_value - previous_sample_value = sample_value + if predictor_order == 0 { + sample_value = residual_sample_value + } else { + //sample_value = residual_sample_value + int(subframe_samples[sample_i]) + predictor := 0 + c := 0 + #reverse for coefficient in fixed_coefficients[predictor_order - 1] { + predictor += int(coefficient) * int(subframe_samples[sample_i + c]) + c += 1 + } - fmt.println("sample_value:", sample_value) + sample_value = predictor + residual_sample_value + } + + + //sample_value := residual_sample_value + (predictor_order == 0 ? 0 : int(subframe_samples[sample_i])) + + append(&subframe_samples, i32(sample_value)) + + sample_i += 1 + + //fmt.printfln(" sample[%d]=%d", sample_i, sample_value) } } } } - // TODO: Maybe this is slow if we have a lot of samples?? - // The cases are ranges because streaminfo can define unusual bits per sample - for sample in subframe_samples { - switch bps { - case 1..=8: - data := []u8{u8(sample)} - os.write(output_file, data) - md5.update(&md5_ctx, data) - case 9..=16: - first_byte := i8(sample & 0xFF) - second_byte := i8((sample >> 8) & 0xFF) - data := slice.reinterpret([]u8, []i8{first_byte, second_byte}) - os.write(output_file, data) - md5.update(&md5_ctx, data) - case 17..=24: - first_byte := i8(sample & 0xFF) - second_byte := i8((sample >> 8) & 0xFF) - third_byte := i8((sample >> 16) & 0xFF) - data := slice.reinterpret([]u8, []i8{first_byte, second_byte, third_byte}) - os.write(output_file, data) - md5.update(&md5_ctx, data) - case 25..=32: - first_byte := i8(sample & 0xFF) - second_byte := i8((sample >> 8) & 0xFF) - third_byte := i8((sample >> 16) & 0xFF) - fourth_byte := i8((sample >> 24) & 0xFF) - data := slice.reinterpret([]u8, []i8{first_byte, second_byte, third_byte, fourth_byte}) - os.write(output_file, data) - md5.update(&md5_ctx, data) - } - } - fmt.println("cur offset", r.i) - fmt.println("cur offset bits", r.bits_read_in_byte) + //fmt.println("cur offset", r.i) + //fmt.println("cur offset bits", r.bits_read_in_byte) fmt.println("FINISHED SUBFRAME:", subframe_type_str) - return nil + return subframe_samples[:], nil } // TODO: remove @@ -698,13 +697,14 @@ decode_frame :: proc(r: ^Reader, streaminfo_bps: u8, streaminfo_sample_rate: u32 // frame_beginning := r.i + fmt.println("STARTING FRAME", frame_num) fmt.println("FRAME START AT OFFSET:", r.i) data := read_data(r, u16be) or_return sync_code := data >> 2 blocking_strategy := BlockingStrategy(data & 1) - fmt.println("blocking strategy:", blocking_strategy) + //fmt.println("blocking strategy:", blocking_strategy) data = read_data(r, u16be) or_return block_size := data >> 12 @@ -801,8 +801,8 @@ decode_frame :: proc(r: ^Reader, streaminfo_bps: u8, streaminfo_sample_rate: u32 fmt.printfln("block size bits: 0b%b", block_size) fmt.printfln("actual block size: %d sample(s)", actual_block_size) - fmt.println("sample rate bits:", sample_rate) - fmt.printfln("actual sample rate: %dHz", actual_sample_rate_in_Hz) + //fmt.println("sample rate bits:", sample_rate) + //fmt.printfln("actual sample rate: %dHz", actual_sample_rate_in_Hz) fmt.println("channel assignment:", channel_assignment) if frame_header_crc != calculated_header_crc { @@ -833,17 +833,51 @@ decode_frame :: proc(r: ^Reader, streaminfo_bps: u8, streaminfo_sample_rate: u32 bps = 32 } - fmt.println("channels:", channels) - fmt.println("bps:", bps) - fmt.println("sample size:", sample_size) - fmt.println("cur offset:", r.i) - fmt.println("cur offset bits:", r.bits_read_in_byte) + //fmt.println("bps:", bps) + //fmt.println("sample size:", sample_size) + //fmt.println("cur offset:", r.i) + //fmt.println("cur offset bits:", r.bits_read_in_byte) if bps != streaminfo_bps { return .Bits_Per_Second_Mismatch } - for i in 0..> 1) + right := (mid - side) >> 1 + + subframes[0][i] = left + subframes[1][i] = right + } + case .STEREO_SIDE_RIGHT: + for i in 0..> 8) & 0xFF) + data := slice.reinterpret([]u8, []i8{first_byte, second_byte}) + os.write(output_file, data) + md5.update(&md5_ctx, data) + case 17..=24: + first_byte := i8(sample & 0xFF) + second_byte := i8((sample >> 8) & 0xFF) + third_byte := i8((sample >> 16) & 0xFF) + data := slice.reinterpret([]u8, []i8{first_byte, second_byte, third_byte}) + os.write(output_file, data) + md5.update(&md5_ctx, data) + case 25..=32: + first_byte := i8(sample & 0xFF) + second_byte := i8((sample >> 8) & 0xFF) + third_byte := i8((sample >> 16) & 0xFF) + fourth_byte := i8((sample >> 24) & 0xFF) + data := slice.reinterpret([]u8, []i8{first_byte, second_byte, third_byte, fourth_byte}) + os.write(output_file, data) + md5.update(&md5_ctx, data) + case: + fmt.println("INVALID BPS") + os.exit(1) + } + } + } + + //fmt.printfln("%d", subframes) + // TODO: for stereo with side channels we must decorrelate the samples depending on the channel assignment // otherwise we'll be saving incorrect samples. @@ -968,7 +1081,7 @@ decode_extended_utf8 :: proc(r: ^Reader) -> (decoded_num: u64, err: Error) { decoded_num = u64(first_byte & 0x7F) } - fmt.println("decoded num:", decoded_num) + //fmt.println("decoded num:", decoded_num) return decoded_num, nil } @@ -1092,7 +1205,7 @@ load_from_bytes :: proc(data: []byte, allocator := context.allocator) -> (err: E } case .PICTURE: // TODO: save the pic data into a struct - // TODO: maybe put all of these into a struct so we do one big read? + // TODO: maybe put all of these into a struct so we do one big read? pic_type := read_data(&r, PictureType) or_return mime_len := read_data(&r, u32be) or_return mimetype := string(read_slice(&r, auto_cast mime_len) or_return)