From d956c111d899c1778ba6fbb8c918f40310075467 Mon Sep 17 00:00:00 2001 From: Joey Hines Date: Sun, 17 Apr 2022 10:19:44 -0600 Subject: [PATCH] Fixed CCSDS parsing + Using bitvec now for parsing binary data + Fixed issues with how some data was handled across byte boundaries + Updated CCSDS def + Added test to verify good ccsds parsing + clippy + fmt --- Cargo.lock | 40 ++++++++++ Cargo.toml | 3 +- formats/ccsds.toml | 31 ++++---- formats/example.toml | 12 ++- src/byte_stream/mod.rs | 50 +++++------- src/error/mod.rs | 3 +- src/formatter/format.rs | 50 ++++++++---- src/formatter/mod.rs | 165 ++++++++++++++++++++++++++++++++++++++++ src/main.rs | 9 +-- 9 files changed, 289 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4bf7c3..9e10df1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,6 +34,18 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitvec" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1489fcb93a5bb47da0462ca93ad252ad6af2145cce58d10d46a83931ba9f016b" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -92,6 +104,7 @@ dependencies = [ name = "formaty" version = "0.1.0" dependencies = [ + "bitvec", "byteorder", "num-bigint", "rust-embed", @@ -100,6 +113,12 @@ dependencies = [ "toml", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "generic-array" version = "0.14.5" @@ -218,6 +237,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rust-embed" version = "6.3.0" @@ -335,6 +360,12 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "textwrap" version = "0.11.0" @@ -430,3 +461,12 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "wyz" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b31594f29d27036c383b53b59ed3476874d518f0efb151b27a4c275141390e" +dependencies = [ + "tap", +] diff --git a/Cargo.toml b/Cargo.toml index 97ce9d4..f27ec7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,4 +11,5 @@ serde = { version = "1.0", features = ["derive"] } toml = "0.5.8" byteorder = "1.4.3" num-bigint = "0.4" -rust-embed="6.3.0" \ No newline at end of file +rust-embed="6.3.0" +bitvec = "1.0.0" \ No newline at end of file diff --git a/formats/ccsds.toml b/formats/ccsds.toml index b6c49aa..3ebe18f 100644 --- a/formats/ccsds.toml +++ b/formats/ccsds.toml @@ -3,7 +3,17 @@ # Ref: https://public.ccsds.org/Pubs/133x0b2e1.pdfa # # Example Packet: -# [0xe0, 0xa1, 0xc0, 0x00, 0x00, 0x05, 0x01, 0x02, 0x03, 0x04, 0x05] +# [0x12, 0x01, 0xc0, 0x00, 0x00, 0x05, 0x01, 0x02, 0x03, 0x04, 0x05] +# +# Output: +# Version Number: 0 +# Packet Type: 1 +# Secondary Header Flag: 0 +# APID: 0x201 +# Sequence Flags: 3 +# Packet Sequence Count: 0 +# Data Length: 5 +# Data: [1, 2, 3, 4, 5] # CCSDS packet w/ primary header + payload [[formats]] @@ -12,33 +22,27 @@ bit_flip = false [[formats.fields]] name = "Version Number" -bit_flip = true field_type = {type = "UInt", bit_width = 3, endianness = "BigEndian"} [[formats.fields]] name = "Packet Type" -bit_flip = true field_type = {type = "UInt", bit_width = 1, endianness = "BigEndian"} [[formats.fields]] name = "Secondary Header Flag" -bit_flip = true field_type = {type = "UInt", bit_width = 1, endianness = "BigEndian"} [[formats.fields]] name = "APID" -bit_flip = true print_type = {print = "Base", base = 16} field_type = {type = "UInt", bit_width = 11, endianness = "BigEndian"} [[formats.fields]] name = "Sequence Flags" -bit_flip = true field_type = {type = "UInt", bit_width = 2, endianness = "BigEndian"} [[formats.fields]] name = "Packet Sequence Count" -bit_flip = true field_type = {type = "UInt", bit_width = 14, endianness = "BigEndian"} [[formats.fields]] @@ -57,42 +61,41 @@ bit_flip = false [[formats.fields]] name = "Version Number" -bit_flip = true field_type = {type = "UInt", bit_width = 3, endianness = "BigEndian"} [[formats.fields]] name = "Packet Type" -bit_flip = true field_type = {type = "UInt", bit_width = 1, endianness = "BigEndian"} [[formats.fields]] name = "Secondary Header Flag" -bit_flip = true field_type = {type = "UInt", bit_width = 1, endianness = "BigEndian"} [[formats.fields]] name = "APID" -bit_flip = true print_type = {print = "Base", base = 16} field_type = {type = "UInt", bit_width = 11, endianness = "BigEndian"} [[formats.fields]] name = "Sequence Flags" -bit_flip = true field_type = {type = "UInt", bit_width = 2, endianness = "BigEndian"} [[formats.fields]] name = "Packet Sequence Count" -bit_flip = true field_type = {type = "UInt", bit_width = 14, endianness = "BigEndian"} [[formats.fields]] name = "Data Length" field_type = {type = "UInt", bit_width = 16, endianness = "BigEndian"} +[[formats.fields]] +name = "Data" +# Allow payloads up to the max size that can be specfied in "Data Length" +field_type = {type = "Bytes", max_len = 65535, endianness = "BigEndian"} + [[formats.fields]] name = "Secondary header" -field_type = {type = "Bytes", max_len = 8, endianness = "BigEndian"} +field_type = {type = "Bytes", max_len = 6, endianness = "BigEndian"} [[formats.fields]] name = "data" diff --git a/formats/example.toml b/formats/example.toml index 6a4e2bc..3c9eb18 100644 --- a/formats/example.toml +++ b/formats/example.toml @@ -1,5 +1,15 @@ # Example fields to show off configuration -# Example data to use "[0x00, 0x55, 0xff, 0x43, 0xd2 0x99 0x90, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00, 0x57, 0x6f, 0x72, 0x6c, 0x64]" +# +# Example data: +# "[0x00, 0x55, 0xff, 0x43, 0xd2 0x99 0x90, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00, 0x57, 0x6f, 0x72, 0x6c, 0x64]" +# +# Example Output: +# int field: 85 +# uint field: 0xf +# uint field: 0xf +# float field: 421.1997 +# string field: [72, 101, 108, 108, 111] +# bytes field: [87, 111, 114, 108, 100] [[formats]] # Format name diff --git a/src/byte_stream/mod.rs b/src/byte_stream/mod.rs index 5dd37af..fbe7a57 100644 --- a/src/byte_stream/mod.rs +++ b/src/byte_stream/mod.rs @@ -1,4 +1,5 @@ use crate::formatter::format::ByteOrderOperations; +use bitvec::prelude::*; use std::fmt::{Display, Formatter}; #[derive(Clone, Debug)] @@ -45,50 +46,33 @@ impl ByteStream { pub fn get_bytes( &self, - bit_ndx: usize, + mut bit_ndx: usize, bit_count: usize, ) -> Result, ByteStreamError> { - let byte_ndx = bit_ndx / 8; - let bits_before = (bit_ndx % 8) as u8; - let bits_in_last_byte = ((bit_ndx + bit_count - bits_before as usize) % 8) as u8; - let byte_count = ((bit_count as f32) / (8.0)).ceil() as usize; - let bytes_needed = if (bits_before as usize + bit_count) % 8 != 0 - && bits_before as usize + bit_count >= 8 - { - byte_count + 1 - } else { - byte_count - }; + let mut bits = self.data.view_bits::().to_bitvec(); - if bytes_needed > self.data.len() || (bytes_needed + byte_ndx) > self.data.len() { + if bit_count + bit_ndx > bits.len() { return Err(ByteStreamError::OutOfRange); } - let mut byte_stream = self.data[byte_ndx..byte_ndx + bytes_needed].to_vec(); - if self.reverse_bits { - for byte in &mut byte_stream { - *byte = byte.reverse_bits(); - } + bits.reverse(); } - let number = T::big_uint(&byte_stream) >> bits_before; + let mut data: Vec = vec![]; - let mut byte_stream = T::big_u_int_to_bytes(number); - - if bytes_needed > byte_count && bytes_needed == byte_stream.len() { - byte_stream.pop(); + if bit_count % 8 != 0 { + let bits_needed = bit_count % 8; + data.push(bits[bit_ndx..bit_ndx + bits_needed].load::()); + bit_ndx += bits_needed; } - if byte_count > byte_stream.len() { - T::pad_bytes(&mut byte_stream, bytes_needed) + for _ in 0..bit_count / 8 { + data.push(bits[bit_ndx..bit_ndx + 8].load::()); + bit_ndx += 8; } - if bits_in_last_byte != 0 { - *byte_stream.last_mut().unwrap() &= bit_mask(bits_in_last_byte); - } - - Ok(byte_stream) + Ok(data) } pub fn len(&self) -> usize { @@ -133,7 +117,7 @@ mod tests { let bit_stream = ByteStream::from(bytes.clone()); let new_bytes = bit_stream.get_bytes::(4, 4).unwrap(); - assert_eq!(vec![0x05], new_bytes); + assert_eq!(vec![0x0f], new_bytes); } #[test] @@ -151,7 +135,7 @@ mod tests { let bit_stream = ByteStream::from(bytes.clone()); let new_bytes = bit_stream.get_bytes::(7, 2).unwrap(); - assert_eq!(vec![0x03], new_bytes); + assert_eq!(vec![0x01], new_bytes); } #[test] @@ -169,6 +153,6 @@ mod tests { let bit_stream = ByteStream::from(bytes.clone()); let new_bytes = bit_stream.get_bytes::(0, 3).unwrap(); - assert_eq!(vec![0x05], new_bytes); + assert_eq!(vec![0x02], new_bytes); } } diff --git a/src/error/mod.rs b/src/error/mod.rs index d22c3a5..4a20d63 100644 --- a/src/error/mod.rs +++ b/src/error/mod.rs @@ -5,11 +5,11 @@ use std::error::Error; use std::fmt::{Display, Formatter}; #[derive(Debug)] +#[allow(clippy::enum_variant_names)] pub enum FormatyError { ByteArrayParseError(ByteArrayParseErr), FormatError(FormatError), FormatConfigError(FormatConfigError), - FormatNotFound(String), } impl From for FormatyError { @@ -36,7 +36,6 @@ impl Display for FormatyError { FormatyError::ByteArrayParseError(e) => e.to_string(), FormatyError::FormatError(e) => e.to_string(), FormatyError::FormatConfigError(e) => e.to_string(), - FormatyError::FormatNotFound(err_msg) => format!("'{}' config not found.", err_msg), }; write!(f, "{}", err_msg) diff --git a/src/formatter/format.rs b/src/formatter/format.rs index b6a8b98..1451a92 100644 --- a/src/formatter/format.rs +++ b/src/formatter/format.rs @@ -12,15 +12,17 @@ use crate::formatter::printers::PrintType; pub trait ByteOrderOperations: ByteOrder { fn last_byte(buf: &mut Vec) -> Option<&mut u8>; - fn big_int(buf: &[u8]) -> BigInt; + fn big_int(buf: &[u8], trim: usize) -> BigInt; - fn big_uint(buf: &[u8]) -> BigUint; + fn big_uint(buf: &[u8], trim: usize) -> BigUint; fn big_u_int_to_bytes(big_int: BigUint) -> Vec; fn big_int_to_bytes(big_int: BigInt) -> Vec; fn pad_bytes(buf: &mut Vec, size: usize); + + fn flip() -> bool; } impl ByteOrderOperations for BigEndian { @@ -28,12 +30,18 @@ impl ByteOrderOperations for BigEndian { buf.first_mut() } - fn big_int(buf: &[u8]) -> BigInt { - BigInt::from_signed_bytes_be(buf) + fn big_int(buf: &[u8], trim: usize) -> BigInt { + let mut buf = buf.to_vec(); + let byte = buf.first_mut().unwrap(); + *byte = (*byte << trim) >> trim; + BigInt::from_signed_bytes_be(&buf) } - fn big_uint(buf: &[u8]) -> BigUint { - BigUint::from_bytes_be(buf) + fn big_uint(buf: &[u8], trim: usize) -> BigUint { + let mut buf = buf.to_vec(); + let byte = buf.first_mut().unwrap(); + *byte = (*byte << trim) >> trim; + BigUint::from_bytes_be(&buf) } fn big_u_int_to_bytes(big_int: BigUint) -> Vec { @@ -52,6 +60,10 @@ impl ByteOrderOperations for BigEndian { let pad_to = size - buf.len(); buf.splice(0..0, vec![0_u8; pad_to].iter().cloned()); } + + fn flip() -> bool { + true + } } impl ByteOrderOperations for LittleEndian { @@ -59,12 +71,12 @@ impl ByteOrderOperations for LittleEndian { buf.last_mut() } - fn big_int(buf: &[u8]) -> BigInt { - BigInt::from_signed_bytes_le(buf) + fn big_int(buf: &[u8], trim: usize) -> BigInt { + BigInt::from_signed_bytes_le(buf) >> trim } - fn big_uint(buf: &[u8]) -> BigUint { - BigUint::from_bytes_le(buf) + fn big_uint(buf: &[u8], trim: usize) -> BigUint { + BigUint::from_bytes_le(buf) >> trim } fn big_u_int_to_bytes(big_int: BigUint) -> Vec { @@ -85,6 +97,10 @@ impl ByteOrderOperations for LittleEndian { let mut pad = vec![0_u8; pad_to]; buf.append(&mut pad); } + + fn flip() -> bool { + false + } } #[derive(Debug, Clone)] @@ -163,6 +179,8 @@ pub struct Field { pub print_type: PrintType, /// Flip Bit Order pub bit_flip: Option, + //#[serde(default = "BitOffset::default")] + //pub bit_offset: BitOffset } impl Field { @@ -183,7 +201,7 @@ impl Field { *last_byte |= !bit_mask(last_bit + 1) } - let big_int = T::big_int(&bytes); + let big_int = T::big_int(&bytes, 0); Ok((self.print_type.print_big_int::(big_int), bit_width)) } else { @@ -199,7 +217,7 @@ impl Field { ) -> Result<(String, usize), FormatError> { let bytes = byte_stream.get_bytes::(bit_ndx, bit_width)?; - let big_int = T::big_uint(&bytes); + let big_int = T::big_uint(&bytes, 0); Ok((self.print_type.print_big_u_int::(big_int), bit_width)) } @@ -411,8 +429,8 @@ mod tests { byte_vec.push((-i) as u8); let mut byte_stream = ByteStream::from(byte_vec); - let (pos_output, width1) = field.format_data(&mut byte_stream, 0).unwrap(); - let (neg_output, width2) = field.format_data(&mut byte_stream, 8).unwrap(); + let (pos_output, width1) = field.format_data(&mut byte_stream, 4).unwrap(); + let (neg_output, width2) = field.format_data(&mut byte_stream, 12).unwrap(); assert_eq!(width1, 4); assert_eq!(width2, 4); @@ -430,7 +448,7 @@ mod tests { endianness: Endianness::LittleEndian, }, name: "test".to_string(), - bit_flip: None, + bit_flip: Some(true), print_type: Default::default(), }; let mut byte_stream = ByteStream::from(vec![0x1B]); @@ -471,7 +489,7 @@ mod tests { print_type: Default::default(), }; - let mut byte_stream = ByteStream::from(vec![0xC0, 0x5F, 0x0A]); + let mut byte_stream = ByteStream::from(vec![0x0C, 0xF5, 0xA0]); let (output, width) = field.format_data(&mut byte_stream, 4).unwrap(); assert_eq!(width, 16); diff --git a/src/formatter/mod.rs b/src/formatter/mod.rs index 427f66c..e8bddb0 100644 --- a/src/formatter/mod.rs +++ b/src/formatter/mod.rs @@ -20,6 +20,7 @@ pub enum FormatConfigError { err: toml::de::Error, }, Utf8Error(std::str::Utf8Error), + FormatNotFound(String), } impl Error for FormatConfigError {} @@ -44,6 +45,7 @@ impl Display for FormatConfigError { format!("Error parsing {}: {}", file_name, err) } FormatConfigError::Utf8Error(e) => e.to_string(), + FormatConfigError::FormatNotFound(s) => format!("{} was not found.", s), }; write!(f, "Format Config Error: {}", err_msg) @@ -96,4 +98,167 @@ impl FormatConfig { Ok(config) } + + pub fn get_format(&self, name: &str) -> Result { + Ok(self + .formats + .iter() + .find(|f| f.name.as_str() == name) + .ok_or_else(|| FormatConfigError::FormatNotFound(name.to_string()))? + .clone()) + } +} + +#[cfg(test)] +mod test { + use crate::formatter::format::Format; + use crate::formatter::printers::print_bytes_as_array; + use crate::formatter::FormatConfig; + + #[derive(PartialOrd, PartialEq, Debug, Clone)] + struct CCSDSPacket { + packet_version: u8, + packet_type: bool, + sec_header_flag: bool, + apid: u16, + seq_flag: u8, + seq: u16, + packet_len: u16, + data: Vec, + } + + impl CCSDSPacket { + fn from_bytes(data: &[u8]) -> Self { + let packet_version = data[0] >> 5; + let packet_type = ((data[0] >> 4) & 0x01) == 1; + let sec_header_flag = ((data[0] >> 3) & 0x01) == 1; + let apid = (((data[0] & 0x7) as u16) << 8) | (data[1] as u16); + + let seq_flag = data[2] >> 6; + let seq = (((data[2] & 0x3F) as u16) << 8) | (data[3] as u16); + + let packet_len = ((data[4] as u16) << 8) | (data[5] as u16); + + CCSDSPacket { + packet_version, + packet_type, + sec_header_flag, + apid, + seq_flag, + seq, + packet_len, + data: data[6..].to_vec(), + } + } + + fn to_bytes(&self) -> Vec { + let mut data = vec![0u8; 6]; + data[0] = (self.packet_version << 5) + | ((self.packet_type as u8) << 4) + | ((self.sec_header_flag as u8) << 3) + | ((self.apid >> 8) as u8); + data[1] = (self.apid & 0xFF) as u8; + data[2] = (self.seq_flag << 6) | ((self.seq >> 8) as u8); + data[3] = (self.seq & 0xFF) as u8; + data[4] = (self.packet_len >> 8) as u8; + data[5] = (self.packet_len & 0xff) as u8; + + data.append(&mut self.data.clone()); + data + } + + // based off formats/ccsds.toml config + fn print(&self) -> String { + format!( + "\ + Version Number: {}\n\ + Packet Type: {}\n\ + Secondary Header Flag: {}\n\ + APID: {:#x}\n\ + Sequence Flags: {}\n\ + Packet Sequence Count: {}\n\ + Data Length: {}\n\ + Data: {}\n", + self.packet_version, + self.packet_type as u8, + self.sec_header_flag as u8, + self.apid, + self.seq_flag, + self.seq, + self.packet_len, + print_bytes_as_array(&self.data) + ) + } + } + + #[test] + fn test_ccsds() { + let ccsds_packet = CCSDSPacket { + packet_version: 0, + packet_type: true, + sec_header_flag: false, + apid: 0x2FF, + seq_flag: 3, + seq: 0, + packet_len: 5, + data: vec![0x01, 0x02, 0x03, 0x04, 0x05], + }; + + let bytes = ccsds_packet.to_bytes(); + + let parsed_packet = CCSDSPacket::from_bytes(&bytes); + + assert_eq!(ccsds_packet, parsed_packet); + + assert_eq!(bytes, parsed_packet.to_bytes()) + } + + fn get_ccsds_format() -> Format { + let format_config = FormatConfig::new(&None).unwrap(); + + format_config.get_format("ccsds").unwrap() + } + + #[test] + fn test_ccsds_parse() { + let ccsds_packet = CCSDSPacket { + packet_version: 0, + packet_type: true, + sec_header_flag: false, + apid: 0x2FF, + seq_flag: 3, + seq: 0, + packet_len: 5, + data: vec![0x01, 0x02, 0x03, 0x04, 0x05], + }; + + let format = get_ccsds_format(); + let data = ccsds_packet.to_bytes(); + let output = format.format_data(&data).unwrap(); + + assert_eq!(ccsds_packet.print(), output) + } + + #[test] + fn test_ccsds_parse_hammer() { + let format = get_ccsds_format(); + + for apid in 0..(2u16.pow(11)) { + let ccsds_packet = CCSDSPacket { + packet_version: 0, + packet_type: (apid % 2) == 0, + sec_header_flag: (apid % 2) == 1, + apid, + seq_flag: (apid % 4) as u8, + seq: (apid % 11), + packet_len: 5, + data: vec![0x01, 0x02, 0x03, 0x04, 0x05], + }; + + let data = ccsds_packet.to_bytes(); + let output = format.format_data(&data).unwrap(); + + assert_eq!(ccsds_packet.print(), output) + } + } } diff --git a/src/main.rs b/src/main.rs index 0542378..0fe106b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -33,14 +33,9 @@ fn init() -> Result<(Vec, Format), FormatyError> { let config = FormatConfig::new(&args.config)?; - let format = config - .formats - .iter() - .find(|f| f.name == args.format) - .ok_or_else(|| FormatyError::FormatNotFound(args.format.to_string()))? - .clone(); + let format = config.get_format(&args.format)?; - let data = parse_bytes_from_input_arg(args.data).unwrap(); + let data = parse_bytes_from_input_arg(args.data)?; Ok((data, format)) }