Fixed CCSDS parsing

+ Using bitvec now for parsing binary data
+ Fixed issues with how some data was handled across byte boundaries
+ Updated CCSDS def
+ Added test to verify good ccsds parsing
+ clippy + fmt

Cargo.lock

@@ -34,6 +34,18 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitvec"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1489fcb93a5bb47da0462ca93ad252ad6af2145cce58d10d46a83931ba9f016b"
+dependencies = [
+ "funty",
+ "radium",
+ "tap",
+ "wyz",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.9.0"
@@ -92,6 +104,7 @@ dependencies = [
 name = "formaty"
 version = "0.1.0"
 dependencies = [
+ "bitvec",
  "byteorder",
  "num-bigint",
  "rust-embed",
@@ -100,6 +113,12 @@ dependencies = [
  "toml",
 ]
 
+[[package]]
+name = "funty"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
+
 [[package]]
 name = "generic-array"
 version = "0.14.5"
@@ -218,6 +237,12 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "radium"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
+
 [[package]]
 name = "rust-embed"
 version = "6.3.0"
@@ -335,6 +360,12 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "tap"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
+
 [[package]]
 name = "textwrap"
 version = "0.11.0"
@@ -430,3 +461,12 @@ name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "wyz"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30b31594f29d27036c383b53b59ed3476874d518f0efb151b27a4c275141390e"
+dependencies = [
+ "tap",
+]

Cargo.toml

@@ -11,4 +11,5 @@ serde = { version = "1.0", features = ["derive"] }
 toml = "0.5.8"
 byteorder = "1.4.3"
 num-bigint = "0.4"
-rust-embed="6.3.0"
+rust-embed="6.3.0"
+bitvec = "1.0.0"

formats/ccsds.toml

@@ -3,7 +3,17 @@
 # Ref: https://public.ccsds.org/Pubs/133x0b2e1.pdfa
 #
 # Example Packet:
-# [0xe0, 0xa1, 0xc0, 0x00, 0x00, 0x05, 0x01, 0x02, 0x03, 0x04, 0x05]
+# [0x12, 0x01, 0xc0, 0x00, 0x00, 0x05, 0x01, 0x02, 0x03, 0x04, 0x05]
+#
+# Output:
+# Version Number: 0
+# Packet Type: 1
+# Secondary Header Flag: 0
+# APID: 0x201
+# Sequence Flags: 3
+# Packet Sequence Count: 0
+# Data Length: 5
+# Data: [1, 2, 3, 4, 5]
 
 # CCSDS packet w/ primary header + payload
 [[formats]]
@@ -12,33 +22,27 @@ bit_flip = false
 
 [[formats.fields]]
 name = "Version Number"
-bit_flip = true
 field_type = {type = "UInt", bit_width = 3, endianness = "BigEndian"}
 
 [[formats.fields]]
 name = "Packet Type"
-bit_flip = true
 field_type = {type = "UInt", bit_width = 1, endianness = "BigEndian"}
 
 [[formats.fields]]
 name = "Secondary Header Flag"
-bit_flip = true
 field_type = {type = "UInt", bit_width = 1, endianness = "BigEndian"}
 
 [[formats.fields]]
 name = "APID"
-bit_flip = true
 print_type = {print = "Base", base = 16}
 field_type = {type = "UInt", bit_width = 11, endianness = "BigEndian"}
 
 [[formats.fields]]
 name = "Sequence Flags"
-bit_flip = true
 field_type = {type = "UInt", bit_width = 2,  endianness = "BigEndian"}
 
 [[formats.fields]]
 name = "Packet Sequence Count"
-bit_flip = true
 field_type = {type = "UInt", bit_width = 14, endianness = "BigEndian"}
 
 [[formats.fields]]
@@ -57,42 +61,41 @@ bit_flip = false
 
 [[formats.fields]]
 name = "Version Number"
-bit_flip = true
 field_type = {type = "UInt", bit_width = 3, endianness = "BigEndian"}
 
 [[formats.fields]]
 name = "Packet Type"
-bit_flip = true
 field_type = {type = "UInt", bit_width = 1, endianness = "BigEndian"}
 
 [[formats.fields]]
 name = "Secondary Header Flag"
-bit_flip = true
 field_type = {type = "UInt", bit_width = 1, endianness = "BigEndian"}
 
 [[formats.fields]]
 name = "APID"
-bit_flip = true
 print_type = {print = "Base", base = 16}
 field_type = {type = "UInt", bit_width = 11, endianness = "BigEndian"}
 
 [[formats.fields]]
 name = "Sequence Flags"
-bit_flip = true
 field_type = {type = "UInt", bit_width = 2,  endianness = "BigEndian"}
 
 [[formats.fields]]
 name = "Packet Sequence Count"
-bit_flip = true
 field_type = {type = "UInt", bit_width = 14, endianness = "BigEndian"}
 
 [[formats.fields]]
 name = "Data Length"
 field_type = {type = "UInt", bit_width = 16, endianness = "BigEndian"}
 
+[[formats.fields]]
+name = "Data"
+# Allow payloads up to the max size that can be specfied in "Data Length"
+field_type = {type = "Bytes", max_len = 65535, endianness = "BigEndian"}
+
 [[formats.fields]]
 name = "Secondary header"
-field_type = {type = "Bytes", max_len = 8, endianness = "BigEndian"}
+field_type = {type = "Bytes", max_len = 6, endianness = "BigEndian"}
 
 [[formats.fields]]
 name = "data"

formats/example.toml

@@ -1,5 +1,15 @@
 # Example fields to show off configuration
-# Example data to use "[0x00, 0x55, 0xff, 0x43, 0xd2 0x99 0x90, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00, 0x57, 0x6f, 0x72, 0x6c, 0x64]"
+#
+# Example data:
+# "[0x00, 0x55, 0xff, 0x43, 0xd2 0x99 0x90, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00, 0x57, 0x6f, 0x72, 0x6c, 0x64]"
+#
+# Example Output:
+# int field: 85
+# uint field: 0xf
+# uint field: 0xf
+# float field: 421.1997
+# string field: [72, 101, 108, 108, 111]
+# bytes field: [87, 111, 114, 108, 100]
 
 [[formats]]
 # Format name

src/byte_stream/mod.rs

@@ -1,4 +1,5 @@
 use crate::formatter::format::ByteOrderOperations;
+use bitvec::prelude::*;
 use std::fmt::{Display, Formatter};
 
 #[derive(Clone, Debug)]
@@ -45,50 +46,33 @@ impl ByteStream {
 
     pub fn get_bytes<T: ByteOrderOperations>(
         &self,
-        bit_ndx: usize,
+        mut bit_ndx: usize,
         bit_count: usize,
     ) -> Result<Vec<u8>, ByteStreamError> {
-        let byte_ndx = bit_ndx / 8;
+        let mut bits = self.data.view_bits::<Msb0>().to_bitvec();
-        let bits_before = (bit_ndx % 8) as u8;
-        let bits_in_last_byte = ((bit_ndx + bit_count - bits_before as usize) % 8) as u8;
-        let byte_count = ((bit_count as f32) / (8.0)).ceil() as usize;
-        let bytes_needed = if (bits_before as usize + bit_count) % 8 != 0
-            && bits_before as usize + bit_count >= 8
-        {
-            byte_count + 1
-        } else {
-            byte_count
-        };
 
-        if bytes_needed > self.data.len() || (bytes_needed + byte_ndx) > self.data.len() {
+        if bit_count + bit_ndx > bits.len() {
             return Err(ByteStreamError::OutOfRange);
         }
 
-        let mut byte_stream = self.data[byte_ndx..byte_ndx + bytes_needed].to_vec();
-
         if self.reverse_bits {
-            for byte in &mut byte_stream {
+            bits.reverse();
-                *byte = byte.reverse_bits();
-            }
         }
 
-        let number = T::big_uint(&byte_stream) >> bits_before;
+        let mut data: Vec<u8> = vec![];
 
-        let mut byte_stream = T::big_u_int_to_bytes(number);
+        if bit_count % 8 != 0 {
-
+            let bits_needed = bit_count % 8;
-        if bytes_needed > byte_count && bytes_needed == byte_stream.len() {
+            data.push(bits[bit_ndx..bit_ndx + bits_needed].load::<u8>());
-            byte_stream.pop();
+            bit_ndx += bits_needed;
         }
 
-        if byte_count > byte_stream.len() {
+        for _ in 0..bit_count / 8 {
-            T::pad_bytes(&mut byte_stream, bytes_needed)
+            data.push(bits[bit_ndx..bit_ndx + 8].load::<u8>());
+            bit_ndx += 8;
         }
 
-        if bits_in_last_byte != 0 {
+        Ok(data)
-            *byte_stream.last_mut().unwrap() &= bit_mask(bits_in_last_byte);
-        }
-
-        Ok(byte_stream)
     }
 
     pub fn len(&self) -> usize {
@@ -133,7 +117,7 @@ mod tests {
         let bit_stream = ByteStream::from(bytes.clone());
 
         let new_bytes = bit_stream.get_bytes::<BigEndian>(4, 4).unwrap();
-        assert_eq!(vec![0x05], new_bytes);
+        assert_eq!(vec![0x0f], new_bytes);
     }
 
     #[test]
@@ -151,7 +135,7 @@ mod tests {
         let bit_stream = ByteStream::from(bytes.clone());
 
         let new_bytes = bit_stream.get_bytes::<LittleEndian>(7, 2).unwrap();
-        assert_eq!(vec![0x03], new_bytes);
+        assert_eq!(vec![0x01], new_bytes);
     }
 
     #[test]
@@ -169,6 +153,6 @@ mod tests {
         let bit_stream = ByteStream::from(bytes.clone());
 
         let new_bytes = bit_stream.get_bytes::<BigEndian>(0, 3).unwrap();
-        assert_eq!(vec![0x05], new_bytes);
+        assert_eq!(vec![0x02], new_bytes);
     }
 }

src/error/mod.rs

@@ -5,11 +5,11 @@ use std::error::Error;
 use std::fmt::{Display, Formatter};
 
 #[derive(Debug)]
+#[allow(clippy::enum_variant_names)]
 pub enum FormatyError {
     ByteArrayParseError(ByteArrayParseErr),
     FormatError(FormatError),
     FormatConfigError(FormatConfigError),
-    FormatNotFound(String),
 }
 
 impl From<ByteArrayParseErr> for FormatyError {
@@ -36,7 +36,6 @@ impl Display for FormatyError {
             FormatyError::ByteArrayParseError(e) => e.to_string(),
             FormatyError::FormatError(e) => e.to_string(),
             FormatyError::FormatConfigError(e) => e.to_string(),
-            FormatyError::FormatNotFound(err_msg) => format!("'{}' config not found.", err_msg),
         };
 
         write!(f, "{}", err_msg)

src/formatter/format.rs

@@ -12,15 +12,17 @@ use crate::formatter::printers::PrintType;
 pub trait ByteOrderOperations: ByteOrder {
     fn last_byte(buf: &mut Vec<u8>) -> Option<&mut u8>;
 
-    fn big_int(buf: &[u8]) -> BigInt;
+    fn big_int(buf: &[u8], trim: usize) -> BigInt;
 
-    fn big_uint(buf: &[u8]) -> BigUint;
+    fn big_uint(buf: &[u8], trim: usize) -> BigUint;
 
     fn big_u_int_to_bytes(big_int: BigUint) -> Vec<u8>;
 
     fn big_int_to_bytes(big_int: BigInt) -> Vec<u8>;
 
     fn pad_bytes(buf: &mut Vec<u8>, size: usize);
+
+    fn flip() -> bool;
 }
 
 impl ByteOrderOperations for BigEndian {
@@ -28,12 +30,18 @@ impl ByteOrderOperations for BigEndian {
         buf.first_mut()
     }
 
-    fn big_int(buf: &[u8]) -> BigInt {
+    fn big_int(buf: &[u8], trim: usize) -> BigInt {
-        BigInt::from_signed_bytes_be(buf)
+        let mut buf = buf.to_vec();
+        let byte = buf.first_mut().unwrap();
+        *byte = (*byte << trim) >> trim;
+        BigInt::from_signed_bytes_be(&buf)
     }
 
-    fn big_uint(buf: &[u8]) -> BigUint {
+    fn big_uint(buf: &[u8], trim: usize) -> BigUint {
-        BigUint::from_bytes_be(buf)
+        let mut buf = buf.to_vec();
+        let byte = buf.first_mut().unwrap();
+        *byte = (*byte << trim) >> trim;
+        BigUint::from_bytes_be(&buf)
     }
 
     fn big_u_int_to_bytes(big_int: BigUint) -> Vec<u8> {
@@ -52,6 +60,10 @@ impl ByteOrderOperations for BigEndian {
         let pad_to = size - buf.len();
         buf.splice(0..0, vec![0_u8; pad_to].iter().cloned());
     }
+
+    fn flip() -> bool {
+        true
+    }
 }
 
 impl ByteOrderOperations for LittleEndian {
@@ -59,12 +71,12 @@ impl ByteOrderOperations for LittleEndian {
         buf.last_mut()
     }
 
-    fn big_int(buf: &[u8]) -> BigInt {
+    fn big_int(buf: &[u8], trim: usize) -> BigInt {
-        BigInt::from_signed_bytes_le(buf)
+        BigInt::from_signed_bytes_le(buf) >> trim
     }
 
-    fn big_uint(buf: &[u8]) -> BigUint {
+    fn big_uint(buf: &[u8], trim: usize) -> BigUint {
-        BigUint::from_bytes_le(buf)
+        BigUint::from_bytes_le(buf) >> trim
     }
 
     fn big_u_int_to_bytes(big_int: BigUint) -> Vec<u8> {
@@ -85,6 +97,10 @@ impl ByteOrderOperations for LittleEndian {
         let mut pad = vec![0_u8; pad_to];
         buf.append(&mut pad);
     }
+
+    fn flip() -> bool {
+        false
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -163,6 +179,8 @@ pub struct Field {
     pub print_type: PrintType,
     /// Flip Bit Order
     pub bit_flip: Option<bool>,
+    //#[serde(default = "BitOffset::default")]
+    //pub bit_offset: BitOffset
 }
 
 impl Field {
@@ -183,7 +201,7 @@ impl Field {
                 *last_byte |= !bit_mask(last_bit + 1)
             }
 
-            let big_int = T::big_int(&bytes);
+            let big_int = T::big_int(&bytes, 0);
 
             Ok((self.print_type.print_big_int::<T>(big_int), bit_width))
         } else {
@@ -199,7 +217,7 @@ impl Field {
     ) -> Result<(String, usize), FormatError> {
         let bytes = byte_stream.get_bytes::<T>(bit_ndx, bit_width)?;
 
-        let big_int = T::big_uint(&bytes);
+        let big_int = T::big_uint(&bytes, 0);
         Ok((self.print_type.print_big_u_int::<T>(big_int), bit_width))
     }
 
@@ -411,8 +429,8 @@ mod tests {
             byte_vec.push((-i) as u8);
 
             let mut byte_stream = ByteStream::from(byte_vec);
-            let (pos_output, width1) = field.format_data(&mut byte_stream, 0).unwrap();
+            let (pos_output, width1) = field.format_data(&mut byte_stream, 4).unwrap();
-            let (neg_output, width2) = field.format_data(&mut byte_stream, 8).unwrap();
+            let (neg_output, width2) = field.format_data(&mut byte_stream, 12).unwrap();
 
             assert_eq!(width1, 4);
             assert_eq!(width2, 4);
@@ -430,7 +448,7 @@ mod tests {
                 endianness: Endianness::LittleEndian,
             },
             name: "test".to_string(),
-            bit_flip: None,
+            bit_flip: Some(true),
             print_type: Default::default(),
         };
         let mut byte_stream = ByteStream::from(vec![0x1B]);
@@ -471,7 +489,7 @@ mod tests {
             print_type: Default::default(),
         };
 
-        let mut byte_stream = ByteStream::from(vec![0xC0, 0x5F, 0x0A]);
+        let mut byte_stream = ByteStream::from(vec![0x0C, 0xF5, 0xA0]);
         let (output, width) = field.format_data(&mut byte_stream, 4).unwrap();
 
         assert_eq!(width, 16);

src/formatter/mod.rs

@@ -20,6 +20,7 @@ pub enum FormatConfigError {
         err: toml::de::Error,
     },
     Utf8Error(std::str::Utf8Error),
+    FormatNotFound(String),
 }
 
 impl Error for FormatConfigError {}
@@ -44,6 +45,7 @@ impl Display for FormatConfigError {
                 format!("Error parsing {}: {}", file_name, err)
             }
             FormatConfigError::Utf8Error(e) => e.to_string(),
+            FormatConfigError::FormatNotFound(s) => format!("{} was not found.", s),
         };
 
         write!(f, "Format Config Error: {}", err_msg)
@@ -96,4 +98,167 @@ impl FormatConfig {
 
         Ok(config)
     }
+
+    pub fn get_format(&self, name: &str) -> Result<Format, FormatConfigError> {
+        Ok(self
+            .formats
+            .iter()
+            .find(|f| f.name.as_str() == name)
+            .ok_or_else(|| FormatConfigError::FormatNotFound(name.to_string()))?
+            .clone())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::formatter::format::Format;
+    use crate::formatter::printers::print_bytes_as_array;
+    use crate::formatter::FormatConfig;
+
+    #[derive(PartialOrd, PartialEq, Debug, Clone)]
+    struct CCSDSPacket {
+        packet_version: u8,
+        packet_type: bool,
+        sec_header_flag: bool,
+        apid: u16,
+        seq_flag: u8,
+        seq: u16,
+        packet_len: u16,
+        data: Vec<u8>,
+    }
+
+    impl CCSDSPacket {
+        fn from_bytes(data: &[u8]) -> Self {
+            let packet_version = data[0] >> 5;
+            let packet_type = ((data[0] >> 4) & 0x01) == 1;
+            let sec_header_flag = ((data[0] >> 3) & 0x01) == 1;
+            let apid = (((data[0] & 0x7) as u16) << 8) | (data[1] as u16);
+
+            let seq_flag = data[2] >> 6;
+            let seq = (((data[2] & 0x3F) as u16) << 8) | (data[3] as u16);
+
+            let packet_len = ((data[4] as u16) << 8) | (data[5] as u16);
+
+            CCSDSPacket {
+                packet_version,
+                packet_type,
+                sec_header_flag,
+                apid,
+                seq_flag,
+                seq,
+                packet_len,
+                data: data[6..].to_vec(),
+            }
+        }
+
+        fn to_bytes(&self) -> Vec<u8> {
+            let mut data = vec![0u8; 6];
+            data[0] = (self.packet_version << 5)
+                | ((self.packet_type as u8) << 4)
+                | ((self.sec_header_flag as u8) << 3)
+                | ((self.apid >> 8) as u8);
+            data[1] = (self.apid & 0xFF) as u8;
+            data[2] = (self.seq_flag << 6) | ((self.seq >> 8) as u8);
+            data[3] = (self.seq & 0xFF) as u8;
+            data[4] = (self.packet_len >> 8) as u8;
+            data[5] = (self.packet_len & 0xff) as u8;
+
+            data.append(&mut self.data.clone());
+            data
+        }
+
+        // based off formats/ccsds.toml config
+        fn print(&self) -> String {
+            format!(
+                "\
+                Version Number: {}\n\
+                Packet Type: {}\n\
+                Secondary Header Flag: {}\n\
+                APID: {:#x}\n\
+                Sequence Flags: {}\n\
+                Packet Sequence Count: {}\n\
+                Data Length: {}\n\
+                Data: {}\n",
+                self.packet_version,
+                self.packet_type as u8,
+                self.sec_header_flag as u8,
+                self.apid,
+                self.seq_flag,
+                self.seq,
+                self.packet_len,
+                print_bytes_as_array(&self.data)
+            )
+        }
+    }
+
+    #[test]
+    fn test_ccsds() {
+        let ccsds_packet = CCSDSPacket {
+            packet_version: 0,
+            packet_type: true,
+            sec_header_flag: false,
+            apid: 0x2FF,
+            seq_flag: 3,
+            seq: 0,
+            packet_len: 5,
+            data: vec![0x01, 0x02, 0x03, 0x04, 0x05],
+        };
+
+        let bytes = ccsds_packet.to_bytes();
+
+        let parsed_packet = CCSDSPacket::from_bytes(&bytes);
+
+        assert_eq!(ccsds_packet, parsed_packet);
+
+        assert_eq!(bytes, parsed_packet.to_bytes())
+    }
+
+    fn get_ccsds_format() -> Format {
+        let format_config = FormatConfig::new(&None).unwrap();
+
+        format_config.get_format("ccsds").unwrap()
+    }
+
+    #[test]
+    fn test_ccsds_parse() {
+        let ccsds_packet = CCSDSPacket {
+            packet_version: 0,
+            packet_type: true,
+            sec_header_flag: false,
+            apid: 0x2FF,
+            seq_flag: 3,
+            seq: 0,
+            packet_len: 5,
+            data: vec![0x01, 0x02, 0x03, 0x04, 0x05],
+        };
+
+        let format = get_ccsds_format();
+        let data = ccsds_packet.to_bytes();
+        let output = format.format_data(&data).unwrap();
+
+        assert_eq!(ccsds_packet.print(), output)
+    }
+
+    #[test]
+    fn test_ccsds_parse_hammer() {
+        let format = get_ccsds_format();
+
+        for apid in 0..(2u16.pow(11)) {
+            let ccsds_packet = CCSDSPacket {
+                packet_version: 0,
+                packet_type: (apid % 2) == 0,
+                sec_header_flag: (apid % 2) == 1,
+                apid,
+                seq_flag: (apid % 4) as u8,
+                seq: (apid % 11),
+                packet_len: 5,
+                data: vec![0x01, 0x02, 0x03, 0x04, 0x05],
+            };
+
+            let data = ccsds_packet.to_bytes();
+            let output = format.format_data(&data).unwrap();
+
+            assert_eq!(ccsds_packet.print(), output)
+        }
+    }
 }

src/main.rs

@@ -33,14 +33,9 @@ fn init() -> Result<(Vec<u8>, Format), FormatyError> {
 
     let config = FormatConfig::new(&args.config)?;
 
-    let format = config
+    let format = config.get_format(&args.format)?;
-        .formats
-        .iter()
-        .find(|f| f.name == args.format)
-        .ok_or_else(|| FormatyError::FormatNotFound(args.format.to_string()))?
-        .clone();
 
-    let data = parse_bytes_from_input_arg(args.data).unwrap();
+    let data = parse_bytes_from_input_arg(args.data)?;
     Ok((data, format))
 }