Fixed CCSDS parsing
ci/woodpecker/push/woodpecker Pipeline was successful Details
ci/woodpecker/tag/woodpecker Pipeline was successful Details

+ Using bitvec now for parsing binary data
+ Fixed issues with how some data was handled across byte boundaries
+ Updated CCSDS def
+ Added test to verify good ccsds parsing
+ clippy + fmt
main
Joey Hines 2022-04-17 10:19:44 -06:00
parent 415a9c6ee1
commit d956c111d8
No known key found for this signature in database
GPG Key ID: 80F567B5C968F91B
9 changed files with 289 additions and 74 deletions

40
Cargo.lock generated
View File

@ -34,6 +34,18 @@ version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitvec"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1489fcb93a5bb47da0462ca93ad252ad6af2145cce58d10d46a83931ba9f016b"
dependencies = [
"funty",
"radium",
"tap",
"wyz",
]
[[package]]
name = "block-buffer"
version = "0.9.0"
@ -92,6 +104,7 @@ dependencies = [
name = "formaty"
version = "0.1.0"
dependencies = [
"bitvec",
"byteorder",
"num-bigint",
"rust-embed",
@ -100,6 +113,12 @@ dependencies = [
"toml",
]
[[package]]
name = "funty"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c"
[[package]]
name = "generic-array"
version = "0.14.5"
@ -218,6 +237,12 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "radium"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
[[package]]
name = "rust-embed"
version = "6.3.0"
@ -335,6 +360,12 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "tap"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
[[package]]
name = "textwrap"
version = "0.11.0"
@ -430,3 +461,12 @@ name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "wyz"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30b31594f29d27036c383b53b59ed3476874d518f0efb151b27a4c275141390e"
dependencies = [
"tap",
]

View File

@ -11,4 +11,5 @@ serde = { version = "1.0", features = ["derive"] }
toml = "0.5.8"
byteorder = "1.4.3"
num-bigint = "0.4"
rust-embed="6.3.0"
rust-embed="6.3.0"
bitvec = "1.0.0"

View File

@ -3,7 +3,17 @@
# Ref: https://public.ccsds.org/Pubs/133x0b2e1.pdfa
#
# Example Packet:
# [0xe0, 0xa1, 0xc0, 0x00, 0x00, 0x05, 0x01, 0x02, 0x03, 0x04, 0x05]
# [0x12, 0x01, 0xc0, 0x00, 0x00, 0x05, 0x01, 0x02, 0x03, 0x04, 0x05]
#
# Output:
# Version Number: 0
# Packet Type: 1
# Secondary Header Flag: 0
# APID: 0x201
# Sequence Flags: 3
# Packet Sequence Count: 0
# Data Length: 5
# Data: [1, 2, 3, 4, 5]
# CCSDS packet w/ primary header + payload
[[formats]]
@ -12,33 +22,27 @@ bit_flip = false
[[formats.fields]]
name = "Version Number"
bit_flip = true
field_type = {type = "UInt", bit_width = 3, endianness = "BigEndian"}
[[formats.fields]]
name = "Packet Type"
bit_flip = true
field_type = {type = "UInt", bit_width = 1, endianness = "BigEndian"}
[[formats.fields]]
name = "Secondary Header Flag"
bit_flip = true
field_type = {type = "UInt", bit_width = 1, endianness = "BigEndian"}
[[formats.fields]]
name = "APID"
bit_flip = true
print_type = {print = "Base", base = 16}
field_type = {type = "UInt", bit_width = 11, endianness = "BigEndian"}
[[formats.fields]]
name = "Sequence Flags"
bit_flip = true
field_type = {type = "UInt", bit_width = 2, endianness = "BigEndian"}
[[formats.fields]]
name = "Packet Sequence Count"
bit_flip = true
field_type = {type = "UInt", bit_width = 14, endianness = "BigEndian"}
[[formats.fields]]
@ -57,42 +61,41 @@ bit_flip = false
[[formats.fields]]
name = "Version Number"
bit_flip = true
field_type = {type = "UInt", bit_width = 3, endianness = "BigEndian"}
[[formats.fields]]
name = "Packet Type"
bit_flip = true
field_type = {type = "UInt", bit_width = 1, endianness = "BigEndian"}
[[formats.fields]]
name = "Secondary Header Flag"
bit_flip = true
field_type = {type = "UInt", bit_width = 1, endianness = "BigEndian"}
[[formats.fields]]
name = "APID"
bit_flip = true
print_type = {print = "Base", base = 16}
field_type = {type = "UInt", bit_width = 11, endianness = "BigEndian"}
[[formats.fields]]
name = "Sequence Flags"
bit_flip = true
field_type = {type = "UInt", bit_width = 2, endianness = "BigEndian"}
[[formats.fields]]
name = "Packet Sequence Count"
bit_flip = true
field_type = {type = "UInt", bit_width = 14, endianness = "BigEndian"}
[[formats.fields]]
name = "Data Length"
field_type = {type = "UInt", bit_width = 16, endianness = "BigEndian"}
[[formats.fields]]
name = "Data"
# Allow payloads up to the max size that can be specfied in "Data Length"
field_type = {type = "Bytes", max_len = 65535, endianness = "BigEndian"}
[[formats.fields]]
name = "Secondary header"
field_type = {type = "Bytes", max_len = 8, endianness = "BigEndian"}
field_type = {type = "Bytes", max_len = 6, endianness = "BigEndian"}
[[formats.fields]]
name = "data"

View File

@ -1,5 +1,15 @@
# Example fields to show off configuration
# Example data to use "[0x00, 0x55, 0xff, 0x43, 0xd2 0x99 0x90, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00, 0x57, 0x6f, 0x72, 0x6c, 0x64]"
#
# Example data:
# "[0x00, 0x55, 0xff, 0x43, 0xd2 0x99 0x90, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00, 0x57, 0x6f, 0x72, 0x6c, 0x64]"
#
# Example Output:
# int field: 85
# uint field: 0xf
# uint field: 0xf
# float field: 421.1997
# string field: [72, 101, 108, 108, 111]
# bytes field: [87, 111, 114, 108, 100]
[[formats]]
# Format name

View File

@ -1,4 +1,5 @@
use crate::formatter::format::ByteOrderOperations;
use bitvec::prelude::*;
use std::fmt::{Display, Formatter};
#[derive(Clone, Debug)]
@ -45,50 +46,33 @@ impl ByteStream {
pub fn get_bytes<T: ByteOrderOperations>(
&self,
bit_ndx: usize,
mut bit_ndx: usize,
bit_count: usize,
) -> Result<Vec<u8>, ByteStreamError> {
let byte_ndx = bit_ndx / 8;
let bits_before = (bit_ndx % 8) as u8;
let bits_in_last_byte = ((bit_ndx + bit_count - bits_before as usize) % 8) as u8;
let byte_count = ((bit_count as f32) / (8.0)).ceil() as usize;
let bytes_needed = if (bits_before as usize + bit_count) % 8 != 0
&& bits_before as usize + bit_count >= 8
{
byte_count + 1
} else {
byte_count
};
let mut bits = self.data.view_bits::<Msb0>().to_bitvec();
if bytes_needed > self.data.len() || (bytes_needed + byte_ndx) > self.data.len() {
if bit_count + bit_ndx > bits.len() {
return Err(ByteStreamError::OutOfRange);
}
let mut byte_stream = self.data[byte_ndx..byte_ndx + bytes_needed].to_vec();
if self.reverse_bits {
for byte in &mut byte_stream {
*byte = byte.reverse_bits();
}
bits.reverse();
}
let number = T::big_uint(&byte_stream) >> bits_before;
let mut data: Vec<u8> = vec![];
let mut byte_stream = T::big_u_int_to_bytes(number);
if bytes_needed > byte_count && bytes_needed == byte_stream.len() {
byte_stream.pop();
if bit_count % 8 != 0 {
let bits_needed = bit_count % 8;
data.push(bits[bit_ndx..bit_ndx + bits_needed].load::<u8>());
bit_ndx += bits_needed;
}
if byte_count > byte_stream.len() {
T::pad_bytes(&mut byte_stream, bytes_needed)
for _ in 0..bit_count / 8 {
data.push(bits[bit_ndx..bit_ndx + 8].load::<u8>());
bit_ndx += 8;
}
if bits_in_last_byte != 0 {
*byte_stream.last_mut().unwrap() &= bit_mask(bits_in_last_byte);
}
Ok(byte_stream)
Ok(data)
}
pub fn len(&self) -> usize {
@ -133,7 +117,7 @@ mod tests {
let bit_stream = ByteStream::from(bytes.clone());
let new_bytes = bit_stream.get_bytes::<BigEndian>(4, 4).unwrap();
assert_eq!(vec![0x05], new_bytes);
assert_eq!(vec![0x0f], new_bytes);
}
#[test]
@ -151,7 +135,7 @@ mod tests {
let bit_stream = ByteStream::from(bytes.clone());
let new_bytes = bit_stream.get_bytes::<LittleEndian>(7, 2).unwrap();
assert_eq!(vec![0x03], new_bytes);
assert_eq!(vec![0x01], new_bytes);
}
#[test]
@ -169,6 +153,6 @@ mod tests {
let bit_stream = ByteStream::from(bytes.clone());
let new_bytes = bit_stream.get_bytes::<BigEndian>(0, 3).unwrap();
assert_eq!(vec![0x05], new_bytes);
assert_eq!(vec![0x02], new_bytes);
}
}

View File

@ -5,11 +5,11 @@ use std::error::Error;
use std::fmt::{Display, Formatter};
#[derive(Debug)]
#[allow(clippy::enum_variant_names)]
pub enum FormatyError {
ByteArrayParseError(ByteArrayParseErr),
FormatError(FormatError),
FormatConfigError(FormatConfigError),
FormatNotFound(String),
}
impl From<ByteArrayParseErr> for FormatyError {
@ -36,7 +36,6 @@ impl Display for FormatyError {
FormatyError::ByteArrayParseError(e) => e.to_string(),
FormatyError::FormatError(e) => e.to_string(),
FormatyError::FormatConfigError(e) => e.to_string(),
FormatyError::FormatNotFound(err_msg) => format!("'{}' config not found.", err_msg),
};
write!(f, "{}", err_msg)

View File

@ -12,15 +12,17 @@ use crate::formatter::printers::PrintType;
pub trait ByteOrderOperations: ByteOrder {
fn last_byte(buf: &mut Vec<u8>) -> Option<&mut u8>;
fn big_int(buf: &[u8]) -> BigInt;
fn big_int(buf: &[u8], trim: usize) -> BigInt;
fn big_uint(buf: &[u8]) -> BigUint;
fn big_uint(buf: &[u8], trim: usize) -> BigUint;
fn big_u_int_to_bytes(big_int: BigUint) -> Vec<u8>;
fn big_int_to_bytes(big_int: BigInt) -> Vec<u8>;
fn pad_bytes(buf: &mut Vec<u8>, size: usize);
fn flip() -> bool;
}
impl ByteOrderOperations for BigEndian {
@ -28,12 +30,18 @@ impl ByteOrderOperations for BigEndian {
buf.first_mut()
}
fn big_int(buf: &[u8]) -> BigInt {
BigInt::from_signed_bytes_be(buf)
fn big_int(buf: &[u8], trim: usize) -> BigInt {
let mut buf = buf.to_vec();
let byte = buf.first_mut().unwrap();
*byte = (*byte << trim) >> trim;
BigInt::from_signed_bytes_be(&buf)
}
fn big_uint(buf: &[u8]) -> BigUint {
BigUint::from_bytes_be(buf)
fn big_uint(buf: &[u8], trim: usize) -> BigUint {
let mut buf = buf.to_vec();
let byte = buf.first_mut().unwrap();
*byte = (*byte << trim) >> trim;
BigUint::from_bytes_be(&buf)
}
fn big_u_int_to_bytes(big_int: BigUint) -> Vec<u8> {
@ -52,6 +60,10 @@ impl ByteOrderOperations for BigEndian {
let pad_to = size - buf.len();
buf.splice(0..0, vec![0_u8; pad_to].iter().cloned());
}
fn flip() -> bool {
true
}
}
impl ByteOrderOperations for LittleEndian {
@ -59,12 +71,12 @@ impl ByteOrderOperations for LittleEndian {
buf.last_mut()
}
fn big_int(buf: &[u8]) -> BigInt {
BigInt::from_signed_bytes_le(buf)
fn big_int(buf: &[u8], trim: usize) -> BigInt {
BigInt::from_signed_bytes_le(buf) >> trim
}
fn big_uint(buf: &[u8]) -> BigUint {
BigUint::from_bytes_le(buf)
fn big_uint(buf: &[u8], trim: usize) -> BigUint {
BigUint::from_bytes_le(buf) >> trim
}
fn big_u_int_to_bytes(big_int: BigUint) -> Vec<u8> {
@ -85,6 +97,10 @@ impl ByteOrderOperations for LittleEndian {
let mut pad = vec![0_u8; pad_to];
buf.append(&mut pad);
}
fn flip() -> bool {
false
}
}
#[derive(Debug, Clone)]
@ -163,6 +179,8 @@ pub struct Field {
pub print_type: PrintType,
/// Flip Bit Order
pub bit_flip: Option<bool>,
//#[serde(default = "BitOffset::default")]
//pub bit_offset: BitOffset
}
impl Field {
@ -183,7 +201,7 @@ impl Field {
*last_byte |= !bit_mask(last_bit + 1)
}
let big_int = T::big_int(&bytes);
let big_int = T::big_int(&bytes, 0);
Ok((self.print_type.print_big_int::<T>(big_int), bit_width))
} else {
@ -199,7 +217,7 @@ impl Field {
) -> Result<(String, usize), FormatError> {
let bytes = byte_stream.get_bytes::<T>(bit_ndx, bit_width)?;
let big_int = T::big_uint(&bytes);
let big_int = T::big_uint(&bytes, 0);
Ok((self.print_type.print_big_u_int::<T>(big_int), bit_width))
}
@ -411,8 +429,8 @@ mod tests {
byte_vec.push((-i) as u8);
let mut byte_stream = ByteStream::from(byte_vec);
let (pos_output, width1) = field.format_data(&mut byte_stream, 0).unwrap();
let (neg_output, width2) = field.format_data(&mut byte_stream, 8).unwrap();
let (pos_output, width1) = field.format_data(&mut byte_stream, 4).unwrap();
let (neg_output, width2) = field.format_data(&mut byte_stream, 12).unwrap();
assert_eq!(width1, 4);
assert_eq!(width2, 4);
@ -430,7 +448,7 @@ mod tests {
endianness: Endianness::LittleEndian,
},
name: "test".to_string(),
bit_flip: None,
bit_flip: Some(true),
print_type: Default::default(),
};
let mut byte_stream = ByteStream::from(vec![0x1B]);
@ -471,7 +489,7 @@ mod tests {
print_type: Default::default(),
};
let mut byte_stream = ByteStream::from(vec![0xC0, 0x5F, 0x0A]);
let mut byte_stream = ByteStream::from(vec![0x0C, 0xF5, 0xA0]);
let (output, width) = field.format_data(&mut byte_stream, 4).unwrap();
assert_eq!(width, 16);

View File

@ -20,6 +20,7 @@ pub enum FormatConfigError {
err: toml::de::Error,
},
Utf8Error(std::str::Utf8Error),
FormatNotFound(String),
}
impl Error for FormatConfigError {}
@ -44,6 +45,7 @@ impl Display for FormatConfigError {
format!("Error parsing {}: {}", file_name, err)
}
FormatConfigError::Utf8Error(e) => e.to_string(),
FormatConfigError::FormatNotFound(s) => format!("{} was not found.", s),
};
write!(f, "Format Config Error: {}", err_msg)
@ -96,4 +98,167 @@ impl FormatConfig {
Ok(config)
}
pub fn get_format(&self, name: &str) -> Result<Format, FormatConfigError> {
Ok(self
.formats
.iter()
.find(|f| f.name.as_str() == name)
.ok_or_else(|| FormatConfigError::FormatNotFound(name.to_string()))?
.clone())
}
}
#[cfg(test)]
mod test {
use crate::formatter::format::Format;
use crate::formatter::printers::print_bytes_as_array;
use crate::formatter::FormatConfig;
#[derive(PartialOrd, PartialEq, Debug, Clone)]
struct CCSDSPacket {
packet_version: u8,
packet_type: bool,
sec_header_flag: bool,
apid: u16,
seq_flag: u8,
seq: u16,
packet_len: u16,
data: Vec<u8>,
}
impl CCSDSPacket {
fn from_bytes(data: &[u8]) -> Self {
let packet_version = data[0] >> 5;
let packet_type = ((data[0] >> 4) & 0x01) == 1;
let sec_header_flag = ((data[0] >> 3) & 0x01) == 1;
let apid = (((data[0] & 0x7) as u16) << 8) | (data[1] as u16);
let seq_flag = data[2] >> 6;
let seq = (((data[2] & 0x3F) as u16) << 8) | (data[3] as u16);
let packet_len = ((data[4] as u16) << 8) | (data[5] as u16);
CCSDSPacket {
packet_version,
packet_type,
sec_header_flag,
apid,
seq_flag,
seq,
packet_len,
data: data[6..].to_vec(),
}
}
fn to_bytes(&self) -> Vec<u8> {
let mut data = vec![0u8; 6];
data[0] = (self.packet_version << 5)
| ((self.packet_type as u8) << 4)
| ((self.sec_header_flag as u8) << 3)
| ((self.apid >> 8) as u8);
data[1] = (self.apid & 0xFF) as u8;
data[2] = (self.seq_flag << 6) | ((self.seq >> 8) as u8);
data[3] = (self.seq & 0xFF) as u8;
data[4] = (self.packet_len >> 8) as u8;
data[5] = (self.packet_len & 0xff) as u8;
data.append(&mut self.data.clone());
data
}
// based off formats/ccsds.toml config
fn print(&self) -> String {
format!(
"\
Version Number: {}\n\
Packet Type: {}\n\
Secondary Header Flag: {}\n\
APID: {:#x}\n\
Sequence Flags: {}\n\
Packet Sequence Count: {}\n\
Data Length: {}\n\
Data: {}\n",
self.packet_version,
self.packet_type as u8,
self.sec_header_flag as u8,
self.apid,
self.seq_flag,
self.seq,
self.packet_len,
print_bytes_as_array(&self.data)
)
}
}
#[test]
fn test_ccsds() {
let ccsds_packet = CCSDSPacket {
packet_version: 0,
packet_type: true,
sec_header_flag: false,
apid: 0x2FF,
seq_flag: 3,
seq: 0,
packet_len: 5,
data: vec![0x01, 0x02, 0x03, 0x04, 0x05],
};
let bytes = ccsds_packet.to_bytes();
let parsed_packet = CCSDSPacket::from_bytes(&bytes);
assert_eq!(ccsds_packet, parsed_packet);
assert_eq!(bytes, parsed_packet.to_bytes())
}
fn get_ccsds_format() -> Format {
let format_config = FormatConfig::new(&None).unwrap();
format_config.get_format("ccsds").unwrap()
}
#[test]
fn test_ccsds_parse() {
let ccsds_packet = CCSDSPacket {
packet_version: 0,
packet_type: true,
sec_header_flag: false,
apid: 0x2FF,
seq_flag: 3,
seq: 0,
packet_len: 5,
data: vec![0x01, 0x02, 0x03, 0x04, 0x05],
};
let format = get_ccsds_format();
let data = ccsds_packet.to_bytes();
let output = format.format_data(&data).unwrap();
assert_eq!(ccsds_packet.print(), output)
}
#[test]
fn test_ccsds_parse_hammer() {
let format = get_ccsds_format();
for apid in 0..(2u16.pow(11)) {
let ccsds_packet = CCSDSPacket {
packet_version: 0,
packet_type: (apid % 2) == 0,
sec_header_flag: (apid % 2) == 1,
apid,
seq_flag: (apid % 4) as u8,
seq: (apid % 11),
packet_len: 5,
data: vec![0x01, 0x02, 0x03, 0x04, 0x05],
};
let data = ccsds_packet.to_bytes();
let output = format.format_data(&data).unwrap();
assert_eq!(ccsds_packet.print(), output)
}
}
}

View File

@ -33,14 +33,9 @@ fn init() -> Result<(Vec<u8>, Format), FormatyError> {
let config = FormatConfig::new(&args.config)?;
let format = config
.formats
.iter()
.find(|f| f.name == args.format)
.ok_or_else(|| FormatyError::FormatNotFound(args.format.to_string()))?
.clone();
let format = config.get_format(&args.format)?;
let data = parse_bytes_from_input_arg(args.data).unwrap();
let data = parse_bytes_from_input_arg(args.data)?;
Ok((data, format))
}