Added deframer support

Cargo.lock

@@ -66,9 +66,9 @@ dependencies = [
 
 [[package]]
 name = "bstr"
-version = "1.9.1"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05efc5cfd9110c8416e471df0e96702d58690178e206e61b7173706673c93706"
+checksum = "40723b8fb387abc38f4f4a37c09073622e41dd12327033091ef8950659e6dc0c"
 dependencies = [
  "memchr",
  "serde",
@@ -159,16 +159,18 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
 
 [[package]]
 name = "formaty"
-version = "0.2.0"
+version = "0.3.0"
 dependencies = [
  "bitvec",
  "byteorder",
  "hex",
  "num-bigint",
  "platform-dirs",
+ "regex",
  "rust-embed",
  "serde",
  "structopt",
+ "thiserror",
  "toml",
 ]
 
@@ -383,6 +385,18 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "regex"
+version = "1.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
 [[package]]
 name = "regex-automata"
 version = "0.4.6"
@@ -553,22 +567,22 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.31"
+version = "1.0.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a"
+checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.31"
+version = "1.0.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a"
+checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.75",
+ "syn 2.0.60",
 ]
 
 [[package]]

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "formaty"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -14,6 +14,8 @@ num-bigint = "0.4.4"
 bitvec = "1.0.1"
 platform-dirs = "0.3.0"
 hex = "0.4.3"
+thiserror = "1.0.63"
+regex = "1.10.6"
 
 [dependencies.rust-embed]
 version = "8.3.0"

README.md

@@ -5,14 +5,15 @@ A simple configurable binary data parser. Data structures are described using TO
 All formats in [formats](./formats) are included in the `formaty` binary. See [formats.md](./formats/formats.md) for
 more info.
 
-### Configuration
+## Configuration
 
-#### Format
+### Format
 A format is a collection of fields that describe the structure of data, so it can be parsed
 
 Members:
 * `name` - name of the format, used as the argument when selecting the format to parse the data as
 * `bit_flip` - should individual bytes have their bits flipped within, individual fields can override this
+* `deframer` - optional list of deframers to use before trying to parse the data
 * `fields` - 1 or more fields to describe the structure of the data
 
 #### Field
@@ -46,6 +47,15 @@ Members:
 * `String`: view fields as a string field
 * `Default`: use the default printer for the source type
 
+### Deframer
+A deframer configures how the data should be unpacked before being parsed. A deframer is made up of one or more
+DeframeOperations. Deframers can be added to `Format` configs or be applied by passing in the `--deframer` option.
+
+#### Deframe Operation
+* `StartSeq` - a start sequence of bytes to check for and remove before parsing
+* `StopSeq` - a stop sequence of bytes to check for and remove before parsing
+* `Replace` - a simple find and replace with Regex support. All matches of `find` are replaced with `replace`
+
 #### Example Config
 [Example](./formats/example.toml)
 
@@ -68,11 +78,11 @@ Data: [1, 2, 3, 4, 5]
 
 ## Help
 ```
-Formaty 0.2.0
+Formaty 0.3.0
 Arbitrary Binary Data Formatting
 
 USAGE:
-    formaty [FLAGS] [OPTIONS] <format> [data]...
+    formaty [FLAGS] [OPTIONS] <format> [--] [data]...
 
 FLAGS:
     -h, --help       Prints help information
@@ -82,6 +92,7 @@ FLAGS:
 OPTIONS:
     -b, --base <base>                      Base of the input values
     -c, --config <config>                  Path to the format config [env: FORMATY_CONFIG=]
+    -d, --deframer <deframer>...           Additional deframers to apply to the data before trying to parse it
     -g, --global_config <global-config>    Path to the global config directory [env: FORMATY_GLOBAL_CONFIG=]
     -f, --file <input-file>                Input data from file
     -i, --input_type <input-type>          Input data type [default: array]

formats/formats.md

@@ -4,5 +4,4 @@ at run time by using the `--config` flag.
 
 ## Formats Included
 * [CCSDS](ccsds.toml) - Standard CCSDS command/tlm packets
-  * `ccsds`: CCSDS packet with primary header + raw payload
+  * `ccsds`: CCSDS packet with primary header + raw payload
-  * `ccsds_sec`: CCSDS packet with primary header + secondary header + raw payload

formats/slip.toml

@@ -0,0 +1,23 @@
+# Serial Line Internet Protocol Deframmer
+# Ref: https://datatracker.ietf.org/doc/html/rfc1055
+
+[[deframers]]
+name = "slip"
+
+[[deframers.operations]]
+operation = "StartSeq"
+start_seq = [0xC0]
+
+[[deframers.operations]]
+operation = "StopSeq"
+stop_seq = [0xC0]
+
+[[deframers.operations]]
+operation = "Replace"
+find = "\\xDB\\xDC"
+replace = [0xC0]
+
+[[deframers.operations]]
+operation = "Replace"
+find = "\\xDB\\xDD"
+replace = [0xDB]

src/byte_stream/mod.rs

@@ -1,20 +1,13 @@
-use crate::formatter::format::ByteOrderOperations;
+use crate::formatter::bytes_operations::ByteOrderOperations;
 use bitvec::prelude::*;
-use std::fmt::{Display, Formatter};
+use thiserror::Error;
 
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Error)]
 pub enum ByteStreamError {
+    #[error("Requested value out of range")]
     OutOfRange,
 }
 
-impl Display for ByteStreamError {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        match self {
-            ByteStreamError::OutOfRange => write!(f, "Requested values out of range"),
-        }
-    }
-}
-
 pub const fn bit_mask(mask: u8) -> u8 {
     match mask {
         0 => 0x00,

src/deframer/mod.rs

@@ -0,0 +1,148 @@
+use regex::bytes::RegexBuilder;
+use serde::Deserialize;
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum DeframeError {
+    #[error("Start sequence not found")]
+    StartSeqInvalid,
+    #[error("Stop sequence not found")]
+    StopSeqInvalid,
+    #[error("Regex Error")]
+    RegexError(#[from] regex::Error),
+}
+
+#[derive(Debug, Deserialize, Clone)]
+#[serde(tag = "operation")]
+pub enum DeframeOperation {
+    StartSeq { start_seq: Vec<u8> },
+    StopSeq { stop_seq: Vec<u8> },
+    Replace { find: String, replace: Vec<u8> },
+}
+
+impl DeframeOperation {
+    pub fn do_operation(&self, mut data: Vec<u8>) -> Result<Vec<u8>, DeframeError> {
+        Ok(match self {
+            DeframeOperation::StartSeq { start_seq } => {
+                if data.starts_with(start_seq) {
+                    data.drain(0..start_seq.len());
+
+                    data
+                } else {
+                    return Err(DeframeError::StartSeqInvalid);
+                }
+            }
+            DeframeOperation::StopSeq { stop_seq } => {
+                if data.ends_with(stop_seq) {
+                    data.drain(data.len() - stop_seq.len()..data.len());
+
+                    data
+                } else {
+                    return Err(DeframeError::StopSeqInvalid);
+                }
+            }
+            DeframeOperation::Replace { find, replace } => {
+                let reg = RegexBuilder::new(find).unicode(false).build()?;
+
+                reg.replace_all(&data, replace).to_vec()
+            }
+        })
+    }
+}
+
+#[derive(Debug, Deserialize, Clone)]
+pub struct Deframer {
+    pub name: String,
+    pub operations: Vec<DeframeOperation>,
+}
+
+impl Deframer {
+    pub fn deframe(&self, mut data: Vec<u8>) -> Vec<u8> {
+        for operation in &self.operations {
+            data = operation.do_operation(data).unwrap();
+        }
+
+        data
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::deframer::{DeframeOperation, Deframer};
+    use crate::formatter::format_config::FormatConfig;
+
+    #[test]
+    fn test_start_seq() {
+        let op = DeframeOperation::StartSeq {
+            start_seq: vec![0x11, 0x00],
+        };
+
+        let data = vec![0x11, 0x00, 0x55, 0x55];
+        let output = op.do_operation(data).unwrap();
+
+        assert_eq!(output, vec![0x55, 0x55]);
+    }
+
+    #[test]
+    fn test_stop_seq() {
+        let op = DeframeOperation::StopSeq {
+            stop_seq: vec![0x11, 0x00],
+        };
+
+        let data = vec![0x55, 0x55, 0x11, 0x00];
+        let output = op.do_operation(data).unwrap();
+
+        assert_eq!(output, vec![0x55, 0x55]);
+    }
+
+    #[test]
+    fn test_replace_seq() {
+        let op = DeframeOperation::Replace {
+            find: "\x55\x11".to_string(),
+            replace: vec![0x11],
+        };
+
+        let data = vec![0x55, 0x11, 0x55, 0x11, 0x00, 0x55];
+        let output = op.do_operation(data).unwrap();
+
+        assert_eq!(output, vec![0x11, 0x11, 0x00, 0x55]);
+    }
+
+    #[test]
+    fn test_deframer() {
+        let deframer = Deframer {
+            name: "Test SLIP Deframmer".to_string(),
+            operations: vec![
+                DeframeOperation::StartSeq {
+                    start_seq: vec![0xc0],
+                },
+                DeframeOperation::StopSeq {
+                    stop_seq: vec![0xc0],
+                },
+                DeframeOperation::Replace {
+                    find: r"\xDB\xDC".to_string(),
+                    replace: vec![0xC0],
+                },
+                DeframeOperation::Replace {
+                    find: r"\xDB\xDD".to_string(),
+                    replace: vec![0xDB],
+                },
+            ],
+        };
+
+        let data = vec![0xC0, 0xDB, 0xDC, 0xDB, 0xDD, 0xC0];
+        let output = deframer.deframe(data);
+
+        assert_eq!(output, vec![0xc0, 0xDB]);
+    }
+
+    #[test]
+    fn deserialize_deframer() {
+        let config = FormatConfig::new(&None, &None).unwrap();
+        let slip = config.get_deframer("slip").unwrap();
+
+        assert_eq!(slip.name, "slip");
+
+        assert_eq!(slip.operations.len(), 4);
+    }
+}

src/formatter/bytes_operations.rs

@@ -0,0 +1,97 @@
+use byteorder::{BigEndian, ByteOrder, LittleEndian};
+use num_bigint::{BigInt, BigUint};
+
+#[allow(dead_code)]
+pub trait ByteOrderOperations: ByteOrder {
+    fn last_byte(buf: &mut Vec<u8>) -> Option<&mut u8>;
+
+    fn big_int(buf: &[u8], trim: usize) -> BigInt;
+
+    fn big_uint(buf: &[u8], trim: usize) -> BigUint;
+
+    fn big_u_int_to_bytes(big_int: BigUint) -> Vec<u8>;
+
+    fn big_int_to_bytes(big_int: BigInt) -> Vec<u8>;
+
+    fn pad_bytes(buf: &mut Vec<u8>, size: usize);
+
+    fn flip() -> bool;
+}
+
+impl ByteOrderOperations for BigEndian {
+    fn last_byte(buf: &mut Vec<u8>) -> Option<&mut u8> {
+        buf.first_mut()
+    }
+
+    fn big_int(buf: &[u8], trim: usize) -> BigInt {
+        let mut buf = buf.to_vec();
+        let byte = buf.first_mut().unwrap();
+        *byte = (*byte << trim) >> trim;
+        BigInt::from_signed_bytes_be(&buf)
+    }
+
+    fn big_uint(buf: &[u8], trim: usize) -> BigUint {
+        let mut buf = buf.to_vec();
+        let byte = buf.first_mut().unwrap();
+        *byte = (*byte << trim) >> trim;
+        BigUint::from_bytes_be(&buf)
+    }
+
+    fn big_u_int_to_bytes(big_int: BigUint) -> Vec<u8> {
+        big_int.to_bytes_be()
+    }
+
+    fn big_int_to_bytes(big_int: BigInt) -> Vec<u8> {
+        big_int.to_signed_bytes_be()
+    }
+
+    fn pad_bytes(buf: &mut Vec<u8>, size: usize) {
+        if size <= buf.len() {
+            return;
+        }
+
+        let pad_to = size - buf.len();
+        buf.splice(0..0, vec![0_u8; pad_to].iter().cloned());
+    }
+
+    fn flip() -> bool {
+        true
+    }
+}
+
+impl ByteOrderOperations for LittleEndian {
+    fn last_byte(buf: &mut Vec<u8>) -> Option<&mut u8> {
+        buf.last_mut()
+    }
+
+    fn big_int(buf: &[u8], trim: usize) -> BigInt {
+        BigInt::from_signed_bytes_le(buf) >> trim
+    }
+
+    fn big_uint(buf: &[u8], trim: usize) -> BigUint {
+        BigUint::from_bytes_le(buf) >> trim
+    }
+
+    fn big_u_int_to_bytes(big_int: BigUint) -> Vec<u8> {
+        big_int.to_bytes_le()
+    }
+
+    fn big_int_to_bytes(big_int: BigInt) -> Vec<u8> {
+        big_int.to_signed_bytes_le()
+    }
+
+    fn pad_bytes(buf: &mut Vec<u8>, size: usize) {
+        if size <= buf.len() {
+            return;
+        }
+
+        let pad_to = size - buf.len();
+
+        let mut pad = vec![0_u8; pad_to];
+        buf.append(&mut pad);
+    }
+
+    fn flip() -> bool {
+        false
+    }
+}

src/formatter/format.rs

@@ -1,142 +1,28 @@
-use std::fmt::{Display, Formatter, Write};
+use std::fmt::Write;
 use std::io::Cursor;
 use std::string::FromUtf8Error;
 
-use byteorder::{BigEndian, ByteOrder, LittleEndian, ReadBytesExt};
-use num_bigint::{BigInt, BigUint};
-use serde::Deserialize;
-
 use crate::byte_stream::{bit_mask, ByteStream, ByteStreamError};
+use crate::formatter::bytes_operations::ByteOrderOperations;
+use crate::formatter::format_config::FormatConfig;
 use crate::formatter::printers::PrintType;
-use crate::FormatConfig;
+use byteorder::{BigEndian, LittleEndian, ReadBytesExt};
+use serde::Deserialize;
+use thiserror::Error;
 
-#[allow(dead_code)]
+#[derive(Debug, Clone, Error)]
-pub trait ByteOrderOperations: ByteOrder {
-    fn last_byte(buf: &mut Vec<u8>) -> Option<&mut u8>;
-
-    fn big_int(buf: &[u8], trim: usize) -> BigInt;
-
-    fn big_uint(buf: &[u8], trim: usize) -> BigUint;
-
-    fn big_u_int_to_bytes(big_int: BigUint) -> Vec<u8>;
-
-    fn big_int_to_bytes(big_int: BigInt) -> Vec<u8>;
-
-    fn pad_bytes(buf: &mut Vec<u8>, size: usize);
-
-    fn flip() -> bool;
-}
-
-impl ByteOrderOperations for BigEndian {
-    fn last_byte(buf: &mut Vec<u8>) -> Option<&mut u8> {
-        buf.first_mut()
-    }
-
-    fn big_int(buf: &[u8], trim: usize) -> BigInt {
-        let mut buf = buf.to_vec();
-        let byte = buf.first_mut().unwrap();
-        *byte = (*byte << trim) >> trim;
-        BigInt::from_signed_bytes_be(&buf)
-    }
-
-    fn big_uint(buf: &[u8], trim: usize) -> BigUint {
-        let mut buf = buf.to_vec();
-        let byte = buf.first_mut().unwrap();
-        *byte = (*byte << trim) >> trim;
-        BigUint::from_bytes_be(&buf)
-    }
-
-    fn big_u_int_to_bytes(big_int: BigUint) -> Vec<u8> {
-        big_int.to_bytes_be()
-    }
-
-    fn big_int_to_bytes(big_int: BigInt) -> Vec<u8> {
-        big_int.to_signed_bytes_be()
-    }
-
-    fn pad_bytes(buf: &mut Vec<u8>, size: usize) {
-        if size <= buf.len() {
-            return;
-        }
-
-        let pad_to = size - buf.len();
-        buf.splice(0..0, vec![0_u8; pad_to].iter().cloned());
-    }
-
-    fn flip() -> bool {
-        true
-    }
-}
-
-impl ByteOrderOperations for LittleEndian {
-    fn last_byte(buf: &mut Vec<u8>) -> Option<&mut u8> {
-        buf.last_mut()
-    }
-
-    fn big_int(buf: &[u8], trim: usize) -> BigInt {
-        BigInt::from_signed_bytes_le(buf) >> trim
-    }
-
-    fn big_uint(buf: &[u8], trim: usize) -> BigUint {
-        BigUint::from_bytes_le(buf) >> trim
-    }
-
-    fn big_u_int_to_bytes(big_int: BigUint) -> Vec<u8> {
-        big_int.to_bytes_le()
-    }
-
-    fn big_int_to_bytes(big_int: BigInt) -> Vec<u8> {
-        big_int.to_signed_bytes_le()
-    }
-
-    fn pad_bytes(buf: &mut Vec<u8>, size: usize) {
-        if size <= buf.len() {
-            return;
-        }
-
-        let pad_to = size - buf.len();
-
-        let mut pad = vec![0_u8; pad_to];
-        buf.append(&mut pad);
-    }
-
-    fn flip() -> bool {
-        false
-    }
-}
-
-#[derive(Debug, Clone)]
 pub enum FormatError {
-    ByteSteamError(ByteStreamError),
+    #[error("Byte Stream Error")]
+    ByteSteamError(#[from] ByteStreamError),
     #[allow(dead_code)]
+    #[error("Not Supported")]
     NotSupported,
-    StringParseError(FromUtf8Error),
+    #[error("String Parse Error")]
+    StringParseError(#[from] FromUtf8Error),
+    #[error("Format Not Found")]
     FormatNotFound(String),
-}
+    #[error("Deframmer Not Found")]
-
+    DeframmerNotFound(String),
-impl From<ByteStreamError> for FormatError {
-    fn from(e: ByteStreamError) -> Self {
-        FormatError::ByteSteamError(e)
-    }
-}
-
-impl From<FromUtf8Error> for FormatError {
-    fn from(e: FromUtf8Error) -> Self {
-        FormatError::StringParseError(e)
-    }
-}
-
-impl Display for FormatError {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        match self {
-            FormatError::ByteSteamError(e) => writeln!(f, "Byte stream error: {}", e),
-            FormatError::NotSupported => write!(f, "Field type not supported"),
-            FormatError::StringParseError(e) => write!(f, "String parse error: {}", e),
-            FormatError::FormatNotFound(format) => {
-                write!(f, "Unable to find sub-format: {}", format)
-            }
-        }
-    }
 }
 
 #[derive(Debug, Deserialize, Clone)]
@@ -393,6 +279,9 @@ pub struct Format {
     pub name: String,
     /// Flip bits
     pub bit_flip: bool,
+    /// Deframmers to apply before parsing,
+    #[serde(default)]
+    pub deframers: Vec<String>,
     /// Elements of the format
     pub fields: Vec<Field>,
 }
@@ -429,7 +318,29 @@ impl Format {
         Ok((format_str, bit_ndx))
     }
 
-    pub fn format_data(&self, data: &[u8], config: &FormatConfig) -> Result<String, FormatError> {
+    pub fn format_data(
+        &self,
+        data: &[u8],
+        config: &FormatConfig,
+        additional_deframers: Option<Vec<String>>,
+    ) -> Result<String, FormatError> {
+        let mut data = data.to_vec();
+
+        let deframers = if let Some(additional_deframers) = additional_deframers {
+            additional_deframers
+                .into_iter()
+                .chain(self.deframers.clone())
+                .collect()
+        } else {
+            self.deframers.clone()
+        };
+
+        for deframer_name in deframers {
+            let deframer = config.get_deframer(&deframer_name)?;
+
+            data = deframer.deframe(data)
+        }
+
         let mut byte_stream = ByteStream::from(data);
 
         let (format_str, _) = self.format_byte_stream(&mut byte_stream, 0, config)?;
@@ -442,7 +353,8 @@ impl Format {
 mod tests {
     use crate::byte_stream::ByteStream;
     use crate::formatter::format::{Endianness, Field, FieldType};
-    use crate::{Format, FormatConfig};
+    use crate::formatter::format_config::FormatConfig;
+    use crate::Format;
 
     #[test]
     fn test_format_int_4_bits() {
@@ -725,9 +637,11 @@ mod tests {
         };
 
         let config = FormatConfig {
+            deframers: vec![],
             formats: vec![Format {
                 name: "sub_format".to_string(),
                 bit_flip: false,
+                deframers: vec![],
                 fields: vec![Field {
                     name: "test".to_string(),
                     field_type: FieldType::String {

src/formatter/format_config.rs

@@ -0,0 +1,132 @@
+use crate::deframer::Deframer;
+use crate::formatter::format::{Format, FormatError};
+use crate::formatter::FormatConfigError;
+use platform_dirs::AppDirs;
+use rust_embed::RustEmbed;
+use serde::Deserialize;
+use std::fs::{read_dir, File};
+use std::io::Read;
+use std::path::PathBuf;
+use std::{fs, str};
+
+#[derive(RustEmbed)]
+#[folder = "formats/"]
+#[include = "*.toml"]
+struct BuiltInFormats;
+
+#[derive(Debug, Deserialize, Clone, Default)]
+pub struct FormatConfig {
+    #[serde(default)]
+    pub deframers: Vec<Deframer>,
+    #[serde(default)]
+    pub formats: Vec<Format>,
+}
+
+impl FormatConfig {
+    fn parse_config_file(
+        src_file_name: &str,
+        config_data: &str,
+    ) -> Result<Self, FormatConfigError> {
+        toml::from_str(config_data).map_err(|e| FormatConfigError::TomlError {
+            file_name: src_file_name.to_string(),
+            err: e,
+        })
+    }
+
+    fn get_built_in_config(&mut self) -> Result<(), FormatConfigError> {
+        for format_file_path in BuiltInFormats::iter() {
+            if format_file_path.ends_with("md") {
+                continue;
+            }
+
+            let format_file = BuiltInFormats::get(&format_file_path).unwrap();
+
+            let config_str = str::from_utf8(&format_file.data).unwrap();
+
+            let mut built_in: FormatConfig =
+                Self::parse_config_file(&format_file_path, config_str)?;
+
+            self.formats.append(&mut built_in.formats);
+            self.deframers.append(&mut built_in.deframers)
+        }
+
+        Ok(())
+    }
+
+    fn get_file_config(&mut self, config_path: &Option<PathBuf>) -> Result<(), FormatConfigError> {
+        if let Some(config_path) = config_path {
+            let mut config_file = File::open(config_path)?;
+            let mut config_data = String::new();
+            config_file.read_to_string(&mut config_data)?;
+            let mut arg_config =
+                Self::parse_config_file(config_path.to_str().unwrap(), &config_data)?;
+
+            self.formats.append(&mut arg_config.formats);
+            self.deframers.append(&mut arg_config.deframers);
+        }
+
+        Ok(())
+    }
+
+    fn parse_directory(
+        &mut self,
+        directory_path: &Option<PathBuf>,
+    ) -> Result<(), FormatConfigError> {
+        if let Some(directory_path) = directory_path {
+            for path in read_dir(directory_path)? {
+                let file_name = path?.file_name();
+                let config_file_path = directory_path.join(file_name);
+
+                self.get_file_config(&Some(config_file_path))?;
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn new(
+        config_path: &Option<PathBuf>,
+        global_config_path: &Option<PathBuf>,
+    ) -> Result<Self, FormatConfigError> {
+        let mut config = FormatConfig::default();
+
+        config.get_built_in_config()?;
+
+        config.get_file_config(config_path)?;
+
+        let global_dir = match global_config_path {
+            Some(g) => g.clone(),
+            None => {
+                let app_dirs = AppDirs::new(Some("formaty"), true).unwrap();
+
+                if !app_dirs.config_dir.exists() {
+                    fs::create_dir(&app_dirs.config_dir)?;
+                }
+
+                app_dirs.config_dir
+            }
+        };
+
+        config.parse_directory(&Some(global_dir))?;
+
+        Ok(config)
+    }
+
+    pub fn get_format(&self, name: &str) -> Result<Format, FormatError> {
+        Ok(self
+            .formats
+            .iter()
+            .find(|f| f.name.as_str() == name)
+            .ok_or_else(|| FormatError::FormatNotFound(name.to_string()))?
+            .clone())
+    }
+
+    pub fn get_deframer(&self, name: &str) -> Result<Deframer, FormatError> {
+        Ok(self
+            .deframers
+            .iter()
+            .find(|f| f.name.as_str() == name)
+            .ok_or_else(|| FormatError::DeframmerNotFound(name.to_string()))?
+            .clone())
+    }
+}

src/formatter/mod.rs

@@ -1,171 +1,33 @@
+pub mod bytes_operations;
 pub mod format;
+pub mod format_config;
 mod printers;
 
-use crate::formatter::format::Format;
+use crate::formatter::format::FormatError;
-use platform_dirs::AppDirs;
+use std::str;
-use rust_embed::RustEmbed;
+use thiserror::Error;
-use serde::Deserialize;
-use std::error::Error;
-use std::fmt::{Display, Formatter};
-use std::fs::{read_dir, File};
-use std::io::Read;
-use std::path::PathBuf;
-use std::{fs, str};
 
-#[derive(Debug)]
+#[derive(Debug, Error)]
 #[allow(clippy::enum_variant_names)]
 pub enum FormatConfigError {
-    IOError(std::io::Error),
+    #[error("IO error")]
+    IOError(#[from] std::io::Error),
+    #[error("TOML Format Error")]
     TomlError {
         file_name: String,
         err: toml::de::Error,
     },
-    Utf8Error(str::Utf8Error),
+    #[error("UTF-8 String Decode Error")]
-    FormatNotFound(String),
+    Utf8Error(#[from] str::Utf8Error),
-}
+    #[error("Format Error")]
-
+    FormatError(#[from] FormatError),
-impl Error for FormatConfigError {}
-
-impl From<std::io::Error> for FormatConfigError {
-    fn from(e: std::io::Error) -> Self {
-        Self::IOError(e)
-    }
-}
-
-impl From<str::Utf8Error> for FormatConfigError {
-    fn from(e: str::Utf8Error) -> Self {
-        Self::Utf8Error(e)
-    }
-}
-
-impl Display for FormatConfigError {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        let err_msg = match self {
-            FormatConfigError::IOError(e) => e.to_string(),
-            FormatConfigError::TomlError { file_name, err } => {
-                format!("Error parsing {}: {}", file_name, err)
-            }
-            FormatConfigError::Utf8Error(e) => e.to_string(),
-            FormatConfigError::FormatNotFound(s) => format!("{} was not found.", s),
-        };
-
-        write!(f, "Format Config Error: {}", err_msg)
-    }
-}
-
-#[derive(RustEmbed)]
-#[folder = "formats/"]
-#[include = "*.toml"]
-struct BuiltInFormats;
-
-#[derive(Debug, Deserialize, Clone, Default)]
-pub struct FormatConfig {
-    pub formats: Vec<Format>,
-}
-
-impl FormatConfig {
-    fn parse_config_file(
-        src_file_name: &str,
-        config_data: &str,
-    ) -> Result<Self, FormatConfigError> {
-        toml::from_str(config_data).map_err(|e| FormatConfigError::TomlError {
-            file_name: src_file_name.to_string(),
-            err: e,
-        })
-    }
-
-    fn get_built_in_config(&mut self) -> Result<(), FormatConfigError> {
-        for format_file_path in BuiltInFormats::iter() {
-            if format_file_path.ends_with("md") {
-                continue;
-            }
-
-            let format_file = BuiltInFormats::get(&format_file_path).unwrap();
-
-            let config_str = str::from_utf8(&format_file.data).unwrap();
-
-            let mut built_in: FormatConfig =
-                Self::parse_config_file(&format_file_path, config_str)?;
-
-            self.formats.append(&mut built_in.formats);
-        }
-
-        Ok(())
-    }
-
-    fn get_file_config(&mut self, config_path: &Option<PathBuf>) -> Result<(), FormatConfigError> {
-        if let Some(config_path) = config_path {
-            let mut config_file = File::open(config_path)?;
-            let mut config_data = String::new();
-            config_file.read_to_string(&mut config_data)?;
-            let mut arg_config =
-                Self::parse_config_file(config_path.to_str().unwrap(), &config_data)?;
-
-            self.formats.append(&mut arg_config.formats);
-        }
-
-        Ok(())
-    }
-
-    fn parse_directory(
-        &mut self,
-        directory_path: &Option<PathBuf>,
-    ) -> Result<(), FormatConfigError> {
-        if let Some(directory_path) = directory_path {
-            for path in read_dir(directory_path)? {
-                let file_name = path?.file_name();
-                let config_file_path = directory_path.join(file_name);
-
-                self.get_file_config(&Some(config_file_path))?;
-            }
-        }
-
-        Ok(())
-    }
-
-    pub fn new(
-        config_path: &Option<PathBuf>,
-        global_config_path: &Option<PathBuf>,
-    ) -> Result<Self, FormatConfigError> {
-        let mut config = FormatConfig::default();
-
-        config.get_built_in_config()?;
-
-        config.get_file_config(config_path)?;
-
-        let global_dir = match global_config_path {
-            Some(g) => g.clone(),
-            None => {
-                let app_dirs = AppDirs::new(Some("formaty"), true).unwrap();
-
-                if !app_dirs.config_dir.exists() {
-                    fs::create_dir(&app_dirs.config_dir)?;
-                }
-
-                app_dirs.config_dir
-            }
-        };
-
-        config.parse_directory(&Some(global_dir))?;
-
-        Ok(config)
-    }
-
-    pub fn get_format(&self, name: &str) -> Result<Format, FormatConfigError> {
-        Ok(self
-            .formats
-            .iter()
-            .find(|f| f.name.as_str() == name)
-            .ok_or_else(|| FormatConfigError::FormatNotFound(name.to_string()))?
-            .clone())
-    }
 }
 
 #[cfg(test)]
 mod test {
     use crate::formatter::format::Format;
+    use crate::formatter::format_config::FormatConfig;
     use crate::formatter::printers::print_bytes_as_array;
-    use crate::formatter::FormatConfig;
 
     #[derive(PartialOrd, PartialEq, Debug, Clone)]
     struct CCSDSPacket {
@@ -286,7 +148,9 @@ mod test {
 
         let format = get_ccsds_format();
         let data = ccsds_packet.to_bytes();
-        let output = format.format_data(&data, &FormatConfig::default()).unwrap();
+        let output = format
+            .format_data(&data, &FormatConfig::default(), None)
+            .unwrap();
 
         assert_eq!(ccsds_packet.print(), output)
     }
@@ -308,7 +172,9 @@ mod test {
             };
 
             let data = ccsds_packet.to_bytes();
-            let output = format.format_data(&data, &FormatConfig::default()).unwrap();
+            let output = format
+                .format_data(&data, &FormatConfig::default(), None)
+                .unwrap();
 
             assert_eq!(ccsds_packet.print(), output)
         }

src/formatter/printers.rs

@@ -2,7 +2,7 @@ use byteorder::WriteBytesExt;
 use num_bigint::{BigInt, BigUint};
 use serde::Deserialize;
 
-use crate::formatter::format::ByteOrderOperations;
+use crate::formatter::bytes_operations::ByteOrderOperations;
 
 pub fn print_bytes_as_array(data: &[u8]) -> String {
     format!("{:?}", data)

src/main.rs

@@ -1,11 +1,12 @@
 use crate::error::FormatyError;
 use crate::formatter::format::Format;
 use crate::parser::InputTypes;
-use formatter::FormatConfig;
+use formatter::format_config::FormatConfig;
 use std::path::PathBuf;
 use structopt::StructOpt;
 
 mod byte_stream;
+mod deframer;
 mod error;
 mod formatter;
 mod parser;
@@ -48,6 +49,13 @@ pub struct Args {
     #[structopt(help = "Input data from stdin", short = "s", long = "stdin")]
     input_stdin: bool,
 
+    #[structopt(
+        help = "Additional deframers to apply to the data before trying to parse it",
+        short = "d",
+        long = "deframer"
+    )]
+    deframer: Option<Vec<String>>,
+
     #[structopt(help = "Format to parse data as")]
     format: String,
 
@@ -55,26 +63,26 @@ pub struct Args {
     data: Vec<String>,
 }
 
-fn init() -> Result<(Vec<u8>, Format, FormatConfig), FormatyError> {
+fn init(args: &Args) -> Result<(Vec<u8>, Format, FormatConfig), FormatyError> {
-    let args: Args = Args::from_args();
-
     let config = FormatConfig::new(&args.config, &args.global_config)?;
 
     let format = config.get_format(&args.format)?;
 
     let data = if args.input_stdin {
         args.input_type.parse_stdin(args.base)?
-    } else if let Some(input_file) = args.input_file {
+    } else if let Some(input_file) = &args.input_file {
-        args.input_type.parse_file(&input_file, args.base)?
+        args.input_type.parse_file(input_file, args.base)?
     } else {
-        args.input_type.parse_arg_input(args.data, args.base)?
+        args.input_type
+            .parse_arg_input(args.data.clone(), args.base)?
     };
 
     Ok((data, format, config))
 }
 
 fn main() {
-    let (data, format, config) = match init() {
+    let args: Args = Args::from_args();
+    let (data, format, config) = match init(&args) {
         Ok((data, format, config)) => (data, format, config),
         Err(e) => {
             println!("Error initializing: {}", e);
@@ -82,8 +90,8 @@ fn main() {
         }
     };
 
-    match format.format_data(&data, &config) {
+    match format.format_data(&data, &config, args.deframer) {
         Ok(data) => println!("{}", data),
-        Err(_) => print!("Unable to parse data"),
+        Err(err) => print!("Unable to parse data: {}", err),
     }
 }