diff --git a/Cargo.lock b/Cargo.lock index dc47dcc..f276db3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,7 @@ dependencies = [ "config", "discord-hooks-rs", "flate2", + "ftp", "log", "regex", "reqwest", @@ -160,13 +161,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.11" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ + "libc", "num-integer", "num-traits 0.2.11", "time", + "winapi 0.3.8", ] [[package]] @@ -332,6 +335,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "ftp" +version = "3.0.1" +source = "git+https://github.com/joeyahines/rust-ftp.git#ab44662b5f27d18116a2721c1ddf76e117ed88be" +dependencies = [ + "chrono", + "lazy_static 1.4.0", + "native-tls", + "openssl", + "regex", +] + [[package]] name = "fuchsia-zircon" version = "0.3.3" @@ -829,12 +844,12 @@ checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" [[package]] name = "openssl" -version = "0.10.29" +version = "0.10.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee6d85f4cb4c4f59a6a85d5b68a233d280c82e29e822913b9c8b129fbf20bdd" +checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70" dependencies = [ "bitflags", - "cfg-if 0.1.10", + "cfg-if 1.0.0", "foreign-types", "lazy_static 1.4.0", "libc", @@ -849,9 +864,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-sys" -version = "0.9.58" +version = "0.9.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" +checksum = "921fc71883267538946025deffb622905ecad223c28efbfdef9bb59a0175f3e6" dependencies = [ "autocfg", "cc", @@ -1031,9 +1046,9 @@ checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" [[package]] name = "regex" -version = "1.3.9" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" dependencies = [ "aho-corasick", "memchr", @@ -1043,9 +1058,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.18" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" [[package]] name = "remove_dir_all" diff --git a/Cargo.toml b/Cargo.toml index 09e3b90..64e38f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,4 @@ reqwest = { version = "0.10", features = ["blocking", "json"] } discord-hooks-rs = { git = "https://github.com/joeyahines/discord-hooks-rs" } anvil-region = "0.4.0" ssh2 = "0.9.1" +ftp = { git = "https://github.com/joeyahines/rust-ftp.git" , features=["secure", "native-tls"]} diff --git a/src/backup.rs b/src/backup.rs index 321721e..271ce2c 100644 --- a/src/backup.rs +++ b/src/backup.rs @@ -1,17 +1,18 @@ use crate::backup; -use crate::config::{AlbatrossConfig, WorldConfig, WorldType}; +use crate::config::{AlbatrossConfig, RemoteBackupConfig, WorldConfig, WorldType}; use crate::discord::send_webhook; use crate::error::Result; use crate::region::Region; -use crate::remote_backup::remote_backup; -use chrono::{NaiveDateTime, Utc}; +use crate::remote::file::FileBackup; +use crate::remote::ftps::FTPSBackup; +use crate::remote::sftp::SFTPBackup; +use crate::remote::RemoteBackupSite; +use chrono::Utc; use flate2::read::GzDecoder; use flate2::write::GzEncoder; use flate2::Compression; use std::convert::TryFrom; -use std::fs::{ - copy, create_dir, create_dir_all, remove_dir_all, remove_file, rename, DirEntry, File, -}; +use std::fs::{copy, create_dir, create_dir_all, remove_dir_all, rename, File}; use std::path::PathBuf; use std::time::Instant; use tar::Archive; @@ -248,60 +249,19 @@ pub fn convert_backup_to_sp( Ok(()) } -pub fn get_time_from_file_name(file_name: &str) -> Result { - let time: Vec<&str> = file_name.split("_backup.tar.gz").collect(); - Ok(chrono::NaiveDateTime::parse_from_str( - time[0], - "%d-%m-%y_%H.%M.%S", - )?) -} - -/// Get the time of the backup from a directory entry -/// -/// # Param -/// * `archive_entry`: archive entry -fn get_time_from_dir_entry(archive_entry: &DirEntry) -> Result { - let file_name = archive_entry.file_name().to_str().unwrap().to_string(); - - get_time_from_file_name(file_name.as_str()) -} - -/// Removes the old backups from the ouput directory -/// -/// # Params -/// * `output_dir` - output directory containing -/// * `keep` - number of backups to keep -fn remove_old_backups(output_dir: &PathBuf, keep: u64) -> Result { - let mut backups = vec![]; - let mut num_of_removed_backups: usize = 0; - - for entry in output_dir.read_dir()? { - let entry = entry?; - - if let Some(ext) = entry.path().extension() { - if ext == "gz" { - backups.push(entry); - } - } +/// Preform a remote backup, if configured +pub fn do_remote_backup( + remote_backup_cfg: &RemoteBackupConfig, + backup_path: PathBuf, +) -> Result<()> { + if let Ok(mut sftp_backup) = SFTPBackup::new(remote_backup_cfg) { + sftp_backup.backup_to_remote(backup_path)?; + sftp_backup.cleanup()?; + } else if let Ok(mut ftps_backup) = FTPSBackup::new(remote_backup_cfg) { + ftps_backup.backup_to_remote(backup_path)?; + ftps_backup.cleanup()?; } - - if backups.len() > keep as usize { - backups.sort_by(|a, b| { - let a_time = get_time_from_dir_entry(a).unwrap(); - let b_time = get_time_from_dir_entry(b).unwrap(); - - b_time.cmp(&a_time) - }); - - num_of_removed_backups = backups.len() - keep as usize; - - for _i in 0..num_of_removed_backups { - let oldest = backups.pop().unwrap(); - remove_file(oldest.path())?; - } - } - - Ok(num_of_removed_backups) + Ok(()) } /// Backup the configured worlds from a minecraft server @@ -342,7 +302,9 @@ pub fn do_backup(cfg: AlbatrossConfig, output: Option) -> Result<()> { remove_dir_all(&tmp_dir)?; - match remove_old_backups(&cfg.backup.output_dir, cfg.backup.backups_to_keep) { + let mut local_backup = FileBackup::new(&cfg.backup).unwrap(); + + match local_backup.cleanup() { Ok(backups_removed) => { if backups_removed > 0 { let msg = format!( @@ -358,8 +320,8 @@ pub fn do_backup(cfg: AlbatrossConfig, output: Option) -> Result<()> { } } - if let Some(remote_backup_cfg) = &cfg.remote { - match remote_backup(output_archive, remote_backup_cfg) { + if let Some(remote_backup_config) = &cfg.remote { + match do_remote_backup(remote_backup_config, output_archive) { Ok(_) => { send_webhook("Remote backup completed!", &cfg); } diff --git a/src/config.rs b/src/config/mod.rs similarity index 78% rename from src/config.rs rename to src/config/mod.rs index 17c638d..7db52bb 100644 --- a/src/config.rs +++ b/src/config/mod.rs @@ -1,3 +1,6 @@ +mod remote; + +use crate::config::remote::{FTPSConfig, SFTPConfig}; use config::{Config, ConfigError, File}; use serde::Deserialize; use std::path::PathBuf; @@ -44,20 +47,9 @@ pub struct BackupConfig { /// Config for remote backups #[derive(Debug, Deserialize, Clone)] pub struct RemoteBackupConfig { - /// Remote server address - pub sftp_server_addr: String, - /// Remote output directory - pub remote_dir: PathBuf, - /// Remote server username - pub username: String, - /// Public key for key auth - pub public_key: Option, - /// Private key for key auth - pub private_key: Option, - /// Password if using password auth - pub password: Option, - /// Remote backups to keep pub backups_to_keep: u64, + pub sftp: Option, + pub ftps: Option, } /// Configs diff --git a/src/config/remote.rs b/src/config/remote.rs new file mode 100644 index 0000000..be825ab --- /dev/null +++ b/src/config/remote.rs @@ -0,0 +1,34 @@ +use serde::Deserialize; +use std::path::PathBuf; + +/// SFTP Config +#[derive(Debug, Deserialize, Clone)] +pub struct SFTPConfig { + /// Remote server address + pub server_addr: String, + /// Remote output directory + pub remote_dir: PathBuf, + /// Remote server username + pub username: String, + /// Public key for key auth + pub public_key: Option, + /// Private key for key auth + pub private_key: Option, + /// Password if using password auth + pub password: Option, +} + +/// FTPS Config +#[derive(Debug, Deserialize, Clone)] +pub struct FTPSConfig { + /// Remote server address + pub server_addr: String, + /// Remote output directory + pub remote_dir: PathBuf, + /// Remote server username + pub username: String, + /// Password + pub password: String, + /// Domain + pub domain: Option, +} diff --git a/src/error.rs b/src/error.rs index 95687bc..88483b3 100644 --- a/src/error.rs +++ b/src/error.rs @@ -10,6 +10,8 @@ pub enum AlbatrossError { RegionParseError(crate::region::RegionParseError), ChronoParseError(chrono::ParseError), NoSSHAuth, + RemoteConfigError(String), + FTPSError(ftp::FtpError), } impl std::error::Error for AlbatrossError {} @@ -25,6 +27,8 @@ impl std::fmt::Display for AlbatrossError { AlbatrossError::RegionParseError(e) => write!(f, "Unable to parse region name: {}", e), AlbatrossError::ChronoParseError(e) => write!(f, "Unable to parse time: {}", e), AlbatrossError::NoSSHAuth => write!(f, "No SSH auth methods provided in the config"), + AlbatrossError::RemoteConfigError(e) => write!(f, "Invalid configuration for {}", e), + AlbatrossError::FTPSError(e) => write!(f, "FTPS error: {}", e), } } } @@ -58,3 +62,9 @@ impl From for AlbatrossError { AlbatrossError::ChronoParseError(e) } } + +impl From for AlbatrossError { + fn from(e: ftp::FtpError) -> Self { + AlbatrossError::FTPSError(e) + } +} diff --git a/src/main.rs b/src/main.rs index a8303ec..0fdbd58 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,7 +7,7 @@ mod config; mod discord; mod error; mod region; -mod remote_backup; +mod remote; mod restore; use crate::backup::{convert_backup_to_sp, do_backup}; @@ -66,7 +66,7 @@ fn main() { let opt = Albatross::from_args(); let cfg = AlbatrossConfig::new(opt.config_path.into_os_string().to_str().unwrap()) - .expect("Config not found"); + .expect("Config error"); if cfg.world_config.is_some() { match opt.sub_command { diff --git a/src/remote/file.rs b/src/remote/file.rs new file mode 100644 index 0000000..7348073 --- /dev/null +++ b/src/remote/file.rs @@ -0,0 +1,47 @@ +use crate::config::BackupConfig; +use crate::error::Result; +use crate::remote::{PathLocation, RemoteBackupSite}; +use std::path::PathBuf; + +pub struct FileBackup { + /// Target directory on the file system + target_dir: PathBuf, + /// Number of backups to keep + backups_to_keep: usize, +} + +impl FileBackup { + /// New FileBackup + pub fn new(config: &BackupConfig) -> Result { + Ok(Self { + target_dir: config.output_dir.clone(), + backups_to_keep: config.backups_to_keep as usize, + }) + } +} + +impl RemoteBackupSite for FileBackup { + type FileType = PathLocation; + + fn backup_to_remote(&mut self, file: PathBuf) -> Result<()> { + let dest = self.target_dir.join(file.file_name().unwrap()); + std::fs::copy(file, dest)?; + Ok(()) + } + + fn get_backups(&mut self) -> Result> { + Ok(self + .target_dir + .read_dir()? + .filter_map(|file| Self::FileType::new(file.unwrap().path())) + .collect()) + } + + fn remove_backup(&mut self, backup: Self::FileType) -> Result<()> { + Ok(std::fs::remove_file(backup.location)?) + } + + fn backups_to_keep(&self) -> usize { + self.backups_to_keep + } +} diff --git a/src/remote/ftps.rs b/src/remote/ftps.rs new file mode 100644 index 0000000..5de5c86 --- /dev/null +++ b/src/remote/ftps.rs @@ -0,0 +1,79 @@ +use ftp::native_tls::TlsConnector; +use ftp::FtpStream; +use std::path::PathBuf; + +use crate::config::RemoteBackupConfig; +use crate::error; +use crate::error::AlbatrossError; +use crate::remote::{PathLocation, RemoteBackupSite}; + +/// FTPS Remote Site +pub struct FTPSBackup { + /// FTP command stream + stream: FtpStream, + /// Remote target directory + target_dir: PathBuf, + /// Number of backups to keep + backups_to_keep: usize, +} + +impl FTPSBackup { + /// New FTPSBackup + pub fn new(config: &RemoteBackupConfig) -> error::Result { + let ftps_config = config + .ftps + .as_ref() + .ok_or_else(|| AlbatrossError::RemoteConfigError("FTPS".to_string()))?; + + let ctx = TlsConnector::new().unwrap(); + let (ftp_stream, _) = FtpStream::connect(&ftps_config.server_addr)?; + + let domain = ftps_config + .domain + .as_ref() + .map_or_else(|| "".to_string(), |s| s.clone()); + let mut ftp_stream = ftp_stream.into_secure(ctx, domain.as_str())?; + ftp_stream.login(&ftps_config.username, &ftps_config.password)?; + + Ok(Self { + stream: ftp_stream, + target_dir: ftps_config.remote_dir.clone(), + backups_to_keep: config.backups_to_keep as usize, + }) + } +} + +impl Drop for FTPSBackup { + fn drop(&mut self) { + self.stream.quit().ok(); + } +} + +impl RemoteBackupSite for FTPSBackup { + type FileType = PathLocation; + + fn backup_to_remote(&mut self, file: PathBuf) -> error::Result<()> { + let mut local_file = std::fs::File::open(&file)?; + let location = self.target_dir.join(file); + self.stream + .put(location.to_str().unwrap(), &mut local_file)?; + + Ok(()) + } + + fn get_backups(&mut self) -> error::Result> { + let files = self.stream.list(Some(self.target_dir.to_str().unwrap()))?; + Ok(files + .into_iter() + .filter_map(|file| Self::FileType::new(PathBuf::from(file))) + .collect()) + } + + fn remove_backup(&mut self, backup: Self::FileType) -> error::Result<()> { + Ok(self.stream.rm(backup.location.to_str().unwrap())?) + } + + fn backups_to_keep(&self) -> usize { + self.backups_to_keep + } +} diff --git a/src/remote/mod.rs b/src/remote/mod.rs new file mode 100644 index 0000000..bd3d897 --- /dev/null +++ b/src/remote/mod.rs @@ -0,0 +1,107 @@ +use std::path::PathBuf; + +use chrono::NaiveDateTime; + +use crate::error::Result; + +pub mod file; +pub mod ftps; +pub mod sftp; + +pub trait RemoteBackupFile { + /// Type containing the location of the remote backup + type LocationType; + + /// Get the underlying location type + fn location(&self) -> Self::LocationType; + + /// Get the time the remote file was created + fn time_created(&self) -> chrono::NaiveDateTime; + + /// Parse the time created from the file name + fn parse_file_name(file_name: &str) -> Option { + let time: Vec<&str> = file_name.split("_backup.tar.gz").collect(); + + if let Some(time_str) = time.get(0) { + chrono::NaiveDateTime::parse_from_str(time_str, "%d-%m-%y_%H.%M.%S").ok() + } else { + None + } + } +} + +pub trait RemoteBackupSite { + /// Struct representing the location of a backup on the site + type FileType: RemoteBackupFile; + + /// Backup a file to the the remote site + fn backup_to_remote(&mut self, file: PathBuf) -> Result<()>; + + /// Get the locations backups contained on the remote site + fn get_backups(&mut self) -> Result>; + + /// Remove a backup from the side + fn remove_backup(&mut self, backup: Self::FileType) -> Result<()>; + + /// Number of backups to keep on the site + fn backups_to_keep(&self) -> usize; + + /// Cleanup old backups on the remote site + fn cleanup(&mut self) -> Result { + let mut backups = self.get_backups()?; + + backups.sort_by_key(|backup| backup.time_created()); + + let mut backups: Vec = backups.into_iter().rev().collect(); + + let mut removed_count: usize = 0; + if backups.len() > self.backups_to_keep() { + for _ in 0..(backups.len() - self.backups_to_keep()) { + if let Some(backup) = backups.pop() { + self.remove_backup(backup)?; + removed_count += 1; + } + } + } + + Ok(removed_count) + } +} + +/// Backup location that can be represented by a path +pub struct PathLocation { + location: PathBuf, + time_created: NaiveDateTime, +} + +impl PathLocation { + /// New PathLocation + fn new(path: PathBuf) -> Option { + if let Some(file_name) = path.file_name() { + let file_name = file_name.to_str().unwrap(); + + if let Some(time) = Self::parse_file_name(file_name) { + Some(Self { + location: path, + time_created: time, + }) + } else { + None + } + } else { + None + } + } +} + +impl RemoteBackupFile for PathLocation { + type LocationType = PathBuf; + + fn location(&self) -> Self::LocationType { + self.location.to_path_buf() + } + + fn time_created(&self) -> NaiveDateTime { + self.time_created + } +} diff --git a/src/remote/sftp.rs b/src/remote/sftp.rs new file mode 100644 index 0000000..6b6b116 --- /dev/null +++ b/src/remote/sftp.rs @@ -0,0 +1,83 @@ +use std::net::TcpStream; +use std::path::PathBuf; + +use ssh2::Session; + +use crate::config::RemoteBackupConfig; +use crate::error; +use crate::error::AlbatrossError; +use crate::remote::{PathLocation, RemoteBackupSite}; + +/// SFTP Remote Site +pub struct SFTPBackup { + /// SSH Session + session: Session, + /// Remote target directory + target_dir: PathBuf, + /// Number of backups to keep + backups_to_keep: usize, +} + +impl SFTPBackup { + /// New SFTPBackup + pub fn new(config: &RemoteBackupConfig) -> error::Result { + let sftp_config = config + .sftp + .as_ref() + .ok_or_else(|| AlbatrossError::RemoteConfigError("SFTP".to_string()))?; + + let tcp = TcpStream::connect(&sftp_config.server_addr)?; + let mut sess = Session::new()?; + sess.set_tcp_stream(tcp); + sess.handshake().unwrap(); + + if let Some(password) = &sftp_config.password { + sess.userauth_password(&sftp_config.username, password)?; + } else if let Some(key) = &sftp_config.private_key { + let public_key = sftp_config.public_key.as_deref(); + sess.userauth_pubkey_file(&sftp_config.username, public_key, key, None)?; + } else { + return Err(AlbatrossError::NoSSHAuth); + } + + Ok(Self { + session: sess, + target_dir: sftp_config.remote_dir.clone(), + backups_to_keep: config.backups_to_keep as usize, + }) + } +} + +impl RemoteBackupSite for SFTPBackup { + type FileType = PathLocation; + + fn backup_to_remote(&mut self, file: PathBuf) -> error::Result<()> { + let remote_path = self.target_dir.join(file.file_name().unwrap()); + + let mut local_file = std::fs::File::open(&file)?; + + let sftp = self.session.sftp()?; + + let mut remote_file = sftp.create(&remote_path)?; + + std::io::copy(&mut local_file, &mut remote_file)?; + + Ok(()) + } + + fn get_backups(&mut self) -> error::Result> { + let files = self.session.sftp()?.readdir(&self.target_dir)?; + Ok(files + .into_iter() + .filter_map(|(file, _)| Self::FileType::new(file)) + .collect()) + } + + fn remove_backup(&mut self, backup: Self::FileType) -> error::Result<()> { + Ok(self.session.sftp()?.unlink(&*backup.location)?) + } + + fn backups_to_keep(&self) -> usize { + self.backups_to_keep + } +} diff --git a/src/remote_backup.rs b/src/remote_backup.rs deleted file mode 100644 index 33c524f..0000000 --- a/src/remote_backup.rs +++ /dev/null @@ -1,73 +0,0 @@ -use crate::backup::get_time_from_file_name; -use crate::config::RemoteBackupConfig; -use crate::error::{AlbatrossError, Result}; -use ssh2::Session; -use std::net::TcpStream; -use std::path::PathBuf; - -/// Open an SSH session -fn open_ssh_session(remote_config: &RemoteBackupConfig) -> Result { - let tcp = TcpStream::connect(&remote_config.sftp_server_addr)?; - let mut sess = Session::new()?; - sess.set_tcp_stream(tcp); - sess.handshake().unwrap(); - - if let Some(password) = &remote_config.password { - sess.userauth_password(&remote_config.username, password)?; - } else if let Some(key) = &remote_config.private_key { - let public_key = remote_config.public_key.as_deref(); - sess.userauth_pubkey_file(&remote_config.username, public_key, key, None)?; - } else { - return Err(AlbatrossError::NoSSHAuth); - } - - Ok(sess) -} - -/// Handle remote backup of a file -pub fn remote_backup(file: PathBuf, remote_config: &RemoteBackupConfig) -> Result<()> { - let sess = open_ssh_session(remote_config)?; - - let remote_path = remote_config.remote_dir.join(file.file_name().unwrap()); - - let mut local_file = std::fs::File::open(&file)?; - - let sftp = sess.sftp()?; - - let mut remote_file = sftp.create(&remote_path)?; - - std::io::copy(&mut local_file, &mut remote_file)?; - - let files = sftp.readdir(&remote_config.remote_dir)?; - - let mut backups: Vec = files - .into_iter() - .map(|(file, _)| file) - .filter(|file| { - if let Some(ext) = file.extension() { - ext == "gz" - } else { - false - } - }) - .collect(); - - backups.sort_by(|file_a, file_b| { - let time_a = - get_time_from_file_name(file_a.file_name().unwrap().to_str().unwrap()).unwrap(); - let time_b = - get_time_from_file_name(file_b.file_name().unwrap().to_str().unwrap()).unwrap(); - - time_b.cmp(&time_a) - }); - - if backups.len() > remote_config.backups_to_keep as usize { - for _ in 0..(backups.len() - remote_config.backups_to_keep as usize) { - if let Some(backup_path) = backups.pop() { - sftp.unlink(&*backup_path)?; - } - } - } - - Ok(()) -}