Minecraft
/
Albatross
6
0
Fork 0
Code Issues 2 Pull Requests Projects Releases 1 Wiki Activity

Added FTPS backups 

+ Made remote backups more generic, should be easier to add new backup types in the future
+ General cleanup
+ clippy + fmt
backup_error_fix
Joey Hines 2021-02-09 20:47:40 -06:00
parent 9c7f9906ad
commit 9a264abe88
No known key found for this signature in database
GPG Key ID: 80F567B5C968F91B
12 changed files with 418 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
Cargo.lock generated
View File

@ -39,6 +39,7 @@ dependencies = [
"config",
"discord-hooks-rs",
"flate2",
"ftp",
"log",
"regex",
"reqwest",
@ -160,13 +161,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.11"
version = "0.4.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2"
checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
dependencies = [
"libc",
"num-integer",
"num-traits 0.2.11",
"time",
"winapi 0.3.8",
]
[[package]]
@ -332,6 +335,18 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "ftp"
version = "3.0.1"
source = "git+https://github.com/joeyahines/rust-ftp.git#ab44662b5f27d18116a2721c1ddf76e117ed88be"
dependencies = [
"chrono",
"lazy_static 1.4.0",
"native-tls",
"openssl",
"regex",
]
[[package]]
name = "fuchsia-zircon"
version = "0.3.3"
@ -829,12 +844,12 @@ checksum = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d"
[[package]]
name = "openssl"
version = "0.10.29"
version = "0.10.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cee6d85f4cb4c4f59a6a85d5b68a233d280c82e29e822913b9c8b129fbf20bdd"
checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70"
dependencies = [
"bitflags",
"cfg-if 0.1.10",
"cfg-if 1.0.0",
"foreign-types",
"lazy_static 1.4.0",
"libc",
@ -849,9 +864,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
[[package]]
name = "openssl-sys"
version = "0.9.58"
version = "0.9.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de"
checksum = "921fc71883267538946025deffb622905ecad223c28efbfdef9bb59a0175f3e6"
dependencies = [
"autocfg",
"cc",
@ -1031,9 +1046,9 @@ checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
[[package]]
name = "regex"
version = "1.3.9"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a"
dependencies = [
"aho-corasick",
"memchr",
@ -1043,9 +1058,9 @@ dependencies = [
[[package]]
name = "regex-syntax"
version = "0.6.18"
version = "0.6.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581"
[[package]]
name = "remove_dir_all"

1
Cargo.toml
View File

@ -19,3 +19,4 @@ reqwest = { version = "0.10", features = ["blocking", "json"] }
discord-hooks-rs = { git = "https://github.com/joeyahines/discord-hooks-rs" }
anvil-region = "0.4.0"
ssh2 = "0.9.1"
ftp = { git = "https://github.com/joeyahines/rust-ftp.git" , features=["secure", "native-tls"]}

86
src/backup.rs
View File

@ -1,17 +1,18 @@
use crate::backup;
use crate::config::{AlbatrossConfig, WorldConfig, WorldType};
use crate::config::{AlbatrossConfig, RemoteBackupConfig, WorldConfig, WorldType};
use crate::discord::send_webhook;
use crate::error::Result;
use crate::region::Region;
use crate::remote_backup::remote_backup;
use chrono::{NaiveDateTime, Utc};
use crate::remote::file::FileBackup;
use crate::remote::ftps::FTPSBackup;
use crate::remote::sftp::SFTPBackup;
use crate::remote::RemoteBackupSite;
use chrono::Utc;
use flate2::read::GzDecoder;
use flate2::write::GzEncoder;
use flate2::Compression;
use std::convert::TryFrom;
use std::fs::{
copy, create_dir, create_dir_all, remove_dir_all, remove_file, rename, DirEntry, File,
};
use std::fs::{copy, create_dir, create_dir_all, remove_dir_all, rename, File};
use std::path::PathBuf;
use std::time::Instant;
use tar::Archive;
@ -248,60 +249,19 @@ pub fn convert_backup_to_sp(
Ok(())
}
pub fn get_time_from_file_name(file_name: &str) -> Result<NaiveDateTime> {
let time: Vec<&str> = file_name.split("_backup.tar.gz").collect();
Ok(chrono::NaiveDateTime::parse_from_str(
time[0],
"%d-%m-%y_%H.%M.%S",
)?)
}
/// Get the time of the backup from a directory entry
///
/// # Param
/// * `archive_entry`: archive entry
fn get_time_from_dir_entry(archive_entry: &DirEntry) -> Result<NaiveDateTime> {
let file_name = archive_entry.file_name().to_str().unwrap().to_string();
get_time_from_file_name(file_name.as_str())
}
/// Removes the old backups from the ouput directory
///
/// # Params
/// * `output_dir` - output directory containing
/// * `keep` - number of backups to keep
fn remove_old_backups(output_dir: &PathBuf, keep: u64) -> Result<usize> {
let mut backups = vec![];
let mut num_of_removed_backups: usize = 0;
for entry in output_dir.read_dir()? {
let entry = entry?;
if let Some(ext) = entry.path().extension() {
if ext == "gz" {
backups.push(entry);
/// Preform a remote backup, if configured
pub fn do_remote_backup(
remote_backup_cfg: &RemoteBackupConfig,
backup_path: PathBuf,
) -> Result<()> {
if let Ok(mut sftp_backup) = SFTPBackup::new(remote_backup_cfg) {
sftp_backup.backup_to_remote(backup_path)?;
sftp_backup.cleanup()?;
} else if let Ok(mut ftps_backup) = FTPSBackup::new(remote_backup_cfg) {
ftps_backup.backup_to_remote(backup_path)?;
ftps_backup.cleanup()?;
}
}
}
if backups.len() > keep as usize {
backups.sort_by(|a, b| {
let a_time = get_time_from_dir_entry(a).unwrap();
let b_time = get_time_from_dir_entry(b).unwrap();
b_time.cmp(&a_time)
});
num_of_removed_backups = backups.len() - keep as usize;
for _i in 0..num_of_removed_backups {
let oldest = backups.pop().unwrap();
remove_file(oldest.path())?;
}
}
Ok(num_of_removed_backups)
Ok(())
}
/// Backup the configured worlds from a minecraft server
@ -342,7 +302,9 @@ pub fn do_backup(cfg: AlbatrossConfig, output: Option<PathBuf>) -> Result<()> {
remove_dir_all(&tmp_dir)?;
match remove_old_backups(&cfg.backup.output_dir, cfg.backup.backups_to_keep) {
let mut local_backup = FileBackup::new(&cfg.backup).unwrap();
match local_backup.cleanup() {
Ok(backups_removed) => {
if backups_removed > 0 {
let msg = format!(
@ -358,8 +320,8 @@ pub fn do_backup(cfg: AlbatrossConfig, output: Option<PathBuf>) -> Result<()> {
}
}
if let Some(remote_backup_cfg) = &cfg.remote {
match remote_backup(output_archive, remote_backup_cfg) {
if let Some(remote_backup_config) = &cfg.remote {
match do_remote_backup(remote_backup_config, output_archive) {
Ok(_) => {
send_webhook("Remote backup completed!", &cfg);
}

18
src/config.rs → src/config/mod.rs
View File

@ -1,3 +1,6 @@
mod remote;
use crate::config::remote::{FTPSConfig, SFTPConfig};
use config::{Config, ConfigError, File};
use serde::Deserialize;
use std::path::PathBuf;
@ -44,20 +47,9 @@ pub struct BackupConfig {
/// Config for remote backups
#[derive(Debug, Deserialize, Clone)]
pub struct RemoteBackupConfig {
/// Remote server address
pub sftp_server_addr: String,
/// Remote output directory
pub remote_dir: PathBuf,
/// Remote server username
pub username: String,
/// Public key for key auth
pub public_key: Option<PathBuf>,
/// Private key for key auth
pub private_key: Option<PathBuf>,
/// Password if using password auth
pub password: Option<String>,
/// Remote backups to keep
pub backups_to_keep: u64,
pub sftp: Option<SFTPConfig>,
pub ftps: Option<FTPSConfig>,
}
/// Configs

34
src/config/remote.rs 100644
View File

@ -0,0 +1,34 @@
use serde::Deserialize;
use std::path::PathBuf;
/// SFTP Config
#[derive(Debug, Deserialize, Clone)]
pub struct SFTPConfig {
/// Remote server address
pub server_addr: String,
/// Remote output directory
pub remote_dir: PathBuf,
/// Remote server username
pub username: String,
/// Public key for key auth
pub public_key: Option<PathBuf>,
/// Private key for key auth
pub private_key: Option<PathBuf>,
/// Password if using password auth
pub password: Option<String>,
}
/// FTPS Config
#[derive(Debug, Deserialize, Clone)]
pub struct FTPSConfig {
/// Remote server address
pub server_addr: String,
/// Remote output directory
pub remote_dir: PathBuf,
/// Remote server username
pub username: String,
/// Password
pub password: String,
/// Domain
pub domain: Option<String>,
}

10
src/error.rs
View File

@ -10,6 +10,8 @@ pub enum AlbatrossError {
RegionParseError(crate::region::RegionParseError),
ChronoParseError(chrono::ParseError),
NoSSHAuth,
RemoteConfigError(String),
FTPSError(ftp::FtpError),
}
impl std::error::Error for AlbatrossError {}
@ -25,6 +27,8 @@ impl std::fmt::Display for AlbatrossError {
AlbatrossError::RegionParseError(e) => write!(f, "Unable to parse region name: {}", e),
AlbatrossError::ChronoParseError(e) => write!(f, "Unable to parse time: {}", e),
AlbatrossError::NoSSHAuth => write!(f, "No SSH auth methods provided in the config"),
AlbatrossError::RemoteConfigError(e) => write!(f, "Invalid configuration for {}", e),
AlbatrossError::FTPSError(e) => write!(f, "FTPS error: {}", e),
}
}
}
@ -58,3 +62,9 @@ impl From<chrono::ParseError> for AlbatrossError {
AlbatrossError::ChronoParseError(e)
}
}
impl From<ftp::FtpError> for AlbatrossError {
fn from(e: ftp::FtpError) -> Self {
AlbatrossError::FTPSError(e)
}
}

4
src/main.rs
View File

@ -7,7 +7,7 @@ mod config;
mod discord;
mod error;
mod region;
mod remote_backup;
mod remote;
mod restore;
use crate::backup::{convert_backup_to_sp, do_backup};
@ -66,7 +66,7 @@ fn main() {
let opt = Albatross::from_args();
let cfg = AlbatrossConfig::new(opt.config_path.into_os_string().to_str().unwrap())
.expect("Config not found");
.expect("Config error");
if cfg.world_config.is_some() {
match opt.sub_command {

47
src/remote/file.rs 100644
View File

@ -0,0 +1,47 @@
use crate::config::BackupConfig;
use crate::error::Result;
use crate::remote::{PathLocation, RemoteBackupSite};
use std::path::PathBuf;
pub struct FileBackup {
/// Target directory on the file system
target_dir: PathBuf,
/// Number of backups to keep
backups_to_keep: usize,
}
impl FileBackup {
/// New FileBackup
pub fn new(config: &BackupConfig) -> Result<Self> {
Ok(Self {
target_dir: config.output_dir.clone(),
backups_to_keep: config.backups_to_keep as usize,
})
}
}
impl RemoteBackupSite for FileBackup {
type FileType = PathLocation;
fn backup_to_remote(&mut self, file: PathBuf) -> Result<()> {
let dest = self.target_dir.join(file.file_name().unwrap());
std::fs::copy(file, dest)?;
Ok(())
}
fn get_backups(&mut self) -> Result<Vec<Self::FileType>> {
Ok(self
.target_dir
.read_dir()?
.filter_map(|file| Self::FileType::new(file.unwrap().path()))
.collect())
}
fn remove_backup(&mut self, backup: Self::FileType) -> Result<()> {
Ok(std::fs::remove_file(backup.location)?)
}
fn backups_to_keep(&self) -> usize {
self.backups_to_keep
}
}

79
src/remote/ftps.rs 100644
View File

@ -0,0 +1,79 @@
use ftp::native_tls::TlsConnector;
use ftp::FtpStream;
use std::path::PathBuf;
use crate::config::RemoteBackupConfig;
use crate::error;
use crate::error::AlbatrossError;
use crate::remote::{PathLocation, RemoteBackupSite};
/// FTPS Remote Site
pub struct FTPSBackup {
/// FTP command stream
stream: FtpStream,
/// Remote target directory
target_dir: PathBuf,
/// Number of backups to keep
backups_to_keep: usize,
}
impl FTPSBackup {
/// New FTPSBackup
pub fn new(config: &RemoteBackupConfig) -> error::Result<Self> {
let ftps_config = config
.ftps
.as_ref()
.ok_or_else(|| AlbatrossError::RemoteConfigError("FTPS".to_string()))?;
let ctx = TlsConnector::new().unwrap();
let (ftp_stream, _) = FtpStream::connect(&ftps_config.server_addr)?;
let domain = ftps_config
.domain
.as_ref()
.map_or_else(|| "".to_string(), |s| s.clone());
let mut ftp_stream = ftp_stream.into_secure(ctx, domain.as_str())?;
ftp_stream.login(&ftps_config.username, &ftps_config.password)?;
Ok(Self {
stream: ftp_stream,
target_dir: ftps_config.remote_dir.clone(),
backups_to_keep: config.backups_to_keep as usize,
})
}
}
impl Drop for FTPSBackup {
fn drop(&mut self) {
self.stream.quit().ok();
}
}
impl RemoteBackupSite for FTPSBackup {
type FileType = PathLocation;
fn backup_to_remote(&mut self, file: PathBuf) -> error::Result<()> {
let mut local_file = std::fs::File::open(&file)?;
let location = self.target_dir.join(file);
self.stream
.put(location.to_str().unwrap(), &mut local_file)?;
Ok(())
}
fn get_backups(&mut self) -> error::Result<Vec<Self::FileType>> {
let files = self.stream.list(Some(self.target_dir.to_str().unwrap()))?;
Ok(files
.into_iter()
.filter_map(|file| Self::FileType::new(PathBuf::from(file)))
.collect())
}
fn remove_backup(&mut self, backup: Self::FileType) -> error::Result<()> {
Ok(self.stream.rm(backup.location.to_str().unwrap())?)
}
fn backups_to_keep(&self) -> usize {
self.backups_to_keep
}
}

107
src/remote/mod.rs 100644
View File

@ -0,0 +1,107 @@
use std::path::PathBuf;
use chrono::NaiveDateTime;
use crate::error::Result;
pub mod file;
pub mod ftps;
pub mod sftp;
pub trait RemoteBackupFile {
/// Type containing the location of the remote backup
type LocationType;
/// Get the underlying location type
fn location(&self) -> Self::LocationType;
/// Get the time the remote file was created
fn time_created(&self) -> chrono::NaiveDateTime;
/// Parse the time created from the file name
fn parse_file_name(file_name: &str) -> Option<chrono::NaiveDateTime> {
let time: Vec<&str> = file_name.split("_backup.tar.gz").collect();
if let Some(time_str) = time.get(0) {
chrono::NaiveDateTime::parse_from_str(time_str, "%d-%m-%y_%H.%M.%S").ok()
} else {
None
}
}
}
pub trait RemoteBackupSite {
/// Struct representing the location of a backup on the site
type FileType: RemoteBackupFile;
/// Backup a file to the the remote site
fn backup_to_remote(&mut self, file: PathBuf) -> Result<()>;
/// Get the locations backups contained on the remote site
fn get_backups(&mut self) -> Result<Vec<Self::FileType>>;
/// Remove a backup from the side
fn remove_backup(&mut self, backup: Self::FileType) -> Result<()>;
/// Number of backups to keep on the site
fn backups_to_keep(&self) -> usize;
/// Cleanup old backups on the remote site
fn cleanup(&mut self) -> Result<usize> {
let mut backups = self.get_backups()?;
backups.sort_by_key(|backup| backup.time_created());
let mut backups: Vec<Self::FileType> = backups.into_iter().rev().collect();
let mut removed_count: usize = 0;
if backups.len() > self.backups_to_keep() {
for _ in 0..(backups.len() - self.backups_to_keep()) {
if let Some(backup) = backups.pop() {
self.remove_backup(backup)?;
removed_count += 1;
}
}
}
Ok(removed_count)
}
}
/// Backup location that can be represented by a path
pub struct PathLocation {
location: PathBuf,
time_created: NaiveDateTime,
}
impl PathLocation {
/// New PathLocation
fn new(path: PathBuf) -> Option<Self> {
if let Some(file_name) = path.file_name() {
let file_name = file_name.to_str().unwrap();
if let Some(time) = Self::parse_file_name(file_name) {
Some(Self {
location: path,
time_created: time,
})
} else {
None
}
} else {
None
}
}
}
impl RemoteBackupFile for PathLocation {
type LocationType = PathBuf;
fn location(&self) -> Self::LocationType {
self.location.to_path_buf()
}
fn time_created(&self) -> NaiveDateTime {
self.time_created
}
}

83
src/remote/sftp.rs 100644
View File

@ -0,0 +1,83 @@
use std::net::TcpStream;
use std::path::PathBuf;
use ssh2::Session;
use crate::config::RemoteBackupConfig;
use crate::error;
use crate::error::AlbatrossError;
use crate::remote::{PathLocation, RemoteBackupSite};
/// SFTP Remote Site
pub struct SFTPBackup {
/// SSH Session
session: Session,
/// Remote target directory
target_dir: PathBuf,
/// Number of backups to keep
backups_to_keep: usize,
}
impl SFTPBackup {
/// New SFTPBackup
pub fn new(config: &RemoteBackupConfig) -> error::Result<Self> {
let sftp_config = config
.sftp
.as_ref()
.ok_or_else(|| AlbatrossError::RemoteConfigError("SFTP".to_string()))?;
let tcp = TcpStream::connect(&sftp_config.server_addr)?;
let mut sess = Session::new()?;
sess.set_tcp_stream(tcp);
sess.handshake().unwrap();
if let Some(password) = &sftp_config.password {
sess.userauth_password(&sftp_config.username, password)?;
} else if let Some(key) = &sftp_config.private_key {
let public_key = sftp_config.public_key.as_deref();
sess.userauth_pubkey_file(&sftp_config.username, public_key, key, None)?;
} else {
return Err(AlbatrossError::NoSSHAuth);
}
Ok(Self {
session: sess,
target_dir: sftp_config.remote_dir.clone(),
backups_to_keep: config.backups_to_keep as usize,
})
}
}
impl RemoteBackupSite for SFTPBackup {
type FileType = PathLocation;
fn backup_to_remote(&mut self, file: PathBuf) -> error::Result<()> {
let remote_path = self.target_dir.join(file.file_name().unwrap());
let mut local_file = std::fs::File::open(&file)?;
let sftp = self.session.sftp()?;
let mut remote_file = sftp.create(&remote_path)?;
std::io::copy(&mut local_file, &mut remote_file)?;
Ok(())
}
fn get_backups(&mut self) -> error::Result<Vec<Self::FileType>> {
let files = self.session.sftp()?.readdir(&self.target_dir)?;
Ok(files
.into_iter()
.filter_map(|(file, _)| Self::FileType::new(file))
.collect())
}
fn remove_backup(&mut self, backup: Self::FileType) -> error::Result<()> {
Ok(self.session.sftp()?.unlink(&*backup.location)?)
}
fn backups_to_keep(&self) -> usize {
self.backups_to_keep
}
}

73
src/remote_backup.rs
View File

@ -1,73 +0,0 @@
use crate::backup::get_time_from_file_name;
use crate::config::RemoteBackupConfig;
use crate::error::{AlbatrossError, Result};
use ssh2::Session;
use std::net::TcpStream;
use std::path::PathBuf;
/// Open an SSH session
fn open_ssh_session(remote_config: &RemoteBackupConfig) -> Result<Session> {
let tcp = TcpStream::connect(&remote_config.sftp_server_addr)?;
let mut sess = Session::new()?;
sess.set_tcp_stream(tcp);
sess.handshake().unwrap();
if let Some(password) = &remote_config.password {
sess.userauth_password(&remote_config.username, password)?;
} else if let Some(key) = &remote_config.private_key {
let public_key = remote_config.public_key.as_deref();
sess.userauth_pubkey_file(&remote_config.username, public_key, key, None)?;
} else {
return Err(AlbatrossError::NoSSHAuth);
}
Ok(sess)
}
/// Handle remote backup of a file
pub fn remote_backup(file: PathBuf, remote_config: &RemoteBackupConfig) -> Result<()> {
let sess = open_ssh_session(remote_config)?;
let remote_path = remote_config.remote_dir.join(file.file_name().unwrap());
let mut local_file = std::fs::File::open(&file)?;
let sftp = sess.sftp()?;
let mut remote_file = sftp.create(&remote_path)?;
std::io::copy(&mut local_file, &mut remote_file)?;
let files = sftp.readdir(&remote_config.remote_dir)?;
let mut backups: Vec<PathBuf> = files
.into_iter()
.map(|(file, _)| file)
.filter(|file| {
if let Some(ext) = file.extension() {
ext == "gz"
} else {
false
}
})
.collect();
backups.sort_by(|file_a, file_b| {
let time_a =
get_time_from_file_name(file_a.file_name().unwrap().to_str().unwrap()).unwrap();
let time_b =
get_time_from_file_name(file_b.file_name().unwrap().to_str().unwrap()).unwrap();
time_b.cmp(&time_a)
});
if backups.len() > remote_config.backups_to_keep as usize {
for _ in 0..(backups.len() - remote_config.backups_to_keep as usize) {
if let Some(backup_path) = backups.pop() {
sftp.unlink(&*backup_path)?;
}
}
}
Ok(())
}