First pass on adding remote backups

+ Added remote_backup.rs to handle remote backup logic
+ Added `remote` section to the config for optional remote backup setup
+ Using ssh2 rust library for SFTP support
+ small tweaks
backup_error_fix
Joey Hines 2021-02-02 21:21:55 -06:00
parent a7e2b260bc
commit d94186c7b8
No known key found for this signature in database
GPG Key ID: 80F567B5C968F91B
6 changed files with 193 additions and 21 deletions

87
Cargo.lock generated
View File

@ -43,6 +43,7 @@ dependencies = [
"regex", "regex",
"reqwest", "reqwest",
"serde 1.0.117", "serde 1.0.117",
"ssh2",
"structopt", "structopt",
"tar", "tar",
] ]
@ -183,6 +184,15 @@ dependencies = [
"vec_map", "vec_map",
] ]
[[package]]
name = "cloudabi"
version = "0.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
dependencies = [
"bitflags",
]
[[package]] [[package]]
name = "config" name = "config"
version = "0.9.3" version = "0.9.3"
@ -579,6 +589,32 @@ version = "0.2.71"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49"
[[package]]
name = "libssh2-sys"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0186af0d8f171ae6b9c4c90ec51898bad5d08a2d5e470903a50d9ad8959cbee"
dependencies = [
"cc",
"libc",
"libz-sys",
"openssl-sys",
"pkg-config",
"vcpkg",
]
[[package]]
name = "libz-sys"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]] [[package]]
name = "linked-hash-map" name = "linked-hash-map"
version = "0.3.0" version = "0.3.0"
@ -595,6 +631,15 @@ version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a"
[[package]]
name = "lock_api"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75"
dependencies = [
"scopeguard",
]
[[package]] [[package]]
name = "log" name = "log"
version = "0.4.11" version = "0.4.11"
@ -815,6 +860,30 @@ dependencies = [
"vcpkg", "vcpkg",
] ]
[[package]]
name = "parking_lot"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
dependencies = [
"cfg-if 0.1.10",
"cloudabi",
"libc",
"redox_syscall",
"smallvec",
"winapi 0.3.8",
]
[[package]] [[package]]
name = "percent-encoding" name = "percent-encoding"
version = "2.1.0" version = "2.1.0"
@ -1050,6 +1119,12 @@ dependencies = [
"winapi 0.3.8", "winapi 0.3.8",
] ]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]] [[package]]
name = "security-framework" name = "security-framework"
version = "0.4.4" version = "0.4.4"
@ -1168,6 +1243,18 @@ dependencies = [
"winapi 0.3.8", "winapi 0.3.8",
] ]
[[package]]
name = "ssh2"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d876d4d57f6bbf2245d43f7ec53759461f801a446d3693704aa6d27b257844d7"
dependencies = [
"bitflags",
"libc",
"libssh2-sys",
"parking_lot",
]
[[package]] [[package]]
name = "strsim" name = "strsim"
version = "0.8.0" version = "0.8.0"

View File

@ -18,3 +18,4 @@ tar = "0.4.28"
reqwest = { version = "0.10", features = ["blocking", "json"] } reqwest = { version = "0.10", features = ["blocking", "json"] }
discord-hooks-rs = { git = "https://github.com/joeyahines/discord-hooks-rs" } discord-hooks-rs = { git = "https://github.com/joeyahines/discord-hooks-rs" }
anvil-region = "0.4.0" anvil-region = "0.4.0"
ssh2 = "0.9.1"

View File

@ -2,6 +2,7 @@ use crate::backup;
use crate::config::{AlbatrossConfig, WorldConfig, WorldType}; use crate::config::{AlbatrossConfig, WorldConfig, WorldType};
use crate::discord::send_webhook; use crate::discord::send_webhook;
use crate::region::Region; use crate::region::Region;
use crate::remote_backup::remote_backup;
use chrono::{NaiveDateTime, Utc}; use chrono::{NaiveDateTime, Utc};
use flate2::read::GzDecoder; use flate2::read::GzDecoder;
use flate2::write::GzEncoder; use flate2::write::GzEncoder;
@ -321,8 +322,42 @@ pub fn do_backup(cfg: AlbatrossConfig, output: Option<PathBuf>) -> Result<(), st
create_dir_all(tmp_dir.clone()).unwrap(); create_dir_all(tmp_dir.clone()).unwrap();
send_webhook("**Albatross is swooping in to backup your worlds!**", &cfg);
let timer = Instant::now(); let timer = Instant::now();
backup_worlds(&cfg, server_base_dir, worlds, &mut tmp_dir)?;
backup::compress_backup(&tmp_dir, &output_archive)?;
remove_dir_all(&tmp_dir)?;
let backups_removed = remove_old_backups(&cfg.backup.output_dir, cfg.backup.backups_to_keep)?;
if backups_removed > 0 {
let msg = format!(
"Albatross mistook **{}** of your old backups for some french fries and ate them!! SKRAWWWW",
backups_removed
);
send_webhook(msg.as_str(), &cfg);
}
if let Some(remote_backup_cfg) = &cfg.remote {
remote_backup(output_archive, remote_backup_cfg);
}
let secs = timer.elapsed().as_secs();
send_webhook(
format!("**Full backup completed in {}s**! *SKREEEEEEEEEE*", secs).as_str(),
&cfg,
);
Ok(())
}
fn backup_worlds(
cfg: &AlbatrossConfig,
server_base_dir: PathBuf,
worlds: Vec<WorldConfig>,
tmp_dir: &mut PathBuf,
) -> Result<(), std::io::Error> {
send_webhook("**Albatross is swooping in to backup your worlds!**", &cfg);
for world in worlds { for world in worlds {
let mut world_dir = server_base_dir.clone(); let mut world_dir = server_base_dir.clone();
let world_name = world.world_name.clone(); let world_name = world.world_name.clone();
@ -363,24 +398,5 @@ pub fn do_backup(cfg: AlbatrossConfig, output: Option<PathBuf>) -> Result<(), st
} }
} }
backup::compress_backup(&tmp_dir, &output_archive)?;
remove_dir_all(&tmp_dir)?;
let backups_removed = remove_old_backups(&cfg.backup.output_dir, cfg.backup.backups_to_keep)?;
if backups_removed > 0 {
let msg = format!(
"Albatross mistook **{}** of your old backups for some french fries and ate them!! SKRAWWWW",
backups_removed
);
send_webhook(msg.as_str(), &cfg);
}
let secs = timer.elapsed().as_secs();
send_webhook(
format!("**Full backup completed in {}s**! *SKREEEEEEEEEE*", secs).as_str(),
&cfg,
);
Ok(()) Ok(())
} }

View File

@ -24,7 +24,7 @@ impl From<String> for WorldType {
} }
} }
/// Config for individual WorldConfig /// Config for individual world configuration
#[derive(Debug, Deserialize, Clone)] #[derive(Debug, Deserialize, Clone)]
pub struct WorldConfig { pub struct WorldConfig {
pub world_name: String, pub world_name: String,
@ -41,11 +41,31 @@ pub struct BackupConfig {
pub discord_webhook: Option<String>, pub discord_webhook: Option<String>,
} }
/// Config for remote backups
#[derive(Debug, Deserialize, Clone)]
pub struct RemoteBackupConfig {
/// Remote server address
pub sftp_server_addr: String,
/// Remote output directory
pub remote_dir: PathBuf,
/// Remote server username
pub username: String,
/// Public key for key auth
pub public_key: Option<PathBuf>,
/// Private key for key auth
pub private_key: Option<PathBuf>,
/// Password if using password auth
pub password: Option<String>,
/// Remote backups to keep
pub backups_to_keep: u64,
}
/// Configs /// Configs
#[derive(Debug, Deserialize, Clone)] #[derive(Debug, Deserialize, Clone)]
pub struct AlbatrossConfig { pub struct AlbatrossConfig {
pub backup: BackupConfig, pub backup: BackupConfig,
pub world_config: Option<Vec<WorldConfig>>, pub world_config: Option<Vec<WorldConfig>>,
pub remote: Option<RemoteBackupConfig>,
} }
impl AlbatrossConfig { impl AlbatrossConfig {

View File

@ -6,6 +6,7 @@ mod chunk_coordinate;
mod config; mod config;
mod discord; mod discord;
mod region; mod region;
mod remote_backup;
mod restore; mod restore;
use crate::backup::{convert_backup_to_sp, do_backup}; use crate::backup::{convert_backup_to_sp, do_backup};

View File

@ -0,0 +1,47 @@
use crate::config::RemoteBackupConfig;
use ssh2::Session;
use std::net::TcpStream;
use std::path::PathBuf;
fn open_ssh_session(remote_config: &RemoteBackupConfig) -> Session {
let tcp = TcpStream::connect(&remote_config.sftp_server_addr).unwrap();
let mut sess = Session::new().unwrap();
sess.set_tcp_stream(tcp);
sess.handshake().unwrap();
if let Some(password) = &remote_config.password {
sess.userauth_password(&remote_config.username, password)
.unwrap();
} else if let Some(key) = &remote_config.private_key {
let public_key =
remote_config
.public_key
.as_ref()
.map(|pub_key| pub_key.as_path().clone());
sess.userauth_pubkey_file(
&remote_config.username,
public_key,
key,
None,
)
.unwrap();
} else {
panic!("No key provided")
}
sess
}
pub fn remote_backup(file: PathBuf, remote_config: &RemoteBackupConfig) {
let sess = open_ssh_session(remote_config);
let remote_path = remote_config.remote_dir.join(file.file_name().unwrap());
let mut local_file = std::fs::File::open(&file).unwrap();
let sftp = sess.sftp().unwrap();
let mut remote_file = sftp.create(&remote_path).unwrap();
std::io::copy(&mut local_file, &mut remote_file).unwrap();
}