From 449af042c3688e9c45699cbe38aad52b837425f1 Mon Sep 17 00:00:00 2001 From: Joey Hines Date: Mon, 31 May 2021 18:52:56 -0600 Subject: [PATCH] Database fixes + Fixed issue where IDs of different models were colliding + Used built in methods to convert u64 to bytes + Added tests for database speed + Converted return time of filter to an iterator + Added better error handling for non-unique values --- .gitignore | 2 + Cargo.lock | 280 +++++++++++++++++++- geoffrey_api/Cargo.toml | 4 +- geoffrey_db/src/database.rs | 88 +++++- geoffrey_db/src/error.rs | 5 +- geoffrey_db/src/lib.rs | 14 - geoffrey_models/src/models/locations/mod.rs | 24 ++ geoffrey_models/src/models/player.rs | 30 +++ 8 files changed, 409 insertions(+), 38 deletions(-) diff --git a/.gitignore b/.gitignore index 150a35b..40e9e7b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ /target .idea /database +/test_database +config.toml diff --git a/Cargo.lock b/Cargo.lock index 19a5ff3..f1b2dad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,40 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + [[package]] name = "autocfg" version = "1.0.1" @@ -63,12 +98,43 @@ checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" dependencies = [ "libc", "num-integer", - "num-traits", - "serde", + "num-traits 0.2.14", + "serde 1.0.124", "time", "winapi", ] +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "config" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1b9d958c2b1368a663f05538fc1b5975adce1e19f435acceae987aceeeb369" +dependencies = [ + "lazy_static", + "nom", + "rust-ini", + "serde 1.0.124", + "serde-hjson", + "serde_json", + "toml", + "yaml-rust", +] + [[package]] name = "cpuid-bool" version = "0.1.2" @@ -228,10 +294,12 @@ dependencies = [ name = "geoffrey_api" version = "0.1.0" dependencies = [ + "config", "geoffrey_db", "geoffrey_models", - "serde", + "serde 1.0.124", "serde_json", + "structopt", "tokio", "warp", ] @@ -243,7 +311,7 @@ dependencies = [ "byteorder", "geoffrey_models", "lazy_static", - "serde", + "serde 1.0.124", "serde_json", "sled", ] @@ -253,7 +321,7 @@ name = "geoffrey_models" version = "0.1.0" dependencies = [ "chrono", - "serde", + "serde 1.0.124", "serde_json", ] @@ -329,6 +397,15 @@ dependencies = [ "http", ] +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.18" @@ -446,12 +523,31 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lexical-core" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" +dependencies = [ + "arrayvec", + "bitflags", + "cfg-if", + "ryu", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03b07a082330a35e43f63177cc01689da34fbffa0105e1246cf0311472cac73a" +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + [[package]] name = "lock_api" version = "0.4.2" @@ -548,6 +644,17 @@ dependencies = [ "twoway", ] +[[package]] +name = "nom" +version = "5.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" +dependencies = [ + "lexical-core", + "memchr", + "version_check", +] + [[package]] name = "ntapi" version = "0.3.6" @@ -564,7 +671,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ "autocfg", - "num-traits", + "num-traits 0.2.14", +] + +[[package]] +name = "num-traits" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" +dependencies = [ + "num-traits 0.2.14", ] [[package]] @@ -667,6 +783,30 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" version = "1.0.24" @@ -781,6 +921,23 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + [[package]] name = "remove_dir_all" version = "0.5.3" @@ -790,6 +947,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "rust-ini" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" + [[package]] name = "ryu" version = "1.0.5" @@ -814,6 +977,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "serde" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" + [[package]] name = "serde" version = "1.0.124" @@ -823,6 +992,18 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-hjson" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8" +dependencies = [ + "lazy_static", + "num-traits 0.1.43", + "regex", + "serde 0.8.23", +] + [[package]] name = "serde_derive" version = "1.0.124" @@ -842,7 +1023,7 @@ checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" dependencies = [ "itoa", "ryu", - "serde", + "serde 1.0.124", ] [[package]] @@ -854,7 +1035,7 @@ dependencies = [ "form_urlencoded", "itoa", "ryu", - "serde", + "serde 1.0.124", ] [[package]] @@ -918,6 +1099,42 @@ dependencies = [ "winapi", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "syn" version = "1.0.61" @@ -943,6 +1160,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "time" version = "0.1.44" @@ -1038,6 +1264,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde 1.0.124", +] + [[package]] name = "tower-service" version = "0.3.1" @@ -1142,6 +1377,18 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + [[package]] name = "unicode-xid" version = "0.2.1" @@ -1166,6 +1413,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + [[package]] name = "version_check" version = "0.9.2" @@ -1200,7 +1453,7 @@ dependencies = [ "percent-encoding", "pin-project", "scoped-tls", - "serde", + "serde 1.0.124", "serde_json", "serde_urlencoded", "tokio", @@ -1245,3 +1498,12 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] diff --git a/geoffrey_api/Cargo.toml b/geoffrey_api/Cargo.toml index 9026419..af35297 100644 --- a/geoffrey_api/Cargo.toml +++ b/geoffrey_api/Cargo.toml @@ -12,4 +12,6 @@ warp = "0.3" serde = "1.0.124" serde_json = "1.0.64" geoffrey_models = { path = "../geoffrey_models" } -geoffrey_db = { path = "../geoffrey_db" } \ No newline at end of file +geoffrey_db = { path = "../geoffrey_db" } +config = "0.11.0" +structopt = "0.3.21" \ No newline at end of file diff --git a/geoffrey_db/src/database.rs b/geoffrey_db/src/database.rs index f2acbfa..e958879 100644 --- a/geoffrey_db/src/database.rs +++ b/geoffrey_db/src/database.rs @@ -1,8 +1,7 @@ -use crate::error::Result; +use crate::error::{Result, GeoffreyDBError}; use std::path::Path; use geoffrey_models::GeoffreyDatabaseModel; -use crate::{u64_from_bytes, u64_to_bytes}; -use sled::Iter; +use std::convert::TryInto; pub struct Database { db: sled::Db, @@ -31,8 +30,16 @@ impl Database { } }; + let match_count = self.filter(|_, o: &T| { + !o.check_unique(&model) + })?.count(); + + if match_count > 0 { + return Err(GeoffreyDBError::NotUnique) + } + let tree = self.get_tree::()?; - let id_bytes = u64_to_bytes(id); + let id_bytes = id.to_be_bytes(); tree.insert(id_bytes, model.to_bytes()?)?; @@ -42,7 +49,7 @@ impl Database { pub fn get(&self, id: u64) -> Result> where T: GeoffreyDatabaseModel { let tree = self.get_tree::()?; - let id_bytes = u64_to_bytes(id); + let id_bytes = id.to_be_bytes(); if let Some(bytes) = tree.get(id_bytes)? { Ok(Some(T::try_from_bytes(&bytes)?)) @@ -53,12 +60,16 @@ impl Database { } - pub fn filter(&self, f: F) -> Result> where T: GeoffreyDatabaseModel, F: Fn(u64, &T) -> bool { + pub fn clear_tree(&self) -> Result<()> where T: GeoffreyDatabaseModel { + Ok(self.db.open_tree(T::tree())?.clear()?) + } + + pub fn filter<'a, T>(&self, f: impl Fn(u64, &T) -> bool + 'a) -> Result + 'a> where T: GeoffreyDatabaseModel { let tree = self.db.open_tree(T::tree())?; - Ok(tree.iter().filter_map(|e| { + Ok(tree.iter().filter_map(move |e| { if let Ok((id, data)) = e { - let id = u64_from_bytes(&mut id.clone()); + let id = u64::from_be_bytes(id.to_vec().try_into().unwrap()); let data = T::try_from_bytes(&data).unwrap(); if f(id, &data) { @@ -71,10 +82,10 @@ impl Database { else { None } - }).collect()) + })) } - pub fn tree_iter(&self) -> Result where T: GeoffreyDatabaseModel { + pub fn tree_iter(&self) -> Result where T: GeoffreyDatabaseModel { Ok(self.db.open_tree(T::tree()).map(|tree| tree.iter())?) } } @@ -86,13 +97,18 @@ mod tests { use geoffrey_models::models::player::{Player, UserID}; use lazy_static::lazy_static; use geoffrey_models::GeoffreyDatabaseModel; + use geoffrey_models::models::locations::{Location, LocationType}; + use geoffrey_models::models::{Position, Dimension}; + use std::time::Instant; lazy_static! { - static ref DB: Database = Database::new(Path::new("../locations")).unwrap(); + static ref DB: Database = Database::new(Path::new("../test_database")).unwrap(); } fn cleanup() { - DB.db.clear().unwrap(); + DB.clear_tree::().unwrap(); + DB.clear_tree::().unwrap(); + DB.db.flush().unwrap(); } #[test] @@ -103,6 +119,19 @@ mod tests { assert!(p2.id().is_some()); assert_eq!(player.name, p2.name); + + + cleanup(); + } + + #[test] + fn test_unique_insert() { + let player1 = Player::new("CoolZero123", UserID::DiscordUUID(0u64)); + let player2 = Player::new("CoolZero123", UserID::DiscordUUID(0u64)); + + DB.insert::(player1.clone()).unwrap(); + assert_eq!(DB.insert::(player2.clone()).is_err(), true); + cleanup(); } @@ -118,5 +147,40 @@ mod tests { assert_eq!(p3.unwrap().name, player.name); cleanup(); } + + #[test] + fn test_filter() { + let player = Player::new("CoolZero123", UserID::DiscordUUID(0u64)); + let player = DB.insert::(player.clone()).unwrap(); + let loc = Location::new("Test Shop", Position::new(0, 0, Dimension::Overworld), player.id.unwrap(), None, LocationType::Shop(0)); + let loc = DB.insert::(loc.clone()).unwrap(); + + let count = DB.filter(|id: u64, l: &Location| { + assert_eq!(id, l.id().unwrap()); + loc.id().unwrap() == id + }).unwrap().count(); + + assert_eq!(count, 1); + DB.db.flush().unwrap(); + cleanup(); + } + + #[test] + fn test_insert_speed() { + cleanup(); + let insert_count = 1000; + let timer = Instant::now(); + for i in 0..insert_count { + let player = Player::new("test", UserID::DiscordUUID(i)); + + DB.insert::(player).unwrap(); + } + + DB.db.flush().unwrap(); + let sec_elapsed = timer.elapsed().as_secs_f32(); + println!("Completed in {}s. {} inserts per second", sec_elapsed, insert_count as f32/sec_elapsed); + + cleanup() + } } diff --git a/geoffrey_db/src/error.rs b/geoffrey_db/src/error.rs index 4d54e49..173f18f 100644 --- a/geoffrey_db/src/error.rs +++ b/geoffrey_db/src/error.rs @@ -1,10 +1,10 @@ - pub type Result = std::result::Result; #[derive(Debug)] pub enum GeoffreyDBError { SledError(sled::Error), SerdeJsonError(serde_json::Error), + NotUnique, } impl std::error::Error for GeoffreyDBError {} @@ -13,7 +13,8 @@ impl std::fmt::Display for GeoffreyDBError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { GeoffreyDBError::SledError(e) => write!(f, "Sled Error: {}", e), - GeoffreyDBError::SerdeJsonError(e) => write!(f, "Serde JSON Error: {}", e) + GeoffreyDBError::SerdeJsonError(e) => write!(f, "Serde JSON Error: {}", e), + GeoffreyDBError::NotUnique => write!(f, "Entry is not unique.") } } } diff --git a/geoffrey_db/src/lib.rs b/geoffrey_db/src/lib.rs index 6a1b26f..cad3bef 100644 --- a/geoffrey_db/src/lib.rs +++ b/geoffrey_db/src/lib.rs @@ -1,18 +1,4 @@ #![allow(dead_code)] -use byteorder::{WriteBytesExt, BigEndian, ReadBytesExt}; -use std::io::Cursor; - pub mod database; pub mod error; -pub fn u64_to_bytes(n: u64) -> Vec { - let mut id_bytes= vec![0u8; 8]; - id_bytes.write_u64::(n).unwrap(); - - id_bytes -} - -pub fn u64_from_bytes(bytes: &mut [u8]) -> u64 { - let mut cursor = Cursor::new(bytes); - cursor.read_u64::().unwrap() -} diff --git a/geoffrey_models/src/models/locations/mod.rs b/geoffrey_models/src/models/locations/mod.rs index 1bf54fb..a8e66e8 100644 --- a/geoffrey_models/src/models/locations/mod.rs +++ b/geoffrey_models/src/models/locations/mod.rs @@ -70,4 +70,28 @@ impl GeoffreyDatabaseModel for Location { fn tree() -> String { "location".to_string() } + + fn check_unique(&self, o: &Self) -> bool { + o.name.to_lowercase() != self.name.to_lowercase() + } +} + +#[cfg(test)] +mod tests { + use crate::GeoffreyDatabaseModel; + use crate::models::locations::{Location, LocationType}; + use crate::models::{Position, Dimension}; + + #[test] + fn test_location_check_unique() { + let l1 = Location::new("Test", Position::new(0, 0, Dimension::Overworld), 0u64, None, LocationType::Base); + let l2 = Location::new("NotTest", Position::new(0, 0, Dimension::Overworld), 0u64, None, LocationType::Base); + + assert!(l1.check_unique(&l2)); + + let l1 = Location::new("Test", Position::new(0, 0, Dimension::Overworld), 0u64, None, LocationType::Base); + let l2 = Location::new("teSt", Position::new(0, 0, Dimension::Overworld), 0u64, None, LocationType::Base); + + assert!(!l1.check_unique(&l2)); + } } diff --git a/geoffrey_models/src/models/player.rs b/geoffrey_models/src/models/player.rs index 8b4883b..0892121 100644 --- a/geoffrey_models/src/models/player.rs +++ b/geoffrey_models/src/models/player.rs @@ -36,4 +36,34 @@ impl GeoffreyDatabaseModel for Player { fn tree() -> String { "player".to_string() } + + fn check_unique(&self, o: &Self) -> bool { + for user1_id in &self.user_ids { + if o.user_ids.contains(user1_id) { + return false; + } + } + + true + } +} + +#[cfg(test)] +mod tests { + use crate::models::player::Player; + use crate::models::player::UserID::{DiscordUUID, MinecraftUUID}; + use crate::GeoffreyDatabaseModel; + + #[test] + fn test_player_check_unique() { + let p1 = Player::new("CoolTest123", DiscordUUID(0u64)); + let p2 = Player::new("NotCoolTest123", DiscordUUID(1u64)); + + assert!(p1.check_unique(&p2)); + + let p1 = Player::new("CoolTest123", MinecraftUUID("0".to_string())); + let p2 = Player::new("NotCoolTest123", MinecraftUUID("0".to_string())); + + assert!(!p1.check_unique(&p2)); + } } \ No newline at end of file