Database fixes

+ Fixed issue where IDs of different models were colliding
+ Used built in methods to convert u64 to bytes
+ Added tests for database speed
+ Converted return time of filter to an iterator
+ Added better error handling for non-unique values
main
Joey Hines 2021-05-31 18:52:56 -06:00
parent 9d104db0d6
commit 449af042c3
No known key found for this signature in database
GPG Key ID: 80F567B5C968F91B
8 changed files with 409 additions and 38 deletions

2
.gitignore vendored
View File

@ -1,3 +1,5 @@
/target
.idea
/database
/test_database
config.toml

280
Cargo.lock generated
View File

@ -1,5 +1,40 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "aho-corasick"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
dependencies = [
"memchr",
]
[[package]]
name = "ansi_term"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
dependencies = [
"winapi",
]
[[package]]
name = "arrayvec"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi",
"libc",
"winapi",
]
[[package]]
name = "autocfg"
version = "1.0.1"
@ -63,12 +98,43 @@ checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
dependencies = [
"libc",
"num-integer",
"num-traits",
"serde",
"num-traits 0.2.14",
"serde 1.0.124",
"time",
"winapi",
]
[[package]]
name = "clap"
version = "2.33.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002"
dependencies = [
"ansi_term",
"atty",
"bitflags",
"strsim",
"textwrap",
"unicode-width",
"vec_map",
]
[[package]]
name = "config"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b1b9d958c2b1368a663f05538fc1b5975adce1e19f435acceae987aceeeb369"
dependencies = [
"lazy_static",
"nom",
"rust-ini",
"serde 1.0.124",
"serde-hjson",
"serde_json",
"toml",
"yaml-rust",
]
[[package]]
name = "cpuid-bool"
version = "0.1.2"
@ -228,10 +294,12 @@ dependencies = [
name = "geoffrey_api"
version = "0.1.0"
dependencies = [
"config",
"geoffrey_db",
"geoffrey_models",
"serde",
"serde 1.0.124",
"serde_json",
"structopt",
"tokio",
"warp",
]
@ -243,7 +311,7 @@ dependencies = [
"byteorder",
"geoffrey_models",
"lazy_static",
"serde",
"serde 1.0.124",
"serde_json",
"sled",
]
@ -253,7 +321,7 @@ name = "geoffrey_models"
version = "0.1.0"
dependencies = [
"chrono",
"serde",
"serde 1.0.124",
"serde_json",
]
@ -329,6 +397,15 @@ dependencies = [
"http",
]
[[package]]
name = "heck"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac"
dependencies = [
"unicode-segmentation",
]
[[package]]
name = "hermit-abi"
version = "0.1.18"
@ -446,12 +523,31 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lexical-core"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
dependencies = [
"arrayvec",
"bitflags",
"cfg-if",
"ryu",
"static_assertions",
]
[[package]]
name = "libc"
version = "0.2.88"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03b07a082330a35e43f63177cc01689da34fbffa0105e1246cf0311472cac73a"
[[package]]
name = "linked-hash-map"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
[[package]]
name = "lock_api"
version = "0.4.2"
@ -548,6 +644,17 @@ dependencies = [
"twoway",
]
[[package]]
name = "nom"
version = "5.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af"
dependencies = [
"lexical-core",
"memchr",
"version_check",
]
[[package]]
name = "ntapi"
version = "0.3.6"
@ -564,7 +671,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db"
dependencies = [
"autocfg",
"num-traits",
"num-traits 0.2.14",
]
[[package]]
name = "num-traits"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31"
dependencies = [
"num-traits 0.2.14",
]
[[package]]
@ -667,6 +783,30 @@ version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
[[package]]
name = "proc-macro-error"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
dependencies = [
"proc-macro-error-attr",
"proc-macro2",
"quote",
"syn",
"version_check",
]
[[package]]
name = "proc-macro-error-attr"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
dependencies = [
"proc-macro2",
"quote",
"version_check",
]
[[package]]
name = "proc-macro2"
version = "1.0.24"
@ -781,6 +921,23 @@ dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a26af418b574bd56588335b3a3659a65725d4e636eb1016c2f9e3b38c7cc759"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
[[package]]
name = "remove_dir_all"
version = "0.5.3"
@ -790,6 +947,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "rust-ini"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2"
[[package]]
name = "ryu"
version = "1.0.5"
@ -814,6 +977,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "serde"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8"
[[package]]
name = "serde"
version = "1.0.124"
@ -823,6 +992,18 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde-hjson"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8"
dependencies = [
"lazy_static",
"num-traits 0.1.43",
"regex",
"serde 0.8.23",
]
[[package]]
name = "serde_derive"
version = "1.0.124"
@ -842,7 +1023,7 @@ checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79"
dependencies = [
"itoa",
"ryu",
"serde",
"serde 1.0.124",
]
[[package]]
@ -854,7 +1035,7 @@ dependencies = [
"form_urlencoded",
"itoa",
"ryu",
"serde",
"serde 1.0.124",
]
[[package]]
@ -918,6 +1099,42 @@ dependencies = [
"winapi",
]
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "strsim"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]]
name = "structopt"
version = "0.3.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5277acd7ee46e63e5168a80734c9f6ee81b1367a7d8772a2d765df2a3705d28c"
dependencies = [
"clap",
"lazy_static",
"structopt-derive",
]
[[package]]
name = "structopt-derive"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ba9cdfda491b814720b6b06e0cac513d922fc407582032e8706e9f137976f90"
dependencies = [
"heck",
"proc-macro-error",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "syn"
version = "1.0.61"
@ -943,6 +1160,15 @@ dependencies = [
"winapi",
]
[[package]]
name = "textwrap"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
dependencies = [
"unicode-width",
]
[[package]]
name = "time"
version = "0.1.44"
@ -1038,6 +1264,15 @@ dependencies = [
"tokio",
]
[[package]]
name = "toml"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa"
dependencies = [
"serde 1.0.124",
]
[[package]]
name = "tower-service"
version = "0.3.1"
@ -1142,6 +1377,18 @@ dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-segmentation"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
[[package]]
name = "unicode-width"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
[[package]]
name = "unicode-xid"
version = "0.2.1"
@ -1166,6 +1413,12 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7"
[[package]]
name = "vec_map"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "version_check"
version = "0.9.2"
@ -1200,7 +1453,7 @@ dependencies = [
"percent-encoding",
"pin-project",
"scoped-tls",
"serde",
"serde 1.0.124",
"serde_json",
"serde_urlencoded",
"tokio",
@ -1245,3 +1498,12 @@ name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "yaml-rust"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
dependencies = [
"linked-hash-map",
]

View File

@ -12,4 +12,6 @@ warp = "0.3"
serde = "1.0.124"
serde_json = "1.0.64"
geoffrey_models = { path = "../geoffrey_models" }
geoffrey_db = { path = "../geoffrey_db" }
geoffrey_db = { path = "../geoffrey_db" }
config = "0.11.0"
structopt = "0.3.21"

View File

@ -1,8 +1,7 @@
use crate::error::Result;
use crate::error::{Result, GeoffreyDBError};
use std::path::Path;
use geoffrey_models::GeoffreyDatabaseModel;
use crate::{u64_from_bytes, u64_to_bytes};
use sled::Iter;
use std::convert::TryInto;
pub struct Database {
db: sled::Db,
@ -31,8 +30,16 @@ impl Database {
}
};
let match_count = self.filter(|_, o: &T| {
!o.check_unique(&model)
})?.count();
if match_count > 0 {
return Err(GeoffreyDBError::NotUnique)
}
let tree = self.get_tree::<T>()?;
let id_bytes = u64_to_bytes(id);
let id_bytes = id.to_be_bytes();
tree.insert(id_bytes, model.to_bytes()?)?;
@ -42,7 +49,7 @@ impl Database {
pub fn get<T>(&self, id: u64) -> Result<Option<T>> where T: GeoffreyDatabaseModel {
let tree = self.get_tree::<T>()?;
let id_bytes = u64_to_bytes(id);
let id_bytes = id.to_be_bytes();
if let Some(bytes) = tree.get(id_bytes)? {
Ok(Some(T::try_from_bytes(&bytes)?))
@ -53,12 +60,16 @@ impl Database {
}
pub fn filter<T, F>(&self, f: F) -> Result<Vec<T>> where T: GeoffreyDatabaseModel, F: Fn(u64, &T) -> bool {
pub fn clear_tree<T>(&self) -> Result<()> where T: GeoffreyDatabaseModel {
Ok(self.db.open_tree(T::tree())?.clear()?)
}
pub fn filter<'a, T>(&self, f: impl Fn(u64, &T) -> bool + 'a) -> Result<impl Iterator<Item=T> + 'a> where T: GeoffreyDatabaseModel {
let tree = self.db.open_tree(T::tree())?;
Ok(tree.iter().filter_map(|e| {
Ok(tree.iter().filter_map(move |e| {
if let Ok((id, data)) = e {
let id = u64_from_bytes(&mut id.clone());
let id = u64::from_be_bytes(id.to_vec().try_into().unwrap());
let data = T::try_from_bytes(&data).unwrap();
if f(id, &data) {
@ -71,10 +82,10 @@ impl Database {
else {
None
}
}).collect())
}))
}
pub fn tree_iter<T>(&self) -> Result<Iter> where T: GeoffreyDatabaseModel {
pub fn tree_iter<T>(&self) -> Result<sled::Iter> where T: GeoffreyDatabaseModel {
Ok(self.db.open_tree(T::tree()).map(|tree| tree.iter())?)
}
}
@ -86,13 +97,18 @@ mod tests {
use geoffrey_models::models::player::{Player, UserID};
use lazy_static::lazy_static;
use geoffrey_models::GeoffreyDatabaseModel;
use geoffrey_models::models::locations::{Location, LocationType};
use geoffrey_models::models::{Position, Dimension};
use std::time::Instant;
lazy_static! {
static ref DB: Database = Database::new(Path::new("../locations")).unwrap();
static ref DB: Database = Database::new(Path::new("../test_database")).unwrap();
}
fn cleanup() {
DB.db.clear().unwrap();
DB.clear_tree::<Player>().unwrap();
DB.clear_tree::<Location>().unwrap();
DB.db.flush().unwrap();
}
#[test]
@ -103,6 +119,19 @@ mod tests {
assert!(p2.id().is_some());
assert_eq!(player.name, p2.name);
cleanup();
}
#[test]
fn test_unique_insert() {
let player1 = Player::new("CoolZero123", UserID::DiscordUUID(0u64));
let player2 = Player::new("CoolZero123", UserID::DiscordUUID(0u64));
DB.insert::<Player>(player1.clone()).unwrap();
assert_eq!(DB.insert::<Player>(player2.clone()).is_err(), true);
cleanup();
}
@ -118,5 +147,40 @@ mod tests {
assert_eq!(p3.unwrap().name, player.name);
cleanup();
}
#[test]
fn test_filter() {
let player = Player::new("CoolZero123", UserID::DiscordUUID(0u64));
let player = DB.insert::<Player>(player.clone()).unwrap();
let loc = Location::new("Test Shop", Position::new(0, 0, Dimension::Overworld), player.id.unwrap(), None, LocationType::Shop(0));
let loc = DB.insert::<Location>(loc.clone()).unwrap();
let count = DB.filter(|id: u64, l: &Location| {
assert_eq!(id, l.id().unwrap());
loc.id().unwrap() == id
}).unwrap().count();
assert_eq!(count, 1);
DB.db.flush().unwrap();
cleanup();
}
#[test]
fn test_insert_speed() {
cleanup();
let insert_count = 1000;
let timer = Instant::now();
for i in 0..insert_count {
let player = Player::new("test", UserID::DiscordUUID(i));
DB.insert::<Player>(player).unwrap();
}
DB.db.flush().unwrap();
let sec_elapsed = timer.elapsed().as_secs_f32();
println!("Completed in {}s. {} inserts per second", sec_elapsed, insert_count as f32/sec_elapsed);
cleanup()
}
}

View File

@ -1,10 +1,10 @@
pub type Result<T> = std::result::Result<T, GeoffreyDBError>;
#[derive(Debug)]
pub enum GeoffreyDBError {
SledError(sled::Error),
SerdeJsonError(serde_json::Error),
NotUnique,
}
impl std::error::Error for GeoffreyDBError {}
@ -13,7 +13,8 @@ impl std::fmt::Display for GeoffreyDBError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
GeoffreyDBError::SledError(e) => write!(f, "Sled Error: {}", e),
GeoffreyDBError::SerdeJsonError(e) => write!(f, "Serde JSON Error: {}", e)
GeoffreyDBError::SerdeJsonError(e) => write!(f, "Serde JSON Error: {}", e),
GeoffreyDBError::NotUnique => write!(f, "Entry is not unique.")
}
}
}

View File

@ -1,18 +1,4 @@
#![allow(dead_code)]
use byteorder::{WriteBytesExt, BigEndian, ReadBytesExt};
use std::io::Cursor;
pub mod database;
pub mod error;
pub fn u64_to_bytes(n: u64) -> Vec<u8> {
let mut id_bytes= vec![0u8; 8];
id_bytes.write_u64::<BigEndian>(n).unwrap();
id_bytes
}
pub fn u64_from_bytes(bytes: &mut [u8]) -> u64 {
let mut cursor = Cursor::new(bytes);
cursor.read_u64::<BigEndian>().unwrap()
}

View File

@ -70,4 +70,28 @@ impl GeoffreyDatabaseModel for Location {
fn tree() -> String {
"location".to_string()
}
fn check_unique(&self, o: &Self) -> bool {
o.name.to_lowercase() != self.name.to_lowercase()
}
}
#[cfg(test)]
mod tests {
use crate::GeoffreyDatabaseModel;
use crate::models::locations::{Location, LocationType};
use crate::models::{Position, Dimension};
#[test]
fn test_location_check_unique() {
let l1 = Location::new("Test", Position::new(0, 0, Dimension::Overworld), 0u64, None, LocationType::Base);
let l2 = Location::new("NotTest", Position::new(0, 0, Dimension::Overworld), 0u64, None, LocationType::Base);
assert!(l1.check_unique(&l2));
let l1 = Location::new("Test", Position::new(0, 0, Dimension::Overworld), 0u64, None, LocationType::Base);
let l2 = Location::new("teSt", Position::new(0, 0, Dimension::Overworld), 0u64, None, LocationType::Base);
assert!(!l1.check_unique(&l2));
}
}

View File

@ -36,4 +36,34 @@ impl GeoffreyDatabaseModel for Player {
fn tree() -> String {
"player".to_string()
}
fn check_unique(&self, o: &Self) -> bool {
for user1_id in &self.user_ids {
if o.user_ids.contains(user1_id) {
return false;
}
}
true
}
}
#[cfg(test)]
mod tests {
use crate::models::player::Player;
use crate::models::player::UserID::{DiscordUUID, MinecraftUUID};
use crate::GeoffreyDatabaseModel;
#[test]
fn test_player_check_unique() {
let p1 = Player::new("CoolTest123", DiscordUUID(0u64));
let p2 = Player::new("NotCoolTest123", DiscordUUID(1u64));
assert!(p1.check_unique(&p2));
let p1 = Player::new("CoolTest123", MinecraftUUID("0".to_string()));
let p2 = Player::new("NotCoolTest123", MinecraftUUID("0".to_string()));
assert!(!p1.check_unique(&p2));
}
}