From 634d6a778d422f3da72608fedcf1aed09978d6d4 Mon Sep 17 00:00:00 2001 From: DaXcess Date: Fri, 3 Mar 2023 10:03:57 +0100 Subject: [PATCH] Added metrics --- .github/workflows/build-push.yml | 1 + Cargo.lock | 45 +++++++++-- Cargo.toml | 8 +- Dockerfile.metrics | 23 ++++++ README.md | 1 + src/bot/events.rs | 13 +++- src/consts.rs | 2 +- src/main.rs | 66 +++++++++------- src/metrics.rs | 124 +++++++++++++++++++++++++++++++ src/session/manager.rs | 2 + src/session/mod.rs | 23 ++++++ src/session/pbi.rs | 24 ++++++ src/stats.rs | 26 ------- src/utils/spotify.rs | 7 ++ 14 files changed, 300 insertions(+), 65 deletions(-) create mode 100644 Dockerfile.metrics create mode 100644 src/metrics.rs delete mode 100644 src/stats.rs diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 580d657..507765c 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -28,6 +28,7 @@ jobs: uses: docker/build-push-action@v2 with: context: . + file: ./Dockerfile.metrics tags: | ${{ secrets.REGISTRY_URL }}/spoticord/spoticord:latest push: ${{ github.ref == 'refs/heads/main' }} diff --git a/Cargo.lock b/Cargo.lock index a494de8..6911e8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1961,6 +1961,37 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de8dacb0873f77e6aefc6d71e044761fcc68060290f5b1089fcdf84626bb69" +dependencies = [ + "bitflags", + "byteorder", + "hex", + "lazy_static", + "rustix", +] + +[[package]] +name = "prometheus" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "libc", + "memchr", + "parking_lot", + "procfs", + "protobuf", + "reqwest", + "thiserror", +] + [[package]] name = "protobuf" version = "2.28.0" @@ -2556,9 +2587,9 @@ dependencies = [ [[package]] name = "songbird" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4c965f6625a2653e0733abfe217679562eb8c4787f000f4f13047541a444217" +checksum = "32637904e4c41947b951853390840471bf8a8a96161a57db5cf7141989e89a6a" dependencies = [ "async-trait", "async-tungstenite", @@ -2604,13 +2635,15 @@ dependencies = [ [[package]] name = "spoticord" -version = "2.0.0-beta" +version = "2.0.0-pre" dependencies = [ "dotenv", "env_logger 0.10.0", "ipc-channel", + "lazy_static", "librespot", "log", + "prometheus", "redis", "reqwest", "samplerate", @@ -2789,9 +2822,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -2804,7 +2837,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a6f9bcf..c6c9007 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "spoticord" -version = "2.0.0-beta" +version = "2.0.0-pre" edition = "2021" rust-version = "1.64.0" @@ -8,12 +8,18 @@ rust-version = "1.64.0" name = "spoticord" path = "src/main.rs" +[features] +default = [] +metrics = ["lazy_static", "prometheus"] + [dependencies] dotenv = "0.15.0" env_logger = "0.10.0" ipc-channel = { version = "0.16.0", features = ["async"] } +lazy_static = { version = "1.4.0", optional = true } librespot = { version = "0.4.2", default-features = false } log = "0.4.17" +prometheus = { version = "0.13.3", optional = true, features = ["push", "process"] } redis = "0.22.3" reqwest = "0.11.14" samplerate = "0.2.4" diff --git a/Dockerfile.metrics b/Dockerfile.metrics new file mode 100644 index 0000000..74d3e3b --- /dev/null +++ b/Dockerfile.metrics @@ -0,0 +1,23 @@ +# Builder +FROM rust:1.64-buster as builder + +WORKDIR /app + +# Add extra build dependencies here +RUN apt-get update && apt-get install -y cmake + +COPY . . +RUN cargo install --path . --features metrics + +# Runtime +FROM debian:buster-slim + +WORKDIR /app + +# Add extra runtime dependencies here +RUN apt-get update && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/* + +# Copy spoticord binary from builder +COPY --from=builder /usr/local/cargo/bin/spoticord ./spoticord + +CMD ["./spoticord"] \ No newline at end of file diff --git a/README.md b/README.md index 0ee15e2..2523bac 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Spoticord uses environment variables to configure itself. The following variable Additionally you can configure the following variables: - `GUILD_ID`: The ID of the Discord server where this bot will create commands for. This is used during testing to prevent the bot from creating slash commands in other servers, as well as getting the commands quicker. This variable is optional, and if not set, the bot will create commands in all servers it is in (this may take up to 15 minutes). - `KV_URL`: The connection URL of a redis-server instance used for storing realtime data. While not required, not providing one will cause the bot to spit out quite a bit of errors. You might want to comment out those error lines in `main.rs`. +- `METRICS_URL`: The connection URL of a Prometheus Push Gateway server used for pushing metrics. This variable is required when compiling with the `metrics` feature. #### Providing environment variables You can provide environment variables in a `.env` file at the root of the working directory of Spoticord. diff --git a/src/bot/events.rs b/src/bot/events.rs index 32d635b..b899fcc 100644 --- a/src/bot/events.rs +++ b/src/bot/events.rs @@ -1,5 +1,7 @@ /* This file implements all events for the Discord gateway */ +use super::commands::CommandManager; +use crate::consts::MOTD; use log::*; use serenity::{ async_trait, @@ -13,9 +15,8 @@ use serenity::{ prelude::{Context, EventHandler}, }; -use crate::consts::MOTD; - -use super::commands::CommandManager; +#[cfg(feature = "metrics")] +use crate::metrics::MetricsManager; // If the GUILD_ID environment variable is set, only allow commands from that guild macro_rules! enforce_guild { @@ -101,6 +102,12 @@ impl Handler { let data = ctx.data.read().await; let command_manager = data.get::().expect("to contain a value"); + #[cfg(feature = "metrics")] + { + let metrics = data.get::().expect("to contain a value"); + metrics.command_exec(&command.data.name); + } + command_manager.execute_command(&ctx, command).await; } diff --git a/src/consts.rs b/src/consts.rs index fe8da6a..4e258aa 100644 --- a/src/consts.rs +++ b/src/consts.rs @@ -1,5 +1,5 @@ pub const VERSION: &str = env!("CARGO_PKG_VERSION"); -pub const MOTD: &str = "OPEN BETA (v2)"; +pub const MOTD: &str = "PRE-RELEASE (v2)"; /// The time it takes for Spoticord to disconnect when no music is being played pub const DISCONNECT_TIME: u64 = 5 * 60; diff --git a/src/main.rs b/src/main.rs index 57cf10a..1d1f17e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,18 +1,20 @@ use dotenv::dotenv; +use crate::{bot::commands::CommandManager, database::Database, session::manager::SessionManager}; use log::*; use serenity::{framework::StandardFramework, prelude::GatewayIntents, Client}; use songbird::SerenityInit; use std::{any::Any, env, process::exit}; -use crate::{ - bot::commands::CommandManager, database::Database, session::manager::SessionManager, - stats::StatsManager, -}; +#[cfg(feature = "metrics")] +use metrics::MetricsManager; #[cfg(unix)] use tokio::signal::unix::SignalKind; +#[cfg(feature = "metrics")] +mod metrics; + mod audio; mod bot; mod consts; @@ -21,7 +23,6 @@ mod ipc; mod librespot_ext; mod player; mod session; -mod stats; mod utils; #[tokio::main] @@ -70,9 +71,13 @@ async fn main() { let token = env::var("DISCORD_TOKEN").expect("a token in the environment"); let db_url = env::var("DATABASE_URL").expect("a database URL in the environment"); - let kv_url = env::var("KV_URL").expect("a redis URL in the environment"); - let stats_manager = StatsManager::new(kv_url).expect("Failed to connect to redis"); + #[cfg(feature = "metrics")] + let metrics_manager = { + let metrics_url = env::var("METRICS_URL").expect("a prometheus pusher URL in the environment"); + MetricsManager::new(metrics_url) + }; + let session_manager = SessionManager::new(); // Create client @@ -92,10 +97,13 @@ async fn main() { data.insert::(Database::new(db_url, None)); data.insert::(CommandManager::new()); data.insert::(session_manager.clone()); + + #[cfg(feature = "metrics")] + data.insert::(metrics_manager.clone()); } let shard_manager = client.shard_manager.clone(); - let cache = client.cache_and_http.cache.clone(); + let _cache = client.cache_and_http.cache.clone(); #[cfg(unix)] let mut term: Option> = Some(Box::new( @@ -111,28 +119,27 @@ async fn main() { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { - let guild_count = cache.guilds().len(); - let active_count = session_manager.get_active_session_count().await; - let total_count = session_manager.get_session_count().await; + #[cfg(feature = "metrics")] + { + let guild_count = _cache.guilds().len(); + let active_count = session_manager.get_active_session_count().await; + let total_count = session_manager.get_session_count().await; - if let Err(why) = stats_manager.set_server_count(guild_count) { - error!("Failed to update server count: {}", why); + metrics_manager.set_server_count(guild_count); + metrics_manager.set_active_sessions(active_count); + metrics_manager.set_total_sessions(total_count); + + // Yes, I like to handle my s's when I'm working with amounts + debug!( + "Updated metrics: {} guild{}, {} active session{}, {} total session{}", + guild_count, + if guild_count == 1 { "" } else { "s" }, + active_count, + if active_count == 1 { "" } else { "s" }, + total_count, + if total_count == 1 { "" } else { "s" } + ); } - - if let Err(why) = stats_manager.set_active_count(active_count) { - error!("Failed to update active count: {}", why); - } - - // Yes, I like to handle my s's when I'm working with amounts - debug!( - "Updated stats: {} guild{}, {} active session{}, {} total session{}", - guild_count, - if guild_count == 1 { "" } else { "s" }, - active_count, - if active_count == 1 { "" } else { "s" }, - total_count, - if total_count == 1 { "" } else { "s" } - ); } _ = tokio::signal::ctrl_c() => { @@ -159,6 +166,9 @@ async fn main() { shard_manager.lock().await.shutdown_all().await; + #[cfg(feature = "metrics")] + metrics_manager.stop(); + break; } } diff --git a/src/metrics.rs b/src/metrics.rs new file mode 100644 index 0000000..00a0658 --- /dev/null +++ b/src/metrics.rs @@ -0,0 +1,124 @@ +use std::{ + collections::hash_map::RandomState, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + thread, + time::Duration, +}; + +use lazy_static::lazy_static; +use prometheus::{ + opts, push_metrics, register_int_counter_vec, register_int_gauge, IntCounterVec, IntGauge, +}; +use serenity::prelude::TypeMapKey; + +use crate::session::pbi::PlaybackInfo; + +lazy_static! { + static ref TOTAL_SERVERS: IntGauge = + register_int_gauge!("total_servers", "Total number of servers Spoticord is in").unwrap(); + static ref ACTIVE_SESSIONS: IntGauge = register_int_gauge!( + "active_sessions", + "Total number of servers with an active Spoticord session" + ) + .unwrap(); + static ref TOTAL_SESSIONS: IntGauge = register_int_gauge!( + "total_sessions", + "Total number of servers with Spoticord in a voice channel" + ) + .unwrap(); + static ref TRACKS_PLAYED: IntCounterVec = register_int_counter_vec!( + opts!("tracks_played", "Tracks Played"), + &["type", "name", "artists", "uri"] + ) + .unwrap(); + static ref COMMANDS_EXECUTED: IntCounterVec = register_int_counter_vec!( + opts!("commands_executed", "Commands Executed"), + &["command"] + ) + .unwrap(); +} + +#[derive(Clone)] +pub struct MetricsManager { + should_stop: Arc, +} + +impl MetricsManager { + pub fn new(pusher_url: impl Into) -> Self { + let instance = Self { + should_stop: Arc::new(AtomicBool::new(false)), + }; + + thread::spawn({ + let instance = instance.clone(); + let pusher_url = pusher_url.into(); + + move || loop { + thread::sleep(Duration::from_secs(5)); + + if instance.should_stop() { + break; + } + + if let Err(why) = push_metrics::( + "spoticord_metrics", + Default::default(), + &pusher_url, + prometheus::gather(), + None, + ) { + log::error!("Failed to push metrics: {}", why); + } + } + }); + + instance + } + + pub fn should_stop(&self) -> bool { + self.should_stop.load(Ordering::Relaxed) + } + + pub fn stop(&self) { + self.should_stop.store(true, Ordering::Relaxed); + } + + pub fn set_server_count(&self, count: usize) { + TOTAL_SERVERS.set(count as i64); + } + + pub fn set_total_sessions(&self, count: usize) { + TOTAL_SESSIONS.set(count as i64); + } + + pub fn set_active_sessions(&self, count: usize) { + ACTIVE_SESSIONS.set(count as i64); + } + + pub fn track_play(&self, track: &PlaybackInfo) { + let track_type = match track.get_type() { + Some(track_type) => track_type, + None => return, + }; + + TRACKS_PLAYED + .with_label_values(&[ + &track_type, + &track.get_name().expect("To have a name"), + &track.get_artists().expect("To have artists"), + &track.get_url().expect("To have a URL"), + ]) + .inc(); + } + + pub fn command_exec(&self, command: &str) { + COMMANDS_EXECUTED.with_label_values(&[command]).inc(); + } +} + +impl TypeMapKey for MetricsManager { + type Value = MetricsManager; +} diff --git a/src/session/manager.rs b/src/session/manager.rs index a2404d1..222a4d3 100644 --- a/src/session/manager.rs +++ b/src/session/manager.rs @@ -173,11 +173,13 @@ impl SessionManager { } /// Get the amount of sessions + #[allow(dead_code)] pub async fn get_session_count(&self) -> usize { self.0.read().await.get_session_count() } /// Get the amount of sessions with an owner + #[allow(dead_code)] pub async fn get_active_session_count(&self) -> usize { self.0.read().await.get_active_session_count().await } diff --git a/src/session/mod.rs b/src/session/mod.rs index 70be9d4..f4be02c 100644 --- a/src/session/mod.rs +++ b/src/session/mod.rs @@ -33,6 +33,9 @@ use std::{ }; use tokio::sync::Mutex; +#[cfg(feature = "metrics")] +use crate::metrics::MetricsManager; + #[derive(Clone)] pub struct SpoticordSession(Arc>); @@ -58,6 +61,9 @@ struct InnerSpoticordSession { /// Whether the session has been disconnected /// If this is true then this instance should no longer be used and dropped disconnected: bool, + + #[cfg(feature = "metrics")] + metrics: MetricsManager, } impl SpoticordSession { @@ -75,6 +81,12 @@ impl SpoticordSession { .expect("to contain a value") .clone(); + #[cfg(feature = "metrics")] + let metrics = data + .get::() + .expect("to contain a value") + .clone(); + // Join the voice channel let songbird = songbird::get(ctx).await.expect("to be present").clone(); @@ -98,6 +110,9 @@ impl SpoticordSession { disconnect_handle: None, client: None, disconnected: false, + + #[cfg(feature = "metrics")] + metrics, }; let mut instance = Self(Arc::new(RwLock::new(inner))); @@ -522,6 +537,14 @@ impl SpoticordSession { pbi.update_track_episode(spotify_id, track, episode); } + // Send track play event to metrics + #[cfg(feature = "metrics")] + { + if let Some(ref pbi) = inner.playback_info { + inner.metrics.track_play(pbi); + } + } + Ok(()) } diff --git a/src/session/pbi.rs b/src/session/pbi.rs index 36df553..293805b 100644 --- a/src/session/pbi.rs +++ b/src/session/pbi.rs @@ -106,4 +106,28 @@ impl PlaybackInfo { None } } + + /// Get the type of audio (track or episode) + #[allow(dead_code)] + pub fn get_type(&self) -> Option { + if self.track.is_some() { + Some("track".into()) + } else if self.episode.is_some() { + Some("episode".into()) + } else { + None + } + } + + /// Get the public facing url of the track or episode + #[allow(dead_code)] + pub fn get_url(&self) -> Option<&str> { + if let Some(ref track) = self.track { + Some(track.external_urls.spotify.as_str()) + } else if let Some(ref episode) = self.episode { + Some(episode.external_urls.spotify.as_str()) + } else { + None + } + } } diff --git a/src/stats.rs b/src/stats.rs deleted file mode 100644 index 93d8a59..0000000 --- a/src/stats.rs +++ /dev/null @@ -1,26 +0,0 @@ -use redis::{Commands, RedisResult}; - -#[derive(Clone)] -pub struct StatsManager { - redis: redis::Client, -} - -impl StatsManager { - pub fn new(url: impl Into) -> RedisResult { - let redis = redis::Client::open(url.into())?; - - Ok(StatsManager { redis }) - } - - pub fn set_server_count(&self, count: usize) -> RedisResult<()> { - let mut con = self.redis.get_connection()?; - - con.set("sc-bot-total-servers", count.to_string()) - } - - pub fn set_active_count(&self, count: usize) -> RedisResult<()> { - let mut con = self.redis.get_connection()?; - - con.set("sc-bot-active-servers", count.to_string()) - } -} diff --git a/src/utils/spotify.rs b/src/utils/spotify.rs index 31929c3..3a7dfe4 100644 --- a/src/utils/spotify.rs +++ b/src/utils/spotify.rs @@ -23,11 +23,17 @@ pub struct Album { pub images: Vec, } +#[derive(Debug, Clone, Deserialize)] +pub struct ExternalUrls { + pub spotify: String, +} + #[derive(Debug, Clone, Deserialize)] pub struct Track { pub name: String, pub artists: Vec, pub album: Album, + pub external_urls: ExternalUrls, } #[derive(Debug, Clone, Deserialize)] @@ -40,6 +46,7 @@ pub struct Show { pub struct Episode { pub name: String, pub show: Show, + pub external_urls: ExternalUrls, } pub async fn get_username(token: impl Into) -> Result {