From 6cf621f15608a4b844c6366b5eb3aa23a9843917 Mon Sep 17 00:00:00 2001 From: Kevin Belisle Date: Thu, 1 Jul 2021 16:10:44 -0400 Subject: [PATCH] Add LiveStatistics table --- app/src/main/kotlin/xyz/etztech/stonks/Api.kt | 93 ++++++++----------- app/src/main/kotlin/xyz/etztech/stonks/App.kt | 12 +++ app/src/main/kotlin/xyz/etztech/stonks/DSL.kt | 10 ++ .../xyz/etztech/stonks/StatisticsImporter.kt | 46 +++++++++ 4 files changed, 106 insertions(+), 55 deletions(-) diff --git a/app/src/main/kotlin/xyz/etztech/stonks/Api.kt b/app/src/main/kotlin/xyz/etztech/stonks/Api.kt index 0511f42..82f6d11 100644 --- a/app/src/main/kotlin/xyz/etztech/stonks/Api.kt +++ b/app/src/main/kotlin/xyz/etztech/stonks/Api.kt @@ -7,7 +7,8 @@ import kotlinx.coroutines.* import kotlinx.serialization.* import kotlinx.serialization.json.Json import org.jetbrains.exposed.sql.* -import org.jetbrains.exposed.sql.transactions.transaction +import org.jetbrains.exposed.sql.transactions.* +import xyz.etztech.stonks.dsl.LiveStatistics import xyz.etztech.stonks.dsl.Statistics private var statisticsCache = "" @@ -22,26 +23,6 @@ fun initApiServer(apiServerPort: Int, database: Database) { val app = Javalin.create().start(apiServerPort) println("Javalin web server started") - app.get("/") { ctx -> - run { - transaction(database) { - addLogger(StdOutSqlLogger) - val dataPoints = - Statistics.slice(Statistics.playerId.count()).selectAll().limit(1).map { - it[Statistics.playerId.count()] - }[0] - - val playerCount = - Statistics.slice(Statistics.playerId.countDistinct()) - .selectAll() - .limit(1) - .map { it[Statistics.playerId.countDistinct()] }[0] - - ctx.result("$dataPoints data points from $playerCount players!") - } - } - } - app.get("/statistics") { ctx -> run { if (statisticsCache.isEmpty() or @@ -51,10 +32,12 @@ fun initApiServer(apiServerPort: Int, database: Database) { addLogger(StdOutSqlLogger) val statistics = - Statistics.slice(Statistics.type, Statistics.name) + LiveStatistics.slice(LiveStatistics.type, LiveStatistics.name) .selectAll() - .groupBy(Statistics.type, Statistics.name) - .map { Statistic(it[Statistics.type], it[Statistics.name]) } + .groupBy(LiveStatistics.type, LiveStatistics.name) + .map { + Statistic(it[LiveStatistics.type], it[LiveStatistics.name]) + } statisticsCache = Json.encodeToString(statistics) } @@ -68,26 +51,25 @@ fun initApiServer(apiServerPort: Int, database: Database) { run { transaction(database) { addLogger(StdOutSqlLogger) - val maxExpr = Statistics.value.max() - val statistics = - Statistics.slice( - Statistics.type, - Statistics.name, - Statistics.playerId, - maxExpr) + LiveStatistics.slice( + LiveStatistics.type, + LiveStatistics.name, + LiveStatistics.playerId, + LiveStatistics.value, + LiveStatistics.rank) .select { - Statistics.type.eq(ctx.pathParam("type")) and - Statistics.name.eq(ctx.pathParam("name")) + LiveStatistics.type.eq(ctx.pathParam("type")) and + LiveStatistics.name.eq(ctx.pathParam("name")) } - .groupBy(Statistics.playerId) - .orderBy(maxExpr, SortOrder.DESC) + .orderBy(LiveStatistics.rank, SortOrder.ASC) .map { StatisticValue( - it[Statistics.playerId], - it[Statistics.type], - it[Statistics.name], - it[maxExpr]!!) + it[LiveStatistics.playerId], + it[LiveStatistics.type], + it[LiveStatistics.name], + it[LiveStatistics.value], + it[LiveStatistics.rank]) } ctx.result(Json { prettyPrint = true }.encodeToString(statistics)) @@ -104,10 +86,10 @@ fun initApiServer(apiServerPort: Int, database: Database) { addLogger(StdOutSqlLogger) val players = - Statistics.slice(Statistics.playerId) + LiveStatistics.slice(LiveStatistics.playerId) .selectAll() - .groupBy(Statistics.playerId) - .map { it[Statistics.playerId] } + .groupBy(LiveStatistics.playerId) + .map { it[LiveStatistics.playerId] } playersCache = Json.encodeToString(players) } } @@ -120,22 +102,22 @@ fun initApiServer(apiServerPort: Int, database: Database) { run { transaction(database) { addLogger(StdOutSqlLogger) - val maxExpr = Statistics.value.max() val statistics = - Statistics.slice( - Statistics.type, - Statistics.name, - Statistics.playerId, - maxExpr) - .select { Statistics.playerId.eq(ctx.pathParam("playerId")) } - .groupBy(Statistics.type, Statistics.name) + LiveStatistics.slice( + LiveStatistics.type, + LiveStatistics.name, + LiveStatistics.playerId, + LiveStatistics.value, + LiveStatistics.rank) + .select { LiveStatistics.playerId.eq(ctx.pathParam("playerId")) } .map { StatisticValue( - it[Statistics.playerId], - it[Statistics.type], - it[Statistics.name], - it[maxExpr]!!) + it[LiveStatistics.playerId], + it[LiveStatistics.type], + it[LiveStatistics.name], + it[LiveStatistics.value], + it[LiveStatistics.rank]) } ctx.result(Json { prettyPrint = true }.encodeToString(statistics)) @@ -182,7 +164,8 @@ data class StatisticValue( val playerId: String, val type: String, val name: String, - val value: Long + val value: Long, + val rank: Int ) @Serializable diff --git a/app/src/main/kotlin/xyz/etztech/stonks/App.kt b/app/src/main/kotlin/xyz/etztech/stonks/App.kt index 73a75fb..b50a317 100644 --- a/app/src/main/kotlin/xyz/etztech/stonks/App.kt +++ b/app/src/main/kotlin/xyz/etztech/stonks/App.kt @@ -10,6 +10,7 @@ import org.jetbrains.exposed.sql.* import org.jetbrains.exposed.sql.transactions.TransactionManager import org.jetbrains.exposed.sql.transactions.transaction import xyz.etztech.stonks.api.initApiServer +import xyz.etztech.stonks.dsl.LiveStatistics import xyz.etztech.stonks.dsl.Statistics import xyz.etztech.stonks.statisticsimporter.StatisticsImporter @@ -76,6 +77,7 @@ fun initH2Server( transaction { addLogger(StdOutSqlLogger) SchemaUtils.create(Statistics) + SchemaUtils.create(LiveStatistics) // Create indexes with explicit SQL because I can't figure out how to do it with exposed // Wrap it in a try block because it throws an exception if index already exists @@ -88,6 +90,16 @@ fun initH2Server( TransactionManager.current() .exec("CREATE INDEX idx_type_name ON Statistics (\"Type\", \"Name\")") } catch (e: ExposedSQLException) {} + + try { + TransactionManager.current() + .exec("CREATE INDEX idx_playerid ON LiveStatistics (\"PlayerId\")") + } catch (e: ExposedSQLException) {} + + try { + TransactionManager.current() + .exec("CREATE INDEX idx_type_name ON LiveStatistics (\"Type\", \"Name\")") + } catch (e: ExposedSQLException) {} } return database diff --git a/app/src/main/kotlin/xyz/etztech/stonks/DSL.kt b/app/src/main/kotlin/xyz/etztech/stonks/DSL.kt index b2092f3..0188e71 100644 --- a/app/src/main/kotlin/xyz/etztech/stonks/DSL.kt +++ b/app/src/main/kotlin/xyz/etztech/stonks/DSL.kt @@ -4,6 +4,16 @@ import java.time.Instant import org.jetbrains.exposed.sql.* import org.jetbrains.exposed.sql.`java-time`.timestamp +object LiveStatistics : Table() { + val playerId: Column = varchar("PlayerId", 150) + val type: Column = varchar("Type", 150) + val name: Column = varchar("Name", 150) + val value: Column = long("Value") + val rank: Column = integer("Rank") + + override val primaryKey = PrimaryKey(playerId, type, name, name = "PK_playerId_type_name") +} + object Statistics : Table() { val playerId: Column = varchar("PlayerId", 150) val type: Column = varchar("Type", 150) diff --git a/app/src/main/kotlin/xyz/etztech/stonks/StatisticsImporter.kt b/app/src/main/kotlin/xyz/etztech/stonks/StatisticsImporter.kt index 20aebda..80a949b 100644 --- a/app/src/main/kotlin/xyz/etztech/stonks/StatisticsImporter.kt +++ b/app/src/main/kotlin/xyz/etztech/stonks/StatisticsImporter.kt @@ -6,6 +6,7 @@ import java.io.File import java.time.Instant import org.jetbrains.exposed.sql.* import org.jetbrains.exposed.sql.`java-time`.timestamp +import org.jetbrains.exposed.sql.transactions.TransactionManager import org.jetbrains.exposed.sql.transactions.transaction import xyz.etztech.stonks.dsl.Statistics @@ -15,7 +16,10 @@ object StatisticsImporter { private val klaxon = Klaxon() fun importStatistics(folder: String, database: Database) { + println("Importing new statistics...") File(folder).listFiles().forEach { readFile(it, database) } + println("Updating live statistics table...") + updateLiveStatistics(database) } fun readFile(file: File, database: Database) { @@ -58,6 +62,48 @@ object StatisticsImporter { } } + fun updateLiveStatistics(database: Database) { + transaction(database) { + TransactionManager.current() + .exec( + """ + MERGE INTO LIVESTATISTICS AS T USING ( + SELECT + STATISTICS."Type", + STATISTICS."Name", + STATISTICS."PlayerId", + STATISTICS."Value", + (RANK () OVER (PARTITION BY STATISTICS."Type", STATISTICS."Name" ORDER BY STATISTICS."Value" DESC)) AS "Rank" + FROM + STATISTICS + JOIN ( + SELECT + "Type", + "Name", + "PlayerId", + MAX("Timestamp") AS "MaxTimestamp", + FROM STATISTICS + GROUP BY + "Type", + "Name", + "PlayerId" + ) MAX_TIMESTAMPS + ON + STATISTICS."Type" = MAX_TIMESTAMPS."Type" + AND STATISTICS."Name" = MAX_TIMESTAMPS."Name" + AND STATISTICS."PlayerId" = MAX_TIMESTAMPS."PlayerId" + AND STATISTICS."Timestamp" = MAX_TIMESTAMPS."MaxTimestamp" + ) AS S + ON (T."Type" = S."Type" AND T."Name" = S."Name" AND T."PlayerId" = S."PlayerId") + + WHEN MATCHED AND (S."Value" <> T."Value" OR S."Rank" <> T."Rank")THEN + UPDATE SET T."Value" = S."Value", T."Rank" = S."Rank" + WHEN NOT MATCHED THEN + INSERT VALUES (S."PlayerId", S."Type", S."Name", S."Value", S."Rank") + """.trimIndent()) + } + } + data class StatsFile(val stats: Map>) { companion object { public fun fromJson(json: String) = klaxon.parse(json)