package xyz.etztech.stonks.statisticsimporter import com.beust.klaxon.* import com.beust.klaxon.Klaxon import io.ktor.client.* import io.ktor.client.engine.java.* import io.ktor.client.request.* import io.ktor.client.statement.* import io.ktor.http.* import java.io.File import java.time.Duration import java.time.Instant import kotlinx.coroutines.* import org.jetbrains.exposed.sql.* import org.jetbrains.exposed.sql.`java-time`.timestamp import org.jetbrains.exposed.sql.transactions.TransactionManager import org.jetbrains.exposed.sql.transactions.transaction import xyz.etztech.stonks.dsl.LiveStatistics import xyz.etztech.stonks.dsl.Players import xyz.etztech.stonks.dsl.Statistics object StatisticsImporter { init {} private val httpClient = HttpClient(Java) private val klaxon = Klaxon() fun importStatistics(folder: String, database: Database) { println("Starting new statistics import.") File(folder).listFiles().forEach { readFile(it, database) } println("Updating live statistics table...") updateLiveStatistics(database) println("Updating aggregate statistics table...") updateAggregateStatistics(database) println("Refreshing player names...") refreshPlayerNames(database) println("Finished new statistics import.") } fun readFile(file: File, database: Database) { val statsFile = StatsFile.fromJson(file.readText()) val playerId = file.nameWithoutExtension val playerStats = emptyMap>().toMutableMap() transaction(database) { LiveStatistics.slice(LiveStatistics.type, LiveStatistics.name, LiveStatistics.value) .select { LiveStatistics.playerId eq playerId } .forEach { if (playerStats.containsKey(it[LiveStatistics.type])) { playerStats[it[LiveStatistics.type]]?.put( it[LiveStatistics.name], it[LiveStatistics.value] ) } else { playerStats.put( it[LiveStatistics.type], mapOf( it[LiveStatistics.name] to it[LiveStatistics.value] ) .toMutableMap() ) } } } transaction(database) { statsFile?.stats?.forEach { type, stats -> stats.forEach { name, value -> if (playerStats.get(type)?.get(name) != value) { Statistics.insert { it[Statistics.playerId] = playerId it[Statistics.type] = type it[Statistics.name] = name it[Statistics.timestamp] = Instant.now() as Instant it[Statistics.value] = value } } } } } } fun updateLiveStatistics(database: Database) { transaction(database) { TransactionManager.current() .exec( """ MERGE INTO LIVESTATISTICS AS T USING ( SELECT STATISTICS."Type", STATISTICS."Name", STATISTICS."PlayerId", STATISTICS."Value", (RANK () OVER (PARTITION BY STATISTICS."Type", STATISTICS."Name" ORDER BY STATISTICS."Value" DESC)) AS "Rank" FROM STATISTICS JOIN ( SELECT "Type", "Name", "PlayerId", MAX("Timestamp") AS "MaxTimestamp", FROM STATISTICS GROUP BY "Type", "Name", "PlayerId" ) MAX_TIMESTAMPS ON STATISTICS."Type" = MAX_TIMESTAMPS."Type" AND STATISTICS."Name" = MAX_TIMESTAMPS."Name" AND STATISTICS."PlayerId" = MAX_TIMESTAMPS."PlayerId" AND STATISTICS."Timestamp" = MAX_TIMESTAMPS."MaxTimestamp" ) AS S ON (T."Type" = S."Type" AND T."Name" = S."Name" AND T."PlayerId" = S."PlayerId") WHEN MATCHED AND (S."Value" <> T."Value" OR S."Rank" <> T."Rank")THEN UPDATE SET T."Value" = S."Value", T."Rank" = S."Rank" WHEN NOT MATCHED THEN INSERT VALUES (S."PlayerId", S."Type", S."Name", S."Value", S."Rank") """.trimIndent() ) } } fun updateAggregateStatistics(database: Database) { transaction(database) { TransactionManager.current() .exec( """ SET @Timestamp = CURRENT_TIMESTAMP; INSERT INTO AGGREGATESTATISTICS ("Type", "Name", "Timestamp", "Value") SELECT LiveMax."Type", '', @Timestamp AS "Timestamp", SUM(LiveMax."Value") FROM ( SELECT Live."Type", Live."Name", MAX(Live."Value") AS "Value" FROM livestatistics as Live WHERE array_contains(array['minecraft:mined'], Live."Type") GROUP BY Live."Type", Live."Name", Live."PlayerId" ) as LiveMax LEFT JOIN ( SELECT AGGREGATESTATISTICS."Type", AGGREGATESTATISTICS."Name", MAX(AGGREGATESTATISTICS."Value") as "Value" FROM AGGREGATESTATISTICS GROUP BY AGGREGATESTATISTICS."Type", AGGREGATESTATISTICS."Name" ) as Agg ON LiveMax."Type" = Agg."Type" GROUP BY LiveMax."Type" HAVING sum(LiveMax."Value") <> max(Agg."Value") OR max(Agg."Value") IS NULL; INSERT INTO AGGREGATESTATISTICS ("Type", "Name", "Timestamp", "Value") SELECT LiveMax."Type", LiveMax."Name", @Timestamp AS "Timestamp", SUM(LiveMax."Value") FROM ( SELECT Live."Type", Live."Name", MAX(Live."Value") AS "Value" FROM livestatistics as Live WHERE array_contains(array['minecraft:animals_bred', 'minecraft:play_one_minute', 'minecraft:deaths', 'minecraft:player_kills', 'minecraft:aviate_one_cm', 'minecraft:boat_one_cm', 'minecraft:crouch_one_cm', 'minecraft:horse_one_cm', 'minecraft:minecart_one_cm', 'minecraft:sprint_one_cm', 'minecraft:strider_one_cm', 'minecraft:swim_one_cm', 'minecraft:walk_on_water_one_cm', 'minecraft:walk_one_cm', 'minecraft:walk_under_water_one_cm' ], Live."Name") OR array_contains(array['minecraft:killed', 'minecraft:killed_by'], Live."Type") GROUP BY Live."Type", Live."Name", Live."PlayerId" ) as LiveMax LEFT JOIN ( SELECT AGGREGATESTATISTICS."Type", AGGREGATESTATISTICS."Name", MAX(AGGREGATESTATISTICS."Value") as "Value" FROM AGGREGATESTATISTICS GROUP BY AGGREGATESTATISTICS."Type", AGGREGATESTATISTICS."Name" ) as Agg ON LiveMax."Type" = Agg."Type" AND LiveMax."Name" = Agg."Name" GROUP BY LiveMax."Type", LiveMax."Name" HAVING SUM(LiveMax."Value") <> MAX(Agg."Value") OR MAX(Agg."Value") IS NULL; """.trimIndent() ) } } fun refreshPlayerNames(database: Database) { transaction(database) { addLogger(StdOutSqlLogger) val savedPlayers = Players.selectAll().map { object { val id = it[Players.id] val timestamp = it[Players.timestamp] } } val players = LiveStatistics.slice(LiveStatistics.playerId) .selectAll() .groupBy(LiveStatistics.playerId) .map { it[LiveStatistics.playerId] } savedPlayers .filter { Duration.between(it.timestamp, Instant.now()) > Duration.ofDays(1) } .forEach { player -> runBlocking { val name = getName(player.id) if (name != null) { println("Updating ${player.id} -> $name") Players.update({ Players.id eq player.id }) { it[Players.name] = name it[Players.timestamp] = Instant.now() } } else { println("Error updating ${player.id}") } } } players .filterNot { player -> savedPlayers.any { savedPlayer -> savedPlayer.id == player } } .forEach { playerId -> runBlocking { val name = getName(playerId) if (name != null) { println("Updating $playerId -> $name") Players.insert { it[Players.id] = playerId it[Players.name] = name it[Players.timestamp] = Instant.now() } } else { println("Error updating $playerId") } } } } } suspend fun getName(id: String): String? { val response: HttpResponse = httpClient.request( "https://sessionserver.mojang.com/session/minecraft/profile/${id.replace("-", "")}" ) { method = HttpMethod.Get } if (response.status.isSuccess()) { try { return MojangProfileResponse.fromJson(response.readText())?.name } catch (e: Exception) {} } return null } data class StatsFile(val stats: Map>) { companion object { public fun fromJson(json: String) = klaxon.parse(json) } } data class MojangProfileResponse(val name: String) { companion object { public fun fromJson(json: String) = klaxon.parse(json) } } }