From 8c550a9000feacc9663f7ab581716fa82b483168 Mon Sep 17 00:00:00 2001 From: Liyan Zhao Date: Sun, 29 Sep 2024 13:26:07 +0800 Subject: [PATCH 01/10] chore: bump dependencies --- gradle/wrapper/gradle-wrapper.properties | 2 +- libs.versions.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index b82aa23a..09523c0e 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME diff --git a/libs.versions.toml b/libs.versions.toml index 5836b88c..02d58aaf 100644 --- a/libs.versions.toml +++ b/libs.versions.toml @@ -48,7 +48,7 @@ wdmcf = { module = "me.bymartrixx:wdmcf", version.ref = "wdmcf" } [plugins] kotlin = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detect" } -loom = { id = "fabric-loom", version = "1.6.+" } +loom = { id = "fabric-loom", version = "1.7.3" } git_hooks = { id = "com.github.jakemarsden.git-hooks", version = "0.0.2" } # https://github.com/johnrengelman/shadow/issues/894 shadow = { id = "io.github.goooler.shadow", version = "8.1.7" } From f27142e5f28dec07bab0aba4655428a850273f72 Mon Sep 17 00:00:00 2001 From: Liyan Zhao Date: Sun, 29 Sep 2024 15:04:28 +0800 Subject: [PATCH 02/10] fix --- .../ledger/database/DatabaseManager.kt | 56 ++++++++++++++----- .../ledger/database/Tables.kt | 14 +++-- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt index cb22f93a..8c52bf57 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt @@ -16,6 +16,7 @@ import com.github.quiltservertools.ledger.utility.Negatable import com.github.quiltservertools.ledger.utility.PlayerResult import com.google.common.cache.Cache import com.mojang.authlib.GameProfile +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap import kotlinx.coroutines.delay import net.minecraft.util.Identifier import net.minecraft.util.math.BlockPos @@ -55,7 +56,6 @@ import org.jetbrains.exposed.sql.update import org.sqlite.SQLiteConfig import org.sqlite.SQLiteDataSource import java.time.Instant -import java.time.temporal.ChronoUnit import java.util.* import java.util.function.Function import javax.sql.DataSource @@ -65,6 +65,7 @@ import kotlin.math.ceil const val MAX_QUERY_RETRIES = 10 const val MIN_RETRY_DELAY = 1000L const val MAX_RETRY_DELAY = 300_000L +const val DAY_SECONDS = 24 * 3600 * 1000 object DatabaseManager { @@ -75,6 +76,7 @@ object DatabaseManager { get() = database.dialect.name private val cache = DatabaseCacheService + private val strings = Long2ObjectOpenHashMap() fun setup(dataSource: DataSource?) { val source = dataSource ?: getDefaultDatasource() @@ -105,6 +107,7 @@ object DatabaseManager { Tables.ObjectIdentifiers, Tables.Sources, Tables.Worlds, + Tables.Strings, withLogs = true ) logInfo("Tables created") @@ -132,8 +135,8 @@ object DatabaseManager { execute { Ledger.logger.info("Purging actions older than ${config[DatabaseSpec.autoPurgeDays]} days") val deleted = Tables.Actions.deleteWhere { - Tables.Actions.timestamp lessEq Instant.now() - .minus(config[DatabaseSpec.autoPurgeDays].toLong(), ChronoUnit.DAYS) + Tables.Actions.timestamp lessEq + System.currentTimeMillis() - config[DatabaseSpec.autoPurgeDays].toLong() * DAY_SECONDS } Ledger.logger.info("Successfully purged $deleted actions") } @@ -194,7 +197,17 @@ object DatabaseManager { } } - private fun getActionsFromQuery(query: Query): List { + private fun Transaction.getString(id: Long?): String? { + if (id == null) return "" + if (id in strings) return strings[id] + Tables.Strings.select(Tables.Strings.value).where { Tables.Strings.id eq id }.firstOrNull()?.let { + strings[id] = it[Tables.Strings.value] + return it[Tables.Strings.value] + } + return null + } + + private fun Transaction.getActionsFromQuery(query: Query): List { val actions = mutableListOf() for (action in query) { @@ -206,20 +219,20 @@ object DatabaseManager { val type = typeSupplier.get() type.id = action[Tables.Actions.id].value - type.timestamp = action[Tables.Actions.timestamp] + type.timestamp = Instant.ofEpochMilli(action[Tables.Actions.timestamp]) type.pos = BlockPos(action[Tables.Actions.x], action[Tables.Actions.y], action[Tables.Actions.z]) type.world = Identifier.tryParse(action[Tables.Worlds.identifier]) type.objectIdentifier = Identifier.of(action[Tables.ObjectIdentifiers.identifier]) type.oldObjectIdentifier = Identifier.of( action[Tables.ObjectIdentifiers.alias("oldObjects")[Tables.ObjectIdentifiers.identifier]] ) - type.objectState = action[Tables.Actions.blockState] - type.oldObjectState = action[Tables.Actions.oldBlockState] + type.objectState = getString(action[Tables.Actions.blockState]?.value) + type.oldObjectState = getString(action[Tables.Actions.oldBlockState]?.value) type.sourceName = action[Tables.Sources.name] type.sourceProfile = action.getOrNull(Tables.Players.playerId)?.let { GameProfile(it, action[Tables.Players.playerName]) } - type.extraData = action[Tables.Actions.extraData] + type.extraData = getString(action[Tables.Actions.extraData]?.value) type.rolledBack = action[Tables.Actions.rolledBack] actions.add(type) @@ -239,12 +252,14 @@ object DatabaseManager { if (params.before != null && params.after != null) { op = op.and { - Tables.Actions.timestamp.greaterEq(params.after) and Tables.Actions.timestamp.lessEq(params.before) + Tables.Actions.timestamp.greaterEq(params.after.toEpochMilli()) and Tables.Actions.timestamp.lessEq( + params.before.toEpochMilli() + ) } } else if (params.before != null) { - op = op.and { Tables.Actions.timestamp.lessEq(params.before) } + op = op.and { Tables.Actions.timestamp.lessEq(params.before.toEpochMilli()) } } else if (params.after != null) { - op = op.and { Tables.Actions.timestamp.greaterEq(params.after) } + op = op.and { Tables.Actions.timestamp.greaterEq(params.after.toEpochMilli()) } } if (params.rolledBack != null) { @@ -447,21 +462,32 @@ object DatabaseManager { } } + private fun getStringId(value: String): Long { + val existing = Tables.Strings.select(Tables.Strings.id).where { + Tables.Strings.hash eq value.hashCode() and (Tables.Strings.value eq value) + }.firstOrNull() + if (existing != null) return existing[Tables.Strings.id].value + return Tables.Strings.insertAndGetId { + it[Tables.Strings.value] = value + it[Tables.Strings.hash] = value.hashCode() + }.value + } + private fun Transaction.insertActions(actions: List) { Tables.Actions.batchInsert(actions, shouldReturnGeneratedValues = false) { action -> this[Tables.Actions.actionIdentifier] = getOrCreateActionId(action.identifier) - this[Tables.Actions.timestamp] = action.timestamp + this[Tables.Actions.timestamp] = action.timestamp.toEpochMilli() this[Tables.Actions.x] = action.pos.x this[Tables.Actions.y] = action.pos.y this[Tables.Actions.z] = action.pos.z this[Tables.Actions.objectId] = getOrCreateRegistryKeyId(action.objectIdentifier) this[Tables.Actions.oldObjectId] = getOrCreateRegistryKeyId(action.oldObjectIdentifier) this[Tables.Actions.world] = getOrCreateWorldId(action.world ?: Ledger.server.overworld.registryKey.value) - this[Tables.Actions.blockState] = action.objectState - this[Tables.Actions.oldBlockState] = action.oldObjectState + this[Tables.Actions.blockState] = action.objectState?.let(::getStringId) + this[Tables.Actions.oldBlockState] = action.oldObjectState?.let(::getStringId) this[Tables.Actions.sourceName] = getOrCreateSourceId(action.sourceName) this[Tables.Actions.sourcePlayer] = action.sourceProfile?.let { getOrCreatePlayerId(it.id) } - this[Tables.Actions.extraData] = action.extraData + this[Tables.Actions.extraData] = action.extraData?.let(::getStringId) } } diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt index 092ca735..77ea1f11 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt @@ -5,6 +5,7 @@ import org.jetbrains.exposed.dao.IntEntity import org.jetbrains.exposed.dao.IntEntityClass import org.jetbrains.exposed.dao.id.EntityID import org.jetbrains.exposed.dao.id.IntIdTable +import org.jetbrains.exposed.dao.id.LongIdTable import org.jetbrains.exposed.sql.alias import org.jetbrains.exposed.sql.javatime.timestamp import java.time.Instant @@ -55,18 +56,18 @@ object Tables { object Actions : IntIdTable("actions") { val actionIdentifier = reference("action_id", ActionIdentifiers.id).index() - val timestamp = timestamp("time") + val timestamp = long("time") val x = integer("x") val y = integer("y") val z = integer("z") val world = reference("world_id", Worlds.id) val objectId = reference("object_id", ObjectIdentifiers.id).index() val oldObjectId = reference("old_object_id", ObjectIdentifiers.id).index() - val blockState = text("block_state").nullable() - val oldBlockState = text("old_block_state").nullable() + val blockState = optReference("block_state_ref", Strings.id) + val oldBlockState = optReference("old_block_state_ref", Strings.id) val sourceName = reference("source", Sources.id).index() val sourcePlayer = optReference("player_id", Players.id).index() - val extraData = text("extra_data").nullable() + val extraData = optReference("extra_data_ref", Strings.id) val rolledBack = bool("rolled_back").clientDefault { false } init { @@ -112,4 +113,9 @@ object Tables { companion object : IntEntityClass(Worlds) } + + object Strings : LongIdTable("strings") { + val hash = integer("java_hash_code").index() + val value = text("value") + } } From 1d354bd8fcc2385ba8a5b4012db7c30447728992 Mon Sep 17 00:00:00 2001 From: Liyan Zhao Date: Mon, 30 Sep 2024 01:23:24 +0800 Subject: [PATCH 03/10] fix: convert --- .../ledger/database/DatabaseManager.kt | 26 ++++++++++++ .../ledger/database/Tables.kt | 42 ++++++++++++++++++- 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt index 8c52bf57..7fdbbbf1 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt @@ -43,6 +43,7 @@ import org.jetbrains.exposed.sql.andWhere import org.jetbrains.exposed.sql.batchInsert import org.jetbrains.exposed.sql.deleteWhere import org.jetbrains.exposed.sql.innerJoin +import org.jetbrains.exposed.sql.insert import org.jetbrains.exposed.sql.insertAndGetId import org.jetbrains.exposed.sql.insertIgnore import org.jetbrains.exposed.sql.or @@ -692,4 +693,29 @@ object DatabaseManager { return Tables.Player.wrapRows(query).toList().map { PlayerResult.fromRow(it) } } + + suspend fun Transaction.convertActions() { + Tables.ActionsLegacy.selectAll().forEach { row -> + newSuspendedTransaction { + // start a new transaction to avoid failing on the same row + Tables.Actions.insert { + it[actionIdentifier] = row[Tables.ActionsLegacy.actionIdentifier] + it[timestamp] = row[Tables.ActionsLegacy.timestamp].toEpochMilli() + it[x] = row[Tables.ActionsLegacy.x] + it[y] = row[Tables.ActionsLegacy.y] + it[z] = row[Tables.ActionsLegacy.z] + it[world] = row[Tables.ActionsLegacy.world] + it[objectId] = row[Tables.ActionsLegacy.objectId] + it[oldObjectId] = row[Tables.ActionsLegacy.oldObjectId] + it[blockState] = row[Tables.ActionsLegacy.blockState]?.let(::getStringId) + it[oldBlockState] = row[Tables.ActionsLegacy.oldBlockState]?.let(::getStringId) + it[sourceName] = row[Tables.ActionsLegacy.sourceName] + it[sourcePlayer] = row[Tables.ActionsLegacy.sourcePlayer] + it[extraData] = row[Tables.ActionsLegacy.extraData]?.let(::getStringId) + it[rolledBack] = row[Tables.ActionsLegacy.rolledBack] + } + Tables.ActionsLegacy.deleteWhere { Tables.ActionsLegacy.id eq row[Tables.ActionsLegacy.id] } + } + } + } } diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt index 77ea1f11..09c14228 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt @@ -54,7 +54,7 @@ object Tables { companion object : IntEntityClass(ObjectIdentifiers) } - object Actions : IntIdTable("actions") { + object Actions : IntIdTable("actions_v2") { val actionIdentifier = reference("action_id", ActionIdentifiers.id).index() val timestamp = long("time") val x = integer("x") @@ -94,6 +94,46 @@ object Tables { companion object : IntEntityClass(Actions) } + object ActionsLegacy : IntIdTable("actions") { + val actionIdentifier = reference("action_id", ActionIdentifiers.id).index() + val timestamp = timestamp("time") + val x = integer("x") + val y = integer("y") + val z = integer("z") + val world = reference("world_id", Worlds.id) + val objectId = reference("object_id", ObjectIdentifiers.id).index() + val oldObjectId = reference("old_object_id", ObjectIdentifiers.id).index() + val blockState = text("block_state").nullable() + val oldBlockState = text("old_block_state").nullable() + val sourceName = reference("source", Sources.id).index() + val sourcePlayer = optReference("player_id", Players.id).index() + val extraData = text("extra_data").nullable() + val rolledBack = bool("rolled_back").clientDefault { false } + + init { + index("actions_by_location", false, x, y, z, world) + } + } + + class ActionOld(id: EntityID) : IntEntity(id) { + var actionIdentifier by ActionIdentifier referencedOn ActionsLegacy.actionIdentifier + var timestamp by ActionsLegacy.timestamp + var x by ActionsLegacy.x + var y by ActionsLegacy.y + var z by ActionsLegacy.z + var world by World referencedOn ActionsLegacy.world + var objectId by ObjectIdentifier referencedOn ActionsLegacy.objectId + var oldObjectId by ObjectIdentifier referencedOn ActionsLegacy.oldObjectId + var blockState by ActionsLegacy.blockState + var oldBlockState by ActionsLegacy.oldBlockState + var sourceName by Source referencedOn ActionsLegacy.sourceName + var sourcePlayer by Player optionalReferencedOn ActionsLegacy.sourcePlayer + var extraData by ActionsLegacy.extraData + var rolledBack by ActionsLegacy.rolledBack + + companion object : IntEntityClass(ActionsLegacy) + } + object Sources : IntIdTable("sources") { val name = varchar("name", MAX_SOURCE_NAME_LENGTH).uniqueIndex() } From 57c0e7960f2bc4692c773baa62ac9defd7d0b65b Mon Sep 17 00:00:00 2001 From: Liyan Zhao Date: Mon, 30 Sep 2024 01:40:52 +0800 Subject: [PATCH 04/10] fix: convert --- .../ledger/database/DatabaseManager.kt | 126 +++++++++++++++++- .../ledger/database/Tables.kt | 1 + 2 files changed, 120 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt index 7fdbbbf1..75065073 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt @@ -57,6 +57,7 @@ import org.jetbrains.exposed.sql.update import org.sqlite.SQLiteConfig import org.sqlite.SQLiteDataSource import java.time.Instant +import java.time.temporal.ChronoUnit import java.util.* import java.util.function.Function import javax.sql.DataSource @@ -88,8 +89,8 @@ object DatabaseManager { val dbFilepath = config.getDatabasePath().resolve("ledger.sqlite").pathString return SQLiteDataSource( SQLiteConfig().apply { - setJournalMode(SQLiteConfig.JournalMode.WAL) - } + setJournalMode(SQLiteConfig.JournalMode.WAL) + } ).apply { url = "jdbc:sqlite:$dbFilepath" } @@ -136,10 +137,14 @@ object DatabaseManager { execute { Ledger.logger.info("Purging actions older than ${config[DatabaseSpec.autoPurgeDays]} days") val deleted = Tables.Actions.deleteWhere { - Tables.Actions.timestamp lessEq + timestamp lessEq System.currentTimeMillis() - config[DatabaseSpec.autoPurgeDays].toLong() * DAY_SECONDS } - Ledger.logger.info("Successfully purged $deleted actions") + val legacyDeleted = Tables.ActionsLegacy.deleteWhere { + timestamp lessEq + Instant.now().minus(config[DatabaseSpec.autoPurgeDays].toLong(), ChronoUnit.DAYS) + } + Ledger.logger.info("Successfully purged ${deleted + legacyDeleted} actions") } } } @@ -242,6 +247,42 @@ object DatabaseManager { return actions } + @Deprecated("legacy") + private fun Transaction.getActionsFromLegacyQuery(query: Query): List { + val actions = mutableListOf() + + for (action in query) { + val typeSupplier = ActionRegistry.getType(action[Tables.ActionIdentifiers.actionIdentifier]) + if (typeSupplier == null) { + logWarn("Unknown action type ${action[Tables.ActionIdentifiers.actionIdentifier]}") + continue + } + + val type = typeSupplier.get() + type.id = action[Tables.ActionsLegacy.id].value + type.timestamp = action[Tables.ActionsLegacy.timestamp] + type.pos = + BlockPos(action[Tables.ActionsLegacy.x], action[Tables.ActionsLegacy.y], action[Tables.ActionsLegacy.z]) + type.world = Identifier.tryParse(action[Tables.Worlds.identifier]) + type.objectIdentifier = Identifier.of(action[Tables.ObjectIdentifiers.identifier]) + type.oldObjectIdentifier = Identifier.of( + action[Tables.ObjectIdentifiers.alias("oldObjects")[Tables.ObjectIdentifiers.identifier]] + ) + type.objectState = action[Tables.ActionsLegacy.blockState] + type.oldObjectState = action[Tables.ActionsLegacy.oldBlockState] + type.sourceName = action[Tables.Sources.name] + type.sourceProfile = action.getOrNull(Tables.Players.playerId)?.let { + GameProfile(it, action[Tables.Players.playerName]) + } + type.extraData = action[Tables.ActionsLegacy.extraData] + type.rolledBack = action[Tables.ActionsLegacy.rolledBack] + + actions.add(type) + } + + return actions + } + private fun buildQueryParams(params: ActionSearchParams): Op { var op: Op = Op.TRUE @@ -306,6 +347,70 @@ object DatabaseManager { return op } + @Deprecated("legacy") + private fun buildQueryParamsLegacy(params: ActionSearchParams): Op { + var op: Op = Op.TRUE + + if (params.bounds != null) { + op = op.and { Tables.ActionsLegacy.x.between(params.bounds.minX, params.bounds.maxX) } + op = op.and { Tables.ActionsLegacy.y.between(params.bounds.minY, params.bounds.maxY) } + op = op.and { Tables.ActionsLegacy.z.between(params.bounds.minZ, params.bounds.maxZ) } + } + + if (params.before != null && params.after != null) { + op = op.and { + Tables.ActionsLegacy.timestamp.greaterEq(params.after) and + Tables.ActionsLegacy.timestamp.lessEq(params.before) + } + } else if (params.before != null) { + op = op.and { Tables.ActionsLegacy.timestamp.lessEq(params.before) } + } else if (params.after != null) { + op = op.and { Tables.ActionsLegacy.timestamp.greaterEq(params.after) } + } + + if (params.rolledBack != null) { + op = op.and { Tables.ActionsLegacy.rolledBack.eq(params.rolledBack) } + } + + op = addParameters( + op, + params.sourceNames, + DatabaseManager::getSourceId, + Tables.ActionsLegacy.sourceName + ) + + op = addParameters( + op, + params.actions, + DatabaseManager::getActionId, + Tables.ActionsLegacy.actionIdentifier + ) + + op = addParameters( + op, + params.worlds, + DatabaseManager::getWorldId, + Tables.ActionsLegacy.world + ) + + op = addParameters( + op, + params.objects, + DatabaseManager::getRegistryKeyId, + Tables.ActionsLegacy.objectId, + Tables.ActionsLegacy.oldObjectId + ) + + op = addParameters( + op, + params.sourcePlayerIds, + DatabaseManager::getPlayerId, + Tables.ActionsLegacy.sourcePlayer + ) + + return op + } + private fun , C : EntityID?, T> addParameters( op: Op, paramSet: Collection>?, @@ -683,7 +788,10 @@ object DatabaseManager { .deleteWhere { Tables.Actions.id inSubQuery Tables.Actions.select(Tables.Actions.id) .where(buildQueryParams(params)) - } + } + Tables.ActionsLegacy.deleteWhere { + Tables.ActionsLegacy.id inSubQuery Tables.ActionsLegacy.select(Tables.ActionsLegacy.id) + .where(buildQueryParamsLegacy(params)) + } private fun Transaction.selectPlayers(players: Set): List { val query = Tables.Players.selectAll() @@ -694,10 +802,12 @@ object DatabaseManager { return Tables.Player.wrapRows(query).toList().map { PlayerResult.fromRow(it) } } - suspend fun Transaction.convertActions() { + suspend fun Transaction.convertActions(progressReporter: (done: Long, total: Long) -> Unit) { + val total = Tables.ActionsLegacy.selectAll().count() + var done = 0L Tables.ActionsLegacy.selectAll().forEach { row -> newSuspendedTransaction { - // start a new transaction to avoid failing on the same row + // start a new transaction to avoid failing on some rows Tables.Actions.insert { it[actionIdentifier] = row[Tables.ActionsLegacy.actionIdentifier] it[timestamp] = row[Tables.ActionsLegacy.timestamp].toEpochMilli() @@ -715,6 +825,8 @@ object DatabaseManager { it[rolledBack] = row[Tables.ActionsLegacy.rolledBack] } Tables.ActionsLegacy.deleteWhere { Tables.ActionsLegacy.id eq row[Tables.ActionsLegacy.id] } + done++ + progressReporter(done, total) } } } diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt index 09c14228..e93141a4 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt @@ -94,6 +94,7 @@ object Tables { companion object : IntEntityClass(Actions) } + @Deprecated("legacy") object ActionsLegacy : IntIdTable("actions") { val actionIdentifier = reference("action_id", ActionIdentifiers.id).index() val timestamp = timestamp("time") From 980d30efeb4058f794cb95823cafca0aebcbb17f Mon Sep 17 00:00:00 2001 From: Liyan Zhao Date: Thu, 3 Oct 2024 20:05:54 +0800 Subject: [PATCH 05/10] build: bump dependencies --- libs.versions.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs.versions.toml b/libs.versions.toml index 02d58aaf..886cea5a 100644 --- a/libs.versions.toml +++ b/libs.versions.toml @@ -1,9 +1,9 @@ [versions] minecraft = "1.21" -yarn-mappings = "1.21+build.1" -fabric-loader = "0.15.11" +yarn-mappings = "1.21+build.9" +fabric-loader = "0.16.5" -fabric-api = "0.100.1+1.21" +fabric-api = "0.102.0+1.21" # Kotlin kotlin = "2.0.0" From bd2ee8dc5f9a3bf0868d22a19c523034b9d027e3 Mon Sep 17 00:00:00 2001 From: Liyan Zhao Date: Sun, 6 Oct 2024 15:05:04 +0800 Subject: [PATCH 06/10] feat: convert subcommand --- .../ledger/commands/LedgerCommand.kt | 3 ++ .../commands/subcommands/ConvertSubcommand.kt | 34 +++++++++++++++++++ .../ledger/database/DatabaseManager.kt | 2 +- 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 src/main/kotlin/com/github/quiltservertools/ledger/commands/subcommands/ConvertSubcommand.kt diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/commands/LedgerCommand.kt b/src/main/kotlin/com/github/quiltservertools/ledger/commands/LedgerCommand.kt index d7c81bd5..61807d4c 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/commands/LedgerCommand.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/commands/LedgerCommand.kt @@ -1,6 +1,7 @@ package com.github.quiltservertools.ledger.commands import com.github.quiltservertools.ledger.api.ExtensionManager +import com.github.quiltservertools.ledger.commands.subcommands.ConvertSubcommand import com.github.quiltservertools.ledger.commands.subcommands.InspectCommand import com.github.quiltservertools.ledger.commands.subcommands.PageCommand import com.github.quiltservertools.ledger.commands.subcommands.PlayerCommand @@ -53,6 +54,8 @@ fun registerCommands(dispatcher: Dispatcher) { rootNode.addChild(PlayerCommand.build()) + rootNode.addChild(ConvertSubcommand.build()) + ExtensionManager.commands.forEach { it.registerSubcommands().forEach { command -> rootNode.addChild(command.build()) diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/commands/subcommands/ConvertSubcommand.kt b/src/main/kotlin/com/github/quiltservertools/ledger/commands/subcommands/ConvertSubcommand.kt new file mode 100644 index 00000000..6f09a12c --- /dev/null +++ b/src/main/kotlin/com/github/quiltservertools/ledger/commands/subcommands/ConvertSubcommand.kt @@ -0,0 +1,34 @@ +package com.github.quiltservertools.ledger.commands.subcommands + +import com.github.quiltservertools.ledger.Ledger +import com.github.quiltservertools.ledger.commands.BuildableCommand +import com.github.quiltservertools.ledger.commands.CommandConsts +import com.github.quiltservertools.ledger.database.DatabaseManager +import com.github.quiltservertools.ledger.utility.LiteralNode +import com.mojang.brigadier.context.CommandContext +import kotlinx.coroutines.launch +import me.lucko.fabric.api.permissions.v0.Permissions +import net.minecraft.server.command.CommandManager.literal +import net.minecraft.server.command.ServerCommandSource +import net.minecraft.text.Text + +private const val PROGRESS_INTERVAL = 30L + +object ConvertSubcommand : BuildableCommand { + override fun build(): LiteralNode = + literal("convert") + .requires(Permissions.require("ledger.commands.convert", CommandConsts.PERMISSION_LEVEL)) + .executes { convertDatabase(it) } + .build() + + private fun convertDatabase(it: CommandContext): Int { + Ledger.launch { + DatabaseManager.convertActions { done, total -> + if (done % PROGRESS_INTERVAL == 0L) { + it.source.sendFeedback({ Text.of("Converted $done/$total actions") }, false) + } + } + } + return 1 + } +} diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt index 75065073..1e549b49 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt @@ -802,7 +802,7 @@ object DatabaseManager { return Tables.Player.wrapRows(query).toList().map { PlayerResult.fromRow(it) } } - suspend fun Transaction.convertActions(progressReporter: (done: Long, total: Long) -> Unit) { + suspend fun convertActions(progressReporter: (done: Long, total: Long) -> Unit) = execute { val total = Tables.ActionsLegacy.selectAll().count() var done = 0L Tables.ActionsLegacy.selectAll().forEach { row -> From b663f2953dbfaa18521c87341b6be42c4bf0345e Mon Sep 17 00:00:00 2001 From: Liyan Zhao Date: Thu, 10 Oct 2024 21:07:36 +0800 Subject: [PATCH 07/10] fix: database compatibility --- .../com/github/quiltservertools/ledger/database/Tables.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt index e93141a4..377fe380 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt @@ -71,7 +71,7 @@ object Tables { val rolledBack = bool("rolled_back").clientDefault { false } init { - index("actions_by_location", false, x, y, z, world) + index("actions_v2_by_location", false, x, y, z, world) } } From efa94cbbb497b408a9ecb3e94f3f3e82d9cbcd5a Mon Sep 17 00:00:00 2001 From: Liyan Zhao Date: Fri, 11 Oct 2024 20:55:39 +0800 Subject: [PATCH 08/10] add gzip; drop old scheme support except converting --- .../ledger/database/DatabaseManager.kt | 211 +++++++----------- .../ledger/database/Tables.kt | 22 +- 2 files changed, 87 insertions(+), 146 deletions(-) diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt index 1e549b49..3ce7669b 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt @@ -18,6 +18,7 @@ import com.google.common.cache.Cache import com.mojang.authlib.GameProfile import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap import kotlinx.coroutines.delay +import kotlinx.coroutines.launch import net.minecraft.util.Identifier import net.minecraft.util.math.BlockPos import org.jetbrains.exposed.dao.Entity @@ -46,20 +47,25 @@ import org.jetbrains.exposed.sql.innerJoin import org.jetbrains.exposed.sql.insert import org.jetbrains.exposed.sql.insertAndGetId import org.jetbrains.exposed.sql.insertIgnore +import org.jetbrains.exposed.sql.not import org.jetbrains.exposed.sql.or import org.jetbrains.exposed.sql.orWhere import org.jetbrains.exposed.sql.selectAll import org.jetbrains.exposed.sql.statements.StatementContext +import org.jetbrains.exposed.sql.statements.api.ExposedBlob import org.jetbrains.exposed.sql.statements.expandArgs import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update import org.sqlite.SQLiteConfig import org.sqlite.SQLiteDataSource +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream import java.time.Instant -import java.time.temporal.ChronoUnit import java.util.* import java.util.function.Function +import java.util.zip.GZIPInputStream +import java.util.zip.GZIPOutputStream import javax.sql.DataSource import kotlin.io.path.pathString import kotlin.math.ceil @@ -68,6 +74,8 @@ const val MAX_QUERY_RETRIES = 10 const val MIN_RETRY_DELAY = 1000L const val MAX_RETRY_DELAY = 300_000L const val DAY_SECONDS = 24 * 3600 * 1000 +const val GZIP_THRESHOLD = 920 +const val MEMORY_100MB = 100_000 // units are KB, sqlite cache size object DatabaseManager { @@ -89,6 +97,8 @@ object DatabaseManager { val dbFilepath = config.getDatabasePath().resolve("ledger.sqlite").pathString return SQLiteDataSource( SQLiteConfig().apply { + enforceForeignKeys(true) + setCacheSize(MEMORY_100MB) setJournalMode(SQLiteConfig.JournalMode.WAL) } ).apply { @@ -140,11 +150,7 @@ object DatabaseManager { timestamp lessEq System.currentTimeMillis() - config[DatabaseSpec.autoPurgeDays].toLong() * DAY_SECONDS } - val legacyDeleted = Tables.ActionsLegacy.deleteWhere { - timestamp lessEq - Instant.now().minus(config[DatabaseSpec.autoPurgeDays].toLong(), ChronoUnit.DAYS) - } - Ledger.logger.info("Successfully purged ${deleted + legacyDeleted} actions") + Ledger.logger.info("Successfully purged $deleted actions") } } } @@ -206,13 +212,58 @@ object DatabaseManager { private fun Transaction.getString(id: Long?): String? { if (id == null) return "" if (id in strings) return strings[id] - Tables.Strings.select(Tables.Strings.value).where { Tables.Strings.id eq id }.firstOrNull()?.let { - strings[id] = it[Tables.Strings.value] - return it[Tables.Strings.value] + Tables.Strings.selectAll().where { Tables.Strings.id eq id }.firstOrNull()?.let { + if (!it[Tables.Strings.gzip]) { + strings[id] = it[Tables.Strings.value].bytes.decodeToString() + return strings[id] + } else { + val bytes = it[Tables.Strings.value].bytes + ByteArrayInputStream(bytes).use { byteArrayInputStream -> + GZIPInputStream(byteArrayInputStream).use { gzip -> + strings[id] = gzip.readBytes().decodeToString() + return strings[id] + } + } + } } return null } + private fun getStringId(value: String): Long { + Tables.Strings.select(Tables.Strings.id).where { + not(Tables.Strings.gzip) and + (Tables.Strings.hash eq value.hashCode()) and + (Tables.Strings.value eq ExposedBlob(value.encodeToByteArray())) + }.firstOrNull()?.let { + return it[Tables.Strings.id].value + } + if (value.length > GZIP_THRESHOLD) { + val bytes = value.encodeToByteArray() + val gzipped = ByteArrayOutputStream().let { arrayOutputStream -> + GZIPOutputStream(arrayOutputStream).use { gzip -> + gzip.write(bytes) + } + arrayOutputStream.toByteArray() + } + Tables.Strings.select(Tables.Strings.id).where { + Tables.Strings.gzip and + (Tables.Strings.hash eq value.hashCode()) and + (Tables.Strings.value eq ExposedBlob(gzipped)) + }.firstOrNull()?.let { + return it[Tables.Strings.id].value + } + return Tables.Strings.insertAndGetId { + it[Tables.Strings.value] = ExposedBlob(gzipped) + it[Tables.Strings.hash] = value.hashCode() + it[Tables.Strings.gzip] = true + }.value + } + return Tables.Strings.insertAndGetId { + it[Tables.Strings.value] = ExposedBlob(value.encodeToByteArray()) + it[Tables.Strings.hash] = value.hashCode() + }.value + } + private fun Transaction.getActionsFromQuery(query: Query): List { val actions = mutableListOf() @@ -247,42 +298,6 @@ object DatabaseManager { return actions } - @Deprecated("legacy") - private fun Transaction.getActionsFromLegacyQuery(query: Query): List { - val actions = mutableListOf() - - for (action in query) { - val typeSupplier = ActionRegistry.getType(action[Tables.ActionIdentifiers.actionIdentifier]) - if (typeSupplier == null) { - logWarn("Unknown action type ${action[Tables.ActionIdentifiers.actionIdentifier]}") - continue - } - - val type = typeSupplier.get() - type.id = action[Tables.ActionsLegacy.id].value - type.timestamp = action[Tables.ActionsLegacy.timestamp] - type.pos = - BlockPos(action[Tables.ActionsLegacy.x], action[Tables.ActionsLegacy.y], action[Tables.ActionsLegacy.z]) - type.world = Identifier.tryParse(action[Tables.Worlds.identifier]) - type.objectIdentifier = Identifier.of(action[Tables.ObjectIdentifiers.identifier]) - type.oldObjectIdentifier = Identifier.of( - action[Tables.ObjectIdentifiers.alias("oldObjects")[Tables.ObjectIdentifiers.identifier]] - ) - type.objectState = action[Tables.ActionsLegacy.blockState] - type.oldObjectState = action[Tables.ActionsLegacy.oldBlockState] - type.sourceName = action[Tables.Sources.name] - type.sourceProfile = action.getOrNull(Tables.Players.playerId)?.let { - GameProfile(it, action[Tables.Players.playerName]) - } - type.extraData = action[Tables.ActionsLegacy.extraData] - type.rolledBack = action[Tables.ActionsLegacy.rolledBack] - - actions.add(type) - } - - return actions - } - private fun buildQueryParams(params: ActionSearchParams): Op { var op: Op = Op.TRUE @@ -347,70 +362,6 @@ object DatabaseManager { return op } - @Deprecated("legacy") - private fun buildQueryParamsLegacy(params: ActionSearchParams): Op { - var op: Op = Op.TRUE - - if (params.bounds != null) { - op = op.and { Tables.ActionsLegacy.x.between(params.bounds.minX, params.bounds.maxX) } - op = op.and { Tables.ActionsLegacy.y.between(params.bounds.minY, params.bounds.maxY) } - op = op.and { Tables.ActionsLegacy.z.between(params.bounds.minZ, params.bounds.maxZ) } - } - - if (params.before != null && params.after != null) { - op = op.and { - Tables.ActionsLegacy.timestamp.greaterEq(params.after) and - Tables.ActionsLegacy.timestamp.lessEq(params.before) - } - } else if (params.before != null) { - op = op.and { Tables.ActionsLegacy.timestamp.lessEq(params.before) } - } else if (params.after != null) { - op = op.and { Tables.ActionsLegacy.timestamp.greaterEq(params.after) } - } - - if (params.rolledBack != null) { - op = op.and { Tables.ActionsLegacy.rolledBack.eq(params.rolledBack) } - } - - op = addParameters( - op, - params.sourceNames, - DatabaseManager::getSourceId, - Tables.ActionsLegacy.sourceName - ) - - op = addParameters( - op, - params.actions, - DatabaseManager::getActionId, - Tables.ActionsLegacy.actionIdentifier - ) - - op = addParameters( - op, - params.worlds, - DatabaseManager::getWorldId, - Tables.ActionsLegacy.world - ) - - op = addParameters( - op, - params.objects, - DatabaseManager::getRegistryKeyId, - Tables.ActionsLegacy.objectId, - Tables.ActionsLegacy.oldObjectId - ) - - op = addParameters( - op, - params.sourcePlayerIds, - DatabaseManager::getPlayerId, - Tables.ActionsLegacy.sourcePlayer - ) - - return op - } - private fun , C : EntityID?, T> addParameters( op: Op, paramSet: Collection>?, @@ -568,17 +519,6 @@ object DatabaseManager { } } - private fun getStringId(value: String): Long { - val existing = Tables.Strings.select(Tables.Strings.id).where { - Tables.Strings.hash eq value.hashCode() and (Tables.Strings.value eq value) - }.firstOrNull() - if (existing != null) return existing[Tables.Strings.id].value - return Tables.Strings.insertAndGetId { - it[Tables.Strings.value] = value - it[Tables.Strings.hash] = value.hashCode() - }.value - } - private fun Transaction.insertActions(actions: List) { Tables.Actions.batchInsert(actions, shouldReturnGeneratedValues = false) { action -> this[Tables.Actions.actionIdentifier] = getOrCreateActionId(action.identifier) @@ -784,13 +724,32 @@ object DatabaseManager { ) // Workaround because can't delete from a join in exposed https://kotlinlang.slack.com/archives/C0CG7E0A1/p1605866974117400 - private fun Transaction.purgeActions(params: ActionSearchParams) = Tables.Actions - .deleteWhere { - Tables.Actions.id inSubQuery Tables.Actions.select(Tables.Actions.id) + private fun Transaction.purgeActions(params: ActionSearchParams): Int { + val refs = Tables.Strings.select(Tables.Strings.id).where { + Tables.Strings.id inSubQuery Tables.Actions.select(Tables.Actions.blockState) + .where(buildQueryParams(params)) + }.map { it[Tables.Strings.id].value } + Tables.Strings.select(Tables.Strings.id).where { + Tables.Strings.id inSubQuery Tables.Actions.select(Tables.Actions.oldBlockState) .where(buildQueryParams(params)) - } + Tables.ActionsLegacy.deleteWhere { - Tables.ActionsLegacy.id inSubQuery Tables.ActionsLegacy.select(Tables.ActionsLegacy.id) - .where(buildQueryParamsLegacy(params)) + }.map { it[Tables.Strings.id].value } + Tables.Strings.select(Tables.Strings.id).where { + Tables.Strings.id inSubQuery Tables.Actions.select(Tables.Actions.extraData) + .where(buildQueryParams(params)) + }.map { it[Tables.Strings.id].value }.toList() + val deleted = Tables.Actions + .deleteWhere { + Tables.Actions.id inSubQuery Tables.Actions.select(Tables.Actions.id) + .where(buildQueryParams(params)) + } + refs.forEach { ref -> + Ledger.launch { + execute { + runCatching { + Tables.Strings.deleteWhere { Tables.Strings.id eq ref } + } + } + } + } + return deleted } private fun Transaction.selectPlayers(players: Set): List { diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt index 377fe380..ed61ea4b 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/Tables.kt @@ -116,25 +116,6 @@ object Tables { } } - class ActionOld(id: EntityID) : IntEntity(id) { - var actionIdentifier by ActionIdentifier referencedOn ActionsLegacy.actionIdentifier - var timestamp by ActionsLegacy.timestamp - var x by ActionsLegacy.x - var y by ActionsLegacy.y - var z by ActionsLegacy.z - var world by World referencedOn ActionsLegacy.world - var objectId by ObjectIdentifier referencedOn ActionsLegacy.objectId - var oldObjectId by ObjectIdentifier referencedOn ActionsLegacy.oldObjectId - var blockState by ActionsLegacy.blockState - var oldBlockState by ActionsLegacy.oldBlockState - var sourceName by Source referencedOn ActionsLegacy.sourceName - var sourcePlayer by Player optionalReferencedOn ActionsLegacy.sourcePlayer - var extraData by ActionsLegacy.extraData - var rolledBack by ActionsLegacy.rolledBack - - companion object : IntEntityClass(ActionsLegacy) - } - object Sources : IntIdTable("sources") { val name = varchar("name", MAX_SOURCE_NAME_LENGTH).uniqueIndex() } @@ -157,6 +138,7 @@ object Tables { object Strings : LongIdTable("strings") { val hash = integer("java_hash_code").index() - val value = text("value") + val value = blob("value") + val gzip = bool("gzip").default(false) } } From 8995779ef165dc6e610a524be405ecf84a500485 Mon Sep 17 00:00:00 2001 From: Liyan Zhao Date: Fri, 11 Oct 2024 21:43:31 +0800 Subject: [PATCH 09/10] feat: smart purge --- .../ledger/config/DatabaseSpec.kt | 13 +++++++ .../ledger/database/DatabaseManager.kt | 37 +++++++++++++++++++ src/main/resources/ledger.toml | 3 ++ 3 files changed, 53 insertions(+) diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/config/DatabaseSpec.kt b/src/main/kotlin/com/github/quiltservertools/ledger/config/DatabaseSpec.kt index 62e953f2..25b64042 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/config/DatabaseSpec.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/config/DatabaseSpec.kt @@ -11,6 +11,19 @@ object DatabaseSpec : ConfigSpec() { val queueTimeoutMin by required() val queueCheckDelaySec by required() val autoPurgeDays by required() + val smartPurge by optional(false) + val smartPurgeThreshold by optional(100) + val smartPurgeFilter by optional>( + listOf( + "action_id", + "world_id", + "x", + "y", + "z", + "object_id", + "player_id" + ) + ) val batchSize by optional(1000) val batchDelay by optional(10) val logSQL by optional(false) diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt index 3ce7669b..25b8f74c 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt @@ -42,6 +42,7 @@ import org.jetbrains.exposed.sql.alias import org.jetbrains.exposed.sql.and import org.jetbrains.exposed.sql.andWhere import org.jetbrains.exposed.sql.batchInsert +import org.jetbrains.exposed.sql.count import org.jetbrains.exposed.sql.deleteWhere import org.jetbrains.exposed.sql.innerJoin import org.jetbrains.exposed.sql.insert @@ -153,6 +154,42 @@ object DatabaseManager { Ledger.logger.info("Successfully purged $deleted actions") } } + if (config[DatabaseSpec.smartPurge]) { + var totalDelete = 0 + Ledger.logger.info( + "Smart purging actions, smart purge threshold: ${config[DatabaseSpec.smartPurgeThreshold]}, " + + "smart purge filter: ${config[DatabaseSpec.smartPurgeFilter]}" + ) + execute { + val count = Tables.Actions.id.count() + val columns = Tables.Actions.columns.filter { + it.name in config[DatabaseSpec.smartPurgeFilter] + } + + @Suppress("SpreadOperator") + val rows = Tables.Actions.select(columns + listOf(count)).having { + count greater config[DatabaseSpec.smartPurgeThreshold].toLong() + }.groupBy(*columns.toTypedArray()).toList() + rows.forEachIndexed { index, row -> + val total = row[count] + Ledger.logger.info("Smart purge found $total actions to delete, params: $row") + val ids = Tables.Actions.select(Tables.Actions.id).where { + columns.map { + if (row[it] != null) { + it eq it.asLiteral(row[it]) + } else { + it.isNull() + } + }.reduce { acc, op -> acc and op } + }.orderBy(Tables.Actions.timestamp, SortOrder.ASC) + .limit(total.toInt() - config[DatabaseSpec.smartPurgeThreshold]) + val deleted = Tables.Actions.deleteWhere { Tables.Actions.id inSubQuery ids } + Ledger.logger.info("Smart purge deleted $deleted actions ($index / ${rows.size})") + totalDelete += deleted + } + } + Ledger.logger.info("Smart purge complete, deleted $totalDelete actions") + } } suspend fun searchActions(params: ActionSearchParams, page: Int): SearchResults = execute { diff --git a/src/main/resources/ledger.toml b/src/main/resources/ledger.toml index d3aaa47d..77bf98ce 100644 --- a/src/main/resources/ledger.toml +++ b/src/main/resources/ledger.toml @@ -12,6 +12,9 @@ batchDelay = 10 # The location of the database file. Defaults to the world folder if not specified #location = "./custom-dir" +# uncomment the following line to enable smart purge +#smartPurge = true + [search] # Number of actions to show per page pageSize = 8 From 66acfe1f4f29a9c629e625060df4d6b9aef2e471 Mon Sep 17 00:00:00 2001 From: Liyan Zhao Date: Fri, 11 Oct 2024 21:50:07 +0800 Subject: [PATCH 10/10] feat: smart purge --- .../ledger/database/DatabaseManager.kt | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt index 25b8f74c..428d184a 100644 --- a/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt +++ b/src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt @@ -184,7 +184,17 @@ object DatabaseManager { }.orderBy(Tables.Actions.timestamp, SortOrder.ASC) .limit(total.toInt() - config[DatabaseSpec.smartPurgeThreshold]) val deleted = Tables.Actions.deleteWhere { Tables.Actions.id inSubQuery ids } - Ledger.logger.info("Smart purge deleted $deleted actions ($index / ${rows.size})") + var strings = 0 + findStringRefs { + Tables.Actions.id inSubQuery ids + }.forEach { ref -> + execute { + runCatching { + strings += Tables.Strings.deleteWhere { Tables.Strings.id eq ref } + } + } + } + Ledger.logger.info("Smart purged $deleted actions & $strings strings ($index / ${rows.size})") totalDelete += deleted } } @@ -762,16 +772,9 @@ object DatabaseManager { // Workaround because can't delete from a join in exposed https://kotlinlang.slack.com/archives/C0CG7E0A1/p1605866974117400 private fun Transaction.purgeActions(params: ActionSearchParams): Int { - val refs = Tables.Strings.select(Tables.Strings.id).where { - Tables.Strings.id inSubQuery Tables.Actions.select(Tables.Actions.blockState) - .where(buildQueryParams(params)) - }.map { it[Tables.Strings.id].value } + Tables.Strings.select(Tables.Strings.id).where { - Tables.Strings.id inSubQuery Tables.Actions.select(Tables.Actions.oldBlockState) - .where(buildQueryParams(params)) - }.map { it[Tables.Strings.id].value } + Tables.Strings.select(Tables.Strings.id).where { - Tables.Strings.id inSubQuery Tables.Actions.select(Tables.Actions.extraData) - .where(buildQueryParams(params)) - }.map { it[Tables.Strings.id].value }.toList() + val refs = findStringRefs { + buildQueryParams(params) + } val deleted = Tables.Actions .deleteWhere { Tables.Actions.id inSubQuery Tables.Actions.select(Tables.Actions.id) @@ -789,6 +792,20 @@ object DatabaseManager { return deleted } + private fun findStringRefs(where: () -> Op): List { + val refs = Tables.Strings.select(Tables.Strings.id).withDistinct().where { + Tables.Strings.id inSubQuery Tables.Actions.select(Tables.Actions.blockState) + .where(where()) + }.map { it[Tables.Strings.id].value } + Tables.Strings.select(Tables.Strings.id).withDistinct().where { + Tables.Strings.id inSubQuery Tables.Actions.select(Tables.Actions.oldBlockState) + .where(where()) + }.map { it[Tables.Strings.id].value } + Tables.Strings.select(Tables.Strings.id).withDistinct().where { + Tables.Strings.id inSubQuery Tables.Actions.select(Tables.Actions.extraData) + .where(where()) + }.map { it[Tables.Strings.id].value }.toList() + return refs.toSet().toList() + } + private fun Transaction.selectPlayers(players: Set): List { val query = Tables.Players.selectAll() for (player in players) {