diff --git a/src/main/kotlin/io/heapy/kotbusta/ktor/routes/books/DownloadBookRoute.kt b/src/main/kotlin/io/heapy/kotbusta/ktor/routes/books/DownloadBookRoute.kt index 677a611..b53c804 100644 --- a/src/main/kotlin/io/heapy/kotbusta/ktor/routes/books/DownloadBookRoute.kt +++ b/src/main/kotlin/io/heapy/kotbusta/ktor/routes/books/DownloadBookRoute.kt @@ -9,7 +9,7 @@ import io.heapy.kotbusta.ktor.routes.requireApprovedUser import io.heapy.kotbusta.ktor.routes.requiredParameter import io.heapy.kotbusta.service.BookFileException import io.heapy.kotbusta.service.ConversionFormat -import io.heapy.kotbusta.util.asciiFallbackFileName +import io.heapy.kotbusta.util.attachmentContentDisposition import io.ktor.http.* import io.ktor.server.response.* import io.ktor.server.routing.* @@ -62,19 +62,5 @@ fun Route.downloadBookRoute() { } } -internal fun downloadContentDisposition(fileName: String): ContentDisposition { - val fallback = asciiFallbackFileName(fileName) - val disposition = ContentDisposition.Attachment.withParameter( - ContentDisposition.Parameters.FileName, - fallback, - ) - - return if (fallback == fileName) { - disposition - } else { - disposition.withParameter( - ContentDisposition.Parameters.FileNameAsterisk, - fileName, - ) - } -} +internal fun downloadContentDisposition(fileName: String): ContentDisposition = + attachmentContentDisposition(fileName, includeAsciiFallback = true) diff --git a/src/main/kotlin/io/heapy/kotbusta/model/ImportJob.kt b/src/main/kotlin/io/heapy/kotbusta/model/ImportJob.kt index 6e7399f..dc3eb61 100644 --- a/src/main/kotlin/io/heapy/kotbusta/model/ImportJob.kt +++ b/src/main/kotlin/io/heapy/kotbusta/model/ImportJob.kt @@ -19,7 +19,6 @@ data class ImportJob( val booksAdded: Int, val bookErrors: Int, val sequentialBookErrors: Int, - val maxSequentialBookErrors: Int, val bookDeleted: Int, val startedAt: Instant, val completedAt: Instant? = null, @@ -98,7 +97,6 @@ fun ImportStats.toImportJob() = ImportJob( booksAdded = booksAdded.load(), bookErrors = bookErrors.load(), sequentialBookErrors = sequentialBookErrors.load(), - maxSequentialBookErrors = ImportStats.MAX_SEQUENTIAL_BOOK_ERRORS, bookDeleted = booksDeleted.load(), startedAt = startedAt.load(), completedAt = completedAt.load(), diff --git a/src/main/kotlin/io/heapy/kotbusta/service/BookFileService.kt b/src/main/kotlin/io/heapy/kotbusta/service/BookFileService.kt index 5a863b2..828b20c 100644 --- a/src/main/kotlin/io/heapy/kotbusta/service/BookFileService.kt +++ b/src/main/kotlin/io/heapy/kotbusta/service/BookFileService.kt @@ -2,6 +2,9 @@ package io.heapy.kotbusta.service import io.heapy.komok.tech.logging.Logger import io.heapy.kotbusta.model.Book +import io.heapy.kotbusta.util.UNDERSCORE_RUN +import io.heapy.kotbusta.util.WHITESPACE_RUN +import io.heapy.kotbusta.util.takeCodePointSafe import java.io.File import java.nio.file.Files import java.nio.file.Path @@ -92,27 +95,15 @@ class ZipBookFileService( .trim() .ifBlank { "book_${book.id}" } .replace(UNSAFE_FILENAME_CHARS, "_") - .replace(WHITESPACE, "_") - .replace(UNDERSCORES, "_") + .replace(WHITESPACE_RUN, "_") + .replace(UNDERSCORE_RUN, "_") .trim('.', '_') .ifBlank { "book_${book.id}" } - .truncateForFilename() - - private fun String.truncateForFilename(): String { - val truncated = take(100) - val withoutTrailingHighSurrogate = if (truncated.lastOrNull()?.isHighSurrogate() == true) { - truncated.dropLast(1) - } else { - truncated - } - - return withoutTrailingHighSurrogate.trimEnd('.', '_') - } + .takeCodePointSafe(100) + .trimEnd('.', '_') private companion object : Logger() { private val UNSAFE_FILENAME_CHARS = Regex("""[\p{Cntrl}/\\:*?"<>|]+""") - private val WHITESPACE = Regex("""\s+""") - private val UNDERSCORES = Regex("""_+""") } } diff --git a/src/main/kotlin/io/heapy/kotbusta/service/EmailService.kt b/src/main/kotlin/io/heapy/kotbusta/service/EmailService.kt index 1f977e1..b4fa2c8 100644 --- a/src/main/kotlin/io/heapy/kotbusta/service/EmailService.kt +++ b/src/main/kotlin/io/heapy/kotbusta/service/EmailService.kt @@ -4,7 +4,11 @@ import aws.sdk.kotlin.services.ses.SesClient import aws.sdk.kotlin.services.ses.model.RawMessage import aws.sdk.kotlin.services.ses.model.SendRawEmailRequest import io.heapy.komok.tech.logging.Logger +import io.heapy.kotbusta.util.WHITESPACE_RUN import io.heapy.kotbusta.util.asciiFallbackFileName +import io.heapy.kotbusta.util.attachmentContentDisposition +import io.heapy.kotbusta.util.takeCodePointSafe +import java.io.ByteArrayOutputStream import java.io.File import java.util.* @@ -19,6 +23,7 @@ interface EmailService { recipientEmail: String, bookFile: File, bookTitle: String, + attachmentFileName: String, format: String, ): EmailResult } @@ -31,6 +36,7 @@ class SesEmailService( recipientEmail: String, bookFile: File, bookTitle: String, + attachmentFileName: String, format: String, ): EmailResult { return try { @@ -46,7 +52,7 @@ class SesEmailService( subject = "Your book: $title", body = "Please find your requested book attached.", attachmentBytes = bookFile.readBytes(), - attachmentName = "$title.${format.lowercase()}", + attachmentName = attachmentFileName, mimeType = mimeType, ) @@ -98,22 +104,15 @@ class SesEmailService( } private val CONTROL_OR_QUOTE = Regex("""[\p{Cntrl}"]+""") -private val WHITESPACE = Regex("""\s+""") private const val MAX_SANITIZED_BOOK_TITLE_LENGTH = 100 internal fun sanitizeBookTitle(title: String): String = title .replace(CONTROL_OR_QUOTE, " ") - .replace(WHITESPACE, " ") + .replace(WHITESPACE_RUN, " ") + .trim() + .takeCodePointSafe(MAX_SANITIZED_BOOK_TITLE_LENGTH) .trim() - .let { sanitizedTitle -> - val truncated = sanitizedTitle.take(MAX_SANITIZED_BOOK_TITLE_LENGTH) - if (truncated.lastOrNull()?.let { Character.isHighSurrogate(it) } == true) { - truncated.dropLast(1) - } else { - truncated - }.trim() - } .ifBlank { "book" } internal fun buildRawEmail( @@ -130,21 +129,14 @@ internal fun buildRawEmail( val boundary = "----=_Part_${System.currentTimeMillis()}" val fallbackName = asciiFallbackFileName(attachmentName) - // For non-ASCII names send ONLY the RFC 2231 filename*. Common parsers - // (Python's email, and likely Amazon's ingestion) prefer a plain - // filename= when both are present, which would replace the real title - // with the ASCII fallback; extended-only is what calibre sends to - // Kindle and is known to decode correctly there. - val disposition = if (attachmentName.all { it.code in 0x20..0x7E }) { - """attachment; filename="$attachmentName"""" - } else { - "attachment; filename*=${rfc2231FileName(attachmentName)}" - } + val disposition = attachmentContentDisposition(attachmentName, includeAsciiFallback = false) - val emailBuilder = StringBuilder() + val out = ByteArrayOutputStream() + fun write(text: String) = out.write(text.toByteArray()) fun line(text: String = "") { // RFC 5322 requires CRLF line endings in raw messages. - emailBuilder.append(text).append("\r\n") + write(text) + write("\r\n") } // Headers @@ -169,13 +161,18 @@ internal fun buildRawEmail( line("Content-Transfer-Encoding: base64") line("Content-Disposition: $disposition") line() - line(Base64.getMimeEncoder().encodeToString(attachmentBytes)) - line() + // Stream-encode the attachment straight into `out` to avoid holding a + // separate full-size base64 String. wrap(...).close() flushes the final + // base64 group; ByteArrayOutputStream.close() is a no-op, so `out` stays + // usable afterwards. + Base64.getMimeEncoder().wrap(out).use { it.write(attachmentBytes) } + line() // terminate the final base64 line + line() // blank line before the closing boundary // End boundary line("--$boundary--") - return emailBuilder.toString().toByteArray() + return out.toByteArray() } /** @@ -213,17 +210,3 @@ internal fun encodeMimeHeaderValue(text: String): String { "=?UTF-8?B?${Base64.getEncoder().encodeToString(it.toByteArray())}?=" } } - -/** - * RFC 2231/5987 extended parameter value (`UTF-8''%D0%9C...`) carrying the - * real Unicode file name alongside the ASCII fallback `filename`. - */ -internal fun rfc2231FileName(fileName: String): String = - fileName.toByteArray().joinToString(separator = "", prefix = "UTF-8''") { byte -> - val char = byte.toInt().toChar() - if (char in 'a'..'z' || char in 'A'..'Z' || char in '0'..'9' || char in "!#$&+-.^_`|~") { - char.toString() - } else { - "%%%02X".format(byte.toInt() and 0xFF) - } - } diff --git a/src/main/kotlin/io/heapy/kotbusta/service/EmbeddingService.kt b/src/main/kotlin/io/heapy/kotbusta/service/EmbeddingService.kt index 1c3642a..942cca4 100644 --- a/src/main/kotlin/io/heapy/kotbusta/service/EmbeddingService.kt +++ b/src/main/kotlin/io/heapy/kotbusta/service/EmbeddingService.kt @@ -78,19 +78,22 @@ class DjlEmbeddingService( } internal class AutoTokenTypeTextEmbeddingTranslator( - tokenizer: HuggingFaceTokenizer, + private val tokenizer: HuggingFaceTokenizer, ) : Translator { - private val withoutTokenTypes = textEmbeddingTranslator(tokenizer, includeTokenTypes = false) - private val withTokenTypes = textEmbeddingTranslator(tokenizer, includeTokenTypes = true) + @Volatile private var delegate: TextEmbeddingTranslator? = null override fun getBatchifier(): Batchifier = Batchifier.STACK override fun prepare(ctx: TranslatorContext) { - val includeTokenTypes = requiresTokenTypes(ctx.model.describeInput().keys()) - delegate = if (includeTokenTypes) withTokenTypes else withoutTokenTypes - selectedDelegate().prepare(ctx) + val translator = delegate ?: synchronized(this) { + delegate ?: textEmbeddingTranslator( + tokenizer, + requiresTokenTypes(ctx.model.describeInput().keys()), + ).also { delegate = it } + } + translator.prepare(ctx) } override fun processInput(ctx: TranslatorContext, input: String): NDList = diff --git a/src/main/kotlin/io/heapy/kotbusta/util/AsciiFileName.kt b/src/main/kotlin/io/heapy/kotbusta/util/AsciiFileName.kt index 24aa374..253d2c2 100644 --- a/src/main/kotlin/io/heapy/kotbusta/util/AsciiFileName.kt +++ b/src/main/kotlin/io/heapy/kotbusta/util/AsciiFileName.kt @@ -22,7 +22,7 @@ internal fun asciiFallbackFileName(fileName: String): String { } } .joinToString("") - .replace(Regex("_+"), "_") + .replace(UNDERSCORE_RUN, "_") .trim('.', '-', '_') .ifBlank { "book" } .take(100) diff --git a/src/main/kotlin/io/heapy/kotbusta/util/AttachmentDisposition.kt b/src/main/kotlin/io/heapy/kotbusta/util/AttachmentDisposition.kt new file mode 100644 index 0000000..c56a6b5 --- /dev/null +++ b/src/main/kotlin/io/heapy/kotbusta/util/AttachmentDisposition.kt @@ -0,0 +1,37 @@ +package io.heapy.kotbusta.util + +import io.ktor.http.ContentDisposition + +/** + * Builds an attachment Content-Disposition for [fileName]. + * + * Pure-ASCII names get a plain `filename=`. A non-ASCII name always carries the + * real name in an RFC 5987 `filename*`; [includeAsciiFallback] controls whether + * a plain ASCII `filename=` fallback is sent alongside it. Kindle wants the + * extended parameter ONLY (false) so mail parsers don't prefer the ASCII + * fallback and show a transliterated title; HTTP downloads send both (true) so + * clients without RFC 5987 support still get a usable name. + */ +internal fun attachmentContentDisposition( + fileName: String, + includeAsciiFallback: Boolean, +): ContentDisposition { + val isAscii = fileName.all { it.code in 0x20..0x7E } + if (isAscii) { + return ContentDisposition.Attachment.withParameter( + ContentDisposition.Parameters.FileName, + fileName, + ) + } + val extendedOnly = ContentDisposition.Attachment.withParameter( + ContentDisposition.Parameters.FileNameAsterisk, + fileName, + ) + return if (includeAsciiFallback) { + ContentDisposition.Attachment + .withParameter(ContentDisposition.Parameters.FileName, asciiFallbackFileName(fileName)) + .withParameter(ContentDisposition.Parameters.FileNameAsterisk, fileName) + } else { + extendedOnly + } +} diff --git a/src/main/kotlin/io/heapy/kotbusta/util/TextSanitization.kt b/src/main/kotlin/io/heapy/kotbusta/util/TextSanitization.kt new file mode 100644 index 0000000..8623776 --- /dev/null +++ b/src/main/kotlin/io/heapy/kotbusta/util/TextSanitization.kt @@ -0,0 +1,20 @@ +package io.heapy.kotbusta.util + +/** Matches a run of whitespace characters. */ +internal val WHITESPACE_RUN = Regex("""\s+""") + +/** Matches a run of underscores. */ +internal val UNDERSCORE_RUN = Regex("""_+""") + +/** + * Takes at most [maxUtf16Units] UTF-16 code units, then drops a trailing + * unpaired high surrogate so truncation never splits a surrogate pair. + */ +internal fun String.takeCodePointSafe(maxUtf16Units: Int): String { + val truncated = take(maxUtf16Units) + return if (truncated.lastOrNull()?.isHighSurrogate() == true) { + truncated.dropLast(1) + } else { + truncated + } +} diff --git a/src/main/kotlin/io/heapy/kotbusta/worker/KindleSendWorker.kt b/src/main/kotlin/io/heapy/kotbusta/worker/KindleSendWorker.kt index 498cbb0..fa08b50 100644 --- a/src/main/kotlin/io/heapy/kotbusta/worker/KindleSendWorker.kt +++ b/src/main/kotlin/io/heapy/kotbusta/worker/KindleSendWorker.kt @@ -187,7 +187,8 @@ class KindleSendWorker( recipientEmail = device.email, bookFile = materialized.file, bookTitle = book.title, - format = item.format, + attachmentFileName = materialized.fileName, + format = materialized.format, ) } finally { materialized.cleanup() diff --git a/src/main/resources/static/js/components/AdminPanel.js b/src/main/resources/static/js/components/AdminPanel.js index 6ae6c52..eb2d7bb 100644 --- a/src/main/resources/static/js/components/AdminPanel.js +++ b/src/main/resources/static/js/components/AdminPanel.js @@ -98,7 +98,6 @@ export function AdminPanel() { : `${processedFiles} files`; const currentFile = jobStatus?.currentInpFile; const sequentialErrors = jobStatus?.sequentialBookErrors || 0; - const maxSequentialErrors = jobStatus?.maxSequentialBookErrors || 100; const messages = Object.entries(jobStatus?.messages || {}) .sort((a, b) => new Date(a[0]) - new Date(b[0])); @@ -186,7 +185,7 @@ export function AdminPanel() { h('div', { className: 'metric-label' }, 'Error Streak'), h('div', { className: `metric-value ${sequentialErrors > 0 ? 'danger-text' : ''}` - }, `${sequentialErrors} / ${maxSequentialErrors}`) + }, `${sequentialErrors}`) ) ), diff --git a/src/test/kotlin/io/heapy/kotbusta/service/EmailServiceTest.kt b/src/test/kotlin/io/heapy/kotbusta/service/EmailServiceTest.kt index ff09a5f..fd53b1c 100644 --- a/src/test/kotlin/io/heapy/kotbusta/service/EmailServiceTest.kt +++ b/src/test/kotlin/io/heapy/kotbusta/service/EmailServiceTest.kt @@ -1,5 +1,6 @@ package io.heapy.kotbusta.service +import org.junit.jupiter.api.Assertions.assertArrayEquals import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertThrows @@ -64,7 +65,7 @@ class EmailServiceTest { val disposition = raw.lines().first { it.startsWith("Content-Disposition: ") } assertEquals( "Content-Disposition: attachment; " + - "filename*=UTF-8''%D0%9C%D0%B5%D0%B4%D0%B8%D1%82%D0%B0%D1%86%D0%B8%D1%8F%20" + + "filename*=utf-8''%D0%9C%D0%B5%D0%B4%D0%B8%D1%82%D0%B0%D1%86%D0%B8%D1%8F%20" + "%D0%B2%D0%B8%D0%BF%D0%B0%D1%81%D1%81%D0%B0%D0%BD%D1%8B.epub", disposition, ) @@ -79,7 +80,7 @@ class EmailServiceTest { ) assertTrue(raw.contains("Subject: Your book: Clean_Title\r\n")) - assertTrue(raw.contains("Content-Disposition: attachment; filename=\"Clean_Title.epub\"\r\n")) + assertTrue(raw.contains("Content-Disposition: attachment; filename=Clean_Title.epub\r\n")) assertFalse(raw.contains("filename*")) } @@ -129,6 +130,37 @@ class EmailServiceTest { assertTrue(raw.endsWith("\r\n")) } + @Test + fun `base64 attachment body decodes and wraps at mime line length`() { + val attachmentBytes = ByteArray(200) { it.toByte() } + val raw = buildRawEmail( + from = "sender@example.com", + to = "device@kindle.com", + subject = "Your book: Clean_Title", + body = "Please find your requested book attached.", + attachmentBytes = attachmentBytes, + attachmentName = "Clean_Title.epub", + mimeType = "application/epub+zip", + ).toString(Charsets.ISO_8859_1) + + val boundary = raw.lines() + .first { it.startsWith("Content-Type: multipart/mixed; boundary=") } + .substringAfter("boundary=\"") + .substringBefore("\"") + val attachmentHeaderBlock = + "Content-Transfer-Encoding: base64\r\n" + + "Content-Disposition: attachment; filename=Clean_Title.epub\r\n\r\n" + val base64Body = raw.substringAfter(attachmentHeaderBlock) + .substringBefore("\r\n--$boundary--") + + assertArrayEquals(attachmentBytes, Base64.getMimeDecoder().decode(base64Body)) + base64Body.split("\r\n") + .filter { it.isNotEmpty() } + .forEach { line -> + assertTrue(line.length <= 76, "base64 line exceeds 76 chars: ${line.length}") + } + } + @Test fun `long subject folds into encoded words of at most 75 chars`() { val raw = rawEmail( diff --git a/src/test/kotlin/io/heapy/kotbusta/worker/KindleSendWorkerTest.kt b/src/test/kotlin/io/heapy/kotbusta/worker/KindleSendWorkerTest.kt index 6e0e273..bce7e53 100644 --- a/src/test/kotlin/io/heapy/kotbusta/worker/KindleSendWorkerTest.kt +++ b/src/test/kotlin/io/heapy/kotbusta/worker/KindleSendWorkerTest.kt @@ -41,6 +41,7 @@ class KindleSendWorkerTest { assertEquals(KindleSendStatus.COMPLETED.name, status(tx, queueId)) assertEquals(1, email.calls) + assertEquals("Worker_Book.epub", email.lastAttachmentFileName) } @Test @@ -170,22 +171,32 @@ class KindleSendWorkerTest { private class FakeEmailService(private val result: EmailResult) : EmailService { var calls = 0 + var lastAttachmentFileName: String? = null + override suspend fun sendBookToKindle( recipientEmail: String, bookFile: File, bookTitle: String, + attachmentFileName: String, format: String, ): EmailResult { calls++ + lastAttachmentFileName = attachmentFileName return result } } private class FakeBookFileService : BookFileService { override suspend fun materialize(book: Book, format: String): MaterializedBook { + val normalizedFormat = format.lowercase() val dir = Files.createTempDirectory("fake-materialize-").toFile() val file = File(dir, "book.$format").apply { writeText("fake") } - return MaterializedBook(file = file, fileName = "book.$format", format = format, tempDir = dir) + return MaterializedBook( + file = file, + fileName = "Worker_Book.$normalizedFormat", + format = normalizedFormat, + tempDir = dir, + ) } }