Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import io.heapy.kotbusta.ktor.routes.requireApprovedUser
import io.heapy.kotbusta.ktor.routes.requiredParameter
import io.heapy.kotbusta.service.BookFileException
import io.heapy.kotbusta.service.ConversionFormat
import io.heapy.kotbusta.util.asciiFallbackFileName
import io.heapy.kotbusta.util.attachmentContentDisposition
import io.ktor.http.*
import io.ktor.server.response.*
import io.ktor.server.routing.*
Expand Down Expand Up @@ -62,19 +62,5 @@ fun Route.downloadBookRoute() {
}
}

internal fun downloadContentDisposition(fileName: String): ContentDisposition {
val fallback = asciiFallbackFileName(fileName)
val disposition = ContentDisposition.Attachment.withParameter(
ContentDisposition.Parameters.FileName,
fallback,
)

return if (fallback == fileName) {
disposition
} else {
disposition.withParameter(
ContentDisposition.Parameters.FileNameAsterisk,
fileName,
)
}
}
internal fun downloadContentDisposition(fileName: String): ContentDisposition =
attachmentContentDisposition(fileName, includeAsciiFallback = true)
2 changes: 0 additions & 2 deletions src/main/kotlin/io/heapy/kotbusta/model/ImportJob.kt
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ data class ImportJob(
val booksAdded: Int,
val bookErrors: Int,
val sequentialBookErrors: Int,
val maxSequentialBookErrors: Int,
val bookDeleted: Int,
val startedAt: Instant,
val completedAt: Instant? = null,
Expand Down Expand Up @@ -98,7 +97,6 @@ fun ImportStats.toImportJob() = ImportJob(
booksAdded = booksAdded.load(),
bookErrors = bookErrors.load(),
sequentialBookErrors = sequentialBookErrors.load(),
maxSequentialBookErrors = ImportStats.MAX_SEQUENTIAL_BOOK_ERRORS,
bookDeleted = booksDeleted.load(),
startedAt = startedAt.load(),
completedAt = completedAt.load(),
Expand Down
23 changes: 7 additions & 16 deletions src/main/kotlin/io/heapy/kotbusta/service/BookFileService.kt
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ package io.heapy.kotbusta.service

import io.heapy.komok.tech.logging.Logger
import io.heapy.kotbusta.model.Book
import io.heapy.kotbusta.util.UNDERSCORE_RUN
import io.heapy.kotbusta.util.WHITESPACE_RUN
import io.heapy.kotbusta.util.takeCodePointSafe
import java.io.File
import java.nio.file.Files
import java.nio.file.Path
Expand Down Expand Up @@ -92,27 +95,15 @@ class ZipBookFileService(
.trim()
.ifBlank { "book_${book.id}" }
.replace(UNSAFE_FILENAME_CHARS, "_")
.replace(WHITESPACE, "_")
.replace(UNDERSCORES, "_")
.replace(WHITESPACE_RUN, "_")
.replace(UNDERSCORE_RUN, "_")
.trim('.', '_')
.ifBlank { "book_${book.id}" }
.truncateForFilename()

private fun String.truncateForFilename(): String {
val truncated = take(100)
val withoutTrailingHighSurrogate = if (truncated.lastOrNull()?.isHighSurrogate() == true) {
truncated.dropLast(1)
} else {
truncated
}

return withoutTrailingHighSurrogate.trimEnd('.', '_')
}
.takeCodePointSafe(100)
.trimEnd('.', '_')

private companion object : Logger() {
private val UNSAFE_FILENAME_CHARS = Regex("""[\p{Cntrl}/\\:*?"<>|]+""")
private val WHITESPACE = Regex("""\s+""")
private val UNDERSCORES = Regex("""_+""")
}
}

Expand Down
63 changes: 23 additions & 40 deletions src/main/kotlin/io/heapy/kotbusta/service/EmailService.kt
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@ import aws.sdk.kotlin.services.ses.SesClient
import aws.sdk.kotlin.services.ses.model.RawMessage
import aws.sdk.kotlin.services.ses.model.SendRawEmailRequest
import io.heapy.komok.tech.logging.Logger
import io.heapy.kotbusta.util.WHITESPACE_RUN
import io.heapy.kotbusta.util.asciiFallbackFileName
import io.heapy.kotbusta.util.attachmentContentDisposition
import io.heapy.kotbusta.util.takeCodePointSafe
import java.io.ByteArrayOutputStream
import java.io.File
import java.util.*

Expand All @@ -19,6 +23,7 @@ interface EmailService {
recipientEmail: String,
bookFile: File,
bookTitle: String,
attachmentFileName: String,
format: String,
): EmailResult
}
Expand All @@ -31,6 +36,7 @@ class SesEmailService(
recipientEmail: String,
bookFile: File,
bookTitle: String,
attachmentFileName: String,
format: String,
): EmailResult {
return try {
Expand All @@ -46,7 +52,7 @@ class SesEmailService(
subject = "Your book: $title",
body = "Please find your requested book attached.",
attachmentBytes = bookFile.readBytes(),
attachmentName = "$title.${format.lowercase()}",
attachmentName = attachmentFileName,
mimeType = mimeType,
)

Expand Down Expand Up @@ -98,22 +104,15 @@ class SesEmailService(
}

private val CONTROL_OR_QUOTE = Regex("""[\p{Cntrl}"]+""")
private val WHITESPACE = Regex("""\s+""")
private const val MAX_SANITIZED_BOOK_TITLE_LENGTH = 100

internal fun sanitizeBookTitle(title: String): String =
title
.replace(CONTROL_OR_QUOTE, " ")
.replace(WHITESPACE, " ")
.replace(WHITESPACE_RUN, " ")
.trim()
.takeCodePointSafe(MAX_SANITIZED_BOOK_TITLE_LENGTH)
.trim()
.let { sanitizedTitle ->
val truncated = sanitizedTitle.take(MAX_SANITIZED_BOOK_TITLE_LENGTH)
if (truncated.lastOrNull()?.let { Character.isHighSurrogate(it) } == true) {
truncated.dropLast(1)
} else {
truncated
}.trim()
}
.ifBlank { "book" }

internal fun buildRawEmail(
Expand All @@ -130,21 +129,14 @@ internal fun buildRawEmail(

val boundary = "----=_Part_${System.currentTimeMillis()}"
val fallbackName = asciiFallbackFileName(attachmentName)
// For non-ASCII names send ONLY the RFC 2231 filename*. Common parsers
// (Python's email, and likely Amazon's ingestion) prefer a plain
// filename= when both are present, which would replace the real title
// with the ASCII fallback; extended-only is what calibre sends to
// Kindle and is known to decode correctly there.
val disposition = if (attachmentName.all { it.code in 0x20..0x7E }) {
"""attachment; filename="$attachmentName""""
} else {
"attachment; filename*=${rfc2231FileName(attachmentName)}"
}
val disposition = attachmentContentDisposition(attachmentName, includeAsciiFallback = false)

val emailBuilder = StringBuilder()
val out = ByteArrayOutputStream()
fun write(text: String) = out.write(text.toByteArray())
fun line(text: String = "") {
// RFC 5322 requires CRLF line endings in raw messages.
emailBuilder.append(text).append("\r\n")
write(text)
write("\r\n")
}

// Headers
Expand All @@ -169,13 +161,18 @@ internal fun buildRawEmail(
line("Content-Transfer-Encoding: base64")
line("Content-Disposition: $disposition")
line()
line(Base64.getMimeEncoder().encodeToString(attachmentBytes))
line()
// Stream-encode the attachment straight into `out` to avoid holding a
// separate full-size base64 String. wrap(...).close() flushes the final
// base64 group; ByteArrayOutputStream.close() is a no-op, so `out` stays
// usable afterwards.
Base64.getMimeEncoder().wrap(out).use { it.write(attachmentBytes) }
line() // terminate the final base64 line
line() // blank line before the closing boundary

// End boundary
line("--$boundary--")

return emailBuilder.toString().toByteArray()
return out.toByteArray()
}

/**
Expand Down Expand Up @@ -213,17 +210,3 @@ internal fun encodeMimeHeaderValue(text: String): String {
"=?UTF-8?B?${Base64.getEncoder().encodeToString(it.toByteArray())}?="
}
}

/**
* RFC 2231/5987 extended parameter value (`UTF-8''%D0%9C...`) carrying the
* real Unicode file name alongside the ASCII fallback `filename`.
*/
internal fun rfc2231FileName(fileName: String): String =
fileName.toByteArray().joinToString(separator = "", prefix = "UTF-8''") { byte ->
val char = byte.toInt().toChar()
if (char in 'a'..'z' || char in 'A'..'Z' || char in '0'..'9' || char in "!#$&+-.^_`|~") {
char.toString()
} else {
"%%%02X".format(byte.toInt() and 0xFF)
}
}
15 changes: 9 additions & 6 deletions src/main/kotlin/io/heapy/kotbusta/service/EmbeddingService.kt
Original file line number Diff line number Diff line change
Expand Up @@ -78,19 +78,22 @@ class DjlEmbeddingService(
}

internal class AutoTokenTypeTextEmbeddingTranslator(
tokenizer: HuggingFaceTokenizer,
private val tokenizer: HuggingFaceTokenizer,
) : Translator<String, FloatArray> {
private val withoutTokenTypes = textEmbeddingTranslator(tokenizer, includeTokenTypes = false)
private val withTokenTypes = textEmbeddingTranslator(tokenizer, includeTokenTypes = true)
@Volatile
private var delegate: TextEmbeddingTranslator? = null

override fun getBatchifier(): Batchifier =
Batchifier.STACK

override fun prepare(ctx: TranslatorContext) {
val includeTokenTypes = requiresTokenTypes(ctx.model.describeInput().keys())
delegate = if (includeTokenTypes) withTokenTypes else withoutTokenTypes
selectedDelegate().prepare(ctx)
val translator = delegate ?: synchronized(this) {
delegate ?: textEmbeddingTranslator(
tokenizer,
requiresTokenTypes(ctx.model.describeInput().keys()),
).also { delegate = it }
}
translator.prepare(ctx)
}

override fun processInput(ctx: TranslatorContext, input: String): NDList =
Expand Down
2 changes: 1 addition & 1 deletion src/main/kotlin/io/heapy/kotbusta/util/AsciiFileName.kt
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ internal fun asciiFallbackFileName(fileName: String): String {
}
}
.joinToString("")
.replace(Regex("_+"), "_")
.replace(UNDERSCORE_RUN, "_")
.trim('.', '-', '_')
.ifBlank { "book" }
.take(100)
Expand Down
37 changes: 37 additions & 0 deletions src/main/kotlin/io/heapy/kotbusta/util/AttachmentDisposition.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package io.heapy.kotbusta.util

import io.ktor.http.ContentDisposition

/**
* Builds an attachment Content-Disposition for [fileName].
*
* Pure-ASCII names get a plain `filename=`. A non-ASCII name always carries the
* real name in an RFC 5987 `filename*`; [includeAsciiFallback] controls whether
* a plain ASCII `filename=` fallback is sent alongside it. Kindle wants the
* extended parameter ONLY (false) so mail parsers don't prefer the ASCII
* fallback and show a transliterated title; HTTP downloads send both (true) so
* clients without RFC 5987 support still get a usable name.
*/
internal fun attachmentContentDisposition(
fileName: String,
includeAsciiFallback: Boolean,
): ContentDisposition {
val isAscii = fileName.all { it.code in 0x20..0x7E }
if (isAscii) {
return ContentDisposition.Attachment.withParameter(
ContentDisposition.Parameters.FileName,
fileName,
)
}
val extendedOnly = ContentDisposition.Attachment.withParameter(
ContentDisposition.Parameters.FileNameAsterisk,
fileName,
)
return if (includeAsciiFallback) {
ContentDisposition.Attachment
.withParameter(ContentDisposition.Parameters.FileName, asciiFallbackFileName(fileName))
.withParameter(ContentDisposition.Parameters.FileNameAsterisk, fileName)
} else {
extendedOnly
}
}
20 changes: 20 additions & 0 deletions src/main/kotlin/io/heapy/kotbusta/util/TextSanitization.kt
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package io.heapy.kotbusta.util

/** Matches a run of whitespace characters. */
internal val WHITESPACE_RUN = Regex("""\s+""")

/** Matches a run of underscores. */
internal val UNDERSCORE_RUN = Regex("""_+""")

/**
* Takes at most [maxUtf16Units] UTF-16 code units, then drops a trailing
* unpaired high surrogate so truncation never splits a surrogate pair.
*/
internal fun String.takeCodePointSafe(maxUtf16Units: Int): String {
val truncated = take(maxUtf16Units)
return if (truncated.lastOrNull()?.isHighSurrogate() == true) {
truncated.dropLast(1)
} else {
truncated
}
}
3 changes: 2 additions & 1 deletion src/main/kotlin/io/heapy/kotbusta/worker/KindleSendWorker.kt
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,8 @@ class KindleSendWorker(
recipientEmail = device.email,
bookFile = materialized.file,
bookTitle = book.title,
format = item.format,
attachmentFileName = materialized.fileName,
format = materialized.format,
)
} finally {
materialized.cleanup()
Expand Down
3 changes: 1 addition & 2 deletions src/main/resources/static/js/components/AdminPanel.js
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,6 @@ export function AdminPanel() {
: `${processedFiles} files`;
const currentFile = jobStatus?.currentInpFile;
const sequentialErrors = jobStatus?.sequentialBookErrors || 0;
const maxSequentialErrors = jobStatus?.maxSequentialBookErrors || 100;
const messages = Object.entries(jobStatus?.messages || {})
.sort((a, b) => new Date(a[0]) - new Date(b[0]));

Expand Down Expand Up @@ -186,7 +185,7 @@ export function AdminPanel() {
h('div', { className: 'metric-label' }, 'Error Streak'),
h('div', {
className: `metric-value ${sequentialErrors > 0 ? 'danger-text' : ''}`
}, `${sequentialErrors} / ${maxSequentialErrors}`)
}, `${sequentialErrors}`)
)
),

Expand Down
Loading